CrossValidation.cpp

Go to the documentation of this file.

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or
 * (at your option) any later version.
 *
 * Written (W) 2011-2012 Heiko Strathmann
 * Copyright (C) 2011 Berlin Institute of Technology and Max-Planck-Society
 */

#include <shogun/evaluation/CrossValidation.h>
#include <shogun/machine/Machine.h>
#include <shogun/evaluation/Evaluation.h>
#include <shogun/evaluation/SplittingStrategy.h>
#include <shogun/base/Parameter.h>
#include <shogun/mathematics/Statistics.h>
#include <shogun/evaluation/CrossValidationOutput.h>
#include <shogun/lib/List.h>

using namespace shogun;

CCrossValidation::CCrossValidation() : CMachineEvaluation()
{
    init();
}

CCrossValidation::CCrossValidation(CMachine* machine, CFeatures* features,
        CLabels* labels, CSplittingStrategy* splitting_strategy,
        CEvaluation* evaluation_criterion, bool autolock) :
        CMachineEvaluation(machine, features, labels, splitting_strategy,
        evaluation_criterion, autolock)
{
    init();
}

CCrossValidation::CCrossValidation(CMachine* machine, CLabels* labels,
        CSplittingStrategy* splitting_strategy,
        CEvaluation* evaluation_criterion, bool autolock) :
        CMachineEvaluation(machine, labels, splitting_strategy, evaluation_criterion,
        autolock)
{
    init();
}

CCrossValidation::~CCrossValidation()
{
    SG_UNREF(m_xval_outputs);
}

void CCrossValidation::init()
{
    m_num_runs=1;

    /* do reference counting for output objects */
    m_xval_outputs=new CList(true);

    SG_ADD(&m_num_runs, "num_runs", "Number of repetitions",
            MS_NOT_AVAILABLE);
    SG_ADD((CSGObject**)&m_xval_outputs, "m_xval_outputs", "List of output "
            "classes for intermediade cross-validation results",
            MS_NOT_AVAILABLE);
}

CEvaluationResult* CCrossValidation::evaluate()
{
    SG_DEBUG("entering %s::evaluate()\n", get_name())

    REQUIRE(m_machine, "%s::evaluate() is only possible if a machine is "
            "attached\n", get_name());

    REQUIRE(m_features, "%s::evaluate() is only possible if features are "
            "attached\n", get_name());

    REQUIRE(m_labels, "%s::evaluate() is only possible if labels are "
            "attached\n", get_name());

    /* if for some reason the do_unlock_frag is set, unlock */
    if (m_do_unlock)
    {
        m_machine->data_unlock();
        m_do_unlock=false;
    }

    /* set labels in any case (no locking needs this) */
    m_machine->set_labels(m_labels);

    if (m_autolock)
    {
        /* if machine supports locking try to do so */
        if (m_machine->supports_locking())
        {
            /* only lock if machine is not yet locked */
            if (!m_machine->is_data_locked())
            {
                m_machine->data_lock(m_labels, m_features);
                m_do_unlock=true;
            }
        }
        else
        {
            SG_WARNING("%s does not support locking. Autolocking is skipped. "
                    "Set autolock flag to false to get rid of warning.\n",
                    m_machine->get_name());
        }
    }

    SGVector<float64_t> results(m_num_runs);

    /* evtl. update xvalidation output class */
    CCrossValidationOutput* current=(CCrossValidationOutput*)
            m_xval_outputs->get_first_element();
    while (current)
    {
        current->init_num_runs(m_num_runs);
        current->init_num_folds(m_splitting_strategy->get_num_subsets());
        current->init_expose_labels(m_labels);
        current->post_init();
        SG_UNREF(current);
        current=(CCrossValidationOutput*)
                m_xval_outputs->get_next_element();
    }

    /* perform all the x-val runs */
    SG_DEBUG("starting %d runs of cross-validation\n", m_num_runs)
    for (index_t i=0; i <m_num_runs; ++i)
    {

        /* evtl. update xvalidation output class */
        current=(CCrossValidationOutput*)m_xval_outputs->get_first_element();
        while (current)
        {
            current->update_run_index(i);
            SG_UNREF(current);
            current=(CCrossValidationOutput*)
                    m_xval_outputs->get_next_element();
        }

        SG_DEBUG("entering cross-validation run %d \n", i)
        results[i]=evaluate_one_run();
        SG_DEBUG("result of cross-validation run %d is %f\n", i, results[i])
    }

    /* construct evaluation result */
    CCrossValidationResult* result = new CCrossValidationResult();
    result->mean=CStatistics::mean(results);
    if (m_num_runs>1)
        result->std_dev=CStatistics::std_deviation(results);
    else
        result->std_dev=0;

    /* unlock machine if it was locked in this method */
    if (m_machine->is_data_locked() && m_do_unlock)
    {
        m_machine->data_unlock();
        m_do_unlock=false;
    }

    SG_DEBUG("leaving %s::evaluate()\n", get_name())

    SG_REF(result);
    return result;
}

void CCrossValidation::set_num_runs(int32_t num_runs)
{
    if (num_runs <1)
        SG_ERROR("%d is an illegal number of repetitions\n", num_runs)

    m_num_runs=num_runs;
}

float64_t CCrossValidation::evaluate_one_run()
{
    SG_DEBUG("entering %s::evaluate_one_run()\n", get_name())
    index_t num_subsets=m_splitting_strategy->get_num_subsets();

    SG_DEBUG("building index sets for %d-fold cross-validation\n", num_subsets)

    /* build index sets */
    m_splitting_strategy->build_subsets();

    /* results array */
    SGVector<float64_t> results(num_subsets);

    /* different behavior whether data is locked or not */
    if (m_machine->is_data_locked())
    {
        SG_DEBUG("starting locked evaluation\n", get_name())
        /* do actual cross-validation */
        for (index_t i=0; i <num_subsets; ++i)
        {
            /* evtl. update xvalidation output class */
            CCrossValidationOutput* current=(CCrossValidationOutput*)
                    m_xval_outputs->get_first_element();
            while (current)
            {
                current->update_fold_index(i);
                SG_UNREF(current);
                current=(CCrossValidationOutput*)
                        m_xval_outputs->get_next_element();
            }

            /* index subset for training, will be freed below */
            SGVector<index_t> inverse_subset_indices =
                    m_splitting_strategy->generate_subset_inverse(i);

            /* train machine on training features */
            m_machine->train_locked(inverse_subset_indices);

            /* feature subset for testing */
            SGVector<index_t> subset_indices =
                    m_splitting_strategy->generate_subset_indices(i);

            /* evtl. update xvalidation output class */
            current=(CCrossValidationOutput*)m_xval_outputs->get_first_element();
            while (current)
            {
                current->update_train_indices(inverse_subset_indices, "\t");
                current->update_trained_machine(m_machine, "\t");
                SG_UNREF(current);
                current=(CCrossValidationOutput*)
                        m_xval_outputs->get_next_element();
            }

            /* produce output for desired indices */
            CLabels* result_labels=m_machine->apply_locked(subset_indices);
            SG_REF(result_labels);

            /* set subset for testing labels */
            m_labels->add_subset(subset_indices);

            /* evaluate against own labels */
            m_evaluation_criterion->set_indices(subset_indices);
            results[i]=m_evaluation_criterion->evaluate(result_labels, m_labels);

            /* evtl. update xvalidation output class */
            current=(CCrossValidationOutput*)m_xval_outputs->get_first_element();
            while (current)
            {
                current->update_test_indices(subset_indices, "\t");
                current->update_test_result(result_labels, "\t");
                current->update_test_true_result(m_labels, "\t");
                current->post_update_results();
                current->update_evaluation_result(results[i], "\t");
                SG_UNREF(current);
                current=(CCrossValidationOutput*)
                        m_xval_outputs->get_next_element();
            }

            /* remove subset to prevent side effects */
            m_labels->remove_subset();

            /* clean up */
            SG_UNREF(result_labels);

            SG_DEBUG("done locked evaluation\n", get_name())
        }
    }
    else
    {
        SG_DEBUG("starting unlocked evaluation\n", get_name())
        /* tell machine to store model internally
         * (otherwise changing subset of features will kaboom the classifier) */
        m_machine->set_store_model_features(true);

        /* do actual cross-validation */
        #pragma omp parallel for
        for (index_t i=0; i <num_subsets; ++i)
        {
            CMachine* machine;
            CFeatures* features;
            CLabels* labels;

            if (get_global_parallel()->get_num_threads()==1)
                machine=m_machine;
            else
                machine=(CMachine*)m_machine->clone();

            /* evtl. update xvalidation output class */
            CCrossValidationOutput* current=(CCrossValidationOutput*)
                    m_xval_outputs->get_first_element();
            #pragma omp critical
            {
            while (current)
            {
                current->update_fold_index(i);
                SG_UNREF(current);
                current=(CCrossValidationOutput*)
                        m_xval_outputs->get_next_element();
            }
            }

            /* set feature subset for training */
            SGVector<index_t> inverse_subset_indices=
                    m_splitting_strategy->generate_subset_inverse(i);
            
            if (get_global_parallel()->get_num_threads()==1)
                features=m_features;
            else
                features=(CFeatures*)m_features->clone();

            features->add_subset(inverse_subset_indices);

            /* set label subset for training */
            if (get_global_parallel()->get_num_threads()==1)
                labels=m_labels;
            else
                labels=machine->get_labels();
            labels->add_subset(inverse_subset_indices);

            SG_DEBUG("training set %d:\n", i)
            if (io->get_loglevel()==MSG_DEBUG)
            {
                SGVector<index_t>::display_vector(inverse_subset_indices.vector,
                        inverse_subset_indices.vlen, "training indices");
            }

            /* train machine on training features and remove subset */
            SG_DEBUG("starting training\n")
            machine->train(features);
            SG_DEBUG("finished training\n")

            /* evtl. update xvalidation output class */
            #pragma omp critical
            {
            current=(CCrossValidationOutput*)m_xval_outputs->get_first_element();
            while (current)
            {
                current->update_train_indices(inverse_subset_indices, "\t");
                current->update_trained_machine(machine, "\t");
                SG_UNREF(current);
                current=(CCrossValidationOutput*)
                        m_xval_outputs->get_next_element();
            }
            }

            features->remove_subset();
            labels->remove_subset();

            /* set feature subset for testing (subset method that stores pointer) */
            SGVector<index_t> subset_indices =
                    m_splitting_strategy->generate_subset_indices(i);
            features->add_subset(subset_indices);

            /* set label subset for testing */
            labels->add_subset(subset_indices);

            SG_DEBUG("test set %d:\n", i)
            if (io->get_loglevel()==MSG_DEBUG)
            {
                SGVector<index_t>::display_vector(subset_indices.vector,
                        subset_indices.vlen, "test indices");
            }

            /* apply machine to test features and remove subset */
            SG_DEBUG("starting evaluation\n")
            SG_DEBUG("%p\n", features)
            CLabels* result_labels=machine->apply(features);
            SG_DEBUG("finished evaluation\n")
            features->remove_subset();
            SG_REF(result_labels);

            /* evaluate */
            #pragma omp critical
            {
            results[i]=m_evaluation_criterion->evaluate(result_labels, labels);
            SG_DEBUG("result on fold %d is %f\n", i, results[i])
            }

            /* evtl. update xvalidation output class */
            #pragma omp critical
            {
            current=(CCrossValidationOutput*)m_xval_outputs->get_first_element();
            while (current)
            {
                current->update_test_indices(subset_indices, "\t");
                current->update_test_result(result_labels, "\t");
                current->update_test_true_result(labels, "\t");
                current->post_update_results();
                current->update_evaluation_result(results[i], "\t");
                SG_UNREF(current);
                current=(CCrossValidationOutput*)
                        m_xval_outputs->get_next_element();
            }
            }

            /* clean up, remove subsets */
            labels->remove_subset();
            if (get_global_parallel()->get_num_threads()!=1)
            {
                SG_UNREF(machine);
                SG_UNREF(features);
                SG_UNREF(labels);
            }
            SG_UNREF(result_labels);
        }

        SG_DEBUG("done unlocked evaluation\n", get_name())
    }

    /* build arithmetic mean of results */
    float64_t mean=CStatistics::mean(results);

    SG_DEBUG("leaving %s::evaluate_one_run()\n", get_name())
    return mean;
}

void CCrossValidation::add_cross_validation_output(
            CCrossValidationOutput* cross_validation_output)
{
    m_xval_outputs->append_element(cross_validation_output);
}