DeepBeliefNetwork.cpp

Go to the documentation of this file.

/*
 * Copyright (c) 2014, Shogun Toolbox Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:

 * 1. Redistributions of source code must retain the above copyright notice,
 * this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.

 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 * Written (W) 2014 Khaled Nasr
 */

#include <shogun/neuralnets/DeepBeliefNetwork.h>


#include <shogun/base/Parameter.h>
#include <shogun/mathematics/Math.h>
#include <shogun/mathematics/eigen3.h>
#include <shogun/lib/SGMatrix.h>
#include <shogun/lib/SGVector.h>
#include <shogun/features/DenseFeatures.h>
#include <shogun/lib/DynamicArray.h>
#include <shogun/neuralnets/NeuralNetwork.h>
#include <shogun/neuralnets/NeuralInputLayer.h>
#include <shogun/neuralnets/NeuralLogisticLayer.h>

using namespace shogun;

CDeepBeliefNetwork::CDeepBeliefNetwork() : CSGObject()
{
    init();
}

CDeepBeliefNetwork::CDeepBeliefNetwork(
    int32_t num_visible_units, ERBMVisibleUnitType unit_type) : CSGObject()
{
    init();
    m_layer_sizes->append_element(num_visible_units);
    m_num_layers++;
    m_visible_units_type = unit_type;
}

CDeepBeliefNetwork::~CDeepBeliefNetwork()
{
    SG_UNREF(m_layer_sizes);
}

void CDeepBeliefNetwork::add_hidden_layer(int32_t num_units)
{
    m_layer_sizes->append_element(num_units);
    m_num_layers++;
}

void CDeepBeliefNetwork::initialize_neural_network(float64_t sigma)
{
    m_bias_index_offsets = SGVector<int32_t>(m_num_layers);
    m_weights_index_offsets = SGVector<int32_t>(m_num_layers-1);

    m_num_params = 0;
    for (int32_t i=0; i<m_num_layers; i++)
    {
        m_bias_index_offsets[i] = m_num_params;
        m_num_params += m_layer_sizes->element(i);

        if (i<m_num_layers-1)
        {
            m_weights_index_offsets[i] = m_num_params;
            m_num_params += m_layer_sizes->element(i+1)*m_layer_sizes->element(i);
        }
    }

    m_params = SGVector<float64_t>(m_num_params);
    for (int32_t i=0; i<m_num_params; i++)
        m_params[i] = CMath::normal_random(0.0,sigma);

    pt_cd_num_steps = SGVector<int32_t>(m_num_layers-1);
    pt_cd_num_steps.set_const(1);

    pt_cd_persistent = SGVector<bool>(m_num_layers-1);
    pt_cd_persistent.set_const(true);

    pt_cd_sample_visible = SGVector<bool>(m_num_layers-1);
    pt_cd_sample_visible.set_const(false);

    pt_l2_coefficient = SGVector<float64_t>(m_num_layers-1);
    pt_l2_coefficient.set_const(0.0);

    pt_l1_coefficient = SGVector<float64_t>(m_num_layers-1);
    pt_l1_coefficient.set_const(0.0);

    pt_monitoring_interval = SGVector<int32_t>(m_num_layers-1);
    pt_monitoring_interval.set_const(10);

    pt_monitoring_method = SGVector<int32_t>(m_num_layers-1);
    pt_monitoring_method.set_const(RBMMM_RECONSTRUCTION_ERROR);

    pt_max_num_epochs = SGVector<int32_t>(m_num_layers-1);
    pt_max_num_epochs.set_const(1);

    pt_gd_mini_batch_size = SGVector<int32_t>(m_num_layers-1);
    pt_gd_mini_batch_size.set_const(0);

    pt_gd_learning_rate = SGVector<float64_t>(m_num_layers-1);
    pt_gd_learning_rate.set_const(0.1);

    pt_gd_learning_rate_decay = SGVector<float64_t>(m_num_layers-1);
    pt_gd_learning_rate_decay.set_const(1.0);

    pt_gd_momentum = SGVector<float64_t>(m_num_layers-1);
    pt_gd_momentum.set_const(0.9);
}

void CDeepBeliefNetwork::set_batch_size(int32_t batch_size)
{
    if (m_batch_size == batch_size) return;

    m_batch_size = batch_size;

    m_states = SGMatrixList<float64_t>(m_num_layers);

    for (int32_t i=0; i<m_num_layers; i++)
        m_states.set_matrix(i, SGMatrix<float64_t>(m_layer_sizes->element(i), m_batch_size));

    reset_chain();
}

void CDeepBeliefNetwork::pre_train(CDenseFeatures< float64_t >* features)
{
    for (int32_t k=0; k<m_num_layers-1; k++)
    {
        SG_INFO("Pre-training RBM %i\n",k);
        pre_train(k, features);
        SG_INFO("Finished pre-training RBM %i\n",k);
    }
}

void CDeepBeliefNetwork::pre_train(int32_t index,
    CDenseFeatures< float64_t >* features)
{
    CRBM rbm(m_layer_sizes->element(index+1));
    if (index == 0)
        rbm.add_visible_group(m_layer_sizes->element(index), m_visible_units_type);
    else
        rbm.add_visible_group(m_layer_sizes->element(index), RBMVUT_BINARY);
    rbm.initialize_neural_network(m_sigma);

    rbm.cd_num_steps = pt_cd_num_steps[index];
    rbm.cd_persistent = pt_cd_persistent[index];
    rbm.cd_sample_visible = pt_cd_sample_visible[index];
    rbm.l2_coefficient = pt_l2_coefficient[index];
    rbm.l1_coefficient = pt_l1_coefficient[index];
    rbm.monitoring_interval = pt_monitoring_interval[index];
    rbm.monitoring_method = ERBMMonitoringMethod(pt_monitoring_method[index]);
    rbm.max_num_epochs = pt_max_num_epochs[index];
    rbm.gd_mini_batch_size = pt_gd_mini_batch_size[index];
    rbm.gd_learning_rate = pt_gd_learning_rate[index];
    rbm.gd_learning_rate_decay = pt_gd_learning_rate_decay[index];
    rbm.gd_momentum = pt_gd_momentum[index];

    if (index > 0)
    {
        CDenseFeatures<float64_t>* transformed_features =
            transform(features, index);
        rbm.train(transformed_features);
        SG_UNREF(transformed_features);
    }
    else
        rbm.train(features);

    SGVector<float64_t> rbm_b = rbm.get_visible_bias();
    SGVector<float64_t> rbm_c = rbm.get_hidden_bias();
    SGMatrix<float64_t> rbm_w = rbm.get_weights();

    SGVector<float64_t> dbn_b = get_biases(index);
    SGVector<float64_t> dbn_c = get_biases(index+1);
    SGMatrix<float64_t> dbn_w = get_weights(index);

    for (int32_t i=0; i<dbn_b.vlen; i++)
        dbn_b[i] = rbm_b[i];

    for (int32_t i=0; i<dbn_c.vlen; i++)
        dbn_c[i] = rbm_c[i];

    for (int32_t i=0; i<dbn_w.num_rows*dbn_w.num_cols; i++)
        dbn_w[i] = rbm_w[i];
}

void CDeepBeliefNetwork::train(CDenseFeatures<float64_t>* features)
{
    REQUIRE(features != NULL, "Invalid (NULL) feature pointer\n");
    REQUIRE(features->get_num_features()==m_layer_sizes->element(0),
        "Number of features (%i) must match the DBN's number of visible units "
        "(%i)\n", features->get_num_features(), m_layer_sizes->element(0));

    SGMatrix<float64_t> inputs = features->get_feature_matrix();

    int32_t training_set_size = inputs.num_cols;
    if (gd_mini_batch_size==0) gd_mini_batch_size = training_set_size;
    set_batch_size(gd_mini_batch_size);

    SGVector<float64_t> rec_params(m_num_params);
    for (int32_t i=0; i<rec_params.vlen; i++)
        rec_params[i] = m_params[i];

    SGVector<float64_t> gradients(m_num_params);
    SGVector<float64_t> rec_gradients(m_num_params);
    gradients.zero();
    rec_gradients.zero();

    SGVector<float64_t> param_updates(m_num_params);
    SGVector<float64_t> rec_param_updates(m_num_params);
    param_updates.zero();
    rec_param_updates.zero();

    SGMatrixList<float64_t> sleep_states = m_states;
    SGMatrixList<float64_t> wake_states(m_num_layers);
    SGMatrixList<float64_t> psleep_states(m_num_layers);
    SGMatrixList<float64_t> pwake_states(m_num_layers);

    for (int32_t i=0; i<m_num_layers; i++)
    {
        wake_states.set_matrix(i, SGMatrix<float64_t>(m_layer_sizes->element(i), m_batch_size));
        psleep_states.set_matrix(i, SGMatrix<float64_t>(m_layer_sizes->element(i), m_batch_size));
        pwake_states.set_matrix(i, SGMatrix<float64_t>(m_layer_sizes->element(i), m_batch_size));
    }

    CRBM top_rbm(m_layer_sizes->element(m_num_layers-1));
    if (m_num_layers > 2)
        top_rbm.add_visible_group(m_layer_sizes->element(m_num_layers-2), RBMVUT_BINARY);
    else
        top_rbm.add_visible_group(m_layer_sizes->element(0), m_visible_units_type);

    top_rbm.initialize_neural_network();
    top_rbm.m_params = SGVector<float64_t>(
        m_params.vector+m_bias_index_offsets[m_num_layers-2],
        top_rbm.get_num_parameters(), false);

    top_rbm.cd_num_steps = cd_num_steps;
    top_rbm.cd_persistent = false;
    top_rbm.set_batch_size(gd_mini_batch_size);

    float64_t alpha = gd_learning_rate;

    int32_t counter = 0;
    for (int32_t i=0; i<max_num_epochs; i++)
    {
        for (int32_t j=0; j < training_set_size; j += gd_mini_batch_size)
        {
            alpha = gd_learning_rate_decay*alpha;

            if (j+gd_mini_batch_size>training_set_size)
                j = training_set_size-gd_mini_batch_size;

            SGMatrix<float64_t> inputs_batch(inputs.matrix+j*inputs.num_rows,
                inputs.num_rows, gd_mini_batch_size, false);

            for (int32_t k=0; k<m_num_params; k++)
            {
                m_params[k] += gd_momentum*param_updates[k];
                rec_params[k] += gd_momentum*rec_param_updates[k];
            }

            wake_sleep(inputs_batch, &top_rbm, sleep_states, wake_states,
                psleep_states, pwake_states, m_params,
                rec_params, gradients, rec_gradients);

            for (int32_t k=0; k<m_num_params; k++)
            {
                param_updates[k] = gd_momentum*param_updates[k]
                        -alpha*gradients[k];
                m_params[k] -= alpha*gradients[k];

                rec_param_updates[k] = gd_momentum*rec_param_updates[k]
                        -alpha*rec_gradients[k];
                rec_params[k] -= alpha*rec_gradients[k];
            }

            if (counter%monitoring_interval == 0)
            {
                SGMatrix<float64_t> reconstruction = sleep_states[0];
                float64_t error = 0;
                for (int32_t k=0; k<inputs_batch.num_rows*inputs_batch.num_cols; k++)
                    error += CMath::pow(reconstruction[k]-inputs_batch[k],2);

                error /= m_batch_size;

                SG_INFO("Epoch %i: reconstruction Error = %f\n",i, error);
            }
            counter++;
        }
    }
}

CDenseFeatures< float64_t >* CDeepBeliefNetwork::transform(
    CDenseFeatures< float64_t >* features, int32_t i)
{
    if (i==-1)
        i = m_num_layers-1;

    SGMatrix<float64_t> transformed_feature_matrix = features->get_feature_matrix();
    for (int32_t h=1; h<=i; h++)
    {
        SGMatrix<float64_t> m(m_layer_sizes->element(h), features->get_num_vectors());
        up_step(h, m_params, transformed_feature_matrix, m, false);
        transformed_feature_matrix = m;
    }

    return new CDenseFeatures<float64_t>(transformed_feature_matrix);
}

CDenseFeatures<float64_t>* CDeepBeliefNetwork::sample(
    int32_t num_gibbs_steps, int32_t batch_size)
{
    set_batch_size(batch_size);

    for (int32_t i=0; i<num_gibbs_steps; i++)
    {
        up_step(m_num_layers-1, m_params, m_states[m_num_layers-2],
            m_states[m_num_layers-1]);
        down_step(m_num_layers-2, m_params, m_states[m_num_layers-1],
            m_states[m_num_layers-2]);
    }

    for (int32_t i=m_num_layers-3; i>=0; i--)
        down_step(i, m_params, m_states[i+1], m_states[i]);

    return new CDenseFeatures<float64_t>(m_states[0]);
}

void CDeepBeliefNetwork::reset_chain()
{
    SGMatrix<float64_t> s = m_states[m_num_layers-2];

    for (int32_t i=0; i<s.num_rows*s.num_cols; i++)
        s[i] = CMath::random(0.0,1.0) > 0.5;
}

CNeuralNetwork* CDeepBeliefNetwork::convert_to_neural_network(
    CNeuralLayer* output_layer, float64_t sigma)
{
    CDynamicObjectArray* layers = new CDynamicObjectArray();

    layers->append_element(new CNeuralInputLayer(m_layer_sizes->element(0)));

    for (int32_t i=1; i<m_num_layers; i++)
        layers->append_element(new CNeuralLogisticLayer(m_layer_sizes->element(i)));

    if (output_layer!=NULL)
        layers->append_element(output_layer);

    CNeuralNetwork* network = new CNeuralNetwork(layers);

    network->quick_connect();
    network->initialize_neural_network(sigma);

    for (int32_t i=1; i<m_num_layers; i++)
    {
        SGMatrix<float64_t> W = get_weights(i-1);
        SGVector<float64_t> b = get_biases(i);

        for (int32_t j=0; j<b.vlen; j++)
            network->m_params[j+network->m_index_offsets[i]] = b[j];

        for (int32_t j=0; j<W.num_rows*W.num_cols; j++)
            network->m_params[j+network->m_index_offsets[i]+b.vlen] = W[j];
    }

    return network;
}

void CDeepBeliefNetwork::down_step(int32_t index, SGVector< float64_t > params,
    SGMatrix< float64_t > input, SGMatrix< float64_t > result, bool sample_states)
{
    typedef Eigen::Map<Eigen::MatrixXd> EMatrix;
    typedef Eigen::Map<Eigen::VectorXd> EVector;

    EMatrix In(input.matrix, input.num_rows, input.num_cols);
    EMatrix Out(result.matrix, result.num_rows, result.num_cols);
    EVector B(get_biases(index,params).vector, m_layer_sizes->element(index));

    Out.colwise() = B;

    if (index < m_num_layers-1)
    {
        EMatrix W(get_weights(index,params).matrix,
            m_layer_sizes->element(index+1), m_layer_sizes->element(index));
        Out += W.transpose()*In;
    }

    if (index > 0 || (index==0 && m_visible_units_type==RBMVUT_BINARY))
    {
        int32_t len = m_layer_sizes->element(index)*m_batch_size;
        for (int32_t i=0; i<len; i++)
                result[i] = 1.0/(1.0+CMath::exp(-1.0*result[i]));
    }

    if (index == 0 && m_visible_units_type==RBMVUT_SOFTMAX)
    {
        float64_t max = Out.maxCoeff();

        for (int32_t j=0; j<m_batch_size; j++)
        {
            float64_t sum = 0;
            for (int32_t i=0; i<m_layer_sizes->element(0); i++)
                sum += CMath::exp(Out(i,j)-max);

            float64_t normalizer = CMath::log(sum);
            for (int32_t k=0; k<m_layer_sizes->element(0); k++)
                Out(k,j) = CMath::exp(Out(k,j)-max-normalizer);
        }
    }

    if (sample_states && index>0)
    {
        int32_t len = m_layer_sizes->element(index)*m_batch_size;
        for (int32_t i=0; i<len; i++)
            result[i] = CMath::random(0.0,1.0) < result[i];
    }
}

void CDeepBeliefNetwork::up_step(int32_t index, SGVector< float64_t > params,
    SGMatrix< float64_t > input, SGMatrix< float64_t > result, bool sample_states)
{
    typedef Eigen::Map<Eigen::MatrixXd> EMatrix;
    typedef Eigen::Map<Eigen::VectorXd> EVector;

    EMatrix In(input.matrix, input.num_rows, input.num_cols);
    EMatrix Out(result.matrix, result.num_rows, result.num_cols);
    EVector C(get_biases(index, params).vector, m_layer_sizes->element(index));

    Out.colwise() = C;

    if (index>0)
    {
        EMatrix W(get_weights(index-1, params).matrix,
            m_layer_sizes->element(index), m_layer_sizes->element(index-1));
        Out += W*In;
    }

    int32_t len = result.num_rows*result.num_cols;
    for (int32_t i=0; i<len; i++)
        result[i] = 1.0/(1.0+CMath::exp(-1.0*result[i]));

    if (sample_states && index>0)
    {
        for (int32_t i=0; i<len; i++)
            result[i] = CMath::random(0.0,1.0) < result[i];
    }
}

void CDeepBeliefNetwork::wake_sleep(SGMatrix< float64_t > data, CRBM* top_rbm,
    SGMatrixList<float64_t> sleep_states, SGMatrixList<float64_t> wake_states,
    SGMatrixList<float64_t> psleep_states, SGMatrixList<float64_t> pwake_states,
    SGVector<float64_t> gen_params,
    SGVector<float64_t> rec_params,
    SGVector<float64_t> gen_gradients,
    SGVector<float64_t> rec_gradients)
{
    typedef Eigen::Map<Eigen::MatrixXd> EMatrix;
    typedef Eigen::Map<Eigen::VectorXd> EVector;

    // Wake phase
    for (int32_t i=0; i<data.num_rows*data.num_cols; i++)
        wake_states[0][i] = data[i];

    for (int32_t i=1; i<m_num_layers-1; i++)
        up_step(i, rec_params, wake_states[i-1], wake_states[i]);

    // Contrastive divergence in the top RBM
    SGVector<float64_t> top_rbm_gradients(
        gen_gradients.vector+m_bias_index_offsets[m_num_layers-2],
        top_rbm->get_num_parameters(), false);
    top_rbm->contrastive_divergence(wake_states[m_num_layers-2], top_rbm_gradients);

    // Sleep phase
    sleep_states.set_matrix(m_num_layers-2, top_rbm->visible_state);
    for (int32_t i=m_num_layers-3; i>=0; i--)
        down_step(i, gen_params, sleep_states[i+1], sleep_states[i]);

    // Predictions
    for (int32_t i=1; i<m_num_layers-1; i++)
        up_step(i, rec_params, sleep_states[i-1], psleep_states[i]);
    for (int32_t i=0; i<m_num_layers-2; i++)
        down_step(i, gen_params, wake_states[i+1], pwake_states[i]);

    // Gradients for generative parameters
    for (int32_t i=0; i<m_num_layers-2; i++)
    {
        EMatrix wake_i(wake_states[i].matrix,
            wake_states[i].num_rows, wake_states[i].num_cols);
        EMatrix wake_i_plus_one(wake_states[i+1].matrix,
            wake_states[i+1].num_rows, wake_states[i+1].num_cols);
        EMatrix pwake_i(pwake_states[i].matrix,
            pwake_states[i].num_rows, pwake_states[i].num_cols);

        EMatrix WG_gen(get_weights(i,gen_gradients).matrix,
            m_layer_sizes->element(i+1), m_layer_sizes->element(i));
        EVector BG_gen(get_biases(i,gen_gradients).vector, m_layer_sizes->element(i));

        pwake_i = pwake_i - wake_i;
        BG_gen = pwake_i.rowwise().sum()/m_batch_size;
        WG_gen = wake_i_plus_one*pwake_i.transpose()/m_batch_size;
    }

    // Gradients for reconstruction parameters
    for (int32_t i=1; i<m_num_layers-1; i++)
    {
        EMatrix sleep_i(sleep_states[i].matrix,
            sleep_states[i].num_rows, sleep_states[i].num_cols);
        EMatrix psleep_i(psleep_states[i].matrix,
            psleep_states[i].num_rows, psleep_states[i].num_cols);
        EMatrix sleep_i_minus_one(sleep_states[i-1].matrix,
            sleep_states[i-1].num_rows, sleep_states[i-1].num_cols);

        EMatrix WG_rec(get_weights(i-1,rec_gradients).matrix,
            m_layer_sizes->element(i), m_layer_sizes->element(i-1));
        EVector BG_rec(get_biases(i,rec_gradients).vector, m_layer_sizes->element(i));

        psleep_i = psleep_i - sleep_i;
        BG_rec = psleep_i.rowwise().sum()/m_batch_size;
        WG_rec = psleep_i*sleep_i_minus_one.transpose()/m_batch_size;
    }
}

SGMatrix< float64_t > CDeepBeliefNetwork::get_weights(int32_t index,
    SGVector< float64_t > p)
{
    if (p.vlen==0)
        return SGMatrix<float64_t>(m_params.vector+m_weights_index_offsets[index],
            m_layer_sizes->element(index+1), m_layer_sizes->element(index), false);
    else
        return SGMatrix<float64_t>(p.vector+m_weights_index_offsets[index],
            m_layer_sizes->element(index+1), m_layer_sizes->element(index), false);
}

SGVector< float64_t > CDeepBeliefNetwork::get_biases(int32_t index,
    SGVector< float64_t > p)
{
    if (p.vlen==0)
        return SGVector<float64_t>(m_params.vector+m_bias_index_offsets[index],
            m_layer_sizes->element(index), false);
    else
        return SGVector<float64_t>(p.vector+m_bias_index_offsets[index],
            m_layer_sizes->element(index), false);;
}

void CDeepBeliefNetwork::init()
{
    cd_num_steps = 1;
    monitoring_interval = 10;

    gd_mini_batch_size = 0;
    max_num_epochs = 1;
    gd_learning_rate = 0.1;
    gd_learning_rate_decay = 1.0;
    gd_momentum = 0.9;

    m_visible_units_type = RBMVUT_BINARY;
    m_num_layers = 0;
    m_layer_sizes = new CDynamicArray<int32_t>();
    m_batch_size = 0;
    m_num_params = 0;
    m_sigma = 0.01;

    SG_ADD((machine_int_t*)&m_visible_units_type, "visible_units_type",
        "Type of the visible units", MS_NOT_AVAILABLE);
    SG_ADD(&m_num_layers, "num_layers",
        "Number of layers", MS_NOT_AVAILABLE);
    SG_ADD((CSGObject**)&m_layer_sizes, "layer_sizes",
        "Size of each hidden layer", MS_NOT_AVAILABLE);

    SG_ADD(&m_params, "params",
        "Parameters of the network", MS_NOT_AVAILABLE);
    SG_ADD(&m_num_params, "num_params",
        "Number of parameters", MS_NOT_AVAILABLE);
    SG_ADD(&m_bias_index_offsets, "bias_index_offsets",
        "Index offsets of the biases", MS_NOT_AVAILABLE);
    SG_ADD(&m_weights_index_offsets, "weights_index_offsets",
        "Index offsets of the weights", MS_NOT_AVAILABLE);

    SG_ADD(&pt_cd_num_steps, "pt_cd_num_steps",
        "Pre-training Number of CD Steps", MS_NOT_AVAILABLE);
    SG_ADD(&pt_cd_persistent, "pt_cd_persistent",
        "Pre-training Persistent CD", MS_NOT_AVAILABLE);
    SG_ADD(&pt_cd_sample_visible, "pt_cd_sample_visible",
        "Pre-training Number of CD Sample Visible", MS_NOT_AVAILABLE);
    SG_ADD(&pt_l2_coefficient, "pt_l2_coefficient",
        "Pre-training L2 regularization coeff", MS_NOT_AVAILABLE);
    SG_ADD(&pt_l1_coefficient, "pt_l1_coefficient",
        "Pre-training L1 regularization coeff", MS_NOT_AVAILABLE);
    SG_ADD(&pt_monitoring_interval, "pt_monitoring_interval",
        "Pre-training Monitoring Interval", MS_NOT_AVAILABLE);
    SG_ADD(&pt_monitoring_method, "pt_monitoring_method",
        "Pre-training Monitoring Method", MS_NOT_AVAILABLE);
    SG_ADD(&pt_cd_num_steps, "pt_gd_mini_batch_size",
        "Pre-training Gradient Descent Mini-batch size", MS_NOT_AVAILABLE);
    SG_ADD(&pt_max_num_epochs, "pt_max_num_epochs",
        "Pre-training Max number of Epochs", MS_NOT_AVAILABLE);
    SG_ADD(&pt_gd_learning_rate, "pt_gd_learning_rate",
        "Pre-training Gradient descent learning rate", MS_NOT_AVAILABLE);
    SG_ADD(&pt_gd_learning_rate_decay, "pt_gd_learning_rate_decay",
        "Pre-training Gradient descent learning rate decay", MS_NOT_AVAILABLE);
    SG_ADD(&pt_gd_momentum, "pt_gd_momentum",
        "Pre-training Gradient Descent Momentum", MS_NOT_AVAILABLE);

    SG_ADD(&cd_num_steps, "cd_num_steps", "Number of CD Steps", MS_NOT_AVAILABLE);
    SG_ADD(&monitoring_interval, "monitoring_interval",
        "Monitoring Interval", MS_NOT_AVAILABLE);

    SG_ADD(&gd_mini_batch_size, "gd_mini_batch_size",
           "Gradient Descent Mini-batch size", MS_NOT_AVAILABLE);
    SG_ADD(&max_num_epochs, "max_num_epochs",
           "Max number of Epochs", MS_NOT_AVAILABLE);
    SG_ADD(&gd_learning_rate, "gd_learning_rate",
           "Gradient descent learning rate", MS_NOT_AVAILABLE);
    SG_ADD(&gd_learning_rate_decay, "gd_learning_rate_decay",
           "Gradient descent learning rate decay", MS_NOT_AVAILABLE);
    SG_ADD(&gd_momentum, "gd_momentum",
           "Gradient Descent Momentum", MS_NOT_AVAILABLE);

    SG_ADD(&m_sigma, "m_sigma", "Initialization Sigma", MS_NOT_AVAILABLE);
}