SHOGUN  4.1.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
StochasticGBMachine.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) The Shogun Machine Learning Toolbox
3  * Written (w) 2014 Parijat Mazumdar
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions are met:
8  *
9  * 1. Redistributions of source code must retain the above copyright notice, this
10  * list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright notice,
12  * this list of conditions and the following disclaimer in the documentation
13  * and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18  * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
19  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *
26  * The views and conclusions contained in the software and documentation are those
27  * of the authors and should not be interpreted as representing official policies,
28  * either expressed or implied, of the Shogun Development Team.
29  */
30 
34 
35 using namespace shogun;
36 
37 CStochasticGBMachine::CStochasticGBMachine(CMachine* machine, CLossFunction* loss, int32_t num_iterations,
38  float64_t learning_rate, float64_t subset_fraction)
39 : CMachine()
40 {
41  init();
42 
43  if (machine!=NULL)
44  {
45  SG_REF(machine);
46  m_machine=machine;
47  }
48 
49  if (loss!=NULL)
50  {
51  SG_REF(loss);
52  m_loss=loss;
53  }
54 
55  m_num_iter=num_iterations;
56  m_subset_frac=subset_fraction;
57  m_learning_rate=learning_rate;
58 }
59 
61 {
66 }
67 
69 {
70  REQUIRE(machine,"Supplied machine is NULL\n")
71 
72  if (m_machine!=NULL)
74 
75  SG_REF(machine);
76  m_machine=machine;
77 }
78 
80 {
81  if (m_machine==NULL)
82  SG_ERROR("machine not set yet!\n")
83 
85  return m_machine;
86 }
87 
89 {
90  REQUIRE(f,"Supplied loss function is NULL\n")
91  if (m_loss!=NULL)
93 
94  SG_REF(f);
95  m_loss=f;
96 }
97 
99 {
100  if (m_loss==NULL)
101  SG_ERROR("Loss function not set yet!\n")
102 
103  SG_REF(m_loss)
104  return m_loss;
105 }
106 
108 {
109  REQUIRE(iter,"Number of iterations\n")
110  m_num_iter=iter;
111 }
112 
114 {
115  return m_num_iter;
116 }
117 
119 {
120  REQUIRE((frac>0)&&(frac<=1),"subset fraction should lie between 0 and 1. Supplied value is %f\n",frac)
121 
122  m_subset_frac=frac;
123 }
124 
126 {
127  return m_subset_frac;
128 }
129 
131 {
132  REQUIRE((lr>0)&&(lr<=1),"learning rate should lie between 0 and 1. Supplied value is %f\n",lr)
133 
134  m_learning_rate=lr;
135 }
136 
138 {
139  return m_learning_rate;
140 }
141 
143 {
144  REQUIRE(data,"test data supplied is NULL\n")
146 
147  SGVector<float64_t> retlabs(feats->get_num_vectors());
148  retlabs.fill_vector(retlabs.vector,retlabs.vlen,0);
149  for (int32_t i=0;i<m_num_iter;i++)
150  {
151  float64_t gamma=m_gamma->get_element(i);
152 
154  REQUIRE(element,"%d element of the array of weak learners is NULL. This is not expected\n",i)
155  CMachine* machine=dynamic_cast<CMachine*>(element);
156 
157  CRegressionLabels* dlabels=machine->apply_regression(feats);
159 
160  for (int32_t j=0;j<retlabs.vlen;j++)
161  retlabs[j]+=delta[j]*gamma*m_learning_rate;
162 
163  SG_UNREF(dlabels);
164  SG_UNREF(element);
165  }
166 
167  return new CRegressionLabels(retlabs);
168 }
169 
171 {
172  REQUIRE(data,"training data not supplied!\n")
173  REQUIRE(m_machine,"machine not set!\n")
174  REQUIRE(m_loss,"loss function not specified\n")
175 
177 
178  // initialize weak learners array and gamma array
180 
181  // cache predicted labels for intermediate models
183  SG_REF(interf);
184  for (int32_t i=0;i<interf->get_num_labels();i++)
185  interf->set_label(i,0);
186 
187  for (int32_t i=0;i<m_num_iter;i++)
188  {
189  // apply subset
190  if (m_subset_frac!=1.0)
191  apply_subset(feats,interf);
192 
193  // compute pseudo-residuals
195 
196  // fit learner
197  CMachine* wlearner=fit_model(feats,pres);
198  m_weak_learners->push_back(wlearner);
199 
200  // compute multiplier
201  CRegressionLabels* hm=wlearner->apply_regression(feats);
202  SG_REF(hm);
203  float64_t gamma=compute_multiplier(interf,hm);
204  m_gamma->push_back(gamma);
205 
206  // remove subset
207  if (m_subset_frac!=1.0)
208  {
209  feats->remove_subset();
211  interf->remove_subset();
212  }
213 
214  // update intermediate function value
215  CRegressionLabels* dlabels=wlearner->apply_regression(feats);
217  for (int32_t j=0;j<interf->get_num_labels();j++)
218  interf->set_label(j,interf->get_label(j)+delta[j]*gamma*m_learning_rate);
219 
220  SG_UNREF(dlabels);
221  SG_UNREF(hm);
222  SG_UNREF(wlearner);
223  }
224 
225  SG_UNREF(interf);
226  return true;
227 }
228 
230 {
231  REQUIRE(f->get_num_labels()==hm->get_num_labels(),"The number of labels in both input parameters should be equal\n")
232 
234  instance->push_back(m_labels);
235  instance->push_back(f);
236  instance->push_back(hm);
237  instance->push_back(m_loss);
238 
239  float64_t ret=get_gamma(instance);
240 
241  SG_UNREF(instance);
242  return ret;
243 }
244 
246 {
247  // clone base machine
248  CSGObject* obj=m_machine->clone();
249  CMachine* c=NULL;
250  if (obj)
251  c=dynamic_cast<CMachine*>(obj);
252  else
253  SG_ERROR("Machine could not be cloned!\n")
254 
255  // train cloned machine
256  c->set_labels(labels);
257  c->train(feats);
258 
259  return c;
260 }
261 
263 {
264  REQUIRE(m_labels,"training labels not set!\n")
265  SGVector<float64_t> labels=(dynamic_cast<CDenseLabels*>(m_labels))->get_labels();
266  SGVector<float64_t> f=inter_f->get_labels();
267 
268  SGVector<float64_t> residuals(f.vlen);
269  for (int32_t i=0;i<residuals.vlen;i++)
270  residuals[i]=-m_loss->first_derivative(f[i],labels[i]);
271 
272  return new CRegressionLabels(residuals);
273 }
274 
276 {
277  int32_t subset_size=m_subset_frac*(f->get_num_vectors());
279  idx.range_fill();
280  CMath::permute(idx);
281 
282  SGVector<index_t> subset(subset_size);
283  memcpy(subset.vector,idx.vector,subset.vlen*sizeof(index_t));
284 
285  f->add_subset(subset);
286  interf->add_subset(subset);
287  m_labels->add_subset(subset);
288 }
289 
291 {
295 
296  SG_UNREF(m_gamma);
298  SG_REF(m_gamma);
299 }
300 
302 {
303  lbfgs_parameter_t lbfgs_param;
304  lbfgs_parameter_init(&lbfgs_param);
305  lbfgs_param.linesearch=2;
306 
307  float64_t gamma=0;
308  lbfgs(1,&gamma,NULL,CStochasticGBMachine::lbfgs_evaluate,NULL,instance,&lbfgs_param);
309 
310  return gamma;
311 }
312 
313 float64_t CStochasticGBMachine::lbfgs_evaluate(void *obj, const float64_t *parameters, float64_t *gradient, const int dim,
314  const float64_t step)
315 {
316  REQUIRE(obj,"object cannot be NULL\n")
317  CDynamicObjectArray* objects=static_cast<CDynamicObjectArray*>(obj);
318  REQUIRE((objects->get_num_elements()==2) || (objects->get_num_elements()==4),"Number of elements in obj array"
319  " (%d) does not match expectations(2 or 4)\n",objects->get_num_elements())
320 
321  if (objects->get_num_elements()==2)
322  {
323  // extract labels
324  CSGObject* element=objects->get_element(0);
325  REQUIRE(element,"0 index element of objects is NULL\n")
326  CDenseLabels* lab=dynamic_cast<CDenseLabels*>(element);
327  SGVector<float64_t> labels=lab->get_labels();
328 
329  // extract loss function
330  element=objects->get_element(1);
331  REQUIRE(element,"1 index element of objects is NULL\n")
332  CLossFunction* lossf=dynamic_cast<CLossFunction*>(element);
333 
334  *gradient=0;
335  float64_t ret=0;
336  for (int32_t i=0;i<labels.vlen;i++)
337  {
338  *gradient+=lossf->first_derivative((*parameters),labels[i]);
339  ret+=lossf->loss((*parameters),labels[i]);
340  }
341 
342  SG_UNREF(lab);
343  SG_UNREF(lossf);
344  return ret;
345  }
346 
347  // extract labels
348  CSGObject* element=objects->get_element(0);
349  REQUIRE(element,"0 index element of objects is NULL\n")
350  CDenseLabels* lab=dynamic_cast<CDenseLabels*>(element);
351  SGVector<float64_t> labels=lab->get_labels();
352 
353  // extract f
354  element=objects->get_element(1);
355  REQUIRE(element,"1 index element of objects is NULL\n")
356  CDenseLabels* func=dynamic_cast<CDenseLabels*>(element);
358 
359  // extract hm
360  element=objects->get_element(2);
361  REQUIRE(element,"2 index element of objects is NULL\n")
362  CDenseLabels* delta=dynamic_cast<CDenseLabels*>(element);
363  SGVector<float64_t> hm=delta->get_labels();
364 
365  // extract loss function
366  element=objects->get_element(3);
367  REQUIRE(element,"3 index element of objects is NULL\n")
368  CLossFunction* lossf=dynamic_cast<CLossFunction*>(element);
369 
370  *gradient=0;
371  float64_t ret=0;
372  for (int32_t i=0;i<labels.vlen;i++)
373  {
374  *gradient+=lossf->first_derivative((*parameters)*hm[i]+f[i],labels[i]);
375  ret+=lossf->loss((*parameters)*hm[i]+f[i],labels[i]);
376  }
377 
378  SG_UNREF(lab);
379  SG_UNREF(delta);
380  SG_UNREF(func);
381  SG_UNREF(lossf)
382  return ret;
383 }
384 
386 {
387  m_machine=NULL;
388  m_loss=NULL;
389  m_num_iter=0;
390  m_subset_frac=0;
391  m_learning_rate=0;
392 
395 
397  SG_REF(m_gamma);
398 
399  SG_ADD((CSGObject**)&m_machine,"m_machine","machine",MS_NOT_AVAILABLE);
400  SG_ADD((CSGObject**)&m_loss,"m_loss","loss function",MS_NOT_AVAILABLE);
401  SG_ADD(&m_num_iter,"m_num_iter","number of iterations",MS_NOT_AVAILABLE);
402  SG_ADD(&m_subset_frac,"m_subset_frac","subset fraction",MS_NOT_AVAILABLE);
403  SG_ADD(&m_learning_rate,"m_learning_rate","learning rate",MS_NOT_AVAILABLE);
404  SG_ADD((CSGObject**)&m_weak_learners,"m_weak_learners","array of weak learners",MS_NOT_AVAILABLE);
405  SG_ADD((CSGObject**)&m_gamma,"m_gamma","array of learner weights",MS_NOT_AVAILABLE);
406 }
void range_fill(T start=0)
Definition: SGVector.cpp:173
static void permute(SGVector< T > v, CRandom *rand=NULL)
Definition: Math.h:1144
virtual CRegressionLabels * apply_regression(CFeatures *data=NULL)
static void fill_vector(T *vec, int32_t len, T value)
Definition: SGVector.cpp:223
Real Labels are real-valued labels.
Class CLossFunction is the base class of all loss functions.
Definition: LossFunction.h:57
int32_t lbfgs(int32_t n, float64_t *x, float64_t *ptr_fx, lbfgs_evaluate_t proc_evaluate, lbfgs_progress_t proc_progress, void *instance, lbfgs_parameter_t *_param, lbfgs_adjust_step_t proc_adjust_step)
Definition: lbfgs.cpp:208
virtual int32_t get_num_labels() const
int32_t index_t
Definition: common.h:62
The class Labels models labels, i.e. class assignments of objects.
Definition: Labels.h:43
virtual CSGObject * clone()
Definition: SGObject.cpp:1361
CLabels * m_labels
Definition: Machine.h:361
#define SG_ERROR(...)
Definition: SGIO.h:129
#define REQUIRE(x,...)
Definition: SGIO.h:206
virtual void set_loss_function(CLossFunction *f)
float64_t get_label(int32_t idx)
#define SG_REF(x)
Definition: SGObject.h:51
A generic learning machine interface.
Definition: Machine.h:143
bool set_label(int32_t idx, float64_t label)
SGVector< float64_t > get_labels()
Definition: DenseLabels.cpp:82
CRegressionLabels * compute_pseudo_residuals(CRegressionLabels *inter_f)
index_t vlen
Definition: SGVector.h:494
Class SGObject is the base class of all shogun objects.
Definition: SGObject.h:112
virtual int32_t get_num_vectors() const
double float64_t
Definition: common.h:50
virtual CRegressionLabels * apply_regression(CFeatures *data=NULL)
Definition: Machine.cpp:222
static float64_t lbfgs_evaluate(void *obj, const float64_t *parameters, float64_t *gradient, const int dim, const float64_t step)
float64_t compute_multiplier(CRegressionLabels *f, CRegressionLabels *hm)
virtual void remove_subset()
Definition: Labels.cpp:49
void set_machine(CMachine *machine)
virtual float64_t loss(float64_t prediction, float64_t label)
Definition: LossFunction.h:79
CMachine * fit_model(CDenseFeatures< float64_t > *feats, CRegressionLabels *labels)
virtual void add_subset(SGVector< index_t > subset)
Definition: Labels.cpp:39
Dynamic array class for CSGObject pointers that creates an array that can be used like a list or an a...
void set_subset_fraction(float64_t frac)
#define SG_UNREF(x)
Definition: SGObject.h:52
all of classes and functions are contained in the shogun namespace
Definition: class_list.h:18
CStochasticGBMachine(CMachine *machine=NULL, CLossFunction *loss=NULL, int32_t num_iterations=100, float64_t learning_rate=1.0, float64_t subset_fraction=0.6)
CDynamicArray< float64_t > * m_gamma
virtual void remove_subset()
Definition: Features.cpp:322
The class Features is the base class of all feature objects.
Definition: Features.h:68
virtual bool train(CFeatures *data=NULL)
Definition: Machine.cpp:47
static CDenseFeatures * obtain_from_generic(CFeatures *const base_features)
float64_t get_gamma(void *instance)
CSGObject * get_element(int32_t index) const
virtual bool train_machine(CFeatures *data=NULL)
#define SG_ADD(...)
Definition: SGObject.h:81
virtual float64_t first_derivative(float64_t prediction, float64_t label)
Definition: LossFunction.h:101
Dense integer or floating point labels.
Definition: DenseLabels.h:35
virtual void set_labels(CLabels *lab)
Definition: Machine.cpp:73
void apply_subset(CDenseFeatures< float64_t > *f, CLabels *interf)
#define delta
Definition: sfa.cpp:23
virtual CLossFunction * get_loss_function() const
const T & get_element(int32_t idx1, int32_t idx2=0, int32_t idx3=0) const
Definition: DynamicArray.h:212
virtual void add_subset(SGVector< index_t > subset)
Definition: Features.cpp:310
CDynamicObjectArray * m_weak_learners
void lbfgs_parameter_init(lbfgs_parameter_t *param)
Definition: lbfgs.cpp:203

SHOGUN Machine Learning Toolbox - Documentation