SHOGUN  3.2.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
StochasticGBMachine.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) The Shogun Machine Learning Toolbox
3  * Written (w) 2014 Parijat Mazumdar
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions are met:
8  *
9  * 1. Redistributions of source code must retain the above copyright notice, this
10  * list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright notice,
12  * this list of conditions and the following disclaimer in the documentation
13  * and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18  * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
19  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *
26  * The views and conclusions contained in the software and documentation are those
27  * of the authors and should not be interpreted as representing official policies,
28  * either expressed or implied, of the Shogun Development Team.
29  */
30 
34 
35 using namespace shogun;
36 
37 CStochasticGBMachine::CStochasticGBMachine(CMachine* machine, CLossFunction* loss, int32_t num_iterations,
38  float64_t learning_rate, float64_t subset_fraction)
39 : CMachine()
40 {
41  init();
42 
43  if (machine!=NULL)
44  {
45  SG_REF(machine);
46  m_machine=machine;
47  }
48 
49  if (loss!=NULL)
50  {
51  SG_REF(loss);
52  m_loss=loss;
53  }
54 
55  m_num_iter=num_iterations;
56  m_subset_frac=subset_fraction;
57  m_learning_rate=learning_rate;
58 }
59 
61 {
66 }
67 
69 {
70  REQUIRE(machine,"Supplied machine is NULL\n")
71 
72  if (m_machine!=NULL)
74 
75  SG_REF(machine);
76  m_machine=machine;
77 }
78 
80 {
81  if (m_machine==NULL)
82  SG_ERROR("machine not set yet!\n")
83 
85  return m_machine;
86 }
87 
89 {
90  REQUIRE(f,"Supplied loss function is NULL\n")
91  if (m_loss!=NULL)
93 
94  SG_REF(f);
95  m_loss=f;
96 }
97 
99 {
100  if (m_loss==NULL)
101  SG_ERROR("Loss function not set yet!\n")
102 
103  SG_REF(m_loss)
104  return m_loss;
105 }
106 
108 {
109  REQUIRE(iter,"Number of iterations\n")
110  m_num_iter=iter;
111 }
112 
114 {
115  return m_num_iter;
116 }
117 
119 {
120  REQUIRE((frac>0)&&(frac<=1),"subset fraction should lie between 0 and 1. Supplied value is %f\n",frac)
121 
122  m_subset_frac=frac;
123 }
124 
126 {
127  return m_subset_frac;
128 }
129 
131 {
132  REQUIRE((lr>0)&&(lr<=1),"learning rate should lie between 0 and 1. Supplied value is %f\n",lr)
133 
134  m_learning_rate=lr;
135 }
136 
138 {
139  return m_learning_rate;
140 }
141 
143 {
144  REQUIRE(data,"test data supplied is NULL\n")
146 
147  SGVector<float64_t> retlabs(feats->get_num_vectors());
148  retlabs.fill_vector(retlabs.vector,retlabs.vlen,0);
149  for (int32_t i=0;i<m_num_iter;i++)
150  {
151  float64_t gamma=m_gamma->get_element(i);
152 
154  REQUIRE(element,"%d element of the array of weak learners is NULL. This is not expected\n",i)
155  CMachine* machine=dynamic_cast<CMachine*>(element);
156 
157  CRegressionLabels* dlabels=machine->apply_regression(feats);
159 
160  for (int32_t j=0;j<retlabs.vlen;j++)
161  retlabs[j]+=delta[j]*gamma*m_learning_rate;
162 
163  SG_UNREF(dlabels);
164  SG_UNREF(element);
165  }
166 
167  return new CRegressionLabels(retlabs);
168 }
169 
171 {
172  REQUIRE(data,"training data not supplied!\n")
173  REQUIRE(m_machine,"machine not set!\n")
174  REQUIRE(m_loss,"loss function not specified\n")
175 
177 
178  // initialize weak learners array and gamma array
180 
181  // cache predicted labels for intermediate models
183  SG_REF(interf);
184  for (int32_t i=0;i<interf->get_num_labels();i++)
185  interf->set_label(i,0);
186 
187  for (int32_t i=0;i<m_num_iter;i++)
188  {
189  // apply subset
190  if (m_subset_frac!=1.0)
191  apply_subset(feats,interf);
192 
193  // compute pseudo-residuals
195 
196  // fit learner
197  CMachine* wlearner=fit_model(feats,pres);
198  m_weak_learners->push_back(wlearner);
199 
200  // compute multiplier
201  CRegressionLabels* hm=wlearner->apply_regression(feats);
202  SG_REF(hm);
203  float64_t gamma=compute_multiplier(interf,hm);
204  m_gamma->push_back(gamma);
205 
206  // remove subset
207  if (m_subset_frac!=1.0)
208  {
209  feats->remove_subset();
211  interf->remove_subset();
212  }
213 
214  // update intermediate function value
215  CRegressionLabels* dlabels=wlearner->apply_regression(feats);
217  for (int32_t j=0;j<interf->get_num_labels();j++)
218  interf->set_label(j,interf->get_label(j)+delta[j]*gamma*m_learning_rate);
219 
220  SG_UNREF(dlabels);
221  SG_UNREF(hm);
222  SG_UNREF(wlearner);
223  }
224 
225  SG_UNREF(interf);
226  return true;
227 }
228 
230 {
231  REQUIRE(f->get_num_labels()==hm->get_num_labels(),"The number of labels in both input parameters should be equal\n")
232 
234  instance->push_back(m_labels);
235  instance->push_back(f);
236  instance->push_back(hm);
237  instance->push_back(m_loss);
238 
239  float64_t ret=get_gamma(instance);
240 
241  SG_UNREF(instance);
242  return ret;
243 }
244 
246 {
247  // clone base machine
248  CSGObject* obj=m_machine->clone();
249  CMachine* c=NULL;
250  if (obj)
251  c=dynamic_cast<CMachine*>(obj);
252  else
253  SG_ERROR("Machine could not be cloned!\n")
254 
255  // train cloned machine
256  c->set_labels(labels);
257  c->train(feats);
258 
259  return c;
260 }
261 
263 {
264  REQUIRE(m_labels,"training labels not set!\n")
265  SGVector<float64_t> labels=(dynamic_cast<CDenseLabels*>(m_labels))->get_labels();
266  SGVector<float64_t> f=inter_f->get_labels();
267 
268  SGVector<float64_t> residuals(f.vlen);
269  for (int32_t i=0;i<residuals.vlen;i++)
270  residuals[i]=-m_loss->first_derivative(f[i],labels[i]);
271 
272  return new CRegressionLabels(residuals);
273 }
274 
276 {
277  int32_t subset_size=m_subset_frac*(f->get_num_vectors());
279  idx.range_fill();
280  CMath::permute(idx);
281 
282  SGVector<index_t> subset(subset_size);
283  memcpy(subset.vector,idx.vector,subset.vlen*sizeof(index_t));
284 
285  f->add_subset(subset);
286  interf->add_subset(subset);
287  m_labels->add_subset(subset);
288 }
289 
291 {
295 
296  SG_UNREF(m_gamma);
298  SG_REF(m_gamma);
299 }
300 
302 {
303  lbfgs_parameter_t lbfgs_param;
304  lbfgs_parameter_init(&lbfgs_param);
305  lbfgs_param.linesearch=2;
306 
307  float64_t gamma=0;
308  lbfgs(1,&gamma,NULL,CStochasticGBMachine::lbfgs_evaluate,NULL,instance,&lbfgs_param);
309 
310  return gamma;
311 }
312 
313 float64_t CStochasticGBMachine::lbfgs_evaluate(void *obj, const float64_t *parameters, float64_t *gradient, const int dim,
314  const float64_t step)
315 {
316  REQUIRE(obj,"object cannot be NULL\n")
317  CDynamicObjectArray* objects=static_cast<CDynamicObjectArray*>(obj);
318  REQUIRE((objects->get_num_elements()==2) || (objects->get_num_elements()==4),"Number of elements in obj array"
319  " (%d) does not match expectations(2 or 4)\n",objects->get_num_elements())
320 
321  if (objects->get_num_elements()==2)
322  {
323  // extract labels
324  CSGObject* element=objects->get_element(0);
325  REQUIRE(element,"0 index element of objects is NULL\n")
326  CDenseLabels* lab=dynamic_cast<CDenseLabels*>(element);
327  SGVector<float64_t> labels=lab->get_labels();
328 
329  // extract loss function
330  element=objects->get_element(1);
331  REQUIRE(element,"1 index element of objects is NULL\n")
332  CLossFunction* lossf=dynamic_cast<CLossFunction*>(element);
333 
334  *gradient=0;
335  float64_t ret=0;
336  for (int32_t i=0;i<labels.vlen;i++)
337  {
338  *gradient+=lossf->first_derivative((*parameters),labels[i]);
339  ret+=lossf->loss((*parameters),labels[i]);
340  }
341 
342  SG_UNREF(lab);
343  SG_UNREF(lossf);
344  return ret;
345  }
346 
347  // extract labels
348  CSGObject* element=objects->get_element(0);
349  REQUIRE(element,"0 index element of objects is NULL\n")
350  CDenseLabels* lab=dynamic_cast<CDenseLabels*>(element);
351  SGVector<float64_t> labels=lab->get_labels();
352 
353  // extract f
354  element=objects->get_element(1);
355  REQUIRE(element,"1 index element of objects is NULL\n")
356  CDenseLabels* func=dynamic_cast<CDenseLabels*>(element);
358 
359  // extract hm
360  element=objects->get_element(2);
361  REQUIRE(element,"2 index element of objects is NULL\n")
362  CDenseLabels* delta=dynamic_cast<CDenseLabels*>(element);
363  SGVector<float64_t> hm=delta->get_labels();
364 
365  // extract loss function
366  element=objects->get_element(3);
367  REQUIRE(element,"3 index element of objects is NULL\n")
368  CLossFunction* lossf=dynamic_cast<CLossFunction*>(element);
369 
370  *gradient=0;
371  float64_t ret=0;
372  for (int32_t i=0;i<labels.vlen;i++)
373  {
374  *gradient+=lossf->first_derivative((*parameters)*hm[i]+f[i],labels[i]);
375  ret+=lossf->loss((*parameters)*hm[i]+f[i],labels[i]);
376  }
377 
378  SG_UNREF(lab);
379  SG_UNREF(delta);
380  SG_UNREF(func);
381  SG_UNREF(lossf)
382  return ret;
383 }
384 
386 {
387  m_machine=NULL;
388  m_loss=NULL;
389  m_num_iter=0;
390  m_subset_frac=0;
391  m_learning_rate=0;
392 
395 
397  SG_REF(m_gamma);
398 
399  SG_ADD((CSGObject**)&m_machine,"m_machine","machine",MS_NOT_AVAILABLE);
400  SG_ADD((CSGObject**)&m_loss,"m_loss","loss function",MS_NOT_AVAILABLE);
401  SG_ADD(&m_num_iter,"m_num_iter","number of iterations",MS_NOT_AVAILABLE);
402  SG_ADD(&m_subset_frac,"m_subset_frac","subset fraction",MS_NOT_AVAILABLE);
403  SG_ADD(&m_learning_rate,"m_learning_rate","learning rate",MS_NOT_AVAILABLE);
404  SG_ADD((CSGObject**)&m_weak_learners,"m_weak_learners","array of weak learners",MS_NOT_AVAILABLE);
405  SG_ADD((CSGObject**)&m_gamma,"m_gamma","array of learner weights",MS_NOT_AVAILABLE);
406 }

SHOGUN Machine Learning Toolbox - Documentation