SHOGUN  3.2.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
StochasticGBMachine.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) The Shogun Machine Learning Toolbox
3  * Written (w) 2014 Parijat Mazumdar
4  * All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions are met:
8  *
9  * 1. Redistributions of source code must retain the above copyright notice, this
10  * list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright notice,
12  * this list of conditions and the following disclaimer in the documentation
13  * and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18  * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
19  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
24  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  *
26  * The views and conclusions contained in the software and documentation are those
27  * of the authors and should not be interpreted as representing official policies,
28  * either expressed or implied, of the Shogun Development Team.
29  */
30 
33 
34 using namespace shogun;
35 
36 CStochasticGBMachine::CStochasticGBMachine(CMachine* machine, CLossFunction* loss, int32_t num_iterations,
37  float64_t learning_rate, float64_t subset_fraction)
38 : CMachine()
39 {
40  init();
41 
42  if (machine!=NULL)
43  {
44  SG_REF(machine);
45  m_machine=machine;
46  }
47 
48  if (loss!=NULL)
49  {
50  SG_REF(loss);
51  m_loss=loss;
52  }
53 
54  m_num_iter=num_iterations;
55  m_subset_frac=subset_fraction;
56  m_learning_rate=learning_rate;
57 }
58 
60 {
65 }
66 
68 {
69  REQUIRE(machine,"Supplied machine is NULL\n")
70 
71  if (m_machine!=NULL)
73 
74  SG_REF(machine);
75  m_machine=machine;
76 }
77 
79 {
80  if (m_machine==NULL)
81  SG_ERROR("machine not set yet!\n")
82 
84  return m_machine;
85 }
86 
88 {
89  REQUIRE(f,"Supplied loss function is NULL\n")
90  if (m_loss!=NULL)
92 
93  SG_REF(f);
94  m_loss=f;
95 }
96 
98 {
99  if (m_loss==NULL)
100  SG_ERROR("Loss function not set yet!\n")
101 
102  SG_REF(m_loss)
103  return m_loss;
104 }
105 
107 {
108  REQUIRE(iter,"Number of iterations\n")
109  m_num_iter=iter;
110 }
111 
113 {
114  return m_num_iter;
115 }
116 
118 {
119  REQUIRE((frac>0)&&(frac<=1),"subset fraction should lie between 0 and 1. Supplied value is %f\n",frac)
120 
121  m_subset_frac=frac;
122 }
123 
125 {
126  return m_subset_frac;
127 }
128 
130 {
131  REQUIRE((lr>0)&&(lr<=1),"learning rate should lie between 0 and 1. Supplied value is %f\n",lr)
132 
133  m_learning_rate=lr;
134 }
135 
137 {
138  return m_learning_rate;
139 }
140 
142 {
143  REQUIRE(data,"test data supplied is NULL\n")
145 
146  SGVector<float64_t> retlabs(feats->get_num_vectors());
147  retlabs.fill_vector(retlabs.vector,retlabs.vlen,0);
148  for (int32_t i=0;i<m_num_iter;i++)
149  {
150  float64_t gamma=m_gamma->get_element(i);
151 
153  REQUIRE(element,"%d element of the array of weak learners is NULL. This is not expected\n",i)
154  CMachine* machine=dynamic_cast<CMachine*>(element);
155 
156  CRegressionLabels* dlabels=machine->apply_regression(feats);
158 
159  for (int32_t j=0;j<retlabs.vlen;j++)
160  retlabs[j]+=delta[j]*gamma*m_learning_rate;
161 
162  SG_UNREF(dlabels);
163  SG_UNREF(element);
164  }
165 
166  return new CRegressionLabels(retlabs);
167 }
168 
170 {
171  REQUIRE(data,"training data not supplied!\n")
172  REQUIRE(m_machine,"machine not set!\n")
173  REQUIRE(m_loss,"loss function not specified\n")
174 
176 
177  // initialize weak learners array and gamma array
179 
180  // cache predicted labels for intermediate models
182  SG_REF(interf);
183  for (int32_t i=0;i<interf->get_num_labels();i++)
184  interf->set_label(i,0);
185 
186  for (int32_t i=0;i<m_num_iter;i++)
187  {
188  // apply subset
189  if (m_subset_frac!=1.0)
190  apply_subset(feats,interf);
191 
192  // compute pseudo-residuals
194 
195  // fit learner
196  CMachine* wlearner=fit_model(feats,pres);
197  m_weak_learners->push_back(wlearner);
198 
199  // compute multiplier
200  CRegressionLabels* hm=wlearner->apply_regression(feats);
201  SG_REF(hm);
202  float64_t gamma=compute_multiplier(interf,hm);
203  m_gamma->push_back(gamma);
204 
205  // remove subset
206  if (m_subset_frac!=1.0)
207  {
208  feats->remove_subset();
210  interf->remove_subset();
211  }
212 
213  // update intermediate function value
214  CRegressionLabels* dlabels=wlearner->apply_regression(feats);
216  for (int32_t j=0;j<interf->get_num_labels();j++)
217  interf->set_label(j,interf->get_label(j)+delta[j]*gamma*m_learning_rate);
218 
219  SG_UNREF(dlabels);
220  SG_UNREF(hm);
221  SG_UNREF(wlearner);
222  }
223 
224  SG_UNREF(interf);
225  return true;
226 }
227 
229 {
230  REQUIRE(f->get_num_labels()==hm->get_num_labels(),"The number of labels in both input parameters should be equal\n")
231 
233  instance->push_back(m_labels);
234  instance->push_back(f);
235  instance->push_back(hm);
236  instance->push_back(m_loss);
237 
238  float64_t ret=get_gamma(instance);
239 
240  SG_UNREF(instance);
241  return ret;
242 }
243 
245 {
246  // clone base machine
247  CSGObject* obj=m_machine->clone();
248  CMachine* c=NULL;
249  if (obj)
250  c=dynamic_cast<CMachine*>(obj);
251  else
252  SG_ERROR("Machine could not be cloned!\n")
253 
254  // train cloned machine
255  c->set_labels(labels);
256  c->train(feats);
257 
258  return c;
259 }
260 
262 {
263  REQUIRE(m_labels,"training labels not set!\n")
264  SGVector<float64_t> labels=(dynamic_cast<CDenseLabels*>(m_labels))->get_labels();
265  SGVector<float64_t> f=inter_f->get_labels();
266 
267  SGVector<float64_t> residuals(f.vlen);
268  for (int32_t i=0;i<residuals.vlen;i++)
269  residuals[i]=-m_loss->first_derivative(f[i],labels[i]);
270 
271  return new CRegressionLabels(residuals);
272 }
273 
275 {
276  int32_t subset_size=m_subset_frac*(f->get_num_vectors());
278  idx.range_fill(0);
279  idx.randperm();
280 
281  SGVector<index_t> subset(subset_size);
282  memcpy(subset.vector,idx.vector,subset.vlen*sizeof(index_t));
283 
284  f->add_subset(subset);
285  interf->add_subset(subset);
286  m_labels->add_subset(subset);
287 }
288 
290 {
294 
295  SG_UNREF(m_gamma);
297  SG_REF(m_gamma);
298 }
299 
301 {
302  lbfgs_parameter_t lbfgs_param;
303  lbfgs_parameter_init(&lbfgs_param);
304  lbfgs_param.linesearch=2;
305 
306  float64_t gamma=0;
307  lbfgs(1,&gamma,NULL,CStochasticGBMachine::lbfgs_evaluate,NULL,instance,&lbfgs_param);
308 
309  return gamma;
310 }
311 
312 float64_t CStochasticGBMachine::lbfgs_evaluate(void *obj, const float64_t *parameters, float64_t *gradient, const int dim,
313  const float64_t step)
314 {
315  REQUIRE(obj,"object cannot be NULL\n")
316  CDynamicObjectArray* objects=static_cast<CDynamicObjectArray*>(obj);
317  REQUIRE((objects->get_num_elements()==2) || (objects->get_num_elements()==4),"Number of elements in obj array"
318  " (%d) does not match expectations(2 or 4)\n",objects->get_num_elements())
319 
320  if (objects->get_num_elements()==2)
321  {
322  // extract labels
323  CSGObject* element=objects->get_element(0);
324  REQUIRE(element,"0 index element of objects is NULL\n")
325  CDenseLabels* lab=dynamic_cast<CDenseLabels*>(element);
326  SGVector<float64_t> labels=lab->get_labels();
327 
328  // extract loss function
329  element=objects->get_element(1);
330  REQUIRE(element,"1 index element of objects is NULL\n")
331  CLossFunction* lossf=dynamic_cast<CLossFunction*>(element);
332 
333  *gradient=0;
334  float64_t ret=0;
335  for (int32_t i=0;i<labels.vlen;i++)
336  {
337  *gradient+=lossf->first_derivative((*parameters),labels[i]);
338  ret+=lossf->loss((*parameters),labels[i]);
339  }
340 
341  SG_UNREF(lab);
342  SG_UNREF(lossf);
343  return ret;
344  }
345 
346  // extract labels
347  CSGObject* element=objects->get_element(0);
348  REQUIRE(element,"0 index element of objects is NULL\n")
349  CDenseLabels* lab=dynamic_cast<CDenseLabels*>(element);
350  SGVector<float64_t> labels=lab->get_labels();
351 
352  // extract f
353  element=objects->get_element(1);
354  REQUIRE(element,"1 index element of objects is NULL\n")
355  CDenseLabels* func=dynamic_cast<CDenseLabels*>(element);
357 
358  // extract hm
359  element=objects->get_element(2);
360  REQUIRE(element,"2 index element of objects is NULL\n")
361  CDenseLabels* delta=dynamic_cast<CDenseLabels*>(element);
362  SGVector<float64_t> hm=delta->get_labels();
363 
364  // extract loss function
365  element=objects->get_element(3);
366  REQUIRE(element,"3 index element of objects is NULL\n")
367  CLossFunction* lossf=dynamic_cast<CLossFunction*>(element);
368 
369  *gradient=0;
370  float64_t ret=0;
371  for (int32_t i=0;i<labels.vlen;i++)
372  {
373  *gradient+=lossf->first_derivative((*parameters)*hm[i]+f[i],labels[i]);
374  ret+=lossf->loss((*parameters)*hm[i]+f[i],labels[i]);
375  }
376 
377  SG_UNREF(lab);
378  SG_UNREF(delta);
379  SG_UNREF(func);
380  SG_UNREF(lossf)
381  return ret;
382 }
383 
385 {
386  m_machine=NULL;
387  m_loss=NULL;
388  m_num_iter=0;
389  m_subset_frac=0;
390  m_learning_rate=0;
391 
394 
396  SG_REF(m_gamma);
397 
398  SG_ADD((CSGObject**)&m_machine,"m_machine","machine",MS_NOT_AVAILABLE);
399  SG_ADD((CSGObject**)&m_loss,"m_loss","loss function",MS_NOT_AVAILABLE);
400  SG_ADD(&m_num_iter,"m_num_iter","number of iterations",MS_NOT_AVAILABLE);
401  SG_ADD(&m_subset_frac,"m_subset_frac","subset fraction",MS_NOT_AVAILABLE);
402  SG_ADD(&m_learning_rate,"m_learning_rate","learning rate",MS_NOT_AVAILABLE);
403  SG_ADD((CSGObject**)&m_weak_learners,"m_weak_learners","array of weak learners",MS_NOT_AVAILABLE);
404  SG_ADD((CSGObject**)&m_gamma,"m_gamma","array of learner weights",MS_NOT_AVAILABLE);
405 }

SHOGUN Machine Learning Toolbox - Documentation