SHOGUN  4.1.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
NeuralConvolutionalLayer.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2014, Shogun Toolbox Foundation
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7 
8  * 1. Redistributions of source code must retain the above copyright notice,
9  * this list of conditions and the following disclaimer.
10  *
11  * 2. Redistributions in binary form must reproduce the above copyright notice,
12  * this list of conditions and the following disclaimer in the documentation
13  * and/or other materials provided with the distribution.
14  *
15  * 3. Neither the name of the copyright holder nor the names of its
16  * contributors may be used to endorse or promote products derived from this
17  * software without specific prior written permission.
18 
19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  *
31  * Written (W) 2014 Khaled Nasr
32  */
33 
36 #include <shogun/lib/SGVector.h>
37 
38 using namespace shogun;
39 
41 {
42  init();
43 }
44 
47  int32_t num_maps,
48  int32_t radius_x, int32_t radius_y,
49  int32_t pooling_width, int32_t pooling_height,
50  int32_t stride_x, int32_t stride_y,
51  EInitializationMode initialization_mode) : CNeuralLayer()
52 {
53  init();
54  m_num_maps = num_maps;
55  m_radius_x = radius_x;
56  m_radius_y = radius_y;
57  m_pooling_width = pooling_width;
58  m_pooling_height = pooling_height;
59  m_stride_x = stride_x;
60  m_stride_y = stride_y;
61  m_initialization_mode = initialization_mode;
62 }
63 
65 {
66  CNeuralLayer::set_batch_size(batch_size);
67 
71  else
74 
76 
78  m_convolution_output.num_rows, m_convolution_output.num_cols);
79 }
80 
81 
83  SGVector< int32_t > input_indices)
84 {
85  CNeuralLayer* first_input_layer =
86  (CNeuralLayer*)layers->element(input_indices[0]);
87 
88  m_input_width = first_input_layer->get_width();
89  m_input_height = first_input_layer->get_height();
90 
91  SG_UNREF(first_input_layer);
92 
94  {
97  }
98  else
99  {
102  }
103 
105 
106  CNeuralLayer::initialize(layers, input_indices);
107 
109  for (int32_t l=0; l<input_indices.vlen; l++)
110  {
111  CNeuralLayer* layer =
112  (CNeuralLayer*)layers->element(input_indices[l]);
113 
115 
116  SG_UNREF(layer);
117  }
118 
119  // one bias for each map and one weight matrix between each map in this
120  // layer and each channel in the input layers
122  m_num_maps*(1 + m_input_num_channels*(2*m_radius_x+1)*(2*m_radius_y+1));
123 }
124 
126  SGVector<bool> parameter_regularizable,
127  float64_t sigma)
128 {
129  int32_t num_parameters_per_map =
131 
132  for (int32_t m=0; m<m_num_maps; m++)
133  {
134  float64_t* map_params = parameters.vector+m*num_parameters_per_map;
135  bool* map_param_regularizable =
136  parameter_regularizable.vector+m*num_parameters_per_map;
137 
138  for (int32_t i=0; i<num_parameters_per_map; i++)
139  {
141  {
142  map_params[i] = CMath::normal_random(0.0, sigma);
143  // turn off regularization for the bias, on for the rest of the parameters
144  map_param_regularizable[i] = (i != 0);
145  }
146  else // for the case when m_initialization_mode = RE_NORMAL
147  {
148  map_params[i] = CMath::normal_random(0.0,
150  // initialize b=0
151  map_param_regularizable[i] = 0;
152  }
153  }
154  }
155 }
156 
158  SGVector<float64_t> parameters,
159  CDynamicObjectArray* layers)
160 {
161  int32_t num_parameters_per_map =
163 
164  for (int32_t m=0; m<m_num_maps; m++)
165  {
166  SGVector<float64_t> map_params(
167  parameters.vector+m*num_parameters_per_map,
168  num_parameters_per_map, false);
169 
173 
174  map.compute_activations(map_params, layers, m_input_indices,
176 
179  }
180 }
181 
183  SGVector<float64_t> parameters,
184  SGMatrix<float64_t> targets,
185  CDynamicObjectArray* layers,
186  SGVector<float64_t> parameter_gradients)
187 {
188  if (targets.num_rows != 0)
189  {
190  // sqaured error measure
191  // local_gradients = activations-targets
192  int32_t length = m_num_neurons*m_batch_size;
193  for (int32_t i=0; i<length; i++)
195  }
196 
197  if (dropout_prop>0.0)
198  {
199  int32_t len = m_num_neurons*m_batch_size;
200  for (int32_t i=0; i<len; i++)
202  }
203 
204  // compute the pre-pooling activation gradients
206  for (int32_t i=0; i<m_num_neurons; i++)
207  for (int32_t j=0; j<m_batch_size; j++)
208  if (m_max_indices(i,j)!=-1.0)
211 
212  int32_t num_parameters_per_map =
214 
215  for (int32_t m=0; m<m_num_maps; m++)
216  {
217  SGVector<float64_t> map_params(
218  parameters.vector+m*num_parameters_per_map,
219  num_parameters_per_map, false);
220 
221  SGVector<float64_t> map_gradients(
222  parameter_gradients.vector+m*num_parameters_per_map,
223  num_parameters_per_map, false);
224 
228 
229  map.compute_gradients(map_params, m_convolution_output,
231  m_input_indices, map_gradients);
232  }
233 }
234 
236 {
237  // error = 0.5*(sum(targets-activations)^2)/batch_size
238  float64_t sum = 0;
239  int32_t length = m_num_neurons*m_batch_size;
240  for (int32_t i=0; i<length; i++)
241  sum += (targets[i]-m_activations[i])*(targets[i]-m_activations[i]);
242  sum *= (0.5/m_batch_size);
243  return sum;
244 }
245 
247  float64_t max_norm)
248 {
249  int32_t num_weights = (2*m_radius_x+1)*(2*m_radius_y+1);
250 
251  int32_t num_parameters_per_map = 1 + m_input_num_channels*num_weights;
252 
253  for (int32_t offset=1; offset<parameters.vlen; offset+=num_parameters_per_map)
254  {
255  float64_t* weights = parameters.vector+offset;
256 
257  float64_t norm =
258  SGVector<float64_t>::twonorm(weights, num_weights);
259 
260  if (norm > max_norm)
261  {
262  float64_t multiplier = max_norm/norm;
263  for (int32_t i=0; i<num_weights; i++)
264  weights[i] *= multiplier;
265  }
266  }
267 }
268 
269 void CNeuralConvolutionalLayer::init()
270 {
271  m_num_maps = 1;
272  m_input_width = 0;
273  m_input_height = 0;
275  m_radius_x = 0;
276  m_radius_y = 0;
277  m_pooling_width = 1;
278  m_pooling_height = 1;
279  m_stride_x = 1;
280  m_stride_y = 1;
283 
284  SG_ADD(&m_num_maps, "num_maps", "Number of maps", MS_NOT_AVAILABLE);
285  SG_ADD(&m_input_width, "input_width", "Input Width", MS_NOT_AVAILABLE);
286  SG_ADD(&m_input_height, "input_height", "Input Height", MS_NOT_AVAILABLE);
287  SG_ADD(&m_input_num_channels, "input_num_channels", "Input's number of channels",
289  SG_ADD(&m_radius_x, "radius_x", "X Radius", MS_NOT_AVAILABLE);
290  SG_ADD(&m_radius_y, "radius_y", "Y Radius", MS_NOT_AVAILABLE);
291  SG_ADD(&m_pooling_width, "pooling_width", "Pooling Width", MS_NOT_AVAILABLE);
292  SG_ADD(&m_pooling_height, "pooling_height", "Pooling Height", MS_NOT_AVAILABLE);
293  SG_ADD(&m_stride_x, "stride_x", "X Stride", MS_NOT_AVAILABLE);
294  SG_ADD(&m_stride_y, "stride_y", "Y Stride", MS_NOT_AVAILABLE);
295 
296  SG_ADD((machine_int_t*) &m_initialization_mode, "initialization_mode", "Initialization Mode",
298 
299  SG_ADD((machine_int_t*) &m_activation_function, "activation_function",
300  "Activation Function", MS_NOT_AVAILABLE);
301 
302  SG_ADD(&m_convolution_output, "convolution_output",
303  "Convolution Output", MS_NOT_AVAILABLE);
304 
305  SG_ADD(&m_convolution_output_gradients, "convolution_output_gradients",
306  "Convolution Output Gradients", MS_NOT_AVAILABLE);
307 }
static T twonorm(const T *x, int32_t len)
|| x ||_2
double norm(double *v, double p, int n)
Definition: epph.cpp:452
virtual void initialize(CDynamicObjectArray *layers, SGVector< int32_t > input_indices)
Definition: NeuralLayer.cpp:61
virtual void initialize(CDynamicObjectArray *layers, SGVector< int32_t > input_indices)
EConvMapActivationFunction m_activation_function
virtual void compute_activations(SGVector< float64_t > parameters, CDynamicObjectArray *layers)
static float32_t normal_random(float32_t mean, float32_t std_dev)
Definition: Math.h:1095
virtual void set_batch_size(int32_t batch_size)
SGMatrix< float64_t > m_activations
Definition: NeuralLayer.h:376
virtual void enforce_max_norm(SGVector< float64_t > parameters, float64_t max_norm)
index_t num_cols
Definition: SGMatrix.h:378
SGMatrix< float64_t > m_convolution_output_gradients
virtual void compute_gradients(SGVector< float64_t > parameters, SGMatrix< float64_t > targets, CDynamicObjectArray *layers, SGVector< float64_t > parameter_gradients)
virtual int32_t get_num_neurons()
Definition: NeuralLayer.h:251
SGVector< int32_t > m_input_indices
Definition: NeuralLayer.h:363
Base class for neural network layers.
Definition: NeuralLayer.h:87
SGMatrix< float64_t > m_activation_gradients
Definition: NeuralLayer.h:381
index_t num_rows
Definition: SGMatrix.h:376
Handles convolution and gradient calculation for a single feature map in a convolutional neural netwo...
virtual int32_t get_height()
Definition: NeuralLayer.h:265
index_t vlen
Definition: SGVector.h:494
CSGObject * element(int32_t idx1, int32_t idx2=0, int32_t idx3=0)
ENLAutoencoderPosition autoencoder_position
Definition: NeuralLayer.h:343
double float64_t
Definition: common.h:50
void compute_activations(SGVector< float64_t > parameters, CDynamicObjectArray *layers, SGVector< int32_t > input_indices, SGMatrix< float64_t > activations)
virtual void initialize_parameters(SGVector< float64_t > parameters, SGVector< bool > parameter_regularizable, float64_t sigma)
Dynamic array class for CSGObject pointers that creates an array that can be used like a list or an a...
virtual int32_t get_width()
Definition: NeuralLayer.h:258
#define SG_UNREF(x)
Definition: SGObject.h:52
all of classes and functions are contained in the shogun namespace
Definition: class_list.h:18
float64_t dropout_prop
Definition: NeuralLayer.h:327
virtual void set_batch_size(int32_t batch_size)
Definition: NeuralLayer.cpp:75
int machine_int_t
Definition: common.h:59
virtual float64_t compute_error(SGMatrix< float64_t > targets)
EConvMapActivationFunction
Determines the activation function for neurons in a convolutional feature map.
#define SG_ADD(...)
Definition: SGObject.h:81
static float32_t sqrt(float32_t x)
Definition: Math.h:459
void compute_gradients(SGVector< float64_t > parameters, SGMatrix< float64_t > activations, SGMatrix< float64_t > activation_gradients, CDynamicObjectArray *layers, SGVector< int32_t > input_indices, SGVector< float64_t > parameter_gradients)
SGMatrix< bool > m_dropout_mask
Definition: NeuralLayer.h:393
void pool_activations(SGMatrix< float64_t > activations, int32_t pooling_width, int32_t pooling_height, SGMatrix< float64_t > pooled_activations, SGMatrix< float64_t > max_indices)

SHOGUN Machine Learning Toolbox - Documentation