SHOGUN  4.2.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Modules Pages
NeuralRectifiedLinearLayer.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2014, Shogun Toolbox Foundation
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions are met:
7 
8  * 1. Redistributions of source code must retain the above copyright notice,
9  * this list of conditions and the following disclaimer.
10  *
11  * 2. Redistributions in binary form must reproduce the above copyright notice,
12  * this list of conditions and the following disclaimer in the documentation
13  * and/or other materials provided with the distribution.
14  *
15  * 3. Neither the name of the copyright holder nor the names of its
16  * contributors may be used to endorse or promote products derived from this
17  * software without specific prior written permission.
18 
19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  *
31  * Written (W) 2014 Khaled Nasr
32  */
33 
36 #include <shogun/lib/SGVector.h>
37 
38 using namespace shogun;
39 
41 {
42 }
43 
45 CNeuralLinearLayer(num_neurons)
46 {
47 }
48 
50  SGVector<float64_t> parameters,
51  CDynamicObjectArray* layers)
52 {
53  CNeuralLinearLayer::compute_activations(parameters, layers);
54 
55  int32_t len = m_num_neurons*m_batch_size;
56  for (int32_t i=0; i<len; i++)
57  {
58  m_activations[i] = CMath::max<float64_t>(0, m_activations[i]);
59  }
60 }
61 
63  SGVector< float64_t > parameters)
64 {
66 
68  m_num_neurons, num_inputs, false);
69 
70  float64_t contraction_term = 0;
71  for (int32_t i=0; i<m_num_neurons; i++)
72  {
73  float64_t sum_j = 0;
74  for (int32_t j=0; j<num_inputs; j++)
75  sum_j += W(i,j)*W(i,j);
76 
77  for (int32_t k = 0; k<m_batch_size; k++)
78  {
79  if (m_activations(i,k) > 0)
80  contraction_term += sum_j;
81  }
82  }
83 
84  return (contraction_coefficient/m_batch_size) * contraction_term;
85 }
86 
88  SGVector< float64_t > parameters, SGVector< float64_t > gradients)
89 {
91 
93  m_num_neurons, num_inputs, false);
95  m_num_neurons, num_inputs, false);
96 
97  for (int32_t k = 0; k<m_batch_size; k++)
98  {
99  for (int32_t i=0; i<m_num_neurons; i++)
100  {
101  if (m_activations(i,k) > 0)
102  {
103  for (int32_t j=0; j<num_inputs; j++)
104  WG(i,j) += 2 * (contraction_coefficient/m_batch_size) * W(i,j);
105  }
106  }
107  }
108 }
109 
110 
112  SGMatrix<float64_t> targets)
113 {
114  if (targets.num_rows != 0)
115  {
116  int32_t length = m_num_neurons*m_batch_size;
117  for (int32_t i=0; i<length; i++)
118  {
119  if (m_activations[i]==0)
120  m_local_gradients[i] = 0;
121  else
122  m_local_gradients[i] = (m_activations[i]-targets[i])/m_batch_size;
123  }
124  }
125  else
126  {
127  int32_t len = m_num_neurons*m_batch_size;
128  for (int32_t i=0; i< len; i++)
129  {
130  if (m_activations[i]==0)
131  m_local_gradients[i] = 0;
132  else
134  }
135  }
136 }
SGVector< int32_t > m_input_sizes
Definition: NeuralLayer.h:368
virtual void compute_contraction_term_gradients(SGVector< float64_t > parameters, SGVector< float64_t > gradients)
SGMatrix< float64_t > m_activations
Definition: NeuralLayer.h:376
SGMatrix< float64_t > m_activation_gradients
Definition: NeuralLayer.h:381
index_t num_rows
Definition: SGMatrix.h:374
SGMatrix< float64_t > m_local_gradients
Definition: NeuralLayer.h:387
index_t vlen
Definition: SGVector.h:494
double float64_t
Definition: common.h:50
virtual void compute_local_gradients(SGMatrix< float64_t > targets)
static T sum(T *vec, int32_t len)
Return sum(vec)
Definition: SGVector.h:354
virtual void compute_activations(SGVector< float64_t > parameters, CDynamicObjectArray *layers)
Dynamic array class for CSGObject pointers that creates an array that can be used like a list or an a...
Neural layer with linear neurons, with an identity activation function. can be used as a hidden layer...
virtual void compute_activations(SGVector< float64_t > parameters, CDynamicObjectArray *layers)
all of classes and functions are contained in the shogun namespace
Definition: class_list.h:18
virtual float64_t compute_contraction_term(SGVector< float64_t > parameters)
float64_t contraction_coefficient
Definition: NeuralLayer.h:338

SHOGUN Machine Learning Toolbox - Documentation