layer.h
1// OpenNN: Open Neural Networks Library
2// www.opennn.net
3//
4// L A Y E R C L A S S H E A D E R
5//
6// Artificial Intelligence Techniques SL
7// artelnics@artelnics.com
8
9#ifndef LAYER_H
10#define LAYER_H
11
12// System includes
13
14#include <cmath>
15#include <cstdlib>
16#include <fstream>
17#include <iostream>
18#include <string>
19#include <sstream>
20#include <ctype.h>
21#include <iostream>
22#include <vector>
23
24// OpenNN includes
25
26#include "config.h"
27//#include "tensor_utilities.h"
28#include "statistics.h"
29#include "data_set.h"
30
31using namespace std;
32using namespace Eigen;
33
34namespace OpenNN {
35
36class Layer;
37
38struct LayerForwardPropagation;
39struct LayerBackPropagation;
40struct LayerBackPropagationLM;
41
42#ifdef OPENNN_CUDA
43 #include "../../opennn-cuda/opennn-cuda/struct_layer_cuda.h"
44#endif
45
46
48
51
52class Layer
53{
54
55public:
56
57 // Enumerations
58
60
61 enum class Type{Scaling, Convolutional, Perceptron, Pooling, Probabilistic,
62 LongShortTermMemory,Recurrent, Unscaling, Bounding};
63
64 // Constructor
65
66 explicit Layer()
67 {
68 const int n = omp_get_max_threads();
69
70 non_blocking_thread_pool = new NonBlockingThreadPool(n);
71 thread_pool_device = new ThreadPoolDevice(non_blocking_thread_pool, n);
72 }
73
74 // Destructor
75
76 virtual ~Layer();
77
78 string get_name() const
79 {
80 return layer_name;
81 }
82
83 // Parameters initialization methods
84
85 virtual void set_parameters_constant(const type&);
86
87 virtual void set_parameters_random();
88
89 // Architecture
90
91 virtual Tensor<type, 1> get_parameters() const;
92 virtual Index get_parameters_number() const;
93
94 virtual void set_parameters(const Tensor<type, 1>&, const Index&);
95
96 void set_threads_number(const int&);
97
98 virtual void insert_gradient(LayerBackPropagation*, const Index&, Tensor<type, 1>&) const {}
99
100 // Outputs
101
102 virtual Tensor<type, 2> calculate_outputs(const Tensor<type, 2>&); // Cannot be const because of Recurrent and LSTM layers
103
104 virtual Tensor<type, 2> calculate_outputs_from4D(const Tensor<type, 4>&) {return Tensor<type, 2>();}
105
106 virtual Tensor<type, 4> calculate_outputs_4D(const Tensor<type, 4>&) {return Tensor<type, 4>();}
107
108 virtual void forward_propagate(const Tensor<type, 2>&, LayerForwardPropagation*) {} // Cannot be const because of Recurrent and LSTM layers
109 virtual void forward_propagate(const Tensor<type, 4>&, LayerForwardPropagation*) {}
110
111 virtual void forward_propagate(const Tensor<type, 4>&, Tensor<type, 1>, LayerForwardPropagation*) {}
112 virtual void forward_propagate(const Tensor<type, 2>&, Tensor<type, 1>, LayerForwardPropagation*) {} // Cannot be const because of Recurrent and LSTM layers
113
114 // Deltas
115
116 virtual void calculate_hidden_delta(LayerForwardPropagation*,
117 LayerBackPropagation*,
118 LayerBackPropagation*) const {}
119
120 virtual void calculate_hidden_delta_lm(LayerForwardPropagation*,
121 LayerBackPropagationLM*,
122 LayerBackPropagationLM*) const {}
123
124 // Error gradient
125
126 virtual void calculate_error_gradient(const Tensor<type, 2>&,
127 LayerForwardPropagation*,
128 LayerBackPropagation*) const {}
129
130 virtual void calculate_error_gradient(const Tensor<type, 4>&,
131 LayerForwardPropagation*,
132 LayerBackPropagation*) const {}
133
134 // Squared errors
135
136 virtual void calculate_squared_errors_Jacobian_lm(const Tensor<type, 2>&,
137 LayerForwardPropagation*,
138 LayerBackPropagationLM*) {}
139
140 virtual void insert_squared_errors_Jacobian_lm(LayerBackPropagationLM*,
141 const Index&,
142 Tensor<type, 2>&) const {}
143
144 // Get neurons number
145
146 virtual Index get_inputs_number() const;
147 virtual Index get_neurons_number() const;
148 virtual Index get_synaptic_weights_number() const;
149 virtual void set_inputs_number(const Index&);
150 virtual void set_neurons_number(const Index&);
151
152 // Layer type
153
154 Type get_type() const;
155
156 string get_type_string() const;
157
158 // Serialization methods
159
160 virtual void from_XML(const tinyxml2::XMLDocument&) {}
161
162 virtual void write_XML(tinyxml2::XMLPrinter&) const {}
163
164 // Expression methods
165
166 virtual string write_expression(const Tensor<string, 1>&, const Tensor<string, 1>&) const {return string();}
167
168 virtual string write_expression_c() const {return string();}
169
170 virtual string write_expression_python() const {return string();}
171
172protected:
173
174 NonBlockingThreadPool* non_blocking_thread_pool = nullptr;
175 ThreadPoolDevice* thread_pool_device = nullptr;
176
178
179 string layer_name = "layer";
180
182
183 Type layer_type = Type::Perceptron;
184
185 // activations 1d (Time Series)
186
187 void hard_sigmoid(const Tensor<type, 1>&, Tensor<type, 1>&) const;
188 void hyperbolic_tangent(const Tensor<type, 1>&, Tensor<type, 1>&) const;
189 void logistic(const Tensor<type, 1>&, Tensor<type, 1>&) const;
190 void linear(const Tensor<type, 1>&, Tensor<type, 1>&) const;
191 void threshold(const Tensor<type, 1>&, Tensor<type, 1>&) const;
192 void symmetric_threshold(const Tensor<type, 1>&, Tensor<type, 1>&) const;
193 void rectified_linear(const Tensor<type, 1>&, Tensor<type, 1>&) const;
194 void scaled_exponential_linear(const Tensor<type, 1>&, Tensor<type, 1>&) const;
195 void soft_plus(const Tensor<type, 1>&, Tensor<type, 1>&) const;
196 void soft_sign(const Tensor<type, 1>&, Tensor<type, 1>&) const;
197 void exponential_linear(const Tensor<type, 1>&, Tensor<type, 1>&) const;
198 void softmax(const Tensor<type, 1>&, Tensor<type, 1>&) const;
199 void binary(const Tensor<type, 1>&, Tensor<type, 1>&) const;
200 void competitive(const Tensor<type, 1>&, Tensor<type, 1>&) const;
201
202 void hard_sigmoid_derivatives(const Tensor<type, 1>&, Tensor<type, 1>&, Tensor<type, 1>&) const;
203 void hyperbolic_tangent_derivatives(const Tensor<type, 1>&, Tensor<type, 1>&, Tensor<type, 1>&) const;
204 void linear_derivatives(const Tensor<type, 1>&, Tensor<type, 1>&, Tensor<type, 1>&) const;
205 void logistic_derivatives(const Tensor<type, 1>&, Tensor<type, 1>&, Tensor<type, 1>&) const;
206 void threshold_derivatives(const Tensor<type, 1>&, Tensor<type, 1>&, Tensor<type, 1>&) const;
207 void symmetric_threshold_derivatives(const Tensor<type, 1>&, Tensor<type, 1>&, Tensor<type, 1>&) const;
208 void rectified_linear_derivatives(const Tensor<type, 1>&, Tensor<type, 1>&, Tensor<type, 1>&) const;
209 void scaled_exponential_linear_derivatives(const Tensor<type, 1>&, Tensor<type, 1>&, Tensor<type, 1>&) const;
210 void soft_plus_derivatives(const Tensor<type, 1>&, Tensor<type, 1>&, Tensor<type, 1>&) const;
211 void soft_sign_derivatives(const Tensor<type, 1>&, Tensor<type, 1>&, Tensor<type, 1>&) const;
212 void exponential_linear_derivatives(const Tensor<type, 1>&, Tensor<type, 1>&, Tensor<type, 1>&) const;
213
214 // activations 2d
215
216 void hard_sigmoid(const Tensor<type, 2>&, Tensor<type, 2>&) const;
217 void hyperbolic_tangent(const Tensor<type, 2>&, Tensor<type, 2>&) const;
218 void logistic(const Tensor<type, 2>&, Tensor<type, 2>&) const;
219 void linear(const Tensor<type, 2>&, Tensor<type, 2>&) const;
220 void threshold(const Tensor<type, 2>&, Tensor<type, 2>&) const;
221 void symmetric_threshold(const Tensor<type, 2>&, Tensor<type, 2>&) const;
222 void rectified_linear(const Tensor<type, 2>&, Tensor<type, 2>&) const;
223 void scaled_exponential_linear(const Tensor<type, 2>&, Tensor<type, 2>&) const;
224 void soft_plus(const Tensor<type, 2>&, Tensor<type, 2>&) const;
225 void soft_sign(const Tensor<type, 2>&, Tensor<type, 2>&) const;
226 void exponential_linear(const Tensor<type, 2>&, Tensor<type, 2>&) const;
227 void softmax(const Tensor<type, 2>&, Tensor<type, 2>&) const;
228 void binary(const Tensor<type, 2>&, Tensor<type, 2>&) const;
229 void competitive(const Tensor<type, 2>&, Tensor<type, 2>&) const;
230
231 void hard_sigmoid_derivatives(const Tensor<type, 2>&, Tensor<type, 2>&, Tensor<type, 2>&) const;
232 void hyperbolic_tangent_derivatives(const Tensor<type, 2>&, Tensor<type, 2>&, Tensor<type, 2>&) const;
233 void linear_derivatives(const Tensor<type, 2>&, Tensor<type, 2>&, Tensor<type, 2>&) const;
234 void logistic_derivatives(const Tensor<type, 2>&, Tensor<type, 2>&, Tensor<type, 2>&) const;
235 void threshold_derivatives(const Tensor<type, 2>&, Tensor<type, 2>&, Tensor<type, 2>&) const;
236 void symmetric_threshold_derivatives(const Tensor<type, 2>&, Tensor<type, 2>&, Tensor<type, 2>&) const;
237 void rectified_linear_derivatives(const Tensor<type, 2>&, Tensor<type, 2>&, Tensor<type, 2>&) const;
238 void scaled_exponential_linear_derivatives(const Tensor<type, 2>&, Tensor<type, 2>&, Tensor<type, 2>&) const;
239 void soft_plus_derivatives(const Tensor<type, 2>&, Tensor<type, 2>&, Tensor<type, 2>&) const;
240 void soft_sign_derivatives(const Tensor<type, 2>&, Tensor<type, 2>&, Tensor<type, 2>&) const;
241 void exponential_linear_derivatives(const Tensor<type, 2>&, Tensor<type, 2>&, Tensor<type, 2>&) const;
242
243 void logistic_derivatives(const Tensor<type, 2>&, Tensor<type, 2>&, Tensor<type, 3>&) const;
244 void softmax_derivatives(const Tensor<type, 2>&, Tensor<type, 2>&, Tensor<type, 3>&) const;
245
246 // activations 4d
247
248 void linear(const Tensor<type, 4>&, Tensor<type, 4>&) const;
249 void logistic(const Tensor<type, 4>&, Tensor<type, 4>&) const;
250 void hyperbolic_tangent(const Tensor<type, 4>&, Tensor<type, 4>&) const;
251 void threshold(const Tensor<type, 4>&, Tensor<type, 4>&) const;
252 void symmetric_threshold(const Tensor<type, 4>&, Tensor<type, 4>&) const;
253 void rectified_linear(const Tensor<type, 4>&, Tensor<type, 4>&) const;
254 void scaled_exponential_linear(const Tensor<type, 4>&, Tensor<type, 4>&) const;
255 void soft_plus(const Tensor<type, 4>&, Tensor<type, 4>&) const;
256 void soft_sign(const Tensor<type, 4>&, Tensor<type, 4>&) const;
257 void hard_sigmoid(const Tensor<type, 4>&, Tensor<type, 4>&) const;
258 void exponential_linear(const Tensor<type, 4>&, Tensor<type, 4>&) const;
259
260 void linear_derivatives(const Tensor<type, 4>&, Tensor<type, 4>&, Tensor<type, 4>&) const;
261 void logistic_derivatives(const Tensor<type, 4>&, Tensor<type, 4>&, Tensor<type, 4>&) const;
262 void hyperbolic_tangent_derivatives(const Tensor<type, 4>&, Tensor<type, 4>&, Tensor<type, 4>&) const;
263 void threshold_derivatives(const Tensor<type, 4>&, Tensor<type, 4>&, Tensor<type, 4>&) const;
264 void symmetric_threshold_derivatives(const Tensor<type, 4>&, Tensor<type, 4>&, Tensor<type, 4>&) const;
265 void rectified_linear_derivatives(const Tensor<type, 4>&, Tensor<type, 4>&, Tensor<type, 4>&) const;
266 void scaled_exponential_linear_derivatives(const Tensor<type, 4>&, Tensor<type, 4>&, Tensor<type, 4>&) const;
267 void soft_plus_derivatives(const Tensor<type, 4>&, Tensor<type, 4>&, Tensor<type, 4>&) const;
268 void soft_sign_derivatives(const Tensor<type, 4>&, Tensor<type, 4>&, Tensor<type, 4>&) const;
269 void hard_sigmoid_derivatives(const Tensor<type, 4>&, Tensor<type, 4>&, Tensor<type, 4>&) const;
270 void exponential_linear_derivatives(const Tensor<type, 4>&, Tensor<type, 4>&, Tensor<type, 4>&) const;
271
272 const Eigen::array<IndexPair<Index>, 1> A_BT = {IndexPair<Index>(1, 1)};
273 const Eigen::array<IndexPair<Index>, 1> AT_B = {IndexPair<Index>(0, 0)};
274 const Eigen::array<IndexPair<Index>, 1> A_B = {IndexPair<Index>(1, 0)};
275
276#ifdef OPENNN_CUDA
277 #include "../../opennn-cuda/opennn-cuda/layer_cuda.h"
278#else
279};
280#endif
281struct LayerForwardPropagation
282{
284
285 explicit LayerForwardPropagation()
286 {
287 }
288
289 virtual ~LayerForwardPropagation() {}
290
291 virtual void set(const Index&, Layer*) {}
292
293 virtual void print() const {}
294
295 Index batch_samples_number = 0;
296
297 Layer* layer_pointer = nullptr;
298};
299
300
301struct LayerBackPropagation
302{
304
305 explicit LayerBackPropagation() {}
306
307 virtual ~LayerBackPropagation() {}
308
309 virtual void set(const Index&, Layer*) {}
310
311 virtual void print() const {}
312
313 Index batch_samples_number = 0;
314
315 Layer* layer_pointer = nullptr;
316};
317
318
319struct LayerBackPropagationLM
320{
322
323 explicit LayerBackPropagationLM() {}
324
325 virtual ~LayerBackPropagationLM() {}
326
327 virtual void set(const Index&, Layer*) {}
328
329 virtual void print() const {}
330
331 Index batch_samples_number = 0;
332
333 Layer* layer_pointer = nullptr;
334};
335
336}
337
338#endif // LAYER_H
OpenNN::Layer
This abstract class represents the concept of layer of neurons in OpenNN.
Definition: layer.h:53
OpenNN::Layer::Type
Type
This enumeration represents the possible types of layers.
Definition: layer.h:61
OpenNN::Layer::get_synaptic_weights_number
virtual Index get_synaptic_weights_number() const
Returns the number of layer's synaptic weights.
Definition: layer.cpp:179
OpenNN::Layer::layer_name
string layer_name
Layer name.
Definition: layer.h:179
OpenNN::Layer::get_inputs_number
virtual Index get_inputs_number() const
Returns the number of inputs.
Definition: layer.cpp:153
OpenNN::Layer::layer_type
Type layer_type
Layer type.
Definition: layer.h:183
OpenNN::Layer::get_type
Type get_type() const
Definition: layer.cpp:25
OpenNN::Layer::get_type_string
string get_type_string() const
Takes the type of layer used by the model.
Definition: layer.cpp:33
std
Extensions to the C++ standard library.
Definition: half.hpp:2325
OpenNN::LayerBackPropagation::LayerBackPropagation
LayerBackPropagation()
Default constructor.
Definition: layer.h:305
OpenNN::LayerBackPropagationLM::LayerBackPropagationLM
LayerBackPropagationLM()
Default constructor.
Definition: layer.h:323
OpenNN::LayerForwardPropagation::LayerForwardPropagation
LayerForwardPropagation()
Default constructor.
Definition: layer.h:285