layer.cpp
1// OpenNN: Open Neural Networks Library
2// www.opennn.net
3//
4// L A Y E R C L A S S
5//
6// Artificial Intelligence Techniques SL
7// artelnics@artelnics.com
8
9#include "layer.h"
10
11namespace OpenNN
12{
13
14Layer::~Layer()
15{
16 delete non_blocking_thread_pool;
17 delete thread_pool_device;
18}
19
20
24
25Layer::Type Layer::get_type () const
26{
27 return layer_type;
28}
29
30
32
33string Layer::get_type_string() const
34{
35 switch(layer_type)
36 {
37 case Type::Perceptron:
38 return "Perceptron";
39
40 case Type::Bounding:
41 return "Bounding";
42
43 case Type::Pooling:
44 return "Pooling";
45
46 case Type::Probabilistic:
47 return "Probabilistic";
48
49 case Type::Convolutional:
50 return "Convolutional";
51
52 case Type::LongShortTermMemory:
53 return "LongShortTermMemory";
54
55 case Type::Recurrent:
56 return "Recurrent";
57
58 case Type::Scaling:
59 return "Scaling";
60
61 case Type::Unscaling:
62 return "Unscaling";
63 }
64
65 return string();
66}
67
68
69void Layer::set_threads_number(const int& new_threads_number)
70{
71 if(non_blocking_thread_pool != nullptr) delete this->non_blocking_thread_pool;
72 if(thread_pool_device != nullptr) delete this->thread_pool_device;
73
74 non_blocking_thread_pool = new NonBlockingThreadPool(new_threads_number);
75 thread_pool_device = new ThreadPoolDevice(non_blocking_thread_pool, new_threads_number);
76}
77
78
79void Layer::set_parameters_constant(const type&)
80{
81 ostringstream buffer;
82
83 buffer << "OpenNN Exception: Layer class.\n"
84 << "set_parameters_constant(const type&) method.\n"
85 << "This method is not implemented in the layer type (" << get_type_string() << ").\n";
86
87 throw logic_error(buffer.str());
88}
89
90
91void Layer::set_parameters_random()
92{
93 ostringstream buffer;
94
95 buffer << "OpenNN Exception: Layer class.\n"
96 << "set_parameters_random() method.\n"
97 << "This method is not implemented in the layer type (" << get_type_string() << ").\n";
98
99 throw logic_error(buffer.str());
100}
101
102
103void Layer::set_parameters(const Tensor<type, 1>&, const Index&)
104{
105 ostringstream buffer;
106
107 buffer << "OpenNN Exception: Layer class.\n"
108 << "set_parameters(const Tensor<type, 1>&) method.\n"
109 << "This method is not implemented in the layer type (" << get_type_string() << ").\n";
110
111 throw logic_error(buffer.str());
112}
113
114
115Index Layer::get_parameters_number() const
116{
117 ostringstream buffer;
118
119 buffer << "OpenNN Exception: Layer class.\n"
120 << "get_parameters_number() method.\n"
121 << "This method is not implemented in the layer type (" << get_type_string() << ").\n";
122
123 throw logic_error(buffer.str());
124}
125
126
127Tensor<type, 1> Layer::get_parameters() const
128{
129 ostringstream buffer;
130
131 buffer << "OpenNN Exception: Layer class.\n"
132 << "get_parameters() method.\n"
133 << "This method is not implemented in the layer type (" << get_type_string() << ").\n";
134
135 throw logic_error(buffer.str());
136}
137
138
139Tensor<type, 2> Layer::calculate_outputs(const Tensor<type, 2>&)
140{
141 ostringstream buffer;
142
143 buffer << "OpenNN Exception: Layer class.\n"
144 << "calculate_outputs(const Tensor<type, 2>&) method.\n"
145 << "This method is not implemented in the layer type (" << get_type_string() << ").\n";
146
147 throw logic_error(buffer.str());
148}
149
150
152
153Index Layer::get_inputs_number() const
154{
155 ostringstream buffer;
156
157 buffer << "OpenNN Exception: Layer class.\n"
158 << "get_inputs_number() const method.\n"
159 << "This method is not implemented in the layer type (" << get_type_string() << ").\n";
160
161 throw logic_error(buffer.str());
162}
163
164
165Index Layer::get_neurons_number() const
166{
167 ostringstream buffer;
168
169 buffer << "OpenNN Exception: Layer class.\n"
170 << "get_neurons_number() const method.\n"
171 << "This method is not implemented in the layer type (" << get_type_string() << ").\n";
172
173 throw logic_error(buffer.str());
174}
175
176
178
179Index Layer::get_synaptic_weights_number() const
180{
181 ostringstream buffer;
182
183 buffer << "OpenNN Exception: Layer class.\n"
184 << "get_synaptic_weight_number() const method.\n"
185 << "This method is not implemented in the layer type (" << get_type_string() << ").\n";
186
187 throw logic_error(buffer.str());
188}
189
190
191void Layer::set_inputs_number(const Index& )
192{
193 ostringstream buffer;
194
195 buffer << "OpenNN Exception: Layer class.\n"
196 << "set_inputs_number(const Index& ) method.\n"
197 << "This method is not implemented in the layer type (" << get_type_string() << ").\n";
198
199 throw logic_error(buffer.str());
200}
201
202
203void Layer::set_neurons_number(const Index& )
204{
205 ostringstream buffer;
206
207 buffer << "OpenNN Exception: Layer class.\n"
208 << "set_neurons_number(const Index& ) method.\n"
209 << "This method is not implemented in the layer type (" << get_type_string() << ").\n";
210
211 throw logic_error(buffer.str());
212}
213
214
215void Layer::hard_sigmoid(const Tensor<type, 1>& x, Tensor<type, 1>& y) const
216{
217 const Tensor<bool, 1> if_sentence = x < x.constant(type(-2.5));
218 const Tensor<bool, 1> elif_sentence = x > x.constant(type(2.5));
219
220 Tensor<type, 1> f1(x.dimension(0));
221 Tensor<type, 1> f2(x.dimension(0));
222 Tensor<type, 1> f3(x.dimension(0));
223
224 f1.setZero();
225 f2.setConstant(type(1));
226 f3 = static_cast<type>(0.2) * x + static_cast<type>(0.5);
227
228 y.device(*thread_pool_device) = if_sentence.select(f1,elif_sentence.select(f2,f3));
229}
230
231
232void Layer::hyperbolic_tangent(const Tensor<type, 1>& x, Tensor<type, 1>& y) const
233{
234 y.device(*thread_pool_device) = x.tanh();
235}
236
237
238void Layer::logistic(const Tensor<type, 1>& x, Tensor<type, 1>& y)const
239{
240 y.device(*thread_pool_device) = (type(1) + x.exp().inverse()).inverse();
241}
242
243
244void Layer::linear(const Tensor<type, 1>& x, Tensor<type, 1>& y) const
245{
246 y = x;
247}
248
249
250void Layer::threshold(const Tensor<type, 1>& x, Tensor<type, 1>& y) const
251{
252 const Tensor<bool, 1> if_sentence = x >= x.constant(type(0));
253
254 Tensor<type, 1> ones(x.dimension(0));
255 ones.setConstant(type(1));
256
257 Tensor<type, 1> zeros(x.dimension(0));
258 zeros.setConstant(type(0));
259
260 y.device(*thread_pool_device) = if_sentence.select(ones, zeros);
261}
262
263
264void Layer::symmetric_threshold(const Tensor<type, 1>& x, Tensor<type, 1>& y) const
265{
266
267 const Tensor<bool, 1> if_sentence = x > x.constant(type(0));
268
269 Tensor<type, 1> ones(x.dimension(0));
270 ones.setConstant(type(1));
271
272 y.device(*thread_pool_device) = if_sentence.select(ones, -ones);
273}
274
275
276void Layer::rectified_linear(const Tensor<type, 1>& x, Tensor<type, 1>& y) const
277{
278 const Tensor<bool, 1> if_sentence = x < x.constant(type(0));
279
280 Tensor<type, 1> zeros(x.dimension(0));
281
282 zeros.setConstant(type(0));
283
284 y.device(*thread_pool_device) = if_sentence.select(zeros, x);
285}
286
287
288void Layer::scaled_exponential_linear(const Tensor<type, 1>& x, Tensor<type, 1>& y) const
289{
290
291 const type lambda = static_cast<type>(1.0507);
292
293 const type alpha = static_cast<type>(1.67326);
294
295 const Tensor<bool, 1> if_sentence = x < x.constant(type(0));
296
297 Tensor<type, 1> f_1(x.dimension(0));
298
299 Tensor<type, 1> f_2(x.dimension(0));
300
301 f_1 = lambda*alpha*(x.exp()-static_cast<type>(1.0));
302
303 f_2 = lambda*x;
304
305 y.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
306}
307
308
309void Layer::soft_plus(const Tensor<type, 1>& x, Tensor<type, 1>& y) const
310{
311 y.device(*thread_pool_device) = (x.constant(type(1)) + x.exp()).log();
312
313}
314
315
316void Layer::soft_sign(const Tensor<type, 1>& x, Tensor<type, 1>& y) const
317{
318
319 const Tensor<bool, 1> if_sentence = x < x.constant(type(0));
320
321 Tensor<type, 1> f_1(x.dimension(0));
322
323 Tensor<type, 1> f_2(x.dimension(0));
324
325 f_1 = x / (static_cast<type>(1) - x);
326
327 f_2 = x / (static_cast<type>(1) + x);
328
329 y.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
330}
331
332
333void Layer::exponential_linear(const Tensor<type, 1>& x, Tensor<type, 1>& y) const
334{
335 const Tensor<bool, 1> if_sentence = x < x.constant(type(0));
336
337 const type alpha = static_cast<type>(1.0);
338
339 Tensor<type, 1> f_1(x.dimension(0));
340
341 Tensor<type, 1> f_2(x.dimension(0));
342
343 f_1.device(*thread_pool_device) = alpha*(x.exp() - static_cast<type>(1));
344
345 f_2 = x;
346
347 y.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
348}
349
350
351void Layer::binary(const Tensor<type, 1>& x, Tensor<type, 1>& y) const
352{
353 const Tensor<bool, 1> if_sentence = x < x.constant(type(0.5));
354
355 Tensor<type, 1> f_1(x.dimension(0));
356
357 Tensor<type, 1> f_2(x.dimension(0));
358
359 f_1 = x.constant(type(false));
360
361 f_2 = x.constant(type(true));
362
363 y.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
364}
365
366
367void Layer::competitive(const Tensor<type, 1>& x, Tensor<type, 1>& y) const
368{
369 y.setZero();
370
371 const Index index = maximal_index(x);
372
373 y(index) = type(1);
374}
375
376
377void Layer::softmax(const Tensor<type, 1>& x, Tensor<type, 1>& y) const
378{
379 Tensor<type, 0> sum;
380
381 sum.device(*thread_pool_device) = x.exp().sum();
382
383 y.device(*thread_pool_device) = x.exp() / sum(0);
384}
385
386
387void Layer::hard_sigmoid_derivatives(const Tensor<type, 1>& combinations,
388 Tensor<type, 1>& activations,
389 Tensor<type, 1>& activations_derivatives) const
390{
391 // Conditions
392
393 const Tensor<bool, 1> if_sentence = combinations < combinations.constant(type(-2.5));
394
395 const Tensor<bool, 1> elif_sentence = combinations > combinations.constant(type(2.5));
396
397 const Tensor<bool, 1> if_sentence_2 = combinations < combinations.constant(type(-2.5)) || combinations > combinations.constant(type(2.5));
398
399 // Sentences
400
401 Tensor<type, 1> f1(combinations.dimension(0));
402 f1.setZero();
403
404 Tensor<type, 1> f2(combinations.dimension(0));
405 f2.setConstant(type(1));
406
407 Tensor<type, 1> f3(combinations.dimension(0));
408 f3 = static_cast<type>(0.2) * combinations + static_cast<type>(0.5);
409
410 Tensor<type, 1> f4(combinations.dimension(0));
411 f4.setConstant(type(0));
412
413 Tensor<type, 1> f5(combinations.dimension(0));
414 f5.setConstant(static_cast<type>(0.2));
415
416 // Activations
417
418 activations.device(*thread_pool_device) = if_sentence.select(f1, elif_sentence.select(f2, f3));
419
420 // Activations Derivatives
421
422 activations_derivatives.device(*thread_pool_device) = if_sentence_2.select(f4, f5);
423
424}
425
426void Layer::hyperbolic_tangent_derivatives(const Tensor<type, 1>& combinations,
427 Tensor<type, 1>& activations,
428 Tensor<type, 1>& activations_derivatives) const
429{
430 // Activations
431
432 activations.device(*thread_pool_device) = combinations.tanh();
433
434 // Activations Derivatives
435
436 activations_derivatives.device(*thread_pool_device) = type(1) - activations.square();
437}
438
439
440void Layer::logistic_derivatives(const Tensor<type, 1>& combinations,
441 Tensor<type, 1>& activations,
442 Tensor<type, 1>& activations_derivatives) const
443{
444 activations.device(*thread_pool_device) = (type(1) + combinations.exp().inverse()).inverse();
445
446 activations_derivatives.device(*thread_pool_device) = activations*(type(1) - activations);
447}
448
449
450void Layer::linear_derivatives(const Tensor<type, 1>& combinations,
451 Tensor<type, 1>& activations,
452 Tensor<type, 1>& activations_derivatives) const
453{
454 activations = combinations;
455
456 activations_derivatives.setConstant(type(1));
457}
458
459
460void Layer::threshold_derivatives(const Tensor<type, 1>& combinations,
461 Tensor<type, 1>& activations,
462 Tensor<type, 1>& activations_derivatives) const
463{
464 const Tensor<bool, 1> if_sentence = combinations > combinations.constant(type(0));
465
466 Tensor<type, 1> ones(combinations.dimension(0));
467 ones.setConstant(type(1));
468
469 Tensor<type, 1> zeros(combinations.dimension(0));
470 zeros.setConstant(type(0));
471
472 // Activations
473
474 activations.device(*thread_pool_device) = if_sentence.select(ones, zeros);
475
476 // Activations Derivatives
477
478 activations_derivatives.setZero();
479}
480
481
482void Layer::symmetric_threshold_derivatives(const Tensor<type, 1>& combinations,
483 Tensor<type, 1>& activations,
484 Tensor<type, 1>& activations_derivatives) const
485{
486 const Tensor<bool, 1> if_sentence = combinations > combinations.constant(type(0));
487
488 Tensor<type, 1> ones(combinations.dimension(0));
489
490 ones.setConstant(type(1));
491
492 // Activations
493
494 activations.device(*thread_pool_device) = if_sentence.select(ones, -ones);
495
496 // Activations Derivatives
497
498 activations_derivatives.setZero();
499}
500
501
502void Layer::rectified_linear_derivatives(const Tensor<type, 1>& combinations,
503 Tensor<type, 1>& activations,
504 Tensor<type, 1>& activations_derivatives) const
505{
506 const Tensor<bool, 1> if_sentence = combinations < combinations.constant(type(0));
507
508 Tensor<type, 1> zeros(combinations.dimension(0));
509 zeros.setZero();
510
511 Tensor<type, 1> ones(combinations.dimension(0));
512 ones.setConstant(type(1));
513
514 // Activations
515
516 activations.device(*thread_pool_device) = if_sentence.select(zeros, combinations);
517
518 // Activations Derivatives
519
520 activations_derivatives.device(*thread_pool_device) = if_sentence.select(zeros, ones);
521}
522
523
524void Layer::scaled_exponential_linear_derivatives(const Tensor<type, 1>& combinations,
525 Tensor<type, 1>& activations,
526 Tensor<type, 1>& activations_derivatives) const
527{
528 const type lambda = static_cast<type>(1.0507);
529
530 const type alpha = static_cast<type>(1.67326);
531
532 const Tensor<bool, 1> if_sentence = combinations < combinations.constant(type(0));
533
534 Tensor<type, 1> f_1(combinations.dimension(0));
535
536 Tensor<type, 1> f_2(combinations.dimension(0));
537
538 f_1 = lambda*alpha*(combinations.exp()-static_cast<type>(1.0));
539
540 f_2 = lambda*combinations;
541
542 // Activations
543
544 activations.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
545
546 // Activations Derivatives
547
548 f_1 = lambda*alpha*combinations.exp();
549
550 f_2 = combinations.constant(type(1))*lambda;
551
552 activations_derivatives.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
553}
554
555
556void Layer::soft_plus_derivatives(const Tensor<type, 1>& combinations,
557 Tensor<type, 1>& activations,
558 Tensor<type, 1>& activations_derivatives) const
559{
560 activations.device(*thread_pool_device) = (combinations.constant(type(1)) + combinations.exp()).log();
561
562 activations_derivatives.device(*thread_pool_device) = static_cast<type>(1.0) / (static_cast<type>(1.0) + combinations.exp().inverse());
563}
564
565
566void Layer::soft_sign_derivatives(const Tensor<type, 1>& combinations,
567 Tensor<type, 1>& activations,
568 Tensor<type, 1>& activations_derivatives) const
569{
570 const Tensor<bool, 1> if_sentence = combinations < combinations.constant(type(0));
571
572 Tensor<type, 1> f_1(combinations.dimension(0));
573
574 Tensor<type, 1> f_2(combinations.dimension(0));
575
576 f_1 = combinations / (static_cast<type>(1) - combinations);
577
578 f_2 = combinations / (static_cast<type>(1) + combinations);
579
580 // Activations
581
582 activations.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
583
584 // Activations Derivatives
585
586 f_1 = static_cast<type>(1.0) / (static_cast<type>(1.0) - combinations).pow(type(2));
587
588 f_2 = static_cast<type>(1.0) / (static_cast<type>(1.0) + combinations).pow(type(2));
589
590 activations_derivatives.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
591}
592
593
594void Layer::exponential_linear_derivatives(const Tensor<type, 1>& combinations,
595 Tensor<type, 1>& activations,
596 Tensor<type, 1>& activations_derivatives) const
597{
598
599 const type alpha = static_cast<type>(1.0);
600
601 const Tensor<bool, 1> if_sentence = combinations < combinations.constant(type(0));
602
603 Tensor<type, 1> f_1(combinations.dimension(0));
604
605 Tensor<type, 1> f_2(combinations.dimension(0));
606
607 f_1 = alpha*(combinations.exp() - static_cast<type>(1));
608
609 f_2 = combinations;
610
611 // Activations
612
613 activations.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
614
615 // Activations Derivatives
616
617 f_1 = alpha * combinations.exp();
618
619 f_2 = combinations.constant(type(1));
620
621 activations_derivatives.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
622}
623
624
625// Activations 2d
626
627void Layer::hard_sigmoid(const Tensor<type, 2>& x, Tensor<type, 2>& y) const
628{
629 const Index n = x.size();
630
631 #pragma omp parallel for
632
633 for(Index i = 0; i < n; i++)
634 {
635 if(x(i) < static_cast<type>(-2.5))
636 {
637 y(i) = type(0);
638 }
639 else if(x(i) > static_cast<type>(2.5))
640 {
641 y(i) = type(1);
642 }
643 else
644 {
645 y(i) = static_cast<type>(0.2) * x(i) + static_cast<type>(0.5);
646 }
647 }
648}
649
650
651void Layer::hyperbolic_tangent(const Tensor<type, 2>& x, Tensor<type, 2>& y) const
652{
653 y.device(*thread_pool_device) = x.tanh();
654}
655
656
657void Layer::logistic(const Tensor<type, 2>& x, Tensor<type, 2>& y)const
658{
659 y.device(*thread_pool_device) = (type(1) + x.exp().inverse()).inverse();
660}
661
662
663void Layer::linear(const Tensor<type, 2>& x, Tensor<type, 2>& y) const
664{
665 y = x;
666}
667
668
669void Layer::threshold(const Tensor<type, 2>& x, Tensor<type, 2>& y) const
670{
671 const Tensor<bool, 2> if_sentence = x >= x.constant(type(0));
672
673 Tensor<type, 2> ones(x.dimension(0), x.dimension(1));
674 ones.setConstant(type(1));
675
676 Tensor<type, 2> zeros(x.dimension(0), x.dimension(1));
677 zeros.setConstant(type(0));
678
679 y.device(*thread_pool_device) = if_sentence.select(ones, zeros);
680}
681
682
683void Layer::symmetric_threshold(const Tensor<type, 2>& x, Tensor<type, 2>& y) const
684{
685 const Tensor<bool, 2> if_sentence = x > x.constant(type(0));
686
687 Tensor<type, 2> ones(x.dimension(0), x.dimension(1));
688
689 ones.setConstant(type(1));
690
691 y.device(*thread_pool_device) = if_sentence.select(ones, -ones);
692}
693
694
695void Layer::rectified_linear(const Tensor<type, 2>& x, Tensor<type, 2>& y) const
696{
697 const Tensor<bool, 2> if_sentence = x < x.constant(type(0));
698
699 Tensor<type, 2> zeros(x.dimension(0), x.dimension(1));
700
701 zeros.setConstant(type(0));
702
703 y.device(*thread_pool_device) = if_sentence.select(zeros, x);
704}
705
706
707void Layer::scaled_exponential_linear(const Tensor<type, 2>& x, Tensor<type, 2>& y) const
708{
709 const type lambda = static_cast<type>(1.0507);
710
711 const type alpha = static_cast<type>(1.67326);
712
713 const Tensor<bool, 2> if_sentence = x < x.constant(type(0));
714
715 Tensor<type, 2> f_1(x.dimension(0), x.dimension(1));
716
717 Tensor<type, 2> f_2(x.dimension(0), x.dimension(1));
718
719 f_1 = lambda*alpha*(x.exp() - static_cast<type>(1.0));
720
721 f_2 = lambda*x;
722
723 y.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
724}
725
726
727void Layer::soft_plus(const Tensor<type, 2>& x, Tensor<type, 2>& y) const
728{
729 y.device(*thread_pool_device) = (x.constant(type(1)) + x.exp()).log();
730}
731
732
733void Layer::soft_sign(const Tensor<type, 2>& x, Tensor<type, 2>& y) const
734{
735 const Tensor<bool, 2> if_sentence = x < x.constant(type(0));
736
737 Tensor<type, 2> f_1(x.dimension(0), x.dimension(1));
738
739 Tensor<type, 2> f_2(x.dimension(0), x.dimension(1));
740
741 f_1 = x / (static_cast<type>(1) - x);
742
743 f_2 = x / (static_cast<type>(1) + x);
744
745 y.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
746}
747
748
749void Layer::exponential_linear(const Tensor<type, 2>& x, Tensor<type, 2>& y) const
750{
751 const Tensor<bool, 2> if_sentence = x < x.constant(type(0));
752
753 const type alpha = static_cast<type>(1.0);
754
755 Tensor<type, 2> f_1(x.dimension(0), x.dimension(1));
756
757 Tensor<type, 2> f_2(x.dimension(0), x.dimension(1));
758
759 f_1 = alpha*(x.exp() - static_cast<type>(1));
760
761 f_2 = x;
762
763 y.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
764}
765
766
767void Layer::binary(const Tensor<type, 2>& x, Tensor<type, 2>& y) const
768{
769 const Tensor<bool, 2> if_sentence = x < x.constant(type(0));
770
771 Tensor<type, 2> f_1(x.dimension(0), x.dimension(1));
772
773 Tensor<type, 2> f_2(x.dimension(0), x.dimension(1));
774
775 f_1 = x.constant(type(false));
776
777 f_2 = x.constant(type(true));
778
779 y.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
780}
781
782
784
785void Layer::competitive(const Tensor<type, 2>& x, Tensor<type, 2>& y) const
786{
787 const Index samples_number = x.dimension(0);
788
789 Index maximum_index = 0;
790
791 y.setZero();
792
793 for(Index i = 0; i < samples_number; i++)
794 {
795 maximum_index = maximal_index(x.chip(i, 1));
796
797 y(i, maximum_index) = type(1);
798 }
799
800}
801
802
803void Layer::softmax(const Tensor<type, 2>& x, Tensor<type, 2>& y) const
804{
805 const Index columns_number = x.dimension(1);
806
807 const Index rows_number = y.dimension(0);
808
809 // Activations
810
811 y.device(*thread_pool_device) = x.exp();
812
813 Tensor<type, 1> sums(rows_number);
814 sums.setZero();
815
816 for (Index i = 0; i < rows_number; i++)
817 {
818 for (Index j = 0; j < columns_number; j++)
819 {
820 sums[i] += y(i, j);
821 }
822 }
823
824 for (Index i = 0; i < rows_number; i++)
825 {
826 for (Index j = 0; j < columns_number; j++)
827 {
828 y(i, j) = y(i, j) / sums(i);
829 }
830 }
831}
832
833
834// Activations derivatives 2d
835
836void Layer::hard_sigmoid_derivatives(const Tensor<type, 2>& combinations,
837 Tensor<type, 2>& activations,
838 Tensor<type, 2>& activations_derivatives) const
839{
840 // Conditions
841
842 const Tensor<bool, 2> if_sentence = combinations < combinations.constant(type(-2.5));
843
844 const Tensor<bool, 2> elif_sentence = combinations > combinations.constant(type(2.5));
845
846 const Tensor<bool, 2> if_sentence_2 = combinations < combinations.constant(type(-2.5)) || combinations > combinations.constant(type(2.5));
847
848 // Sentences
849
850 Tensor<type, 2> f1(combinations.dimension(0), combinations.dimension(1));
851 f1.setZero();
852
853 Tensor<type, 2> f2(combinations.dimension(0), combinations.dimension(1));
854 f2.setConstant(type(1));
855
856 Tensor<type, 2> f3(combinations.dimension(0), combinations.dimension(1));
857 f3 = static_cast<type>(0.2) * combinations + static_cast<type>(0.5);
858
859 Tensor<type, 2> f4(combinations.dimension(0), combinations.dimension(1));
860 f4.setConstant(type(0));
861
862 Tensor<type, 2> f5(combinations.dimension(0), combinations.dimension(1));
863 f5.setConstant(static_cast<type>(0.2));
864
865 // Activations
866
867 activations.device(*thread_pool_device) = if_sentence.select(f1, elif_sentence.select(f2, f3));
868
869 // Activations Derivatives
870
871 activations_derivatives.device(*thread_pool_device) = if_sentence_2.select(f4, f5);
872}
873
874void Layer::hyperbolic_tangent_derivatives(const Tensor<type, 2>& combinations,
875 Tensor<type, 2>& activations,
876 Tensor<type, 2>& activations_derivatives) const
877{
878 // Activations
879
880 activations.device(*thread_pool_device) = combinations.tanh();
881
882 // Activations Derivatives
883
884 activations_derivatives.device(*thread_pool_device) = type(1) - activations.square();
885}
886
887
888void Layer::logistic_derivatives(const Tensor<type, 2>& combinations,
889 Tensor<type, 2>& activations,
890 Tensor<type, 2>& activations_derivatives) const
891{
892 // Activations
893
894 activations.device(*thread_pool_device) = (type(1) + combinations.exp().inverse()).inverse();
895
896 // Activations Derivatives
897
898 activations_derivatives.device(*thread_pool_device) = activations*(type(1) - activations);
899}
900
901
902void Layer::linear_derivatives(const Tensor<type, 2>& combinations,
903 Tensor<type, 2>& activations,
904 Tensor<type, 2>& activations_derivatives) const
905{
906 activations = combinations;
907
908 activations_derivatives.setConstant(type(1));
909}
910
911
912
913void Layer::threshold_derivatives(const Tensor<type, 2>& combinations,
914 Tensor<type, 2>& activations,
915 Tensor<type, 2>& activations_derivatives) const
916{
917
918 const Tensor<bool, 2> if_sentence = combinations > combinations.constant(type(0));
919
920 Tensor<type, 2> ones(combinations.dimension(0), combinations.dimension(1));
921 ones.setConstant(type(1));
922
923 Tensor<type, 2> zeros(combinations.dimension(0), combinations.dimension(1));
924 zeros.setConstant(type(0));
925
926 // Activations
927
928 activations.device(*thread_pool_device) = if_sentence.select(ones, zeros);
929
930 // Activations Derivatives
931
932 activations_derivatives.setZero();
933
934}
935
936
937void Layer::symmetric_threshold_derivatives(const Tensor<type, 2>& combinations,
938 Tensor<type, 2>& activations,
939 Tensor<type, 2>& activations_derivatives) const
940{
941
942 const Tensor<bool, 2> if_sentence = combinations > combinations.constant(type(0));
943
944 Tensor<type, 2> ones(combinations.dimension(0), combinations.dimension(1));
945
946 ones.setConstant(type(1));
947
948 // Activations
949
950 activations.device(*thread_pool_device) = if_sentence.select(ones, -ones);
951
952 // Activations Derivatives
953
954 activations_derivatives.setZero();
955}
956
957
958void Layer::rectified_linear_derivatives(const Tensor<type, 2>& combinations,
959 Tensor<type, 2>& activations,
960 Tensor<type, 2>& activations_derivatives) const
961{
962 const Tensor<bool, 2> if_sentence = combinations < combinations.constant(type(0));
963
964 Tensor<type, 2> zeros(combinations.dimension(0), combinations.dimension(1));
965 zeros.setZero();
966
967 Tensor<type, 2> ones(combinations.dimension(0), combinations.dimension(1));
968 ones.setConstant(type(1));
969
970 // Activations
971
972 activations.device(*thread_pool_device) = if_sentence.select(zeros, combinations);
973
974 // Activations Derivatives
975
976 activations_derivatives.device(*thread_pool_device) = if_sentence.select(zeros, ones);
977}
978
979
980void Layer::scaled_exponential_linear_derivatives(const Tensor<type, 2>& combinations,
981 Tensor<type, 2>& activations,
982 Tensor<type, 2>& activations_derivatives) const
983{
984 const type lambda = static_cast<type>(1.0507);
985
986 const type alpha = static_cast<type>(1.67326);
987
988 const Tensor<bool, 2> if_sentence = combinations < combinations.constant(type(0));
989
990 Tensor<type, 2> f_1(combinations.dimension(0), combinations.dimension(1));
991
992 Tensor<type, 2> f_2(combinations.dimension(0), combinations.dimension(1));
993
994 f_1 = lambda*alpha*(combinations.exp()-static_cast<type>(1.0));
995
996 f_2 = lambda*combinations;
997
998 // Activations
999
1000 activations.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1001
1002 // Activations Derivatives
1003
1004 f_1 = lambda*alpha*combinations.exp();
1005
1006 f_2 = combinations.constant(type(1))*lambda;
1007
1008 activations_derivatives.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1009
1010}
1011
1012
1013void Layer::soft_plus_derivatives(const Tensor<type, 2>& combinations,
1014 Tensor<type, 2>& activations,
1015 Tensor<type, 2>& activations_derivatives) const
1016{
1017 activations.device(*thread_pool_device)
1018 = (combinations.constant(type(1)) + combinations.exp()).log();
1019
1020 activations_derivatives.device(*thread_pool_device)
1021 = static_cast<type>(1.0) / (static_cast<type>(1.0) + combinations.exp().inverse());
1022}
1023
1024
1025void Layer::soft_sign_derivatives(const Tensor<type, 2>& combinations,
1026 Tensor<type, 2>& activations,
1027 Tensor<type, 2>& activations_derivatives) const
1028{
1029
1030 const Tensor<bool, 2> if_sentence = combinations < combinations.constant(type(0));
1031
1032 Tensor<type, 2> f_1(combinations.dimension(0), combinations.dimension(1));
1033
1034 Tensor<type, 2> f_2(combinations.dimension(0), combinations.dimension(1));
1035
1036 f_1 = combinations / (static_cast<type>(1) - combinations);
1037
1038 f_2 = combinations / (static_cast<type>(1) + combinations);
1039
1040 // Activations
1041
1042 activations.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1043
1044 // Activations Derivatives
1045
1046 f_1 = static_cast<type>(1.0) / (static_cast<type>(1.0) - combinations).pow(type(2));
1047
1048 f_2 = static_cast<type>(1.0) / (static_cast<type>(1.0) + combinations).pow(type(2));
1049
1050 activations_derivatives.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1051}
1052
1053
1054void Layer::exponential_linear_derivatives(const Tensor<type, 2>& combinations,
1055 Tensor<type, 2>& activations,
1056 Tensor<type, 2>& activations_derivatives) const
1057{
1058 const type alpha = static_cast<type>(1.0);
1059
1060 const Tensor<bool, 2> if_sentence = combinations < combinations.constant(type(0));
1061
1062 Tensor<type, 2> f_1(combinations.dimension(0), combinations.dimension(1));
1063
1064 Tensor<type, 2> f_2(combinations.dimension(0), combinations.dimension(1));
1065
1066 f_1 = alpha*(combinations.exp() - static_cast<type>(1));
1067
1068 f_2 = combinations;
1069
1070 // Activations
1071
1072 activations.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1073
1074 // Activations Derivatives
1075
1076 f_1 = alpha * combinations.exp();
1077
1078 f_2 = combinations.constant(type(1));
1079
1080 activations_derivatives.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1081}
1082
1083
1084void Layer::logistic_derivatives(const Tensor<type, 2>& combinations,
1085 Tensor<type, 2>& activations,
1086 Tensor<type, 3>& activations_derivatives) const
1087{
1088 // Activations
1089
1090 activations.device(*thread_pool_device) = (type(1) + combinations.exp().inverse()).inverse();
1091
1092 // Activations Derivatives
1093
1094 Tensor<type, 2> derivatives_2d(activations.dimension(0), activations.dimension(1));
1095
1096 derivatives_2d.device(*thread_pool_device) = activations*(type(1) - activations);
1097
1098 memcpy(activations_derivatives.data(), derivatives_2d.data(), static_cast<size_t>(derivatives_2d.size())*sizeof(type));
1099}
1100
1101
1102void Layer::softmax_derivatives(const Tensor<type, 2>& combinations,
1103 Tensor<type, 2>& activations,
1104 Tensor<type, 3>& activations_derivatives) const
1105{
1106 const Index dim = combinations.dimension(1);
1107
1108 const Index matrix_number = activations.dimension(0);
1109
1110 //Activations
1111
1112 softmax(combinations, activations);
1113
1114 /*type delta = type(0);
1115 Index index= 0;
1116
1117 for(Index i = 0; i < activations_derivatives.dimension(0); i++)
1118 {
1119 for(Index j = 0; j < activations_derivatives.dimension(1); j++)
1120 {
1121 for(Index k = 0; k < activations_derivatives.dimension(2); k++)
1122 {
1123 (i == j) ? delta = type(1) : delta = type(0);
1124
1125 activations_derivatives(i,j,k) = activations(i,k) * (delta - activations(k,j));
1126 }
1127 }
1128 }*/
1129
1130 type delta = type(0);
1131 Index index= 0;
1132
1133 for(Index row = 0; row < matrix_number; row++)
1134 {
1135 for(Index i = 0; i < dim; i++)
1136 {
1137 for(Index j = 0; j < dim; j++)
1138 {
1139 (i == j) ? delta = type(1) : delta = type(0);
1140
1141 // row, i, j
1142
1143 activations_derivatives(index) = activations(row,j) * (delta - activations(row,i));
1144 index++;
1145 }
1146 }
1147 }
1148/*
1149 // Activations derivatives
1150
1151 Tensor<type,2> diagonal_matrix(dim,dim);
1152 Tensor<type,1> row(dim);
1153 Tensor<type,2> temp_matrix(dim,dim);
1154
1155 diagonal_matrix.setZero();
1156 sum_diagonal(diagonal_matrix,type(1));
1157
1158 for(Index row_index = 0; row_index < matrix_number; row_index++)
1159 {
1160 row = activations.chip(row_index,0);
1161
1162 temp_matrix =
1163 row.reshape(Eigen::array<Index,2>({dim,1})).broadcast(Eigen::array<Index,2>({1,dim}))
1164 * (diagonal_matrix- row.reshape(Eigen::array<Index,2>({1,dim})).broadcast(Eigen::array<Index,2>({dim,1})));
1165
1166 for(Index i = 0; i < dim_2 ; i++)
1167 {
1168 activations_derivatives(i + dim_2 * row_index) = temp_matrix(i);
1169 }
1170
1171 memcpy(activations_derivatives.data()+(dim_2)*row_index,
1172 temp_matrix.data(),
1173 static_cast<size_t>(dim_2)*sizeof(type));
1174 }
1175 */
1176}
1177
1178
1179// Activations 4d
1180
1181void Layer::linear(const Tensor<type, 4>& x, Tensor<type, 4>& y) const
1182{
1183 y = x;
1184}
1185
1186
1187void Layer::logistic(const Tensor<type, 4>& x, Tensor<type, 4>& y)const
1188{
1189 y.device(*thread_pool_device) = (type(1) + x.exp().inverse()).inverse();
1190}
1191
1192
1193void Layer::hyperbolic_tangent(const Tensor<type, 4>& x, Tensor<type, 4>& y) const
1194{
1195 y.device(*thread_pool_device) = x.tanh();
1196}
1197
1198
1199void Layer::threshold(const Tensor<type, 4>& x, Tensor<type, 4>& y) const
1200{
1201 const Tensor<bool, 4> if_sentence = (x >= x.constant(type(0)));
1202
1203 Tensor<type, 4> ones(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1204 ones.setConstant(type(1));
1205
1206 Tensor<type, 4> zeros(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1207 zeros.setConstant(type(0));
1208
1209 y.device(*thread_pool_device) = if_sentence.select(ones, zeros);
1210}
1211
1212
1213void Layer::symmetric_threshold(const Tensor<type, 4>& x, Tensor<type, 4>& y) const
1214{
1215 const Tensor<bool, 4> if_sentence = x > x.constant(type(0));
1216
1217 Tensor<type, 4> ones(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1218
1219 ones.setConstant(type(1));
1220
1221 y.device(*thread_pool_device) = if_sentence.select(ones, -ones);
1222}
1223
1224
1225void Layer::rectified_linear(const Tensor<type, 4>& x, Tensor<type, 4>& y) const
1226{
1227 const Tensor<bool, 4> if_sentence = x < x.constant(type(0));
1228
1229 Tensor<type, 4> zeros(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1230
1231 zeros.setConstant(type(0));
1232
1233 y.device(*thread_pool_device) = if_sentence.select(zeros, x);
1234}
1235
1236
1237void Layer::scaled_exponential_linear(const Tensor<type, 4>& x, Tensor<type, 4>& y) const
1238{
1239 const type lambda = static_cast<type>(1.0507);
1240
1241 const type alpha = static_cast<type>(1.67326);
1242
1243 const Tensor<bool, 4> if_sentence = x < x.constant(type(0));
1244
1245 Tensor<type, 4> f_1(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1246
1247 Tensor<type, 4> f_2(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1248
1249 f_1 = lambda*alpha*(x.exp() - static_cast<type>(1.0));
1250
1251 f_2 = lambda*x;
1252
1253 y.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1254}
1255
1256
1257void Layer::soft_plus(const Tensor<type, 4>& x, Tensor<type, 4>& y) const
1258{
1259 y.device(*thread_pool_device) = (x.constant(type(1)) + x.exp()).log();
1260}
1261
1262
1263void Layer::soft_sign(const Tensor<type, 4>& x, Tensor<type, 4>& y) const
1264{
1265 const Tensor<bool, 4> if_sentence = x < x.constant(type(0));
1266
1267 Tensor<type, 4> f_1(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1268
1269 Tensor<type, 4> f_2(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1270
1271 f_1 = x / (static_cast<type>(1) - x);
1272
1273 f_2 = x / (static_cast<type>(1) + x);
1274
1275 y.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1276}
1277
1278
1280
1281void Layer::hard_sigmoid(const Tensor<type, 4>& x, Tensor<type, 4>& y) const
1282{
1283 const Tensor<bool, 4> if_lower = x < x.constant(type(-2.5));
1284 const Tensor<bool, 4> if_greater = x > x.constant(type(2.5));
1285 const Tensor<bool, 4> if_middle = x < x.constant(type(-2.5)) && x > x.constant(type(2.5));
1286
1287 Tensor<type, 4> f_lower(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1288 Tensor<type, 4> f_greater(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1289 Tensor<type, 4> f_middle(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1290 Tensor<type, 4> f_equal(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1291
1292 f_lower = x.constant(type(0));
1293 f_greater = x.constant(type(1));
1294 f_middle = static_cast<type>(0.2) * x + static_cast<type>(0.5);
1295 f_equal = x;
1296
1297 y.device(*thread_pool_device) = if_lower.select(f_lower, f_equal);
1298 y.device(*thread_pool_device) = if_greater.select(f_greater, f_equal);
1299 y.device(*thread_pool_device) = if_middle.select(f_middle, f_equal);
1300}
1301
1302
1303void Layer::exponential_linear(const Tensor<type, 4>& x, Tensor<type, 4>& y) const
1304{
1305
1306 const Tensor<bool, 4> if_sentence = x < x.constant(type(0));
1307
1308 const type alpha = static_cast<type>(1.0);
1309
1310 Tensor<type, 4> f_1(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1311
1312 Tensor<type, 4> f_2(x.dimension(0), x.dimension(1), x.dimension(2), x.dimension(3));
1313
1314 f_1 = alpha*(x.exp() - static_cast<type>(1));
1315
1316 f_2 = x;
1317
1318 y.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1319}
1320
1321
1322void Layer::linear_derivatives(const Tensor<type, 4>& combinations,
1323 Tensor<type, 4>& activations,
1324 Tensor<type, 4>& activations_derivatives) const
1325{
1326 activations = combinations;
1327
1328 activations_derivatives.setConstant(type(1));
1329}
1330
1331
1332void Layer::logistic_derivatives(const Tensor<type, 4>& combinations,
1333 Tensor<type, 4>& activations,
1334 Tensor<type, 4>& activations_derivatives) const
1335{
1336 // Activations
1337
1338 activations.device(*thread_pool_device) = (type(1) + combinations.exp().inverse()).inverse();
1339
1340 // Activations Derivatives
1341
1342 activations_derivatives.device(*thread_pool_device) = activations*(type(1) - activations);
1343}
1344
1345
1346void Layer::hyperbolic_tangent_derivatives(const Tensor<type, 4>& combinations,
1347 Tensor<type, 4>& activations,
1348 Tensor<type, 4>& activations_derivatives) const
1349{
1350 // Activations
1351
1352 activations.device(*thread_pool_device) = combinations.tanh();
1353
1354 // Activations Derivatives
1355
1356 activations_derivatives.device(*thread_pool_device) = type(1) - activations.square();
1357}
1358
1359
1360void Layer::threshold_derivatives(const Tensor<type, 4>& combinations,
1361 Tensor<type, 4>& activations,
1362 Tensor<type, 4>& activations_derivatives) const
1363{
1364 const Tensor<bool, 4> if_sentence = combinations > combinations.constant(type(0));
1365
1366 Tensor<type, 4> ones(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1367 ones.setConstant(type(1));
1368
1369 Tensor<type, 4> zeros(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1370 zeros.setConstant(type(0));
1371
1372 // Activations
1373
1374 activations.device(*thread_pool_device) = if_sentence.select(ones, zeros);
1375
1376 // Activations Derivatives
1377
1378 activations_derivatives.setZero();
1379}
1380
1381
1382void Layer::symmetric_threshold_derivatives(const Tensor<type, 4>& combinations,
1383 Tensor<type, 4>& activations,
1384 Tensor<type, 4>& activations_derivatives) const
1385{
1386 const Tensor<bool, 4> if_sentence = combinations > combinations.constant(type(0));
1387
1388 Tensor<type, 4> ones(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1389
1390 ones.setConstant(type(1));
1391
1392 // Activations
1393
1394 activations.device(*thread_pool_device) = if_sentence.select(ones, -ones);
1395
1396 // Activations Derivatives
1397
1398 activations_derivatives.setZero();
1399}
1400
1401
1402void Layer::rectified_linear_derivatives(const Tensor<type, 4>& combinations,
1403 Tensor<type, 4>& activations,
1404 Tensor<type, 4>& activations_derivatives) const
1405{
1406
1407 const Tensor<bool, 4> if_sentence = combinations < combinations.constant(type(0));
1408
1409 Tensor<type, 4> zeros(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1410 zeros.setZero();
1411
1412 Tensor<type, 4> ones(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1413 ones.setConstant(type(1));
1414
1415 // Activations
1416
1417 activations.device(*thread_pool_device) = if_sentence.select(zeros, combinations);
1418
1419 // Activations Derivatives
1420
1421 activations_derivatives.device(*thread_pool_device) = if_sentence.select(zeros, ones);
1422}
1423
1424
1425void Layer::scaled_exponential_linear_derivatives(const Tensor<type, 4>& combinations,
1426 Tensor<type, 4>& activations,
1427 Tensor<type, 4>& activations_derivatives) const
1428{
1429
1430 const type lambda = static_cast<type>(1.0507);
1431
1432 const type alpha = static_cast<type>(1.67326);
1433
1434 const Tensor<bool, 4> if_sentence = combinations < combinations.constant(type(0));
1435
1436 Tensor<type, 4> f_1(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1437
1438 Tensor<type, 4> f_2(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1439
1440 f_1 = lambda*alpha*(combinations.exp()-static_cast<type>(1.0));
1441
1442 f_2 = lambda*combinations;
1443
1444 // Activations
1445
1446 activations.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1447
1448 // Activations Derivatives
1449
1450 f_1 = lambda*alpha*combinations.exp();
1451
1452 f_2 = combinations.constant(type(1))*lambda;
1453
1454 activations_derivatives.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1455
1456}
1457
1458
1459void Layer::soft_plus_derivatives(const Tensor<type, 4>& combinations,
1460 Tensor<type, 4>& activations,
1461 Tensor<type, 4>& activations_derivatives) const
1462{
1463
1464 // Activations
1465
1466 activations.device(*thread_pool_device) = (combinations.constant(type(1)) + combinations.exp()).log();
1467
1468 // Activations Derivatives
1469
1470 activations_derivatives.device(*thread_pool_device) = static_cast<type>(1.0) / (static_cast<type>(1.0) + combinations.exp().inverse());
1471
1472}
1473
1474
1475void Layer::soft_sign_derivatives(const Tensor<type, 4>& combinations,
1476 Tensor<type, 4>& activations,
1477 Tensor<type, 4>& activations_derivatives) const
1478{
1479
1480 const Tensor<bool, 4> if_sentence = combinations < combinations.constant(type(0));
1481
1482 Tensor<type, 4> f_1(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1483
1484 Tensor<type, 4> f_2(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1485
1486 f_1 = combinations / (static_cast<type>(1) - combinations);
1487
1488 f_2 = combinations / (static_cast<type>(1) + combinations);
1489
1490 // Activations
1491
1492 activations.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1493
1494 // Activations Derivatives
1495
1496 f_1 = static_cast<type>(1.0) / (static_cast<type>(1.0) - combinations).pow(type(2));
1497
1498 f_2 = static_cast<type>(1.0) / (static_cast<type>(1.0) + combinations).pow(type(2));
1499
1500 activations_derivatives.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1501
1502}
1503
1504
1505void Layer::hard_sigmoid_derivatives(const Tensor<type, 4>& combinations,
1506 Tensor<type, 4>& activations,
1507 Tensor<type, 4>& activations_derivatives) const
1508{
1509
1510 // Conditions
1511
1512 const Tensor<bool, 4> if_sentence = combinations < combinations.constant(type(-2.5));
1513 const Tensor<bool, 4> elif_sentence = combinations > combinations.constant(type(2.5));
1514 const Tensor<bool, 4> if_sentence_2 = combinations < combinations.constant(type(-2.5)) || combinations > combinations.constant(type(2.5));
1515
1516 // Sentences
1517
1518 Tensor<type, 4> f1(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1519 f1.setZero();
1520
1521 Tensor<type, 4> f2(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1522 f2.setConstant(type(1));
1523
1524 Tensor<type, 4> f3(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1525 f3 = static_cast<type>(0.2) * combinations + static_cast<type>(0.5);
1526
1527 Tensor<type, 4> f4(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1528 f4.setConstant(type(0));
1529
1530 Tensor<type, 4> f5(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1531 f5.setConstant(static_cast<type>(0.2));
1532
1533 // Activations
1534
1535 activations.device(*thread_pool_device) = if_sentence.select(f1, elif_sentence.select(f2, f3));
1536
1537 // Activations derivatives
1538
1539 activations_derivatives.device(*thread_pool_device) = if_sentence_2.select(f4, f5);
1540}
1541
1542
1543void Layer::exponential_linear_derivatives(const Tensor<type, 4>& combinations,
1544 Tensor<type, 4>& activations,
1545 Tensor<type, 4>& activations_derivatives) const
1546{
1547
1548 const type alpha = static_cast<type>(1.0);
1549
1550 const Tensor<bool, 4> if_sentence = combinations < combinations.constant(type(0));
1551
1552 Tensor<type, 4> f_1(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1553
1554 Tensor<type, 4> f_2(combinations.dimension(0), combinations.dimension(1), combinations.dimension(2), combinations.dimension(3));
1555
1556 f_1 = alpha*(combinations.exp() - static_cast<type>(1));
1557
1558 f_2 = combinations;
1559
1560 // Activations
1561
1562 activations.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1563
1564 // Activations Derivatives
1565
1566 f_1 = alpha * combinations.exp();
1567
1568 f_2 = combinations.constant(type(1));
1569
1570 activations_derivatives.device(*thread_pool_device) = if_sentence.select(f_1, f_2);
1571}
1572
1573}
1574
1575// OpenNN: Open Neural Networks Library.
1576// Copyright(C) 2005-2021 Artificial Intelligence Techniques, SL.
1577//
1578// This library is free software; you can redistribute it and/or
1579// modify it under the terms of the GNU Lesser General Public
1580// License as published by the Free Software Foundation; either
1581// version 2.1 of the License, or any later version.
1582//
1583// This library is distributed in the hope that it will be useful,
1584// but WITHOUT ANY WARRANTY; without even the implied warranty of
1585// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
1586// Lesser General Public License for more details.
1587
1588// You should have received a copy of the GNU Lesser General Public
1589// License along with this library; if not, write to the Free Software
1590// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
OpenNN::Layer::Type
Type
This enumeration represents the possible types of layers.
Definition: layer.h:61
OpenNN::Layer::get_synaptic_weights_number
virtual Index get_synaptic_weights_number() const
Returns the number of layer's synaptic weights.
Definition: layer.cpp:179
OpenNN::Layer::get_inputs_number
virtual Index get_inputs_number() const
Returns the number of inputs.
Definition: layer.cpp:153
OpenNN::Layer::layer_type
Type layer_type
Layer type.
Definition: layer.h:183
OpenNN::Layer::get_type
Type get_type() const
Definition: layer.cpp:25
OpenNN::Layer::get_type_string
string get_type_string() const
Takes the type of layer used by the model.
Definition: layer.cpp:33
half_float::log
half log(half arg)
Definition: half.hpp:3050
half_float::pow
half pow(half x, half y)
Definition: half.hpp:3427