testing_analysis.cpp
1// OpenNN: Open Neural Networks Library
2// www.opennn.net
3//
4// T E S T I N G A N A L Y S I S C L A S S
5//
6// Artificial Intelligence Techniques SL
7// artelnics@artelnics.com
8
9#include "testing_analysis.h"
10
11namespace OpenNN
12{
13
17
19 : neural_network_pointer(nullptr),
20 data_set_pointer(nullptr)
21{
23}
24
25
31
32TestingAnalysis::TestingAnalysis(NeuralNetwork* new_neural_network_pointer, DataSet* new_data_set_pointer)
33{
34 data_set_pointer = new_data_set_pointer;
35
36 neural_network_pointer = new_neural_network_pointer;
37
39}
40
41
44
46{
47 delete thread_pool;
48 delete thread_pool_device;
49}
50
51
53
55{
56#ifdef OPENNN_DEBUG
57
59 {
60 ostringstream buffer;
61
62 buffer << "OpenNN Exception: TestingAnalysis class.\n"
63 << "NeuralNetwork* get_neural_network_pointer() const method.\n"
64 << "Neural network pointer is nullptr.\n";
65
66 throw logic_error(buffer.str());
67 }
68
69#endif
70
72}
73
74
76
78{
79#ifdef OPENNN_DEBUG
80
82 {
83 ostringstream buffer;
84
85 buffer << "OpenNN Exception: TestingAnalysis class.\n"
86 << "DataSet* get_data_set_pointer() const method.\n"
87 << "Data set pointer is nullptr.\n";
88
89 throw logic_error(buffer.str());
90 }
91
92#endif
93
94 return data_set_pointer;
95}
96
97
100
102{
103 return display;
104}
105
106
111
113{
114 delete thread_pool;
115 delete thread_pool_device;
116
117 const int n = omp_get_max_threads();
118 thread_pool = new ThreadPool(n);
119 thread_pool_device = new ThreadPoolDevice(thread_pool, n);
120}
121
122
123void TestingAnalysis::set_threads_number(const int& new_threads_number)
124{
125 if(thread_pool != nullptr) delete this->thread_pool;
126 if(thread_pool_device != nullptr) delete this->thread_pool_device;
127
128 thread_pool = new ThreadPool(new_threads_number);
129 thread_pool_device = new ThreadPoolDevice(thread_pool, new_threads_number);
130}
131
132
135
137{
138 neural_network_pointer = new_neural_network_pointer;
139}
140
141
144
146{
147 data_set_pointer = new_data_set_pointer;
148}
149
150
155
156void TestingAnalysis::set_display(const bool& new_display)
157{
158 display = new_display;
159}
160
161
167
169{
170 ostringstream buffer;
171
173 {
174 buffer << "OpenNN Exception: TestingAnalysis class.\n"
175 << "void check() const method.\n"
176 << "Neural network pointer is nullptr.\n";
177
178 throw logic_error(buffer.str());
179 }
180
182 {
183 buffer << "OpenNN Exception: TestingAnalysis class.\n"
184 << "void check() const method.\n"
185 << "Data set pointer is nullptr.\n";
186
187 throw logic_error(buffer.str());
188 }
189}
190
191
198
199Tensor<Correlation, 1> TestingAnalysis::linear_correlation() const
200{
201#ifdef OPENNN_DEBUG
202
203 check();
204
205 const Index testing_samples_number = data_set_pointer->get_testing_samples_number();
206
207 ostringstream buffer;
208
209 if(testing_samples_number == 0)
210 {
211 buffer << "OpenNN Exception: TestingAnalysis class.\n"
212 << "Tensor<Correlation, 1> linear_correlation() const method.\n"
213 << "Number of testing samples is zero.\n";
214
215 throw logic_error(buffer.str());
216 }
217
218#endif
219
220 // Calculate regression parameters
221
222 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
223
224 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
225
226 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
227
228 return linear_correlation(targets,outputs);
229}
230
231
232Tensor<Correlation, 1> TestingAnalysis::linear_correlation(const Tensor<type, 2>& target, const Tensor<type, 2>& output) const
233{
234 const Index outputs_number = data_set_pointer->get_target_variables_number();
235
236 Tensor<Correlation, 1> linear_correlation(outputs_number);
237
238 for(Index i = 0; i < outputs_number; i++)
239 {
240 linear_correlation[i] = OpenNN::linear_correlation(thread_pool_device, output.chip(i,1), target.chip(i,1));
241 }
242
243 return linear_correlation;
244}
245
246
247void TestingAnalysis::print_linear_regression_correlations() const
248{
249 const Tensor<Correlation, 1> linear_correlation = this->linear_correlation();
250
251 const Tensor<string, 1> targets_name = data_set_pointer->get_target_variables_names();
252
253 const Index targets_number = linear_correlation.size();
254
255 for(Index i = 0; i < targets_number; i++)
256 {
257 cout << targets_name[i] << " correlation: " << linear_correlation[i].r << endl;
258 }
259}
260
261
268
269Tensor<TestingAnalysis::LinearRegressionAnalysis, 1> TestingAnalysis::perform_linear_regression_analysis() const
270{
271 check();
272
273 const Index outputs_number = neural_network_pointer->get_outputs_number();
274
275 const Index testing_samples_number = data_set_pointer->get_testing_samples_number();
276
277 if(testing_samples_number == 0)
278 {
279 ostringstream buffer;
280
281 buffer << "OpenNN Exception: TestingAnalysis class.\n"
282 << "LinearRegressionResults perform_linear_regression_analysis() const method.\n"
283 << "Number of testing samples is zero.\n";
284
285 throw logic_error(buffer.str());
286 }
287
288 const Tensor<type, 2> testing_inputs = data_set_pointer->get_testing_input_data();
289
290 const Tensor<type, 2> testing_targets = data_set_pointer->get_testing_target_data();
291
292 // Neural network
293
294 const Tensor<type, 2> testing_outputs = neural_network_pointer->calculate_outputs(testing_inputs);
295
296 // Approximation testing stuff
297
298 Tensor<LinearRegressionAnalysis, 1> linear_regression_results(outputs_number);
299
300 for(Index i = 0; i < outputs_number; i++)
301 {
302 const Tensor<type, 1> targets = testing_targets.chip(i,1);
303 const Tensor<type, 1> outputs = testing_outputs.chip(i,1);
304
305 const Correlation linear_correlation = OpenNN::linear_correlation(thread_pool_device, outputs, targets);
306
307 linear_regression_results[i].targets = targets;
308 linear_regression_results[i].outputs = outputs;
309
310 linear_regression_results[i].intercept = linear_correlation.a;
311 linear_regression_results[i].slope = linear_correlation.b;
312 linear_regression_results[i].correlation = linear_correlation.r;
313 }
314
315 return linear_regression_results;
316}
317
318
319void TestingAnalysis::print_linear_regression_analysis() const
320{
321 const Tensor<LinearRegressionAnalysis, 1> linear_regression_analysis = perform_linear_regression_analysis();
322
323 for(Index i = 0; i < linear_regression_analysis.size(); i++)
324 {
325 linear_regression_analysis(i).print();
326 }
327}
328
329
339
341{
342 // Data set
343
344#ifdef OPENNN_DEBUG
345
346 check();
347
348#endif
349
350 const Index testing_samples_number = data_set_pointer->get_testing_samples_number();
351
352#ifdef OPENNN_DEBUG
353
354 ostringstream buffer;
355
356 if(testing_samples_number == 0)
357 {
358 buffer << "OpenNN Exception: TestingAnalysis class.\n"
359 << "Tensor<Tensor<type, 2>, 1> calculate_error_data() const.\n"
360 << "Number of testing samples is zero.\n";
361
362 throw logic_error(buffer.str());
363 }
364
365#endif
366
367 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
368
369 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
370
371 // Neural network
372
373 const Index outputs_number = neural_network_pointer->get_outputs_number();
374
375 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
376
377 const UnscalingLayer* unscaling_layer_pointer = neural_network_pointer->get_unscaling_layer_pointer();
378
379#ifdef OPENNN_DEBUG
380
381 if(!unscaling_layer_pointer)
382 {
383 buffer << "OpenNN Exception: TestingAnalysis class.\n"
384 << "Tensor<Tensor<type, 2>, 1> calculate_error_data() const.\n"
385 << "Unscaling layer is nullptr.\n";
386
387 throw logic_error(buffer.str());
388 }
389
390#endif
391
392 const Tensor<type, 1>& outputs_minimum = unscaling_layer_pointer->get_minimums();
393
394 const Tensor<type, 1>& outputs_maximum = unscaling_layer_pointer->get_maximums();
395
396 // Error data
397
398 Tensor<type, 3> error_data(testing_samples_number, 3, outputs_number);
399
400 // Absolute error
401
402 Tensor<type, 2> difference_absolute_value = (targets - outputs).abs();
403
404 for(Index i = 0; i < outputs_number; i++)
405 {
406 for(Index j = 0; j < testing_samples_number; j++)
407 {
408 // Absolute error
409 error_data(j,0,i) = difference_absolute_value(j,i);
410 // Relative error
411 error_data(j,1,i) = difference_absolute_value(j,i)/abs(outputs_maximum(i)-outputs_minimum(i));
412 // Percentage error
413 error_data(j,2,i) = (difference_absolute_value(j,i)*static_cast<type>(100.0))/abs(outputs_maximum(i)-outputs_minimum(i));
414 }
415 }
416
417 return error_data;
418}
419
420
423
425{
426 // Data set
427
428#ifdef OPENNN_DEBUG
429
430 check();
431
432#endif
433
434 const Index testing_samples_number = data_set_pointer->get_testing_samples_number();
435
436#ifdef OPENNN_DEBUG
437
438 ostringstream buffer;
439
440 if(testing_samples_number == 0)
441 {
442 buffer << "OpenNN Exception: TestingAnalysis class.\n"
443 << "Tensor<Tensor<type, 2>, 1> calculate_error_data() const.\n"
444 << "Number of testing samples is zero.\n";
445
446 throw logic_error(buffer.str());
447 }
448
449#endif
450
451 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
452
453 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
454
455 // Neural network
456
457 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
458
459 const UnscalingLayer* unscaling_layer_pointer = neural_network_pointer->get_unscaling_layer_pointer();
460
461#ifdef OPENNN_DEBUG
462
463 if(!unscaling_layer_pointer)
464 {
465 buffer << "OpenNN Exception: TestingAnalysis class.\n"
466 << "Tensor<Tensor<type, 1>, 1> calculate_percentage_error_data() const.\n"
467 << "Unscaling layer is nullptr.\n";
468
469 throw logic_error(buffer.str());
470 }
471
472#endif
473
474 const Tensor<type, 1>& outputs_minimum = unscaling_layer_pointer->get_minimums();
475 const Tensor<type, 1>& outputs_maximum = unscaling_layer_pointer->get_maximums();
476
477 const Index outputs_number = neural_network_pointer->get_outputs_number();
478
479 // Error data
480
481 Tensor<type, 2> error_data(testing_samples_number, outputs_number);
482
483 Tensor<type, 2> difference_value = (targets - outputs);
484
485 for(Index i = 0; i < testing_samples_number; i++)
486 {
487 for(Index j = 0; j < outputs_number; j++)
488 {
489 error_data(i,j) = (difference_value(i,j)*static_cast<type>(100.0))/abs(outputs_maximum(j)-outputs_minimum(j));
490 }
491 }
492
493 return error_data;
494}
495
496
497Tensor<Descriptives, 1> TestingAnalysis::calculate_absolute_errors_descriptives() const
498{
499 // Data set
500
501 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
502 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
503
504 // Neural network
505
506 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
507
508 // Error descriptives
509
510 return calculate_absolute_errors_descriptives(targets, outputs);
511}
512
513
514Tensor<Descriptives, 1> TestingAnalysis::calculate_absolute_errors_descriptives(const Tensor<type, 2>& targets,
515 const Tensor<type, 2>& outputs) const
516{
517 const Tensor<type, 2> diff = (targets-outputs).abs();
518
519 return descriptives(diff);
520}
521
522
523Tensor<Descriptives, 1> TestingAnalysis::calculate_percentage_errors_descriptives() const
524{
525 // Data set
526
527 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
528 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
529
530 // Neural network
531
532 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
533
534 // Error descriptives
535
536 return calculate_percentage_errors_descriptives(targets,outputs);
537}
538
539
540Tensor<Descriptives, 1> TestingAnalysis::calculate_percentage_errors_descriptives(const Tensor<type, 2>& targets,
541 const Tensor<type, 2>& outputs) const
542{
543 const Tensor<type, 2> diff = (static_cast<type>(100)*(targets-outputs).abs())/targets;
544
545 return descriptives(diff);
546}
547
548
556
557Tensor<Tensor<Descriptives, 1>, 1> TestingAnalysis::calculate_error_data_descriptives() const
558{
559 // Neural network
560
561 const Index outputs_number = neural_network_pointer->get_outputs_number();
562
563 const Index testing_samples_number = data_set_pointer->get_testing_samples_number();
564
565 // Testing analysis stuff
566
567 Tensor<Tensor<Descriptives, 1>, 1> descriptives(outputs_number);
568
569 Tensor<type, 3> error_data = calculate_error_data();
570
571 Index index = 0;
572
573 for(Index i = 0; i < outputs_number; i++)
574 {
575 TensorMap< Tensor<type, 2> > matrix_error(error_data.data()+index, testing_samples_number, 3);
576
577 Tensor<type, 2> matrix(matrix_error);
578
579 descriptives[i] = OpenNN::descriptives(matrix);
580
581 index += testing_samples_number*3;
582 }
583
584 return descriptives;
585}
586
587
588void TestingAnalysis::print_error_data_descriptives() const
589{
590 const Index targets_number = data_set_pointer->get_target_variables_number();
591
592 const Tensor<string, 1> targets_name = data_set_pointer->get_target_variables_names();
593
594 const Tensor<Tensor<Descriptives, 1>, 1> error_data_statistics = calculate_error_data_descriptives();
595
596 for(Index i = 0; i < targets_number; i++)
597 {
598 cout << targets_name[i] << endl;
599 cout << "Minimum error: " << error_data_statistics[i][0].minimum << endl;
600 cout << "Maximum error: " << error_data_statistics[i][0].maximum << endl;
601 cout << "Mean error: " << error_data_statistics[i][0].mean << " " << endl;
602 cout << "Standard deviation error: " << error_data_statistics[i][0].standard_deviation << " " << endl;
603
604 cout << "Minimum percentage error: " << error_data_statistics[i][2].minimum << " %" << endl;
605 cout << "Maximum percentage error: " << error_data_statistics[i][2].maximum << " %" << endl;
606 cout << "Mean percentage error: " << error_data_statistics[i][2].mean << " %" << endl;
607 cout << "Standard deviation percentage error: " << error_data_statistics[i][2].standard_deviation << " %" << endl;
608 cout << endl;
609 }
610}
611
612
616
617Tensor<Histogram, 1> TestingAnalysis::calculate_error_data_histograms(const Index& bins_number) const
618{
619 const Tensor<type, 2> error_data = calculate_percentage_error_data();
620
621 const Index outputs_number = error_data.dimension(1);
622
623 Tensor<Histogram, 1> histograms(outputs_number);
624
625 for(Index i = 0; i < outputs_number; i++)
626 {
627 histograms(i) = histogram_centered(error_data.chip(i,1), type(0), bins_number);
628 }
629
630 return histograms;
631}
632
633
636
637Tensor<Tensor<Index, 1>, 1> TestingAnalysis::calculate_maximal_errors(const Index& samples_number) const
638{
639 Tensor<type, 3> error_data = calculate_error_data();
640
641 const Index outputs_number = error_data.dimension(2);
642 const Index testing_samples_number = error_data.dimension(0);
643
644 Tensor<Tensor<Index, 1>, 1> maximal_errors(samples_number);
645
646 Index index = 0;
647
648 for(Index i = 0; i < outputs_number; i++)
649 {
650 TensorMap< Tensor<type, 2> > matrix_error(error_data.data()+index, testing_samples_number, 3);
651
652 maximal_errors[i] = maximal_indices(matrix_error.chip(0,1), samples_number);
653
654 index += testing_samples_number*3;
655 }
656
657 return maximal_errors;
658}
659
660
663
665{
666 Tensor<type, 2> errors(5,3);
667
668 const Tensor<type, 1> training_errors = calculate_training_errors();
669 const Tensor<type, 1> selection_errors = calculate_selection_errors();
670 const Tensor<type, 1> testing_errors = calculate_testing_errors();
671
672 errors(0,0) = training_errors(0);
673 errors(1,0) = training_errors(1);
674 errors(2,0) = training_errors(2);
675 errors(3,0) = training_errors(3);
676 errors(4,0) = training_errors(4);
677
678 errors(0,1) = selection_errors(0);
679 errors(1,1) = selection_errors(1);
680 errors(2,1) = selection_errors(2);
681 errors(3,1) = selection_errors(3);
682 errors(4,1) = selection_errors(4);
683
684 errors(0,2) = testing_errors(0);
685 errors(1,2) = testing_errors(1);
686 errors(2,2) = testing_errors(2);
687 errors(3,2) = testing_errors(3);
688 errors(4,2) = testing_errors(4);
689
690 return errors;
691}
692
693
694Tensor<type, 2> TestingAnalysis::calculate_binary_classification_errors() const
695{
696 Tensor<type, 2> errors(7, 3);
697
698 const Tensor<type, 1> training_errors = calculate_binary_classification_training_errors();
699 const Tensor<type, 1> selection_errors = calculate_binary_classification_selection_errors();
700 const Tensor<type, 1> testing_errors = calculate_binary_classification_testing_errors();
701
702 errors(0,0) = training_errors(0);
703 errors(1,0) = training_errors(1);
704 errors(2,0) = training_errors(2);
705 errors(3,0) = training_errors(3);
706 errors(4,0) = training_errors(4);
707 errors(5,0) = training_errors(5);
708 errors(6,0) = training_errors(6);
709
710 errors(0,1) = selection_errors(0);
711 errors(1,1) = selection_errors(1);
712 errors(2,1) = selection_errors(2);
713 errors(3,1) = selection_errors(3);
714 errors(4,1) = selection_errors(4);
715 errors(5,1) = selection_errors(5);
716 errors(6,1) = selection_errors(6);
717
718 errors(0,2) = testing_errors(0);
719 errors(1,2) = testing_errors(1);
720 errors(2,2) = testing_errors(2);
721 errors(3,2) = testing_errors(3);
722 errors(4,2) = testing_errors(4);
723 errors(5,2) = testing_errors(5);
724 errors(6,2) = testing_errors(6);
725
726 return errors;
727}
728
729
730Tensor<type, 2> TestingAnalysis::calculate_multiple_classification_errors() const
731{
732 Tensor<type, 2> errors(6,3);
733
734 const Tensor<type, 1> training_errors = calculate_multiple_classification_training_errors();
735 const Tensor<type, 1> selection_errors = calculate_multiple_classification_selection_errors();
736 const Tensor<type, 1> testing_errors = calculate_multiple_classification_testing_errors();
737
738 errors(0,0) = training_errors(0);
739 errors(1,0) = training_errors(1);
740 errors(2,0) = training_errors(2);
741 errors(3,0) = training_errors(3);
742 errors(4,0) = training_errors(4);
743 errors(5,0) = training_errors(5);
744
745 errors(0,1) = selection_errors(0);
746 errors(1,1) = selection_errors(1);
747 errors(2,1) = selection_errors(2);
748 errors(3,1) = selection_errors(3);
749 errors(4,1) = selection_errors(4);
750 errors(5,1) = selection_errors(5);
751
752 errors(0,2) = testing_errors(0);
753 errors(1,2) = testing_errors(1);
754 errors(2,2) = testing_errors(2);
755 errors(3,2) = testing_errors(3);
756 errors(4,2) = testing_errors(4);
757 errors(5,2) = testing_errors(5);
758
759 return errors;
760}
761
762
763Tensor<type, 1> TestingAnalysis::calculate_training_errors() const
764{
765 // Data set
766
767#ifdef OPENNN_DEBUG
768
769 check();
770
771#endif
772
773 const Index training_samples_number = data_set_pointer->get_training_samples_number();
774
775#ifdef OPENNN_DEBUG
776
777 ostringstream buffer;
778
779 if(training_samples_number == 0)
780 {
781 buffer << "OpenNN Exception: TestingAnalysis class.\n"
782 << "Tensor<type, 1> calculate_training_errors() const.\n"
783 << "Number of training samples is zero.\n";
784
785 throw logic_error(buffer.str());
786 }
787
788#endif
789
790 const Tensor<type, 2> inputs = data_set_pointer->get_training_input_data();
791
792 const Tensor<type, 2> targets = data_set_pointer->get_training_target_data();
793
794 // Neural network
795
796 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
797
798 Tensor<type, 1> errors(4);
799
800 // Results
801
802 const Tensor<type, 0> sum_squared_error = (outputs-targets).square().sum().sqrt();
803
804 errors(0) = sum_squared_error(0);
805 errors(1) = errors(0)/training_samples_number;
806 errors(2) = sqrt(errors(1));
807 errors(3) = calculate_normalized_squared_error(targets, outputs);
808
809 return errors;
810}
811
812
813Tensor<type, 1> TestingAnalysis::calculate_binary_classification_training_errors() const
814{
815 // Data set
816
817#ifdef OPENNN_DEBUG
818
819 check();
820
821#endif
822
823 const Index training_samples_number = data_set_pointer->get_training_samples_number();
824
825#ifdef OPENNN_DEBUG
826
827 ostringstream buffer;
828
829 if(training_samples_number == 0)
830 {
831 buffer << "OpenNN Exception: TestingAnalysis class.\n"
832 << "Tensor<type, 1> calculate_binary_classification_training_errors() const.\n"
833 << "Number of training samples is zero.\n";
834
835 throw logic_error(buffer.str());
836 }
837
838#endif
839
840 const Tensor<type, 2> inputs = data_set_pointer->get_training_input_data();
841
842 const Tensor<type, 2> targets = data_set_pointer->get_training_target_data();
843
844 // Neural network
845
846 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
847
848 Tensor<type, 1> errors(6);
849
850 // Results
851 const Tensor<type, 0> sum_squared_error = (outputs-targets).square().sum().sqrt();
852
853 // SSE
854 errors(0) = sum_squared_error(0);
855
856 // MSE
857 errors(1) = errors(0)/training_samples_number;
858
859 // RMSE
860 errors(2) = sqrt(errors(1));
861
862 // NSE
863 errors(3) = calculate_normalized_squared_error(targets, outputs);
864
865 // CE
866 errors(4) = calculate_cross_entropy_error(targets, outputs);
867
868 // WSE
869 errors(5) = calculate_weighted_squared_error(targets, outputs);
870
871 return errors;
872}
873
874Tensor<type, 1> TestingAnalysis::calculate_multiple_classification_training_errors() const
875{
876 // Data set
877
878#ifdef OPENNN_DEBUG
879
880 check();
881
882#endif
883
884 const Index training_samples_number = data_set_pointer->get_training_samples_number();
885
886#ifdef OPENNN_DEBUG
887
888 ostringstream buffer;
889
890 if(training_samples_number == 0)
891 {
892 buffer << "OpenNN Exception: TestingAnalysis class.\n"
893 << "Tensor<type, 1> calculate_multiple_classification_training_errors() const.\n"
894 << "Number of training samples is zero.\n";
895
896 throw logic_error(buffer.str());
897 }
898
899#endif
900
901 const Tensor<type, 2> inputs = data_set_pointer->get_training_input_data();
902
903 const Tensor<type, 2> targets = data_set_pointer->get_training_target_data();
904
905 // Neural network
906
907 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
908
909 Tensor<type, 1> errors(5);
910
911 // Results
912
913 const Tensor<type, 0> sum_squared_error = (outputs-targets).square().sum().sqrt();
914
915 errors(0) = sum_squared_error(0);
916 errors(1) = errors(0)/training_samples_number;
917 errors(2) = sqrt(errors(1));
918 errors(3) = calculate_normalized_squared_error(targets, outputs);
919 errors(4) = calculate_cross_entropy_error(targets, outputs); // NO
920
921 return errors;
922}
923
924Tensor<type, 1> TestingAnalysis::calculate_selection_errors() const
925{
926 // Data set
927
928#ifdef OPENNN_DEBUG
929
930 check();
931
932#endif
933
934 const Index selection_samples_number = data_set_pointer->get_selection_samples_number();
935
936#ifdef OPENNN_DEBUG
937
938 ostringstream buffer;
939
940 if(selection_samples_number == 0)
941 {
942 buffer << "OpenNN Exception: TestingAnalysis class.\n"
943 << "Tensor<type, 1> calculate_selection_errors() const.\n"
944 << "Number of selection samples is zero.\n";
945
946 throw logic_error(buffer.str());
947 }
948
949#endif
950
951 const Tensor<type, 2> inputs = data_set_pointer->get_selection_input_data();
952
953 const Tensor<type, 2> targets = data_set_pointer->get_selection_target_data();
954
955 // Neural network
956
957 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
958
959 Tensor<type, 1> errors(4);
960
961 // Results
962
963 const Tensor<type, 0> sum_squared_error = (outputs-targets).square().sum().sqrt();
964
965 errors(0) = sum_squared_error(0);
966 errors(1) = errors(0)/selection_samples_number;
967 errors(2) = sqrt(errors(1));
968 errors(3) = calculate_normalized_squared_error(targets, outputs);
969
970 return errors;
971}
972
973
974Tensor<type, 1> TestingAnalysis::calculate_binary_classification_selection_errors() const
975{
976 // Data set
977
978#ifdef OPENNN_DEBUG
979
980 check();
981
982#endif
983
984 const Index selection_samples_number = data_set_pointer->get_selection_samples_number();
985
986#ifdef OPENNN_DEBUG
987
988 ostringstream buffer;
989
990 if(selection_samples_number == 0)
991 {
992 buffer << "OpenNN Exception: TestingAnalysis class.\n"
993 << "Tensor<type, 1> calculate_binary_classification_selection_errors() const.\n"
994 << "Number of selection samples is zero.\n";
995
996 throw logic_error(buffer.str());
997 }
998
999#endif
1000
1001 const Tensor<type, 2> inputs = data_set_pointer->get_selection_input_data();
1002
1003 const Tensor<type, 2> targets = data_set_pointer->get_selection_target_data();
1004
1005 // Neural network
1006
1007 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
1008
1009 Tensor<type, 1> errors(6);
1010
1011 // Results
1012
1013 const Tensor<type, 0> sum_squared_error = (outputs-targets).square().sum().sqrt();
1014
1015 errors(0) = sum_squared_error(0);
1016 errors(1) = errors(0)/selection_samples_number;
1017 errors(2) = sqrt(errors(1));
1018 errors(3) = calculate_normalized_squared_error(targets, outputs);
1019 errors(4) = calculate_cross_entropy_error(targets, outputs);
1020 errors(5) = calculate_weighted_squared_error(targets, outputs);
1021
1022 return errors;
1023}
1024
1025
1026Tensor<type, 1> TestingAnalysis::calculate_multiple_classification_selection_errors() const
1027{
1028 // Data set
1029
1030#ifdef OPENNN_DEBUG
1031
1032 check();
1033
1034#endif
1035
1036 const Index selection_samples_number = data_set_pointer->get_selection_samples_number();
1037
1038#ifdef OPENNN_DEBUG
1039
1040 ostringstream buffer;
1041
1042 if(selection_samples_number == 0)
1043 {
1044 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1045 << "Tensor<type, 1> calculate_multiple_classification_selection_errors() const.\n"
1046 << "Number of selection samples is zero.\n";
1047
1048 throw logic_error(buffer.str());
1049 }
1050
1051#endif
1052
1053 const Tensor<type, 2> inputs = data_set_pointer->get_selection_input_data();
1054
1055 const Tensor<type, 2> targets = data_set_pointer->get_selection_target_data();
1056
1057 // Neural network
1058
1059 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
1060
1061 Tensor<type, 1> errors(5);
1062
1063 // Results
1064
1065 const Tensor<type, 0> sum_squared_error = (outputs-targets).square().sum().sqrt();
1066
1067 errors(0) = sum_squared_error(0);
1068 errors(1) = errors(0)/selection_samples_number;
1069 errors(2) = sqrt(errors(1));
1070 errors(3) = calculate_normalized_squared_error(targets, outputs);
1071 errors(4) = calculate_cross_entropy_error(targets, outputs);
1072
1073 return errors;
1074}
1075
1076
1085
1087{
1088 // Data set
1089
1090#ifdef OPENNN_DEBUG
1091
1092 check();
1093
1094#endif
1095
1096 const Index testing_samples_number = data_set_pointer->get_testing_samples_number();
1097
1098#ifdef OPENNN_DEBUG
1099
1100 ostringstream buffer;
1101
1102 if(testing_samples_number == 0)
1103 {
1104 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1105 << "Tensor<Tensor<type, 2>, 1> calculate_testing_errors() const.\n"
1106 << "Number of testing samples is zero.\n";
1107
1108 throw logic_error(buffer.str());
1109 }
1110
1111#endif
1112
1113 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
1114
1115 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
1116
1117 // Neural network
1118
1119 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
1120
1121 Tensor<type, 1> errors(4);
1122
1123 // Results
1124
1125 const Tensor<type, 0> sum_squared_error = (outputs-targets).square().sum().sqrt();
1126
1127 errors(0) = sum_squared_error(0);
1128 errors(1) = errors(0)/testing_samples_number;
1129 errors(2) = sqrt(errors(1));
1130 errors(3) = calculate_normalized_squared_error(targets, outputs);
1131
1132 return errors;
1133}
1134
1135
1146
1148{
1149 // Data set
1150
1151#ifdef OPENNN_DEBUG
1152
1153 check();
1154
1155#endif
1156
1157 const Index testing_samples_number = data_set_pointer->get_testing_samples_number();
1158
1159#ifdef OPENNN_DEBUG
1160
1161 ostringstream buffer;
1162
1163 if(testing_samples_number == 0)
1164 {
1165 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1166 << "Tensor<type, 1> calculate_binary_classification_testing_errors() const.\n"
1167 << "Number of testing samples is zero.\n";
1168
1169 throw logic_error(buffer.str());
1170 }
1171
1172#endif
1173
1174 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
1175
1176 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
1177
1178 // Neural network
1179
1180 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
1181
1182 Tensor<type, 1> errors(6);
1183
1184 // Results
1185
1186 const Tensor<type, 0> sum_squared_error = (outputs-targets).square().sum().sqrt();
1187
1188 errors(0) = sum_squared_error(0);
1189 errors(1) = errors(0)/testing_samples_number;
1190 errors(2) = sqrt(errors(1));
1191 errors(3) = calculate_normalized_squared_error(targets, outputs);
1192 errors(4) = calculate_cross_entropy_error(targets, outputs);
1193 errors(5) = calculate_weighted_squared_error(targets, outputs);
1194
1195 return errors;
1196}
1197
1198
1208
1210{
1211 // Data set
1212
1213#ifdef OPENNN_DEBUG
1214
1215 check();
1216
1217#endif
1218
1219 const Index testing_samples_number = data_set_pointer->get_testing_samples_number();
1220
1221#ifdef OPENNN_DEBUG
1222
1223 ostringstream buffer;
1224
1225 if(testing_samples_number == 0)
1226 {
1227 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1228 << "Tensor<type, 1> calculate_multiple_classification_testing_errors() const.\n"
1229 << "Number of testing samples is zero.\n";
1230
1231 throw logic_error(buffer.str());
1232 }
1233
1234#endif
1235
1236 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
1237
1238 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
1239
1240 // Neural network
1241
1242 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
1243
1244 Tensor<type, 1> errors(4);
1245
1246 // Results
1247
1248 const Tensor<type, 0> sum_squared_error = (outputs-targets).square().sum().sqrt();
1249
1250 errors(0) = sum_squared_error(0);
1251 errors(1) = errors(0)/testing_samples_number;
1252 errors(2) = sqrt(errors(1));
1253 errors(3) = calculate_normalized_squared_error(targets, outputs);
1254
1255 return errors;
1256}
1257
1258
1262
1263type TestingAnalysis::calculate_normalized_squared_error(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs) const
1264{
1265 const Index samples_number = targets.dimension(0);
1266
1267 const Tensor<type, 1> targets_mean = mean(targets);
1268
1269 Tensor<type, 0> sum_squared_error = (outputs - targets).square().sum();
1270
1271 type normalization_coefficient = type(0);
1272
1273#pragma omp parallel for reduction(+: normalization_coefficient)
1274
1275 for(Index i = 0; i < samples_number; i++)
1276 {
1277 const Tensor<type, 0> norm_1 = (targets.chip(i,0) - targets_mean).square().sum();
1278
1279 normalization_coefficient += norm_1(0);
1280 }
1281
1282 return sum_squared_error()/normalization_coefficient;
1283}
1284
1285
1290
1291type TestingAnalysis::calculate_cross_entropy_error(const Tensor<type, 2>& targets,
1292 const Tensor<type, 2>& outputs) const
1293{
1294 const Index testing_samples_number = targets.dimension(0);
1295 const Index outputs_number = targets.dimension(1);
1296
1297 Tensor<type, 1> targets_row(outputs_number);
1298 Tensor<type, 1> outputs_row(outputs_number);
1299
1300 type cross_entropy_error = type(0);
1301
1302#pragma omp parallel for reduction(+:cross_entropy_error)
1303
1304 for(Index i = 0; i < testing_samples_number; i++)
1305 {
1306 outputs_row = outputs.chip(i, 0);
1307 targets_row = targets.chip(i, 0);
1308
1309 for(Index j = 0; j < outputs_number; j++)
1310 {
1311 if(outputs_row(j) < type(NUMERIC_LIMITS_MIN))
1312 {
1313 outputs_row(j) = static_cast<type>(1.0e-6);
1314 }
1315 else if(static_cast<double>(outputs_row(j)) == 1.0)
1316 {
1317 outputs_row(j) = numeric_limits<type>::max();
1318 }
1319
1320 cross_entropy_error -=
1321 targets_row(j)*log(outputs_row(j)) + (static_cast<type>(1) - targets_row(j))*log(static_cast<type>(1) - outputs_row(j));
1322 }
1323 }
1324
1325 return cross_entropy_error/static_cast<type>(testing_samples_number);
1326}
1327
1328
1334
1335type TestingAnalysis::calculate_weighted_squared_error(const Tensor<type, 2>& targets,
1336 const Tensor<type, 2>& outputs,
1337 const Tensor<type, 1>& weights) const
1338{
1339#ifdef OPENNN_DEBUG
1340
1341 const Index outputs_number = outputs.dimension(1);
1342
1343 ostringstream buffer;
1344
1345 if(outputs_number != 1)
1346 {
1347 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1348 << "type calculate_testing_weighted_squared_error(const Tensor<type, 2>&, const Tensor<type, 2>&, const Tensor<type, 1>&) const.\n"
1349 << "Number of outputs must be one.\n";
1350
1351 throw logic_error(buffer.str());
1352 }
1353
1354#endif
1355
1356 type negatives_weight;
1357 type positives_weight;
1358
1359 if(weights.size() != 2)
1360 {
1361 const Tensor<Index, 1> target_distribution = data_set_pointer->calculate_target_distribution();
1362
1363 const Index negatives_number = target_distribution[0];
1364 const Index positives_number = target_distribution[1];
1365
1366 negatives_weight = type(1);
1367 positives_weight = static_cast<type>(negatives_number/positives_number);
1368 }
1369 else
1370 {
1371 positives_weight = weights[0];
1372 negatives_weight = weights[1];
1373 }
1374
1375 const Tensor<bool, 2> if_sentence = targets == targets.constant(type(1));
1376 const Tensor<bool, 2> else_sentence = targets == targets.constant(type(0));
1377
1378 Tensor<type, 2> f_1(targets.dimension(0), targets.dimension(1));
1379
1380 Tensor<type, 2> f_2(targets.dimension(0), targets.dimension(1));
1381
1382 Tensor<type, 2> f_3(targets.dimension(0), targets.dimension(1));
1383
1384 f_1 = (targets - outputs).square() * positives_weight;
1385
1386 f_2 = (targets - outputs).square()*negatives_weight;
1387
1388 f_3 = targets.constant(type(0));
1389
1390 Tensor<type, 0> sum_squared_error = (if_sentence.select(f_1, else_sentence.select(f_2, f_3))).sum();
1391
1392 Index negatives = 0;
1393
1394 Tensor<type, 1> target_column = targets.chip(0,1);
1395
1396 for(Index i = 0; i < target_column.size(); i++)
1397 {
1398 if(static_cast<double>(target_column(i)) == 0.0) negatives++;
1399 }
1400
1401 const type normalization_coefficient = type(negatives)*negatives_weight*static_cast<type>(0.5);
1402
1403 return sum_squared_error(0)/normalization_coefficient;
1404}
1405
1406
1407type TestingAnalysis::calculate_Minkowski_error(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs, const type minkowski_parameter) const
1408{
1409 Tensor<type, 0> Minkoski_error = (outputs - targets).abs().pow(minkowski_parameter).sum().pow(static_cast<type>(1.0)/minkowski_parameter);
1410
1411 return Minkoski_error();
1412}
1413
1414
1419
1420Tensor<Index, 2> TestingAnalysis::calculate_confusion_binary_classification(const Tensor<type, 2>& targets,
1421 const Tensor<type, 2>& outputs,
1422 const type& decision_threshold) const
1423{
1424 const Index testing_samples_number = targets.dimension(0);
1425
1426 Tensor<Index, 2> confusion(2, 2);
1427
1428 Index true_positive = 0;
1429 Index false_negative = 0;
1430 Index false_positive = 0;
1431 Index true_negative = 0;
1432
1433 type target = type(0);
1434 type output = type(0);
1435
1436 for(Index i = 0; i < testing_samples_number; i++)
1437 {
1438 target = targets(i,0);
1439 output = outputs(i,0);
1440
1441 if(target > decision_threshold && output > decision_threshold)
1442 {
1443 true_positive++;
1444 }
1445 else if(target >= decision_threshold && output < decision_threshold)
1446 {
1447 false_negative++;
1448 }
1449 else if(target <= decision_threshold && output > decision_threshold)
1450 {
1451 false_positive++;
1452 }
1453 else if(target < decision_threshold && output < decision_threshold)
1454 {
1455 true_negative++;
1456 }
1457 else
1458 {
1459 ostringstream buffer;
1460
1461 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1462 << "Tensor<Index, 2> calculate_confusion_binary_classification(const Tensor<type, 2>&, const Tensor<type, 2>&, const type&) const method.\n"
1463 << "Unknown case.\n";
1464
1465 throw logic_error(buffer.str());
1466 }
1467 }
1468
1469 confusion(0,0) = true_positive;
1470 confusion(0,1) = false_negative;
1471 confusion(1,0) = false_positive;
1472 confusion(1,1) = true_negative;
1473
1474 const Index confusion_sum = true_positive + false_negative + false_positive + true_negative;
1475
1476 if(confusion_sum != testing_samples_number)
1477 {
1478 ostringstream buffer;
1479
1480 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1481 << "Tensor<Index, 2> calculate_confusion_binary_classification(const Tensor<type, 2>&, const Tensor<type, 2>&, const type&) const method.\n"
1482 << "Number of elements in confusion matrix (" << confusion_sum << ") must be equal to number of testing samples (" << testing_samples_number << ").\n";
1483
1484 throw logic_error(buffer.str());
1485 }
1486
1487 return confusion;
1488}
1489
1490
1494
1495Tensor<Index, 2> TestingAnalysis::calculate_confusion_multiple_classification(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs) const
1496{
1497 const Index samples_number = targets.dimension(0);
1498 const Index targets_number = targets.dimension(1);
1499
1500 if(targets_number != outputs.dimension(1))
1501 {
1502 ostringstream buffer;
1503
1504 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1505 << "Tensor<Index, 2> calculate_confusion_multiple_classification(const Tensor<type, 2>&, const Tensor<type, 2>&) const method.\n"
1506 << "Number of targets (" << targets_number << ") must be equal to number of outputs (" << outputs.dimension(1) << ").\n";
1507
1508 throw logic_error(buffer.str());
1509 }
1510
1511 Tensor<Index, 2> confusion(targets_number, targets_number);
1512 confusion.setZero();
1513
1514 Index target_index = 0;
1515 Index output_index = 0;
1516
1517 for(Index i = 0; i < samples_number; i++)
1518 {
1519 target_index = maximal_index(targets.chip(i, 0));
1520 output_index = maximal_index(outputs.chip(i, 0));
1521
1522 confusion(target_index,output_index)++;
1523 }
1524
1525 return confusion;
1526}
1527
1528
1536
1537Tensor<Index, 1> TestingAnalysis::calculate_positives_negatives_rate(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs) const
1538{
1539 const Tensor<Index, 2> confusion = calculate_confusion_binary_classification(targets, outputs, type(0.5));
1540 Tensor<Index, 1> positives_negatives_rate(2);
1541
1542 positives_negatives_rate[0] = confusion(0,0) + confusion(0,1);
1543 positives_negatives_rate[1] = confusion(1,0) + confusion(1,1);
1544
1545 return positives_negatives_rate;
1546}
1547
1548
1552
1554{
1555 const Index outputs_number = neural_network_pointer->get_outputs_number();
1556
1557#ifdef OPENNN_DEBUG
1558
1559 check();
1560
1562 {
1563 ostringstream buffer;
1564
1565 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1566 << "Tensor<Index, 2> calculate_confusion() const method.\n"
1567 << "Pointer to neural network in neural network is nullptr.\n";
1568
1569 throw logic_error(buffer.str());
1570 }
1571
1572 const Index inputs_number = neural_network_pointer->get_inputs_number();
1573
1574 // Control sentence
1575
1576 if(inputs_number != data_set_pointer->get_input_variables_number())
1577 {
1578 ostringstream buffer;
1579
1580 buffer << "OpenNN Exception: TestingAnalysis class." << endl
1581 << "Tensor<Index, 2> calculate_confusion() const method." << endl
1582 << "Number of inputs in neural network must be equal to number of inputs in data set." << endl;
1583
1584 throw logic_error(buffer.str());
1585 }
1586
1587 if(outputs_number != data_set_pointer->get_target_variables_number())
1588 {
1589 ostringstream buffer;
1590
1591 buffer << "OpenNN Exception: TestingAnalysis class." << endl
1592 << "Tensor<Index, 2> calculate_confusion() const method." << endl
1593 << "Number of outputs in neural network must be equal to number of targets in data set." << endl;
1594
1595 throw logic_error(buffer.str());
1596 }
1597
1598#endif
1599
1600 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
1601
1602 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
1603
1604 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
1605
1606 if(outputs_number == 1)
1607 {
1608 type decision_threshold;
1609
1611 {
1613 }
1614 else
1615 {
1616 decision_threshold = type(0.5);
1617 }
1618
1619 return calculate_confusion_binary_classification(targets, outputs, decision_threshold);
1620 }
1621 else
1622 {
1623 return calculate_confusion_multiple_classification(targets, outputs);
1624 }
1625}
1626
1627
1635
1637{
1638#ifdef OPENNN_DEBUG
1639
1640 check();
1641
1643 {
1644 ostringstream buffer;
1645
1646 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1647 << "RocCurveResults perform_roc_analysis() const method.\n"
1648 << "Pointer to neural network in neural network is nullptr.\n";
1649
1650 throw logic_error(buffer.str());
1651 }
1652
1653 const Index inputs_number = neural_network_pointer->get_inputs_number();
1654
1655 // Control sentence
1656
1657 if(inputs_number != data_set_pointer->get_input_variables_number())
1658 {
1659 ostringstream buffer;
1660
1661 buffer << "OpenNN Exception: TestingAnalysis class." << endl
1662 << "RocCurveResults perform_roc_analysis() const method." << endl
1663 << "Number of inputs in neural network must be equal to number of inputs in data set." << endl;
1664
1665 throw logic_error(buffer.str());
1666 }
1667
1668 const Index outputs_number = neural_network_pointer->get_outputs_number();
1669
1670 if(outputs_number != data_set_pointer->get_target_variables_number())
1671 {
1672 ostringstream buffer;
1673
1674 buffer << "OpenNN Exception: TestingAnalysis class." << endl
1675 << "RocCurveResults perform_roc_analysis() const method." << endl
1676 << "Number of outputs in neural network must be equal to number of targets in data set." << endl;
1677
1678 throw logic_error(buffer.str());
1679 }
1680
1681#endif
1682
1683 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
1684
1685 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
1686
1687 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
1688
1689 RocAnalysisResults roc_analysis_results;
1690
1691 cout << "Calculating ROC curve..." << endl;
1692
1693 roc_analysis_results.roc_curve = calculate_roc_curve(targets, outputs);
1694
1695 cout << "Calculating area under curve..." << endl;
1696
1697 roc_analysis_results.area_under_curve = calculate_area_under_curve(roc_analysis_results.roc_curve);
1698
1699 cout << "Calculating confidence limits..." << endl;
1700
1701 roc_analysis_results.confidence_limit = calculate_area_under_curve_confidence_limit(targets, outputs, roc_analysis_results.area_under_curve);
1702
1703 cout << "Calculating optimal threshold..." << endl;
1704
1705 roc_analysis_results.optimal_threshold = calculate_optimal_threshold(targets, outputs, roc_analysis_results.roc_curve);
1706
1707 return roc_analysis_results;
1708}
1709
1710
1715
1716type TestingAnalysis::calculate_Wilcoxon_parameter(const type& x, const type& y) const
1717{
1718 if(x > y)
1719 {
1720 return type(1);
1721 }
1722 else if(x < y)
1723 {
1724 return type(0);
1725 }
1726 else
1727 {
1728 return type(0.5);
1729 }
1730}
1731
1732
1739
1740Tensor<type, 2> TestingAnalysis::calculate_roc_curve(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs) const
1741{
1742 const Tensor<Index, 1> positives_negatives_rate = calculate_positives_negatives_rate(targets, outputs);
1743
1744 const Index total_positives = positives_negatives_rate(0);
1745 const Index total_negatives = positives_negatives_rate(1);
1746
1747 if(total_positives == 0)
1748 {
1749 ostringstream buffer;
1750
1751 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1752 << "Tensor<type, 2> calculate_roc_curve(const Tensor<type, 2>&, const Tensor<type, 2>&) const.\n"
1753 << "Number of positive samples ("<< total_positives <<") must be greater than zero.\n";
1754
1755 throw logic_error(buffer.str());
1756 }
1757
1758 if(total_negatives == 0)
1759 {
1760 ostringstream buffer;
1761
1762 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1763 << "Tensor<type, 2> calculate_roc_curve(const Tensor<type, 2>&, const Tensor<type, 2>&) const.\n"
1764 << "Number of negative samples ("<< total_negatives <<") must be greater than zero.\n";
1765
1766 throw logic_error(buffer.str());
1767 }
1768
1769 const Index maximum_points_number = 501;
1770
1771 Index step_size;
1772
1773 const Index testing_samples_number = targets.dimension(0);
1774 Index points_number;
1775
1776 if(testing_samples_number > maximum_points_number)
1777 {
1778 step_size = static_cast<Index>(static_cast<type>(testing_samples_number)/static_cast<type>(maximum_points_number));
1779 points_number = static_cast<Index>(static_cast<type>(testing_samples_number)/static_cast<type>(step_size));
1780 }
1781 else
1782 {
1783 points_number = testing_samples_number;
1784 step_size = 1;
1785 }
1786
1787 if(targets.dimension(1) != 1)
1788 {
1789 ostringstream buffer;
1790
1791 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1792 << "Tensor<type, 2> calculate_roc_curve(const Tensor<type, 2>&, const Tensor<type, 2>&) const.\n"
1793 << "Number of of target variables ("<< targets.dimension(1) <<") must be one.\n";
1794
1795 throw logic_error(buffer.str());
1796 }
1797
1798 if(outputs.dimension(1) != 1)
1799 {
1800 ostringstream buffer;
1801
1802 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1803 << "Tensor<type, 2> calculate_roc_curve(const Tensor<type, 2>&, const Tensor<type, 2>&) const.\n"
1804 << "Number of of output variables ("<< targets.dimension(1) <<") must be one.\n";
1805
1806 throw logic_error(buffer.str());
1807 }
1808
1809 // Sort by ascending values of outputs vector
1810
1811 Tensor<Index, 1> sorted_indices(outputs.dimension(0));
1812 iota(sorted_indices.data(), sorted_indices.data() + sorted_indices.size(), 0);
1813
1814 stable_sort(sorted_indices.data(), sorted_indices.data()+sorted_indices.size(), [outputs](Index i1, Index i2) {return outputs(i1,0) < outputs(i2,0);});
1815
1816 Tensor<type, 1> sorted_targets(testing_samples_number);
1817 Tensor<type, 1> sorted_outputs(testing_samples_number);
1818
1819 for(Index i = 0; i < testing_samples_number; i++)
1820 {
1821 sorted_targets(i) = targets(sorted_indices(i),0);
1822 sorted_outputs(i) = outputs(sorted_indices(i),0);
1823 }
1824
1825 Tensor<type, 2> roc_curve(points_number+1, 3);
1826 roc_curve.setZero();
1827
1828 #pragma omp parallel for schedule(dynamic)
1829
1830 for(Index i = 0; i < static_cast<Index>(points_number); i++)
1831 {
1832 Index positives = 0;
1833 Index negatives = 0;
1834
1835 const Index current_index = i*step_size;
1836
1837 const type threshold = sorted_outputs(current_index);
1838
1839 for(Index j = 0; j < static_cast<Index>(current_index); j++)
1840 {
1841 if(sorted_outputs(j) < threshold && static_cast<double>(sorted_targets(j)) == 1.0)
1842 {
1843 positives++;
1844 }
1845 if(sorted_outputs(j) < threshold && sorted_targets(j) < type(NUMERIC_LIMITS_MIN))
1846 {
1847 negatives++;
1848 }
1849 }
1850
1851 roc_curve(i,0) = static_cast<type>(positives)/static_cast<type>(total_positives);
1852 roc_curve(i,1) = static_cast<type>(negatives)/static_cast<type>(total_negatives);
1853 roc_curve(i,2) = static_cast<type>(threshold);
1854 }
1855
1856 roc_curve(points_number, 0) = type(1);
1857 roc_curve(points_number, 1) = type(1);
1858 roc_curve(points_number, 2) = type(1);
1859
1860 return roc_curve;
1861}
1862
1863
1867
1868type TestingAnalysis::calculate_area_under_curve(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs) const
1869{
1870 const Tensor<Index, 1> positives_negatives_rate = calculate_positives_negatives_rate(targets, outputs);
1871
1872 const Index total_positives = positives_negatives_rate[0];
1873 const Index total_negatives = positives_negatives_rate[1];
1874
1875 if(total_positives == 0)
1876 {
1877 ostringstream buffer;
1878
1879 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1880 << "Tensor<type, 2> calculate_roc_curve(const Tensor<type, 2>&, const Tensor<type, 2>&) const.\n"
1881 << "Number of positive samples("<< total_positives <<") must be greater than zero.\n";
1882
1883 throw logic_error(buffer.str());
1884 }
1885
1886 if(total_negatives == 0)
1887 {
1888 ostringstream buffer;
1889
1890 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1891 << "Tensor<type, 2> calculate_roc_curve(const Tensor<type, 2>&, const Tensor<type, 2>&) const.\n"
1892 << "Number of negative samples("<< total_negatives <<") must be greater than zero.\n";
1893
1894 throw logic_error(buffer.str());
1895 }
1896
1897 Index testing_samples_number = targets.dimension(0);
1898
1899 type sum = type(0);
1900
1901 type area_under_curve;
1902
1903 #pragma omp parallel for reduction(+ : sum) schedule(dynamic)
1904
1905 for(Index i = 0; i < testing_samples_number; i++)
1906 {
1907 if(abs(targets(i,0) - static_cast<type>(1.0)) < type(NUMERIC_LIMITS_MIN))
1908 {
1909 for(Index j = 0; j < testing_samples_number; j++)
1910 {
1911 if(abs(targets(j,0)) < type(NUMERIC_LIMITS_MIN))
1912 {
1913 sum += calculate_Wilcoxon_parameter(outputs(i,0),outputs(j,0));
1914 }
1915 }
1916 }
1917 }
1918
1919 area_under_curve = static_cast<type>(sum)/static_cast<type>(total_positives*total_negatives);
1920
1921 return area_under_curve;
1922}
1923
1924
1927
1928type TestingAnalysis::calculate_area_under_curve(const Tensor<type, 2>& roc_curve) const
1929{
1930 type area_under_curve = type(0);
1931
1932 for(Index i = 1; i < roc_curve.dimension(0); i++)
1933 {
1934 area_under_curve += (roc_curve(i,0)-roc_curve(i-1,0))*(roc_curve(i,1)+roc_curve(i-1,1));
1935 }
1936
1937 return area_under_curve/ type(2);
1938}
1939
1940
1944
1945type TestingAnalysis::calculate_area_under_curve_confidence_limit(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs) const
1946{
1947 const Tensor<Index, 1> positives_negatives_rate = calculate_positives_negatives_rate(targets, outputs);
1948
1949 const Index total_positives = positives_negatives_rate[0];
1950 const Index total_negatives = positives_negatives_rate[1];
1951
1952 if(total_positives == 0)
1953 {
1954 ostringstream buffer;
1955
1956 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1957 << "Tensor<type, 2> calculate_roc_curve_confidence_limit(const Tensor<type, 2>&, const Tensor<type, 2>&) const.\n"
1958 << "Number of positive samples("<< total_positives <<") must be greater than zero.\n";
1959
1960 throw logic_error(buffer.str());
1961 }
1962
1963 if(total_negatives == 0)
1964 {
1965 ostringstream buffer;
1966
1967 buffer << "OpenNN Exception: TestingAnalysis class.\n"
1968 << "Tensor<type, 2> calculate_roc_curve_confidence_limit(const Tensor<type, 2>&, const Tensor<type, 2>&) const.\n"
1969 << "Number of negative samples("<< total_negatives <<") must be greater than zero.\n";
1970
1971 throw logic_error(buffer.str());
1972 }
1973
1974 const type area_under_curve = calculate_area_under_curve(targets, outputs);
1975
1976 const type Q_1 = area_under_curve/(static_cast<type>(2.0) - area_under_curve);
1977 const type Q_2 = (static_cast<type>(2.0) *area_under_curve*area_under_curve)/(static_cast<type>(1.0) *area_under_curve);
1978
1979 const type confidence_limit = type(type(1.64485)*sqrt((area_under_curve*(type(1) - area_under_curve)
1980 + (type(total_positives) - type(1))*(Q_1-area_under_curve*area_under_curve)
1981 + (type(total_negatives) - type(1))*(Q_2-area_under_curve*area_under_curve))/(type(total_positives*total_negatives))));
1982
1983 return confidence_limit;
1984}
1985
1986
1991
1992type TestingAnalysis::calculate_area_under_curve_confidence_limit(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs, const type& area_under_curve) const
1993{
1994 const Tensor<Index, 1> positives_negatives_rate = calculate_positives_negatives_rate(targets, outputs);
1995
1996 const Index total_positives = positives_negatives_rate[0];
1997 const Index total_negatives = positives_negatives_rate[1];
1998
1999 if(total_positives == 0)
2000 {
2001 ostringstream buffer;
2002
2003 buffer << "OpenNN Exception: TestingAnalysis class.\n"
2004 << "calculate_area_under_curve_confidence_limit(const Tensor<type, 2>&, const Tensor<type, 2>&, const type&) const.\n"
2005 << "Number of positive samples("<< total_positives <<") must be greater than zero.\n";
2006
2007 throw logic_error(buffer.str());
2008 }
2009
2010 if(total_negatives == 0)
2011 {
2012 ostringstream buffer;
2013
2014 buffer << "OpenNN Exception: TestingAnalysis class.\n"
2015 << "calculate_area_under_curve_confidence_limit(const Tensor<type, 2>&, const Tensor<type, 2>&, const type&) const.\n"
2016 << "Number of negative samples("<< total_negatives <<") must be greater than zero.\n";
2017
2018 throw logic_error(buffer.str());
2019 }
2020
2021 const type Q_1 = area_under_curve/(static_cast<type>(2.0) -area_under_curve);
2022 const type Q_2 = (static_cast<type>(2.0) *area_under_curve*area_under_curve)/(static_cast<type>(1.0) *area_under_curve);
2023
2024 const type confidence_limit = static_cast<type>(1.64485) *sqrt((area_under_curve*(static_cast<type>(1.0) - area_under_curve)
2025 + (type(total_positives)-static_cast<type>(1.0))*(Q_1-area_under_curve*area_under_curve)
2026 + (type(total_negatives)-static_cast<type>(1.0))*(Q_2-area_under_curve*area_under_curve))/(type(total_positives*total_negatives)));
2027
2028 return confidence_limit;
2029}
2030
2031
2035
2036type TestingAnalysis::calculate_optimal_threshold(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs) const
2037{
2038 const Index maximum_points_number = 1000;
2039
2040 Index step_size;
2041
2042 const Index testing_samples_number = targets.dimension(0);
2043 Index points_number;
2044
2045 if(testing_samples_number > maximum_points_number)
2046 {
2047 step_size = testing_samples_number/maximum_points_number;
2048 points_number = testing_samples_number/step_size;
2049 }
2050 else
2051 {
2052 points_number = testing_samples_number;
2053 step_size = 1;
2054 }
2055
2056 Tensor<type, 2> targets_outputs(targets.dimension(0), targets.dimension(1)+outputs.dimension(1));
2057
2058 for(Index i = 0; i < targets.dimension(1)+outputs.dimension(1); i++)
2059 {
2060 for(Index j = 0; j < targets.dimension(0); j++)
2061 {
2062 if(i < targets.dimension(1)) targets_outputs(j,i) = targets(j,i);
2063 else targets_outputs(j,i) = outputs(j,i);
2064 }
2065 }
2066
2067 // Sort by ascending values
2068
2069 sort(targets_outputs.data(), targets_outputs.data()+targets.size(), less<type>());
2070
2071 const TensorMap< Tensor<type, 2> > sorted_targets(targets_outputs.data(), targets.dimension(0), targets.dimension(1));
2072 const TensorMap< Tensor<type, 2> > sorted_outputs(targets_outputs.data()+targets.size(), outputs.dimension(0), outputs.dimension(1));
2073
2074 const Tensor<type, 2> roc_curve = calculate_roc_curve(sorted_targets, sorted_outputs);
2075
2076 type threshold = type(0);
2077 type optimal_threshold = type(0.5);
2078
2079 type minimun_distance = numeric_limits<type>::max();
2080 type distance;
2081
2082 Index current_index;
2083
2084 for(Index i = 0; i < points_number; i++)
2085 {
2086 current_index = i*step_size;
2087
2088 threshold = sorted_outputs(current_index, 0);
2089
2090 distance = static_cast<type>(sqrt(roc_curve(i,0)*roc_curve(i,0) + (roc_curve(i,1) - type(1))*(roc_curve(i,1) - type(1))));
2091
2092 if(distance < minimun_distance)
2093 {
2094 optimal_threshold = threshold;
2095
2096 minimun_distance = distance;
2097 }
2098 }
2099
2100 return optimal_threshold;
2101}
2102
2103
2108
2109type TestingAnalysis::calculate_optimal_threshold(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs, const Tensor<type, 2>& roc_curve) const
2110{
2111 const Index points_number = roc_curve.dimension(0);
2112
2113 type optimal_threshold = type(0.5);
2114
2115 type minimun_distance = numeric_limits<type>::max();
2116 type distance;
2117
2118 for(Index i = 0; i < points_number; i++)
2119 {
2120 distance = sqrt(roc_curve(i,0)*roc_curve(i,0) + (roc_curve(i,1) - static_cast<type>(1))*(roc_curve(i,1) - static_cast<type>(1)));
2121
2122 if(distance < minimun_distance)
2123 {
2124 optimal_threshold = roc_curve(i,2);
2125
2126 minimun_distance = distance;
2127 }
2128 }
2129
2130 return optimal_threshold;
2131}
2132
2133
2136
2138{
2139#ifdef OPENNN_DEBUG
2140
2141 check();
2142
2143#endif
2144
2145#ifdef OPENNN_DEBUG
2146
2148 {
2149 ostringstream buffer;
2150
2151 buffer << "OpenNN Exception: TestingAnalysis class.\n"
2152 << "Tensor<type, 2> perform_cumulative_gain_analysis() const method.\n"
2153 << "Pointer to neural network in neural network is nullptr.\n";
2154
2155 throw logic_error(buffer.str());
2156 }
2157
2158#endif
2159
2160#ifdef OPENNN_DEBUG
2161
2162 const Index inputs_number = neural_network_pointer->get_inputs_number();
2163
2164 // Control sentence
2165
2166 if(inputs_number != data_set_pointer->get_input_variables_number())
2167 {
2168 ostringstream buffer;
2169
2170 buffer << "OpenNN Exception: TestingAnalysis class." << endl
2171 << "Tensor<type, 2> perform_cumulative_gain_analysis() const method." << endl
2172 << "Number of inputs in neural network must be equal to number of inputs in data set." << endl;
2173
2174 throw logic_error(buffer.str());
2175 }
2176
2177 const Index outputs_number = neural_network_pointer->get_outputs_number();
2178
2179 if(outputs_number != data_set_pointer->get_target_variables_number())
2180 {
2181 ostringstream buffer;
2182
2183 buffer << "OpenNN Exception: TestingAnalysis class." << endl
2184 << "Tensor<type, 2> perform_cumulative_gain_analysis() const method." << endl
2185 << "Number of outputs in neural network must be equal to number of targets in data set." << endl;
2186
2187 throw logic_error(buffer.str());
2188 }
2189
2190#endif
2191
2192 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
2193 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
2194
2195 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
2196
2197 const Tensor<type, 2> cumulative_gain = calculate_cumulative_gain(targets, outputs);
2198
2199 return cumulative_gain;
2200}
2201
2202
2207
2208Tensor<type, 2> TestingAnalysis::calculate_cumulative_gain(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs) const
2209{
2210 const Index total_positives = calculate_positives_negatives_rate(targets, outputs)[0];
2211
2212 if(total_positives == 0)
2213 {
2214 ostringstream buffer;
2215
2216 buffer << "OpenNN Exception: TestingAnalysis class.\n"
2217 << "Tensor<type, 2> calculate_cumulative_gain(const Tensor<type, 2>&, const Tensor<type, 2>&) const.\n"
2218 << "Number of positive samples(" << total_positives << ") must be greater than zero.\n";
2219
2220 throw logic_error(buffer.str());
2221 }
2222
2223 const Index testing_samples_number = targets.dimension(0);
2224
2225 // Sort by ascending values of outputs vector
2226
2227 Tensor<Index, 1> sorted_indices(outputs.dimension(0));
2228 iota(sorted_indices.data(), sorted_indices.data() + sorted_indices.size(), 0);
2229
2230 stable_sort(sorted_indices.data(),
2231 sorted_indices.data()+sorted_indices.size(),
2232 [outputs](Index i1, Index i2) {return outputs(i1,0) > outputs(i2,0);});
2233
2234 Tensor<type, 1> sorted_targets(testing_samples_number);
2235
2236 for(Index i = 0; i < testing_samples_number; i++)
2237 {
2238 sorted_targets(i) = targets(sorted_indices(i),0);
2239 }
2240
2241 const Index points_number = 21;
2242 const type percentage_increment = static_cast<type>(0.05);
2243
2244 Tensor<type, 2> cumulative_gain(points_number, 2);
2245
2246 cumulative_gain(0,0) = type(0);
2247 cumulative_gain(0,1) = type(0);
2248
2249 Index positives = 0;
2250
2251 type percentage = type(0);
2252
2253 Index maximum_index;
2254
2255 for(Index i = 0; i < points_number - 1; i++)
2256 {
2257 percentage += percentage_increment;
2258
2259 positives = 0;
2260
2261 maximum_index = static_cast<Index>(percentage* type(testing_samples_number));
2262
2263 for(Index j = 0; j < maximum_index; j++)
2264 {
2265 if(static_cast<double>(sorted_targets(j)) == 1.0)
2266 {
2267 positives++;
2268 }
2269 }
2270
2271 cumulative_gain(i + 1, 0) = percentage;
2272 cumulative_gain(i + 1, 1) = static_cast<type>(positives)/static_cast<type>(total_positives);
2273 }
2274
2275 return cumulative_gain;
2276}
2277
2278
2283
2284Tensor<type, 2> TestingAnalysis::calculate_negative_cumulative_gain(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs) const
2285{
2286 const Index total_negatives = calculate_positives_negatives_rate(targets, outputs)[1];
2287
2288 if(total_negatives == 0)
2289 {
2290 ostringstream buffer;
2291
2292 buffer << "OpenNN Exception: TestingAnalysis class.\n"
2293 << "Tensor<type, 2> calculate_negative_cumulative_gain(const Tensor<type, 2>&, const Tensor<type, 2>&) const.\n"
2294 << "Number of negative samples(" << total_negatives << ") must be greater than zero.\n";
2295
2296 throw logic_error(buffer.str());
2297 }
2298
2299 const Index testing_samples_number = targets.dimension(0);
2300
2301 // Sort by ascending values of outputs vector
2302
2303 Tensor<Index, 1> sorted_indices(outputs.dimension(0));
2304 iota(sorted_indices.data(), sorted_indices.data() + sorted_indices.size(), 0);
2305
2306 stable_sort(sorted_indices.data(), sorted_indices.data()+sorted_indices.size(), [outputs](Index i1, Index i2) {return outputs(i1,0) > outputs(i2,0);});
2307
2308 Tensor<type, 1> sorted_targets(testing_samples_number);
2309
2310 for(Index i = 0; i < testing_samples_number; i++)
2311 {
2312 sorted_targets(i) = targets(sorted_indices(i),0);
2313 }
2314
2315 const Index points_number = 21;
2316 const type percentage_increment = static_cast<type>(0.05);
2317
2318 Tensor<type, 2> negative_cumulative_gain(points_number, 2);
2319
2320 negative_cumulative_gain(0,0) = type(0);
2321 negative_cumulative_gain(0,1) = type(0);
2322
2323 Index negatives = 0;
2324
2325 type percentage = type(0);
2326
2327 Index maximum_index;
2328
2329 for(Index i = 0; i < points_number - 1; i++)
2330 {
2331 percentage += percentage_increment;
2332
2333 negatives = 0;
2334
2335 maximum_index = static_cast<Index>(percentage* type(testing_samples_number));
2336
2337 for(Index j = 0; j < maximum_index; j++)
2338 {
2339 if(sorted_targets(j) < type(NUMERIC_LIMITS_MIN))
2340 {
2341 negatives++;
2342 }
2343 }
2344
2345 negative_cumulative_gain(i + 1, 0) = percentage;
2346
2347 negative_cumulative_gain(i + 1, 1) = static_cast<type>(negatives)/static_cast<type>(total_negatives);
2348 }
2349
2350 return negative_cumulative_gain;
2351}
2352
2353
2356
2358{
2359#ifdef OPENNN_DEBUG
2360
2361 check();
2362
2364 {
2365 ostringstream buffer;
2366
2367 buffer << "OpenNN Exception: TestingAnalysis class.\n"
2368 << "Tensor<type, 2> perform_lift_chart_analysis() const method.\n"
2369 << "Pointer to neural network in neural network is nullptr.\n";
2370
2371 throw logic_error(buffer.str());
2372 }
2373
2374 const Index inputs_number = neural_network_pointer->get_inputs_number();
2375
2376 // Control sentence
2377
2378 if(inputs_number != data_set_pointer->get_input_variables_number())
2379 {
2380 ostringstream buffer;
2381
2382 buffer << "OpenNN Exception: TestingAnalysis class." << endl
2383 << "Tensor<type, 2> perform_lift_chart_analysis() const method." << endl
2384 << "Number of inputs in neural network must be equal to number of inputs in data set." << endl;
2385
2386 throw logic_error(buffer.str());
2387 }
2388
2389 const Index outputs_number = neural_network_pointer->get_outputs_number();
2390
2391 if(outputs_number != data_set_pointer->get_target_variables_number())
2392 {
2393 ostringstream buffer;
2394
2395 buffer << "OpenNN Exception: TestingAnalysis class." << endl
2396 << "Tensor<type, 2> perform_lift_chart_analysis() const method." << endl
2397 << "Number of outputs in neural network must be equal to number of targets in data set." << endl;
2398
2399 throw logic_error(buffer.str());
2400 }
2401
2402#endif
2403
2404 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
2405 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
2406
2407 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
2408
2409 const Tensor<type, 2> cumulative_gain = calculate_cumulative_gain(targets, outputs);
2410 const Tensor<type, 2> lift_chart = calculate_lift_chart(cumulative_gain);
2411
2412 return lift_chart;
2413}
2414
2415
2419
2420Tensor<type, 2> TestingAnalysis::calculate_lift_chart(const Tensor<type, 2>& cumulative_gain) const
2421{
2422 const Index rows_number = cumulative_gain.dimension(0);
2423 const Index columns_number = cumulative_gain.dimension(1);
2424
2425 Tensor<type, 2> lift_chart(rows_number, columns_number);
2426
2427 lift_chart(0,0) = type(0);
2428 lift_chart(0,1) = type(1);
2429
2430 #pragma omp parallel for
2431
2432 for(Index i = 1; i < rows_number; i++)
2433 {
2434 lift_chart(i, 0) = static_cast<type>(cumulative_gain(i, 0));
2435 lift_chart(i, 1) = static_cast<type>(cumulative_gain(i, 1))/static_cast<type>(cumulative_gain(i, 0));
2436 }
2437
2438 return lift_chart;
2439}
2440
2441
2449
2451{
2452#ifdef OPENNN_DEBUG
2453
2454 check();
2455
2457 {
2458 ostringstream buffer;
2459
2460 buffer << "OpenNN Exception: TestingAnalysis class.\n"
2461 << "Tensor<type, 2> perform_Kolmogorov_Smirnov_analysis() const method.\n"
2462 << "Pointer to neural network in neural network is nullptr.\n";
2463
2464 throw logic_error(buffer.str());
2465 }
2466
2467#endif
2468
2469#ifdef OPENNN_DEBUG
2470
2471 const Index inputs_number = neural_network_pointer->get_inputs_number();
2472
2473 // Control sentence
2474
2475 if(inputs_number != data_set_pointer->get_input_variables_number())
2476 {
2477 ostringstream buffer;
2478
2479 buffer << "OpenNN Exception: TestingAnalysis class." << endl
2480 << "Tensor<type, 2> perform_Kolmogorov_Smirnov_analysis() const method." << endl
2481 << "Number of inputs in neural network must be equal to number of inputs in data set." << endl;
2482
2483 throw logic_error(buffer.str());
2484 }
2485
2486 const Index outputs_number = neural_network_pointer->get_outputs_number();
2487
2488 if(outputs_number != data_set_pointer->get_target_variables_number())
2489 {
2490 ostringstream buffer;
2491
2492 buffer << "OpenNN Exception: TestingAnalysis class." << endl
2493 << "Tensor<type, 2> perform_Kolmogorov_Smirnov_analysis() const method." << endl
2494 << "Number of outputs in neural network must be equal to number of targets in data set." << endl;
2495
2496 throw logic_error(buffer.str());
2497 }
2498
2499#endif
2500
2501 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
2502 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
2503
2504 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
2505
2506 TestingAnalysis::KolmogorovSmirnovResults Kolmogorov_Smirnov_results;
2507
2508 Kolmogorov_Smirnov_results.positive_cumulative_gain = calculate_cumulative_gain(targets, outputs);
2509 Kolmogorov_Smirnov_results.negative_cumulative_gain = calculate_negative_cumulative_gain(targets, outputs);
2510 Kolmogorov_Smirnov_results.maximum_gain =
2511 calculate_maximum_gain(Kolmogorov_Smirnov_results.positive_cumulative_gain,Kolmogorov_Smirnov_results.negative_cumulative_gain);
2512
2513 return Kolmogorov_Smirnov_results;
2514}
2515
2516
2521
2522Tensor<type, 1> TestingAnalysis::calculate_maximum_gain(const Tensor<type, 2>& positive_cumulative_gain, const Tensor<type, 2>& negative_cumulative_gain) const
2523{
2524 const Index points_number = positive_cumulative_gain.dimension(0);
2525
2526#ifdef OPENNN_DEBUG
2527
2528 if(points_number != negative_cumulative_gain.dimension(0))
2529 {
2530 ostringstream buffer;
2531
2532 buffer << "OpenNN Exception: TestingAnalysis class.\n"
2533 << "Tensor<type, 2> calculate_maximum_gain() const method.\n"
2534 << "Positive and negative cumulative gain matrix must have the same rows number.\n";
2535
2536 throw logic_error(buffer.str());
2537 }
2538
2539#endif
2540
2541 Tensor<type, 1> maximum_gain(2);
2542
2543 const type percentage_increment = static_cast<type>(0.05);
2544
2545 type percentage = type(0);
2546
2547 for(Index i = 0; i < points_number - 1; i++)
2548 {
2549 percentage += percentage_increment;
2550
2551 if(positive_cumulative_gain(i+1,1)-negative_cumulative_gain(i+1,1) > maximum_gain[1]
2552 && positive_cumulative_gain(i+1,1)-negative_cumulative_gain(i+1,1) > static_cast<type>(0.0))
2553 {
2554 maximum_gain(1) = positive_cumulative_gain(i+1,1)-negative_cumulative_gain(i+1,1);
2555 maximum_gain(0) = percentage;
2556 }
2557 }
2558
2559 return maximum_gain;
2560}
2561
2562
2564
2566{
2567#ifdef OPENNN_DEBUG
2568
2569 check();
2570
2572 {
2573 ostringstream buffer;
2574
2575 buffer << "OpenNN Exception: TestingAnalysis class.\n"
2576 << "Tensor<type, 2> perform_calibration_plot_analysis() const method.\n"
2577 << "Pointer to neural network in neural network is nullptr.\n";
2578
2579 throw logic_error(buffer.str());
2580 }
2581
2582 const Index inputs_number = neural_network_pointer->get_inputs_number();
2583
2584 // Control sentence
2585
2586 if(inputs_number != data_set_pointer->get_input_variables_number())
2587 {
2588 ostringstream buffer;
2589
2590 buffer << "OpenNN Exception: TestingAnalysis class." << endl
2591 << "Tensor<type, 2> perform_calibration_plot_analysis() const method." << endl
2592 << "Number of inputs in neural network must be equal to number of inputs in data set." << endl;
2593
2594 throw logic_error(buffer.str());
2595 }
2596
2597 const Index outputs_number = neural_network_pointer->get_outputs_number();
2598
2599 if(outputs_number != data_set_pointer->get_target_variables_number())
2600 {
2601 ostringstream buffer;
2602
2603 buffer << "OpenNN Exception: TestingAnalysis class." << endl
2604 << "Tensor<type, 2> perform_calibration_plot_analysis() const method." << endl
2605 << "Number of outputs in neural network must be equal to number of targets in data set." << endl;
2606
2607 throw logic_error(buffer.str());
2608 }
2609
2610#endif
2611
2612 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
2613 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
2614
2615 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
2616
2617 const Tensor<type, 2> calibration_plot = calculate_calibration_plot(targets, outputs);
2618
2619 return calibration_plot;
2620
2621}
2622
2623
2628
2629Tensor<type, 2> TestingAnalysis::calculate_calibration_plot(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs) const
2630{
2631 const Index rows_number = targets.dimension(0);
2632
2633 const Index points_number = 10;
2634
2635 Tensor<type, 2> calibration_plot(points_number+2, 2);
2636
2637 // First point
2638
2639 calibration_plot(0,0) = type(0);
2640 calibration_plot(0,1) = type(0);
2641
2642 Index positives = 0;
2643
2644 Index count = 0;
2645
2646 type probability = type(0);
2647
2648 type sum = type(0);
2649
2650 for(Index i = 1; i < points_number+1; i++)
2651 {
2652 count = 0;
2653
2654 positives = 0;
2655 sum = type(0);
2656
2657 probability += static_cast<type>(0.1);
2658
2659 for(Index j = 0; j < rows_number; j++)
2660 {
2661 if(outputs(j, 0) >= (probability - static_cast<type>(0.1)) && outputs(j, 0) < probability)
2662 {
2663 count++;
2664
2665 sum += outputs(j, 0);
2666
2667 if(static_cast<Index>(targets(j, 0)) == 1)
2668 {
2669 positives++;
2670 }
2671 }
2672 }
2673
2674 if(count == 0)
2675 {
2676 calibration_plot(i, 0) = type(-1);
2677 calibration_plot(i, 1) = type(-1);
2678 }
2679 else
2680 {
2681 calibration_plot(i, 0) = sum/static_cast<type>(count);
2682 calibration_plot(i, 1) = static_cast<type>(positives)/static_cast<type>(count);
2683 }
2684 }
2685
2686 // Last point
2687
2688 calibration_plot(points_number+1,0) = type(1);
2689 calibration_plot(points_number+1,1) = type(1);
2690
2691 // Subtracts calibration plot rows with value -1
2692
2693 Index points_number_subtracted = 0;
2694
2695 while(contains(calibration_plot.chip(0,1), type(-1)))
2696 {
2697 for(Index i = 1; i < points_number - points_number_subtracted+1; i++)
2698 {
2699 if(abs(calibration_plot(i, 0) + type(1)) < type(NUMERIC_LIMITS_MIN))
2700 {
2701 calibration_plot = delete_row(calibration_plot, i);
2702
2703 points_number_subtracted++;
2704 }
2705 }
2706 }
2707
2708 return calibration_plot;
2709}
2710
2711
2715
2716Tensor<Histogram, 1> TestingAnalysis::calculate_output_histogram(const Tensor<type, 2>& outputs, const Index& bins_number) const
2717{
2718 Tensor<Histogram, 1> output_histogram(1);
2719
2720 Tensor<type, 1> output_column = outputs.chip(0,1);
2721
2722 output_histogram (0) = histogram(output_column, bins_number);
2723
2724 return output_histogram;
2725}
2726
2727
2735
2737{
2738#ifdef OPENNN_DEBUG
2739
2740 check();
2741
2743 {
2744 ostringstream buffer;
2745
2746 buffer << "OpenNN Exception: TestingAnalysis class.\n"
2747 << "BinaryClassificationRates calculate_binary_classification_rates() const method.\n"
2748 << "Pointer to neural network in neural network is nullptr.\n";
2749
2750 throw logic_error(buffer.str());
2751 }
2752
2753 const Index inputs_number = neural_network_pointer->get_inputs_number();
2754
2755 // Control sentence
2756
2757 if(inputs_number != data_set_pointer->get_input_variables_number())
2758 {
2759 ostringstream buffer;
2760
2761 buffer << "OpenNN Exception: TestingAnalysis class." << endl
2762 << "BinaryClassificationRates calculate_binary_classification_rates() const method." << endl
2763 << "Number of inputs in neural network must be equal to number of inputs in data set." << endl;
2764
2765 throw logic_error(buffer.str());
2766 }
2767
2768 const Index outputs_number = neural_network_pointer->get_outputs_number();
2769
2770 if(outputs_number != data_set_pointer->get_target_variables_number())
2771 {
2772 ostringstream buffer;
2773
2774 buffer << "OpenNN Exception: TestingAnalysis class." << endl
2775 << "BinaryClassificationRates calculate_binary_classification_rates() const method." << endl
2776 << "Number of outputs in neural network must be equal to number of targets in data set." << endl;
2777
2778 throw logic_error(buffer.str());
2779 }
2780
2781#endif
2782
2783 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
2784 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
2785
2786 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
2787
2788 const Tensor<Index, 1> testing_indices = data_set_pointer->get_testing_samples_indices();
2789
2790 type decision_threshold;
2791
2793 {
2795 }
2796 else
2797 {
2798 decision_threshold = type(0.5);
2799 }
2800
2801 BinaryClassifcationRates binary_classification_rates;
2802
2803 binary_classification_rates.true_positives_indices = calculate_true_positive_samples(targets, outputs, testing_indices, decision_threshold);
2804 binary_classification_rates.false_positives_indices = calculate_false_positive_samples(targets, outputs, testing_indices, decision_threshold);
2805 binary_classification_rates.false_negatives_indices = calculate_false_negative_samples(targets, outputs, testing_indices, decision_threshold);
2806 binary_classification_rates.true_negatives_indices = calculate_true_negative_samples(targets, outputs, testing_indices, decision_threshold);
2807
2808 return binary_classification_rates;
2809}
2810
2811
2818
2819Tensor<Index, 1> TestingAnalysis::calculate_true_positive_samples(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs,
2820 const Tensor<Index, 1>& testing_indices, const type& decision_threshold) const
2821{
2822 const Index rows_number = targets.dimension(0);
2823
2824 Tensor<Index, 1> true_positives_indices_copy(rows_number);
2825
2826 Index index = 0;
2827
2828 for(Index i = 0; i < rows_number; i++)
2829 {
2830 Tensor<Index, 1> copy;
2831 if(targets(i,0) >= decision_threshold && outputs(i,0) >= decision_threshold)
2832 {
2833 true_positives_indices_copy(index) = testing_indices(i);
2834 index++;
2835 }
2836 }
2837
2838 Tensor<Index, 1> true_positives_indices(index);
2839
2840 memcpy(true_positives_indices.data(), true_positives_indices_copy.data(), static_cast<size_t>(index)*sizeof(Index));
2841
2842 return true_positives_indices;
2843}
2844
2845
2852
2853Tensor<Index, 1> TestingAnalysis::calculate_false_positive_samples(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs,
2854 const Tensor<Index, 1>& testing_indices, const type& decision_threshold) const
2855{
2856 const Index rows_number = targets.dimension(0);
2857
2858 Tensor<Index, 1> false_positives_indices_copy(rows_number);
2859
2860 Index index = 0;
2861
2862 for(Index i = 0; i < rows_number; i++)
2863 {
2864 if(targets(i,0) < decision_threshold && outputs(i,0) >= decision_threshold)
2865 {
2866 false_positives_indices_copy(index) = testing_indices(i);
2867 index++;
2868 }
2869 }
2870
2871 Tensor<Index, 1> false_positives_indices(index);
2872
2873 memcpy(false_positives_indices.data(), false_positives_indices_copy.data(), static_cast<size_t>(index)*sizeof(Index));
2874
2875 return false_positives_indices;
2876}
2877
2878
2885
2886Tensor<Index, 1> TestingAnalysis::calculate_false_negative_samples(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs,
2887 const Tensor<Index, 1>& testing_indices, const type& decision_threshold) const
2888{
2889 const Index rows_number = targets.dimension(0);
2890
2891 Tensor<Index, 1> false_negatives_indices_copy(rows_number);
2892
2893 Index index = 0;
2894
2895 for(Index i = 0; i < rows_number; i++)
2896 {
2897 if(targets(i,0) > decision_threshold && outputs(i,0) < decision_threshold)
2898 {
2899 false_negatives_indices_copy(index) = testing_indices(i);
2900 index++;
2901 }
2902 }
2903
2904 Tensor<Index, 1> false_negatives_indices(index);
2905
2906 memcpy(false_negatives_indices.data(), false_negatives_indices_copy.data(), static_cast<size_t>(index)*sizeof(Index));
2907
2908 return false_negatives_indices;
2909}
2910
2911
2918
2919Tensor<Index, 1> TestingAnalysis::calculate_true_negative_samples(const Tensor<type, 2>& targets, const Tensor<type, 2>& outputs,
2920 const Tensor<Index, 1>& testing_indices, const type& decision_threshold) const
2921{
2922 const Index rows_number = targets.dimension(0);
2923
2924 Tensor<Index, 1> true_negatives_indices_copy(rows_number);
2925
2926 Index index = 0;
2927
2928 for(Index i = 0; i < rows_number; i++)
2929 {
2930 if(targets(i,0) < decision_threshold && outputs(i,0) < decision_threshold)
2931 {
2932 true_negatives_indices_copy(index) = testing_indices(i);
2933 index++;
2934 }
2935 }
2936
2937 Tensor<Index, 1> true_negatives_indices(index);
2938
2939 memcpy(true_negatives_indices.data(), true_negatives_indices_copy.data(), static_cast<size_t>(index)*sizeof(Index));
2940
2941 return true_negatives_indices;
2942}
2943
2944
2945Tensor<type, 1> TestingAnalysis::calculate_multiple_classification_tests() const
2946{
2947 Tensor<type, 1> multiple_classification_tests(2);
2948
2949 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
2950 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
2951
2952 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
2953
2954 const Tensor<Index, 2> confusion_matrix = calculate_confusion_multiple_classification(targets, outputs);
2955
2956 type diagonal_sum = type(0);
2957 type off_diagonal_sum = type(0);
2958 const Tensor<Index, 0> total_sum = confusion_matrix.sum();
2959
2960 for(Index i = 0; i < confusion_matrix.dimension(0); i++)
2961 {
2962 for(Index j = 0; j < confusion_matrix.dimension(1); j++)
2963 {
2964 i == j ? diagonal_sum += static_cast<type>(confusion_matrix(i,j)) : off_diagonal_sum += static_cast<type>(confusion_matrix(i,j));
2965 }
2966 }
2967
2968 multiple_classification_tests(0) = diagonal_sum/static_cast<type>(total_sum());
2969 multiple_classification_tests(1) = off_diagonal_sum/static_cast<type>(total_sum());
2970
2971 return multiple_classification_tests;
2972}
2973
2974
2975void TestingAnalysis::save_confusion(const string& confusion_file_name) const
2976{
2977 const Tensor<Index, 2> confusion = calculate_confusion();
2978
2979 const Index columns_number = confusion.dimension(0);
2980
2981 ofstream confusion_file(confusion_file_name);
2982
2983 Tensor<string, 1> target_variable_names = data_set_pointer->get_target_variables_names();
2984
2985 confusion_file << ",";
2986
2987 for(Index i = 0; i < confusion.dimension(0); i++)
2988 {
2989 confusion_file << target_variable_names(i);
2990
2991 if(i != target_variable_names.dimension(0) -1)
2992 {
2993 confusion_file << ",";
2994 }
2995 }
2996
2997 confusion_file << endl;
2998
2999 for(Index i = 0; i < columns_number; i++)
3000 {
3001 confusion_file << target_variable_names(i) << ",";
3002
3003 for(Index j = 0; j < columns_number; j++)
3004 {
3005 if(j == columns_number - 1)
3006 {
3007 confusion_file << confusion(i,j) << endl;
3008 }
3009 else
3010 {
3011 confusion_file << confusion(i,j) << ",";
3012 }
3013 }
3014 }
3015 confusion_file.close();
3016}
3017
3018
3019void TestingAnalysis::save_multiple_classification_tests(const string& classification_tests_file_name) const
3020{
3021 const Tensor<type, 1> multiple_classification_tests = calculate_multiple_classification_tests();
3022
3023 ofstream multiple_classifiaction_tests_file(classification_tests_file_name);
3024
3025 multiple_classifiaction_tests_file << "accuracy,error" << endl;
3026 multiple_classifiaction_tests_file << multiple_classification_tests(0)* type(100) << "," << multiple_classification_tests(1)* type(100) << endl;
3027
3028 multiple_classifiaction_tests_file.close();
3029}
3030
3031
3033
3035{
3036#ifdef OPENNN_DEBUG
3037
3038 check();
3039
3041 {
3042 ostringstream buffer;
3043
3044 buffer << "OpenNN Exception: TestingAnalysis class.\n"
3045 << "BinaryClassificationRates calculate_binary_classification_rates() const method.\n"
3046 << "Pointer to neural network in neural network is nullptr.\n";
3047
3048 throw logic_error(buffer.str());
3049 }
3050
3051 const Index inputs_number = neural_network_pointer->get_inputs_number();
3052
3053 // Control sentence
3054
3055 if(inputs_number != data_set_pointer->get_input_variables_number())
3056 {
3057 ostringstream buffer;
3058
3059 buffer << "OpenNN Exception: TestingAnalysis class." << endl
3060 << "BinaryClassificationRates calculate_binary_classification_rates() const method." << endl
3061 << "Number of inputs in neural network must be equal to number of inputs in data set." << endl;
3062
3063 throw logic_error(buffer.str());
3064 }
3065
3066 const Index outputs_number = neural_network_pointer->get_outputs_number();
3067
3068 if(outputs_number != data_set_pointer->get_target_variables_number())
3069 {
3070 ostringstream buffer;
3071
3072 buffer << "OpenNN Exception: TestingAnalysis class." << endl
3073 << "BinaryClassificationRates calculate_binary_classification_rates() const method." << endl
3074 << "Number of outputs in neural network must be equal to number of targets in data set." << endl;
3075
3076 throw logic_error(buffer.str());
3077 }
3078
3079#endif
3080
3081 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
3082 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
3083
3084 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
3085
3086 const Tensor<Index, 1> testing_indices = data_set_pointer->get_testing_samples_indices();
3087
3088 return calculate_multiple_classification_rates(targets, outputs, testing_indices);
3089}
3090
3091
3095
3096Tensor<Tensor<Index,1>, 2> TestingAnalysis::calculate_multiple_classification_rates(const Tensor<type, 2>& targets,
3097 const Tensor<type, 2>& outputs,
3098 const Tensor<Index, 1>& testing_indices) const
3099{
3100 const Index samples_number = targets.dimension(0);
3101 const Index targets_number = targets.dimension(1);
3102
3103 Tensor< Tensor<Index, 1>, 2> multiple_classification_rates(targets_number, targets_number);
3104
3105 // Count instances per class
3106
3107 const Tensor<Index, 2> confusion = calculate_confusion_multiple_classification(targets, outputs);
3108
3109 for(Index i = 0; i < targets_number; i++)
3110 {
3111 for(Index j = 0; j < targets_number; j++)
3112 {
3113 multiple_classification_rates(i,j).resize(confusion(i,j));
3114 }
3115 }
3116
3117 // Save indices
3118
3119 Index target_index;
3120 Index output_index;
3121
3122 Tensor<Index, 2> indices(targets_number, targets_number);
3123 indices.setZero();
3124
3125 for(Index i = 0; i < samples_number; i++)
3126 {
3127 target_index = maximal_index(targets.chip(i, 0));
3128 output_index = maximal_index(outputs.chip(i, 0));
3129
3130 multiple_classification_rates(target_index, output_index)(indices(target_index, output_index)) = testing_indices(i);
3131
3132 indices(target_index, output_index)++;
3133 }
3134
3135 return multiple_classification_rates;
3136}
3137
3138
3139Tensor<string, 2> TestingAnalysis::calculate_well_classified_samples(const Tensor<type, 2>& targets,
3140 const Tensor<type, 2>& outputs,
3141 const Tensor<string, 1>& labels)
3142{
3143 const Index samples_number = targets.dimension(0);
3144
3145 Tensor<string, 2> well_lassified_samples(samples_number, 4);
3146
3147 Index predicted_class, actual_class;
3148 Index number_of_well_classified = 0;
3149 string class_name;
3150
3151 const Tensor<string, 1> target_variables_names = data_set_pointer->get_target_variables_names();
3152
3153 for(Index i = 0; i < samples_number; i++)
3154 {
3155 predicted_class = maximal_index(outputs.chip(i, 0));
3156 actual_class = maximal_index(targets.chip(i, 0));
3157
3158 if(actual_class != predicted_class)
3159 {
3160 continue;
3161 }
3162 else
3163 {
3164 well_lassified_samples(number_of_well_classified, 0) = labels(i);
3165 class_name = target_variables_names(actual_class);
3166 well_lassified_samples(number_of_well_classified, 1) = class_name;
3167 class_name = target_variables_names(predicted_class);
3168 well_lassified_samples(number_of_well_classified, 2) = class_name;
3169 well_lassified_samples(number_of_well_classified, 3) = to_string(double(outputs(i, predicted_class)));
3170
3171 number_of_well_classified ++;
3172 }
3173 }
3174
3175 Eigen::array<Index, 2> offsets = {0, 0};
3176 Eigen::array<Index, 2> extents = {number_of_well_classified, 4};
3177
3178 return well_lassified_samples.slice(offsets, extents);
3179}
3180
3181
3182Tensor<string, 2> TestingAnalysis::calculate_misclassified_samples(const Tensor<type, 2>& targets,
3183 const Tensor<type, 2>& outputs,
3184 const Tensor<string, 1>& labels)
3185{
3186 const Index samples_number = targets.dimension(0);
3187
3188 Index predicted_class, actual_class;
3189 string class_name;
3190
3191 const Tensor<string, 1> target_variables_names = neural_network_pointer->get_outputs_names();
3192
3193 Index count_misclassified = 0;
3194
3195 for(Index i = 0; i < samples_number; i++)
3196 {
3197 predicted_class = maximal_index(outputs.chip(i, 0));
3198 actual_class = maximal_index(targets.chip(i, 0));
3199
3200 if(actual_class != predicted_class) count_misclassified++;
3201 }
3202
3203 Tensor<string, 2> misclassified_samples(count_misclassified, 4);
3204
3205 Index j = 0;
3206
3207 for(Index i = 0; i < samples_number; i++)
3208 {
3209 predicted_class = maximal_index(outputs.chip(i, 0));
3210 actual_class = maximal_index(targets.chip(i, 0));
3211
3212 if(actual_class == predicted_class)
3213 {
3214 continue;
3215 }
3216 else
3217 {
3218 misclassified_samples(j, 0) = labels(i);
3219 class_name = target_variables_names(actual_class);
3220 misclassified_samples(j, 1) = class_name;
3221 class_name = target_variables_names(predicted_class);
3222 misclassified_samples(j, 2) = class_name;
3223 misclassified_samples(j, 3) = to_string(double(outputs(i, predicted_class)));
3224 j++;
3225 }
3226 }
3227
3228// Eigen::array<Index, 2> offsets = {0, 0};
3229// Eigen::array<Index, 2> extents = {number_of_misclassified, 4};
3230
3231// return misclassified_samples.slice(offsets, extents);
3232 return misclassified_samples;
3233}
3234
3235
3236void TestingAnalysis::save_well_classified_samples(const Tensor<type, 2>& targets,
3237 const Tensor<type, 2>& outputs,
3238 const Tensor<string, 1>& labels,
3239 const string& well_classified_samples_file_name)
3240{
3241 const Tensor<string,2> well_classified_samples = calculate_well_classified_samples(targets,
3242 outputs,
3243 labels);
3244
3245 ofstream well_classified_samples_file(well_classified_samples_file_name);
3246 well_classified_samples_file << "sample_name,actual_class,predicted_class,probability" << endl;
3247 for(Index i = 0; i < well_classified_samples.dimension(0); i++)
3248 {
3249 well_classified_samples_file << well_classified_samples(i, 0) << ",";
3250 well_classified_samples_file << well_classified_samples(i, 1) << ",";
3251 well_classified_samples_file << well_classified_samples(i, 2) << ",";
3252 well_classified_samples_file << well_classified_samples(i, 3) << endl;
3253 }
3254 well_classified_samples_file.close();
3255}
3256
3257
3258void TestingAnalysis::save_misclassified_samples(const Tensor<type, 2>& targets,
3259 const Tensor<type, 2>& outputs,
3260 const Tensor<string, 1>& labels,
3261 const string& misclassified_samples_file_name)
3262{
3263 const Tensor<string,2> misclassified_samples = calculate_misclassified_samples(targets,
3264 outputs,
3265 labels);
3266
3267 ofstream misclassified_samples_file(misclassified_samples_file_name);
3268 misclassified_samples_file << "sample_name,actual_class,predicted_class,probability" << endl;
3269 for(Index i = 0; i < misclassified_samples.dimension(0); i++)
3270 {
3271 misclassified_samples_file << misclassified_samples(i, 0) << ",";
3272 misclassified_samples_file << misclassified_samples(i, 1) << ",";
3273 misclassified_samples_file << misclassified_samples(i, 2) << ",";
3274 misclassified_samples_file << misclassified_samples(i, 3) << endl;
3275 }
3276 misclassified_samples_file.close();
3277}
3278
3279
3280void TestingAnalysis::save_well_classified_samples_statistics(const Tensor<type, 2>& targets,
3281 const Tensor<type, 2>& outputs,
3282 const Tensor<string, 1>& labels,
3283 const string& statistics_file_name)
3284{
3285 const Tensor<string, 2> well_classified_samples = calculate_well_classified_samples(targets,
3286 outputs,
3287 labels);
3288
3289 Tensor<type, 1> well_classified_numerical_probabilities(well_classified_samples.dimension(0));
3290
3291 for(Index i = 0; i < well_classified_numerical_probabilities.size(); i++)
3292 {
3293 well_classified_numerical_probabilities(i) = type(::atof(well_classified_samples(i, 3).c_str()));
3294 }
3295
3296 ofstream classification_statistics_file(statistics_file_name);
3297 classification_statistics_file << "minimum,maximum,mean,std" << endl;
3298 classification_statistics_file << well_classified_numerical_probabilities.minimum() << ",";
3299 classification_statistics_file << well_classified_numerical_probabilities.maximum() << ",";
3300
3301 classification_statistics_file << well_classified_numerical_probabilities.mean() << ",";
3302
3303 classification_statistics_file << standard_deviation(well_classified_numerical_probabilities);
3304
3305}
3306
3307
3308void TestingAnalysis::save_misclassified_samples_statistics(const Tensor<type, 2>& targets,
3309 const Tensor<type, 2>& outputs,
3310 const Tensor<string, 1>& labels,
3311 const string& statistics_file_name)
3312{
3313 const Tensor<string, 2> misclassified_samples = calculate_misclassified_samples(targets,
3314 outputs,
3315 labels);
3316
3317 Tensor<type, 1> misclassified_numerical_probabilities(misclassified_samples.dimension(0));
3318
3319 for(Index i = 0; i < misclassified_numerical_probabilities.size(); i++)
3320 {
3321 misclassified_numerical_probabilities(i) = type(::atof(misclassified_samples(i, 3).c_str()));
3322 }
3323 ofstream classification_statistics_file(statistics_file_name);
3324 classification_statistics_file << "minimum,maximum,mean,std" << endl;
3325 classification_statistics_file << misclassified_numerical_probabilities.minimum() << ",";
3326 classification_statistics_file << misclassified_numerical_probabilities.maximum() << ",";
3327/*
3328 classification_statistics_file << misclassified_numerical_probabilities.mean() << ",";
3329*/
3330 classification_statistics_file << standard_deviation(misclassified_numerical_probabilities);
3331}
3332
3333
3334void TestingAnalysis::save_well_classified_samples_probability_histogram(const Tensor<type, 2>& targets,
3335 const Tensor<type, 2>& outputs,
3336 const Tensor<string, 1>& labels,
3337 const string& histogram_file_name)
3338{
3339 const Tensor<string, 2> well_classified_samples = calculate_well_classified_samples(targets,
3340 outputs,
3341 labels);
3342
3343 Tensor<type, 1> well_classified_numerical_probabilities(well_classified_samples.dimension(0));
3344
3345 for(Index i = 0; i < well_classified_numerical_probabilities.size(); i++)
3346 {
3347 well_classified_numerical_probabilities(i) = type(::atof(well_classified_samples(i, 3).c_str()));
3348 }
3349
3350 Histogram misclassified_samples_histogram(well_classified_numerical_probabilities);
3351 misclassified_samples_histogram.save(histogram_file_name);
3352}
3353
3354
3355void TestingAnalysis::save_well_classified_samples_probability_histogram(const Tensor<string, 2>& well_classified_samples,
3356 const string& histogram_file_name)
3357{
3358
3359 Tensor<type, 1> well_classified_numerical_probabilities(well_classified_samples.dimension(0));
3360
3361 for(Index i = 0; i < well_classified_numerical_probabilities.size(); i++)
3362 {
3363 well_classified_numerical_probabilities(i) = type(::atof(well_classified_samples(i, 3).c_str()));
3364 }
3365
3366 Histogram misclassified_samples_histogram(well_classified_numerical_probabilities);
3367 misclassified_samples_histogram.save(histogram_file_name);
3368}
3369
3370
3371void TestingAnalysis::save_misclassified_samples_probability_histogram(const Tensor<type, 2>& targets,
3372 const Tensor<type, 2>& outputs,
3373 const Tensor<string, 1>& labels,
3374 const string& histogram_file_name)
3375{
3376 const Tensor<string, 2> misclassified_samples = calculate_misclassified_samples(targets,
3377 outputs,
3378 labels);
3379
3380 Tensor<type, 1> misclassified_numerical_probabilities(misclassified_samples.dimension(0));
3381
3382 for(Index i = 0; i < misclassified_numerical_probabilities.size(); i++)
3383 {
3384 misclassified_numerical_probabilities(i) = type(::atof(misclassified_samples(i, 3).c_str()));
3385 }
3386
3387 Histogram misclassified_samples_histogram(misclassified_numerical_probabilities);
3388 misclassified_samples_histogram.save(histogram_file_name);
3389}
3390
3391
3392void TestingAnalysis::save_misclassified_samples_probability_histogram(const Tensor<string, 2>& misclassified_samples,
3393 const string& histogram_file_name)
3394{
3395
3396 Tensor<type, 1> misclassified_numerical_probabilities(misclassified_samples.dimension(0));
3397
3398 for(Index i = 0; i < misclassified_numerical_probabilities.size(); i++)
3399 {
3400 misclassified_numerical_probabilities(i) = type(::atof(misclassified_samples(i, 3).c_str()));
3401 }
3402
3403 Histogram misclassified_samples_histogram(misclassified_numerical_probabilities);
3404 misclassified_samples_histogram.save(histogram_file_name);
3405}
3406
3407
3413
3414Tensor<Tensor<type, 1>, 1> TestingAnalysis::calculate_error_autocorrelation(const Index& maximum_lags_number) const
3415{
3416#ifdef OPENNN_DEBUG
3417
3418 check();
3419
3421 {
3422 ostringstream buffer;
3423
3424 buffer << "OpenNN Exception: TestingAnalysis class.\n"
3425 << "Tensor<type, 1> calculate_error_autocorrelation() const method.\n"
3426 << "Pointer to neural network in neural network is nullptr.\n";
3427
3428 throw logic_error(buffer.str());
3429 }
3430
3431 const Index inputs_number = neural_network_pointer->get_inputs_number();
3432
3433 // Control sentence
3434
3435 if(inputs_number != data_set_pointer->get_input_variables_number())
3436 {
3437 ostringstream buffer;
3438
3439 buffer << "OpenNN Exception: TestingAnalysis class." << endl
3440 << "Tensor<type, 1> calculate_error_autocorrelation() const method." << endl
3441 << "Number of inputs in neural network must be equal to number of inputs in data set." << endl;
3442
3443 throw logic_error(buffer.str());
3444 }
3445
3446 const Index outputs_number = neural_network_pointer->get_outputs_number();
3447
3448 if(outputs_number != data_set_pointer->get_target_variables_number())
3449 {
3450 ostringstream buffer;
3451
3452 buffer << "OpenNN Exception: TestingAnalysis class." << endl
3453 << "Tensor<type, 1> calculate_error_autocorrelation() const method." << endl
3454 << "Number of outputs in neural network must be equal to number of targets in data set." << endl;
3455
3456 throw logic_error(buffer.str());
3457 }
3458
3459#endif
3460
3461 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
3462 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
3463
3464 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
3465
3466 const Index targets_number = data_set_pointer->get_target_variables_number();
3467
3468 const Tensor<type, 2> error = targets - outputs;
3469
3470 Tensor<Tensor<type, 1>, 1> error_autocorrelations(targets_number);
3471
3472 for(Index i = 0; i < targets_number; i++)
3473 {
3474 error_autocorrelations[i] = autocorrelations(this->thread_pool_device, error.chip(i,1), maximum_lags_number);
3475 }
3476
3477 return error_autocorrelations;
3478}
3479
3480
3486
3487Tensor<Tensor<type, 1>, 1> TestingAnalysis::calculate_inputs_errors_cross_correlation(const Index& lags_number) const
3488{
3489#ifdef OPENNN_DEBUG
3490
3491 check();
3492
3494 {
3495 ostringstream buffer;
3496
3497 buffer << "OpenNN Exception: TestingAnalysis class.\n"
3498 << "Tensor<type, 1> calculate_inputs_errors_cross_correlation() const method.\n"
3499 << "Pointer to neural network in neural network is nullptr.\n";
3500
3501 throw logic_error(buffer.str());
3502 }
3503
3504 const Index inputs_number = neural_network_pointer->get_inputs_number();
3505
3506 // Control sentence
3507
3508
3509 if(inputs_number != data_set_pointer->get_input_variables_number())
3510 {
3511 ostringstream buffer;
3512
3513 buffer << "OpenNN Exception: TestingAnalysis class." << endl
3514 << "Tensor<type, 1> calculate_inputs_errors_cross_correlation() const method." << endl
3515 << "Number of inputs in neural network must be equal to number of inputs in data set." << endl;
3516
3517 throw logic_error(buffer.str());
3518 }
3519
3520 const Index outputs_number = neural_network_pointer->get_outputs_number();
3521
3522 if(outputs_number != data_set_pointer->get_target_variables_number())
3523 {
3524 ostringstream buffer;
3525
3526 buffer << "OpenNN Exception: TestingAnalysis class." << endl
3527 << "Tensor<type, 1> calculate_inputs_errors_cross_correlation() const method." << endl
3528 << "Number of outputs in neural network must be equal to number of targets in data set." << endl;
3529
3530 throw logic_error(buffer.str());
3531 }
3532
3533#endif
3534
3535 const Index targets_number = data_set_pointer->get_target_variables_number();
3536
3537 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
3538
3539 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
3540
3541 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
3542
3543 const Tensor<type, 2> errors = targets - outputs;
3544
3545 Tensor<Tensor<type, 1>, 1> inputs_errors_cross_correlation(targets_number);
3546
3547 for(Index i = 0; i < targets_number; i++)
3548 {
3549 inputs_errors_cross_correlation[i] = cross_correlations(this->thread_pool_device, inputs.chip(i,1), errors.chip(i,1), lags_number);
3550 }
3551
3552 return inputs_errors_cross_correlation;
3553}
3554
3555
3576
3578{
3579#ifdef OPENNN_DEBUG
3580
3581 const Index inputs_number = neural_network_pointer->get_inputs_number();
3582
3583 if(!data_set_pointer)
3584 {
3585 ostringstream buffer;
3586
3587 buffer << "OpenNN Exception: TestingAnalysis class." << endl
3588 << "Tensor<type, 1> calculate_binary_classification_tests() const." << endl
3589 << "Data set is nullptr." << endl;
3590
3591 throw logic_error(buffer.str());
3592 }
3593
3594 const Index targets_number = data_set_pointer->get_target_variables_number();
3595
3596 const Index outputs_number = neural_network_pointer->get_outputs_number();
3597
3598 // Control sentence
3599
3600 if(inputs_number != data_set_pointer->get_input_variables_number())
3601 {
3602 ostringstream buffer;
3603
3604 buffer << "OpenNN Exception: TestingAnalysis class." << endl
3605 << "Tensor<type, 1> calculate_binary_classification_tests() const." << endl
3606 << "Number of inputs in neural network is not equal to number of inputs in data set." << endl;
3607
3608 throw logic_error(buffer.str());
3609 }
3610 else if(outputs_number != 1)
3611 {
3612 ostringstream buffer;
3613
3614 buffer << "OpenNN Exception: TestingAnalysis class." << endl
3615 << "Tensor<type, 1> calculate_binary_classification_tests() const." << endl
3616 << "Number of outputs in neural network must be one." << endl;
3617
3618 throw logic_error(buffer.str());
3619 }
3620 else if(targets_number != 1)
3621 {
3622 ostringstream buffer;
3623
3624 buffer << "OpenNN Exception: TestingAnalysis class." << endl
3625 << "Tensor<type, 1> calculate_binary_classification_tests() const." << endl
3626 << "Number of targets in data set must be one." << endl;
3627
3628 throw logic_error(buffer.str());
3629 }
3630
3631#endif
3632
3633 // Confusion matrix
3634
3635 const Tensor<Index, 2> confusion = calculate_confusion();
3636
3637 const Index true_positive = confusion(0,0);
3638 const Index false_positive = confusion(1,0);
3639 const Index false_negative = confusion(0,1);
3640 const Index true_negative = confusion(1,1);
3641
3642 // Classification accuracy
3643
3644 type classification_accuracy;
3645
3646 if(true_positive + true_negative + false_positive + false_negative == 0)
3647 {
3648 classification_accuracy = type(0);
3649 }
3650 else
3651 {
3652 classification_accuracy = static_cast<type>(true_positive + true_negative)/static_cast<type>(true_positive + true_negative + false_positive + false_negative);
3653 }
3654
3655 // Error rate
3656
3657 type error_rate;
3658
3659 if(true_positive + true_negative + false_positive + false_negative == 0)
3660 {
3661 error_rate = type(0);
3662 }
3663 else
3664 {
3665 error_rate = static_cast<type>(false_positive + false_negative)/static_cast<type>(true_positive + true_negative + false_positive + false_negative);
3666 }
3667
3668 // Sensitivity
3669
3670 type sensitivity;
3671
3672 if(true_positive + false_negative == 0)
3673 {
3674 sensitivity = type(0);
3675 }
3676 else
3677 {
3678 sensitivity = static_cast<type>(true_positive)/static_cast<type>(true_positive + false_negative);
3679 }
3680
3681 // Specificity
3682
3683 type specificity;
3684
3685 if(true_negative + false_positive == 0)
3686 {
3687 specificity = type(0);
3688 }
3689 else
3690 {
3691 specificity = static_cast<type>(true_negative)/static_cast<type>(true_negative + false_positive);
3692 }
3693
3694 // Precision
3695
3696 type precision;
3697
3698 if(true_positive + false_positive == 0)
3699 {
3700 precision = type(0);
3701 }
3702 else
3703 {
3704 precision = static_cast<type>(true_positive) /static_cast<type>(true_positive + false_positive);
3705 }
3706
3707 // Positive likelihood
3708
3709 type positive_likelihood;
3710
3711 if(abs(classification_accuracy - static_cast<type>(1.0)) < type(NUMERIC_LIMITS_MIN))
3712 {
3713 positive_likelihood = type(1);
3714 }
3715 else if(abs(static_cast<type>(1.0) - specificity) < type(NUMERIC_LIMITS_MIN))
3716 {
3717 positive_likelihood = type(0);
3718 }
3719 else
3720 {
3721 positive_likelihood = sensitivity/(static_cast<type>(1.0) - specificity);
3722 }
3723
3724 // Negative likelihood
3725
3726 type negative_likelihood;
3727
3728 if(static_cast<Index>(classification_accuracy) == 1)
3729 {
3730 negative_likelihood = type(1);
3731 }
3732 else if(abs(type(1) - sensitivity) < type(NUMERIC_LIMITS_MIN))
3733 {
3734 negative_likelihood = type(0);
3735 }
3736 else
3737 {
3738 negative_likelihood = specificity/(static_cast<type>(1.0) - sensitivity);
3739 }
3740
3741 // F1 score
3742
3743 type f1_score;
3744
3745 if(2*true_positive + false_positive + false_negative == 0)
3746 {
3747 f1_score = type(0);
3748 }
3749 else
3750 {
3751 f1_score = static_cast<type>(2.0)* type(true_positive)/(static_cast<type>(2.0)* type(true_positive) + type(false_positive) + type(false_negative));
3752 }
3753
3754 // False positive rate
3755
3756 type false_positive_rate;
3757
3758 if(false_positive + true_negative == 0)
3759 {
3760 false_positive_rate = type(0);
3761 }
3762 else
3763 {
3764 false_positive_rate = static_cast<type>(false_positive)/static_cast<type>(false_positive + true_negative);
3765 }
3766
3767 // False discovery rate
3768
3769 type false_discovery_rate;
3770
3771 if(false_positive + true_positive == 0)
3772 {
3773 false_discovery_rate = type(0);
3774 }
3775 else
3776 {
3777 false_discovery_rate = static_cast<type>(false_positive) /static_cast<type>(false_positive + true_positive);
3778 }
3779
3780 // False negative rate
3781
3782 type false_negative_rate;
3783
3784 if(false_negative + true_positive == 0)
3785 {
3786 false_negative_rate = type(0);
3787 }
3788 else
3789 {
3790 false_negative_rate = static_cast<type>(false_negative)/static_cast<type>(false_negative + true_positive);
3791 }
3792
3793 // Negative predictive value
3794
3795 type negative_predictive_value;
3796
3797 if(true_negative + false_negative == 0)
3798 {
3799 negative_predictive_value = type(0);
3800 }
3801 else
3802 {
3803 negative_predictive_value = static_cast<type>(true_negative)/static_cast<type>(true_negative + false_negative);
3804 }
3805
3806 // Matthews correlation coefficient
3807
3808 type Matthews_correlation_coefficient;
3809
3810 if((true_positive + false_positive) *(true_positive + false_negative) *(true_negative + false_positive) *(true_negative + false_negative) == 0)
3811 {
3812 Matthews_correlation_coefficient = type(0);
3813 }
3814 else
3815 {
3816 Matthews_correlation_coefficient = static_cast<type>(true_positive * true_negative - false_positive * false_negative) / static_cast<type>(sqrt(((true_positive + false_positive) *(true_positive + false_negative) *(true_negative + false_positive) *(true_negative + false_negative))))
3817 ;
3818 }
3819
3820 //Informedness
3821
3822 type informedness = sensitivity + specificity - type(1);
3823
3824 //Markedness
3825
3826 type markedness;
3827
3828 if(true_negative + false_positive == 0)
3829 {
3830 markedness = precision - type(1);
3831 }
3832 else
3833 {
3834 markedness = precision + static_cast<type>(true_negative)/static_cast<type>(true_negative + false_positive) - static_cast<type>(1.0);
3835 }
3836
3837 //Arrange vector
3838
3839 Tensor<type, 1> binary_classification_test(15);
3840
3841 binary_classification_test[0] = classification_accuracy;
3842 binary_classification_test[1] = error_rate;
3843 binary_classification_test[2] = sensitivity;
3844 binary_classification_test[3] = specificity;
3845 binary_classification_test[4] = precision;
3846 binary_classification_test[5] = positive_likelihood;
3847 binary_classification_test[6] = negative_likelihood;
3848 binary_classification_test[7] = f1_score;
3849 binary_classification_test[8] = false_positive_rate;
3850 binary_classification_test[9] = false_discovery_rate;
3851 binary_classification_test[10] = false_negative_rate;
3852 binary_classification_test[11] = negative_predictive_value;
3853 binary_classification_test[12] = Matthews_correlation_coefficient;
3854 binary_classification_test[13] = informedness;
3855 binary_classification_test[14] = markedness;
3856
3857 return binary_classification_test;
3858}
3859
3860
3861void TestingAnalysis::print_binary_classification_tests() const
3862{
3863 const Tensor<type, 1> binary_classification_tests = calculate_binary_classification_tests();
3864
3865 cout << "Binary classification tests: " << endl;
3866 cout << "Classification accuracy : " << binary_classification_tests[0] << endl;
3867 cout << "Error rate : " << binary_classification_tests[1] << endl;
3868 cout << "Sensitivity : " << binary_classification_tests[2] << endl;
3869 cout << "Specificity : " << binary_classification_tests[3] << endl;
3870}
3871
3872
3874
3876{
3877#ifdef OPENNN_DEBUG
3878
3879 check();
3880
3882 {
3883 ostringstream buffer;
3884
3885 buffer << "OpenNN Exception: TestingAnalysis class.\n"
3886 << "Tensor<type, 1> calculate_inputs_errors_cross_correlation() const method.\n"
3887 << "Pointer to neural network in neural network is nullptr.\n";
3888
3889 throw logic_error(buffer.str());
3890 }
3891
3892 const Index inputs_number = neural_network_pointer->get_inputs_number();
3893
3894 // Control sentence
3895
3896 if(inputs_number != data_set_pointer->get_input_variables_number())
3897 {
3898 ostringstream buffer;
3899
3900 buffer << "OpenNN Exception: TestingAnalysis class." << endl
3901 << "Tensor<type, 1> calculate_inputs_errors_cross_correlation() const method." << endl
3902 << "Number of inputs in neural network must be equal to number of inputs in data set." << endl;
3903
3904 throw logic_error(buffer.str());
3905 }
3906
3907 const Index outputs_number = neural_network_pointer->get_outputs_number();
3908
3909 if(outputs_number != data_set_pointer->get_target_variables_number())
3910 {
3911 ostringstream buffer;
3912
3913 buffer << "OpenNN Exception: TestingAnalysis class." << endl
3914 << "Tensor<type, 1> calculate_inputs_errors_cross_correlation() const method." << endl
3915 << "Number of outputs in neural network must be equal to number of targets in data set." << endl;
3916
3917 throw logic_error(buffer.str());
3918 }
3919
3920#endif
3921
3922 const Tensor<type, 2> inputs = data_set_pointer->get_testing_input_data();
3923 const Tensor<type, 2> targets = data_set_pointer->get_testing_target_data();
3924
3925 const Tensor<type, 2> outputs = neural_network_pointer->calculate_outputs(inputs);
3926
3927 const Index testing_samples_number = data_set_pointer->get_testing_samples_number();
3928
3929 type logloss = type(0);
3930
3931 for(Index i = 0; i < testing_samples_number; i++)
3932 {
3933 logloss += targets(i,0)*log(outputs(i,0)) + (type(1) - targets(i,0))*log(type(1) - outputs(i,0));
3934 }
3935
3936 return -logloss/type(testing_samples_number);
3937}
3938
3939
3941
3943{
3944}
3945
3946
3949
3951{
3952 ostringstream buffer;
3953 file_stream.OpenElement("TestingAnalysis");
3954
3955 // Display
3956
3957 file_stream.OpenElement("Display");
3958
3959 buffer.str("");
3960 buffer << display;
3961
3962 file_stream.PushText(buffer.str().c_str());
3963
3964 file_stream.CloseElement();
3965
3966
3967 file_stream.CloseElement();
3968}
3969
3970
3973
3975{
3976 ostringstream buffer;
3977
3978 const tinyxml2::XMLElement* root_element = document.FirstChildElement("TestingAnalysis");
3979
3980 if(!root_element)
3981 {
3982 buffer << "OpenNN Exception: TestingAnalysis class.\n"
3983 << "void from_XML(const tinyxml2::XMLDocument&) method.\n"
3984 << "Testing analysis element is nullptr.\n";
3985
3986 throw logic_error(buffer.str());
3987 }
3988
3989 // Display
3990
3991 const tinyxml2::XMLElement* element = root_element->FirstChildElement("Display");
3992
3993 if(element)
3994 {
3995 string new_display_string = element->GetText();
3996
3997 try
3998 {
3999 set_display(new_display_string != "0");
4000 }
4001 catch(const logic_error& e)
4002 {
4003 cerr << e.what() << endl;
4004 }
4005 }
4006}
4007
4008
4011
4012void TestingAnalysis::save(const string& file_name) const
4013{
4014 FILE *pFile;
4015
4016 pFile = fopen(file_name.c_str(), "w");
4017
4018 tinyxml2::XMLPrinter document(pFile);
4019
4020 write_XML(document);
4021
4022 fclose(pFile);
4023}
4024
4025
4028
4029void TestingAnalysis::load(const string& file_name)
4030{
4031 set_default();
4032
4033 tinyxml2::XMLDocument document;
4034
4035 if(document.LoadFile(file_name.c_str()))
4036 {
4037 ostringstream buffer;
4038
4039 buffer << "OpenNN Exception: Testing analysis class.\n"
4040 << "void load(const string&) method.\n"
4041 << "Cannot load XML file " << file_name << ".\n";
4042
4043 throw logic_error(buffer.str());
4044 }
4045
4046 from_XML(document);
4047}
4048
4049bool TestingAnalysis::contains(const Tensor<type, 1>& tensor, const type& value) const
4050{
4051 Tensor<type, 1> copy(tensor);
4052
4053 type* it = find(copy.data(), copy.data()+copy.size(), value);
4054
4055 return it != (copy.data()+copy.size());
4056}
4057
4058Tensor<type, 2> TestingAnalysis::delete_row(const Tensor<type, 2>& tensor, const Index& row_index) const
4059{
4060 const Index rows_number = tensor.dimension(0);
4061 const Index columns_number = tensor.dimension(1);
4062 #ifdef OPENNN_DEBUG
4063
4064 if(row_index > rows_number)
4065 {
4066 ostringstream buffer;
4067
4068 buffer << "OpenNN Exception: Matrix Template.\n"
4069 << "Matrix<T> delete_row(const size_t&) const.\n"
4070 << "Index of row must be less than number of rows.\n"
4071 << "row index: " << row_index << "rows_number" << rows_number << "\n";
4072
4073 throw logic_error(buffer.str());
4074 }
4075 else if(rows_number < 2)
4076 {
4077 ostringstream buffer;
4078
4079 buffer << "OpenNN Exception: Matrix Template.\n"
4080 << "Matrix<T> delete_row(const size_t&) const.\n"
4081 << "Number of rows must be equal or greater than two.\n";
4082
4083 throw logic_error(buffer.str());
4084 }
4085
4086 #endif
4087
4088 Tensor<type, 2> new_matrix(rows_number-1, columns_number);
4089
4090 for(Index i = 0; i < row_index; i++)
4091 {
4092 for(Index j = 0; j < columns_number; j++)
4093 {
4094 new_matrix(i,j) = tensor(i,j);
4095 }
4096 }
4097
4098 for(Index i = row_index+1; i < rows_number; i++)
4099 {
4100 for(Index j = 0; j < columns_number; j++)
4101 {
4102 new_matrix(i-1,j) = tensor(i,j);
4103 }
4104 }
4105
4106 return new_matrix;
4107}
4108
4109}
4110
4111
4112// OpenNN: Open Neural Networks Library.
4113// Copyright(C) 2005-2021 Artificial Intelligence Techniques, SL.
4114//
4115// This library is free software; you can redistribute it and/or
4116// modify it under the terms of the GNU Lesser General Public
4117// License as published by the Free Software Foundation; either
4118// version 2.1 of the License, or any later version.
4119//
4120// This library is distributed in the hope that it will be useful,
4121// but WITHOUT ANY WARRANTY; without even the implied warranty of
4122// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
4123// Lesser General Public License for more details.
4124
4125// You should have received a copy of the GNU Lesser General Public
4126// License along with this library; if not, write to the Free Software
4127// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
This class represents the concept of a data set for data modelling problems, such as approximation,...
Definition: data_set.h:56
Index get_training_samples_number() const
Returns the number of samples in the data set which will be used for training.
Definition: data_set.cpp:1386
Index get_target_variables_number() const
Returns the number of target variables of the data set.
Definition: data_set.cpp:2888
Tensor< Index, 1 > get_testing_samples_indices() const
Returns the indices of the samples which will be used for testing.
Definition: data_set.cpp:1124
Tensor< Index, 1 > calculate_target_distribution() const
Definition: data_set.cpp:8430
Index get_selection_samples_number() const
Returns the number of samples in the data set which will be used for selection.
Definition: data_set.cpp:1406
Tensor< type, 2 > get_selection_target_data() const
Definition: data_set.cpp:4005
Tensor< type, 2 > get_testing_input_data() const
Definition: data_set.cpp:4019
Index get_input_variables_number() const
Definition: data_set.cpp:2863
Index get_testing_samples_number() const
Returns the number of samples in the data set which will be used for testing.
Definition: data_set.cpp:1426
Tensor< type, 2 > get_selection_input_data() const
Definition: data_set.cpp:3991
Tensor< string, 1 > get_target_variables_names() const
Definition: data_set.cpp:2219
Tensor< type, 2 > get_training_input_data() const
Definition: data_set.cpp:3963
Tensor< type, 2 > get_testing_target_data() const
Definition: data_set.cpp:4033
Tensor< type, 2 > get_training_target_data() const
Definition: data_set.cpp:3977
This class represents the concept of neural network in the OpenNN library.
const Tensor< string, 1 > & get_outputs_names() const
Returns a string vector with the names of the variables used as outputs.
Index get_inputs_number() const
Returns the number of inputs to the neural network.
Tensor< type, 2 > calculate_outputs(const Tensor< type, 2 > &)
UnscalingLayer * get_unscaling_layer_pointer() const
Returns a pointer to the unscaling layers object composing this neural network object.
ProbabilisticLayer * get_probabilistic_layer_pointer() const
Returns a pointer to the first probabilistic layer composing this neural network.
const type & get_decision_threshold() const
Returns the decision threshold.
Tensor< type, 2 > calculate_roc_curve(const Tensor< type, 2 > &, const Tensor< type, 2 > &) const
Tensor< type, 2 > perform_calibration_plot_analysis() const
Performs a calibration plot analysis.
KolmogorovSmirnovResults perform_Kolmogorov_Smirnov_analysis() const
DataSet * data_set_pointer
Pointer to a data set object.
Tensor< type, 2 > calculate_lift_chart(const Tensor< type, 2 > &) const
NeuralNetwork * neural_network_pointer
Pointer to the neural network object to be tested.
type calculate_optimal_threshold(const Tensor< type, 2 > &, const Tensor< type, 2 > &) const
Tensor< Tensor< Index, 1 >, 2 > calculate_multiple_classification_rates() const
Returns a matrix of subvectors which have the rates for a multiple classification problem.
Tensor< type, 2 > calculate_percentage_error_data() const
void set_data_set_pointer(DataSet *)
Tensor< type, 2 > calculate_errors() const
const bool & get_display() const
Tensor< type, 1 > calculate_multiple_classification_testing_errors() const
Tensor< Index, 1 > calculate_false_negative_samples(const Tensor< type, 2 > &, const Tensor< type, 2 > &, const Tensor< Index, 1 > &, const type &) const
Tensor< Index, 1 > calculate_true_positive_samples(const Tensor< type, 2 > &, const Tensor< type, 2 > &, const Tensor< Index, 1 > &, const type &) const
type calculate_weighted_squared_error(const Tensor< type, 2 > &, const Tensor< type, 2 > &, const Tensor< type, 1 > &=Tensor< type, 1 >()) const
type calculate_cross_entropy_error(const Tensor< type, 2 > &, const Tensor< type, 2 > &) const
virtual void from_XML(const tinyxml2::XMLDocument &)
void load(const string &)
Tensor< type, 3 > calculate_error_data() const
Tensor< Index, 1 > calculate_positives_negatives_rate(const Tensor< type, 2 > &, const Tensor< type, 2 > &) const
Tensor< Index, 1 > calculate_false_positive_samples(const Tensor< type, 2 > &, const Tensor< type, 2 > &, const Tensor< Index, 1 > &, const type &) const
bool display
Display messages to screen.
Tensor< type, 2 > perform_cumulative_gain_analysis() const
Tensor< Index, 2 > calculate_confusion_multiple_classification(const Tensor< type, 2 > &, const Tensor< type, 2 > &) const
Tensor< type, 2 > perform_lift_chart_analysis() const
type calculate_Wilcoxon_parameter(const type &, const type &) const
type calculate_area_under_curve_confidence_limit(const Tensor< type, 2 > &, const Tensor< type, 2 > &) const
Tensor< type, 2 > calculate_calibration_plot(const Tensor< type, 2 > &, const Tensor< type, 2 > &) const
type calculate_area_under_curve(const Tensor< type, 2 > &, const Tensor< type, 2 > &) const
RocAnalysisResults perform_roc_analysis() const
Tensor< Index, 2 > calculate_confusion_binary_classification(const Tensor< type, 2 > &, const Tensor< type, 2 > &, const type &) const
Tensor< Tensor< Index, 1 >, 1 > calculate_maximal_errors(const Index &=10) const
void save(const string &) const
Tensor< Tensor< type, 1 >, 1 > calculate_inputs_errors_cross_correlation(const Index &=10) const
NeuralNetwork * get_neural_network_pointer() const
Returns a pointer to the neural network object which is to be tested.
type calculate_normalized_squared_error(const Tensor< type, 2 > &, const Tensor< type, 2 > &) const
Tensor< Index, 2 > calculate_confusion() const
Tensor< type, 2 > calculate_negative_cumulative_gain(const Tensor< type, 2 > &, const Tensor< type, 2 > &) const
void set_neural_network_pointer(NeuralNetwork *)
Tensor< Tensor< type, 1 >, 1 > calculate_error_autocorrelation(const Index &=10) const
Tensor< LinearRegressionAnalysis, 1 > perform_linear_regression_analysis() const
type calculate_logloss() const
Returns the logloss for a binary classification problem.
Tensor< type, 1 > calculate_maximum_gain(const Tensor< type, 2 > &, const Tensor< type, 2 > &) const
Tensor< type, 1 > calculate_binary_classification_tests() const
void print() const
Prints to the standard output the string representation of this testing analysis object.
Tensor< Histogram, 1 > calculate_error_data_histograms(const Index &=10) const
void set_display(const bool &)
virtual void write_XML(tinyxml2::XMLPrinter &) const
Tensor< type, 1 > calculate_binary_classification_testing_errors() const
Tensor< type, 1 > calculate_testing_errors() const
Tensor< Index, 1 > calculate_true_negative_samples(const Tensor< type, 2 > &, const Tensor< type, 2 > &, const Tensor< Index, 1 > &, const type &) const
Tensor< Correlation, 1 > linear_correlation() const
BinaryClassifcationRates calculate_binary_classification_rates() const
Tensor< type, 2 > calculate_cumulative_gain(const Tensor< type, 2 > &, const Tensor< type, 2 > &) const
DataSet * get_data_set_pointer() const
Returns a pointer to the data set object on which the neural network is tested.
Tensor< Histogram, 1 > calculate_output_histogram(const Tensor< type, 2 > &, const Index &=10) const
Tensor< Tensor< Descriptives, 1 >, 1 > calculate_error_data_descriptives() const
This class represents a layer of unscaling neurons.
Tensor< type, 1 > get_minimums() const
Tensor< type, 1 > get_maximums() const
friend half pow(half, half)
Definition: half.hpp:3427
void PushText(const char *text, bool cdata=false)
Add a text node.
Definition: tinyxml2.cpp:2878
virtual void CloseElement(bool compactMode=false)
If streaming, close the Element.
Definition: tinyxml2.cpp:2834
uint32 sqrt(uint32 &r, int &exp)
Definition: half.hpp:1480
HALF_CONSTEXPR half abs(half arg)
Definition: half.hpp:2735
half log(half arg)
Definition: half.hpp:3050
This structure provides the results obtained from the regression analysis.
Definition: correlations.h:40
This structure contains the binary classification rates.
Tensor< Index, 1 > true_negatives_indices
Vector with the indices of the samples which are true negative.
Tensor< Index, 1 > true_positives_indices
Vector with the indices of the samples which are true positive.
Tensor< Index, 1 > false_positives_indices
Vector with the indices of the samples which are false positive.
Tensor< Index, 1 > false_negatives_indices
Vector with the indices of the samples which are false negative.
This structure contains the results of a Kolmogorov-Smirnov analysis.
Tensor< type, 2 > positive_cumulative_gain
Matrix containing the data of a positive cumulative gain.
Tensor< type, 1 > maximum_gain
Maximum gain of the cumulative gain analysis.
Tensor< type, 2 > negative_cumulative_gain
Matrix containing the data of a negative cumulative gain.
This structure contains the results of a roc curve analysis.
type optimal_threshold
Optimal threshold of a ROC curve.
Tensor< type, 2 > roc_curve
Matrix containing the data of a ROC curve.
type area_under_curve
Area under a ROC curve.