OpenNN  2.2
Open Neural Networks Library
testing_analysis.cpp
1 /****************************************************************************************************************/
2 /* */
3 /* OpenNN: Open Neural Networks Library */
4 /* www.opennn.net */
5 /* */
6 /* T E S T I N G A N A L Y S I S C L A S S */
7 /* */
8 /* Roberto Lopez */
9 /* Artelnics - Making intelligent use of data */
10 /* robertolopez@artelnics.com */
11 /* */
12 /****************************************************************************************************************/
13 
14 // OpenNN includes
15 
16 #include "testing_analysis.h"
17 
18 namespace OpenNN
19 {
20 
21 // DEFAULT CONSTRUCTOR
22 
26 
28  : neural_network_pointer(NULL),
29  data_set_pointer(NULL),
30  mathematical_model_pointer(NULL)
31 {
32  set_default();
33 }
34 
35 
36 // NEURAL NETWORK CONSTRUCTOR
37 
42 
43 TestingAnalysis::TestingAnalysis(NeuralNetwork* new_neural_network_pointer)
44 : neural_network_pointer(new_neural_network_pointer),
45  data_set_pointer(NULL),
47 {
48  set_default();
49 }
50 
51 
52 // MATHEMATICAL MODEL CONSTRUCTOR
53 
58 
59 TestingAnalysis::TestingAnalysis(MathematicalModel* new_mathematical_model_pointer)
61  data_set_pointer(NULL),
62  mathematical_model_pointer(new_mathematical_model_pointer)
63 {
64  set_default();
65 }
66 
67 
68 // DATA SET CONSTRUCTOR
69 
74 
77  data_set_pointer(new_data_set_pointer),
79 {
80  set_default();
81 }
82 
83 
84 // NEURAL NETWORK AND MATHEMATICAL MODEL CONSTRUCTOR
85 
91 
92 TestingAnalysis::TestingAnalysis(NeuralNetwork* new_neural_network_pointer, MathematicalModel* new_mathematical_model_pointer)
93  : neural_network_pointer(new_neural_network_pointer),
94  data_set_pointer(NULL),
95  mathematical_model_pointer(new_mathematical_model_pointer)
96 {
97  set_default();
98 }
99 
100 
101 // NEURAL NETWORK AND DATA SET CONSTRUCTOR
102 
108 
109 TestingAnalysis::TestingAnalysis(NeuralNetwork* new_neural_network_pointer, DataSet* new_data_set_pointer)
110  : neural_network_pointer(new_neural_network_pointer),
111  data_set_pointer(new_data_set_pointer),
113 {
114  set_default();
115 }
116 
117 
118 // NEURAL NETWORK, MATHEMATICAL MODEL AND DATA SET CONSTRUCTOR
119 
126 
127 TestingAnalysis::TestingAnalysis(NeuralNetwork* new_neural_network_pointer, DataSet* new_data_set_pointer, MathematicalModel* new_mathematical_model_pointer)
128  : neural_network_pointer(new_neural_network_pointer),
129  data_set_pointer(new_data_set_pointer),
130  mathematical_model_pointer(new_mathematical_model_pointer)
131 {
132  set_default();
133 }
134 
135 
136 // XML CONSTRUCTOR
137 
142 
143 TestingAnalysis::TestingAnalysis(const tinyxml2::XMLDocument& testing_analysis_document)
144  : neural_network_pointer(NULL),
145  data_set_pointer(NULL),
147 {
148  set_default();
149 
150  from_XML(testing_analysis_document);
151 }
152 
153 
154 // FILE CONSTRUCTOR
155 
160 
161 TestingAnalysis::TestingAnalysis(const std::string& file_name)
162  : neural_network_pointer(NULL),
163  data_set_pointer(NULL),
165 {
166  set_default();
167 
168  load(file_name);
169 }
170 
171 // DESTRUCTOR
172 
175 
177 {
178 }
179 
180 
181 // METHODS
182 
183 // NeuralNetwork* get_neural_network_pointer(void) const method
184 
186 
188 {
189  #ifdef __OPENNN_DEBUG__
190 
192  {
193  std::ostringstream buffer;
194 
195  buffer << "OpenNN Exception: TestingAnalysis class.\n"
196  << "NeuralNetwork* get_neural_network_pointer(void) const method.\n"
197  << "Neural network pointer is NULL.\n";
198 
199  throw std::logic_error(buffer.str());
200  }
201 
202  #endif
203 
204  return(neural_network_pointer);
205 }
206 
207 
208 // DataSet* get_data_set_pointer(void) const method
209 
211 
213 {
214  #ifdef __OPENNN_DEBUG__
215 
216  if(!data_set_pointer)
217  {
218  std::ostringstream buffer;
219 
220  buffer << "OpenNN Exception: TestingAnalysis class.\n"
221  << "DataSet* get_data_set_pointer(void) const method.\n"
222  << "Data set pointer is NULL.\n";
223 
224  throw std::logic_error(buffer.str());
225  }
226 
227  #endif
228 
229  return(data_set_pointer);
230 }
231 
232 
233 // MathematicalModel* get_mathematical_model_pointer(void) const method
234 
236 
238 {
239  #ifdef __OPENNN_DEBUG__
240 
242  {
243  std::ostringstream buffer;
244 
245  buffer << "OpenNN Exception: TestingAnalysis class.\n"
246  << "MathematicalModel* get_mathematical_model_pointer(void) const method.\n"
247  << "Mathematical model pointer is NULL.\n";
248 
249  throw std::logic_error(buffer.str());
250  }
251 
252  #endif
253 
255 }
256 
257 
258 // const bool& get_display(void) const method
259 
262 
263 const bool& TestingAnalysis::get_display(void) const
264 {
265  return(display);
266 }
267 
268 
269 // void set_default(void) method
270 
275 
277 {
278  display = true;
279 }
280 
281 
282 // void set_neural_network_pointer(NeuralNetwork*) method
283 
286 
288 {
289  neural_network_pointer = new_neural_network_pointer;
290 }
291 
292 
293 // void set_mathematical_model_pointer(MathematicalModel*) method
294 
297 
299 {
300  mathematical_model_pointer = new_mathematical_model_pointer;
301 }
302 
303 
304 // void set_data_set_pointer(DataSet*) method
305 
308 
310 {
311  data_set_pointer = new_data_set_pointer;
312 }
313 
314 
315 // void set_display(const bool&) method
316 
321 
322 void TestingAnalysis::set_display(const bool& new_display)
323 {
324  display = new_display;
325 }
326 
327 
328 // void check(void) const method
329 
335 
336 void TestingAnalysis::check(void) const
337 {
338  std::ostringstream buffer;
339 
341  {
342  buffer << "OpenNN Exception: TestingAnalysis class.\n"
343  << "void check(void) const method.\n"
344  << "Neural network pointer is NULL.\n";
345 
346  throw std::logic_error(buffer.str());
347  }
348 
349  if(!data_set_pointer)
350  {
351  buffer << "OpenNN Exception: TestingAnalysis class.\n"
352  << "void check(void) const method.\n"
353  << "Data set pointer is NULL.\n";
354 
355  throw std::logic_error(buffer.str());
356  }
357 }
358 
359 
360 // Vector< Matrix<double> > calculate_target_output_data(void) const method
361 
364 
366 {
367  // Control sentence (if debug)
368 
369  #ifdef __OPENNN_DEBUG__
370 
371  check();
372 
373  #endif
374 
375  // Data set stuff
376 
377  const Instances& instances = data_set_pointer->get_instances();
378 
379  const size_t testing_instances_number = instances.count_testing_instances_number();
380 
381  const Matrix<double> testing_input_data = data_set_pointer->arrange_testing_input_data();
382 
384 
385  // Neural network stuff
386 
387  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
388 
389  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
390 
391  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(testing_input_data);
392 
393  // Approximation testing stuff
394 
395  Vector< Matrix<double> > target_output_data(outputs_number);
396 
397  for(size_t i = 0; i < outputs_number; i++)
398  {
399  target_output_data[i].set(testing_instances_number, 2);
400 
401  target_output_data[i].set_column(0, target_data.arrange_column(i));
402  target_output_data[i].set_column(1, output_data.arrange_column(i));
403  }
404 
405  return(target_output_data);
406 }
407 
408 
409 
410 // Vector< LinearRegressionParameters<double> > calculate_linear_regression_parameters(void) const method
411 
418 
420 {
421  // Control sentence (if debug)
422 
423  #ifdef __OPENNN_DEBUG__
424 
425  check();
426 
427  #endif
428 
429  // Data set stuff
430 
431  const Instances& instances = data_set_pointer->get_instances();
432 
433  const size_t testing_instances_number = instances.count_testing_instances_number();
434 
435  // Neural network stuff
436 
437  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
438 
439  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
440 
441  #ifdef __OPENNN_DEBUG__
442 
443  std::ostringstream buffer;
444 
445  if(testing_instances_number == 0)
446  {
447  buffer << "OpenNN Exception: TestingAnalysis class.\n"
448  << "Vector< LinearRegressionParameters<double> > calculate_linear_regression_parameters(void) const method.\n"
449  << "Number of testing instances is zero.\n";
450 
451  throw std::logic_error(buffer.str());
452  }
453 
454  #endif
455 
456  // Calculate regression parameters
457 
460  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
461 
462  Vector<double> target_variable(testing_instances_number);
463  Vector<double> output_variable(testing_instances_number);
464 
465  Vector< LinearRegressionParameters<double> > linear_regression_parameters(outputs_number);
466 
467  for(size_t i = 0; i < outputs_number; i++)
468  {
469  target_variable = target_data.arrange_column(i);
470  output_variable = output_data.arrange_column(i);
471 
472  linear_regression_parameters[i] = output_variable.calculate_linear_regression_parameters(target_variable);
473  }
474 
475  return(linear_regression_parameters);
476 }
477 
478 
479 // TestingAnalysis::LinearRegressionResults TestingAnalysis::perform_linear_regression_analysis(void) const
480 
487 
489 {
490  check();
491 
492  const Instances& instances = data_set_pointer->get_instances();
493 
494  const size_t testing_instances_number = instances.count_testing_instances_number();
495 
496  if(testing_instances_number == 0)
497  {
498  std::ostringstream buffer;
499 
500  buffer << "OpenNN Exception: TestingAnalysis class.\n"
501  << "LinearRegressionResults perform_linear_regression_analysis(void) const method.\n"
502  << "Number of testing instances is zero.\n";
503 
504  throw std::logic_error(buffer.str());
505  }
506 
507  LinearRegressionResults linear_regression_results;
508 
509  linear_regression_results.target_output_data = calculate_target_output_data();
510  linear_regression_results.linear_regression_parameters = calculate_linear_regression_parameters();
511 
512  return(linear_regression_results);
513 }
514 
515 
516 // void LinearRegressionResults::save(const std::string&) const method
517 
520 
521 void TestingAnalysis::LinearRegressionResults::save(const std::string& file_name) const
522 {
523  std::ofstream file(file_name.c_str());
524 
525  file << linear_regression_parameters
526  << "Target-output data:\n"
527  << target_output_data;
528 
529  file.close();
530 }
531 
532 
533 // Matrix<double> calculate_error_data(void) const method
534 
544 
546 {
547  // Data set stuff
548 
549  #ifdef __OPENNN_DEBUG__
550 
551  check();
552 
553  #endif
554 
555  const size_t testing_instances_number = data_set_pointer->get_instances().count_testing_instances_number();
556 
557  #ifdef __OPENNN_DEBUG__
558 
559  std::ostringstream buffer;
560 
561  if(testing_instances_number == 0)
562  {
563  buffer << "OpenNN Exception: TestingAnalysis class.\n"
564  << "Vector< Matrix<double> > calculate_error_data(void) const.\n"
565  << "Number of testing instances is zero.\n";
566 
567  throw std::logic_error(buffer.str());
568  }
569 
570  #endif
571 
572 
574 
576 
577  // Neural network stuff
578 
579  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
580 
581  const UnscalingLayer* unscaling_layer_pointer = neural_network_pointer->get_unscaling_layer_pointer();
582 
583  #ifdef __OPENNN_DEBUG__
584 
585  if(!unscaling_layer_pointer)
586  {
587  buffer << "OpenNN Exception: TestingAnalysis class.\n"
588  << "Vector< Matrix<double> > calculate_error_data(void) const.\n"
589  << "Unscaling layer is NULL.\n";
590 
591  throw std::logic_error(buffer.str());
592  }
593 
594  #endif
595 
596  const Vector<double>& outputs_minimum = unscaling_layer_pointer->arrange_minimums();
597  const Vector<double>& outputs_maximum = unscaling_layer_pointer->arrange_maximums();
598 
599  const size_t outputs_number = unscaling_layer_pointer->get_unscaling_neurons_number();
600 
601  // Error data
602 
603  Vector< Matrix<double> > error_data(outputs_number);
604 
605  Vector<double> targets(testing_instances_number);
606  Vector<double> outputs(testing_instances_number);
607 
608  Vector<double> difference_absolute_value(testing_instances_number);
609 
610  for(size_t i = 0; i < outputs_number; i++)
611  {
612  error_data[i].set(testing_instances_number, 3, 0.0);
613 
614  // Absolute error
615 
616  targets = target_data.arrange_column(i);
617  outputs = output_data.arrange_column(i);
618 
619  difference_absolute_value = (targets - outputs).calculate_absolute_value();
620 
621  error_data[i].set_column(0, difference_absolute_value);
622 
623  // Relative error
624 
625  error_data[i].set_column(1, difference_absolute_value/std::abs(outputs_maximum[i]-outputs_minimum[i]));
626 
627  // Percentage error
628 
629  error_data[i].set_column(2, difference_absolute_value*100.0/std::abs(outputs_maximum[i]-outputs_minimum[i]));
630  }
631 
632  return(error_data);
633 }
634 
635 
636 // Vector< Vector< Statistics<double> > > calculate_error_data_statistics(void) const method
637 
645 
647 {
648  // Neural network stuff
649 
650  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
651 
652  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
653 
654  // Testing analysis stuff
655 
656  Vector< Vector< Statistics<double> > > statistics(outputs_number);
657 
658  const Vector< Matrix<double> > error_data = calculate_error_data();
659 
660  for(size_t i = 0; i < outputs_number; i++)
661  {
662  statistics[i] = error_data[i].calculate_statistics();
663  }
664 
665  return(statistics);
666 }
667 
668 
669 // Vector< Matrix<double> > calculate_error_data_statistics_matrices(void) const method
670 
675 
677 {
678  const Vector< Vector< Statistics<double> > > error_data_statistics = calculate_error_data_statistics();
679 
680  const size_t outputs_number = error_data_statistics.size();
681 
682  Vector< Matrix<double> > statistics(outputs_number);
683 
684  for(size_t i = 0; i < outputs_number; i++)
685  {
686  statistics[i].set(3, 4);
687  statistics[i].set_row(0, error_data_statistics[i][0].to_vector());
688  statistics[i].set_row(1, error_data_statistics[i][1].to_vector());
689  statistics[i].set_row(2, error_data_statistics[i][2].to_vector());
690  }
691 
692  return(statistics);
693 }
694 
695 
696 // Vector< Histogram<double> > calculate_error_data_histograms(const size_t&) const method
697 
701 
703 {
704  const Vector< Matrix<double> > error_data = calculate_error_data();
705 
706  const size_t outputs_number = error_data.size();
707 
708  Vector< Histogram<double> > histograms(outputs_number);
709 
710  for(size_t i = 0; i < outputs_number; i++)
711  {
712  histograms[i] = error_data[i].arrange_column(0).calculate_histogram(bins_number);
713  }
714 
715  return(histograms);
716 }
717 
718 
719 // Vector< Vector<size_t> > calculate_maximal_errors(const size_t&) const method
720 
724 
726 {
727  const Vector< Matrix<double> > error_data = calculate_error_data();
728 
729  const size_t outputs_number = error_data.size();
730 
731  Vector< Vector<size_t> > maximal_errors(outputs_number);
732 
733  for(size_t i = 0; i < outputs_number; i++)
734  {
735  maximal_errors[i] = error_data[i].arrange_column(0).calculate_maximal_indices(instances_number);
736  }
737 
738  return(maximal_errors);
739 }
740 
741 
742 // Vector<double> calculate_testing_errors(void) const method
743 
752 
754 {
755  // Data set stuff
756 
757  #ifdef __OPENNN_DEBUG__
758 
759  check();
760 
761  #endif
762 
763  const size_t testing_instances_number = data_set_pointer->get_instances().count_testing_instances_number();
764 
765  #ifdef __OPENNN_DEBUG__
766 
767  std::ostringstream buffer;
768 
769  if(testing_instances_number == 0)
770  {
771  buffer << "OpenNN Exception: TestingAnalysis class.\n"
772  << "Vector< Matrix<double> > calculate_errors(void) const.\n"
773  << "Number of testing instances is zero.\n";
774 
775  throw std::logic_error(buffer.str());
776  }
777 
778  #endif
779 
781 
783 
784  // Neural network stuff
785 
786  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
787 
788  #ifdef __OPENNN_DEBUG__
789 
790 // if(!unscaling_layer_pointer)
791 // {
792 // buffer << "OpenNN Exception: TestingAnalysis class.\n"
793 // << "Vector< Matrix<double> > calculate_errors(void) const.\n"
794 // << "Unscaling layer is NULL.\n";
795 
796 // throw std::logic_error(buffer.str());
797 // }
798 
799  #endif
800 
801  Vector<double> errors(4,0.0);
802 
803  // Results
804 
805  errors[0] = output_data.calculate_sum_squared_error(target_data);
806  errors[1] = output_data.calculate_sum_squared_error(target_data)/testing_instances_number;
807  errors[2] = sqrt(errors[1]);
808  errors[3] = calculate_testing_normalized_squared_error(target_data, output_data);
809 
810  return errors;
811 }
812 
813 
814 // Vector<double> calculate_classification_testing_errors(void) const method
815 
825 
827 {
828  // Data set stuff
829 
830  #ifdef __OPENNN_DEBUG__
831 
832  check();
833 
834  #endif
835 
836  const size_t testing_instances_number = data_set_pointer->get_instances().count_testing_instances_number();
837 
838  #ifdef __OPENNN_DEBUG__
839 
840  std::ostringstream buffer;
841 
842  if(testing_instances_number == 0)
843  {
844  buffer << "OpenNN Exception: TestingAnalysis class.\n"
845  << "Vector< Matrix<double> > calculate_errors(void) const.\n"
846  << "Number of testing instances is zero.\n";
847 
848  throw std::logic_error(buffer.str());
849  }
850 
851  #endif
852 
854 
856 
857  // Neural network stuff
858 
859  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
860 
861  Vector<double> errors(5,0.0);
862 
863  // Results
864 
865  errors[0] = output_data.calculate_sum_squared_error(target_data);
866  errors[1] = output_data.calculate_sum_squared_error(target_data)/testing_instances_number;
867  errors[2] = sqrt(errors[1]);
868  errors[3] = calculate_testing_normalized_squared_error(target_data, output_data);
869  errors[4] = calculate_testing_cross_entropy_error(target_data, output_data);
870 
871  return errors;
872 }
873 
874 
875 // double calculate_testing_normalized_squared_error(const Matrix<double>&, const Matrix<double>&) const method
876 
880 
882 {
883  const size_t testing_instances_number = target_data.get_rows_number();
884 
885  const Vector<double> testing_target_data_mean = data_set_pointer->calculate_testing_target_data_mean();
886 
887  double normalization_coefficient = 0.0;
888  double sum_squared_error = 0.0;
889 
890  for(size_t i = 0; i < testing_instances_number; i++)
891  {
892  sum_squared_error += output_data.arrange_row(i).calculate_sum_squared_error(target_data.arrange_row(i));
893 
894  normalization_coefficient += target_data.arrange_row(i).calculate_sum_squared_error(testing_target_data_mean);
895  }
896 
897  return sum_squared_error/normalization_coefficient;
898 }
899 
900 
901 // double calculate_testing_cross_entropy_error(const Matrix<double>&, const Matrix<double>&) const method
902 
907 
909 {
910  const size_t testing_instances_number = target_data.get_rows_number();
911  const size_t outputs_number = target_data.get_columns_number();
912 
913  Vector<double> targets(outputs_number);
914  Vector<double> outputs(outputs_number);
915 
916  double cross_entropy_error = 0.0;
917 
918  for(size_t i = 0; i < testing_instances_number; i++)
919  {
920  outputs = output_data.arrange_row(i);
921  targets = target_data.arrange_row(i);
922 
923  for(size_t j = 0; j < outputs_number; j++)
924  {
925  if(outputs[j] == 0.0)
926  {
927  outputs[j] = 1.0e-6;
928  }
929  else if(outputs[j] == 1.0)
930  {
931  outputs[j] = 0.999999;
932  }
933 
934  cross_entropy_error -= targets[j]*log(outputs[j]) + (1.0 - targets[j])*log(1.0 - outputs[j]);
935  }
936  }
937 
938  return cross_entropy_error;
939 }
940 
941 
942 // double calculate_testing_weighted_squared_error(const Matrix<double>&, const Matrix<double>&) const method
943 
948 
950 {
951  const size_t testing_instances_number = target_data.get_rows_number();
952 
953  #ifdef __OPENNN_DEBUG__
954 
955 // std::ostringstream buffer;
956 
957 // if(outputs_number != 1)
958 // {
959 // buffer << "OpenNN Exception: TestingAnalysis class.\n"
960 // << "double calculate_testing_weighted_squared_error(const Matrix<double>&, const Matrix<double>&) const.\n"
961 // << "Number of outputs must be one.\n";
962 
963 // throw std::logic_error(buffer.str());
964 // }
965 
966  #endif
967 
968  const Vector<size_t> target_distribution = data_set_pointer->calculate_target_distribution();
969 
970  const size_t negatives_number = target_distribution[0];
971  const size_t positives_number = target_distribution[1];
972 
973  const double negatives_weight = 1.0;
974  const double positives_weight = (double)negatives_number/positives_number;
975 
976  double error = 0.0;
977  double sum_squared_error = 0.0;
978 
979  for(size_t i = 0; i < testing_instances_number; i++)
980  {
981  if(target_data(0,i) == 1.0)
982  {
983  error = positives_weight*output_data.arrange_column(i).calculate_sum_squared_error(target_data.arrange_column(i));
984  }
985  else if(target_data(0,i) == 0.0)
986  {
987  error = negatives_weight*output_data.arrange_column(i).calculate_sum_squared_error(target_data.arrange_column(i));
988  }
989  else
990  {
991  std::ostringstream buffer;
992 
993  buffer << "OpenNN Exception: TestingAnalysis class.\n"
994  << "double calculate_testing_weighted_squared_error(const Matrix<double>&, const Matrix<double>&) const method.\n"
995  << "Target is neither a positive nor a negative.\n";
996 
997  throw std::logic_error(buffer.str());
998  }
999 
1000  sum_squared_error += error;
1001  }
1002 
1004 
1005  const size_t negatives = data_set_pointer->calculate_training_negatives(targets_indices[0]);
1006 
1007  const double normalization_coefficient = negatives*negatives_weight*0.5;
1008 
1009  return(sum_squared_error/normalization_coefficient);
1010 }
1011 
1012 
1013 // Matrix<size_t> calculate_confusion_binary_classification(const Matrix<double>&, const Matrix<double>&, const double&) const method
1014 
1019 
1020 Matrix<size_t> TestingAnalysis::calculate_confusion_binary_classification(const Matrix<double>& target_data, const Matrix<double>& output_data, const double& decision_threshold) const
1021 {
1022  const size_t rows_number = target_data.get_rows_number();
1023 
1024  Matrix<size_t> confusion(2, 2);
1025 
1026  size_t true_positive = 0;
1027  size_t false_negative = 0;
1028  size_t false_positive = 0;
1029  size_t true_negative = 0;
1030 
1031  for(size_t i = 0; i < rows_number; i++)
1032  {
1033 
1034  if(decision_threshold == 0.0 && target_data(i,0) == 0.0 )
1035  {
1036  false_positive++;
1037 
1038  }
1039  else if (decision_threshold == 0.0 && target_data(i,0) == 1.0)
1040  {
1041  true_positive++;
1042 
1043  }
1044  else if(target_data(i,0) >= decision_threshold && output_data(i,0) >= decision_threshold)
1045  {
1046  // True positive
1047 
1048  true_positive++;
1049 
1050  }
1051  else if(target_data(i,0) >= decision_threshold && output_data(i,0) < decision_threshold)
1052  {
1053  // False negative
1054 
1055  false_negative++;
1056 
1057  }
1058  else if(target_data(i,0) < decision_threshold && output_data(i,0) >= decision_threshold)
1059  {
1060  // False positive
1061 
1062  false_positive++;
1063 
1064  }
1065  else if(target_data(i,0) < decision_threshold && output_data(i,0) < decision_threshold)
1066  {
1067  // True negative
1068 
1069  true_negative++;
1070  }
1071  }
1072 
1073  confusion(0,0) = true_positive;
1074  confusion(0,1) = false_negative;
1075  confusion(1,0) = false_positive;
1076  confusion(1,1) = true_negative;
1077 
1078  if(confusion.calculate_sum() != rows_number)
1079  {
1080  std::ostringstream buffer;
1081 
1082  buffer << "OpenNN Exception: TestingAnalysis class.\n"
1083  << "Matrix<size_t> calculate_confusion_binary_classification(const Matrix<double>&, const Matrix<double>&, const double&) const method.\n"
1084  << "Number of elements in confusion matrix must be equal to number of testing instances.\n";
1085 
1086  throw std::logic_error(buffer.str());
1087  }
1088 
1089  return(confusion);
1090 }
1091 
1092 
1093 // Matrix<size_t> calculate_confusion_multiple_classification(const Matrix<double>&, const Matrix<double>&) const method
1094 
1098 
1100 {
1101  const size_t rows_number = target_data.get_rows_number();
1102  const size_t columns_number = target_data.get_columns_number();
1103 
1104  Matrix<size_t> confusion(columns_number, columns_number, 0);
1105 
1106  size_t target_index = 0;
1107  size_t output_index = 0;
1108 
1109  for(size_t i = 0; i < rows_number; i++)
1110  {
1111  target_index = target_data.arrange_row(i).calculate_maximal_index();
1112  output_index = output_data.arrange_row(i).calculate_maximal_index();
1113 
1114  confusion(target_index,output_index)++;
1115  }
1116 
1117  return(confusion);
1118 }
1119 
1120 
1121 // Vector<size_t> calculate_positives_negatives_rate(const Matrix<double>&, const Matrix<double>&) const
1122 
1130 
1132 {
1133  const Matrix<size_t> confusion = calculate_confusion_binary_classification(target_data, output_data, 0.5);
1134  Vector<size_t> positives_negatives_rate(2);
1135 
1136  positives_negatives_rate[0] = confusion(0,0) + confusion(0,1);
1137  positives_negatives_rate[1] = confusion(1,0) + confusion(1,1);
1138 
1139  return(positives_negatives_rate);
1140 }
1141 
1142 
1143 // Matrix<size_t> calculate_confusion(void) const method
1144 
1148 
1150 {
1151  #ifdef __OPENNN_DEBUG__
1152 
1153  check();
1154 
1155  #endif
1156 
1157  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
1158 
1159  #ifdef __OPENNN_DEBUG__
1160 
1161  if(!multilayer_perceptron_pointer)
1162  {
1163  std::ostringstream buffer;
1164 
1165  buffer << "OpenNN Exception: TestingAnalysis class.\n"
1166  << "Matrix<size_t> calculate_confusion(void) const method.\n"
1167  << "Pointer to multilayer perceptron in neural network is NULL.\n";
1168 
1169  throw std::logic_error(buffer.str());
1170  }
1171 
1172  #endif
1173 
1174  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
1175 
1176  #ifdef __OPENNN_DEBUG__
1177 
1178  const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
1179 
1180  // Control sentence
1181 
1182  const Variables& variables = data_set_pointer->get_variables();
1183 
1184  if(inputs_number != variables.count_inputs_number())
1185  {
1186  std::ostringstream buffer;
1187 
1188  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
1189  << "Matrix<size_t> calculate_confusion(void) const method." << std::endl
1190  << "Number of inputs in neural network must be equal to number of inputs in data set." << std::endl;
1191 
1192  throw std::logic_error(buffer.str());
1193  }
1194 
1195  if(outputs_number != variables.count_targets_number())
1196  {
1197  std::ostringstream buffer;
1198 
1199  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
1200  << "Matrix<size_t> calculate_confusion(void) const method." << std::endl
1201  << "Number of outputs in neural network must be equal to number of targets in data set." << std::endl;
1202 
1203  throw std::logic_error(buffer.str());
1204  }
1205 
1206  #endif
1207 
1210 
1211  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
1212 
1213 // output_data.save("../data/output_data.dat");
1214 
1215  if(outputs_number == 1)
1216  {
1217  double decision_threshold;
1218 
1220  {
1222  }
1223  else
1224  {
1225  decision_threshold = 0.5;
1226  }
1227 
1228  return(calculate_confusion_binary_classification(target_data, output_data, decision_threshold));
1229  }
1230  else
1231  {
1232  return(calculate_confusion_multiple_classification(target_data, output_data));
1233  }
1234 }
1235 
1236 
1237 //TestingAnalysis::RocCurveResults perform_roc_analysis (void) const
1238 
1246 
1248 {
1249  #ifdef __OPENNN_DEBUG__
1250 
1251  check();
1252 
1253  #endif
1254 
1255  #ifdef __OPENNN_DEBUG__
1256 
1257  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
1258 
1259  if(!multilayer_perceptron_pointer)
1260  {
1261  std::ostringstream buffer;
1262 
1263  buffer << "OpenNN Exception: TestingAnalysis class.\n"
1264  << "RocCurveResults perform_roc_analysis(void) const method.\n"
1265  << "Pointer to multilayer perceptron in neural network is NULL.\n";
1266 
1267  throw std::logic_error(buffer.str());
1268  }
1269 
1270  #endif
1271 
1272  #ifdef __OPENNN_DEBUG__
1273 
1274  const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
1275 
1276  // Control sentence
1277 
1278  const Variables& variables = data_set_pointer->get_variables();
1279 
1280  if(inputs_number != variables.count_inputs_number())
1281  {
1282  std::ostringstream buffer;
1283 
1284  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
1285  << "RocCurveResults perform_roc_analysis(void) const method." << std::endl
1286  << "Number of inputs in neural network must be equal to number of inputs in data set." << std::endl;
1287 
1288  throw std::logic_error(buffer.str());
1289  }
1290 
1291  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
1292 
1293  if(outputs_number != variables.count_targets_number())
1294  {
1295  std::ostringstream buffer;
1296 
1297  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
1298  << "RocCurveResults perform_roc_analysis(void) const method." << std::endl
1299  << "Number of outputs in neural network must be equal to number of targets in data set." << std::endl;
1300 
1301  throw std::logic_error(buffer.str());
1302  }
1303 
1304  #endif
1305 
1308 
1309  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
1310 
1311  RocAnalysisResults roc_analysis_results;
1312 
1313  roc_analysis_results.roc_curve = calculate_roc_curve(target_data, output_data);
1314  roc_analysis_results.area_under_curve = calculate_area_under_curve(target_data, output_data);
1315  roc_analysis_results.optimal_threshold = calculate_optimal_threshold(target_data, output_data, roc_analysis_results.roc_curve);
1316 
1317  return(roc_analysis_results);
1318 }
1319 
1320 
1321 //double calculate_Wilcoxon_parameter(const double& ,const double& ) const
1322 
1327 
1328 double TestingAnalysis::calculate_Wilcoxon_parameter (const double& x, const double& y) const
1329 {
1330  if(x > y)
1331  {
1332  return (1);
1333  }
1334  else if(x < y)
1335  {
1336  return (0);
1337  }
1338  else
1339  {
1340  return (0.5);
1341  }
1342 }
1343 
1344 
1345 // Matrix<double> calculate_roc_curve (const Matrix<double>& , const Matrix<double>& output_data) const
1346 
1353 
1355 {
1356  const Vector<size_t> positives_negatives_rate = calculate_positives_negatives_rate(target_data, output_data);
1357 
1358  const size_t total_positives = positives_negatives_rate[0];
1359  const size_t total_negatives = positives_negatives_rate[1];
1360 
1361  if(total_positives == 0)
1362  {
1363  std::ostringstream buffer;
1364 
1365  buffer << "OpenNN Exception: TestingAnalysis class.\n"
1366  << "Matrix<double> calculate_roc_curve(const Matrix<double>&, const Matrix<double>&) const.\n"
1367  << "Number of positive instances ("<< total_positives <<") must be greater than zero.\n";
1368 
1369  throw std::logic_error(buffer.str());
1370  }
1371 
1372  if(total_negatives == 0)
1373  {
1374  std::ostringstream buffer;
1375 
1376  buffer << "OpenNN Exception: TestingAnalysis class.\n"
1377  << "Matrix<double> calculate_roc_curve(const Matrix<double>&, const Matrix<double>&) const.\n"
1378  << "Number of negative instances ("<< total_negatives <<") must be greater than zero.\n";
1379 
1380  throw std::logic_error(buffer.str());
1381  }
1382 
1383  const size_t maximum_points_number = 1000;
1384 
1385  size_t step_size;
1386 
1387  const size_t testing_instances_number = target_data.get_rows_number();
1388  size_t points_number;
1389 
1390  if(testing_instances_number > maximum_points_number)
1391  {
1392  step_size = (size_t)((double)testing_instances_number/(double)maximum_points_number);
1393  points_number = (size_t)((double)testing_instances_number/(double)step_size);
1394  }
1395  else
1396  {
1397  points_number = testing_instances_number;
1398  step_size = 1;
1399  }
1400 
1401  Matrix<double> target_output_data = output_data.assemble_columns(target_data);
1402 
1403  Matrix<double> sorted_target_output_data = target_output_data.sort_less_rows(0);
1404 
1405  const Vector<size_t> columns_output_indices(1,0);
1406  const Vector<size_t> columns_target_indices(1,1);
1407 
1408  const Matrix<double> sorted_target_data = sorted_target_output_data.arrange_submatrix_columns(columns_target_indices);
1409  const Matrix<double> sorted_output_data = sorted_target_output_data.arrange_submatrix_columns(columns_output_indices);
1410 
1411  Matrix<double> roc_curve(points_number+1, 3, 0.0);
1412 
1413  double threshold = 0;
1414 
1415  size_t positives;
1416  size_t negatives;
1417 
1418  size_t current_index;
1419 
1420  int j = 0;
1421  int i = 0;
1422 
1423  const size_t step_s = step_size;
1424 
1425 #pragma omp parallel for private(i, j, positives, negatives, threshold, current_index) schedule(dynamic)
1426 
1427  for(i = 0; i < (int)points_number; i++)
1428  {
1429  positives = 0;
1430  negatives = 0;
1431 
1432  current_index = i*step_s;
1433 
1434  threshold = sorted_output_data(current_index, 0);
1435 
1436  for(j = 0; j < (int)current_index; j++)
1437  {
1438  if(sorted_output_data(j,0) < threshold && sorted_target_data(j,0) == 1.0)
1439  {
1440  positives++;
1441  }
1442  if(sorted_output_data(j,0) < threshold && sorted_target_data(j,0) == 0.0)
1443  {
1444  negatives++;
1445  }
1446  }
1447 
1448  roc_curve(i,0) = (double)positives/(double)(total_positives);
1449  roc_curve(i,1) = (double)negatives/(double)(total_negatives);
1450  roc_curve(i,2) = (double)threshold;
1451  }
1452 
1453  roc_curve(points_number, 0) = 1.0;
1454  roc_curve(points_number, 1) = 1.0;
1455  roc_curve(points_number, 2) = 1.0;
1456 
1457  return (roc_curve);
1458 }
1459 
1460 
1461 // double calculate_area_under_curve(const Matrix<double>& , const Matrix<double>& ) const
1462 
1466 
1467 double TestingAnalysis::calculate_area_under_curve (const Matrix<double>& target_data, const Matrix<double>& output_data) const
1468 {
1469  const Vector<size_t> positives_negatives_rate = calculate_positives_negatives_rate(target_data, output_data);
1470 
1471  const size_t total_positives = positives_negatives_rate[0];
1472  const size_t total_negatives = positives_negatives_rate[1];
1473 
1474  if(total_positives == 0)
1475  {
1476  std::ostringstream buffer;
1477 
1478  buffer << "OpenNN Exception: TestingAnalysis class.\n"
1479  << "Matrix<double> calculate_roc_curve(const Matrix<double>&, const Matrix<double>&) const.\n"
1480  << "Number of positive instances ("<< total_positives <<") must be greater than zero.\n";
1481 
1482  throw std::logic_error(buffer.str());
1483  }
1484 
1485  if(total_negatives == 0)
1486  {
1487  std::ostringstream buffer;
1488 
1489  buffer << "OpenNN Exception: TestingAnalysis class.\n"
1490  << "Matrix<double> calculate_roc_curve(const Matrix<double>&, const Matrix<double>&) const.\n"
1491  << "Number of negative instances ("<< total_negatives <<") must be greater than zero.\n";
1492 
1493  throw std::logic_error(buffer.str());
1494  }
1495 
1496  size_t testing_instances_number = target_data.get_rows_number();
1497 
1498  double sum = 0.0;
1499 
1500  double area_under_curve;
1501 
1502  int i,j;
1503 
1504 #pragma omp parallel for private(i, j) reduction(+ : sum) schedule(dynamic)
1505 
1506  for(i = 0; i < testing_instances_number; i++)
1507  {
1508  if(target_data(i,0) == 1)
1509  {
1510  for(j = 0; j < testing_instances_number; j++)
1511  {
1512  if(target_data(j,0) == 0)
1513  {
1514  sum += calculate_Wilcoxon_parameter(output_data(i,0),output_data(j,0));
1515  }
1516  }
1517  }
1518  }
1519 
1520  area_under_curve = (double)sum/(double)(total_positives*total_negatives);
1521 
1522  return (area_under_curve);
1523 }
1524 
1525 
1526 // double calculate_optimal_threshold (const Matrix<double>& , const Matrix<double>& ) const
1527 
1531 
1532 double TestingAnalysis::calculate_optimal_threshold (const Matrix<double>& target_data, const Matrix<double>& output_data ) const
1533 {
1534  const size_t rows_number = target_data.get_rows_number();
1535  const size_t columns_number = target_data.get_columns_number();
1536 
1537  const size_t maximum_points_number = 1000;
1538 
1539  size_t step_size;
1540  size_t points_number;
1541 
1542  if(rows_number > maximum_points_number)
1543  {
1544  step_size = rows_number/maximum_points_number;
1545  points_number = rows_number/step_size;
1546  }
1547  else
1548  {
1549  points_number = rows_number;
1550  step_size = 1;
1551  }
1552 
1553  Matrix<double> target_output_data = output_data.assemble_columns(target_data);
1554 
1555  Matrix<double> sorted_target_output_data = target_output_data.sort_less_rows(0);
1556 
1557  const Vector<size_t> columns_output_indices(0, 1, columns_number - 1);
1558  const Vector<size_t> columns_target_indices(columns_number, 1, 2*columns_number - 1);
1559 
1560  const Matrix<double> sorted_target_data = sorted_target_output_data.arrange_submatrix_columns(columns_target_indices);
1561  const Matrix<double> sorted_output_data = sorted_target_output_data.arrange_submatrix_columns(columns_output_indices);
1562 
1563  const Matrix<double> roc_curve = calculate_roc_curve(sorted_target_data, sorted_output_data);
1564 
1565  double threshold = 0.0;
1566  double optimal_threshold = 0.5;
1567 
1568  double minimun_distance = std::numeric_limits<double>::max();
1569  double distance;
1570 
1571  size_t current_index;
1572 
1573  for(size_t i = 0; i < points_number; i++)
1574  {
1575  current_index = i*step_size;
1576 
1577  threshold = sorted_output_data(current_index, 0);
1578 
1579  distance = sqrt(roc_curve(i,0)*roc_curve(i,0) + (roc_curve(i,1) - 1.0)*(roc_curve(i,1) - 1.0));
1580 
1581  if(distance < minimun_distance)
1582  {
1583  optimal_threshold = threshold;
1584 
1585  minimun_distance = distance;
1586  }
1587  }
1588 
1589  return (optimal_threshold);
1590 }
1591 
1592 // double calculate_optimal_threshold (const Matrix<double>& , const Matrix<double>&, const Matrix<double>&) const
1593 
1598 
1599 double TestingAnalysis::calculate_optimal_threshold (const Matrix<double>& target_data, const Matrix<double>& output_data, const Matrix<double>& roc_curve) const
1600 {
1601  const size_t rows_number = target_data.get_rows_number();
1602  const size_t columns_number = target_data.get_columns_number();
1603 
1604  size_t step_size;
1605  const size_t points_number = roc_curve.get_rows_number();
1606 
1607  if(rows_number > points_number)
1608  {
1609  step_size = rows_number/points_number;
1610  }
1611  else
1612  {
1613  step_size = 1;
1614  }
1615 
1616  Matrix<double> target_output_data = output_data.assemble_columns(target_data);
1617 
1618  Matrix<double> sorted_target_output_data = target_output_data.sort_less_rows(0);
1619 
1620  const Vector<size_t> columns_output_indices(0, 1, columns_number - 1);
1621 
1622  const Matrix<double> sorted_output_data = sorted_target_output_data.arrange_submatrix_columns(columns_output_indices);
1623 
1624  double threshold = 0.0;
1625  double optimal_threshold = 0.5;
1626 
1627  double minimun_distance = std::numeric_limits<double>::max();
1628  double distance;
1629 
1630  size_t current_index;
1631 
1632  for(size_t i = 0; i < points_number; i++)
1633  {
1634  current_index = i*step_size;
1635 
1636  threshold = sorted_output_data(current_index, 0);
1637 
1638  distance = sqrt(roc_curve(i,0)*roc_curve(i,0) + (roc_curve(i,1) - 1.0)*(roc_curve(i,1) - 1.0));
1639 
1640  if(distance < minimun_distance)
1641  {
1642  optimal_threshold = threshold;
1643 
1644  minimun_distance = distance;
1645  }
1646  }
1647 
1648  return (optimal_threshold);
1649 }
1650 
1651 // Matrix<double> perform_cumulative_gain_analysis(void) const
1652 
1655 
1657 {
1658  #ifdef __OPENNN_DEBUG__
1659 
1660  check();
1661 
1662  #endif
1663 
1664  #ifdef __OPENNN_DEBUG__
1665 
1666  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
1667 
1668  if(!multilayer_perceptron_pointer)
1669  {
1670  std::ostringstream buffer;
1671 
1672  buffer << "OpenNN Exception: TestingAnalysis class.\n"
1673  << "Matrix<double> perform_cumulative_gain_analysis(void) const method.\n"
1674  << "Pointer to multilayer perceptron in neural network is NULL.\n";
1675 
1676  throw std::logic_error(buffer.str());
1677  }
1678 
1679  #endif
1680 
1681  #ifdef __OPENNN_DEBUG__
1682 
1683  const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
1684 
1685  // Control sentence
1686 
1687  const Variables& variables = data_set_pointer->get_variables();
1688 
1689  if(inputs_number != variables.count_inputs_number())
1690  {
1691  std::ostringstream buffer;
1692 
1693  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
1694  << "Matrix<double> perform_cumulative_gain_analysis(void) const method." << std::endl
1695  << "Number of inputs in neural network must be equal to number of inputs in data set." << std::endl;
1696 
1697  throw std::logic_error(buffer.str());
1698  }
1699 
1700  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
1701 
1702  if(outputs_number != variables.count_targets_number())
1703  {
1704  std::ostringstream buffer;
1705 
1706  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
1707  << "Matrix<double> perform_cumulative_gain_analysis(void) const method." << std::endl
1708  << "Number of outputs in neural network must be equal to number of targets in data set." << std::endl;
1709 
1710  throw std::logic_error(buffer.str());
1711  }
1712 
1713  #endif
1714 
1717 
1718  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
1719 
1720  const Matrix<double> cumulative_gain = calculate_cumulative_gain(target_data, output_data);
1721 
1722  return(cumulative_gain);
1723 }
1724 
1725 
1726 // Matrix<double> calculate_cumulative_gain(const Matrix<double>& , const Matrix<double>&) const
1727 
1732 
1734 {
1735  const size_t total_positives = calculate_positives_negatives_rate(target_data, output_data)[0];
1736 
1737  if(total_positives == 0)
1738  {
1739  std::ostringstream buffer;
1740 
1741  buffer << "OpenNN Exception: TestingAnalysis class.\n"
1742  << "Matrix<double> calculate_cumulative_gain(const Matrix<double>&, const Matrix<double>&) const.\n"
1743  << "Number of positive instances (" << total_positives << ") must be greater than zero.\n";
1744 
1745  throw std::logic_error(buffer.str());
1746  }
1747 
1748  const size_t rows_number = target_data.get_rows_number();
1749 
1750  const Matrix<double> target_output_data = output_data.assemble_columns(target_data);
1751 
1752  const Matrix<double> sorted_target_output_data = target_output_data.sort_greater_rows(0);
1753 
1754  const Vector<size_t> target_indices(1,1);
1755 
1756  const Matrix<double> sorted_target_data = sorted_target_output_data.arrange_submatrix_columns(target_indices);
1757 
1758  const size_t points_number = 21;
1759  const double percentage_increment = 0.05;
1760 
1761  Matrix<double> cumulative_gain(points_number, 2);
1762 
1763  cumulative_gain(0,0) = 0.0;
1764  cumulative_gain(0,1) = 0.0;
1765 
1766  size_t positives = 0;
1767  size_t negatives = 0;
1768 
1769  double percentage = 0.0;
1770 
1771  size_t maximum_index;
1772 
1773  for(size_t i = 0; i < points_number - 1; i++)
1774  {
1775  percentage += percentage_increment;
1776  positives = 0;
1777  negatives = 0;
1778  maximum_index = (size_t)(percentage*rows_number);
1779 
1780  for(size_t j = 0; j < maximum_index; j++)
1781  {
1782  if(sorted_target_data(j, 0) == 1.0)
1783  {
1784  positives++;
1785  }
1786  }
1787 
1788  cumulative_gain(i + 1, 0) = (double) percentage;
1789  cumulative_gain(i + 1, 1) = (double) positives/(double)(total_positives);
1790  }
1791 
1792  return(cumulative_gain);
1793 }
1794 
1795 
1796 // Matrix<double> calculate_cumulative_gain(const Matrix<double>& , const Matrix<double>&) const
1797 
1802 
1804 {
1805  const size_t total_negatives = calculate_positives_negatives_rate(target_data, output_data)[1];
1806 
1807  if(total_negatives == 0)
1808  {
1809  std::ostringstream buffer;
1810 
1811  buffer << "OpenNN Exception: TestingAnalysis class.\n"
1812  << "Matrix<double> calculate_negative_cumulative_gain(const Matrix<double>&, const Matrix<double>&) const.\n"
1813  << "Number of negative instances (" << total_negatives << ") must be greater than zero.\n";
1814 
1815  throw std::logic_error(buffer.str());
1816  }
1817 
1818  const size_t rows_number = target_data.get_rows_number();
1819 
1820  const Matrix<double> target_output_data = output_data.assemble_columns(target_data);
1821 
1822  const Matrix<double> sorted_target_output_data = target_output_data.sort_greater_rows(0);
1823 
1824  const Vector<size_t> target_indices(1,1);
1825 
1826  const Matrix<double> sorted_target_data = sorted_target_output_data.arrange_submatrix_columns(target_indices);
1827 
1828  const size_t points_number = 21;
1829  const double percentage_increment = 0.05;
1830 
1831  Matrix<double> negative_cumulative_gain(points_number, 2);
1832 
1833  negative_cumulative_gain(0,0) = 0.0;
1834  negative_cumulative_gain(0,1) = 0.0;
1835 
1836  size_t negatives = 0;
1837 
1838  double percentage = 0.0;
1839 
1840  size_t maximum_index;
1841 
1842  for(size_t i = 0; i < points_number - 1; i++)
1843  {
1844  percentage += percentage_increment;
1845  negatives = 0;
1846  maximum_index = (size_t)(percentage*rows_number);
1847 
1848  for(size_t j = 0; j < maximum_index; j++)
1849  {
1850  if(sorted_target_data(j, 0) == 0.0)
1851  {
1852  negatives++;
1853  }
1854  }
1855 
1856  negative_cumulative_gain(i + 1, 0) = (double) percentage;
1857  negative_cumulative_gain(i + 1, 1) = (double) negatives/(double)(total_negatives);
1858  }
1859 
1860  return(negative_cumulative_gain);
1861 }
1862 
1863 
1864 // Matrix<double> perform_lift_chart_analysis(void) const
1865 
1868 
1870 {
1871  #ifdef __OPENNN_DEBUG__
1872 
1873  check();
1874 
1875  #endif
1876 
1877  #ifdef __OPENNN_DEBUG__
1878 
1879  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
1880 
1881  if(!multilayer_perceptron_pointer)
1882  {
1883  std::ostringstream buffer;
1884 
1885  buffer << "OpenNN Exception: TestingAnalysis class.\n"
1886  << "Matrix<double> perform_lift_chart_analysis(void) const method.\n"
1887  << "Pointer to multilayer perceptron in neural network is NULL.\n";
1888 
1889  throw std::logic_error(buffer.str());
1890  }
1891 
1892  #endif
1893 
1894  #ifdef __OPENNN_DEBUG__
1895 
1896  const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
1897 
1898  // Control sentence
1899 
1900  const Variables& variables = data_set_pointer->get_variables();
1901 
1902  if(inputs_number != variables.count_inputs_number())
1903  {
1904  std::ostringstream buffer;
1905 
1906  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
1907  << "Matrix<double> perform_lift_chart_analysis(void) const method." << std::endl
1908  << "Number of inputs in neural network must be equal to number of inputs in data set." << std::endl;
1909 
1910  throw std::logic_error(buffer.str());
1911  }
1912 
1913  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
1914 
1915  if(outputs_number != variables.count_targets_number())
1916  {
1917  std::ostringstream buffer;
1918 
1919  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
1920  << "Matrix<double> perform_lift_chart_analysis(void) const method." << std::endl
1921  << "Number of outputs in neural network must be equal to number of targets in data set." << std::endl;
1922 
1923  throw std::logic_error(buffer.str());
1924  }
1925 
1926  #endif
1927 
1930 
1931  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
1932 
1933  const Matrix<double> cumulative_gain = calculate_cumulative_gain(target_data, output_data);
1934  const Matrix<double> lift_chart = calculate_lift_chart(cumulative_gain);
1935 
1936  return(lift_chart);
1937 }
1938 
1939 
1940 // Matrix<double> calculate_lift_chart(const Matrix<double>& , const Matrix<double>&) const
1941 
1945 
1947 {
1948  const size_t rows_number = cumulative_gain.get_rows_number();
1949  const size_t columns_number = cumulative_gain.get_columns_number();
1950 
1951  Matrix<double> lift_chart(rows_number, columns_number);
1952 
1953  lift_chart(0,0) = 0.0;
1954  lift_chart(0,1) = 1.0;
1955 
1956 #pragma omp parallel for
1957 
1958  for(int i = 1; i < rows_number; i++)
1959  {
1960  lift_chart(i, 0) = cumulative_gain(i, 0);
1961  lift_chart(i, 1) = (double) cumulative_gain(i, 1)/(double)cumulative_gain(i, 0);
1962  }
1963 
1964  return(lift_chart);
1965 }
1966 
1967 
1968 // Matrix<double> perform_Kolmogorov_Smirnov_analysis(void) const
1969 
1977 
1979 {
1980  #ifdef __OPENNN_DEBUG__
1981 
1982  check();
1983 
1984  #endif
1985 
1986  #ifdef __OPENNN_DEBUG__
1987 
1988  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
1989 
1990  if(!multilayer_perceptron_pointer)
1991  {
1992  std::ostringstream buffer;
1993 
1994  buffer << "OpenNN Exception: TestingAnalysis class.\n"
1995  << "Matrix<double> perform_Kolmogorov_Smirnov_analysis(void) const method.\n"
1996  << "Pointer to multilayer perceptron in neural network is NULL.\n";
1997 
1998  throw std::logic_error(buffer.str());
1999  }
2000 
2001  #endif
2002 
2003  #ifdef __OPENNN_DEBUG__
2004 
2005  const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
2006 
2007  // Control sentence
2008 
2009  const Variables& variables = data_set_pointer->get_variables();
2010 
2011  if(inputs_number != variables.count_inputs_number())
2012  {
2013  std::ostringstream buffer;
2014 
2015  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2016  << "Matrix<double> perform_Kolmogorov_Smirnov_analysis(void) const method." << std::endl
2017  << "Number of inputs in neural network must be equal to number of inputs in data set." << std::endl;
2018 
2019  throw std::logic_error(buffer.str());
2020  }
2021 
2022  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
2023 
2024  if(outputs_number != variables.count_targets_number())
2025  {
2026  std::ostringstream buffer;
2027 
2028  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2029  << "Matrix<double> perform_Kolmogorov_Smirnov_analysis(void) const method." << std::endl
2030  << "Number of outputs in neural network must be equal to number of targets in data set." << std::endl;
2031 
2032  throw std::logic_error(buffer.str());
2033  }
2034 
2035  #endif
2036 
2039 
2040  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
2041 
2042  TestingAnalysis::KolmogorovSmirnovResults Kolmogorov_Smirnov_results;
2043 
2044  Kolmogorov_Smirnov_results.positive_cumulative_gain = calculate_cumulative_gain(target_data, output_data);
2045  Kolmogorov_Smirnov_results.negative_cumulative_gain = calculate_negative_cumulative_gain(target_data, output_data);
2046  Kolmogorov_Smirnov_results.maximum_gain =
2047  calculate_maximum_gain(Kolmogorov_Smirnov_results.positive_cumulative_gain,Kolmogorov_Smirnov_results.negative_cumulative_gain);
2048 
2049  return(Kolmogorov_Smirnov_results);
2050 }
2051 
2052 
2053 // double calculate_Komogorov_Smirnov_score(const Matrix<double>&, const Matrix<double>&) const
2054 
2059 
2060 Vector<double> TestingAnalysis::calculate_maximum_gain(const Matrix<double>& positive_cumulative_gain, const Matrix<double>& negative_cumulative_gain) const
2061 {
2062  const size_t points_number = positive_cumulative_gain.get_rows_number();
2063 
2064  #ifdef __OPENNN_DEBUG__
2065 
2066  if(points_number != negative_cumulative_gain.get_rows_number())
2067  {
2068  std::ostringstream buffer;
2069 
2070  buffer << "OpenNN Exception: TestingAnalysis class.\n"
2071  << "Matrix<double> calculate_maximum_gain(void) const method.\n"
2072  << "Positive and negative cumulative gain matrix must have the same rows number.\n";
2073 
2074  throw std::logic_error(buffer.str());
2075  }
2076 
2077  #endif
2078 
2079  Vector<double> maximum_gain(2, 0.0);
2080 
2081  const double percentage_increment = 0.05;
2082 
2083  double percentage = 0.0;
2084 
2085  for(size_t i = 0; i < points_number - 1; i++)
2086  {
2087  percentage += percentage_increment;
2088 
2089  if(positive_cumulative_gain(i+1,1)-negative_cumulative_gain(i+1,1) > maximum_gain[1]
2090  && positive_cumulative_gain(i+1,1)-negative_cumulative_gain(i+1,1) > 0.0)
2091  {
2092  maximum_gain[1] = positive_cumulative_gain(i+1,1)-negative_cumulative_gain(i+1,1);
2093  maximum_gain[0] = percentage;
2094  }
2095  }
2096 
2097  return maximum_gain;
2098 }
2099 
2100 
2101 // Matrix<double> perform_calibration_plot_analysis(void) const
2102 
2104 
2106 {
2107  #ifdef __OPENNN_DEBUG__
2108 
2109  check();
2110 
2111  #endif
2112 
2113  #ifdef __OPENNN_DEBUG__
2114 
2115  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
2116 
2117  if(!multilayer_perceptron_pointer)
2118  {
2119  std::ostringstream buffer;
2120 
2121  buffer << "OpenNN Exception: TestingAnalysis class.\n"
2122  << "Matrix<double> perform_calibration_plot_analysis(void) const method.\n"
2123  << "Pointer to multilayer perceptron in neural network is NULL.\n";
2124 
2125  throw std::logic_error(buffer.str());
2126  }
2127 
2128  #endif
2129 
2130  #ifdef __OPENNN_DEBUG__
2131 
2132  const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
2133 
2134  // Control sentence
2135 
2136  const Variables& variables = data_set_pointer->get_variables();
2137 
2138  if(inputs_number != variables.count_inputs_number())
2139  {
2140  std::ostringstream buffer;
2141 
2142  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2143  << "Matrix<double> perform_calibration_plot_analysis(void) const method." << std::endl
2144  << "Number of inputs in neural network must be equal to number of inputs in data set." << std::endl;
2145 
2146  throw std::logic_error(buffer.str());
2147  }
2148 
2149  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
2150 
2151  if(outputs_number != variables.count_targets_number())
2152  {
2153  std::ostringstream buffer;
2154 
2155  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2156  << "Matrix<double> perform_calibration_plot_analysis(void) const method." << std::endl
2157  << "Number of outputs in neural network must be equal to number of targets in data set." << std::endl;
2158 
2159  throw std::logic_error(buffer.str());
2160  }
2161 
2162  #endif
2163 
2166 
2167  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
2168 
2169  const Matrix<double> calibration_plot = calculate_calibration_plot(target_data, output_data);
2170 
2171  return(calibration_plot);
2172 
2173 }
2174 
2175 //Matrix<double> calculate_calibration_plot(const Matrix<double>&, const Matrix<double>&) const
2176 
2181 
2183 {
2184  std::cout << "Calibration plot" << std::endl;
2185 
2186  const size_t rows_number = target_data.get_rows_number();
2187 
2188  std::cout << "Rows number: " << rows_number << std::endl;
2189 
2190  const size_t points_number = 10;
2191 
2192  Matrix<double> calibration_plot(points_number+2, 2);
2193 
2194  // First point
2195 
2196  calibration_plot(0,0) = 0.0;
2197  calibration_plot(0,1) = 0.0;
2198 
2199  size_t positives = 0;
2200 
2201  size_t count = 0;
2202 
2203  double probability = 0.0;
2204 
2205  double sum = 0.0;
2206 
2207  for(size_t i = 1; i < points_number+1; i++)
2208  {
2209  count = 0;
2210  positives = 0;
2211  sum = 0.0;
2212  probability += 0.1;
2213 
2214  for(size_t j = 0; j < rows_number; j++)
2215  {
2216  if(output_data(j, 0) >= (probability - 0.1) && output_data(j, 0) < probability)
2217  {
2218  count++;
2219 
2220  sum += output_data(j, 0);
2221 
2222  if(target_data(j, 0) == 1.0)
2223  {
2224  positives++;
2225  }
2226  }
2227  }
2228 
2229  if(count == 0)
2230  {
2231  calibration_plot(i, 0) = -1;
2232  calibration_plot(i, 1) = -1;
2233  }
2234  else
2235  {
2236  calibration_plot(i, 0) = (double)sum/(double)count;
2237  calibration_plot(i, 1) = (double)positives/(double)count;
2238  }
2239  }
2240 
2241  // Last point
2242 
2243  calibration_plot(points_number+1,0) = 1.0;
2244  calibration_plot(points_number+1,1) = 1.0;
2245 
2246  // Subtracts calibration plot rows with value -1
2247 
2248  size_t points_number_subtracted = 0;
2249 
2250  size_t current_rows_number;
2251 
2252  while(calibration_plot.arrange_column(0).contains(-1))
2253  {
2254  current_rows_number = calibration_plot.get_rows_number();
2255 
2256  for(size_t i = 1; i < points_number - points_number_subtracted+1; i++)
2257  {
2258  if(calibration_plot(i, 0) == -1)
2259  {
2260  calibration_plot.subtract_row(i);
2261 
2262  points_number_subtracted++;
2263  }
2264  }
2265  }
2266 
2267  return(calibration_plot);
2268 }
2269 
2270 
2271 // Vector< Histogram <double> > calculate_output_histogram(const Matrix<double>&, const size_t&) const;
2272 
2276 
2277 Vector< Histogram <double> > TestingAnalysis::calculate_output_histogram(const Matrix<double>& output_data, const size_t& bins_number) const
2278 {
2279  Vector< Histogram <double> > output_histogram (1);
2280 
2281  output_histogram [0] = output_data.arrange_column(0).calculate_histogram(bins_number);
2282 
2283  return(output_histogram);
2284 }
2285 
2286 
2287 // TestingAnalysis::BinaryClassificationRates TestingAnalysis::calculate_binary_classification_rates(void) const
2288 
2296 
2298 {
2299  #ifdef __OPENNN_DEBUG__
2300 
2301  check();
2302 
2303  #endif
2304 
2305  #ifdef __OPENNN_DEBUG__
2306 
2307  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
2308 
2309  if(!multilayer_perceptron_pointer)
2310  {
2311  std::ostringstream buffer;
2312 
2313  buffer << "OpenNN Exception: TestingAnalysis class.\n"
2314  << "BinaryClassificationRates calculate_binary_classification_rates(void) const method.\n"
2315  << "Pointer to multilayer perceptron in neural network is NULL.\n";
2316 
2317  throw std::logic_error(buffer.str());
2318  }
2319 
2320  #endif
2321 
2322  #ifdef __OPENNN_DEBUG__
2323 
2324  const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
2325 
2326  // Control sentence
2327 
2328  const Variables& variables = data_set_pointer->get_variables();
2329 
2330  if(inputs_number != variables.count_inputs_number())
2331  {
2332  std::ostringstream buffer;
2333 
2334  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2335  << "BinaryClassificationRates calculate_binary_classification_rates(void) const method." << std::endl
2336  << "Number of inputs in neural network must be equal to number of inputs in data set." << std::endl;
2337 
2338  throw std::logic_error(buffer.str());
2339  }
2340 
2341  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
2342 
2343  if(outputs_number != variables.count_targets_number())
2344  {
2345  std::ostringstream buffer;
2346 
2347  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2348  << "BinaryClassificationRates calculate_binary_classification_rates(void) const method." << std::endl
2349  << "Number of outputs in neural network must be equal to number of targets in data set." << std::endl;
2350 
2351  throw std::logic_error(buffer.str());
2352  }
2353 
2354  #endif
2355 
2358 
2359  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
2360 
2362 
2363  double decision_threshold;
2364 
2366  {
2368  }
2369  else
2370  {
2371  decision_threshold = 0.5;
2372  }
2373 
2374  BinaryClassifcationRates binary_classification_rates;
2375 
2376  binary_classification_rates.true_positive_instances = calculate_true_positive_instances(target_data, output_data, testing_indices, decision_threshold);
2377  binary_classification_rates.false_positive_instances = calculate_false_positive_instances(target_data, output_data, testing_indices, decision_threshold);
2378  binary_classification_rates.false_negative_instances = calculate_false_negative_instances(target_data, output_data, testing_indices, decision_threshold);
2379  binary_classification_rates.true_negative_instances = calculate_true_negative_instances(target_data, output_data, testing_indices, decision_threshold);
2380 
2381  return(binary_classification_rates);
2382 }
2383 
2384 
2385 // Vector<size_t> calculate_true_positive_instances (const Matrix<double>&, const Matrix<double>&, const Vector<size_t>&, const double&) const
2386 
2393 
2394 Vector<size_t> TestingAnalysis::calculate_true_positive_instances(const Matrix<double>& target_data, const Matrix<double>& output_data, const Vector<size_t>& testing_indices, const double& decision_threshold) const
2395 {
2396  const size_t rows_number = target_data.get_rows_number();
2397 
2398  Vector<size_t> true_positive_instances;
2399 
2400  for(size_t i = 0; i < rows_number; i++)
2401  {
2402  if(target_data(i,0) >= decision_threshold && output_data(i,0) >= decision_threshold)
2403  {
2404  true_positive_instances.push_back(testing_indices[i]);
2405  }
2406  }
2407 
2408  return(true_positive_instances);
2409 }
2410 
2411 
2412 // Vector<size_t> calculate_false_positive_instances(const Matrix<double>& , const Matrix<double>&, const Vector<size_t>&, const double&) const
2413 
2420 
2421 Vector<size_t> TestingAnalysis::calculate_false_positive_instances(const Matrix<double>& target_data, const Matrix<double>& output_data, const Vector<size_t>& testing_indices, const double& decision_threshold) const
2422 {
2423  const size_t rows_number = target_data.get_rows_number();
2424 
2425  Vector<size_t> false_positive_instances;
2426 
2427  for(size_t i = 0; i < rows_number; i++)
2428  {
2429  if(target_data(i,0) < decision_threshold && output_data(i,0) >= decision_threshold)
2430  {
2431  false_positive_instances.push_back(testing_indices[i]);
2432  }
2433  }
2434 
2435  return(false_positive_instances);
2436 }
2437 
2438 
2439 //Vector<size_t> calculate_false_negative_instances(const Matrix<double>& , const Matrix<double>&, const Vector<size_t>&, const double&)
2440 
2447 
2448 Vector<size_t> TestingAnalysis::calculate_false_negative_instances(const Matrix<double>& target_data, const Matrix<double>& output_data, const Vector<size_t>& testing_indices, const double& decision_threshold) const
2449 {
2450  const size_t rows_number = target_data.get_rows_number();
2451 
2452  Vector<size_t> false_negative_instances;
2453 
2454  for(size_t i = 0; i < rows_number; i++)
2455  {
2456  if(target_data(i,0) > decision_threshold && output_data(i,0) < decision_threshold)
2457  {
2458  false_negative_instances.push_back(testing_indices[i]);
2459  }
2460  }
2461 
2462  return(false_negative_instances);
2463 }
2464 
2465 
2466 // Vector<size_t> calculate_true_negative_instances(const Matrix<double>& , const Matrix<double>&, const Vector<size_t>&, const double&) const
2467 
2474 
2475 Vector<size_t> TestingAnalysis::calculate_true_negative_instances(const Matrix<double>& target_data, const Matrix<double>& output_data, const Vector<size_t>& testing_indices, const double& decision_threshold) const
2476 {
2477  Vector<size_t> true_negative_instances;
2478 
2479  const size_t rows_number = target_data.get_rows_number();
2480 
2481  for(size_t i = 0; i < rows_number; i++)
2482  {
2483  if(target_data(i,0) < decision_threshold && output_data(i,0) < decision_threshold)
2484  {
2485  true_negative_instances.push_back(testing_indices[i]);
2486  }
2487  }
2488 
2489  return(true_negative_instances);
2490 }
2491 
2492 
2493 // Matrix< Vector<size_t> > calculate_multiple_classification_rates(void) const
2494 
2496 
2498 {
2499  #ifdef __OPENNN_DEBUG__
2500 
2501  check();
2502 
2503  #endif
2504 
2505  #ifdef __OPENNN_DEBUG__
2506 
2507  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
2508 
2509  if(!multilayer_perceptron_pointer)
2510  {
2511  std::ostringstream buffer;
2512 
2513  buffer << "OpenNN Exception: TestingAnalysis class.\n"
2514  << "BinaryClassificationRates calculate_binary_classification_rates(void) const method.\n"
2515  << "Pointer to multilayer perceptron in neural network is NULL.\n";
2516 
2517  throw std::logic_error(buffer.str());
2518  }
2519 
2520  #endif
2521 
2522  #ifdef __OPENNN_DEBUG__
2523 
2524  const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
2525 
2526  // Control sentence
2527 
2528  const Variables& variables = data_set_pointer->get_variables();
2529 
2530  if(inputs_number != variables.count_inputs_number())
2531  {
2532  std::ostringstream buffer;
2533 
2534  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2535  << "BinaryClassificationRates calculate_binary_classification_rates(void) const method." << std::endl
2536  << "Number of inputs in neural network must be equal to number of inputs in data set." << std::endl;
2537 
2538  throw std::logic_error(buffer.str());
2539  }
2540 
2541  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
2542 
2543  if(outputs_number != variables.count_targets_number())
2544  {
2545  std::ostringstream buffer;
2546 
2547  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2548  << "BinaryClassificationRates calculate_binary_classification_rates(void) const method." << std::endl
2549  << "Number of outputs in neural network must be equal to number of targets in data set." << std::endl;
2550 
2551  throw std::logic_error(buffer.str());
2552  }
2553 
2554  #endif
2555 
2558 
2559  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
2560 
2562 
2563  return(calculate_multiple_classification_rates(target_data, output_data, testing_indices));
2564 }
2565 
2566 
2567 // Matrix< Vector<size_t> > calculate_multiple_classification_rates(void) method
2568 
2572 
2574 {
2575  const size_t rows_number = target_data.get_rows_number();
2576  const size_t columns_number = output_data.get_columns_number();
2577 
2578  Matrix< Vector<size_t> > multiple_classification_rates(rows_number, columns_number);
2579 
2580  size_t target_index;
2581  size_t output_index;
2582 
2583  for(size_t i = 0; i < rows_number; i++)
2584  {
2585  target_index = target_data.arrange_row(i).calculate_maximal_index();
2586  output_index = output_data.arrange_row(i).calculate_maximal_index();
2587 
2588  multiple_classification_rates(target_index, output_index).push_back(testing_indices[i]);
2589  }
2590 
2591  return(multiple_classification_rates);
2592 }
2593 
2594 
2595 // Vector< Vector<double> > calculate_error_autocorrelation(const size_t&) const
2596 
2602 
2604 {
2605  #ifdef __OPENNN_DEBUG__
2606 
2607  check();
2608 
2609  #endif
2610 
2611  #ifdef __OPENNN_DEBUG__
2612 
2613  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
2614 
2615  if(!multilayer_perceptron_pointer)
2616  {
2617  std::ostringstream buffer;
2618 
2619  buffer << "OpenNN Exception: TestingAnalysis class.\n"
2620  << "Vector<double> calculate_error_autocorrelation(void) const method.\n"
2621  << "Pointer to multilayer perceptron in neural network is NULL.\n";
2622 
2623  throw std::logic_error(buffer.str());
2624  }
2625 
2626  #endif
2627 
2628  #ifdef __OPENNN_DEBUG__
2629 
2630  const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
2631 
2632  // Control sentence
2633 
2634  const Variables& variables = data_set_pointer->get_variables();
2635 
2636  if(inputs_number != variables.count_inputs_number())
2637  {
2638  std::ostringstream buffer;
2639 
2640  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2641  << "Vector<double> calculate_error_autocorrelation(void) const method." << std::endl
2642  << "Number of inputs in neural network must be equal to number of inputs in data set." << std::endl;
2643 
2644  throw std::logic_error(buffer.str());
2645  }
2646 
2647  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
2648 
2649  if(outputs_number != variables.count_targets_number())
2650  {
2651  std::ostringstream buffer;
2652 
2653  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2654  << "Vector<double> calculate_error_autocorrelation(void) const method." << std::endl
2655  << "Number of outputs in neural network must be equal to number of targets in data set." << std::endl;
2656 
2657  throw std::logic_error(buffer.str());
2658  }
2659 
2660  #endif
2661 
2664 
2665  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
2666 
2667  const size_t targets_number = target_data.get_columns_number();
2668 
2669  Vector< Vector<double> > error_autocorrelation(targets_number);
2670 
2671  Matrix<double> error = target_data - output_data;
2672 
2673  for(size_t i = 0; i < targets_number; i++)
2674  {
2675  error_autocorrelation[i] = error.arrange_column(i).calculate_autocorrelation(maximum_lags_number);
2676  }
2677 
2678  return error_autocorrelation;
2679 }
2680 
2681 
2682 // Vector< Vector<double> > calculate_input_error_cross_correlation(const size_t&) const
2683 
2689 
2691 {
2692  #ifdef __OPENNN_DEBUG__
2693 
2694  check();
2695 
2696  #endif
2697 
2698  #ifdef __OPENNN_DEBUG__
2699 
2700  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
2701 
2702  if(!multilayer_perceptron_pointer)
2703  {
2704  std::ostringstream buffer;
2705 
2706  buffer << "OpenNN Exception: TestingAnalysis class.\n"
2707  << "Vector<double> calculate_input_error_cross_correlation(void) const method.\n"
2708  << "Pointer to multilayer perceptron in neural network is NULL.\n";
2709 
2710  throw std::logic_error(buffer.str());
2711  }
2712 
2713  #endif
2714 
2715  #ifdef __OPENNN_DEBUG__
2716 
2717  const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
2718 
2719  // Control sentence
2720 
2721  const Variables& variables = data_set_pointer->get_variables();
2722 
2723  if(inputs_number != variables.count_inputs_number())
2724  {
2725  std::ostringstream buffer;
2726 
2727  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2728  << "Vector<double> calculate_input_error_cross_correlation(void) const method." << std::endl
2729  << "Number of inputs in neural network must be equal to number of inputs in data set." << std::endl;
2730 
2731  throw std::logic_error(buffer.str());
2732  }
2733 
2734  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
2735 
2736  if(outputs_number != variables.count_targets_number())
2737  {
2738  std::ostringstream buffer;
2739 
2740  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2741  << "Vector<double> calculate_input_error_cross_correlation(void) const method." << std::endl
2742  << "Number of outputs in neural network must be equal to number of targets in data set." << std::endl;
2743 
2744  throw std::logic_error(buffer.str());
2745  }
2746 
2747  #endif
2748 
2751 
2752  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
2753 
2754  const size_t targets_number = target_data.get_columns_number();
2755 
2756  const Matrix<double> error = target_data - output_data;
2757 
2758  Vector<double> input_column;
2759 
2760  Vector< Vector<double> > input_error_cross_correlation(targets_number);
2761 
2762  for(size_t i = 0; i < targets_number; i++)
2763  {
2764  input_column = input_data.arrange_column(i);
2765 
2766  input_error_cross_correlation[i] = input_column.calculate_cross_correlation(error.arrange_column(i), maximum_lags_number);
2767  }
2768 
2769  return (input_error_cross_correlation);
2770 }
2771 
2772 
2773 // Vector<double> calculate_binary_classification_tests(void) method
2774 
2795 
2797 {
2798  // Control sentence (if debug)
2799 
2800  #ifdef __OPENNN_DEBUG__
2801 
2802  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
2803 
2804  const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
2805 
2806  if(!data_set_pointer)
2807  {
2808  std::ostringstream buffer;
2809 
2810  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2811  << "Vector<double> calculate_binary_classification_tests(void) const." << std::endl
2812  << "Data set is NULL." << std::endl;
2813 
2814  throw std::logic_error(buffer.str());
2815  }
2816 
2817  const Variables& variables = data_set_pointer->get_variables();
2818 
2819  const size_t targets_number = variables.count_targets_number();
2820 
2821  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
2822 
2823  // Control sentence
2824 
2825  if(inputs_number != variables.count_inputs_number())
2826  {
2827  std::ostringstream buffer;
2828 
2829  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2830  << "Vector<double> calculate_binary_classification_tests(void) const." << std::endl
2831  << "Number of inputs in neural network is not equal to number of inputs in data set." << std::endl;
2832 
2833  throw std::logic_error(buffer.str());
2834  }
2835  else if(outputs_number != 1)
2836  {
2837  std::ostringstream buffer;
2838 
2839  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2840  << "Vector<double> calculate_binary_classification_tests(void) const." << std::endl
2841  << "Number of outputs in neural network must be one." << std::endl;
2842 
2843  throw std::logic_error(buffer.str());
2844  }
2845  else if(targets_number != 1)
2846  {
2847  std::ostringstream buffer;
2848 
2849  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
2850  << "Vector<double> calculate_binary_classification_tests(void) const." << std::endl
2851  << "Number of targets in data set must be one." << std::endl;
2852 
2853  throw std::logic_error(buffer.str());
2854  }
2855 
2856  #endif
2857 
2858  // Confusion matrix
2859 
2860  const Matrix<size_t> confusion = calculate_confusion();
2861 
2862  const size_t true_positive = confusion(0,0);
2863  const size_t false_positive = confusion(1,0);
2864  const size_t false_negative = confusion(0,1);
2865  const size_t true_negative = confusion(1,1);
2866 
2867  // Classification accuracy
2868 
2869  double classification_accuracy;
2870 
2871  if(true_positive + true_negative + false_positive + false_negative == 0)
2872  {
2873  classification_accuracy = 0.0;
2874  }
2875  else
2876  {
2877  classification_accuracy = (double)(true_positive + true_negative)/(double)(true_positive + true_negative + false_positive + false_negative);
2878  }
2879 
2880  // Error rate
2881 
2882  double error_rate;
2883 
2884  if(true_positive + true_negative + false_positive + false_negative == 0)
2885  {
2886  error_rate = 0.0;
2887  }
2888  else
2889  {
2890  error_rate = (double)(false_positive + false_negative)/(double)(true_positive + true_negative + false_positive + false_negative);
2891  }
2892 
2893  // Sensitivity
2894 
2895  double sensitivity;
2896 
2897  if(true_positive + false_negative == 0)
2898  {
2899  sensitivity = 0.0;
2900  }
2901  else
2902  {
2903  sensitivity = (double)true_positive/(double)(true_positive + false_negative);
2904  }
2905 
2906  // Specificity
2907 
2908  double specificity;
2909 
2910  if(true_negative + false_positive == 0)
2911  {
2912  specificity = 0.0;
2913  }
2914  else
2915  {
2916  specificity = (double)true_negative/(double)(true_negative + false_positive);
2917  }
2918 
2919  // Precision
2920 
2921  double precision;
2922 
2923  if(true_positive + false_positive == 0)
2924  {
2925  precision = 0.0;
2926  }
2927  else
2928  {
2929  precision = (double) true_positive / (double)(true_positive + false_positive);
2930  }
2931 
2932  // Positive likelihood
2933 
2934  double positive_likelihood;
2935 
2936  if(classification_accuracy == 1.0)
2937  {
2938  positive_likelihood = 1.0;
2939  }
2940  else if(1.0 - specificity == 0.0)
2941  {
2942  positive_likelihood = 0.0;
2943  }
2944  else
2945  {
2946  positive_likelihood = sensitivity/(1.0 - specificity);
2947  }
2948 
2949  // Negative likelihood
2950 
2951  double negative_likelihood;
2952 
2953  if(classification_accuracy == 1.0)
2954  {
2955  negative_likelihood = 1.0;
2956  }
2957  else if(1.0 - sensitivity == 0.0)
2958  {
2959  negative_likelihood = 0.0;
2960  }
2961  else
2962  {
2963  negative_likelihood = specificity/(1.0 - sensitivity);
2964  }
2965 
2966  // F1 score
2967 
2968  double F1_score;
2969 
2970  if(2*true_positive + false_positive + false_negative == 0)
2971  {
2972  F1_score = 0.0;
2973  }
2974  else
2975  {
2976  F1_score = (double) 2*true_positive/(double) (2*true_positive + false_positive + false_negative);
2977  }
2978 
2979  // False positive rate
2980 
2981  double false_positive_rate;
2982 
2983  if(false_positive + true_negative == 0)
2984  {
2985  false_positive_rate = 0.0;
2986  }
2987  else
2988  {
2989  false_positive_rate = (double) false_positive/(double) (false_positive + true_negative);
2990  }
2991 
2992  // False discovery rate
2993 
2994  double false_discovery_rate;
2995 
2996  if(false_positive + true_positive == 0)
2997  {
2998  false_discovery_rate = 0.0;
2999  }
3000  else
3001  {
3002  false_discovery_rate = (double) false_positive /(double) (false_positive + true_positive);
3003  }
3004 
3005  // False negative rate
3006 
3007  double false_negative_rate;
3008 
3009  if(false_negative + true_positive == 0)
3010  {
3011  false_negative_rate = 0.0;
3012  }
3013  else
3014  {
3015  false_negative_rate = (double) false_negative /(double) (false_negative + true_positive);
3016  }
3017 
3018  // Negative predictive value
3019 
3020  double negative_predictive_value;
3021 
3022  if(true_negative + false_negative == 0)
3023  {
3024  negative_predictive_value = 0.0;
3025  }
3026  else
3027  {
3028  negative_predictive_value = (double) true_negative/(double) (true_negative + false_negative);
3029  }
3030 
3031  //Matthews correlation coefficient
3032 
3033  double Matthews_correlation_coefficient;
3034 
3035  if((true_positive + false_positive) * (true_positive + false_negative) * (true_negative + false_positive) * (true_negative + false_negative) == 0)
3036  {
3037  Matthews_correlation_coefficient = 0.0;
3038  }
3039  else
3040  {
3041  Matthews_correlation_coefficient = (double) (true_positive * true_negative - false_positive * false_negative) /(double) sqrt((double)((true_positive + false_positive) * (true_positive + false_negative) * (true_negative + false_positive) * (true_negative + false_negative)));
3042  }
3043 
3044  //Informedness
3045 
3046  double informedness = sensitivity + specificity - 1;
3047 
3048  //Markedness
3049 
3050  double markedness;
3051 
3052  if(true_negative + false_positive == 0)
3053  {
3054  markedness = precision - 1;
3055  }
3056  else
3057  {
3058  markedness = precision + (double) true_negative/(double) (true_negative + false_positive) - 1;
3059  }
3060 
3061  //Arrange vector
3062 
3063  Vector<double> binary_classification_test(15);
3064 
3065  binary_classification_test[0] = classification_accuracy;
3066  binary_classification_test[1] = error_rate;
3067  binary_classification_test[2] = sensitivity;
3068  binary_classification_test[3] = specificity;
3069  binary_classification_test[4] = precision;
3070  binary_classification_test[5] = positive_likelihood;
3071  binary_classification_test[6] = negative_likelihood;
3072  binary_classification_test[7] = F1_score;
3073  binary_classification_test[8] = false_positive_rate;
3074  binary_classification_test[9] = false_discovery_rate;
3075  binary_classification_test[10] = false_negative_rate;
3076  binary_classification_test[11] = negative_predictive_value;
3077  binary_classification_test[12] = Matthews_correlation_coefficient;
3078  binary_classification_test[13] = informedness;
3079  binary_classification_test[14] = markedness;
3080 
3081  return(binary_classification_test);
3082 }
3083 
3084 
3085 // double calculate_logloss(void) const method
3086 
3088 
3090 {
3091  #ifdef __OPENNN_DEBUG__
3092 
3093  check();
3094 
3095  #endif
3096 
3097  #ifdef __OPENNN_DEBUG__
3098 
3099  const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
3100 
3101  if(!multilayer_perceptron_pointer)
3102  {
3103  std::ostringstream buffer;
3104 
3105  buffer << "OpenNN Exception: TestingAnalysis class.\n"
3106  << "Vector<double> calculate_input_error_cross_correlation(void) const method.\n"
3107  << "Pointer to multilayer perceptron in neural network is NULL.\n";
3108 
3109  throw std::logic_error(buffer.str());
3110  }
3111 
3112  #endif
3113 
3114  #ifdef __OPENNN_DEBUG__
3115 
3116  const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
3117 
3118  // Control sentence
3119 
3120  const Variables& variables = data_set_pointer->get_variables();
3121 
3122  if(inputs_number != variables.count_inputs_number())
3123  {
3124  std::ostringstream buffer;
3125 
3126  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
3127  << "Vector<double> calculate_input_error_cross_correlation(void) const method." << std::endl
3128  << "Number of inputs in neural network must be equal to number of inputs in data set." << std::endl;
3129 
3130  throw std::logic_error(buffer.str());
3131  }
3132 
3133  const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
3134 
3135  if(outputs_number != variables.count_targets_number())
3136  {
3137  std::ostringstream buffer;
3138 
3139  buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
3140  << "Vector<double> calculate_input_error_cross_correlation(void) const method." << std::endl
3141  << "Number of outputs in neural network must be equal to number of targets in data set." << std::endl;
3142 
3143  throw std::logic_error(buffer.str());
3144  }
3145 
3146  #endif
3147 
3150 
3151  const Matrix<double> output_data = neural_network_pointer->calculate_output_data(input_data);
3152 
3153  const size_t testing_instances_number = target_data.get_rows_number();
3154 
3155  double logloss = 0.0;
3156 
3157  for(size_t i = 0; i < testing_instances_number; i++)
3158  {
3159  logloss += target_data(i,0)*log(output_data(i,0)) + (1-target_data(i,0))*log(1-output_data(i,0));
3160  }
3161 
3162  return(-logloss/testing_instances_number);
3163 }
3164 
3165 
3166 // std::string to_string(void) const method
3167 
3169 
3170 std::string TestingAnalysis::to_string(void) const
3171 {
3172  std::ostringstream buffer;
3173 
3174  buffer << "Testing analysis\n"
3175  << "Display: " << display << "\n";
3176 
3177  return(buffer.str());
3178 }
3179 
3180 
3181 // void print(void) const method
3182 
3184 
3185 void TestingAnalysis::print(void) const
3186 {
3187  std::cout << to_string();
3188 }
3189 
3190 
3191 // tinyxml2::XMLDocument* to_XML(void) const method
3192 
3195 
3196 tinyxml2::XMLDocument* TestingAnalysis::to_XML(void) const
3197 {
3198  tinyxml2::XMLDocument* document = new tinyxml2::XMLDocument;
3199 
3200  std::ostringstream buffer;
3201 
3202  // Root element
3203 
3204  tinyxml2::XMLElement* testing_analysis_element = document->NewElement("TestingAnalysis");
3205 
3206  document->InsertFirstChild(testing_analysis_element);
3207 
3208  // Display
3209 
3210  tinyxml2::XMLElement* display_element = document->NewElement("Display");
3211  testing_analysis_element->LinkEndChild(display_element);
3212 
3213  buffer.str("");
3214  buffer << display;
3215 
3216  tinyxml2::XMLText* display_text = document->NewText(buffer.str().c_str());
3217  testing_analysis_element->LinkEndChild(display_text);
3218 
3219  return(document);
3220 }
3221 
3222 
3223 // void write_XML(tinyxml2::XMLPrinter&) const method
3224 
3227 
3228 void TestingAnalysis::write_XML(tinyxml2::XMLPrinter& file_stream) const
3229 {
3230  std::ostringstream buffer;
3231  file_stream.OpenElement("TestingAnalysis");
3232 
3233  // Display
3234 
3235  file_stream.OpenElement("Display");
3236 
3237  buffer.str("");
3238  buffer << display;
3239 
3240  file_stream.PushText(buffer.str().c_str());
3241 
3242  file_stream.CloseElement();
3243 
3244 
3245  file_stream.CloseElement();
3246 }
3247 
3248 
3249 // void from_XML(const tinyxml2::XMLDocument&) method
3250 
3253 
3254 void TestingAnalysis::from_XML(const tinyxml2::XMLDocument& document)
3255 {
3256  std::ostringstream buffer;
3257 
3258  const tinyxml2::XMLElement* root_element = document.FirstChildElement("TestingAnalysis");
3259 
3260  if(!root_element)
3261  {
3262  buffer << "OpenNN Exception: TestingAnalysis class.\n"
3263  << "void from_XML(const tinyxml2::XMLDocument&) method.\n"
3264  << "Testing analysis element is NULL.\n";
3265 
3266  throw std::logic_error(buffer.str());
3267  }
3268 
3269  // Display
3270 
3271  const tinyxml2::XMLElement* element = root_element->FirstChildElement("Display");
3272 
3273  if(element)
3274  {
3275  std::string new_display_string = element->GetText();
3276 
3277  try
3278  {
3279  set_display(new_display_string != "0");
3280  }
3281  catch(const std::logic_error& e)
3282  {
3283  std::cout << e.what() << std::endl;
3284  }
3285  }
3286 }
3287 
3288 
3289 // void save(const std::string&) const method
3290 
3293 
3294 void TestingAnalysis::save(const std::string& file_name) const
3295 {
3296  tinyxml2::XMLDocument* document = to_XML();
3297 
3298  document->SaveFile(file_name.c_str());
3299 
3300  delete document;
3301 }
3302 
3303 
3304 // void load(const std::string&) method
3305 
3308 
3309 void TestingAnalysis::load(const std::string& file_name)
3310 {
3311  set_default();
3312 
3313  tinyxml2::XMLDocument document;
3314 
3315  if(document.LoadFile(file_name.c_str()))
3316  {
3317  std::ostringstream buffer;
3318 
3319  buffer << "OpenNN Exception: Testing analysis class.\n"
3320  << "void load(const std::string&) method.\n"
3321  << "Cannot load XML file " << file_name << ".\n";
3322 
3323  throw std::logic_error(buffer.str());
3324  }
3325 
3326  from_XML(document);
3327 }
3328 
3329 }
3330 
3331 
3332 // OpenNN: Open Neural Networks Library.
3333 // Copyright (c) 2005-2016 Roberto Lopez.
3334 //
3335 // This library is free software; you can redistribute it and/or
3336 // modify it under the terms of the GNU Lesser General Public
3337 // License as published by the Free Software Foundation; either
3338 // version 2.1 of the License, or any later version.
3339 //
3340 // This library is distributed in the hope that it will be useful,
3341 // but WITHOUT ANY WARRANTY; without even the implied warranty of
3342 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
3343 // Lesser General Public License for more details.
3344 
3345 // You should have received a copy of the GNU Lesser General Public
3346 // License along with this library; if not, write to the Free Software
3347 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Matrix< T > arrange_submatrix_columns(const Vector< size_t > &) const
Definition: matrix.h:1509
Vector< T > arrange_row(const size_t &) const
Definition: matrix.h:1537
Matrix< double > arrange_testing_target_data(void) const
Definition: data_set.cpp:941
double calculate_sum_squared_error(const Matrix< double > &) const
Definition: matrix.h:5239
Matrix< T > sort_less_rows(const size_t &) const
Definition: matrix.h:2476
double calculate_testing_normalized_squared_error(const Matrix< double > &, const Matrix< double > &) const
MathematicalModel * get_mathematical_model_pointer(void) const
Returns a pointer to the mathematical model object on which the neural network is tested...
Vector< T > arrange_column(const size_t &) const
Definition: matrix.h:1612
double calculate_testing_cross_entropy_error(const Matrix< double > &, const Matrix< double > &) const
Histogram< T > calculate_histogram(const size_t &=10) const
Definition: vector.h:1667
Vector< size_t > calculate_target_distribution(void) const
Definition: data_set.cpp:5775
double calculate_area_under_curve(const Matrix< double > &, const Matrix< double > &) const
Vector< Histogram< double > > calculate_error_data_histograms(const size_t &=10) const
bool display
Display messages to screen.
std::string to_string(void) const
Returns a string representation of the testing analysis object.
Vector< double > calculate_testing_errors(void) const
Matrix< double > calculate_calibration_plot(const Matrix< double > &, const Matrix< double > &) const
Matrix< size_t > calculate_confusion_multiple_classification(const Matrix< double > &, const Matrix< double > &) const
Vector< size_t > calculate_true_positive_instances(const Matrix< double > &, const Matrix< double > &, const Vector< size_t > &, const double &) const
NeuralNetwork * get_neural_network_pointer(void) const
Returns a pointer to the neural network object which is to be tested.
void set_mathematical_model_pointer(MathematicalModel *)
void set(void)
Sets the size of a vector to zero.
Definition: vector.h:686
Vector< double > calculate_autocorrelation(const size_t &=10) const
Definition: vector.h:3653
void set_display(const bool &)
Vector< size_t > arrange_testing_indices(void) const
Returns the indices of the instances which will be used for testing.
Definition: instances.cpp:625
double calculate_Wilcoxon_parameter(const double &, const double &) const
Vector< size_t > calculate_true_negative_instances(const Matrix< double > &, const Matrix< double > &, const Vector< size_t > &, const double &) const
Vector< Vector< size_t > > calculate_maximal_errors(const size_t &=10) const
double calculate_sum_squared_error(const Vector< double > &) const
Definition: vector.h:3328
Vector< LinearRegressionParameters< double > > calculate_linear_regression_parameters(void) const
size_t count_targets_number(void) const
Returns the number of target variables of the data set.
Definition: variables.cpp:272
T calculate_sum(void) const
Returns the sum of all the elements in the matrix.
Definition: matrix.h:2948
size_t calculate_training_negatives(const size_t &) const
Definition: data_set.cpp:2353
Vector< size_t > calculate_false_negative_instances(const Matrix< double > &, const Matrix< double > &, const Vector< size_t > &, const double &) const
size_t get_unscaling_neurons_number(void) const
Returns the number of unscaling neurons in this layer.
Vector< double > calculate_binary_classification_tests(void) const
double area_under_curve
Area under a ROC curve.
virtual void write_XML(tinyxml2::XMLPrinter &) const
LinearRegressionResults perform_linear_regression_analysis(void) const
void set_data_set_pointer(DataSet *)
LinearRegressionParameters< T > calculate_linear_regression_parameters(const Vector< T > &) const
Definition: vector.h:3756
DataSet * get_data_set_pointer(void) const
Returns a pointer to the data set object on which the neural network is tested.
Matrix< T > assemble_columns(const Matrix< T > &) const
Definition: matrix.h:2559
Matrix< size_t > calculate_confusion_binary_classification(const Matrix< double > &, const Matrix< double > &, const double &) const
Vector< Vector< double > > calculate_error_autocorrelation(const size_t &=10) const
const Instances & get_instances(void) const
Returns a constant reference to the instances object composing this data set object.
Definition: data_set.cpp:220
Matrix< double > calculate_lift_chart(const Matrix< double > &) const
Matrix< double > arrange_testing_input_data(void) const
Definition: data_set.cpp:925
BinaryClassifcationRates calculate_binary_classification_rates(void) const
Vector< size_t > false_positive_instances
Vector with the indices of the instances which are false positive.
ProbabilisticLayer * get_probabilistic_layer_pointer(void) const
Returns a pointer to the probabilistic layer composing this neural network.
Matrix< double > negative_cumulative_gain
Matrix containing the data of a negative cumulative gain.
size_t count_inputs_number(void) const
Returns the number of input variables of the data set.
Definition: variables.cpp:250
double calculate_logloss(void) const
Returns the logloss for a binary classification problem.
Vector< double > calculate_maximum_gain(const Matrix< double > &, const Matrix< double > &) const
Vector< double > calculate_cross_correlation(const Vector< double > &, const size_t &=10) const
Definition: vector.h:3695
Matrix< double > calculate_output_data(const Matrix< double > &) const
size_t get_inputs_number(void) const
Returns the number of inputs to the multilayer perceptron.
Vector< double > calculate_classification_testing_errors(void) const
Vector< double > calculate_testing_target_data_mean(void) const
Returns the mean values of the target variables on the testing instances.
Definition: data_set.cpp:2915
Vector< Matrix< double > > calculate_error_data_statistics_matrices(void) const
Matrix< double > positive_cumulative_gain
Matrix containing the data of a positive cumulative gain.
double optimal_threshold
Optimal threshold of a ROC curve.
KolmogorovSmirnovResults perform_Kolmogorov_Smirnov_analysis(void) const
NeuralNetwork * neural_network_pointer
Pointer to the neural network object to be tested.
void set_neural_network_pointer(NeuralNetwork *)
Vector< Matrix< double > > calculate_target_output_data(void) const
size_t count_testing_instances_number(void) const
Returns the number of instances in the data set which will be used for testing.
Definition: instances.cpp:447
UnscalingLayer * get_unscaling_layer_pointer(void) const
Returns a pointer to the unscaling layer composing this neural network.
Vector< size_t > calculate_false_positive_instances(const Matrix< double > &, const Matrix< double > &, const Vector< size_t > &, const double &) const
Vector< size_t > calculate_maximal_indices(const size_t &) const
Definition: vector.h:1941
Vector< size_t > true_negative_instances
Vector with the indices of the instances which are true negative.
void print(void) const
Prints to the standard output the string representation of this testing analysis object.
double calculate_optimal_threshold(const Matrix< double > &, const Matrix< double > &) const
Matrix< Vector< size_t > > calculate_multiple_classification_rates(void) const
Returns a matrix of subvectors which have the rates for a multiple classification problem...
void load(const std::string &)
Vector< Histogram< double > > calculate_output_histogram(const Matrix< double > &, const size_t &=10) const
Vector< Matrix< double > > target_output_data
Target data from data set and output data from neural network.
Vector< double > arrange_minimums(void) const
const bool & get_display(void) const
Matrix< double > perform_calibration_plot_analysis(void) const
Performs a calibration plot analysis.
Matrix< double > perform_lift_chart_analysis(void) const
Matrix< double > calculate_cumulative_gain(const Matrix< double > &, const Matrix< double > &) const
Matrix< double > perform_cumulative_gain_analysis(void) const
MultilayerPerceptron * get_multilayer_perceptron_pointer(void) const
Returns a pointer to the multilayer perceptron composing this neural network.
Matrix< size_t > calculate_confusion(void) const
DataSet * data_set_pointer
Pointer to a data set object.
Vector< double > maximum_gain
Maximum gain of the cumulative gain analysis.
size_t get_outputs_number(void) const
Returns the number of outputs neurons in the multilayer perceptron.
virtual void from_XML(const tinyxml2::XMLDocument &)
const size_t & get_columns_number(void) const
Returns the number of columns in the matrix.
Definition: matrix.h:1122
size_t calculate_maximal_index(void) const
Returns the index of the largest element in the vector.
Definition: vector.h:1893
virtual tinyxml2::XMLDocument * to_XML(void) const
double calculate_testing_weighted_squared_error(const Matrix< double > &, const Matrix< double > &) const
Statistics< T > calculate_statistics(void) const
Definition: vector.h:2929
bool contains(const T &) const
Returns true if the vector contains a certain value, and false otherwise.
Definition: vector.h:988
const Variables & get_variables(void) const
Returns a constant reference to the variables object composing this data set object.
Definition: data_set.cpp:200
const double & get_decision_threshold(void) const
Returns the decision threshold.
RocAnalysisResults perform_roc_analysis(void) const
Vector< Vector< Statistics< double > > > calculate_error_data_statistics(void) const
Vector< Vector< double > > calculate_input_error_cross_correlation(const size_t &=10) const
Matrix< double > calculate_negative_cumulative_gain(const Matrix< double > &, const Matrix< double > &) const
Vector< size_t > arrange_targets_indices(void) const
Returns the indices of the target variables.
Definition: variables.cpp:660
MathematicalModel * mathematical_model_pointer
Pointer to a mathematical model object.
Vector< size_t > true_positive_instances
Vector with the indices of the instances which are true positive.
Matrix< double > roc_curve
Matrix containing the data of a ROC curve.
Vector< double > arrange_maximums(void) const
void save(const std::string &) const
Matrix< T > sort_greater_rows(const size_t &) const
Definition: matrix.h:2516
Matrix< double > calculate_roc_curve(const Matrix< double > &, const Matrix< double > &) const
Vector< Matrix< double > > calculate_error_data(void) const
void subtract_row(const size_t &)
Definition: matrix.h:2315
Vector< size_t > false_negative_instances
Vector with the indices of the instances which are false negative.
const size_t & get_rows_number(void) const
Returns the number of rows in the matrix.
Definition: matrix.h:1111
Vector< size_t > calculate_positives_negatives_rate(const Matrix< double > &, const Matrix< double > &) const