9#include "neural_network.h"
34 set(model_type, architecture);
40 Tensor<Index, 1> architecture(architecture_list.size());
41 architecture.setValues(architecture_list);
43 set(model_type, architecture);
52 const Index& new_blocks_number,
53 const Tensor<Index, 1>& new_filters_dimensions,
54 const Index& new_outputs_number)
56 set(new_inputs_dimensions, new_blocks_number, new_filters_dimensions, new_outputs_number);
101void NeuralNetwork::delete_layers()
105 for(Index i = 0; i < layers_number; i++)
123 ostringstream buffer;
125 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
126 <<
"NeuralNetwork::add_layer() method.\n"
127 <<
"No layers can be added after a bounding layer.\n";
131 throw logic_error(buffer.str());
136 ostringstream buffer;
138 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
139 <<
"NeuralNetwork::add_layer() method.\n"
140 <<
"No layers can be added after a probabilistic layer.\n";
142 throw logic_error(buffer.str());
147 ostringstream buffer;
149 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
150 <<
"NeuralNetwork::add_layer() method.\n"
151 <<
"Pooling Layer is not available yet. It will be included in future versions.\n";
153 throw logic_error(buffer.str());
158 ostringstream buffer;
160 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
161 <<
"NeuralNetwork::add_layer() method.\n"
162 <<
"Convolutional Layer is not available yet. It will be included in future versions.\n";
164 throw logic_error(buffer.str());
177 for(Index i = 0; i < old_layers_number; i++)
layers_pointers(i) = old_layers_pointers(i);
183 ostringstream buffer;
185 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
186 <<
"void add_layer(const Layer*) method.\n"
188 <<
" in the neural network architecture.\n";
190 throw logic_error(buffer.str());
203 if(layers_number > 1 && (layer_type == Layer::Type::Recurrent || layer_type == Layer::Type::LongShortTermMemory))
207 else if(layers_number == 1 && (layer_type == Layer::Type::Recurrent || layer_type == Layer::Type::LongShortTermMemory))
211 if(first_layer_type != Layer::Type::Scaling)
return false;
225 for(Index i = 0; i < layers_number; i++)
227 if(
layers_pointers[i]->get_type() == Layer::Type::Scaling)
return true;
241 for(Index i = 0; i < layers_number; i++)
243 if(
layers_pointers[i]->get_type() == Layer::Type::LongShortTermMemory)
return true;
257 for(Index i = 0; i < layers_number; i++)
259 if(
layers_pointers[i]->get_type() == Layer::Type::Convolutional)
return true;
273 for(Index i = 0; i < layers_number; i++)
275 if(
layers_pointers[i]->get_type() == Layer::Type::Recurrent)
return true;
289 for(Index i = 0; i < layers_number; i++)
291 if(
layers_pointers[i]->get_type() == Layer::Type::Unscaling)
return true;
305 for(Index i = 0; i < layers_number; i++)
307 if(
layers_pointers[i]->get_type() == Layer::Type::Bounding)
return true;
321 for(Index i = 0; i < layers_number; i++)
323 if(
layers_pointers[i]->get_type() == Layer::Type::Probabilistic)
return true;
411Layer* NeuralNetwork::get_layer_pointer(
const Index& layer_index)
const
423 const Index trainable_layers_number = get_trainable_layers_number();
425 Tensor<Layer*, 1> trainable_layers_pointers(trainable_layers_number);
429 for(Index i = 0; i < layers_number; i++)
440 return trainable_layers_pointers;
450 const Index trainable_layers_number = get_trainable_layers_number();
452 Tensor<Index, 1> trainable_layers_indices(trainable_layers_number);
454 Index trainable_layer_index = 0;
456 for(Index i = 0; i < layers_number; i++)
462 trainable_layers_indices[trainable_layer_index] = i;
463 trainable_layer_index++;
467 return trainable_layers_indices;
477 for(Index i = 0; i < layers_number; i++)
485 ostringstream buffer;
487 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
488 <<
"ScalingLayer* get_scaling_layer_pointer() const method.\n"
489 <<
"No scaling layer in neural network.\n";
491 throw logic_error(buffer.str());
501 for(Index i = 0; i < layers_number; i++)
509 ostringstream buffer;
511 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
512 <<
"UnscalingLayer* get_unscaling_layer_pointer() const method.\n"
513 <<
"No unscaling layer in neural network.\n";
515 throw logic_error(buffer.str());
525 for(Index i = 0; i < layers_number; i++)
533 ostringstream buffer;
535 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
536 <<
"BoundingLayer* get_bounding_layer_pointer() const method.\n"
537 <<
"No bounding layer in neural network.\n";
539 throw logic_error(buffer.str());
549 for(Index i = 0; i < layers_number; i++)
557 ostringstream buffer;
559 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
560 <<
"ProbabilisticLayer* get_probabilistic_layer_pointer() const method.\n"
561 <<
"No probabilistic layer in neural network.\n";
563 throw logic_error(buffer.str());
572 for(Index i = 0; i < layers_number; i++)
580 ostringstream buffer;
582 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
583 <<
"LongShortTermMemoryLayer* get_long_short_term_memory_layer_pointer() const method.\n"
584 <<
"No long-short term memory layer in neural network.\n";
586 throw logic_error(buffer.str());
596 for(Index i = 0; i < layers_number; i++)
604 ostringstream buffer;
606 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
607 <<
"RecurrentLayer* get_recurrent_layer_pointer() const method.\n"
608 <<
"No recurrent layer in neural network.\n";
610 throw logic_error(buffer.str());
643void NeuralNetwork::set(
const NeuralNetwork::ProjectType& model_type,
const Tensor<Index, 1>& architecture)
647 if(architecture.size() <= 1)
return;
649 const Index size = architecture.size();
651 const Index inputs_number = architecture[0];
652 const Index outputs_number = architecture[size-1];
660 if(model_type == ProjectType::Approximation)
662 for(Index i = 0; i < size-1; i++)
665 perceptron_layer_pointer->set_name(
"perceptron_layer_" + to_string(i+1));
667 this->
add_layer(perceptron_layer_pointer);
669 if(i == size-2) perceptron_layer_pointer->
set_activation_function(PerceptronLayer::ActivationFunction::Linear);
674 this->
add_layer(unscaling_layer_pointer);
680 else if(model_type == ProjectType::Classification)
682 for(Index i = 0; i < size-2; i++)
686 perceptron_layer_pointer->set_name(
"perceptron_layer_" + to_string(i+1));
688 this->
add_layer(perceptron_layer_pointer);
693 this->
add_layer(probabilistic_layer_pointer);
695 else if(model_type == ProjectType::Forecasting)
700 this->
add_layer(long_short_term_memory_layer_pointer);
702 for(Index i = 1; i < size-1; i++)
706 perceptron_layer_pointer->set_name(
"perceptron_layer_" + to_string(i));
708 this->
add_layer(perceptron_layer_pointer);
710 if(i == size-2) perceptron_layer_pointer->
set_activation_function(PerceptronLayer::ActivationFunction::Linear);
715 this->
add_layer(unscaling_layer_pointer);
728void NeuralNetwork::set(
const NeuralNetwork::ProjectType& model_type,
const initializer_list<Index>& architecture_list)
730 Tensor<Index, 1> architecture(architecture_list.size());
731 architecture.setValues(architecture_list);
733 set(model_type, architecture);
745 const Index& blocks_number,
746 const Tensor<Index, 1>& filters_dimensions,
747 const Index& outputs_number)
754 Tensor<Index, 1> outputs_dimensions = scaling_layer->get_outputs_dimensions();
756 for(Index i = 0; i < blocks_number; i++)
771 const Tensor<Index, 0> outputs_dimensions_sum = outputs_dimensions.sum();
819 if(new_inputs_number == 0)
821 ostringstream buffer;
823 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
824 <<
"void set_inputs_number(const Index&) method.\n"
825 <<
"The number of inputs (" << new_inputs_number <<
") must be greater than 0.\n";
827 throw logic_error(buffer.str());
838 scaling_layer_pointer->set_inputs_number(new_inputs_number);
841 const Index trainable_layers_number = get_trainable_layers_number();
844 if(trainable_layers_number > 0)
846 trainable_layers_pointers[0]->set_inputs_number(new_inputs_number);
858 Index new_inputs_number = 0;
860 for(Index i = 0; i < inputs.dimension(0); i++)
862 if(inputs(i)) new_inputs_number++;
877void NeuralNetwork::set_threads_number(
const int& new_threads_number)
881 for(Index i = 0; i < layers_number; i++)
888void NeuralNetwork::set_layers_pointers(Tensor<Layer*, 1>& new_layers_pointers)
894PerceptronLayer* NeuralNetwork::get_first_perceptron_layer_pointer()
const
898 for(Index i = 0; i < layers_number; i++)
923Index NeuralNetwork::get_outputs_number()
const
929 return last_layer->get_neurons_number();
936Tensor<Index, 1> NeuralNetwork::get_trainable_layers_neurons_numbers()
const
938 const Index trainable_layers_number = get_trainable_layers_number();
940 Tensor<Index, 1> layers_neurons_number(trainable_layers_number);
950 layers_neurons_number(count) =
layers_pointers[i]->get_neurons_number();
957 return layers_neurons_number;
961Tensor<Index, 1> NeuralNetwork::get_trainable_layers_inputs_numbers()
const
963 const Index trainable_layers_number = get_trainable_layers_number();
965 Tensor<Index, 1> layers_neurons_number(trainable_layers_number);
975 layers_neurons_number(count) =
layers_pointers[i]->get_inputs_number();
981 return layers_neurons_number;
985Tensor<Index, 1> NeuralNetwork::get_trainable_layers_synaptic_weight_numbers()
const
987 const Index trainable_layers_number = get_trainable_layers_number();
989 Tensor<Index, 1> layers_neurons_number(trainable_layers_number);
999 layers_neurons_number(count) =
layers_pointers[i]->get_synaptic_weights_number();
1005 return layers_neurons_number;
1024 Tensor<Index, 1> architecture(layers_number);
1028 if(inputs_number == 0)
return architecture;
1030 if(layers_number > 0)
1032 for(Index i = 0; i < layers_number; i++)
1038 return architecture;
1049 Index parameters_number = 0;
1051 for(Index i = 0; i < trainable_layers_pointers.size(); i++)
1053 parameters_number += trainable_layers_pointers[i]->get_parameters_number();
1056 return parameters_number;
1067 Tensor<type, 1> parameters(parameters_number);
1069 const Index trainable_layers_number = get_trainable_layers_number();
1075 for(Index i = 0; i < trainable_layers_number; i++)
1077 const Tensor<type, 1> layer_parameters = trainable_layers_pointers(i)->get_parameters();
1079 for(Index j = 0; j < layer_parameters.size(); j++)
1081 parameters(j + position) = layer_parameters(j);
1084 position += layer_parameters.size();
1091Tensor<Index, 1> NeuralNetwork::get_trainable_layers_parameters_numbers()
const
1093 const Index trainable_layers_number = get_trainable_layers_number();
1097 Tensor<Index, 1> trainable_layers_parameters_number(trainable_layers_number);
1099 for(Index i = 0; i < trainable_layers_number; i++)
1101 trainable_layers_parameters_number[i] = trainable_layers_pointers[i]->get_parameters_number();
1104 return trainable_layers_parameters_number;
1108Tensor<Tensor<type, 1>, 1> NeuralNetwork::get_trainable_layers_parameters(
const Tensor<type, 1>& parameters)
const
1110 const Index trainable_layers_number = get_trainable_layers_number();
1112 const Tensor<Index, 1> trainable_layers_parameters_number = get_trainable_layers_parameters_numbers();
1114 Tensor<Tensor<type, 1>, 1> trainable_layers_parameters(trainable_layers_number);
1118 for(Index i = 0; i < trainable_layers_number; i++)
1121 trainable_layers_parameters(i).resize(trainable_layers_parameters_number(i));
1123 trainable_layers_parameters(i) = parameters.slice(Eigen::array<Eigen::Index, 1>({index}), Eigen::array<Eigen::Index, 1>({trainable_layers_parameters_number(i)}));
1125 index += trainable_layers_parameters_number(i);
1129 return trainable_layers_parameters;
1140 const Index size = new_parameters.size();
1144 if(size < parameters_number)
1146 ostringstream buffer;
1148 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
1149 <<
"void set_parameters(const Tensor<type, 1>&) method.\n"
1150 <<
"Size (" << size <<
") must be grater or equal to number of parameters (" << parameters_number <<
").\n";
1152 throw logic_error(buffer.str());
1157 const Index trainable_layers_number = get_trainable_layers_number();
1161 const Tensor<Index, 1> trainable_layers_parameters_numbers = get_trainable_layers_parameters_numbers();
1165 for(Index i = 0; i < trainable_layers_number; i++)
1167 trainable_layers_pointers(i)->set_parameters(new_parameters, index);
1169 index += trainable_layers_parameters_numbers(i);
1194Tensor<Index, 1> NeuralNetwork::get_layers_neurons_numbers()
const
1203 return layers_neurons_number;
1207Index NeuralNetwork::get_trainable_layers_number()
const
1213 for(Index i = 0; i < layers_number; i++)
1227Index NeuralNetwork::get_perceptron_layers_number()
const
1233 for(Index i = 0; i < layers_number; i++)
1245Index NeuralNetwork::get_probabilistic_layers_number()
const
1251 for(Index i = 0; i < layers_number; i++)
1263Index NeuralNetwork::get_long_short_term_memory_layers_number()
const
1269 for(Index i = 0; i < layers_number; i++)
1271 if(
layers_pointers(i)->get_type() == Layer::Type::LongShortTermMemory)
1280Index NeuralNetwork::get_recurrent_layers_number()
const
1286 for(Index i = 0; i < layers_number; i++)
1301 const Index trainable_layers_number = get_trainable_layers_number();
1305 for(Index i = 0; i < trainable_layers_number; i++)
1307 trainable_layers_pointers[i]->set_parameters_constant(value);
1319 const Index trainable_layers_number = get_trainable_layers_number();
1323 for(Index i = 0; i < trainable_layers_number; i++)
1325 trainable_layers_pointers[i]->set_parameters_random();
1336 const Tensor<type, 0> parameters_norm = parameters.square().sum().sqrt();
1338 return parameters_norm(0);
1349 if(perturbation < 0)
1351 ostringstream buffer;
1353 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
1354 <<
"void perturbate_parameters(const type&) method.\n"
1355 <<
"Perturbation must be equal or greater than 0.\n";
1357 throw logic_error(buffer.str());
1364 for (Index i = 0; i < parameters.size(); i++) parameters(i) += perturbation;
1379 const Index trainable_layers_number = trainable_layers_pointers.size();
1381 if(trainable_layers_pointers(0)->get_type() == Layer::Type::Convolutional)
1383 trainable_layers_pointers(0)->forward_propagate(batch.inputs_4d, forward_propagation.layers(0));
1387 trainable_layers_pointers(0)->forward_propagate(batch.inputs_2d, forward_propagation.layers(0));
1390 for(Index i = 1; i < trainable_layers_number; i++)
1392 switch(trainable_layers_pointers(i-1)->get_type())
1394 case Layer::Type::Perceptron:
1396 trainable_layers_pointers(i)
1398 forward_propagation.layers(i));
1401 case Layer::Type::Probabilistic:
1403 trainable_layers_pointers(i)
1405 forward_propagation.layers(i));
1408 case Layer::Type::Recurrent:
1410 trainable_layers_pointers(i)
1412 forward_propagation.layers(i));
1416 case Layer::Type::LongShortTermMemory:
1418 trainable_layers_pointers(i)
1420 forward_propagation.layers(i));
1423 case Layer::Type::Pooling:
1427 case Layer::Type::Convolutional:
1443 Tensor<type, 1>& parameters,
1448 const Index trainable_layers_number = trainable_layers_pointers.size();
1450 const Index parameters_number = trainable_layers_pointers(0)->get_parameters_number();
1452 const TensorMap<Tensor<type, 1>> potential_parameters(parameters.data(), parameters_number);
1454 if(trainable_layers_pointers(0)->get_type() == Layer::Type::Convolutional)
1456 trainable_layers_pointers(0)->forward_propagate(batch.inputs_4d, potential_parameters, forward_propagation.layers(0));
1460 trainable_layers_pointers(0)->forward_propagate(batch.inputs_2d, potential_parameters, forward_propagation.layers(0));
1463 Index index = parameters_number;
1465 for(Index i = 1; i < trainable_layers_number; i++)
1467 const Index parameters_number = trainable_layers_pointers(i)->get_parameters_number();
1469 const TensorMap<Tensor<type, 1>> potential_parameters(parameters.data() + index, parameters_number);
1471 switch(trainable_layers_pointers(i-1)->get_type())
1473 case Layer::Type::Perceptron:
1475 trainable_layers_pointers(i)
1477 potential_parameters,
1478 forward_propagation.layers(i));
1482 case Layer::Type::Probabilistic:
1484 trainable_layers_pointers(i)
1486 potential_parameters,
1487 forward_propagation.layers(i));
1491 case Layer::Type::Recurrent:
1493 trainable_layers_pointers(i)
1495 potential_parameters,
1496 forward_propagation.layers(i));
1500 case Layer::Type::LongShortTermMemory:
1502 trainable_layers_pointers(i)
1504 potential_parameters,
1505 forward_propagation.layers(i));
1509 case Layer::Type::Convolutional:
1521 index += parameters_number;
1542 const Index inputs_dimensions_number = inputs.rank();
1544 if(inputs_dimensions_number != 2 && inputs_dimensions_number != 4)
1546 ostringstream buffer;
1548 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
1549 <<
"Tensor<type, 2> calculate_outputs(const Tensor<type, 2>&) const method.\n"
1550 <<
"Inputs dimensions number (" << inputs_dimensions_number <<
") must be 2 or 4.\n";
1552 throw logic_error(buffer.str());
1557 Tensor<type, 2> outputs;
1561 if(layers_number == 0)
return inputs;
1565 for(Index i = 1; i < layers_number; i++)
1578 const Index inputs_dimensions_number = inputs.rank();
1580 if(inputs_dimensions_number != 2 && inputs_dimensions_number != 4)
1582 ostringstream buffer;
1584 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
1585 <<
"Tensor<type, 2> calculate_outputs(const Tensor<type, 2>&) const method.\n"
1586 <<
"Inputs dimensions number (" << inputs_dimensions_number <<
") must be 2 or 4.\n";
1588 throw logic_error(buffer.str());
1593 Tensor<type, 4> outputs_4d;
1594 Tensor<type, 2> outputs, inputs_2d;
1598 if(layers_number == 0)
return inputs_2d;
1611 for(Index i = 1; i < layers_number; i++)
1644 const Tensor<type, 1>& point,
1645 const type& minimum,
1646 const type& maximum,
1647 const Index& points_number)
const
1651 Tensor<type, 2> directional_inputs(points_number, inputs_number);
1653 Tensor<type, 1> inputs(inputs_number);
1657 for(Index i = 0; i < points_number; i++)
1659 inputs(direction) = minimum + (maximum - minimum)*
static_cast<type
>(i)/
static_cast<type
>(points_number-1);
1661 for(Index j = 0; j < inputs_number; j++)
1663 directional_inputs(i,j) = inputs(j);
1667 return directional_inputs;
1676 const Index trainable_layers_number = get_trainable_layers_number();
1678 Tensor<string, 2> information(trainable_layers_number, 3);
1682 for(Index i = 0; i < trainable_layers_number; i++)
1684 information(i,0) = to_string(trainable_layers_pointers(i)->
get_inputs_number());
1685 information(i,1) = to_string(trainable_layers_pointers(i)->get_neurons_number());
1687 const string layer_type = trainable_layers_pointers(i)->get_type_string();
1689 if(layer_type ==
"Perceptron")
1709 const Index trainable_layers_number = get_trainable_layers_number();
1711 const Index perceptron_layers_number = get_perceptron_layers_number();
1713 Tensor<string, 2> information(perceptron_layers_number, 3);
1717 Index perceptron_layer_index = 0;
1719 for(Index i = 0; i < trainable_layers_number; i++)
1721 const string layer_type = trainable_layers_pointers(i)->get_type_string();
1723 if(layer_type ==
"Perceptron")
1725 information(perceptron_layer_index,0) = to_string(trainable_layers_pointers(i)->
get_inputs_number());
1726 information(perceptron_layer_index,1) = to_string(trainable_layers_pointers(i)->get_neurons_number());
1732 perceptron_layer_index++;
1744 const Index trainable_layers_number = get_trainable_layers_number();
1746 const Index probabilistic_layers_number = get_probabilistic_layers_number();
1748 Tensor<string, 2> information(probabilistic_layers_number, 3);
1752 Index probabilistic_layer_index = 0;
1754 for(Index i = 0; i < trainable_layers_number; i++)
1756 const string layer_type = trainable_layers_pointers(i)->get_type_string();
1758 if(layer_type ==
"Probabilistic")
1760 information(probabilistic_layer_index,0) = to_string(trainable_layers_pointers(i)->
get_inputs_number());
1761 information(probabilistic_layer_index,1) = to_string(trainable_layers_pointers(i)->get_neurons_number());
1767 probabilistic_layer_index++;
1780 ostringstream buffer;
1782 file_stream.OpenElement(
"NeuralNetwork");
1786 file_stream.OpenElement(
"Inputs");
1790 file_stream.OpenElement(
"InputsNumber");
1795 file_stream.
PushText(buffer.str().c_str());
1803 file_stream.OpenElement(
"Input");
1818 file_stream.OpenElement(
"Layers");
1822 file_stream.OpenElement(
"LayersTypes");
1832 file_stream.
PushText(buffer.str().c_str());
1849 file_stream.OpenElement(
"Outputs");
1855 file_stream.OpenElement(
"OutputsNumber");
1858 buffer << outputs_number;
1860 file_stream.
PushText(buffer.str().c_str());
1866 for(Index i = 0; i < outputs_number; i++)
1868 file_stream.OpenElement(
"Output");
1892 ostringstream buffer;
1898 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
1899 <<
"void from_XML(const tinyxml2::XMLDocument&) method.\n"
1900 <<
"Neural network element is nullptr.\n";
1902 throw logic_error(buffer.str());
1914 element_clone = element->DeepClone(&inputs_document);
1916 inputs_document.InsertFirstChild(element_clone);
1918 inputs_from_XML(inputs_document);
1931 element_clone = element->DeepClone(&layers_document);
1933 layers_document.InsertFirstChild(element_clone);
1935 layers_from_XML(layers_document);
1949 element_clone = element->DeepClone(&outputs_document);
1951 outputs_document.InsertFirstChild(element_clone);
1953 outputs_from_XML(outputs_document);
1964 const string new_display_string = element->GetText();
1970 catch(
const logic_error& e)
1972 cerr << e.what() << endl;
1981 ostringstream buffer;
1987 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
1988 <<
"void inputs_from_XML(const tinyxml2::XMLDocument&) method.\n"
1989 <<
"Inputs element is nullptr.\n";
1991 throw logic_error(buffer.str());
1996 const tinyxml2::XMLElement* inputs_number_element = root_element->FirstChildElement(
"InputsNumber");
1998 if(!inputs_number_element)
2000 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
2001 <<
"void inputs_from_XML(const tinyxml2::XMLDocument&) method.\n"
2002 <<
"Inputs number element is nullptr.\n";
2004 throw logic_error(buffer.str());
2007 Index new_inputs_number = 0;
2009 if(inputs_number_element->GetText())
2011 new_inputs_number =
static_cast<Index
>(atoi(inputs_number_element->GetText()));
2020 if(new_inputs_number > 0)
2022 for(Index i = 0; i < new_inputs_number; i++)
2025 start_element = input_element;
2027 if(input_element->Attribute(
"Index") != to_string(i+1))
2029 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
2030 <<
"void inputs_from_XML(const tinyxml2::XMLDocument&) method.\n"
2031 <<
"Input index number (" << i+1 <<
") does not match (" << input_element->Attribute(
"Item") <<
").\n";
2033 throw logic_error(buffer.str());
2036 if(!input_element->GetText())
2051 ostringstream buffer;
2057 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
2058 <<
"void layers_from_XML(const tinyxml2::XMLDocument&) method.\n"
2059 <<
"Layers element is nullptr.\n";
2061 throw logic_error(buffer.str());
2066 const tinyxml2::XMLElement* layers_types_element = root_element->FirstChildElement(
"LayersTypes");
2068 if(!layers_types_element)
2070 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
2071 <<
"void layers_from_XML(const tinyxml2::XMLDocument&) method.\n"
2072 <<
"Layers types element is nullptr.\n";
2074 throw logic_error(buffer.str());
2077 Tensor<string, 1> layers_types;
2079 if(layers_types_element->GetText())
2081 layers_types = get_tokens(layers_types_element->GetText(),
' ');
2088 for(Index i = 0; i < layers_types.size(); i++)
2090 if(layers_types(i) ==
"Scaling")
2092 ScalingLayer* scaling_layer =
new ScalingLayer();
2095 start_element = scaling_element;
2102 element_clone = scaling_element->DeepClone(&scaling_document);
2104 scaling_document.InsertFirstChild(element_clone);
2106 scaling_layer->from_XML(scaling_document);
2111 else if(layers_types(i) ==
"Convolutional")
2134 else if(layers_types(i) ==
"Perceptron")
2136 PerceptronLayer* perceptron_layer =
new PerceptronLayer();
2139 start_element = perceptron_element;
2141 if(perceptron_element)
2146 element_clone = perceptron_element->DeepClone(&perceptron_document);
2148 perceptron_document.InsertFirstChild(element_clone);
2150 perceptron_layer->from_XML(perceptron_document);
2155 else if(layers_types(i) ==
"Pooling")
2178 else if(layers_types(i) ==
"Probabilistic")
2180 ProbabilisticLayer* probabilistic_layer =
new ProbabilisticLayer();
2183 start_element = probabilistic_element;
2185 if(probabilistic_element)
2190 element_clone = probabilistic_element->DeepClone(&probabilistic_document);
2192 probabilistic_document.InsertFirstChild(element_clone);
2193 probabilistic_layer->from_XML(probabilistic_document);
2198 else if(layers_types(i) ==
"LongShortTermMemory")
2200 LongShortTermMemoryLayer* long_short_term_memory_layer =
new LongShortTermMemoryLayer();
2203 start_element = long_short_term_memory_element;
2205 if(long_short_term_memory_element)
2210 element_clone = long_short_term_memory_element->DeepClone(&long_short_term_memory_document);
2212 long_short_term_memory_document.InsertFirstChild(element_clone);
2214 long_short_term_memory_layer->from_XML(long_short_term_memory_document);
2217 add_layer(long_short_term_memory_layer);
2219 else if(layers_types(i) ==
"Recurrent")
2221 RecurrentLayer* recurrent_layer =
new RecurrentLayer();
2224 start_element = recurrent_element;
2226 if(recurrent_element)
2231 element_clone = recurrent_element->DeepClone(&recurrent_document);
2233 recurrent_document.InsertFirstChild(element_clone);
2235 recurrent_layer->from_XML(recurrent_document);
2240 else if(layers_types(i) ==
"Unscaling")
2242 UnscalingLayer* unscaling_layer =
new UnscalingLayer();
2245 start_element = unscaling_element;
2247 if(unscaling_element)
2252 element_clone = unscaling_element->DeepClone(&unscaling_document);
2254 unscaling_document.InsertFirstChild(element_clone);
2256 unscaling_layer->from_XML(unscaling_document);
2261 else if(layers_types(i) ==
"Bounding")
2263 BoundingLayer* bounding_layer =
new BoundingLayer();
2267 start_element = bounding_element;
2269 if(bounding_element)
2274 element_clone = bounding_element->DeepClone(&bounding_document);
2276 bounding_document.InsertFirstChild(element_clone);
2278 bounding_layer->from_XML(bounding_document);
2289 ostringstream buffer;
2295 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
2296 <<
"void outputs_from_XML(const tinyxml2::XMLDocument&) method.\n"
2297 <<
"Outputs element is nullptr.\n";
2299 throw logic_error(buffer.str());
2304 const tinyxml2::XMLElement* outputs_number_element = root_element->FirstChildElement(
"OutputsNumber");
2306 if(!outputs_number_element)
2308 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
2309 <<
"void inputs_from_XML(const tinyxml2::XMLDocument&) method.\n"
2310 <<
"Outputs number element is nullptr.\n";
2312 throw logic_error(buffer.str());
2315 Index new_outputs_number = 0;
2317 if(outputs_number_element->GetText())
2319 new_outputs_number =
static_cast<Index
>(atoi(outputs_number_element->GetText()));
2326 if(new_outputs_number > 0)
2330 for(Index i = 0; i < new_outputs_number; i++)
2333 start_element = output_element;
2335 if(output_element->Attribute(
"Index") != to_string(i+1))
2337 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
2338 <<
"void outputs_from_XML(const tinyxml2::XMLDocument&) method.\n"
2339 <<
"Output index number (" << i+1 <<
") does not match (" << output_element->Attribute(
"Item") <<
").\n";
2341 throw logic_error(buffer.str());
2344 if(!output_element->GetText())
2361 cout <<
"Neural network" << endl;
2365 cout <<
"Layers number: " << layers_number << endl;
2367 for(Index i = 0; i < layers_number; i++)
2369 cout <<
"Layer " << i+1 <<
": " <<
layers_pointers[i]->get_neurons_number()
2370 <<
" " <<
layers_pointers[i]->get_type_string() <<
" neurons" << endl;
2380 FILE * file = fopen(file_name.c_str(),
"w");
2395 ofstream file(file_name.c_str());
2399 ostringstream buffer;
2401 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
2402 <<
"void save_parameters(const string&) const method.\n"
2403 <<
"Cannot open parameters data file.\n";
2405 throw logic_error(buffer.str());
2410 file << parameters << endl;
2428 if(document.LoadFile(file_name.c_str()))
2430 ostringstream buffer;
2432 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
2433 <<
"void load(const string&) method.\n"
2434 <<
"Cannot load XML file " << file_name <<
".\n";
2436 throw logic_error(buffer.str());
2452 file.open(file_name.c_str(), ios::binary);
2456 ostringstream buffer;
2458 buffer <<
"OpenNN Exception: NeuralNetwork template.\n"
2459 <<
"void load_parameters_binary(const string&) method.\n"
2460 <<
"Cannot open binary file: " << file_name <<
"\n";
2462 throw logic_error(buffer.str());
2465 streamsize size =
sizeof(double);
2469 Tensor<type, 1> new_parameters(parameters_number);
2473 for(Index i = 0; i < parameters_number; i++)
2475 file.read(
reinterpret_cast<char*
>(&value), size);
2477 new_parameters(i) = value;
2491 const Tensor<string, 1> layers_names = get_layers_names();
2493 ostringstream buffer;
2495 buffer <<
"// Artificial Intelligence Techniques SL\t"<<endl;
2496 buffer <<
"// artelnics@artelnics.com\t"<<endl;
2497 buffer <<
"// "<<endl;
2498 buffer <<
"// Your model has been exported to this file." <<endl;
2499 buffer <<
"// You can manage it with the 'neural network' method.\t"<<endl;
2500 buffer <<
"// Example:"<<endl;
2501 buffer <<
"// "<<endl;
2502 buffer <<
"// \tvector<float> sample(n);\t"<<endl;
2503 buffer <<
"// \tsample[0] = 1;\t"<<endl;
2504 buffer <<
"// \tsample[1] = 2;\t"<<endl;
2505 buffer <<
"// \tsample[n] = 10;\t"<<endl;
2506 buffer <<
"// \tvector<float> outputs = neural_network(sample);"<<endl;
2507 buffer <<
"// "<<endl;
2508 buffer <<
"// Notice that only one sample is allowed as input. DataSetBatch of inputs are not yet implement,\t"<<endl;
2509 buffer <<
"// however you can loop through neural network function in order to get multiple outputs.\t"<<endl;
2512 buffer <<
"#include <vector>\n" << endl;
2514 buffer <<
"using namespace std;\n" << endl;
2519 Index timestep = long_short_term_memory_pointer->
get_timesteps();
2522 buffer <<
"class LSTMNetwork\n";
2523 buffer <<
"{\n" << endl;
2524 buffer <<
"public:\n" << endl;
2525 buffer <<
" LSTMNetwork()\n";
2527 buffer <<
" hidden_states.resize(" << neurons_number <<
");\n";
2528 buffer <<
" cell_states.resize(" << neurons_number <<
");\n";
2529 buffer <<
" }\n" << endl;
2530 buffer <<
" vector<vector<float>> neural_network_batch(const vector<vector<float>>& inputs)\n";
2532 buffer <<
" vector<vector<float>> outputs(inputs.size());\n" << endl;
2533 buffer <<
" for(size_t i = 0; i < inputs.size(); i++)\n";
2535 buffer <<
" if(i % " << timestep <<
" == 0)\n";
2537 buffer <<
" fill(hidden_states.begin(), hidden_states.end(), 0.0);\n";
2538 buffer <<
" fill(cell_states.begin(), cell_states.end(), 0.0);\n";
2539 buffer <<
" }\n" << endl;
2540 buffer <<
" outputs[i] = neural_network(inputs[i]);\n";
2541 buffer <<
" }\n" << endl;
2542 buffer <<
" return outputs;\n";
2543 buffer <<
" }\n" << endl << endl;
2544 buffer <<
"private:\n" << endl;
2545 buffer <<
" vector<float> hidden_states;\n";
2546 buffer <<
" vector<float> cell_states;\n" << endl << endl;
2549 for(Index i = 0; i < layers_number; i++)
2554 buffer <<
"vector<float> neural_network(const vector<float>& inputs)\n{" << endl;
2556 buffer <<
"\tvector<float> outputs;\n" << endl;
2558 if(layers_number > 0)
2560 buffer <<
"\toutputs = " << layers_names[0] <<
"(inputs);\n";
2563 for(Index i = 1; i < layers_number; i++)
2565 buffer <<
"\toutputs = " << layers_names[i] <<
"(outputs);\n";
2568 buffer <<
"\n\treturn outputs;\n}" << endl;
2572 buffer <<
"int main(){return 0;}" << endl;
2574 string expression = buffer.str();
2576 replace(expression,
"+-",
"-");
2577 replace(expression,
"-+",
"-");
2578 replace(expression,
"--",
"+");
2584string NeuralNetwork::write_expression()
const
2589 const Tensor<string, 1> layers_names = get_layers_names();
2591 Tensor<string, 1> outputs_names_vector;
2592 Tensor<string, 1> inputs_names_vector;
2595 Index layer_neurons_number;
2597 ostringstream buffer;
2599 for(Index i = 0; i < layers_number; i++)
2601 if(i == layers_number-1)
2604 buffer <<
layers_pointers[i]->write_expression(inputs_names_vector, outputs_names_vector) << endl;
2609 outputs_names_vector.resize(layer_neurons_number);
2611 for(Index j = 0; j < layer_neurons_number; j++)
2613 if(layers_names(i) ==
"scaling_layer")
2619 outputs_names_vector(j) = layers_names(i) +
"_output_" + to_string(j);
2622 buffer <<
layers_pointers[i]->write_expression(inputs_names_vector, outputs_names_vector) << endl;
2624 inputs_names_vector = outputs_names_vector;
2628 string expression = buffer.str();
2630 replace(expression,
"+-",
"-");
2642 const Tensor<string, 1> layers_names = get_layers_names();
2644 ostringstream buffer;
2646 buffer <<
"'''"<<endl;
2647 buffer <<
"Artificial Intelligence Techniques SL\t"<<endl;
2648 buffer <<
"artelnics@artelnics.com\t"<<endl;
2650 buffer <<
"Your model has been exported to this python file." <<endl;
2651 buffer <<
"You can manage it with the 'NeuralNetwork' class.\t"<<endl;
2652 buffer <<
"Example:"<<endl;
2654 buffer <<
"\tmodel = NeuralNetwork()\t"<<endl;
2655 buffer <<
"\tsample = [input_1, input_2, input_3, input_4, ...] \t"<<endl;
2656 buffer <<
"\toutputs = model.calculate_output(sample)"<<endl;
2658 buffer <<
"\tInputs Names: \t"<<endl;
2662 for(
int i = 0; i < inputs.dimension(0); i++)
2666 buffer <<
"\t" << to_string(1+i) +
" )" <<
"input_"+ to_string(1+i) << endl;
2670 buffer <<
"\t" << to_string(1+i) +
" )" << inputs[i] << endl;
2675 buffer <<
"You can predict with a batch of samples using calculate_batch_output method\t" <<endl;
2676 buffer <<
"IMPORTANT: input batch must be <class 'numpy.ndarray'> type\t" <<endl;
2677 buffer <<
"Example_1:\t" <<endl;
2678 buffer <<
"\tmodel = NeuralNetwork()\t"<<endl;
2679 buffer <<
"\tinput_batch = np.array([[1, 2], [4, 5]], np.int32)\t" <<endl;
2680 buffer <<
"\toutputs = model.calculate_batch_output(input_batch)"<<endl;
2681 buffer <<
"Example_2:\t" <<endl;
2682 buffer <<
"\tinput_batch = pd.DataFrame( {'col1': [1, 2], 'col2': [3, 4]})\t" <<endl;
2683 buffer <<
"\toutputs = model.calculate_batch_output(input_batch.values)"<<endl;
2684 buffer <<
"'''"<<endl;
2686 buffer <<
"import numpy as np\n" << endl;
2687 buffer <<
"class NeuralNetwork:\n " << endl;
2688 buffer <<
"\tdef __init__(self):\n " << endl;
2705 for(Index i = 0; i < layers_number; i++)
2710 buffer <<
"\tdef calculate_output(self, inputs):\n" << endl;
2714 for(Index i = 1; i < layers_number; i++)
2719 buffer <<
"\t\treturn output_" +
layers_pointers[layers_number-1]->get_name()<<endl;
2721 buffer <<
"\n\n\tdef calculate_batch_output(self, input_batch):\n" << endl;
2723 buffer <<
"\t\toutput = []\n" << endl;
2725 buffer <<
"\t\tfor i in range(input_batch.shape[0]):\n" << endl;
2729 buffer <<
"\t\t\tif(i%self.timestep==0):\n" << endl;
2736 buffer <<
"\t\t\tif(i%self.timestep==0):\n" << endl;
2744 buffer <<
"\t\t\tinputs = list(input_batch[i])\n" << endl;
2748 for(Index i = 1; i < layers_number; i++)
2753 buffer <<
"\t\t\toutput = np.append(output,output_" +
layers_pointers[layers_number-1]->get_name()+
", axis=0)\n"<< endl;
2755 buffer <<
"\t\treturn output"<<endl;
2757 string expression = buffer.str();
2759 replace(expression,
"+-",
"-");
2760 replace(expression,
"-+",
"-");
2761 replace(expression,
"--",
"+");
2772 ofstream file(file_name.c_str());
2776 ostringstream buffer;
2778 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
2779 <<
"void save_expression(const string&) method.\n"
2780 <<
"Cannot open expression text file.\n";
2782 throw logic_error(buffer.str());
2796 ofstream file(file_name.c_str());
2800 ostringstream buffer;
2802 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
2803 <<
"void save_expression_python(const string&) method.\n"
2804 <<
"Cannot open expression text file.\n";
2806 throw logic_error(buffer.str());
2823 ofstream file(file_name.c_str());
2827 ostringstream buffer;
2829 buffer <<
"OpenNN Exception: NeuralNetwork class.\n"
2830 <<
"void save_expression_python(const string&) method.\n"
2831 <<
"Cannot open expression text file.\n";
2833 throw logic_error(buffer.str());
2838 const Index outputs_number = get_outputs_number();
2839 const Index samples_number = inputs.dimension(0);
2841 for(Index i = 0; i < outputs_number; i++)
2850 for(Index i = 0; i < samples_number; i++)
2852 for(Index j = 0; j < outputs_number; j++)
2854 file << outputs(i,j);
2856 if(j != outputs_number-1) file <<
";";
2866Tensor<string, 1> NeuralNetwork::get_layers_names()
const
2870 Tensor<string, 1> layers_names(layers_number);
2872 for(Index i = 0; i < layers_number; i++)
2877 return layers_names;
2881Layer* NeuralNetwork::get_last_trainable_layer_pointer()
const
2887 const Index trainable_layers_number = get_trainable_layers_number();
2889 return trainable_layers_pointers(trainable_layers_number-1);
This class represents a layer of bounding neurons.
This abstract class represents the concept of a layer of neurons in OpenNN.
Type
This enumeration represents the possible types of layers.
string get_type_string() const
Takes the type of layer used by the model.
This class represents a layer of LSTM neurons.
Index get_neurons_number() const
Returns the size of the neurons vector.
Index get_timesteps() const
Returns the number of timesteps.
ScalingLayer * get_scaling_layer_pointer() const
Returns a pointer to the scaling layers object composing this neural network object.
string write_expression_c() const
Returns a string with the c function of the expression represented by the neural network.
Tensor< string, 1 > outputs_names
Names of ouputs.
Tensor< Layer *, 1 > get_trainable_layers_pointers() const
Returns a pointer to the trainable layers object composing this neural network object.
bool has_long_short_term_memory_layer() const
void load_parameters_binary(const string &)
bool has_scaling_layer() const
void set_parameters_constant(const type &)
Initializes all the neural and the independent parameters with a given value.
bool check_layer_type(const Layer::Type)
Tensor< Index, 1 > get_trainable_layers_indices() const
Returns a vector with the indices of the trainable layers.
const Tensor< string, 1 > & get_outputs_names() const
Returns a string vector with the names of the variables used as outputs.
type calculate_parameters_norm() const
Returns the norm of the vector of parameters.
const bool & get_display() const
Tensor< Layer *, 1 > layers_pointers
Layers.
Index get_inputs_number() const
Returns the number of inputs to the neural network.
LongShortTermMemoryLayer * get_long_short_term_memory_layer_pointer() const
Returns a pointer to the long short term memory layer of this neural network, if exits.
virtual void from_XML(const tinyxml2::XMLDocument &)
virtual void set_default()
Sets those members which are not pointer to their default values.
bool has_probabilistic_layer() const
virtual void load(const string &)
string write_expression_python() const
Returns a string with the python function of the expression represented by the neural network.
bool has_unscaling_layer() const
bool display
Display messages to screen.
Tensor< type, 2 > calculate_directional_inputs(const Index &, const Tensor< type, 1 > &, const type &, const type &, const Index &=101) const
bool has_recurrent_layer() const
bool has_convolutional_layer() const
bool has_bounding_layer() const
Tensor< Layer *, 1 > get_layers_pointers() const
Returns a pointer to the layers object composing this neural network object.
Index get_input_index(const string &) const
Tensor< string, 2 > get_information() const
const Tensor< string, 1 > & get_inputs_names() const
Returns a string vector with the names of the variables used as inputs.
void save_expression_python(const string &)
virtual ~NeuralNetwork()
Destructor.
void save_parameters(const string &) const
void forward_propagate(const DataSetBatch &, NeuralNetworkForwardPropagation &) const
Calculate forward propagation in neural network.
void save(const string &) const
Tensor< Index, 1 > get_architecture() const
void set_inputs_number(const Index &)
Index get_layers_number() const
string get_output_name(const Index &) const
Index get_output_index(const string &) const
Tensor< type, 2 > calculate_outputs(const Tensor< type, 2 > &)
void set_parameters(Tensor< type, 1 > &)
void save_expression_c(const string &)
Tensor< string, 2 > get_perceptron_layers_information() const
For each perceptron layer: inputs, neurons, activation function.
string get_input_name(const Index &) const
Tensor< string, 2 > get_probabilistic_layer_information() const
For each probabilistic layer: inputs, neurons, activation function.
void set_parameters_random()
BoundingLayer * get_bounding_layer_pointer() const
Returns a pointer to the bounding layers object composing this neural network object.
Tensor< string, 1 > inputs_names
Names of inputs.
void print() const
Prints to the screen the most important information about the neural network object.
UnscalingLayer * get_unscaling_layer_pointer() const
Returns a pointer to the unscaling layers object composing this neural network object.
void set_display(const bool &)
ProbabilisticLayer * get_probabilistic_layer_pointer() const
Returns a pointer to the first probabilistic layer composing this neural network.
void set_inputs_names(const Tensor< string, 1 > &)
virtual void write_XML(tinyxml2::XMLPrinter &) const
RecurrentLayer * get_recurrent_layer_pointer() const
Returns a pointer to the recurrent layer of this neural network, if exits.
Index get_parameters_number() const
void perturbate_parameters(const type &)
void save_outputs(const Tensor< type, 2 > &, const string &)
void set_outputs_names(const Tensor< string, 1 > &)
Tensor< type, 1 > get_parameters() const
This class represents a layer of perceptrons.
string write_activation_function() const
void set_activation_function(const ActivationFunction &)
Index get_neurons_number() const
Returns the number of neurons in the layer.
This class represents a layer of probabilistic neurons.
string write_activation_function() const
This class represents a recurrent layer of neurons.
This class represents a layer of scaling neurons.
This class represents a layer of unscaling neurons.
const XMLElement * NextSiblingElement(const char *name=nullptr) const
Get the next(right) sibling element of this node, with an optionally supplied name.
void PushText(const char *text, bool cdata=false)
Add a text node.
void PushAttribute(const char *name, const char *value)
If streaming, add an attribute to an open element.
virtual void CloseElement(bool compactMode=false)
If streaming, close the Element.
This structure contains the batches of the data set.
This structure contains information for the forward propagation of the LSTM layer.
This structure contains information for the forward propagation of the neural network.
This structure contains information for the forward propagation of the perceptron layer.
This structure contains information for the forward propagation of the probabilistic layer.
This structure contains information for the forward propagation of the recurrent layer.