loss_index.cpp

//   OpenNN: Open Neural Networks Library
//   www.opennn.net
//
//   L O S S   I N D E X   C L A S S
//
//   Artificial Intelligence Techniques SL
//   artelnics@artelnics.com
 
#include "loss_index.h"
 
namespace OpenNN
{
 
 
LossIndex::LossIndex()
    : neural_network_pointer(nullptr),
      data_set_pointer(nullptr)
{
    set_default();
}
 
 
 
LossIndex::LossIndex(NeuralNetwork* new_neural_network_pointer, DataSet* new_data_set_pointer)
    : neural_network_pointer(new_neural_network_pointer),
      data_set_pointer(new_data_set_pointer)
{
    set_default();
}
 
 
 
LossIndex::~LossIndex()
{
    delete non_blocking_thread_pool;
    delete thread_pool_device;
}
 
 
 
const type& LossIndex::get_regularization_weight() const
{
    return regularization_weight;
}
 
 
 
const bool& LossIndex::get_display() const
{
    return display;
}
 
 
 
bool LossIndex::has_neural_network() const
{
    if(neural_network_pointer)
    {
        return true;
    }
    else
    {
        return false;
    }
}
 
 
 
bool LossIndex::has_data_set() const
{
    if(data_set_pointer)
    {
        return true;
    }
    else
    {
        return false;
    }
}
 
 
 
LossIndex::RegularizationMethod LossIndex::get_regularization_method() const
{
    return regularization_method;
}
 
 
 
void LossIndex::set()
{
    neural_network_pointer = nullptr;
    data_set_pointer = nullptr;
 
    set_default();
}
 
 
 
void LossIndex::set(NeuralNetwork* new_neural_network_pointer)
{
    neural_network_pointer = new_neural_network_pointer;
    data_set_pointer = nullptr;
 
    set_default();
}
 
 
 
void LossIndex::set(DataSet* new_data_set_pointer)
{
    neural_network_pointer = nullptr;
    data_set_pointer = new_data_set_pointer;
 
    set_default();
}
 
 
 
void LossIndex::set(NeuralNetwork* new_neural_network_pointer, DataSet* new_data_set_pointer)
{
    neural_network_pointer = new_neural_network_pointer;
 
    data_set_pointer = new_data_set_pointer;
 
    set_default();
}
 
 
 
void LossIndex::set(const LossIndex& other_error_term)
{
    neural_network_pointer = other_error_term.neural_network_pointer;
 
    data_set_pointer = other_error_term.data_set_pointer;
 
    regularization_method = other_error_term.regularization_method;
 
    display = other_error_term.display;
}
 
 
void LossIndex::set_threads_number(const int& new_threads_number)
{
    if(non_blocking_thread_pool != nullptr) delete this->non_blocking_thread_pool;
    if(thread_pool_device != nullptr) delete this->thread_pool_device;
 
    non_blocking_thread_pool = new NonBlockingThreadPool(new_threads_number);
    thread_pool_device = new ThreadPoolDevice(non_blocking_thread_pool, new_threads_number);
}
 
 
 
void LossIndex::set_neural_network_pointer(NeuralNetwork* new_neural_network_pointer)
{
    neural_network_pointer = new_neural_network_pointer;
}
 
 
 
void LossIndex::set_data_set_pointer(DataSet* new_data_set_pointer)
{
    data_set_pointer = new_data_set_pointer;
}
 
 
 
void LossIndex::set_default()
{
    delete non_blocking_thread_pool;
    delete thread_pool_device;
 
    const int n = omp_get_max_threads();
 
    non_blocking_thread_pool = new NonBlockingThreadPool(n);
    thread_pool_device = new ThreadPoolDevice(non_blocking_thread_pool, n);
 
    regularization_method = RegularizationMethod::L2;
}
 
 
 
void LossIndex::set_regularization_method(const string& new_regularization_method)
{
    if(new_regularization_method == "L1_NORM")
    {
        set_regularization_method(RegularizationMethod::L1);
    }
    else if(new_regularization_method == "L2_NORM")
    {
        set_regularization_method(RegularizationMethod::L2);
    }
    else if(new_regularization_method == "NO_REGULARIZATION")
    {
        set_regularization_method(RegularizationMethod::NoRegularization);
    }
    else
    {
        ostringstream buffer;
 
        buffer << "OpenNN Exception: LossIndex class.\n"
               << "void set_regularization_method(const string&) const method.\n"
               << "Unknown regularization method: " << new_regularization_method << ".";
 
        throw logic_error(buffer.str());
    }
}
 
 
 
void LossIndex::set_regularization_method(const LossIndex::RegularizationMethod& new_regularization_method)
{
    regularization_method = new_regularization_method;
}
 
 
 
void LossIndex::set_regularization_weight(const type& new_regularization_weight)
{
    regularization_weight = new_regularization_weight;
}
 
 
 
void LossIndex::set_display(const bool& new_display)
{
    display = new_display;
}
 
 
 
bool LossIndex::has_selection() const
{
    if(data_set_pointer->get_selection_samples_number() != 0)
    {
        return true;
    }
    else
    {
        return false;
    }
}
 
 
 
void LossIndex::check() const
{
    ostringstream buffer;
 
    if(!neural_network_pointer)
    {
        buffer << "OpenNN Exception: LossIndex class.\n"
               << "void check() const.\n"
               << "Pointer to neural network is nullptr.\n";
 
        throw logic_error(buffer.str());
    }
 
    // Data set
 
    if(!data_set_pointer)
    {
        buffer << "OpenNN Exception: LossIndex class.\n"
               << "void check() const method.\n"
               << "Pointer to data set is nullptr.\n";
 
        throw logic_error(buffer.str());
    }
}
 
 
void LossIndex::calculate_errors(const DataSetBatch& batch,
                                 const NeuralNetworkForwardPropagation& forward_propagation,
                                 LossIndexBackPropagation& back_propagation) const
{
    const Index trainable_layers_number = neural_network_pointer->get_trainable_layers_number();
 
    switch(forward_propagation.layers(trainable_layers_number-1)->layer_pointer->get_type())
    {
    case Layer::Type::Perceptron:
    {
        back_propagation.errors.device(*thread_pool_device) =
                static_cast<PerceptronLayerForwardPropagation*>(forward_propagation.layers(trainable_layers_number-1))->activations -
                batch.targets_2d;
    }
        break;
 
    case Layer::Type::Probabilistic:
    {
        back_propagation.errors.device(*thread_pool_device) =
                static_cast<ProbabilisticLayerForwardPropagation*>(forward_propagation.layers(trainable_layers_number-1))->activations -
                batch.targets_2d;
    }
        break;
 
    case Layer::Type::Recurrent:
    {
        back_propagation.errors.device(*thread_pool_device) =
                static_cast<RecurrentLayerForwardPropagation*>(forward_propagation.layers(trainable_layers_number-1))->activations -
                batch.targets_2d;
    }
        break;
 
    case Layer::Type::LongShortTermMemory:
    {
        back_propagation.errors.device(*thread_pool_device) =
                static_cast<LongShortTermMemoryLayerForwardPropagation*>(forward_propagation.layers(trainable_layers_number-1))->activations -
                batch.targets_2d;
    }
        break;
 
    default: break;
    }
}
 
 
void LossIndex::calculate_errors_lm(const DataSetBatch& batch,
                                 const NeuralNetworkForwardPropagation & neural_network_forward_propagation,
                                 LossIndexBackPropagationLM & loss_index_back_propagation) const
{
    const Index trainable_layers_number = neural_network_pointer->get_trainable_layers_number();
 
    switch(neural_network_forward_propagation.layers(trainable_layers_number-1)->layer_pointer->get_type())
    {
    case Layer::Type::Perceptron:
 
        loss_index_back_propagation.errors.device(*thread_pool_device) =
                static_cast<PerceptronLayerForwardPropagation*>(neural_network_forward_propagation.layers(trainable_layers_number-1))->activations -
                batch.targets_2d;
 
        break;
 
    case Layer::Type::Probabilistic:
 
        loss_index_back_propagation.errors.device(*thread_pool_device) =
                static_cast<ProbabilisticLayerForwardPropagation*>(neural_network_forward_propagation.layers(trainable_layers_number-1))->activations -
                batch.targets_2d;
 
        break;
 
    case Layer::Type::Recurrent:
 
        loss_index_back_propagation.errors.device(*thread_pool_device) =
                static_cast<RecurrentLayerForwardPropagation*>(neural_network_forward_propagation.layers(trainable_layers_number-1))->activations -
                batch.targets_2d;
 
        break;
 
    case Layer::Type::LongShortTermMemory:
 
        loss_index_back_propagation.errors.device(*thread_pool_device) =
                static_cast<LongShortTermMemoryLayerForwardPropagation*>(neural_network_forward_propagation.layers(trainable_layers_number-1))->activations -
                batch.targets_2d;
 
        break;
 
    default: break;
    }
}
 
 
void LossIndex::calculate_squared_errors_lm(const DataSetBatch& ,
                                            const NeuralNetworkForwardPropagation& ,
                                            LossIndexBackPropagationLM& loss_index_back_propagation_lm) const
{
    loss_index_back_propagation_lm.squared_errors.device(*thread_pool_device) = loss_index_back_propagation_lm.errors.square().sum(rows_sum).sqrt();
}
 
 
void LossIndex::back_propagate(const DataSetBatch& batch,
                               NeuralNetworkForwardPropagation& forward_propagation,
                               LossIndexBackPropagation& back_propagation) const
{
    // Loss index
 
    calculate_errors(batch, forward_propagation, back_propagation);
 
    calculate_error(batch, forward_propagation, back_propagation);
 
    calculate_layers_delta(batch, forward_propagation, back_propagation);
 
    calculate_error_gradient(batch, forward_propagation, back_propagation);
 
    // Loss
 
    back_propagation.loss = back_propagation.error;
 
    // Regularization
 
    if(regularization_method != RegularizationMethod::NoRegularization)
    {
        const type regularization = calculate_regularization(back_propagation.parameters);
 
        back_propagation.loss += regularization_weight*regularization;
 
        calculate_regularization_gradient(back_propagation.parameters, back_propagation.regularization_gradient);
 
        back_propagation.gradient.device(*thread_pool_device) += regularization_weight*back_propagation.regularization_gradient;
    }
}
 
 
 
void LossIndex::back_propagate_lm(const DataSetBatch& batch,
                                  NeuralNetworkForwardPropagation& forward_propagation,
                                  LossIndexBackPropagationLM& loss_index_back_propagation_lm) const
{
    calculate_errors_lm(batch, forward_propagation, loss_index_back_propagation_lm);
 
    calculate_squared_errors_lm(batch, forward_propagation, loss_index_back_propagation_lm);
 
    calculate_error_lm(batch, forward_propagation, loss_index_back_propagation_lm);
 
    calculate_layers_delta_lm(batch, forward_propagation, loss_index_back_propagation_lm);
 
    calculate_squared_errors_jacobian_lm(batch, forward_propagation, loss_index_back_propagation_lm);
 
    calculate_error_gradient_lm(batch, loss_index_back_propagation_lm);
 
    calculate_error_hessian_lm(batch, loss_index_back_propagation_lm);
 
    // Loss
 
    loss_index_back_propagation_lm.loss = loss_index_back_propagation_lm.error;
 
    // Regularization
 
    if(regularization_method != RegularizationMethod::NoRegularization)
    {
        const type regularization = calculate_regularization(loss_index_back_propagation_lm.parameters);
 
        loss_index_back_propagation_lm.loss += regularization_weight*regularization;
 
        calculate_regularization_gradient(loss_index_back_propagation_lm.parameters, loss_index_back_propagation_lm.regularization_gradient);
 
        loss_index_back_propagation_lm.gradient.device(*thread_pool_device) += regularization_weight*loss_index_back_propagation_lm.regularization_gradient;
 
        calculate_regularization_hessian(loss_index_back_propagation_lm.parameters, loss_index_back_propagation_lm.regularization_hessian);
 
        loss_index_back_propagation_lm.hessian += regularization_weight*loss_index_back_propagation_lm.regularization_hessian;
    }
}
 
 
 
void LossIndex::calculate_squared_errors_jacobian_lm(const DataSetBatch& batch,
                                                  NeuralNetworkForwardPropagation& forward_propagation,
                                                  LossIndexBackPropagationLM& loss_index_back_propagation_lm) const
{
    const Index trainable_layers_number = neural_network_pointer->get_trainable_layers_number();
 
    loss_index_back_propagation_lm.squared_errors_jacobian.setZero();
 
    const Index batch_samples_number = batch.get_samples_number();
 
    Index mem_index = 0;
 
    Tensor<Layer*, 1> trainable_layers_pointers = neural_network_pointer->get_trainable_layers_pointers();
 
    const Tensor<Index, 1> trainable_layers_parameters_number = neural_network_pointer->get_trainable_layers_parameters_numbers();
 
    // Layer 0
 
    if(trainable_layers_pointers(0)->get_type() != Layer::Type::Perceptron && trainable_layers_pointers(0)->get_type() != Layer::Type::Probabilistic)
    {
        ostringstream buffer;
 
        buffer << "OpenNN Exception: LossIndex class.\n"
               << "void calculate_squared_errors_jacobian_lm(const DataSetBatch&, NeuralNetworkForwardPropagation&, LossIndexBackPropagationLM&) const method "
               << "Levenberg - Marquardt algorithm can only be used with Perceptron and Probabilistic layers.\n";
 
        throw logic_error(buffer.str());
    }
    else
    {
        trainable_layers_pointers(0)->calculate_squared_errors_Jacobian_lm(batch.inputs_2d,
                                                                           forward_propagation.layers(0),
                                                                           loss_index_back_propagation_lm.neural_network.layers(0));
 
        trainable_layers_pointers(0)->insert_squared_errors_Jacobian_lm(loss_index_back_propagation_lm.neural_network.layers(0),
                                                                        mem_index,
                                                                        loss_index_back_propagation_lm.squared_errors_jacobian);
 
        mem_index += trainable_layers_parameters_number(0)*batch_samples_number;
    }
 
    // Rest of the layers
 
    for(Index i = 1; i < trainable_layers_number; i++)
    {
        switch (forward_propagation.layers(i-1)->layer_pointer->get_type())
        {
        case Layer::Type::Perceptron:
        {
            PerceptronLayerForwardPropagation* perceptron_layer_forward_propagation
                    = static_cast<PerceptronLayerForwardPropagation*>(forward_propagation.layers(i-1));
 
            trainable_layers_pointers(i)->calculate_squared_errors_Jacobian_lm(perceptron_layer_forward_propagation->activations,
                                                                               forward_propagation.layers(i),
                                                                               loss_index_back_propagation_lm.neural_network.layers(i));
 
            trainable_layers_pointers(i)->insert_squared_errors_Jacobian_lm(loss_index_back_propagation_lm.neural_network.layers(i),
                                                                            mem_index,
                                                                            loss_index_back_propagation_lm.squared_errors_jacobian);
 
            mem_index += trainable_layers_parameters_number(i)*batch_samples_number;
        }
            break;
 
        case Layer::Type::Probabilistic:
        {
            ostringstream buffer;
 
            buffer << "OpenNN Exception: LossIndex class.\n"
                   << "void calculate_squared_errors_jacobian_lm(const DataSetBatch&, NeuralNetworkForwardPropagation&, LossIndexBackPropagationLM&) const method "
                   << "Probabilistic layer can only occupy the last position in the neural network. Please, check network structure.\n";
 
            throw logic_error(buffer.str());
        }
 
        default:
        {
            ostringstream buffer;
 
            buffer << "OpenNN Exception: LossIndex class.\n"
                   << "void calculate_squared_errors_jacobian_lm(const DataSetBatch&, NeuralNetworkForwardPropagation&, LossIndexBackPropagationLM&) const method "
                   << "Levenberg - Marquardt algorithm can only be used with Perceptron and Probabilistic layers.\n";
 
            throw logic_error(buffer.str());
        }
        }
    }
}
 
 
void LossIndex::calculate_error_gradient_lm(const DataSetBatch& batch,
                                      LossIndexBackPropagationLM& loss_index_back_propagation_lm) const
{
    loss_index_back_propagation_lm.gradient.device(*thread_pool_device)
            = loss_index_back_propagation_lm.squared_errors_jacobian.contract(loss_index_back_propagation_lm.squared_errors, AT_B);
}
 
 
 
string LossIndex::get_error_type() const
{
    return "USER_ERROR_TERM";
}
 
 
 
string LossIndex::get_error_type_text() const
{
    return "USER_ERROR_TERM";
}
 
 
 
string LossIndex::write_regularization_method() const
{
    switch(regularization_method)
    {
    case RegularizationMethod::NoRegularization:
        return "NO_REGULARIZATION";
 
    case RegularizationMethod::L1:
        return "L1_NORM";
 
    case RegularizationMethod::L2:
        return "L2_NORM";
    }
 
    return string();
}
 
 
 
type LossIndex::calculate_regularization(const Tensor<type, 1>& parameters) const
{
    switch(regularization_method)
    {
        case RegularizationMethod::NoRegularization: return type(0);
 
        case RegularizationMethod::L1: return l1_norm(thread_pool_device, parameters);
 
        case RegularizationMethod::L2: return l2_norm(thread_pool_device, parameters);
    }
 
    return type(0);
}
 
 
 
void LossIndex::calculate_regularization_gradient(const Tensor<type, 1>& parameters, Tensor<type, 1>& regularization_gradient) const
{
    switch(regularization_method)
    {
    case RegularizationMethod::L1: l1_norm_gradient(thread_pool_device, parameters, regularization_gradient); return;
 
    case RegularizationMethod::L2: l2_norm_gradient(thread_pool_device, parameters, regularization_gradient); return;
 
    default: return;
    }
}
 
 
 
void LossIndex::calculate_regularization_hessian(const Tensor<type, 1>& parameters, Tensor<type, 2>& regularization_hessian) const
{
    switch(regularization_method)
    {
    case RegularizationMethod::L1: l1_norm_hessian(thread_pool_device, parameters, regularization_hessian); return;
 
    case RegularizationMethod::L2: l2_norm_hessian(thread_pool_device, parameters, regularization_hessian); return;
 
    default: return;
    }
}
 
 
void LossIndex::calculate_layers_delta(const DataSetBatch& batch,
                                       NeuralNetworkForwardPropagation& forward_propagation,
                                       LossIndexBackPropagation& back_propagation) const
{
    const Index trainable_layers_number = neural_network_pointer->get_trainable_layers_number();
 
    if(trainable_layers_number == 0) return;
 
    const Tensor<Layer*, 1> trainable_layers_pointers = neural_network_pointer->get_trainable_layers_pointers();
 
    // Output layer
 
    calculate_output_delta(batch,
                           forward_propagation,
                           back_propagation);
 
    // Hidden layers
 
    for(Index i = static_cast<Index>(trainable_layers_number)-2; i >= 0; i--)
    {
        trainable_layers_pointers(i)
                ->calculate_hidden_delta(forward_propagation.layers(i+1),
                                         back_propagation.neural_network.layers(i+1),
                                         back_propagation.neural_network.layers(i));
    }
}
 
 
void LossIndex::calculate_layers_delta_lm(const DataSetBatch& batch,
                                          NeuralNetworkForwardPropagation& forward_propagation,
                                          LossIndexBackPropagationLM& back_propagation) const
{
    const Index trainable_layers_number = neural_network_pointer->get_trainable_layers_number();
 
    if(trainable_layers_number == 0) return;
 
    const Tensor<Layer*, 1> trainable_layers_pointers = neural_network_pointer->get_trainable_layers_pointers();
 
    // Output layer
 
    calculate_output_delta_lm(batch,
                              forward_propagation,
                              back_propagation);
 
    // Hidden layers
 
    for(Index i = static_cast<Index>(trainable_layers_number)-2; i >= 0; i--)
    {
        trainable_layers_pointers(i)
                ->calculate_hidden_delta_lm(forward_propagation.layers(i+1),
                                            back_propagation.neural_network.layers(i+1),
                                            back_propagation.neural_network.layers(i));
    }
}
 
 
void LossIndex::calculate_error_gradient(const DataSetBatch& batch,
                                         const NeuralNetworkForwardPropagation& forward_propagation,
                                         LossIndexBackPropagation& back_propagation) const
{
    #ifdef OPENNN_DEBUG
 
    check();
 
    #endif
 
    const Tensor<Layer*, 1> trainable_layers_pointers = neural_network_pointer->get_trainable_layers_pointers();
 
    const Index trainable_layers_number = trainable_layers_pointers.size();
 
    const Tensor<Index, 1> trainable_layers_parameters_number
            = neural_network_pointer->get_trainable_layers_parameters_numbers();
 
    if(trainable_layers_pointers(0)->get_type() == Layer::Type::Convolutional)
    {
        trainable_layers_pointers(0)->calculate_error_gradient(batch.inputs_4d,
                                                               forward_propagation.layers(0),
                                                               back_propagation.neural_network.layers(0));
    }
    else
    {
        trainable_layers_pointers(0)->calculate_error_gradient(batch.inputs_2d,
                                                               forward_propagation.layers(0),
                                                               back_propagation.neural_network.layers(0));
    }
 
    Index index = 0;
 
    trainable_layers_pointers(0)->insert_gradient(back_propagation.neural_network.layers(0),
                                                  index,
                                                  back_propagation.gradient);
 
    index += trainable_layers_parameters_number(0);
 
    for(Index i = 1; i < trainable_layers_number; i++)
    {
        switch(forward_propagation.layers(i-1)->layer_pointer->get_type())
        {
        case Layer::Type::Perceptron:
        {
            PerceptronLayerForwardPropagation* perceptron_layer_forward_propagation
                    = static_cast<PerceptronLayerForwardPropagation*>(forward_propagation.layers(i-1));
 
            trainable_layers_pointers(i)->
                    calculate_error_gradient(perceptron_layer_forward_propagation->activations,
                                             forward_propagation.layers(i),
                                             back_propagation.neural_network.layers(i));
        }
            break;
 
        case Layer::Type::Probabilistic:
        {
            ProbabilisticLayerForwardPropagation* probabilistic_layer_forward_propagation
                    = static_cast<ProbabilisticLayerForwardPropagation*>(forward_propagation.layers(i-1));
 
            trainable_layers_pointers(i)->
                    calculate_error_gradient(probabilistic_layer_forward_propagation->activations,
                                             forward_propagation.layers(i),
                                             back_propagation.neural_network.layers(i));
        }
            break;
 
        case Layer::Type::Recurrent:
        {
            RecurrentLayerForwardPropagation* recurrent_layer_forward_propagation
                    = static_cast<RecurrentLayerForwardPropagation*>(forward_propagation.layers(i-1));
 
            trainable_layers_pointers(i)->
                    calculate_error_gradient(recurrent_layer_forward_propagation->activations,
                                             forward_propagation.layers(i),
                                             back_propagation.neural_network.layers(i));
        }
            break;
 
        case Layer::Type::LongShortTermMemory:
        {
            LongShortTermMemoryLayerForwardPropagation* long_short_term_memory_layer_forward_propagation
                    = static_cast<LongShortTermMemoryLayerForwardPropagation*>(forward_propagation.layers(i-1));
 
            trainable_layers_pointers(i)->
                    calculate_error_gradient(long_short_term_memory_layer_forward_propagation->activations,
                                             forward_propagation.layers(i),
                                             back_propagation.neural_network.layers(i));
        }
            break;
 
        case Layer::Type::Convolutional:
        {
            // @todo
 
            //trainable_layers_pointers(i)->
            //        calculate_error_gradient(static_cast<ConvolutionalLayer::ConvolutionalLayerForwardPropagation*>(forward_propagation.layers(i-1))->activations,
            //                                 forward_propagation.layers(i),
            //                                 back_propagation.neural_network.layers(i));
        }
            break;
 
        case Layer::Type::Pooling:
        {
            // @todo
        }
            break;
 
        default: break;
 
        }
 
        trainable_layers_pointers(i)->insert_gradient(back_propagation.neural_network.layers(i),
                                                      index,
                                                      back_propagation.gradient);
 
        index += trainable_layers_parameters_number(i);
    }
}
 
 
 
 
void LossIndex::write_XML(tinyxml2::XMLPrinter& file_stream) const
{
    ostringstream buffer;
 
    file_stream.OpenElement("LossIndex");
 
    file_stream.CloseElement();
}
 
 
void LossIndex::regularization_from_XML(const tinyxml2::XMLDocument& document)
{
    const tinyxml2::XMLElement* root_element = document.FirstChildElement("Regularization");
 
    if(!root_element)
    {
        ostringstream buffer;
 
        buffer << "OpenNN Exception: LossIndex class.\n"
               << "void from_XML(const tinyxml2::XMLDocument&) method.\n"
               << "Regularization tag not found.\n";
 
        throw logic_error(buffer.str());
    }
 
    const string new_regularization_method = root_element->Attribute("Type");
 
    set_regularization_method(new_regularization_method);
 
    const tinyxml2::XMLElement* element = root_element->FirstChildElement("RegularizationWeight");
 
    if(element)
    {
        const type new_regularization_weight = static_cast<type>(atof(element->GetText()));
 
        try
        {
            set_regularization_weight(new_regularization_weight);
        }
        catch(const logic_error& e)
        {
            cerr << e.what() << endl;
        }
    }
}
 
 
void LossIndex::write_regularization_XML(tinyxml2::XMLPrinter& file_stream) const
{
    ostringstream buffer;
 
    file_stream.OpenElement("Regularization");
 
    // Regularization method
 
    switch(regularization_method)
    {
    case RegularizationMethod::L1:
    {
        file_stream.PushAttribute("Type", "L1_NORM");
    }
    break;
 
    case RegularizationMethod::L2:
    {
        file_stream.PushAttribute("Type", "L2_NORM");
    }
    break;
 
    case RegularizationMethod::NoRegularization:
    {
        file_stream.PushAttribute("Type", "NO_REGULARIZATION");
    }
    break;
    }
 
    // Regularization weight
 
    file_stream.OpenElement("RegularizationWeight");
 
    buffer.str("");
    buffer << regularization_weight;
 
    file_stream.PushText(buffer.str().c_str());
 
    // Close regularization weight
 
    file_stream.CloseElement();
 
    // Close regularization
 
    file_stream.CloseElement();
}
 
 
 
void LossIndex::from_XML(const tinyxml2::XMLDocument& document)
{
    const tinyxml2::XMLElement* root_element = document.FirstChildElement("MeanSquaredError");
 
    if(!root_element)
    {
        ostringstream buffer;
 
        buffer << "OpenNN Exception: MeanSquaredError class.\n"
               << "void from_XML(const tinyxml2::XMLDocument&) method.\n"
               << "Mean squared element is nullptr.\n";
 
        throw logic_error(buffer.str());
    }
 
    // Regularization
 
    tinyxml2::XMLDocument regularization_document;
    tinyxml2::XMLNode* element_clone;
 
    const tinyxml2::XMLElement* regularization_element = root_element->FirstChildElement("Regularization");
 
    element_clone = regularization_element->DeepClone(&regularization_document);
 
    regularization_document.InsertFirstChild(element_clone);
 
    regularization_from_XML(regularization_document);
}
 
 
 
LossIndexBackPropagation::~LossIndexBackPropagation()
{
}
 
 
Tensor<type, 1> LossIndex::calculate_gradient_numerical_differentiation()
{
    const Index samples_number = data_set_pointer->get_training_samples_number();
 
    const Tensor<Index, 1> samples_indices = data_set_pointer->get_training_samples_indices();
    const Tensor<Index, 1> input_variables_indices = data_set_pointer->get_input_variables_indices();
    const Tensor<Index, 1> target_variables_indices = data_set_pointer->get_target_variables_indices();
 
    DataSetBatch batch(samples_number, data_set_pointer);
    batch.fill(samples_indices, input_variables_indices, target_variables_indices);
 
    NeuralNetworkForwardPropagation forward_propagation(samples_number, neural_network_pointer);
 
    LossIndexBackPropagation back_propagation(samples_number, this);
 
    const Tensor<type, 1> parameters = neural_network_pointer->get_parameters();
 
    const Index parameters_number = parameters.size();
 
    type h;
    Tensor<type, 1> parameters_forward(parameters);
    Tensor<type, 1> parameters_backward(parameters);
 
    type error_forward;
    type error_backward;
 
    Tensor<type, 1> gradient_numerical_differentiation(parameters_number);
 
    for(Index i = 0; i < parameters_number; i++)
    {
       h = calculate_h(parameters(i));
 
       parameters_forward(i) += h;
 
       neural_network_pointer->forward_propagate(batch, parameters_forward, forward_propagation);
 
       calculate_errors(batch, forward_propagation, back_propagation);
 
       calculate_error(batch, forward_propagation, back_propagation);
 
       error_forward = back_propagation.error;
 
       parameters_forward(i) -= h;
 
       parameters_backward(i) -= h;
 
       neural_network_pointer->forward_propagate(batch, parameters_backward, forward_propagation);
       calculate_errors(batch, forward_propagation, back_propagation);
       calculate_error(batch, forward_propagation, back_propagation);
       error_backward = back_propagation.error;
 
       parameters_backward(i) += h;
 
       gradient_numerical_differentiation(i) = (error_forward - error_backward)/(type(2)*h);
    }
 
    return gradient_numerical_differentiation;
}
 
 
Tensor<type, 2> LossIndex::calculate_jacobian_numerical_differentiation()
{
    LossIndexBackPropagationLM back_propagation_lm;
 
    const Index samples_number = data_set_pointer->get_training_samples_number();
 
    DataSetBatch batch(samples_number, data_set_pointer);
 
    const Tensor<Index, 1> samples_indices = data_set_pointer->get_training_samples_indices();
 
    const Tensor<Index, 1> input_variables_indices = data_set_pointer->get_input_variables_indices();
    const Tensor<Index, 1> target_variables_indices = data_set_pointer->get_target_variables_indices();
 
    batch.fill(samples_indices, input_variables_indices, target_variables_indices);
 
    NeuralNetworkForwardPropagation forward_propagation(samples_number, neural_network_pointer);
 
    LossIndexBackPropagation back_propagation(samples_number, this);
 
    Tensor<type, 1> parameters = neural_network_pointer->get_parameters();
 
    const Index parameters_number = parameters.size();
 
    back_propagation_lm.set(samples_number, this);
 
    neural_network_pointer->forward_propagate(batch, parameters, forward_propagation);
    calculate_errors_lm(batch, forward_propagation, back_propagation_lm);
    calculate_squared_errors_lm(batch, forward_propagation, back_propagation_lm);
 
    type h;
 
    Tensor<type, 1> parameters_forward(parameters);
    Tensor<type, 1> parameters_backward(parameters);
 
    Tensor<type, 1> error_terms_forward(parameters_number);
    Tensor<type, 1> error_terms_backward(parameters_number);
 
    Tensor<type, 2> jacobian(samples_number,parameters_number);
 
    for(Index j = 0; j < parameters_number; j++)
    {
        h = calculate_h(parameters(j));
 
        parameters_backward(j) -= h;
        neural_network_pointer->forward_propagate(batch, parameters_backward, forward_propagation);
        calculate_errors_lm(batch, forward_propagation, back_propagation_lm);
        calculate_squared_errors_lm(batch, forward_propagation, back_propagation_lm);
        error_terms_backward = back_propagation_lm.squared_errors;
        parameters_backward(j) += h;
 
        parameters_forward(j) += h;
        neural_network_pointer->forward_propagate(batch, parameters_forward, forward_propagation);
        calculate_errors_lm(batch, forward_propagation, back_propagation_lm);
        calculate_squared_errors_lm(batch, forward_propagation, back_propagation_lm);
        error_terms_forward = back_propagation_lm.squared_errors;
        parameters_forward(j) -= h;
 
        for(Index i = 0; i < samples_number; i++)
        {
            jacobian(i,j) = (error_terms_forward(i) - error_terms_backward(i))/(static_cast<type>(2.0)*h);
        }
    }
 
    return jacobian;
}
 
 
type LossIndex::calculate_eta() const
{
    const Index precision_digits = 6;
 
    return pow(static_cast<type>(10.0), static_cast<type>(-1.0*precision_digits));
}
 
 
 
type LossIndex::calculate_h(const type& x) const
{
    const type eta = calculate_eta();
 
    return sqrt(eta)*(static_cast<type>(1.0) + abs(x));
}
 
}
 
// OpenNN: Open Neural Networks Library.
// Copyright(C) 2005-2021 Artificial Intelligence Techniques, SL.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
 
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA