cross_entropy_error.cpp

//   OpenNN: Open Neural Networks Library
//   www.opennn.net
//
//   C R O S S   E N T R O P Y   E R R O R   C L A S S
//
//   Artificial Intelligence Techniques SL
//   artelnics@artelnics.com
 
#include "cross_entropy_error.h"
 
namespace OpenNN
{
 
 
CrossEntropyError::CrossEntropyError() : LossIndex()
{
}
 
 
 
CrossEntropyError::CrossEntropyError(NeuralNetwork* new_neural_network_pointer, DataSet* new_data_set_pointer)
    : LossIndex(new_neural_network_pointer, new_data_set_pointer)
{
}
 
 
 
CrossEntropyError::~CrossEntropyError()
{
}
 
 
// \brief CrossEntropyError::calculate_error.
// \param batch
// \param forward_propagation
// \param back_propagation
 
void CrossEntropyError::calculate_error(const DataSetBatch& batch,
                     const NeuralNetworkForwardPropagation& forward_propagation,
                     LossIndexBackPropagation& back_propagation) const
{      
#ifdef OPENNN_DEBUG
 
    Layer* last_trainable_layer_pointer = forward_propagation.neural_network_pointer->get_last_trainable_layer_pointer();
 
    if(last_trainable_layer_pointer->get_type() != Layer::Probabilistic)
    {
        ostringstream buffer;
 
        buffer << "OpenNN Exception: CrossEntropyError class.\n"
               << "calculate_error() method.\n"
               << "Last trainable layer is not probabilistic: " << last_trainable_layer_pointer->get_type_string() << endl;
 
        throw logic_error(buffer.str());
    }
 
#endif
 
    const Index outputs_number = neural_network_pointer->get_outputs_number();
 
    if(outputs_number == 1)
    {
        calculate_binary_error(batch, forward_propagation, back_propagation);
    }
    else
    {
        calculate_multiple_error(batch, forward_propagation, back_propagation);
    }
}
 
 
 
void CrossEntropyError::calculate_binary_error(const DataSetBatch& batch,
                                               const NeuralNetworkForwardPropagation& forward_propagation,
                                               LossIndexBackPropagation& back_propagation) const
{
    const Index batch_samples_number = batch.inputs_2d.dimension(0);
 
    const Index trainable_layers_number = neural_network_pointer->get_trainable_layers_number();
 
    const Tensor<type, 2>& outputs =
            static_cast<ProbabilisticLayerForwardPropagation*>(forward_propagation.layers(trainable_layers_number-1))->activations;
 
    const Tensor<type, 2>& targets = batch.targets_2d;
 
    Tensor<type, 0> cross_entropy_error;
 
    cross_entropy_error.device(*thread_pool_device) = -(targets*(outputs.log())).sum() - ((type(1)-targets)*((type(1)-outputs).log())).sum();
 
    back_propagation.error = cross_entropy_error()/static_cast<type>(batch_samples_number);
}
 
 
void CrossEntropyError::calculate_multiple_error(const DataSetBatch& batch,
                                                 const NeuralNetworkForwardPropagation& forward_propagation,
                                                 LossIndexBackPropagation& back_propagation) const
{
    const Index batch_samples_number = batch.inputs_2d.dimension(0);
 
    const Index trainable_layers_number = neural_network_pointer->get_trainable_layers_number();
 
    const Tensor<type, 2>& outputs =
            static_cast<ProbabilisticLayerForwardPropagation*>(forward_propagation.layers(trainable_layers_number-1))->activations;
 
    const Tensor<type, 2>& targets = batch.targets_2d;
 
    Tensor<type, 0> cross_entropy_error;
    cross_entropy_error.device(*thread_pool_device) = -(targets*(outputs.log())).sum();
 
    back_propagation.error = cross_entropy_error()/static_cast<type>(batch_samples_number);
}
 
 
void CrossEntropyError::calculate_output_delta(const DataSetBatch& batch,
                                               NeuralNetworkForwardPropagation& forward_propagation,
                                               LossIndexBackPropagation& back_propagation) const
{
     #ifdef OPENNN_DEBUG
 
     check();
 
     #endif
 
     const Index outputs_number = neural_network_pointer->get_outputs_number();
 
     if(outputs_number == 1)
     {
         calculate_binary_output_delta(batch, forward_propagation, back_propagation);
     }
     else
     {
         calculate_multiple_output_delta(batch, forward_propagation, back_propagation);
     }
}
 
 
void CrossEntropyError::calculate_binary_output_delta(const DataSetBatch& batch,
                                                      NeuralNetworkForwardPropagation& forward_propagation,
                                                      LossIndexBackPropagation& back_propagation) const
{
    const Index trainable_layers_number = neural_network_pointer->get_trainable_layers_number();
 
    ProbabilisticLayerForwardPropagation* probabilistic_layer_forward_propagation
            = static_cast<ProbabilisticLayerForwardPropagation*>(forward_propagation.layers(trainable_layers_number-1));
 
    ProbabilisticLayerBackPropagation* probabilistic_layer_back_propagation
            = static_cast<ProbabilisticLayerBackPropagation*>(back_propagation.neural_network.layers(trainable_layers_number-1));
 
    const Index batch_samples_number = batch.inputs_2d.dimension(0);
 
    const Tensor<type, 2>& targets = batch.targets_2d;
 
    const Tensor<type, 2>& outputs = probabilistic_layer_forward_propagation->activations;
 
    probabilistic_layer_back_propagation->delta.device(*thread_pool_device)
            = static_cast<type>(1)/static_cast<type>(batch_samples_number) *
            (static_cast<type>(-1)*(targets/outputs) + (static_cast<type>(1) - targets)/(static_cast<type>(1) - outputs));
}
 
 
void CrossEntropyError::calculate_multiple_output_delta(const DataSetBatch& batch,
                                                        NeuralNetworkForwardPropagation& forward_propagation,
                                                        LossIndexBackPropagation& back_propagation) const
{
    const Index trainable_layers_number = neural_network_pointer->get_trainable_layers_number();
 
    ProbabilisticLayerBackPropagation* probabilistic_layer_back_propagation
            = static_cast<ProbabilisticLayerBackPropagation*>(back_propagation.neural_network.layers(trainable_layers_number-1));
 
    const Index batch_samples_number = batch.inputs_2d.dimension(0);
 
    const Tensor<type, 2>& targets = batch.targets_2d;
 
    const Tensor<type, 2>& outputs =
            static_cast<ProbabilisticLayerForwardPropagation*>(forward_propagation.layers(trainable_layers_number-1))->activations;
 
    probabilistic_layer_back_propagation->delta.device(*thread_pool_device)
            = static_cast<type>(1)/static_cast<type>(batch_samples_number) *(-targets/outputs);
}
 
 
 
string CrossEntropyError::get_error_type() const
{
    return "CROSS_ENTROPY_ERROR";
}
 
 
 
string CrossEntropyError::get_error_type_text() const
{
    return "Cross entropy error";
}
 
 
 
void CrossEntropyError::write_XML(tinyxml2::XMLPrinter& file_stream) const
{
    // Error type
 
    file_stream.OpenElement("CrossEntropyError");
 
    file_stream.CloseElement();
}
 
 
 
void CrossEntropyError::from_XML(const tinyxml2::XMLDocument& document)
{
    const tinyxml2::XMLElement* root_element = document.FirstChildElement("CrossEntropyError");
 
    if(!root_element)
    {
        ostringstream buffer;
 
        buffer << "OpenNN Exception: CrossEntropyError class.\n"
               << "void from_XML(const tinyxml2::XMLDocument&) method.\n"
               << "Cross entropy error element is nullptr.\n";
 
        throw logic_error(buffer.str());
    }
 
    // Regularization
 
    tinyxml2::XMLDocument regularization_document;
    tinyxml2::XMLNode* element_clone;
 
    const tinyxml2::XMLElement* regularization_element = root_element->FirstChildElement("Regularization");
 
    element_clone = regularization_element->DeepClone(&regularization_document);
 
    regularization_document.InsertFirstChild(element_clone);
 
    regularization_from_XML(regularization_document);
}
 
}
 
// OpenNN: Open Neural Networks Library.
// Copyright(C) 2005-2021 Artificial Intelligence Techniques, SL.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
 
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA