optimization_algorithm.h

//   OpenNN: Open Neural Networks Library
//   www.opennn.net
//
//   O P T I M I Z A T I O N   A L G O R I T H M   C L A S S   H E A D E R 
//
//   Artificial Intelligence Techniques SL
//   artelnics@artelnics.com
 
#ifndef OPTIMIZATIONALGORITHM_H
#define OPTIMIZATIONALGORITHM_H
 
// System includes
 
#include <iostream>
#include <fstream>
#include <algorithm>
#include <functional>
#include <limits>
#include <cmath>
#include <ctime>
#include <iomanip>
 
// OpenNN includes
 
#include "config.h"
#include "tensor_utilities.h"
#include "loss_index.h"
 
using namespace std;
using namespace Eigen;
 
 
namespace OpenNN
{
 
struct TrainingResults;
 
 
class OptimizationAlgorithm
{
 
public:
 
   explicit OptimizationAlgorithm();
 
   explicit OptimizationAlgorithm(LossIndex*);
 
   virtual ~OptimizationAlgorithm();
 
 
    enum class StoppingCondition{MinimumLossDecrease, LossGoal,
                           MaximumSelectionErrorIncreases, MaximumEpochsNumber, MaximumTime};
 
   // Get methods
 
   LossIndex* get_loss_index_pointer() const;
 
   string get_hardware_use() const;
   void set_hardware_use(const string&);
 
   bool has_loss_index() const;
 
   // Utilities
 
   const bool& get_display() const;
 
   const Index& get_display_period() const;
 
   const Index& get_save_period() const;
 
   const string& get_neural_network_file_name() const;
 
 
   const string write_time(const type&) const;
 
   // Set methods
 
   void set();
 
   virtual void set_default();
 
   virtual void set_threads_number(const int&);
 
   virtual void set_loss_index_pointer(LossIndex*);
 
   virtual void set_display(const bool&);
 
   void set_display_period(const Index&);
 
   void set_save_period(const Index&);
   void set_neural_network_file_name(const string&);
 
   // Training methods
 
   virtual void check() const;
 
 
   virtual TrainingResults perform_training() = 0;
 
   virtual string write_optimization_algorithm_type() const {return string();}
 
   // Serialization methods
 
   virtual void print() const;
 
   virtual Tensor<string, 2> to_string_matrix() const;
 
   virtual void from_XML(const tinyxml2::XMLDocument&);
 
   virtual void write_XML(tinyxml2::XMLPrinter&) const;
 
   void save(const string&) const;
   void load(const string&);
 
protected:
 
   NonBlockingThreadPool* non_blocking_thread_pool = nullptr;
   ThreadPoolDevice* thread_pool_device;
 
 
   LossIndex* loss_index_pointer = nullptr;
 
 
   Index epochs_number = 10000;
 
   // UTILITIES
 
 
   string hardware_use = "Multi-core";
 
 
   Index display_period = 10;
 
 
   Index save_period = numeric_limits<Index>::max();
 
 
   string neural_network_file_name = "neural_network.xml";
 
 
   bool display = true;
 
   const Eigen::array<IndexPair<Index>, 1> AT_B = {IndexPair<Index>(0, 0)};
   const Eigen::array<IndexPair<Index>, 1> product_vector_matrix = {IndexPair<Index>(0, 1)}; // Normal product vector times matrix
   const Eigen::array<IndexPair<Index>, 1> A_B = {IndexPair<Index>(1, 0)};
 
#ifdef OPENNN_CUDA
    #include "../../opennn-cuda/opennn-cuda/optimization_algorithm_cuda.h"
#endif
 
};
 
 
struct OptimizationAlgorithmData
{
    explicit OptimizationAlgorithmData()
    {
    }
 
    virtual ~OptimizationAlgorithmData()
    {
    }
 
    void print() const
    {
        cout << "Potential parameters:" << endl;
        cout << potential_parameters << endl;
 
        cout << "Training direction:" << endl;
        cout << training_direction << endl;
 
        cout << "Initial learning rate:" << endl;
        cout << initial_learning_rate << endl;
    }
 
    Tensor<type, 1> potential_parameters;
    Tensor<type, 1> training_direction;
    type initial_learning_rate = type(0);
};
 
 
 
struct TrainingResults
{
 
    explicit TrainingResults()
    {
    }
 
    explicit TrainingResults(const Index& epochs_number)
    {
        training_error_history.resize(1+epochs_number);
        training_error_history.setConstant(type(-1.0));
 
        selection_error_history.resize(1+epochs_number);
        selection_error_history.setConstant(type(-1.0));
    }
 
 
    virtual ~TrainingResults() {}
 
    string write_stopping_condition() const;
 
    type get_training_error()
    {
        const Index size = training_error_history.size();
 
        return training_error_history(size-1);
    }
 
    type get_selection_error()
    {
        const Index size = selection_error_history.size();
 
        return selection_error_history(size-1);
    }
 
 
    void save(const string&) const;
 
    void print(const string& message = string())
    {             
        cout << message << endl;
 
        const Index epochs_number = training_error_history.size();
 
        cout << "Training results" << endl;
        cout << "Epochs number: " << epochs_number-1 << endl;
 
        cout << "Training error: " << training_error_history(epochs_number-1) << endl;
 
        if(abs(training_error_history(epochs_number-1) + type(1))  < type(NUMERIC_LIMITS_MIN))
            cout << "Selection error: " << selection_error_history(epochs_number-1) << endl;
 
        cout << "Stopping condition: " << write_stopping_condition() << endl;
    }
 
 
    OptimizationAlgorithm::StoppingCondition stopping_condition = OptimizationAlgorithm::StoppingCondition::MaximumTime;
 
 
    Tensor<string, 2> write_final_results(const Index& = 3) const;
 
 
    void resize_training_error_history(const Index&);
 
 
    void resize_selection_error_history(const Index&);
 
    // Training history
 
 
    Tensor<type, 1> training_error_history;
 
 
    Tensor<type, 1> selection_error_history;
 
 
    string elapsed_time;
 
};
 
}
 
#endif
 
 
// OpenNN: Open Neural Networks Library.
// Copyright(C) 2005-2021 Artificial Intelligence Techniques, SL.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
 
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA