adaptive_moment_estimation.h

//   OpenNN: Open Neural Networks Library
//   www.opennn.net
//
//   A D A P T I V E   M O M E N T   E S T I M A T I O N
//
//   Artificial Intelligence Techniques SL
//   artelnics@artelnics.com
 
#ifndef ADAPTIVEMOMENTESTIMATION_H
#define ADAPTIVEMOMENTESTIMATION_H
 
// System includes
 
#include <string>
#include <sstream>
#include <iostream>
#include <fstream>
#include <algorithm>
#include <functional>
#include <limits>
#include <cmath>
#include <ctime>
#include <chrono>
#include <time.h>
#include <iostream>
#include <ctime>
#include <ratio>
#include <chrono>
 
// OpenNN includes
 
#include "loss_index.h"
#include "optimization_algorithm.h"
#include "config.h"
 
namespace OpenNN
{
 
struct AdaptiveMomentEstimationData;
 
 
 
class AdaptiveMomentEstimation : public OptimizationAlgorithm
{
 
public:
 
   // Constructors
 
   explicit AdaptiveMomentEstimation();
 
   explicit AdaptiveMomentEstimation(LossIndex*);   
 
   virtual ~AdaptiveMomentEstimation();
   
   // Training operators
 
   const type& get_initial_learning_rate() const;
   const type& get_beta_1() const;
   const type& get_beta_2() const;
   const type& get_epsilon() const;
 
   // Stopping criteria
 
   const type& get_loss_goal() const;
   const type& get_maximum_time() const;
 
   // Set methods
 
   void set_loss_index_pointer(LossIndex*);
 
   void set_batch_samples_number(const Index& new_batch_samples_number);
 
   void set_default();
 
   // Get methods
 
   Index get_batch_samples_number() const;
 
   // Training operators
 
   void set_initial_learning_rate(const type&);
   void set_beta_1(const type&);
   void set_beta_2(const type&);
   void set_epsilon(const type&);
 
   // Training parameters
 
   void set_maximum_epochs_number(const Index&);
 
   // Stopping criteria
 
   void set_loss_goal(const type&);
   void set_maximum_time(const type&);
 
   // Training methods
 
   TrainingResults perform_training();
 
 
   string write_optimization_algorithm_type() const;
 
   // Serialization methods
 
   Tensor<string, 2> to_string_matrix() const;
 
   void from_XML(const tinyxml2::XMLDocument&);
 
   void write_XML(tinyxml2::XMLPrinter&) const;
 
   void update_parameters(LossIndexBackPropagation& ,
                         AdaptiveMomentEstimationData&);
 
private:
 
   // TRAINING OPERATORS
 
 
   type initial_learning_rate = static_cast<type>(0.001);
 
 
   type initial_decay = type(0);
 
 
   type beta_1 = static_cast<type>(0.9);
 
 
   type beta_2 = static_cast<type>(0.999);
 
 
   type epsilon =static_cast<type>(1.e-7);
 
    // Stopping criteria
 
 
   type training_loss_goal = type(0);
 
 
   Index maximum_epochs_number = 10000;
 
 
   Index maximum_selection_failures = numeric_limits<Index>::max();
 
 
   type maximum_time = type(3600);
 
 
   Index batch_samples_number = 1000;
 
#ifdef OPENNN_CUDA
    #include "../../opennn-cuda/opennn-cuda/adaptive_moment_estimation_cuda.h"
#endif
 
};
 
 
struct AdaptiveMomentEstimationData : public OptimizationAlgorithmData
{
 
    explicit AdaptiveMomentEstimationData();
 
    explicit AdaptiveMomentEstimationData(AdaptiveMomentEstimation*);
 
    virtual ~AdaptiveMomentEstimationData();
 
    void set(AdaptiveMomentEstimation*);
 
    void print() const;
 
    AdaptiveMomentEstimation* adaptive_moment_estimation_pointer = nullptr;
 
    Index learning_rate_iteration = 0;
 
    Tensor<type, 1> gradient_exponential_decay;
    Tensor<type, 1> square_gradient_exponential_decay;
 
    Index iteration;
};
 
}
 
#endif