optimizer.h

Go to the documentation of this file.

//   OpenNN: Open Neural Networks Library
//   www.opennn.net
//
//   O P T I M I Z A T I O N   A L G O R I T H M   C L A S S   H E A D E R
//
//   Artificial Intelligence Techniques SL
//   artelnics@artelnics.com

 
#pragma once
 
#include <functional>
#include "json.h"
#include "tensor_utilities.h"
#include "thread_safe_queue.h"
 
namespace opennn
{
 
class Loss;
struct Batch;
struct Buffer;
struct ForwardPropagation;
struct BackPropagation;
 
struct TrainingResults;

struct EpochStats
{
    float error = 0.0f;
    float accuracy = 0.0f;
};

class Optimizer
{
 
public:

    Optimizer(Loss* loss = nullptr);
    virtual ~Optimizer() = default;

    enum class StoppingCondition{None,                            
                                 MinimumLossDecrease,             
                                 LossGoal,                        
                                 MaximumSelectionErrorIncreases,  
                                 MaximumEpochsNumber,             
                                 MaximumTime};                    

    const Loss* get_loss() const { return loss; }

    bool get_display() const { return display; }

    void set(Loss* new_loss) { loss = new_loss; }

    virtual void set_loss(Loss* new_loss) { loss = new_loss; }

    virtual void set_display(bool new_display) { display = new_display; }

    void set_display_period(const Index new_display_period) { display_period = new_display_period; }

    void set_maximum_epochs(const Index new_maximum_epochs) { maximum_epochs = new_maximum_epochs; }
    void set_maximum_time(const float new_maximum_time) { maximum_time = new_maximum_time; }

    void set_loss_goal(const float new_loss_goal) { training_loss_goal = new_loss_goal; }
    void set_maximum_validation_failures(const Index new_maximum_validation_failures) { maximum_validation_failures = new_maximum_validation_failures; }
    virtual TrainingResults train() = 0;

    const string& get_name() const { return name; }

    virtual void print() const {}

    virtual void from_JSON(const JsonDocument&);

    virtual void to_JSON(JsonWriter&) const;

    void save(const filesystem::path&) const;
    void load(const filesystem::path&);

    static float get_elapsed_time(const time_t& beginning_time);
 
protected:

    void set_names();
    void set_scaling();
    void set_unscaling();

    bool check_stopping_condition(TrainingResults& results, Index epoch, float elapsed_time,
                                   float training_error, Index validation_failures) const;

    void write_common_xml(JsonWriter&) const;
    void read_common_xml(const Json*);

    void setup_device_training();
    void teardown_device_training();

    void prefetch_batch(Batch& batch, Index sample_count, int slot);

    void wait_prefetch(int slot);

    void sync_device();

    static void clip_gradient_norm(Buffer& gradient, float max_norm);

    bool should_display(Index epoch) const { return display && epoch % display_period == 0; }

    EpochStats train_epoch(bool is_classification,
                           ForwardPropagation& forward_propagation,
                           BackPropagation& back_propagation,
                           ThreadSafeQueue<Batch*>& empty_queue,
                           ThreadSafeQueue<Batch*>& ready_queue,
                           const vector<vector<Index>>& batches,
                           const vector<Index>& input_feature_indices,
                           const vector<Index>& decoder_feature_indices,
                           const vector<Index>& target_feature_indices,
                           const std::function<void(BackPropagation&)>& update);

    EpochStats evaluate_epoch(bool is_classification,
                              ForwardPropagation& forward_propagation,
                              ThreadSafeQueue<Batch*>& empty_queue,
                              ThreadSafeQueue<Batch*>& ready_queue,
                              const vector<vector<Index>>& batches,
                              const vector<Index>& input_feature_indices,
                              const vector<Index>& decoder_feature_indices,
                              const vector<Index>& target_feature_indices);

    Loss* loss = nullptr;

    float training_loss_goal = 0.0f;

    Index maximum_validation_failures = numeric_limits<Index>::max();

    Index maximum_epochs = 10000;

    float maximum_time = 360000.0f;

    Index display_period = 10;

    bool display = true;

    string name;

    cudaStream_t memory_stream = nullptr;
    cudaEvent_t batch_ready_event[2] = {nullptr, nullptr};
};

struct OptimizerData
{
    OptimizerData() = default;
    virtual ~OptimizerData() = default;

    virtual void print() const;

    void set(const vector<Shape>& slot_shapes, Device device = Device::CPU);

    Buffer data;
    vector<TensorView> views;

    VectorR potential_parameters;
    VectorR training_direction;
    float initial_learning_rate = 0.0f;
    Index iteration = 0;
};

struct TrainingResults
{
    TrainingResults(const Index expected_epochs = 0);
    virtual ~TrainingResults() = default;

    string write_stopping_condition() const;

    float get_training_error() const;

    float get_validation_error() const;

    Index get_epochs_number() const;

    void save(const filesystem::path&) const;

    void print(const string& message = string()) const;

    Optimizer::StoppingCondition stopping_condition = Optimizer::StoppingCondition::None;

    Tensor<string, 2> write_override_results(const Index decimals = 3) const;

    void resize_training_error_history(const Index);

    void resize_validation_error_history(const Index);

    VectorR training_error_history;

    VectorR validation_error_history;

    string elapsed_time;

    float loss = NAN;

    Index validation_failures = 0;

    float loss_decrease = 0.0f;
};
 
}
 
// OpenNN: Open Neural Networks Library.
// Copyright(C) 2005-2026 Artificial Intelligence Techniques, SL.
// Licensed under the GNU Lesser General Public License v2.1 or later.