Adam optimizer with first/second gradient moments for stochastic minibatch training. More...

#include <adaptive_moment_estimation.h>

Inheritance diagram for opennn::AdaptiveMomentEstimation:

Public Types
enum	DataSlot { GradientMoment , SquareGradientMoment }
	Slot indices into the optimizer scratch buffer (m_t and v_t moments). More...

Public Types inherited from opennn::Optimizer
enum class	StoppingCondition { None , MinimumLossDecrease , LossGoal , MaximumSelectionErrorIncreases , MaximumEpochsNumber , MaximumTime }
	Reason that training was halted in the last call to train(). More...

Public Member Functions
	AdaptiveMomentEstimation (Loss *=nullptr)
	Constructs Adam optionally bound to a Loss instance.

Index	get_samples_number () const
	Returns the number of training samples seen by the bound dataset.

void	set_batch_size (const Index)
	Sets the minibatch size used by train().

void	set_default ()
	Resets all hyperparameters (learning rate, betas, stopping criteria) to library defaults.

void	set_learning_rate (const float)
	Sets the base learning rate alpha.

void	set_beta_1 (const float)
	Sets the first-moment decay rate beta_1.

void	set_beta_2 (const float)
	Sets the second-moment decay rate beta_2.

TrainingResults	train () override
	Runs the Adam training loop and returns the recorded error history.

void	update_parameters (BackPropagation &, OptimizerData &) const
	Applies one Adam update to the network parameters using the gradient in back_propagation.

void	from_JSON (const JsonDocument &) override
	Restores hyperparameters from a JSON document.

void	to_JSON (JsonWriter &) const override
	Serializes hyperparameters to JSON.

Public Member Functions inherited from opennn::Optimizer
	Optimizer (Loss *=nullptr)
	Constructs an optimizer optionally bound to a Loss instance.

virtual	~Optimizer ()=default

const Loss *	get_loss () const

bool	get_display () const

void	set (Loss *new_loss)
	Binds the optimizer to a new Loss instance.

virtual void	set_loss (Loss *new_loss)
	Binds the optimizer to a new Loss instance (virtual hook for derived classes).

virtual void	set_display (bool new_display)
	Enables or disables console progress reporting during training.

void	set_display_period (const Index new_display_period)

void	set_num_workers (int n)

int	get_num_workers () const

void	set_maximum_epochs (const Index new_maximum_epochs)

void	set_maximum_time (const float new_maximum_time)

void	set_loss_goal (const float new_loss_goal)

void	set_maximum_validation_failures (const Index new_maximum_validation_failures)

Index	get_maximum_batch_size () const
	Largest batch size compatible with the dataset and configured memory budget.

const string &	get_name () const

virtual void	print () const
	Prints a human-readable description of the optimizer (no-op default).

void	save (const filesystem::path &) const
	Writes the current optimizer configuration to a JSON file at the given path.

void	load (const filesystem::path &)
	Loads the optimizer configuration from a JSON file at the given path.

Additional Inherited Members
Static Public Member Functions inherited from opennn::Optimizer
static float	get_elapsed_time (const time_t &beginning_time)
	Returns seconds elapsed since the given start time.

Protected Member Functions inherited from opennn::Optimizer
void	set_names ()

void	set_scaling ()

void	set_unscaling ()

bool	check_stopping_condition (TrainingResults &, Index epoch, float elapsed_time, float training_error, Index validation_failures) const

void	write_common_json (JsonWriter &) const

void	read_common_json (const Json *)

void	setup_device_training ()

void	teardown_device_training ()

void	prefetch_batch (Batch &batch, Index sample_count, int slot)

void	wait_prefetch (int slot)

void	record_batch_reuse (Batch &batch)

void	clear_batch_reuse_events ()

void	sync_device ()

bool	should_display (Index epoch) const

void	warn_dropped_samples (Index batch_size, Index samples_number, const char *context) const

EpochStats	train_epoch (bool tracks_accuracy, ForwardPropagation &forward_propagation, BackPropagation &back_propagation, ThreadSafeQueue< Batch * > &empty_queue, const vector< vector< Index > > &batches, const vector< Index > &input_feature_indices, const vector< Index > &decoder_feature_indices, const vector< Index > &target_feature_indices, const function< void(BackPropagation &)> &update, bool show_progress=true)

EpochStats	evaluate_epoch (bool tracks_accuracy, ForwardPropagation &forward_propagation, ThreadSafeQueue< Batch * > &empty_queue, const vector< vector< Index > > &batches, const vector< Index > &input_feature_indices, const vector< Index > &decoder_feature_indices, const vector< Index > &target_feature_indices)

Static Protected Member Functions inherited from opennn::Optimizer
static void	clip_gradient_norm (Buffer &gradient, float max_norm)

Protected Attributes inherited from opennn::Optimizer
Loss *	loss = nullptr

float	training_loss_goal = 0.0f

Index	maximum_validation_failures = numeric_limits<Index>::max()

Index	maximum_epochs = 10000

float	maximum_time = 360000.0f

Index	display_period = 10

bool	display = true

string	name

int	num_workers = 2

cudaStream_t	memory_stream = nullptr

cudaEvent_t	batch_ready_event [2] = {nullptr, nullptr}

unordered_map< Batch *, cudaEvent_t >	batch_reuse_events

unordered_set< Batch * >	batch_reuse_recorded

Buffer	prefetch_fp32_staging {Device::CUDA}

Detailed Description

Adam optimizer with first/second gradient moments for stochastic minibatch training.

Member Enumeration Documentation

◆ DataSlot

enum opennn::AdaptiveMomentEstimation::DataSlot

Slot indices into the optimizer scratch buffer (m_t and v_t moments).

Enumerator
GradientMoment
SquareGradientMoment

Constructor & Destructor Documentation

◆ AdaptiveMomentEstimation()

opennn::AdaptiveMomentEstimation::AdaptiveMomentEstimation ( Loss * = nullptr )

Constructs Adam optionally bound to a Loss instance.

Member Function Documentation

◆ from_JSON()

void opennn::AdaptiveMomentEstimation::from_JSON ( const JsonDocument & )

overridevirtual

Restores hyperparameters from a JSON document.

Reimplemented from opennn::Optimizer.

◆ get_samples_number()

Index opennn::AdaptiveMomentEstimation::get_samples_number ( ) const

Returns the number of training samples seen by the bound dataset.

◆ set_batch_size()

void opennn::AdaptiveMomentEstimation::set_batch_size ( const Index )

Sets the minibatch size used by train().

◆ set_beta_1()

void opennn::AdaptiveMomentEstimation::set_beta_1 ( const float )

Sets the first-moment decay rate beta_1.

◆ set_beta_2()

void opennn::AdaptiveMomentEstimation::set_beta_2 ( const float )

Sets the second-moment decay rate beta_2.

◆ set_default()

void opennn::AdaptiveMomentEstimation::set_default ( )

Resets all hyperparameters (learning rate, betas, stopping criteria) to library defaults.

◆ set_learning_rate()

void opennn::AdaptiveMomentEstimation::set_learning_rate ( const float )

Sets the base learning rate alpha.

◆ to_JSON()

void opennn::AdaptiveMomentEstimation::to_JSON ( JsonWriter & ) const

overridevirtual

Serializes hyperparameters to JSON.

Reimplemented from opennn::Optimizer.

◆ train()

TrainingResults opennn::AdaptiveMomentEstimation::train ( )

overridevirtual

Runs the Adam training loop and returns the recorded error history.

Implements opennn::Optimizer.

◆ update_parameters()

void opennn::AdaptiveMomentEstimation::update_parameters	(	BackPropagation &	,
		OptimizerData &	) const

Applies one Adam update to the network parameters using the gradient in back_propagation.

Public Types

Public Member Functions

Additional Inherited Members

Detailed Description

Member Enumeration Documentation

◆ DataSlot

Constructor & Destructor Documentation

◆ AdaptiveMomentEstimation()

Member Function Documentation

◆ from_JSON()

◆ get_samples_number()

◆ set_batch_size()

◆ set_beta_1()

◆ set_beta_2()

◆ set_default()

◆ set_learning_rate()

◆ to_JSON()

◆ train()

◆ update_parameters()