opennn::Optimizer Class Reference

Abstract base class for training optimizers (Adam, SGD, Quasi-Newton, Levenberg-Marquardt). More...

#include <optimizer.h>

Inheritance diagram for opennn::Optimizer:

Classes
struct	EpochStats
	Aggregated per-epoch error and accuracy returned by training/evaluation passes. More...

Public Types
enum class	StoppingCondition {   None , MinimumLossDecrease , LossGoal , MaximumSelectionErrorIncreases ,   MaximumEpochsNumber , MaximumTime }
	Reason that training was halted in the last call to train(). More...

Public Member Functions
	Optimizer (Loss *=nullptr)
	Constructs an optimizer optionally bound to a Loss instance.
virtual	~Optimizer ()=default
const Loss *	get_loss () const
bool	get_display () const
void	set (Loss *new_loss)
	Binds the optimizer to a new Loss instance.
virtual void	set_loss (Loss *new_loss)
	Binds the optimizer to a new Loss instance (virtual hook for derived classes).
virtual void	set_display (bool new_display)
	Enables or disables console progress reporting during training.
void	set_display_period (const Index new_display_period)
void	set_num_workers (int n)
int	get_num_workers () const
void	set_maximum_epochs (const Index new_maximum_epochs)
void	set_maximum_time (const float new_maximum_time)
void	set_loss_goal (const float new_loss_goal)
void	set_maximum_validation_failures (const Index new_maximum_validation_failures)
virtual TrainingResults	train ()=0
	Runs the training loop and returns the recorded results (must be implemented by subclasses).
Index	get_maximum_batch_size () const
	Largest batch size compatible with the dataset and configured memory budget.
const string &	get_name () const
virtual void	print () const
	Prints a human-readable description of the optimizer (no-op default).
virtual void	from_JSON (const JsonDocument &)
	Restores optimizer configuration from a JSON document.
virtual void	to_JSON (JsonWriter &) const
	Serializes the optimizer configuration to JSON.
void	save (const filesystem::path &) const
	Writes the current optimizer configuration to a JSON file at the given path.
void	load (const filesystem::path &)
	Loads the optimizer configuration from a JSON file at the given path.

Static Public Member Functions
static float	get_elapsed_time (const time_t &beginning_time)
	Returns seconds elapsed since the given start time.

Protected Member Functions
void	set_names ()
void	set_scaling ()
void	set_unscaling ()
bool	check_stopping_condition (TrainingResults &, Index epoch, float elapsed_time, float training_error, Index validation_failures) const
void	write_common_json (JsonWriter &) const
void	read_common_json (const Json *)
void	setup_device_training ()
void	teardown_device_training ()
void	prefetch_batch (Batch &batch, Index sample_count, int slot)
void	wait_prefetch (int slot)
void	record_batch_reuse (Batch &batch)
void	clear_batch_reuse_events ()
void	sync_device ()
bool	should_display (Index epoch) const
void	warn_dropped_samples (Index batch_size, Index samples_number, const char *context) const
EpochStats	train_epoch (bool tracks_accuracy, ForwardPropagation &forward_propagation, BackPropagation &back_propagation, ThreadSafeQueue< Batch * > &empty_queue, const vector< vector< Index > > &batches, const vector< Index > &input_feature_indices, const vector< Index > &decoder_feature_indices, const vector< Index > &target_feature_indices, const function< void(BackPropagation &)> &update, bool show_progress=true)
EpochStats	evaluate_epoch (bool tracks_accuracy, ForwardPropagation &forward_propagation, ThreadSafeQueue< Batch * > &empty_queue, const vector< vector< Index > > &batches, const vector< Index > &input_feature_indices, const vector< Index > &decoder_feature_indices, const vector< Index > &target_feature_indices)

Static Protected Member Functions
static void	clip_gradient_norm (Buffer &gradient, float max_norm)

Protected Attributes
Loss *	loss = nullptr
float	training_loss_goal = 0.0f
Index	maximum_validation_failures = numeric_limits<Index>::max()
Index	maximum_epochs = 10000
float	maximum_time = 360000.0f
Index	display_period = 10
bool	display = true
string	name
int	num_workers = 2
cudaStream_t	memory_stream = nullptr
cudaEvent_t	batch_ready_event [2] = {nullptr, nullptr}
unordered_map< Batch *, cudaEvent_t >	batch_reuse_events
unordered_set< Batch * >	batch_reuse_recorded
Buffer	prefetch_fp32_staging {Device::CUDA}

Detailed Description

Abstract base class for training optimizers (Adam, SGD, Quasi-Newton, Levenberg-Marquardt).

Member Enumeration Documentation

StoppingCondition

enum class opennn::Optimizer::StoppingCondition

strong

Reason that training was halted in the last call to train().

Enumerator
None
MinimumLossDecrease
LossGoal
MaximumSelectionErrorIncreases
MaximumEpochsNumber
MaximumTime

Constructor & Destructor Documentation

Optimizer()

opennn::Optimizer::Optimizer

(

Loss *

= nullptr

)

Constructs an optimizer optionally bound to a Loss instance.

~Optimizer()

virtual opennn::Optimizer::~Optimizer ( )

virtualdefault

Member Function Documentation

check_stopping_condition()

bool opennn::Optimizer::check_stopping_condition ( TrainingResults & , Index epoch, float elapsed_time, float training_error, Index validation_failures ) const

protected

clear_batch_reuse_events()

void opennn::Optimizer::clear_batch_reuse_events ( )

protected

clip_gradient_norm()

static void opennn::Optimizer::clip_gradient_norm ( Buffer & gradient, float max_norm )

staticprotected

evaluate_epoch()

EpochStats opennn::Optimizer::evaluate_epoch ( bool tracks_accuracy, ForwardPropagation & forward_propagation, ThreadSafeQueue< Batch * > & empty_queue, const vector< vector< Index > > & batches, const vector< Index > & input_feature_indices, const vector< Index > & decoder_feature_indices, const vector< Index > & target_feature_indices )

protected

from_JSON()

virtual void opennn::Optimizer::from_JSON ( const JsonDocument & )

virtual

Restores optimizer configuration from a JSON document.

Reimplemented in opennn::AdaptiveMomentEstimation, opennn::LevenbergMarquardtAlgorithm, opennn::QuasiNewtonMethod, and opennn::StochasticGradientDescent.

get_display()

bool opennn::Optimizer::get_display ( ) const

inline

get_elapsed_time()

static float opennn::Optimizer::get_elapsed_time ( const time_t & beginning_time )

static

Returns seconds elapsed since the given start time.

get_loss()

const Loss * opennn::Optimizer::get_loss ( ) const

inline

get_maximum_batch_size()

Index opennn::Optimizer::get_maximum_batch_size

(

)

const

Largest batch size compatible with the dataset and configured memory budget.

get_name()

const string & opennn::Optimizer::get_name ( ) const

inline

get_num_workers()

int opennn::Optimizer::get_num_workers ( ) const

inline

load()

void opennn::Optimizer::load

(

const filesystem::path &

)

Loads the optimizer configuration from a JSON file at the given path.

prefetch_batch()

void opennn::Optimizer::prefetch_batch ( Batch & batch, Index sample_count, int slot )

protected

print()

virtual void opennn::Optimizer::print ( ) const

inlinevirtual

Prints a human-readable description of the optimizer (no-op default).

read_common_json()

void opennn::Optimizer::read_common_json ( const Json * )

protected

record_batch_reuse()

void opennn::Optimizer::record_batch_reuse ( Batch & batch )

protected

save()

void opennn::Optimizer::save

(

const filesystem::path &

)

const

Writes the current optimizer configuration to a JSON file at the given path.

set()

void opennn::Optimizer::set ( Loss * new_loss )

inline

Binds the optimizer to a new Loss instance.

set_display()

virtual void opennn::Optimizer::set_display ( bool new_display )

inlinevirtual

Enables or disables console progress reporting during training.

set_display_period()

void opennn::Optimizer::set_display_period ( const Index new_display_period )

inline

set_loss()

virtual void opennn::Optimizer::set_loss ( Loss * new_loss )

inlinevirtual

Binds the optimizer to a new Loss instance (virtual hook for derived classes).

set_loss_goal()

void opennn::Optimizer::set_loss_goal ( const float new_loss_goal )

inline

set_maximum_epochs()

void opennn::Optimizer::set_maximum_epochs ( const Index new_maximum_epochs )

inline

set_maximum_time()

void opennn::Optimizer::set_maximum_time ( const float new_maximum_time )

inline

set_maximum_validation_failures()

void opennn::Optimizer::set_maximum_validation_failures ( const Index new_maximum_validation_failures )

inline

set_names()

void opennn::Optimizer::set_names ( )

protected

set_num_workers()

void opennn::Optimizer::set_num_workers ( int n )

inline

set_scaling()

void opennn::Optimizer::set_scaling ( )

protected

set_unscaling()

void opennn::Optimizer::set_unscaling ( )

protected

setup_device_training()

void opennn::Optimizer::setup_device_training ( )

protected

should_display()

bool opennn::Optimizer::should_display ( Index epoch ) const

inlineprotected

sync_device()

void opennn::Optimizer::sync_device ( )

protected

teardown_device_training()

void opennn::Optimizer::teardown_device_training ( )

protected

to_JSON()

virtual void opennn::Optimizer::to_JSON ( JsonWriter & ) const

virtual

Serializes the optimizer configuration to JSON.

Reimplemented in opennn::AdaptiveMomentEstimation, opennn::LevenbergMarquardtAlgorithm, opennn::QuasiNewtonMethod, and opennn::StochasticGradientDescent.

train()

virtual TrainingResults opennn::Optimizer::train ( )

pure virtual

Runs the training loop and returns the recorded results (must be implemented by subclasses).

Implemented in opennn::AdaptiveMomentEstimation, opennn::LevenbergMarquardtAlgorithm, opennn::QuasiNewtonMethod, and opennn::StochasticGradientDescent.

train_epoch()

EpochStats opennn::Optimizer::train_epoch ( bool tracks_accuracy, ForwardPropagation & forward_propagation, BackPropagation & back_propagation, ThreadSafeQueue< Batch * > & empty_queue, const vector< vector< Index > > & batches, const vector< Index > & input_feature_indices, const vector< Index > & decoder_feature_indices, const vector< Index > & target_feature_indices, const function< void(BackPropagation &)> & update, bool show_progress = true )

protected

wait_prefetch()

void opennn::Optimizer::wait_prefetch ( int slot )

protected

warn_dropped_samples()

void opennn::Optimizer::warn_dropped_samples ( Index batch_size, Index samples_number, const char * context ) const

protected

write_common_json()

void opennn::Optimizer::write_common_json ( JsonWriter & ) const

protected

Member Data Documentation

batch_ready_event

cudaEvent_t opennn::Optimizer::batch_ready_event[2] = {nullptr, nullptr}

protected

batch_reuse_events

unordered_map<Batch*, cudaEvent_t> opennn::Optimizer::batch_reuse_events

protected

batch_reuse_recorded

unordered_set<Batch*> opennn::Optimizer::batch_reuse_recorded

protected

display

bool opennn::Optimizer::display = true

protected

display_period

Index opennn::Optimizer::display_period = 10

protected

loss

Loss* opennn::Optimizer::loss = nullptr

protected

maximum_epochs

Index opennn::Optimizer::maximum_epochs = 10000

protected

maximum_time

float opennn::Optimizer::maximum_time = 360000.0f

protected

maximum_validation_failures

Index opennn::Optimizer::maximum_validation_failures = numeric_limits<Index>::max()

protected

memory_stream

cudaStream_t opennn::Optimizer::memory_stream = nullptr

protected

name

string opennn::Optimizer::name

protected

num_workers

int opennn::Optimizer::num_workers = 2

protected

prefetch_fp32_staging

Buffer opennn::Optimizer::prefetch_fp32_staging {Device::CUDA}

protected

training_loss_goal

float opennn::Optimizer::training_loss_goal = 0.0f

protected