#include <opennn/training/optimizer.h>

Inheritance diagram for opennn::Optimizer:

Classes
struct	BestModelSnapshot

struct	EpochReport

struct	FullBatchContext

struct	FullBatchHooks

struct	FullBatchStep

struct	TrainingSession

Public Types
using	StoppingCondition = opennn::StoppingCondition

Public Member Functions
	Optimizer (Loss *=nullptr)

virtual	~Optimizer ()

const Loss *	get_loss () const noexcept

bool	get_display () const noexcept

void	set (Loss *new_loss)

virtual void	set_display (bool new_display)

void	set_display_period (const Index new_display_period)

void	set_workers_number (int new_workers_number)

void	set_cuda_graph (bool enabled)

bool	get_cuda_graph_capture_failed () const noexcept

void	set_shuffle (bool enabled)

void	set_batch_pool_size (int size)

void	set_maximum_epochs (const Index new_maximum_epochs)

Index	get_maximum_epochs () const noexcept

void	set_maximum_time (const float new_maximum_time)

void	set_loss_goal (const float new_loss_goal)

void	set_maximum_validation_failures (const Index new_maximum_validation_failures)

void	set_validation_period (const Index n)

void	set_restore_best (bool enabled)

void	set_gradient_clip_norm (const float new_clip)

void	set_joint_gradient_arena (bool enabled) noexcept

Index	get_batch_size () const noexcept

void	set_batch_size (const Index new_batch_size)

virtual void	configure_for_task (NetworkTask)

virtual TrainingResult	train ()

Index	get_maximum_batch_size () const

const string &	get_name () const noexcept

virtual void	from_JSON (const JsonDocument &)

virtual void	to_JSON (JsonWriter &) const

void	save (const filesystem::path &) const

void	load (const filesystem::path &)

Public Attributes
function< void(Index, float, float, Network *)>	post_epoch_callback

function< void(Network *)>	post_batch_callback

function< void(Index, float)>	post_best_callback

Protected Types
enum class	UpdateMode { Standard , Capturable }

Protected Member Functions
void	set_names ()

void	prepare_training_artifacts ()

Index	get_epoch_budget () const noexcept

optional< StoppingCondition >	find_stopping_condition (const EpochReport &, float minimum_loss_decrease=-MAX) const

void	close_training_result (TrainingResult &, const EpochReport &, bool has_validation) const

void	update_best_parameters (Network *, float, Index, Index &, BestModelSnapshot &)

void	restore_best_parameters (Network *, TrainingResult &, const BestModelSnapshot &)

void	write_common_json (JsonWriter &) const

void	read_common_json (const Json *)

void	setup_device_training ()

void	teardown_device_training ()

void	warmup_device_training (TrainingContext &, ThreadSafeQueue< Batch * > &, const vector< vector< Index > > &, const FeatureSelection &, TrainingSession &, OptimizerData &, ForwardPropagation validation_forward_propagation=nullptr, ThreadSafeQueue< Batch > validation_empty_queue=nullptr, const vector< vector< Index > > validation_batches=nullptr)

void	prefetch_batch (Batch &)

void	sync_device (bool on_gpu, bool has_recurrent_layers, TrainingSession &)

bool	should_display (Index epoch) const noexcept

void	display_epoch_results (Index epoch, float training_error, float training_accuracy, float validation_error, float validation_accuracy, bool has_validation, bool validation_fresh, bool is_token_cross_entropy, float elapsed_time) const

void	setup_batch_pools (BatchPools &pools, Dataset &dataset, Network &network, Index training_batch_size, Index validation_batch_size, bool has_validation, TrainingSession &training_session)

unique_ptr< BatchPrefetchSession >	start_batch_prefetch (ThreadSafeQueue< Batch * > &, const vector< vector< Index > > &, const FeatureSelection &, FillMode, WorkerProfileCounters *profile_counters=nullptr)

int	get_batch_pool_size (const Network &) const

Loss::EvaluationResult	run_epoch_loop (EpochLoopContext &)

void	prepare_full_batch_training (FullBatchContext &, const char *banner)

TrainingResult	train_full_batch (FullBatchContext &, const FullBatchHooks &)

virtual string	get_display_name () const

virtual void	setup_optimizer_data (OptimizerData &, Index, Device)

virtual void	update_parameters (BackPropagation &, OptimizerData &, UpdateMode=UpdateMode::Standard)

virtual bool	supports_cuda_graph () const noexcept

bool	network_has_active_dropout () const

bool	can_use_cuda_graph () const

virtual void	on_epoch_begin (Index, OptimizerData &)

bool	uses_joint_gradient_arena () const noexcept

Loss::EvaluationResult	run_graph_epoch (TrainingSession &, OptimizerData &, ForwardPropagation &, BackPropagation &, ThreadSafeQueue< Batch * > &, const vector< vector< Index > > &, const FeatureSelection &)

Loss::EvaluationResult	train_epoch (TrainingContext &, ThreadSafeQueue< Batch * > &, const vector< vector< Index > > &, const FeatureSelection &, TrainingSession &, OptimizerData &)

Loss::EvaluationResult	evaluate_epoch (ForwardPropagation &, ThreadSafeQueue< Batch * > &, const vector< vector< Index > > &, const FeatureSelection &, TrainingSession &)

Static Protected Member Functions
static void	clip_gradient_norm (BackPropagation &, float)

Protected Attributes
Loss *	loss = nullptr

float	training_loss_goal = 0.0f

Index	maximum_validation_failures = numeric_limits<Index>::max()

Index	validation_period = 1

float	gradient_clip_norm = 0.0f

bool	joint_gradient_arena = false

bool	restore_best = true

Index	maximum_epochs = 10000

Index	batch_size = 0

float	maximum_time = 360000.0f

Index	display_period = 10

bool	display = true

bool	shuffle_samples = true

int	batch_pool_size_override = 0

int	workers_number = default_workers_number

bool	use_cuda_graph = false

bool	cuda_graph_capture_failed = false

string	name

Static Protected Attributes
static const int	default_workers_number

Member Typedef Documentation

◆ StoppingCondition

using opennn::Optimizer::StoppingCondition = opennn::StoppingCondition

Member Enumeration Documentation

◆ UpdateMode

enum class opennn::Optimizer::UpdateMode

strongprotected

Enumerator
Standard
Capturable

Constructor & Destructor Documentation

◆ Optimizer()

opennn::Optimizer::Optimizer ( Loss * = nullptr )

explicit

◆ ~Optimizer()

virtual opennn::Optimizer::~Optimizer ( )

virtual

Member Function Documentation

◆ can_use_cuda_graph()

bool opennn::Optimizer::can_use_cuda_graph ( ) const

inlineprotected

◆ clip_gradient_norm()

static void opennn::Optimizer::clip_gradient_norm	(	BackPropagation &	,
		float	)

staticprotected

◆ close_training_result()

void opennn::Optimizer::close_training_result	(	TrainingResult &	,
		const EpochReport &	,
		bool	has_validation ) const

protected

◆ configure_for_task()

virtual void opennn::Optimizer::configure_for_task ( NetworkTask )

virtual

Reimplemented in opennn::Adam.

◆ display_epoch_results()

void opennn::Optimizer::display_epoch_results	(	Index	epoch,
		float	training_error,
		float	training_accuracy,
		float	validation_error,
		float	validation_accuracy,
		bool	has_validation,
		bool	validation_fresh,
		bool	is_token_cross_entropy,
		float	elapsed_time ) const

protected

◆ evaluate_epoch()

Loss::EvaluationResult opennn::Optimizer::evaluate_epoch	(	ForwardPropagation &	,
		ThreadSafeQueue< Batch * > &	,
		const vector< vector< Index > > &	,
		const FeatureSelection &	,
		TrainingSession &	)

protected

◆ find_stopping_condition()

optional< StoppingCondition > opennn::Optimizer::find_stopping_condition	(	const EpochReport &	,
		float	minimum_loss_decrease = -MAX ) const

protected

◆ from_JSON()

virtual void opennn::Optimizer::from_JSON ( const JsonDocument & )

virtual

Reimplemented in opennn::Adam, opennn::LevenbergMarquardt, opennn::QuasiNewton, and opennn::SGD.

◆ get_batch_pool_size()

int opennn::Optimizer::get_batch_pool_size ( const Network & ) const

protected

◆ get_batch_size()

Index opennn::Optimizer::get_batch_size ( ) const

inlinenoexcept

◆ get_cuda_graph_capture_failed()

bool opennn::Optimizer::get_cuda_graph_capture_failed ( ) const

inlinenoexcept

◆ get_display()

bool opennn::Optimizer::get_display ( ) const

inlinenoexcept

◆ get_display_name()

virtual string opennn::Optimizer::get_display_name ( ) const

inlineprotectedvirtual

◆ get_epoch_budget()

Index opennn::Optimizer::get_epoch_budget ( ) const

inlineprotectednoexcept

◆ get_loss()

const Loss * opennn::Optimizer::get_loss ( ) const

inlinenoexcept

◆ get_maximum_batch_size()

Index opennn::Optimizer::get_maximum_batch_size ( ) const

◆ get_maximum_epochs()

Index opennn::Optimizer::get_maximum_epochs ( ) const

inlinenoexcept

◆ get_name()

const string & opennn::Optimizer::get_name ( ) const

inlinenoexcept

◆ load()

void opennn::Optimizer::load ( const filesystem::path & )

◆ network_has_active_dropout()

bool opennn::Optimizer::network_has_active_dropout ( ) const

protected

◆ on_epoch_begin()

virtual void opennn::Optimizer::on_epoch_begin	(	Index	,
		OptimizerData &	)

inlineprotectedvirtual

Reimplemented in opennn::Adam.

◆ prefetch_batch()

void opennn::Optimizer::prefetch_batch ( Batch & )

protected

◆ prepare_full_batch_training()

void opennn::Optimizer::prepare_full_batch_training	(	FullBatchContext &	,
		const char *	banner )

protected

◆ prepare_training_artifacts()

void opennn::Optimizer::prepare_training_artifacts ( )

protected

◆ read_common_json()

void opennn::Optimizer::read_common_json ( const Json * )

protected

◆ restore_best_parameters()

void opennn::Optimizer::restore_best_parameters	(	Network *	,
		TrainingResult &	,
		const BestModelSnapshot &	)

protected

◆ run_epoch_loop()

Loss::EvaluationResult opennn::Optimizer::run_epoch_loop ( EpochLoopContext & )

protected

◆ run_graph_epoch()

Loss::EvaluationResult opennn::Optimizer::run_graph_epoch	(	TrainingSession &	,
		OptimizerData &	,
		ForwardPropagation &	,
		BackPropagation &	,
		ThreadSafeQueue< Batch * > &	,
		const vector< vector< Index > > &	,
		const FeatureSelection &	)

protected

◆ save()

void opennn::Optimizer::save ( const filesystem::path & ) const

◆ set()

void opennn::Optimizer::set ( Loss * new_loss )

inline

◆ set_batch_pool_size()

void opennn::Optimizer::set_batch_pool_size ( int size )

inline

◆ set_batch_size()

void opennn::Optimizer::set_batch_size ( const Index new_batch_size )

inline

◆ set_cuda_graph()

void opennn::Optimizer::set_cuda_graph ( bool enabled )

inline

◆ set_display()

virtual void opennn::Optimizer::set_display ( bool new_display )

inlinevirtual

◆ set_display_period()

void opennn::Optimizer::set_display_period ( const Index new_display_period )

inline

◆ set_gradient_clip_norm()

void opennn::Optimizer::set_gradient_clip_norm ( const float new_clip )

inline

◆ set_joint_gradient_arena()

void opennn::Optimizer::set_joint_gradient_arena ( bool enabled )

inlinenoexcept

◆ set_loss_goal()

void opennn::Optimizer::set_loss_goal ( const float new_loss_goal )

inline

◆ set_maximum_epochs()

void opennn::Optimizer::set_maximum_epochs ( const Index new_maximum_epochs )

inline

◆ set_maximum_time()

void opennn::Optimizer::set_maximum_time ( const float new_maximum_time )

inline

◆ set_maximum_validation_failures()

void opennn::Optimizer::set_maximum_validation_failures ( const Index new_maximum_validation_failures )

inline

◆ set_names()

void opennn::Optimizer::set_names ( )

protected

◆ set_restore_best()

void opennn::Optimizer::set_restore_best ( bool enabled )

inline

◆ set_shuffle()

void opennn::Optimizer::set_shuffle ( bool enabled )

inline

◆ set_validation_period()

void opennn::Optimizer::set_validation_period ( const Index n )

inline

◆ set_workers_number()

void opennn::Optimizer::set_workers_number ( int new_workers_number )

inline

◆ setup_batch_pools()

void opennn::Optimizer::setup_batch_pools	(	BatchPools &	pools,
		Dataset &	dataset,
		Network &	network,
		Index	training_batch_size,
		Index	validation_batch_size,
		bool	has_validation,
		TrainingSession &	training_session )

protected

◆ setup_device_training()

void opennn::Optimizer::setup_device_training ( )

protected

◆ setup_optimizer_data()

virtual void opennn::Optimizer::setup_optimizer_data	(	OptimizerData &	,
		Index	,
		Device	)

inlineprotectedvirtual

◆ should_display()

bool opennn::Optimizer::should_display ( Index epoch ) const

inlineprotectednoexcept

◆ start_batch_prefetch()

unique_ptr< BatchPrefetchSession > opennn::Optimizer::start_batch_prefetch	(	ThreadSafeQueue< Batch * > &	,
		const vector< vector< Index > > &	,
		const FeatureSelection &	,
		FillMode	,
		WorkerProfileCounters *	profile_counters = nullptr )

protected

◆ supports_cuda_graph()

virtual bool opennn::Optimizer::supports_cuda_graph ( ) const

inlineprotectedvirtualnoexcept

◆ sync_device()

void opennn::Optimizer::sync_device	(	bool	on_gpu,
		bool	has_recurrent_layers,
		TrainingSession &	)

protected

◆ teardown_device_training()

void opennn::Optimizer::teardown_device_training ( )

protected

◆ to_JSON()

virtual void opennn::Optimizer::to_JSON ( JsonWriter & ) const

virtual

Reimplemented in opennn::Adam, opennn::LevenbergMarquardt, opennn::QuasiNewton, and opennn::SGD.

◆ train()

virtual TrainingResult opennn::Optimizer::train ( )

virtual

Reimplemented in opennn::LevenbergMarquardt, and opennn::QuasiNewton.

◆ train_epoch()

Loss::EvaluationResult opennn::Optimizer::train_epoch	(	TrainingContext &	,
		ThreadSafeQueue< Batch * > &	,
		const vector< vector< Index > > &	,
		const FeatureSelection &	,
		TrainingSession &	,
		OptimizerData &	)

protected

◆ train_full_batch()

TrainingResult opennn::Optimizer::train_full_batch	(	FullBatchContext &	,
		const FullBatchHooks &	)

protected

◆ update_best_parameters()

void opennn::Optimizer::update_best_parameters	(	Network *	,
		float	,
		Index	,
		Index &	,
		BestModelSnapshot &	)

protected

◆ update_parameters()

virtual void opennn::Optimizer::update_parameters	(	BackPropagation &	,
		OptimizerData &	,
		UpdateMode	= UpdateMode::Standard )

inlineprotectedvirtual

Reimplemented in opennn::Adam, and opennn::SGD.

◆ uses_joint_gradient_arena()

bool opennn::Optimizer::uses_joint_gradient_arena ( ) const

protectednoexcept

◆ warmup_device_training()

void opennn::Optimizer::warmup_device_training	(	TrainingContext &	,
		ThreadSafeQueue< Batch * > &	,
		const vector< vector< Index > > &	,
		const FeatureSelection &	,
		TrainingSession &	,
		OptimizerData &	,
		ForwardPropagation *	validation_forward_propagation = nullptr,
		ThreadSafeQueue< Batch * > *	validation_empty_queue = nullptr,
		const vector< vector< Index > > *	validation_batches = nullptr )

protected

◆ write_common_json()

void opennn::Optimizer::write_common_json ( JsonWriter & ) const

protected

Member Data Documentation

◆ batch_pool_size_override

int opennn::Optimizer::batch_pool_size_override = 0

protected

◆ batch_size

Index opennn::Optimizer::batch_size = 0

protected

◆ cuda_graph_capture_failed

bool opennn::Optimizer::cuda_graph_capture_failed = false

protected

◆ default_workers_number

const int opennn::Optimizer::default_workers_number

staticprotected

◆ display

bool opennn::Optimizer::display = true

protected

◆ display_period

Index opennn::Optimizer::display_period = 10

protected

◆ gradient_clip_norm

float opennn::Optimizer::gradient_clip_norm = 0.0f

protected

◆ joint_gradient_arena

bool opennn::Optimizer::joint_gradient_arena = false

protected

◆ loss

Loss* opennn::Optimizer::loss = nullptr

protected

◆ maximum_epochs

Index opennn::Optimizer::maximum_epochs = 10000

protected

◆ maximum_time

float opennn::Optimizer::maximum_time = 360000.0f

protected

◆ maximum_validation_failures

Index opennn::Optimizer::maximum_validation_failures = numeric_limits<Index>::max()

protected

◆ name

string opennn::Optimizer::name

protected

◆ post_batch_callback

function<void(Network*)> opennn::Optimizer::post_batch_callback

◆ post_best_callback

function<void(Index, float)> opennn::Optimizer::post_best_callback

◆ post_epoch_callback

function<void(Index, float, float, Network*)> opennn::Optimizer::post_epoch_callback

◆ restore_best

bool opennn::Optimizer::restore_best = true

protected

◆ shuffle_samples

bool opennn::Optimizer::shuffle_samples = true

protected

◆ training_loss_goal

float opennn::Optimizer::training_loss_goal = 0.0f

protected

◆ use_cuda_graph

bool opennn::Optimizer::use_cuda_graph = false

protected

◆ validation_period

Index opennn::Optimizer::validation_period = 1

protected

◆ workers_number

int opennn::Optimizer::workers_number = default_workers_number

protected

Classes

Public Types

Public Member Functions

Public Attributes

Protected Types

Protected Member Functions

Static Protected Member Functions

Protected Attributes

Static Protected Attributes

Member Typedef Documentation

◆ StoppingCondition

Member Enumeration Documentation

◆ UpdateMode

Constructor & Destructor Documentation

◆ Optimizer()

◆ ~Optimizer()

Member Function Documentation

◆ can_use_cuda_graph()

◆ clip_gradient_norm()

◆ close_training_result()

◆ configure_for_task()

◆ display_epoch_results()

◆ evaluate_epoch()

◆ find_stopping_condition()

◆ from_JSON()

◆ get_batch_pool_size()

◆ get_batch_size()

◆ get_cuda_graph_capture_failed()

◆ get_display()

◆ get_display_name()

◆ get_epoch_budget()

◆ get_loss()

◆ get_maximum_batch_size()

◆ get_maximum_epochs()

◆ get_name()

◆ load()

◆ network_has_active_dropout()

◆ on_epoch_begin()

◆ prefetch_batch()

◆ prepare_full_batch_training()

◆ prepare_training_artifacts()

◆ read_common_json()

◆ restore_best_parameters()

◆ run_epoch_loop()

◆ run_graph_epoch()

◆ save()

◆ set()

◆ set_batch_pool_size()

◆ set_batch_size()

◆ set_cuda_graph()

◆ set_display()

◆ set_display_period()

◆ set_gradient_clip_norm()

◆ set_joint_gradient_arena()

◆ set_loss_goal()

◆ set_maximum_epochs()

◆ set_maximum_time()

◆ set_maximum_validation_failures()

◆ set_names()

◆ set_restore_best()

◆ set_shuffle()

◆ set_validation_period()

◆ set_workers_number()

◆ setup_batch_pools()

◆ setup_device_training()

◆ setup_optimizer_data()

◆ should_display()

◆ start_batch_prefetch()

◆ supports_cuda_graph()

◆ sync_device()

◆ teardown_device_training()

◆ to_JSON()

◆ train()

◆ train_epoch()

◆ train_full_batch()

◆ update_best_parameters()

◆ update_parameters()

◆ uses_joint_gradient_arena()

◆ warmup_device_training()

◆ write_common_json()