#include <conjugate_gradient.h>

Inheritance diagram for ConjugateGradient:

Public Types
enum class	TrainingDirectionMethod { PR , FR }
	Enumeration of the available training operators for obtaining the training direction. More...

Public Types inherited from OptimizationAlgorithm
enum class	StoppingCondition { MinimumLossDecrease , LossGoal , MaximumSelectionErrorIncreases , MaximumEpochsNumber , MaximumTime }
	Enumeration of all possibles condition of stop for the algorithms. More...

Public Member Functions
	ConjugateGradient ()

	ConjugateGradient (LossIndex *)

virtual	~ConjugateGradient ()
	Destructor. More...

const LearningRateAlgorithm &	get_learning_rate_algorithm () const
	Returns a constant reference to the learning rate algorithm object inside the conjugate gradient method object. More...

LearningRateAlgorithm *	get_learning_rate_algorithm_pointer ()
	Returns a pointer to the learning rate algorithm object inside the conjugate gradient method object. More...

const TrainingDirectionMethod &	get_training_direction_method () const
	Returns the conjugate gradient training direction method used for training. More...

string	write_training_direction_method () const
	Returns a string with the name of the training direction. More...

const type &	get_minimum_loss_decrease () const
	Returns the minimum loss improvement during training. More...

const type &	get_loss_goal () const

const Index &	get_maximum_selection_failures () const
	Returns the maximum number of selection error increases during the training process. More...

const Index &	get_maximum_epochs_number () const
	Returns the maximum number of epochs for training. More...

const type &	get_maximum_time () const
	Returns the maximum training time. More...

void	set_default ()

void	set_loss_index_pointer (LossIndex *)

void	set_training_direction_method (const TrainingDirectionMethod &)

void	set_training_direction_method (const string &)

void	set_loss_goal (const type &)

void	set_minimum_loss_decrease (const type &)

void	set_maximum_selection_failures (const Index &)

void	set_maximum_epochs_number (const Index &)

void	set_maximum_time (const type &)

void	set_save_period (const Index &)

type	calculate_PR_parameter (const Tensor< type, 1 > &, const Tensor< type, 1 > &) const

type	calculate_FR_parameter (const Tensor< type, 1 > &, const Tensor< type, 1 > &) const

void	calculate_PR_training_direction (const Tensor< type, 1 > &, const Tensor< type, 1 > &, const Tensor< type, 1 > &, Tensor< type, 1 > &) const

void	calculate_FR_training_direction (const Tensor< type, 1 > &, const Tensor< type, 1 > &, const Tensor< type, 1 > &, Tensor< type, 1 > &) const

void	calculate_gradient_descent_training_direction (const Tensor< type, 1 > &, Tensor< type, 1 > &) const

void	calculate_conjugate_gradient_training_direction (const Tensor< type, 1 > &, const Tensor< type, 1 > &, const Tensor< type, 1 > &, Tensor< type, 1 > &) const

TrainingResults	perform_training ()

string	write_optimization_algorithm_type () const
	Write a string with best algorithm type for the model. More...

Tensor< string, 2 >	to_string_matrix () const
	Writes as matrix of strings the most representative atributes. More...

void	from_XML (const tinyxml2::XMLDocument &)

void	write_XML (tinyxml2::XMLPrinter &) const

void	update_parameters (const DataSetBatch &batch, NeuralNetworkForwardPropagation &forward_propagation, LossIndexBackPropagation &back_propagation, ConjugateGradientData &optimization_data)
	ConjugateGradient::update_parameters. More...

Public Member Functions inherited from OptimizationAlgorithm
	OptimizationAlgorithm ()

	OptimizationAlgorithm (LossIndex *)

virtual	~OptimizationAlgorithm ()
	Destructor. More...

LossIndex *	get_loss_index_pointer () const

string	get_hardware_use () const
	Hardware use. More...

void	set_hardware_use (const string &)
	Set hardware to use. Default: Multi-core. More...

bool	has_loss_index () const

const bool &	get_display () const

const Index &	get_display_period () const
	Returns the number of iterations between the training showing progress. More...

const Index &	get_save_period () const
	Returns the number of iterations between the training saving progress. More...

const string &	get_neural_network_file_name () const
	Returns the file name where the neural network will be saved. More...

const string	write_time (const type &) const
	Writes the time from seconds in format HH:mm:ss. More...

void	set ()

virtual void	set_threads_number (const int &)

virtual void	set_display (const bool &)

void	set_display_period (const Index &)

void	set_save_period (const Index &)

void	set_neural_network_file_name (const string &)

virtual void	check () const

virtual void	print () const
	Prints to the screen the XML-type representation of the optimization algorithm object. More...

void	save (const string &) const

void	load (const string &)

Private Attributes
type	first_learning_rate = static_cast<type>(0.01)

TrainingDirectionMethod	training_direction_method = ConjugateGradient::TrainingDirectionMethod::FR
	Applied method for calculating the conjugate gradient direction. More...

LearningRateAlgorithm	learning_rate_algorithm
	Learning rate algorithm object for one-dimensional minimization. More...

type	minimum_loss_decrease = type(0)
	Minimum loss improvement between two successive iterations. It is used as a stopping criterion. More...

type	training_loss_goal = type(0)
	Goal value for the loss. It is used as a stopping criterion. More...

Index	maximum_selection_failures

Index	maximum_epochs_number
	Maximum number of epochs to perform_training. It is used as a stopping criterion. More...

type	maximum_time
	Maximum training time. It is used as a stopping criterion. More...

Additional Inherited Members
Protected Attributes inherited from OptimizationAlgorithm
NonBlockingThreadPool *	non_blocking_thread_pool = nullptr

ThreadPoolDevice *	thread_pool_device

LossIndex *	loss_index_pointer = nullptr
	Pointer to a loss index for a neural network object. More...

Index	epochs_number = 10000
	Number of training epochs in the neural network. More...

string	hardware_use = "Multi-core"
	Hardware use. More...

Index	display_period = 10
	Number of iterations between the training showing progress. More...

Index	save_period = numeric_limits<Index>::max()
	Number of iterations between the training saving progress. More...

string	neural_network_file_name = "neural_network.xml"
	Path where the neural network is saved. More...

bool	display = true
	Display messages to screen. More...

const Eigen::array< IndexPair< Index >, 1 >	AT_B = {IndexPair<Index>(0, 0)}

const Eigen::array< IndexPair< Index >, 1 >	product_vector_matrix = {IndexPair<Index>(0, 1)}

const Eigen::array< IndexPair< Index >, 1 >	A_B = {IndexPair<Index>(1, 0)}

Detailed Description

In the conjugate gradient algorithms a search is performed along conjugate directions, which produces generally faster convergence than a search along the steepest descent directions. This concrete class represents a conjugate gradient training algorithm, based on solving sparse systems.

[1] https://www.neuraldesigner.com/blog/5_algorithms_to_train_a_neural_network

[2] D.P. O'Leary "The Block Conjugate Gradient Algorithm and Related Methods."

Definition at line 45 of file conjugate_gradient.h.

Member Enumeration Documentation

◆ TrainingDirectionMethod

enum class TrainingDirectionMethod

strong

Enumeration of the available training operators for obtaining the training direction.

Definition at line 54 of file conjugate_gradient.h.

Constructor & Destructor Documentation

◆ ConjugateGradient() [1/2]

ConjugateGradient ( )

explicit

Default constructor. It creates a conjugate gradient optimization algorithm object not associated to any loss index object. It also initializes the class members to their default values.

Definition at line 19 of file conjugate_gradient.cpp.

◆ ConjugateGradient() [2/2]

ConjugateGradient ( LossIndex * new_loss_index_pointer )

explicit

Destructor. It creates a conjugate gradient optimization algorithm associated to a loss index object. It also initializes the rest of class members to their default values.

Parameters

new_loss_index_pointer Pointer to a loss index object.

Definition at line 31 of file conjugate_gradient.cpp.

◆ ~ConjugateGradient()

~ConjugateGradient ( )

virtual

Destructor.

Definition at line 42 of file conjugate_gradient.cpp.

Member Function Documentation

◆ calculate_conjugate_gradient_training_direction()

void calculate_conjugate_gradient_training_direction	(	const Tensor< type, 1 > &	old_gradient,
		const Tensor< type, 1 > &	gradient,
		const Tensor< type, 1 > &	old_training_direction,
		Tensor< type, 1 > &	training_direction
	)		const

Returns the conjugate gradient training direction.

Parameters

old_gradient	Gradient vector in the previous iteration.
gradient	Current gradient vector.
old_training_direction	Training direction in the previous iteration.

Definition at line 641 of file conjugate_gradient.cpp.

◆ calculate_FR_parameter()

type calculate_FR_parameter	(	const Tensor< type, 1 > &	old_gradient,
		const Tensor< type, 1 > &	gradient
	)		const

Returns the Fletcher-Reeves parameter used to calculate the training direction.

Parameters

old_gradient	Previous error function gradient.
gradient	Current error function gradient.

Definition at line 327 of file conjugate_gradient.cpp.

◆ calculate_FR_training_direction()

void calculate_FR_training_direction	(	const Tensor< type, 1 > &	old_gradient,
		const Tensor< type, 1 > &	gradient,
		const Tensor< type, 1 > &	old_training_direction,
		Tensor< type, 1 > &	training_direction
	)		const

Returns the training direction using the Fletcher-Reeves update.

Parameters

old_gradient	Previous error function gradient.
gradient	Current error function gradient.
old_training_direction	Previous training direction vector.

Definition at line 561 of file conjugate_gradient.cpp.

◆ calculate_gradient_descent_training_direction()

void calculate_gradient_descent_training_direction	(	const Tensor< type, 1 > &	gradient,
		Tensor< type, 1 > &	training_direction
	)		const

Definition at line 630 of file conjugate_gradient.cpp.

◆ calculate_PR_parameter()

type calculate_PR_parameter	(	const Tensor< type, 1 > &	old_gradient,
		const Tensor< type, 1 > &	gradient
	)		const

Returns the Polak-Ribiere parameter used to calculate the training direction.

Parameters

old_gradient	Previous error function gradient.
gradient	Current error function gradient.

Definition at line 409 of file conjugate_gradient.cpp.

◆ calculate_PR_training_direction()

void calculate_PR_training_direction	(	const Tensor< type, 1 > &	old_gradient,
		const Tensor< type, 1 > &	gradient,
		const Tensor< type, 1 > &	old_training_direction,
		Tensor< type, 1 > &	training_direction
	)		const

Returns the training direction using the Polak-Ribiere update.

Parameters

old_gradient	Previous error function gradient.
gradient	Current error function gradient.
old_training_direction	Previous training direction vector.

Definition at line 492 of file conjugate_gradient.cpp.

◆ from_XML()

void from_XML ( const tinyxml2::XMLDocument & document )

virtual

Deserializes the conjugate gradient object from a XML document of the TinyXML library.

Parameters

document TinyXML document containing the member data.

Reimplemented from OptimizationAlgorithm.

Definition at line 1086 of file conjugate_gradient.cpp.

◆ get_learning_rate_algorithm()

const LearningRateAlgorithm & get_learning_rate_algorithm ( ) const

Returns a constant reference to the learning rate algorithm object inside the conjugate gradient method object.

Definition at line 49 of file conjugate_gradient.cpp.

◆ get_learning_rate_algorithm_pointer()

LearningRateAlgorithm * get_learning_rate_algorithm_pointer ( )

Returns a pointer to the learning rate algorithm object inside the conjugate gradient method object.

Definition at line 57 of file conjugate_gradient.cpp.

◆ get_loss_goal()

const type & get_loss_goal ( ) const

Returns the goal value for the loss. This is used as a stopping criterion when training a neural network

Definition at line 99 of file conjugate_gradient.cpp.

◆ get_maximum_epochs_number()

const Index & get_maximum_epochs_number ( ) const

Returns the maximum number of epochs for training.

Definition at line 115 of file conjugate_gradient.cpp.

◆ get_maximum_selection_failures()

const Index & get_maximum_selection_failures ( ) const

Returns the maximum number of selection error increases during the training process.

Definition at line 107 of file conjugate_gradient.cpp.

◆ get_maximum_time()

const type & get_maximum_time ( ) const

Returns the maximum training time.

Definition at line 123 of file conjugate_gradient.cpp.

◆ get_minimum_loss_decrease()

const type & get_minimum_loss_decrease ( ) const

Returns the minimum loss improvement during training.

Definition at line 90 of file conjugate_gradient.cpp.

◆ get_training_direction_method()

const ConjugateGradient::TrainingDirectionMethod & get_training_direction_method ( ) const

Returns the conjugate gradient training direction method used for training.

Definition at line 65 of file conjugate_gradient.cpp.

◆ perform_training()

TrainingResults perform_training ( )

virtual

Trains a neural network with an associated loss index according to the conjugate gradient algorithm. Training occurs according to the training operators, training parameters and stopping criteria.

Implements OptimizationAlgorithm.

Definition at line 715 of file conjugate_gradient.cpp.

◆ set_default()

void set_default ( )

virtual

Sets the default values into a conjugate gradient object. Training operators:

Training direction method = Polak-Ribiere;
Learning rate method = Brent;

Training parameters:

First learning rate: 1.0.
Bracketing factor: 2.0.
Learning rate tolerance: 1.0e-3.

Stopping criteria:

Loss goal: -numeric_limits<type>::max().
Maximum training time: 1.0e6.
Maximum number of epochs: 100.

User stuff:

Warning learning rate: 1.0e6.
Error learning rate: 1.0e12.
Display: true.
Display period: 10.
Save period: 0.

Reserve:

Reserve training error history: false.
Reserve training direction norm history: false.

Reimplemented from OptimizationAlgorithm.

Definition at line 216 of file conjugate_gradient.cpp.

◆ set_loss_goal()

void set_loss_goal ( const type & new_loss_goal )

Sets a new goal value for the loss. This is used as a stopping criterion when training a neural network

Parameters

new_loss_goal Goal value for the loss.

Definition at line 248 of file conjugate_gradient.cpp.

◆ set_loss_index_pointer()

void set_loss_index_pointer ( LossIndex * new_loss_index_pointer )

virtual

Sets a pointer to a loss index object to be associated to the conjugate gradient object. It also sets that loss index to the learning rate algorithm.

Parameters

new_loss_index_pointer Pointer to a loss index object.

Reimplemented from OptimizationAlgorithm.

Definition at line 133 of file conjugate_gradient.cpp.

◆ set_maximum_epochs_number()

void set_maximum_epochs_number ( const Index & new_maximum_epochs_number )

Sets a maximum number of epochs for training.

Parameters

new_maximum_iterations_number Maximum number of epochs for training.

Definition at line 266 of file conjugate_gradient.cpp.

◆ set_maximum_selection_failures()

void set_maximum_selection_failures ( const Index & new_maximum_selection_failures )

Sets a new maximum number of selection error increases.

Parameters

new_maximum_selection_failures Maximum number of epochs in which the selection evalutation increases.

Definition at line 257 of file conjugate_gradient.cpp.

◆ set_maximum_time()

void set_maximum_time ( const type & new_maximum_time )

Sets a new maximum training time.

Parameters

new_maximum_time Maximum training time.

Definition at line 275 of file conjugate_gradient.cpp.

◆ set_minimum_loss_decrease()

void set_minimum_loss_decrease ( const type & new_minimum_loss_decrease )

Sets a new minimum loss improvement during training.

Parameters

new_minimum_loss_decrease Minimum improvement in the loss between two iterations.

Definition at line 238 of file conjugate_gradient.cpp.

◆ set_save_period()

void set_save_period ( const Index & new_save_period )

Sets a new number of iterations between the training saving progress.

Parameters

new_save_period Number of iterations between the training saving progress.

Definition at line 302 of file conjugate_gradient.cpp.

◆ set_training_direction_method() [1/2]

void set_training_direction_method ( const string & new_training_direction_method_name )

Sets a new conjugate gradient training direction from a string representation. Possible values are:

"PR"
"FR"

Parameters

new_training_direction_method_name String with the name of the training direction method.

Definition at line 159 of file conjugate_gradient.cpp.

◆ set_training_direction_method() [2/2]

void set_training_direction_method ( const TrainingDirectionMethod & new_training_direction_method )

Sets a new training direction method to be used for training.

Parameters

new_training_direction_method Conjugate gradient training direction method.

Definition at line 144 of file conjugate_gradient.cpp.

◆ to_string_matrix()

Tensor< string, 2 > to_string_matrix ( ) const

virtual

Writes as matrix of strings the most representative atributes.

Reimplemented from OptimizationAlgorithm.

Definition at line 933 of file conjugate_gradient.cpp.

◆ update_parameters()

void update_parameters	(	const DataSetBatch &	batch,
		NeuralNetworkForwardPropagation &	forward_propagation,
		LossIndexBackPropagation &	back_propagation,
		ConjugateGradientData &	optimization_data
	)

ConjugateGradient::update_parameters.

Parameters

batch
forward_propagation
back_propagation
optimization_data

Definition at line 1335 of file conjugate_gradient.cpp.

◆ write_optimization_algorithm_type()

string write_optimization_algorithm_type ( ) const

virtual

Write a string with best algorithm type for the model.

Reimplemented from OptimizationAlgorithm.

Definition at line 925 of file conjugate_gradient.cpp.

◆ write_training_direction_method()

string write_training_direction_method ( ) const

Returns a string with the name of the training direction.

Definition at line 73 of file conjugate_gradient.cpp.

◆ write_XML()

void write_XML ( tinyxml2::XMLPrinter & file_stream ) const

virtual

Serializes the conjugate gradient object into a XML document of the TinyXML library without keep the DOM tree in memory. See the OpenNN manual for more information about the format of this element.

Reimplemented from OptimizationAlgorithm.

Definition at line 984 of file conjugate_gradient.cpp.

Member Data Documentation

◆ first_learning_rate

type first_learning_rate = static_cast<type>(0.01)

private

Definition at line 150 of file conjugate_gradient.h.

◆ learning_rate_algorithm

LearningRateAlgorithm learning_rate_algorithm

private

Learning rate algorithm object for one-dimensional minimization.

Definition at line 158 of file conjugate_gradient.h.

◆ maximum_epochs_number

Index maximum_epochs_number

private

Maximum number of epochs to perform_training. It is used as a stopping criterion.

Definition at line 177 of file conjugate_gradient.h.

◆ maximum_selection_failures

Index maximum_selection_failures

private

Maximum number of epochs at which the selection error increases. This is an early stopping method for improving selection.

Definition at line 173 of file conjugate_gradient.h.

◆ maximum_time

type maximum_time

private

Maximum training time. It is used as a stopping criterion.

Definition at line 181 of file conjugate_gradient.h.

◆ minimum_loss_decrease

type minimum_loss_decrease = type(0)

private

Minimum loss improvement between two successive iterations. It is used as a stopping criterion.

Definition at line 164 of file conjugate_gradient.h.

◆ training_direction_method

TrainingDirectionMethod training_direction_method = ConjugateGradient::TrainingDirectionMethod::FR

private

Applied method for calculating the conjugate gradient direction.

Definition at line 154 of file conjugate_gradient.h.

◆ training_loss_goal

type training_loss_goal = type(0)

private

Goal value for the loss. It is used as a stopping criterion.

Definition at line 168 of file conjugate_gradient.h.

The documentation for this class was generated from the following files:

Public Types

Public Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Member Enumeration Documentation

◆ TrainingDirectionMethod

Constructor & Destructor Documentation

◆ ConjugateGradient() [1/2]

◆ ConjugateGradient() [2/2]

◆ ~ConjugateGradient()

Member Function Documentation

◆ calculate_conjugate_gradient_training_direction()

◆ calculate_FR_parameter()

◆ calculate_FR_training_direction()

◆ calculate_gradient_descent_training_direction()

◆ calculate_PR_parameter()

◆ calculate_PR_training_direction()

◆ from_XML()

◆ get_learning_rate_algorithm()

◆ get_learning_rate_algorithm_pointer()

◆ get_loss_goal()

◆ get_maximum_epochs_number()

◆ get_maximum_selection_failures()

◆ get_maximum_time()

◆ get_minimum_loss_decrease()

◆ get_training_direction_method()

◆ perform_training()

◆ set_default()

◆ set_loss_goal()

◆ set_loss_index_pointer()

◆ set_maximum_epochs_number()

◆ set_maximum_selection_failures()

◆ set_maximum_time()

◆ set_minimum_loss_decrease()

◆ set_save_period()

◆ set_training_direction_method() [1/2]

◆ set_training_direction_method() [2/2]

◆ to_string_matrix()

◆ update_parameters()

◆ write_optimization_algorithm_type()

◆ write_training_direction_method()

◆ write_XML()

Member Data Documentation

◆ first_learning_rate

◆ learning_rate_algorithm

◆ maximum_epochs_number

◆ maximum_selection_failures

◆ maximum_time

◆ minimum_loss_decrease

◆ training_direction_method

◆ training_loss_goal