#include <quasi_newton_method.h>

Inheritance diagram for QuasiNewtonMethod:

Public Types
enum class	InverseHessianApproximationMethod { DFP , BFGS }
	Enumeration of the available training operators for obtaining the approximation to the inverse hessian. More...

Public Types inherited from OptimizationAlgorithm
enum class	StoppingCondition { MinimumLossDecrease , LossGoal , MaximumSelectionErrorIncreases , MaximumEpochsNumber , MaximumTime }
	Enumeration of all possibles condition of stop for the algorithms. More...

Public Member Functions
	QuasiNewtonMethod ()

	QuasiNewtonMethod (LossIndex *)

virtual	~QuasiNewtonMethod ()

const LearningRateAlgorithm &	get_learning_rate_algorithm () const
	Returns a constant reference to the learning rate algorithm object inside the quasi-Newton method object. More...

LearningRateAlgorithm *	get_learning_rate_algorithm_pointer ()
	Returns a pointer to the learning rate algorithm object inside the quasi-Newton method object. More...

const InverseHessianApproximationMethod &	get_inverse_hessian_approximation_method () const
	Returns the method for approximating the inverse hessian matrix to be used when training. More...

string	write_inverse_hessian_approximation_method () const
	Returns the name of the method for the approximation of the inverse hessian. More...

const Index &	get_epochs_number () const

const type &	get_minimum_loss_decrease () const
	Returns the minimum loss improvement during training. More...

const type &	get_loss_goal () const

const Index &	get_maximum_selection_failures () const
	Returns the maximum number of selection error increases during the training process. More...

const Index &	get_maximum_epochs_number () const
	Returns the maximum number of epochs for training. More...

const type &	get_maximum_time () const
	Returns the maximum training time. More...

void	set_loss_index_pointer (LossIndex *)

void	set_inverse_hessian_approximation_method (const InverseHessianApproximationMethod &)

void	set_inverse_hessian_approximation_method (const string &)

void	set_display (const bool &)

void	set_default ()
	Sets the members of the optimization algorithm object to their default values. More...

void	set_minimum_loss_decrease (const type &)

void	set_loss_goal (const type &)

void	set_maximum_selection_failures (const Index &)

void	set_maximum_epochs_number (const Index &)

void	set_maximum_time (const type &)

void	calculate_DFP_inverse_hessian (QuasiNewtonMehtodData &) const

void	calculate_BFGS_inverse_hessian (QuasiNewtonMehtodData &) const

void	initialize_inverse_hessian_approximation (QuasiNewtonMehtodData &) const

void	calculate_inverse_hessian_approximation (QuasiNewtonMehtodData &) const

const Tensor< type, 2 >	kronecker_product (Tensor< type, 2 > &, Tensor< type, 2 > &) const

const Tensor< type, 2 >	kronecker_product (Tensor< type, 1 > &, Tensor< type, 1 > &) const

void	update_parameters (const DataSetBatch &batch, NeuralNetworkForwardPropagation &forward_propagation, LossIndexBackPropagation &back_propagation, QuasiNewtonMehtodData &optimization_data)
	QuasiNewtonMethod::update_parameters. More...

TrainingResults	perform_training ()

string	write_optimization_algorithm_type () const

void	from_XML (const tinyxml2::XMLDocument &)

void	write_XML (tinyxml2::XMLPrinter &) const

Tensor< string, 2 >	to_string_matrix () const
	Writes as matrix of strings the most representative atributes. More...

Public Member Functions inherited from OptimizationAlgorithm
	OptimizationAlgorithm ()

	OptimizationAlgorithm (LossIndex *)

virtual	~OptimizationAlgorithm ()
	Destructor. More...

LossIndex *	get_loss_index_pointer () const

string	get_hardware_use () const
	Hardware use. More...

void	set_hardware_use (const string &)
	Set hardware to use. Default: Multi-core. More...

bool	has_loss_index () const

const bool &	get_display () const

const Index &	get_display_period () const
	Returns the number of iterations between the training showing progress. More...

const Index &	get_save_period () const
	Returns the number of iterations between the training saving progress. More...

const string &	get_neural_network_file_name () const
	Returns the file name where the neural network will be saved. More...

const string	write_time (const type &) const
	Writes the time from seconds in format HH:mm:ss. More...

void	set ()

virtual void	set_threads_number (const int &)

void	set_display_period (const Index &)

void	set_save_period (const Index &)

void	set_neural_network_file_name (const string &)

virtual void	check () const

virtual void	print () const
	Prints to the screen the XML-type representation of the optimization algorithm object. More...

void	save (const string &) const

void	load (const string &)

Private Attributes
LearningRateAlgorithm	learning_rate_algorithm

InverseHessianApproximationMethod	inverse_hessian_approximation_method
	Variable containing the actual method used to obtain a suitable learning rate. More...

type	first_learning_rate = static_cast<type>(0.01)

type	minimum_loss_decrease
	Minimum loss improvement between two successive epochs. It is used as a stopping criterion. More...

type	training_loss_goal
	Goal value for the loss. It is used as a stopping criterion. More...

Index	maximum_selection_failures

Index	maximum_epochs_number
	Maximum number of epochs to perform_training. It is used as a stopping criterion. More...

type	maximum_time
	Maximum training time. It is used as a stopping criterion. More...

Additional Inherited Members
Protected Attributes inherited from OptimizationAlgorithm
NonBlockingThreadPool *	non_blocking_thread_pool = nullptr

ThreadPoolDevice *	thread_pool_device

LossIndex *	loss_index_pointer = nullptr
	Pointer to a loss index for a neural network object. More...

Index	epochs_number = 10000
	Number of training epochs in the neural network. More...

string	hardware_use = "Multi-core"
	Hardware use. More...

Index	display_period = 10
	Number of iterations between the training showing progress. More...

Index	save_period = numeric_limits<Index>::max()
	Number of iterations between the training saving progress. More...

string	neural_network_file_name = "neural_network.xml"
	Path where the neural network is saved. More...

bool	display = true
	Display messages to screen. More...

const Eigen::array< IndexPair< Index >, 1 >	AT_B = {IndexPair<Index>(0, 0)}

const Eigen::array< IndexPair< Index >, 1 >	product_vector_matrix = {IndexPair<Index>(0, 1)}

const Eigen::array< IndexPair< Index >, 1 >	A_B = {IndexPair<Index>(1, 0)}

Detailed Description

Class of optimization algorithm based on Newton's method. An approximate Hessian matrix is computed at each iteration of the algorithm based on the gradients. This concrete class represents a quasi-Newton training algorithm[1], used to minimize loss function.

[1] Neural Designer "5 Algorithms to Train a Neural Network." https://www.neuraldesigner.com/blog/5_algorithms_to_train_a_neural_network

Definition at line 52 of file quasi_newton_method.h.

Member Enumeration Documentation

◆ InverseHessianApproximationMethod

enum class InverseHessianApproximationMethod

strong

Enumeration of the available training operators for obtaining the approximation to the inverse hessian.

Definition at line 61 of file quasi_newton_method.h.

Constructor & Destructor Documentation

◆ QuasiNewtonMethod() [1/2]

QuasiNewtonMethod ( )

explicit

Default constructor. It creates a quasi-Newton method optimization algorithm not associated to any loss index. It also initializes the class members to their default values.

Definition at line 18 of file quasi_newton_method.cpp.

◆ QuasiNewtonMethod() [2/2]

QuasiNewtonMethod ( LossIndex * new_loss_index_pointer )

explicit

Loss index constructor. It creates a quasi-Newton method optimization algorithm associated to a loss index. It also initializes the class members to their default values.

Parameters

new_loss_index_pointer Pointer to a loss index object.

Definition at line 30 of file quasi_newton_method.cpp.

◆ ~QuasiNewtonMethod()

~QuasiNewtonMethod ( )

virtual

Destructor. It does not delete any object.

Definition at line 42 of file quasi_newton_method.cpp.

Member Function Documentation

◆ calculate_BFGS_inverse_hessian()

void calculate_BFGS_inverse_hessian ( QuasiNewtonMehtodData & optimization_data ) const

Returns an approximation of the inverse hessian matrix according to the Broyden-Fletcher-Goldfarb-Shanno(BGFS) algorithm.

Parameters

old_parameters	A previous set of parameters.
old_gradient	The gradient of the error function for that previous set of parameters.
old_inverse_hessian	The hessian of the error function for that previous set of parameters.
parameters	Actual set of parameters.
gradient	The gradient of the error function for the actual set of parameters.

Todo:: Add thread pool.

Definition at line 430 of file quasi_newton_method.cpp.

◆ calculate_DFP_inverse_hessian()

void calculate_DFP_inverse_hessian ( QuasiNewtonMehtodData & optimization_data ) const

Returns an approximation of the inverse hessian matrix according to the Davidon-Fletcher-Powel (DFP) algorithm.

Parameters

old_parameters	A previous set of parameters.
old_gradient	The gradient of the error function for that previous set of parameters.
old_inverse_hessian	The hessian of the error function for that previous set of parameters.
parameters	Actual set of parameters.
gradient	The gradient of the error function for the actual set of parameters.

Definition at line 390 of file quasi_newton_method.cpp.

◆ calculate_inverse_hessian_approximation()

void calculate_inverse_hessian_approximation ( QuasiNewtonMehtodData & optimization_data ) const

Calculates an approximation of the inverse hessian, accoring to the method used.

Parameters

old_parameters	Another point of the error function.
parameters	Current point of the error function
old_gradient	Gradient at the other point.
gradient	Gradient at the current point.
old_inverse_hessian	Inverse hessian at the other point of the error function.

Definition at line 307 of file quasi_newton_method.cpp.

◆ from_XML()

void from_XML ( const tinyxml2::XMLDocument & document )

virtual

Loads a default optimization algorithm from a XML document.

Parameters

document TinyXML document containing the error term members.

Reimplemented from OptimizationAlgorithm.

Definition at line 953 of file quasi_newton_method.cpp.

◆ get_epochs_number()

const Index & get_epochs_number ( ) const

Definition at line 94 of file quasi_newton_method.cpp.

◆ get_inverse_hessian_approximation_method()

const QuasiNewtonMethod::InverseHessianApproximationMethod & get_inverse_hessian_approximation_method ( ) const

Returns the method for approximating the inverse hessian matrix to be used when training.

Definition at line 65 of file quasi_newton_method.cpp.

◆ get_learning_rate_algorithm()

const LearningRateAlgorithm & get_learning_rate_algorithm ( ) const

Returns a constant reference to the learning rate algorithm object inside the quasi-Newton method object.

Definition at line 49 of file quasi_newton_method.cpp.

◆ get_learning_rate_algorithm_pointer()

LearningRateAlgorithm * get_learning_rate_algorithm_pointer ( )

Returns a pointer to the learning rate algorithm object inside the quasi-Newton method object.

Definition at line 57 of file quasi_newton_method.cpp.

◆ get_loss_goal()

const type & get_loss_goal ( ) const

Returns the goal value for the loss. This is used as a stopping criterion when training a neural network

Definition at line 111 of file quasi_newton_method.cpp.

◆ get_maximum_epochs_number()

const Index & get_maximum_epochs_number ( ) const

Returns the maximum number of epochs for training.

Definition at line 127 of file quasi_newton_method.cpp.

◆ get_maximum_selection_failures()

const Index & get_maximum_selection_failures ( ) const

Returns the maximum number of selection error increases during the training process.

Definition at line 119 of file quasi_newton_method.cpp.

◆ get_maximum_time()

const type & get_maximum_time ( ) const

Returns the maximum training time.

Definition at line 135 of file quasi_newton_method.cpp.

◆ get_minimum_loss_decrease()

const type & get_minimum_loss_decrease ( ) const

Returns the minimum loss improvement during training.

Definition at line 102 of file quasi_newton_method.cpp.

◆ initialize_inverse_hessian_approximation()

void initialize_inverse_hessian_approximation ( QuasiNewtonMehtodData & optimization_data ) const

Definition at line 290 of file quasi_newton_method.cpp.

◆ kronecker_product() [1/2]

const Tensor< type, 2 > kronecker_product	(	Tensor< type, 1 > &	left_matrix,
		Tensor< type, 1 > &	right_matrix
	)		const

Definition at line 333 of file quasi_newton_method.cpp.

◆ kronecker_product() [2/2]

const Tensor< type, 2 > kronecker_product	(	Tensor< type, 2 > &	left_matrix,
		Tensor< type, 2 > &	right_matrix
	)		const

This method calculates the kronecker product between two matrices. Its return a direct matrix.

Parameters

left_matrix	Matrix.
right_matrix	Matrix.

Definition at line 360 of file quasi_newton_method.cpp.

◆ perform_training()

TrainingResults perform_training ( )

virtual

Trains a neural network with an associated loss index according to the quasi-Newton method. Training occurs according to the training operators, training parameters and stopping criteria.

Implements OptimizationAlgorithm.

Definition at line 592 of file quasi_newton_method.cpp.

◆ set_default()

void set_default ( )

virtual

Sets the members of the optimization algorithm object to their default values.

Reimplemented from OptimizationAlgorithm.

Definition at line 205 of file quasi_newton_method.cpp.

◆ set_display()

void set_display ( const bool & new_display )

virtual

Sets a new display value. If it is set to true messages from this class are to be displayed on the screen; if it is set to false messages from this class are not to be displayed on the screen.

Parameters

new_display Display value.

Reimplemented from OptimizationAlgorithm.

Definition at line 199 of file quasi_newton_method.cpp.

◆ set_inverse_hessian_approximation_method() [1/2]

void set_inverse_hessian_approximation_method ( const InverseHessianApproximationMethod & new_inverse_hessian_approximation_method )

Sets a new inverse hessian approximatation method value.

Parameters

new_inverse_hessian_approximation_method Inverse hessian approximation method value.

Definition at line 156 of file quasi_newton_method.cpp.

◆ set_inverse_hessian_approximation_method() [2/2]

void set_inverse_hessian_approximation_method ( const string & new_inverse_hessian_approximation_method_name )

Sets a new method for approximating the inverse of the hessian matrix from a string containing the name. Possible values are:

"DFP"
"BFGS"

Parameters

new_inverse_hessian_approximation_method_name Name of inverse hessian approximation method.

Definition at line 171 of file quasi_newton_method.cpp.

◆ set_loss_goal()

void set_loss_goal ( const type & new_loss_goal )

Sets a new goal value for the loss. This is used as a stopping criterion when training a neural network

Parameters

new_loss_goal Goal value for the loss.

Definition at line 240 of file quasi_newton_method.cpp.

◆ set_loss_index_pointer()

void set_loss_index_pointer ( LossIndex * new_loss_index_pointer )

virtual

Sets a pointer to a loss index object to be associated to the quasi-Newton method object. It also sets that loss index to the learning rate algorithm.

Parameters

new_loss_index_pointer Pointer to a loss index object.

Reimplemented from OptimizationAlgorithm.

Definition at line 145 of file quasi_newton_method.cpp.

◆ set_maximum_epochs_number()

void set_maximum_epochs_number ( const Index & new_maximum_epochs_number )

Sets a new maximum number of epochs number.

Parameters

new_maximum_epochs_number Maximum number of epochs in which the selection evalutation decreases.

Definition at line 258 of file quasi_newton_method.cpp.

◆ set_maximum_selection_failures()

void set_maximum_selection_failures ( const Index & new_maximum_selection_failures )

Sets a new maximum number of selection error increases.

Parameters

new_maximum_selection_failures Maximum number of epochs in which the selection evalutation increases.

Definition at line 249 of file quasi_newton_method.cpp.

◆ set_maximum_time()

void set_maximum_time ( const type & new_maximum_time )

Sets a new maximum training time.

Parameters

new_maximum_time Maximum training time.

Definition at line 267 of file quasi_newton_method.cpp.

◆ set_minimum_loss_decrease()

void set_minimum_loss_decrease ( const type & new_minimum_loss_decrease )

Sets a new minimum loss improvement during training.

Parameters

new_minimum_loss_decrease Minimum improvement in the loss between two epochs.

Definition at line 230 of file quasi_newton_method.cpp.

◆ to_string_matrix()

Tensor< string, 2 > to_string_matrix ( ) const

virtual

Writes as matrix of strings the most representative atributes.

Reimplemented from OptimizationAlgorithm.

Definition at line 905 of file quasi_newton_method.cpp.

◆ update_parameters()

void update_parameters	(	const DataSetBatch &	batch,
		NeuralNetworkForwardPropagation &	forward_propagation,
		LossIndexBackPropagation &	back_propagation,
		QuasiNewtonMehtodData &	optimization_data
	)

QuasiNewtonMethod::update_parameters.

Parameters

batch
forward_propagation
back_propagation
optimization_data

Definition at line 479 of file quasi_newton_method.cpp.

◆ write_inverse_hessian_approximation_method()

string write_inverse_hessian_approximation_method ( ) const

Returns the name of the method for the approximation of the inverse hessian.

Definition at line 73 of file quasi_newton_method.cpp.

◆ write_optimization_algorithm_type()

string write_optimization_algorithm_type ( ) const

virtual

Reimplemented from OptimizationAlgorithm.

Definition at line 806 of file quasi_newton_method.cpp.

◆ write_XML()

void write_XML ( tinyxml2::XMLPrinter & file_stream ) const

virtual

Serializes the quasi Newton method object into a XML document of the TinyXML library without keep the DOM tree in memory. See the OpenNN manual for more information about the format of this document.

Reimplemented from OptimizationAlgorithm.

Definition at line 815 of file quasi_newton_method.cpp.

Member Data Documentation

◆ first_learning_rate

type first_learning_rate = static_cast<type>(0.01)

private

Definition at line 157 of file quasi_newton_method.h.

◆ inverse_hessian_approximation_method

InverseHessianApproximationMethod inverse_hessian_approximation_method

private

Variable containing the actual method used to obtain a suitable learning rate.

Definition at line 155 of file quasi_newton_method.h.

◆ learning_rate_algorithm

LearningRateAlgorithm learning_rate_algorithm

private

Learning rate algorithm object. It is used to calculate the step for the quasi-Newton training direction.

Definition at line 151 of file quasi_newton_method.h.

◆ maximum_epochs_number

Index maximum_epochs_number

private

Maximum number of epochs to perform_training. It is used as a stopping criterion.

Definition at line 176 of file quasi_newton_method.h.

◆ maximum_selection_failures

Index maximum_selection_failures

private

Maximum number of epochs at which the selection error increases. This is an early stopping method for improving selection.

Definition at line 172 of file quasi_newton_method.h.

◆ maximum_time

type maximum_time

private

Maximum training time. It is used as a stopping criterion.

Definition at line 180 of file quasi_newton_method.h.

◆ minimum_loss_decrease

type minimum_loss_decrease

private

Minimum loss improvement between two successive epochs. It is used as a stopping criterion.

Definition at line 163 of file quasi_newton_method.h.

◆ training_loss_goal

type training_loss_goal

private

Goal value for the loss. It is used as a stopping criterion.

Definition at line 167 of file quasi_newton_method.h.

The documentation for this class was generated from the following files:

Public Types

Public Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Member Enumeration Documentation

◆ InverseHessianApproximationMethod

Constructor & Destructor Documentation

◆ QuasiNewtonMethod() [1/2]

◆ QuasiNewtonMethod() [2/2]

◆ ~QuasiNewtonMethod()

Member Function Documentation

◆ calculate_BFGS_inverse_hessian()

◆ calculate_DFP_inverse_hessian()

◆ calculate_inverse_hessian_approximation()

◆ from_XML()

◆ get_epochs_number()

◆ get_inverse_hessian_approximation_method()

◆ get_learning_rate_algorithm()

◆ get_learning_rate_algorithm_pointer()

◆ get_loss_goal()

◆ get_maximum_epochs_number()

◆ get_maximum_selection_failures()

◆ get_maximum_time()

◆ get_minimum_loss_decrease()

◆ initialize_inverse_hessian_approximation()

◆ kronecker_product() [1/2]

◆ kronecker_product() [2/2]

◆ perform_training()

◆ set_default()

◆ set_display()

◆ set_inverse_hessian_approximation_method() [1/2]

◆ set_inverse_hessian_approximation_method() [2/2]

◆ set_loss_goal()

◆ set_loss_index_pointer()

◆ set_maximum_epochs_number()

◆ set_maximum_selection_failures()

◆ set_maximum_time()

◆ set_minimum_loss_decrease()

◆ to_string_matrix()

◆ update_parameters()

◆ write_inverse_hessian_approximation_method()

◆ write_optimization_algorithm_type()

◆ write_XML()

Member Data Documentation

◆ first_learning_rate

◆ inverse_hessian_approximation_method

◆ learning_rate_algorithm

◆ maximum_epochs_number

◆ maximum_selection_failures

◆ maximum_time

◆ minimum_loss_decrease

◆ training_loss_goal