#include <gradient_descent.h>

Inheritance diagram for GradientDescent:

Public Member Functions
	GradientDescent ()

	GradientDescent (LossIndex *)

virtual	~GradientDescent ()
	Destructor. More...

const LearningRateAlgorithm &	get_learning_rate_algorithm () const
	Returns a constant reference to the learning rate algorithm object inside the gradient descent object. More...

LearningRateAlgorithm *	get_learning_rate_algorithm_pointer ()
	Returns a pointer to the learning rate algorithm object inside the gradient descent object. More...

string	get_hardware_use () const
	Returns the hardware used. Default: Multi-core. More...

const type &	get_minimum_loss_decrease () const
	Returns the minimum loss improvement during training. More...

const type &	get_loss_goal () const

const Index &	get_maximum_selection_failures () const
	Returns the maximum number of selection error increases during the training process. More...

const Index &	get_maximum_epochs_number () const
	Returns the maximum number of iterations for training. More...

const type &	get_maximum_time () const
	Returns the maximum training time. More...

void	set_loss_index_pointer (LossIndex *)

void	set_learning_rate_algorithm (const LearningRateAlgorithm &)

void	set_default ()
	Sets the members of the optimization algorithm object to their default values. More...

void	set_maximum_epochs_number (const Index &)

void	set_minimum_loss_decrease (const type &)

void	set_loss_goal (const type &)

void	set_maximum_selection_failures (const Index &)

void	set_maximum_time (const type &)

void	calculate_training_direction (const Tensor< type, 1 > &, Tensor< type, 1 > &) const

void	update_parameters (const DataSetBatch &batch, NeuralNetworkForwardPropagation &forward_propagation, LossIndexBackPropagation &back_propagation, GradientDescentData &optimization_data)
	GradientDescent::update_parameters. More...

TrainingResults	perform_training ()

string	write_optimization_algorithm_type () const

Tensor< string, 2 >	to_string_matrix () const
	Writes as matrix of strings the most representative atributes. More...

void	from_XML (const tinyxml2::XMLDocument &)

void	write_XML (tinyxml2::XMLPrinter &) const

Public Member Functions inherited from OptimizationAlgorithm
	OptimizationAlgorithm ()

	OptimizationAlgorithm (LossIndex *)

virtual	~OptimizationAlgorithm ()
	Destructor. More...

LossIndex *	get_loss_index_pointer () const

string	get_hardware_use () const
	Hardware use. More...

void	set_hardware_use (const string &)
	Set hardware to use. Default: Multi-core. More...

bool	has_loss_index () const

const bool &	get_display () const

const Index &	get_display_period () const
	Returns the number of iterations between the training showing progress. More...

const Index &	get_save_period () const
	Returns the number of iterations between the training saving progress. More...

const string &	get_neural_network_file_name () const
	Returns the file name where the neural network will be saved. More...

const string	write_time (const type &) const
	Writes the time from seconds in format HH:mm:ss. More...

void	set ()

virtual void	set_threads_number (const int &)

virtual void	set_display (const bool &)

void	set_display_period (const Index &)

void	set_save_period (const Index &)

void	set_neural_network_file_name (const string &)

virtual void	check () const

virtual void	print () const
	Prints to the screen the XML-type representation of the optimization algorithm object. More...

void	save (const string &) const

void	load (const string &)

Private Attributes
LearningRateAlgorithm	learning_rate_algorithm
	Learning rate algorithm object for one-dimensional minimization. More...

const type	first_learning_rate = static_cast<type>(0.01)

type	minimum_loss_decrease
	Minimum loss improvement between two successive iterations. It is used as a stopping criterion. More...

type	training_loss_goal
	Goal value for the loss. It is used as a stopping criterion. More...

Index	maximum_selection_failures

Index	maximum_epochs_number
	Maximum epochs number. More...

type	maximum_time
	Maximum training time. It is used as a stopping criterion. More...

Additional Inherited Members
Public Types inherited from OptimizationAlgorithm
enum class	StoppingCondition { MinimumLossDecrease , LossGoal , MaximumSelectionErrorIncreases , MaximumEpochsNumber , MaximumTime }
	Enumeration of all possibles condition of stop for the algorithms. More...

Protected Attributes inherited from OptimizationAlgorithm
NonBlockingThreadPool *	non_blocking_thread_pool = nullptr

ThreadPoolDevice *	thread_pool_device

LossIndex *	loss_index_pointer = nullptr
	Pointer to a loss index for a neural network object. More...

Index	epochs_number = 10000
	Number of training epochs in the neural network. More...

string	hardware_use = "Multi-core"
	Hardware use. More...

Index	display_period = 10
	Number of iterations between the training showing progress. More...

Index	save_period = numeric_limits<Index>::max()
	Number of iterations between the training saving progress. More...

string	neural_network_file_name = "neural_network.xml"
	Path where the neural network is saved. More...

bool	display = true
	Display messages to screen. More...

const Eigen::array< IndexPair< Index >, 1 >	AT_B = {IndexPair<Index>(0, 0)}

const Eigen::array< IndexPair< Index >, 1 >	product_vector_matrix = {IndexPair<Index>(0, 1)}

const Eigen::array< IndexPair< Index >, 1 >	A_B = {IndexPair<Index>(1, 0)}

Detailed Description

The process of making changes to weights and biases, where the changes are propotyional to derivatives of network error with respect to those weights and biases. This is done to minimize network error. This concrete class represents the gradient descent optimization algorithm[1], used to minimize loss function.

[1] Neural Designer "5 Algorithms to Train a Neural Network." https://www.neuraldesigner.com/blog/5_algorithms_to_train_a_neural_network

Definition at line 47 of file gradient_descent.h.

Constructor & Destructor Documentation

◆ GradientDescent() [1/2]

GradientDescent ( )

explicit

Default constructor. It creates a gradient descent optimization algorithm not associated to any loss index object. It also initializes the class members to their default values.

Definition at line 18 of file gradient_descent.cpp.

◆ GradientDescent() [2/2]

GradientDescent ( LossIndex * new_loss_index_pointer )

explicit

Loss index constructor. It creates a gradient descent optimization algorithm associated to a loss index. It also initializes the class members to their default values.

Parameters

new_loss_index_pointer Pointer to a loss index object.

Definition at line 30 of file gradient_descent.cpp.

◆ ~GradientDescent()

~GradientDescent ( )

virtual

Destructor.

Definition at line 41 of file gradient_descent.cpp.

Member Function Documentation

◆ calculate_training_direction()

void calculate_training_direction	(	const Tensor< type, 1 > &	gradient,
		Tensor< type, 1 > &	training_direction
	)		const

Returns the gradient descent training direction, which is the negative of the normalized gradient.

Parameters

gradient Loss index gradient.

Definition at line 224 of file gradient_descent.cpp.

◆ from_XML()

void from_XML ( const tinyxml2::XMLDocument & document )

virtual

Loads a default optimization algorithm from a XML document.

Parameters

document TinyXML document containing the error term members.

Reimplemented from OptimizationAlgorithm.

Definition at line 690 of file gradient_descent.cpp.

◆ get_hardware_use()

string get_hardware_use ( ) const

Returns the hardware used. Default: Multi-core.

Definition at line 64 of file gradient_descent.cpp.

◆ get_learning_rate_algorithm()

const LearningRateAlgorithm & get_learning_rate_algorithm ( ) const

Returns a constant reference to the learning rate algorithm object inside the gradient descent object.

Definition at line 48 of file gradient_descent.cpp.

◆ get_learning_rate_algorithm_pointer()

LearningRateAlgorithm * get_learning_rate_algorithm_pointer ( )

Returns a pointer to the learning rate algorithm object inside the gradient descent object.

Definition at line 56 of file gradient_descent.cpp.

◆ get_loss_goal()

const type & get_loss_goal ( ) const

Returns the goal value for the loss. This is used as a stopping criterion when training a neural network.

Definition at line 81 of file gradient_descent.cpp.

◆ get_maximum_epochs_number()

const Index & get_maximum_epochs_number ( ) const

Returns the maximum number of iterations for training.

Definition at line 97 of file gradient_descent.cpp.

◆ get_maximum_selection_failures()

const Index & get_maximum_selection_failures ( ) const

Returns the maximum number of selection error increases during the training process.

Definition at line 89 of file gradient_descent.cpp.

◆ get_maximum_time()

const type & get_maximum_time ( ) const

Returns the maximum training time.

Definition at line 105 of file gradient_descent.cpp.

◆ get_minimum_loss_decrease()

const type & get_minimum_loss_decrease ( ) const

Returns the minimum loss improvement during training.

Definition at line 72 of file gradient_descent.cpp.

◆ perform_training()

TrainingResults perform_training ( )

virtual

Trains a neural network with an associated loss index, according to the gradient descent method. Training occurs according to the training parameters and stopping criteria. It returns a results structure with the history and the final values of the reserved variables.

Implements OptimizationAlgorithm.

Definition at line 341 of file gradient_descent.cpp.

◆ set_default()

void set_default ( )

virtual

Sets the members of the optimization algorithm object to their default values.

Reimplemented from OptimizationAlgorithm.

Definition at line 123 of file gradient_descent.cpp.

◆ set_loss_goal()

void set_loss_goal ( const type & new_loss_goal )

Sets a new goal value for the loss. This is used as a stopping criterion when training a neural network.

Parameters

new_loss_goal Goal value for the loss.

Definition at line 178 of file gradient_descent.cpp.

◆ set_loss_index_pointer()

void set_loss_index_pointer ( LossIndex * new_loss_index_pointer )

virtual

Sets a pointer to a loss index object to be associated to the gradient descent object. It also sets that loss index to the learning rate algorithm.

Parameters

new_loss_index_pointer Pointer to a loss index object.

Reimplemented from OptimizationAlgorithm.

Definition at line 115 of file gradient_descent.cpp.

◆ set_maximum_epochs_number()

void set_maximum_epochs_number ( const Index & new_maximum_epochs_number )

Set the a new maximum for the epochs number.

Parameters

new_maximum_epochs number New maximum epochs number.

Definition at line 144 of file gradient_descent.cpp.

◆ set_maximum_selection_failures()

void set_maximum_selection_failures ( const Index & new_maximum_selection_failures )

Sets a new maximum number of selection error increases.

Parameters

new_maximum_selection_failures Maximum number of epochs in which the selection evalutation increases.

Definition at line 188 of file gradient_descent.cpp.

◆ set_maximum_time()

void set_maximum_time ( const type & new_maximum_time )

Sets a new maximum training time.

Parameters

new_maximum_time Maximum training time.

Definition at line 197 of file gradient_descent.cpp.

◆ set_minimum_loss_decrease()

void set_minimum_loss_decrease ( const type & new_minimum_loss_decrease )

Sets a new minimum loss improvement during training.

Parameters

new_minimum_loss_decrease Minimum improvement in the loss between two iterations.

Definition at line 168 of file gradient_descent.cpp.

◆ to_string_matrix()

Tensor< string, 2 > to_string_matrix ( ) const

virtual

Writes as matrix of strings the most representative atributes.

Reimplemented from OptimizationAlgorithm.

Definition at line 563 of file gradient_descent.cpp.

◆ update_parameters()

void update_parameters	(	const DataSetBatch &	batch,
		NeuralNetworkForwardPropagation &	forward_propagation,
		LossIndexBackPropagation &	back_propagation,
		GradientDescentData &	optimization_data
	)

GradientDescent::update_parameters.

Parameters

batch
forward_propagation
back_propagation
optimization_data

Definition at line 267 of file gradient_descent.cpp.

◆ write_optimization_algorithm_type()

string write_optimization_algorithm_type ( ) const

virtual

Reimplemented from OptimizationAlgorithm.

Definition at line 555 of file gradient_descent.cpp.

◆ write_XML()

void write_XML ( tinyxml2::XMLPrinter & file_stream ) const

virtual

Serializes the gradient descent object into a XML document of the TinyXML library without keep the DOM tree in memory. See the OpenNN manual for more information about the format of this document.

Reimplemented from OptimizationAlgorithm.

Definition at line 610 of file gradient_descent.cpp.

Member Data Documentation

◆ first_learning_rate

const type first_learning_rate = static_cast<type>(0.01)

private

Definition at line 128 of file gradient_descent.h.

◆ learning_rate_algorithm

LearningRateAlgorithm learning_rate_algorithm

private

Learning rate algorithm object for one-dimensional minimization.

Definition at line 126 of file gradient_descent.h.

◆ maximum_epochs_number

Index maximum_epochs_number

private

Maximum epochs number.

Definition at line 147 of file gradient_descent.h.

◆ maximum_selection_failures

Index maximum_selection_failures

private

Maximum number of epochs at which the selection error increases. This is an early stopping method for improving selection.

Definition at line 143 of file gradient_descent.h.

◆ maximum_time

type maximum_time

private

Maximum training time. It is used as a stopping criterion.

Definition at line 151 of file gradient_descent.h.

◆ minimum_loss_decrease

type minimum_loss_decrease

private

Minimum loss improvement between two successive iterations. It is used as a stopping criterion.

Definition at line 134 of file gradient_descent.h.

◆ training_loss_goal

type training_loss_goal

private

Goal value for the loss. It is used as a stopping criterion.

Definition at line 138 of file gradient_descent.h.

The documentation for this class was generated from the following files:

Public Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ GradientDescent() [1/2]

◆ GradientDescent() [2/2]

◆ ~GradientDescent()

Member Function Documentation

◆ calculate_training_direction()

◆ from_XML()

◆ get_hardware_use()

◆ get_learning_rate_algorithm()

◆ get_learning_rate_algorithm_pointer()

◆ get_loss_goal()

◆ get_maximum_epochs_number()

◆ get_maximum_selection_failures()

◆ get_maximum_time()

◆ get_minimum_loss_decrease()

◆ perform_training()

◆ set_default()

◆ set_loss_goal()

◆ set_loss_index_pointer()

◆ set_maximum_epochs_number()

◆ set_maximum_selection_failures()

◆ set_maximum_time()

◆ set_minimum_loss_decrease()

◆ to_string_matrix()

◆ update_parameters()

◆ write_optimization_algorithm_type()

◆ write_XML()

Member Data Documentation

◆ first_learning_rate

◆ learning_rate_algorithm

◆ maximum_epochs_number

◆ maximum_selection_failures

◆ maximum_time

◆ minimum_loss_decrease

◆ training_loss_goal