Batch normalization with learnable scale/shift and running statistics for inference. More...

#include <operators.h>

Inheritance diagram for opennn::BatchNormOp:

Public Member Functions
bool	active () const
	Returns true when the operator has been configured (features > 0).

void	set (Index new_features, float new_momentum=0.1f)
	Configures the per-feature normalization.

vector< TensorSpec >	parameter_specs () const override
	Returns the tensor specs of trainable parameters owned by this operator.

vector< TensorSpec >	state_specs () const override
	Returns the tensor specs of persistent state owned by this operator.

void	link_parameters (span< const TensorView > views) override
	Binds parameter views provided by the hosting layer.

void	link_gradients (span< const TensorView > views) override
	Binds gradient views provided by the hosting layer.

void	link_states (span< const TensorView > views) override
	Binds state views provided by the hosting layer.

void	set_parameters_random () override
	Initializes parameters with random values.

void	set_parameters_glorot () override
	Initializes parameters using Glorot (Xavier) initialization.

void	init_defaults ()
	Resets gamma to one, beta to zero, and running stats to identity values.

void	forward_propagate (ForwardPropagation &fp, size_t layer, bool is_training) noexcept override
	Runs the operator's forward computation.

void	back_propagate (ForwardPropagation &fp, BackPropagation &bp, size_t layer) const noexcept override
	Runs the operator's backward computation, accumulating into gradient/delta buffers.

void	apply_delta (const TensorView &input, const TensorView &mean, const TensorView &inverse_variance, TensorView &delta) const
	Computes the input gradient given the cached normalization statistics from forward.

void	update_inference_cache ()
	Rebuilds the fused scale/shift cache used by the inference path from running stats.

void	invalidate_inference_cache ()
	Marks the inference cache as stale so it is rebuilt on the next inference call.

void	to_JSON (JsonWriter &w) const override
	Serializes the operator configuration to a JSON writer.

void	from_JSON (const Json *parent) override
	Restores the operator configuration from a JSON node.

void	load_state_from_JSON (const Json *parent) override
	Restores persistent state (e.g. running statistics) from a JSON node.

Public Member Functions inherited from opennn::Operator
virtual	~Operator ()=default

virtual void	destroy_cuda ()
	Releases CUDA resources owned by the operator; called from destructors.

TensorView &	get_input (ForwardPropagation &fp, size_t layer, size_t i=0) const noexcept

vector< TensorView > &	get_inputs (ForwardPropagation &fp, size_t layer, size_t i=0) const noexcept

TensorView &	get_output (ForwardPropagation &fp, size_t layer, size_t i=0) const noexcept

TensorView &	get_output_delta (BackPropagation &bp, size_t layer, size_t i=0) const noexcept

TensorView &	get_input_delta (BackPropagation &bp, size_t layer, size_t i=0) const noexcept

Public Attributes
Index	features = 0

float	momentum = 0.1f

TensorView	gamma

TensorView	beta

TensorView	running_mean

TensorView	running_variance

TensorView	gamma_gradient

TensorView	beta_gradient

Public Attributes inherited from opennn::Operator
vector< size_t >	input_slots = {0}

vector< size_t >	output_slots = {1}

vector< size_t >	input_delta_slots = {1}

vector< size_t >	output_delta_slots = {0}

Detailed Description

Batch normalization with learnable scale/shift and running statistics for inference.

Member Function Documentation

◆ active()

bool opennn::BatchNormOp::active ( ) const

inline

Returns true when the operator has been configured (features > 0).

◆ apply_delta()

void opennn::BatchNormOp::apply_delta	(	const TensorView &	input,
		const TensorView &	mean,
		const TensorView &	inverse_variance,
		TensorView &	delta ) const

Computes the input gradient given the cached normalization statistics from forward.

Parameters

input	Forward-pass input tensor.
mean	Per-feature mean cached during forward.
inverse_variance	Per-feature inverse standard deviation cached during forward.
delta	In/out gradient: output_delta on input, input_delta on output.

◆ back_propagate()

void opennn::BatchNormOp::back_propagate	(	ForwardPropagation &	fp,
		BackPropagation &	bp,
		size_t	layer ) const

overridevirtualnoexcept

Runs the operator's backward computation, accumulating into gradient/delta buffers.

Parameters

fp	Forward propagation workspace (read-only).
bp	Back propagation workspace receiving gradients and deltas.
layer	Index of the hosting layer in the workspace.

Reimplemented from opennn::Operator.

◆ forward_propagate()

void opennn::BatchNormOp::forward_propagate	(	ForwardPropagation &	fp,
		size_t	layer,
		bool	is_training )

overridevirtualnoexcept

Runs the operator's forward computation.

Parameters

fp	Forward propagation workspace.
layer	Index of the hosting layer in the workspace.
is_training	If true, enables training-only behavior (e.g. dropout sampling).

Reimplemented from opennn::Operator.

◆ from_JSON()

void opennn::BatchNormOp::from_JSON ( const Json * )

overridevirtual

Restores the operator configuration from a JSON node.

Reimplemented from opennn::Operator.

◆ init_defaults()

void opennn::BatchNormOp::init_defaults ( )

Resets gamma to one, beta to zero, and running stats to identity values.

◆ invalidate_inference_cache()

void opennn::BatchNormOp::invalidate_inference_cache ( )

inline

Marks the inference cache as stale so it is rebuilt on the next inference call.

◆ link_gradients()

void opennn::BatchNormOp::link_gradients ( span< const TensorView > )

overridevirtual

Binds gradient views provided by the hosting layer.

Reimplemented from opennn::Operator.

◆ link_parameters()

void opennn::BatchNormOp::link_parameters ( span< const TensorView > )

overridevirtual

Binds parameter views provided by the hosting layer.

Reimplemented from opennn::Operator.

◆ link_states()

void opennn::BatchNormOp::link_states ( span< const TensorView > )

overridevirtual

Binds state views provided by the hosting layer.

Reimplemented from opennn::Operator.

◆ load_state_from_JSON()

void opennn::BatchNormOp::load_state_from_JSON ( const Json * )

overridevirtual

Restores persistent state (e.g. running statistics) from a JSON node.

Reimplemented from opennn::Operator.

◆ parameter_specs()

vector< TensorSpec > opennn::BatchNormOp::parameter_specs ( ) const

overridevirtual

Returns the tensor specs of trainable parameters owned by this operator.

Reimplemented from opennn::Operator.

◆ set()

void opennn::BatchNormOp::set	(	Index	new_features,
		float	new_momentum = 0.1f )

Configures the per-feature normalization.

Parameters

new_features	Number of channels / features to normalize independently.
new_momentum	Exponential-moving-average momentum for running statistics.

◆ set_parameters_glorot()

void opennn::BatchNormOp::set_parameters_glorot ( )

inlineoverridevirtual

Initializes parameters using Glorot (Xavier) initialization.

Reimplemented from opennn::Operator.

◆ set_parameters_random()

void opennn::BatchNormOp::set_parameters_random ( )

inlineoverridevirtual

Initializes parameters with random values.

Reimplemented from opennn::Operator.

◆ state_specs()

vector< TensorSpec > opennn::BatchNormOp::state_specs ( ) const

overridevirtual

Returns the tensor specs of persistent state owned by this operator.

Reimplemented from opennn::Operator.

◆ to_JSON()

void opennn::BatchNormOp::to_JSON ( JsonWriter & ) const

overridevirtual

Serializes the operator configuration to a JSON writer.

Reimplemented from opennn::Operator.

◆ update_inference_cache()

void opennn::BatchNormOp::update_inference_cache ( )

Rebuilds the fused scale/shift cache used by the inference path from running stats.

Member Data Documentation

◆ beta

TensorView opennn::BatchNormOp::beta

◆ beta_gradient

TensorView opennn::BatchNormOp::beta_gradient

◆ features

Index opennn::BatchNormOp::features = 0

◆ gamma

TensorView opennn::BatchNormOp::gamma

◆ gamma_gradient

TensorView opennn::BatchNormOp::gamma_gradient

◆ momentum

float opennn::BatchNormOp::momentum = 0.1f

◆ running_mean

TensorView opennn::BatchNormOp::running_mean

◆ running_variance

TensorView opennn::BatchNormOp::running_variance

Public Member Functions

Public Attributes

Detailed Description

Member Function Documentation

◆ active()

◆ apply_delta()

◆ back_propagate()

◆ forward_propagate()

◆ from_JSON()

◆ init_defaults()

◆ invalidate_inference_cache()

◆ link_gradients()

◆ link_parameters()

◆ link_states()

◆ load_state_from_JSON()

◆ parameter_specs()

◆ set()

◆ set_parameters_glorot()

◆ set_parameters_random()

◆ state_specs()

◆ to_JSON()

◆ update_inference_cache()

Member Data Documentation

◆ beta

◆ beta_gradient

◆ features

◆ gamma

◆ gamma_gradient

◆ momentum

◆ running_mean

◆ running_variance