Layer normalization across the embedding dimension of rank-2 inputs. More...

#include <normalization_layer_3d.h>

Inheritance diagram for opennn::Normalization3d:

Public Member Functions
	Normalization3d (const Shape &input_shape=Shape({0, 0}), const string &label="normalization_layer_3d")
	Constructs a Normalization3d layer.

Shape	get_input_shape () const override
	Returns the per-sample input shape (sequence_length, embedding_dimension).

Shape	get_output_shape () const override
	Returns the per-sample output shape (same as input).

Index	get_sequence_length () const
	Sequence length of the input.

Index	get_embedding_dimension () const
	Embedding dimension along which normalization is applied.

vector< Operator * >	get_operators () override
	Returns the single LayerNorm operator.

vector< pair< Shape, Type > >	get_forward_specs (Index batch_size) const override
	Specifications of the forward intermediate buffers.

void	set (Index sequence_length=0, Index embedding_dimension=0, const string &label="normalization_layer_3d")
	Re-initializes the layer.

void	set_input_shape (const Shape &new_input_shape) override
	Updates the input shape (sequence_length, embedding_dimension).

void	back_propagate (ForwardPropagation &, BackPropagation &, size_t) const noexcept override
	Backward pass through layer normalization.

void	read_JSON_body (const Json *) override
	Reads the layer-specific JSON body (sequence length and embedding dimension).

Public Member Functions inherited from opennn::Layer
virtual	~Layer ()=default
	Virtual destructor; subclasses are owned via unique_ptr<Layer>.

const string &	get_label () const
	Returns the user-assigned label of this layer.

const string &	get_name () const
	Returns the canonical type name of this layer.

LayerType	get_type () const
	Returns the LayerType enumerator for this layer.

virtual void	set_output_shape (const Shape &)
	Sets the per-sample output shape of this layer.

void	set_label (string new_label)
	Sets the human-readable label of this layer.

Index	get_parameters_number () const
	Total number of trainable parameters in this layer.

virtual vector< pair< Shape, Type > >	get_parameter_specs () const
	Specifications of the trainable parameter tensors owned by this layer.

virtual vector< pair< Shape, Type > >	get_state_specs () const
	Specifications of the persistent state tensors of this layer.

virtual vector< pair< Shape, Type > >	get_backward_specs (Index batch_size) const
	Specifications of the backward intermediate buffers for one batch.

vector< Shape >	get_parameter_shapes () const
	Shape-only view of get_parameter_specs().

vector< Shape >	get_state_shapes () const
	Shape-only view of get_state_specs().

vector< Shape >	get_forward_shapes (Index b) const
	Shape-only view of get_forward_specs() for batch size `b`.

vector< Shape >	get_backward_shapes (Index b) const
	Shape-only view of get_backward_specs() for batch size `b`.

vector< Type >	get_parameter_dtypes () const
	Dtype-only view of get_parameter_specs().

vector< Type >	get_forward_dtypes (Index b) const
	Dtype-only view of get_forward_specs() for batch size `b`.

vector< Type >	get_backward_dtypes (Index b) const
	Dtype-only view of get_backward_specs() for batch size `b`.

virtual Activation::Function	get_output_activation () const
	Activation function fused at the end of this layer, if any.

Index	get_inputs_number () const
	Total number of scalar inputs per sample (product of input dims).

Index	get_outputs_number () const
	Total number of scalar outputs per sample (product of output dims).

virtual void	forward_propagate (ForwardPropagation &fp, size_t layer, bool is_training) noexcept
	Forward pass: reads inputs from `fp` and writes outputs into `fp`.

virtual void	from_JSON (const JsonDocument &document)
	Loads the layer configuration (hyperparameters) from JSON.

virtual void	load_state_from_JSON (const JsonDocument &document)
	Loads parameter and state tensors from a JSON document.

virtual void	to_JSON (JsonWriter &writer) const
	Writes the layer configuration to JSON.

virtual void	write_JSON_body (JsonWriter &) const
	Subclass hook for emitting the body of to_JSON().

virtual void	print () const
	Prints a human-readable summary of the layer to stdout.

bool	get_is_trainable () const
	Whether this layer has trainable parameters.

Type	get_compute_dtype () const
	Numerical type used for forward/backward computation.

void	set_compute_dtype (Type new_compute_dtype)
	Sets the compute dtype and triggers on_compute_dtype_changed().

virtual void	on_compute_dtype_changed ()
	Hook invoked after set_compute_dtype() mutates the dtype.

virtual float *	link_parameters (float *pointer)
	Wires this layer's parameter TensorViews onto an external buffer.

virtual float *	link_states (float *pointer)
	Wires this layer's state TensorViews onto an external buffer.

vector< TensorView > &	get_parameter_views ()
	Mutable access to this layer's parameter TensorViews.

const vector< TensorView > &	get_parameter_views () const
	Read-only access to this layer's parameter TensorViews.

vector< TensorView > &	get_state_views ()
	Mutable access to this layer's state TensorViews.

const vector< TensorView > &	get_state_views () const
	Read-only access to this layer's state TensorViews.

void	redistribute_parameters_to_operators ()
	Forwards the current parameter views down to each composing Operator.

void	redistribute_parameter_gradients_to_operators (vector< TensorView > &gradient_views)
	Forwards externally provided gradient views down to each Operator.

void	redistribute_states_to_operators ()
	Forwards the current state views down to each composing Operator.

Additional Inherited Members
Protected Member Functions inherited from opennn::Layer
	Layer ()=default
	Default constructor; only invoked by subclasses.

float *	link_views (float pointer, const vector< Shape > &shapes, vector< TensorView > &views, const char tag) const
	Builds `views` over a contiguous float buffer using `shapes`.

void	distribute_to_operators (vector< TensorView > &views, void(Operator::link)(const vector< TensorView > &), vector< pair< Shape, Type > >(Operator::specs)() const)
	Generic helper used by the redistribute_*_to_operators() routines.

Protected Attributes inherited from opennn::Layer
string	label = "my_layer"
	User-visible label for this layer instance (default "my_layer").

string	name = "layer"
	Canonical type name set by the subclass (e.g. "dense").

LayerType	layer_type = LayerType::Dense
	Layer type tag set by the subclass.

bool	is_trainable = true
	True if the layer has parameters that participate in training.

bool	is_first_layer = false
	True if this layer is the network's input layer.

Type	compute_dtype = Type::FP32
	Numerical type used for forward and backward computation.

vector< TensorView >	parameters
	Parameter TensorViews bound to the network's parameter arena.

vector< TensorView >	states
	State TensorViews bound to the network's state arena.

vector< unique_ptr< Layer > >	layers
	Sub-layers, when this layer is itself a composite.

Detailed Description

Layer normalization across the embedding dimension of rank-2 inputs.

Inputs are rank-2 tensors (sequence_length, embedding_dimension). The layer computes the mean and standard deviation along the embedding axis for each (sample, position) pair and rescales the input accordingly. Trainable affine parameters (gain, bias) are exposed via the underlying LayerNorm operator.

Constructor & Destructor Documentation

◆ Normalization3d()

opennn::Normalization3d::Normalization3d	(	const Shape &	input_shape = Shape({0, 0}),
		const string &	label = "normalization_layer_3d" )

Constructs a Normalization3d layer.

Parameters

input_shape	Per-sample input shape (sequence_length, embedding_dimension).
label	Human-readable label assigned to this layer.

Member Function Documentation

◆ back_propagate()

void opennn::Normalization3d::back_propagate	(	ForwardPropagation &	,
		BackPropagation &	,
		size_t	) const

overridevirtualnoexcept

Backward pass through layer normalization.

Receives the forward intermediates, the BackPropagation buffer and this layer's index inside the network.

Reimplemented from opennn::Layer.

◆ get_embedding_dimension()

Index opennn::Normalization3d::get_embedding_dimension ( ) const

inline

Embedding dimension along which normalization is applied.

◆ get_forward_specs()

vector< pair< Shape, Type > > opennn::Normalization3d::get_forward_specs ( Index batch_size ) const

overridevirtual

Specifications of the forward intermediate buffers.

Parameters

batch_size Batch size used for sizing.

Returns: One spec per slot in the Forward enum.

Reimplemented from opennn::Layer.

◆ get_input_shape()

Shape opennn::Normalization3d::get_input_shape ( ) const

overridevirtual

Returns the per-sample input shape (sequence_length, embedding_dimension).

Implements opennn::Layer.

◆ get_operators()

vector< Operator * > opennn::Normalization3d::get_operators ( )

overridevirtual

Returns the single LayerNorm operator.

Reimplemented from opennn::Layer.

◆ get_output_shape()

Shape opennn::Normalization3d::get_output_shape ( ) const

overridevirtual

Returns the per-sample output shape (same as input).

Implements opennn::Layer.

◆ get_sequence_length()

Index opennn::Normalization3d::get_sequence_length ( ) const

inline

Sequence length of the input.

◆ read_JSON_body()

void opennn::Normalization3d::read_JSON_body ( const Json * )

overridevirtual

Reads the layer-specific JSON body (sequence length and embedding dimension).

Reimplemented from opennn::Layer.

◆ set()

void opennn::Normalization3d::set	(	Index	sequence_length = 0,
		Index	embedding_dimension = 0,
		const string &	label = "normalization_layer_3d" )

Re-initializes the layer.

Parameters

sequence_length	Sequence length of the input.
embedding_dimension	Embedding dimension along which to normalize.
label	Human-readable label.

◆ set_input_shape()

void opennn::Normalization3d::set_input_shape ( const Shape & new_input_shape )

inlineoverridevirtual

Updates the input shape (sequence_length, embedding_dimension).

Parameters

new_input_shape New per-sample input shape; ignored if rank < 2.

Reimplemented from opennn::Layer.

Public Member Functions

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ Normalization3d()

Member Function Documentation

◆ back_propagate()

◆ get_embedding_dimension()

◆ get_forward_specs()

◆ get_input_shape()

◆ get_operators()

◆ get_output_shape()

◆ get_sequence_length()

◆ read_JSON_body()

◆ set()

◆ set_input_shape()