Token embedding lookup with optional scaling and additive positional encoding. More...

#include <operators.h>

Inheritance diagram for opennn::EmbeddingLookupOp:

Public Member Functions
void	set (Index new_vocabulary_size, Index new_sequence_length, Index new_embedding_dimension)
	Configures the lookup table dimensions.

vector< TensorSpec >	parameter_specs () const override
	Returns the tensor specs of trainable parameters owned by this operator.

vector< TensorSpec >	state_specs () const override
	Returns the tensor specs of persistent state owned by this operator.

void	link_parameters (span< const TensorView > views) override
	Binds parameter views provided by the hosting layer.

void	link_gradients (span< const TensorView > views) override
	Binds gradient views provided by the hosting layer.

void	link_states (span< const TensorView > views) override
	Binds state views provided by the hosting layer.

void	set_parameters_random () override
	Initializes parameters with random values.

void	set_parameters_glorot () override
	Initializes parameters using Glorot (Xavier) initialization.

void	init_positional_encoding ()
	Fills the positional-encoding state tensor with the standard sinusoidal pattern.

void	forward_propagate (ForwardPropagation &fp, size_t layer, bool is_training) noexcept override
	Runs the operator's forward computation.

void	back_propagate (ForwardPropagation &fp, BackPropagation &bp, size_t layer) const noexcept override
	Runs the operator's backward computation, accumulating into gradient/delta buffers.

Public Member Functions inherited from opennn::Operator
virtual	~Operator ()=default

virtual void	to_JSON (JsonWriter &) const
	Serializes the operator configuration to a JSON writer.

virtual void	from_JSON (const Json *)
	Restores the operator configuration from a JSON node.

virtual void	load_state_from_JSON (const Json *)
	Restores persistent state (e.g. running statistics) from a JSON node.

virtual void	destroy_cuda ()
	Releases CUDA resources owned by the operator; called from destructors.

TensorView &	get_input (ForwardPropagation &fp, size_t layer, size_t i=0) const noexcept

vector< TensorView > &	get_inputs (ForwardPropagation &fp, size_t layer, size_t i=0) const noexcept

TensorView &	get_output (ForwardPropagation &fp, size_t layer, size_t i=0) const noexcept

TensorView &	get_output_delta (BackPropagation &bp, size_t layer, size_t i=0) const noexcept

TensorView &	get_input_delta (BackPropagation &bp, size_t layer, size_t i=0) const noexcept

Public Attributes
Index	vocabulary_size = 0

Index	sequence_length = 0

Index	embedding_dimension = 0

bool	scale_embedding = false

bool	add_positional_encoding = false

float	embedding_scale = 1.0f

TensorView	weights

TensorView	positional_encoding

TensorView	weight_gradient

Public Attributes inherited from opennn::Operator
vector< size_t >	input_slots = {0}

vector< size_t >	output_slots = {1}

vector< size_t >	input_delta_slots = {1}

vector< size_t >	output_delta_slots = {0}

Detailed Description

Token embedding lookup with optional scaling and additive positional encoding.

Member Function Documentation

◆ back_propagate()

void opennn::EmbeddingLookupOp::back_propagate	(	ForwardPropagation &	fp,
		BackPropagation &	bp,
		size_t	layer ) const

overridevirtualnoexcept

Runs the operator's backward computation, accumulating into gradient/delta buffers.

Parameters

fp	Forward propagation workspace (read-only).
bp	Back propagation workspace receiving gradients and deltas.
layer	Index of the hosting layer in the workspace.

Reimplemented from opennn::Operator.

◆ forward_propagate()

void opennn::EmbeddingLookupOp::forward_propagate	(	ForwardPropagation &	fp,
		size_t	layer,
		bool	is_training )

overridevirtualnoexcept

Runs the operator's forward computation.

Parameters

fp	Forward propagation workspace.
layer	Index of the hosting layer in the workspace.
is_training	If true, enables training-only behavior (e.g. dropout sampling).

Reimplemented from opennn::Operator.

◆ init_positional_encoding()

void opennn::EmbeddingLookupOp::init_positional_encoding ( )

Fills the positional-encoding state tensor with the standard sinusoidal pattern.

◆ link_gradients()

void opennn::EmbeddingLookupOp::link_gradients ( span< const TensorView > )

overridevirtual

Binds gradient views provided by the hosting layer.

Reimplemented from opennn::Operator.

◆ link_parameters()

void opennn::EmbeddingLookupOp::link_parameters ( span< const TensorView > )

overridevirtual

Binds parameter views provided by the hosting layer.

Reimplemented from opennn::Operator.

◆ link_states()

void opennn::EmbeddingLookupOp::link_states ( span< const TensorView > )

overridevirtual

Binds state views provided by the hosting layer.

Reimplemented from opennn::Operator.

◆ parameter_specs()

vector< TensorSpec > opennn::EmbeddingLookupOp::parameter_specs ( ) const

overridevirtual

Returns the tensor specs of trainable parameters owned by this operator.

Reimplemented from opennn::Operator.

◆ set()

void opennn::EmbeddingLookupOp::set	(	Index	new_vocabulary_size,
		Index	new_sequence_length,
		Index	new_embedding_dimension )

Configures the lookup table dimensions.

Parameters

new_vocabulary_size	Number of unique tokens in the vocabulary.
new_sequence_length	Length of the input token sequence.
new_embedding_dimension	Size of each embedding vector.

◆ set_parameters_glorot()

void opennn::EmbeddingLookupOp::set_parameters_glorot ( )

overridevirtual

Initializes parameters using Glorot (Xavier) initialization.

Reimplemented from opennn::Operator.

◆ set_parameters_random()

void opennn::EmbeddingLookupOp::set_parameters_random ( )

overridevirtual

Initializes parameters with random values.

Reimplemented from opennn::Operator.

◆ state_specs()

vector< TensorSpec > opennn::EmbeddingLookupOp::state_specs ( ) const

overridevirtual

Returns the tensor specs of persistent state owned by this operator.

Reimplemented from opennn::Operator.

Member Data Documentation

◆ add_positional_encoding

bool opennn::EmbeddingLookupOp::add_positional_encoding = false

◆ embedding_dimension

Index opennn::EmbeddingLookupOp::embedding_dimension = 0

◆ embedding_scale

float opennn::EmbeddingLookupOp::embedding_scale = 1.0f

◆ positional_encoding

TensorView opennn::EmbeddingLookupOp::positional_encoding

◆ scale_embedding

bool opennn::EmbeddingLookupOp::scale_embedding = false

◆ sequence_length

Index opennn::EmbeddingLookupOp::sequence_length = 0

◆ vocabulary_size

Index opennn::EmbeddingLookupOp::vocabulary_size = 0

◆ weight_gradient

TensorView opennn::EmbeddingLookupOp::weight_gradient

◆ weights

TensorView opennn::EmbeddingLookupOp::weights

Public Member Functions

Public Attributes

Detailed Description

Member Function Documentation

◆ back_propagate()

◆ forward_propagate()

◆ init_positional_encoding()

◆ link_gradients()

◆ link_parameters()

◆ link_states()

◆ parameter_specs()

◆ set()

◆ set_parameters_glorot()

◆ set_parameters_random()

◆ state_specs()

Member Data Documentation

◆ add_positional_encoding

◆ embedding_dimension

◆ embedding_scale

◆ positional_encoding

◆ scale_embedding

◆ sequence_length

◆ vocabulary_size

◆ weight_gradient

◆ weights