opennn::Embedding Class Reference

Token-id to dense vector embedding layer with optional scaling and positional encoding. More...

#include <embedding_layer.h>

Inheritance diagram for opennn::Embedding:

Public Member Functions
	Embedding (const Shape &={0, 0}, Index=0, const string &="embedding_layer")
	Constructs an embedding layer.
Shape	get_input_shape () const override
	Returns the input tensor shape (sequence_length of token ids).
Shape	get_output_shape () const override
	Returns the output tensor shape (sequence_length, embedding_dimension).
Index	get_vocabulary_size () const
Index	get_sequence_length () const
Index	get_embedding_dimension () const
vector< TensorSpec >	get_backward_specs (Index) const override
	Returns the tensor specs of the backward workspace; empty for non-trainable layers.
void	set (Index=0, Index=0, Index=0, const string &="embedding_layer")
	Reconfigures the layer with vocabulary size, sequence length, embedding dimension and name.
void	set_scale_embedding (bool enabled)
	Enables or disables embedding scaling by sqrt(embedding_dimension).
void	set_add_positional_encoding (bool enabled)
	Enables or disables sinusoidal positional encoding added to the embeddings.
void	set_dropout_rate (float rate)
	Sets the dropout rate applied to the embedding output.
void	read_JSON_body (const Json *) override
	Reads the layer configuration from a JSON node.
void	write_JSON_body (JsonWriter &) const override
	Writes the layer configuration to a JSON writer.
Public Member Functions inherited from opennn::Layer
virtual	~Layer ()=default
const string &	get_label () const
const string &	get_name () const
LayerType	get_type () const
virtual void	set_input_shape (const Shape &)
	Sets the input shape; subclasses override to derive dependent dimensions.
virtual void	set_output_shape (const Shape &)
	Sets the output shape; subclasses override when the output is user-configurable.
void	set_label (string new_label)
Index	get_parameters_number () const
	Returns the total number of trainable parameters owned by this layer.
const vector< Operator * > &	get_operators () const
virtual vector< TensorSpec >	get_parameter_specs () const
	Returns the tensor specs of trainable parameters; subclasses override.
virtual vector< TensorSpec >	get_state_specs () const
	Returns the tensor specs of persistent state (e.g. running mean/variance).
virtual vector< TensorSpec >	get_forward_specs (Index batch_size) const
	Returns the tensor specs of the forward workspace; defaults to a single output tensor.
virtual ActivationOp::Function	get_output_activation () const
	Returns the layer's output activation (Identity for most layers; overridden by Dense/Bounding).
Index	get_inputs_number () const
Index	get_outputs_number () const
virtual void	forward_propagate (ForwardPropagation &fp, size_t layer, bool is_training) noexcept
	Runs the forward pass by chaining the layer's operators in order.
virtual void	back_propagate (ForwardPropagation &fp, BackPropagation &bp, size_t i) const noexcept
	Runs the backward pass by chaining the layer's operators in reverse order.
virtual void	from_JSON (const JsonDocument &document)
	Restores layer configuration and parameters from a JSON document.
virtual void	load_state_from_JSON (const JsonDocument &document)
	Restores persistent state (e.g. running statistics) from a JSON document.
virtual void	to_JSON (JsonWriter &writer) const
	Serializes layer configuration and parameters to a JSON writer.
virtual string	write_expression (const vector< string > &, const vector< string > &) const
	Returns a human-readable mathematical expression for this layer (empty by default).
virtual void	print () const
	Prints a human-readable summary of the layer to standard output.
bool	get_is_trainable () const
Type	get_compute_dtype () const
void	set_compute_dtype (Type new_compute_dtype)
	Sets the compute dtype and notifies subclasses via on_compute_dtype_changed().
virtual void	on_compute_dtype_changed ()
	Subclass hook invoked when the compute dtype changes; default is no-op.
virtual float *	link_states (float *pointer)
	Binds the persistent-state region of the shared buffer to operator views.
float *	link_gradients (float *pointer, vector< TensorView > &gradient_views)
	Binds the gradient slice of the shared buffer to operator gradient views.
vector< TensorView > &	get_parameter_views ()
const vector< TensorView > &	get_parameter_views () const
void	redistribute_parameters_to_operators ()
	Re-binds operator parameter views after the parameter buffer has been resized or moved.

Additional Inherited Members
Protected Types inherited from opennn::Layer
enum	Forward { Input , Output }
enum	Backward { OutputDelta , InputDelta }
Protected Member Functions inherited from opennn::Layer
	Layer ()=default
	Layer (LayerType t, bool trainable=true)
float *	link_views_to_operators (vector< TensorView > &views, float *pointer, vector< TensorSpec >(Operator::*specs_fn)() const, void(Operator::*link_fn)(span< const TensorView >))
Protected Attributes inherited from opennn::Layer
string	label = "my_layer"
LayerType	layer_type = LayerType::Dense
bool	is_trainable = true
Shape	input_shape
Type	compute_dtype = Type::FP32
vector< TensorView >	parameters
vector< TensorView >	states
vector< Operator * >	operators

Detailed Description

Token-id to dense vector embedding layer with optional scaling and positional encoding.

Constructor & Destructor Documentation

Embedding()

opennn::Embedding::Embedding	(	const Shape &	= {0, 0},
		Index	= 0,
		const string &	= "embedding_layer" )

Constructs an embedding layer.

Parameters

input_shape	Input shape as (vocabulary_size, sequence_length).
embedding_dimension	Size of each embedding vector.
name	Layer name used for serialization.

Member Function Documentation

get_backward_specs()

vector< TensorSpec > opennn::Embedding::get_backward_specs ( Index batch_size ) const

inlineoverridevirtual

Returns the tensor specs of the backward workspace; empty for non-trainable layers.

Parameters

batch_size

Batch size used to size each tensor.

Reimplemented from opennn::Layer.

get_embedding_dimension()

Index opennn::Embedding::get_embedding_dimension ( ) const

inline

get_input_shape()

Shape opennn::Embedding::get_input_shape ( ) const

inlineoverridevirtual

Returns the input tensor shape (sequence_length of token ids).

Reimplemented from opennn::Layer.

get_output_shape()

Shape opennn::Embedding::get_output_shape ( ) const

overridevirtual

Returns the output tensor shape (sequence_length, embedding_dimension).

Implements opennn::Layer.

get_sequence_length()

Index opennn::Embedding::get_sequence_length ( ) const

inline

get_vocabulary_size()

Index opennn::Embedding::get_vocabulary_size ( ) const

inline

read_JSON_body()

void opennn::Embedding::read_JSON_body ( const Json * )

overridevirtual

Reads the layer configuration from a JSON node.

Reimplemented from opennn::Layer.

set()

void opennn::Embedding::set	(	Index	= 0,
		Index	= 0,
		Index	= 0,
		const string &	= "embedding_layer" )

Reconfigures the layer with vocabulary size, sequence length, embedding dimension and name.

set_add_positional_encoding()

void opennn::Embedding::set_add_positional_encoding ( bool enabled )

inline

Enables or disables sinusoidal positional encoding added to the embeddings.

set_dropout_rate()

void opennn::Embedding::set_dropout_rate ( float rate )

inline

Sets the dropout rate applied to the embedding output.

set_scale_embedding()

void opennn::Embedding::set_scale_embedding ( bool enabled )

inline

Enables or disables embedding scaling by sqrt(embedding_dimension).

write_JSON_body()

void opennn::Embedding::write_JSON_body ( JsonWriter & ) const

overridevirtual

Writes the layer configuration to a JSON writer.

Reimplemented from opennn::Layer.