#include <opennn/network/layers/multihead_attention_layer.h>

Inheritance diagram for opennn::MultiHeadAttention:

Public Member Functions
	MultiHeadAttention (const Shape &=Shape({0, 0}), Index=0, const string &={})

	MultiHeadAttention (const Shape &, const Shape &, Index=0, const string &={})

Shape	get_input_shape () const noexcept override

Shape	get_output_shape () const override

Index	get_query_sequence_length () const noexcept

Index	get_source_sequence_length () const noexcept

Index	get_embedding_dimension () const noexcept

Index	get_heads_number () const noexcept

Index	get_head_dimension () const noexcept

Index	get_sources_number () const noexcept override

vector< TensorSpec >	get_forward_specs (Index) const override

vector< TensorSpec >	get_backward_specs (Index) const override

ForwardSlotKind	get_forward_slot_kind (size_t slot) const override

bool	backward_uses_forward_output () const noexcept override

bool	preserves_output_delta_during_backward () const noexcept override

bool	folds_input_delta_addend (size_t input) const noexcept override

void	set (Index=0, Index=0, Index=0, Index=0, CausalMask=CausalMask::No, const string &="multihead_attention_layer")

bool	accepts_input_rank (Index rank) const override

void	apply_input_shape (const Shape &) override

void	on_compute_dtype_changed () override

void	set_dropout_rate (float new_dropout_rate)

void	set_zero_padded_queries (bool new_zero_padded_queries)

void	set_sdpa_auto (bool new_sdpa_auto)

void	set_sdpa_min_sequence_length (Index)

bool	should_use_sdpa () const

void	apply_sdpa_choice ()

void	read_JSON_body (const Json *) override

void	write_JSON_body (JsonWriter &) const override

Public Member Functions inherited from opennn::Layer
virtual	~Layer ()=default

const string &	get_label () const noexcept

const string &	get_name () const

LayerType	get_type () const noexcept

void	set_input_shape (const Shape &new_input_shape)

virtual void	set_output_shape (const Shape &)

void	set_label (string new_label)

Index	get_parameters_number () const

const vector< Operator * > &	get_operators () const noexcept

virtual vector< TensorSpec >	get_parameter_specs () const

virtual vector< TensorSpec >	get_state_specs () const

virtual bool	is_forward_slot_inference_elidable (size_t, Device) const noexcept

virtual size_t	get_recomputable_forward_slot () const noexcept

virtual void	recompute_forward_slot (ForwardPropagation &, size_t)

virtual bool	backward_uses_input (size_t) const noexcept

virtual bool	allows_input_delta_alias () const noexcept

virtual ActivationFunction	get_output_activation () const

virtual bool	allows_successors () const noexcept

virtual bool	is_recurrent () const noexcept

virtual bool	skip_for_pre_scaled_input () const noexcept

virtual bool	uses_sequence_position () const noexcept

virtual bool	allows_bf16_input_cast (size_t) const noexcept

Index	get_inputs_number () const noexcept

Index	get_outputs_number () const

virtual void	forward_propagate (ForwardPropagation &forward_propagation, size_t layer, ForwardPropagationMode pass)

virtual void	back_propagate (ForwardPropagation &forward_propagation, BackPropagation &back_propagation, size_t i) const

virtual void	from_JSON (const JsonDocument &)

virtual void	on_loaded ()

virtual void	load_state_from_JSON (const JsonDocument &)

virtual void	to_JSON (JsonWriter &) const

virtual string	write_expression (const vector< string > &, const vector< string > &) const

bool	get_is_trainable () const noexcept

void	set_is_trainable (bool trainable)

Type	get_compute_dtype () const noexcept

Device	get_compute_device () const noexcept

void	set_compute_dtype (Type new_compute_dtype)

void	set_compute_device (Device new_compute_device)

virtual float *	link_states (float *, Device)

float *	link_gradients (float *, Device)

vector< TensorView > &	get_parameter_views ()

const vector< TensorView > &	get_parameter_views () const noexcept

vector< TensorView > &	get_parameter_scales ()

const vector< TensorView > &	get_parameter_scales () const noexcept

vector< Operator::SlotQuantization >	get_parameter_quantization () const

virtual TiedWeight	get_tied_weight () const

virtual void	set_tied_weight (const TiedWeight &tied_weight)

void	redistribute_parameters_to_operators ()

Static Public Attributes
static const Index	default_sdpa_min_sequence_length

Additional Inherited Members
Protected Types inherited from opennn::Layer
enum	Forward { Input , Output }

Protected Member Functions inherited from opennn::Layer
	Layer (LayerType type, Trainability trainability=Trainability::Trainable)

float *	link_views_to_operators (vector< TensorView > &, float , vector< TensorSpec >(Operator::specs_fn)() const, void(Operator::*link_fn)(span< const TensorView >), Device)

Static Protected Member Functions inherited from opennn::Layer
static bool	refresh_feature_storage (Buffer &storage, bool &dirty, Device device, Index features, Index columns, const function< void(float *)> &fill)

Protected Attributes inherited from opennn::Layer
string	label = "my_layer"

bool	is_trainable = true

Shape	input_shape

Type	compute_dtype = Type::FP32

Type	weights_dtype = Type::FP32

Device	compute_device = Device::CPU

vector< TensorView >	parameters

vector< TensorView >	parameter_scales

vector< TensorView >	states

vector< Operator * >	operators

Constructor & Destructor Documentation

◆ MultiHeadAttention() [1/2]

opennn::MultiHeadAttention::MultiHeadAttention	(	const Shape &	= Shape({0, 0}),
		Index	= 0,
		const string &	= {} )

◆ MultiHeadAttention() [2/2]

opennn::MultiHeadAttention::MultiHeadAttention	(	const Shape &	,
		const Shape &	,
		Index	= 0,
		const string &	= {} )

Member Function Documentation

◆ accepts_input_rank()

bool opennn::MultiHeadAttention::accepts_input_rank ( Index rank ) const

inlineoverridevirtual

Reimplemented from opennn::Layer.

◆ apply_input_shape()

void opennn::MultiHeadAttention::apply_input_shape ( const Shape & )

overridevirtual

Reimplemented from opennn::Layer.

◆ apply_sdpa_choice()

void opennn::MultiHeadAttention::apply_sdpa_choice ( )

◆ backward_uses_forward_output()

bool opennn::MultiHeadAttention::backward_uses_forward_output ( ) const

inlineoverridevirtualnoexcept

Reimplemented from opennn::Layer.

◆ folds_input_delta_addend()

bool opennn::MultiHeadAttention::folds_input_delta_addend ( size_t input ) const

inlineoverridevirtualnoexcept

Reimplemented from opennn::Layer.

◆ get_backward_specs()

vector< TensorSpec > opennn::MultiHeadAttention::get_backward_specs ( Index ) const

overridevirtual

Reimplemented from opennn::Layer.

◆ get_embedding_dimension()

Index opennn::MultiHeadAttention::get_embedding_dimension ( ) const

inlinenoexcept

◆ get_forward_slot_kind()

ForwardSlotKind opennn::MultiHeadAttention::get_forward_slot_kind ( size_t slot ) const

inlineoverridevirtual

Reimplemented from opennn::Layer.

◆ get_forward_specs()

vector< TensorSpec > opennn::MultiHeadAttention::get_forward_specs ( Index ) const

overridevirtual

Reimplemented from opennn::Layer.

◆ get_head_dimension()

Index opennn::MultiHeadAttention::get_head_dimension ( ) const

inlinenoexcept

◆ get_heads_number()

Index opennn::MultiHeadAttention::get_heads_number ( ) const

inlinenoexcept

◆ get_input_shape()

Shape opennn::MultiHeadAttention::get_input_shape ( ) const

overridevirtualnoexcept

Reimplemented from opennn::Layer.

◆ get_output_shape()

Shape opennn::MultiHeadAttention::get_output_shape ( ) const

inlineoverridevirtual

Implements opennn::Layer.

◆ get_query_sequence_length()

Index opennn::MultiHeadAttention::get_query_sequence_length ( ) const

inlinenoexcept

◆ get_source_sequence_length()

Index opennn::MultiHeadAttention::get_source_sequence_length ( ) const

inlinenoexcept

◆ get_sources_number()

Index opennn::MultiHeadAttention::get_sources_number ( ) const

inlineoverridevirtualnoexcept

Reimplemented from opennn::Layer.

◆ on_compute_dtype_changed()

void opennn::MultiHeadAttention::on_compute_dtype_changed ( )

overridevirtual

Reimplemented from opennn::Layer.

◆ preserves_output_delta_during_backward()

bool opennn::MultiHeadAttention::preserves_output_delta_during_backward ( ) const

inlineoverridevirtualnoexcept

Reimplemented from opennn::Layer.

◆ read_JSON_body()

void opennn::MultiHeadAttention::read_JSON_body ( const Json * )

overridevirtual

Reimplemented from opennn::Layer.

◆ set()

void opennn::MultiHeadAttention::set	(	Index	= 0,
		Index	= 0,
		Index	= 0,
		Index	= 0,
		CausalMask	= CausalMask::No,
		const string &	= "multihead_attention_layer" )

◆ set_dropout_rate()

void opennn::MultiHeadAttention::set_dropout_rate ( float new_dropout_rate )

inline

◆ set_sdpa_auto()

void opennn::MultiHeadAttention::set_sdpa_auto ( bool new_sdpa_auto )

◆ set_sdpa_min_sequence_length()

void opennn::MultiHeadAttention::set_sdpa_min_sequence_length ( Index )

◆ set_zero_padded_queries()

void opennn::MultiHeadAttention::set_zero_padded_queries ( bool new_zero_padded_queries )

◆ should_use_sdpa()

bool opennn::MultiHeadAttention::should_use_sdpa ( ) const

◆ write_JSON_body()

void opennn::MultiHeadAttention::write_JSON_body ( JsonWriter & ) const

overridevirtual

Reimplemented from opennn::Layer.

Member Data Documentation

◆ default_sdpa_min_sequence_length

const Index opennn::MultiHeadAttention::default_sdpa_min_sequence_length

static

Public Member Functions

Static Public Attributes

Additional Inherited Members

Constructor & Destructor Documentation

◆ MultiHeadAttention() [1/2]

◆ MultiHeadAttention() [2/2]

Member Function Documentation

◆ accepts_input_rank()

◆ apply_input_shape()

◆ apply_sdpa_choice()

◆ backward_uses_forward_output()

◆ folds_input_delta_addend()

◆ get_backward_specs()

◆ get_embedding_dimension()

◆ get_forward_slot_kind()

◆ get_forward_specs()

◆ get_head_dimension()

◆ get_heads_number()

◆ get_input_shape()

◆ get_output_shape()

◆ get_query_sequence_length()

◆ get_source_sequence_length()

◆ get_sources_number()

◆ on_compute_dtype_changed()

◆ preserves_output_delta_during_backward()

◆ read_JSON_body()

◆ set()

◆ set_dropout_rate()

◆ set_sdpa_auto()

◆ set_sdpa_min_sequence_length()

◆ set_zero_padded_queries()

◆ should_use_sdpa()

◆ write_JSON_body()

Member Data Documentation

◆ default_sdpa_min_sequence_length