opennn::Tokenizer Class Reference

#include <opennn/network/layers/tokenizer_layer.h>

Inheritance diagram for opennn::Tokenizer:

Public Member Functions
	Tokenizer (const Shape &={}, const string &="tokenizer", VariableRole=VariableRole::Input)
Shape	get_output_shape () const noexcept override
bool	accepts_input_rank (Index rank) const override
bool	allows_bf16_input_cast (size_t) const noexcept override
vector< TensorSpec >	get_forward_specs (Index) const override
void	set_tokenizer (unique_ptr< TokenizerOperator >)
const TokenizerOperator *	get_tokenizer () const noexcept
VariableRole	get_variable_role () const noexcept
void	set_vocabulary (const vector< string > &)
const vector< string > &	get_vocabulary () const
const TokenizerOperator::VocabularyMap &	get_vocabulary_map () const
Index	get_vocabulary_size () const
void	read_JSON_body (const Json *) override
void	write_JSON_body (JsonWriter &) const override
Public Member Functions inherited from opennn::Layer
virtual	~Layer ()=default
const string &	get_label () const noexcept
const string &	get_name () const
LayerType	get_type () const noexcept
void	set_input_shape (const Shape &new_input_shape)
virtual void	set_output_shape (const Shape &)
void	set_label (string new_label)
Index	get_parameters_number () const
const vector< Operator * > &	get_operators () const noexcept
virtual vector< TensorSpec >	get_parameter_specs () const
virtual vector< TensorSpec >	get_state_specs () const
virtual vector< TensorSpec >	get_backward_specs (Index batch_size) const
virtual ForwardSlotKind	get_forward_slot_kind (size_t) const
virtual bool	is_forward_slot_inference_elidable (size_t, Device) const noexcept
virtual size_t	get_recomputable_forward_slot () const noexcept
virtual void	recompute_forward_slot (ForwardPropagation &, size_t)
virtual bool	backward_uses_forward_output () const noexcept
virtual bool	backward_uses_input (size_t) const noexcept
virtual bool	preserves_output_delta_during_backward () const noexcept
virtual bool	allows_input_delta_alias () const noexcept
virtual bool	folds_input_delta_addend (size_t) const noexcept
virtual Shape	get_input_shape () const noexcept
virtual ActivationFunction	get_output_activation () const
virtual Index	get_sources_number () const noexcept
virtual bool	allows_successors () const noexcept
virtual bool	is_recurrent () const noexcept
virtual bool	skip_for_pre_scaled_input () const noexcept
virtual bool	uses_sequence_position () const noexcept
Index	get_inputs_number () const noexcept
Index	get_outputs_number () const
virtual void	forward_propagate (ForwardPropagation &forward_propagation, size_t layer, ForwardPropagationMode pass)
virtual void	back_propagate (ForwardPropagation &forward_propagation, BackPropagation &back_propagation, size_t i) const
virtual void	from_JSON (const JsonDocument &)
virtual void	on_loaded ()
virtual void	load_state_from_JSON (const JsonDocument &)
virtual void	to_JSON (JsonWriter &) const
virtual string	write_expression (const vector< string > &, const vector< string > &) const
bool	get_is_trainable () const noexcept
void	set_is_trainable (bool trainable)
Type	get_compute_dtype () const noexcept
Device	get_compute_device () const noexcept
void	set_compute_dtype (Type new_compute_dtype)
void	set_compute_device (Device new_compute_device)
virtual void	on_compute_dtype_changed ()
virtual float *	link_states (float *, Device)
float *	link_gradients (float *, Device)
vector< TensorView > &	get_parameter_views ()
const vector< TensorView > &	get_parameter_views () const noexcept
vector< TensorView > &	get_parameter_scales ()
const vector< TensorView > &	get_parameter_scales () const noexcept
vector< Operator::SlotQuantization >	get_parameter_quantization () const
virtual TiedWeight	get_tied_weight () const
virtual void	set_tied_weight (const TiedWeight &tied_weight)
void	redistribute_parameters_to_operators ()

Additional Inherited Members
Protected Types inherited from opennn::Layer
enum	Forward { Input , Output }
Protected Member Functions inherited from opennn::Layer
virtual void	apply_input_shape (const Shape &new_input_shape)
	Layer (LayerType type, Trainability trainability=Trainability::Trainable)
float *	link_views_to_operators (vector< TensorView > &, float *, vector< TensorSpec >(Operator::*specs_fn)() const, void(Operator::*link_fn)(span< const TensorView >), Device)
Static Protected Member Functions inherited from opennn::Layer
static bool	refresh_feature_storage (Buffer &storage, bool &dirty, Device device, Index features, Index columns, const function< void(float *)> &fill)
Protected Attributes inherited from opennn::Layer
string	label = "my_layer"
bool	is_trainable = true
Shape	input_shape
Type	compute_dtype = Type::FP32
Type	weights_dtype = Type::FP32
Device	compute_device = Device::CPU
vector< TensorView >	parameters
vector< TensorView >	parameter_scales
vector< TensorView >	states
vector< Operator * >	operators

Constructor & Destructor Documentation

Tokenizer()

opennn::Tokenizer::Tokenizer	(	const Shape &	= {},
		const string &	= "tokenizer",
		VariableRole	= VariableRole::Input )

Member Function Documentation

accepts_input_rank()

bool opennn::Tokenizer::accepts_input_rank ( Index rank ) const

inlineoverridevirtual

Reimplemented from opennn::Layer.

allows_bf16_input_cast()

bool opennn::Tokenizer::allows_bf16_input_cast ( size_t ) const

inlineoverridevirtualnoexcept

Reimplemented from opennn::Layer.

get_forward_specs()

vector< TensorSpec > opennn::Tokenizer::get_forward_specs ( Index ) const

inlineoverridevirtual

Reimplemented from opennn::Layer.

get_output_shape()

Shape opennn::Tokenizer::get_output_shape ( ) const

inlineoverridevirtualnoexcept

Implements opennn::Layer.

get_tokenizer()

const TokenizerOperator * opennn::Tokenizer::get_tokenizer ( ) const

inlinenoexcept

get_variable_role()

VariableRole opennn::Tokenizer::get_variable_role ( ) const

inlinenoexcept

get_vocabulary()

const vector< string > & opennn::Tokenizer::get_vocabulary

(

)

const

get_vocabulary_map()

const TokenizerOperator::VocabularyMap & opennn::Tokenizer::get_vocabulary_map

(

)

const

get_vocabulary_size()

Index opennn::Tokenizer::get_vocabulary_size ( ) const

inline

read_JSON_body()

void opennn::Tokenizer::read_JSON_body ( const Json * )

overridevirtual

Reimplemented from opennn::Layer.

set_tokenizer()

void opennn::Tokenizer::set_tokenizer

(

unique_ptr< TokenizerOperator >

)

set_vocabulary()

void opennn::Tokenizer::set_vocabulary

(

const vector< string > &

)

write_JSON_body()

void opennn::Tokenizer::write_JSON_body ( JsonWriter & ) const

overridevirtual

Reimplemented from opennn::Layer.