opennn::TokenizerOperator Class Reference

#include <opennn/network/operators/tokenizer_operator.h>

Inheritance diagram for opennn::TokenizerOperator:

Public Types
using	VocabularyMap = StringMap<Index>

Public Member Functions
virtual vector< string >	tokenize (string_view text) const =0
virtual void	build_vocabulary (const vector< vector< string > > &documents, Index maximum_vocabulary_size, Index minimum_token_frequency)
virtual void	set_vocabulary (const vector< string > &)
const vector< string > &	get_vocabulary () const noexcept
const vector< string > &	get_reserved_tokens () const noexcept
const VocabularyMap &	get_vocabulary_map () const noexcept
Index	get_vocabulary_size () const noexcept
Index	token_to_id (string_view token) const
const string &	id_to_token (Index id) const
virtual vector< Index >	encode (string_view text) const
vector< Index >	encode_sequence (const vector< string > &tokens, Index sequence_length) const
vector< Index >	encode_sequence (string_view text, Index sequence_length) const
virtual string	decode (const vector< Index > &ids) const
virtual string	decode_token (Index id) const
virtual bool	supports_incremental_decode () const noexcept
Index	get_unk_id () const noexcept
virtual unique_ptr< TokenizerOperator >	clone () const =0
virtual string_view	get_kind () const =0
virtual uint64_t	fingerprint () const
void	to_JSON (JsonWriter &) const override
void	from_JSON (const Json *) override
Public Member Functions inherited from opennn::Operator
virtual	~Operator ()=default
virtual vector< TensorSpec >	parameter_specs () const
virtual vector< TensorSpec >	state_specs () const
virtual vector< SlotQuantization >	parameter_quantization () const
virtual vector< ParameterSlot >	parameter_slots ()
virtual void	link_parameters (span< const TensorView > views)
virtual void	link_gradients (span< const TensorView > views)
virtual void	link_states (span< const TensorView >)
virtual void	link_parameter_scales (span< const TensorView >)
bool	link_slots (span< const TensorView > views, TensorView *ParameterSlot::*member)
virtual void	initialize_states ()
virtual void	set_weights_dtype (Type new_weights_dtype)
virtual void	set_parameters_random ()
virtual void	set_parameters_glorot ()
virtual void	set_parameters_pytorch ()
virtual void	forward_propagate (ForwardPropagation &, size_t, ForwardPropagationMode)
virtual void	back_propagate (ForwardPropagation &, BackPropagation &, size_t) const
virtual void	load_state_from_JSON (const Json *)
TensorView &	get_input (ForwardPropagation &forward_propagation, size_t layer, size_t slot_index=0) const noexcept
vector< TensorView > &	get_inputs (ForwardPropagation &forward_propagation, size_t layer) const noexcept
TensorView &	get_output (ForwardPropagation &forward_propagation, size_t layer, size_t slot_index=0) const noexcept
TensorView &	get_output_delta (BackPropagation &back_propagation, size_t layer, size_t slot_index=0) const noexcept
TensorView &	get_input_delta (BackPropagation &back_propagation, size_t layer, size_t slot_index=0) const noexcept

Static Public Attributes
static constexpr Index	UNK_INDEX = 1
static constexpr Index	START_INDEX = 2
static constexpr Index	END_INDEX = 3

Protected Member Functions
void	rebuild_map ()
vector< Index >	frame_sequence (span< const Index >, Index sequence_length) const

Protected Attributes
vector< string >	reserved_tokens
Index	unk_id = 0
Index	start_id = -1
Index	end_id = -1
vector< string >	vocabulary
VocabularyMap	vocabulary_map

Additional Inherited Members
Public Attributes inherited from opennn::Operator
Type	compute_dtype = Type::FP32
Type	weights_dtype = Type::FP32
vector< size_t >	input_slots = {0}
vector< size_t >	output_slots = {1}
vector< size_t >	input_delta_slots = {1}
vector< size_t >	output_delta_slots = {0}

Member Typedef Documentation

VocabularyMap

using opennn::TokenizerOperator::VocabularyMap = StringMap<Index>

Member Function Documentation

build_vocabulary()

virtual void opennn::TokenizerOperator::build_vocabulary ( const vector< vector< string > > & documents, Index maximum_vocabulary_size, Index minimum_token_frequency )

virtual

Reimplemented in opennn::BytePairTokenizer, and opennn::WordPieceTokenizer.

clone()

virtual unique_ptr< TokenizerOperator > opennn::TokenizerOperator::clone ( ) const

pure virtual

Implemented in opennn::BytePairTokenizer, opennn::Qwen3Tokenizer, opennn::WordLevelTokenizer, and opennn::WordPieceTokenizer.

decode()

virtual string opennn::TokenizerOperator::decode ( const vector< Index > & ids ) const

virtual

Reimplemented in opennn::BytePairTokenizer.

decode_token()

virtual string opennn::TokenizerOperator::decode_token ( Index id ) const

virtual

Reimplemented in opennn::BytePairTokenizer.

encode()

virtual vector< Index > opennn::TokenizerOperator::encode ( string_view text ) const

virtual

Reimplemented in opennn::BytePairTokenizer, opennn::WordLevelTokenizer, and opennn::WordPieceTokenizer.

encode_sequence() [1/2]

vector< Index > opennn::TokenizerOperator::encode_sequence	(	const vector< string > &	tokens,
		Index	sequence_length ) const

encode_sequence() [2/2]

vector< Index > opennn::TokenizerOperator::encode_sequence	(	string_view	text,
		Index	sequence_length ) const

fingerprint()

virtual uint64_t opennn::TokenizerOperator::fingerprint ( ) const

virtual

Reimplemented in opennn::BytePairTokenizer, opennn::WordLevelTokenizer, and opennn::WordPieceTokenizer.

frame_sequence()

vector< Index > opennn::TokenizerOperator::frame_sequence ( span< const Index > , Index sequence_length ) const

protected

from_JSON()

void opennn::TokenizerOperator::from_JSON ( const Json * )

overridevirtual

Reimplemented from opennn::Operator.

Reimplemented in opennn::WordLevelTokenizer, and opennn::WordPieceTokenizer.

get_kind()

virtual string_view opennn::TokenizerOperator::get_kind ( ) const

pure virtual

Implemented in opennn::BytePairTokenizer, opennn::Qwen3Tokenizer, opennn::WordLevelTokenizer, and opennn::WordPieceTokenizer.

get_reserved_tokens()

const vector< string > & opennn::TokenizerOperator::get_reserved_tokens ( ) const

inlinenoexcept

get_unk_id()

Index opennn::TokenizerOperator::get_unk_id ( ) const

inlinenoexcept

get_vocabulary()

const vector< string > & opennn::TokenizerOperator::get_vocabulary ( ) const

inlinenoexcept

get_vocabulary_map()

const VocabularyMap & opennn::TokenizerOperator::get_vocabulary_map ( ) const

inlinenoexcept

get_vocabulary_size()

Index opennn::TokenizerOperator::get_vocabulary_size ( ) const

inlinenoexcept

id_to_token()

const string & opennn::TokenizerOperator::id_to_token

(

Index

id

)

const

rebuild_map()

void opennn::TokenizerOperator::rebuild_map ( )

protected

set_vocabulary()

virtual void opennn::TokenizerOperator::set_vocabulary ( const vector< string > & )

virtual

Reimplemented in opennn::BytePairTokenizer, and opennn::WordPieceTokenizer.

supports_incremental_decode()

virtual bool opennn::TokenizerOperator::supports_incremental_decode ( ) const

inlinevirtualnoexcept

Reimplemented in opennn::BytePairTokenizer.

to_JSON()

void opennn::TokenizerOperator::to_JSON ( JsonWriter & ) const

overridevirtual

Reimplemented from opennn::Operator.

Reimplemented in opennn::WordLevelTokenizer, and opennn::WordPieceTokenizer.

token_to_id()

Index opennn::TokenizerOperator::token_to_id

(

string_view

token

)

const

tokenize()

virtual vector< string > opennn::TokenizerOperator::tokenize ( string_view text ) const

pure virtual

Implemented in opennn::BytePairTokenizer, opennn::WordLevelTokenizer, and opennn::WordPieceTokenizer.

Member Data Documentation

end_id

Index opennn::TokenizerOperator::end_id = -1

protected

END_INDEX

Index opennn::TokenizerOperator::END_INDEX = 3

staticconstexpr

reserved_tokens

vector<string> opennn::TokenizerOperator::reserved_tokens

protected

start_id

Index opennn::TokenizerOperator::start_id = -1

protected

START_INDEX

Index opennn::TokenizerOperator::START_INDEX = 2

staticconstexpr

unk_id

Index opennn::TokenizerOperator::unk_id = 0

protected

UNK_INDEX

Index opennn::TokenizerOperator::UNK_INDEX = 1

staticconstexpr

vocabulary

vector<string> opennn::TokenizerOperator::vocabulary

protected

vocabulary_map

VocabularyMap opennn::TokenizerOperator::vocabulary_map

protected