#include <opennn/network/operators/tokenizer_operator.h>

Inheritance diagram for opennn::BytePairTokenizer:

Public Member Functions
	BytePairTokenizer ()

	BytePairTokenizer (const filesystem::path &vocabulary_json, const filesystem::path &merges_txt)

void	load (const filesystem::path &vocabulary_json, const filesystem::path &merges_txt)

void	set_vocabulary (const vector< string > &) override

vector< string >	tokenize (string_view text) const override

vector< Index >	encode (string_view text) const override

string	decode (const vector< Index > &ids) const override

string	decode_token (Index id) const override

bool	supports_incremental_decode () const noexcept override

void	build_vocabulary (const vector< vector< string > > &, Index, Index) override

vector< string >	get_merges () const

void	set_merges (const vector< string > &)

void	set_special_tokens (const vector< string > &)

Index	get_special_token_id (string_view) const

unique_ptr< TokenizerOperator >	clone () const override

string_view	get_kind () const override

uint64_t	fingerprint () const override

void	to_JSON (JsonWriter &) const override

void	from_JSON (const Json *) override

Public Member Functions inherited from opennn::TokenizerOperator
const vector< string > &	get_vocabulary () const noexcept

const vector< string > &	get_reserved_tokens () const noexcept

const VocabularyMap &	get_vocabulary_map () const noexcept

Index	get_vocabulary_size () const noexcept

Index	token_to_id (string_view token) const

const string &	id_to_token (Index id) const

vector< Index >	encode_sequence (const vector< string > &tokens, Index sequence_length) const

vector< Index >	encode_sequence (string_view text, Index sequence_length) const

Index	get_unk_id () const noexcept

void	to_JSON (JsonWriter &) const override

void	from_JSON (const Json *) override

Public Member Functions inherited from opennn::Operator
virtual	~Operator ()=default

virtual vector< TensorSpec >	parameter_specs () const

virtual vector< TensorSpec >	state_specs () const

virtual vector< SlotQuantization >	parameter_quantization () const

virtual vector< ParameterSlot >	parameter_slots ()

virtual void	link_parameters (span< const TensorView > views)

virtual void	link_gradients (span< const TensorView > views)

virtual void	link_states (span< const TensorView >)

virtual void	link_parameter_scales (span< const TensorView >)

bool	link_slots (span< const TensorView > views, TensorView ParameterSlot::member)

virtual void	initialize_states ()

virtual void	set_weights_dtype (Type new_weights_dtype)

virtual void	set_parameters_random ()

virtual void	set_parameters_glorot ()

virtual void	set_parameters_pytorch ()

virtual void	forward_propagate (ForwardPropagation &, size_t, ForwardPropagationMode)

virtual void	back_propagate (ForwardPropagation &, BackPropagation &, size_t) const

virtual void	load_state_from_JSON (const Json *)

TensorView &	get_input (ForwardPropagation &forward_propagation, size_t layer, size_t slot_index=0) const noexcept

vector< TensorView > &	get_inputs (ForwardPropagation &forward_propagation, size_t layer) const noexcept

TensorView &	get_output (ForwardPropagation &forward_propagation, size_t layer, size_t slot_index=0) const noexcept

TensorView &	get_output_delta (BackPropagation &back_propagation, size_t layer, size_t slot_index=0) const noexcept

TensorView &	get_input_delta (BackPropagation &back_propagation, size_t layer, size_t slot_index=0) const noexcept

Static Public Attributes
static constexpr string_view	PAD_TOKEN = "[PAD]"

Static Public Attributes inherited from opennn::TokenizerOperator
static constexpr Index	UNK_INDEX = 1

static constexpr Index	START_INDEX = 2

static constexpr Index	END_INDEX = 3

Protected Member Functions
virtual vector< string >	pre_tokenize (string_view text) const

vector< string >	bpe (const string &token) const

void	tokenize_into (string_view, vector< string > , vector< Index > ) const

Protected Member Functions inherited from opennn::TokenizerOperator
void	rebuild_map ()

vector< Index >	frame_sequence (span< const Index >, Index sequence_length) const

Protected Attributes
array< uint32_t, 256 >	byte_encoder {}

unordered_map< uint32_t, unsigned char >	byte_decoder

StringMap< int >	merge_ranks

shared_ptr< Cache >	bpe_cache

vector< string >	special_strings

unordered_set< Index >	special_ids

Protected Attributes inherited from opennn::TokenizerOperator
vector< string >	reserved_tokens

Index	unk_id = 0

Index	start_id = -1

Index	end_id = -1

vector< string >	vocabulary

VocabularyMap	vocabulary_map

Additional Inherited Members
Public Types inherited from opennn::TokenizerOperator
using	VocabularyMap = StringMap<Index>

Public Attributes inherited from opennn::Operator
Type	compute_dtype = Type::FP32

Type	weights_dtype = Type::FP32

vector< size_t >	input_slots = {0}

vector< size_t >	output_slots = {1}

vector< size_t >	input_delta_slots = {1}

vector< size_t >	output_delta_slots = {0}

Constructor & Destructor Documentation

◆ BytePairTokenizer() [1/2]

opennn::BytePairTokenizer::BytePairTokenizer ( )

◆ BytePairTokenizer() [2/2]

opennn::BytePairTokenizer::BytePairTokenizer	(	const filesystem::path &	vocabulary_json,
		const filesystem::path &	merges_txt )

Member Function Documentation

◆ bpe()

vector< string > opennn::BytePairTokenizer::bpe ( const string & token ) const

protected

◆ build_vocabulary()

void opennn::BytePairTokenizer::build_vocabulary	(	const vector< vector< string > > &	,
		Index	,
		Index	)

inlineoverridevirtual

Reimplemented from opennn::TokenizerOperator.

◆ clone()

unique_ptr< TokenizerOperator > opennn::BytePairTokenizer::clone ( ) const

inlineoverridevirtual

Implements opennn::TokenizerOperator.

Reimplemented in opennn::Qwen3Tokenizer.

◆ decode()

string opennn::BytePairTokenizer::decode ( const vector< Index > & ids ) const

overridevirtual

Reimplemented from opennn::TokenizerOperator.

◆ decode_token()

string opennn::BytePairTokenizer::decode_token ( Index id ) const

overridevirtual

Reimplemented from opennn::TokenizerOperator.

◆ encode()

vector< Index > opennn::BytePairTokenizer::encode ( string_view text ) const

overridevirtual

Reimplemented from opennn::TokenizerOperator.

◆ fingerprint()

uint64_t opennn::BytePairTokenizer::fingerprint ( ) const

overridevirtual

Reimplemented from opennn::TokenizerOperator.

◆ from_JSON()

void opennn::BytePairTokenizer::from_JSON ( const Json * )

overridevirtual

Reimplemented from opennn::Operator.

◆ get_kind()

string_view opennn::BytePairTokenizer::get_kind ( ) const

inlineoverridevirtual

Implements opennn::TokenizerOperator.

Reimplemented in opennn::Qwen3Tokenizer.

◆ get_merges()

vector< string > opennn::BytePairTokenizer::get_merges ( ) const

◆ get_special_token_id()

Index opennn::BytePairTokenizer::get_special_token_id ( string_view ) const

◆ load()

void opennn::BytePairTokenizer::load	(	const filesystem::path &	vocabulary_json,
		const filesystem::path &	merges_txt )

◆ pre_tokenize()

virtual vector< string > opennn::BytePairTokenizer::pre_tokenize ( string_view text ) const

protectedvirtual

Reimplemented in opennn::Qwen3Tokenizer.

◆ set_merges()

void opennn::BytePairTokenizer::set_merges ( const vector< string > & )

◆ set_special_tokens()

void opennn::BytePairTokenizer::set_special_tokens ( const vector< string > & )

◆ set_vocabulary()

void opennn::BytePairTokenizer::set_vocabulary ( const vector< string > & )

overridevirtual

Reimplemented from opennn::TokenizerOperator.

◆ supports_incremental_decode()

bool opennn::BytePairTokenizer::supports_incremental_decode ( ) const

inlineoverridevirtualnoexcept

Reimplemented from opennn::TokenizerOperator.

◆ to_JSON()

void opennn::BytePairTokenizer::to_JSON ( JsonWriter & ) const

overridevirtual

Reimplemented from opennn::Operator.

◆ tokenize()

vector< string > opennn::BytePairTokenizer::tokenize ( string_view text ) const

overridevirtual

Implements opennn::TokenizerOperator.

◆ tokenize_into()

void opennn::BytePairTokenizer::tokenize_into	(	string_view	,
		vector< string > *	,
		vector< Index > *	) const

protected

Member Data Documentation

◆ bpe_cache

shared_ptr<Cache> opennn::BytePairTokenizer::bpe_cache

protected

◆ byte_decoder

unordered_map<uint32_t, unsigned char> opennn::BytePairTokenizer::byte_decoder

protected

◆ byte_encoder

array<uint32_t, 256> opennn::BytePairTokenizer::byte_encoder {}

protected

◆ merge_ranks

StringMap<int> opennn::BytePairTokenizer::merge_ranks

protected

◆ PAD_TOKEN

string_view opennn::BytePairTokenizer::PAD_TOKEN = "[PAD]"

staticconstexpr

◆ special_ids

unordered_set<Index> opennn::BytePairTokenizer::special_ids

protected

◆ special_strings

vector<string> opennn::BytePairTokenizer::special_strings

protected

Public Member Functions

Static Public Attributes

Protected Member Functions

Protected Attributes

Additional Inherited Members

Constructor & Destructor Documentation

◆ BytePairTokenizer() [1/2]

◆ BytePairTokenizer() [2/2]

Member Function Documentation

◆ bpe()

◆ build_vocabulary()

◆ clone()

◆ decode()

◆ decode_token()

◆ encode()

◆ fingerprint()

◆ from_JSON()

◆ get_kind()

◆ get_merges()

◆ get_special_token_id()

◆ load()

◆ pre_tokenize()

◆ set_merges()

◆ set_special_tokens()

◆ set_vocabulary()

◆ supports_incremental_decode()

◆ to_JSON()

◆ tokenize()

◆ tokenize_into()

Member Data Documentation

◆ bpe_cache

◆ byte_decoder

◆ byte_encoder

◆ merge_ranks

◆ PAD_TOKEN

◆ special_ids

◆ special_strings