#include <opennn/network/operators/tokenizer_operator.h>

Inheritance diagram for opennn::Qwen3Tokenizer:

Public Member Functions
	Qwen3Tokenizer ()=default

	Qwen3Tokenizer (const filesystem::path &directory)

void	load (const filesystem::path &vocabulary_json, const filesystem::path &merges_txt, const filesystem::path &special_tokens_tsv)

Index	get_im_start_id () const

Index	get_im_end_id () const

Index	get_endoftext_id () const

unique_ptr< TokenizerOperator >	clone () const override

string_view	get_kind () const override

Public Member Functions inherited from opennn::BytePairTokenizer
	BytePairTokenizer ()

	BytePairTokenizer (const filesystem::path &vocabulary_json, const filesystem::path &merges_txt)

void	load (const filesystem::path &vocabulary_json, const filesystem::path &merges_txt)

void	set_vocabulary (const vector< string > &) override

vector< string >	tokenize (string_view text) const override

vector< Index >	encode (string_view text) const override

string	decode (const vector< Index > &ids) const override

string	decode_token (Index id) const override

bool	supports_incremental_decode () const noexcept override

void	build_vocabulary (const vector< vector< string > > &, Index, Index) override

vector< string >	get_merges () const

void	set_merges (const vector< string > &)

void	set_special_tokens (const vector< string > &)

Index	get_special_token_id (string_view) const

uint64_t	fingerprint () const override

void	to_JSON (JsonWriter &) const override

void	from_JSON (const Json *) override

Public Member Functions inherited from opennn::TokenizerOperator
const vector< string > &	get_vocabulary () const noexcept

const vector< string > &	get_reserved_tokens () const noexcept

const VocabularyMap &	get_vocabulary_map () const noexcept

Index	get_vocabulary_size () const noexcept

Index	token_to_id (string_view token) const

const string &	id_to_token (Index id) const

vector< Index >	encode_sequence (const vector< string > &tokens, Index sequence_length) const

vector< Index >	encode_sequence (string_view text, Index sequence_length) const

Index	get_unk_id () const noexcept

void	to_JSON (JsonWriter &) const override

void	from_JSON (const Json *) override

Public Member Functions inherited from opennn::Operator
virtual	~Operator ()=default

virtual vector< TensorSpec >	parameter_specs () const

virtual vector< TensorSpec >	state_specs () const

virtual vector< SlotQuantization >	parameter_quantization () const

virtual vector< ParameterSlot >	parameter_slots ()

virtual void	link_parameters (span< const TensorView > views)

virtual void	link_gradients (span< const TensorView > views)

virtual void	link_states (span< const TensorView >)

virtual void	link_parameter_scales (span< const TensorView >)

bool	link_slots (span< const TensorView > views, TensorView ParameterSlot::member)

virtual void	initialize_states ()

virtual void	set_weights_dtype (Type new_weights_dtype)

virtual void	set_parameters_random ()

virtual void	set_parameters_glorot ()

virtual void	set_parameters_pytorch ()

virtual void	forward_propagate (ForwardPropagation &, size_t, ForwardPropagationMode)

virtual void	back_propagate (ForwardPropagation &, BackPropagation &, size_t) const

virtual void	load_state_from_JSON (const Json *)

TensorView &	get_input (ForwardPropagation &forward_propagation, size_t layer, size_t slot_index=0) const noexcept

vector< TensorView > &	get_inputs (ForwardPropagation &forward_propagation, size_t layer) const noexcept

TensorView &	get_output (ForwardPropagation &forward_propagation, size_t layer, size_t slot_index=0) const noexcept

TensorView &	get_output_delta (BackPropagation &back_propagation, size_t layer, size_t slot_index=0) const noexcept

TensorView &	get_input_delta (BackPropagation &back_propagation, size_t layer, size_t slot_index=0) const noexcept

Protected Member Functions
vector< string >	pre_tokenize (string_view text) const override

Protected Member Functions inherited from opennn::BytePairTokenizer
vector< string >	bpe (const string &token) const

void	tokenize_into (string_view, vector< string > , vector< Index > ) const

Protected Member Functions inherited from opennn::TokenizerOperator
void	rebuild_map ()

vector< Index >	frame_sequence (span< const Index >, Index sequence_length) const

Additional Inherited Members
Public Types inherited from opennn::TokenizerOperator
using	VocabularyMap = StringMap<Index>

Public Attributes inherited from opennn::Operator
Type	compute_dtype = Type::FP32

Type	weights_dtype = Type::FP32

vector< size_t >	input_slots = {0}

vector< size_t >	output_slots = {1}

vector< size_t >	input_delta_slots = {1}

vector< size_t >	output_delta_slots = {0}

Static Public Attributes inherited from opennn::BytePairTokenizer
static constexpr string_view	PAD_TOKEN = "[PAD]"

Static Public Attributes inherited from opennn::TokenizerOperator
static constexpr Index	UNK_INDEX = 1

static constexpr Index	START_INDEX = 2

static constexpr Index	END_INDEX = 3

Protected Attributes inherited from opennn::BytePairTokenizer
array< uint32_t, 256 >	byte_encoder {}

unordered_map< uint32_t, unsigned char >	byte_decoder

StringMap< int >	merge_ranks

shared_ptr< Cache >	bpe_cache

vector< string >	special_strings

unordered_set< Index >	special_ids

Protected Attributes inherited from opennn::TokenizerOperator
vector< string >	reserved_tokens

Index	unk_id = 0

Index	start_id = -1

Index	end_id = -1

vector< string >	vocabulary

VocabularyMap	vocabulary_map

Constructor & Destructor Documentation

◆ Qwen3Tokenizer() [1/2]

opennn::Qwen3Tokenizer::Qwen3Tokenizer ( )

default

◆ Qwen3Tokenizer() [2/2]

opennn::Qwen3Tokenizer::Qwen3Tokenizer ( const filesystem::path & directory )

explicit

Member Function Documentation

◆ clone()

unique_ptr< TokenizerOperator > opennn::Qwen3Tokenizer::clone ( ) const

inlineoverridevirtual

Reimplemented from opennn::BytePairTokenizer.

◆ get_endoftext_id()

Index opennn::Qwen3Tokenizer::get_endoftext_id ( ) const

inline

◆ get_im_end_id()

Index opennn::Qwen3Tokenizer::get_im_end_id ( ) const

inline

◆ get_im_start_id()

Index opennn::Qwen3Tokenizer::get_im_start_id ( ) const

inline

◆ get_kind()

string_view opennn::Qwen3Tokenizer::get_kind ( ) const

inlineoverridevirtual

Reimplemented from opennn::BytePairTokenizer.

◆ load()

void opennn::Qwen3Tokenizer::load	(	const filesystem::path &	vocabulary_json,
		const filesystem::path &	merges_txt,
		const filesystem::path &	special_tokens_tsv )

◆ pre_tokenize()

vector< string > opennn::Qwen3Tokenizer::pre_tokenize ( string_view text ) const

overrideprotectedvirtual

Reimplemented from opennn::BytePairTokenizer.

Public Member Functions

Protected Member Functions

Additional Inherited Members

Constructor & Destructor Documentation

◆ Qwen3Tokenizer() [1/2]

◆ Qwen3Tokenizer() [2/2]

Member Function Documentation

◆ clone()

◆ get_endoftext_id()

◆ get_im_end_id()

◆ get_im_start_id()

◆ get_kind()

◆ load()

◆ pre_tokenize()