| back_propagate(ForwardPropagation &, BackPropagation &, size_t) const | opennn::Operator | inlinevirtual |
| bpe(const string &token) const | opennn::BytePairTokenizer | protected |
| bpe_cache | opennn::BytePairTokenizer | protected |
| build_vocabulary(const vector< vector< string > > &, Index, Index) override | opennn::BytePairTokenizer | inlinevirtual |
| byte_decoder | opennn::BytePairTokenizer | protected |
| byte_encoder | opennn::BytePairTokenizer | protected |
| BytePairTokenizer() | opennn::BytePairTokenizer | |
| BytePairTokenizer(const filesystem::path &vocabulary_json, const filesystem::path &merges_txt) | opennn::BytePairTokenizer | |
| clone() const override | opennn::Qwen3Tokenizer | inlinevirtual |
| compute_dtype | opennn::Operator | |
| decode(const vector< Index > &ids) const override | opennn::BytePairTokenizer | virtual |
| decode_token(Index id) const override | opennn::BytePairTokenizer | virtual |
| encode(string_view text) const override | opennn::BytePairTokenizer | virtual |
| encode_sequence(const vector< string > &tokens, Index sequence_length) const | opennn::TokenizerOperator | |
| encode_sequence(string_view text, Index sequence_length) const | opennn::TokenizerOperator | |
| end_id | opennn::TokenizerOperator | protected |
| END_INDEX | opennn::TokenizerOperator | static |
| fingerprint() const override | opennn::BytePairTokenizer | virtual |
| forward_propagate(ForwardPropagation &, size_t, ForwardPropagationMode) | opennn::Operator | inlinevirtual |
| frame_sequence(span< const Index >, Index sequence_length) const | opennn::TokenizerOperator | protected |
| from_JSON(const Json *) override | opennn::BytePairTokenizer | virtual |
| get_endoftext_id() const | opennn::Qwen3Tokenizer | inline |
| get_im_end_id() const | opennn::Qwen3Tokenizer | inline |
| get_im_start_id() const | opennn::Qwen3Tokenizer | inline |
| get_input(ForwardPropagation &forward_propagation, size_t layer, size_t slot_index=0) const noexcept | opennn::Operator | inline |
| get_input_delta(BackPropagation &back_propagation, size_t layer, size_t slot_index=0) const noexcept | opennn::Operator | inline |
| get_inputs(ForwardPropagation &forward_propagation, size_t layer) const noexcept | opennn::Operator | inline |
| get_kind() const override | opennn::Qwen3Tokenizer | inlinevirtual |
| get_merges() const | opennn::BytePairTokenizer | |
| get_output(ForwardPropagation &forward_propagation, size_t layer, size_t slot_index=0) const noexcept | opennn::Operator | inline |
| get_output_delta(BackPropagation &back_propagation, size_t layer, size_t slot_index=0) const noexcept | opennn::Operator | inline |
| get_reserved_tokens() const noexcept | opennn::TokenizerOperator | inline |
| get_special_token_id(string_view) const | opennn::BytePairTokenizer | |
| get_unk_id() const noexcept | opennn::TokenizerOperator | inline |
| get_vocabulary() const noexcept | opennn::TokenizerOperator | inline |
| get_vocabulary_map() const noexcept | opennn::TokenizerOperator | inline |
| get_vocabulary_size() const noexcept | opennn::TokenizerOperator | inline |
| id_to_token(Index id) const | opennn::TokenizerOperator | |
| initialize_states() | opennn::Operator | inlinevirtual |
| input_delta_slots | opennn::Operator | |
| input_slots | opennn::Operator | |
| link_gradients(span< const TensorView > views) | opennn::Operator | inlinevirtual |
| link_parameter_scales(span< const TensorView >) | opennn::Operator | inlinevirtual |
| link_parameters(span< const TensorView > views) | opennn::Operator | inlinevirtual |
| link_slots(span< const TensorView > views, TensorView *ParameterSlot::*member) | opennn::Operator | inline |
| link_states(span< const TensorView >) | opennn::Operator | inlinevirtual |
| load(const filesystem::path &vocabulary_json, const filesystem::path &merges_txt, const filesystem::path &special_tokens_tsv) | opennn::Qwen3Tokenizer | |
| opennn::BytePairTokenizer::load(const filesystem::path &vocabulary_json, const filesystem::path &merges_txt) | opennn::BytePairTokenizer | |
| load_state_from_JSON(const Json *) | opennn::Operator | inlinevirtual |
| merge_ranks | opennn::BytePairTokenizer | protected |
| output_delta_slots | opennn::Operator | |
| output_slots | opennn::Operator | |
| PAD_TOKEN | opennn::BytePairTokenizer | static |
| parameter_quantization() const | opennn::Operator | inlinevirtual |
| parameter_slots() | opennn::Operator | inlinevirtual |
| parameter_specs() const | opennn::Operator | inlinevirtual |
| pre_tokenize(string_view text) const override | opennn::Qwen3Tokenizer | protectedvirtual |
| Qwen3Tokenizer()=default | opennn::Qwen3Tokenizer | |
| Qwen3Tokenizer(const filesystem::path &directory) | opennn::Qwen3Tokenizer | explicit |
| rebuild_map() | opennn::TokenizerOperator | protected |
| reserved_tokens | opennn::TokenizerOperator | protected |
| set_merges(const vector< string > &) | opennn::BytePairTokenizer | |
| set_parameters_glorot() | opennn::Operator | inlinevirtual |
| set_parameters_pytorch() | opennn::Operator | inlinevirtual |
| set_parameters_random() | opennn::Operator | inlinevirtual |
| set_special_tokens(const vector< string > &) | opennn::BytePairTokenizer | |
| set_vocabulary(const vector< string > &) override | opennn::BytePairTokenizer | virtual |
| set_weights_dtype(Type new_weights_dtype) | opennn::Operator | inlinevirtual |
| special_ids | opennn::BytePairTokenizer | protected |
| special_strings | opennn::BytePairTokenizer | protected |
| start_id | opennn::TokenizerOperator | protected |
| START_INDEX | opennn::TokenizerOperator | static |
| state_specs() const | opennn::Operator | inlinevirtual |
| supports_incremental_decode() const noexcept override | opennn::BytePairTokenizer | inlinevirtual |
| to_JSON(JsonWriter &) const override | opennn::BytePairTokenizer | virtual |
| token_to_id(string_view token) const | opennn::TokenizerOperator | |
| tokenize(string_view text) const override | opennn::BytePairTokenizer | virtual |
| tokenize_into(string_view, vector< string > *, vector< Index > *) const | opennn::BytePairTokenizer | protected |
| unk_id | opennn::TokenizerOperator | protected |
| UNK_INDEX | opennn::TokenizerOperator | static |
| vocabulary | opennn::TokenizerOperator | protected |
| vocabulary_map | opennn::TokenizerOperator | protected |
| VocabularyMap typedef | opennn::TokenizerOperator | |
| weights_dtype | opennn::Operator | |
| ~Operator()=default | opennn::Operator | virtual |
