tokenizer_operator.h File Reference

#include <array>
#include <memory>
#include <unordered_set>
#include "opennn/network/operators/operator.h"
#include "opennn/core/string_utilities.h"

Go to the source code of this file.

Classes
class	opennn::TokenizerOperator
class	opennn::WordLevelTokenizer
class	opennn::WordPieceTokenizer
class	opennn::BytePairTokenizer
class	opennn::Qwen3Tokenizer

Namespaces
namespace	opennn

Functions
vector< string >	opennn::make_vocabulary (const unordered_map< string_view, size_t > &, span< const string > reserved_tokens, Index maximum_size, Index minimum_frequency)
constexpr size_t	opennn::utf8_sequence_length (unsigned char lead) noexcept
constexpr bool	opennn::is_utf8_continuation (unsigned char byte) noexcept
unique_ptr< TokenizerOperator >	opennn::make_tokenizer_operator (string_view kind)