opennn::WordPieceTokenizer Class Reference

#include <opennn/network/operators/tokenizer_operator.h>

Inheritance diagram for opennn::WordPieceTokenizer:

Public Member Functions
	WordPieceTokenizer ()=default
	WordPieceTokenizer (const vector< string > &vocabulary)
void	load_vocabulary (const filesystem::path &vocabulary_file)
void	set_vocabulary (const vector< string > &) override
vector< string >	tokenize (string_view text) const override
vector< Index >	encode (string_view text) const override
void	build_vocabulary (const vector< vector< string > > &, Index, Index) override
void	set_lower_case (bool value) noexcept
unique_ptr< TokenizerOperator >	clone () const override
string_view	get_kind () const override
uint64_t	fingerprint () const override
void	to_JSON (JsonWriter &) const override
void	from_JSON (const Json *) override
Public Member Functions inherited from opennn::TokenizerOperator
const vector< string > &	get_vocabulary () const noexcept
const vector< string > &	get_reserved_tokens () const noexcept
const VocabularyMap &	get_vocabulary_map () const noexcept
Index	get_vocabulary_size () const noexcept
Index	token_to_id (string_view token) const
const string &	id_to_token (Index id) const
vector< Index >	encode_sequence (const vector< string > &tokens, Index sequence_length) const
vector< Index >	encode_sequence (string_view text, Index sequence_length) const
virtual string	decode (const vector< Index > &ids) const
virtual string	decode_token (Index id) const
virtual bool	supports_incremental_decode () const noexcept
Index	get_unk_id () const noexcept
Public Member Functions inherited from opennn::Operator
virtual	~Operator ()=default
virtual vector< TensorSpec >	parameter_specs () const
virtual vector< TensorSpec >	state_specs () const
virtual vector< SlotQuantization >	parameter_quantization () const
virtual vector< ParameterSlot >	parameter_slots ()
virtual void	link_parameters (span< const TensorView > views)
virtual void	link_gradients (span< const TensorView > views)
virtual void	link_states (span< const TensorView >)
virtual void	link_parameter_scales (span< const TensorView >)
bool	link_slots (span< const TensorView > views, TensorView *ParameterSlot::*member)
virtual void	initialize_states ()
virtual void	set_weights_dtype (Type new_weights_dtype)
virtual void	set_parameters_random ()
virtual void	set_parameters_glorot ()
virtual void	set_parameters_pytorch ()
virtual void	forward_propagate (ForwardPropagation &, size_t, ForwardPropagationMode)
virtual void	back_propagate (ForwardPropagation &, BackPropagation &, size_t) const
virtual void	load_state_from_JSON (const Json *)
TensorView &	get_input (ForwardPropagation &forward_propagation, size_t layer, size_t slot_index=0) const noexcept
vector< TensorView > &	get_inputs (ForwardPropagation &forward_propagation, size_t layer) const noexcept
TensorView &	get_output (ForwardPropagation &forward_propagation, size_t layer, size_t slot_index=0) const noexcept
TensorView &	get_output_delta (BackPropagation &back_propagation, size_t layer, size_t slot_index=0) const noexcept
TensorView &	get_input_delta (BackPropagation &back_propagation, size_t layer, size_t slot_index=0) const noexcept

Additional Inherited Members
Public Types inherited from opennn::TokenizerOperator
using	VocabularyMap = StringMap<Index>
Public Attributes inherited from opennn::Operator
Type	compute_dtype = Type::FP32
Type	weights_dtype = Type::FP32
vector< size_t >	input_slots = {0}
vector< size_t >	output_slots = {1}
vector< size_t >	input_delta_slots = {1}
vector< size_t >	output_delta_slots = {0}
Static Public Attributes inherited from opennn::TokenizerOperator
static constexpr Index	UNK_INDEX = 1
static constexpr Index	START_INDEX = 2
static constexpr Index	END_INDEX = 3
Protected Member Functions inherited from opennn::TokenizerOperator
void	rebuild_map ()
vector< Index >	frame_sequence (span< const Index >, Index sequence_length) const
Protected Attributes inherited from opennn::TokenizerOperator
vector< string >	reserved_tokens
Index	unk_id = 0
Index	start_id = -1
Index	end_id = -1
vector< string >	vocabulary
VocabularyMap	vocabulary_map

Constructor & Destructor Documentation

WordPieceTokenizer() [1/2]

opennn::WordPieceTokenizer::WordPieceTokenizer ( )

default

WordPieceTokenizer() [2/2]

opennn::WordPieceTokenizer::WordPieceTokenizer ( const vector< string > & vocabulary )

explicit

Member Function Documentation

build_vocabulary()

void opennn::WordPieceTokenizer::build_vocabulary ( const vector< vector< string > > & , Index , Index  )

inlineoverridevirtual

Reimplemented from opennn::TokenizerOperator.

clone()

unique_ptr< TokenizerOperator > opennn::WordPieceTokenizer::clone ( ) const

inlineoverridevirtual

Implements opennn::TokenizerOperator.

encode()

vector< Index > opennn::WordPieceTokenizer::encode ( string_view text ) const

overridevirtual

Reimplemented from opennn::TokenizerOperator.

fingerprint()

uint64_t opennn::WordPieceTokenizer::fingerprint ( ) const

overridevirtual

Reimplemented from opennn::TokenizerOperator.

from_JSON()

void opennn::WordPieceTokenizer::from_JSON ( const Json * )

overridevirtual

Reimplemented from opennn::TokenizerOperator.

get_kind()

string_view opennn::WordPieceTokenizer::get_kind ( ) const

inlineoverridevirtual

Implements opennn::TokenizerOperator.

load_vocabulary()

void opennn::WordPieceTokenizer::load_vocabulary

(

const filesystem::path &

vocabulary_file

)

set_lower_case()

void opennn::WordPieceTokenizer::set_lower_case ( bool value )

inlinenoexcept

set_vocabulary()

void opennn::WordPieceTokenizer::set_vocabulary ( const vector< string > & )

overridevirtual

Reimplemented from opennn::TokenizerOperator.

to_JSON()

void opennn::WordPieceTokenizer::to_JSON ( JsonWriter & ) const

overridevirtual

Reimplemented from opennn::TokenizerOperator.

tokenize()

vector< string > opennn::WordPieceTokenizer::tokenize ( string_view text ) const

overridevirtual

Implements opennn::TokenizerOperator.