#include <opennn/dataset/text_dataset.h>

Inheritance diagram for opennn::TextDataset:

Classes
struct	Column

struct	Options

struct	Record

Public Types
enum class	Task { Classification , SequenceToSequence , NextToken }

enum class	InputLayout { Tokens , TokensAndMask }

Public Types inherited from opennn::Dataset
enum class	Codification { UTF8 , SHIFT_JIS }

enum class	Separator { Space , Tab , Comma , Semicolon }

enum class	StorageMode { Matrix , BinaryFile , GPUPersistantData }

Public Member Functions
	TextDataset ()

	TextDataset (Options)

const Options &	get_options () const noexcept

void	read_txt (const filesystem::path &)

void	set_tokenizer (unique_ptr< TokenizerOperator >, VariableRole=VariableRole::Input)

const TokenizerOperator *	get_tokenizer (VariableRole=VariableRole::Input) const noexcept

const vector< string > &	get_vocabulary (VariableRole=VariableRole::Input) const noexcept

Index	get_vocabulary_size (VariableRole role=VariableRole::Input) const noexcept

Index	get_sequence_length (VariableRole role=VariableRole::Input) const

const vector< Column > &	get_columns () const noexcept

void	set_columns (vector< Column >)

vector< string >	get_text_columns () const

vector< string >	get_text_column_markers () const

vector< Index >	encode_text (span< const string > texts) const

void	for_each_record (const function< bool(Index, const Record &)> &) const

vector< Record >	read_records () const

optional< Record >	split_line (string_view line, bool with_id=true) const

const TokenizerOperator *	get_training_tokenizer () const override

const TokenizerOperator *	get_training_tokenizer (VariableRole role) const override

bool	supports_bf16_inputs () const override

VectorI	calculate_target_distribution () const override

void	set_storage_mode (StorageMode) override

void	to_JSON (JsonWriter &) const override

void	from_JSON (const JsonDocument &) override

void	fill_inputs (const vector< Index > &, const vector< Index > &, float *, FillMode, ColumnContiguity=ColumnContiguity::Unknown) const override

void	fill_targets (const vector< Index > &, const vector< Index > &, float *, FillMode, ColumnContiguity=ColumnContiguity::Unknown) const override

void	fill_decoder (const vector< Index > &, const vector< Index > &, float *, FillMode, ColumnContiguity=ColumnContiguity::Unknown) const override

virtual void	set_storage_mode (const string &)

Public Member Functions inherited from opennn::Dataset
virtual	~Dataset ()=default

virtual Index	get_samples_number () const noexcept

Index	get_samples_number (SampleRole role_type) const

Index	get_samples_number (string_view role) const

Index	get_used_samples_number () const

vector< Index >	get_sample_indices (SampleRole) const

vector< Index >	get_sample_indices (string_view role) const

vector< Index >	get_used_sample_indices () const

const vector< SampleRole > &	get_sample_roles () const noexcept

void	set_fold_split (const vector< Index > &training, const vector< Index > &validation)

void	clear_fold_split () noexcept

Index	get_variables_number () const noexcept

Index	get_variables_number (VariableRole) const

Index	get_variables_number (string_view role) const

Index	get_used_variables_number () const

const vector< Variable > &	get_variables () const noexcept

vector< Variable >	get_variables (VariableRole) const

vector< Variable >	get_variables (string_view role) const

Index	get_variable_index (const string &) const

Index	get_variable_index (const Index) const

vector< Index >	get_variable_indices (VariableRole) const

vector< Index >	get_variable_indices (string_view role) const

vector< Index >	get_used_variables_indices () const

vector< string >	get_variable_names () const

vector< string >	get_variable_names (VariableRole) const

vector< string >	get_variable_names (string_view role) const

VariableType	get_variable_type (const Index index) const

Index	get_features_number () const

Index	get_features_number (VariableRole) const

Index	get_features_number (string_view role) const

vector< string >	get_feature_names () const

vector< string >	get_feature_names (VariableRole) const

vector< string >	get_feature_names (string_view role) const

virtual vector< Variable >	get_model_input_variables () const

virtual bool	sample_order_matters () const noexcept

virtual MatrixR	calculate_input_target_correlation_values () const

virtual FeatureScaling	calculate_used_feature_scaling (VariableRole) const

vector< vector< Index > >	get_feature_indices () const

vector< Index >	get_feature_indices (const Index) const

vector< Index >	get_feature_indices (VariableRole) const

vector< Index >	get_feature_indices (string_view role) const

vector< Index >	get_used_feature_indices () const

Shape	get_shape (VariableRole) const

Shape	get_shape (string_view role) const

void	get_batches (const vector< Index > &, Index, Shuffle, vector< vector< Index > > &, optional< unsigned > shuffle_seed=nullopt) const

const vector< vector< string > > &	get_data_file_preview () const noexcept

const filesystem::path &	get_data_path () const noexcept

void	set_cache_directory (const filesystem::path &new_cache_directory)

StorageMode	get_storage_mode () const noexcept

string	get_storage_mode_string () const

string	get_separator_string () const

string	get_separator_name () const

string	get_codification_string () const

bool	get_display () const noexcept

bool	is_empty () const noexcept

Shape	get_input_shape () const noexcept

Shape	get_target_shape () const noexcept

void	record_memory (const string &stage) const

const MatrixR &	get_data () const noexcept

void	set_data (const MatrixR &)

void	set_data (MatrixR &&)

void	set_data_constant (float new_value)

virtual void	enable_device_residency ()

void	disable_device_residency ()

bool	is_device_resident () const noexcept

bool	requests_device_residency () const noexcept

bool	uses_device_residency () const noexcept

const float *	get_device_data () const

Index	get_device_data_columns () const noexcept

virtual FeatureScaling	prepare_training_scaling (VariableRole, const FeatureScaling &, Index)

virtual void	clear_training_scaling () noexcept

void	set_sample_roles (SampleRole)

void	set_sample_roles (string_view role)

void	set_sample_role (Index, SampleRole)

void	set_sample_role (Index index, string_view role)

void	set_sample_roles (const vector< string > &)

void	set_sample_roles (const vector< Index > &, SampleRole)

void	set_sample_roles (const vector< Index > &indices, string_view role)

void	set_default_variable_names ()

void	set_default_variable_roles ()

void	set_variable_roles (const vector< string > &)

void	set_variable_indices (const vector< Index > &, const vector< Index > &)

void	set_input_variables_unused ()

void	set_variable_role (Index, VariableRole)

void	set_variable_role (Index index, string_view role)

void	set_variable_role (const string &, VariableRole)

void	set_variable_role (const string &name, string_view role)

void	set_variable_names (const vector< string > &)

void	set_variables_number (const Index new_size)

void	set_variable_roles (VariableRole)

void	set_variable_roles (string_view role)

void	set_shape (VariableRole, const Shape &)

void	set_shape (string_view role, const Shape &shape)

virtual void	resize_input_shape (Index input_features_count)

virtual void	set_data_path (const filesystem::path &new_data_path)

void	set_has_header (bool new_has_header)

void	set_has_ids (bool new_has_ids)

bool	get_header_line () const

bool	get_has_sample_ids () const

const vector< string > &	get_sample_ids () const

VectorI	filter_data (const VectorR &, const VectorR &)

bool	has_categorical_variables () const

void	set_separator (const Separator &new_separator)

void	set_separator_string (const string &)

void	set_separator_name (const string &)

void	set_codification (const Codification &new_codification)

void	set_codification (const string &)

void	set_display (bool new_display)

bool	has_validation () const

void	split_samples (const float training_ratio=0.6f, float validation_ratio=0.2f, float testing_ratio=0.2f, bool shuffle=true)

void	split_samples_sequential (const float training_ratio=0.6f, float validation_ratio=0.2f, float testing_ratio=0.2f)

void	split_samples_random (const float training_ratio=0.6f, float validation_ratio=0.2f, float testing_ratio=0.2f)

virtual bool	has_nan () const

virtual void	scrub_missing_values ()

pair< Index, Index >	count_binary_targets (const string &sample_role) const

void	save (const filesystem::path &) const

void	load (const filesystem::path &)

FeatureSelection	get_feature_selection () const

virtual void	fill_batch (Batch &, const vector< Index > &sample_indices, const FeatureSelection &, FillMode) const

bool	can_device_gather (const Batch &, const FeatureSelection &) const

Static Public Member Functions
static vector< Column >	columns_from_JSON (const Json *)

Additional Inherited Members
Protected Member Functions inherited from opennn::Dataset
	Dataset ()=default

void	fill_batch_host (Batch &, const vector< Index > &sample_indices, const FeatureSelection &, FillMode) const

DeviceGather &	start_device_gather (Batch &, const vector< Index > &sample_indices, const FeatureSelection &) const

void	read_data_file_preview (const vector< string_view > &, char, bool has_quotes=false)

void	check_separators (string_view) const

void	samples_from_JSON (const Json *)

virtual void	resize_data_from_JSON (Index)

virtual void	on_used_samples_changed ()

void	require_in_memory_data (string_view what) const

void	upload_device_matrix (const MatrixR &)

void	upload_device_samples ()

void	invalidate_data ()

const vector< SampleRole > &	active_sample_roles () const noexcept

void	variables_to_JSON (JsonWriter &) const

void	samples_to_JSON (JsonWriter &) const

void	preview_data_to_JSON (JsonWriter &) const

void	variables_from_JSON (const Json *)

void	preview_data_from_JSON (const Json *)

virtual void	missing_values_from_JSON (const Json *)

void	write_json_header (JsonWriter &, initializer_list< pair< const char *, Json > >) const

void	write_json_footer (JsonWriter &) const

void	read_json_blocks (const Json *)

Protected Attributes inherited from opennn::Dataset
StorageMode	storage_mode = StorageMode::Matrix

MatrixR	data

Buffer	data_device {Device::CUDA}

Index	device_data_columns = 0

Shape	input_shape

Shape	target_shape

Shape	decoder_shape

vector< SampleRole >	sample_roles

vector< string >	sample_ids

optional< vector< SampleRole > >	fold_split_roles

vector< Variable >	variables

filesystem::path	data_path

filesystem::path	cache_directory

Separator	separator = Separator::Comma

bool	has_header = false

bool	has_sample_ids = false

Codification	codification = Codification::UTF8

vector< vector< string > >	data_file_preview

bool	display = true

Member Enumeration Documentation

◆ InputLayout

enum class opennn::TextDataset::InputLayout

strong

Enumerator
Tokens
TokensAndMask

◆ Task

enum class opennn::TextDataset::Task

strong

Enumerator
Classification
SequenceToSequence
NextToken

Constructor & Destructor Documentation

◆ TextDataset() [1/2]

opennn::TextDataset::TextDataset ( )

◆ TextDataset() [2/2]

opennn::TextDataset::TextDataset ( Options )

explicit

Member Function Documentation

◆ calculate_target_distribution()

VectorI opennn::TextDataset::calculate_target_distribution ( ) const

overridevirtual

Reimplemented from opennn::Dataset.

◆ columns_from_JSON()

static vector< Column > opennn::TextDataset::columns_from_JSON ( const Json * )

static

◆ encode_text()

vector< Index > opennn::TextDataset::encode_text ( span< const string > texts ) const

◆ fill_decoder()

void opennn::TextDataset::fill_decoder	(	const vector< Index > &	,
		const vector< Index > &	,
		float *	,
		FillMode	,
		ColumnContiguity	= ColumnContiguity::Unknown ) const

overridevirtual

Reimplemented from opennn::Dataset.

◆ fill_inputs()

void opennn::TextDataset::fill_inputs	(	const vector< Index > &	,
		const vector< Index > &	,
		float *	,
		FillMode	,
		ColumnContiguity	= ColumnContiguity::Unknown ) const

overridevirtual

Reimplemented from opennn::Dataset.

◆ fill_targets()

void opennn::TextDataset::fill_targets	(	const vector< Index > &	,
		const vector< Index > &	,
		float *	,
		FillMode	,
		ColumnContiguity	= ColumnContiguity::Unknown ) const

overridevirtual

Reimplemented from opennn::Dataset.

◆ for_each_record()

void opennn::TextDataset::for_each_record ( const function< bool(Index, const Record &)> & ) const

◆ from_JSON()

void opennn::TextDataset::from_JSON ( const JsonDocument & )

overridevirtual

Implements opennn::Dataset.

◆ get_columns()

const vector< Column > & opennn::TextDataset::get_columns ( ) const

inlinenoexcept

◆ get_options()

const Options & opennn::TextDataset::get_options ( ) const

inlinenoexcept

◆ get_sequence_length()

Index opennn::TextDataset::get_sequence_length ( VariableRole role = VariableRole::Input ) const

inline

◆ get_text_column_markers()

vector< string > opennn::TextDataset::get_text_column_markers ( ) const

◆ get_text_columns()

vector< string > opennn::TextDataset::get_text_columns ( ) const

◆ get_tokenizer()

const TokenizerOperator * opennn::TextDataset::get_tokenizer ( VariableRole = VariableRole::Input ) const

noexcept

◆ get_training_tokenizer() [1/2]

const TokenizerOperator * opennn::TextDataset::get_training_tokenizer ( ) const

inlineoverridevirtual

Reimplemented from opennn::Dataset.

◆ get_training_tokenizer() [2/2]

const TokenizerOperator * opennn::TextDataset::get_training_tokenizer ( VariableRole role ) const

inlineoverridevirtual

Reimplemented from opennn::Dataset.

◆ get_vocabulary()

const vector< string > & opennn::TextDataset::get_vocabulary ( VariableRole = VariableRole::Input ) const

noexcept

◆ get_vocabulary_size()

Index opennn::TextDataset::get_vocabulary_size ( VariableRole role = VariableRole::Input ) const

inlinenoexcept

◆ read_records()

vector< Record > opennn::TextDataset::read_records ( ) const

◆ read_txt()

void opennn::TextDataset::read_txt ( const filesystem::path & )

◆ set_columns()

void opennn::TextDataset::set_columns ( vector< Column > )

◆ set_storage_mode() [1/2]

virtual void opennn::Dataset::set_storage_mode ( const string & )

virtual

Reimplemented from opennn::Dataset.

◆ set_storage_mode() [2/2]

void opennn::TextDataset::set_storage_mode ( StorageMode )

overridevirtual

Reimplemented from opennn::Dataset.

◆ set_tokenizer()

void opennn::TextDataset::set_tokenizer	(	unique_ptr< TokenizerOperator >	,
		VariableRole	= VariableRole::Input )

◆ split_line()

optional< Record > opennn::TextDataset::split_line	(	string_view	line,
		bool	with_id = true ) const

◆ supports_bf16_inputs()

bool opennn::TextDataset::supports_bf16_inputs ( ) const

inlineoverridevirtual

Reimplemented from opennn::Dataset.

◆ to_JSON()

void opennn::TextDataset::to_JSON ( JsonWriter & ) const

overridevirtual

Reimplemented from opennn::Dataset.

Classes

Public Types

Public Member Functions

Static Public Member Functions

Additional Inherited Members

Member Enumeration Documentation

◆ InputLayout

◆ Task

Constructor & Destructor Documentation

◆ TextDataset() [1/2]

◆ TextDataset() [2/2]

Member Function Documentation

◆ calculate_target_distribution()

◆ columns_from_JSON()

◆ encode_text()

◆ fill_decoder()

◆ fill_inputs()

◆ fill_targets()

◆ for_each_record()

◆ from_JSON()

◆ get_columns()

◆ get_options()

◆ get_sequence_length()

◆ get_text_column_markers()

◆ get_text_columns()

◆ get_tokenizer()

◆ get_training_tokenizer() [1/2]

◆ get_training_tokenizer() [2/2]

◆ get_vocabulary()

◆ get_vocabulary_size()

◆ read_records()

◆ read_txt()

◆ set_columns()

◆ set_storage_mode() [1/2]

◆ set_storage_mode() [2/2]

◆ set_tokenizer()

◆ split_line()

◆ supports_bf16_inputs()

◆ to_JSON()