#include <opennn/dataset/dataset.h>

Inheritance diagram for opennn::Dataset:

Public Types
enum class	Codification { UTF8 , SHIFT_JIS }

enum class	Separator { Space , Tab , Comma , Semicolon }

enum class	StorageMode { Matrix , BinaryFile , GPUPersistantData }

Public Member Functions
virtual	~Dataset ()=default

virtual Index	get_samples_number () const noexcept

Index	get_samples_number (SampleRole role_type) const

Index	get_samples_number (string_view role) const

Index	get_used_samples_number () const

vector< Index >	get_sample_indices (SampleRole) const

vector< Index >	get_sample_indices (string_view role) const

vector< Index >	get_used_sample_indices () const

const vector< SampleRole > &	get_sample_roles () const noexcept

void	set_fold_split (const vector< Index > &training, const vector< Index > &validation)

void	clear_fold_split () noexcept

Index	get_variables_number () const noexcept

Index	get_variables_number (VariableRole) const

Index	get_variables_number (string_view role) const

Index	get_used_variables_number () const

const vector< Variable > &	get_variables () const noexcept

vector< Variable >	get_variables (VariableRole) const

vector< Variable >	get_variables (string_view role) const

Index	get_variable_index (const string &) const

Index	get_variable_index (const Index) const

vector< Index >	get_variable_indices (VariableRole) const

vector< Index >	get_variable_indices (string_view role) const

vector< Index >	get_used_variables_indices () const

vector< string >	get_variable_names () const

vector< string >	get_variable_names (VariableRole) const

vector< string >	get_variable_names (string_view role) const

VariableType	get_variable_type (const Index index) const

Index	get_features_number () const

Index	get_features_number (VariableRole) const

Index	get_features_number (string_view role) const

vector< string >	get_feature_names () const

vector< string >	get_feature_names (VariableRole) const

vector< string >	get_feature_names (string_view role) const

virtual vector< Variable >	get_model_input_variables () const

virtual bool	sample_order_matters () const noexcept

virtual MatrixR	calculate_input_target_correlation_values () const

virtual FeatureScaling	calculate_used_feature_scaling (VariableRole) const

vector< vector< Index > >	get_feature_indices () const

vector< Index >	get_feature_indices (const Index) const

vector< Index >	get_feature_indices (VariableRole) const

vector< Index >	get_feature_indices (string_view role) const

vector< Index >	get_used_feature_indices () const

Shape	get_shape (VariableRole) const

Shape	get_shape (string_view role) const

void	get_batches (const vector< Index > &, Index, Shuffle, vector< vector< Index > > &, optional< unsigned > shuffle_seed=nullopt) const

const vector< vector< string > > &	get_data_file_preview () const noexcept

const filesystem::path &	get_data_path () const noexcept

void	set_cache_directory (const filesystem::path &new_cache_directory)

StorageMode	get_storage_mode () const noexcept

string	get_storage_mode_string () const

string	get_separator_string () const

string	get_separator_name () const

string	get_codification_string () const

bool	get_display () const noexcept

bool	is_empty () const noexcept

Shape	get_input_shape () const noexcept

Shape	get_target_shape () const noexcept

void	record_memory (const string &stage) const

const MatrixR &	get_data () const noexcept

void	set_data (const MatrixR &)

void	set_data (MatrixR &&)

void	set_data_constant (float new_value)

virtual void	enable_device_residency ()

void	disable_device_residency ()

bool	is_device_resident () const noexcept

bool	requests_device_residency () const noexcept

bool	uses_device_residency () const noexcept

const float *	get_device_data () const

Index	get_device_data_columns () const noexcept

virtual FeatureScaling	prepare_training_scaling (VariableRole, const FeatureScaling &, Index)

virtual void	clear_training_scaling () noexcept

virtual const TokenizerOperator *	get_training_tokenizer () const

virtual const TokenizerOperator *	get_training_tokenizer (VariableRole role) const

void	set_sample_roles (SampleRole)

void	set_sample_roles (string_view role)

void	set_sample_role (Index, SampleRole)

void	set_sample_role (Index index, string_view role)

void	set_sample_roles (const vector< string > &)

void	set_sample_roles (const vector< Index > &, SampleRole)

void	set_sample_roles (const vector< Index > &indices, string_view role)

void	set_default_variable_names ()

void	set_default_variable_roles ()

void	set_variable_roles (const vector< string > &)

void	set_variable_indices (const vector< Index > &, const vector< Index > &)

void	set_input_variables_unused ()

void	set_variable_role (Index, VariableRole)

void	set_variable_role (Index index, string_view role)

void	set_variable_role (const string &, VariableRole)

void	set_variable_role (const string &name, string_view role)

void	set_variable_names (const vector< string > &)

void	set_variables_number (const Index new_size)

void	set_variable_roles (VariableRole)

void	set_variable_roles (string_view role)

void	set_shape (VariableRole, const Shape &)

void	set_shape (string_view role, const Shape &shape)

virtual void	resize_input_shape (Index input_features_count)

virtual void	set_data_path (const filesystem::path &new_data_path)

virtual void	set_storage_mode (StorageMode)

virtual void	set_storage_mode (const string &)

void	set_has_header (bool new_has_header)

void	set_has_ids (bool new_has_ids)

bool	get_header_line () const

bool	get_has_sample_ids () const

const vector< string > &	get_sample_ids () const

VectorI	filter_data (const VectorR &, const VectorR &)

bool	has_categorical_variables () const

void	set_separator (const Separator &new_separator)

void	set_separator_string (const string &)

void	set_separator_name (const string &)

void	set_codification (const Codification &new_codification)

void	set_codification (const string &)

void	set_display (bool new_display)

bool	has_validation () const

void	split_samples (const float training_ratio=0.6f, float validation_ratio=0.2f, float testing_ratio=0.2f, bool shuffle=true)

void	split_samples_sequential (const float training_ratio=0.6f, float validation_ratio=0.2f, float testing_ratio=0.2f)

void	split_samples_random (const float training_ratio=0.6f, float validation_ratio=0.2f, float testing_ratio=0.2f)

virtual void	from_JSON (const JsonDocument &)=0

virtual void	to_JSON (JsonWriter &) const

virtual bool	has_nan () const

virtual void	scrub_missing_values ()

virtual VectorI	calculate_target_distribution () const

pair< Index, Index >	count_binary_targets (const string &sample_role) const

void	save (const filesystem::path &) const

void	load (const filesystem::path &)

virtual void	fill_inputs (const vector< Index > &, const vector< Index > &, float *, FillMode, ColumnContiguity column_contiguity=ColumnContiguity::Unknown) const

virtual void	fill_decoder (const vector< Index > &, const vector< Index > &, float *, FillMode, ColumnContiguity column_contiguity=ColumnContiguity::Unknown) const

virtual void	fill_targets (const vector< Index > &, const vector< Index > &, float *, FillMode, ColumnContiguity column_contiguity=ColumnContiguity::Unknown) const

virtual bool	supports_bf16_inputs () const

FeatureSelection	get_feature_selection () const

virtual void	fill_batch (Batch &, const vector< Index > &sample_indices, const FeatureSelection &, FillMode) const

bool	can_device_gather (const Batch &, const FeatureSelection &) const

Protected Member Functions
	Dataset ()=default

void	fill_batch_host (Batch &, const vector< Index > &sample_indices, const FeatureSelection &, FillMode) const

DeviceGather &	start_device_gather (Batch &, const vector< Index > &sample_indices, const FeatureSelection &) const

void	read_data_file_preview (const vector< string_view > &, char, bool has_quotes=false)

void	check_separators (string_view) const

void	samples_from_JSON (const Json *)

virtual void	resize_data_from_JSON (Index)

virtual void	on_used_samples_changed ()

void	require_in_memory_data (string_view what) const

void	upload_device_matrix (const MatrixR &)

void	upload_device_samples ()

void	invalidate_data ()

const vector< SampleRole > &	active_sample_roles () const noexcept

void	variables_to_JSON (JsonWriter &) const

void	samples_to_JSON (JsonWriter &) const

void	preview_data_to_JSON (JsonWriter &) const

void	variables_from_JSON (const Json *)

void	preview_data_from_JSON (const Json *)

virtual void	missing_values_from_JSON (const Json *)

void	write_json_header (JsonWriter &, initializer_list< pair< const char *, Json > >) const

void	write_json_footer (JsonWriter &) const

void	read_json_blocks (const Json *)

Protected Attributes
StorageMode	storage_mode = StorageMode::Matrix

MatrixR	data

Buffer	data_device {Device::CUDA}

Index	device_data_columns = 0

Shape	input_shape

Shape	target_shape

Shape	decoder_shape

vector< SampleRole >	sample_roles

vector< string >	sample_ids

optional< vector< SampleRole > >	fold_split_roles

vector< Variable >	variables

filesystem::path	data_path

filesystem::path	cache_directory

Separator	separator = Separator::Comma

bool	has_header = false

bool	has_sample_ids = false

Codification	codification = Codification::UTF8

vector< vector< string > >	data_file_preview

bool	display = true

Member Enumeration Documentation

◆ Codification

enum class opennn::Dataset::Codification

strong

Enumerator
UTF8
SHIFT_JIS

◆ Separator

enum class opennn::Dataset::Separator

strong

Enumerator
Space
Tab
Comma
Semicolon

◆ StorageMode

enum class opennn::Dataset::StorageMode

strong

Enumerator
Matrix
BinaryFile
GPUPersistantData

Constructor & Destructor Documentation

◆ ~Dataset()

virtual opennn::Dataset::~Dataset ( )

virtualdefault

◆ Dataset()

opennn::Dataset::Dataset ( )

protecteddefault

Member Function Documentation

◆ active_sample_roles()

const vector< SampleRole > & opennn::Dataset::active_sample_roles ( ) const

inlineprotectednoexcept

◆ calculate_input_target_correlation_values()

virtual MatrixR opennn::Dataset::calculate_input_target_correlation_values ( ) const

inlinevirtual

Reimplemented in opennn::TabularDataset.

◆ calculate_target_distribution()

virtual VectorI opennn::Dataset::calculate_target_distribution ( ) const

inlinevirtual

Reimplemented in opennn::ImageDataset, opennn::TabularDataset, and opennn::TextDataset.

◆ calculate_used_feature_scaling()

virtual FeatureScaling opennn::Dataset::calculate_used_feature_scaling ( VariableRole ) const

virtual

Reimplemented in opennn::TabularDataset.

◆ can_device_gather()

bool opennn::Dataset::can_device_gather	(	const Batch &	,
		const FeatureSelection &	) const

◆ check_separators()

void opennn::Dataset::check_separators ( string_view ) const

protected

◆ clear_fold_split()

void opennn::Dataset::clear_fold_split ( )

inlinenoexcept

◆ clear_training_scaling()

virtual void opennn::Dataset::clear_training_scaling ( )

inlinevirtualnoexcept

Reimplemented in opennn::ImageDataset, and opennn::TabularDataset.

◆ count_binary_targets()

pair< Index, Index > opennn::Dataset::count_binary_targets ( const string & sample_role ) const

◆ disable_device_residency()

void opennn::Dataset::disable_device_residency ( )

inline

◆ enable_device_residency()

virtual void opennn::Dataset::enable_device_residency ( )

virtual

Reimplemented in opennn::ImageDataset, opennn::TabularDataset, and opennn::YoloDataset.

◆ fill_batch()

virtual void opennn::Dataset::fill_batch	(	Batch &	,
		const vector< Index > &	sample_indices,
		const FeatureSelection &	,
		FillMode	) const

virtual

Reimplemented in opennn::TabularDataset.

◆ fill_batch_host()

void opennn::Dataset::fill_batch_host	(	Batch &	,
		const vector< Index > &	sample_indices,
		const FeatureSelection &	,
		FillMode	) const

protected

◆ fill_decoder()

virtual void opennn::Dataset::fill_decoder	(	const vector< Index > &	,
		const vector< Index > &	,
		float *	,
		FillMode	,
		ColumnContiguity	column_contiguity = ColumnContiguity::Unknown ) const

virtual

Reimplemented in opennn::TabularDataset, and opennn::TextDataset.

◆ fill_inputs()

virtual void opennn::Dataset::fill_inputs	(	const vector< Index > &	,
		const vector< Index > &	,
		float *	,
		FillMode	,
		ColumnContiguity	column_contiguity = ColumnContiguity::Unknown ) const

virtual

Reimplemented in opennn::ImageDataset, opennn::TabularDataset, opennn::TextDataset, and opennn::YoloDataset.

◆ fill_targets()

virtual void opennn::Dataset::fill_targets	(	const vector< Index > &	,
		const vector< Index > &	,
		float *	,
		FillMode	,
		ColumnContiguity	column_contiguity = ColumnContiguity::Unknown ) const

virtual

Reimplemented in opennn::ImageDataset, opennn::TabularDataset, opennn::TextDataset, and opennn::YoloDataset.

◆ filter_data()

VectorI opennn::Dataset::filter_data	(	const VectorR &	,
		const VectorR &	)

◆ from_JSON()

virtual void opennn::Dataset::from_JSON ( const JsonDocument & )

pure virtual

Implemented in opennn::ImageDataset, opennn::TabularDataset, opennn::TextDataset, and opennn::YoloDataset.

◆ get_batches()

void opennn::Dataset::get_batches	(	const vector< Index > &	,
		Index	,
		Shuffle	,
		vector< vector< Index > > &	,
		optional< unsigned >	shuffle_seed = nullopt ) const

◆ get_codification_string()

string opennn::Dataset::get_codification_string ( ) const

◆ get_data()

const MatrixR & opennn::Dataset::get_data ( ) const

inlinenoexcept

◆ get_data_file_preview()

const vector< vector< string > > & opennn::Dataset::get_data_file_preview ( ) const

inlinenoexcept

◆ get_data_path()

const filesystem::path & opennn::Dataset::get_data_path ( ) const

inlinenoexcept

◆ get_device_data()

const float * opennn::Dataset::get_device_data ( ) const

inline

◆ get_device_data_columns()

Index opennn::Dataset::get_device_data_columns ( ) const

inlinenoexcept

◆ get_display()

bool opennn::Dataset::get_display ( ) const

inlinenoexcept

◆ get_feature_indices() [1/4]

vector< vector< Index > > opennn::Dataset::get_feature_indices ( ) const

◆ get_feature_indices() [2/4]

vector< Index > opennn::Dataset::get_feature_indices ( const Index ) const

◆ get_feature_indices() [3/4]

vector< Index > opennn::Dataset::get_feature_indices ( string_view role ) const

inline

◆ get_feature_indices() [4/4]

vector< Index > opennn::Dataset::get_feature_indices ( VariableRole ) const

◆ get_feature_names() [1/3]

vector< string > opennn::Dataset::get_feature_names ( ) const

◆ get_feature_names() [2/3]

vector< string > opennn::Dataset::get_feature_names ( string_view role ) const

inline

◆ get_feature_names() [3/3]

vector< string > opennn::Dataset::get_feature_names ( VariableRole ) const

◆ get_feature_selection()

FeatureSelection opennn::Dataset::get_feature_selection ( ) const

◆ get_features_number() [1/3]

Index opennn::Dataset::get_features_number ( ) const

◆ get_features_number() [2/3]

Index opennn::Dataset::get_features_number ( string_view role ) const

inline

◆ get_features_number() [3/3]

Index opennn::Dataset::get_features_number ( VariableRole ) const

◆ get_has_sample_ids()

bool opennn::Dataset::get_has_sample_ids ( ) const

inline

◆ get_header_line()

bool opennn::Dataset::get_header_line ( ) const

inline

◆ get_input_shape()

Shape opennn::Dataset::get_input_shape ( ) const

inlinenoexcept

◆ get_model_input_variables()

virtual vector< Variable > opennn::Dataset::get_model_input_variables ( ) const

inlinevirtual

Reimplemented in opennn::TabularDataset.

◆ get_sample_ids()

const vector< string > & opennn::Dataset::get_sample_ids ( ) const

inline

◆ get_sample_indices() [1/2]

vector< Index > opennn::Dataset::get_sample_indices ( SampleRole ) const

◆ get_sample_indices() [2/2]

vector< Index > opennn::Dataset::get_sample_indices ( string_view role ) const

inline

◆ get_sample_roles()

const vector< SampleRole > & opennn::Dataset::get_sample_roles ( ) const

inlinenoexcept

◆ get_samples_number() [1/3]

virtual Index opennn::Dataset::get_samples_number ( ) const

inlinevirtualnoexcept

Reimplemented in opennn::TabularDataset, and opennn::YoloDataset.

◆ get_samples_number() [2/3]

Index opennn::Dataset::get_samples_number ( SampleRole role_type ) const

inline

◆ get_samples_number() [3/3]

Index opennn::Dataset::get_samples_number ( string_view role ) const

inline

◆ get_separator_name()

string opennn::Dataset::get_separator_name ( ) const

◆ get_separator_string()

string opennn::Dataset::get_separator_string ( ) const

◆ get_shape() [1/2]

Shape opennn::Dataset::get_shape ( string_view role ) const

inline

◆ get_shape() [2/2]

Shape opennn::Dataset::get_shape ( VariableRole ) const

◆ get_storage_mode()

StorageMode opennn::Dataset::get_storage_mode ( ) const

inlinenoexcept

◆ get_storage_mode_string()

string opennn::Dataset::get_storage_mode_string ( ) const

◆ get_target_shape()

Shape opennn::Dataset::get_target_shape ( ) const

inlinenoexcept

◆ get_training_tokenizer() [1/2]

virtual const TokenizerOperator * opennn::Dataset::get_training_tokenizer ( ) const

inlinevirtual

Reimplemented in opennn::TextDataset.

◆ get_training_tokenizer() [2/2]

virtual const TokenizerOperator * opennn::Dataset::get_training_tokenizer ( VariableRole role ) const

inlinevirtual

Reimplemented in opennn::TextDataset.

◆ get_used_feature_indices()

vector< Index > opennn::Dataset::get_used_feature_indices ( ) const

◆ get_used_sample_indices()

vector< Index > opennn::Dataset::get_used_sample_indices ( ) const

◆ get_used_samples_number()

Index opennn::Dataset::get_used_samples_number ( ) const

inline

◆ get_used_variables_indices()

vector< Index > opennn::Dataset::get_used_variables_indices ( ) const

◆ get_used_variables_number()

Index opennn::Dataset::get_used_variables_number ( ) const

◆ get_variable_index() [1/2]

Index opennn::Dataset::get_variable_index ( const Index ) const

◆ get_variable_index() [2/2]

Index opennn::Dataset::get_variable_index ( const string & ) const

◆ get_variable_indices() [1/2]

vector< Index > opennn::Dataset::get_variable_indices ( string_view role ) const

inline

◆ get_variable_indices() [2/2]

vector< Index > opennn::Dataset::get_variable_indices ( VariableRole ) const

◆ get_variable_names() [1/3]

vector< string > opennn::Dataset::get_variable_names ( ) const

◆ get_variable_names() [2/3]

vector< string > opennn::Dataset::get_variable_names ( string_view role ) const

inline

◆ get_variable_names() [3/3]

vector< string > opennn::Dataset::get_variable_names ( VariableRole ) const

◆ get_variable_type()

VariableType opennn::Dataset::get_variable_type ( const Index index ) const

inline

◆ get_variables() [1/3]

const vector< Variable > & opennn::Dataset::get_variables ( ) const

inlinenoexcept

◆ get_variables() [2/3]

vector< Variable > opennn::Dataset::get_variables ( string_view role ) const

inline

◆ get_variables() [3/3]

vector< Variable > opennn::Dataset::get_variables ( VariableRole ) const

◆ get_variables_number() [1/3]

Index opennn::Dataset::get_variables_number ( ) const

inlinenoexcept

◆ get_variables_number() [2/3]

Index opennn::Dataset::get_variables_number ( string_view role ) const

inline

◆ get_variables_number() [3/3]

Index opennn::Dataset::get_variables_number ( VariableRole ) const

◆ has_categorical_variables()

bool opennn::Dataset::has_categorical_variables ( ) const

inline

◆ has_nan()

virtual bool opennn::Dataset::has_nan ( ) const

inlinevirtual

Reimplemented in opennn::TabularDataset.

◆ has_validation()

bool opennn::Dataset::has_validation ( ) const

inline

◆ invalidate_data()

void opennn::Dataset::invalidate_data ( )

inlineprotected

◆ is_device_resident()

bool opennn::Dataset::is_device_resident ( ) const

inlinenoexcept

◆ is_empty()

bool opennn::Dataset::is_empty ( ) const

inlinenoexcept

◆ load()

void opennn::Dataset::load ( const filesystem::path & )

◆ missing_values_from_JSON()

virtual void opennn::Dataset::missing_values_from_JSON ( const Json * )

inlineprotectedvirtual

Reimplemented in opennn::TabularDataset.

◆ on_used_samples_changed()

virtual void opennn::Dataset::on_used_samples_changed ( )

inlineprotectedvirtual

Reimplemented in opennn::TabularDataset.

◆ prepare_training_scaling()

virtual FeatureScaling opennn::Dataset::prepare_training_scaling	(	VariableRole	,
		const FeatureScaling &	,
		Index	)

virtual

Reimplemented in opennn::ImageDataset, and opennn::TabularDataset.

◆ preview_data_from_JSON()

void opennn::Dataset::preview_data_from_JSON ( const Json * )

protected

◆ preview_data_to_JSON()

void opennn::Dataset::preview_data_to_JSON ( JsonWriter & ) const

protected

◆ read_data_file_preview()

void opennn::Dataset::read_data_file_preview	(	const vector< string_view > &	,
		char	,
		bool	has_quotes = false )

protected

◆ read_json_blocks()

void opennn::Dataset::read_json_blocks ( const Json * )

protected

◆ record_memory()

void opennn::Dataset::record_memory ( const string & stage ) const

◆ requests_device_residency()

bool opennn::Dataset::requests_device_residency ( ) const

inlinenoexcept

◆ require_in_memory_data()

void opennn::Dataset::require_in_memory_data ( string_view what ) const

inlineprotected

◆ resize_data_from_JSON()

virtual void opennn::Dataset::resize_data_from_JSON ( Index )

inlineprotectedvirtual

Reimplemented in opennn::TabularDataset.

◆ resize_input_shape()

virtual void opennn::Dataset::resize_input_shape ( Index input_features_count )

inlinevirtual

Reimplemented in opennn::TabularDataset.

◆ sample_order_matters()

virtual bool opennn::Dataset::sample_order_matters ( ) const

inlinevirtualnoexcept

Reimplemented in opennn::TabularDataset.

◆ samples_from_JSON()

void opennn::Dataset::samples_from_JSON ( const Json * )

protected

◆ samples_to_JSON()

void opennn::Dataset::samples_to_JSON ( JsonWriter & ) const

protected

◆ save()

void opennn::Dataset::save ( const filesystem::path & ) const

◆ scrub_missing_values()

virtual void opennn::Dataset::scrub_missing_values ( )

inlinevirtual

Reimplemented in opennn::TabularDataset.

◆ set_cache_directory()

void opennn::Dataset::set_cache_directory ( const filesystem::path & new_cache_directory )

inline

◆ set_codification() [1/2]

void opennn::Dataset::set_codification ( const Codification & new_codification )

inline

◆ set_codification() [2/2]

void opennn::Dataset::set_codification ( const string & )

◆ set_data() [1/2]

void opennn::Dataset::set_data ( const MatrixR & )

◆ set_data() [2/2]

void opennn::Dataset::set_data ( MatrixR && )

◆ set_data_constant()

void opennn::Dataset::set_data_constant ( float new_value )

inline

◆ set_data_path()

virtual void opennn::Dataset::set_data_path ( const filesystem::path & new_data_path )

inlinevirtual

◆ set_default_variable_names()

void opennn::Dataset::set_default_variable_names ( )

◆ set_default_variable_roles()

void opennn::Dataset::set_default_variable_roles ( )

◆ set_display()

void opennn::Dataset::set_display ( bool new_display )

inline

◆ set_fold_split()

void opennn::Dataset::set_fold_split	(	const vector< Index > &	training,
		const vector< Index > &	validation )

◆ set_has_header()

void opennn::Dataset::set_has_header ( bool new_has_header )

inline

◆ set_has_ids()

void opennn::Dataset::set_has_ids ( bool new_has_ids )

inline

◆ set_input_variables_unused()

void opennn::Dataset::set_input_variables_unused ( )

◆ set_sample_role() [1/2]

void opennn::Dataset::set_sample_role	(	Index	index,
		string_view	role )

inline

◆ set_sample_role() [2/2]

void opennn::Dataset::set_sample_role	(	Index	,
		SampleRole	)

◆ set_sample_roles() [1/5]

void opennn::Dataset::set_sample_roles	(	const vector< Index > &	,
		SampleRole	)

◆ set_sample_roles() [2/5]

void opennn::Dataset::set_sample_roles	(	const vector< Index > &	indices,
		string_view	role )

inline

◆ set_sample_roles() [3/5]

void opennn::Dataset::set_sample_roles ( const vector< string > & )

◆ set_sample_roles() [4/5]

void opennn::Dataset::set_sample_roles ( SampleRole )

◆ set_sample_roles() [5/5]

void opennn::Dataset::set_sample_roles ( string_view role )

inline

◆ set_separator()

void opennn::Dataset::set_separator ( const Separator & new_separator )

inline

◆ set_separator_name()

void opennn::Dataset::set_separator_name ( const string & )

◆ set_separator_string()

void opennn::Dataset::set_separator_string ( const string & )

◆ set_shape() [1/2]

void opennn::Dataset::set_shape	(	string_view	role,
		const Shape &	shape )

inline

◆ set_shape() [2/2]

void opennn::Dataset::set_shape	(	VariableRole	,
		const Shape &	)

◆ set_storage_mode() [1/2]

virtual void opennn::Dataset::set_storage_mode ( const string & )

virtual

Reimplemented in opennn::TabularDataset, opennn::TextDataset, and opennn::YoloDataset.

◆ set_storage_mode() [2/2]

virtual void opennn::Dataset::set_storage_mode ( StorageMode )

virtual

Reimplemented in opennn::TabularDataset, opennn::TextDataset, and opennn::YoloDataset.

◆ set_variable_indices()

void opennn::Dataset::set_variable_indices	(	const vector< Index > &	,
		const vector< Index > &	)

◆ set_variable_names()

void opennn::Dataset::set_variable_names ( const vector< string > & )

◆ set_variable_role() [1/4]

void opennn::Dataset::set_variable_role	(	const string &	,
		VariableRole	)

◆ set_variable_role() [2/4]

void opennn::Dataset::set_variable_role	(	const string &	name,
		string_view	role )

inline

◆ set_variable_role() [3/4]

void opennn::Dataset::set_variable_role	(	Index	index,
		string_view	role )

inline

◆ set_variable_role() [4/4]

void opennn::Dataset::set_variable_role	(	Index	,
		VariableRole	)

◆ set_variable_roles() [1/3]

void opennn::Dataset::set_variable_roles ( const vector< string > & )

◆ set_variable_roles() [2/3]

void opennn::Dataset::set_variable_roles ( string_view role )

inline

◆ set_variable_roles() [3/3]

void opennn::Dataset::set_variable_roles ( VariableRole )

◆ set_variables_number()

void opennn::Dataset::set_variables_number ( const Index new_size )

inline

◆ split_samples()

void opennn::Dataset::split_samples	(	const float	training_ratio = 0.6f,
		float	validation_ratio = 0.2f,
		float	testing_ratio = 0.2f,
		bool	shuffle = true )

◆ split_samples_random()

void opennn::Dataset::split_samples_random	(	const float	training_ratio = 0.6f,
		float	validation_ratio = 0.2f,
		float	testing_ratio = 0.2f )

◆ split_samples_sequential()

void opennn::Dataset::split_samples_sequential	(	const float	training_ratio = 0.6f,
		float	validation_ratio = 0.2f,
		float	testing_ratio = 0.2f )

◆ start_device_gather()

DeviceGather & opennn::Dataset::start_device_gather	(	Batch &	,
		const vector< Index > &	sample_indices,
		const FeatureSelection &	) const

protected

◆ supports_bf16_inputs()

virtual bool opennn::Dataset::supports_bf16_inputs ( ) const

inlinevirtual

Reimplemented in opennn::TextDataset.

◆ to_JSON()

virtual void opennn::Dataset::to_JSON ( JsonWriter & ) const

inlinevirtual

Reimplemented in opennn::ImageDataset, opennn::TabularDataset, opennn::TextDataset, and opennn::YoloDataset.

◆ upload_device_matrix()

void opennn::Dataset::upload_device_matrix ( const MatrixR & )

protected

◆ upload_device_samples()

void opennn::Dataset::upload_device_samples ( )

protected

◆ uses_device_residency()

bool opennn::Dataset::uses_device_residency ( ) const

inlinenoexcept

◆ variables_from_JSON()

void opennn::Dataset::variables_from_JSON ( const Json * )

protected

◆ variables_to_JSON()

void opennn::Dataset::variables_to_JSON ( JsonWriter & ) const

protected

◆ write_json_footer()

void opennn::Dataset::write_json_footer ( JsonWriter & ) const

protected

◆ write_json_header()

void opennn::Dataset::write_json_header	(	JsonWriter &	,
		initializer_list< pair< const char *, Json > >	) const

protected

Member Data Documentation

◆ cache_directory

filesystem::path opennn::Dataset::cache_directory

protected

◆ codification

Codification opennn::Dataset::codification = Codification::UTF8

protected

◆ data

MatrixR opennn::Dataset::data

protected

◆ data_device

Buffer opennn::Dataset::data_device {Device::CUDA}

protected

◆ data_file_preview

vector<vector<string> > opennn::Dataset::data_file_preview

protected

◆ data_path

filesystem::path opennn::Dataset::data_path

protected

◆ decoder_shape

Shape opennn::Dataset::decoder_shape

protected

◆ device_data_columns

Index opennn::Dataset::device_data_columns = 0

protected

◆ display

bool opennn::Dataset::display = true

protected

◆ fold_split_roles

optional<vector<SampleRole> > opennn::Dataset::fold_split_roles

protected

◆ has_header

bool opennn::Dataset::has_header = false

protected

◆ has_sample_ids

bool opennn::Dataset::has_sample_ids = false

protected

◆ input_shape

Shape opennn::Dataset::input_shape

protected

◆ sample_ids

vector<string> opennn::Dataset::sample_ids

protected

◆ sample_roles

vector<SampleRole> opennn::Dataset::sample_roles

protected

◆ separator

Separator opennn::Dataset::separator = Separator::Comma

protected

◆ storage_mode

StorageMode opennn::Dataset::storage_mode = StorageMode::Matrix

protected

◆ target_shape

Shape opennn::Dataset::target_shape

protected

◆ variables

vector<Variable> opennn::Dataset::variables

protected

Public Types

Public Member Functions

Protected Member Functions

Protected Attributes

Member Enumeration Documentation

◆ Codification

◆ Separator

◆ StorageMode

Constructor & Destructor Documentation

◆ ~Dataset()

◆ Dataset()

Member Function Documentation

◆ active_sample_roles()

◆ calculate_input_target_correlation_values()

◆ calculate_target_distribution()

◆ calculate_used_feature_scaling()

◆ can_device_gather()

◆ check_separators()

◆ clear_fold_split()

◆ clear_training_scaling()

◆ count_binary_targets()

◆ disable_device_residency()

◆ enable_device_residency()

◆ fill_batch()

◆ fill_batch_host()

◆ fill_decoder()

◆ fill_inputs()

◆ fill_targets()

◆ filter_data()

◆ from_JSON()

◆ get_batches()

◆ get_codification_string()

◆ get_data()

◆ get_data_file_preview()

◆ get_data_path()

◆ get_device_data()

◆ get_device_data_columns()

◆ get_display()

◆ get_feature_indices() [1/4]

◆ get_feature_indices() [2/4]

◆ get_feature_indices() [3/4]

◆ get_feature_indices() [4/4]

◆ get_feature_names() [1/3]

◆ get_feature_names() [2/3]

◆ get_feature_names() [3/3]

◆ get_feature_selection()

◆ get_features_number() [1/3]

◆ get_features_number() [2/3]

◆ get_features_number() [3/3]

◆ get_has_sample_ids()

◆ get_header_line()

◆ get_input_shape()

◆ get_model_input_variables()

◆ get_sample_ids()

◆ get_sample_indices() [1/2]

◆ get_sample_indices() [2/2]

◆ get_sample_roles()

◆ get_samples_number() [1/3]

◆ get_samples_number() [2/3]

◆ get_samples_number() [3/3]

◆ get_separator_name()

◆ get_separator_string()

◆ get_shape() [1/2]

◆ get_shape() [2/2]

◆ get_storage_mode()

◆ get_storage_mode_string()

◆ get_target_shape()

◆ get_training_tokenizer() [1/2]

◆ get_training_tokenizer() [2/2]

◆ get_used_feature_indices()

◆ get_used_sample_indices()

◆ get_used_samples_number()

◆ get_used_variables_indices()

◆ get_used_variables_number()

◆ get_variable_index() [1/2]

◆ get_variable_index() [2/2]

◆ get_variable_indices() [1/2]

◆ get_variable_indices() [2/2]

◆ get_variable_names() [1/3]

◆ get_variable_names() [2/3]