Loading...
Searching...
No Matches
tabular_dataset.h
Go to the documentation of this file.
1// SPDX-License-Identifier: AGPL-3.0-only
2// Copyright (C) 2026 Artificial Intelligence Techniques, S.L. and Roberto López González
3
4#pragma once
5
11
12#include <utility>
13
14namespace opennn
15{
16
17class TabularDataset : public Dataset
18{
19
20public:
21
23
24 TabularDataset(const Index = 0,
25 const Shape& = {0},
26 const Shape& = {0});
27
28 TabularDataset(const filesystem::path&,
29 const string&,
30 bool has_header = true,
31 bool has_sample_ids = false,
33
34 Index get_samples_number() const noexcept override
35 {
37 ? ssize(sample_roles)
38 : data.rows();
39 }
41
43 MatrixR get_data(const string& sample_role, const string& variable_role) const { return get_data_from_indices(get_sample_indices(sample_role), get_feature_indices(variable_role)); }
44 MatrixR get_data_from_indices(const vector<Index>&, const vector<Index>&) const;
45
47 MatrixR get_variable_data(Index, const vector<Index>&) const;
48
49 MatrixR get_feature_data(const string&) const;
50
51 // Windows use the current feature roles; sample indices remain raw row starts.
52 // Configuration creates chronological splits with complete windows in each split.
53 void configure_forecasting(Index past, Index future = 1, bool multi_target = false);
55 bool is_forecasting() const noexcept { return past_time_steps > 0; }
56 Index get_past_time_steps() const noexcept { return past_time_steps; }
57 Index get_future_time_steps() const noexcept { return future_time_steps; }
58 bool get_multi_target() const noexcept { return multi_target; }
59 // Returns a copy of the selected input windows in sample/time/feature order.
60 Tensor3 get_sequence_data(const string& sample_role, const string& feature_role) const;
61
64 vector<Variable> get_model_input_variables() const override;
65 bool sample_order_matters() const noexcept override { return is_forecasting(); }
66 void resize_input_shape(Index) override;
67
68 void set(Index = 0, const Shape& = {}, const Shape& = {});
69 void set(const filesystem::path&,
70 const string&,
71 bool new_has_header = true,
72 bool new_has_ids = false,
74 void set(const filesystem::path&);
75
78
79 void set_binary_cache_path(const filesystem::path&);
80
81 vector<string> get_feature_scalers(const string&) const;
82
83 void set_variable_scalers(const string&);
84 void set_variable_scalers(const vector<string>&);
86
89 const string& get_missing_values_label() const { return missing_values_label; }
91
92 const NumberFormat& get_number_format() const { return number_format; }
93
94 void set_number_format(const NumberFormat& new_number_format)
95 {
96 number_format = new_number_format;
98 }
99
101
102 void set_missing_values_label(string label) { missing_values_label = std::move(label); }
104 void set_missing_values_method(const string&);
105
106 void scrub_missing_values() override;
112
113 vector<string> unuse_uncorrelated_variables(const float = 0.25f);
114 vector<string> unuse_collinear_variables(const float = 0.95f);
115 vector<string> unuse_least_correlated_variables(const Index inputs_to_keep);
116
117 vector<Descriptives> calculate_feature_descriptives() const;
118 vector<Descriptives> calculate_feature_descriptives(const string&) const;
119 vector<Descriptives> calculate_feature_descriptives(const string&, const vector<Index>&) const;
120
123 vector<Descriptives> calculate_variable_descriptives_categories(Index) const;
124
125 vector<Histogram> calculate_variable_distributions(const Index = 10) const;
126 vector<BoxPlot> calculate_variables_box_plots() const;
127
129 Correlation (*)(const MatrixR&, const MatrixR&), Correlation::Method, const string&) const;
132
134 Correlation (*)(const MatrixR&, const MatrixR&), const string&) const;
137
139
141
142 vector<Descriptives> scale_data();
143 vector<Descriptives> scale_features(const string&);
144 void unscale_features(const string&, const vector<Descriptives>&);
145
148 const FeatureScaling&,
149 Index) override;
150 void clear_training_scaling() noexcept override;
151 void enable_device_residency() override;
152
154 vector<vector<Index>> calculate_Tukey_outliers(float cleaning_parameter = 1.5f, bool replace_with_nan = false);
155 vector<vector<Index>> replace_Tukey_outliers_with_NaN(const float = 1.5f);
156
157 bool has_nan() const override;
158 bool has_nan_row(Index row_index) const { return data.row(row_index).array().isNaN().any(); }
159
161 Index count_rows_with_nan() const { return data.array().isNaN().rowwise().any().count(); }
162 Index count_nan() const { return data.array().isNaN().count(); }
163
165 void set_data_integer(const Index);
167
168 void from_JSON(const JsonDocument&) override;
169 void to_JSON(JsonWriter&) const override;
170
171 void read_csv();
172
173 void fill_inputs(const vector<Index>&,
174 const vector<Index>&,
175 float*,
176 FillMode,
177 ColumnContiguity column_contiguity = ColumnContiguity::Unknown) const override;
178
179 void fill_decoder(const vector<Index>&,
180 const vector<Index>&,
181 float*,
182 FillMode,
183 ColumnContiguity column_contiguity = ColumnContiguity::Unknown) const override;
184
185 void fill_targets(const vector<Index>&,
186 const vector<Index>&,
187 float*,
188 FillMode,
189 ColumnContiguity column_contiguity = ColumnContiguity::Unknown) const override;
190
191 void fill_batch(Batch&, const vector<Index>&, const FeatureSelection&, FillMode) const override;
192
193protected:
194
197 bool multi_target = false;
198
199 void set_forecasting_window(Index, Index, bool);
201 pair<vector<Index>, Index> correlation_lags(Index) const;
202 void fill_window_features(const vector<Index>&, const vector<Index>&, float*,
203 Index row_offset, Index steps, bool feature_major,
204 ColumnContiguity) const;
205
206 string missing_values_label = "NA";
213
215 void missing_values_from_JSON(const Json*) override;
216
217 void resize_data_from_JSON(Index) override;
218
219 filesystem::path cache_file_path() const;
220
221 void fill_features(const vector<Index>&,
222 const vector<Index>&,
223 float*,
224 ColumnContiguity column_contiguity = ColumnContiguity::Unknown) const;
225
226 void fill_from_binary_cache(const vector<Index>&,
227 const vector<Index>&,
228 float*,
229 ColumnContiguity column_contiguity = ColumnContiguity::Unknown) const;
230
231 float apply_training_scaling(Index, float) const;
232 void apply_training_scaling(const vector<Index>&,
233 float*,
234 Index) const;
235
236 vector<Index> filter_used_samples_by_column(Index column, bool positive) const;
237
238 vector<Descriptives> calculate_variable_descriptives_samples(bool positive) const;
239
240 vector<ScalerMethod> get_feature_scaler_methods() const;
241 vector<ScalerMethod> get_feature_scaler_methods(VariableRole) const;
242
243 void unuse_samples_with_missing_targets(const vector<Index>&, const vector<Index>&);
244
245 void on_used_samples_changed() override;
248 vector<Descriptives> compute_descriptives_streaming(const vector<Index>&) const;
249
250 DateFormat configure_csv_columns(vector<string_view>&, char, bool);
251 void load_csv_data(const vector<string_view>&, char, bool, DateFormat);
252
253 filesystem::path cache_path;
254 filesystem::path cache_path_override;
257
258 vector<Descriptives> cache_feature_descriptives;
259 vector<Descriptives> cache_transform_descriptives;
261 vector<ScalerMethod> cache_feature_transforms;
262
264 {
267 float min_range = -1.0f;
268 float max_range = 1.0f;
269 bool configured = false;
270
271 float apply(float value) const
272 {
273 return configured
275 : value;
276 }
277 };
278
279 vector<TrainingTransform> training_transforms;
280
281 DateFormat infer_column_types(const vector<string_view>&, char, bool has_quotes = false);
282
283 void apply_scaler(Index feature_index, ScalerMethod scaler, const Descriptives& descriptives, bool unscale);
284};
285
286}
Dataset()=default
bool has_header
Definition dataset.h:377
MatrixR data
Definition dataset.h:353
bool has_sample_ids
Definition dataset.h:378
vector< SampleRole > sample_roles
Definition dataset.h:362
vector< vector< Index > > get_feature_indices() const
Codification
Definition dataset.h:52
@ UTF8
Definition dataset.h:52
virtual void set_storage_mode(StorageMode)
const MatrixR & get_data() const noexcept
Definition dataset.h:151
virtual Index get_samples_number() const noexcept
Definition dataset.h:59
StorageMode storage_mode
Definition dataset.h:340
vector< Index > get_sample_indices(SampleRole) const
StorageMode
Definition dataset.h:57
@ BinaryFile
Definition dataset.h:57
Definition io_utilities.h:30
Definition json.h:75
Definition json.h:92
Definition json.h:24
vector< Descriptives > scale_data()
void set_storage_mode(StorageMode) override
VectorI count_nans_per_variable() const
void set_missing_values_label(string label)
Definition tabular_dataset.h:102
vector< string > unuse_least_correlated_variables(const Index inputs_to_keep)
MatrixR get_data_from_indices(const vector< Index > &, const vector< Index > &) const
void fill_inputs(const vector< Index > &, const vector< Index > &, float *, FillMode, ColumnContiguity column_contiguity=ColumnContiguity::Unknown) const override
void missing_values_to_JSON(JsonWriter &) const
vector< TrainingTransform > training_transforms
Definition tabular_dataset.h:279
TabularDataset(const Index=0, const Shape &={0}, const Shape &={0})
string get_missing_values_method_string() const
vector< ScalerMethod > get_feature_scaler_methods() const
vector< float > cache_feature_replacement
Definition tabular_dataset.h:260
MatrixR get_variable_data(Index) const
FeatureScaling calculate_used_feature_scaling(VariableRole) const override
void set(Index=0, const Shape &={}, const Shape &={})
bool is_forecasting() const noexcept
Definition tabular_dataset.h:55
void set_data_binary_classification()
Index count_rows_with_nan() const
Definition tabular_dataset.h:161
vector< Descriptives > calculate_variable_descriptives_negative_samples() const
Definition tabular_dataset.h:122
vector< Descriptives > scale_features(const string &)
vector< vector< Index > > replace_Tukey_outliers_with_NaN(const float=1.5f)
void fill_from_binary_cache(const vector< Index > &, const vector< Index > &, float *, ColumnContiguity column_contiguity=ColumnContiguity::Unknown) const
vector< Descriptives > calculate_variable_descriptives_positive_samples() const
Definition tabular_dataset.h:121
Index cache_columns_number
Definition tabular_dataset.h:256
void fill_features(const vector< Index > &, const vector< Index > &, float *, ColumnContiguity column_contiguity=ColumnContiguity::Unknown) const
vector< Descriptives > calculate_variable_descriptives_samples(bool positive) const
bool multi_target
Definition tabular_dataset.h:197
vector< BoxPlot > calculate_variables_box_plots() const
void unscale_features(const string &, const vector< Descriptives > &)
void from_JSON(const JsonDocument &) override
Tensor< Correlation, 2 > calculate_input_target_variable_pearson_correlations() const
Definition tabular_dataset.h:135
vector< Descriptives > calculate_variable_descriptives_categories(Index) const
const NumberFormat & get_number_format() const
Definition tabular_dataset.h:92
DateFormat configure_csv_columns(vector< string_view > &, char, bool)
Index get_missing_values_number() const
Definition tabular_dataset.h:90
Tensor3 get_sequence_data(const string &sample_role, const string &feature_role) const
void set_variable_scalers(const string &)
const string & get_missing_values_label() const
Definition tabular_dataset.h:89
vector< Descriptives > compute_descriptives_streaming(const vector< Index > &) const
Tensor< Correlation, 2 > calculate_input_variable_pearson_correlations() const
Definition tabular_dataset.h:130
void calculate_missing_values_statistics()
vector< vector< Index > > calculate_Tukey_outliers(float cleaning_parameter=1.5f, bool replace_with_nan=false)
MissingValuesMethod missing_values_method
Definition tabular_dataset.h:209
void to_JSON(JsonWriter &) const override
vector< string > unuse_uncorrelated_variables(const float=0.25f)
TabularDataset(const filesystem::path &, const string &, bool has_header=true, bool has_sample_ids=false, const Codification &=Codification::UTF8)
void unuse_samples_with_missing_targets(const vector< Index > &, const vector< Index > &)
void load_csv_data(const vector< string_view > &, char, bool, DateFormat)
void set_default_variable_scalers()
MatrixR calculate_autocorrelations(Index=10) const
Index past_time_steps
Definition tabular_dataset.h:195
vector< ScalerMethod > get_feature_scaler_methods(VariableRole) const
vector< Descriptives > cache_feature_descriptives
Definition tabular_dataset.h:258
Tensor< Correlation, 2 > calculate_input_variable_correlations(Correlation(*)(const MatrixR &, const MatrixR &), Correlation::Method, const string &) const
Tensor< Correlation, 2 > calculate_input_target_variable_correlations(Correlation(*)(const MatrixR &, const MatrixR &), const string &) const
void set(const filesystem::path &, const string &, bool new_has_header=true, bool new_has_ids=false, const Codification &=Codification::UTF8)
vector< string > get_feature_scalers(const string &) const
void fill_window_features(const vector< Index > &, const vector< Index > &, float *, Index row_offset, Index steps, bool feature_major, ColumnContiguity) const
vector< Index > filter_used_samples_by_column(Index column, bool positive) const
bool sample_order_matters() const noexcept override
Definition tabular_dataset.h:65
void set_number_format_auto()
Definition tabular_dataset.h:100
void fill_targets(const vector< Index > &, const vector< Index > &, float *, FillMode, ColumnContiguity column_contiguity=ColumnContiguity::Unknown) const override
virtual void impute_missing_values_interpolate()
Index future_time_steps
Definition tabular_dataset.h:196
MissingValuesMethod
Definition tabular_dataset.h:22
@ Unuse
Definition tabular_dataset.h:22
@ Mean
Definition tabular_dataset.h:22
@ Interpolation
Definition tabular_dataset.h:22
@ Median
Definition tabular_dataset.h:22
vector< ScalerMethod > cache_feature_transforms
Definition tabular_dataset.h:261
void apply_scaler(Index feature_index, ScalerMethod scaler, const Descriptives &descriptives, bool unscale)
Index get_future_time_steps() const noexcept
Definition tabular_dataset.h:57
void clear_training_scaling() noexcept override
bool has_nan() const override
Tensor< Correlation, 2 > calculate_input_variable_spearman_correlations() const
Definition tabular_dataset.h:131
void set_forecasting_window(Index, Index, bool)
Tensor3 calculate_cross_correlations(Index=10) const
pair< vector< Index >, Index > correlation_lags(Index) const
void fill_decoder(const vector< Index > &, const vector< Index > &, float *, FillMode, ColumnContiguity column_contiguity=ColumnContiguity::Unknown) const override
void impute_missing_values_statistic(const MissingValuesMethod &)
bool get_multi_target() const noexcept
Definition tabular_dataset.h:58
void fill_batch(Batch &, const vector< Index > &, const FeatureSelection &, FillMode) const override
void set_number_format(const NumberFormat &new_number_format)
Definition tabular_dataset.h:94
MatrixR get_feature_data(const string &) const
string missing_values_label
Definition tabular_dataset.h:206
bool number_format_automatic
Definition tabular_dataset.h:208
void enable_device_residency() override
vector< Descriptives > calculate_feature_descriptives(const string &, const vector< Index > &) const
void set_data_integer(const Index)
Index rows_missing_values_number
Definition tabular_dataset.h:212
MatrixR get_variable_data(Index, const vector< Index > &) const
filesystem::path cache_path_override
Definition tabular_dataset.h:254
VectorI calculate_target_distribution() const override
vector< Variable > get_model_input_variables() const override
void resize_input_shape(Index) override
void set_missing_values_method(const MissingValuesMethod &method)
Definition tabular_dataset.h:103
void set_missing_values_method(const string &)
MatrixR get_data(const string &sample_role, const string &variable_role) const
Definition tabular_dataset.h:43
float apply_training_scaling(Index, float) const
MissingValuesMethod get_missing_values_method() const
Definition tabular_dataset.h:87
Tensor< Correlation, 2 > calculate_input_target_variable_spearman_correlations() const
Definition tabular_dataset.h:136
filesystem::path cache_path
Definition tabular_dataset.h:253
void set_variable_scalers(const vector< string > &)
vector< string > unuse_collinear_variables(const float=0.95f)
Index missing_values_number
Definition tabular_dataset.h:210
vector< Descriptives > calculate_feature_descriptives(const string &) const
VectorI variables_missing_values_number
Definition tabular_dataset.h:211
void set(const filesystem::path &)
NumberFormat number_format
Definition tabular_dataset.h:207
FileReader cache_reader
Definition tabular_dataset.h:255
Index get_samples_number() const noexcept override
Definition tabular_dataset.h:34
filesystem::path cache_file_path() const
MatrixR calculate_input_target_correlation_values() const override
Definition tabular_dataset.h:138
bool has_nan_row(Index row_index) const
Definition tabular_dataset.h:158
vector< Histogram > calculate_variable_distributions(const Index=10) const
vector< Descriptives > cache_transform_descriptives
Definition tabular_dataset.h:259
virtual void impute_missing_values_unuse()
void on_used_samples_changed() override
void resize_data_from_JSON(Index) override
void missing_values_from_JSON(const Json *) override
void configure_forecasting(Index past, Index future=1, bool multi_target=false)
void scrub_missing_values() override
void set_binary_cache_path(const filesystem::path &)
vector< Descriptives > calculate_feature_descriptives() const
Index get_past_time_steps() const noexcept
Definition tabular_dataset.h:56
Index count_nan() const
Definition tabular_dataset.h:162
DateFormat infer_column_types(const vector< string_view > &, char, bool has_quotes=false)
void reuse_input_incomplete_rows_binary()
void apply_training_scaling(const vector< Index > &, float *, Index) const
FeatureScaling prepare_training_scaling(VariableRole, const FeatureScaling &, Index) override
Definition configuration.h:10
Correlation correlation_spearman(const MatrixR &, const MatrixR &)
Correlation correlation(const MatrixR &, const MatrixR &)
FillMode
Definition batch.h:20
VariableRole
Definition variable.h:72
void unscale(const TensorView &, const TensorView &, const TensorView &, const TensorView &, const TensorView &, const TensorView &, float, float, TensorView &)
ScalerMethod
Definition variable.h:39
@ None
Definition variable.h:40
Eigen::array< T, N > array
Definition tensor_types.h:736
vector< Descriptives > descriptives(const MatrixR &)
float scale_value(ScalerMethod method, const Descriptives &desc, float value, float min_range=-1.0f, float max_range=1.0f)
Definition scaling.h:58
ColumnContiguity
Definition tensor_types.h:747
@ Unknown
Definition tensor_types.h:747
DateFormat
Definition field_parsing.h:81
MatrixR get_correlation_values(const Tensor< Correlation, 2 > &)
Eigen::Matrix< float, Eigen::Dynamic, Eigen::Dynamic, Layout > MatrixR
Definition opennn_types.h:331
Eigen::Tensor< float, 3, Layout|Eigen::AlignedMax > Tensor3
Definition opennn_types.h:352
Eigen::Matrix< Index, Eigen::Dynamic, 1 > VectorI
Definition opennn_types.h:336
Definition batch.h:60
Definition correlations.h:12
Method
Definition correlations.h:13
@ Pearson
Definition correlations.h:13
@ Spearman
Definition correlations.h:13
Definition statistics.h:12
Definition scaling.h:12
Definition batch.h:24
Definition field_parsing.h:41
Definition tensor_types.h:172
Definition tabular_dataset.h:264
bool configured
Definition tabular_dataset.h:269
ScalerMethod scaler
Definition tabular_dataset.h:266
float max_range
Definition tabular_dataset.h:268
float min_range
Definition tabular_dataset.h:267
Descriptives descriptives
Definition tabular_dataset.h:265
float apply(float value) const
Definition tabular_dataset.h:271