Loading...
Searching...
No Matches
tabular_dataset.h
Go to the documentation of this file.
43 MatrixR get_data(const string& sample_role, const string& variable_role) const { return get_data_from_indices(get_sample_indices(sample_role), get_feature_indices(variable_role)); }
103 void set_missing_values_method(const MissingValuesMethod& method) { missing_values_method = method; }
121 vector<Descriptives> calculate_variable_descriptives_positive_samples() const { return calculate_variable_descriptives_samples(true); }
122 vector<Descriptives> calculate_variable_descriptives_negative_samples() const { return calculate_variable_descriptives_samples(false); }
130 Tensor<Correlation, 2> calculate_input_variable_pearson_correlations() const { return calculate_input_variable_correlations(correlation, Correlation::Method::Pearson, "pearson"); }
131 Tensor<Correlation, 2> calculate_input_variable_spearman_correlations() const { return calculate_input_variable_correlations(correlation_spearman, Correlation::Method::Spearman, "spearman"); }
135 Tensor<Correlation, 2> calculate_input_target_variable_pearson_correlations() const { return calculate_input_target_variable_correlations(correlation, "pearson"); }
136 Tensor<Correlation, 2> calculate_input_target_variable_spearman_correlations() const { return calculate_input_target_variable_correlations(correlation_spearman, "spearman"); }
138 MatrixR calculate_input_target_correlation_values() const override { return get_correlation_values(calculate_input_target_variable_pearson_correlations()); }
154 vector<vector<Index>> calculate_Tukey_outliers(float cleaning_parameter = 1.5f, bool replace_with_nan = false);
191 void fill_batch(Batch&, const vector<Index>&, const FeatureSelection&, FillMode) const override;
283 void apply_scaler(Index feature_index, ScalerMethod scaler, const Descriptives& descriptives, bool unscale);
Dataset()=default
vector< vector< Index > > get_feature_indices() const
virtual void set_storage_mode(StorageMode)
virtual Index get_samples_number() const noexcept
Definition dataset.h:59
vector< Index > get_sample_indices(SampleRole) const
Definition io_utilities.h:30
Definition json.h:75
Definition json.h:92
Definition json.h:24
vector< Descriptives > scale_data()
void set_storage_mode(StorageMode) override
VectorI count_nans_per_variable() const
void set_missing_values_label(string label)
Definition tabular_dataset.h:102
vector< string > unuse_least_correlated_variables(const Index inputs_to_keep)
MatrixR get_data_from_indices(const vector< Index > &, const vector< Index > &) const
void fill_inputs(const vector< Index > &, const vector< Index > &, float *, FillMode, ColumnContiguity column_contiguity=ColumnContiguity::Unknown) const override
void missing_values_to_JSON(JsonWriter &) const
vector< TrainingTransform > training_transforms
Definition tabular_dataset.h:279
TabularDataset(const Index=0, const Shape &={0}, const Shape &={0})
string get_missing_values_method_string() const
vector< ScalerMethod > get_feature_scaler_methods() const
vector< float > cache_feature_replacement
Definition tabular_dataset.h:260
MatrixR get_variable_data(Index) const
FeatureScaling calculate_used_feature_scaling(VariableRole) const override
void set(Index=0, const Shape &={}, const Shape &={})
bool is_forecasting() const noexcept
Definition tabular_dataset.h:55
void set_data_binary_classification()
Index count_rows_with_nan() const
Definition tabular_dataset.h:161
void read_csv()
vector< Descriptives > calculate_variable_descriptives_negative_samples() const
Definition tabular_dataset.h:122
void set_data_random()
vector< Descriptives > scale_features(const string &)
vector< vector< Index > > replace_Tukey_outliers_with_NaN(const float=1.5f)
void fill_from_binary_cache(const vector< Index > &, const vector< Index > &, float *, ColumnContiguity column_contiguity=ColumnContiguity::Unknown) const
vector< Descriptives > calculate_variable_descriptives_positive_samples() const
Definition tabular_dataset.h:121
Index cache_columns_number
Definition tabular_dataset.h:256
void fill_features(const vector< Index > &, const vector< Index > &, float *, ColumnContiguity column_contiguity=ColumnContiguity::Unknown) const
vector< Descriptives > calculate_variable_descriptives_samples(bool positive) const
vector< BoxPlot > calculate_variables_box_plots() const
void unscale_features(const string &, const vector< Descriptives > &)
void from_JSON(const JsonDocument &) override
Tensor< Correlation, 2 > calculate_input_target_variable_pearson_correlations() const
Definition tabular_dataset.h:135
vector< Descriptives > calculate_variable_descriptives_categories(Index) const
const NumberFormat & get_number_format() const
Definition tabular_dataset.h:92
DateFormat configure_csv_columns(vector< string_view > &, char, bool)
Index get_missing_values_number() const
Definition tabular_dataset.h:90
Tensor3 get_sequence_data(const string &sample_role, const string &feature_role) const
void set_variable_scalers(const string &)
const string & get_missing_values_label() const
Definition tabular_dataset.h:89
vector< Descriptives > compute_descriptives_streaming(const vector< Index > &) const
Tensor< Correlation, 2 > calculate_input_variable_pearson_correlations() const
Definition tabular_dataset.h:130
void calculate_missing_values_statistics()
vector< vector< Index > > calculate_Tukey_outliers(float cleaning_parameter=1.5f, bool replace_with_nan=false)
MissingValuesMethod missing_values_method
Definition tabular_dataset.h:209
void to_JSON(JsonWriter &) const override
vector< string > unuse_uncorrelated_variables(const float=0.25f)
TabularDataset(const filesystem::path &, const string &, bool has_header=true, bool has_sample_ids=false, const Codification &=Codification::UTF8)
void unuse_samples_with_missing_targets(const vector< Index > &, const vector< Index > &)
void load_csv_data(const vector< string_view > &, char, bool, DateFormat)
void set_default_variable_scalers()
MatrixR calculate_autocorrelations(Index=10) const
vector< ScalerMethod > get_feature_scaler_methods(VariableRole) const
vector< Descriptives > cache_feature_descriptives
Definition tabular_dataset.h:258
Tensor< Correlation, 2 > calculate_input_variable_correlations(Correlation(*)(const MatrixR &, const MatrixR &), Correlation::Method, const string &) const
Tensor< Correlation, 2 > calculate_input_target_variable_correlations(Correlation(*)(const MatrixR &, const MatrixR &), const string &) const
void clear_cache_derived_state()
void set(const filesystem::path &, const string &, bool new_has_header=true, bool new_has_ids=false, const Codification &=Codification::UTF8)
vector< string > get_feature_scalers(const string &) const
void fill_window_features(const vector< Index > &, const vector< Index > &, float *, Index row_offset, Index steps, bool feature_major, ColumnContiguity) const
vector< Index > filter_used_samples_by_column(Index column, bool positive) const
bool sample_order_matters() const noexcept override
Definition tabular_dataset.h:65
void set_number_format_auto()
Definition tabular_dataset.h:100
void fill_targets(const vector< Index > &, const vector< Index > &, float *, FillMode, ColumnContiguity column_contiguity=ColumnContiguity::Unknown) const override
virtual void impute_missing_values_interpolate()
@ Interpolation
Definition tabular_dataset.h:22
vector< ScalerMethod > cache_feature_transforms
Definition tabular_dataset.h:261
void apply_scaler(Index feature_index, ScalerMethod scaler, const Descriptives &descriptives, bool unscale)
Index get_future_time_steps() const noexcept
Definition tabular_dataset.h:57
void clear_training_scaling() noexcept override
bool has_nan() const override
Tensor< Correlation, 2 > calculate_input_variable_spearman_correlations() const
Definition tabular_dataset.h:131
void set_forecasting_window(Index, Index, bool)
Tensor3 calculate_cross_correlations(Index=10) const
pair< vector< Index >, Index > correlation_lags(Index) const
void fill_decoder(const vector< Index > &, const vector< Index > &, float *, FillMode, ColumnContiguity column_contiguity=ColumnContiguity::Unknown) const override
void impute_missing_values_statistic(const MissingValuesMethod &)
bool get_multi_target() const noexcept
Definition tabular_dataset.h:58
void fill_batch(Batch &, const vector< Index > &, const FeatureSelection &, FillMode) const override
void set_number_format(const NumberFormat &new_number_format)
Definition tabular_dataset.h:94
MatrixR get_feature_data(const string &) const
string missing_values_label
Definition tabular_dataset.h:206
bool number_format_automatic
Definition tabular_dataset.h:208
void enable_device_residency() override
vector< Descriptives > calculate_feature_descriptives(const string &, const vector< Index > &) const
void set_data_integer(const Index)
Index rows_missing_values_number
Definition tabular_dataset.h:212
MatrixR get_variable_data(Index, const vector< Index > &) const
filesystem::path cache_path_override
Definition tabular_dataset.h:254
VectorI calculate_target_distribution() const override
vector< Variable > get_model_input_variables() const override
void resize_input_shape(Index) override
void set_missing_values_method(const MissingValuesMethod &method)
Definition tabular_dataset.h:103
void refresh_forecasting_roles()
void set_missing_values_method(const string &)
MatrixR get_data(const string &sample_role, const string &variable_role) const
Definition tabular_dataset.h:43
float apply_training_scaling(Index, float) const
MissingValuesMethod get_missing_values_method() const
Definition tabular_dataset.h:87
Tensor< Correlation, 2 > calculate_input_target_variable_spearman_correlations() const
Definition tabular_dataset.h:136
void set_variable_scalers(const vector< string > &)
vector< string > unuse_collinear_variables(const float=0.95f)
Index missing_values_number
Definition tabular_dataset.h:210
vector< Descriptives > calculate_feature_descriptives(const string &) const
VectorI variables_missing_values_number
Definition tabular_dataset.h:211
void set(const filesystem::path &)
void refresh_cache_statistics()
Index get_samples_number() const noexcept override
Definition tabular_dataset.h:34
filesystem::path cache_file_path() const
MatrixR calculate_input_target_correlation_values() const override
Definition tabular_dataset.h:138
bool has_nan_row(Index row_index) const
Definition tabular_dataset.h:158
vector< Histogram > calculate_variable_distributions(const Index=10) const
vector< Descriptives > cache_transform_descriptives
Definition tabular_dataset.h:259
virtual void impute_missing_values_unuse()
void on_used_samples_changed() override
void clear_forecasting()
void resize_data_from_JSON(Index) override
void missing_values_from_JSON(const Json *) override
void configure_forecasting(Index past, Index future=1, bool multi_target=false)
void scrub_missing_values() override
void set_binary_cache_path(const filesystem::path &)
vector< Descriptives > calculate_feature_descriptives() const
Index get_past_time_steps() const noexcept
Definition tabular_dataset.h:56
DateFormat infer_column_types(const vector< string_view > &, char, bool has_quotes=false)
void reuse_input_incomplete_rows_binary()
void apply_training_scaling(const vector< Index > &, float *, Index) const
FeatureScaling prepare_training_scaling(VariableRole, const FeatureScaling &, Index) override
Definition configuration.h:10
Correlation correlation_spearman(const MatrixR &, const MatrixR &)
Correlation correlation(const MatrixR &, const MatrixR &)
void unscale(const TensorView &, const TensorView &, const TensorView &, const TensorView &, const TensorView &, const TensorView &, float, float, TensorView &)
vector< Descriptives > descriptives(const MatrixR &)
float scale_value(ScalerMethod method, const Descriptives &desc, float value, float min_range=-1.0f, float max_range=1.0f)
Definition scaling.h:58
MatrixR get_correlation_values(const Tensor< Correlation, 2 > &)
Eigen::Matrix< float, Eigen::Dynamic, Eigen::Dynamic, Layout > MatrixR
Definition opennn_types.h:331
Definition batch.h:60
Definition correlations.h:12
Definition statistics.h:12
Definition scaling.h:12
Definition batch.h:24
Definition field_parsing.h:41
Definition tensor_types.h:172
Definition tabular_dataset.h:264
bool configured
Definition tabular_dataset.h:269
ScalerMethod scaler
Definition tabular_dataset.h:266
Descriptives descriptives
Definition tabular_dataset.h:265
float apply(float value) const
Definition tabular_dataset.h:271