dataset.h

Go to the documentation of this file.

//   OpenNN: Open Neural Networks Library
//   www.opennn.net
//
//   D A T A   S E T   C L A S S   H E A D E R
//
//   Artificial Intelligence Techniques SL
//   artelnics@artelnics.com

 
#pragma once
 
#include "correlations.h"
#include "statistics.h"
#include "tensor_utilities.h"
#include "enum_map.h"
#include "string_utilities.h"
#include "variable.h"
 
namespace opennn
{

enum class SampleRole
{
    Training,
    Validation,
    Testing,
    None
};

inline const EnumMap<SampleRole>& sample_role_map()
{
    static const vector<pair<SampleRole, string>> entries = {
        {SampleRole::Training,   "Training"},
        {SampleRole::Validation, "Validation"},
        {SampleRole::Testing,    "Testing"},
        {SampleRole::None,       "None"}
    };
    static const EnumMap<SampleRole> map{entries};
    return map;
}

inline const string& sample_role_to_string(SampleRole role)
{
    return sample_role_map().to_string(role);
}

inline SampleRole string_to_sample_role(const string& name)
{
    if (name == "0") return SampleRole::Training;
    if (name == "1") return SampleRole::Validation;
    if (name == "2") return SampleRole::Testing;
    if (name == "3") return SampleRole::None;
    return sample_role_map().from_string(name);
}

class Dataset
{
 
public:

    enum class Codification { UTF8, SHIFT_JIS };

    Dataset(const Index samples_number = 0,
            const Shape& input_shape = {0},
            const Shape& target_shape = {0});

    Dataset(const filesystem::path& data_path,
            const string& separator,
            bool has_header = true,
            bool has_ids = false,
            const Codification& codification = Codification::UTF8);
 
    // Enumerations

    enum class Separator{Space, Tab, Comma, Semicolon};

    enum class MissingValuesMethod{Unuse, Mean, Median, Interpolation};

    Index get_samples_number() const {return data.rows();}

    Index get_samples_number(const string& role_name) const;

    Index get_used_samples_number() const;

    vector<Index> get_sample_indices(const string& role_name) const;

    vector<Index> get_used_sample_indices() const;

    const vector<SampleRole>& get_sample_roles() const { return sample_roles; }

    vector<Index> get_sample_roles_vector() const;

    VectorI get_sample_role_numbers() const;

    Index get_variables_number() const { return variables.size(); }

    Index get_variables_number(const string& role_name) const;

    Index get_used_variables_number() const;

    const vector<Variable>& get_variables() const { return variables; }

    vector<Variable> get_variables(const string& role_name) const;

    Index get_variable_index(const string& name) const;

    Index get_variable_index(const Index id) const;

    vector<Index> get_variable_indices(const string& role_name) const;

    vector<Index> get_used_variables_indices() const;

    vector<string> get_variable_names() const;

    vector<string> get_variable_names(const string& role_name) const;

    VariableType get_variable_type(const Index index) const { return variables[index].type; }

    vector<VariableType> get_variable_types(const vector<Index> indices) const;

    Index get_features_number() const;

    Index get_features_number(const string& role_name) const;

    Index get_used_features_number() const;

    vector<string> get_feature_names() const;

    vector<string> get_feature_names(const string& role_name) const;

    vector<vector<Index>> get_feature_indices() const;

    vector<Index> get_feature_indices(const Index variable_index) const;

    vector<Index> get_feature_indices(const string& role_name) const;

    vector<Index> get_used_feature_indices() const;

    vector<Index> get_feature_dimensions() const;

    Shape get_shape(const string& role_name) const;

    vector<string> get_feature_scalers(const string& role_name) const;

    virtual void get_batches(const vector<Index>& sample_indices, Index batch_size, bool shuffle, vector<vector<Index>>& batches) const;

    const MatrixR& get_data() const { return data; }

    MatrixR get_feature_data(const string& role_name) const;

    MatrixR get_data(const string& sample_role, const string& variable_role) const;

    MatrixR get_data_from_indices(const vector<Index>& sample_indices, const vector<Index>& variable_indices) const;

    VectorR get_sample_data(const Index sample_index) const;

    MatrixR get_variable_data(const Index variable_index) const;

    MatrixR get_variable_data(const Index variable_index, const vector<Index>& sample_indices) const;

    MatrixR get_variable_data(const string& variable_name) const;

    const vector<vector<string>>& get_data_file_preview() const { return data_file_preview; }

    MissingValuesMethod get_missing_values_method() const { return missing_values_method; }

    string get_missing_values_method_string() const;

    const filesystem::path& get_data_path() const { return data_path; }

    const Separator& get_separator() const { return separator; }

    string get_separator_string() const;

    string get_separator_name() const;

    const Codification& get_codification() const { return codification; }

    const string get_codification_string() const;

    const string& get_missing_values_label() const { return missing_values_label; }

    bool get_display() const { return display; }

    bool is_empty() const { return data.size() == 0; }

    Shape get_input_shape() const { return input_shape; }

    Shape get_target_shape() const { return target_shape; }

    void set(const Index samples_number = 0, const Shape& input_shape = {}, const Shape& target_shape = {});

    void set(const filesystem::path& data_path,
             const string& separator,
             bool has_header = true,
             bool has_ids = false,
             const Dataset::Codification& codification = Codification::UTF8);

    void set(const filesystem::path& file_name);

    void set_default();

    void set_sample_roles(const string& role_name);

    void set_sample_role(const Index sample_index, const string& role_name);

    void set_sample_roles(const vector<string>& role_names);

    void set_sample_roles(const vector<Index>& sample_indices, const string& role_name);

    void set_variables(const vector<Variable>& new_variables) { variables = new_variables; }

    void set_default_variable_names();

    virtual void set_variable_roles(const vector<string>& role_names);

    void set_variables(const string& description);

    void set_variable_indices(const vector<Index>& input_indices, const vector<Index>& target_indices);

    void set_input_variables_unused();

    void set_variable_role(const Index variable_index, const string& role_name);

    void set_variable_role(const string& variable_name, const string& role_name);

    void set_variable_type(const Index variable_index, const VariableType& type);

    void set_variable_type(const string& variable_name, const VariableType& type);

    void set_variable_types(const VariableType& type);

    void set_variable_names(const vector<string>& new_variable_names);

    void set_variables_number(const Index new_size) { variables.resize(new_size); }

    void set_variable_scalers(const string& scaler_name);

    void set_variable_scalers(const vector<string>& scaler_names);

    void set_binary_variables();

    void set_feature_names(const vector<string>& new_feature_names);

    void set_variable_roles(const string& role_name);

    void set_shape(const string& role_name, const Shape& new_shape);

    void set_data(const MatrixR& new_data);

    void set_data_path(const filesystem::path& new_data_path) { data_path = new_data_path; }

    void set_has_header(bool new_has_header) { has_header = new_has_header; }

    void set_has_ids(bool new_has_ids) { has_sample_ids = new_has_ids; }

    void set_separator(const Separator& new_separator) { separator = new_separator; }

    void set_separator_string(const string& new_separator_string);

    void set_separator_name(const string& new_separator_name);

    void set_codification(const Codification& new_codification) { codification = new_codification; }

    void set_codification(const string& new_codification);

    void set_missing_values_label(string label) { missing_values_label = move(label); }

    void set_missing_values_method(const MissingValuesMethod& method) { missing_values_method = method; }

    void set_missing_values_method(const string& method_name);

    void set_gmt(const Index new_gmt) { gmt = new_gmt; }

    void set_display(bool new_display) { display = new_display; }

    bool is_sample_used(const Index i) const { return sample_roles[i] != SampleRole::None; }

    bool has_binary_variables() const;

    bool has_categorical_variables() const;

    bool has_binary_or_categorical_variables() const;

    bool has_time_variable() const;

    bool has_validation() const;

    bool has_missing_values(const vector<string>& labels) const;

    void split_samples(const float training_ratio = 0.6f, float selection_ratio = 0.2f, float testing_ratio = 0.2f, bool shuffle = true);

    void split_samples_sequential(const float training_ratio = 0.6f, float selection_ratio = 0.2f, float testing_ratio = 0.2f);

    void split_samples_random(const float training_ratio = 0.6f, float selection_ratio = 0.2f, float testing_ratio = 0.2f);

    vector<string> unuse_uncorrelated_variables(const float minimum_correlation = 0.25f);

    vector<string> unuse_collinear_variables(const float maximum_correlation = 0.95f);

    void set_data_constant(const float value);

    vector<Descriptives> calculate_feature_descriptives() const;

    vector<Descriptives> calculate_variable_descriptives_positive_samples() const;

    vector<Descriptives> calculate_variable_descriptives_negative_samples() const;

    vector<Descriptives> calculate_variable_descriptives_categories(const Index variable_index) const;

    vector<Descriptives> calculate_feature_descriptives(const string& role_name) const;

    vector<Histogram> calculate_variable_distributions(const Index bins_number = 10) const;

    vector<BoxPlot> calculate_variables_box_plots() const;

    Tensor<Correlation, 2> calculate_input_variable_correlations(
        Correlation (*correlation_function)(const MatrixR&, const MatrixR&), Correlation::Method method, const string& samples_role) const;

    Tensor<Correlation, 2> calculate_input_variable_pearson_correlations() const;

    Tensor<Correlation, 2> calculate_input_variable_spearman_correlations() const;

    Tensor<Correlation, 2> calculate_input_target_variable_correlations(
        Correlation (*correlation_function)(const MatrixR&, const MatrixR&), const string& samples_role) const;

    Tensor<Correlation, 2> calculate_input_target_variable_pearson_correlations() const;

    Tensor<Correlation, 2> calculate_input_target_variable_spearman_correlations() const;

    VectorI calculate_correlations_rank() const;

    void set_default_variable_scalers();

    vector<Descriptives> scale_data();

    virtual vector<Descriptives> scale_features(const string& role_name);

    void unscale_features(const string& role_name, const vector<Descriptives>& feature_descriptives);

    VectorI calculate_target_distribution() const;

    vector<vector<Index>> calculate_Tukey_outliers(const float tukey_factor = 1.5f, bool replace = false);

    vector<vector<Index>> replace_Tukey_outliers_with_NaN(const float tukey_factor = 1.5f);

    void unuse_Tukey_outliers(const float tukey_factor = 1.5f);

    virtual void set_data_random();

    virtual void set_data_integer(const Index vocabulary_size);

    void set_data_rosenbrock();

    void set_data_binary_classification();

    virtual void from_JSON(const JsonDocument& document);

    virtual void to_JSON(JsonWriter& writer) const;

    void save(const filesystem::path& file_name) const;

    void load(const filesystem::path& file_name);

    void save_data() const;

    void save_data_binary(const filesystem::path& file_name) const;

    void load_data_binary();

    Index get_missing_values_number() const { return missing_values_number; }

    bool has_nan() const;

    bool has_nan_row(const Index row_index) const;

    virtual void impute_missing_values_unuse();

    void impute_missing_values_statistic(const MissingValuesMethod& method);

    virtual void impute_missing_values_interpolate();

    void scrub_missing_values();

    void calculate_missing_values_statistics();

    VectorI count_nans_per_variable() const;

    Index count_variables_with_nan() const;

    Index count_rows_with_nan() const;

    Index count_nan() const;

    vector<vector<Index>> split_samples(const vector<Index>& indices, Index parts_number) const;

    virtual void read_csv();

    DateFormat infer_dataset_date_format(const vector<Variable>& variables, const vector<vector<string>>& data_file_preview, bool has_header, const string& missing_values_label);

    virtual void fill_inputs(const vector<Index>& sample_indices, const vector<Index>& feature_indices, float* buffer, bool transpose = true, int contiguous = -1) const;

    virtual void augment_inputs(float* buffer, Index batch_size) const {}

    virtual void fill_decoder(const vector<Index>& sample_indices,
                              const vector<Index>& feature_indices,
                              float* buffer,
                              bool transpose = true,
                              int contiguous = -1) const;

    virtual void fill_targets(const vector<Index>& sample_indices, const vector<Index>& feature_indices, float* buffer, bool transpose = true, int contiguous = -1) const;
 
private:

    void infer_variable_types_from_data();

    void unuse_constant_variables();

    string get_sample_role(const Index i) const { return sample_role_to_string(sample_roles[i]); }

    SampleRole get_sample_role_type(const Index i) const { return sample_roles[i]; }

    vector<Index> filter_used_samples_by_column(Index, bool) const;

    void apply_scaler(Index feature_index, const string& scaler, const Descriptives& descriptives, bool inverse);

    void infer_column_types(const vector<vector<string>>& data_file_preview);

    void read_data_file_preview(const vector<vector<string>>& data_file_preview);

    void check_separators(const string& file_path) const;
 
protected:

    void set_default_variable_roles();

    void set_default_variable_roles_forecasting();
 
    // DATA

    MatrixR data;
 
    // Dimensions

    Shape input_shape;
    Shape target_shape;
    Shape decoder_shape;
 
    // Samples

    vector<SampleRole> sample_roles;

    vector<string> sample_ids;
 
    // Variables

    vector<Variable> variables;
 
    // Data File

    filesystem::path data_path;

    Separator separator = Separator::Comma;

    string missing_values_label = "NA";
 
    //VectorB nans_variables;

    bool has_header = false;

    bool has_sample_ids = false;

    Codification codification = Codification::UTF8;

    vector<vector<string>> data_file_preview;

    Index gmt = 0;
 
    // Missing Values

    MissingValuesMethod missing_values_method = MissingValuesMethod::Mean;

    Index missing_values_number = 0;

    VectorI variables_missing_values_number;

    Index rows_missing_values_number = 0;
 
    // Display

    bool display = true;

    const vector<string> positive_words = {"1", "yes", "positive", "+", "true", "good", "si", "sí", "Sí"};
    const vector<string> negative_words = {"0", "no", "negative", "-", "false", "bad", "not", "No"};

    void variables_to_JSON(JsonWriter&) const;
    void samples_to_JSON(JsonWriter&) const;
    void missing_values_to_JSON(JsonWriter&) const;
    void preview_data_to_JSON(JsonWriter&) const;

    void variables_from_JSON(const Json*);
    void samples_from_JSON(const Json*);
    void missing_values_from_JSON(const Json*);
    void preview_data_from_JSON(const Json*);
};
 
}
 
#define STRINGIFY_ENUM(x) #x
 
#define ENUM_TO_STRING(x) STRINGIFY_ENUM(x)
 
// OpenNN: Open Neural Networks Library.
// Copyright(C) 2005-2026 Artificial Intelligence Techniques, SL.
// Licensed under the GNU Lesser General Public License v2.1 or later.