Computes diagnostic metrics for a trained network on testing data. More...

#include <testing_analysis.h>

Classes
struct	BinaryClassificationRates
	Sample indices grouped by binary-classification outcome. More...

struct	GoodnessOfFitAnalysis
	Per-output regression goodness-of-fit summary. More...

struct	KolmogorovSmirnovResults
	Output of perform_Kolmogorov_Smirnov_analysis(). More...

struct	RocAnalysis
	Output of perform_roc_analysis(). More...

Public Member Functions
	TestingAnalysis (NeuralNetwork new_neural_network=nullptr, Dataset new_dataset=nullptr)
	Constructs a testing analysis bound to a network and dataset.

const NeuralNetwork *	get_neural_network () const
	Returns the network being evaluated.

const Dataset *	get_dataset () const
	Returns the dataset providing testing samples.

void	set_neural_network (NeuralNetwork *new_neural_network)
	Replaces the network being evaluated.

void	set_dataset (Dataset *new_dataset)
	Replaces the dataset providing testing samples.

void	set_batch_size (Index new_batch_size)
	Sets the batch size used when running the network.

Index	get_batch_size () const
	Returns the batch size used when running the network.

void	check () const
	Validates that network and dataset are configured.

MatrixR	calculate_error () const
	Computes the per-output mean error on the testing samples.

Tensor3	calculate_error_data () const
	Computes raw error data for every testing sample.

MatrixR	calculate_percentage_error_data () const
	Computes percentage errors for every testing sample.

vector< Descriptives >	calculate_absolute_errors_descriptives () const
	Computes descriptive statistics of absolute errors on the testing partition.

vector< Descriptives >	calculate_absolute_errors_descriptives (const MatrixR &targets, const MatrixR &outputs) const
	Computes descriptive statistics of absolute errors between the supplied tensors.

vector< Descriptives >	calculate_percentage_errors_descriptives () const
	Computes descriptive statistics of percentage errors on the testing partition.

vector< Descriptives >	calculate_percentage_errors_descriptives (const MatrixR &targets, const MatrixR &outputs) const
	Computes descriptive statistics of percentage errors between the supplied tensors.

vector< vector< Descriptives > >	calculate_error_data_descriptives () const
	Computes descriptive statistics for every error type on the testing partition.

void	print_error_data_descriptives () const
	Prints the result of calculate_error_data_descriptives() to standard output.

vector< Histogram >	calculate_error_data_histograms (const Index bins_number=10) const
	Builds histograms of the per-sample errors.

Tensor< VectorI, 1 >	calculate_maximal_errors (const Index maximal_number=10) const
	Returns the indices of the samples with the largest errors.

MatrixR	calculate_errors () const
	Computes per-output regression error metrics on the testing partition.

VectorR	calculate_errors (const MatrixR &targets, const MatrixR &outputs) const
	Computes regression error metrics between the supplied tensors.

VectorR	calculate_errors (const string &samples_role) const
	Computes regression error metrics on a specific samples subset.

MatrixR	calculate_binary_classification_errors () const
	Computes binary-classification error metrics on the testing partition.

VectorR	calculate_binary_classification_errors (const string &samples_role) const
	Computes binary-classification error metrics on a specific samples subset.

MatrixR	calculate_multiple_classification_errors () const
	Computes multi-class classification error metrics on the testing partition.

VectorR	calculate_multiple_classification_errors (const string &samples_role) const
	Computes multi-class classification error metrics on a specific samples subset.

float	calculate_masked_accuracy (const Tensor3 &outputs, const MatrixR &mask) const
	Computes accuracy ignoring positions flagged by a mask (e.g. padding tokens).

float	calculate_determination (const VectorR &targets, const VectorR &outputs) const
	Computes the coefficient of determination R^2.

Tensor< Correlation, 1 >	linear_correlation (const MatrixR &targets, const MatrixR &outputs) const
	Computes per-output linear correlation between targets and outputs.

void	print_linear_correlations () const
	Prints the linear correlations to standard output.

Tensor< GoodnessOfFitAnalysis, 1 >	perform_goodness_of_fit_analysis () const
	Runs goodness-of-fit analysis for every output.

void	print_goodness_of_fit_analysis () const
	Prints the result of perform_goodness_of_fit_analysis() to standard output.

VectorR	calculate_binary_classification_tests (const float decision_threshold=0.50) const
	Computes a battery of binary-classification metrics at a given threshold.

void	print_binary_classification_tests () const
	Prints the result of calculate_binary_classification_tests() to standard output.

vector< MatrixI >	calculate_multilabel_confusion (const float decision_threshold) const
	Computes one confusion matrix per label for a multilabel problem.

MatrixI	calculate_confusion (const MatrixR &targets, const MatrixR &outputs, float decision_threshold=0.50) const
	Computes a confusion matrix from the supplied targets and outputs.

MatrixI	calculate_confusion (const float decision_threshold=0.50) const
	Computes the confusion matrix on the testing partition.

VectorI	calculate_positives_negatives_rate (const MatrixR &targets, const MatrixR &outputs) const
	Counts samples that are positive vs. negative per class.

RocAnalysis	perform_roc_analysis () const
	Runs full ROC analysis on the testing partition.

MatrixR	calculate_roc_curve (const MatrixR &targets, const MatrixR &outputs) const
	Computes the ROC curve from the supplied targets and outputs.

float	calculate_area_under_curve (const MatrixR &roc_curve) const
	Computes the area under a ROC curve.

float	calculate_area_under_curve_confidence_limit (const MatrixR &targets, const MatrixR &outputs) const
	Computes the 95% confidence limit on the AUC.

float	calculate_optimal_threshold (const MatrixR &roc_curve) const
	Returns the threshold that maximizes Youden's J on a ROC curve.

MatrixR	perform_cumulative_gain_analysis () const
	Runs cumulative-gain analysis on the testing partition.

MatrixR	calculate_cumulative_gain (const MatrixR &targets, const MatrixR &outputs) const
	Computes the positive-class cumulative-gain curve.

MatrixR	calculate_negative_cumulative_gain (const MatrixR &targets, const MatrixR &outputs) const
	Computes the negative-class cumulative-gain curve.

MatrixR	perform_lift_chart_analysis () const
	Runs lift-chart analysis on the testing partition.

MatrixR	calculate_lift_chart (const MatrixR &cumulative_gain) const
	Computes the lift curve from a cumulative-gain curve.

KolmogorovSmirnovResults	perform_Kolmogorov_Smirnov_analysis () const
	Runs the Kolmogorov-Smirnov analysis on the testing partition.

VectorR	calculate_maximum_gain (const MatrixR &positive_cumulative_gain, const MatrixR &negative_cumulative_gain) const
	Computes the maximum vertical distance between two cumulative-gain curves.

vector< Histogram >	calculate_output_histogram (const MatrixR &outputs, Index bins_number=10) const
	Builds histograms of the network output values.

BinaryClassificationRates	calculate_binary_classification_rates (const float decision_threshold=0.50) const
	Splits the testing samples into TP/FP/FN/TN groups.

vector< Index >	calculate_true_positive_samples (const MatrixR &targets, const MatrixR &outputs, const vector< Index > &indices, float decision_threshold) const
	Returns the indices of the true-positive samples in the supplied subset.

vector< Index >	calculate_false_positive_samples (const MatrixR &targets, const MatrixR &outputs, const vector< Index > &indices, float decision_threshold) const
	Returns the indices of the false-positive samples in the supplied subset.

vector< Index >	calculate_false_negative_samples (const MatrixR &targets, const MatrixR &outputs, const vector< Index > &indices, float decision_threshold) const
	Returns the indices of the false-negative samples in the supplied subset.

vector< Index >	calculate_true_negative_samples (const MatrixR &targets, const MatrixR &outputs, const vector< Index > &indices, float decision_threshold) const
	Returns the indices of the true-negative samples in the supplied subset.

VectorR	calculate_multiple_classification_precision () const
	Computes the per-class precision in a multi-class problem.

MatrixR	calculate_multiple_classification_tests () const
	Computes a battery of multi-class classification metrics.

Tensor< VectorI, 2 >	calculate_multiple_classification_rates () const
	Splits multi-class testing samples by (true class, predicted class).

Tensor< VectorI, 2 >	calculate_multiple_classification_rates (const MatrixR &targets, const MatrixR &outputs, const vector< Index > &indices) const
	Splits multi-class samples by (true class, predicted class) on supplied data.

Tensor< string, 2 >	calculate_well_classified_samples (const MatrixR &targets, const MatrixR &outputs, const vector< string > &labels_names) const
	Returns the well-classified samples annotated with predicted/target labels.

Tensor< string, 2 >	calculate_misclassified_samples (const MatrixR &targets, const MatrixR &outputs, const vector< string > &labels_names) const
	Returns the misclassified samples annotated with predicted/target labels.

void	save_confusion (const filesystem::path &file_name) const
	Saves the testing-partition confusion matrix to a CSV file.

void	save_multiple_classification_tests (const filesystem::path &file_name) const
	Saves the multi-class classification tests to a CSV file.

void	save_well_classified_samples (const MatrixR &targets, const MatrixR &outputs, const vector< string > &labels_names, const filesystem::path &file_name) const
	Saves the well-classified samples to a CSV file.

void	save_misclassified_samples (const MatrixR &targets, const MatrixR &outputs, const vector< string > &labels_names, const filesystem::path &file_name) const
	Saves the misclassified samples to a CSV file.

void	save_well_classified_samples_statistics (const MatrixR &targets, const MatrixR &outputs, const vector< string > &labels_names, const filesystem::path &file_name) const
	Saves descriptive statistics of the well-classified samples to CSV.

void	save_misclassified_samples_statistics (const MatrixR &targets, const MatrixR &outputs, const vector< string > &labels_names, const filesystem::path &file_name) const
	Saves descriptive statistics of the misclassified samples to CSV.

vector< VectorR >	calculate_error_autocorrelation (const Index maximum_lags_number=10) const
	Computes the autocorrelation of the residuals at increasing lags.

vector< VectorR >	calculate_inputs_errors_cross_correlation (const Index maximum_lags_number=10) const
	Computes the cross-correlation between inputs and residuals at increasing lags.

pair< float, float >	test_transformer () const
	Runs perplexity-style evaluation on a transformer-like network.

string	test_transformer (const vector< string > &context_string, bool imported_vocabulary) const
	Runs free-form generation on a transformer-like network.

void	from_JSON (const JsonDocument &document)
	Restores the analysis state from a JSON document.

void	to_JSON (JsonWriter &writer) const
	Serializes the analysis state to JSON.

void	save (const filesystem::path &file_name) const
	Saves the analysis state to a JSON file on disk.

void	load (const filesystem::path &file_name)
	Loads the analysis state from a JSON file on disk.

Detailed Description

Computes diagnostic metrics for a trained network on testing data.

Holds non-owning pointers to a NeuralNetwork and a Dataset and runs the network on the dataset's testing partition. Exposes:

regression diagnostics: errors, error histograms, descriptives, goodness-of-fit, linear correlation;
binary-classification diagnostics: confusion matrix, ROC, AUC, cumulative gain, lift, Kolmogorov-Smirnov;
multi-class diagnostics: precision, multilabel confusion, per-class rates;
utilities to enumerate well-classified and misclassified samples.

Constructor & Destructor Documentation

◆ TestingAnalysis()

opennn::TestingAnalysis::TestingAnalysis	(	NeuralNetwork *	new_neural_network = nullptr,
		Dataset *	new_dataset = nullptr )

Constructs a testing analysis bound to a network and dataset.

Parameters

new_neural_network	Non-owning pointer to the trained network.
new_dataset	Non-owning pointer to the dataset providing testing samples.

Member Function Documentation

◆ calculate_absolute_errors_descriptives() [1/2]

vector< Descriptives > opennn::TestingAnalysis::calculate_absolute_errors_descriptives ( ) const

Computes descriptive statistics of absolute errors on the testing partition.

Returns: One Descriptives entry per network output.

◆ calculate_absolute_errors_descriptives() [2/2]

vector< Descriptives > opennn::TestingAnalysis::calculate_absolute_errors_descriptives	(	const MatrixR &	targets,
		const MatrixR &	outputs ) const

Computes descriptive statistics of absolute errors between the supplied tensors.

Parameters

targets	Target values.
outputs	Network outputs.

Returns: One Descriptives entry per output column.

◆ calculate_area_under_curve()

float opennn::TestingAnalysis::calculate_area_under_curve ( const MatrixR & roc_curve ) const

Computes the area under a ROC curve.

Parameters

roc_curve Curve produced by calculate_roc_curve().

Returns: AUC value in [0, 1].

◆ calculate_area_under_curve_confidence_limit()

float opennn::TestingAnalysis::calculate_area_under_curve_confidence_limit	(	const MatrixR &	targets,
		const MatrixR &	outputs ) const

Computes the 95% confidence limit on the AUC.

Parameters

targets	Target values.
outputs	Network outputs.

Returns: Confidence limit on the AUC.

◆ calculate_binary_classification_errors() [1/2]

MatrixR opennn::TestingAnalysis::calculate_binary_classification_errors ( ) const

Computes binary-classification error metrics on the testing partition.

Returns: Matrix of metrics suitable for downstream reporting.

◆ calculate_binary_classification_errors() [2/2]

VectorR opennn::TestingAnalysis::calculate_binary_classification_errors ( const string & samples_role ) const

Computes binary-classification error metrics on a specific samples subset.

Parameters

samples_role Sample-role filter.

Returns: Vector of metrics for the requested partition.

◆ calculate_binary_classification_rates()

BinaryClassificationRates opennn::TestingAnalysis::calculate_binary_classification_rates ( const float decision_threshold = 0.50 ) const

Splits the testing samples into TP/FP/FN/TN groups.

Parameters

decision_threshold Probability threshold.

Returns: BinaryClassificationRates with sample indices for each group.

◆ calculate_binary_classification_tests()

VectorR opennn::TestingAnalysis::calculate_binary_classification_tests ( const float decision_threshold = 0.50 ) const

Computes a battery of binary-classification metrics at a given threshold.

Parameters

decision_threshold Probability threshold separating the positive and negative class.

Returns: Vector with accuracy, precision, recall, specificity, F1, MCC, etc.

◆ calculate_confusion() [1/2]

MatrixI opennn::TestingAnalysis::calculate_confusion ( const float decision_threshold = 0.50 ) const

Computes the confusion matrix on the testing partition.

Parameters

decision_threshold Probability threshold (binary case only).

Returns: Integer confusion matrix [true class x predicted class].

◆ calculate_confusion() [2/2]

MatrixI opennn::TestingAnalysis::calculate_confusion	(	const MatrixR &	targets,
		const MatrixR &	outputs,
		float	decision_threshold = 0.50 ) const

Computes a confusion matrix from the supplied targets and outputs.

Parameters

targets	Target values.
outputs	Network outputs.
decision_threshold	Probability threshold (binary case only).

Returns: Integer confusion matrix [true class x predicted class].

◆ calculate_cumulative_gain()

MatrixR opennn::TestingAnalysis::calculate_cumulative_gain	(	const MatrixR &	targets,
		const MatrixR &	outputs ) const

Computes the positive-class cumulative-gain curve.

Parameters

targets	Target values.
outputs	Network outputs.

Returns: Cumulative-gain curve.

◆ calculate_determination()

float opennn::TestingAnalysis::calculate_determination	(	const VectorR &	targets,
		const VectorR &	outputs ) const

Computes the coefficient of determination R^2.

Parameters

targets	Target values.
outputs	Network outputs.

Returns: R^2 between the two vectors.

◆ calculate_error()

MatrixR opennn::TestingAnalysis::calculate_error ( ) const

Computes the per-output mean error on the testing samples.

Returns: Matrix of mean errors (one row per error type, one column per output).

◆ calculate_error_autocorrelation()

vector< VectorR > opennn::TestingAnalysis::calculate_error_autocorrelation ( const Index maximum_lags_number = 10 ) const

Computes the autocorrelation of the residuals at increasing lags.

Parameters

maximum_lags_number Maximum lag to compute.

Returns: One autocorrelation vector per output.

◆ calculate_error_data()

Tensor3 opennn::TestingAnalysis::calculate_error_data ( ) const

Computes raw error data for every testing sample.

Returns: Rank-3 tensor [error_type, sample, output] of error values.

◆ calculate_error_data_descriptives()

vector< vector< Descriptives > > opennn::TestingAnalysis::calculate_error_data_descriptives ( ) const

Computes descriptive statistics for every error type on the testing partition.

Returns: Outer vector indexed by error type, inner vector by output.

◆ calculate_error_data_histograms()

vector< Histogram > opennn::TestingAnalysis::calculate_error_data_histograms ( const Index bins_number = 10 ) const

Builds histograms of the per-sample errors.

Parameters

bins_number Number of bins per histogram.

Returns: One Histogram per network output.

◆ calculate_errors() [1/3]

MatrixR opennn::TestingAnalysis::calculate_errors ( ) const

Computes per-output regression error metrics on the testing partition.

Returns: Matrix [metric x output] containing absolute, relative and percentage errors.

◆ calculate_errors() [2/3]

VectorR opennn::TestingAnalysis::calculate_errors	(	const MatrixR &	targets,
		const MatrixR &	outputs ) const

Computes regression error metrics between the supplied tensors.

Parameters

targets	Target values.
outputs	Network outputs.

Returns: Vector of metrics aggregated across outputs.

◆ calculate_errors() [3/3]

VectorR opennn::TestingAnalysis::calculate_errors ( const string & samples_role ) const

Computes regression error metrics on a specific samples subset.

Parameters

samples_role Sample-role filter ("Training", "Validation", "Testing").

Returns: Vector of metrics for the requested partition.

◆ calculate_false_negative_samples()

vector< Index > opennn::TestingAnalysis::calculate_false_negative_samples	(	const MatrixR &	targets,
		const MatrixR &	outputs,
		const vector< Index > &	indices,
		float	decision_threshold ) const

Returns the indices of the false-negative samples in the supplied subset.

Parameters

targets	Target values.
outputs	Network outputs.
indices	Index subset to consider.
decision_threshold	Probability threshold.

Returns: Indices of false-negative samples.

◆ calculate_false_positive_samples()

vector< Index > opennn::TestingAnalysis::calculate_false_positive_samples	(	const MatrixR &	targets,
		const MatrixR &	outputs,
		const vector< Index > &	indices,
		float	decision_threshold ) const

Returns the indices of the false-positive samples in the supplied subset.

Parameters

targets	Target values.
outputs	Network outputs.
indices	Index subset to consider.
decision_threshold	Probability threshold.

Returns: Indices of false-positive samples.

◆ calculate_inputs_errors_cross_correlation()

vector< VectorR > opennn::TestingAnalysis::calculate_inputs_errors_cross_correlation ( const Index maximum_lags_number = 10 ) const

Computes the cross-correlation between inputs and residuals at increasing lags.

Parameters

maximum_lags_number Maximum lag to compute.

Returns: One cross-correlation vector per input.

◆ calculate_lift_chart()

MatrixR opennn::TestingAnalysis::calculate_lift_chart ( const MatrixR & cumulative_gain ) const

Computes the lift curve from a cumulative-gain curve.

Parameters

cumulative_gain Curve produced by calculate_cumulative_gain().

Returns: Lift curve.

◆ calculate_masked_accuracy()

float opennn::TestingAnalysis::calculate_masked_accuracy	(	const Tensor3 &	outputs,
		const MatrixR &	mask ) const

Computes accuracy ignoring positions flagged by a mask (e.g. padding tokens).

Parameters

outputs	Network outputs as a rank-3 tensor (e.g. [batch, time, vocab]).
mask	Binary mask matching the [batch, time] dims; non-zero positions count.

Returns: Fraction of unmasked positions correctly classified.

◆ calculate_maximal_errors()

Tensor< VectorI, 1 > opennn::TestingAnalysis::calculate_maximal_errors ( const Index maximal_number = 10 ) const

Returns the indices of the samples with the largest errors.

Parameters

maximal_number Number of indices to return.

Returns: Tensor of vectors of indices (one entry per error type).

◆ calculate_maximum_gain()

VectorR opennn::TestingAnalysis::calculate_maximum_gain	(	const MatrixR &	positive_cumulative_gain,
		const MatrixR &	negative_cumulative_gain ) const

Computes the maximum vertical distance between two cumulative-gain curves.

Parameters

positive_cumulative_gain	Positive-class curve.
negative_cumulative_gain	Negative-class curve.

Returns: Vector containing the maximum gain value and its index.

◆ calculate_misclassified_samples()

Tensor< string, 2 > opennn::TestingAnalysis::calculate_misclassified_samples	(	const MatrixR &	targets,
		const MatrixR &	outputs,
		const vector< string > &	labels_names ) const

Returns the misclassified samples annotated with predicted/target labels.

Parameters

targets	Target values.
outputs	Network outputs.
labels_names	Class names matching the target columns.

Returns: Per-sample table of predicted vs. target labels.

◆ calculate_multilabel_confusion()

vector< MatrixI > opennn::TestingAnalysis::calculate_multilabel_confusion ( const float decision_threshold ) const

Computes one confusion matrix per label for a multilabel problem.

Parameters

decision_threshold Probability threshold per label.

Returns: One MatrixI per output label.

◆ calculate_multiple_classification_errors() [1/2]

MatrixR opennn::TestingAnalysis::calculate_multiple_classification_errors ( ) const

Computes multi-class classification error metrics on the testing partition.

Returns: Matrix of metrics suitable for downstream reporting.

◆ calculate_multiple_classification_errors() [2/2]

VectorR opennn::TestingAnalysis::calculate_multiple_classification_errors ( const string & samples_role ) const

Computes multi-class classification error metrics on a specific samples subset.

Parameters

samples_role Sample-role filter.

Returns: Vector of metrics for the requested partition.

◆ calculate_multiple_classification_precision()

VectorR opennn::TestingAnalysis::calculate_multiple_classification_precision ( ) const

Computes the per-class precision in a multi-class problem.

Returns: Vector of precision values, one per class.

◆ calculate_multiple_classification_rates() [1/2]

Tensor< VectorI, 2 > opennn::TestingAnalysis::calculate_multiple_classification_rates ( ) const

Splits multi-class testing samples by (true class, predicted class).

Returns: Matrix of vectors of sample indices.

◆ calculate_multiple_classification_rates() [2/2]

Tensor< VectorI, 2 > opennn::TestingAnalysis::calculate_multiple_classification_rates	(	const MatrixR &	targets,
		const MatrixR &	outputs,
		const vector< Index > &	indices ) const

Splits multi-class samples by (true class, predicted class) on supplied data.

Parameters

targets	Target values.
outputs	Network outputs.
indices	Index subset to consider.

Returns: Matrix of vectors of sample indices.

◆ calculate_multiple_classification_tests()

MatrixR opennn::TestingAnalysis::calculate_multiple_classification_tests ( ) const

Computes a battery of multi-class classification metrics.

Returns: Matrix of metrics suitable for downstream reporting.

◆ calculate_negative_cumulative_gain()

MatrixR opennn::TestingAnalysis::calculate_negative_cumulative_gain	(	const MatrixR &	targets,
		const MatrixR &	outputs ) const

Computes the negative-class cumulative-gain curve.

Parameters

targets	Target values.
outputs	Network outputs.

Returns: Cumulative-gain curve.

◆ calculate_optimal_threshold()

float opennn::TestingAnalysis::calculate_optimal_threshold ( const MatrixR & roc_curve ) const

Returns the threshold that maximizes Youden's J on a ROC curve.

Parameters

roc_curve Curve produced by calculate_roc_curve().

Returns: Optimal probability threshold.

◆ calculate_output_histogram()

vector< Histogram > opennn::TestingAnalysis::calculate_output_histogram	(	const MatrixR &	outputs,
		Index	bins_number = 10 ) const

Builds histograms of the network output values.

Parameters

outputs	Network outputs.
bins_number	Number of bins per histogram.

Returns: One Histogram per output.

◆ calculate_percentage_error_data()

MatrixR opennn::TestingAnalysis::calculate_percentage_error_data ( ) const

Computes percentage errors for every testing sample.

Returns: Matrix of percentage errors (sample x output).

◆ calculate_percentage_errors_descriptives() [1/2]

vector< Descriptives > opennn::TestingAnalysis::calculate_percentage_errors_descriptives ( ) const

Computes descriptive statistics of percentage errors on the testing partition.

Returns: One Descriptives entry per network output.

◆ calculate_percentage_errors_descriptives() [2/2]

vector< Descriptives > opennn::TestingAnalysis::calculate_percentage_errors_descriptives	(	const MatrixR &	targets,
		const MatrixR &	outputs ) const

Computes descriptive statistics of percentage errors between the supplied tensors.

Parameters

targets	Target values.
outputs	Network outputs.

Returns: One Descriptives entry per output column.

◆ calculate_positives_negatives_rate()

VectorI opennn::TestingAnalysis::calculate_positives_negatives_rate	(	const MatrixR &	targets,
		const MatrixR &	outputs ) const

Counts samples that are positive vs. negative per class.

Parameters

targets	Target values.
outputs	Network outputs.

Returns: Vector with the positive and negative counts.

◆ calculate_roc_curve()

MatrixR opennn::TestingAnalysis::calculate_roc_curve	(	const MatrixR &	targets,
		const MatrixR &	outputs ) const

Computes the ROC curve from the supplied targets and outputs.

Parameters

targets	Target values.
outputs	Network outputs.

Returns: Matrix with [false-positive-rate, true-positive-rate] rows.

◆ calculate_true_negative_samples()

vector< Index > opennn::TestingAnalysis::calculate_true_negative_samples	(	const MatrixR &	targets,
		const MatrixR &	outputs,
		const vector< Index > &	indices,
		float	decision_threshold ) const

Returns the indices of the true-negative samples in the supplied subset.

Parameters

targets	Target values.
outputs	Network outputs.
indices	Index subset to consider.
decision_threshold	Probability threshold.

Returns: Indices of true-negative samples.

◆ calculate_true_positive_samples()

vector< Index > opennn::TestingAnalysis::calculate_true_positive_samples	(	const MatrixR &	targets,
		const MatrixR &	outputs,
		const vector< Index > &	indices,
		float	decision_threshold ) const

Returns the indices of the true-positive samples in the supplied subset.

Parameters

targets	Target values.
outputs	Network outputs.
indices	Index subset to consider.
decision_threshold	Probability threshold.

Returns: Indices of true-positive samples.

◆ calculate_well_classified_samples()

Tensor< string, 2 > opennn::TestingAnalysis::calculate_well_classified_samples	(	const MatrixR &	targets,
		const MatrixR &	outputs,
		const vector< string > &	labels_names ) const

Returns the well-classified samples annotated with predicted/target labels.

Parameters

targets	Target values.
outputs	Network outputs.
labels_names	Class names matching the target columns.

Returns: Per-sample table of predicted vs. target labels.

◆ check()

void opennn::TestingAnalysis::check ( ) const

Validates that network and dataset are configured.

Exceptions

runtime_error if either is missing.

◆ from_JSON()

void opennn::TestingAnalysis::from_JSON ( const JsonDocument & document )

Restores the analysis state from a JSON document.

Parameters

document Parsed JSON produced by to_JSON().

◆ get_batch_size()

Index opennn::TestingAnalysis::get_batch_size ( ) const

inline

Returns the batch size used when running the network.

Returns: Configured batch size.

◆ get_dataset()

const Dataset * opennn::TestingAnalysis::get_dataset ( ) const

inline

Returns the dataset providing testing samples.

Returns: Const pointer to the dataset (may be nullptr).

◆ get_neural_network()

const NeuralNetwork * opennn::TestingAnalysis::get_neural_network ( ) const

inline

Returns the network being evaluated.

Returns: Const pointer to the network (may be nullptr).

◆ linear_correlation()

Tensor< Correlation, 1 > opennn::TestingAnalysis::linear_correlation	(	const MatrixR &	targets,
		const MatrixR &	outputs ) const

Computes per-output linear correlation between targets and outputs.

Parameters

targets	Target values (samples x outputs).
outputs	Network outputs (samples x outputs).

Returns: One Correlation per output column.

◆ load()

void opennn::TestingAnalysis::load ( const filesystem::path & file_name )

Loads the analysis state from a JSON file on disk.

Parameters

file_name Source path.

◆ perform_cumulative_gain_analysis()

MatrixR opennn::TestingAnalysis::perform_cumulative_gain_analysis ( ) const

Runs cumulative-gain analysis on the testing partition.

Returns: Matrix with the positive class cumulative-gain curve.

◆ perform_goodness_of_fit_analysis()

Tensor< GoodnessOfFitAnalysis, 1 > opennn::TestingAnalysis::perform_goodness_of_fit_analysis ( ) const

Runs goodness-of-fit analysis for every output.

Returns: One GoodnessOfFitAnalysis per output.

◆ perform_Kolmogorov_Smirnov_analysis()

KolmogorovSmirnovResults opennn::TestingAnalysis::perform_Kolmogorov_Smirnov_analysis ( ) const

Runs the Kolmogorov-Smirnov analysis on the testing partition.

Returns: KolmogorovSmirnovResults with cumulative-gain curves and maximum gain.

◆ perform_lift_chart_analysis()

MatrixR opennn::TestingAnalysis::perform_lift_chart_analysis ( ) const

Runs lift-chart analysis on the testing partition.

Returns: Matrix with the lift curve.

◆ perform_roc_analysis()

RocAnalysis opennn::TestingAnalysis::perform_roc_analysis ( ) const

Runs full ROC analysis on the testing partition.

Returns: RocAnalysis with curve, AUC, confidence limit and optimal threshold.

◆ print_binary_classification_tests()

void opennn::TestingAnalysis::print_binary_classification_tests ( ) const

Prints the result of calculate_binary_classification_tests() to standard output.

◆ print_error_data_descriptives()

void opennn::TestingAnalysis::print_error_data_descriptives ( ) const

Prints the result of calculate_error_data_descriptives() to standard output.

◆ print_goodness_of_fit_analysis()

void opennn::TestingAnalysis::print_goodness_of_fit_analysis ( ) const

Prints the result of perform_goodness_of_fit_analysis() to standard output.

◆ print_linear_correlations()

void opennn::TestingAnalysis::print_linear_correlations ( ) const

Prints the linear correlations to standard output.

◆ save()

void opennn::TestingAnalysis::save ( const filesystem::path & file_name ) const

Saves the analysis state to a JSON file on disk.

Parameters

file_name Destination path.

◆ save_confusion()

void opennn::TestingAnalysis::save_confusion ( const filesystem::path & file_name ) const

Saves the testing-partition confusion matrix to a CSV file.

Parameters

file_name Destination path.

◆ save_misclassified_samples()

void opennn::TestingAnalysis::save_misclassified_samples	(	const MatrixR &	targets,
		const MatrixR &	outputs,
		const vector< string > &	labels_names,
		const filesystem::path &	file_name ) const

Saves the misclassified samples to a CSV file.

Parameters

targets	Target values.
outputs	Network outputs.
labels_names	Class names matching the target columns.
file_name	Destination path.

◆ save_misclassified_samples_statistics()

void opennn::TestingAnalysis::save_misclassified_samples_statistics	(	const MatrixR &	targets,
		const MatrixR &	outputs,
		const vector< string > &	labels_names,
		const filesystem::path &	file_name ) const

Saves descriptive statistics of the misclassified samples to CSV.

Parameters

targets	Target values.
outputs	Network outputs.
labels_names	Class names matching the target columns.
file_name	Destination path.

◆ save_multiple_classification_tests()

void opennn::TestingAnalysis::save_multiple_classification_tests ( const filesystem::path & file_name ) const

Saves the multi-class classification tests to a CSV file.

Parameters

file_name Destination path.

◆ save_well_classified_samples()

void opennn::TestingAnalysis::save_well_classified_samples	(	const MatrixR &	targets,
		const MatrixR &	outputs,
		const vector< string > &	labels_names,
		const filesystem::path &	file_name ) const

Saves the well-classified samples to a CSV file.

Parameters

targets	Target values.
outputs	Network outputs.
labels_names	Class names matching the target columns.
file_name	Destination path.

◆ save_well_classified_samples_statistics()

void opennn::TestingAnalysis::save_well_classified_samples_statistics	(	const MatrixR &	targets,
		const MatrixR &	outputs,
		const vector< string > &	labels_names,
		const filesystem::path &	file_name ) const

Saves descriptive statistics of the well-classified samples to CSV.

Parameters

targets	Target values.
outputs	Network outputs.
labels_names	Class names matching the target columns.
file_name	Destination path.

◆ set_batch_size()

void opennn::TestingAnalysis::set_batch_size ( Index new_batch_size )

inline

Sets the batch size used when running the network.

Parameters

new_batch_size Number of samples per batch (0 = full pass).

◆ set_dataset()

void opennn::TestingAnalysis::set_dataset ( Dataset * new_dataset )

inline

Replaces the dataset providing testing samples.

Parameters

new_dataset Non-owning pointer to the new dataset.

◆ set_neural_network()

void opennn::TestingAnalysis::set_neural_network ( NeuralNetwork * new_neural_network )

inline

Replaces the network being evaluated.

Parameters

new_neural_network Non-owning pointer to the new network.

◆ test_transformer() [1/2]

pair< float, float > opennn::TestingAnalysis::test_transformer ( ) const

Runs perplexity-style evaluation on a transformer-like network.

Returns: Pair of (loss, accuracy) on the testing partition.

◆ test_transformer() [2/2]

string opennn::TestingAnalysis::test_transformer	(	const vector< string > &	context_string,
		bool	imported_vocabulary ) const

Runs free-form generation on a transformer-like network.

Parameters

context_string	Tokens prepended to the model as context.
imported_vocabulary	Whether to load the vocabulary from disk.

Returns: Generated text continuation.

◆ to_JSON()

void opennn::TestingAnalysis::to_JSON ( JsonWriter & writer ) const

Serializes the analysis state to JSON.

Parameters

writer JSON writer that receives the state tree.

Classes

Public Member Functions

Detailed Description

Constructor & Destructor Documentation

◆ TestingAnalysis()

Member Function Documentation

◆ calculate_absolute_errors_descriptives() [1/2]

◆ calculate_absolute_errors_descriptives() [2/2]

◆ calculate_area_under_curve()

◆ calculate_area_under_curve_confidence_limit()

◆ calculate_binary_classification_errors() [1/2]

◆ calculate_binary_classification_errors() [2/2]

◆ calculate_binary_classification_rates()

◆ calculate_binary_classification_tests()

◆ calculate_confusion() [1/2]

◆ calculate_confusion() [2/2]

◆ calculate_cumulative_gain()

◆ calculate_determination()

◆ calculate_error()

◆ calculate_error_autocorrelation()

◆ calculate_error_data()

◆ calculate_error_data_descriptives()

◆ calculate_error_data_histograms()

◆ calculate_errors() [1/3]

◆ calculate_errors() [2/3]

◆ calculate_errors() [3/3]

◆ calculate_false_negative_samples()

◆ calculate_false_positive_samples()

◆ calculate_inputs_errors_cross_correlation()

◆ calculate_lift_chart()

◆ calculate_masked_accuracy()

◆ calculate_maximal_errors()

◆ calculate_maximum_gain()

◆ calculate_misclassified_samples()

◆ calculate_multilabel_confusion()

◆ calculate_multiple_classification_errors() [1/2]

◆ calculate_multiple_classification_errors() [2/2]

◆ calculate_multiple_classification_precision()

◆ calculate_multiple_classification_rates() [1/2]

◆ calculate_multiple_classification_rates() [2/2]

◆ calculate_multiple_classification_tests()

◆ calculate_negative_cumulative_gain()

◆ calculate_optimal_threshold()

◆ calculate_output_histogram()

◆ calculate_percentage_error_data()

◆ calculate_percentage_errors_descriptives() [1/2]

◆ calculate_percentage_errors_descriptives() [2/2]

◆ calculate_positives_negatives_rate()

◆ calculate_roc_curve()

◆ calculate_true_negative_samples()

◆ calculate_true_positive_samples()

◆ calculate_well_classified_samples()

◆ check()

◆ from_JSON()

◆ get_batch_size()

◆ get_dataset()

◆ get_neural_network()

◆ linear_correlation()

◆ load()

◆ perform_cumulative_gain_analysis()

◆ perform_goodness_of_fit_analysis()

◆ perform_Kolmogorov_Smirnov_analysis()

◆ perform_lift_chart_analysis()

◆ perform_roc_analysis()

◆ print_binary_classification_tests()

◆ print_error_data_descriptives()

◆ print_goodness_of_fit_analysis()

◆ print_linear_correlations()

◆ save()

◆ save_confusion()

◆ save_misclassified_samples()

◆ save_misclassified_samples_statistics()

◆ save_multiple_classification_tests()

◆ save_well_classified_samples()

◆ save_well_classified_samples_statistics()

◆ set_batch_size()

◆ set_dataset()

◆ set_neural_network()

◆ test_transformer() [1/2]

◆ test_transformer() [2/2]