tensor_operations.h

Go to the documentation of this file.

// SPDX-License-Identifier: AGPL-3.0-only
// Copyright (C) 2026 Artificial Intelligence Techniques, S.L. and Roberto López González
 
#pragma once
 
#include "opennn/core/opennn_types.h"
#include "opennn/core/tensor_types.h"
#include "opennn/core/enum_map.h"
 
namespace opennn
{
 
inline constexpr float INV_SQRT_2      = 0.70710678118654752440f;
inline constexpr float INV_SQRT_2_PI   = 0.39894228040143267794f;
inline constexpr float SQRT_2_OVER_PI  = 0.7978845608028654f;
inline constexpr float GELU_TANH_CUBIC = 0.044715f;
 
inline float gelu_value(float x)
{
    return 0.5f * x * (1.0f + erff(x * INV_SQRT_2));
}
 
inline float gelu_tanh_value(float x)
{
    return 0.5f * x * (1.0f + tanhf(SQRT_2_OVER_PI * (x + GELU_TANH_CUBIC * x * x * x)));
}
 
inline float silu_value(float x)
{
    return x / (1.0f + exp(-x));
}
 
inline float gelu_derivative(float x)
{
    const float cdf = 0.5f * (1.0f + erff(x * INV_SQRT_2));
    const float pdf = INV_SQRT_2_PI * expf(-0.5f * x * x);
    return cdf + x * pdf;
}
 
inline float gelu_tanh_derivative(float x)
{
    const float x2 = x * x;
    const float u = SQRT_2_OVER_PI * (x + GELU_TANH_CUBIC * x * x2);
    const float t = tanhf(u);
    const float du = SQRT_2_OVER_PI * (1.0f + 3.0f * GELU_TANH_CUBIC * x2);
    return 0.5f * (1.0f + t) + 0.5f * x * (1.0f - t * t) * du;
}
 
inline float silu_derivative(float x)
{
    const float s = 1.0f / (1.0f + expf(-x));
    return s * (1.0f + x * (1.0f - s));
}
 
const EnumMap<ActivationFunction>& activation_function_map();
const string& activation_function_to_string(ActivationFunction);
ActivationFunction activation_function_from_string(const string&);
 
bool activation_needs_input(ActivationFunction function);
 
// Whether this build has the MKL CPU kernels at all. `Configuration::set_blas`
// accepts `Blas::Mkl` either way; this is how a caller finds out whether that
// request has anywhere to go.
bool blas_mkl_available();
 
inline float activation_forward_value(ActivationFunction function, float x)
{
    using enum ActivationFunction;
    switch (function)
    {
    case Identity:  return x;
    case Sigmoid:   return 1.0f / (1.0f + exp(-x));
    case Tanh:      return tanh(x);
    case ReLU:      return max(0.0f, x);
    case LeakyReLU: return x >= 0.0f ? x : x * LEAKY_RELU_SLOPE;
    case GELU:      return gelu_value(x);
    case GELUTanh:  return gelu_tanh_value(x);
    case SiLU:      return silu_value(x);
    case Softmax:   break;
    }
 
    throw runtime_error("activation_forward_value: Softmax must be handled separately.");
}
 
inline float activation_derivative_from_output_value(ActivationFunction function, float y)
{
    using enum ActivationFunction;
    switch (function)
    {
    case Identity:  return 1.0f;
    case Sigmoid:   return y * (1.0f - y);
    case Tanh:      return 1.0f - y * y;
    case ReLU:      return y > 0.0f ? 1.0f : 0.0f;
    case LeakyReLU: return y >= 0.0f ? 1.0f : LEAKY_RELU_SLOPE;
    case Softmax:   break;
    case GELU:
    case GELUTanh:
    case SiLU:      break;
    }
 
    throw runtime_error("activation_derivative_from_output_value: Softmax/GELU/GELUTanh/SiLU must be handled separately.");
}
 
template<typename TensorType>
typename TensorType::PlainObject activation_forward_values(ActivationFunction function, const TensorType& values)
{
    return values.unaryExpr([function](float value) { return activation_forward_value(function, value); });
}
 
template<typename TensorType>
typename TensorType::PlainObject activation_derivative_from_output_values(ActivationFunction function, const TensorType& values)
{
    return values.unaryExpr([function](float value) { return activation_derivative_from_output_value(function, value); });
}
 
void copy(const TensorView&, TensorView&);
 
void add(const TensorView&, const TensorView&, TensorView&);
 
enum class Transpose { No, Yes };
 
void multiply(const TensorView&, Transpose, const TensorView&, Transpose, TensorView&,
              float alpha = 1.0f, float beta = 0.0f);
 
void softmax(TensorView&);
void softmax_backward(const TensorView&, TensorView&, float alpha = 1.0f);
 
void activation_forward(TensorView&, ActivationFunction);
void activation_backward(const TensorView&, TensorView&, ActivationFunction);
 
void linear_forward(const TensorView&, const TensorView&, const TensorView&,
                    TensorView&, LinearEpilogue epilogue = LinearEpilogue::Bias,
                    TensorView* pre_activation = nullptr,
                    const TensorView& weight_scale = {},
                    ActivationFunction fused_activation = ActivationFunction::Identity);
struct LinearBackwardOptions
{
    const TensorView* drelu_mask = nullptr;
 
    const TensorView* addend = nullptr;
 
    bool* fused_input_relu = nullptr;
};
 
void linear_backward(const TensorView&, const TensorView&, const TensorView&,
                     const TensorView&, const TensorView&,
                     TensorView&, bool accumulate_input_delta = false,
                     const LinearBackwardOptions& options = {});
 
void linear_forward_transposed(const TensorView& input, const TensorView& embed_weight, TensorView& output,
                          const TensorView& weight_scale = {});
 
MatrixR append_rows(const MatrixR&, const MatrixR&);
MatrixR append_columns(const MatrixR&, const MatrixR&);
pair<MatrixR, MatrixR> append_rows(const pair<MatrixR, MatrixR>&, const pair<MatrixR, MatrixR>&);
MatrixR append_columns(const pair<MatrixR, MatrixR>&);
 
VectorR slice_rows(const VectorR&, const vector<Index>&);
MatrixR slice_rows(const MatrixR&, const vector<Index>&);
pair<MatrixR, MatrixR> slice_rows(const pair<MatrixR, MatrixR>&, const vector<Index>&);
 
MatrixR calculate_distances(const MatrixR&);
VectorI get_nearest_points(const MatrixR&, const VectorR&, Index = 1);
 
bool row_dominates(const MatrixR&, Index, Index);
 
}