Dense + ReLU fused into a single forward op. More...

#include <dense_relu_layer.h>

Inheritance diagram for opennn::DenseRelu:

Public Member Functions
	DenseRelu (const Shape &input_shape={}, const Shape &output_shape={}, const string &label="dense_relu_layer")
	Constructs a DenseRelu layer.

Shape	get_input_shape () const override
	Returns the per-sample input shape.

Shape	get_output_shape () const override
	Returns the per-sample output shape.

Index	get_input_features () const
	Number of input features (last dimension of the input shape).

Index	get_sequence_length () const
	Sequence length when the input is 2D, 1 otherwise.

Activation::Function	get_output_activation () const override
	Activation function fused at the end of this layer (always ReLU).

vector< Operator * >	get_operators () override
	Returns the single operator (Combination with fused ReLU).

vector< pair< Shape, Type > >	get_forward_specs (Index batch_size) const override
	Specifications of the forward intermediate buffers.

void	set (const Shape &input_shape={}, const Shape &output_shape={}, const string &label="dense_relu_layer")
	Re-initializes the layer; same arguments as the constructor.

void	set_input_shape (const Shape &) override
	Updates the input shape and re-shapes weight tensors accordingly.

void	set_output_shape (const Shape &) override
	Updates the output features and re-shapes weight tensors accordingly.

void	on_compute_dtype_changed () override
	Reconfigures inner operators when the compute dtype changes.

void	back_propagate (ForwardPropagation &fp, BackPropagation &bp, size_t layer) const noexcept override
	Backward pass through the fused ReLU and Combination operators.

Public Member Functions inherited from opennn::Layer
virtual	~Layer ()=default
	Virtual destructor; subclasses are owned via unique_ptr<Layer>.

const string &	get_label () const
	Returns the user-assigned label of this layer.

const string &	get_name () const
	Returns the canonical type name of this layer.

LayerType	get_type () const
	Returns the LayerType enumerator for this layer.

void	set_label (string new_label)
	Sets the human-readable label of this layer.

Index	get_parameters_number () const
	Total number of trainable parameters in this layer.

virtual vector< pair< Shape, Type > >	get_parameter_specs () const
	Specifications of the trainable parameter tensors owned by this layer.

virtual vector< pair< Shape, Type > >	get_state_specs () const
	Specifications of the persistent state tensors of this layer.

virtual vector< pair< Shape, Type > >	get_backward_specs (Index batch_size) const
	Specifications of the backward intermediate buffers for one batch.

vector< Shape >	get_parameter_shapes () const
	Shape-only view of get_parameter_specs().

vector< Shape >	get_state_shapes () const
	Shape-only view of get_state_specs().

vector< Shape >	get_forward_shapes (Index b) const
	Shape-only view of get_forward_specs() for batch size `b`.

vector< Shape >	get_backward_shapes (Index b) const
	Shape-only view of get_backward_specs() for batch size `b`.

vector< Type >	get_parameter_dtypes () const
	Dtype-only view of get_parameter_specs().

vector< Type >	get_forward_dtypes (Index b) const
	Dtype-only view of get_forward_specs() for batch size `b`.

vector< Type >	get_backward_dtypes (Index b) const
	Dtype-only view of get_backward_specs() for batch size `b`.

Index	get_inputs_number () const
	Total number of scalar inputs per sample (product of input dims).

Index	get_outputs_number () const
	Total number of scalar outputs per sample (product of output dims).

virtual void	forward_propagate (ForwardPropagation &fp, size_t layer, bool is_training) noexcept
	Forward pass: reads inputs from `fp` and writes outputs into `fp`.

virtual void	from_JSON (const JsonDocument &document)
	Loads the layer configuration (hyperparameters) from JSON.

virtual void	read_JSON_body (const Json *)
	Subclass hook for parsing the body of from_JSON().

virtual void	load_state_from_JSON (const JsonDocument &document)
	Loads parameter and state tensors from a JSON document.

virtual void	to_JSON (JsonWriter &writer) const
	Writes the layer configuration to JSON.

virtual void	write_JSON_body (JsonWriter &) const
	Subclass hook for emitting the body of to_JSON().

virtual void	print () const
	Prints a human-readable summary of the layer to stdout.

bool	get_is_trainable () const
	Whether this layer has trainable parameters.

Type	get_compute_dtype () const
	Numerical type used for forward/backward computation.

void	set_compute_dtype (Type new_compute_dtype)
	Sets the compute dtype and triggers on_compute_dtype_changed().

virtual float *	link_parameters (float *pointer)
	Wires this layer's parameter TensorViews onto an external buffer.

virtual float *	link_states (float *pointer)
	Wires this layer's state TensorViews onto an external buffer.

vector< TensorView > &	get_parameter_views ()
	Mutable access to this layer's parameter TensorViews.

const vector< TensorView > &	get_parameter_views () const
	Read-only access to this layer's parameter TensorViews.

vector< TensorView > &	get_state_views ()
	Mutable access to this layer's state TensorViews.

const vector< TensorView > &	get_state_views () const
	Read-only access to this layer's state TensorViews.

void	redistribute_parameters_to_operators ()
	Forwards the current parameter views down to each composing Operator.

void	redistribute_parameter_gradients_to_operators (vector< TensorView > &gradient_views)
	Forwards externally provided gradient views down to each Operator.

void	redistribute_states_to_operators ()
	Forwards the current state views down to each composing Operator.

Additional Inherited Members
Protected Member Functions inherited from opennn::Layer
	Layer ()=default
	Default constructor; only invoked by subclasses.

float *	link_views (float pointer, const vector< Shape > &shapes, vector< TensorView > &views, const char tag) const
	Builds `views` over a contiguous float buffer using `shapes`.

void	distribute_to_operators (vector< TensorView > &views, void(Operator::link)(const vector< TensorView > &), vector< pair< Shape, Type > >(Operator::specs)() const)
	Generic helper used by the redistribute_*_to_operators() routines.

Protected Attributes inherited from opennn::Layer
string	label = "my_layer"
	User-visible label for this layer instance (default "my_layer").

string	name = "layer"
	Canonical type name set by the subclass (e.g. "dense").

LayerType	layer_type = LayerType::Dense
	Layer type tag set by the subclass.

bool	is_trainable = true
	True if the layer has parameters that participate in training.

bool	is_first_layer = false
	True if this layer is the network's input layer.

Type	compute_dtype = Type::FP32
	Numerical type used for forward and backward computation.

vector< TensorView >	parameters
	Parameter TensorViews bound to the network's parameter arena.

vector< TensorView >	states
	State TensorViews bound to the network's state arena.

vector< unique_ptr< Layer > >	layers
	Sub-layers, when this layer is itself a composite.

Detailed Description

Dense + ReLU fused into a single forward op.

Uses the cuBLASLt RELU_BIAS epilogue on GPU and a ReLU baked into Combination::apply_cpu when the epilogue is RELU_BIAS. No batch normalization, no dropout, activation hard-wired to ReLU — keeps forward_propagate() branch-free for CUDA Graph capture.

Use this layer instead of Dense when ReLU is the desired activation and the runtime cost of branching matters (e.g. latency-bound inference).

Constructor & Destructor Documentation

◆ DenseRelu()

opennn::DenseRelu::DenseRelu	(	const Shape &	input_shape = {},
		const Shape &	output_shape = {},
		const string &	label = "dense_relu_layer" )

Constructs a DenseRelu layer.

Parameters

input_shape	Per-sample input shape; empty means "set later".
output_shape	Per-sample output shape; the trailing dimension is the number of output features.
label	Human-readable label assigned to this layer.

Member Function Documentation

◆ back_propagate()

void opennn::DenseRelu::back_propagate	(	ForwardPropagation &	fp,
		BackPropagation &	bp,
		size_t	layer ) const

overridevirtualnoexcept

Backward pass through the fused ReLU and Combination operators.

Parameters

fp	Forward intermediates from the matching forward pass.
bp	BackPropagation buffer in which to accumulate gradients.
layer	Index of this layer inside the network.

Reimplemented from opennn::Layer.

◆ get_forward_specs()

vector< pair< Shape, Type > > opennn::DenseRelu::get_forward_specs ( Index batch_size ) const

overridevirtual

Specifications of the forward intermediate buffers.

Parameters

batch_size Batch size used for sizing.

Returns: Specs for the Input and Output slots in the Forward enum.

Reimplemented from opennn::Layer.

◆ get_input_features()

Index opennn::DenseRelu::get_input_features ( ) const

inline

Number of input features (last dimension of the input shape).

Returns: 0 if the input shape is empty; the trailing dimension otherwise.

◆ get_input_shape()

Shape opennn::DenseRelu::get_input_shape ( ) const

inlineoverridevirtual

Returns the per-sample input shape.

Implements opennn::Layer.

◆ get_operators()

vector< Operator * > opennn::DenseRelu::get_operators ( )

inlineoverridevirtual

Returns the single operator (Combination with fused ReLU).

Reimplemented from opennn::Layer.

◆ get_output_activation()

Activation::Function opennn::DenseRelu::get_output_activation ( ) const

inlineoverridevirtual

Activation function fused at the end of this layer (always ReLU).

Reimplemented from opennn::Layer.

◆ get_output_shape()

Shape opennn::DenseRelu::get_output_shape ( ) const

overridevirtual

Returns the per-sample output shape.

Returns: Same leading dimensions as the input with the configured number of output features as the trailing dimension.

Implements opennn::Layer.

◆ get_sequence_length()

Index opennn::DenseRelu::get_sequence_length ( ) const

inline

Sequence length when the input is 2D, 1 otherwise.

Returns: Leading dimension of the input shape if rank-2, else 1.

◆ on_compute_dtype_changed()

void opennn::DenseRelu::on_compute_dtype_changed ( )

inlineoverridevirtual

Reconfigures inner operators when the compute dtype changes.

Reimplemented from opennn::Layer.

◆ set()

void opennn::DenseRelu::set	(	const Shape &	input_shape = {},
		const Shape &	output_shape = {},
		const string &	label = "dense_relu_layer" )

Re-initializes the layer; same arguments as the constructor.

Parameters

input_shape	Per-sample input shape.
output_shape	Per-sample output shape.
label	Human-readable label.

◆ set_input_shape()

void opennn::DenseRelu::set_input_shape ( const Shape & )

overridevirtual

Updates the input shape and re-shapes weight tensors accordingly.

Reimplemented from opennn::Layer.

◆ set_output_shape()

void opennn::DenseRelu::set_output_shape ( const Shape & )

overridevirtual

Updates the output features and re-shapes weight tensors accordingly.

Reimplemented from opennn::Layer.

Public Member Functions

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ DenseRelu()

Member Function Documentation

◆ back_propagate()

◆ get_forward_specs()

◆ get_input_features()

◆ get_input_shape()

◆ get_operators()

◆ get_output_activation()

◆ get_output_shape()

◆ get_sequence_length()

◆ on_compute_dtype_changed()

◆ set()

◆ set_input_shape()

◆ set_output_shape()