opennn::ForwardPropagation Struct Reference

#include <opennn/network/forward_propagation.h>

Public Member Functions
	ForwardPropagation ()=default
	ForwardPropagation (Index, Network *, ForwardPropagationMode=ForwardPropagationMode::Training, InferenceShapePolicy={}, bool inputs_pre_scaled=false, span< const MemoryPoolEntry > co_planned_lifetimes={}, bool exhaustive_training_plan=false)
	~ForwardPropagation ()
	ForwardPropagation (const ForwardPropagation &)=delete
ForwardPropagation &	operator= (const ForwardPropagation &)=delete
void	set (Index, Network *, Buffer *external_storage=nullptr, ForwardPropagationMode=ForwardPropagationMode::Training, InferenceShapePolicy={}, bool inputs_pre_scaled=false, span< const MemoryPoolEntry > co_planned_lifetimes={}, bool exhaustive_training_plan=false)
void	stage_position (DeviceStream stream)
void	set_active_sequence_length (Index length)
void	share_session_state_from (const ForwardPropagation &source)
bool	reserve_kv_cache (Index required, Index preserved_tokens)
void	set_output_sequence_window (Index start, Index count)
void	gather_output_window ()
Index	get_sequence_capacity () const noexcept
Index	get_final_output_capacity () const noexcept
Index	get_final_output_layer () const noexcept
Index	get_execution_start_layer () const noexcept
bool	needs_position_staging () const noexcept
TensorView	get_last_trainable_layer_outputs () const
TensorView	get_outputs () const
void	enable_binary_logits (Buffer *shared_storage=nullptr)
void	recompute_for_backward (Index layer_index)
void	set_cuda_graph (bool enabled)
void	reset_cuda_graph () noexcept
void	prepare_cuda_graph_workspaces ()
bool	cuda_graph_workspaces_need_growth () const noexcept
device::GraphWorkspaceViews	get_cuda_graph_workspace_views () const noexcept
const vector< Index > *	input_valid_lengths (size_t layer, size_t input_ordinal) const
const int *	input_device_valid_lengths (size_t layer, size_t input_ordinal) const
SequenceLengths	input_sequence_lengths (size_t layer, size_t input_ordinal) const
int *	device_valid_lengths_slot (size_t layer, Index batch_size)
void	inherit_valid_lengths (size_t layer)
Index	valid_lengths_source (size_t layer, size_t input_ordinal) const
uint64_t	get_parameters_version () const

Public Attributes
vector< Index >	co_planned_offsets
Index	batch_size = 0
ForwardPropagationMode	mode = ForwardPropagationMode::Training
Index	past_length = 0
Buffer	arena
vector< Buffer >	layer_state_storage
shared_ptr< vector< Buffer > >	layer_session_state_storage
vector< device::PinnedBuffer >	layer_pinned_storage
vector< Buffer >	staged_input_storage
vector< TensorView >	staged_inputs
Buffer	loss_workspace {Device::CUDA}
Buffer	loss_target_workspace {Device::CUDA}
Buffer	binary_logits_storage
TensorView	binary_logits
Index	binary_logits_layer = -1
vector< vector< uint16_t > >	host_bf16_input_scratch
vector< uint16_t >	host_bf16_output_scratch
Buffer	position_device {Device::CUDA}
device::PinnedBuffer	position_pinned
vector< vector< TensorView > >	inputs
vector< vector< TensorView > >	slots
vector< uint8_t >	drelu_fused_by_layer
vector< tuple< size_t, size_t, size_t > >	passthrough_overrides
vector< vector< Index > >	valid_lengths
vector< const int * >	device_valid_lengths
vector< Buffer >	device_valid_length_storage
bool	use_cuda_graph = false
bool	cuda_graph_failed = false
Index	cuda_graph_warmup_calls = 0
device::GraphExecHandle	inference_graph_exec
vector< const void * >	captured_input_pointers
device::GraphWorkspaceRequirements	inference_graph_workspace_requirements {}
array< Buffer, size_t(device::GraphWorkspaceKind::Count)>	inference_graph_workspaces = cuda_workspace_buffers(make_index_sequence<size_t(device::GraphWorkspaceKind::Count)>{})

Friends
struct	TrainingContext
class	ChatSession

Constructor & Destructor Documentation

ForwardPropagation() [1/3]

opennn::ForwardPropagation::ForwardPropagation ( )

default

ForwardPropagation() [2/3]

opennn::ForwardPropagation::ForwardPropagation	(	Index	,
		Network *	,
		ForwardPropagationMode	= ForwardPropagationMode::Training,
		InferenceShapePolicy	= {},
		bool	inputs_pre_scaled = false,
		span< const MemoryPoolEntry >	co_planned_lifetimes = {},
		bool	exhaustive_training_plan = false )

~ForwardPropagation()

opennn::ForwardPropagation::~ForwardPropagation

(

)

ForwardPropagation() [3/3]

opennn::ForwardPropagation::ForwardPropagation ( const ForwardPropagation & )

delete

Member Function Documentation

cuda_graph_workspaces_need_growth()

bool opennn::ForwardPropagation::cuda_graph_workspaces_need_growth ( ) const

noexcept

device_valid_lengths_slot()

int * opennn::ForwardPropagation::device_valid_lengths_slot	(	size_t	layer,
		Index	batch_size )

enable_binary_logits()

void opennn::ForwardPropagation::enable_binary_logits

(

Buffer *

shared_storage = nullptr

)

gather_output_window()

void opennn::ForwardPropagation::gather_output_window

(

)

get_cuda_graph_workspace_views()

device::GraphWorkspaceViews opennn::ForwardPropagation::get_cuda_graph_workspace_views ( ) const

noexcept

get_execution_start_layer()

Index opennn::ForwardPropagation::get_execution_start_layer ( ) const

inlinenoexcept

get_final_output_capacity()

Index opennn::ForwardPropagation::get_final_output_capacity ( ) const

inlinenoexcept

get_final_output_layer()

Index opennn::ForwardPropagation::get_final_output_layer ( ) const

inlinenoexcept

get_last_trainable_layer_outputs()

TensorView opennn::ForwardPropagation::get_last_trainable_layer_outputs

(

)

const

get_outputs()

TensorView opennn::ForwardPropagation::get_outputs

(

)

const

get_parameters_version()

uint64_t opennn::ForwardPropagation::get_parameters_version

(

)

const

get_sequence_capacity()

Index opennn::ForwardPropagation::get_sequence_capacity ( ) const

inlinenoexcept

inherit_valid_lengths()

void opennn::ForwardPropagation::inherit_valid_lengths

(

size_t

layer

)

input_device_valid_lengths()

const int * opennn::ForwardPropagation::input_device_valid_lengths	(	size_t	layer,
		size_t	input_ordinal ) const

input_sequence_lengths()

SequenceLengths opennn::ForwardPropagation::input_sequence_lengths	(	size_t	layer,
		size_t	input_ordinal ) const

input_valid_lengths()

const vector< Index > * opennn::ForwardPropagation::input_valid_lengths	(	size_t	layer,
		size_t	input_ordinal ) const

needs_position_staging()

bool opennn::ForwardPropagation::needs_position_staging ( ) const

inlinenoexcept

operator=()

ForwardPropagation & opennn::ForwardPropagation::operator= ( const ForwardPropagation & )

delete

prepare_cuda_graph_workspaces()

void opennn::ForwardPropagation::prepare_cuda_graph_workspaces

(

)

recompute_for_backward()

void opennn::ForwardPropagation::recompute_for_backward

(

Index

layer_index

)

reserve_kv_cache()

bool opennn::ForwardPropagation::reserve_kv_cache	(	Index	required,
		Index	preserved_tokens )

reset_cuda_graph()

void opennn::ForwardPropagation::reset_cuda_graph ( )

noexcept

set()

void opennn::ForwardPropagation::set	(	Index	,
		Network *	,
		Buffer *	external_storage = nullptr,
		ForwardPropagationMode	= ForwardPropagationMode::Training,
		InferenceShapePolicy	= {},
		bool	inputs_pre_scaled = false,
		span< const MemoryPoolEntry >	co_planned_lifetimes = {},
		bool	exhaustive_training_plan = false )

set_active_sequence_length()

void opennn::ForwardPropagation::set_active_sequence_length

(

Index

length

)

set_cuda_graph()

void opennn::ForwardPropagation::set_cuda_graph

(

bool

enabled

)

set_output_sequence_window()

void opennn::ForwardPropagation::set_output_sequence_window	(	Index	start,
		Index	count )

share_session_state_from()

void opennn::ForwardPropagation::share_session_state_from

(

const ForwardPropagation &

source

)

stage_position()

void opennn::ForwardPropagation::stage_position

(

DeviceStream

stream

)

valid_lengths_source()

Index opennn::ForwardPropagation::valid_lengths_source	(	size_t	layer,
		size_t	input_ordinal ) const

Friends And Related Symbol Documentation

ChatSession

friend class ChatSession

friend

TrainingContext

friend struct TrainingContext

friend

Member Data Documentation

arena

Buffer opennn::ForwardPropagation::arena

batch_size

Index opennn::ForwardPropagation::batch_size = 0

binary_logits

TensorView opennn::ForwardPropagation::binary_logits

binary_logits_layer

Index opennn::ForwardPropagation::binary_logits_layer = -1

binary_logits_storage

Buffer opennn::ForwardPropagation::binary_logits_storage

captured_input_pointers

vector<const void*> opennn::ForwardPropagation::captured_input_pointers

co_planned_offsets

vector<Index> opennn::ForwardPropagation::co_planned_offsets

cuda_graph_failed

bool opennn::ForwardPropagation::cuda_graph_failed = false

cuda_graph_warmup_calls

Index opennn::ForwardPropagation::cuda_graph_warmup_calls = 0

device_valid_length_storage

vector<Buffer> opennn::ForwardPropagation::device_valid_length_storage

device_valid_lengths

vector<const int*> opennn::ForwardPropagation::device_valid_lengths

drelu_fused_by_layer

vector<uint8_t> opennn::ForwardPropagation::drelu_fused_by_layer

host_bf16_input_scratch

vector<vector<uint16_t> > opennn::ForwardPropagation::host_bf16_input_scratch

host_bf16_output_scratch

vector<uint16_t> opennn::ForwardPropagation::host_bf16_output_scratch

inference_graph_exec

device::GraphExecHandle opennn::ForwardPropagation::inference_graph_exec

inference_graph_workspace_requirements

device::GraphWorkspaceRequirements opennn::ForwardPropagation::inference_graph_workspace_requirements {}

inference_graph_workspaces

array<Buffer, size_t(device::GraphWorkspaceKind::Count)> opennn::ForwardPropagation::inference_graph_workspaces = cuda_workspace_buffers(make_index_sequence<size_t(device::GraphWorkspaceKind::Count)>{})

inputs

vector<vector<TensorView> > opennn::ForwardPropagation::inputs

layer_pinned_storage

vector<device::PinnedBuffer> opennn::ForwardPropagation::layer_pinned_storage

layer_session_state_storage

shared_ptr<vector<Buffer> > opennn::ForwardPropagation::layer_session_state_storage

layer_state_storage

vector<Buffer> opennn::ForwardPropagation::layer_state_storage

loss_target_workspace

Buffer opennn::ForwardPropagation::loss_target_workspace {Device::CUDA}

mutable

loss_workspace

Buffer opennn::ForwardPropagation::loss_workspace {Device::CUDA}

mutable

mode

ForwardPropagationMode opennn::ForwardPropagation::mode = ForwardPropagationMode::Training

passthrough_overrides

vector<tuple<size_t, size_t, size_t> > opennn::ForwardPropagation::passthrough_overrides

past_length

Index opennn::ForwardPropagation::past_length = 0

position_device

Buffer opennn::ForwardPropagation::position_device {Device::CUDA}

position_pinned

device::PinnedBuffer opennn::ForwardPropagation::position_pinned

slots

vector<vector<TensorView> > opennn::ForwardPropagation::slots

staged_input_storage

vector<Buffer> opennn::ForwardPropagation::staged_input_storage

staged_inputs

vector<TensorView> opennn::ForwardPropagation::staged_inputs

use_cuda_graph

bool opennn::ForwardPropagation::use_cuda_graph = false

valid_lengths

vector<vector<Index> > opennn::ForwardPropagation::valid_lengths