opennn::device Namespace Reference

Classes
class	CublasMathModeGuard
class	CublasPointerModeGuard
class	CudaAllocationGrowthGuard
class	CudaBlockCacheBypass
class	CudaEvent
class	CudaGraphWorkspaceScope
struct	GraphExecDeleter
struct	GraphWorkspaceView
class	PinnedBuffer
class	StreamCapture

Typedefs
using	GraphWorkspaceRequirements = std::array<Index, static_cast<size_t>(GraphWorkspaceKind::Count)>
using	GraphWorkspaceViews = std::array<GraphWorkspaceView, static_cast<size_t>(GraphWorkspaceKind::Count)>
using	GraphExecHandle = unique_ptr<remove_pointer_t<DeviceGraphExec>, GraphExecDeleter>

Enumerations
enum class	CopyKind { HostToHost , HostToDevice , DeviceToHost , DeviceToDevice }
enum class	GraphWorkspaceKind {   SharedScratch , Bf16Input , Bf16Gradient , Bf16ToFp32 ,   Int8Dequant , PoolingMask , NormPartials , GradientPartials ,   FlashAttention , Count }
enum class	BatchNormBackwardRung { Auto , StagedFp32 , PlainNative , OwnKernel }
enum class	BatchNormForwardRung { Auto , CudnnGraph , OwnKernel }
enum class	MaxPoolingRung { Auto , Cudnn , OwnKernel }
enum class	AttentionRung { Auto , CudnnGraph , FlashAttention }

Functions
bool	energy_meter_available () noexcept
unsigned long long	energy_meter_begin () noexcept
double	energy_meter_window_watts (unsigned long long begin, int &samples) noexcept
constexpr bool	is_cuda_build () noexcept
bool	has_cuda_device () noexcept
string	cuda_unavailable_reason () noexcept
int	cuda_compute_capability () noexcept
size_t	available_memory ()
string	gpu_info_string () noexcept
string	lt_plan_cache_directory () noexcept
bool	cuda_allocation_growth_forbidden () noexcept
void	set_cuda_allocation_growth_forbidden (bool forbidden) noexcept
void	begin_cuda_shutdown () noexcept
bool	cuda_matmul_plan_creation_forbidden () noexcept
optional< void * >	graph_workspace_override (GraphWorkspaceKind, Index minimum_bytes)
int64_t	conv_workspace_limit_bytes () noexcept
void	set_conv_workspace_cap (int64_t mode) noexcept
void	set_conv_workspace_auto_limit_bytes (int64_t) noexcept
bool	conv_autotune_enabled () noexcept
void	set_conv_autotune (bool enabled) noexcept
bool	allow_tf32 () noexcept
void	set_allow_tf32 (bool enabled) noexcept
void	refresh_blas_math_mode ()
template<typename Rung>
Rung	rung () noexcept
template<typename Rung>
void	set_rung (Rung) noexcept
void *	allocate (Device, Index)
void	deallocate (Device, void *, Index) noexcept
void	set_zero (void *, Index, Device)
void	set_zero_async (void *, Index, DeviceStream=nullptr)
void	copy_async (void *, const void *, Index, CopyKind, DeviceStream=nullptr)
void	copy_async (void *, const void *, Index, Device, Device, DeviceStream=nullptr)
void	synchronize (DeviceStream=nullptr)
void	check_last_error ()
void	reset_last_error () noexcept
void	record_event (DeviceEvent, DeviceStream)
void	synchronize_event (DeviceEvent)
void	stream_wait_event (DeviceStream, DeviceEvent)
void	launch_graph (const GraphExecHandle &, DeviceStream)
int	lanes_available () noexcept
int	active_lane () noexcept
void	set_active_lane (int lane)
DeviceStream	lane_stream (int lane)
DeviceStream	get_compute_stream ()
DeviceStream	get_transfer_stream ()
BlasHandle	get_cublas_handle ()
BlasLtHandle	get_cublas_lt_handle ()
DnnHandle	get_cudnn_handle ()
DnnOpTensorDescriptor	get_op_tensor_add_descriptor ()

Variables
constexpr std::array< const char *, static_cast< size_t >(GraphWorkspaceKind::Count)>	graph_workspace_labels
constexpr int	MAX_LANES = 4

Typedef Documentation

GraphExecHandle

using opennn::device::GraphExecHandle = unique_ptr<remove_pointer_t<DeviceGraphExec>, GraphExecDeleter>

GraphWorkspaceRequirements

using opennn::device::GraphWorkspaceRequirements = std::array<Index, static_cast<size_t>(GraphWorkspaceKind::Count)>

GraphWorkspaceViews

using opennn::device::GraphWorkspaceViews = std::array<GraphWorkspaceView, static_cast<size_t>(GraphWorkspaceKind::Count)>

Enumeration Type Documentation

AttentionRung

enum class opennn::device::AttentionRung

strong

Enumerator
Auto
CudnnGraph
FlashAttention

BatchNormBackwardRung

enum class opennn::device::BatchNormBackwardRung

strong

Enumerator
Auto
StagedFp32
PlainNative
OwnKernel

BatchNormForwardRung

enum class opennn::device::BatchNormForwardRung

strong

Enumerator
Auto
CudnnGraph
OwnKernel

CopyKind

enum class opennn::device::CopyKind

strong

Enumerator
HostToHost
HostToDevice
DeviceToHost
DeviceToDevice

GraphWorkspaceKind

enum class opennn::device::GraphWorkspaceKind

strong

Enumerator
SharedScratch
Bf16Input
Bf16Gradient
Bf16ToFp32
Int8Dequant
PoolingMask
NormPartials
GradientPartials
FlashAttention
Count

MaxPoolingRung

enum class opennn::device::MaxPoolingRung

strong

Enumerator
Auto
Cudnn
OwnKernel

Function Documentation

active_lane()

int opennn::device::active_lane ( )

noexcept

allocate()

void * opennn::device::allocate	(	Device	,
		Index	)

allow_tf32()

bool opennn::device::allow_tf32 ( )

noexcept

available_memory()

size_t opennn::device::available_memory

(

)

begin_cuda_shutdown()

void opennn::device::begin_cuda_shutdown ( )

noexcept

check_last_error()

void opennn::device::check_last_error

(

)

conv_autotune_enabled()

bool opennn::device::conv_autotune_enabled ( )

noexcept

conv_workspace_limit_bytes()

int64_t opennn::device::conv_workspace_limit_bytes ( )

noexcept

copy_async() [1/2]

void opennn::device::copy_async	(	void *	,
		const void *	,
		Index	,
		CopyKind	,
		DeviceStream	= nullptr )

copy_async() [2/2]

void opennn::device::copy_async	(	void *	,
		const void *	,
		Index	,
		Device	,
		Device	,
		DeviceStream	= nullptr )

cuda_allocation_growth_forbidden()

bool opennn::device::cuda_allocation_growth_forbidden ( )

noexcept

cuda_compute_capability()

int opennn::device::cuda_compute_capability ( )

noexcept

cuda_matmul_plan_creation_forbidden()

bool opennn::device::cuda_matmul_plan_creation_forbidden ( )

noexcept

cuda_unavailable_reason()

string opennn::device::cuda_unavailable_reason ( )

noexcept

deallocate()

void opennn::device::deallocate ( Device , void * , Index  )

noexcept

energy_meter_available()

bool opennn::device::energy_meter_available ( )

noexcept

energy_meter_begin()

unsigned long long opennn::device::energy_meter_begin ( )

noexcept

energy_meter_window_watts()

double opennn::device::energy_meter_window_watts ( unsigned long long begin, int & samples )

noexcept

get_compute_stream()

DeviceStream opennn::device::get_compute_stream

(

)

get_cublas_handle()

BlasHandle opennn::device::get_cublas_handle

(

)

get_cublas_lt_handle()

BlasLtHandle opennn::device::get_cublas_lt_handle

(

)

get_cudnn_handle()

DnnHandle opennn::device::get_cudnn_handle

(

)

get_op_tensor_add_descriptor()

DnnOpTensorDescriptor opennn::device::get_op_tensor_add_descriptor

(

)

get_transfer_stream()

DeviceStream opennn::device::get_transfer_stream

(

)

gpu_info_string()

string opennn::device::gpu_info_string ( )

noexcept

graph_workspace_override()

optional< void * > opennn::device::graph_workspace_override	(	GraphWorkspaceKind	,
		Index	minimum_bytes )

has_cuda_device()

bool opennn::device::has_cuda_device ( )

noexcept

is_cuda_build()

bool opennn::device::is_cuda_build ( )

constexprnoexcept

lane_stream()

DeviceStream opennn::device::lane_stream

(

int

lane

)

lanes_available()

int opennn::device::lanes_available ( )

noexcept

launch_graph()

void opennn::device::launch_graph	(	const GraphExecHandle &	,
		DeviceStream	)

lt_plan_cache_directory()

string opennn::device::lt_plan_cache_directory ( )

noexcept

record_event()

void opennn::device::record_event	(	DeviceEvent	,
		DeviceStream	)

refresh_blas_math_mode()

void opennn::device::refresh_blas_math_mode

(

)

reset_last_error()

void opennn::device::reset_last_error ( )

noexcept

rung()

template<typename Rung>

Rung opennn::device::rung ( )

noexcept

set_active_lane()

void opennn::device::set_active_lane

(

int

lane

)

set_allow_tf32()

void opennn::device::set_allow_tf32 ( bool enabled )

noexcept

set_conv_autotune()

void opennn::device::set_conv_autotune ( bool enabled )

noexcept

set_conv_workspace_auto_limit_bytes()

void opennn::device::set_conv_workspace_auto_limit_bytes ( int64_t )

noexcept

set_conv_workspace_cap()

void opennn::device::set_conv_workspace_cap ( int64_t mode )

noexcept

set_cuda_allocation_growth_forbidden()

void opennn::device::set_cuda_allocation_growth_forbidden ( bool forbidden )

noexcept

set_rung()

template<typename Rung>

void opennn::device::set_rung ( Rung )

noexcept

set_zero()

void opennn::device::set_zero	(	void *	,
		Index	,
		Device	)

set_zero_async()

void opennn::device::set_zero_async	(	void *	,
		Index	,
		DeviceStream	= nullptr )

stream_wait_event()

void opennn::device::stream_wait_event	(	DeviceStream	,
		DeviceEvent	)

synchronize()

void opennn::device::synchronize

(

DeviceStream

= nullptr

)

synchronize_event()

void opennn::device::synchronize_event

(

DeviceEvent

)

Variable Documentation

graph_workspace_labels

std::array<const char*, static_cast<size_t>(GraphWorkspaceKind::Count)> opennn::device::graph_workspace_labels

inlineconstexpr

Initial value:

= {"shared_scratch", "bf16_input", "bf16_gradient",
                          "bf16_to_fp32", "int8_dequant", "pooling_mask", "norm_partials",
                          "gradient_partials", "flash_attention"}

MAX_LANES

int opennn::device::MAX_LANES = 4

constexpr