sample Namespace Reference

Namespaces
	anonymous_namespace{logging.h}
	anonymous_namespace{sampleDevice.h}
	anonymous_namespace{sampleEngines.cpp}
	anonymous_namespace{sampleInference.cpp}
	anonymous_namespace{sampleOptions.cpp}
	anonymous_namespace{sampleReporting.cpp}

Classes
struct	AllOptions
struct	BaseModelOptions
struct	Binding
class	Bindings
struct	BuildOptions
struct	DeviceAllocator
struct	DeviceDeallocator
struct	HostAllocator
struct	HostDeallocator
struct	InferenceEnvironment
struct	InferenceOptions
struct	InferenceTime
	Measurement times in milliseconds. More...
struct	InferenceTrace
	Measurement points in milliseconds. More...
struct	LayerProfile
	Layer profile information. More...
class	Logger
	Class which manages logging of TensorRT tools and samples. More...
class	LogStreamConsumer
	Convenience object used to facilitate use of C++ stream syntax when logging messages. Order of base classes is LogStreamConsumerBase and then std::ostream. This is because the LogStreamConsumerBase class is used to initialize the LogStreamConsumerBuffer member field in LogStreamConsumer and then the address of the buffer is passed to std::ostream. This is necessary to prevent the address of an uninitialized buffer from being passed to std::ostream. Please do not change the order of the parent classes. More...
class	LogStreamConsumerBase
	Convenience object used to initialize LogStreamConsumerBuffer before std::ostream in LogStreamConsumer. More...
class	LogStreamConsumerBuffer
class	MirroredBuffer
	Coupled host and device buffers. More...
struct	ModelOptions
struct	Options
struct	Parser
class	Profiler
	Collect per-layer profile information, assuming times are reported in the same order. More...
struct	ReportingOptions
struct	SystemOptions
class	TrtCudaBuffer
	Managed buffer for host and device. More...
class	TrtCudaEvent
	Managed CUDA event. More...
class	TrtCudaGraph
	Managed CUDA graph. More...
class	TrtCudaStream
	Managed CUDA stream. More...
struct	TrtDestroyer
struct	UffInput

Typedefs
using	Severity = nvinfer1::ILogger::Severity
using	TrtDeviceBuffer = TrtCudaBuffer< DeviceAllocator, DeviceDeallocator >
using	TrtHostBuffer = TrtCudaBuffer< HostAllocator, HostDeallocator >
using	Arguments = std::unordered_multimap< std::string, std::string >
using	IOFormat = std::pair< nvinfer1::DataType, nvinfer1::TensorFormats >
using	ShapeRange = std::array< std::vector< int >, nvinfer1::EnumMax< nvinfer1::OptProfileSelector >()>
template<typename T >
using	TrtUniquePtr = std::unique_ptr< T, TrtDestroyer< T > >

Enumerations
enum	ModelFormat {   ModelFormat::kANY,   ModelFormat::kCAFFE,   ModelFormat::kONNX,   ModelFormat::kUFF }

Functions
void	setReportableSeverity (Logger::Severity severity)
void	cudaCheck (cudaError_t ret, std::ostream &err=std::cerr)
void	setCudaDevice (int device, std::ostream &os)
Parser	modelToNetwork (const ModelOptions &model, nvinfer1::INetworkDefinition &network, std::ostream &err)
	Generate a network definition for a given model. More...
ICudaEngine *	networkToEngine (const BuildOptions &build, const SystemOptions &sys, nvinfer1::IBuilder &builder, nvinfer1::INetworkDefinition &network, std::ostream &err)
	Create an engine for a network defintion. More...
ICudaEngine *	modelToEngine (const ModelOptions &model, const BuildOptions &build, const SystemOptions &sys, std::ostream &err)
	Create an engine for a given model. More...
void	dumpRefittable (nvinfer1::ICudaEngine &engine)
	Log refittable layers and weights of a refittable engine. More...
ICudaEngine *	loadEngine (const std::string &engine, int DLACore, std::ostream &err)
	Load a serialized engine. More...
bool	saveEngine (const nvinfer1::ICudaEngine &engine, const std::string &fileName, std::ostream &err)
	Save an engine into a file. More...
TrtUniquePtr< nvinfer1::ICudaEngine >	getEngine (const ModelOptions &model, const BuildOptions &build, const SystemOptions &sys, std::ostream &err)
	Create an engine from model or serialized file, and optionally save engine. More...
bool	setUpInference (InferenceEnvironment &iEnv, const InferenceOptions &inference)
	Set up contexts and bindings for inference. More...
void	runInference (const InferenceOptions &inference, InferenceEnvironment &iEnv, int device, std::vector< InferenceTrace > &trace)
	Run inference and collect timing. More...
Arguments	argsToArgumentsMap (int argc, char *argv[])
bool	parseHelp (Arguments &arguments)
std::ostream &	operator<< (std::ostream &os, const BaseModelOptions &options)
std::ostream &	operator<< (std::ostream &os, const UffInput &input)
std::ostream &	operator<< (std::ostream &os, const ModelOptions &options)
std::ostream &	operator<< (std::ostream &os, const IOFormat &format)
std::ostream &	operator<< (std::ostream &os, const ShapeRange &dims)
std::ostream &	operator<< (std::ostream &os, const BuildOptions &options)
std::ostream &	operator<< (std::ostream &os, const SystemOptions &options)
std::ostream &	operator<< (std::ostream &os, const InferenceOptions &options)
std::ostream &	operator<< (std::ostream &os, const ReportingOptions &options)
std::ostream &	operator<< (std::ostream &os, const AllOptions &options)
void	helpHelp (std::ostream &os)
void	printProlog (int warmups, int timings, float warmupMs, float walltime, std::ostream &os)
	Print benchmarking time and number of traces collected. More...
void	printTiming (const std::vector< InferenceTime > &timings, int runsPerAvg, std::ostream &os)
	Print a timing trace. More...
void	printEpilog (std::vector< InferenceTime > timings, float walltimeMs, float percentile, int queries, std::ostream &os)
void	printPerformanceReport (const std::vector< InferenceTrace > &trace, const ReportingOptions &reporting, float warmupMs, int queries, std::ostream &os)
	Print and summarize a timing trace. More...
void	exportJSONTrace (const std::vector< InferenceTrace > &trace, const std::string &fileName)
	Printed format: [ value, ...] value ::= { "start enq : time, "end enq" : time, "start in" : time, "end in" : time, "start compute" : time, "end compute" : time, "start out" : time, "in" : time, "compute" : time, "out" : time, "latency" : time, "end to end" : time}. More...
void	dumpInputs (const nvinfer1::IExecutionContext &context, const Bindings &bindings, std::ostream &os)
	Print input tensors to stream. More...
void	dumpOutputs (const nvinfer1::IExecutionContext &context, const Bindings &bindings, std::ostream &os)
	Print output tensors to stream. More...
void	exportJSONOutput (const nvinfer1::IExecutionContext &context, const Bindings &bindings, const std::string &fileName)
	Export output tensors to JSON file. More...
InferenceTime	operator+ (const InferenceTime &a, const InferenceTime &b)
InferenceTime	operator+= (InferenceTime &a, const InferenceTime &b)
void	printEpilog (std::vector< InferenceTime > timings, float percentile, int queries, std::ostream &os)
	Print the performance summary of a trace. More...
int	dataTypeSize (nvinfer1::DataType dataType)
template<typename T >
T	roundUp (T m, T n)
int	volume (const nvinfer1::Dims &d)
int	volume (const nvinfer1::Dims &dims, const nvinfer1::Dims &strides, int vecDim, int comps, int batch)
int	volume (nvinfer1::Dims dims, int vecDim, int comps, int batch)
std::ostream &	operator<< (std::ostream &os, const nvinfer1::Dims &dims)
std::ostream &	operator<< (std::ostream &os, const std::vector< int > &vec)
std::ostream &	operator<< (std::ostream &os, const nvinfer1::WeightsRole role)
nvinfer1::Dims	toDims (const std::vector< int > &vec)
template<typename T >
void	fillBuffer (void *buffer, int volume, T min, T max)
template<typename H >
void	fillBufferHalf (void *buffer, int volume, H min, H max)
template<>
void	fillBuffer< half_float::half > (void *buffer, int volume, half_float::half min, half_float::half max)
template<typename T >
void	dumpBuffer (const void *buffer, int volume, const std::string &separator, std::ostream &os)
bool	broadcastIOFormats (const std::vector< IOFormat > &formats, size_t nbBindings, bool isInput=true)

Variables
Logger	gLogger {Logger::Severity::kINFO}
LogStreamConsumer	gLogVerbose {LOG_VERBOSE(gLogger)}
LogStreamConsumer	gLogInfo {LOG_INFO(gLogger)}
LogStreamConsumer	gLogWarning {LOG_WARN(gLogger)}
LogStreamConsumer	gLogError {LOG_ERROR(gLogger)}
LogStreamConsumer	gLogFatal {LOG_FATAL(gLogger)}
constexpr int	defaultMaxBatch {1}
constexpr int	defaultWorkspace {16}
constexpr int	defaultMinTiming {1}
constexpr int	defaultAvgTiming {8}
constexpr int	defaultDevice {0}
constexpr int	defaultBatch {1}
constexpr int	defaultStreams {1}
constexpr int	defaultIterations {10}
constexpr int	defaultWarmUp {200}
constexpr int	defaultDuration {3}
constexpr int	defaultSleep {0}
constexpr int	defaultAvgRuns {10}
constexpr float	defaultPercentile {99}

Typedef Documentation

Severity

using sample::Severity = typedef nvinfer1::ILogger::Severity

TrtDeviceBuffer

using sample::TrtDeviceBuffer = typedef TrtCudaBuffer<DeviceAllocator, DeviceDeallocator>

TrtHostBuffer

using sample::TrtHostBuffer = typedef TrtCudaBuffer<HostAllocator, HostDeallocator>

Arguments

using sample::Arguments = typedef std::unordered_multimap<std::string, std::string>

IOFormat

using sample::IOFormat = typedef std::pair<nvinfer1::DataType, nvinfer1::TensorFormats>

ShapeRange

using sample::ShapeRange = typedef std::array<std::vector<int>, nvinfer1::EnumMax<nvinfer1::OptProfileSelector>()>

TrtUniquePtr

template<typename T >

using sample::TrtUniquePtr = typedef std::unique_ptr<T, TrtDestroyer<T> >

Enumeration Type Documentation

ModelFormat

enum sample::ModelFormat

strong

Enumerator
kANY
kCAFFE
kONNX
kUFF

Function Documentation

setReportableSeverity()

void sample::setReportableSeverity

(

Logger::Severity

severity

)

Here is the call graph for this function:

cudaCheck()

void sample::cudaCheck ( cudaError_t  ret, std::ostream &  err = std::cerr  )

inline

Here is the caller graph for this function:

setCudaDevice()

void sample::setCudaDevice ( int  device, std::ostream &  os  )

inline

Here is the call graph for this function:

modelToNetwork()

Parser sample::modelToNetwork	(	const ModelOptions &	model,
		nvinfer1::INetworkDefinition &	network,
		std::ostream &	err
	)

Generate a network definition for a given model.

Returns

Parser The parser used to initialize the network and that holds the weights for the network, or an invalid parser (the returned parser converts to false if tested)

See also

Parser::operator bool()

Here is the call graph for this function:

Here is the caller graph for this function:

networkToEngine()

nvinfer1::ICudaEngine * sample::networkToEngine	(	const BuildOptions &	build,
		const SystemOptions &	sys,
		nvinfer1::IBuilder &	builder,
		nvinfer1::INetworkDefinition &	network,
		std::ostream &	err
	)

Create an engine for a network defintion.

Returns

Pointer to the engine created or nullptr if the creation failed

Here is the call graph for this function:

Here is the caller graph for this function:

modelToEngine()

nvinfer1::ICudaEngine * sample::modelToEngine	(	const ModelOptions &	model,
		const BuildOptions &	build,
		const SystemOptions &	sys,
		std::ostream &	err
	)

Create an engine for a given model.

Returns

Pointer to the engine created or nullptr if the creation failed

Here is the call graph for this function:

Here is the caller graph for this function:

dumpRefittable()

void sample::dumpRefittable

(

nvinfer1::ICudaEngine &

engine

)

Log refittable layers and weights of a refittable engine.

Here is the call graph for this function:

loadEngine()

nvinfer1::ICudaEngine * sample::loadEngine	(	const std::string &	engine,
		int	DLACore,
		std::ostream &	err
	)

Load a serialized engine.

Returns

Pointer to the engine loaded or nullptr if the operation failed

Here is the call graph for this function:

Here is the caller graph for this function:

saveEngine()

bool sample::saveEngine	(	const nvinfer1::ICudaEngine &	engine,
		const std::string &	fileName,
		std::ostream &	err
	)

Save an engine into a file.

Returns

boolean Return true if the engine was successfully saved

Here is the call graph for this function:

Here is the caller graph for this function:

getEngine()

TrtUniquePtr< nvinfer1::ICudaEngine > sample::getEngine	(	const ModelOptions &	model,
		const BuildOptions &	build,
		const SystemOptions &	sys,
		std::ostream &	err
	)

Create an engine from model or serialized file, and optionally save engine.

Returns

Pointer to the engine created or nullptr if the creation failed

Here is the call graph for this function:

setUpInference()

bool sample::setUpInference	(	InferenceEnvironment &	iEnv,
		const InferenceOptions &	inference
	)

Set up contexts and bindings for inference.

Here is the call graph for this function:

runInference()

void sample::runInference	(	const InferenceOptions &	inference,
		InferenceEnvironment &	iEnv,
		int	device,
		std::vector< InferenceTrace > &	trace
	)

Run inference and collect timing.

argsToArgumentsMap()

Arguments sample::argsToArgumentsMap	(	int	argc,
		char *	argv[]
	)

parseHelp()

bool sample::parseHelp

(

Arguments &

arguments

)

Here is the caller graph for this function:

operator<<() [1/13]

std::ostream & sample::operator<<	(	std::ostream &	os,
		const BaseModelOptions &	options
	)

operator<<() [2/13]

std::ostream & sample::operator<<	(	std::ostream &	os,
		const UffInput &	input
	)

operator<<() [3/13]

std::ostream & sample::operator<<	(	std::ostream &	os,
		const ModelOptions &	options
	)

operator<<() [4/13]

std::ostream & sample::operator<<	(	std::ostream &	os,
		const IOFormat &	format
	)

operator<<() [5/13]

std::ostream & sample::operator<<	(	std::ostream &	os,
		const ShapeRange &	dims
	)

operator<<() [6/13]

std::ostream & sample::operator<<	(	std::ostream &	os,
		const BuildOptions &	options
	)

operator<<() [7/13]

std::ostream & sample::operator<<	(	std::ostream &	os,
		const SystemOptions &	options
	)

operator<<() [8/13]

std::ostream & sample::operator<<	(	std::ostream &	os,
		const InferenceOptions &	options
	)

operator<<() [9/13]

std::ostream & sample::operator<<	(	std::ostream &	os,
		const ReportingOptions &	options
	)

operator<<() [10/13]

std::ostream & sample::operator<<	(	std::ostream &	os,
		const AllOptions &	options
	)

helpHelp()

void sample::helpHelp

(

std::ostream &

os

)

Here is the call graph for this function:

Here is the caller graph for this function:

printProlog()

void sample::printProlog	(	int	warmups,
		int	timings,
		float	warmupMs,
		float	benchTimeMs,
		std::ostream &	os
	)

Print benchmarking time and number of traces collected.

Here is the call graph for this function:

Here is the caller graph for this function:

printTiming()

void sample::printTiming	(	const std::vector< InferenceTime > &	timings,
		int	runsPerAvg,
		std::ostream &	os
	)

Print a timing trace.

Here is the call graph for this function:

Here is the caller graph for this function:

printEpilog() [1/2]

void sample::printEpilog	(	std::vector< InferenceTime >	timings,
		float	walltimeMs,
		float	percentile,
		int	queries,
		std::ostream &	os
	)

Here is the call graph for this function:

Here is the caller graph for this function:

printPerformanceReport()

void sample::printPerformanceReport	(	const std::vector< InferenceTrace > &	trace,
		const ReportingOptions &	reporting,
		float	warmupMs,
		int	queries,
		std::ostream &	os
	)

Print and summarize a timing trace.

Here is the call graph for this function:

Here is the caller graph for this function:

exportJSONTrace()

void sample::exportJSONTrace	(	const std::vector< InferenceTrace > &	trace,
		const std::string &	fileName
	)

Printed format: [ value, ...] value ::= { "start enq : time, "end enq" : time, "start in" : time, "end in" : time, "start compute" : time, "end compute" : time, "start out" : time, "in" : time, "compute" : time, "out" : time, "latency" : time, "end to end" : time}.

Export a timing trace to JSON file.

Here is the call graph for this function:

Here is the caller graph for this function:

dumpInputs()

void sample::dumpInputs	(	const nvinfer1::IExecutionContext &	context,
		const Bindings &	bindings,
		std::ostream &	os
	)

Print input tensors to stream.

Here is the call graph for this function:

Here is the caller graph for this function:

dumpOutputs()

void sample::dumpOutputs	(	const nvinfer1::IExecutionContext &	context,
		const Bindings &	bindings,
		std::ostream &	os
	)

Print output tensors to stream.

Here is the call graph for this function:

Here is the caller graph for this function:

exportJSONOutput()

void sample::exportJSONOutput	(	const nvinfer1::IExecutionContext &	context,
		const Bindings &	bindings,
		const std::string &	fileName
	)

Export output tensors to JSON file.

Here is the call graph for this function:

Here is the caller graph for this function:

operator+()

InferenceTime sample::operator+ ( const InferenceTime &  a, const InferenceTime &  b  )

inline

operator+=()

InferenceTime sample::operator+= ( InferenceTime &  a, const InferenceTime &  b  )

inline

printEpilog() [2/2]

void sample::printEpilog	(	std::vector< InferenceTime >	timings,
		float	percentile,
		int	queries,
		std::ostream &	os
	)

Print the performance summary of a trace.

dataTypeSize()

int sample::dataTypeSize ( nvinfer1::DataType  dataType )

inline

Here is the caller graph for this function:

roundUp()

template<typename T >

T sample::roundUp ( T  m, T  n  )

inline

Here is the caller graph for this function:

volume() [1/3]

int sample::volume ( const nvinfer1::Dims &  d )

inline

Here is the caller graph for this function:

volume() [2/3]

int sample::volume ( const nvinfer1::Dims &  dims, const nvinfer1::Dims &  strides, int  vecDim, int  comps, int  batch  )

inline

Here is the call graph for this function:

volume() [3/3]

int sample::volume ( nvinfer1::Dims  dims, int  vecDim, int  comps, int  batch  )

inline

Here is the call graph for this function:

operator<<() [11/13]

std::ostream& sample::operator<< ( std::ostream &  os, const nvinfer1::Dims &  dims  )

inline

operator<<() [12/13]

std::ostream& sample::operator<< ( std::ostream &  os, const std::vector< int > &  vec  )

inline

operator<<() [13/13]

std::ostream& sample::operator<< ( std::ostream &  os, const nvinfer1::WeightsRole  role  )

inline

toDims()

nvinfer1::Dims sample::toDims ( const std::vector< int > &  vec )

inline

Here is the caller graph for this function:

fillBuffer()

template<typename T >

void sample::fillBuffer ( void *  buffer, int  volume, T  min, T  max  )

inline

Here is the call graph for this function:

fillBufferHalf()

template<typename H >

void sample::fillBufferHalf ( void *  buffer, int  volume, H  min, H  max  )

inline

Here is the call graph for this function:

Here is the caller graph for this function:

fillBuffer< half_float::half >()

template<>

void sample::fillBuffer< half_float::half > ( void *  buffer, int  volume, half_float::half  min, half_float::half  max  )

inline

Here is the call graph for this function:

dumpBuffer()

template<typename T >

void sample::dumpBuffer ( const void *  buffer, int  volume, const std::string &  separator, std::ostream &  os  )

inline

Here is the call graph for this function:

broadcastIOFormats()

bool sample::broadcastIOFormats ( const std::vector< IOFormat > &  formats, size_t  nbBindings, bool  isInput = true  )

inline

Here is the caller graph for this function:

Variable Documentation

gLogger

Logger sample::gLogger {Logger::Severity::kINFO}

gLogVerbose

LogStreamConsumer sample::gLogVerbose {LOG_VERBOSE(gLogger)}

gLogInfo

LogStreamConsumer sample::gLogInfo {LOG_INFO(gLogger)}

gLogWarning

LogStreamConsumer sample::gLogWarning {LOG_WARN(gLogger)}

gLogError

LogStreamConsumer sample::gLogError {LOG_ERROR(gLogger)}

gLogFatal

LogStreamConsumer sample::gLogFatal {LOG_FATAL(gLogger)}

defaultMaxBatch

constexpr int sample::defaultMaxBatch {1}

constexpr

defaultWorkspace

constexpr int sample::defaultWorkspace {16}

constexpr

defaultMinTiming

constexpr int sample::defaultMinTiming {1}

constexpr

defaultAvgTiming

constexpr int sample::defaultAvgTiming {8}

constexpr

defaultDevice

constexpr int sample::defaultDevice {0}

constexpr

defaultBatch

constexpr int sample::defaultBatch {1}

constexpr

defaultStreams

constexpr int sample::defaultStreams {1}

constexpr

defaultIterations

constexpr int sample::defaultIterations {10}

constexpr

defaultWarmUp

constexpr int sample::defaultWarmUp {200}

constexpr

defaultDuration

constexpr int sample::defaultDuration {3}

constexpr

defaultSleep

constexpr int sample::defaultSleep {0}

constexpr

defaultAvgRuns

constexpr int sample::defaultAvgRuns {10}

constexpr

defaultPercentile

constexpr float sample::defaultPercentile {99}

constexpr