13 #include <type_traits>
17 #define ORT_CXX_RETURN_ON_API_FAIL(expression) \
19 auto ort_status = (expression); \
21 return Ort::Status(ort_status); \
25 #ifdef __cpp_if_constexpr
26 #define ORT_CXX_IF_CONSTEXPR if constexpr
28 #define ORT_CXX_IF_CONSTEXPR if
54 inline Status::Status(OrtStatus* status) noexcept : Base<OrtStatus>{status} {
58 p_ =
GetApi().CreateStatus(ORT_FAIL, e.what());
62 p_ =
GetApi().CreateStatus(e.GetOrtErrorCode(), e.what());
79 return (
p_ ==
nullptr);
84 struct TypeToTensorType;
86 struct TypeToTensorType<
float> {
87 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT;
91 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16;
95 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16;
98 struct TypeToTensorType<double> {
99 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE;
102 struct TypeToTensorType<int8_t> {
103 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8;
106 struct TypeToTensorType<int16_t> {
107 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16;
110 struct TypeToTensorType<int32_t> {
111 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32;
114 struct TypeToTensorType<int64_t> {
115 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64;
118 struct TypeToTensorType<uint8_t> {
119 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8;
122 struct TypeToTensorType<uint16_t> {
123 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16;
126 struct TypeToTensorType<uint32_t> {
127 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32;
130 struct TypeToTensorType<uint64_t> {
131 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64;
134 struct TypeToTensorType<
bool> {
135 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL;
140 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FN;
144 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E4M3FNUZ;
148 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2;
152 static constexpr ONNXTensorElementDataType
type = ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT8E5M2FNUZ;
156 if (IsNaN() || rhs.IsNaN()) {
160 return val == rhs.val;
164 if (IsNaN() || rhs.IsNaN()) {
169 const bool left_is_negative = IsNegative();
170 if (left_is_negative != rhs.IsNegative()) {
174 return left_is_negative && !AreZero(*
this, rhs);
176 return (
val != rhs.val) && ((
val < rhs.val) ^ left_is_negative);
180 : allocator_(allocator), p_(p), size_(size) {
186 auto ret =
GetApi().AllocatorFree(allocator_, p_);
187 static_cast<void>(ret);
192 *
this = std::move(o);
196 OrtAllocator* alloc =
nullptr;
217 template <
typename T>
224 template <
typename T>
232 template <
typename T>
237 template <
typename T>
256 template <
typename T>
258 const char*
name =
nullptr;
260 return std::string(name);
263 template <
typename T>
265 OrtAllocatorType
type;
270 template <
typename T>
277 template <
typename T>
279 OrtMemoryInfoDeviceType
type;
280 GetApi().MemoryInfoGetDeviceType(this->p_, &type);
284 template <
typename T>
291 template <
typename T>
292 template <
typename U>
296 return comp_result == 0;
312 template <
typename T>
318 template <
typename T>
323 template <
typename T>
329 template <
typename T>
334 template <
typename T>
339 template <
typename T>
344 template <
typename T>
349 template <
typename T>
351 GetApi().ClearBoundInputs(this->p_);
354 template <
typename T>
356 GetApi().ClearBoundOutputs(this->p_);
359 template <
typename T>
364 template <
typename T>
369 namespace binding_utils {
371 std::vector<std::string>
result;
373 using Ptr = std::unique_ptr<void, decltype(free_fn)>;
378 ThrowOnError(
GetApi().GetBoundOutputNames(binding, allocator, &buffer, &lengths, &count));
384 Ptr buffer_g(buffer, free_fn);
385 Ptr lengths_g(lengths, free_fn);
387 result.reserve(count);
388 for (
size_t i = 0; i < count; ++i) {
390 result.emplace_back(buffer, sz);
398 std::vector<Value>
result;
400 size_t output_count = 0;
403 auto free_fn = [&owned, &output_count, allocator](
OrtValue**
buffer) {
405 while (owned < output_count) {
406 auto* p =
buffer + owned++;
407 GetApi().ReleaseValue(*p);
409 allocator->Free(allocator,
buffer);
412 using Ptr = std::unique_ptr<OrtValue*, decltype(free_fn)>;
415 ThrowOnError(
GetApi().GetBoundOutputValues(binding, allocator, &output_buffer, &output_count));
416 if (output_count == 0) {
420 Ptr buffer_g(output_buffer, free_fn);
422 result.reserve(output_count);
423 for (
size_t i = 0; i < output_count; ++i) {
424 result.emplace_back(output_buffer[i]);
437 inline ArenaCfg::ArenaCfg(
size_t max_mem,
int arena_extend_strategy,
int initial_chunk_size_bytes,
int max_dead_bytes_per_chunk) {
438 ThrowOnError(
GetApi().CreateArenaCfg(max_mem, arena_extend_strategy, initial_chunk_size_bytes, max_dead_bytes_per_chunk, &
p_));
480 inline Env::Env(OrtLoggingLevel logging_level, _In_
const char* logid) {
482 if (strcmp(logid,
"onnxruntime-node") == 0) {
485 ThrowOnError(
GetApi().SetLanguageProjection(
p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));
489 inline Env::Env(OrtLoggingLevel logging_level,
const char* logid, OrtLoggingFunction logging_function,
void* logger_param) {
490 ThrowOnError(
GetApi().CreateEnvWithCustomLogger(logging_function, logger_param, logging_level, logid, &
p_));
491 if (strcmp(logid,
"onnxruntime-node") == 0) {
494 ThrowOnError(
GetApi().SetLanguageProjection(
p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));
498 inline Env::Env(
const OrtThreadingOptions* tp_options, OrtLoggingLevel logging_level, _In_
const char* logid) {
500 if (strcmp(logid,
"onnxruntime-node") == 0) {
503 ThrowOnError(
GetApi().SetLanguageProjection(
p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));
507 inline Env::Env(
const OrtThreadingOptions* tp_options, OrtLoggingFunction logging_function,
void* logger_param,
508 OrtLoggingLevel logging_level, _In_
const char* logid) {
509 ThrowOnError(
GetApi().CreateEnvWithCustomLoggerAndGlobalThreadPools(logging_function, logger_param, logging_level, logid, tp_options, &
p_));
510 if (strcmp(logid,
"onnxruntime-node") == 0) {
513 ThrowOnError(
GetApi().SetLanguageProjection(
p_, OrtLanguageProjection::ORT_PROJECTION_CPLUSPLUS));
517 inline Env& Env::EnableTelemetryEvents() {
538 std::vector<const char*> keys,
values;
539 auto num_entries = options.size();
540 if (num_entries > 0) {
541 keys.reserve(num_entries);
542 values.reserve(num_entries);
543 for (
const auto& entry : options) {
544 keys.push_back(entry.first.c_str());
545 values.push_back(entry.second.c_str());
561 OrtAllocator* allocator) {
568 OrtAllocator* allocator) {
633 template <
typename T>
635 OrtSessionOptions* out;
640 template <
typename T>
649 out.resize(size - 1);
654 template <
typename T>
658 return static_cast<bool>(out);
661 template <
typename T>
663 if (!this->HasConfigEntry(config_key)) {
667 return this->GetConfigEntry(config_key);
670 template <
typename T>
676 template <
typename T>
682 template <
typename T>
684 ThrowOnError(
GetApi().SetSessionGraphOptimizationLevel(this->p_, graph_optimization_level));
688 template <
typename T>
694 template <
typename T>
700 template <
typename T>
706 template <
typename T>
712 template <
typename T>
718 template <
typename T>
724 template <
typename T>
730 template <
typename T>
736 template <
typename T>
742 template <
typename T>
748 template <
typename T>
754 template <
typename T>
760 template <
typename T>
766 template <
typename T>
772 template <
typename T>
778 template <
typename T>
784 template <
typename T>
786 const std::vector<Value>& ort_values) {
787 const size_t inputs_num = names.size();
788 if (inputs_num != ort_values.size()) {
789 ORT_CXX_API_THROW(
"Expecting names and ort_values to have the same length", ORT_INVALID_ARGUMENT);
791 std::vector<const char*> names_ptr;
792 std::vector<const OrtValue*> ort_values_ptrs;
793 names_ptr.reserve(inputs_num);
794 ort_values_ptrs.reserve(inputs_num);
795 for (
size_t i = 0; i < inputs_num; ++i) {
796 names_ptr.push_back(names[i].
c_str());
797 ort_values_ptrs.push_back(ort_values[i]);
799 ThrowOnError(
GetApi().AddExternalInitializers(this->p_, names_ptr.data(), ort_values_ptrs.data(), inputs_num));
803 template <
typename T>
805 const std::vector<char*>& buffer_array,
806 const std::vector<size_t>& file_lengths) {
807 const size_t inputs_num = file_names.size();
808 if (inputs_num != buffer_array.size()) {
809 ORT_CXX_API_THROW(
"Expecting names and buffer_array to have the same length", ORT_INVALID_ARGUMENT);
811 if (inputs_num != file_lengths.size()) {
812 ORT_CXX_API_THROW(
"Expecting names and file_lengths to have the same length", ORT_INVALID_ARGUMENT);
814 std::vector<const ORTCHAR_T*> names_ptr;
815 names_ptr.reserve(inputs_num);
816 for (
size_t i = 0; i < inputs_num; ++i) {
817 names_ptr.push_back(file_names[i].
c_str());
819 ThrowOnError(
GetApi().AddExternalInitializersFromFilesInMemory(this->p_, names_ptr.data(), buffer_array.data(),
820 file_lengths.data(), inputs_num));
824 template <
typename T>
826 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_CUDA(this->p_, &provider_options));
830 template <
typename T>
832 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_CUDA_V2(this->p_, &provider_options));
836 template <
typename T>
838 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_ROCM(this->p_, &provider_options));
842 template <
typename T>
844 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_TensorRT(this->p_, &provider_options));
848 template <
typename T>
850 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_TensorRT_V2(this->p_, &provider_options));
854 template <
typename T>
856 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_MIGraphX(this->p_, &provider_options));
860 template <
typename T>
862 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_CANN(this->p_, &provider_options));
866 template <
typename T>
868 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_Dnnl(this->p_, &provider_options));
872 template <
typename T>
874 const std::string& provider_name,
875 const std::unordered_map<std::string, std::string>& provider_options) {
876 auto num_entries = provider_options.size();
877 std::vector<const char*> keys,
values;
878 if (num_entries > 0) {
879 keys.reserve(num_entries);
880 values.reserve(num_entries);
882 for (
const auto& entry : provider_options) {
883 keys.push_back(entry.first.c_str());
884 values.push_back(entry.second.c_str());
888 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider(this->p_, provider_name.c_str(),
889 keys.data(), values.data(), num_entries));
894 template <
typename T>
896 ThrowOnError(
GetApi().SessionOptionsSetCustomCreateThreadFn(this->p_, ort_custom_create_thread_fn));
900 template <
typename T>
902 ThrowOnError(
GetApi().SessionOptionsSetCustomThreadCreationOptions(this->p_, ort_custom_thread_creation_options));
906 template <
typename T>
908 ThrowOnError(
GetApi().SessionOptionsSetCustomJoinThreadFn(this->p_, ort_custom_join_thread_fn));
912 template <
typename T>
914 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_OpenVINO(this->p_, &provider_options));
918 template <
typename T>
920 auto num_entries = provider_options.size();
921 std::vector<const char*> keys,
values;
922 if (num_entries > 0) {
923 keys.reserve(num_entries);
924 values.reserve(num_entries);
926 for (
const auto& entry : provider_options) {
927 keys.push_back(entry.first.c_str());
928 values.push_back(entry.second.c_str());
933 keys.data(), values.data(), num_entries));
938 template <
typename T>
940 auto num_entries = provider_options.size();
941 std::vector<const char*> keys,
values;
942 if (num_entries > 0) {
943 keys.reserve(num_entries);
944 values.reserve(num_entries);
946 for (
const auto& entry : provider_options) {
947 keys.push_back(entry.first.c_str());
948 values.push_back(entry.second.c_str());
952 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_VitisAI(this->p_, keys.data(), values.data(), num_entries));
957 template <
typename T>
963 AddConfigEntry(config_iter.first.c_str(), config_iter.second.c_str());
970 template <
typename T>
972 ThrowOnError(
GetApi().RegisterCustomOpsUsingFunction(this->p_, registration_function_name));
977 template <
typename T>
984 template <
typename T>
991 template <
typename T>
998 template <
typename T>
1005 template <
typename T>
1012 template <
typename T>
1015 ThrowOnError(
GetApi().SessionGetOverridableInitializerName(this->p_, index, allocator, &out));
1019 template <
typename T>
1026 template <
typename T>
1028 OrtModelMetadata* out;
1033 template <
typename T>
1040 template <
typename T>
1047 template <
typename T>
1050 ThrowOnError(
GetApi().SessionGetOverridableInitializerTypeInfo(this->p_, index, &out));
1054 template <
typename T>
1056 const char*
const* output_names,
size_t output_count) {
1057 std::vector<Value> output_values;
1058 output_values.reserve(output_count);
1059 for (
size_t i = 0; i < output_count; i++)
1060 output_values.emplace_back(
nullptr);
1061 Run(run_options, input_names, input_values, input_count, output_names, output_values.data(), output_count);
1062 return output_values;
1065 template <
typename T>
1067 const char*
const* output_names,
Value* output_values,
size_t output_count) {
1068 static_assert(
sizeof(
Value) ==
sizeof(
OrtValue*),
"Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely");
1069 auto ort_input_values =
reinterpret_cast<const OrtValue* const*
>(input_values);
1070 auto ort_output_values =
reinterpret_cast<OrtValue**
>(output_values);
1071 ThrowOnError(
GetApi().Run(this->p_, run_options, input_names, ort_input_values, input_count, output_names, output_count, ort_output_values));
1074 template <
typename T>
1079 template <
typename T>
1081 const char*
const* output_names,
Value* output_values,
size_t output_count, RunAsyncCallbackFn callback,
void* user_data) {
1082 auto ort_input_values =
reinterpret_cast<const OrtValue* const*
>(input_values);
1083 auto ort_output_values =
reinterpret_cast<OrtValue**
>(output_values);
1085 ort_input_values, input_count, output_names, output_count,
1086 ort_output_values, callback, user_data));
1089 template <
typename T>
1091 char* out =
nullptr;
1096 template <
typename T>
1109 std::string config_key =
"custom_op.";
1111 config_key += custom_op_name;
1113 config_key += config;
1120 flat_configs_[full_flat_key] = config_value;
1125 return flat_configs_;
1133 OrtPrepackedWeightsContainer* prepacked_weights_container) {
1134 ThrowOnError(
GetApi().CreateSessionWithPrepackedWeightsContainer(env, model_path, options, prepacked_weights_container, &this->
p_));
1138 ThrowOnError(
GetApi().CreateSessionFromArray(env, model_data, model_data_length, options, &this->
p_));
1142 const SessionOptions& options, OrtPrepackedWeightsContainer* prepacked_weights_container) {
1143 ThrowOnError(
GetApi().CreateSessionFromArrayWithPrepackedWeightsContainer(env, model_data, model_data_length, options,
1144 prepacked_weights_container, &this->
p_));
1147 inline AllocatedStringPtr ModelMetadata::GetProducerNameAllocated(OrtAllocator* allocator)
const {
1185 std::vector<AllocatedStringPtr>
result;
1187 char** out =
nullptr;
1188 int64_t num_keys = 0;
1190 if (num_keys <= 0) {
1195 std::unique_ptr<void, decltype(deletor)> array_guard(out, deletor);
1197 auto strings_deletor = [&deletor, num_keys](
char** out) {
for(int64_t i = 0; i < num_keys; ++i) deletor(out[i]); };
1198 std::unique_ptr<char*, decltype(strings_deletor)> strings_guard(out, strings_deletor);
1199 result.reserve(static_cast<size_t>(num_keys));
1200 strings_guard.release();
1201 for (int64_t i = 0; i < num_keys; ++i) {
1216 template <
typename T>
1218 ONNXTensorElementDataType out;
1223 template <
typename T>
1227 return static_cast<size_t>(out);
1230 template <
typename T>
1237 template <
typename T>
1242 template <
typename T>
1247 template <
typename T>
1249 std::vector<int64_t> out(GetDimensionsCount(), 0);
1254 template <
typename T>
1256 const OrtTensorTypeAndShapeInfo* out;
1261 template <
typename T>
1263 const OrtSequenceTypeInfo* out;
1268 template <
typename T>
1270 const OrtMapTypeInfo* out;
1275 template <
typename T>
1282 template <
typename T>
1284 OrtTypeInfo* output;
1289 template <
typename T>
1296 template <
typename T>
1298 ONNXTensorElementDataType out;
1303 template <
typename T>
1305 OrtTypeInfo* output;
1310 template <
typename T>
1312 const OrtOptionalTypeInfo* info;
1321 template <
typename T>
1322 template <
typename R>
1327 template <
typename T>
1334 template <
typename T>
1341 template <
typename T>
1348 template <
typename T>
1355 template <
typename T>
1362 template <
typename T>
1369 template <
typename T>
1370 template <
typename R>
1373 ThrowOnError(
GetApi().GetTensorMutableData(const_cast<OrtValue*>(this->p_), (
void**)&out));
1377 template <
typename T>
1384 template <
typename T>
1386 OrtTypeInfo* output;
1391 template <
typename T>
1393 OrtTensorTypeAndShapeInfo* output;
1398 template <
typename T>
1405 template <
typename T>
1407 ThrowOnError(
GetApi().GetStringTensorElement(this->p_, buffer_length, element_index, buffer));
1410 template <
typename T>
1412 size_t buffer_length;
1413 ThrowOnError(
GetApi().GetStringTensorElementLength(this->p_, element_index, &buffer_length));
1416 s.resize(buffer_length);
1417 ThrowOnError(
GetApi().GetStringTensorElement(this->p_, buffer_length, element_index, &s[0]));
1421 template <
typename T>
1423 ThrowOnError(
GetApi().GetStringTensorContent(this->p_, buffer, buffer_length, offsets, offsets_count));
1426 #if !defined(DISABLE_SPARSE_TENSORS)
1427 template <
typename T>
1434 template <
typename T>
1436 OrtTensorTypeAndShapeInfo* output;
1441 template <
typename T>
1443 OrtTensorTypeAndShapeInfo* output;
1444 ThrowOnError(
GetApi().GetSparseTensorIndicesTypeShape(this->p_, indices_format, &output));
1448 template <
typename T>
1449 template <
typename R>
1452 ThrowOnError(
GetApi().GetSparseTensorIndices(this->p_, indices_format, &num_indices, &out));
1453 return reinterpret_cast<const R*
>(out);
1456 template <
typename T>
1463 template <
typename T>
1464 template <
typename R>
1468 return reinterpret_cast<const R*
>(out);
1473 template <
typename T>
1478 template <
typename T>
1483 template <
typename T>
1486 ThrowOnError(
GetApi().GetResizedStringTensorElementBuffer(this->p_, index, buffer_length, &result));
1490 template <
typename T>
1497 template <
typename T>
1498 template <
typename R>
1505 template <
typename T>
1506 template <
typename R>
1510 ThrowOnError(
GetApi().TensorAt(this->p_, location.data(), location.size(), (
void**)&out));
1514 #if !defined(DISABLE_SPARSE_TENSORS)
1515 template <
typename T>
1520 template <
typename T>
1522 ThrowOnError(
GetApi().UseCsrIndices(this->p_, inner_data, inner_num, outer_data, outer_num));
1525 template <
typename T>
1530 template <
typename T>
1532 const int64_t* indices_data,
size_t indices_num) {
1535 indices_data, indices_num));
1538 template <
typename T>
1541 const int64_t* inner_indices_data,
size_t inner_indices_num,
1542 const int64_t* outer_indices_data,
size_t outer_indices_num) {
1544 inner_indices_data, inner_indices_num,
1545 outer_indices_data, outer_indices_num));
1548 template <
typename T>
1551 const Shape& indices_shape,
1552 const int32_t* indices_data) {
1558 #endif // !defined(DISABLE_SPARSE_TENSORS)
1562 template <
typename T>
1568 ONNXTensorElementDataType type) {
1570 ThrowOnError(
GetApi().CreateTensorWithDataAsOrtValue(info, p_data, p_data_byte_count, shape, shape_len, type, &out));
1574 template <
typename T>
1581 ThrowOnError(
GetApi().CreateTensorAsOrtValue(allocator, shape, shape_len, type, &out));
1585 #if !defined(DISABLE_SPARSE_TENSORS)
1587 template <
typename T>
1589 const Shape& values_shape) {
1594 const Shape& values_shape, ONNXTensorElementDataType type) {
1601 template <
typename T>
1607 ONNXTensorElementDataType type) {
1612 #endif // !defined(DISABLE_SPARSE_TENSORS)
1616 const OrtValue* inputs[2] = {keys, values};
1623 std::vector<const OrtValue*> values_ort{values.data(), values.data() + values.size()};
1624 ThrowOnError(
GetApi().CreateValue(values_ort.data(), values_ort.size(), ONNX_TYPE_SEQUENCE, &out));
1628 template <
typename T>
1631 ThrowOnError(
GetApi().CreateOpaqueValue(domain, type_name, &data_container,
sizeof(
T), &out));
1643 return cached_severity_level_;
1647 const char* func_name,
const char*
message)
const noexcept {
1648 OrtStatus* status =
GetApi().Logger_LogMessage(logger_, log_severity_level,
message, file_path, line_number,
1656 #if defined(__GNUC__)
1657 #pragma GCC diagnostic push
1658 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
1659 #pragma GCC diagnostic ignored "-Wformat-security"
1660 #elif defined(__clang__)
1661 #pragma clang diagnostic push
1662 #pragma clang diagnostic ignored "-Wformat-nonliteral"
1663 #pragma clang diagnostic ignored "-Wformat-security"
1665 template <
typename... Args>
1667 int line_number,
const char* func_name,
const char*
format,
1668 Args&&...
args) const noexcept {
1669 int msg_len = std::snprintf(
nullptr, 0U, format, std::forward<Args>(
args)...);
1672 return Status(
"Failed to log message due to formatting error", OrtErrorCode::ORT_FAIL);
1675 OrtStatus* status =
nullptr;
1676 const size_t buffer_size =
static_cast<size_t>(msg_len) + 1U;
1678 constexpr
size_t kStackBufferSize = 1024;
1680 if (buffer_size < kStackBufferSize) {
1681 char buffer[kStackBufferSize];
1682 snprintf(buffer, kStackBufferSize, format, std::forward<Args>(
args)...);
1683 status =
GetApi().Logger_LogMessage(logger_, log_severity_level, buffer, file_path, line_number, func_name);
1686 #if (__cplusplus >= 201402L) || (_MSC_VER >= 1900)
1687 auto buffer = std::make_unique<char[]>(buffer_size);
1689 std::unique_ptr<char[]>
buffer(
new char[buffer_size]);
1691 std::snprintf(buffer.get(), buffer_size,
format, std::forward<Args>(
args)...);
1692 status =
GetApi().Logger_LogMessage(logger_, log_severity_level, buffer.get(), file_path, line_number, func_name);
1698 #if defined(__GNUC__)
1699 #pragma GCC diagnostic pop
1700 #elif defined(__clang__)
1701 #pragma clang diagnostic pop
1738 void* out =
nullptr;
1744 OrtAllocator* out =
nullptr;
1750 const OrtLogger* out =
nullptr;
1756 ThrowOnError(
GetApi().KernelContext_ParallelFor(ctx_, fn, total, num_batch, usr_data));
1764 template <
typename T>
1766 OrtKernelInfo* info_copy =
nullptr;
1771 template <
typename T>
1778 template <
typename T>
1785 template <
typename T>
1795 out.resize(size - 1);
1800 template <
typename T>
1810 out.resize(size - 1);
1815 template <
typename T>
1817 OrtTypeInfo* out =
nullptr;
1822 template <
typename T>
1824 OrtTypeInfo* out =
nullptr;
1829 template <
typename T>
1832 ThrowOnError(
GetApi().KernelInfoGetAttribute_tensor(this->p_, name, allocator, &out));
1836 template <
typename T>
1839 ThrowOnError(
GetApi().KernelInfoGetConstantInput_tensor(this->p_, index, is_constant, &out));
1843 template <
typename T>
1853 out.resize(size - 1);
1858 template <
typename T>
1860 const OrtLogger* out =
nullptr;
1881 out.resize(size - 1);
1890 std::vector<float> out;
1902 std::vector<int64_t> out;
1911 inline Op::Op(OrtOp* p) : Base<OrtOp>(p) {}
1913 inline Op Op::Create(
const OrtKernelInfo* info,
const char* op_name,
const char* domain,
int version,
1914 const char** type_constraint_names,
1915 const ONNXTensorElementDataType* type_constraint_values,
1916 size_t type_constraint_count,
1917 const OpAttr* attr_values,
size_t attr_count,
1918 size_t input_count,
size_t output_count) {
1919 static_assert(
sizeof(OpAttr) ==
sizeof(OrtOpAttr*),
1920 "OpAttr's is expected to be just an array of OrtOpAttr in memory so we can reinterpret safely");
1921 auto attr_input_values =
reinterpret_cast<const OrtOpAttr* const*
>(attr_values);
1923 Ort::ThrowOnError(
GetApi().CreateOp(info, op_name, domain, version, type_constraint_names, type_constraint_values,
1924 static_cast<int>(type_constraint_count),
1926 static_cast<int>(attr_count),
1927 static_cast<int>(input_count),
1928 static_cast<int>(output_count), &op));
1932 inline void Op::Invoke(
const OrtKernelContext* context,
1933 const Value* input_values,
1935 Value* output_values,
1936 size_t output_count) {
1938 "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely");
1939 auto ort_input_values =
reinterpret_cast<const OrtValue* const*
>(input_values);
1940 auto ort_output_values =
reinterpret_cast<OrtValue**
>(output_values);
1942 ort_output_values, static_cast<int>(output_count)));
1945 inline void Op::Invoke(
const OrtKernelContext* context,
1946 const OrtValue*
const* input_values,
1949 size_t output_count) {
1951 output_values, static_cast<int>(output_count)));
1955 return OrtGetApiBase()->GetVersionString();
1959 return GetApi().GetBuildInfoString();
1966 auto release_fn = [&len](
char** providers) {
1972 std::unique_ptr<char*, decltype(release_fn)> guard(providers, release_fn);
1973 std::vector<std::string> available_providers;
1974 available_providers.reserve(static_cast<size_t>(len));
1975 for (
int i = 0; i < len; ++i) {
1976 available_providers.emplace_back(providers[i]);
1978 return available_providers;
1981 template <
typename TOp,
typename TKernel,
bool WithStatus>
1984 const TOp* derived =
static_cast<const TOp*
>(
this);
1985 std::vector<std::string> keys = derived->GetSessionConfigKeys();
1987 out.reserve(keys.size());
1990 const size_t prefix_size = config_entry_key.length();
1992 for (
const auto& key : keys) {
1993 config_entry_key.resize(prefix_size);
1994 config_entry_key.append(key);
1995 out[key] = options.GetConfigEntryOrDefault(config_entry_key.c_str(),
"");
2000 OrtShapeInferContext* ctx) : ort_api_(ort_api),
ctx_(ctx) {
2001 size_t input_count = 0;
2003 for (
size_t ith_input = 0; ith_input < input_count; ++ith_input) {
2004 OrtTensorTypeAndShapeInfo* info{};
2005 Ort::ThrowOnError(ort_api_->ShapeInferContext_GetInputTypeShape(ctx, ith_input, &info));
2007 auto integer_shape = type_shape_info.GetShape();
2008 std::vector<const char*> symbolic_shape(integer_shape.size(), {});
2009 if (!integer_shape.empty()) {
2010 type_shape_info.GetSymbolicDimensions(&symbolic_shape[0], integer_shape.size());
2013 for (
size_t ith = 0; ith < integer_shape.size(); ++ith) {
2014 if (symbolic_shape[ith] && std::string{symbolic_shape[ith]}.size() > 0) {
2015 shape.emplace_back(symbolic_shape[ith]);
2017 shape.emplace_back(integer_shape[ith]);
2020 input_shapes_.push_back(std::move(shape));
2021 type_shape_info.release();
2025 inline Status ShapeInferContext::SetOutputShape(
size_t indice,
const Shape& shape, ONNXTensorElementDataType type) {
2026 OrtTensorTypeAndShapeInfo* info = {};
2030 using InfoPtr = std::unique_ptr<OrtTensorTypeAndShapeInfo, std::function<void(OrtTensorTypeAndShapeInfo*)>>;
2032 InfoPtr
info_ptr(info, [
this](OrtTensorTypeAndShapeInfo* obj) {
2033 ort_api_->ReleaseTensorTypeAndShapeInfo(obj);
2036 std::vector<int64_t> integer_dims;
2037 std::vector<const char*> symbolic_dims;
2039 for (
const auto dim : shape) {
2041 integer_dims.push_back(dim.AsInt());
2042 symbolic_dims.push_back(
"");
2044 if (!dim.AsSym() || std::string{dim.AsSym()}.empty()) {
2045 ORT_CXX_API_THROW(
"Symbolic dim must not be an empty string", ORT_INVALID_ARGUMENT);
2047 integer_dims.push_back(SymbolicInteger::INVALID_INT_DIM);
2048 symbolic_dims.push_back(dim.AsSym());
2058 inline int64_t ShapeInferContext::GetAttrInt(
const char* attr_name) {
2059 const auto* attr = GetAttrHdl(attr_name);
2062 Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_INT, &i,
sizeof(i), &out));
2067 const auto* attr = GetAttrHdl(attr_name);
2074 auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_INTS, &i,
sizeof(i), &out);
2076 size_t num_i = out /
sizeof(int64_t);
2078 Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_INTS, ints.data(), out, &out));
2088 inline float ShapeInferContext::GetAttrFloat(
const char* attr_name) {
2089 const auto* attr = GetAttrHdl(attr_name);
2092 Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_FLOAT, &f,
sizeof(f), &out));
2097 const auto* attr = GetAttrHdl(attr_name);
2104 auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_FLOATS, &f,
sizeof(f), &out);
2106 size_t num_f = out /
sizeof(
float);
2108 Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_FLOATS, floats.data(), out, &out));
2118 inline std::string ShapeInferContext::GetAttrString(
const char* attr_name) {
2119 const auto* attr = GetAttrHdl(attr_name);
2123 auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRING, &c,
sizeof(
char), &out);
2125 std::vector<char> chars(out,
'\0');
2126 Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRING, chars.data(), out, &out));
2127 return {chars.data()};
2134 const auto* attr = GetAttrHdl(attr_name);
2141 auto status = ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRINGS, &c,
sizeof(
char), &out);
2143 std::vector<char> chars(out,
'\0');
2144 Ort::ThrowOnError(ort_api_->ReadOpAttr(attr, ORT_OP_ATTR_STRINGS, chars.data(), out, &out));
2146 char* char_st = chars.data();
2147 char* char_ed = char_st + out;
2148 while (char_st < char_ed) {
2149 strings.emplace_back(char_st);
2150 while (*char_st !=
'\0') {
2160 return {std::string{c}};
2164 inline const OrtOpAttr* ShapeInferContext::GetAttrHdl(
const char* attr_name)
const {
2165 const OrtOpAttr* attr_hdl = {};
OrtMemType GetMemoryType() const
OrtMemoryInfoDeviceType GetDeviceType() const
Status LogFormattedMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T *file_path, int line_number, const char *func_name, const char *format, Args &&...args) const noexcept
std::vector< int64_t > Ints
void Invoke(const OrtKernelContext *context, const Value *input_values, size_t input_count, Value *output_values, size_t output_count)
std::string GetBuildInfoString()
This function returns the onnxruntime build information: including git branch, git commit id...
SessionOptionsImpl & SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn)
Wraps OrtApi::SessionOptionsSetCustomJoinThreadFn.
GLuint GLsizei const GLchar * message
size_t GetElementCount() const
Wraps OrtApi::GetTensorShapeElementCount.
ONNXType GetONNXType() const
MemoryAllocation & operator=(const MemoryAllocation &)=delete
Env & DisableTelemetryEvents()
Wraps OrtApi::EnableTelemetryEvents.
AllocatedStringPtr GetOverridableInitializerNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of the overridable initializer name at then specified index.
ThreadingOptions & SetGlobalSpinControl(int allow_spinning)
Wraps OrtApi::SetGlobalSpinControl.
OrtAllocator * GetAllocator(const OrtMemoryInfo &memory_info) const
std::string GetErrorMessage() const
png_const_structrp png_const_inforp info_ptr
size_t GetInputCount() const
Returns the number of model inputs.
SessionOptionsImpl & DisablePerSessionThreads()
Wraps OrtApi::DisablePerSessionThreads.
std::vector< std::string > Strings
Env & UpdateEnvWithCustomLogLevel(OrtLoggingLevel log_severity_level)
Wraps OrtApi::UpdateEnvWithCustomLogLevel.
void UseBlockSparseIndices(const Shape &indices_shape, int32_t *indices_data)
Supplies BlockSparse format specific indices and marks the contained sparse tensor as being a BlockSp...
RunOptions & AddConfigEntry(const char *config_key, const char *config_value)
Wraps OrtApi::AddRunConfigEntry.
ConstMapTypeInfo GetMapTypeInfo() const
Wraps OrtApi::CastTypeInfoToMapTypeInfo.
void FillSparseTensorCsr(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values, const int64_t *inner_indices_data, size_t inner_indices_num, const int64_t *outer_indices_data, size_t outer_indices_num)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
TypeInfo GetInputTypeInfo(size_t index) const
Wraps OrtApi::SessionGetInputTypeInfo.
size_t GetOutputCount() const
void GetSymbolicDimensions(const char **values, size_t values_count) const
Wraps OrtApi::GetSymbolicDimensions.
Type information that may contain either TensorTypeAndShapeInfo or the information about contained se...
SessionOptionsImpl & EnableMemPattern()
Wraps OrtApi::EnableMemPattern.
SessionOptionsImpl & EnableCpuMemArena()
Wraps OrtApi::EnableCpuMemArena.
static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1)
std::vector< float > Floats
bool IsTensor() const
Returns true if Value is a tensor, false for other types like map/sequence/etc.
AllocatedStringPtr GetOutputNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of output name at then specified index.
void GetOpaqueData(const char *domain, const char *type_name, R &) const
Obtains a pointer to a user defined data for experimental purposes
const void * GetTensorRawData() const
Returns a non-typed pointer to a tensor contained data.
ConstMemoryInfo GetInfo() const
void GetStringTensorElement(size_t buffer_length, size_t element_index, void *buffer) const
The API copies UTF-8 encoded bytes for the requested string element contained within a tensor or a sp...
SessionOptionsImpl & SetLogId(const char *logid)
Wraps OrtApi::SetSessionLogId.
void swap(UT::ArraySet< Key, MULTI, MAX_LOAD_FACTOR_256, Clearer, Hash, KeyEqual > &a, UT::ArraySet< Key, MULTI, MAX_LOAD_FACTOR_256, Clearer, Hash, KeyEqual > &b)
Value GetValue(int index, OrtAllocator *allocator) const
uint64_t GetProfilingStartTimeNs() const
Wraps OrtApi::SessionGetProfilingStartTimeNs.
SessionOptionsImpl & AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_CANN.
UnownedValue GetOutput(size_t index, const int64_t *dim_values, size_t dim_count) const
static Value CreateSequence(const std::vector< Value > &values)
Creates an OrtValue with a Sequence Onnx type representation. The API would ref-count the supplied Or...
void ThrowOnError(OrtStatus *ort_status)
GLsizei const GLfloat * value
void BindInput(const char *name, const Value &)
ONNXTensorElementDataType GetMapKeyType() const
Wraps OrtApi::GetMapKeyType.
ConstValue GetInput(size_t index) const
SessionOptionsImpl & AppendExecutionProvider_CANN(const OrtCANNProviderOptions &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_Dnnl.
MemoryAllocation GetAllocation(size_t size)
std::unique_ptr< char, detail::AllocatedFree > AllocatedStringPtr
unique_ptr typedef used to own strings allocated by OrtAllocators and release them at the end of the ...
SessionOptionsImpl & DisableCpuMemArena()
Wraps OrtApi::DisableCpuMemArena.
bool HasConfigEntry(const char *config_key) const
Wraps OrtApi::HasSessionConfigEntry.
void UseCsrIndices(int64_t *inner_data, size_t inner_num, int64_t *outer_data, size_t outer_num)
Supplies CSR format specific indices and marks the contained sparse tensor as being a CSR format tens...
ONNXTensorElementDataType GetElementType() const
Wraps OrtApi::GetTensorElementType.
static Value CreateOpaque(const char *domain, const char *type_name, const T &value)
Creates an OrtValue wrapping an Opaque type. This is used for experimental support of non-tensor type...
Env & CreateAndRegisterAllocator(const OrtMemoryInfo *mem_info, const OrtArenaCfg *arena_cfg)
Wraps OrtApi::CreateAndRegisterAllocator.
void FillStringTensorElement(const char *s, size_t index)
Set a single string in a string tensor
union Ort::detail::OrtSparseValuesParam::@166 data
std::vector< SymbolicInteger > Shape
std::vector< Value > Run(const RunOptions &run_options, const char *const *input_names, const Value *input_values, size_t input_count, const char *const *output_names, size_t output_count)
Run the model returning results in an Ort allocated vector.
SessionOptionsImpl & AddInitializer(const char *name, const OrtValue *ort_val)
Wraps OrtApi::AddInitializer.
const R * GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t &num_indices) const
The API retrieves a pointer to the internal indices buffer. The API merely performs a convenience dat...
void SynchronizeOutputs()
std::string GetConfigEntryOrDefault(const char *config_key, const std::string &def)
std::string GetNodeName() const
OpAttr(const char *name, const void *data, int len, OrtOpAttrType type)
SessionOptionsImpl & AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT.
SessionOptionsImpl & EnableOrtCustomOps()
Wraps OrtApi::EnableOrtCustomOps.
size_t GetDimensionsCount() const
Wraps OrtApi::GetDimensionsCount.
R & At(const std::vector< int64_t > &location)
std::string GetAllocatorName() const
void RunAsync(const RunOptions &run_options, const char *const *input_names, const Value *input_values, size_t input_count, const char *const *output_names, Value *output_values, size_t output_count, RunAsyncCallbackFn callback, void *user_data)
Run the model asynchronously in a thread owned by intra op thread pool.
SessionOptionsImpl & AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA.
const int64_t * values_shape
SessionOptionsImpl & SetIntraOpNumThreads(int intra_op_num_threads)
Wraps OrtApi::SetIntraOpNumThreads.
**But if you need a result
SessionOptionsImpl & RegisterCustomOpsUsingFunction(const char *function_name)
Wraps OrtApi::RegisterCustomOpsUsingFunction.
IoBinding(std::nullptr_t)
Create an empty object for convenience. Sometimes, we want to initialize members later.
OrtLoggingLevel GetLoggingSeverityLevel() const noexcept
ConstOptionalTypeInfo GetOptionalTypeInfo() const
wraps OrtApi::CastTypeInfoToOptionalTypeInfo
SessionOptionsImpl & SetOptimizedModelFilePath(const ORTCHAR_T *optimized_model_file)
Wraps OrtApi::SetOptimizedModelFilePath.
static LoraAdapter CreateLoraAdapterFromArray(const void *bytes, size_t num_bytes, OrtAllocator *allocator)
Wraps OrtApi::CreateLoraAdapterFromArray.
Env(std::nullptr_t)
Create an empty Env object, must be assigned a valid one to be used.
ThreadingOptions & SetGlobalCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn)
Wraps OrtApi::SetGlobalCustomJoinThreadFn.
float8e4m3fnuz (Float8 Floating Point) data type
const R * GetSparseTensorValues() const
The API returns a pointer to an internal buffer of the sparse tensor containing non-zero values...
void * Alloc(size_t size)
TensorTypeAndShapeInfo GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat format) const
The API returns type and shape information for the specified indices. Each supported indices have the...
SessionOptionsImpl & SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level)
Wraps OrtApi::SetSessionGraphOptimizationLevel.
const R * GetTensorData() const
Returns a const typed pointer to the tensor contained data. No type checking is performed, the caller must ensure the type matches the tensor type.
SessionOptionsImpl & SetCustomThreadCreationOptions(void *ort_custom_thread_creation_options)
Wraps OrtApi::SessionOptionsSetCustomThreadCreationOptions.
void * GetTensorMutableRawData()
Returns a non-typed non-const pointer to a tensor contained data.
static Op Create(const OrtKernelInfo *info, const char *op_name, const char *domain, int version, const char **type_constraint_names, const ONNXTensorElementDataType *type_constraint_values, size_t type_constraint_count, const OpAttr *attr_values, size_t attr_count, size_t input_count, size_t output_count)
std::string GetConfigEntry(const char *config_key) const
Wraps OrtApi::GetSessionConfigEntry.
void ThrowStatus(const Status &st)
std::vector< std::string > GetOutputNames() const
float8e4m3fn (Float8 Floating Point) data type
SessionOptionsImpl & AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2 &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT.
SessionOptionsImpl & AddExternalInitializersFromFilesInMemory(const std::vector< std::basic_string< ORTCHAR_T >> &external_initializer_file_names, const std::vector< char * > &external_initializer_file_buffer_array, const std::vector< size_t > &external_initializer_file_lengths)
Wraps OrtApi::AddExternalInitializersFromFilesInMemory.
ConstValue GetTensorConstantInput(size_t index, int *is_constant) const
GLuint GLsizei const GLuint const GLintptr * offsets
const OrtApi & GetApi() noexcept
This returns a reference to the OrtApi interface in use.
bool IsSparseTensor() const
Returns true if the OrtValue contains a sparse tensor
std::vector< Value > GetOutputValuesHelper(const OrtIoBinding *binding, OrtAllocator *)
ModelMetadata GetModelMetadata() const
Wraps OrtApi::SessionGetModelMetadata.
ThreadingOptions & SetGlobalInterOpNumThreads(int inter_op_num_threads)
Wraps OrtApi::SetGlobalInterOpNumThreads.
void FillSparseTensorBlockSparse(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values, const Shape &indices_shape, const int32_t *indices_data)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
TypeInfo GetOutputTypeInfo(size_t index) const
std::vector< int64_t > GetShape() const
Uses GetDimensionsCount & GetDimensions to return a std::vector of the shape.
Wrapper around OrtMemoryInfo.
void SetEpDynamicOptions(const char *const *keys, const char *const *values, size_t kv_len)
Set DynamicOptions for EPs (Execution Providers)
std::vector< std::string > GetAvailableProviders()
This is a C++ wrapper for OrtApi::GetAvailableProviders() and returns a vector of strings representin...
GLint GLint GLsizei GLint GLenum GLenum type
Op(std::nullptr_t)
Create an empty Operator object, must be assigned a valid one to be used.
bool operator<(const BFloat16_t &rhs) const noexcept
SessionOptions Clone() const
Creates and returns a copy of this SessionOptions object. Wraps OrtApi::CloneSessionOptions.
static LoraAdapter CreateLoraAdapter(const std::basic_string< ORTCHAR_T > &adapter_path, OrtAllocator *allocator)
Wraps OrtApi::CreateLoraAdapter.
Wrapper around ::OrtIoBinding.
void GetAttrs(const OrtKernelInfo *p, const char *name, std::vector< float > &)
char * GetResizedStringTensorElementBuffer(size_t index, size_t buffer_length)
Allocate if necessary and obtain a pointer to a UTF-8 encoded string element buffer indexed by the fl...
SessionOptionsImpl & SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn)
Wraps OrtApi::SessionOptionsSetCustomCreateThreadFn.
The Status that holds ownership of OrtStatus received from C API Use it to safely destroy OrtStatus* ...
void GetStringTensorContent(void *buffer, size_t buffer_length, size_t *offsets, size_t offsets_count) const
The API copies all of the UTF-8 encoded string data contained within a tensor or a sparse tensor into...
void GetDimensions(int64_t *values, size_t values_count) const
Wraps OrtApi::GetDimensions.
A generic, discriminated value, whose type may be queried dynamically.
void GetAttr(const OrtKernelInfo *p, const char *name, float &)
CustomOpDomain(std::nullptr_t)
Create an empty CustomOpDomain object, must be assigned a valid one to be used.
SessionOptionsImpl & AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO_V2.
const std::unordered_map< std::string, std::string > & GetFlattenedConfigs() const
Returns a flattened map of custom operator configuration entries and their values.
SessionOptionsImpl & AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2 &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA_V2.
constexpr std::enable_if< I< type_count_base< T >::value, int >::type tuple_type_size(){return subtype_count< typename std::tuple_element< I, T >::type >::value+tuple_type_size< T, I+1 >);}template< typename T > struct type_count< T, typename std::enable_if< is_tuple_like< T >::value >::type >{static constexpr int value{tuple_type_size< T, 0 >)};};template< typename T > struct subtype_count{static constexpr int value{is_mutable_container< T >::value?expected_max_vector_size:type_count< T >::value};};template< typename T, typename Enable=void > struct type_count_min{static const int value{0};};template< typename T >struct type_count_min< T, typename std::enable_if<!is_mutable_container< T >::value &&!is_tuple_like< T >::value &&!is_wrapper< T >::value &&!is_complex< T >::value &&!std::is_void< T >::value >::type >{static constexpr int value{type_count< T >::value};};template< typename T > struct type_count_min< T, typename std::enable_if< is_complex< T >::value >::type >{static constexpr int value{1};};template< typename T >struct type_count_min< T, typename std::enable_if< is_wrapper< T >::value &&!is_complex< T >::value &&!is_tuple_like< T >::value >::type >{static constexpr int value{subtype_count_min< typename T::value_type >::value};};template< typename T, std::size_t I >constexpr typename std::enable_if< I==type_count_base< T >::value, int >::type tuple_type_size_min(){return 0;}template< typename T, std::size_t I > constexpr typename std::enable_if< I< type_count_base< T >::value, int >::type tuple_type_size_min(){return subtype_count_min< typename std::tuple_element< I, T >::type >::value+tuple_type_size_min< T, I+1 >);}template< typename T > struct type_count_min< T, typename std::enable_if< is_tuple_like< T >::value >::type >{static constexpr int value{tuple_type_size_min< T, 0 >)};};template< typename T > struct subtype_count_min{static constexpr int value{is_mutable_container< T >::value?((type_count< T >::value< expected_max_vector_size)?type_count< T >::value:0):type_count_min< T >::value};};template< typename T, typename Enable=void > struct expected_count{static const int value{0};};template< typename T >struct expected_count< T, typename std::enable_if<!is_mutable_container< T >::value &&!is_wrapper< T >::value &&!std::is_void< T >::value >::type >{static constexpr int value{1};};template< typename T > struct expected_count< T, typename std::enable_if< is_mutable_container< T >::value >::type >{static constexpr int value{expected_max_vector_size};};template< typename T >struct expected_count< T, typename std::enable_if<!is_mutable_container< T >::value &&is_wrapper< T >::value >::type >{static constexpr int value{expected_count< typename T::value_type >::value};};enum class object_category:int{char_value=1, integral_value=2, unsigned_integral=4, enumeration=6, boolean_value=8, floating_point=10, number_constructible=12, double_constructible=14, integer_constructible=16, string_assignable=23, string_constructible=24, other=45, wrapper_value=50, complex_number=60, tuple_value=70, container_value=80,};template< typename T, typename Enable=void > struct classify_object{static constexpr object_category value{object_category::other};};template< typename T >struct classify_object< T, typename std::enable_if< std::is_integral< T >::value &&!std::is_same< T, char >::value &&std::is_signed< T >::value &&!is_bool< T >::value &&!std::is_enum< T >::value >::type >{static constexpr object_category value{object_category::integral_value};};template< typename T >struct classify_object< T, typename std::enable_if< std::is_integral< T >::value &&std::is_unsigned< T >::value &&!std::is_same< T, char >::value &&!is_bool< T >::value >::type >{static constexpr object_category value{object_category::unsigned_integral};};template< typename T >struct classify_object< T, typename std::enable_if< std::is_same< T, char >::value &&!std::is_enum< T >::value >::type >{static constexpr object_category value{object_category::char_value};};template< typename T > struct classify_object< T, typename std::enable_if< is_bool< T >::value >::type >{static constexpr object_category value{object_category::boolean_value};};template< typename T > struct classify_object< T, typename std::enable_if< std::is_floating_point< T >::value >::type >{static constexpr object_category value{object_category::floating_point};};template< typename T >struct classify_object< T, typename std::enable_if<!std::is_floating_point< T >::value &&!std::is_integral< T >::value &&std::is_assignable< T &, std::string >::value >::type >{static constexpr object_category value{object_category::string_assignable};};template< typename T >struct classify_object< T, typename std::enable_if<!std::is_floating_point< T >::value &&!std::is_integral< T >::value &&!std::is_assignable< T &, std::string >::value &&(type_count< T >::value==1)&&std::is_constructible< T, std::string >::value >::type >{static constexpr object_category value{object_category::string_constructible};};template< typename T > struct classify_object< T, typename std::enable_if< std::is_enum< T >::value >::type >{static constexpr object_category value{object_category::enumeration};};template< typename T > struct classify_object< T, typename std::enable_if< is_complex< T >::value >::type >{static constexpr object_category value{object_category::complex_number};};template< typename T > struct uncommon_type{using type=typename std::conditional<!std::is_floating_point< T >::value &&!std::is_integral< T >::value &&!std::is_assignable< T &, std::string >::value &&!std::is_constructible< T, std::string >::value &&!is_complex< T >::value &&!is_mutable_container< T >::value &&!std::is_enum< T >::value, std::true_type, std::false_type >::type;static constexpr bool value=type::value;};template< typename T >struct classify_object< T, typename std::enable_if<(!is_mutable_container< T >::value &&is_wrapper< T >::value &&!is_tuple_like< T >::value &&uncommon_type< T >::value)>::type >{static constexpr object_category value{object_category::wrapper_value};};template< typename T >struct classify_object< T, typename std::enable_if< uncommon_type< T >::value &&type_count< T >::value==1 &&!is_wrapper< T >::value &&is_direct_constructible< T, double >::value &&is_direct_constructible< T, int >::value >::type >{static constexpr object_category value{object_category::number_constructible};};template< typename T >struct classify_object< T, typename std::enable_if< uncommon_type< T >::value &&type_count< T >::value==1 &&!is_wrapper< T >::value &&!is_direct_constructible< T, double >::value &&is_direct_constructible< T, int >::value >::type >{static constexpr object_category value{object_category::integer_constructible};};template< typename T >struct classify_object< T, typename std::enable_if< uncommon_type< T >::value &&type_count< T >::value==1 &&!is_wrapper< T >::value &&is_direct_constructible< T, double >::value &&!is_direct_constructible< T, int >::value >::type >{static constexpr object_category value{object_category::double_constructible};};template< typename T >struct classify_object< T, typename std::enable_if< is_tuple_like< T >::value &&((type_count< T >::value >=2 &&!is_wrapper< T >::value)||(uncommon_type< T >::value &&!is_direct_constructible< T, double >::value &&!is_direct_constructible< T, int >::value)||(uncommon_type< T >::value &&type_count< T >::value >=2))>::type >{static constexpr object_category value{object_category::tuple_value};};template< typename T > struct classify_object< T, typename std::enable_if< is_mutable_container< T >::value >::type >{static constexpr object_category value{object_category::container_value};};template< typename T, enable_if_t< classify_object< T >::value==object_category::char_value, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"CHAR";}template< typename T, enable_if_t< classify_object< T >::value==object_category::integral_value||classify_object< T >::value==object_category::integer_constructible, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"INT";}template< typename T, enable_if_t< classify_object< T >::value==object_category::unsigned_integral, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"UINT";}template< typename T, enable_if_t< classify_object< T >::value==object_category::floating_point||classify_object< T >::value==object_category::number_constructible||classify_object< T >::value==object_category::double_constructible, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"FLOAT";}template< typename T, enable_if_t< classify_object< T >::value==object_category::enumeration, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"ENUM";}template< typename T, enable_if_t< classify_object< T >::value==object_category::boolean_value, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"BOOLEAN";}template< typename T, enable_if_t< classify_object< T >::value==object_category::complex_number, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"COMPLEX";}template< typename T, enable_if_t< classify_object< T >::value >=object_category::string_assignable &&classify_object< T >::value<=object_category::other, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"TEXT";}template< typename T, enable_if_t< classify_object< T >::value==object_category::tuple_value &&type_count_base< T >::value >=2, detail::enabler >=detail::dummy >std::string type_name();template< typename T, enable_if_t< classify_object< T >::value==object_category::container_value||classify_object< T >::value==object_category::wrapper_value, detail::enabler >=detail::dummy >std::string type_name();template< typename T, enable_if_t< classify_object< T >::value==object_category::tuple_value &&type_count_base< T >::value==1, detail::enabler >=detail::dummy >inline std::string type_name(){return type_name< typename std::decay< typename std::tuple_element< 0, T >::type >::type >);}template< typename T, std::size_t I >inline typename std::enable_if< I==type_count_base< T >::value, std::string >::type tuple_name(){return std::string{};}template< typename T, std::size_t I >inline typename std::enable_if<(I< type_count_base< T >::value), std::string >::type tuple_name(){auto str=std::string{type_name< typename std::decay< typename std::tuple_element< I, T >::type >::type >)}+ ','+tuple_name< T, I+1 >);if(str.back()== ',') str.pop_back();return str;}template< typename T, enable_if_t< classify_object< T >::value==object_category::tuple_value &&type_count_base< T >::value >=2, detail::enabler > > std::string type_name()
Recursively generate the tuple type name.
static Value CreateTensor(const OrtMemoryInfo *info, T *p_data, size_t p_data_element_count, const int64_t *shape, size_t shape_len)
Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue.
GLint GLint GLsizei GLint GLenum format
ThreadingOptions & SetGlobalIntraOpNumThreads(int intra_op_num_threads)
Wraps OrtApi::SetGlobalIntraOpNumThreads.
Value GetTensorAttribute(const char *name, OrtAllocator *allocator) const
void Add(const OrtCustomOp *op)
Wraps CustomOpDomain_Add.
KernelInfo(std::nullptr_t)
Create an empty instance to initialize later.
Represents native memory allocation coming from one of the OrtAllocators registered with OnnxRuntime...
Session(std::nullptr_t)
Create an empty Session object, must be assigned a valid one to be used.
ThreadingOptions & SetGlobalDenormalAsZero()
Wraps OrtApi::SetGlobalDenormalAsZero.
IEEE 754 half-precision floating point data type.
size_t GetInputCount() const
static Value CreateMap(const Value &keys, const Value &values)
Creates an OrtValue with a Map Onnx type representation. The API would ref-count the supplied OrtValu...
SessionOptionsImpl & SetInterOpNumThreads(int inter_op_num_threads)
Wraps OrtApi::SetInterOpNumThreads.
size_t GetOutputCount() const
float8e5m2fnuz (Float8 Floating Point) data type
Options for the TensorRT provider that are passed to SessionOptionsAppendExecutionProvider_TensorRT_V...
R * GetTensorMutableData()
Returns a non-const typed pointer to an OrtValue/Tensor contained buffer No type checking is performe...
SessionOptionsImpl & AppendExecutionProvider_Dnnl(const OrtDnnlProviderOptions &provider_options)
SessionOptions()
Wraps OrtApi::CreateSessionOptions.
void * GetGPUComputeStream() const
void BindOutput(const char *name, const Value &)
GLuint const GLchar * name
RunOptions & SetRunTag(const char *run_tag)
wraps OrtApi::RunOptionsSetRunTag
Allocator(std::nullptr_t)
Convenience to create a class member and then replace with an instance.
SessionOptionsImpl & EnableProfiling(const ORTCHAR_T *profile_file_prefix)
Wraps OrtApi::EnableProfiling.
size_t GetStringTensorDataLength() const
This API returns a full length of string data contained within either a tensor or a sparse Tensor...
detail::ConstSessionOptionsImpl< detail::Unowned< const OrtSessionOptions >> ConstSessionOptions
GLsizei const GLchar *const * strings
RunOptions & SetTerminate()
Terminates all currently executing Session::Run calls that were made using this RunOptions instance...
OrtAllocatorType GetAllocatorType() const
SessionOptionsImpl & AppendExecutionProvider_OpenVINO_V2(const std::unordered_map< std::string, std::string > &provider_options={})
size_t GetInputCount() const
TypeInfo GetSequenceElementType() const
Wraps OrtApi::GetSequenceElementType.
static Value CreateSparseTensor(const OrtMemoryInfo *info, T *p_data, const Shape &dense_shape, const Shape &values_shape)
This is a simple forwarding method to the other overload that helps deducing data type enum value fro...
void GetSessionConfigs(std::unordered_map< std::string, std::string > &out, ConstSessionOptions options) const
TypeInfo GetOutputTypeInfo(size_t index) const
Wraps OrtApi::SessionGetOutputTypeInfo.
size_t GetStringTensorElementLength(size_t element_index) const
The API returns a byte length of UTF-8 encoded string element contained in either a tensor or a spare...
detail::ValueImpl< detail::Unowned< OrtValue >> UnownedValue
SessionOptionsImpl & Add(OrtCustomOpDomain *custom_op_domain)
Wraps OrtApi::AddCustomOpDomain.
TensorTypeAndShapeInfo GetSparseTensorValuesTypeAndShapeInfo() const
The API returns type and shape information for stored non-zero values of the sparse tensor...
GT_API const UT_StringHolder version
TypeInfo GetTypeInfo() const
The API returns type information for data contained in a tensor. For sparse tensors it returns type i...
RunOptions & SetRunLogSeverityLevel(int)
Wraps OrtApi::RunOptionsSetRunLogSeverityLevel.
ConstTensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const
Wraps OrtApi::CastTypeInfoToTensorInfo.
bool operator==(const MemoryInfoImpl< U > &o) const
TypeInfo GetMapValueType() const
Wraps OrtApi::GetMapValueType.
RunOptions()
Wraps OrtApi::CreateRunOptions.
SessionOptionsImpl & AppendExecutionProvider_VitisAI(const std::unordered_map< std::string, std::string > &provider_options={})
This struct owns the OrtKernInfo* pointer when a copy is made. For convenient wrapping of OrtKernelIn...
void FillSparseTensorCoo(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values_param, const int64_t *indices_data, size_t indices_num)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
std::vector< Value > GetOutputValues() const
OrtErrorCode GetErrorCode() const
void FillStringTensor(const char *const *s, size_t s_len)
Set all strings at once in a string tensor
void ParallelFor(void(*fn)(void *, size_t), size_t total, size_t num_batch, void *usr_data) const
KernelContext(OrtKernelContext *context)
This class represents an ONNX Runtime logger that can be used to log information with an associated s...
Env & CreateAndRegisterAllocatorV2(const std::string &provider_type, const OrtMemoryInfo *mem_info, const std::unordered_map< std::string, std::string > &options, const OrtArenaCfg *arena_cfg)
Wraps OrtApi::CreateAndRegisterAllocatorV2.
IMATH_NAMESPACE::V2f IMATH_NAMESPACE::Box2i std::string this attribute is obsolete as of OpenEXR v3 float
ArenaCfg(std::nullptr_t)
Create an empty ArenaCfg object, must be assigned a valid one to be used.
SessionOptionsImpl & SetDeterministicCompute(bool value)
Wraps OrtApi::SetDeterministicCompute.
GT_API const UT_StringHolder st
ThreadingOptions()
Wraps OrtApi::CreateThreadingOptions.
GLenum GLsizei GLsizei GLint * values
std::string GetOutputName(size_t index) const
LoraAdapter holds a set of Lora Parameters loaded from a single file.
bool operator==(const BFloat16_t &rhs) const noexcept
TypeInfo GetOverridableInitializerTypeInfo(size_t index) const
Wraps OrtApi::SessionGetOverridableInitializerTypeInfo.
SessionOptionsImpl & AddConfigEntry(const char *config_key, const char *config_value)
Wraps OrtApi::AddSessionConfigEntry.
bool IsOK() const noexcept
Returns true if instance represents an OK (non-error) status.
SessionOptionsImpl & SetExecutionMode(ExecutionMode execution_mode)
Wraps OrtApi::SetSessionExecutionMode.
AllocatorWithDefaultOptions()
AllocatedStringPtr EndProfilingAllocated(OrtAllocator *allocator)
End profiling and return a copy of the profiling file name.
ShapeInferContext(const OrtApi *ort_api, OrtShapeInferContext *ctx)
SessionOptionsImpl & AppendExecutionProvider(const std::string &provider_name, const std::unordered_map< std::string, std::string > &provider_options={})
Wraps OrtApi::SessionOptionsAppendExecutionProvider. Currently supports QNN, SNPE and XNNPACK...
SessionOptionsImpl & DisableProfiling()
Wraps OrtApi::DisableProfiling.
float8e5m2 (Float8 Floating Point) data type
**If you just want to fire and args
RunOptions & UnsetTerminate()
Clears the terminate flag so this RunOptions instance can be used in a new Session::Run call without ...
size_t GetOverridableInitializerCount() const
Returns the number of inputs that have defaults that can be overridden.
const char * GetRunTag() const
Wraps OrtApi::RunOptionsGetRunTag.
#define ORT_CXX_RETURN_ON_API_FAIL(expression)
int GetRunLogVerbosityLevel() const
Wraps OrtApi::RunOptionsGetRunLogVerbosityLevel.
OIIO_UTIL_API const char * c_str(string_view str)
std::string GetVersionString()
This function returns the onnxruntime version string
size_t GetOutputCount() const
Returns the number of model outputs.
Status LogMessage(OrtLoggingLevel log_severity_level, const ORTCHAR_T *file_path, int line_number, const char *func_name, const char *message) const noexcept
SessionOptionsImpl & AddExternalInitializers(const std::vector< std::string > &names, const std::vector< Value > &ort_values)
Wraps OrtApi::AddExternalInitializers.
TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const
The API returns type information for data contained in a tensor. For sparse tensors it returns type i...
int GetRunLogSeverityLevel() const
Wraps OrtApi::RunOptionsGetRunLogSeverityLevel.
ConstMemoryInfo GetTensorMemoryInfo() const
This API returns information about the memory allocation used to hold data.
AllocatedStringPtr GetInputNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of input name at the specified index.
Wrapper around ::OrtTensorTypeAndShapeInfo.
MemoryInfo(std::nullptr_t)
No instance is created.
std::string MakeCustomOpConfigEntryKey(const char *custom_op_name, const char *config)
CustomOpConfigs.
size_t GetCount() const
< Return true if OrtValue contains data and returns false if the OrtValue is a None ...
std::string GetInputName(size_t index) const
ThreadingOptions & SetGlobalCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn)
Wraps OrtApi::SetGlobalCustomCreateThreadFn.
Wrapper around ::OrtSessionOptions.
OrtSparseFormat GetSparseFormat() const
The API returns the sparse data format this OrtValue holds in a sparse tensor. If the sparse tensor w...
detail::MemoryInfoImpl< detail::Unowned< const OrtMemoryInfo >> ConstMemoryInfo
void UseCooIndices(int64_t *indices_data, size_t indices_num)
Supplies COO format specific indices and marks the contained sparse tensor as being a COO format tens...
MemoryAllocation(OrtAllocator *allocator, void *p, size_t size)
CustomOpConfigs & AddConfig(const char *custom_op_name, const char *config_key, const char *config_value)
Adds a session configuration entry/value for a specific custom operator.
TypeInfo GetInputTypeInfo(size_t index) const
SessionOptionsImpl & SetLogSeverityLevel(int level)
Wraps OrtApi::SetSessionLogSeverityLevel.
Wrapper around ::OrtAllocator default instance that is owned by Onnxruntime.
Wrapper around ::OrtSession.
SessionOptionsImpl & DisableMemPattern()
Wraps OrtApi::DisableMemPattern.
SessionOptionsImpl & AppendExecutionProvider_ROCM(const OrtROCMProviderOptions &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_ROCM.
Options for the CUDA provider that are passed to SessionOptionsAppendExecutionProvider_CUDA_V2. Please note that this struct is similar to OrtCUDAProviderOptions but only to be used internally. Going forward, new cuda provider options are to be supported via this struct and usage of the publicly defined OrtCUDAProviderOptions will be deprecated over time. User can only get the instance of OrtCUDAProviderOptionsV2 via CreateCUDAProviderOptions.
RunOptions & SetRunLogVerbosityLevel(int)
Wraps OrtApi::RunOptionsSetRunLogVerbosityLevel.
ThreadingOptions & SetGlobalCustomThreadCreationOptions(void *ort_custom_thread_creation_options)
Wraps OrtApi::SetGlobalCustomThreadCreationOptions.
TypeInfo GetOptionalElementType() const
Wraps OrtApi::CastOptionalTypeToContainedTypeInfo.
bfloat16 (Brain Floating Point) data type
RunOptions & AddActiveLoraAdapter(const LoraAdapter &adapter)
Add the LoraAdapter to the list of active adapters. The setting does not affect RunWithBinding() call...
Class that represents session configuration entries for one or more custom operators.
Status(std::nullptr_t) noexcept
Create an empty object, must be assigned a valid one to be used.
std::vector< std::string > GetOutputNamesHelper(const OrtIoBinding *binding, OrtAllocator *)
#define ORT_CXX_API_THROW(string, code)
SessionOptionsImpl & RegisterCustomOpsLibrary(const ORTCHAR_T *library_name, const CustomOpConfigs &custom_op_configs={})
GLsizei GLenum GLenum GLuint GLenum GLsizei * lengths
ConstSequenceTypeInfo GetSequenceTypeInfo() const
Wraps OrtApi::CastTypeInfoToSequenceTypeInfo.