55 struct TypeToTensorType;
57 struct TypeToTensorType<
float> {
69 struct TypeToTensorType<double> {
73 struct TypeToTensorType<int8_t> {
77 struct TypeToTensorType<int16_t> {
81 struct TypeToTensorType<int32_t> {
85 struct TypeToTensorType<int64_t> {
89 struct TypeToTensorType<uint8_t> {
93 struct TypeToTensorType<uint16_t> {
97 struct TypeToTensorType<uint32_t> {
101 struct TypeToTensorType<uint64_t> {
105 struct TypeToTensorType<bool> {
110 : allocator_(allocator), p_(p), size_(size) {
116 auto ret =
GetApi().AllocatorFree(allocator_, p_);
117 static_cast<void>(ret);
122 *
this = std::move(o);
147 template <
typename T>
154 template <
typename T>
162 template <
typename T>
167 template <
typename T>
186 template <
typename T>
188 const char*
name =
nullptr;
193 template <
typename T>
200 template <
typename T>
207 template <
typename T>
210 GetApi().MemoryInfoGetDeviceType(this->p_, &type);
214 template <
typename T>
221 template <
typename T>
222 template <
typename U>
226 return comp_result == 0;
242 template <
typename T>
248 template <
typename T>
253 template <
typename T>
259 template <
typename T>
264 template <
typename T>
269 template <
typename T>
274 template <
typename T>
279 template <
typename T>
281 GetApi().ClearBoundInputs(this->p_);
284 template <
typename T>
286 GetApi().ClearBoundOutputs(this->p_);
289 template <
typename T>
294 template <
typename T>
299 namespace binding_utils {
301 std::vector<std::string>
result;
303 using Ptr = std::unique_ptr<void, decltype(free_fn)>;
308 ThrowOnError(
GetApi().GetBoundOutputNames(binding, allocator, &buffer, &lengths, &count));
314 Ptr buffer_g(buffer, free_fn);
315 Ptr lengths_g(lengths, free_fn);
317 result.reserve(count);
318 for (
size_t i = 0; i <
count; ++i) {
320 result.emplace_back(buffer, sz);
328 std::vector<Value>
result;
330 size_t output_count = 0;
333 auto free_fn = [&owned, &output_count, allocator](
OrtValue**
buffer) {
335 while (owned < output_count) {
336 auto* p =
buffer + owned++;
337 GetApi().ReleaseValue(*p);
339 allocator->Free(allocator,
buffer);
342 using Ptr = std::unique_ptr<OrtValue*, decltype(free_fn)>;
345 ThrowOnError(
GetApi().GetBoundOutputValues(binding, allocator, &output_buffer, &output_count));
346 if (output_count == 0) {
350 Ptr buffer_g(output_buffer, free_fn);
352 result.reserve(output_count);
353 for (
size_t i = 0; i < output_count; ++i) {
354 result.emplace_back(output_buffer[i]);
367 inline ArenaCfg::ArenaCfg(
size_t max_mem,
int arena_extend_strategy,
int initial_chunk_size_bytes,
int max_dead_bytes_per_chunk) {
368 ThrowOnError(
GetApi().CreateArenaCfg(max_mem, arena_extend_strategy, initial_chunk_size_bytes, max_dead_bytes_per_chunk, &
p_));
412 if (strcmp(logid,
"onnxruntime-node") == 0) {
420 ThrowOnError(
GetApi().CreateEnvWithCustomLogger(logging_function, logger_param, logging_level, logid, &
p_));
421 if (strcmp(logid,
"onnxruntime-node") == 0) {
430 if (strcmp(logid,
"onnxruntime-node") == 0) {
439 ThrowOnError(
GetApi().CreateEnvWithCustomLoggerAndGlobalThreadPools(logging_function, logger_param, logging_level, logid, tp_options, &
p_));
440 if (strcmp(logid,
"onnxruntime-node") == 0) {
447 inline Env& Env::EnableTelemetryEvents() {
529 template <
typename T>
531 OrtSessionOptions* out;
536 template <
typename T>
545 out.resize(size - 1);
550 template <
typename T>
554 return static_cast<bool>(out);
557 template <
typename T>
559 if (!this->HasConfigEntry(config_key)) {
563 return this->GetConfigEntry(config_key);
566 template <
typename T>
572 template <
typename T>
578 template <
typename T>
580 ThrowOnError(
GetApi().SetSessionGraphOptimizationLevel(this->p_, graph_optimization_level));
584 template <
typename T>
590 template <
typename T>
596 template <
typename T>
602 template <
typename T>
608 template <
typename T>
614 template <
typename T>
620 template <
typename T>
626 template <
typename T>
632 template <
typename T>
638 template <
typename T>
644 template <
typename T>
650 template <
typename T>
656 template <
typename T>
662 template <
typename T>
668 template <
typename T>
674 template <
typename T>
676 const std::vector<Value>& ort_values) {
677 const size_t inputs_num = names.size();
678 if (inputs_num != ort_values.size()) {
681 std::vector<const char*> names_ptr;
682 std::vector<const OrtValue*> ort_values_ptrs;
683 names_ptr.reserve(inputs_num);
684 ort_values_ptrs.reserve(inputs_num);
685 for (
size_t i = 0; i < inputs_num; ++i) {
686 names_ptr.push_back(names[i].c_str());
687 ort_values_ptrs.push_back(ort_values[i]);
689 ThrowOnError(
GetApi().AddExternalInitializers(this->p_, names_ptr.data(), ort_values_ptrs.data(), inputs_num));
693 template <
typename T>
695 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_CUDA(this->p_, &provider_options));
699 template <
typename T>
701 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_CUDA_V2(this->p_, &provider_options));
705 template <
typename T>
707 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_ROCM(this->p_, &provider_options));
711 template <
typename T>
713 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_TensorRT(this->p_, &provider_options));
717 template <
typename T>
719 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_TensorRT_V2(this->p_, &provider_options));
723 template <
typename T>
725 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_MIGraphX(this->p_, &provider_options));
729 template <
typename T>
731 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_CANN(this->p_, &provider_options));
735 template <
typename T>
738 const std::unordered_map<std::string, std::string>& provider_options) {
739 auto num_entries = provider_options.size();
740 std::vector<const char*> keys,
values;
741 if (num_entries > 0) {
742 keys.reserve(num_entries);
743 values.reserve(num_entries);
745 for (
const auto& entry : provider_options) {
746 keys.push_back(entry.first.c_str());
747 values.push_back(entry.second.c_str());
751 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider(this->p_, provider_name.c_str(),
752 keys.data(), values.data(), num_entries));
757 template <
typename T>
759 ThrowOnError(
GetApi().SessionOptionsSetCustomCreateThreadFn(this->p_, ort_custom_create_thread_fn));
763 template <
typename T>
765 ThrowOnError(
GetApi().SessionOptionsSetCustomThreadCreationOptions(this->p_, ort_custom_thread_creation_options));
769 template <
typename T>
771 ThrowOnError(
GetApi().SessionOptionsSetCustomJoinThreadFn(this->p_, ort_custom_join_thread_fn));
775 template <
typename T>
777 ThrowOnError(
GetApi().SessionOptionsAppendExecutionProvider_OpenVINO(this->p_, &provider_options));
781 template <
typename T>
787 AddConfigEntry(config_iter.first.c_str(), config_iter.second.c_str());
794 template <
typename T>
796 ThrowOnError(
GetApi().RegisterCustomOpsUsingFunction(this->p_, registration_function_name));
801 template <
typename T>
808 template <
typename T>
815 template <
typename T>
822 template <
typename T>
829 template <
typename T>
836 template <
typename T>
839 ThrowOnError(
GetApi().SessionGetOverridableInitializerName(this->p_, index, allocator, &out));
843 template <
typename T>
850 template <
typename T>
852 OrtModelMetadata* out;
857 template <
typename T>
864 template <
typename T>
871 template <
typename T>
874 ThrowOnError(
GetApi().SessionGetOverridableInitializerTypeInfo(this->p_, index, &out));
878 template <
typename T>
880 const char*
const* output_names,
size_t output_count) {
881 std::vector<Value> output_values;
882 output_values.reserve(output_count);
883 for (
size_t i = 0; i < output_count; i++)
884 output_values.emplace_back(
nullptr);
885 Run(run_options, input_names, input_values, input_count, output_names, output_values.data(), output_count);
886 return output_values;
889 template <
typename T>
891 const char*
const* output_names,
Value* output_values,
size_t output_count) {
892 static_assert(
sizeof(
Value) ==
sizeof(
OrtValue*),
"Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely");
893 auto ort_input_values =
reinterpret_cast<const OrtValue*
const*
>(input_values);
894 auto ort_output_values =
reinterpret_cast<OrtValue**
>(output_values);
895 ThrowOnError(
GetApi().Run(this->p_, run_options, input_names, ort_input_values, input_count, output_names, output_count, ort_output_values));
898 template <
typename T>
903 template <
typename T>
920 config_key += custom_op_name;
922 config_key += config;
929 flat_configs_[full_flat_key] = config_value;
934 return flat_configs_;
942 OrtPrepackedWeightsContainer* prepacked_weights_container) {
943 ThrowOnError(
GetApi().CreateSessionWithPrepackedWeightsContainer(env, model_path, options, prepacked_weights_container, &this->
p_));
947 ThrowOnError(
GetApi().CreateSessionFromArray(env, model_data, model_data_length, options, &this->
p_));
951 const SessionOptions& options, OrtPrepackedWeightsContainer* prepacked_weights_container) {
952 ThrowOnError(
GetApi().CreateSessionFromArrayWithPrepackedWeightsContainer(env, model_data, model_data_length, options,
953 prepacked_weights_container, &this->
p_));
994 std::vector<AllocatedStringPtr>
result;
996 char** out =
nullptr;
997 int64_t num_keys = 0;
1004 std::unique_ptr<void, decltype(deletor)> array_guard(out, deletor);
1006 auto strings_deletor = [&deletor, num_keys](
char** out) {
for(int64_t i = 0; i < num_keys; ++i) deletor(out[i]); };
1007 std::unique_ptr<char*, decltype(strings_deletor)> strings_guard(out, strings_deletor);
1008 result.reserve(static_cast<size_t>(num_keys));
1009 strings_guard.release();
1010 for (int64_t i = 0; i < num_keys; ++i) {
1025 template <
typename T>
1032 template <
typename T>
1036 return static_cast<size_t>(out);
1039 template <
typename T>
1046 template <
typename T>
1051 template <
typename T>
1056 template <
typename T>
1058 std::vector<int64_t> out(GetDimensionsCount(), 0);
1066 template <
typename T>
1068 const OrtTensorTypeAndShapeInfo* out;
1073 template <
typename T>
1075 const OrtSequenceTypeInfo* out;
1080 template <
typename T>
1082 const OrtMapTypeInfo* out;
1087 template <
typename T>
1097 template <
typename T>
1099 OrtTypeInfo* output;
1107 template <
typename T>
1114 template <
typename T>
1116 OrtTypeInfo* output;
1124 template <
typename T>
1125 template <
typename R>
1130 template <
typename T>
1137 template <
typename T>
1144 template <
typename T>
1151 template <
typename T>
1158 template <
typename T>
1165 template <
typename T>
1172 template <
typename T>
1173 template <
typename R>
1176 ThrowOnError(
GetApi().GetTensorMutableData(const_cast<OrtValue*>(this->p_), (
void**)&out));
1180 template <
typename T>
1187 template <
typename T>
1189 OrtTypeInfo* output;
1194 template <
typename T>
1196 OrtTensorTypeAndShapeInfo* output;
1201 template <
typename T>
1208 template <
typename T>
1210 ThrowOnError(
GetApi().GetStringTensorElement(this->p_, buffer_length, element_index, buffer));
1213 template <
typename T>
1215 ThrowOnError(
GetApi().GetStringTensorContent(this->p_, buffer, buffer_length, offsets, offsets_count));
1218 #if !defined(DISABLE_SPARSE_TENSORS)
1219 template <
typename T>
1226 template <
typename T>
1228 OrtTensorTypeAndShapeInfo* output;
1233 template <
typename T>
1235 OrtTensorTypeAndShapeInfo* output;
1236 ThrowOnError(
GetApi().GetSparseTensorIndicesTypeShape(this->p_, indices_format, &output));
1240 template <
typename T>
1241 template <
typename R>
1244 ThrowOnError(
GetApi().GetSparseTensorIndices(this->p_, indices_format, &num_indices, &out));
1245 return reinterpret_cast<const R*
>(out);
1248 template <
typename T>
1255 template <
typename T>
1256 template <
typename R>
1260 return reinterpret_cast<const R*
>(out);
1265 template <
typename T>
1270 template <
typename T>
1275 template <
typename T>
1282 template <
typename T>
1283 template <
typename R>
1290 template <
typename T>
1291 template <
typename R>
1295 ThrowOnError(
GetApi().TensorAt(this->p_, location.data(), location.size(), (
void**)&out));
1299 #if !defined(DISABLE_SPARSE_TENSORS)
1300 template <
typename T>
1305 template <
typename T>
1307 ThrowOnError(
GetApi().UseCsrIndices(this->p_, inner_data, inner_num, outer_data, outer_num));
1310 template <
typename T>
1315 template <
typename T>
1317 const int64_t* indices_data,
size_t indices_num) {
1320 indices_data, indices_num));
1323 template <
typename T>
1326 const int64_t* inner_indices_data,
size_t inner_indices_num,
1327 const int64_t* outer_indices_data,
size_t outer_indices_num) {
1329 inner_indices_data, inner_indices_num,
1330 outer_indices_data, outer_indices_num));
1333 template <
typename T>
1336 const Shape& indices_shape,
1337 const int32_t* indices_data) {
1343 #endif // !defined(DISABLE_SPARSE_TENSORS)
1347 template <
typename T>
1355 ThrowOnError(
GetApi().CreateTensorWithDataAsOrtValue(info, p_data, p_data_byte_count, shape, shape_len, type, &out));
1359 template <
typename T>
1366 ThrowOnError(
GetApi().CreateTensorAsOrtValue(allocator, shape, shape_len, type, &out));
1370 #if !defined(DISABLE_SPARSE_TENSORS)
1372 template <
typename T>
1374 const Shape& values_shape) {
1386 template <
typename T>
1397 #endif // !defined(DISABLE_SPARSE_TENSORS)
1401 OrtValue* inputs[2] = {keys, values};
1408 std::vector<OrtValue*> values_ort{values.data(), values.data() + values.size()};
1413 template <
typename T>
1416 ThrowOnError(
GetApi().CreateOpaqueValue(domain, type_name, &data_container,
sizeof(
T), &out));
1457 void* out =
nullptr;
1467 template <
typename T>
1474 template <
typename T>
1481 template <
typename T>
1488 template <
typename T>
1498 out.resize(size - 1);
1503 template <
typename T>
1513 out.resize(size - 1);
1518 template <
typename T>
1520 OrtTypeInfo* out =
nullptr;
1525 template <
typename T>
1527 OrtTypeInfo* out =
nullptr;
1532 template <
typename T>
1535 ThrowOnError(
GetApi().KernelInfoGetAttribute_tensor(this->p_, name, allocator, &out));
1555 out.resize(size - 1);
1564 std::vector<float> out;
1576 std::vector<int64_t> out;
1585 inline Op::Op(OrtOp* p) : Base<OrtOp>(p) {}
1588 const char** type_constraint_names,
1590 size_t type_constraint_count,
1591 const OpAttr* attr_values,
size_t attr_count,
1592 size_t input_count,
size_t output_count) {
1593 static_assert(
sizeof(OpAttr) ==
sizeof(OrtOpAttr*),
1594 "OpAttr's is expected to be just an array of OrtOpAttr in memory so we can reinterpret safely");
1595 auto attr_input_values =
reinterpret_cast<const OrtOpAttr*
const*
>(attr_values);
1597 Ort::ThrowOnError(
GetApi().CreateOp(info, op_name, domain, version, type_constraint_names, type_constraint_values,
1598 static_cast<int>(type_constraint_count),
1600 static_cast<int>(attr_count),
1601 static_cast<int>(input_count),
1602 static_cast<int>(output_count), &op));
1607 const Value* input_values,
1609 Value* output_values,
1610 size_t output_count) {
1612 "Value is really just an array of OrtValue* in memory, so we can reinterpret_cast safely");
1613 auto ort_input_values =
reinterpret_cast<const OrtValue*
const*
>(input_values);
1614 auto ort_output_values =
reinterpret_cast<OrtValue**
>(output_values);
1616 ort_output_values, static_cast<int>(output_count)));
1620 const OrtValue*
const* input_values,
1623 size_t output_count) {
1625 output_values, static_cast<int>(output_count)));
1633 inline float CustomOpApi::KernelInfoGetAttribute<float>(
_In_ const OrtKernelInfo* info,
_In_ const char* name) {
1640 inline int64_t CustomOpApi::KernelInfoGetAttribute<int64_t>(
_In_ const OrtKernelInfo* info,
_In_ const char* name) {
1652 OrtStatus* status = api_.KernelInfoGetAttribute_string(info, name,
nullptr, &size);
1654 if (status ==
nullptr) {
1656 Ort::ThrowOnError(api_.KernelInfoGetAttribute_string(info, name, &out[0], &size));
1657 out.resize(size - 1);
1667 std::vector<float> out;
1670 OrtStatus* status = api_.KernelInfoGetAttributeArray_float(info, name,
nullptr, &size);
1672 if (status ==
nullptr) {
1684 std::vector<int64_t> out;
1687 OrtStatus* status = api_.KernelInfoGetAttributeArray_int64(info, name,
nullptr, &size);
1689 if (status ==
nullptr) {
1698 OrtTensorTypeAndShapeInfo* out;
1729 template <
typename T>
1732 Ort::ThrowOnError(api_.GetTensorMutableData(value, reinterpret_cast<void**>(&data)));
1742 template <
typename T>
1745 Ort::ThrowOnError(api_.GetTensorMutableData(const_cast<OrtValue*>(value), reinterpret_cast<void**>(&data)));
1752 std::vector<int64_t> output(out);
1758 api_.ReleaseTensorTypeAndShapeInfo(input);
1780 _In_ const int64_t* dim_values,
size_t dim_count) {
1782 Ort::ThrowOnError(api_.KernelContext_GetOutput(context, index, dim_values, dim_count, &out));
1793 _In_ const void* data,
1796 OrtOpAttr* op_attr{};
1802 api_.ReleaseOpAttr(op_attr);
1806 _In_ const char* op_name,
1807 _In_ const char* domain,
1809 _In_opt_ const char** type_constraint_names,
1811 _In_opt_ int type_constraint_count,
1812 _In_opt_ const OrtOpAttr*
const* attr_values,
1814 _In_ int input_count,
1815 _In_ int output_count) {
1817 Ort::ThrowOnError(api_.CreateOp(info, op_name, domain, version, type_constraint_names, type_constraint_values,
1818 type_constraint_count, attr_values, attr_count, input_count, output_count, &ort_op));
1823 _In_ const OrtOp* ort_op,
1825 _In_ int input_count,
1827 _In_ int output_count) {
1828 Ort::ThrowOnError(api_.InvokeOp(context, ort_op, input_values, input_count, output_values, output_count));
1832 api_.ReleaseOp(ort_op);
1842 api_.ReleaseKernelInfo(info_copy);
1849 std::vector<std::string> available_providers(providers, providers + len);
1851 return available_providers;
1854 SessionOptions& AddInitializer(
const char* name,
const OrtValue* ort_val);
1856 template <
typename TOp,
typename TKernel>
1859 const TOp* derived =
static_cast<const TOp*
>(
this);
1860 std::vector<std::string> keys = derived->GetSessionConfigKeys();
1862 out.reserve(keys.size());
1865 const size_t prefix_size = config_entry_key.length();
1867 for (
const auto& key : keys) {
1868 config_entry_key.resize(prefix_size);
1869 config_entry_key.append(key);
OrtMemType GetMemoryType() const
OrtMemoryInfoDeviceType GetDeviceType() const
ONNXTensorElementDataType GetTensorElementType(const OrtTensorTypeAndShapeInfo *info)
void * KernelContext_GetGPUComputeStream(const OrtKernelContext *context)
void Invoke(const OrtKernelContext *context, const Value *input_values, size_t input_count, Value *output_values, size_t output_count)
SessionOptionsImpl & SetCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn)
Wraps OrtApi::SessionOptionsSetCustomJoinThreadFn.
GLuint GLsizei const GLchar * message
size_t GetElementCount() const
Wraps OrtApi::GetTensorShapeElementCount.
ONNXType GetONNXType() const
static Value CreateOpaque(const char *domain, const char *type_name, const T &)
Wraps OrtApi::CreateOpaqueValue.
MemoryAllocation & operator=(const MemoryAllocation &)=delete
Env & DisableTelemetryEvents()
Wraps OrtApi::EnableTelemetryEvents.
AllocatedStringPtr GetOverridableInitializerNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of the overridable initializer name at then specified index.
ThreadingOptions & SetGlobalSpinControl(int allow_spinning)
Wraps OrtApi::SetGlobalSpinControl.
OrtCustomThreadHandle(* OrtCustomCreateThreadFn)(void *ort_custom_thread_creation_options, OrtThreadWorkerFn ort_thread_worker_fn, void *ort_worker_fn_param)
Ort custom thread creation function.
std::string GetErrorMessage() const
size_t GetInputCount() const
Returns the number of model inputs.
SessionOptionsImpl & DisablePerSessionThreads()
Wraps OrtApi::DisablePerSessionThreads.
Env & UpdateEnvWithCustomLogLevel(OrtLoggingLevel log_severity_level)
Wraps OrtApi::UpdateEnvWithCustomLogLevel.
void InvokeOp(_In_ const OrtKernelContext *context, _In_ const OrtOp *ort_op, _In_ const OrtValue *const *input_values, _In_ int input_count, _Inout_ OrtValue *const *output_values, _In_ int output_count)
void UseBlockSparseIndices(const Shape &indices_shape, int32_t *indices_data)
Supplies BlockSparse format specific indices and marks the contained sparse tensor as being a BlockSp...
RunOptions & AddConfigEntry(const char *config_key, const char *config_value)
Wraps OrtApi::AddRunConfigEntry.
ConstMapTypeInfo GetMapTypeInfo() const
Wraps OrtApi::CastTypeInfoToMapTypeInfo.
void FillSparseTensorCsr(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values, const int64_t *inner_indices_data, size_t inner_indices_num, const int64_t *outer_indices_data, size_t outer_indices_num)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
TypeInfo GetInputTypeInfo(size_t index) const
Wraps OrtApi::SessionGetInputTypeInfo.
size_t GetOutputCount() const
void GetSymbolicDimensions(const char **values, size_t values_count) const
Wraps OrtApi::GetSymbolicDimensions.
Type information that may contain either TensorTypeAndShapeInfo or the information about contained se...
SessionOptionsImpl & EnableMemPattern()
Wraps OrtApi::EnableMemPattern.
SessionOptionsImpl & EnableCpuMemArena()
Wraps OrtApi::EnableCpuMemArena.
static MemoryInfo CreateCpu(OrtAllocatorType type, OrtMemType mem_type1)
bool IsTensor() const
Returns true if Value is a tensor, false for other types like map/sequence/etc.
AllocatedStringPtr GetOutputNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of output name at then specified index.
void GetOpaqueData(const char *domain, const char *type_name, R &) const
Obtains a pointer to a user defined data for experimental purposes
const void * GetTensorRawData() const
Returns a non-typed pointer to a tensor contained data.
ConstMemoryInfo GetInfo() const
T * GetTensorMutableData(_Inout_ OrtValue *value)
void GetStringTensorElement(size_t buffer_length, size_t element_index, void *buffer) const
The API copies UTF-8 encoded bytes for the requested string element contained within a tensor or a sp...
SessionOptionsImpl & SetLogId(const char *logid)
Wraps OrtApi::SetSessionLogId.
void swap(UT::ArraySet< Key, MULTI, MAX_LOAD_FACTOR_256, Clearer, Hash, KeyEqual > &a, UT::ArraySet< Key, MULTI, MAX_LOAD_FACTOR_256, Clearer, Hash, KeyEqual > &b)
Value GetValue(int index, OrtAllocator *allocator) const
uint64_t GetProfilingStartTimeNs() const
Wraps OrtApi::SessionGetProfilingStartTimeNs.
Status(std::nullptr_t)
Create an empty object, must be assigned a valid one to be used.
SessionOptionsImpl & AppendExecutionProvider_MIGraphX(const OrtMIGraphXProviderOptions &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_CANN.
UnownedValue GetOutput(size_t index, const int64_t *dim_values, size_t dim_count) const
GLsizei const GLchar *const * string
void ThrowOnError(OrtStatus *ort_status)
GLsizei const GLfloat * value
void BindInput(const char *name, const Value &)
ONNXTensorElementDataType GetMapKeyType() const
Wraps OrtApi::GetMapKeyType.
OrtMemoryInfoDeviceType
This mimics OrtDevice type constants so they can be returned in the API.
ConstValue GetInput(size_t index) const
SessionOptionsImpl & AppendExecutionProvider_CANN(const OrtCANNProviderOptions &provider_options)
MemoryAllocation GetAllocation(size_t size)
std::unique_ptr< char, detail::AllocatedFree > AllocatedStringPtr
unique_ptr typedef used to own strings allocated by OrtAllocators and release them at the end of the ...
SessionOptionsImpl & DisableCpuMemArena()
Wraps OrtApi::DisableCpuMemArena.
bool HasConfigEntry(const char *config_key) const
Wraps OrtApi::HasSessionConfigEntry.
void UseCsrIndices(int64_t *inner_data, size_t inner_num, int64_t *outer_data, size_t outer_num)
Supplies CSR format specific indices and marks the contained sparse tensor as being a CSR format tens...
ONNXTensorElementDataType GetElementType() const
Wraps OrtApi::GetTensorElementType.
Env & CreateAndRegisterAllocator(const OrtMemoryInfo *mem_info, const OrtArenaCfg *arena_cfg)
Wraps OrtApi::CreateAndRegisterAllocator.
void FillStringTensorElement(const char *s, size_t index)
Set a single string in a string tensor
void ReleaseOp(_Frees_ptr_opt_ OrtOp *ort_op)
std::vector< Value > Run(const RunOptions &run_options, const char *const *input_names, const Value *input_values, size_t input_count, const char *const *output_names, size_t output_count)
Run the model returning results in an Ort allocated vector.
void GetDimensions(_In_ const OrtTensorTypeAndShapeInfo *info, _Out_ int64_t *dim_values, size_t dim_values_length)
SessionOptionsImpl & AddInitializer(const char *name, const OrtValue *ort_val)
Wraps OrtApi::AddInitializer.
const R * GetSparseTensorIndicesData(OrtSparseIndicesFormat indices_format, size_t &num_indices) const
The API retrieves a pointer to the internal indices buffer. The API merely performs a convenience dat...
void SynchronizeOutputs()
std::string GetConfigEntryOrDefault(const char *config_key, const std::string &def)
OpAttr(const char *name, const void *data, int len, OrtOpAttrType type)
SessionOptionsImpl & AppendExecutionProvider_TensorRT(const OrtTensorRTProviderOptions &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT.
SessionOptionsImpl & EnableOrtCustomOps()
Wraps OrtApi::EnableOrtCustomOps.
size_t GetDimensionsCount() const
Wraps OrtApi::GetDimensionsCount.
R & At(const std::vector< int64_t > &location)
std::string GetAllocatorName() const
SessionOptionsImpl & AppendExecutionProvider_CUDA(const OrtCUDAProviderOptions &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA.
const int64_t * values_shape
SessionOptionsImpl & SetIntraOpNumThreads(int intra_op_num_threads)
Wraps OrtApi::SetIntraOpNumThreads.
static Value CreateSequence(std::vector< Value > &values)
Wraps OrtApi::CreateValue.
**But if you need a result
SessionOptionsImpl & RegisterCustomOpsUsingFunction(const char *function_name)
Wraps OrtApi::RegisterCustomOpsUsingFunction.
IoBinding(std::nullptr_t)
Create an empty object for convenience. Sometimes, we want to initialize members later.
SessionOptionsImpl & SetOptimizedModelFilePath(const ORTCHAR_T *optimized_model_file)
Wraps OrtApi::SetOptimizedModelFilePath.
Env(std::nullptr_t)
Create an empty Env object, must be assigned a valid one to be used.
ThreadingOptions & SetGlobalCustomJoinThreadFn(OrtCustomJoinThreadFn ort_custom_join_thread_fn)
Wraps OrtApi::SetGlobalCustomJoinThreadFn.
const R * GetSparseTensorValues() const
The API returns a pointer to an internal buffer of the sparse tensor containing non-zero values...
void * Alloc(size_t size)
TensorTypeAndShapeInfo GetSparseTensorIndicesTypeShapeInfo(OrtSparseIndicesFormat format) const
The API returns type and shape information for the specified indices. Each supported indices have the...
SessionOptionsImpl & SetGraphOptimizationLevel(GraphOptimizationLevel graph_optimization_level)
Wraps OrtApi::SetSessionGraphOptimizationLevel.
const R * GetTensorData() const
Returns a const typed pointer to the tensor contained data. No type checking is performed, the caller must ensure the type matches the tensor type.
SessionOptionsImpl & SetCustomThreadCreationOptions(void *ort_custom_thread_creation_options)
Wraps OrtApi::SessionOptionsSetCustomThreadCreationOptions.
void * GetTensorMutableRawData()
Returns a non-typed non-const pointer to a tensor contained data.
union Ort::detail::OrtSparseValuesParam::@93 data
static Op Create(const OrtKernelInfo *info, const char *op_name, const char *domain, int version, const char **type_constraint_names, const ONNXTensorElementDataType *type_constraint_values, size_t type_constraint_count, const OpAttr *attr_values, size_t attr_count, size_t input_count, size_t output_count)
std::string GetConfigEntry(const char *config_key) const
Wraps OrtApi::GetSessionConfigEntry.
void ThrowStatus(const Status &st)
std::vector< std::string > GetOutputNames() const
void(ORT_API_CALL * OrtLoggingFunction)(void *param, OrtLoggingLevel severity, const char *category, const char *logid, const char *code_location, const char *message)
SessionOptionsImpl & AppendExecutionProvider_TensorRT_V2(const OrtTensorRTProviderOptionsV2 &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_TensorRT.
GLuint GLsizei const GLuint const GLintptr * offsets
bool IsSparseTensor() const
Returns true if the OrtValue contains a sparse tensor
std::vector< Value > GetOutputValuesHelper(const OrtIoBinding *binding, OrtAllocator *)
void ReleaseKernelInfo(_Frees_ptr_opt_ OrtKernelInfo *info_copy)
ModelMetadata GetModelMetadata() const
Wraps OrtApi::SessionGetModelMetadata.
size_t GetDimensionsCount(_In_ const OrtTensorTypeAndShapeInfo *info)
ThreadingOptions & SetGlobalInterOpNumThreads(int inter_op_num_threads)
Wraps OrtApi::SetGlobalInterOpNumThreads.
void FillSparseTensorBlockSparse(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values, const Shape &indices_shape, const int32_t *indices_data)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
TypeInfo GetOutputTypeInfo(size_t index) const
std::vector< int64_t > GetShape() const
Uses GetDimensionsCount & GetDimensions to return a std::vector of the shape.
const T * GetTensorData(_Inout_ const OrtValue *value)
const OrtValue * KernelContext_GetInput(const OrtKernelContext *context, _In_ size_t index)
std::vector< int64_t > GetTensorShape(const OrtTensorTypeAndShapeInfo *info)
Wrapper around OrtMemoryInfo.
const OrtMemoryInfo * GetTensorMemoryInfo(_In_ const OrtValue *value)
std::vector< std::string > GetAvailableProviders()
This is a C++ wrapper for OrtApi::GetAvailableProviders() and returns a vector of strings representin...
Op(std::nullptr_t)
Create an empty Operator object, must be assigned a valid one to be used.
size_t GetTensorShapeElementCount(_In_ const OrtTensorTypeAndShapeInfo *info)
SessionOptions Clone() const
Creates and returns a copy of this SessionOptions object. Wraps OrtApi::CloneSessionOptions.
Wrapper around ::OrtIoBinding.
void GetAttrs(const OrtKernelInfo *p, const char *name, std::vector< float > &)
SessionOptionsImpl & SetCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn)
Wraps OrtApi::SessionOptionsSetCustomCreateThreadFn.
OrtKernelInfo * CopyKernelInfo(_In_ const OrtKernelInfo *info)
The Status that holds ownership of OrtStatus received from C API Use it to safely destroy OrtStatus* ...
OrtOpAttr * CreateOpAttr(_In_ const char *name, _In_ const void *data, _In_ int len, _In_ OrtOpAttrType type)
void GetStringTensorContent(void *buffer, size_t buffer_length, size_t *offsets, size_t offsets_count) const
The API copies all of the UTF-8 encoded string data contained within a tensor or a sparse tensor into...
void GetDimensions(int64_t *values, size_t values_count) const
Wraps OrtApi::GetDimensions.
A generic, discriminated value, whose type may be queried dynamically.
void ReleaseTensorTypeAndShapeInfo(OrtTensorTypeAndShapeInfo *input)
void GetAttr(const OrtKernelInfo *p, const char *name, float &)
CustomOpDomain(std::nullptr_t)
Create an empty CustomOpDomain object, must be assigned a valid one to be used.
SessionOptionsImpl & AppendExecutionProvider_OpenVINO(const OrtOpenVINOProviderOptions &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_OpenVINO.
const std::unordered_map< std::string, std::string > & GetFlattenedConfigs() const
Returns a flattened map of custom operator configuration entries and their values.
SessionOptionsImpl & AppendExecutionProvider_CUDA_V2(const OrtCUDAProviderOptionsV2 &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_CUDA_V2.
constexpr std::enable_if< I< type_count_base< T >::value, int >::type tuple_type_size(){return subtype_count< typename std::tuple_element< I, T >::type >::value+tuple_type_size< T, I+1 >);}template< typename T > struct type_count< T, typename std::enable_if< is_tuple_like< T >::value >::type >{static constexpr int value{tuple_type_size< T, 0 >)};};template< typename T > struct subtype_count{static constexpr int value{is_mutable_container< T >::value?expected_max_vector_size:type_count< T >::value};};template< typename T, typename Enable=void > struct type_count_min{static const int value{0};};template< typename T >struct type_count_min< T, typename std::enable_if<!is_mutable_container< T >::value &&!is_tuple_like< T >::value &&!is_wrapper< T >::value &&!is_complex< T >::value &&!std::is_void< T >::value >::type >{static constexpr int value{type_count< T >::value};};template< typename T > struct type_count_min< T, typename std::enable_if< is_complex< T >::value >::type >{static constexpr int value{1};};template< typename T >struct type_count_min< T, typename std::enable_if< is_wrapper< T >::value &&!is_complex< T >::value &&!is_tuple_like< T >::value >::type >{static constexpr int value{subtype_count_min< typename T::value_type >::value};};template< typename T, std::size_t I >constexpr typename std::enable_if< I==type_count_base< T >::value, int >::type tuple_type_size_min(){return 0;}template< typename T, std::size_t I > constexpr typename std::enable_if< I< type_count_base< T >::value, int >::type tuple_type_size_min(){return subtype_count_min< typename std::tuple_element< I, T >::type >::value+tuple_type_size_min< T, I+1 >);}template< typename T > struct type_count_min< T, typename std::enable_if< is_tuple_like< T >::value >::type >{static constexpr int value{tuple_type_size_min< T, 0 >)};};template< typename T > struct subtype_count_min{static constexpr int value{is_mutable_container< T >::value?((type_count< T >::value< expected_max_vector_size)?type_count< T >::value:0):type_count_min< T >::value};};template< typename T, typename Enable=void > struct expected_count{static const int value{0};};template< typename T >struct expected_count< T, typename std::enable_if<!is_mutable_container< T >::value &&!is_wrapper< T >::value &&!std::is_void< T >::value >::type >{static constexpr int value{1};};template< typename T > struct expected_count< T, typename std::enable_if< is_mutable_container< T >::value >::type >{static constexpr int value{expected_max_vector_size};};template< typename T >struct expected_count< T, typename std::enable_if<!is_mutable_container< T >::value &&is_wrapper< T >::value >::type >{static constexpr int value{expected_count< typename T::value_type >::value};};enum class object_category:int{char_value=1, integral_value=2, unsigned_integral=4, enumeration=6, boolean_value=8, floating_point=10, number_constructible=12, double_constructible=14, integer_constructible=16, string_assignable=23, string_constructible=24, other=45, wrapper_value=50, complex_number=60, tuple_value=70, container_value=80,};template< typename T, typename Enable=void > struct classify_object{static constexpr object_category value{object_category::other};};template< typename T >struct classify_object< T, typename std::enable_if< std::is_integral< T >::value &&!std::is_same< T, char >::value &&std::is_signed< T >::value &&!is_bool< T >::value &&!std::is_enum< T >::value >::type >{static constexpr object_category value{object_category::integral_value};};template< typename T >struct classify_object< T, typename std::enable_if< std::is_integral< T >::value &&std::is_unsigned< T >::value &&!std::is_same< T, char >::value &&!is_bool< T >::value >::type >{static constexpr object_category value{object_category::unsigned_integral};};template< typename T >struct classify_object< T, typename std::enable_if< std::is_same< T, char >::value &&!std::is_enum< T >::value >::type >{static constexpr object_category value{object_category::char_value};};template< typename T > struct classify_object< T, typename std::enable_if< is_bool< T >::value >::type >{static constexpr object_category value{object_category::boolean_value};};template< typename T > struct classify_object< T, typename std::enable_if< std::is_floating_point< T >::value >::type >{static constexpr object_category value{object_category::floating_point};};template< typename T >struct classify_object< T, typename std::enable_if<!std::is_floating_point< T >::value &&!std::is_integral< T >::value &&std::is_assignable< T &, std::string >::value >::type >{static constexpr object_category value{object_category::string_assignable};};template< typename T >struct classify_object< T, typename std::enable_if<!std::is_floating_point< T >::value &&!std::is_integral< T >::value &&!std::is_assignable< T &, std::string >::value &&(type_count< T >::value==1)&&std::is_constructible< T, std::string >::value >::type >{static constexpr object_category value{object_category::string_constructible};};template< typename T > struct classify_object< T, typename std::enable_if< std::is_enum< T >::value >::type >{static constexpr object_category value{object_category::enumeration};};template< typename T > struct classify_object< T, typename std::enable_if< is_complex< T >::value >::type >{static constexpr object_category value{object_category::complex_number};};template< typename T > struct uncommon_type{using type=typename std::conditional<!std::is_floating_point< T >::value &&!std::is_integral< T >::value &&!std::is_assignable< T &, std::string >::value &&!std::is_constructible< T, std::string >::value &&!is_complex< T >::value &&!is_mutable_container< T >::value &&!std::is_enum< T >::value, std::true_type, std::false_type >::type;static constexpr bool value=type::value;};template< typename T >struct classify_object< T, typename std::enable_if<(!is_mutable_container< T >::value &&is_wrapper< T >::value &&!is_tuple_like< T >::value &&uncommon_type< T >::value)>::type >{static constexpr object_category value{object_category::wrapper_value};};template< typename T >struct classify_object< T, typename std::enable_if< uncommon_type< T >::value &&type_count< T >::value==1 &&!is_wrapper< T >::value &&is_direct_constructible< T, double >::value &&is_direct_constructible< T, int >::value >::type >{static constexpr object_category value{object_category::number_constructible};};template< typename T >struct classify_object< T, typename std::enable_if< uncommon_type< T >::value &&type_count< T >::value==1 &&!is_wrapper< T >::value &&!is_direct_constructible< T, double >::value &&is_direct_constructible< T, int >::value >::type >{static constexpr object_category value{object_category::integer_constructible};};template< typename T >struct classify_object< T, typename std::enable_if< uncommon_type< T >::value &&type_count< T >::value==1 &&!is_wrapper< T >::value &&is_direct_constructible< T, double >::value &&!is_direct_constructible< T, int >::value >::type >{static constexpr object_category value{object_category::double_constructible};};template< typename T >struct classify_object< T, typename std::enable_if< is_tuple_like< T >::value &&((type_count< T >::value >=2 &&!is_wrapper< T >::value)||(uncommon_type< T >::value &&!is_direct_constructible< T, double >::value &&!is_direct_constructible< T, int >::value)||(uncommon_type< T >::value &&type_count< T >::value >=2))>::type >{static constexpr object_category value{object_category::tuple_value};};template< typename T > struct classify_object< T, typename std::enable_if< is_mutable_container< T >::value >::type >{static constexpr object_category value{object_category::container_value};};template< typename T, enable_if_t< classify_object< T >::value==object_category::char_value, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"CHAR";}template< typename T, enable_if_t< classify_object< T >::value==object_category::integral_value||classify_object< T >::value==object_category::integer_constructible, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"INT";}template< typename T, enable_if_t< classify_object< T >::value==object_category::unsigned_integral, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"UINT";}template< typename T, enable_if_t< classify_object< T >::value==object_category::floating_point||classify_object< T >::value==object_category::number_constructible||classify_object< T >::value==object_category::double_constructible, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"FLOAT";}template< typename T, enable_if_t< classify_object< T >::value==object_category::enumeration, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"ENUM";}template< typename T, enable_if_t< classify_object< T >::value==object_category::boolean_value, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"BOOLEAN";}template< typename T, enable_if_t< classify_object< T >::value==object_category::complex_number, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"COMPLEX";}template< typename T, enable_if_t< classify_object< T >::value >=object_category::string_assignable &&classify_object< T >::value<=object_category::other, detail::enabler >=detail::dummy >constexpr const char *type_name(){return"TEXT";}template< typename T, enable_if_t< classify_object< T >::value==object_category::tuple_value &&type_count_base< T >::value >=2, detail::enabler >=detail::dummy >std::string type_name();template< typename T, enable_if_t< classify_object< T >::value==object_category::container_value||classify_object< T >::value==object_category::wrapper_value, detail::enabler >=detail::dummy >std::string type_name();template< typename T, enable_if_t< classify_object< T >::value==object_category::tuple_value &&type_count_base< T >::value==1, detail::enabler >=detail::dummy >inline std::string type_name(){return type_name< typename std::decay< typename std::tuple_element< 0, T >::type >::type >);}template< typename T, std::size_t I >inline typename std::enable_if< I==type_count_base< T >::value, std::string >::type tuple_name(){return std::string{};}template< typename T, std::size_t I >inline typename std::enable_if<(I< type_count_base< T >::value), std::string >::type tuple_name(){auto str=std::string{type_name< typename std::decay< typename std::tuple_element< I, T >::type >::type >)}+ ','+tuple_name< T, I+1 >);if(str.back()== ',') str.pop_back();return str;}template< typename T, enable_if_t< classify_object< T >::value==object_category::tuple_value &&type_count_base< T >::value >=2, detail::enabler > > std::string type_name()
Recursively generate the tuple type name.
static Value CreateTensor(const OrtMemoryInfo *info, T *p_data, size_t p_data_element_count, const int64_t *shape, size_t shape_len)
Creates a tensor with a user supplied buffer. Wraps OrtApi::CreateTensorWithDataAsOrtValue.
GLint GLint GLsizei GLint GLenum format
const OrtApi & GetApi()
This returns a reference to the OrtApi interface in use.
ThreadingOptions & SetGlobalIntraOpNumThreads(int intra_op_num_threads)
Wraps OrtApi::SetGlobalIntraOpNumThreads.
GraphOptimizationLevel
Graph optimization level.
size_t KernelContext_GetOutputCount(const OrtKernelContext *context)
Value GetTensorAttribute(const char *name, OrtAllocator *allocator) const
void Add(const OrtCustomOp *op)
Wraps CustomOpDomain_Add.
KernelInfo(std::nullptr_t)
Create an empty instance to initialize later.
Represents native memory allocation coming from one of the OrtAllocators registered with OnnxRuntime...
Session(std::nullptr_t)
Create an empty Session object, must be assigned a valid one to be used.
ThreadingOptions & SetGlobalDenormalAsZero()
Wraps OrtApi::SetGlobalDenormalAsZero.
IEEE 754 half-precision floating point data type.
OpenVINO Provider Options.
size_t GetInputCount() const
SessionOptionsImpl & SetInterOpNumThreads(int inter_op_num_threads)
Wraps OrtApi::SetInterOpNumThreads.
size_t GetOutputCount() const
Options for the TensorRT provider that are passed to SessionOptionsAppendExecutionProvider_TensorRT_V...
OrtStatus *ORT_API_CALL * CreateStatus(OrtErrorCode code, _In_ const char *msg) NO_EXCEPTION ORT_ALL_ARGS_NONNULL
Create an OrtStatus from a null terminated string.
R * GetTensorMutableData()
Returns a non-const typed pointer to an OrtValue/Tensor contained buffer No type checking is performe...
SessionOptions()
Wraps OrtApi::CreateSessionOptions.
void * GetGPUComputeStream() const
void BindOutput(const char *name, const Value &)
GLuint const GLchar * name
RunOptions & SetRunTag(const char *run_tag)
wraps OrtApi::RunOptionsSetRunTag
Allocator(std::nullptr_t)
Convenience to create a class member and then replace with an instance.
SessionOptionsImpl & EnableProfiling(const ORTCHAR_T *profile_file_prefix)
Wraps OrtApi::EnableProfiling.
size_t GetStringTensorDataLength() const
This API returns a full length of string data contained within either a tensor or a sparse Tensor...
RunOptions & SetTerminate()
Terminates all currently executing Session::Run calls that were made using this RunOptions instance...
OrtAllocatorType GetAllocatorType() const
size_t GetInputCount() const
TypeInfo GetSequenceElementType() const
Wraps OrtApi::GetSequenceElementType.
static Value CreateSparseTensor(const OrtMemoryInfo *info, T *p_data, const Shape &dense_shape, const Shape &values_shape)
This is a simple forwarding method to the other overload that helps deducing data type enum value fro...
TypeInfo GetOutputTypeInfo(size_t index) const
Wraps OrtApi::SessionGetOutputTypeInfo.
size_t GetStringTensorElementLength(size_t element_index) const
The API returns a byte length of UTF-8 encoded string element contained in either a tensor or a spare...
detail::ValueImpl< detail::Unowned< OrtValue >> UnownedValue
SessionOptionsImpl & Add(OrtCustomOpDomain *custom_op_domain)
Wraps OrtApi::AddCustomOpDomain.
TensorTypeAndShapeInfo GetSparseTensorValuesTypeAndShapeInfo() const
The API returns type and shape information for stored non-zero values of the sparse tensor...
GT_API const UT_StringHolder version
TypeInfo GetTypeInfo() const
The API returns type information for data contained in a tensor. For sparse tensors it returns type i...
RunOptions & SetRunLogSeverityLevel(int)
Wraps OrtApi::RunOptionsSetRunLogSeverityLevel.
ConstTensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const
Wraps OrtApi::CastTypeInfoToTensorInfo.
TensorRT Provider Options.
bool operator==(const MemoryInfoImpl< U > &o) const
TypeInfo GetMapValueType() const
Wraps OrtApi::GetMapValueType.
RunOptions()
Wraps OrtApi::CreateRunOptions.
OrtTensorTypeAndShapeInfo * GetTensorTypeAndShape(_In_ const OrtValue *value)
This struct owns the OrtKernInfo* pointer when a copy is made. For convenient wrapping of OrtKernelIn...
void FillSparseTensorCoo(const OrtMemoryInfo *data_mem_info, const OrtSparseValuesParam &values_param, const int64_t *indices_data, size_t indices_num)
The API will allocate memory using the allocator instance supplied to the CreateSparseTensor() API an...
std::vector< Value > GetOutputValues() const
OrtErrorCode GetErrorCode() const
const char * what() const noexceptoverride
struct OrtKernelInfo OrtKernelInfo
void FillStringTensor(const char *const *s, size_t s_len)
Set all strings at once in a string tensor
KernelContext(OrtKernelContext *context)
ArenaCfg(std::nullptr_t)
Create an empty ArenaCfg object, must be assigned a valid one to be used.
GT_API const UT_StringHolder st
ThreadingOptions()
Wraps OrtApi::CreateThreadingOptions.
void ReleaseOpAttr(_Frees_ptr_opt_ OrtOpAttr *op_attr)
GLenum GLsizei GLsizei GLint * values
void SetDimensions(OrtTensorTypeAndShapeInfo *info, _In_ const int64_t *dim_values, size_t dim_count)
std::string GetOutputName(size_t index) const
static Value CreateMap(Value &keys, Value &values)
Wraps OrtApi::CreateValue.
void GetSessionConfigs(std::unordered_map< std::string, std::string > &out, ConstSessionOptions options) const
TypeInfo GetOverridableInitializerTypeInfo(size_t index) const
Wraps OrtApi::SessionGetOverridableInitializerTypeInfo.
SessionOptionsImpl & AddConfigEntry(const char *config_key, const char *config_value)
Wraps OrtApi::AddSessionConfigEntry.
Memory allocation interface.
OrtOp * CreateOp(_In_ const OrtKernelInfo *info, _In_ const char *op_name, _In_ const char *domain, _In_ int version, _In_opt_ const char **type_constraint_names, _In_opt_ const ONNXTensorElementDataType *type_constraint_values, _In_opt_ int type_constraint_count, _In_opt_ const OrtOpAttr *const *attr_values, _In_opt_ int attr_count, _In_ int input_count, _In_ int output_count)
SessionOptionsImpl & SetExecutionMode(ExecutionMode execution_mode)
Wraps OrtApi::SetSessionExecutionMode.
AllocatorWithDefaultOptions()
AllocatedStringPtr EndProfilingAllocated(OrtAllocator *allocator)
End profiling and return a copy of the profiling file name.
SessionOptionsImpl & AppendExecutionProvider(const std::string &provider_name, const std::unordered_map< std::string, std::string > &provider_options={})
Wraps OrtApi::SessionOptionsAppendExecutionProvider. Currently supports SNPE and XNNPACK.
SessionOptionsImpl & DisableProfiling()
Wraps OrtApi::DisableProfiling.
OrtValue * KernelContext_GetOutput(OrtKernelContext *context, _In_ size_t index, _In_ const int64_t *dim_values, size_t dim_count)
RunOptions & UnsetTerminate()
Clears the terminate flag so this RunOptions instance can be used in a new Session::Run call without ...
size_t GetOverridableInitializerCount() const
Returns the number of inputs that have defaults that can be overridden.
const char * GetRunTag() const
Wraps OrtApi::RunOptionsGetRunTag.
int GetRunLogVerbosityLevel() const
Wraps OrtApi::RunOptionsGetRunLogVerbosityLevel.
size_t GetOutputCount() const
Returns the number of model outputs.
SessionOptionsImpl & AddExternalInitializers(const std::vector< std::string > &names, const std::vector< Value > &ort_values)
Wraps OrtApi::AddExternalInitializers.
TensorTypeAndShapeInfo GetTensorTypeAndShapeInfo() const
The API returns type information for data contained in a tensor. For sparse tensors it returns type i...
ONNXTensorElementDataType
int GetRunLogSeverityLevel() const
Wraps OrtApi::RunOptionsGetRunLogSeverityLevel.
ConstMemoryInfo GetTensorMemoryInfo() const
This API returns information about the memory allocation used to hold data.
AllocatedStringPtr GetInputNameAllocated(size_t index, OrtAllocator *allocator) const
Returns a copy of input name at the specified index.
Wrapper around ::OrtTensorTypeAndShapeInfo.
MemoryInfo(std::nullptr_t)
No instance is created.
std::string MakeCustomOpConfigEntryKey(const char *custom_op_name, const char *config)
CustomOpConfigs.
size_t GetCount() const
< Return true if OrtValue contains data and returns false if the OrtValue is a None ...
std::string GetInputName(size_t index) const
ThreadingOptions & SetGlobalCustomCreateThreadFn(OrtCustomCreateThreadFn ort_custom_create_thread_fn)
Wraps OrtApi::SetGlobalCustomCreateThreadFn.
Wrapper around ::OrtSessionOptions.
OrtSparseFormat GetSparseFormat() const
The API returns the sparse data format this OrtValue holds in a sparse tensor. If the sparse tensor w...
detail::MemoryInfoImpl< detail::Unowned< const OrtMemoryInfo >> ConstMemoryInfo
void UseCooIndices(int64_t *indices_data, size_t indices_num)
Supplies COO format specific indices and marks the contained sparse tensor as being a COO format tens...
MemoryAllocation(OrtAllocator *allocator, void *p, size_t size)
CustomOpConfigs & AddConfig(const char *custom_op_name, const char *config_key, const char *config_value)
Adds a session configuration entry/value for a specific custom operator.
void(* OrtCustomJoinThreadFn)(OrtCustomThreadHandle ort_custom_thread_handle)
Custom thread join function.
void ThrowOnError(OrtStatus *result)
TypeInfo GetInputTypeInfo(size_t index) const
SessionOptionsImpl & SetLogSeverityLevel(int level)
Wraps OrtApi::SetSessionLogSeverityLevel.
Wrapper around OrtAllocator default instance that is owned by Onnxruntime.
Wrapper around ::OrtSession.
SessionOptionsImpl & DisableMemPattern()
Wraps OrtApi::DisableMemPattern.
SessionOptionsImpl & AppendExecutionProvider_ROCM(const OrtROCMProviderOptions &provider_options)
Wraps OrtApi::SessionOptionsAppendExecutionProvider_ROCM.
Options for the CUDA provider that are passed to SessionOptionsAppendExecutionProvider_CUDA_V2. Please note that this struct is similar to OrtCUDAProviderOptions but only to be used internally. Going forward, new cuda provider options are to be supported via this struct and usage of the publicly defined OrtCUDAProviderOptions will be deprecated over time. User can only get the instance of OrtCUDAProviderOptionsV2 via CreateCUDAProviderOptions.
RunOptions & SetRunLogVerbosityLevel(int)
Wraps OrtApi::RunOptionsSetRunLogVerbosityLevel.
ThreadingOptions & SetGlobalCustomThreadCreationOptions(void *ort_custom_thread_creation_options)
Wraps OrtApi::SetGlobalCustomThreadCreationOptions.
struct OrtKernelContext OrtKernelContext
bfloat16 (Brain Floating Point) data type
OrtLoggingLevel
Logging severity levels.
Class that represents session configuration entries for one or more custom operators.
size_t KernelContext_GetInputCount(const OrtKernelContext *context)
std::vector< std::string > GetOutputNamesHelper(const OrtIoBinding *binding, OrtAllocator *)
#define ORT_CXX_API_THROW(string, code)
OrtMemType
Memory types for allocated memory, execution provider specific types should be extended in each provi...
SessionOptionsImpl & RegisterCustomOpsLibrary(const ORTCHAR_T *library_name, const CustomOpConfigs &custom_op_configs={})
T KernelInfoGetAttribute(_In_ const OrtKernelInfo *info, _In_ const char *name)
GLsizei GLenum GLenum GLuint GLenum GLsizei * lengths
ConstSequenceTypeInfo GetSequenceTypeInfo() const
Wraps OrtApi::CastTypeInfoToSequenceTypeInfo.
MIGraphX Provider Options.