Posts ONNX runtime
Post
Cancel

ONNX runtime

Posted Mar 1 2022-03-01T18:45:47+09:00 by dnchoi
Updated Mar 13 2022-03-13T00:39:53+09:00

ONNX Runtime Example

1. Export ONNX

pytorch

  • static batch size

고정된 batch size의 onnx모델로 변환하는 방법은 input tensor의 shape을 넣어줄 때 원하는 size의 batch를 설정해서 export해주면 된다.

모델은 기본적으로 pytorch에서 제공해 주는 resnet18을 load했고, 생성될 onnx 모델의 이름, input과 output 이름, 그리고 input tensor를 원하는 shape로 생성하여 설정했다.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

import os
import torch
import torchvision.models as models
from loguru import logger

net = models.resnet18(pretrained=True)
net.eval()
logger.info('Finished loading model!')
device = torch.device("cuda:0")
net = net.to(device)

output_onnx = 'static.onnx'
input_names = ["input_0"]
output_names = ["output_0"]

inputs = torch.randn(2, 3, 256, 256).to(device)

torch_out = torch.onnx._export(net, inputs, output_onnx, export_params=True, verbose=False, 
                               input_names=input_names, output_names=output_names, 
                               opset_version=11)

static

  • dynamic batch size

dynamic batch로 설정하고 싶으면 export모델을 사용할 때 option에 dynamic_axes를 설정해 주면 된다.

dynamic_axes를 설정해 주는 것 말고 위와 같기 때문에 모델과 output 이름 등등은 같게 설정하였다.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

import os
import argparse
import torch
import torchvision.models as models
from loguru import logger

net = models.resnet18(pretrained=True)
net.eval()
logger.info('Finished loading model!')
device = torch.device("cuda:0")
net = net.to(device)

output_onnx = 'dynamic.onnx'
input_names = ["input_0"]
output_names = ["output_0"]
inputs = torch.randn(1, 3, 256, 256).to(device)
logger.info(net(inputs))

dynamic_axes = {'input_0' : {0 : 'batch_size'},
                    'output_0' : {0 : 'batch_size'}}

torch_out = torch.onnx._export(net, inputs, output_onnx, export_params=True, verbose=False,
                               input_names=input_names, output_names=output_names, 
                               opset_version=11, dynamic_axes = dynamic_axes)

logger.info(torch_out)

dynamic

tensorflow - todo

keras - todo

2. Inference ONNX-Runtime engine

Inference

  • PYTHON
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

import torch
import torchvision.transforms as transforms

import onnx
import onnxruntime
import os
from loguru import logger
import cv2

from glob import glob
from tqdm import tqdm 

logger.level("DEBUG")

@logger.catch()
class verify_face():
    def __init__(self, model):
        logger.trace("pytorch: {}".format(torch.__version__))
        logger.trace("onnxruntime: {}".format(onnxruntime.__version__))
        logger.trace("onnx: {}".format(onnx.__version__))
        model_path = model
        
        logger.trace("model file is exits : {}".format(os.path.isfile(model_path)))
        onnx_model = onnx.load(model_path)
        logger.trace("onnx model check : {}".format(onnx.checker.check_model(onnx_model)))

        self.ort_session = onnxruntime.InferenceSession(model_path)
        
        self.to_tensor = transforms.ToTensor()
        
    def _resize(self, img, model_size):
        _shape = img.shape
        re = None
        if model_size != _shape:
            re = cv2.resize(img, dsize=model_size[0:2])
        else:
            re = img
        return re
    
    def _tensor(self, img):
        img = self._resize(img, (112, 112, 3))
        output = self.to_tensor(img)
        output.unsqueeze_(0)
        logger.trace(output.shape)
        output = self.to_numpy(output)
        ort_inputs = {self.ort_session.get_inputs()[0].name: output}
        return ort_inputs
    
    def to_numpy(self, tensor):
        return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
    
    def do_inference(self, img):
        ort_outs = self.ort_session.run(None, self._tensor(img))    
        
        return ort_outs[0]



def foo(verify, img):
    img_out = verify.do_inference(img)
    logger.info(img_out.shape)


if __name__ == "__main__":
    model_path = "model.onnx"
    _verify = verify_face(model_path)
    file_path = "./DATASET/LFW/lfw-py/lfw_funneled"
    
    files = glob(os.path.join(file_path, "*", "*"))
    for i in tqdm(files):    
        img = cv2.imread(i)
        foo(_verify, img)
  • C++
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

|—— 12.png
|—— CMakeLists.txt
|—— build_run.sh
|—— example.cpp
|—— libs
|    |—— configparser
|        |—— CMakeLists.txt
|        |—— configparser.cpp
|        |—— include
|            |—— configparser.h
|    |—— onnxruntime
|        |—— CMakeLists.txt
|        |—— frvf_onnx.cpp
|        |—— include
|            |—— frvf_onnx.h
|    |—— spdlog
|        |—— include
|            |—— spdlog
|            |—— .
|            |—— .
|            |—— .
|            |—— .
|—— main.cpp
|—— model.onnx
|—— test.ini

Build

build_run.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

#!/bin/sh

# exit on first error
set -e
rm -rf build
mkdir -p build
cd build

# Generate a Makefile for GCC (or Clang, depanding on CC/CXX envvar)
cmake ..

# Build (ie 'make')
# cmake --build .
make all
cd ..

./build/main

CMakeFiles

CMakeLists.txt

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78

cmake_minimum_required(VERSION 3.13)

# 프로젝트 정보
project(
  main
  VERSION 0.1
  DESCRIPTION "Face recognition verify test"
  LANGUAGES CXX
)
add_compile_definitions(_DEBUG_)

set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
set(CMAKE_BUILD_TYPE RelWithDebInfo)
set(CMAKE_CXX_FLAGS -pthread)
set(CMAKE_CXX_STANDARD 14)

message(STATUS CMAKE_BUILD_TYPE)                     # -- CMAKE_BUILD_TYPE
message(STATUS ${CMAKE_BUILD_TYPE})                  # -- Debug
message(STATUS "Configuration: ${CMAKE_BUILD_TYPE}") # -- Configuration: Debug
message(STATUS "Compiler")
message(STATUS " - ID       \t: ${CMAKE_CXX_COMPILER_ID}")
message(STATUS " - Version  \t: ${CMAKE_CXX_COMPILER_VERSION}")
message(STATUS " - Path     \t: ${CMAKE_CXX_COMPILER}\n\n")


set (onnx_Header "${CMAKE_CURRENT_SOURCE_DIR}/libs/onnxruntime/include")
set (logger_Header "${CMAKE_SOURCE_DIR}/libs/spdlog/include")
set (config_Header "${CMAKE_SOURCE_DIR}/libs/configparser/include")

message("${onnx_Header}")
message("${logger_Header}")
message("${config_Header}")

include_directories(
    ${onnx_Header}
    ${logger_Header}
    ${config_Header}
)

find_package(OpenCV REQUIRED)
find_path(ONNX_RUNTIME_SESSION_INCLUDE_DIRS onnxruntime_cxx_api.h HINTS /usr/local/include/onnxruntime/core/session/)
find_library(ONNX_RUNTIME_LIB onnxruntime HINTS /usr/local/lib)

#INIT
# Lib 등록을 위한 작업 -> 해당 Lib Cmake 동작 후 build
add_subdirectory(
    libs/onnxruntime
)
message("ONNX runtime lib")
add_subdirectory(
    libs/configparser
)
message("configparser lib")
# add_subdirectory(
#     lib/ThreadPool
# )
# message("Thread pool shared lib build done")
# #INIT

add_executable (${PROJECT_NAME} main.cpp)

# include lib file in program
target_link_libraries(
    ${PROJECT_NAME}
    frvf_onnx
    configparser
    pthread
)
target_include_directories(
    ${PROJECT_NAME} 
    PUBLIC 
        ${onnx_Header} 
        ${logger_Header}
        ${config_Header}
)

target_compile_features(${PROJECT_NAME} PUBLIC cxx_std_17)

libs/onnxruntime/CMakeLists.txt

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

cmake_minimum_required(VERSION 3.13)
project(
  frvf_onnx
  VERSION 0.1
  LANGUAGES CXX
)
message("@@ frvf_onnx CMake Start @@")
set(CMAKE_BUILD_TYPE RelWithDebInfo)
set(CMAKE_CXX_STANDARD 14)
message("This lib name : ${PROJECT_NAME}")

include_directories(include)

add_library(${PROJECT_NAME} SHARED frvf_onnx.cpp)
message("ONNX RUNTIME INCLUDE DIRS : ${ONNX_RUNTIME_SESSION_INCLUDE_DIRS}")
message("OPENCV INCLUDE DIRS : ${OpenCV_INCLUDE_DIRS}")
message("ONNX RUNTIME LIB : ${ONNX_RUNTIME_LIB}")
message("OpenCV LIBRARIES : ${OpenCV_LIBRARIES}")
target_include_directories(${PROJECT_NAME} PUBLIC ${ONNX_RUNTIME_SESSION_INCLUDE_DIRS} ${OpenCV_INCLUDE_DIRS})
target_link_libraries(${PROJECT_NAME} PUBLIC ${ONNX_RUNTIME_LIB} ${OpenCV_LIBRARIES})
target_compile_features(${PROJECT_NAME} PUBLIC cxx_std_17)

message("@@ frvf_onnx CMake Finish @@\n\n")

Code

src/main.cpp

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

#include "iostream"
#include "frvf_onnx.h"
#include <spdlog/spdlog.h>
#include "configparser.h"

// #include "spdlog/sinks/basic_file_sink.h"

#define SPDLOG_ACTIVE_LEVEL SPDLOG_LEVEL_DEBUG

int main(int argc, char* argv[]){
    /**
     * @brief Construct a new frvf onnx::frvf onnx object and get Instence
     * @arg model_path, useCUDA, optimizer
     * @param model_path type string 
     * @param useCUDA type bool 
     * @param optimizer type int 
     * 0 = ORT_DISABLE_ALL : To disable all optimizations
     * 1 = ORT_ENABLE_BASIC : To enable basic optimizations (Such as redundant node removals) 
     * 2 = ORT_ENABLE_EXTENDED : To enable extended optimizations(Includes level 1 + more complex optimizations like node fusions)
     * 3 = ORT_ENABLE_ALL : To Enable All possible optimizations
     */
    
    // auto file_logger = spdlog::basic_logger_mt("basic_logger", "logs/basic.txt");
    // spdlog::set_default_logger(file_logger);
    struct Arg
    {
        int _B;
        int _W;
        int _H;
        int _C;
        int _iter;
        int _acc;
        int _opti;
        std::string _model;
        std::string _engine;
    };
    
    Arg args;
    CConfigParser config("test.ini");
	if (config.IsSuccess()) {
        args._B = config.GetInt("B");
        args._W = config.GetInt("W");
        args._H = config.GetInt("H");
        args._C = config.GetInt("C");
        args._iter = config.GetInt("ITERATION");
        args._acc = config.GetInt("ACCELERATOR");

        args._model = config.GetString("MODEL");
        args._engine = config.GetString("ENGINE");
	}
    
    SPDLOG_INFO("batch size : {}", args._B);
    SPDLOG_INFO("input width : {}", args._W);
    SPDLOG_INFO("input height : {}", args._H);
    SPDLOG_INFO("input channel : {}", args._C);
    SPDLOG_INFO("iteration number : {}", args._iter);
    SPDLOG_INFO("accelerator : {}", args._acc);
    SPDLOG_INFO("model path : {}", args._model);
    SPDLOG_INFO("optimizer : {}", args._engine);
    if(args._engine == "onnx"){
        onnx_frvf::frvf_onnx *onnx;
        onnx = new onnx_frvf::frvf_onnx("model.onnx", true, 0);
        std::vector<float> result;
        float avg_ms = 0.0;
        for(int i = 0; i < 1000; i++){
            result.push_back(onnx->do_inference("12.png"));
        }
        for(int q = 0; q < result.size(); q++)
        {
            avg_ms += result[q];
        }
        avg_ms = avg_ms / 1000.0;
        SPDLOG_CRITICAL("{:03.8f}", avg_ms);
    }
    else{
        return 0;
    }
	return 0;
}

libs/onnxruntime/include/frvf_onnx.h

#ifndef __FRVF_ONNX_H__
#define __FRVF_ONNX_H__

#include <iostream>
#include <onnxruntime_cxx_api.h>
#include <opencv2/dnn/dnn.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/imgproc.hpp>
#include <chrono>
#include <cmath>
#include <exception>
#include <fstream>
#include <iostream>
#include <limits>
#include <numeric>
#include <string>
#include <vector>
#include "spdlog/spdlog.h"

namespace onnx_frvf{
    
    class frvf_onnx
    {
    private:
        Ort::Env *env;
        Ort::Session *sess;
        Ort::SessionOptions *sessionOptions;
        Ort::AllocatorWithDefaultOptions *allocator;
        GraphOptimizationLevel optimizer_selector(int expression);
        size_t numInputNodes;
        const char* inputName;
        std::vector<int64_t> inputDims;
        size_t numOutputNodes;
        const char* outputName;
        std::vector<int64_t> outputDims;
        
        std::vector<const char*> inputNames;
        std::vector<const char*> outputNames;
        // std::vector<Ort::Value> inputTensors;
        // std::vector<Ort::Value> outputTensors;

    public:
        frvf_onnx(std::string file_path, bool useCUDA, int OPT_OPTION);
        ~frvf_onnx();

        void _Instance(std::string file_path, bool useCUDA, int OPT_OPTION);
        float do_inference(std::string imageFilepath);
    };

}

#endif // __FRVF_ONNX_H__

libs/onnxruntime/frvf_onnx.cpp

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227

#include <frvf_onnx.h>

using namespace onnx_frvf;

template <typename T>
T vectorProduct(const std::vector<T>& v)
{
    return accumulate(v.begin(), v.end(), 1, std::multiplies<T>());
}

template <typename T>
std::ostream& operator<<(std::ostream& os, const std::vector<T>& v)
{
    os << "[";
    for (int i = 0; i < v.size(); ++i)
    {
        os << v[i];
        if (i != v.size() - 1)
        {
            os << ", ";
        }
    }
    os << "]";
    return os;
}


std::ostream& operator<<(std::ostream& os,
                         const ONNXTensorElementDataType& type)
{
    switch (type)
    {
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED:
            os << "undefined";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT:
            os << "float";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8:
            os << "uint8_t";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8:
            os << "int8_t";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16:
            os << "uint16_t";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16:
            os << "int16_t";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32:
            os << "int32_t";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64:
            os << "int64_t";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING:
            os << "std::string";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL:
            os << "bool";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT16:
            os << "float16";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE:
            os << "double";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32:
            os << "uint32_t";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64:
            os << "uint64_t";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX64:
            os << "float real + float imaginary";
            break;
        case ONNXTensorElementDataType::
            ONNX_TENSOR_ELEMENT_DATA_TYPE_COMPLEX128:
            os << "double real + float imaginary";
            break;
        case ONNXTensorElementDataType::ONNX_TENSOR_ELEMENT_DATA_TYPE_BFLOAT16:
            os << "bfloat16";
            break;
        default:
            break;
    }

    return os;
}

/**
 * @brief Construct a new frvf onnx::frvf onnx object
 * 
 * @param file_path 
 * @param useCUDA 
 * @param OPT_OPTION 
 */
frvf_onnx::frvf_onnx(std::string file_path, bool useCUDA, int OPT_OPTION){
    
    this->_Instance(file_path, useCUDA, OPT_OPTION);
}

GraphOptimizationLevel frvf_onnx::optimizer_selector(int expression){
    GraphOptimizationLevel a;
    switch (expression)
    {
    case 0:
        a = GraphOptimizationLevel::ORT_DISABLE_ALL;
        break;
    case 1:
        a = GraphOptimizationLevel::ORT_ENABLE_BASIC;
        break;
    case 2:
        a = GraphOptimizationLevel::ORT_ENABLE_EXTENDED;
        break;
    case 3:
        a = GraphOptimizationLevel::ORT_ENABLE_ALL;
        break;
    default:
        a = GraphOptimizationLevel::ORT_DISABLE_ALL;
        break;
    }

    return a;
}

void frvf_onnx::_Instance(std::string file_path, bool useCUDA, int OPT_OPTION)
{
#ifdef _DEBUG_
    std::string modelFilepath = file_path;
    std::string instanceName{"ONNX-face-recognition"};
    sessionOptions = new Ort::SessionOptions;
    sessionOptions->SetIntraOpNumThreads(1);
    if (useCUDA)
    {
        OrtCUDAProviderOptions cuda_options{0};
        sessionOptions->AppendExecutionProvider_CUDA(cuda_options);
    }
    env = new Ort::Env(OrtLoggingLevel::ORT_LOGGING_LEVEL_WARNING, instanceName.c_str());

    sessionOptions->SetGraphOptimizationLevel(optimizer_selector(OPT_OPTION));
    sess = new Ort::Session(*env, modelFilepath.c_str(), *sessionOptions);
    allocator = new Ort::AllocatorWithDefaultOptions;
    numInputNodes = sess->GetInputCount();
    numOutputNodes = sess->GetOutputCount();
    inputName = sess->GetInputName(0, *allocator);
    Ort::TypeInfo inputTypeInfo = sess->GetInputTypeInfo(0);
    auto inputTensorInfo = inputTypeInfo.GetTensorTypeAndShapeInfo();
    ONNXTensorElementDataType inputType = inputTensorInfo.GetElementType();
    inputDims = inputTensorInfo.GetShape();
    outputName = sess->GetOutputName(0, *allocator);
    Ort::TypeInfo outputTypeInfo = sess->GetOutputTypeInfo(0);
    auto outputTensorInfo = outputTypeInfo.GetTensorTypeAndShapeInfo();
    ONNXTensorElementDataType outputType = outputTensorInfo.GetElementType();
    outputDims = outputTensorInfo.GetShape();

    SPDLOG_INFO("Number of Input Nodes : {}",numInputNodes);
    SPDLOG_INFO("Number of Output Nodes : {}",numOutputNodes);
    SPDLOG_INFO("Input Name : {}",inputName);
    SPDLOG_INFO("Input Type : {}",inputType);
    SPDLOG_INFO("Input Dimensions : {} {} {} {}",inputDims[0],inputDims[1],inputDims[2],inputDims[3]);
    SPDLOG_INFO("Output Name : {}",outputName);
    SPDLOG_INFO("Output Type : {}",outputType);
    SPDLOG_INFO("Output Dimensions : {} {} {} {}",outputDims[0],outputDims[1],outputDims[2],outputDims[3]);

    inputNames.push_back(inputName);
    outputNames.push_back(outputName);

#else
#endif
}

float frvf_onnx::do_inference(std::string imageFilepath){
    cv::Mat imageBGR2= cv::Mat::zeros(1, 1, CV_64F);
    cv::Mat imageBGR = cv::imread(imageFilepath, cv::ImreadModes::IMREAD_COLOR);
    cv::Mat resizedImageBGR, resizedImageRGB, resizedImage, preprocessedImage;
    cv::resize(imageBGR, resizedImageBGR,
               cv::Size(inputDims.at(2), inputDims.at(3)),
               cv::InterpolationFlags::INTER_CUBIC);
    cv::cvtColor(resizedImageBGR, resizedImageRGB,
                 cv::ColorConversionCodes::COLOR_BGR2RGB);
    resizedImageRGB.convertTo(resizedImage, CV_32F, 1.0 / 255);

    cv::dnn::blobFromImage(resizedImage, preprocessedImage);

    size_t inputTensorSize = vectorProduct(inputDims);
    std::vector<float> inputTensorValues(inputTensorSize);
    inputTensorValues.assign(preprocessedImage.begin<float>(),
                             preprocessedImage.end<float>());
                             
    size_t outputTensorSize = vectorProduct(outputDims);
    std::vector<float> outputTensorValues(outputTensorSize);

    std::vector<Ort::Value> inputTensors;
    std::vector<Ort::Value> outputTensors;
    
    Ort::MemoryInfo memoryInfo = Ort::MemoryInfo::CreateCpu(
        OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);
    inputTensors.push_back(Ort::Value::CreateTensor<float>(
        memoryInfo, inputTensorValues.data(), inputTensorSize, inputDims.data(),
        inputDims.size()));
    outputTensors.push_back(Ort::Value::CreateTensor<float>(
        memoryInfo, outputTensorValues.data(), outputTensorSize,
        outputDims.data(), outputDims.size()));
   
    std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();

    sess->Run(Ort::RunOptions{nullptr}, inputNames.data(),
                inputTensors.data(), 1, outputNames.data(),
                outputTensors.data(), 1);

    for (int _i=0; _i < outputDims[1]; _i ++)
    {
        SPDLOG_TRACE("{}",outputTensorValues.at(_i));
    }

    std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now();

    float processtime = std::chrono::duration_cast<std::chrono::milliseconds>(end - begin).count();

    return processtime;
}

frvf_onnx::~frvf_onnx(){
    
}
onnx, python, c++
onnx python c++
This post is licensed under CC BY 4.0 by the author.
Recent Update
Contents

ONNX runtime install

TensorRT runtime installer