Note

This is the documentation for the latest development branch and may refer to features that are not available in released versions. If you are looking for the documentation for a specific release, use the drop-down menu on the left and select the desired version.

nncase Model Simulator API Manual

Overview

In addition to the model compilation API, nncase also provides APIs for inferring models, using Python on PC to infer kmodel generated by compiled models, to verify whether the inference results of nncase are consistent with the results generated by the runtime of the corresponding deep learning framework. The API provided in this document is used to verify the correctness of kmodel conversion on a local PC, and is not the code running on k230. For learning about nncase, please refer to: nncase github repo .

API Introduction

MemoryRange

【Description】

MemoryRange class, used to represent memory range.

【Definition】

py::class_<memory_range>(m, "MemoryRange")
    .def_readwrite("location", &memory_range::memory_location)
    .def_property(
        "dtype", [](const memory_range &range) { return to_dtype(range.datatype); },
        [](memory_range &range, py::object dtype) { range.datatype = from_dtype(py::dtype::from_args(dtype)); })
    .def_readwrite("start", &memory_range::start)
    .def_readwrite("size", &memory_range::size);

【Properties】

Name	Type	Description
location	int	Memory location, 0 means input, 1 means output, 2 means rdata, 3 means data, 4 means shared_data
dtype	python data type	Data type
start	int	Memory start address
Size	int	Memory size

【Example】

mr = nncase.MemoryRange()

RuntimeTensor

【Description】

RuntimeTensor class, used to represent runtime tensor.

【Definition】

py::class_<runtime_tensor>(m, "RuntimeTensor")
    .def_static("from_numpy", [](py::array arr) {
        auto src_buffer = arr.request();
        auto datatype = from_dtype(arr.dtype());
        auto tensor = host_runtime_tensor::create(
            datatype,
            to_rt_shape(src_buffer.shape),
            to_rt_strides(src_buffer.itemsize, src_buffer.strides),
            gsl::make_span(reinterpret_cast<gsl::byte *>(src_buffer.ptr), src_buffer.size * src_buffer.itemsize),
            [=](gsl::byte *) { arr.dec_ref(); })
                          .unwrap_or_throw();
        arr.inc_ref();
        return tensor;
    })
    .def("copy_to", [](runtime_tensor &from, runtime_tensor &to) {
        from.copy_to(to).unwrap_or_throw();
    })
    .def("to_numpy", [](runtime_tensor &tensor) {
        auto host = tensor.as_host().unwrap_or_throw();
        auto src_map = std::move(hrt::map(host, hrt::map_read).unwrap_or_throw());
        auto src_buffer = src_map.buffer();
        return py::array(
            to_dtype(tensor.datatype()),
            tensor.shape(),
            to_py_strides(runtime::get_bytes(tensor.datatype()), tensor.strides()),
            src_buffer.data());
    })
    .def_property_readonly("dtype", [](runtime_tensor &tensor) {
        return to_dtype(tensor.datatype());
    })
    .def_property_readonly("shape", [](runtime_tensor &tensor) {
        return to_py_shape(tensor.shape());
    })

【Properties】

Name	Type	Description
dtype	python data type	Tensor’s data type
shape	list	Tensor’s shape

from_numpy

【Description】

Construct a RuntimeTensor object from numpy.ndarray.

【Definition】

from_numpy(py::array arr)

【Parameters】

Name	Type	Description
Arr	numpy.ndarray	numpy.ndarray object

【Return Value】

RuntimeTensor object.

【Example】

tensor = nncase.RuntimeTensor.from_numpy(self.inputs[i]['data'])

copy_to

【Description】

Copy RuntimeTensor.

【Definition】

copy_to(RuntimeTensor to)

【Parameters】

Name	Type	Description
to	RuntimeTensor	RuntimeTensor object

【Return Value】

None.

【Example】

sim.get_output_tensor(i).copy_to(to)

to_numpy

【Description】

Convert RuntimeTensor to numpy.ndarray object.

【Definition】

to_numpy()

【Parameters】

None.

【Return Value】

numpy.ndarray object.

【Example】

arr = sim.get_output_tensor(i).to_numpy()

Simulator

【Description】

Simulator class, used to infer kmodel on PC.

【Definition】

py::class_<interpreter>(m, "Simulator")
    .def(py::init())
    .def("load_model", [](interpreter &interp, gsl::span<const gsl::byte> buffer) { interp.load_model(buffer).unwrap_or_throw(); })
    .def_property_readonly("inputs_size", &interpreter::inputs_size)
    .def_property_readonly("outputs_size", &interpreter::outputs_size)
    .def("get_input_desc", &interpreter::input_desc)
    .def("get_output_desc", &interpreter::output_desc)
    .def("get_input_tensor", [](interpreter &interp, size_t index) { return interp.input_tensor(index).unwrap_or_throw(); })
    .def("set_input_tensor", [](interpreter &interp, size_t index, runtime_tensor tensor) { return interp.input_tensor(index, tensor).unwrap_or_throw(); })
    .def("get_output_tensor", [](interpreter &interp, size_t index) { return interp.output_tensor(index).unwrap_or_throw(); })
    .def("set_output_tensor", [](interpreter &interp, size_t index, runtime_tensor tensor) { return interp.output_tensor(index, tensor).unwrap_or_throw(); })
    .def("run", [](interpreter &interp) { interp.run().unwrap_or_throw(); })

【Properties】

Name	Type	Description
inputs_size	int	Number of inputs
outputs_size	int	Number of outputs

【Example】

sim = nncase.Simulator()

load_model

【Description】

Load kmodel.

【Definition】

load_model(model_content)

【Parameters】

Name	Type	Description
model_content	byte[]	kmodel byte stream

【Return Value】

None.

【Example】

sim.load_model(kmodel)

get_input_desc

【Description】

Get the description information of the input at the specified index.

【Definition】

get_input_desc(index)

【Parameters】

Name	Type	Description
index	int	Index of the input

【Return Value】

MemoryRange

【Example】

input_desc_0 = sim.get_input_desc(0)

get_output_desc

【Description】

Get the description information of the output at the specified index.

【Definition】

get_output_desc(index)

【Parameters】

Name	Type	Description
index	int	Index of the output

【Return Value】

MemoryRange

【Example】

output_desc_0 = sim.get_output_desc(0)

get_input_tensor

【Description】

Get the RuntimeTensor of the input at the specified index.

【Definition】

get_input_tensor(index)

【Parameters】

Name	Type	Description
index	int	Index of the input tensor

【Return Value】

RuntimeTensor

【Example】

input_tensor_0 = sim.get_input_tensor(0)

set_input_tensor

【Description】

Set the RuntimeTensor of the input at the specified index.

【Definition】

set_input_tensor(index, tensor)

【Parameters】

Name	Type	Description
index	int	Index of the input tensor
tensor	RuntimeTensor	Input tensor

【Return Value】

None.

【Example】

sim.set_input_tensor(0, nncase.RuntimeTensor.from_numpy(self.inputs[0]['data']))

get_output_tensor

【Description】

Get the RuntimeTensor of the output at the specified index.

【Definition】

get_output_tensor(index)

【Parameters】

Name	Type	Description
index	int	Index of the output tensor

【Return Value】

RuntimeTensor

【Example】

output_arr_0 = sim.get_output_tensor(0).to_numpy()

set_output_tensor

【Description】

Set the RuntimeTensor of the output at the specified index.

【Definition】

set_output_tensor(index, tensor)

【Parameters】

Name	Type	Description
index	int	Index of the output tensor
tensor	RuntimeTensor	Output tensor

【Return Value】

None.

【Example】

sim.set_output_tensor(0, tensor)

run

【Description】

Run kmodel inference.

【Definition】

run()

【Parameters】

None.

【Return Value】

None.

【Example】

sim.run()

Example

Prerequisites: The yolov5s.onnx model has been compiled by the yolov5s_onnx.py script.

yolov5s_onnx_simu.py is located in the src/rtsmart/libs/nncase/examples/scripts subdirectory, with the following content:

import os
import copy
import argparse
import numpy as np
import onnx
import onnxruntime as ort
import nncase

def read_model_file(model_file):
    with open(model_file, 'rb') as f:
        model_content = f.read()
    return model_content

def cosine(gt, pred):
    return (gt @ pred) / (np.linalg.norm(gt, 2) * np.linalg.norm(pred, 2))

def main():
    parser = argparse.ArgumentParser(prog="nncase")
    parser.add_argument("--model", type=str, help='original model file')
    parser.add_argument("--model_input", type=str, help='input bin file for original model')
    parser.add_argument("--kmodel", type=str, help='kmodel file')
    parser.add_argument("--kmodel_input", type=str, help='input bin file for kmodel')
    args = parser.parse_args()

    # cpu inference
    ort_session = ort.InferenceSession(args.model)
    output_names = []
    model_outputs = ort_session.get_outputs()
    for i in range(len(model_outputs)):
        output_names.append(model_outputs[i].name)
    model_input = ort_session.get_inputs()[0]
    model_input_name = model_input.name
    model_input_type = np.float32
    model_input_shape = model_input.shape
    model_input_data = np.fromfile(args.model_input, model_input_type).reshape(model_input_shape)
    cpu_results = []
    cpu_results = ort_session.run(output_names, { model_input_name : model_input_data })

    # create simulator
    sim = nncase.Simulator()

    # read kmodel
    kmodel = read_model_file(args.kmodel)

    # load kmodel
    sim.load_model(kmodel)

    # read input.bin
    # input_tensor=sim.get_input_tensor(0).to_numpy()
    dtype = sim.get_input_desc(0).dtype
    input = np.fromfile(args.kmodel_input, dtype).reshape([1, 3, 320, 320])

    # set input for simulator
    sim.set_input_tensor(0, nncase.RuntimeTensor.from_numpy(input))

    # simulator inference
    nncase_results = []
    sim.run()
    for i in range(sim.outputs_size):
        nncase_result = sim.get_output_tensor(i).to_numpy()
        nncase_results.append(copy.deepcopy(nncase_result))

    # compare
    for i in range(sim.outputs_size):
        cos = cosine(np.reshape(nncase_results[i], (-1)), np.reshape(cpu_results[i], (-1)))
        print('output {0} cosine similarity : {1}'.format(i, cos))

if __name__ == '__main__':
    main()

Execute the inference script

root@5f718e19f8a7:/mnt/# cd rtos_sdk/src/rtsmart/libs/nncase/examples
root@5f718e19f8a7:/mnt/rtos_sdk/src/rtsmart/libs/nncase/examples # export PATH=$PATH:/usr/local/lib/python3.8/dist-packages/
root@5f718e19f8a7:/mnt/rtos_sdk/src/rtsmart/libs/nncase/examples # python3 scripts/yolov5s_onnx_simu.py --model models/yolov5s.onnx --model_input object_detect/data/input_fp32.bin --kmodel tmp/yolov5s_onnx/test.kmodel --kmodel_input object_detect/data/input_uint8.bin

The comparison between the nncase simulator and CPU inference results is as follows:

output 0 cosine similarity : 0.9997244477272034
output 1 cosine similarity : 0.999757707118988
output 2 cosine similarity : 0.9997308850288391