Note

This is the documentation for the latest development branch and may refer to features that are not available in released versions. If you are looking for the documentation for a specific release, use the drop-down menu on the left and select the desired version.

nncase Model Simulator API Manual

Overview

In addition to the model compilation API, nncase also provides an API for inferring models. It uses Python to infer the kmodel generated by compiling the model on a PC, which is used to verify whether the inference results of nncase are consistent with the results generated by the runtime of the corresponding deep learning framework. The API provided in this document is used to verify the correctness of kmodel conversion on a local PC, and is not code that runs on k230. For learning about nncase, please refer to: nncase github repo.

API Introduction

MemoryRange

【Description】

The MemoryRange class, used to represent a memory range.

【Definition】

py::class_<memory_range>(m, "MemoryRange")
    .def_readwrite("location", &memory_range::memory_location)
    .def_property(
        "dtype", [](const memory_range &range) { return to_dtype(range.datatype); },
        [](memory_range &range, py::object dtype) { range.datatype = from_dtype(py::dtype::from_args(dtype)); })
    .def_readwrite("start", &memory_range::start)
    .def_readwrite("size", &memory_range::size);

【Properties】

Name	Type	Description
location	int	Memory location, 0 indicates input, 1 indicates output, 2 indicates rdata, 3 indicates data, 4 indicates shared_data
dtype	python data type	Data type
start	int	Memory start address
size	int	Memory size

【Example】

mr = nncase.MemoryRange()

RuntimeTensor

【Description】

The RuntimeTensor class, used to represent a runtime tensor.

【Definition】

py::class_<runtime_tensor>(m, "RuntimeTensor")
    .def_static("from_numpy", [](py::array arr) {
        auto src_buffer = arr.request();
        auto datatype = from_dtype(arr.dtype());
        auto tensor = host_runtime_tensor::create(
            datatype,
            to_rt_shape(src_buffer.shape),
            to_rt_strides(src_buffer.itemsize, src_buffer.strides),
            gsl::make_span(reinterpret_cast<gsl::byte *>(src_buffer.ptr), src_buffer.size * src_buffer.itemsize),
            [=](gsl::byte *) { arr.dec_ref(); })
                          .unwrap_or_throw();
        arr.inc_ref();
        return tensor;
    })
    .def("copy_to", [](runtime_tensor &from, runtime_tensor &to) {
        from.copy_to(to).unwrap_or_throw();
    })
    .def("to_numpy", [](runtime_tensor &tensor) {
        auto host = tensor.as_host().unwrap_or_throw();
        auto src_map = std::move(hrt::map(host, hrt::map_read).unwrap_or_throw());
        auto src_buffer = src_map.buffer();
        return py::array(
            to_dtype(tensor.datatype()),
            tensor.shape(),
            to_py_strides(runtime::get_bytes(tensor.datatype()), tensor.strides()),
            src_buffer.data());
    })
    .def_property_readonly("dtype", [](runtime_tensor &tensor) {
        return to_dtype(tensor.datatype());
    })
    .def_property_readonly("shape", [](runtime_tensor &tensor) {
        return to_py_shape(tensor.shape());
    })

【Properties】

Name	Type	Description
dtype	python data type	Tensor’s data type
shape	list	tensor’s shape

from_numpy

【Description】

Constructs a RuntimeTensor object from a numpy.ndarray.

【Definition】

from_numpy(py::array arr)

【Parameters】

Name	Type	Description
arr	numpy.ndarray	numpy.ndarray object

【Return Value】

A RuntimeTensor object.

【Example】

tensor = nncase.RuntimeTensor.from_numpy(self.inputs[i]['data'])

copy_to

【Description】

Copies the RuntimeTensor.

【Definition】

copy_to(RuntimeTensor to)

【Parameters】

Name	Type	Description
to	RuntimeTensor	RuntimeTensor object

【Return Value】

None.

【Example】

sim.get_output_tensor(i).copy_to(to)

to_numpy

【Description】

Converts the RuntimeTensor to a numpy.ndarray object.

【Definition】

to_numpy()

【Parameters】

None.

【Return Value】

A numpy.ndarray object.

【Example】

arr = sim.get_output_tensor(i).to_numpy()

Simulator

【Description】

The Simulator class, used to infer a kmodel on a PC.

【Definition】

py::class_<interpreter>(m, "Simulator")
    .def(py::init())
    .def("load_model", [](interpreter &interp, gsl::span<const gsl::byte> buffer) { interp.load_model(buffer).unwrap_or_throw(); })
    .def_property_readonly("inputs_size", &interpreter::inputs_size)
    .def_property_readonly("outputs_size", &interpreter::outputs_size)
    .def("get_input_desc", &interpreter::input_desc)
    .def("get_output_desc", &interpreter::output_desc)
    .def("get_input_tensor", [](interpreter &interp, size_t index) { return interp.input_tensor(index).unwrap_or_throw(); })
    .def("set_input_tensor", [](interpreter &interp, size_t index, runtime_tensor tensor) { return interp.input_tensor(index, tensor).unwrap_or_throw(); })
    .def("get_output_tensor", [](interpreter &interp, size_t index) { return interp.output_tensor(index).unwrap_or_throw(); })
    .def("set_output_tensor", [](interpreter &interp, size_t index, runtime_tensor tensor) { return interp.output_tensor(index, tensor).unwrap_or_throw(); })
    .def("run", [](interpreter &interp) { interp.run().unwrap_or_throw(); })

【Properties】

Name	Type	Description
inputs_size	int	Number of inputs
outputs_size	int	Number of outputs

【Example】

sim = nncase.Simulator()

load_model

【Description】

Loads a kmodel.

【Definition】

load_model(model_content)

【Parameters】

Name	Type	Description
model_content	byte[]	kmodel byte stream

【Return Value】

None.

【Example】

sim.load_model(kmodel)

get_input_desc

【Description】

Gets the description information of the input at the specified index.

【Definition】

get_input_desc(index)

【Parameters】

Name	Type	Description
index	int	Index of the input

【Return Value】

MemoryRange

【Example】

input_desc_0 = sim.get_input_desc(0)

get_output_desc

【Description】

Gets the description information of the output at the specified index.

【Definition】

get_output_desc(index)

【Parameters】

Name	Type	Description
index	int	Index of the output

【Return Value】

MemoryRange

【Example】

output_desc_0 = sim.get_output_desc(0)

get_input_tensor

【Description】

Gets the RuntimeTensor of the input at the specified index.

【Definition】

get_input_tensor(index)

【Parameters】

Name	Type	Description
index	int	Index of the input tensor

【Return Value】

RuntimeTensor

【Example】

input_tensor_0 = sim.get_input_tensor(0)

set_input_tensor

【Description】

Sets the RuntimeTensor of the input at the specified index.

【Definition】

set_input_tensor(index, tensor)

【Parameters】

Name	Type	Description
index	int	Index of the input tensor
tensor	RuntimeTensor	Input tensor

【Return Value】

None.

【Example】

sim.set_input_tensor(0, nncase.RuntimeTensor.from_numpy(self.inputs[0]['data']))

get_output_tensor

【Description】

Gets the RuntimeTensor of the output at the specified index.

【Definition】

get_output_tensor(index)

【Parameters】

Name	Type	Description
index	int	Index of the output tensor

【Return Value】

RuntimeTensor

【Example】

output_arr_0 = sim.get_output_tensor(0).to_numpy()

set_output_tensor

【Description】

Sets the RuntimeTensor of the output at the specified index.

【Definition】

set_output_tensor(index, tensor)

【Parameters】

Name	Type	Description
index	int	Index of the output tensor
tensor	RuntimeTensor	Output tensor

【Return Value】

None.

【Example】

sim.set_output_tensor(0, tensor)

run

【Description】

Runs kmodel inference.

【Definition】

run()

【Parameters】

None.

【Return Value】

None.

【Example】

sim.run()

Example

Assuming that a certain onnx model has been compiled into a kmodel model, the simulation verification script is as follows:

import os
import copy
import argparse
import numpy as np
import onnx
import onnxruntime as ort
import nncase

def read_model_file(model_file):
    with open(model_file, 'rb') as f:
        model_content = f.read()
    return model_content

def cosine(gt, pred):
    return (gt @ pred) / (np.linalg.norm(gt, 2) * np.linalg.norm(pred, 2))

def main():
    parser = argparse.ArgumentParser(prog="nncase")
    parser.add_argument("--model", type=str, help='original model file')
    parser.add_argument("--model_input", type=str, help='input bin file for original model')
    parser.add_argument("--kmodel", type=str, help='kmodel file')
    parser.add_argument("--kmodel_input", type=str, help='input bin file for kmodel')
    args = parser.parse_args()

    # cpu inference
    ort_session = ort.InferenceSession(args.model)
    output_names = []
    model_outputs = ort_session.get_outputs()
    for i in range(len(model_outputs)):
        output_names.append(model_outputs[i].name)
    model_input = ort_session.get_inputs()[0]
    model_input_name = model_input.name
    model_input_type = np.float32
    model_input_shape = model_input.shape
    model_input_data = np.fromfile(args.model_input, model_input_type).reshape(model_input_shape)
    cpu_results = []
    cpu_results = ort_session.run(output_names, { model_input_name : model_input_data })

    # create simulator
    sim = nncase.Simulator()

    # read kmodel
    kmodel = read_model_file(args.kmodel)

    # load kmodel
    sim.load_model(kmodel)

    # read input.bin
    # input_tensor=sim.get_input_tensor(0).to_numpy()
    dtype = sim.get_input_desc(0).dtype
    input = np.fromfile(args.kmodel_input, dtype).reshape([1, 3, 320, 320])

    # set input for simulator
    sim.set_input_tensor(0, nncase.RuntimeTensor.from_numpy(input))

    # simulator inference
    nncase_results = []
    sim.run()
    for i in range(sim.outputs_size):
        nncase_result = sim.get_output_tensor(i).to_numpy()
        nncase_results.append(copy.deepcopy(nncase_result))

    # compare
    for i in range(sim.outputs_size):
        cos = cosine(np.reshape(nncase_results[i], (-1)), np.reshape(cpu_results[i], (-1)))
        print('output {0} cosine similarity : {1}'.format(i, cos))

if __name__ == '__main__':
    main()

Using the input data from the saved bin file, calculate the cosine similarity between the inference results of onnx and kmodel, and you can verify the correctness of the model conversion.