Note

This is the documentation for the latest development branch and may refer to features that are not available in released versions. If you are looking for the documentation for a specific release, use the drop-down menu on the left and select the desired version.

AI2D Runtime API Manual

Overview

The AI2D runtime APIs are used to configure AI2D parameters on the AI ​​device, generate relevant register configurations, and perform AI2D preprocessing calculations. The API provided by this document is used to use C++ to write code to run on k230 on the local PC, compile it into an executable file and copy it to run on k230.

Notice:

1. The Affine and Resize functions are mutually exclusive and cannot be enabled at the same time;

2. The input format of the Shift function can only be Raw16;

3. Pad value is configured by channel, and the number of corresponding list elements must be equal to the number of channels;

4. In the current version, when only one function of AI2D is needed, other parameters also need to be configured. Just set the flag to false, and other fields do not need to be configured;

5. When multiple functions are configured, the execution order is Crop->Shift->Resize/Affine->Pad. Pay attention to matching when configuring parameters.

Supported format conversion

Input format	Output format	Remark
YUV420_NV12	RGB_planar/YUV420_NV12
YUV420_NV21	RGB_planar/YUV420_NV21
YUV420_I420	RGB_planar/YUV420_I420
YUV400	YUV400
NCHW(RGB_planar)	NCHW(RGB_planar)
RGB_packed	RGB_planar/RGB_packed
RAW16	RAW16/8	Depth map, perform shift operation

Functional Description

Function	Description	Remark
Affine	Support input format YUV420, YUV400, RGB (planar/packed) Support depth map RAW16 format Support output format YUV400, RGB, depth map
Crop/Resize/Padding	Support input YUV420, YUV400, RGB Support depth map RAW16 format Resize supports intermediate NCHW arrangement format Support output format YUV420, YUV400, RGB	Only padding constants are supported
Shift	Supports input format Raw16 and supports output format Raw8
sign bit	Supports signed and unsigned input

Introduction to parameter types

ai2d_format

Description

ai2d_format is used to configure optional data formats for input and output.

definition

enum class ai2d_format
{
    YUV420_NV12 = 0,
    YUV420_NV21 = 1,
    YUV420_I420 = 2,
    NCHW_FMT = 3,
    RGB_packed = 4,
    RAW16 = 5,
}

ai2d_interp_method

Description

ai2d_interp_method is used to configure optional interpolation methods.

definition

 enum class ai2d_interp_method
{
    tf_nearest = 0,
    tf_bilinear = 1,
    cv2_nearest = 2,
    cv2_bilinear = 3,
}

ai2d_interp_mode

Description

ai2d_interp_mode is used to configure optional interpolation mode.

definition

enum class ai2d_interp_mode
{
    none = 0,
    align_corner = 1,
    half_pixel = 2,
}

ai2d_pad_mode

Description

ai2d_pad_mode is used to configure optional padding mode. Currently, only constant padding is supported.

definition

enum class ai2d_pad_mode
{
    constant = 0,
    copy = 1,
    mirror = 2,
}

ai2d_datatype_t

Description

ai2d_datatype_t is used to set the data type during AI2D calculation.

definition

struct ai2d_datatype_t
{
    ai2d_format src_format;
    ai2d_format dst_format;
    datatype_t src_type;
    datatype_t dst_type;
    ai2d_data_loc src_loc = ai2d_data_loc::ddr;
    ai2d_data_loc dst_loc = ai2d_data_loc::ddr;
}

Parameters

name	type	Description
src_format	ai2d_format	Input data format
dst_format	ai2d_format	Output data format
src_type	datatype_t	input data type
dst_type	datatype_t	Output data type
src_loc	ai2d_data_loc	Enter the data location, the default is ddr
dst_loc	ai2d_data_loc	Output data location, default ddr

Example

ai2d_datatype_t ai2d_dtype { ai2d_format::RAW16, ai2d_format::NCHW_FMT, datatype_t::dt_uint16, datatype_t::dt_uint8 };

ai2d_crop_param_t

Description

ai2d_crop_param_t is used to configure crop-related parameters.

definition

struct ai2d_crop_param_t
{
    bool crop_flag = false;
    int32_t start_x = 0;
    int32_t start_y = 0;
    int32_t width = 0;
    int32_t height = 0;
}

Parameters

name	type	Description
crop_flag	bool	Whether to enable crop function
start_x	int	Starting pixel in width direction
start_y	int	Starting pixel in height direction
width	int	Crop length in width direction
height	int	Crop length in height direction

Example

ai2d_crop_param_t crop_param { true, 40, 30, 400, 600 };

ai2d_shift_param_t

Description

ai2d_shift_param_t is used to configure shift-related parameters.

definition

struct ai2d_shift_param_t
{
    bool shift_flag = false;
    int32_t shift_val = 0;
}

Parameters

name	type	Description
shift_flag	bool	Whether to enable shift function
shift_val	int	Number of bits to shift right

Example

ai2d_shift_param_t shift_param { true, 2 };

ai2d_pad_param_t

Description

ai2d_pad_param_t is used to configure pad-related parameters.

definition

struct ai2d_pad_param_t
{
    bool pad_flag = false;
    runtime_paddings_t paddings;
    ai2d_pad_mode pad_mode = ai2d_pad_mode::constant;
    std::vector<int32_t> pad_val; // by channel
}

Parameters

name	type	Description
pad_flag	bool	Whether to enable the pad function
paddings	runtime_paddings_t	The padding in each dimension, shape=[4, 2], respectively represents the number of padding before and after dim0 to dim4, where dim0/dim1 has a fixed configuration of {0, 0}
pad_mode	ai2d_pad_mode	padding mode, only supports constant padding
pad_val	std::vector<int32_t>	padding value for each channel

Example

ai2d_pad_param_t pad_param { false, { { 0, 0 }, { 0, 0 }, { 0, 0 }, { 60, 60 } }, ai2d_pad_mode::constant, { 255 } };

ai2d_resize_param_t

Description

ai2d_resize_param_t is used to configure resize-related parameters.

definition

struct ai2d_resize_param_t
{
    bool resize_flag = false;
    ai2d_interp_method interp_method = ai2d_interp_method::tf_bilinear;
    ai2d_interp_mode interp_mode = ai2d_interp_mode::none;
}

Parameters

name	type	Description
resize_flag	bool	Whether to enable the resize function
interp_method	ai2d_interp_method	resize interpolation method
interp_mode	ai2d_interp_mode	resize mode

Example

ai2d_resize_param_t resize_param { true, ai2d_interp_method::tf_bilinear, ai2d_interp_mode::half_pixel };

ai2d_affine_param_t

Description

ai2d_affine_param_t is used to configure affine-related parameters.

definition

struct ai2d_affine_param_t
{
    bool affine_flag = false;
    ai2d_interp_method interp_method = ai2d_interp_method::cv2_bilinear;
    uint32_t cord_round = 0;
    uint32_t bound_ind = 0;
    int32_t bound_val = 0;
    uint32_t bound_smooth = 0;
    std::vector<float> M;
}

Parameters

name	type	Description
affine_flag	bool	Whether to enable the affine function
interp_method	ai2d_interp_method	The interpolation method used by Affine
cord_round	uint32_t	Integer bounds 0 or 1
bound_ind	uint32_t	Border pixel mode 0 or 1
bound_val	uint32_t	border padding value
bound_smooth	uint32_t	Boundary smoothing 0 or 1
M	std::vector<float>	The vector corresponding to the affine transformation matrix, the affine transformation is \(Y=\[a_0, a_1; a_2, a_3\] \cdot X + \[b_0, b_1\] \), then \( M=\{a_0,a_1,b_0,a_2,a_3,b_1\} \)

Example

ai2d_affine_param_t affine_param { true, ai2d_interp_method::cv2_bilinear, 0, 0, 127, 1, { 0.5, 0.1, 0.0, 0.1, 0.5, 0.0 } };

API introduction

ai2d_builder:: ai2d_builder

Description

Constructor of ai2d_builder.

definition

ai2d_builder(dims_t &input_shape, dims_t &output_shape, ai2d_datatype_t ai2d_dtype, ai2d_crop_param_t crop_param, ai2d_shift_param_t shift_param, ai2d_pad_param_t pad_param, ai2d_resize_param_t resize_param, ai2d_affine_param_t affine_param);

Parameters

name	type	Description
input_shape	dims_t	Enter shape
output_shape	dims_t	Output shape
ai2d_dtype	ai2d_datatype_t	ai2d data type
crop_param	ai2d_crop_param_t	crop related parameters
shift_param	ai2d_shift_param_t	shift related parameters
pad_param	ai2d_pad_param_t	pad related parameters
resize_param	ai2d_resize_param_t	resize related parameters
affine_param	ai2d_affine_param_t	affine related parameters

return value

None

Example

dims_t in_shape { 1, ai2d_input_c_, ai2d_input_h_, ai2d_input_w_ };
auto out_span = ai2d_out_tensor_.shape();
dims_t out_shape { out_span.begin(), out_span.end() };
ai2d_datatype_t ai2d_dtype { ai2d_format::NCHW_FMT, ai2d_format::NCHW_FMT, typecode_t::dt_uint8, typecode_t::dt_uint8 };
ai2d_crop_param_t crop_param { false, 0, 0, 0, 0 };
ai2d_shift_param_t shift_param { false, 0 };
ai2d_pad_param_t pad_param { true, { { 0, 0 }, { 0, 0 }, { 0, 0 }, { 70, 70 } }, ai2d_pad_mode::constant, { 0, 0, 0 } };
ai2d_resize_param_t resize_param { true, ai2d_interp_method::tf_bilinear, ai2d_interp_mode::half_pixel };
ai2d_affine_param_t affine_param { false };
ai2d_builder_.reset(new ai2d_builder(in_shape, out_shape, ai2d_dtype, crop_param, shift_param, pad_param, resize_param, affine_param));

ai2d_builder:: build_schedule

Description

Generate parameters required for AI2D calculations.

definition

result<void> build_schedule();

Parameters

None.

Return Value

result<void>

Example

ai2d_builder_->build_schedule();

ai2d_builder::invoke

Description

Configure registers and start AI2D calculations.

definition

result<void> invoke(runtime_tensor &input, runtime_tensor &output);

Parameters

name	type	Description
input	runtime_tensor	input tensor
output	runtime_tensor	output tensor

return value

result<void>.

Example

// run ai2d
ai2d_builder_->invoke(ai2d_in_tensor, ai2d_out_tensor_).expect("error occurred in ai2d running");

Example

static void test_pad_mini_test(const char *gmodel_file, const char *expect_file)
{
    // input tensor
    dims_t in_shape { 1, 100, 150, 3 };
    auto in_tensor = host_runtime_tensor::create(dt_uint8, in_shape, hrt::pool_shared).expect("cannot create input tensor");
    auto mapped_in_buf = std::move(hrt::map(in_tensor, map_access_t::map_write).unwrap());
    read_binary_file(gmodel_file, reinterpret_cast<char *>(mapped_in_buf.buffer().data()));
    mapped_in_buf.unmap().expect("unmap input tensor failed");
    hrt::sync(in_tensor, sync_op_t::sync_write_back, true).expect("write back input failed");

    // output tensor
    dims_t out_shape { 1, 100, 160, 3 };
    auto out_tensor = host_runtime_tensor::create(dt_uint8, out_shape, hrt::pool_shared).expect("cannot create output tensor");

    // Configure ai2d; this sets padding preprocessing
    ai2d_datatype_t ai2d_dtype { ai2d_format::RGB_packed, ai2d_format::RGB_packed, dt_uint8, dt_uint8 };
    ai2d_crop_param_t crop_param { false, 0, 0, 0, 0 };
    ai2d_shift_param_t shift_param { false, 0 };
    ai2d_pad_param_t pad_param { true, { { 0, 0 }, { 0, 0 }, { 0, 0 }, { 10, 0 } }, ai2d_pad_mode::constant, { 255, 10, 5 } };
    ai2d_resize_param_t resize_param { false, ai2d_interp_method::tf_bilinear, ai2d_interp_mode::half_pixel };
    ai2d_affine_param_t affine_param { false };

    // run
    ai2d_builder builder { in_shape, out_shape, ai2d_dtype, crop_param, shift_param, pad_param, resize_param, affine_param };
    auto start = std::chrono::steady_clock::now();
    builder.build_schedule().expect("error occurred in ai2d build_schedule");
    builder.invoke(in_tensor, out_tensor).expect("error occurred in ai2d invoke");
    auto stop = std::chrono::steady_clock::now();
    double duration = std::chrono::duration<double, std::milli>(stop - start).count();
    std::cout << "ai2d run: duration = " << duration << " ms, fps = " << 1000 / duration << std::endl;

    // compare
    auto mapped_out_buf = std::move(hrt::map(out_tensor, map_access_t::map_read).unwrap());
    auto actual = mapped_out_buf.buffer().data();
    auto expected = read_binary_file<unsigned char>(expect_file);
    int ret = memcmp(reinterpret_cast<void *>(actual), reinterpret_cast<void *>(expected.data()), expected.size());
    if (!ret)
    {
        std::cout << "compare output succeed!" << std::endl;
    }
    else
    {
        auto cos = cosine(reinterpret_cast<const uint8_t *>(actual), reinterpret_cast<const uint8_t *>(expected.data()), expected.size());
        std::cerr << "compare output failed: cosine similarity = " << cos << std::endl;
    }
}

The above code needs to be compiled into a elf executable file using compilation tools in the k230 sdk environment, and then copied to the development board for running.