问题描述
【代码】我参考 reference\ai_poc\dec_ai_enc\dec_enc\dec_enc.cc 代码，去掉ai和编码部分，仅保留解码部分代码。
【操作步骤】分别解码640480和160120分辨率的mjpeg格式视频，发现两种分辨率下均为约100帧每秒的解码速度。
【需求】
可否在你们的开发板上验证一下解码速度
【疑问】
1、160*120分辨率下100帧每秒的解码速度与k230芯片手册1.2.9节相差很大。手册中为 20MBit/s。如何调整能接近芯片的理论解码速度
2、为何两种分辨率下解码速度接近
复现步骤
执行命令示例为：
./dec_enc.elf -i 160x120_700fps.mjpeg -o /sharefs/test_gen.h265
代码更新后命令示例为：
./dec_enc_raw.elf -i 160x120_700fps.mjpeg
硬件板卡
CanMV-K230-LP4 V3.0
其他信息
代码及视频文件见 https://github.com/gdyshi/k230_dec_enc
如果还需要其他信息，请提出。我会尽量提供
其他信息
代码及视频文件见 https://github.com/gdyshi/k230_dec_enc
如果还需要其他信息，请提出。我会尽量提供
其他信息
代码及视频文件见 https://github.com/gdyshi/k230_dec_enc
如果还需要其他信息，请提出。我会尽量提供
其他信息
代码及视频文件见 https://github.com/gdyshi/k230_dec_enc
如果还需要其他信息，请提出。我会尽量提供
其他信息
代码及视频文件见 https://github.com/gdyshi/k230_dec_enc
如果还需要其他信息，请提出。我会尽量提供
其他信息
代码及视频文件见 https://github.com/gdyshi/k230_dec_enc
如果还需要其他信息，请提出。我会尽量提供
其他信息
代码及视频文件见 https://github.com/gdyshi/k230_dec_enc
如果还需要其他信息，请提出。我会尽量提供
其他信息
代码及视频文件见 https://github.com/gdyshi/k230_dec_enc
其他信息
源代码：
/* Copyright (c) 2023, Canaan Bright Sight Co., Ltd
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/mman.h>
#include <pthread.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <signal.h>
#include <opencv2/opencv.hpp>
#include <opencv2/highgui/highgui.hpp>
#include "k_module.h"
#include "k_type.h"
#include "k_vb_comm.h"
#include "k_video_comm.h"
#include "k_sys_comm.h"
#include "mpi_vb_api.h"
#include "mpi_vdec_api.h"
#include "mpi_vo_api.h"
#include "mpi_sys_api.h"
#include "k_vdec_comm.h"
#include "k_vvo_comm.h"
#include "mpi_venc_api.h"
#include "k_venc_comm.h"
#include "mpi_vvi_api.h"
#include "vo_test_case.h"
#define ENABLE_VDEC_DEBUG 1
#define BIND_VO_LAYER 1
#ifdef ENABLE_VDEC_DEBUG
#define vdec_debug printf
#else
#define vdec_debug(ARGS...)
#endif
#define MAX_WIDTH 1088
#define MAX_HEIGHT 1920
#define STREAM_BUF_SIZE MAX_WIDTH*MAX_HEIGHT
#define FRAME_BUF_SIZE MAX_WIDTH*MAX_HEIGHT*2
#define INPUT_BUF_CNT 4
#define OUTPUT_BUF_CNT 6
#define VENC_MAX_IN_FRAMES 30
#define ENABLE_VENC_DEBUG 1
#ifdef ENABLE_VDSS
#include "k_vdss_comm.h"
#include "mpi_vdss_api.h"
#else
#include "mpi_vicap_api.h"
#endif
#ifdef ENABLE_VENC_DEBUG
#define venc_debug printf
#else
#define venc_debug(ARGS...)
#endif
#define VE_MAX_WIDTH 1920
#define VE_MAX_HEIGHT 1080
#define VE_STREAM_BUF_SIZE ((VE_MAX_WIDTH*VE_MAX_HEIGHT/2 + 0xfff) & ~0xfff)
#define VE_FRAME_BUF_SIZE ((VE_MAX_WIDTH*VE_MAX_HEIGHT*2 + 0xfff) & ~0xfff)
#define OSD_MAX_WIDTH 1920
#define OSD_MAX_HEIGHT 1088
#define OSD_BUF_SIZE OSD_MAX_WIDTH*OSD_MAX_HEIGHT*4
#define VE_INPUT_BUF_CNT 6
#define VE_OUTPUT_BUF_CNT 15
#define OSD_BUF_CNT 20
typedef struct
{
k_pixel_format chn_format;
k_u32 file_size;
k_s32 pool_id;
pthread_t input_tid;
pthread_t output_tid;
FILE *input_file;
k_u32 ch_id;
char *dump_frame;
k_u32 dump_frame_size;
k_bool done;
k_payload_type type;
k_vb_blk_handle vb_handle[INPUT_BUF_CNT];
k_pixel_format pic_format;
k_u32 act_width;
k_u32 act_height;
k_u32 input_pool_id;
k_u32 output_pool_id;
} sample_vdec_conf_t;
static sample_vdec_conf_t g_vdec_conf[VDEC_MAX_CHN_NUMS];
//****************function***********************************
static inline void CHECK_RET(k_s32 ret, const char *func, const int line)
{
if (ret)
printf("error ret %d, func %s line %d\n", ret, func, line);
}
/**
*单独创建pool给解码器的输入输出使用
*/
static k_s32 vb_create_pool(int ch)
{
k_vb_pool_config pool_config;
memset(&pool_config, 0, sizeof(pool_config));
pool_config.blk_cnt = INPUT_BUF_CNT;
pool_config.blk_size =STREAM_BUF_SIZE,VICAP_ALIGN_1K;
pool_config.mode = VB_REMAP_MODE_NOCACHE;
g_vdec_conf[ch].input_pool_id = kd_mpi_vb_create_pool(&pool_config);
vdec_debug("input_pool_id %d\n", g_vdec_conf[ch].input_pool_id);
memset(&pool_config, 0, sizeof(pool_config));
pool_config.blk_cnt = OUTPUT_BUF_CNT;
pool_config.blk_size = FRAME_BUF_SIZE,VICAP_ALIGN_1K;
pool_config.mode = VB_REMAP_MODE_NOCACHE;
g_vdec_conf[ch].output_pool_id = kd_mpi_vb_create_pool(&pool_config);
vdec_debug("output_pool_id %d\n", g_vdec_conf[ch].output_pool_id);
return 0;
}
/**
*销毁创建的pool
*/
static k_s32 vb_destory_pool(int ch)
{
vdec_debug("destory_pool input %d \n", g_vdec_conf[ch].input_pool_id);
kd_mpi_vb_destory_pool(g_vdec_conf[ch].input_pool_id);
vdec_debug("destory_pool output %d \n", g_vdec_conf[ch].output_pool_id);
kd_mpi_vb_destory_pool(g_vdec_conf[ch].output_pool_id);
return 0;
}
/**
*vb初始化，pool个数为5，先初始化3个pool给编码器，后面再创建两个pool给解码器
*/
static k_s32 sample_vb_init(k_u32 ch_cnt, k_bool osd_enable)
{
k_s32 ret;
k_vb_config config;
memset(&config, 0, sizeof(config));
config.max_pool_cnt = 5;
config.comm_pool[0].blk_cnt = VE_INPUT_BUF_CNT * ch_cnt;
config.comm_pool[0].blk_size = VE_FRAME_BUF_SIZE;
config.comm_pool[0].mode = VB_REMAP_MODE_NOCACHE;
config.comm_pool[1].blk_cnt = VE_OUTPUT_BUF_CNT * ch_cnt;
config.comm_pool[1].blk_size =VE_STREAM_BUF_SIZE;
config.comm_pool[1].mode = VB_REMAP_MODE_NOCACHE;
config.comm_pool[2].blk_cnt = 4;
config.comm_pool[2].blk_size =OSD_BUF_SIZE;
config.comm_pool[2].mode = VB_REMAP_MODE_NOCACHE;
ret = kd_mpi_vb_set_config(&config);
venc_debug("-----------venc sample test------------------------\n");
if (ret)
venc_debug("vb_set_config failed ret:%d\n", ret);
ret = kd_mpi_vb_init();
if (ret)
venc_debug("vb_init failed ret:%d\n", ret);
return ret;
}
/**
*vb退出
*/
static k_s32 sample_vb_exit(void)
{
k_s32 ret;
ret = kd_mpi_vb_exit();
if (ret)
vdec_debug("vb_exit failed ret:%d\n", ret);
return ret;
}
/**
*解码器输入线程逻辑
*/
static void *input_thread(void *arg)
{
sample_vdec_conf_t *vdec_conf;
k_vdec_stream stream;
k_vb_blk_handle handle;
k_s32 pool_id = 0;
k_u64 phys_addr = 0;
k_u8 *virt_addr;
k_u8 *file_buffer = NULL;
k_u32 blk_size, stream_len;
k_u32 file_size = 0;
int i = 0;
k_s32 ret;
vdec_conf = (sample_vdec_conf_t *)arg;
if (vdec_conf->input_file)
{
fseek(vdec_conf->input_file, 0L, SEEK_END);
vdec_conf->file_size = ftell(vdec_conf->input_file);
fseek(vdec_conf->input_file, 0, SEEK_SET);
}
else
vdec_conf->file_size = 0;
vdec_debug("input file size %d\n", vdec_conf->file_size);
blk_size = STREAM_BUF_SIZE;
int poolid = vdec_conf->input_pool_id;
vdec_debug("%s>poolid:%d \n", __func__, poolid);
// 一次性读取整个文件到内存
if (vdec_conf->input_file && vdec_conf->file_size > 0) {
file_buffer = (k_u8 *)malloc(vdec_conf->file_size);
if (file_buffer == NULL) {
vdec_debug("%s malloc file buffer failed\n", __func__);
return arg;
}
// 将文件指针移动到开头
fseek(vdec_conf->input_file, 0, SEEK_SET);
// 一次性读取整个文件
size_t bytes_read = fread(file_buffer, 1, vdec_conf->file_size, vdec_conf->input_file);
if (bytes_read != vdec_conf->file_size) {
vdec_debug("%s read file failed, expected %d bytes, got %zu bytes\n",
__func__, vdec_conf->file_size, bytes_read);
free(file_buffer);
return arg;
}
vdec_debug("%s read file to memory successfully, size: %d bytes\n",
__func__, vdec_conf->file_size);
}
// 从内存循环获取数据
while (file_size < vdec_conf->file_size && file_buffer != NULL)
{
memset(&stream, 0, sizeof(k_vdec_stream));
handle = kd_mpi_vb_get_block(poolid, blk_size, NULL);
if (handle == VB_INVALID_HANDLE)
{
usleep(3000);
continue;
}
pool_id = kd_mpi_vb_handle_to_pool_id(handle);
if (pool_id == VB_INVALID_POOLID)
{
vdec_debug("%s get pool id error\n", __func__);
break;
}
if(i >= INPUT_BUF_CNT)
i = 0;
vdec_conf->pool_id = pool_id;
vdec_conf->vb_handle[i] = handle;
phys_addr = kd_mpi_vb_handle_to_phyaddr(handle);
if (phys_addr == 0)
{
vdec_debug("%s get phys addr error\n", __func__);
break;
}
virt_addr = (k_u8 *)kd_mpi_sys_mmap_cached(phys_addr, blk_size);
if (virt_addr == NULL)
{
vdec_debug("%s mmap error\n", __func__);
break;
}
// 从内存缓冲区复制数据，而不是从文件读取
if (file_size + blk_size > vdec_conf->file_size)
{
memcpy(virt_addr, file_buffer + file_size, (vdec_conf->file_size - file_size));
stream_len = vdec_conf->file_size - file_size;
stream.end_of_stream = K_TRUE;
}
else
{
memcpy(virt_addr, file_buffer + file_size, blk_size);
stream_len = blk_size;
}
ret = kd_mpi_sys_mmz_flush_cache(phys_addr, virt_addr, stream_len);
CHECK_RET(ret, __func__, __LINE__);
file_size += stream_len;
stream.phy_addr = phys_addr;
stream.len = stream_len;
ret = kd_mpi_vdec_send_stream(vdec_conf->ch_id, &stream, -1);
CHECK_RET(ret, __func__, __LINE__);
ret = kd_mpi_sys_munmap((void *)virt_addr, blk_size);
CHECK_RET(ret, __func__, __LINE__);
ret = kd_mpi_vb_release_block(handle);
CHECK_RET(ret, __func__, __LINE__);
i++;
if (vdec_conf->done)
break;
}
// 释放内存缓冲区
if (file_buffer) {
free(file_buffer);
file_buffer = NULL;
}
vdec_debug("%s>done, total processed %d bytes\n", __func__, file_size);
return arg;
}
/**
* NV12(YUV420)数据是解码器输出格式，需要将其转换为RGB格式给AI使用
*/
cv::Mat nv12ToRGBHWC(const uint8_t* nv12Data, int width, int height, uint8_t* rgbChwData) {
cv::Mat nv12Mat(height + height / 2, width, CV_8UC1, const_cast<uint8_t*>(nv12Data));
cv::Mat rgbMat(height, width, CV_8UC3, rgbChwData);
cv::cvtColor(nv12Mat, rgbMat, cv::COLOR_YUV2RGB_NV12);
// cv::imwrite("test.jpg",rgbMat);
return rgbMat;
}
/**
* AI计算后得到的RGB图像要转换成ARGB格式发送给编码器
*/
cv::Mat convertToARGB(const cv::Mat& src) {
CV_Assert(src.channels() == 3); // 输入图像应该是 3 通道的 RGB 图像
cv::Mat dst(src.rows, src.cols, CV_8UC4);
for (int y = 0; y < src.rows; ++y) {
const cv::Vec3b* src_row = src.ptr<cv::Vec3b>(y);
cv::Vec4b* dst_row = dst.ptr<cv::Vec4b>(y);
for (int x = 0; x < src.cols; ++x) {
// RGB 到 ARGB 的通道转换
dst_row[x] = cv::Vec4b(255,src_row[x][0], src_row[x][1], src_row[x][2]);
}
}
return dst;
}
/**
*解码器输出线程逻辑
*/
static void *output_thread(void *arg)
{
sample_vdec_conf_t *vdec_conf;
k_s32 ret;
int out_cnt;
k_video_frame_info output;
k_vdec_supplement_info supplement;
FILE *output_file=NULL;
void *virt_addr = NULL;
k_u32 frame_size=0;
int frame_number=0;
vdec_conf = (sample_vdec_conf_t *)arg;
//循环解码
while (1)
{
//获取解码器通道状态
k_vdec_chn_status status;
ret = kd_mpi_vdec_query_status(vdec_conf->ch_id, &status);
CHECK_RET(ret, __func__, __LINE__);
if (status.end_of_stream)
{
vdec_debug("%s>ch %d, receive eos\n", __func__, vdec_conf->ch_id);
break;
}
else
{
//获取一帧数据
ret = kd_mpi_vdec_get_frame(vdec_conf->ch_id, &output, &supplement, -1);
CHECK_RET(ret, __func__, __LINE__);
if (supplement.is_valid_frame)
{
out_cnt++;
}
//获取nv12格式数据大小
frame_size = status.width*status.height*3/2;
//获取数据的虚拟地址，从帧内的物理地址做映射
virt_addr = kd_mpi_sys_mmap_cached(output.v_frame.phys_addr[0], frame_size);
// todo 算法运算
if(0==out_cnt%100){
printf("phrase %d frames. width:%d, height:%d\n",out_cnt,status.width, status.height);
// // nv12(YUV420)转换成RGB_HWC数据
// uint8_t *rgb_buffer = (uint8_t *)malloc(status.width * status.height * 3);
// cv::Mat rgb_image=nv12ToRGBHWC((uint8_t *)virt_addr,status.width, status.height,rgb_buffer);
// // 保存为 BMP 格式
// std::string bmp_filename = "decode_"+std::to_string(out_cnt)+".bmp";
// if (!cv::imwrite(bmp_filename, rgb_image)) {
// std::cerr << "Failed to save image: " << bmp_filename << std::endl;
// } else {
// std::cout << "Image saved successfully: " << bmp_filename << std::endl;
// }
}
kd_mpi_sys_munmap(virt_addr, frame_size);
ret = kd_mpi_vdec_release_frame(vdec_conf->ch_id, &output);
CHECK_RET(ret, __func__, __LINE__);
if (supplement.end_of_stream)
{
vdec_debug("%s>ch %d, type %d, receive eos\n", __func__, supplement.type, vdec_conf->ch_id);
break;
}
}
}
vdec_conf->done = K_TRUE;
return arg;
}
int main(int argc, char *argv[])
{
k_s32 ret;
//**********************encoder****************************************
//编码器配置，编码通道编号为0
int chnum = 1;
int ve_ch = 0;
k_u32 output_frames = 10;
k_u32 bitrate = 4000; //kbps
int width = 1920;
int height = 1080;
pthread_t exit_thread_handle;
//**********************decoder****************************************
//解码器配置，解码通道编号为1
int ch = 1;
k_vdec_chn_attr attr;
k_payload_type type = K_PT_BUTT;
int j;
int i;
FILE *input_file = NULL;
memset(g_vdec_conf, 0, sizeof(sample_vdec_conf_t)*VDEC_MAX_CHN_NUMS);
//参数解析
for (i = 1; i < argc; i += 2)
{
if (strcmp(argv[i], "-help") == 0)
{
printf("Please input:\n");
printf("-i: input file name\n");
printf("-o: output file name\n");
printf("./sample_vdec.elf -i input_file.h265 -o output_file.h265\n");
return -1;
}
else if (strcmp(argv[i], "-i") == 0)
{
printf("The answer is: %d\n", 1);
char *ptr = strchr(argv[i + 1], '.');
vdec_debug("infilename: %s\n", argv[i + 1]);
if ((input_file = fopen(argv[i + 1], "rb")) == NULL)
{
vdec_debug("Cannot open input file!!!\n");
return -1;
}
// 计算文件大小
fseek(input_file, 0L, SEEK_END);
k_u32 file_size = ftell(input_file);
fseek(input_file, 0, SEEK_SET);
// 设置文件大小到解码器配置中
g_vdec_conf[ch].file_size = file_size;
vdec_debug("input file size: %d bytes\n", file_size);
if (strcmp(ptr, ".h265") == 0 || strcmp(ptr, ".hevc") == 0 || strcmp(ptr, ".265") == 0)
{
type = K_PT_H265;
vdec_debug("file type is H265\n");
}
else if (strcmp(ptr, ".jpeg") == 0 || strcmp(ptr, ".mjpeg") == 0 || strcmp(ptr, ".jpg") == 0)
{
type = K_PT_JPEG;
vdec_debug("file type is JPEG\n");
}
else
{
vdec_debug("Error input type\n");
return -1;
}
}
else
{
vdec_debug("Error :Invalid arguments %s\n", argv[i]);
return -1;
}
}
//vb初始化，申请两个缓冲池
sample_vb_init(chnum, K_FALSE);
//在编码器中已完成vb初始化，增加缓冲池用于解码器
vb_create_pool(ch);
//解码器设置
g_vdec_conf[ch].ch_id = ch;
for (j = 0; j < INPUT_BUF_CNT; j++)
{
g_vdec_conf[ch].vb_handle[j] = VB_INVALID_HANDLE;
}
g_vdec_conf[ch].input_file = input_file;
attr.pic_width = MAX_WIDTH;
attr.pic_height = MAX_HEIGHT;
attr.frame_buf_cnt = OUTPUT_BUF_CNT;
attr.frame_buf_size = FRAME_BUF_SIZE;
attr.stream_buf_size = STREAM_BUF_SIZE;
attr.type = K_PT_JPEG;
attr.frame_buf_pool_id = g_vdec_conf[ch].output_pool_id;
//创建解码器通道
ret = kd_mpi_vdec_create_chn(ch, &attr);
CHECK_RET(ret, __func__, __LINE__);
//启动解码器通道
ret = kd_mpi_vdec_start_chn(ch);
CHECK_RET(ret, __func__, __LINE__);
//启动两个线程，一个线程从h265文件中读取数据，另一个线程输出NV12（YUV420）格式的数据
pthread_create(&g_vdec_conf[ch].input_tid, NULL, input_thread, &g_vdec_conf[ch]);
pthread_create(&g_vdec_conf[ch].output_tid, NULL, output_thread, &g_vdec_conf[ch]);
//解码结束后，停止并销毁解码器通道
while (1)
{
if (g_vdec_conf[ch].done == K_TRUE)
{
ret = kd_mpi_vdec_stop_chn(ch);
CHECK_RET(ret, __func__, __LINE__);
ret = kd_mpi_vdec_destroy_chn(ch);
CHECK_RET(ret, __func__, __LINE__);
pthread_kill(g_vdec_conf[ch].input_tid, SIGALRM);
pthread_join(g_vdec_conf[ch].input_tid, NULL);
pthread_kill(g_vdec_conf[ch].output_tid, SIGALRM);
pthread_join(g_vdec_conf[ch].output_tid, NULL);
fclose(g_vdec_conf[ch].input_file);
vb_destory_pool(ch);
vdec_debug("kill ch %d thread done!\n", ch);
usleep(10000);
break;
}
else
usleep(50000);
}
ret = kd_mpi_vdec_close_fd();
CHECK_RET(ret, __func__, __LINE__);
//vb退出
sample_vb_exit();
vdec_debug("sample decode done!\n");
return 0;
}
