【米尔-全志T113-i开发板试用】JPG硬件解码的实现和对比测试

MYC-YT113i核心板及开发板

真正的国产核心板，100%国产物料认证

• 国产T113-i处理器配备2*Cortex-A7@1.2GHz ，RISC-V；
• 外置DDR3接口、支持视频编解码器、HiFi4 DSP；
• 接口丰富：视频采集接口、显示器接口、USB2.0 接口、CAN 接口、千兆以太网接口；
• 工业级：-40℃~+85℃、尺寸37mm*39mm；
• 邮票孔+LGA，140+50PIN；

全志T113-i 硬件解码支持情况

datasheet中清楚的写着

• H.265 MP@L5.0 up to 4K@30fps
• H.264 BP/MP/HP@L5.0 up to 4K@24fps
• H.263 BP up to 1080p@60fps
• MPEG-4 SP/ASP L5.0 up to 1080p@60fps
• MPEG-2 MP/HL up to 1080p@60fps
• MPEG-1 MP/HL up to 1080p@60fps
• JPEG/Xvid/Sorenson Spark up to 1080p@60fps
• MJPEG up to 1080p@30fps

开发环境配置

基础开发环境搭建参考上一篇

https://bbs.elecfans.com/jishu_2408808_1_1.html

我们需要调用全志的视频引擎库进行硬件编解码，具体的sdk下载自

https://github.com/MYIR-ALLWINNER/framework/tree/develop-yt113-framework/libcedarc

so库文件可以直接从开发板上提取出来

adb pull /usr/lib/libaftertreatment.so
adb pull /usr/lib/libcdc_base.so
adb pull /usr/lib/libfbm.so
adb pull /usr/lib/libMemAdapter.so
adb pull /usr/lib/libsbm.so
adb pull /usr/lib/libscaledown.so
adb pull /usr/lib/libvdecoder.so
adb pull /usr/lib/libVE.so
adb pull /usr/lib/libvideoengine.so

基于stb_image的jpg软件解码

https://github.com/nothings/stb

stb_image 是应用十分广泛的图片解码实现，纯C语言，无任何依赖，接口简单易用，支持读取jpg/png/bmp等图片格式，适合用于嵌入式环境中

先从文件读取jpg数据到内存中

// read file
std::vector<unsigned char> buf;
{
    FILE* fp = fopen("in.jpg", "rb");
    fseek(fp, 0, SEEK_END);
    int size = ftell(fp);
    rewind(fp);
    buf.resize(size);
    fread(buf.data(), 1, size, fp);
    fclose(fp);
    fprintf(stderr, "size = %d\n", size);
}

使用 stb_image 解码图片为 RGB pixeldata 完成jpg软件解码

int desired_channels = 3;
int w;
int h;
int c;
unsigned char* pixeldata = stbi_load_from_memory(buf.data(), buf.size(), &w, &h, &c, desired_channels);

基于cedarc的jpg硬件解码

1. 读取图片到内存
2. 解析jpg头部，获取 width height 基本信息
3. 初始化 cedarc 引擎和解码插件
4. 创建 videodecoder 初始化
5. 获取缓存区，将jpg数据送到 videodecoder
6. 开始解码，获取返回的 YUV420 数据
7. YUV420转RGB

其中步骤 3~6 是基于cedarc库完成，剩余步骤是软件实现

步骤3可以初始化一次后，不需要再重复初始化

cedarc库的使用方法参考

https://github.com/MYIR-ALLWINNER/framework/tree/develop-yt113-framework/libcedarc/demo/vdecoderDemo

硬件解码核心关键代码

#include <memoryAdapter.h>
#include <vdecoder.h>

// 步骤3
AddVDPlugin();

// 步骤4
struct ScMemOpsS* memops = MemAdapterGetOpsS();

CdcMemOpen(memops);

VideoDecoder* vdec = CreateVideoDecoder();

{
    VideoStreamInfo videoInfo;
    memset(&videoInfo, 0, sizeof(videoInfo));
    videoInfo.eCodecFormat = VIDEO_CODEC_FORMAT_MJPEG;
    videoInfo.nWidth = width;
    videoInfo.nHeight = height;

    VConfig vconfig;
    memset(&vconfig, 0, sizeof(vconfig));
    vconfig.eOutputPixelFormat = PIXEL_FORMAT_NV21;
    vconfig.eSecOutputPixelFormat = PIXEL_FORMAT_NV21;
    vconfig.nDeInterlaceHoldingFrameBufferNum = 1;
    vconfig.nDisplayHoldingFrameBufferNum = 1;
    vconfig.nRotateHoldingFrameBufferNum = 0;
    vconfig.nDecodeSmoothFrameBufferNum = 1;
    vconfig.memops = memops;
    vconfig.eCtlAfbcMode = DISABLE_AFBC_ALL_SIZE;
    vconfig.eCtlIptvMode = DISABLE_IPTV_ALL_SIZE;
    vconfig.nSupportMaxWidth = width;
    vconfig.nSupportMaxHeight = height;

    InitializeVideoDecoder(vdec, &videoInfo, &vconfig);
}

// 步骤5
char* pBuf = 0;
{
    int bufSize = 0;
    char* pRingBuf = 0;
    int ringBufSize = 0;
    RequestVideoStreamBuffer(vdec, jpgsize, &pBuf, &bufSize, &pRingBuf, &ringBufSize, 0);

    // copy to vdec sbm
    if (bufSize >= jpgsize)
    {
        memcpy(pBuf, jpgdata, jpgsize);
    }
    else
    {
        memcpy(pBuf, jpgdata, bufSize);
        memcpy(pRingBuf, jpgdata + bufSize, jpgsize - bufSize);
    }
}

{
    VideoStreamDataInfo dataInfo;
    memset(&dataInfo, 0, sizeof(dataInfo));
    dataInfo.pData = pBuf;
    dataInfo.nLength = jpgsize;
    dataInfo.bIsFirstPart = 1;
    dataInfo.bIsLastPart = 1;

    SubmitVideoStreamData(vdec, &dataInfo, 0);
}

// 步骤6
DecodeVideoStream(vdec, 1/*endofstream*/, 0, 0, 0);

VideoPicture* vpic = RequestPicture(vdec, 0);

// 步骤7
yuv420sp2rgb_neon((const unsigned char*)vpic->pData0, (const unsigned char*)vpic->pData1, vpic->nLineStride, width, height, outrgb);

输出的rgb数据，可以再利用 stb_image_write 写入新的jpg图片，下载到本地查看解码是否正常

stbi_write_jpg("testout.jpg", width, height, 3, outrgb, 90);

1080p图片解码性能测试

./jpgd 1920x1080.jpg

sh-4.4# ./jpgd 1920x1080.jpg 2>&1 | grep jpgd
jpgd stbi 143.520
jpgd stbi 144.460
jpgd stbi 143.488
jpgd stbi 144.909
jpgd stbi 178.350
jpgd stbi 144.221
jpgd stbi 143.416
jpgd stbi 144.427
jpgd vdec 66.448
jpgd vdec 62.489
jpgd vdec 63.168
jpgd vdec 63.128
jpgd vdec 62.299
jpgd vdec 63.151
jpgd vdec 65.188
jpgd vdec 78.007

对1080p图片，分别使用 stb_image 和 cedarc 硬件解码库做解码测试，循环调用记录最低耗时

可以看到米尔-全志T113-i开发板的jpg硬件解码能有效加速 50% 以上

附录全流程代码

#include <stdio.h>
#include <string>
#include <vector>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>

#define STB_IMAGE_IMPLEMENTATION
#define STBI_NEON
#include "stb_image.h"

#define STB_IMAGE_WRITE_IMPLEMENTATION
#include "stb_image_write.h"

// aw header
#include <memoryAdapter.h>
#include <vdecoder.h>

void yuv420sp2rgb_neon(const unsigned char* _yptr, const unsigned char* _vuptr, int stride, int w, int h, unsigned char* rgb)
{
    // 此处省略 yuv420sp 转rgb neon加速代码
    // 具体可参考 ncnn项目中的相关实现
}

double get_current_time()
{
    struct timeval tv;
    gettimeofday(&tv, NULL);

    return tv.tv_sec * 1000.0 + tv.tv_usec / 1000.0;
}

class jpeg_decoder_aw
{
public:
    jpeg_decoder_aw(const unsigned char* jpgdata, size_t jpgsize);

    int ping();

    int decode(unsigned char* outrgb);

protected:
    const unsigned char* jpgdata;
    size_t jpgsize;

public:
    int corrupted; // 0=fine
    int width;
    int height;
    int components; // 1=gray 3=yuv
    int sampling_factor; // 0=444 1=422h 2=422v 3=420 4=400
    int progressive;
};

jpeg_decoder_aw::jpeg_decoder_aw(const unsigned char* _jpgdata, size_t _jpgsize)
{
    jpgdata = _jpgdata;
    jpgsize = _jpgsize;

    corrupted = 1;
    width = 0;
    height = 0;
    components = 0;
    sampling_factor = -1;
    progressive = 0;
}

int jpeg_decoder_aw::ping()
{
    if (!jpgdata || jpgsize < 4)
        return -1;

    // jpg magic
    if (jpgdata[0] != 0xFF || jpgdata[1] != 0xD8)
        return -1;

    // parse jpg for width height components sampling-factor progressive

    // 此处省略解析 exif 部分代码

    if (corrupted)
        return -1;

    return 0;
}

int jpeg_decoder_aw::decode(unsigned char* outrgb)
{
    if (!outrgb)
        return -1;

    // corrupted file
    if (corrupted)
        return -1;

    // progressive not supported
    if (progressive)
        return -1;

    struct ScMemOpsS* memops = MemAdapterGetOpsS();

    CdcMemOpen(memops);

    VideoDecoder* vdec = CreateVideoDecoder();

    {
        VideoStreamInfo videoInfo;
        memset(&videoInfo, 0, sizeof(videoInfo));
        videoInfo.eCodecFormat = VIDEO_CODEC_FORMAT_MJPEG;
        videoInfo.nWidth = width;
        videoInfo.nHeight = height;

        VConfig vconfig;
        memset(&vconfig, 0, sizeof(vconfig));

        // PIXEL_FORMAT_NV21 = 5
        vconfig.eOutputPixelFormat = PIXEL_FORMAT_NV21;
        vconfig.eSecOutputPixelFormat = PIXEL_FORMAT_NV21;

        vconfig.nDeInterlaceHoldingFrameBufferNum = 1;
        vconfig.nDisplayHoldingFrameBufferNum = 1;
        vconfig.nRotateHoldingFrameBufferNum = 0;
        vconfig.nDecodeSmoothFrameBufferNum = 1;
        vconfig.memops = memops;
        vconfig.eCtlAfbcMode = DISABLE_AFBC_ALL_SIZE;
        vconfig.eCtlIptvMode = DISABLE_IPTV_ALL_SIZE;
        vconfig.nSupportMaxWidth = width;
        vconfig.nSupportMaxHeight = height;

        InitializeVideoDecoder(vdec, &videoInfo, &vconfig);
    }

    char* pBuf = 0;
    {
        int bufSize = 0;
        char* pRingBuf = 0;
        int ringBufSize = 0;

        RequestVideoStreamBuffer(vdec, jpgsize, &pBuf, &bufSize, &pRingBuf, &ringBufSize, 0);

        // copy to vdec sbm
        if (bufSize >= jpgsize)
        {
            memcpy(pBuf, jpgdata, jpgsize);
        }
        else
        {
            memcpy(pBuf, jpgdata, bufSize);
            memcpy(pRingBuf, jpgdata + bufSize, jpgsize - bufSize);
        }
    }

    {
        VideoStreamDataInfo dataInfo;
        memset(&dataInfo, 0, sizeof(dataInfo));
        dataInfo.pData = pBuf;
        dataInfo.nLength = jpgsize;
        dataInfo.bIsFirstPart = 1;
        dataInfo.bIsLastPart = 1;

        SubmitVideoStreamData(vdec, &dataInfo, 0);
    }

    DecodeVideoStream(vdec, 1/*endofstream*/, 0, 0, 0);

    VideoPicture* vpic = RequestPicture(vdec, 0);

    yuv420sp2rgb_neon((const unsigned char*)vpic->pData0, (const unsigned char*)vpic->pData1, vpic->nLineStride, width, height, outrgb);

    ReturnPicture(vdec, vpic);

    DestroyVideoDecoder(vdec);

    CdcMemClose(memops);

    return 0;
}

int main(int argc, char** argv)
{
    const char* path = argc >= 2 ? argv[1] : "in.jpg";

    AddVDPlugin();

    // read file
    std::vector<unsigned char> buf;
    {
        FILE* fp = fopen(path, "rb");
        fseek(fp, 0, SEEK_END);
        int size = ftell(fp);
        rewind(fp);
        buf.resize(size);
        fread(buf.data(), 1, size, fp);
        fclose(fp);
        fprintf(stderr, "size = %d\n", size);
    }

    for (int j = 0; j < 8; j++)
    {
        double t0 = get_current_time();

        int desired_channels = 3;
        int w;
        int h;
        int c;
        unsigned char* pixeldata = stbi_load_from_memory(buf.data(), buf.size(), &w, &h, &c, desired_channels);

        double t1 = get_current_time();

        stbi_image_free(pixeldata);

        fprintf(stderr, "jpgd stbi %.3f\n", t1-t0);
    }

    for (int j = 0; j < 8; j++)
    {
        std::vector<unsigned char> outrgb;

        double t0 = get_current_time();

        jpeg_decoder_aw jpg_decoder(buf.data(), buf.size());

        jpg_decoder.ping();

        const int width = jpg_decoder.width;
        const int height = jpg_decoder.height;

        outrgb.resize(width * height * 3);

        jpg_decoder.decode(outrgb.data());

        double t1 = get_current_time();

        fprintf(stderr, "jpgd vdec %.3f\n", t1-t0);

        if (j == 7)
        {
            // save to out image
            stbi_write_jpg("testout.jpg", width, height, 3, outrgb.data(), 90);
        }
    }

    return 0;
}