前言

解碼作為渲染模塊和拉流模塊的中間模塊，它一方面要不停的從拉流模塊的壓縮數(shù)據(jù)緩沖區(qū)中獲取待解碼數(shù)據(jù)包，讓后將這個(gè)數(shù)據(jù)包送入自己的解碼模塊，獲得解碼數(shù)據(jù)后再送入自己的解碼緩沖區(qū)，這就是整個(gè)解碼模塊的工作流程，所以解碼模塊包括解碼和解碼緩沖區(qū)兩個(gè)部分，畫一下流程圖大致就是這樣：

image.png

• 我的思考：

1、解碼線程也是整個(gè)解碼模塊的一部分，它是獨(dú)立的線程，這里音頻、視頻、字幕是三個(gè)獨(dú)立的線程，所以這里只分析其中任何一個(gè)的實(shí)現(xiàn)流程，這里以視頻為例?？梢院苋菀椎南氲接靡粋€(gè)for循環(huán)讓此線程不停的工作，同時(shí)滿足，當(dāng)壓縮數(shù)據(jù)隊(duì)列為空時(shí)此線程等待，當(dāng)視頻幀緩沖區(qū)滿時(shí)也等待，這樣兩個(gè)條件保證解碼線程不會(huì)空轉(zhuǎn)浪費(fèi)cpu資源

2、視頻、音頻、字幕緩沖區(qū)的設(shè)計(jì)。因?yàn)檫@個(gè)緩沖區(qū)是用于渲染的，渲染線程和解碼線程又是獨(dú)立的，所以它要滿足線程安全，同時(shí)又有一定的容量保證緩沖區(qū)不能無限增長

ffplay.c的實(shí)現(xiàn)

由于這里音視頻字幕流程差不多，這里以視頻為例

• 解碼線程的工作流程

解碼肯定是在獲取到壓縮數(shù)據(jù)包之后開始工作才有意義，所以解碼模塊是在拉流模塊準(zhǔn)備工作做好之后讓其進(jìn)行初始化并開始工作，這里根據(jù)是否有對應(yīng)的流決定是否打開對應(yīng)的解碼模塊

static int read_thread(void *arg)
{
  .......省略代碼拉流模塊初始化相關(guān)代碼........
  /** 學(xué)習(xí)：解碼模塊開始工作
     *  分析：解碼肯定是在獲取到壓縮數(shù)據(jù)包之后開始工作才有意義，所以解碼模塊是在拉流模塊準(zhǔn)備工作做好之后讓其進(jìn)行初始化并開始工作
     *  這里根據(jù)是否有對應(yīng)的流決定是否打開對應(yīng)的解碼模塊
     */
    /* open the streams */
    if (st_index[AVMEDIA_TYPE_AUDIO] >= 0) {
        stream_component_open(is, st_index[AVMEDIA_TYPE_AUDIO]);
    }

    ret = -1;
    if (st_index[AVMEDIA_TYPE_VIDEO] >= 0) {
        ret = stream_component_open(is, st_index[AVMEDIA_TYPE_VIDEO]);
    }
    if (is->show_mode == SHOW_MODE_NONE)
        is->show_mode = ret >= 0 ? SHOW_MODE_VIDEO : SHOW_MODE_RDFT;

    if (st_index[AVMEDIA_TYPE_SUBTITLE] >= 0) {
        stream_component_open(is, st_index[AVMEDIA_TYPE_SUBTITLE]);
    }

    ......省略代碼.......
}

接下來看一下stream_component_open()函數(shù)

/* open a given stream. Return 0 if OK */
static int stream_component_open(VideoState *is, int stream_index)
{
     ..............省略代碼解碼器初始化相關(guān)的代碼.......

    is->eof = 0;
    ic->streams[stream_index]->discard = AVDISCARD_DEFAULT;
    switch (avctx->codec_type) {
    case AVMEDIA_TYPE_AUDIO:
#if CONFIG_AVFILTER
        {
            AVFilterContext *sink;

            is->audio_filter_src.freq           = avctx->sample_rate;
            is->audio_filter_src.channels       = avctx->channels;
            is->audio_filter_src.channel_layout = get_valid_channel_layout(avctx->channel_layout, avctx->channels);
            is->audio_filter_src.fmt            = avctx->sample_fmt;
            if ((ret = configure_audio_filters(is, afilters, 0)) < 0)
                goto fail;
            sink = is->out_audio_filter;
            sample_rate    = av_buffersink_get_sample_rate(sink);
            nb_channels    = av_buffersink_get_channels(sink);
            channel_layout = av_buffersink_get_channel_layout(sink);
        }
#else
        sample_rate    = avctx->sample_rate;
        nb_channels    = avctx->channels;
        channel_layout = avctx->channel_layout;
#endif

        /* prepare audio output */
        if ((ret = audio_open(is, channel_layout, nb_channels, sample_rate, &is->audio_tgt)) < 0)
            goto fail;
        is->audio_hw_buf_size = ret;
        is->audio_src = is->audio_tgt;
        is->audio_buf_size  = 0;
        is->audio_buf_index = 0;

        /* init averaging filter */
        is->audio_diff_avg_coef  = exp(log(0.01) / AUDIO_DIFF_AVG_NB);
        is->audio_diff_avg_count = 0;
        /* since we do not have a precise anough audio FIFO fullness,
           we correct audio sync only if larger than this threshold */
        is->audio_diff_threshold = (double)(is->audio_hw_buf_size) / is->audio_tgt.bytes_per_sec;

        is->audio_stream = stream_index;
        is->audio_st = ic->streams[stream_index];

        decoder_init(&is->auddec, avctx, &is->audioq, is->continue_read_thread);
        if ((is->ic->iformat->flags & (AVFMT_NOBINSEARCH | AVFMT_NOGENSEARCH | AVFMT_NO_BYTE_SEEK)) && !is->ic->iformat->read_seek) {
            is->auddec.start_pts = is->audio_st->start_time;
            is->auddec.start_pts_tb = is->audio_st->time_base;
        }
        if ((ret = decoder_start(&is->auddec, audio_thread, "audio_decoder", is)) < 0)
            goto out;
        SDL_PauseAudioDevice(audio_dev, 0);
        break;
    case AVMEDIA_TYPE_VIDEO:
        is->video_stream = stream_index;
        is->video_st = ic->streams[stream_index];

        decoder_init(&is->viddec, avctx, &is->videoq, is->continue_read_thread);
        if ((ret = decoder_start(&is->viddec, video_thread, "video_decoder", is)) < 0)
            goto out;
        is->queue_attachments_req = 1;
        break;
    case AVMEDIA_TYPE_SUBTITLE:
        is->subtitle_stream = stream_index;
        is->subtitle_st = ic->streams[stream_index];

        decoder_init(&is->subdec, avctx, &is->subtitleq, is->continue_read_thread);
        // 打開視頻解碼線程
        if ((ret = decoder_start(&is->subdec, subtitle_thread, "subtitle_decoder", is)) < 0)
            goto out;
        break;
    default:
        break;
    }
    goto out;

fail:
    avcodec_free_context(&avctx);
out:
    av_dict_free(&opts);

    return ret;
}

這個(gè)函數(shù)分為兩部分，前面是對解碼器的初始化工作，直到
if ((ret = decoder_start(&is->viddec, video_thread, "video_decoder", is)) < 0) 這里打開解碼線程，從這塊代碼可以看到，ffplay.c用三個(gè)線程分別處理音頻、視頻、字幕，這里分別對應(yīng)audio_thread()、video_thread()、subtitle_thread()三個(gè)函數(shù)

接下來重點(diǎn)看一下video_thread()函數(shù)

/** 視頻解碼線程的工作機(jī)制
 *  1、通過get_video_frame(is, frame);不停的向解碼器獲取已解碼的視頻數(shù)據(jù)；如果未獲取到則結(jié)束
 *  2、如果獲取到了解碼的視頻數(shù)據(jù)，則給其賦值pts，并通過ret = queue_picture(is, frame, pts, duration, frame->pkt_pos, is->viddec.pkt_serial);將frame插入視頻FrameQueue隊(duì)列
 */
static int video_thread(void *arg)
{
    VideoState *is = arg;
    AVFrame *frame = av_frame_alloc();
    double pts;
    double duration;
    int ret;
    AVRational tb = is->video_st->time_base;
    AVRational frame_rate = av_guess_frame_rate(is->ic, is->video_st, NULL);

#if CONFIG_AVFILTER
    AVFilterGraph *graph = NULL;
    AVFilterContext *filt_out = NULL, *filt_in = NULL;
    int last_w = 0;
    int last_h = 0;
    enum AVPixelFormat last_format = -2;
    int last_serial = -1;
    int last_vfilter_idx = 0;
#endif

    if (!frame)
        return AVERROR(ENOMEM);

    /** for 循環(huán)保證視頻解碼線程不會(huì)退出。這里通過get_video_frame()函數(shù)從解碼器獲取待已解碼的數(shù)據(jù)，此函數(shù)不停地從視頻PacketQueue獲取未解碼數(shù)據(jù)然后送入解碼器，同時(shí)不停地從
     *  解碼器中獲取已解碼數(shù)據(jù)，當(dāng)視頻PacketQueue中沒有數(shù)據(jù)時(shí)會(huì)讓此線程進(jìn)入等待狀態(tài)(釋放cpu)；獲得解碼數(shù)據(jù)后通過queue_picture()函數(shù)將解碼數(shù)據(jù)Frame插入視頻FrameQueue隊(duì)列
     *  當(dāng)隊(duì)列滿時(shí)也會(huì)讓此線程進(jìn)入等待狀態(tài)(釋放cpu)
     *
     *  學(xué)習(xí)：線程for循環(huán)+條件變量等待鎖 保證cpu不會(huì)一直被占用，同時(shí)線程不會(huì)退出
     */
    for (;;) {
        ret = get_video_frame(is, frame);
        if (ret < 0)
            goto the_end;
        if (!ret)
            continue;

#if CONFIG_AVFILTER
        if (   last_w != frame->width
            || last_h != frame->height
            || last_format != frame->format
            || last_serial != is->viddec.pkt_serial
            || last_vfilter_idx != is->vfilter_idx) {
            av_log(NULL, AV_LOG_DEBUG,
                   "Video frame changed from size:%dx%d format:%s serial:%d to size:%dx%d format:%s serial:%d\n",
                   last_w, last_h,
                   (const char *)av_x_if_null(av_get_pix_fmt_name(last_format), "none"), last_serial,
                   frame->width, frame->height,
                   (const char *)av_x_if_null(av_get_pix_fmt_name(frame->format), "none"), is->viddec.pkt_serial);
            avfilter_graph_free(&graph);
            graph = avfilter_graph_alloc();
            if (!graph) {
                ret = AVERROR(ENOMEM);
                goto the_end;
            }
            graph->nb_threads = filter_nbthreads;
            if ((ret = configure_video_filters(graph, is, vfilters_list ? vfilters_list[is->vfilter_idx] : NULL, frame)) < 0) {
                SDL_Event event;
                event.type = FF_QUIT_EVENT;
                event.user.data1 = is;
                SDL_PushEvent(&event);
                goto the_end;
            }
            filt_in  = is->in_video_filter;
            filt_out = is->out_video_filter;
            last_w = frame->width;
            last_h = frame->height;
            last_format = frame->format;
            last_serial = is->viddec.pkt_serial;
            last_vfilter_idx = is->vfilter_idx;
            frame_rate = av_buffersink_get_frame_rate(filt_out);
        }

        ret = av_buffersrc_add_frame(filt_in, frame);
        if (ret < 0)
            goto the_end;

        while (ret >= 0) {
            is->frame_last_returned_time = av_gettime_relative() / 1000000.0;

            ret = av_buffersink_get_frame_flags(filt_out, frame, 0);
            if (ret < 0) {
                if (ret == AVERROR_EOF)
                    is->viddec.finished = is->viddec.pkt_serial;
                ret = 0;
                break;
            }

            is->frame_last_filter_delay = av_gettime_relative() / 1000000.0 - is->frame_last_returned_time;
            if (fabs(is->frame_last_filter_delay) > AV_NOSYNC_THRESHOLD / 10.0)
                is->frame_last_filter_delay = 0;
            tb = av_buffersink_get_time_base(filt_out);
#endif
            duration = (frame_rate.num && frame_rate.den ? av_q2d((AVRational){frame_rate.den, frame_rate.num}) : 0);
            pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
            ret = queue_picture(is, frame, pts, duration, frame->pkt_pos, is->viddec.pkt_serial);
            av_frame_unref(frame);
#if CONFIG_AVFILTER
            if (is->videoq.serial != is->viddec.pkt_serial)
                break;
        }
#endif

        if (ret < 0)
            goto the_end;
    }
 the_end:
#if CONFIG_AVFILTER
    avfilter_graph_free(&graph);
#endif
    av_frame_free(&frame);
    return 0;
}

for 循環(huán)保證視頻解碼線程不會(huì)退出。這里通過get_video_frame()函數(shù)從解碼器獲取待已解碼的數(shù)據(jù)，此函數(shù)不停地從視頻PacketQueue獲取未解碼數(shù)據(jù)然后送入解碼器，同時(shí)不停地從解碼器中獲取已解碼數(shù)據(jù)，當(dāng)視頻PacketQueue中沒有數(shù)據(jù)時(shí)會(huì)讓此線程進(jìn)入等待狀態(tài)(釋放cpu)；獲得解碼數(shù)據(jù)后通過queue_picture()函數(shù)將解碼數(shù)據(jù)Frame插入視頻FrameQueue隊(duì)列當(dāng)隊(duì)列滿時(shí)也會(huì)讓此線程進(jìn)入等待狀態(tài)(釋放cpu)

解碼成功后，代碼ret = queue_picture(is, frame, pts, duration, frame->pkt_pos, is->viddec.pkt_serial);代表將解碼得到的視頻幀插入解碼緩沖區(qū)

以上就是解碼線程的啟動(dòng)以及工作機(jī)制

• 視頻、音頻、字幕緩沖區(qū)設(shè)計(jì)原理

前面說道，解碼后的視頻、音頻、字幕緩沖區(qū)也是解碼器的一部分，ffplay.c是如何實(shí)現(xiàn)的呢？這里仍然以視頻為例
首先看一下它的結(jié)構(gòu)體：

/** 這是一個(gè)用數(shù)組實(shí)現(xiàn)的環(huán)形緩沖區(qū)，rindex和windex分別代表了讀寫指針?biāo)饕?max_size代表了緩沖區(qū)的最大節(jié)點(diǎn)數(shù)量(其值不大于FRAME_QUEUE_SIZE)
 *  size代表緩沖區(qū)中目前存儲(chǔ)的節(jié)點(diǎn)數(shù)量
 */
typedef struct FrameQueue {
    Frame queue[FRAME_QUEUE_SIZE];
    int rindex;
    int windex;
    int size;
    int max_size;
    int keep_last;
    int rindex_shown;
    SDL_mutex *mutex;
    SDL_cond *cond;
    PacketQueue *pktq;
} FrameQueue;

這是一個(gè)用數(shù)組實(shí)現(xiàn)的環(huán)形緩沖區(qū)，rindex和windex分別代表了讀寫指針?biāo)饕?max_size代表了緩沖區(qū)的最大節(jié)點(diǎn)數(shù)量(其值不大于FRAME_QUEUE_SIZE)size代表緩沖區(qū)中目前存儲(chǔ)的節(jié)點(diǎn)數(shù)量。這個(gè)緩沖區(qū)滿足了前面對于緩沖區(qū)設(shè)計(jì)的思考，即線程安全通過SDL_mutex鎖保證，限制了緩沖區(qū)的容量大小FRAME_QUEUE_SIZE。

思考：這里的緩沖區(qū)為什么用數(shù)組實(shí)現(xiàn)的環(huán)形隊(duì)列，首先環(huán)形隊(duì)列在每次出隊(duì)時(shí)不需要移動(dòng)其它數(shù)據(jù)這種額外的操作，其次對于單線程讀單線程寫這樣的生產(chǎn)者消費(fèi)者模型，用數(shù)組實(shí)現(xiàn)可以不需要枷鎖，以上兩點(diǎn)其實(shí)就是將效率做到極致

這里主要看一下出隊(duì)和入隊(duì)操作相關(guān)函數(shù)
入隊(duì)：

static Frame *frame_queue_peek_writable(FrameQueue *f)
{
    /** 疑問：為什么環(huán)形緩沖隊(duì)列枷鎖和解鎖操作只針對f->size變量
     *  分析：根據(jù)ffplay.c的架構(gòu)設(shè)計(jì)，解碼線程向此隊(duì)列寫入數(shù)據(jù)，渲染線程向此隊(duì)列讀取數(shù)據(jù)，讀或?qū)懖僮鞣謩e在獨(dú)立的線程，就沒有資源競爭的問題，
     *  所以對應(yīng)的讀寫操作的變量就不需要加鎖了，體現(xiàn)了加鎖最小粒度的原則提高效率
     */
    /* wait until we have space to put a new frame */
    SDL_LockMutex(f->mutex);
    while (f->size >= f->max_size &&
           !f->pktq->abort_request) {
        SDL_CondWait(f->cond, f->mutex);
    }
    SDL_UnlockMutex(f->mutex);

    if (f->pktq->abort_request)
        return NULL;

    return &f->queue[f->windex];
}
static void frame_queue_push(FrameQueue *f)
{
    if (++f->windex == f->max_size)
        f->windex = 0;
    SDL_LockMutex(f->mutex);
    f->size++;
    SDL_CondSignal(f->cond);
    SDL_UnlockMutex(f->mutex);
}

它的入隊(duì)操作要由兩個(gè)函數(shù)來完成，首先通過frame_queue_peek_writable()獲取一個(gè)可寫入數(shù)據(jù)的Frame對象，然后往這個(gè)對象里面寫入數(shù)據(jù)，最后再通過frame_queue_push()更新寫指針以及數(shù)據(jù)的大小

出隊(duì)：

static Frame *frame_queue_peek_readable(FrameQueue *f)
{
    /* wait until we have a readable a new frame */
    SDL_LockMutex(f->mutex);
    while (f->size - f->rindex_shown <= 0 &&
           !f->pktq->abort_request) {
        SDL_CondWait(f->cond, f->mutex);
    }
    SDL_UnlockMutex(f->mutex);

    if (f->pktq->abort_request)
        return NULL;

    return &f->queue[(f->rindex + f->rindex_shown) % f->max_size];
}
static void frame_queue_next(FrameQueue *f)
{
    if (f->keep_last && !f->rindex_shown) {
        f->rindex_shown = 1;
        return;
    }
    frame_queue_unref_item(&f->queue[f->rindex]);
    if (++f->rindex == f->max_size)
        f->rindex = 0;
    SDL_LockMutex(f->mutex);
    f->size--;
    SDL_CondSignal(f->cond);
    SDL_UnlockMutex(f->mutex);
}

它的出隊(duì)操作要由兩個(gè)函數(shù)來完成，首先通過frame_queue_peek_readable()獲取一個(gè)有數(shù)據(jù)的Frame對象，使用完該數(shù)據(jù)后，最后再通過frame_queue_next()更新讀指針以及數(shù)據(jù)的大小

讀完ffplay.c解碼緩沖區(qū)的實(shí)現(xiàn)后發(fā)現(xiàn)值得學(xué)習(xí)的地方還挺多的，首先利用數(shù)組實(shí)現(xiàn)的環(huán)形緩沖區(qū)保證效率，其次它只對size這個(gè)變量上鎖，保證了最低粒度的加解鎖保證效率。但是我覺得這個(gè)隊(duì)列可以實(shí)現(xiàn)無鎖隊(duì)列，效率應(yīng)該更高？

以上就是整個(gè)解碼模塊的實(shí)現(xiàn)原理

思考

如果視頻、音頻、字幕緩沖區(qū)采用無鎖隊(duì)列應(yīng)該如何實(shí)現(xiàn)、效率會(huì)提升多少？