上一篇讓我們能運(yùn)行起來(lái)kxmovie 項(xiàng)目.接下來(lái)我們就應(yīng)該分析下如何播放音頻和視頻了.
文件概覽
KxMovieViewController.h
KxMovieViewController.m
KxMovieGLView.h
KxMovieGLView.m
KxMovieDecoder.h
KxMovieDecoder.m
KxAudioManager.h
KxAudioManager.m
KxLogger.h
類(lèi)概覽
KxMovieViewController
KxMovieFrame
KxAudioFrame
KxVideoFrame
KxVideoFrameRGB
KxVideoFrameYUV
KxArtworkFrame
KxSubtitleFrame
KxMovieDecoder
KxMovieSubtitleASSParser
KxAudioManager
KxAudioManagerImpl
KxMovieGLView
KxMovieGLRenderer_RGB
KxMovieGLRenderer_YUV
類(lèi)結(jié)構(gòu)圖
類(lèi)分析
KxMovieViewController
屬性和變量
該類(lèi)相當(dāng)于播放器,屬性比較多,比較龐大.
入口方法
+(id) movieViewControllerWithContentPath: (NSString *) path
parameters: (NSDictionary *) parameters;
該類(lèi)是通過(guò)該方法進(jìn)行創(chuàng)建類(lèi)的
static NSMutableDictionary * gHistory;
+ (void)initialize
{
if (!gHistory)
gHistory = [NSMutableDictionary dictionary];
}
該類(lèi)在加載的時(shí)候,初始化了一個(gè)全局變量gHistory
+ (id) movieViewControllerWithContentPath: (NSString *) path
parameters: (NSDictionary *) parameters
{
id<KxAudioManager> audioManager = [KxAudioManager audioManager];
[audioManager activateAudioSession];
return [[KxMovieViewController alloc] initWithContentPath: path parameters: parameters];
}
初始化的時(shí)候,我們初始化了類(lèi)
KxAudioManager.該類(lèi)是單利類(lèi).
接著調(diào)用方法- (id) initWithContentPath: (NSString *) path parameters: (NSDictionary *) parameters
- (id) initWithContentPath: (NSString *) path
parameters: (NSDictionary *) parameters
{
NSAssert(path.length > 0, @"empty path");
self = [super initWithNibName:nil bundle:nil];
if (self) {
_moviePosition = 0;
// self.wantsFullScreenLayout = YES;
_parameters = parameters;
__weak KxMovieViewController *weakSelf = self;
KxMovieDecoder *decoder = [[KxMovieDecoder alloc] init];
decoder.interruptCallback = ^BOOL(){
__strong KxMovieViewController *strongSelf = weakSelf;
return strongSelf ? [strongSelf interruptDecoder] : YES;
};
dispatch_async(dispatch_get_global_queue(0, 0), ^{
NSError *error = nil;
[decoder openFile:path error:&error];
__strong KxMovieViewController *strongSelf = weakSelf;
if (strongSelf) {
dispatch_sync(dispatch_get_main_queue(), ^{
[strongSelf setMovieDecoder:decoder withError:error];
});
}
});
}
return self;
}
1 .該類(lèi)初始化了KxMovieDecoder(解碼器).并且設(shè)置是否需要中斷回調(diào)函數(shù).
2.開(kāi)啟全局線程 解碼器打開(kāi)文件
3 調(diào)用- (void) setMovieDecoder: (KxMovieDecoder *) decoder withError: (NSError *) error
4 .返回 self
- (void) setMovieDecoder: (KxMovieDecoder *) decoder
withError: (NSError *) error
{
LoggerStream(2, @"setMovieDecoder");
if (!error && decoder) {
_decoder = decoder;
_dispatchQueue = dispatch_queue_create("KxMovie", DISPATCH_QUEUE_SERIAL);
_videoFrames = [NSMutableArray array];
_audioFrames = [NSMutableArray array];
if (_decoder.subtitleStreamsCount) {
_subtitles = [NSMutableArray array];
}
if (_decoder.isNetwork) {
_minBufferedDuration = NETWORK_MIN_BUFFERED_DURATION;
_maxBufferedDuration = NETWORK_MAX_BUFFERED_DURATION;
} else {
_minBufferedDuration = LOCAL_MIN_BUFFERED_DURATION;
_maxBufferedDuration = LOCAL_MAX_BUFFERED_DURATION;
}
if (!_decoder.validVideo)
_minBufferedDuration *= 10.0; // increase for audio
// allow to tweak some parameters at runtime
if (_parameters.count) {
id val;
val = [_parameters valueForKey: KxMovieParameterMinBufferedDuration];
if ([val isKindOfClass:[NSNumber class]])
_minBufferedDuration = [val floatValue];
val = [_parameters valueForKey: KxMovieParameterMaxBufferedDuration];
if ([val isKindOfClass:[NSNumber class]])
_maxBufferedDuration = [val floatValue];
val = [_parameters valueForKey: KxMovieParameterDisableDeinterlacing];
if ([val isKindOfClass:[NSNumber class]])
_decoder.disableDeinterlacing = [val boolValue];
if (_maxBufferedDuration < _minBufferedDuration)
_maxBufferedDuration = _minBufferedDuration * 2;
}
LoggerStream(2, @"buffered limit: %.1f - %.1f", _minBufferedDuration, _maxBufferedDuration);
if (self.isViewLoaded) {
[self setupPresentView];
_progressLabel.hidden = NO;
_progressSlider.hidden = NO;
_leftLabel.hidden = NO;
_infoButton.hidden = NO;
if (_activityIndicatorView.isAnimating) {
[_activityIndicatorView stopAnimating];
// if (self.view.window)
[self restorePlay];
}
}
} else {
if (self.isViewLoaded && self.view.window) {
[_activityIndicatorView stopAnimating];
if (!_interrupted)
[self handleDecoderMovieError: error];
}
}
}
該函數(shù)根據(jù)解碼器解析數(shù)據(jù)的結(jié)構(gòu)設(shè)置相關(guān)變量
在該函數(shù)里調(diào)用- (void) setupPresentView.該函數(shù)是設(shè)置播放視頻的view的
而- (void) restorePlay函數(shù)是對(duì)視頻進(jìn)行播放
- (void) setupPresentView
{
CGRect bounds = self.view.bounds;
if (_decoder.validVideo) {
_glView = [[KxMovieGLView alloc] initWithFrame:bounds decoder:_decoder];
}
if (!_glView) {
LoggerVideo(0, @"fallback to use RGB video frame and UIKit");
[_decoder setupVideoFrameFormat:KxVideoFrameFormatRGB];
_imageView = [[UIImageView alloc] initWithFrame:bounds];
_imageView.backgroundColor = [UIColor blackColor];
}
UIView *frameView = [self frameView];
frameView.contentMode = UIViewContentModeScaleAspectFit;
frameView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleTopMargin | UIViewAutoresizingFlexibleRightMargin | UIViewAutoresizingFlexibleLeftMargin | UIViewAutoresizingFlexibleHeight | UIViewAutoresizingFlexibleBottomMargin;
[self.view insertSubview:frameView atIndex:0];
if (_decoder.validVideo) {
[self setupUserInteraction];
} else {
_imageView.image = [UIImage imageNamed:@"kxmovie.bundle/music_icon.png"];
_imageView.contentMode = UIViewContentModeCenter;
}
self.view.backgroundColor = [UIColor clearColor];
if (_decoder.duration == MAXFLOAT) {
_leftLabel.text = @"\u221E"; // infinity
_leftLabel.font = [UIFont systemFontOfSize:14];
CGRect frame;
frame = _leftLabel.frame;
frame.origin.x += 40;
frame.size.width -= 40;
_leftLabel.frame = frame;
frame =_progressSlider.frame;
frame.size.width += 40;
_progressSlider.frame = frame;
} else {
[_progressSlider addTarget:self
action:@selector(progressDidChange:)
forControlEvents:UIControlEventValueChanged];
}
if (_decoder.subtitleStreamsCount) {
CGSize size = self.view.bounds.size;
_subtitlesLabel = [[UILabel alloc] initWithFrame:CGRectMake(0, size.height, size.width, 0)];
_subtitlesLabel.numberOfLines = 0;
_subtitlesLabel.backgroundColor = [UIColor clearColor];
_subtitlesLabel.opaque = NO;
_subtitlesLabel.adjustsFontSizeToFitWidth = NO;
_subtitlesLabel.textAlignment = NSTextAlignmentCenter;
_subtitlesLabel.autoresizingMask = UIViewAutoresizingFlexibleWidth;
_subtitlesLabel.textColor = [UIColor whiteColor];
_subtitlesLabel.font = [UIFont systemFontOfSize:16];
_subtitlesLabel.hidden = YES;
[self.view addSubview:_subtitlesLabel];
}
}
該函數(shù)就是設(shè)置播放視頻的view和UI繪制
- (void) restorePlay
{
NSNumber *n = [gHistory valueForKey:_decoder.path];
if (n)
[self updatePosition:n.floatValue playMode:YES];
else
[self play];
}
播放視頻
-(void) play
{
if (self.playing)
return;
if (!_decoder.validVideo &&
!_decoder.validAudio) {
return;
}
if (_interrupted)
return;
self.playing = YES;
_interrupted = NO;
_disableUpdateHUD = NO;
_tickCorrectionTime = 0;
_tickCounter = 0;
#ifdef DEBUG
_debugStartTime = -1;
#endif
[self asyncDecodeFrames];
[self updatePlayButton];
dispatch_time_t popTime = dispatch_time(DISPATCH_TIME_NOW, 0.1 * NSEC_PER_SEC);
dispatch_after(popTime, dispatch_get_main_queue(), ^(void){
[self tick];
});
if (_decoder.validAudio)
[self enableAudio:YES];
LoggerStream(1, @"play movie");
}
該函數(shù) 調(diào)用asyncDecodeFrames 解碼視頻
調(diào)用- (void) enableAudio: (BOOL) on 解碼音頻
- (void) asyncDecodeFrames
{
if (self.decoding)
return;
__weak KxMovieViewController *weakSelf = self;
__weak KxMovieDecoder *weakDecoder = _decoder;
const CGFloat duration = _decoder.isNetwork ? .0f : 0.1f;
self.decoding = YES;
dispatch_async(_dispatchQueue, ^{
{
__strong KxMovieViewController *strongSelf = weakSelf;
if (!strongSelf.playing)
return;
}
BOOL good = YES;
while (good) {
good = NO;
@autoreleasepool {
__strong KxMovieDecoder *decoder = weakDecoder;
if (decoder && (decoder.validVideo || decoder.validAudio)) {
NSArray *frames = [decoder decodeFrames:duration];
if (frames.count) {
__strong KxMovieViewController *strongSelf = weakSelf;
if (strongSelf)
good = [strongSelf addFrames:frames];
}
}
}
}
{
__strong KxMovieViewController *strongSelf = weakSelf;
if (strongSelf) strongSelf.decoding = NO;
}
});
}
在該函數(shù)中讓解碼器decoder對(duì)間隔時(shí)間進(jìn)行解碼.獲取解碼數(shù)組圖像
調(diào)用 - (BOOL) addFrames: (NSArray *)frames.將解碼的視頻增加到緩存中
- (BOOL) addFrames: (NSArray *)frames
{
if (_decoder.validVideo) {
@synchronized(_videoFrames) {
for (KxMovieFrame *frame in frames)
if (frame.type == KxMovieFrameTypeVideo) {
[_videoFrames addObject:frame];
_bufferedDuration += frame.duration;
}
}
}
if (_decoder.validAudio) {
@synchronized(_audioFrames) {
for (KxMovieFrame *frame in frames)
if (frame.type == KxMovieFrameTypeAudio) {
[_audioFrames addObject:frame];
if (!_decoder.validVideo)
_bufferedDuration += frame.duration;
}
}
if (!_decoder.validVideo) {
for (KxMovieFrame *frame in frames)
if (frame.type == KxMovieFrameTypeArtwork)
self.artworkFrame = (KxArtworkFrame *)frame;
}
}
if (_decoder.validSubtitles) {
@synchronized(_subtitles) {
for (KxMovieFrame *frame in frames)
if (frame.type == KxMovieFrameTypeSubtitle) {
[_subtitles addObject:frame];
}
}
}
return self.playing && _bufferedDuration < _maxBufferedDuration;
}
將音視頻對(duì)象加入到指定緩存中
- (void) enableAudio: (BOOL) on
{
id<KxAudioManager> audioManager = [KxAudioManager audioManager];
if (on && _decoder.validAudio) {
audioManager.outputBlock = ^(float *outData, UInt32 numFrames, UInt32 numChannels) {
[self audioCallbackFillData: outData numFrames:numFrames numChannels:numChannels];
};
[audioManager play];
LoggerAudio(2, @"audio device smr: %d fmt: %d chn: %d",
(int)audioManager.samplingRate,
(int)audioManager.numBytesPerSample,
(int)audioManager.numOutputChannels);
} else {
[audioManager pause];
audioManager.outputBlock = nil;
}
}
播放音頻
這就是完整的播放邏輯. UML時(shí)序圖
時(shí)序圖中只列舉了相關(guān)關(guān)鍵函數(shù)的調(diào)用.許多變量的初始化都沒(méi)詳細(xì)書(shū)寫(xiě)
變量
_dispatchQueue 解碼隊(duì)列
_videoFrames 視頻幀數(shù)組
_audioFrames 音頻鎮(zhèn)數(shù)組
解碼器 KxMovieDecoder
屬性和變量
![]()
KxMovieDecoder屬性和變量
入口方法
+ (id) movieDecoderWithContentPath: (NSString *) path
error: (NSError **) perror;
- (BOOL) openFile: (NSString *) path
error: (NSError **) perror;
-init;
其實(shí)第一個(gè)方法就是下面兩個(gè)方法的組合體
+ (id) movieDecoderWithContentPath: (NSString *) path
error: (NSError **) perror
{
KxMovieDecoder *mp = [[KxMovieDecoder alloc] init];
if (mp) {
[mp openFile:path error:perror];
}
return mp;
}
該類(lèi)在加載 的時(shí)候初始化了ffmpeg
+ (void)initialize
{
av_log_set_callback(FFLog);
av_register_all();
avformat_network_init();
}
- av_log_set_callback 設(shè)置log回調(diào)函數(shù)
- av_register_all 注冊(cè)ffmpeg組件
- avformat_network_init 默認(rèn)開(kāi)始網(wǎng)絡(luò)模塊
我們知道av_register_all 是在文件av_format.h文件中.我們看看這個(gè)類(lèi)的介紹
av_format.h頭文件有該類(lèi)的基本使用
該庫(kù)的主要作用是I/O 和 Muxing/Demuxing
該庫(kù)有兩個(gè)目的就是編解碼音視頻
使用該庫(kù)我們首先必須要調(diào)用av_register_all()函數(shù)注冊(cè)所有的編解碼組件,如果我們需要網(wǎng)絡(luò)模塊,那么需要調(diào)用avformat_network_init()
AVInputFormat struct代表input format .AVOutputFormat struct代表 output format. 我們可以通過(guò)av_iformat_next()和av_oformat_next()查看我們注冊(cè)的所有的input 和output format
我們可以通過(guò)avio_enum_protocols()查看該庫(kù)支持的協(xié)議
AVFormatContext 是muxing 的和 demuxing 橋梁,用來(lái)讀寫(xiě)文件的.
AVFormatContext 必須通過(guò)avformat_alloc_context()來(lái)創(chuàng)建.不能在棧上或者 用av_malloc()來(lái)創(chuàng)建.
AVFormatContext.iformat是對(duì)輸入的引用
AVFormatContext.oformat是對(duì)輸出的引用
input(iformat) 可以自動(dòng)設(shè)置或者用戶(hù)設(shè)置.但是output(oformat) 總是用戶(hù)設(shè)置
AVFormatContext.streams是個(gè)數(shù)組,裝有AVStream.該屬性一般是通過(guò)下標(biāo)來(lái)訪問(wèn)的
AVFormatContext.pb 是I/o 的上下文,可以打開(kāi)該庫(kù)和用戶(hù)設(shè)置input和output.
url 字符串是代表scheme/協(xié)議. url需要有
:.如果沒(méi)有:,那么認(rèn)為是本地文件,但是這種寫(xiě)法被廢棄了.想代表本地文件,需要使用file:格式
請(qǐng)注意，一些scheme/protocol非常強(qiáng)大，允許訪問(wèn)本地和遠(yuǎn)程文件、部分文件、本地音頻和視頻設(shè)備等。
解碼 Demuxing
Demuxers 從音視頻文件中讀取數(shù)據(jù)并將其分成chunks 數(shù)據(jù) ,該分割的數(shù)據(jù)用
AVPacket代表.AVPacket包含一幀或者多幀.
我們通過(guò)avformat_open_input()打開(kāi)輸入源.
av_read_frame()讀取AVPacket
avformat_close_input()關(guān)閉數(shù)據(jù)源
代碼片段打開(kāi)媒體文件
const char *url = "file:in.mp3"; AVFormatContext *s = NULL; int ret = avformat_open_input(&s, url, NULL, NULL); if (ret < 0) abort();首先該函數(shù)先給AVFormatContext 分配內(nèi)存,然后打開(kāi)文件讀取頭文件,自動(dòng)將其賦值到AVFormatContext 的iformat上.對(duì)于沒(méi)有頭文件的輸入格式或者信息不夠的,我們可以調(diào)用
avformat_find_stream_info函數(shù),該函數(shù)能試著讀取和解碼一些frame,從中找到我們丟失的信息.
如果預(yù)先通過(guò)avformat_alloc_context函數(shù)創(chuàng)建了AVFormatContext.我們?cè)趥鬟f給avformat_open_input之前需要做一些調(diào)整. 如果我們想用自定義函數(shù)來(lái)讀取輸入的數(shù)據(jù).那么我們需要通過(guò)函數(shù)avio_alloc_context創(chuàng)建自己的AVIOContext,然后通過(guò)callbacks 函數(shù)來(lái)完成自定義函數(shù)對(duì)數(shù)據(jù)的解析.我們需要通過(guò)AVFormatContext.pb 來(lái)指向我們創(chuàng)建的新的AVIOContext
許多輸入格式在avformat_open_input 返回之前我們是不知道具體的格式信息的.因此我們?cè)谝呀?jīng)預(yù)先創(chuàng)建的AVFormatContext不能設(shè)置demuxer private. 因此想要設(shè)置這些參數(shù),我們需要通過(guò)
avformat_open_input函數(shù)將這些參數(shù)設(shè)定進(jìn)去.AVDictionary *options = NULL; av_dict_set(&options, "video_size", "640x480", 0); av_dict_set(&options, "pixel_format", "rgb24", 0); if (avformat_open_input(&s, url, NULL, &options) < 0) abort(); av_dict_free(&options);上述代碼我們給demuxer設(shè)置私有參數(shù)video_size 和pixel_format .
對(duì)于the rawvideo demuxer 是必須的,要不他不知道怎么解釋視頻源數(shù)據(jù). 如果這些參數(shù)設(shè)置與原始數(shù)據(jù)的不同,那么這些參數(shù)也就不起作用了.不會(huì)識(shí)別的選項(xiàng)會(huì)在可選字典中返回(識(shí)別的會(huì)在字典中刪除).
下面的代碼片段是處理不認(rèn)識(shí)的可選參數(shù)AVDictionaryEntry *e; if (e = av_dict_get(options, "", NULL, AV_DICT_IGNORE_SUFFIX)) { fprintf(stderr, "Option %s not recognized by the demuxer.\n", e->key); abort(); }
如何讀取數(shù)據(jù)
我們可以反復(fù)調(diào)用av_read_frame函數(shù)獲取數(shù)據(jù).調(diào)用該函數(shù),要是成功就返回AVPacket數(shù)據(jù).我們可以通過(guò)調(diào)用avcodec_send_packet或者avcodec_decode_subtitle2來(lái)解碼AVPacket
AVPacket.pts, AVPacket.dts 和 AVPacket.duration 的時(shí)間信息會(huì)在解碼的時(shí)候設(shè)置.(解碼的時(shí)候知道的前提).如果流中沒(méi)有提供這些信息,這些值可能被重新設(shè)置.這些時(shí)間信息的基本單位是AVStream.time_base(都是該值的整數(shù)倍).需要和該值相乘換算成秒
要是AVPacket.buf的在返回的packet中被設(shè)置.那么該空間是動(dòng)態(tài)分配的,用戶(hù)可以無(wú)限期的持有它.如果AVPacket.buf是NULL,那么返回的是一個(gè)在demuxer中的靜態(tài)區(qū)域.該AVPacket有效期到下一次調(diào)用av_read_frame.如果調(diào)用者需要一個(gè)長(zhǎng)的時(shí)間周期保持AVPacket.那么我們可以通過(guò)調(diào)用av_dup_packet 來(lái)復(fù)制一份出來(lái)
我們需要調(diào)用av_packet_unref來(lái)釋放該空間.
解碼 Muxing
Muxers 可以解碼AVPacket 數(shù)據(jù)將其寫(xiě)入文件或者指定的輸出格式數(shù)據(jù)
avformat_write_header()寫(xiě)文件的頭文件
av_write_frame() / av_interleaved_write_frame()寫(xiě)packets數(shù)據(jù)到文件
av_write_trailer()寫(xiě)文件結(jié)尾
開(kāi)啟解碼Muxing ,首先要調(diào)用avformat_alloc_context創(chuàng)建上下文.
我們需要給AVFormatContext.oformat附上該值
并且我們也要設(shè)置AVFormatContext.pb 的值,該值必須是avio_open2()返回的上下文或者自定義的上下文
我們需要通過(guò)avformat_new_stream()創(chuàng)建一個(gè)stream.我們應(yīng)該給創(chuàng)建好的steam的屬性codecpar 賦值(該參數(shù)是流編碼需要的參數(shù)).
AVStream.time_base 也需要設(shè)置.
大概流程如下
這里是真正的初始化函數(shù)
- (BOOL) openFile: (NSString *) path
error: (NSError **) perror
{
NSAssert(path, @"nil path");
NSAssert(!_formatCtx, @"already open");
_isNetwork = isNetworkPath(path);
static BOOL needNetworkInit = YES;
if (needNetworkInit && _isNetwork) {
needNetworkInit = NO;
avformat_network_init();
}
_path = path;
kxMovieError errCode = [self openInput: path];
if (errCode == kxMovieErrorNone) {
kxMovieError videoErr = [self openVideoStream];
kxMovieError audioErr = [self openAudioStream];
_subtitleStream = -1;
if (videoErr != kxMovieErrorNone &&
audioErr != kxMovieErrorNone) {
errCode = videoErr; // both fails
} else {
_subtitleStreams = collectStreams(_formatCtx, AVMEDIA_TYPE_SUBTITLE);
}
}
if (errCode != kxMovieErrorNone) {
[self closeFile];
NSString *errMsg = errorMessage(errCode);
LoggerStream(0, @"%@, %@", errMsg, path.lastPathComponent);
if (perror)
*perror = kxmovieError(errCode, errMsg);
return NO;
}
return YES;
}
分步解釋
1 isNetworkPath 是判斷是否是需要網(wǎng)絡(luò)模塊. 需要網(wǎng)絡(luò)模塊加載
avformat_network_init
- 調(diào)用函數(shù)
- (kxMovieError) openInput: (NSString *) path打開(kāi)文件
3 打開(kāi)openVideoStream
4 打開(kāi)openAudioStream
5 _subtitleStreams 參數(shù)設(shè)置
流程很清晰,分別看
- (kxMovieError) openInput: (NSString *) path
{
AVFormatContext *formatCtx = NULL;
if (_interruptCallback) {
formatCtx = avformat_alloc_context();
if (!formatCtx)
return kxMovieErrorOpenFile;
AVIOInterruptCB cb = {interrupt_callback, (__bridge void *)(self)};
formatCtx->interrupt_callback = cb;
}
if (avformat_open_input(&formatCtx, [path cStringUsingEncoding: NSUTF8StringEncoding], NULL, NULL) < 0) {
if (formatCtx)
avformat_free_context(formatCtx);
return kxMovieErrorOpenFile;
}
if (avformat_find_stream_info(formatCtx, NULL) < 0) {
avformat_close_input(&formatCtx);
return kxMovieErrorStreamInfoNotFound;
}
av_dump_format(formatCtx, 0, [path.lastPathComponent cStringUsingEncoding: NSUTF8StringEncoding], false);
_formatCtx = formatCtx;
return kxMovieErrorNone;
}
這里我們?cè)谕饨缭O(shè)置了_interruptCallback 參數(shù),因此,這里我們采用的是用戶(hù)創(chuàng)建AVFormatContext 結(jié)構(gòu)體
1.創(chuàng)建AVFormatContext 對(duì)象,并且設(shè)置interrupt_callback回調(diào)函數(shù)
2.調(diào)用avformat_open_input打開(kāi)文件
3.調(diào)用函數(shù)avformat_find_stream_info檢查對(duì)象AVFormatContext 是否有對(duì)應(yīng)的解碼器
4.調(diào)用av_dump_format 打印信息
- (kxMovieError) openVideoStream
{
kxMovieError errCode = kxMovieErrorStreamNotFound;
_videoStream = -1;
_artworkStream = -1;
_videoStreams = collectStreams(_formatCtx, AVMEDIA_TYPE_VIDEO);
for (NSNumber *n in _videoStreams) {
const NSUInteger iStream = n.integerValue;
if (0 == (_formatCtx->streams[iStream]->disposition & AV_DISPOSITION_ATTACHED_PIC)) {
errCode = [self openVideoStream: iStream];
if (errCode == kxMovieErrorNone)
break;
} else {
_artworkStream = iStream;
}
}
return errCode;
}
1 從_formatCtx->stream中查找包含AVMEDIA_TYPE_VIDEO的AVStream
2.依次判斷AVStream的disposition 是否支持 AV_DISPOSITION_ATTACHED_PIC(圖像)
3 支持圖像調(diào)用- (kxMovieError) openVideoStream: (NSInteger) videoStream
- (kxMovieError) openVideoStream: (NSInteger) videoStream
{
// get a pointer to the codec context for the video stream
AVCodecContext *codecCtx = _formatCtx->streams[videoStream]->codec;
// find the decoder for the video stream
AVCodec *codec = avcodec_find_decoder(codecCtx->codec_id);
if (!codec)
return kxMovieErrorCodecNotFound;
// inform the codec that we can handle truncated bitstreams -- i.e.,
// bitstreams where frame boundaries can fall in the middle of packets
//if(codec->capabilities & CODEC_CAP_TRUNCATED)
// _codecCtx->flags |= CODEC_FLAG_TRUNCATED;
// open codec
if (avcodec_open2(codecCtx, codec, NULL) < 0)
return kxMovieErrorOpenCodec;
_videoFrame = av_frame_alloc();
if (!_videoFrame) {
avcodec_close(codecCtx);
return kxMovieErrorAllocateFrame;
}
_videoStream = videoStream;
_videoCodecCtx = codecCtx;
// determine fps
AVStream *st = _formatCtx->streams[_videoStream];
avStreamFPSTimeBase(st, 0.04, &_fps, &_videoTimeBase);
LoggerVideo(1, @"video codec size: %d:%d fps: %.3f tb: %f",
self.frameWidth,
self.frameHeight,
_fps,
_videoTimeBase);
LoggerVideo(1, @"video start time %f", st->start_time * _videoTimeBase);
LoggerVideo(1, @"video disposition %d", st->disposition);
return kxMovieErrorNone;
}
1獲取AVCodecContext 對(duì)象
2 調(diào)用avcodec_find_decoder查找解碼器
3.avcodec_open2打開(kāi)解碼器
- 調(diào)用avStreamFPSTimeBase 設(shè)基準(zhǔn)時(shí)間
- (kxMovieError) openAudioStream
{
kxMovieError errCode = kxMovieErrorStreamNotFound;
_audioStream = -1;
_audioStreams = collectStreams(_formatCtx, AVMEDIA_TYPE_AUDIO);
for (NSNumber *n in _audioStreams) {
errCode = [self openAudioStream: n.integerValue];
if (errCode == kxMovieErrorNone)
break;
}
return errCode;
}
音頻也是一樣的查找過(guò)程只不過(guò)查找的格式是AVMEDIA_TYPE_AUDIO
- (kxMovieError) openAudioStream: (NSInteger) audioStream
{
AVCodecContext *codecCtx = _formatCtx->streams[audioStream]->codec;
SwrContext *swrContext = NULL;
AVCodec *codec = avcodec_find_decoder(codecCtx->codec_id);
if(!codec)
return kxMovieErrorCodecNotFound;
if (avcodec_open2(codecCtx, codec, NULL) < 0)
return kxMovieErrorOpenCodec;
if (!audioCodecIsSupported(codecCtx)) {
id<KxAudioManager> audioManager = [KxAudioManager audioManager];
swrContext = swr_alloc_set_opts(NULL,
av_get_default_channel_layout(audioManager.numOutputChannels),
AV_SAMPLE_FMT_S16,
audioManager.samplingRate,
av_get_default_channel_layout(codecCtx->channels),
codecCtx->sample_fmt,
codecCtx->sample_rate,
0,
NULL);
if (!swrContext ||
swr_init(swrContext)) {
if (swrContext)
swr_free(&swrContext);
avcodec_close(codecCtx);
return kxMovieErroReSampler;
}
}
_audioFrame = av_frame_alloc();
if (!_audioFrame) {
if (swrContext)
swr_free(&swrContext);
avcodec_close(codecCtx);
return kxMovieErrorAllocateFrame;
}
_audioStream = audioStream;
_audioCodecCtx = codecCtx;
_swrContext = swrContext;
AVStream *st = _formatCtx->streams[_audioStream];
avStreamFPSTimeBase(st, 0.025, 0, &_audioTimeBase);
LoggerAudio(1, @"audio codec smr: %.d fmt: %d chn: %d tb: %f %@",
_audioCodecCtx->sample_rate,
_audioCodecCtx->sample_fmt,
_audioCodecCtx->channels,
_audioTimeBase,
_swrContext ? @"resample" : @"");
return kxMovieErrorNone;
}
1.調(diào)用
avcodec_find_decoder查找音頻解碼器
2.調(diào)用avcodec_open2 打開(kāi)音頻解碼器
- 設(shè)置音頻到KxAudioManager對(duì)象
- 設(shè)置片段基本時(shí)間
流程框圖如下
從上面的時(shí)序圖我們能看看出來(lái)avformat 解碼的基本流程
注冊(cè)所有組件 -> 打開(kāi)網(wǎng)絡(luò)組件-> 打開(kāi)文件-> 查找解碼器->解碼
這里我們需要看下函數(shù)static void avStreamFPSTimeBase(AVStream *st, CGFloat defaultTimeBase, CGFloat *pFPS, CGFloat *pTimeBase)設(shè)置解碼片段時(shí)間
static void avStreamFPSTimeBase(AVStream *st, CGFloat defaultTimeBase, CGFloat *pFPS, CGFloat *pTimeBase)
{
CGFloat fps, timebase;
if (st->time_base.den && st->time_base.num)
timebase = av_q2d(st->time_base);
else if(st->codec->time_base.den && st->codec->time_base.num)
timebase = av_q2d(st->codec->time_base);
else
timebase = defaultTimeBase;
if (st->codec->ticks_per_frame != 1) {
LoggerStream(0, @"WARNING: st.codec.ticks_per_frame=%d", st->codec->ticks_per_frame);
//timebase *= st->codec->ticks_per_frame;
}
if (st->avg_frame_rate.den && st->avg_frame_rate.num)
fps = av_q2d(st->avg_frame_rate);
else if (st->r_frame_rate.den && st->r_frame_rate.num)
fps = av_q2d(st->r_frame_rate);
else
fps = 1.0 / timebase;
if (pFPS)
*pFPS = fps;
if (pTimeBase)
*pTimeBase = timebase;
}
AVStream.time_base 解碼過(guò)程中是被avformat設(shè)置的代表時(shí)間戳
AVStream. avg_frame_rate 設(shè)置解碼的fps
以上沒(méi)有真正的解碼過(guò)程.這里我們看看視頻的真正解碼
- (NSArray *) decodeFrames: (CGFloat) minDuration
{
if (_videoStream == -1 &&
_audioStream == -1)
return nil;
NSMutableArray *result = [NSMutableArray array];
AVPacket packet;
CGFloat decodedDuration = 0;
BOOL finished = NO;
while (!finished) {
if (av_read_frame(_formatCtx, &packet) < 0) {
_isEOF = YES;
break;
}
if (packet.stream_index ==_videoStream) {
int pktSize = packet.size;
while (pktSize > 0) {
int gotframe = 0;
int len = avcodec_decode_video2(_videoCodecCtx,
_videoFrame,
&gotframe,
&packet);
if (len < 0) {
LoggerVideo(0, @"decode video error, skip packet");
break;
}
if (gotframe) {
if (!_disableDeinterlacing &&
_videoFrame->interlaced_frame) {
// avpicture_deinterlace((AVPicture*)_videoFrame,
// (AVPicture*)_videoFrame,
// _videoCodecCtx->pix_fmt,
// _videoCodecCtx->width,
// _videoCodecCtx->height);
}
KxVideoFrame *frame = [self handleVideoFrame];
if (frame) {
[result addObject:frame];
_position = frame.position;
decodedDuration += frame.duration;
if (decodedDuration > minDuration)
finished = YES;
}
}
if (0 == len)
break;
pktSize -= len;
}
} else if (packet.stream_index == _audioStream) {
int pktSize = packet.size;
while (pktSize > 0) {
int gotframe = 0;
int len = avcodec_decode_audio4(_audioCodecCtx,
_audioFrame,
&gotframe,
&packet);
if (len < 0) {
LoggerAudio(0, @"decode audio error, skip packet");
break;
}
if (gotframe) {
KxAudioFrame * frame = [self handleAudioFrame];
if (frame) {
[result addObject:frame];
if (_videoStream == -1) {
_position = frame.position;
decodedDuration += frame.duration;
if (decodedDuration > minDuration)
finished = YES;
}
}
}
if (0 == len)
break;
pktSize -= len;
}
} else if (packet.stream_index == _artworkStream) {
if (packet.size) {
KxArtworkFrame *frame = [[KxArtworkFrame alloc] init];
frame.picture = [NSData dataWithBytes:packet.data length:packet.size];
[result addObject:frame];
}
} else if (packet.stream_index == _subtitleStream) {
int pktSize = packet.size;
while (pktSize > 0) {
AVSubtitle subtitle;
int gotsubtitle = 0;
int len = avcodec_decode_subtitle2(_subtitleCodecCtx,
&subtitle,
&gotsubtitle,
&packet);
if (len < 0) {
LoggerStream(0, @"decode subtitle error, skip packet");
break;
}
if (gotsubtitle) {
KxSubtitleFrame *frame = [self handleSubtitle: &subtitle];
if (frame) {
[result addObject:frame];
}
avsubtitle_free(&subtitle);
}
if (0 == len)
break;
pktSize -= len;
}
}
av_free_packet(&packet);
}
return result;
}
這個(gè)函數(shù)的調(diào)用是在KxMovieViewController類(lèi)的專(zhuān)有_dispatchQueue中進(jìn)行解碼的.并且執(zhí)行該函數(shù),我們已經(jīng)找到了音視頻的專(zhuān)有的解碼器了.這里講解我們?nèi)绾潍@取音視頻的AVPacket
調(diào)用
av_read_frame獲取AVPacket
判斷AVPacket是否是我們需要解碼的包,不是就繼續(xù)讀取包
對(duì)于視頻調(diào)用avcodec_decode_video2對(duì)視頻解碼.數(shù)據(jù)幀存放在_videoFrame變量 中. 調(diào)用handleVideoFrame獲取KxVideoFrame對(duì)象存入到數(shù)組中avcodec_decode_video2 在最新的4.3 版本被廢棄了.可以使用avcodec_send_packet() and avcodec_receive_frame()來(lái)解碼
音頻和視頻都是同樣的道理
這里我們看如何處理vidoeFrame的
- (KxVideoFrame *) handleVideoFrame
{
if (!_videoFrame->data[0])
return nil;
KxVideoFrame *frame;
if (_videoFrameFormat == KxVideoFrameFormatYUV) {
KxVideoFrameYUV * yuvFrame = [[KxVideoFrameYUV alloc] init];
yuvFrame.luma = copyFrameData(_videoFrame->data[0],
_videoFrame->linesize[0],
_videoCodecCtx->width,
_videoCodecCtx->height);
yuvFrame.chromaB = copyFrameData(_videoFrame->data[1],
_videoFrame->linesize[1],
_videoCodecCtx->width / 2,
_videoCodecCtx->height / 2);
yuvFrame.chromaR = copyFrameData(_videoFrame->data[2],
_videoFrame->linesize[2],
_videoCodecCtx->width / 2,
_videoCodecCtx->height / 2);
frame = yuvFrame;
} else {
if (!_swsContext &&
![self setupScaler]) {
LoggerVideo(0, @"fail setup video scaler");
return nil;
}
sws_scale(_swsContext,
(const uint8_t **)_videoFrame->data,
_videoFrame->linesize,
0,
_videoCodecCtx->height,
_picture.data,
_picture.linesize);
KxVideoFrameRGB *rgbFrame = [[KxVideoFrameRGB alloc] init];
rgbFrame.linesize = _picture.linesize[0];
rgbFrame.rgb = [NSData dataWithBytes:_picture.data[0]
length:rgbFrame.linesize * _videoCodecCtx->height];
frame = rgbFrame;
}
frame.width = _videoCodecCtx->width;
frame.height = _videoCodecCtx->height;
frame.position = av_frame_get_best_effort_timestamp(_videoFrame) * _videoTimeBase;
const int64_t frameDuration = av_frame_get_pkt_duration(_videoFrame);
if (frameDuration) {
frame.duration = frameDuration * _videoTimeBase;
frame.duration += _videoFrame->repeat_pict * _videoTimeBase * 0.5;
//if (_videoFrame->repeat_pict > 0) {
// LoggerVideo(0, @"_videoFrame.repeat_pict %d", _videoFrame->repeat_pict);
//}
} else {
// sometimes, ffmpeg unable to determine a frame duration
// as example yuvj420p stream from web camera
frame.duration = 1.0 / _fps;
}
#if 0
LoggerVideo(2, @"VFD: %.4f %.4f | %lld ",
frame.position,
frame.duration,
av_frame_get_pkt_pos(_videoFrame));
#endif
return frame;
}
這里我們采用的是_videoFrameFormat 變量的值是KxVideoFrameFormatYUV.在KxMovieGLView 中設(shè)置的
KxVideoFrameFormatYUV對(duì)應(yīng)的avformat中的AV_PIX_FMT_YUV420P
AV_PIX_FMT_YUV420P的排列格式是< planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
我們對(duì)_videoFrame中的數(shù)據(jù)解析到到KxVideoFrameYUV 中
這里我們需要了解下yuv420 的視頻流格式.可以參考這里
bpp 位圖的單位
16 bit per pixel
每個(gè)像素點(diǎn)分解為16個(gè)字節(jié)來(lái)表示
接下來(lái)看音頻編碼
- (KxAudioFrame *) handleAudioFrame
{
if (!_audioFrame->data[0])
return nil;
id<KxAudioManager> audioManager = [KxAudioManager audioManager];
const NSUInteger numChannels = audioManager.numOutputChannels;
NSInteger numFrames;
void * audioData;
if (_swrContext) {
const NSUInteger ratio = MAX(1, audioManager.samplingRate / _audioCodecCtx->sample_rate) *
MAX(1, audioManager.numOutputChannels / _audioCodecCtx->channels) * 2;
const int bufSize = av_samples_get_buffer_size(NULL,
audioManager.numOutputChannels,
_audioFrame->nb_samples * ratio,
AV_SAMPLE_FMT_S16,
1);
if (!_swrBuffer || _swrBufferSize < bufSize) {
_swrBufferSize = bufSize;
_swrBuffer = realloc(_swrBuffer, _swrBufferSize);
}
Byte *outbuf[2] = { _swrBuffer, 0 };
numFrames = swr_convert(_swrContext,
outbuf,
_audioFrame->nb_samples * ratio,
(const uint8_t **)_audioFrame->data,
_audioFrame->nb_samples);
if (numFrames < 0) {
LoggerAudio(0, @"fail resample audio");
return nil;
}
//int64_t delay = swr_get_delay(_swrContext, audioManager.samplingRate);
//if (delay > 0)
// LoggerAudio(0, @"resample delay %lld", delay);
audioData = _swrBuffer;
} else {
if (_audioCodecCtx->sample_fmt != AV_SAMPLE_FMT_S16) {
NSAssert(false, @"bucheck, audio format is invalid");
return nil;
}
audioData = _audioFrame->data[0];
numFrames = _audioFrame->nb_samples;
}
const NSUInteger numElements = numFrames * numChannels;
NSMutableData *data = [NSMutableData dataWithLength:numElements * sizeof(float)];
float scale = 1.0 / (float)INT16_MAX ;
vDSP_vflt16((SInt16 *)audioData, 1, data.mutableBytes, 1, numElements);
vDSP_vsmul(data.mutableBytes, 1, &scale, data.mutableBytes, 1, numElements);
KxAudioFrame *frame = [[KxAudioFrame alloc] init];
frame.position = av_frame_get_best_effort_timestamp(_audioFrame) * _audioTimeBase;
frame.duration = av_frame_get_pkt_duration(_audioFrame) * _audioTimeBase;
frame.samples = data;
if (frame.duration == 0) {
// sometimes ffmpeg can't determine the duration of audio frame
// especially of wma/wmv format
// so in this case must compute duration
frame.duration = frame.samples.length / (sizeof(float) * numChannels * audioManager.samplingRate);
}
#if 0
LoggerAudio(2, @"AFD: %.4f %.4f | %.4f ",
frame.position,
frame.duration,
frame.samples.length / (8.0 * 44100.0));
#endif
return frame;
}
- (NSArray *) decodeFrames: (CGFloat) minDuration 函數(shù)就是對(duì)AVPacket 數(shù)據(jù)進(jìn)行解碼,保存到各自的音頻和視頻解碼數(shù)組中
