上一篇直接使用YUV數(shù)據(jù)播放視頻，但是YUV轉(zhuǎn)換成可視化的圖片是在CPU上完成的，這一篇要把這些工作通過MetalKit放到GPU上進(jìn)行渲染。
//PlayESView.h 導(dǎo)入MetalKit庫創(chuàng)建MTKView子類PlayESView
#import <Cocoa/Cocoa.h>
#import <MetalKit/MetalKit.h>
@interface PlayESView : MTKView
- (void)renderWithPixelBuffer:(CVPixelBufferRef)buffer; //傳入存儲(chǔ)YUV數(shù)據(jù)的pixelbuffer進(jìn)行渲染
@end
//PlayESView.m
@implementation PlayESView {
id<MTLComputePipelineState> _pipelineState;
id<MTLCommandQueue> _commandQueue;
CVMetalTextureCacheRef _textCache;
}
- (instancetype)initWithCoder:(NSCoder *)coder {
self = [super initWithCoder:coder];
if (self) {
id<MTLDevice> device = MTLCreateSystemDefaultDevice();
_commandQueue = [device newCommandQueue];
id<MTLLibrary> library = [device newDefaultLibrary];
/*這里使用的是計(jì)算管線，需要在metal文件中定義kernel關(guān)鍵字的yuvToRGB方法
還可以使用渲染管線[device newRenderPipelineStateWithDescriptor:error:]
需要?jiǎng)?chuàng)建MTLRenderPipelineDescriptor然后對(duì)vertexFunction和fragmentFunction進(jìn)行賦值，分別對(duì)應(yīng)的metal文件中vertex關(guān)鍵字的頂點(diǎn)著色器和fragment關(guān)鍵字的片段著色器
*/
id<MTLFunction> function = [library newFunctionWithName:@"yuvToRGB"];
NSError * error = NULL;
_pipelineState = [device newComputePipelineStateWithFunction:function error:&error];
CVReturn ret = CVMetalTextureCacheCreate(kCFAllocatorDefault, nil, device, nil, &_textCache);
if (ret != kCVReturnSuccess) {
NSLog(@"Unable to allocate texture cache");
return nil;
}
self.device = device;
self.framebufferOnly = NO;
self.autoResizeDrawable = NO;
}
return self;
}
- (void)renderWithPixelBuffer:(CVPixelBufferRef)buffer {
if (buffer == nil) return;
CVMetalTextureRef y_texture ;
//獲取pixelbuffer中y數(shù)據(jù)的寬和高，然后創(chuàng)建包含y數(shù)據(jù)的MetalTexture，注意pixelformat為MTLPixelFormatR8Unorm
size_t y_width = CVPixelBufferGetWidthOfPlane(buffer, 0);
size_t y_height = CVPixelBufferGetHeightOfPlane(buffer, 0);
CVReturn ret = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _textCache, buffer, nil, MTLPixelFormatR8Unorm, y_width, y_height, 0, &y_texture);
if (ret != kCVReturnSuccess) {
NSLog(@"fail to create texture");
}
id<MTLTexture> y_inputTexture = CVMetalTextureGetTexture(y_texture);
if (y_inputTexture == nil) {
NSLog(@"failed to create metal texture");
}
CVMetalTextureRef uv_texture ;
//獲取pixelbuffer中uv數(shù)據(jù)的寬和高，然后創(chuàng)建包含uv數(shù)據(jù)的MetalTexture，注意pixelformat為MTLPixelFormatRG8Unorm
size_t uv_width = CVPixelBufferGetWidthOfPlane(buffer, 1);
size_t uv_height = CVPixelBufferGetHeightOfPlane(buffer, 1);
ret = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, _textCache, buffer, nil, MTLPixelFormatRG8Unorm, uv_width, uv_height, 1, &uv_texture);
if (ret != kCVReturnSuccess) {
NSLog(@"fail to create texture");
}
id<MTLTexture> uv_inputTexture = CVMetalTextureGetTexture(uv_texture);
if (uv_inputTexture == nil) {
NSLog(@"failed to create metal texture");
}
CAMetalLayer * metalLayer = (CAMetalLayer *)self.layer;
id<CAMetalDrawable> drawable = metalLayer.nextDrawable;
id<MTLCommandBuffer> commandBuffer = [_commandQueue commandBuffer];
id<MTLComputeCommandEncoder> computeCommandEncoder = [commandBuffer computeCommandEncoder];
[computeCommandEncoder setComputePipelineState:_pipelineState];
//把包含y數(shù)據(jù)的texture、uv數(shù)據(jù)的texture和承載渲染圖像的texture傳入
[computeCommandEncoder setTexture:y_inputTexture atIndex:0];
[computeCommandEncoder setTexture:uv_inputTexture atIndex:1];
[computeCommandEncoder setTexture:drawable.texture atIndex:2];
MTLSize threadgroupSize = MTLSizeMake(16, 16, 1);
MTLSize threadgroupCount = MTLSizeMake((y_width + threadgroupSize.width - 1) / threadgroupSize.width, (y_width + threadgroupSize.height - 1) / threadgroupSize.height, 1);
[computeCommandEncoder dispatchThreadgroups:threadgroupCount threadsPerThreadgroup: threadgroupSize];
[computeCommandEncoder endEncoding];
[commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> _Nonnull cmdBuffer) {
CVBufferRelease(y_texture); //銷毀texture防止內(nèi)存泄露
CVBufferRelease(uv_texture);
}];
[commandBuffer presentDrawable:drawable];
[commandBuffer commit];
}
//Metal.metal
kernel void yuvToRGB(texture2d<float, access::read> y_inTexture [[ texture(0) ]],
texture2d<float, access::read> uv_inTexture [[ texture(1) ]],
texture2d<float, access::write> outTexture [[ texture(2) ]],
uint2 gid [[ thread_position_in_grid ]]) {
float4 yFloat4 = y_inTexture.read(gid);
float4 uvFloat4 = uv_inTexture.read(gid/2); //這里使用yuv420格式進(jìn)行像素計(jì)算
float y = yFloat4.x;
float u = uvFloat4.x - 0.5;
float v = uvFloat4.y - 0.5;
float r = y + 1.403 * v;
r = (r < 0.0) ? 0.0 : ((r > 1.0) ? 1.0 : r);
r = 1 - r;
float g = y - 0.343 * u - 0.714 * v;
g = (g < 0.0) ? 0.0 : ((g > 1.0) ? 1.0 : g);
g = 1 - g;
float b = y + 1.770 * u;
b = (b < 0.0) ? 0.0 : ((b > 1.0) ? 1.0 : b);
b = 1 - b;
outTexture.write(float4(r, g, b, 1.0), gid);
}
修改后的解碼代碼，將ViewController的View設(shè)置為PlayESView
- (void)decodeVideo {
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ //在全局隊(duì)列中解碼
AVPacket * packet = av_packet_alloc();
if (av_read_frame(self->pFormatCtx, packet) >= 0) {
if (packet->stream_index == self->videoIndex) { //解碼視頻流
//FFmpeg 3.0之后avcodec_send_packet和avcodec_receive_frame成對(duì)出現(xiàn)用于解碼，包括音頻和視頻的解碼，avcodec_decode_video2和avcodec_decode_audio4被廢棄
NSInteger ret = avcodec_send_packet(self->pCodecCtx, packet);
if (ret < 0) {
NSLog(@"send packet error");
av_packet_free(&packet);
return;
}
AVFrame * frame = av_frame_alloc();
ret = avcodec_receive_frame(self->pCodecCtx, frame);
if (ret < 0) {
NSLog(@"receive frame error");
av_frame_free(&frame);
return;
}
//frame中data存放解碼出的yuv數(shù)據(jù)，data[0]中是y數(shù)據(jù)，data[1]中是u數(shù)據(jù)，data[2]中是v數(shù)據(jù)，linesize對(duì)應(yīng)的數(shù)據(jù)長(zhǎng)度
float time = packet->pts * av_q2d(self->pFormatCtx->streams[self->videoIndex]->time_base); //計(jì)算當(dāng)前幀時(shí)間
av_packet_free(&packet);
CVReturn theError;
if (!self->pixelBufferPool){ //創(chuàng)建pixelBuffer緩存池，從緩存池中創(chuàng)建pixelBuffer以便復(fù)用
NSMutableDictionary* attributes = [NSMutableDictionary dictionary];
[attributes setObject:[NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange] forKey:(NSString*)kCVPixelBufferPixelFormatTypeKey];
[attributes setObject:[NSNumber numberWithInt:frame->width] forKey: (NSString*)kCVPixelBufferWidthKey];
[attributes setObject:[NSNumber numberWithInt:frame->height] forKey: (NSString*)kCVPixelBufferHeightKey];
[attributes setObject:@(frame->linesize[0]) forKey:(NSString*)kCVPixelBufferBytesPerRowAlignmentKey];
[attributes setObject:[NSDictionary dictionary] forKey:(NSString*)kCVPixelBufferIOSurfacePropertiesKey];
theError = CVPixelBufferPoolCreate(kCFAllocatorDefault, NULL, (__bridge CFDictionaryRef) attributes, &self->pixelBufferPool);
if (theError != kCVReturnSuccess){
NSLog(@"CVPixelBufferPoolCreate Failed");
}
}
CVPixelBufferRef pixelBuffer = nil;
theError = CVPixelBufferPoolCreatePixelBuffer(NULL, self->pixelBufferPool, &pixelBuffer);
if(theError != kCVReturnSuccess){
NSLog(@"CVPixelBufferPoolCreatePixelBuffer Failed");
}
theError = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
if (theError != kCVReturnSuccess) {
NSLog(@"lock error");
}
/*
PixelBuffer中Y數(shù)據(jù)存放在Plane0中，UV數(shù)據(jù)存放在Plane1中，數(shù)據(jù)格式如下
frame->data[0] ......... YYYYYYYYY
frame->data[1] ......... UUUUUUUU
frame->data[2] ......... VVVVVVVVV
PixelBuffer->Plane0 ....... YYYYYYYY
PixelBuffer->Plane1 ....... UVUVUVUVUV
所以需要把Y數(shù)據(jù)拷貝到Plane0上，把U和V數(shù)據(jù)交叉拷到Plane1上
*/
size_t bytePerRowY = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 0);
size_t bytesPerRowUV = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, 1);
//獲取Plane0的起始地址
void* base = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
memcpy(base, frame->data[0], bytePerRowY * frame->height);
//獲取Plane1的起始地址
base = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
uint32_t size = frame->linesize[1] * frame->height / 2;
//把UV數(shù)據(jù)交叉存儲(chǔ)到dstData然后拷貝到Plane1上
uint8_t* dstData = new uint8_t[2 * size];
uint8_t * firstData = new uint8_t[size];
memcpy(firstData, frame->data[1], size);
uint8_t * secondData = new uint8_t[size];
memcpy(secondData, frame->data[2], size);
for (int i = 0; i < 2 * size; i++){
if (i % 2 == 0){
dstData[i] = firstData[i/2];
}else {
dstData[i] = secondData[i/2];
}
}
memcpy(base, dstData, bytesPerRowUV * frame->height/2);
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
av_frame_free(&frame);
free(dstData);
free(firstData);
free(secondData);
// CIImage *coreImage = [CIImage imageWithCVPixelBuffer:pixelBuffer];
// CGImageRef videoImage = [self->context createCGImage:coreImage
// fromRect:CGRectMake(0, 0, self->pCodecCtx->width, self->pCodecCtx->height)];
// NSImage * image = [[NSImage alloc] initWithCGImage:videoImage size:NSSizeFromCGSize(CGSizeMake(self->pCodecCtx->width, self->pCodecCtx->height))];
// CVPixelBufferRelease(pixelBuffer);
// CGImageRelease(videoImage);
dispatch_async(dispatch_get_main_queue(), ^{
self.label.stringValue = [NSString stringWithFormat:@"%.2d:%.2d", (int)time/60, (int)time%60];
// self.imageView.image = image;
PlayESView * esView = (PlayESView *)self.view;
[esView renderWithPixelBuffer:pixelBuffer];
self.slider.floatValue = time / (float)self->videoDuration;
});
}
} else {
avcodec_free_context(&self->pCodecCtx);
avformat_close_input(&self->pFormatCtx);
avformat_free_context(self->pFormatCtx);
[self->timer invalidate];
}
});
}
使用CoreImage CPU利用率.png
使用MetalKit CPU利用率.png
