這篇文章處理的是編碼后的視頻圖像數(shù)據(jù)CVPixelBufferRef，我們使用CAEAGLLayer來作為圖像的渲染圖層。
該功能使用到的是OpenGL ES的相關(guān)技術(shù)，如果對(duì)該技術(shù)不了解的伙伴，請(qǐng)先了解相關(guān)技術(shù)。
一、著色器
1.1頂點(diǎn)著色器：
const GLchar *shader_vsh = (const GLchar*)"attribute vec4 position;"
"attribute vec2 texCoord;"
"uniform float preferredRotation;"
"varying vec2 texCoordVarying;"
"void main()"
"{"
" mat4 rotationMatrix = mat4(cos(preferredRotation), -sin(preferredRotation), 0.0, 0.0,"
" sin(preferredRotation), cos(preferredRotation), 0.0, 0.0,"
" 0.0, 0.0, 1.0, 0.0,"
" 0.0, 0.0, 0.0, 1.0);"
" gl_Position = position * rotationMatrix;"
" texCoordVarying = texCoord;"
"}";
-
"attribute vec4 position：頂點(diǎn)坐標(biāo)
attribute vec2 texCoord：紋理坐標(biāo)
uniform float preferredRotation：旋轉(zhuǎn)角度
attribute vec2 texCoordVarying：用于把texCoord（紋理坐標(biāo)）傳遞到片元著器。
mat4 rotationMatrix4x4 的旋轉(zhuǎn)矩陣
gl_Position：內(nèi)建變量，用來接收頂點(diǎn)坐標(biāo)。
1.1片元著色器：
const GLchar *shader_fsh = (const GLchar*)"varying highp vec2 texCoordVarying;"
"precision mediump float;"
"uniform sampler2D SamplerY;"
"uniform sampler2D SamplerUV;"
"uniform mat3 colorConversionMatrix;"
"void main()"
"{"
" mediump vec3 yuv;"
" lowp vec3 rgb;"
// Subtract constants to map the video range start at 0
" yuv.x = (texture2D(SamplerY, texCoordVarying).r - (16.0/255.0));"
" yuv.yz = (texture2D(SamplerUV, texCoordVarying).rg - vec2(0.5, 0.5));"
" rgb = colorConversionMatrix * yuv;"
" gl_FragColor = vec4(rgb, 1);"
"}";
-
varying highp vec2 texCoordVarying：從頂點(diǎn)著色器傳遞過來的紋理坐標(biāo)。
precision mediump float;：精度設(shè)置，中間精度float。
uniform sampler2D SamplerY;：Y亮度紋理采樣器。
uniform sampler2D SamplerUV;：UV色度紋理采樣器。
uniform mat3 colorConversionMatrix;：顏色轉(zhuǎn)換矩陣。
gl_FragColor：內(nèi)建函數(shù)，輸出的顏色用于隨后的像素操作。
二、初始化
2.1繼承之CAEAGLLayer自定義類CQEAGLLayer。
CGSize size = UIScreen.mainScreen.bounds.size;
_displayLayer = [[CQEAGLLayer alloc] initWithFrame:CGRectMake(size.width*0.5, 100, size.width*0.5, size.height*0.5)];
[self.view.layer addSublayer:_displayLayer];
2.2初始化代碼：
- (instancetype)initWithFrame:(CGRect)frame {
self = [super init];
if (self) {
CGFloat scale = [[UIScreen mainScreen] scale];
self.contentsScale = scale;
//一個(gè)布爾值，指示層是否包含完全不透明的內(nèi)容.默認(rèn)為NO
self.opaque = TRUE;
//kEAGLDrawablePropertyRetainedBacking指定可繪制表面在顯示后是否保留其內(nèi)容的鍵.默認(rèn)為NO.
self.drawableProperties = @{ kEAGLDrawablePropertyRetainedBacking :[NSNumber numberWithBool:YES]};
[self setFrame:frame];
// 1、設(shè)置繪制框架的上下文.
_context = [[EAGLContext alloc] initWithAPI:kEAGLRenderingAPIOpenGLES2];
if (!_context || ![EAGLContext setCurrentContext:_context]) {
return nil;
}
// 將默認(rèn)轉(zhuǎn)換設(shè)置為BT.709，這是HDTV的標(biāo)準(zhǔn)
_preferredConversion = kColorConversion709;
//2、設(shè)置緩沖區(qū)
[self setupBuffers];
//3、加載shaders 著色器
[self loadShaders];
//4、使用著色程序
glUseProgram(self.program);
// 0 和 1 分別代表_lumaTexture 和 _chromaTexture的紋理 ID。
glUniform1i(uniforms[UNIFORM_Y], 0);
glUniform1i(uniforms[UNIFORM_UV], 1);
glUniform1f(uniforms[UNIFORM_ROTATION_ANGLE], 0);
glUniformMatrix3fv(uniforms[UNIFORM_COLOR_CONVERSION_MATRIX], 1, GL_FALSE, _preferredConversion);
}
return self;
}
- 1、創(chuàng)建上下文。
- 2、設(shè)置緩沖區(qū)。
- 3、使用
program
2.3 設(shè)置緩沖區(qū)
- (void)setupBuffers {
glDisable(GL_DEPTH_TEST);
//默認(rèn)頂點(diǎn)屬性(position)是關(guān)閉的,所以使用前要手動(dòng)打開
glEnableVertexAttribArray(ATTRIB_VERTEX);
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(GLfloat), 0);
//打開紋理數(shù)據(jù)讀取通道
glEnableVertexAttribArray(ATTRIB_TEXCOORD);
glVertexAttribPointer(ATTRIB_TEXCOORD, 2, GL_FLOAT, GL_FALSE, 2 * sizeof(GLfloat), 0);
//創(chuàng)建buffer
[self createBuffers];
}
- (void) createBuffers {
//創(chuàng)建幀緩存區(qū) frameBuffer
glGenFramebuffers(1, &_frameBufferHandle);
glBindFramebuffer(GL_FRAMEBUFFER, _frameBufferHandle);
//創(chuàng)建顏色緩存區(qū) RenderBuffer
glGenRenderbuffers(1, &_colorBufferHandle);
glBindRenderbuffer(GL_RENDERBUFFER, _colorBufferHandle);
//綁定渲染緩存區(qū)
[_context renderbufferStorage:GL_RENDERBUFFER fromDrawable:self];
//設(shè)置渲染緩存區(qū)的尺寸
glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_WIDTH, &_backingWidth);
glGetRenderbufferParameteriv(GL_RENDERBUFFER, GL_RENDERBUFFER_HEIGHT, &_backingHeight);
//綁定renderBuffer到FrameBuffer
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, _colorBufferHandle);
//檢查FrameBuffer狀態(tài)
if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) {
NSLog(@"Failed to make complete framebuffer object %x", glCheckFramebufferStatus(GL_FRAMEBUFFER));
}
}
-
glVertexAttribPointer頂點(diǎn)數(shù)據(jù)解析方式:
參數(shù)1: 指定從索引0開始取數(shù)據(jù)，與頂點(diǎn)著色器對(duì)應(yīng)
參數(shù)2: 頂點(diǎn)屬性大小
參數(shù)3: 數(shù)據(jù)類型
參數(shù)4: 歸一化
參數(shù)5: 步長（Stride)
參數(shù)6: 數(shù)據(jù)在緩沖區(qū)起始位置的偏移量
2.4加載著色器
- (BOOL)loadShaders {
GLuint vertShader = 0, fragShader = 0;
// 創(chuàng)建著色program.
self.program = glCreateProgram();
//編譯頂點(diǎn)著色器
if(![self compileShaderString:&vertShader type:GL_VERTEX_SHADER shaderString:shader_vsh]) {
NSLog(@"Failed to compile vertex shader");
return NO;
}
//編譯片元著色器
if(![self compileShaderString:&fragShader type:GL_FRAGMENT_SHADER shaderString:shader_fsh]) {
NSLog(@"Failed to compile fragment shader");
return NO;
}
// 附著頂點(diǎn)著色器到program.
glAttachShader(self.program, vertShader);
// 附著片元著色器到program.
glAttachShader(self.program, fragShader);
// 綁定屬性位置。這需要在鏈接之前完成.(讓ATTRIB_VERTEX/ATTRIB_TEXCOORD 與position/texCoord產(chǎn)生連接)
glBindAttribLocation(self.program, ATTRIB_VERTEX, "position");
glBindAttribLocation(self.program, ATTRIB_TEXCOORD, "texCoord");
// Link the program.
if (![self linkProgram:self.program]) {
NSLog(@"Failed to link program: %d", self.program);
if (vertShader) {
glDeleteShader(vertShader);
vertShader = 0;
}
if (fragShader) {
glDeleteShader(fragShader);
fragShader = 0;
}
if (self.program) {
glDeleteProgram(self.program);
self.program = 0;
}
return NO;
}
//獲取uniform的位置
//Y亮度紋理
uniforms[UNIFORM_Y] = glGetUniformLocation(self.program, "SamplerY");
//UV色量紋理
uniforms[UNIFORM_UV] = glGetUniformLocation(self.program, "SamplerUV");
//旋轉(zhuǎn)角度preferredRotation
uniforms[UNIFORM_ROTATION_ANGLE] = glGetUniformLocation(self.program, "preferredRotation");
//YUV->RGB
uniforms[UNIFORM_COLOR_CONVERSION_MATRIX] = glGetUniformLocation(self.program, "colorConversionMatrix");
// Release vertex and fragment shaders.
if (vertShader) {
glDetachShader(self.program, vertShader);
glDeleteShader(vertShader);
}
if (fragShader) {
glDetachShader(self.program, fragShader);
glDeleteShader(fragShader);
}
return YES;
}
2.5編譯
- (BOOL)compileShaderString:(GLuint *)shader type:(GLenum)type shaderString:(const GLchar*)shaderString
{
*shader = glCreateShader(type);
glShaderSource(*shader, 1, &shaderString, NULL);
glCompileShader(*shader);
#if defined(DEBUG)
GLint logLength;
glGetShaderiv(*shader, GL_INFO_LOG_LENGTH, &logLength);
if (logLength > 0) {
GLchar *log = (GLchar *)malloc(logLength);
glGetShaderInfoLog(*shader, logLength, &logLength, log);
NSLog(@"Shader compile log:\n%s", log);
free(log);
}
#endif
GLint status = 0;
glGetShaderiv(*shader, GL_COMPILE_STATUS, &status);
if (status == 0) {
glDeleteShader(*shader);
return NO;
}
return YES;
}
2.5鏈接
- (BOOL)linkProgram:(GLuint)prog {
GLint status;
glLinkProgram(prog);
#if defined(DEBUG)
GLint logLength;
glGetProgramiv(prog, GL_INFO_LOG_LENGTH, &logLength);
if (logLength > 0) {
GLchar *log = (GLchar *)malloc(logLength);
glGetProgramInfoLog(prog, logLength, &logLength, log);
NSLog(@"Program link log:\n%s", log);
free(log);
}
#endif
glGetProgramiv(prog, GL_LINK_STATUS, &status);
if (status == 0) {
return NO;
}
return YES;
}
三、繪制
3.1設(shè)置CVPixelBufferRef
- (void)setPixelBuffer:(CVPixelBufferRef)pb {
if(_pixelBuffer) {
CVPixelBufferRelease(_pixelBuffer);
}
_pixelBuffer = CVPixelBufferRetain(pb);
//獲取視頻幀的寬與高
int frameWidth = (int)CVPixelBufferGetWidth(_pixelBuffer);
int frameHeight = (int)CVPixelBufferGetHeight(_pixelBuffer);
//顯示_pixelBuffer
[self display:_pixelBuffer width:frameWidth height:frameHeight];
}
- 在iOS里，我們經(jīng)常能看到
CVPixelBufferRef這個(gè)類型，在Camera 采集返回的數(shù)據(jù)里得到一個(gè)CMSampleBufferRef，而每個(gè)CMSampleBufferRef里則包含一個(gè)CVPixelBufferRef，在視頻硬解碼的返回?cái)?shù)據(jù)里也是一個(gè)CVPixelBufferRef（里面包好了所有的壓縮的圖片信息）。CVPixelBufferRef：是一種像素圖片類型，由于CV開頭，所以它是屬于CoreVideo模塊的。
3.2顯示
- (void)display:(CVPixelBufferRef)pixelBuffer width:(uint32_t)frameWidth height:(uint32_t)frameHeight {
if (!_context || ![EAGLContext setCurrentContext:_context]) {
return;
}
if(pixelBuffer == NULL) {
NSLog(@"PixelBuffer is null");
return;
}
CVReturn errer;
size_t planeCount = CVPixelBufferGetPlaneCount(pixelBuffer);
CFTypeRef colorAttachments = CVBufferGetAttachment(pixelBuffer, kCVImageBufferYCbCrMatrixKey, NULL);
if (CFStringCompare(colorAttachments, kCVImageBufferYCbCrMatrix_ITU_R_601_4, 0) == kCFCompareEqualTo) {
_preferredConversion = kColorConversion601;
}
else {
_preferredConversion = kColorConversion709;
}
CVOpenGLESTextureCacheRef _videoTextureCache;
//創(chuàng)建 CVOpenGLESTextureCacheRef 創(chuàng)建新的紋理緩存
errer = CVOpenGLESTextureCacheCreate(kCFAllocatorDefault, NULL, _context, NULL, &_videoTextureCache);
if (error != noErr) {
NSLog(@"CVOpenGLESTextureCacheCreate error : %d", errer);
return;
}
glActiveTexture(GL_TEXTURE0); //激活紋理
//1.創(chuàng)建Y紋理(亮度紋理)
errer = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_videoTextureCache,
pixelBuffer,
NULL,
GL_TEXTURE_2D,
GL_RED_EXT,
frameWidth,
frameHeight,
GL_RED_EXT,
GL_UNSIGNED_BYTE,
0,
&_lumaTexture);
if (errer) {
NSLog(@"CVOpenGLESTextureCacheCreateTextureFromImage error: %d", errer);
}
//2.配置Y亮度紋理屬性
//綁定紋理.
glBindTexture(CVOpenGLESTextureGetTarget(_lumaTexture), CVOpenGLESTextureGetName(_lumaTexture));
//配置紋理放大/縮小過濾方式以及紋理圍繞S/T環(huán)繞方式
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
//3.UV-plane 紋理
//如果顏色通道個(gè)數(shù)>1,則除了Y還有UV-Plane.
if(planeCount == 2) {
// UV-plane.
glActiveTexture(GL_TEXTURE1); //激活UV-plane紋理
//4.創(chuàng)建UV-plane紋理
errer = CVOpenGLESTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
_videoTextureCache,
pixelBuffer,
NULL,
GL_TEXTURE_2D,
GL_RG_EXT,
frameWidth / 2,
frameHeight / 2,
GL_RG_EXT,
GL_UNSIGNED_BYTE,
1,
&_chromaTexture);
if (error) {
NSLog(@"CVOpenGLESTextureCacheCreateTextureFromImage error: %d", error);
}
//5.綁定紋理
glBindTexture(CVOpenGLESTextureGetTarget(_chromaTexture), CVOpenGLESTextureGetName(_chromaTexture));
//6.配置紋理放大/縮小過濾方式以及紋理圍繞S/T環(huán)繞方式
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
//綁定幀緩存區(qū)
glBindFramebuffer(GL_FRAMEBUFFER, _frameBufferHandle);
//設(shè)置視口.
glViewport(0, 0, _backingWidth, _backingHeight);
//清理屏幕
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
//使用shaderProgram
glUseProgram(self.program);
//傳遞Uniform屬性到shader
//UNIFORM_ROTATION_ANGLE 旋轉(zhuǎn)角度
glUniform1f(uniforms[UNIFORM_ROTATION_ANGLE], 0);
//UNIFORM_COLOR_CONVERSION_MATRIX YUV->RGB顏色矩陣
glUniformMatrix3fv(uniforms[UNIFORM_COLOR_CONVERSION_MATRIX], 1, GL_FALSE, _preferredConversion);
// 根據(jù)視頻的方向和縱橫比設(shè)置四邊形頂點(diǎn)
CGRect viewBounds = self.bounds;
CGSize contentSize = CGSizeMake(frameWidth, frameHeight);
/*
AVMakeRectWithAspectRatioInsideRect
功能: 返回一個(gè)按比例縮放的CGRect，該CGRect保持由邊界CGRect內(nèi)的CGSize指定的縱橫比
參數(shù)1:希望保持的寬高比或縱橫比
參數(shù)2:填充的rect
*/
CGRect vertexSamplingRect = AVMakeRectWithAspectRatioInsideRect(contentSize, viewBounds);
// 計(jì)算標(biāo)準(zhǔn)化的四邊形坐標(biāo)以將幀繪制到其中
//標(biāo)準(zhǔn)化采樣大小
CGSize normalizedSamplingSize = CGSizeMake(0.0, 0.0);
//標(biāo)準(zhǔn)化規(guī)模
CGSize cropScaleAmount = CGSizeMake(vertexSamplingRect.size.width/viewBounds.size.width,vertexSamplingRect.size.height/viewBounds.size.height);
// 規(guī)范化四元頂點(diǎn)
if (cropScaleAmount.width > cropScaleAmount.height) {
normalizedSamplingSize.width = 1.0;
normalizedSamplingSize.height = cropScaleAmount.height/cropScaleAmount.width;
}
else {
normalizedSamplingSize.width = cropScaleAmount.width/cropScaleAmount.height;
normalizedSamplingSize.height = 1.0;;
}
/*
四頂點(diǎn)數(shù)據(jù)定義了我們繪制像素緩沖區(qū)的二維平面區(qū)域。
使用（-1，-1）和（1,1）分別作為左下角和右上角坐標(biāo)形成的頂點(diǎn)數(shù)據(jù)覆蓋整個(gè)屏幕。
*/
GLfloat quadVertexData [] = {
-1 * normalizedSamplingSize.width, -1 * normalizedSamplingSize.height,
normalizedSamplingSize.width, -1 * normalizedSamplingSize.height,
-1 * normalizedSamplingSize.width, normalizedSamplingSize.height,
normalizedSamplingSize.width, normalizedSamplingSize.height,
};
// 更新屬性值.
//坐標(biāo)數(shù)據(jù)
glVertexAttribPointer(ATTRIB_VERTEX, 2, GL_FLOAT, 0, 0, quadVertexData);
glEnableVertexAttribArray(ATTRIB_VERTEX);
/*
紋理頂點(diǎn)的設(shè)置使我們垂直翻轉(zhuǎn)紋理。這使得我們的左上角原點(diǎn)緩沖區(qū)匹配OpenGL的左下角紋理坐標(biāo)系
*/
CGRect textureSamplingRect = CGRectMake(0, 0, 1, 1);
GLfloat quadTextureData[] = {
CGRectGetMinX(textureSamplingRect), CGRectGetMaxY(textureSamplingRect),
CGRectGetMaxX(textureSamplingRect), CGRectGetMaxY(textureSamplingRect),
CGRectGetMinX(textureSamplingRect), CGRectGetMinY(textureSamplingRect),
CGRectGetMaxX(textureSamplingRect), CGRectGetMinY(textureSamplingRect)
};
//更新紋理坐標(biāo)屬性值
glVertexAttribPointer(ATTRIB_TEXCOORD, 2, GL_FLOAT, 0, 0, quadTextureData);
glEnableVertexAttribArray(ATTRIB_TEXCOORD);
//繪制圖形
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
//綁定渲染緩存區(qū)->顯示到屏幕
glBindRenderbuffer(GL_RENDERBUFFER, _colorBufferHandle);
[_context presentRenderbuffer:GL_RENDERBUFFER];
//清理紋理,方便下一幀紋理顯示
[self cleanUpTextures];
// 定期紋理緩存刷新每幀
CVOpenGLESTextureCacheFlush(_videoTextureCache, 0);
if(_videoTextureCache) {
CFRelease(_videoTextureCache);
}
}
-
CVBufferGetAttachment使用像素緩沖區(qū)的顏色附件確定適當(dāng)?shù)念伾D(zhuǎn)換矩陣:
參數(shù)1: 像素緩存區(qū)
參數(shù)2: kCVImageBufferYCbCrMatrixKey YCbCr->RGB
參數(shù)3: 附件模式,NULL -
CVOpenGLESTextureCacheCreate創(chuàng)建新的紋理緩存：
參數(shù)1: kCFAllocatorDefault默認(rèn)內(nèi)存分配器.
參數(shù)2: NULL
參數(shù)3: EAGLContext 圖形上下文
參數(shù)4: NULL
參數(shù)5: 新創(chuàng)建的紋理緩存 -
CVOpenGLESTextureCacheCreateTextureFromImage創(chuàng)建亮度/色度紋理-Y/UV紋理:
參數(shù)1: 內(nèi)存分配器,kCFAllocatorDefault
參數(shù)2: 紋理緩存.紋理緩存將管理紋理的紋理緩存對(duì)象
參數(shù)3: sourceImage.
參數(shù)4: 紋理屬性.默認(rèn)給NULL
參數(shù)5: 目標(biāo)紋理,GL_TEXTURE_2D
參數(shù)6: 指定紋理中顏色組件的數(shù)量(GL_RGBA, GL_LUMINANCE, GL_RGBA8_OES, GL_RG, and GL_RED (NOTE: 在 GLES3 使用 GL_R8 替代 GL_RED).)
參數(shù)7: 幀寬度
參數(shù)8: 幀高度
參數(shù)9: 格式指定像素?cái)?shù)據(jù)的格式
參數(shù)10: 指定像素?cái)?shù)據(jù)的數(shù)據(jù)類型,GL_UNSIGNED_BYTE
參數(shù)11: planeIndex
參數(shù)12: 紋理輸出新創(chuàng)建的紋理對(duì)象將放置在此處。
