diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 62f0eb7c3b..9ca6ab796b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -184,7 +184,12 @@ endif() include("${SDL3_SOURCE_DIR}/cmake/FindFFmpeg.cmake") if(FFmpeg_FOUND AND FFmpeg_AVCODEC_VERSION VERSION_GREATER_EQUAL "60") - add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c ${icon_bmp_header}) + if(APPLE) + add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c testffmpeg_videotoolbox.m ${icon_bmp_header}) + target_link_options(testffmpeg PRIVATE "-Wl,-framework,CoreFoundation" "-Wl,-framework,CoreVideo" "-Wl,-framework,Metal") + else() + add_sdl_test_executable(testffmpeg NO_C90 SOURCES testffmpeg.c ${icon_bmp_header}) + endif() target_link_libraries(testffmpeg PRIVATE ${FFMPEG_LIBRARIES}) if(HAVE_OPENGLES_V2) #message(STATUS "Enabling EGL support in testffmpeg") diff --git a/test/testffmpeg.c b/test/testffmpeg.c index 301a819fbb..5e2dbfc1d1 100644 --- a/test/testffmpeg.c +++ b/test/testffmpeg.c @@ -36,6 +36,10 @@ #endif #endif +#ifdef __APPLE__ +#include "testffmpeg_videotoolbox.h" +#endif + #include #include #include @@ -64,6 +68,9 @@ static SDL_bool has_EGL_EXT_image_dma_buf_import; static PFNGLACTIVETEXTUREARBPROC glActiveTextureARBFunc; static PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOESFunc; #endif +#ifdef __APPLE__ +static SDL_bool has_videotoolbox_output; +#endif static int done; static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL) @@ -87,7 +94,7 @@ static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL) SDL_GL_SetAttribute(SDL_GL_GREEN_SIZE, 6); SDL_GL_SetAttribute(SDL_GL_BLUE_SIZE, 5); - if (SDL_CreateWindowAndRenderer(WINDOW_WIDTH, WINDOW_HEIGHT, SDL_WINDOW_HIDDEN, &window, &renderer) < 0) { + if (SDL_CreateWindowAndRenderer(WINDOW_WIDTH, WINDOW_HEIGHT, window_flags, &window, &renderer) < 0) { return SDL_FALSE; } @@ -116,6 +123,10 @@ static SDL_bool CreateWindow(Uint32 window_flags, SDL_bool useEGL) } #endif /* HAVE_EGL */ +#ifdef __APPLE__ + has_videotoolbox_output = SetupVideoToolboxOutput(renderer); +#endif + return SDL_TRUE; } @@ -222,6 +233,11 @@ static SDL_bool SupportedPixelFormat(enum AVPixelFormat format) (format == AV_PIX_FMT_VAAPI || format == AV_PIX_FMT_DRM_PRIME)) { return SDL_TRUE; } +#ifdef __APPLE__ + if (has_videotoolbox_output && format == AV_PIX_FMT_VIDEOTOOLBOX) { + return SDL_TRUE; + } +#endif if (GetTextureFormat(format) != SDL_PIXELFORMAT_UNKNOWN) { return SDL_TRUE; @@ -474,11 +490,41 @@ static SDL_bool GetTextureForFrame(AVFrame *frame, SDL_Texture **texture) } } -static void HandleVideoFrame(AVFrame *frame, double pts) +static void DisplayVideoTexture(AVFrame *frame) { /* Update the video texture */ GetTextureForFrame(frame, &video_texture); + if (frame->linesize[0] < 0) { + SDL_RenderTextureRotated(renderer, video_texture, NULL, NULL, 0.0, NULL, SDL_FLIP_VERTICAL); + } else { + SDL_RenderTexture(renderer, video_texture, NULL, NULL); + } +} + +static void DisplayVideoToolbox(AVFrame *frame) +{ +#ifdef __APPLE__ + SDL_Rect viewport; + SDL_GetRenderViewport(renderer, &viewport); + DisplayVideoToolboxFrame(renderer, frame->data[3], 0, 0, frame->width, frame->height, viewport.x, viewport.y, viewport.w, viewport.h); +#endif +} + +static void DisplayVideoFrame(AVFrame *frame) +{ + switch (frame->format) { + case AV_PIX_FMT_VIDEOTOOLBOX: + DisplayVideoToolbox(frame); + break; + default: + DisplayVideoTexture(frame); + break; + } +} + +static void HandleVideoFrame(AVFrame *frame, double pts) +{ /* Quick and dirty PTS handling */ if (!video_start) { video_start = SDL_GetTicks(); @@ -489,11 +535,7 @@ static void HandleVideoFrame(AVFrame *frame, double pts) now = (double)(SDL_GetTicks() - video_start) / 1000.0; } - if (frame->linesize[0] < 0) { - SDL_RenderTextureRotated(renderer, video_texture, NULL, NULL, 0.0, NULL, SDL_FLIP_VERTICAL); - } else { - SDL_RenderTexture(renderer, video_texture, NULL, NULL); - } + DisplayVideoFrame(frame); /* Render any bouncing balls */ MoveSprite(); @@ -658,7 +700,7 @@ int main(int argc, char *argv[]) goto quit; } - window_flags = SDL_WINDOW_HIDDEN; + window_flags = SDL_WINDOW_HIDDEN | SDL_WINDOW_HIGH_PIXEL_DENSITY; #ifdef __APPLE__ window_flags |= SDL_WINDOW_METAL; #elif !defined(__WIN32__) @@ -832,6 +874,9 @@ int main(int argc, char *argv[]) } return_code = 0; quit: +#ifdef __APPLE__ + CleanupVideoToolboxOutput(); +#endif SDL_free(positions); SDL_free(velocities); av_frame_free(&frame); diff --git a/test/testffmpeg_videotoolbox.h b/test/testffmpeg_videotoolbox.h new file mode 100644 index 0000000000..21242955cb --- /dev/null +++ b/test/testffmpeg_videotoolbox.h @@ -0,0 +1,15 @@ +/* + Copyright (C) 1997-2023 Sam Lantinga + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely. +*/ + +extern SDL_bool SetupVideoToolboxOutput(SDL_Renderer *renderer); +extern SDL_bool DisplayVideoToolboxFrame(SDL_Renderer *renderer, void *buffer, int srcX, int srcY, int srcW, int srcH, int dstX, int dstY, int dstW, int dstH ); +extern void CleanupVideoToolboxOutput(); diff --git a/test/testffmpeg_videotoolbox.m b/test/testffmpeg_videotoolbox.m new file mode 100644 index 0000000000..e7d580cae0 --- /dev/null +++ b/test/testffmpeg_videotoolbox.m @@ -0,0 +1,147 @@ +/* + Copyright (C) 1997-2023 Sam Lantinga + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely. +*/ +#include + +#include "testffmpeg_videotoolbox.h" + +#include +#include +#include +#include + + +// Metal BT.601 to RGB conversion shader +static NSString *drawMetalShaderSource = +@" using namespace metal;\n" +"\n" +" struct Vertex\n" +" {\n" +" float4 position [[position]];\n" +" float2 texCoords;\n" +" };\n" +"\n" +" constexpr sampler s(coord::normalized, address::clamp_to_edge, filter::linear);\n" +"\n" +" vertex Vertex draw_vs(constant Vertex *vertices [[ buffer(0) ]], uint vid [[ vertex_id ]])\n" +" {\n" +" return vertices[ vid ];\n" +" }\n" +"\n" +" fragment float4 draw_ps_bt601(Vertex in [[ stage_in ]],\n" +" texture2d textureY [[ texture(0) ]],\n" +" texture2d textureUV [[ texture(1) ]])\n" +" {\n" +" float3 yuv = float3(textureY.sample(s, in.texCoords).r, textureUV.sample(s, in.texCoords).rg);\n" +" float3 rgb;\n" +" yuv += float3(-0.0627451017, -0.501960814, -0.501960814);\n" +" rgb.r = dot(yuv, float3(1.1644, 0.000, 1.596));\n" +" rgb.g = dot(yuv, float3(1.1644, -0.3918, -0.813));\n" +" rgb.b = dot(yuv, float3(1.1644, 2.0172, 0.000));\n" +" return float4(rgb, 1.0);\n" +" }\n" +; + +// keep this structure aligned with the proceeding drawMetalShaderSource's struct Vertex +typedef struct Vertex +{ + vector_float4 position; + vector_float2 texCoord; +} Vertex; + +static void SetVertex(Vertex *vertex, float x, float y, float s, float t) +{ + vertex->position[ 0 ] = x; + vertex->position[ 1 ] = y; + vertex->position[ 2 ] = 0.0f; + vertex->position[ 3 ] = 1.0f; + vertex->texCoord[ 0 ] = s; + vertex->texCoord[ 1 ] = t; +} + +static CAMetalLayer *metal_layer; +static id library; +static id video_pipeline; + +SDL_bool SetupVideoToolboxOutput(SDL_Renderer *renderer) +{ @autoreleasepool { + NSError *error; + + // Create the metal view + metal_layer = (CAMetalLayer *)SDL_GetRenderMetalLayer(renderer); + if (!metal_layer) { + return SDL_FALSE; + } + + // FIXME: Handle other colorspaces besides BT.601 + library = [metal_layer.device newLibraryWithSource:drawMetalShaderSource options:nil error:&error]; + + MTLRenderPipelineDescriptor *videoPipelineDescriptor = [[MTLRenderPipelineDescriptor new] autorelease]; + videoPipelineDescriptor.vertexFunction = [library newFunctionWithName:@"draw_vs"]; + videoPipelineDescriptor.fragmentFunction = [library newFunctionWithName:@"draw_ps_bt601"]; + videoPipelineDescriptor.colorAttachments[ 0 ].pixelFormat = metal_layer.pixelFormat; + + video_pipeline = [metal_layer.device newRenderPipelineStateWithDescriptor:videoPipelineDescriptor error:nil]; + if (!video_pipeline) { + SDL_SetError("Couldn't create video pipeline"); + return SDL_FALSE; + } + + return true; +}} + +SDL_bool DisplayVideoToolboxFrame(SDL_Renderer *renderer, void *buffer, int srcX, int srcY, int srcW, int srcH, int dstX, int dstY, int dstW, int dstH ) +{ @autoreleasepool { + CVPixelBufferRef pPixelBuffer = (CVPixelBufferRef)buffer; + size_t nPixelBufferWidth = CVPixelBufferGetWidthOfPlane(pPixelBuffer, 0); + size_t nPixelBufferHeight = CVPixelBufferGetHeightOfPlane(pPixelBuffer, 0); + id videoFrameTextureY = nil; + id videoFrameTextureUV = nil; + + IOSurfaceRef pSurface = CVPixelBufferGetIOSurface(pPixelBuffer); + + MTLTextureDescriptor *textureDescriptorY = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatR8Unorm width:nPixelBufferWidth height:nPixelBufferHeight mipmapped:NO]; + MTLTextureDescriptor *textureDescriptorUV = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRG8Unorm width:CVPixelBufferGetWidthOfPlane(pPixelBuffer, 1) height:CVPixelBufferGetHeightOfPlane(pPixelBuffer, 1) mipmapped:NO]; + + videoFrameTextureY = [[metal_layer.device newTextureWithDescriptor:textureDescriptorY iosurface:pSurface plane:0] autorelease]; + videoFrameTextureUV = [[metal_layer.device newTextureWithDescriptor:textureDescriptorUV iosurface:pSurface plane:1] autorelease]; + + float flMinSrcX = ( srcX + 0.5f ) / nPixelBufferWidth; + float flMaxSrcX = ( srcX + srcW + 0.5f ) / nPixelBufferWidth; + float flMinSrcY = ( srcY + 0.5f ) / nPixelBufferHeight; + float flMaxSrcY = ( srcY + srcH + 0.5f ) / nPixelBufferHeight; + + int nOutputWidth, nOutputHeight; + nOutputWidth = metal_layer.drawableSize.width; + nOutputHeight = metal_layer.drawableSize.height; + float flMinDstX = 2.0f * ( ( dstX + 0.5f ) / nOutputWidth ) - 1.0f; + float flMaxDstX = 2.0f * ( ( dstX + dstW + 0.5f ) / nOutputWidth ) - 1.0f; + float flMinDstY = 2.0f * ( ( nOutputHeight - dstY - 0.5f ) / nOutputHeight ) - 1.0f; + float flMaxDstY = 2.0f * ( ( nOutputHeight - ( dstY + dstH ) - 0.5f ) / nOutputHeight ) - 1.0f; + + Vertex arrVerts[4]; + SetVertex(&arrVerts[0], flMinDstX, flMaxDstY, flMinSrcX, flMaxSrcY); + SetVertex(&arrVerts[1], flMinDstX, flMinDstY, flMinSrcX, flMinSrcY); + SetVertex(&arrVerts[2], flMaxDstX, flMaxDstY, flMaxSrcX, flMaxSrcY); + SetVertex(&arrVerts[3], flMaxDstX, flMinDstY, flMaxSrcX, flMinSrcY); + + id renderEncoder = (id)SDL_GetRenderMetalCommandEncoder(renderer); + [renderEncoder setRenderPipelineState:video_pipeline]; + [renderEncoder setFragmentTexture:videoFrameTextureY atIndex:0]; + [renderEncoder setFragmentTexture:videoFrameTextureUV atIndex:1]; + [renderEncoder setVertexBytes:arrVerts length:sizeof(arrVerts) atIndex:0]; + [renderEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:SDL_arraysize(arrVerts)]; + return SDL_TRUE; +}} + +void CleanupVideoToolboxOutput() +{ +}