Added support for decoding MJPG into NV12 textures

src/render/SDL_render.c

@@ -1282,7 +1282,12 @@ static SDL_PixelFormat GetClosestSupportedFormat(SDL_Renderer *renderer, SDL_Pix
     int i;
 
     if (format == SDL_PIXELFORMAT_MJPG) {
-        // We'll decode to SDL_PIXELFORMAT_RGBA32
+        // We'll decode to SDL_PIXELFORMAT_NV12 or SDL_PIXELFORMAT_RGBA32
+        for (i = 0; i < renderer->num_texture_formats; ++i) {
+            if (renderer->texture_formats[i] == SDL_PIXELFORMAT_NV12) {
+                return renderer->texture_formats[i];
+            }
+        }
         for (i = 0; i < renderer->num_texture_formats; ++i) {
             if (renderer->texture_formats[i] == SDL_PIXELFORMAT_RGBA32) {
                 return renderer->texture_formats[i];
@@ -1417,12 +1422,16 @@ SDL_Texture *SDL_CreateTextureWithProperties(SDL_Renderer *renderer, SDL_Propert
             closest_format = renderer->texture_formats[0];
         }
 
+        if (format == SDL_PIXELFORMAT_MJPG && closest_format == SDL_PIXELFORMAT_NV12) {
+            SDL_SetNumberProperty(native_props, SDL_PROP_TEXTURE_CREATE_COLORSPACE_NUMBER, SDL_COLORSPACE_JPEG);
+        } else {
             default_colorspace = SDL_GetDefaultColorspaceForFormat(closest_format);
             if (SDL_COLORSPACETYPE(texture->colorspace) == SDL_COLORSPACETYPE(default_colorspace)) {
                 SDL_SetNumberProperty(native_props, SDL_PROP_TEXTURE_CREATE_COLORSPACE_NUMBER, texture->colorspace);
             } else {
                 SDL_SetNumberProperty(native_props, SDL_PROP_TEXTURE_CREATE_COLORSPACE_NUMBER, default_colorspace);
             }
+        }
         SDL_SetNumberProperty(native_props, SDL_PROP_TEXTURE_CREATE_FORMAT_NUMBER, closest_format);
         SDL_SetNumberProperty(native_props, SDL_PROP_TEXTURE_CREATE_ACCESS_NUMBER, texture->access);
         SDL_SetNumberProperty(native_props, SDL_PROP_TEXTURE_CREATE_WIDTH_NUMBER, texture->w);

src/video/SDL_stb.c

@@ -58,31 +58,60 @@
 #define STBI_ASSERT SDL_assert
 #define STB_IMAGE_IMPLEMENTATION
 #include "stb_image.h"
+
+#undef memset
 #endif
 
+#ifdef SDL_HAVE_STB
+static bool SDL_ConvertPixels_MJPG_to_NV12(int width, int height, const void *src, int src_pitch, void *dst, int dst_pitch)
+{
+    int w = 0, h = 0, format = 0;
+    stbi__context s;
+    stbi__start_mem(&s, src, src_pitch);
+
+    stbi__result_info ri;
+    SDL_zero(ri);
+    ri.bits_per_channel = 8;
+    ri.channel_order = STBI_ORDER_RGB;
+    ri.num_channels = 0;
+
+    stbi__nv12 nv12;
+    nv12.w = width;
+    nv12.h = height;
+    nv12.pitch = dst_pitch;
+    nv12.y = (stbi_uc *)dst;
+    nv12.uv = nv12.y + (nv12.h * nv12.pitch);
+
+    void *pixels = stbi__jpeg_load(&s, &w, &h, &format, 4, &nv12, &ri);
+    if (!pixels) {
+        return SDL_SetError("Couldn't decode image: %s", stbi_failure_reason());
+    }
+    return true;
+}
+#endif // SDL_HAVE_STB
+
 bool SDL_ConvertPixels_STB(int width, int height,
                            SDL_PixelFormat src_format, SDL_Colorspace src_colorspace, SDL_PropertiesID src_properties, const void *src, int src_pitch,
                            SDL_PixelFormat dst_format, SDL_Colorspace dst_colorspace, SDL_PropertiesID dst_properties, void *dst, int dst_pitch)
 {
 #ifdef SDL_HAVE_STB
-    if (src_colorspace != dst_colorspace) {
+    if (src_format == SDL_PIXELFORMAT_MJPG && dst_format == SDL_PIXELFORMAT_NV12) {
-        return SDL_SetError("SDL_ConvertPixels_STB: colorspace conversion not supported");
+        return SDL_ConvertPixels_MJPG_to_NV12(width, height, src, src_pitch, dst, dst_pitch);
-    }
-
-    if (src_format == dst_format) {
-        if (src == dst) {
-            // Nothing to do
-            return true;
-        }
     }
 
+    bool result;
     int w = 0, h = 0, format = 0;
-    stbi_uc *pixels = stbi_load_from_memory((const stbi_uc *)src, src_pitch, &w, &h, &format, 4);
+    int len = (src_format == SDL_PIXELFORMAT_MJPG) ? src_pitch : (height * src_pitch);
+    void *pixels = stbi_load_from_memory(src, len, &w, &h, &format, 4);
     if (!pixels) {
         return SDL_SetError("Couldn't decode image: %s", stbi_failure_reason());
     }
 
-    bool result = SDL_ConvertPixelsAndColorspace(w, h, SDL_PIXELFORMAT_RGBA32, src_colorspace, src_properties, pixels, width * 4, dst_format, dst_colorspace, dst_properties, dst, dst_pitch);
+    if (w == width && h == height) {
+        result = SDL_ConvertPixelsAndColorspace(w, h, SDL_PIXELFORMAT_RGBA32, SDL_COLORSPACE_SRGB, 0, pixels, width * 4, dst_format, dst_colorspace, dst_properties, dst, dst_pitch);
+    } else {
+        result = SDL_SetError("Expected image size %dx%d, actual size %dx%d", width, height, w, h);
+    }
     stbi_image_free(pixels);
 
     return result;

src/video/stb_image.h

@@ -912,6 +912,15 @@ enum
    STBI_ORDER_BGR
 };
 
+typedef struct
+{
+    int w;
+    int h;
+    int pitch;
+    stbi_uc *y;
+    stbi_uc *uv;
+} stbi__nv12;
+
 typedef struct
 {
    int bits_per_channel;
@@ -921,7 +930,7 @@ typedef struct
 
 #ifndef STBI_NO_JPEG
 static int      stbi__jpeg_test(stbi__context *s);
-static void    *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri);
+static void    *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__nv12 *nv12, stbi__result_info *ri);
 #ifndef STB_INTERNAL_SDL
 static int      stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp);
 #endif
@@ -1185,7 +1194,7 @@ static void *stbi__load_main(stbi__context *s, int *x, int *y, int *comp, int re
    // bytes matching expectations; these are prone to false positives, so
    // try them later
    #ifndef STBI_NO_JPEG
-   if (stbi__jpeg_test(s)) return stbi__jpeg_load(s,x,y,comp,req_comp, ri);
+   if (stbi__jpeg_test(s)) return stbi__jpeg_load(s,x,y,comp,req_comp,0, ri);
    #endif
    #ifndef STBI_NO_PNM
    if (stbi__pnm_test(s))  return stbi__pnm_load(s,x,y,comp,req_comp, ri);
@@ -3896,7 +3905,49 @@ static stbi_uc stbi__blinn_8x8(stbi_uc x, stbi_uc y)
    return (stbi_uc) ((t + (t >>8)) >> 8);
 }
 
-static stbi_uc *load_jpeg_image(stbi__jpeg *z, int *out_x, int *out_y, int *comp, int req_comp)
+static stbi_uc *output_jpeg_nv12(stbi__jpeg *z, stbi__nv12 *nv12)
+{
+   unsigned int i,j;
+
+   // Copy the Y plane
+   if (nv12->pitch == (int)z->s->img_x) {
+      memcpy(nv12->y, z->img_comp[0].data, z->s->img_y * z->s->img_x);
+   } else {
+      for (i=0; i < z->s->img_y; ++i) {
+         memcpy(nv12->y + i * nv12->pitch, z->img_comp[0].data + i * z->s->img_x, z->s->img_x);
+      }
+   }
+
+   if (z->s->img_n == 3) {
+      // NV12: U and V are interleaved, each subsampled by 2
+      const int nv12_hs = 2;
+      const int nv12_vs = 2;
+      const int u_hs = (z->img_h_max / z->img_comp[1].h);
+      const int u_vs = (z->img_v_max / z->img_comp[1].v);
+      const int v_hs = (z->img_h_max / z->img_comp[2].h);
+      const int v_vs = (z->img_v_max / z->img_comp[2].v);
+      for (i=0; i < (z->s->img_y + 1) / 2; ++i) {
+         stbi_uc *src_u = z->img_comp[1].data + i * (1 + (nv12_vs - u_vs)) * z->img_comp[1].x;
+         stbi_uc *src_v = z->img_comp[2].data + i * (1 + (nv12_vs - v_vs)) * z->img_comp[2].x;
+         stbi_uc *dst = nv12->uv + i * nv12->pitch;
+         for (j=0; j < (z->s->img_x + 1) / 2; ++j) {
+            *dst++ = *src_u;
+            src_u += 1 + (nv12_hs - u_hs);
+            *dst++ = *src_v;
+            src_v += 1 + (nv12_hs - v_hs);
+         }
+      }
+   } else {
+      // Grayscale
+      for (i=0; i < (z->s->img_y + 1) / 2; ++i) {
+         memset(nv12->uv + i * nv12->pitch, 0x80808080, ((z->s->img_x + 1) / 2) * 2);
+      }
+   }
+
+   return nv12->y;
+}
+
+static stbi_uc *load_jpeg_image(stbi__jpeg *z, int *out_x, int *out_y, int *comp, int req_comp, stbi__nv12 *nv12)
 {
    int n, decode_n, is_rgb;
    z->s->img_n = 0; // make stbi__cleanup_jpeg safe
@@ -3930,6 +3981,19 @@ static stbi_uc *load_jpeg_image(stbi__jpeg *z, int *out_x, int *out_y, int *comp
 
       stbi__resample res_comp[4];
 
+      if (nv12) {
+         if (nv12->w != (int)z->s->img_x || nv12->h != (int)z->s->img_y) {
+             stbi__cleanup_jpeg(z);
+             return stbi__errpuc("badsize", "Unexpected size");
+         }
+
+         if (is_rgb) {
+             stbi__cleanup_jpeg(z);
+             return stbi__errpuc("rgbtonv12", "Can't convert RGB to NV12");
+         }
+
+         output = output_jpeg_nv12(z, nv12);
+      } else {
          for (k=0; k < decode_n; ++k) {
             stbi__resample *r = &res_comp[k];
 
@@ -4051,6 +4115,7 @@ static stbi_uc *load_jpeg_image(stbi__jpeg *z, int *out_x, int *out_y, int *comp
                }
             }
          }
+      }
       stbi__cleanup_jpeg(z);
       *out_x = z->s->img_x;
       *out_y = z->s->img_y;
@@ -4059,7 +4124,7 @@ static stbi_uc *load_jpeg_image(stbi__jpeg *z, int *out_x, int *out_y, int *comp
    }
 }
 
-static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri)
+static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__nv12 *nv12, stbi__result_info *ri)
 {
    unsigned char* result;
    stbi__jpeg* j = (stbi__jpeg*) stbi__malloc(sizeof(stbi__jpeg));
@@ -4068,7 +4133,7 @@ static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int re
    STBI_NOTUSED(ri);
    j->s = s;
    stbi__setup_jpeg(j);
-   result = load_jpeg_image(j, x,y,comp,req_comp);
+   result = load_jpeg_image(j, x,y,comp,req_comp,nv12);
    STBI_FREE(j);
    return result;
 }