Here's a complete patch for hacks according to @Jojojoris's investigation and changes to make video playback work with un-extended GLES2. I have only tested it with the GStreamer backend where it does work in my situation: VP8 video resulting in NV12 format, i.MX6 DualLite SoC, Linux 5.15.74, etnaviv driver from Mesa 22.1.7.
Since it isn't possible to paste files here, I'm pasting it as a code block. The file name would be: 0001-HACK-disable-texture-and-pixel-formats-that-don-t-wo.patch
From 7a194147152534efd213346771ce3af135e3df02 Mon Sep 17 00:00:00 2001
From: Andreas Hartmetz <andreas@ixgreen.de>
Date: Thu, 1 Aug 2024 14:10:27 +0200
Subject: [PATCH] HACK: disable texture and pixel formats that don't work with
GLES2
Un-extended GLES2 doesn't support R8 and RG8 texture formats. For
R8, RHI's RED_OR_ALPHA8 is a suitable workaround resulting in
A[lpha]8, but for RG8, there is no simple workaround. To avoid the
need for RG8, crudely mark the NV12 and NV21 pixel formats whose
OpenGL pixel conversion shaders require it as unsupported in the
commonly used GStreamer and FFMPEG backends.
Symptoms without this hack, on i.MX6 DualLite / etnaviv: OpenGL
error messages about unsupported texture format R8 and the video
output is just solid green.
Mostly based on a post of user Jojojoris on forum.qt.io.
---
src/multimedia/shaders/yuv_triplanar.frag | 6 +++---
src/multimedia/shaders/yuv_triplanar_p10.frag | 6 +++---
src/multimedia/shaders/yvu_triplanar.frag | 6 +++---
src/multimedia/video/qvideotexturehelper.cpp | 20 +++++++++----------
.../multimedia/ffmpeg/qffmpegvideobuffer.cpp | 9 +++++++++
.../common/qgstvideorenderersink.cpp | 6 ++++++
6 files changed, 34 insertions(+), 19 deletions(-)
diff --git a/src/multimedia/shaders/yuv_triplanar.frag b/src/multimedia/shaders/yuv_triplanar.frag
index 5b83b05b5..0d3ab7a2c 100644
--- a/src/multimedia/shaders/yuv_triplanar.frag
+++ b/src/multimedia/shaders/yuv_triplanar.frag
@@ -6,20 +6,20 @@
layout(location = 0) in vec2 texCoord;
layout(location = 0) out vec4 fragColor;
layout(binding = 1) uniform sampler2D plane1Texture;
layout(binding = 2) uniform sampler2D plane2Texture;
layout(binding = 3) uniform sampler2D plane3Texture;
void main()
{
- float Y = texture(plane1Texture, texCoord).r;
- float U = texture(plane2Texture, texCoord).r;
- float V = texture(plane3Texture, texCoord).r;
+ float Y = texture(plane1Texture, texCoord).a;
+ float U = texture(plane2Texture, texCoord).a;
+ float V = texture(plane3Texture, texCoord).a;
vec4 color = vec4(Y, U, V, 1.);
fragColor = ubuf.colorMatrix * color * ubuf.opacity;
#ifdef QMM_OUTPUTSURFACE_LINEAR
fragColor = convertSRGBToLinear(fragColor);
#endif
}
diff --git a/src/multimedia/shaders/yuv_triplanar_p10.frag b/src/multimedia/shaders/yuv_triplanar_p10.frag
index bac34c72c..fbbed3506 100644
--- a/src/multimedia/shaders/yuv_triplanar_p10.frag
+++ b/src/multimedia/shaders/yuv_triplanar_p10.frag
@@ -6,20 +6,20 @@
layout(location = 0) in vec2 texCoord;
layout(location = 0) out vec4 fragColor;
layout(binding = 1) uniform sampler2D plane1Texture;
layout(binding = 2) uniform sampler2D plane2Texture;
layout(binding = 3) uniform sampler2D plane3Texture;
void main()
{
- float Y = texture(plane1Texture, texCoord).r * 64;
- float U = texture(plane2Texture, texCoord).r * 64;
- float V = texture(plane3Texture, texCoord).r * 64;
+ float Y = texture(plane1Texture, texCoord).a * 64;
+ float U = texture(plane2Texture, texCoord).a * 64;
+ float V = texture(plane3Texture, texCoord).a * 64;
vec4 color = vec4(Y, U, V, 1.);
fragColor = ubuf.colorMatrix * color * ubuf.opacity;
#ifdef QMM_OUTPUTSURFACE_LINEAR
fragColor = convertSRGBToLinear(fragColor);
#endif
}
diff --git a/src/multimedia/shaders/yvu_triplanar.frag b/src/multimedia/shaders/yvu_triplanar.frag
index 37fc3a18d..cfe406f2a 100644
--- a/src/multimedia/shaders/yvu_triplanar.frag
+++ b/src/multimedia/shaders/yvu_triplanar.frag
@@ -6,20 +6,20 @@
layout(location = 0) in vec2 texCoord;
layout(location = 0) out vec4 fragColor;
layout(binding = 1) uniform sampler2D plane1Texture;
layout(binding = 2) uniform sampler2D plane2Texture;
layout(binding = 3) uniform sampler2D plane3Texture;
void main()
{
- float Y = texture(plane1Texture, texCoord).r;
- float V = texture(plane2Texture, texCoord).r;
- float U = texture(plane3Texture, texCoord).r;
+ float Y = texture(plane1Texture, texCoord).a;
+ float V = texture(plane2Texture, texCoord).a;
+ float U = texture(plane3Texture, texCoord).a;
vec4 color = vec4(Y, U, V, 1.);
fragColor = ubuf.colorMatrix * color * ubuf.opacity;
#ifdef QMM_OUTPUTSURFACE_LINEAR
fragColor = convertSRGBToLinear(fragColor);
#endif
}
diff --git a/src/multimedia/video/qvideotexturehelper.cpp b/src/multimedia/video/qvideotexturehelper.cpp
index 3675e157f..0627cf556 100644
--- a/src/multimedia/video/qvideotexturehelper.cpp
+++ b/src/multimedia/video/qvideotexturehelper.cpp
@@ -88,97 +88,97 @@ static const TextureDescription descriptions[QVideoFrameFormat::NPixelFormats] =
},
// Format_AYUV_Premultiplied
{ 1, 4,
[](int stride, int height) { return stride*height; },
{ QRhiTexture::RGBA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat },
{ { 1, 1 }, { 1, 1 }, { 1, 1 } }
},
// Format_YUV420P
{ 3, 1,
[](int stride, int height) { return stride * ((height * 3 / 2 + 1) & ~1); },
- { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 },
+ { QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RED_OR_ALPHA8 },
{ { 1, 1 }, { 2, 2 }, { 2, 2 } }
},
// Format_YUV422P
{ 3, 1,
[](int stride, int height) { return stride * ((height * 3 / 2 + 1) & ~1); },
- { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 },
+ { QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RED_OR_ALPHA8 },
{ { 1, 1 }, { 2, 1 }, { 2, 1 } }
},
// Format_YV12
{ 3, 1,
[](int stride, int height) { return stride * ((height * 3 / 2 + 1) & ~1); },
- { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 },
+ { QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RED_OR_ALPHA8 },
{ { 1, 1 }, { 2, 2 }, { 2, 2 } }
},
// Format_UYVY
{ 1, 2,
[](int stride, int height) { return stride*height; },
{ QRhiTexture::RGBA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat },
{ { 2, 1 }, { 1, 1 }, { 1, 1 } }
},
// Format_YUYV
{ 1, 2,
[](int stride, int height) { return stride*height; },
{ QRhiTexture::RGBA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat },
{ { 2, 1 }, { 1, 1 }, { 1, 1 } }
},
// Format_NV12
{ 2, 1,
[](int stride, int height) { return stride * ((height * 3 / 2 + 1) & ~1); },
- { QRhiTexture::R8, QRhiTexture::RG8, QRhiTexture::UnknownFormat },
+ { QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RG8, QRhiTexture::UnknownFormat },
{ { 1, 1 }, { 2, 2 }, { 1, 1 } }
},
// Format_NV21
{ 2, 1,
[](int stride, int height) { return stride * ((height * 3 / 2 + 1) & ~1); },
- { QRhiTexture::R8, QRhiTexture::RG8, QRhiTexture::UnknownFormat },
+ { QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RG8, QRhiTexture::UnknownFormat },
{ { 1, 1 }, { 2, 2 }, { 1, 1 } }
},
// Format_IMC1
{ 3, 1,
[](int stride, int height) {
// IMC1 requires that U and V components are aligned on a multiple of 16 lines
int h = (height + 15) & ~15;
h += 2*(((h/2) + 15) & ~15);
return stride * h;
},
- { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 },
+ { QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RED_OR_ALPHA8 },
{ { 1, 1 }, { 2, 2 }, { 2, 2 } }
},
// Format_IMC2
{ 2, 1,
[](int stride, int height) { return 2*stride*height; },
- { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::UnknownFormat },
+ { QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RED_OR_ALPHA8, QRhiTexture::UnknownFormat },
{ { 1, 1 }, { 1, 2 }, { 1, 1 } }
},
// Format_IMC3
{ 3, 1,
[](int stride, int height) {
// IMC3 requires that U and V components are aligned on a multiple of 16 lines
int h = (height + 15) & ~15;
h += 2*(((h/2) + 15) & ~15);
return stride * h;
},
- { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::R8 },
+ { QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RED_OR_ALPHA8 },
{ { 1, 1 }, { 2, 2 }, { 2, 2 } }
},
// Format_IMC4
{ 2, 1,
[](int stride, int height) { return 2*stride*height; },
- { QRhiTexture::R8, QRhiTexture::R8, QRhiTexture::UnknownFormat },
+ { QRhiTexture::RED_OR_ALPHA8, QRhiTexture::RED_OR_ALPHA8, QRhiTexture::UnknownFormat },
{ { 1, 1 }, { 1, 2 }, { 1, 1 } }
},
// Format_Y8
{ 1, 1,
[](int stride, int height) { return stride*height; },
- { QRhiTexture::R8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat },
+ { QRhiTexture::RED_OR_ALPHA8, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat },
{ { 1, 1 }, { 1, 1 }, { 1, 1 } }
},
// Format_Y16
{ 1, 2,
[](int stride, int height) { return stride*height; },
{ QRhiTexture::R16, QRhiTexture::UnknownFormat, QRhiTexture::UnknownFormat },
{ { 1, 1 }, { 1, 1 }, { 1, 1 } }
},
// Format_P010
{ 2, 2,
diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
index 7b72a3cfe..d70e9cb9c 100644
--- a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
+++ b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp
@@ -264,24 +264,33 @@ QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::toQtPixelFormat(AVPixelFormat
case AV_PIX_FMT_YUV422P:
return QVideoFrameFormat::Format_YUV422P;
case AV_PIX_FMT_YUV420P:
return QVideoFrameFormat::Format_YUV420P;
case AV_PIX_FMT_YUV420P10:
return QVideoFrameFormat::Format_YUV420P10;
case AV_PIX_FMT_UYVY422:
return QVideoFrameFormat::Format_UYVY;
case AV_PIX_FMT_YUYV422:
return QVideoFrameFormat::Format_YUYV;
+// The following two are disabled because the pixel format conversion shader for these requires the RG8 pixel
+// format, which is not supported in OpenGL ES2 without GL_EXT_texture_rg, which etnaviv does not currently
+// (July 2024) support. It will probably come with OpenGL ES3 support in etnaviv, which seems to be the focus
+// of its development. ES3 supports RG8 textures in the base specification.
+// (UNTESTED. It is possible that this crude approach breaks something or everything video-related. It might
+// be necessary to recognize these formats in situations other than "which Qt format should we choose for the
+// output buffer".)
+#if 0
case AV_PIX_FMT_NV12:
return QVideoFrameFormat::Format_NV12;
case AV_PIX_FMT_NV21:
return QVideoFrameFormat::Format_NV21;
+#endif
case AV_PIX_FMT_GRAY8:
return QVideoFrameFormat::Format_Y8;
case AV_PIX_FMT_GRAY16:
return QVideoFrameFormat::Format_Y16;
case AV_PIX_FMT_P010:
return QVideoFrameFormat::Format_P010;
case AV_PIX_FMT_P016:
return QVideoFrameFormat::Format_P016;
case AV_PIX_FMT_MEDIACODEC:
diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp
index d67319fdd..e6a572210 100644
--- a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp
+++ b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp
@@ -55,22 +55,28 @@ void QGstVideoRenderer::createSurfaceCaps()
auto caps = QGstCaps::create();
// All the formats that both we and gstreamer support
auto formats = QList<QVideoFrameFormat::PixelFormat>()
<< QVideoFrameFormat::Format_YUV420P
<< QVideoFrameFormat::Format_YUV422P
<< QVideoFrameFormat::Format_YV12
<< QVideoFrameFormat::Format_UYVY
<< QVideoFrameFormat::Format_YUYV
+// The following two are disabled because the pixel format conversion shader for these requires the RG8
+// pixel format, which is not supported in OpenGL ES2 without GL_EXT_texture_rg, which etnaviv does not
+// currently (July 2024) support. It will probably come with OpenGL ES3 support in etnaviv, which seems to
+// be the focus of its development. ES3 supports RG8 textures in the base specification.
+#if 0
<< QVideoFrameFormat::Format_NV12
<< QVideoFrameFormat::Format_NV21
+#endif
<< QVideoFrameFormat::Format_AYUV
<< QVideoFrameFormat::Format_P010
<< QVideoFrameFormat::Format_XRGB8888
<< QVideoFrameFormat::Format_XBGR8888
<< QVideoFrameFormat::Format_RGBX8888
<< QVideoFrameFormat::Format_BGRX8888
<< QVideoFrameFormat::Format_ARGB8888
<< QVideoFrameFormat::Format_ABGR8888
<< QVideoFrameFormat::Format_RGBA8888
<< QVideoFrameFormat::Format_BGRA8888
--
2.43.0
