Browse Source

Add GLVideoWidget for displaying frames of YUV data directly from GPU memory.

chirpmodem-v8.0.0
Jon Beniston 1 month ago
parent
commit
d9febf6281
  1. 19
      sdrgui/CMakeLists.txt
  2. 123
      sdrgui/gui/glvideoframe.h
  3. 71
      sdrgui/gui/glvideointerop.h
  4. 332
      sdrgui/gui/glvideointeropcuda.cpp
  5. 510
      sdrgui/gui/glvideointeropd3d11.cpp
  6. 356
      sdrgui/gui/glvideointeropdmabuf.cpp
  7. 145
      sdrgui/gui/glvideointeropiosurface.cpp
  8. 764
      sdrgui/gui/glvideorenderer.cpp
  9. 109
      sdrgui/gui/glvideorenderer.h
  10. 275
      sdrgui/gui/glvideowidget.cpp
  11. 87
      sdrgui/gui/glvideowidget.h

19
sdrgui/CMakeLists.txt

@ -60,6 +60,12 @@ set(sdrgui_SOURCES
gui/glshaderspectrogram.cpp gui/glshaderspectrogram.cpp
gui/glshadertextured.cpp gui/glshadertextured.cpp
gui/glshadertvarray.cpp gui/glshadertvarray.cpp
gui/glvideorenderer.cpp
gui/glvideowidget.cpp
gui/glvideointeropd3d11.cpp
gui/glvideointeropcuda.cpp
gui/glvideointeropdmabuf.cpp
gui/glvideointeropiosurface.cpp
gui/glspectrum.cpp gui/glspectrum.cpp
gui/glspectrumgui.cpp gui/glspectrumgui.cpp
gui/glspectrumview.cpp gui/glspectrumview.cpp
@ -194,6 +200,10 @@ set(sdrgui_HEADERS
gui/glshadersimple.h gui/glshadersimple.h
gui/glshaderspectrogram.h gui/glshaderspectrogram.h
gui/glshadertvarray.h gui/glshadertvarray.h
gui/glvideoframe.h
gui/glvideointerop.h
gui/glvideorenderer.h
gui/glvideowidget.h
gui/glshadertextured.h gui/glshadertextured.h
gui/glspectrum.h gui/glspectrum.h
gui/glspectrumgui.h gui/glspectrumgui.h
@ -366,6 +376,15 @@ if(ANDROID)
target_link_libraries(sdrgui GLESv3) target_link_libraries(sdrgui GLESv3)
endif() endif()
# GPU video frames shown without copying them: D3D11 on Windows, IOSurface on macOS
if(WIN32 AND MSVC)
target_link_libraries(sdrgui d3d11)
endif()
if(APPLE)
target_link_libraries(sdrgui "-framework IOSurface" "-framework OpenGL")
endif()
install(TARGETS sdrgui DESTINATION ${INSTALL_LIB_DIR}) install(TARGETS sdrgui DESTINATION ${INSTALL_LIB_DIR})
if(WIN32 AND Qt6_FOUND) if(WIN32 AND Qt6_FOUND)

123
sdrgui/gui/glvideoframe.h

@ -0,0 +1,123 @@
///////////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2026 Jon Beniston, M7RCE <jon@beniston.com> //
// Some code by AI //
// //
// This program is free software; you can redistribute it and/or modify //
// it under the terms of the GNU General Public License as published by //
// the Free Software Foundation as version 3 of the License, or //
// (at your option) any later version. //
// //
// This program is distributed in the hope that it will be useful, //
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
// GNU General Public License V3 for more details. //
// //
// You should have received a copy of the GNU General Public License //
// along with this program. If not, see <http://www.gnu.org/licenses/>. //
///////////////////////////////////////////////////////////////////////////////////
#ifndef SDRGUI_GUI_GLVIDEOFRAME_H_
#define SDRGUI_GUI_GLVIDEOFRAME_H_
#include <cstdint>
#include <functional>
#include <memory>
#include <vector>
// A video frame for GLVideoWidget / GLVideoRenderer: YUV 4:2:0 in memory, or on the GPU as a native handle, which
// the renderer imports into OpenGL without copying it through the CPU. Whatever keeps the frame's memory valid (e.g.
// a decoder's frame reference) is held by a subclass, and released when the last pointer to the frame goes. No
// dependency on any decoder, so usable by any video plugin
// An image drawn over the picture, such as subtitles: premultiplied RGBA pixels (m_width x m_height), at (m_x, m_y)
// of a canvas (m_canvasWidth x m_canvasHeight, e.g. a DVB subtitle display) that is stretched over the picture
struct GLVideoSubtitle
{
int m_x = 0;
int m_y = 0;
int m_width = 0;
int m_height = 0;
int m_canvasWidth = 720;
int m_canvasHeight = 576;
std::vector<uint8_t> m_rgba;
};
typedef std::shared_ptr<const GLVideoSubtitle> GLVideoSubtitlePtr;
class GLVideoFrame
{
public:
enum Type
{
Planar420, //!< Y, U, V planes in memory (e.g. AV_PIX_FMT_YUV420P)
NV12, //!< Y, interleaved UV planes in memory
D3D11, //!< NV12 texture array slice of a D3D11 device (Windows, D3D11VA)
DMABuf, //!< NV12 as DMA-BUF planes (Linux, VAAPI)
IOSurface, //!< NV12 IOSurface (macOS, VideoToolbox)
CUDA, //!< NV12 in CUDA device memory (NVIDIA)
NbTypes
};
enum Matrix { BT601, BT709, BT2020 }; //!< BT2020 non-constant luminance
//! Where 4:2:0 chroma samples are, relative to luma: left is co-sited with even columns, between rows (MPEG-2,
//! H.264 and HEVC's default), centre between columns too (MPEG-1, JPEG), top left co-sited with even rows too
enum ChromaLocation { ChromaLeft, ChromaCenter, ChromaTopLeft };
virtual ~GLVideoFrame() {}
Type m_type = Planar420;
int m_width = 0;
int m_height = 0;
double m_sampleAspectRatio = 1.0; //!< Pixel width / height: displayed width is m_width * this
Matrix m_matrix = BT601;
ChromaLocation m_chromaLocation = ChromaLeft; //!< Not used for D3D11 frames, as the video processor converts them
bool m_fullRange = false; //!< Else limited (16 to 235)
bool m_interlaced = false; //!< For deinterlacing by the renderer, which only D3D11 frames need
bool m_topFieldFirst = true;
double m_timestamp = 0.0; //!< Seconds, for information
GLVideoSubtitlePtr m_subtitle; //!< Drawn over the picture, if set. Frames share it while it's shown
// Planar420 and NV12 in memory
const uint8_t *m_data[3] = {};
int m_linesize[3] = {};
// D3D11: a slice of an NV12 texture array, of a device whose immediate context the decoder uses too, so it's
// used between m_lock() and m_unlock()
void *m_d3d11Device = nullptr; //!< ID3D11Device *
void *m_d3d11Texture = nullptr; //!< ID3D11Texture2D *
int m_d3d11Index = 0;
std::function<void()> m_lock;
std::function<void()> m_unlock;
// DMABuf: Y (DRM_FORMAT_R8) and UV (DRM_FORMAT_GR88) planes
int m_dmaBufFd[2] = {-1, -1};
int m_dmaBufOffset[2] = {};
int m_dmaBufPitch[2] = {};
uint32_t m_dmaBufFourcc[2] = {};
uint64_t m_dmaBufModifier[2] = {};
bool m_dmaBufHasModifier = false;
// Which surface (as a VASurfaceID) of which pool (as a decoder's frames context) the planes are, so the display
// can keep them imported for when the surface comes again, rather than import them for each frame (their file
// descriptors differ each time). And what keeps the pool alive (so its address can't be reused for another) while
// the display keeps them. m_dmaBufPool null if not known: then imported for each frame
uint64_t m_dmaBufSurface = 0;
const void *m_dmaBufPool = nullptr;
std::shared_ptr<void> m_dmaBufPoolOwner;
// IOSurface (IOSurfaceRef), NV12 (biplanar)
void *m_ioSurface = nullptr;
// CUDA: Y and UV device pointers (CUdeviceptr), in a context (CUcontext), and what keeps the context alive (e.g. a
// reference to the decoder's device), for the display to hold while it has textures registered in the context, which
// would otherwise be destroyed with the decoder and its last frame
uint64_t m_cudaPlane[2] = {};
int m_cudaPitch[2] = {};
void *m_cudaContext = nullptr;
std::shared_ptr<void> m_cudaContextOwner;
bool isGPU() const { return (m_type != Planar420) && (m_type != NV12); }
};
typedef std::shared_ptr<GLVideoFrame> GLVideoFramePtr;
#endif // SDRGUI_GUI_GLVIDEOFRAME_H_

71
sdrgui/gui/glvideointerop.h

@ -0,0 +1,71 @@
///////////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2026 Jon Beniston, M7RCE <jon@beniston.com> //
// Some code by AI //
// //
// This program is free software; you can redistribute it and/or modify //
// it under the terms of the GNU General Public License as published by //
// the Free Software Foundation as version 3 of the License, or //
// (at your option) any later version. //
// //
// This program is distributed in the hope that it will be useful, //
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
// GNU General Public License V3 for more details. //
// //
// You should have received a copy of the GNU General Public License //
// along with this program. If not, see <http://www.gnu.org/licenses/>. //
///////////////////////////////////////////////////////////////////////////////////
#ifndef SDRGUI_GUI_GLVIDEOINTEROP_H_
#define SDRGUI_GUI_GLVIDEOINTEROP_H_
#include <memory>
#include <QtGui/qopengl.h>
#include "gui/glvideoframe.h"
class QOpenGLContext;
// Internal to GLVideoRenderer: imports GPU frames of one type into OpenGL textures, in the renderer's context
class GLVideoInterop
{
public:
//! How the textures acquired are to be sampled
enum Layout
{
LayoutNV12, //!< Unit 0 Y (R), unit 1 UV (RG)
LayoutNV12Rect, //!< As NV12, as rectangle textures (in pixels)
LayoutRGBA //!< Unit 0 RGBA, already converted (and deinterlaced) on the GPU
};
virtual ~GLVideoInterop() {}
virtual GLVideoFrame::Type type() const = 0;
//! Previous frames wanted, for deinterlacing
virtual int pastFrames() const { return 0; }
//! False if interlaced frames to be deinterlaced are shown as they are, as the GPU can't deinterlace them. Known
//! once a frame has been acquired
virtual bool canDeinterlace() const { return true; }
//! Make the frame available as textures, which are bound to units 0 and 1. past: the previous frames, most
//! recent first, of the same type. False if it can't be shown
virtual bool acquire(const GLVideoFrame& frame, const GLVideoFrame *const *past, int nbPast, bool deinterlace,
GLuint textures[2], Layout& layout) = 0;
//! The textures of the frame last acquired, again, as to draw it again, without importing (or deinterlacing) it
//! again. False if they aren't kept, so it's to be acquired
virtual bool reacquire(GLuint textures[2], Layout& layout) { (void) textures; (void) layout; return false; }
//! After drawing
virtual void release() {}
//! Let go of whatever is held of the frames acquired (as views of a decoder's surfaces), with nothing shown
virtual void releaseFrames() {}
//! Let go of everything without calling OpenGL or its platform's API (WGL, CUDA's OpenGL interop), as the context
//! isn't current or has gone: what's in the context is left to it. Then it can only be destroyed
virtual void abandon() = 0;
};
// Each returns null if the platform, driver or context doesn't support it
std::unique_ptr<GLVideoInterop> createGLVideoInteropD3D11(QOpenGLContext *context);
std::unique_ptr<GLVideoInterop> createGLVideoInteropCUDA(QOpenGLContext *context);
std::unique_ptr<GLVideoInterop> createGLVideoInteropDMABuf(QOpenGLContext *context);
std::unique_ptr<GLVideoInterop> createGLVideoInteropIOSurface(QOpenGLContext *context);
#endif // SDRGUI_GUI_GLVIDEOINTEROP_H_

332
sdrgui/gui/glvideointeropcuda.cpp

@ -0,0 +1,332 @@
///////////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2026 Jon Beniston, M7RCE <jon@beniston.com> //
// Some code by AI //
// //
// This program is free software; you can redistribute it and/or modify //
// it under the terms of the GNU General Public License as published by //
// the Free Software Foundation as version 3 of the License, or //
// (at your option) any later version. //
// //
// This program is distributed in the hope that it will be useful, //
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
// GNU General Public License V3 for more details. //
// //
// You should have received a copy of the GNU General Public License //
// along with this program. If not, see <http://www.gnu.org/licenses/>. //
///////////////////////////////////////////////////////////////////////////////////
// CUDA frames (NVIDIA, NVDEC decoding, deinterlaced by bwdif_cuda): each NV12 plane is copied, within the GPU, into
// an OpenGL texture registered with CUDA. The CUDA driver is loaded when needed, so there's no dependency on it
#include "gui/glvideointerop.h"
#if defined(_WIN32) || (defined(__linux__) && !defined(ANDROID))
#include <cstddef>
#include <QDebug>
#include <QLibrary>
#include <QOpenGLContext>
#include <QOpenGLFunctions>
#ifndef GL_RED
#define GL_RED 0x1903
#endif
#ifndef GL_RG
#define GL_RG 0x8227
#endif
#ifndef GL_R8
#define GL_R8 0x8229
#endif
#ifndef GL_RG8
#define GL_RG8 0x822B
#endif
namespace {
// The parts of the CUDA driver API (cuda.h) used
typedef int CUresult;
typedef void *CUcontext;
typedef void *CUgraphicsResource;
typedef void *CUarray;
typedef void *CUstream;
typedef unsigned long long CUdeviceptr;
enum { CU_MEMORYTYPE_DEVICE = 2, CU_MEMORYTYPE_ARRAY = 3 };
enum { CU_GRAPHICS_REGISTER_FLAGS_WRITE_DISCARD = 2 };
struct CUDAMemcpy2D
{
size_t srcXInBytes;
size_t srcY;
int srcMemoryType;
const void *srcHost;
CUdeviceptr srcDevice;
CUarray srcArray;
size_t srcPitch;
size_t dstXInBytes;
size_t dstY;
int dstMemoryType;
void *dstHost;
CUdeviceptr dstDevice;
CUarray dstArray;
size_t dstPitch;
size_t WidthInBytes;
size_t Height;
};
#if defined(_WIN32)
#define CUDAAPI __stdcall
#else
#define CUDAAPI
#endif
typedef CUresult (CUDAAPI *CtxPushCurrent)(CUcontext);
typedef CUresult (CUDAAPI *CtxPopCurrent)(CUcontext *);
typedef CUresult (CUDAAPI *GraphicsGLRegisterImage)(CUgraphicsResource *, GLuint, GLenum, unsigned int);
typedef CUresult (CUDAAPI *GraphicsUnregisterResource)(CUgraphicsResource);
typedef CUresult (CUDAAPI *GraphicsMapResources)(unsigned int, CUgraphicsResource *, CUstream);
typedef CUresult (CUDAAPI *GraphicsUnmapResources)(unsigned int, CUgraphicsResource *, CUstream);
typedef CUresult (CUDAAPI *GraphicsSubResourceGetMappedArray)(CUarray *, CUgraphicsResource, unsigned int, unsigned int);
typedef CUresult (CUDAAPI *Memcpy2D)(const CUDAMemcpy2D *);
class GLVideoInteropCUDA : public GLVideoInterop
{
public:
GLVideoInteropCUDA() :
m_gl(nullptr),
m_context(nullptr),
m_width(0),
m_height(0),
m_textures{},
m_resources{},
m_acquired(false)
{
}
~GLVideoInteropCUDA() override
{
unregister();
if (m_textures[0]) {
m_gl->glDeleteTextures(2, m_textures);
}
}
bool init(QOpenGLContext *context)
{
#if defined(_WIN32)
m_library.setFileName("nvcuda");
#else
m_library.setFileName("cuda");
m_library.setFileNameAndVersion("cuda", 1);
#endif
if (!m_library.load()) {
return false;
}
m_push = reinterpret_cast<CtxPushCurrent>(m_library.resolve("cuCtxPushCurrent_v2"));
m_pop = reinterpret_cast<CtxPopCurrent>(m_library.resolve("cuCtxPopCurrent_v2"));
m_register = reinterpret_cast<GraphicsGLRegisterImage>(m_library.resolve("cuGraphicsGLRegisterImage"));
m_unregister = reinterpret_cast<GraphicsUnregisterResource>(m_library.resolve("cuGraphicsUnregisterResource"));
m_map = reinterpret_cast<GraphicsMapResources>(m_library.resolve("cuGraphicsMapResources"));
m_unmap = reinterpret_cast<GraphicsUnmapResources>(m_library.resolve("cuGraphicsUnmapResources"));
m_mappedArray = reinterpret_cast<GraphicsSubResourceGetMappedArray>(m_library.resolve("cuGraphicsSubResourceGetMappedArray"));
m_memcpy2D = reinterpret_cast<Memcpy2D>(m_library.resolve("cuMemcpy2D_v2"));
if (!m_push || !m_pop || !m_register || !m_unregister || !m_map || !m_unmap || !m_mappedArray || !m_memcpy2D) {
return false;
}
m_gl = context->functions();
m_gl->glGenTextures(2, m_textures);
return true;
}
GLVideoFrame::Type type() const override { return GLVideoFrame::CUDA; }
bool acquire(const GLVideoFrame& frame, const GLVideoFrame *const *, int, bool, GLuint textures[2], Layout& layout) override
{
CUcontext context = static_cast<CUcontext>(frame.m_cudaContext);
m_acquired = false;
if (!context || (m_push(context) != 0)) {
return false;
}
bool ok = setup(context, frame.m_cudaContextOwner, frame.m_width, frame.m_height);
if (ok)
{
ok = m_map(2, m_resources, nullptr) == 0;
if (ok)
{
for (int plane = 0; (plane < 2) && ok; plane++)
{
CUarray array = nullptr;
ok = m_mappedArray(&array, m_resources[plane], 0, 0) == 0;
if (ok)
{
CUDAMemcpy2D copy = {};
copy.srcMemoryType = CU_MEMORYTYPE_DEVICE;
copy.srcDevice = frame.m_cudaPlane[plane];
copy.srcPitch = frame.m_cudaPitch[plane];
copy.dstMemoryType = CU_MEMORYTYPE_ARRAY;
copy.dstArray = array;
copy.WidthInBytes = plane == 0 ? frame.m_width : 2 * ((frame.m_width + 1) / 2);
copy.Height = plane == 0 ? frame.m_height : (frame.m_height + 1) / 2;
ok = m_memcpy2D(&copy) == 0;
}
}
m_unmap(2, m_resources, nullptr);
}
}
CUcontext popped;
m_pop(&popped);
if (!ok)
{
qWarning("GLVideoInteropCUDA: copy to OpenGL failed");
return false;
}
m_acquired = true;
textures[0] = m_textures[0];
textures[1] = m_textures[1];
layout = LayoutNV12;
return true;
}
// The textures keep the last frame copied
bool reacquire(GLuint textures[2], Layout& layout) override
{
if (!m_acquired) {
return false;
}
textures[0] = m_textures[0];
textures[1] = m_textures[1];
layout = LayoutNV12;
return true;
}
void abandon() override
{
// The registrations go with the CUDA context, the textures with the OpenGL context
m_resources[0] = m_resources[1] = nullptr;
m_textures[0] = m_textures[1] = 0;
unregister();
}
private:
// In the context they were registered in, which is kept alive until they are unregistered
void unregister()
{
if (m_context && (m_resources[0] || m_resources[1]) && (m_push(m_context) == 0))
{
for (CUgraphicsResource& resource : m_resources)
{
if (resource) {
m_unregister(resource);
}
}
CUcontext popped;
m_pop(&popped);
}
m_resources[0] = m_resources[1] = nullptr;
m_acquired = false;
m_context = nullptr;
m_contextOwner.reset();
m_width = 0;
m_height = 0;
}
// Textures of the frame's size, registered with its context. In the context
bool setup(CUcontext context, const std::shared_ptr<void>& contextOwner, int width, int height)
{
if ((width == m_width) && (height == m_height) && (context == m_context) && m_resources[0] && m_resources[1]) {
return true;
}
unregister();
m_context = context;
m_contextOwner = contextOwner;
const int uvWidth = (width + 1) / 2, uvHeight = (height + 1) / 2;
for (int plane = 0; plane < 2; plane++)
{
m_gl->glBindTexture(GL_TEXTURE_2D, m_textures[plane]);
m_gl->glTexImage2D(GL_TEXTURE_2D, 0, plane == 0 ? GL_R8 : GL_RG8, plane == 0 ? width : uvWidth,
plane == 0 ? height : uvHeight, 0, plane == 0 ? GL_RED : GL_RG, GL_UNSIGNED_BYTE, nullptr);
m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
if (m_register(&m_resources[plane], m_textures[plane], GL_TEXTURE_2D, CU_GRAPHICS_REGISTER_FLAGS_WRITE_DISCARD) != 0)
{
qWarning("GLVideoInteropCUDA: cuGraphicsGLRegisterImage failed");
m_resources[plane] = nullptr;
return false;
}
}
m_gl->glBindTexture(GL_TEXTURE_2D, 0);
m_width = width;
m_height = height;
return true;
}
QLibrary m_library;
CtxPushCurrent m_push = nullptr;
CtxPopCurrent m_pop = nullptr;
GraphicsGLRegisterImage m_register = nullptr;
GraphicsUnregisterResource m_unregister = nullptr;
GraphicsMapResources m_map = nullptr;
GraphicsUnmapResources m_unmap = nullptr;
GraphicsSubResourceGetMappedArray m_mappedArray = nullptr;
Memcpy2D m_memcpy2D = nullptr;
QOpenGLFunctions *m_gl;
CUcontext m_context; //!< Of the resources
std::shared_ptr<void> m_contextOwner; //!< Keeps m_context alive while resources are registered in it
int m_width;
int m_height;
GLuint m_textures[2];
CUgraphicsResource m_resources[2];
bool m_acquired; //!< The textures have the last frame acquired
};
} // namespace
std::unique_ptr<GLVideoInterop> createGLVideoInteropCUDA(QOpenGLContext *context)
{
// Registering textures needs R8 and RG8, so desktop OpenGL 3 or core
if (context->isOpenGLES() || ((context->format().majorVersion() < 3) && !context->hasExtension("GL_ARB_texture_rg"))) {
return nullptr;
}
std::unique_ptr<GLVideoInteropCUDA> interop(new GLVideoInteropCUDA());
if (!interop->init(context)) {
return nullptr;
}
return std::unique_ptr<GLVideoInterop>(interop.release());
}
#else
std::unique_ptr<GLVideoInterop> createGLVideoInteropCUDA(QOpenGLContext *)
{
return nullptr;
}
#endif

510
sdrgui/gui/glvideointeropd3d11.cpp

@ -0,0 +1,510 @@
///////////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2026 Jon Beniston, M7RCE <jon@beniston.com> //
// Some code by AI //
// //
// This program is free software; you can redistribute it and/or modify //
// it under the terms of the GNU General Public License as published by //
// the Free Software Foundation as version 3 of the License, or //
// (at your option) any later version. //
// //
// This program is distributed in the hope that it will be useful, //
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
// GNU General Public License V3 for more details. //
// //
// You should have received a copy of the GNU General Public License //
// along with this program. If not, see <http://www.gnu.org/licenses/>. //
///////////////////////////////////////////////////////////////////////////////////
// D3D11 frames (Windows, D3D11VA decoding): the GPU's video processor deinterlaces and converts each NV12 frame to RGB
// into a texture of the same device, which is copied to one WGL_NV_DX_interop2 shares with OpenGL. Nothing goes
// through the CPU
#include "gui/glvideointerop.h"
#if defined(_WIN32) && !defined(__MINGW32__)
#include <map>
#include <windows.h>
#include <d3d11.h>
#include <wrl/client.h>
#include <QDebug>
#include <QOpenGLContext>
#include <QOpenGLFunctions>
using Microsoft::WRL::ComPtr;
namespace {
typedef HANDLE (WINAPI *DXOpenDevice)(void *dxDevice);
typedef BOOL (WINAPI *DXCloseDevice)(HANDLE device);
typedef HANDLE (WINAPI *DXRegisterObject)(HANDLE device, void *dxObject, GLuint name, GLenum type, GLenum access);
typedef BOOL (WINAPI *DXUnregisterObject)(HANDLE device, HANDLE object);
typedef BOOL (WINAPI *DXLockObjects)(HANDLE device, GLint count, HANDLE *objects);
typedef BOOL (WINAPI *DXUnlockObjects)(HANDLE device, GLint count, HANDLE *objects);
const GLenum accessReadOnly = 0x0000; // WGL_ACCESS_READ_ONLY_NV
class GLVideoInteropD3D11 : public GLVideoInterop
{
public:
GLVideoInteropD3D11() :
m_gl(nullptr),
m_interopDevice(nullptr),
m_width(0),
m_height(0),
m_pastFrames(0),
m_canDeinterlace(false),
m_glTexture(0),
m_glObject(nullptr),
m_locked(false),
m_acquired(false),
m_outputFrame(0),
m_lastTexture(nullptr)
{
}
~GLVideoInteropD3D11() override
{
closeDevice();
}
bool init(QOpenGLContext *context)
{
m_gl = context->functions();
m_open = reinterpret_cast<DXOpenDevice>(context->getProcAddress("wglDXOpenDeviceNV"));
m_close = reinterpret_cast<DXCloseDevice>(context->getProcAddress("wglDXCloseDeviceNV"));
m_register = reinterpret_cast<DXRegisterObject>(context->getProcAddress("wglDXRegisterObjectNV"));
m_unregister = reinterpret_cast<DXUnregisterObject>(context->getProcAddress("wglDXUnregisterObjectNV"));
m_dxLockObjects = reinterpret_cast<DXLockObjects>(context->getProcAddress("wglDXLockObjectsNV"));
m_dxUnlockObjects = reinterpret_cast<DXUnlockObjects>(context->getProcAddress("wglDXUnlockObjectsNV"));
return m_open && m_close && m_register && m_unregister && m_dxLockObjects && m_dxUnlockObjects;
}
GLVideoFrame::Type type() const override { return GLVideoFrame::D3D11; }
int pastFrames() const override { return m_pastFrames; }
bool canDeinterlace() const override { return m_canDeinterlace; }
bool acquire(const GLVideoFrame& frame, const GLVideoFrame *const *past, int nbPast, bool deinterlace,
GLuint textures[2], Layout& layout) override
{
ID3D11Device *device = static_cast<ID3D11Device *>(frame.m_d3d11Device);
ID3D11Texture2D *texture = static_cast<ID3D11Texture2D *>(frame.m_d3d11Texture);
m_acquired = false;
if (!device || !texture || !openDevice(device) || !setupProcessor(frame.m_width, frame.m_height)) {
return false;
}
// If the video processor can't deinterlace, the frame's shown as it is: the renderer then has the decoder
// deinterlace in memory, while deinterlacing's on
const bool deinterlaceFrame = deinterlace && frame.m_interlaced && m_canDeinterlace;
if (texture != m_lastTexture)
{
m_inputViews.clear(); // A new decoder: its texture array
m_lastTexture = texture;
}
if (frame.m_lock) {
frame.m_lock();
}
bool ok = false;
ID3D11VideoProcessorInputView *input = inputView(texture, frame.m_d3d11Index);
ID3D11VideoProcessorInputView *pastViews[4];
int nbPastViews = 0;
for (int i = 0; deinterlaceFrame && (i < nbPast) && (i < 4); i++)
{
ID3D11Texture2D *pastTexture = static_cast<ID3D11Texture2D *>(past[i]->m_d3d11Texture);
if (pastTexture == texture)
{
ID3D11VideoProcessorInputView *view = inputView(pastTexture, past[i]->m_d3d11Index);
if (view) {
pastViews[nbPastViews++] = view;
}
}
}
if (input)
{
const D3D11_VIDEO_FRAME_FORMAT format = !deinterlaceFrame ? D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE
: frame.m_topFieldFirst ? D3D11_VIDEO_FRAME_FORMAT_INTERLACED_TOP_FIELD_FIRST : D3D11_VIDEO_FRAME_FORMAT_INTERLACED_BOTTOM_FIELD_FIRST;
m_videoContext->VideoProcessorSetStreamFrameFormat(m_processor.Get(), 0, format);
// Only the picture, unscaled: the decoder's textures (and so the past frames', of the same array) are
// larger, aligned to its blocks (as 1080 lines in 1088), and the padding would otherwise be scaled in
const RECT rect = {0, 0, (LONG) frame.m_width, (LONG) frame.m_height};
m_videoContext->VideoProcessorSetStreamSourceRect(m_processor.Get(), 0, TRUE, &rect);
m_videoContext->VideoProcessorSetStreamDestRect(m_processor.Get(), 0, TRUE, &rect);
m_videoContext->VideoProcessorSetOutputTargetRect(m_processor.Get(), TRUE, &rect);
// A frame out for each frame in, as bwdif's send_frame
m_videoContext->VideoProcessorSetStreamOutputRate(m_processor.Get(), 0, D3D11_VIDEO_PROCESSOR_OUTPUT_RATE_HALF, FALSE, nullptr);
m_videoContext->VideoProcessorSetStreamAutoProcessingMode(m_processor.Get(), 0, FALSE);
D3D11_VIDEO_PROCESSOR_COLOR_SPACE in = {};
// BT.2020 needs ID3D11VideoContext1's DXGI colour spaces: BT.709 is the nearest of these
in.YCbCr_Matrix = frame.m_matrix == GLVideoFrame::BT601 ? 0 : 1;
in.Nominal_Range = frame.m_fullRange ? D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_0_255 : D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_16_235;
m_videoContext->VideoProcessorSetStreamColorSpace(m_processor.Get(), 0, &in);
D3D11_VIDEO_PROCESSOR_COLOR_SPACE out = {};
out.RGB_Range = 0; // 0 to 255
m_videoContext->VideoProcessorSetOutputColorSpace(m_processor.Get(), &out);
D3D11_VIDEO_PROCESSOR_STREAM stream = {};
stream.Enable = TRUE;
stream.InputFrameOrField = 0;
stream.PastFrames = nbPastViews;
stream.ppPastSurfaces = nbPastViews > 0 ? pastViews : nullptr;
stream.pInputSurface = input;
const HRESULT hr = m_videoContext->VideoProcessorBlt(m_processor.Get(), m_outputView.Get(), m_outputFrame++, 1, &stream);
ok = SUCCEEDED(hr);
if (ok) {
m_context->CopyResource(m_shared.Get(), m_output.Get());
}
m_context->Flush();
if (!ok) {
qWarning("GLVideoInteropD3D11: VideoProcessorBlt failed: 0x%08lx", (unsigned long) hr);
}
}
if (frame.m_unlock) {
frame.m_unlock();
}
if (!ok) {
return false;
}
if (!m_dxLockObjects(m_interopDevice, 1, &m_glObject))
{
qWarning("GLVideoInteropD3D11: wglDXLockObjectsNV failed");
return false;
}
m_locked = true;
m_acquired = true;
textures[0] = m_glTexture;
textures[1] = 0;
layout = LayoutRGBA;
return true;
}
// The shared texture keeps the last frame processed
bool reacquire(GLuint textures[2], Layout& layout) override
{
if (!m_acquired || !m_glObject || m_locked) {
return false;
}
if (!m_dxLockObjects(m_interopDevice, 1, &m_glObject))
{
qWarning("GLVideoInteropD3D11: wglDXLockObjectsNV failed");
return false;
}
m_locked = true;
textures[0] = m_glTexture;
textures[1] = 0;
layout = LayoutRGBA;
return true;
}
void release() override
{
if (m_locked)
{
m_dxUnlockObjects(m_interopDevice, 1, &m_glObject);
m_locked = false;
}
}
// The input views hold the decoder's texture array, and the device (and the interop's) the decoder's device.
// Opened again for the next frame
void releaseFrames() override
{
closeDevice();
}
void abandon() override
{
// The registration, OpenGL texture and interop device go with the context. Then only Direct3D's references
m_locked = false;
m_glObject = nullptr;
m_glTexture = 0;
m_interopDevice = nullptr;
closeDevice();
}
private:
bool openDevice(ID3D11Device *device)
{
if (device == m_device.Get()) {
return m_interopDevice != nullptr;
}
closeDevice();
m_device = device;
m_interopDevice = m_open(device);
if (!m_interopDevice)
{
qWarning("GLVideoInteropD3D11: wglDXOpenDeviceNV failed");
return false;
}
device->GetImmediateContext(&m_context);
if (FAILED(device->QueryInterface(IID_PPV_ARGS(&m_videoDevice))) || FAILED(m_context.As(&m_videoContext)))
{
qWarning("GLVideoInteropD3D11: no video device");
return false;
}
return true;
}
void closeProcessor()
{
release();
if (m_glObject)
{
m_unregister(m_interopDevice, m_glObject);
m_glObject = nullptr;
}
if (m_glTexture)
{
m_gl->glDeleteTextures(1, &m_glTexture);
m_glTexture = 0;
}
m_inputViews.clear();
m_acquired = false;
m_canDeinterlace = false;
m_outputView.Reset();
m_output.Reset();
m_shared.Reset();
m_processor.Reset();
m_enumerator.Reset();
m_width = 0;
m_height = 0;
}
void closeDevice()
{
closeProcessor();
if (m_interopDevice)
{
m_close(m_interopDevice);
m_interopDevice = nullptr;
}
m_videoContext.Reset();
m_videoDevice.Reset();
m_context.Reset();
m_device.Reset();
m_lastTexture = nullptr;
}
// A video processor that can deinterlace, preferably adaptively (with previous frames), and its output texture,
// shared with OpenGL
bool setupProcessor(int width, int height)
{
if ((width == m_width) && (height == m_height) && m_processor) {
return true;
}
closeProcessor();
D3D11_VIDEO_PROCESSOR_CONTENT_DESC desc = {};
desc.InputFrameFormat = D3D11_VIDEO_FRAME_FORMAT_INTERLACED_TOP_FIELD_FIRST;
desc.InputFrameRate = {25, 1};
desc.InputWidth = width;
desc.InputHeight = height;
desc.OutputFrameRate = {25, 1};
desc.OutputWidth = width;
desc.OutputHeight = height;
desc.Usage = D3D11_VIDEO_USAGE_PLAYBACK_NORMAL;
if (FAILED(m_videoDevice->CreateVideoProcessorEnumerator(&desc, &m_enumerator))) {
return false;
}
D3D11_VIDEO_PROCESSOR_CAPS caps;
m_enumerator->GetVideoProcessorCaps(&caps);
UINT index = 0;
int best = -1;
for (UINT i = 0; i < caps.RateConversionCapsCount; i++)
{
D3D11_VIDEO_PROCESSOR_RATE_CONVERSION_CAPS rate;
if (FAILED(m_enumerator->GetVideoProcessorRateConversionCaps(i, &rate))) {
continue;
}
const int score = (rate.ProcessorCaps & D3D11_VIDEO_PROCESSOR_PROCESSOR_CAPS_DEINTERLACE_ADAPTIVE) ? 2
: (rate.ProcessorCaps & D3D11_VIDEO_PROCESSOR_PROCESSOR_CAPS_DEINTERLACE_BOB) ? 1 : 0;
if (score > best)
{
best = score;
index = i;
m_pastFrames = (int) std::min<UINT>(rate.PastFrames, 4);
}
}
if (FAILED(m_videoDevice->CreateVideoProcessor(m_enumerator.Get(), index, &m_processor))) {
return false;
}
UINT support = 0;
DXGI_FORMAT format = DXGI_FORMAT_B8G8R8A8_UNORM;
if (FAILED(m_enumerator->CheckVideoProcessorFormat(format, &support)) || !(support & D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT)) {
format = DXGI_FORMAT_R8G8B8A8_UNORM;
}
D3D11_TEXTURE2D_DESC texture = {};
texture.Width = width;
texture.Height = height;
texture.MipLevels = 1;
texture.ArraySize = 1;
texture.Format = format;
texture.SampleDesc.Count = 1;
texture.Usage = D3D11_USAGE_DEFAULT;
texture.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE;
if (FAILED(m_device->CreateTexture2D(&texture, nullptr, &m_output))) {
return false;
}
D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC view = {};
view.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D;
if (FAILED(m_videoDevice->CreateVideoProcessorOutputView(m_output.Get(), m_enumerator.Get(), &view, &m_outputView))) {
return false;
}
// The video processor can't write to a texture registered with OpenGL: NVIDIA's driver (2026) crashes later
// (in its worker thread), and OpenGL sees nothing. So its output is copied to another texture, that is
texture.BindFlags = D3D11_BIND_SHADER_RESOURCE;
if (FAILED(m_device->CreateTexture2D(&texture, nullptr, &m_shared))) {
return false;
}
m_gl->glGenTextures(1, &m_glTexture);
m_glObject = m_register(m_interopDevice, m_shared.Get(), m_glTexture, GL_TEXTURE_2D, accessReadOnly);
if (!m_glObject)
{
qWarning("GLVideoInteropD3D11: wglDXRegisterObjectNV failed");
return false;
}
m_gl->glBindTexture(GL_TEXTURE_2D, m_glTexture);
m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
m_gl->glBindTexture(GL_TEXTURE_2D, 0);
m_width = width;
m_height = height;
m_canDeinterlace = best > 0;
qDebug("GLVideoInteropD3D11: %dx%d, deinterlacing %s, %d past frames", width, height,
best == 2 ? "adaptive" : best == 1 ? "bob" : "none", m_pastFrames);
return true;
}
// Input views of the decoder's texture array slices, made once each
ID3D11VideoProcessorInputView *inputView(ID3D11Texture2D *texture, int index)
{
auto it = m_inputViews.find(index);
if (it != m_inputViews.end()) {
return it->second.Get();
}
D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC desc = {};
desc.FourCC = 0;
desc.ViewDimension = D3D11_VPIV_DIMENSION_TEXTURE2D;
desc.Texture2D.MipSlice = 0;
desc.Texture2D.ArraySlice = index;
ComPtr<ID3D11VideoProcessorInputView> view;
if (FAILED(m_videoDevice->CreateVideoProcessorInputView(texture, m_enumerator.Get(), &desc, &view)))
{
qWarning("GLVideoInteropD3D11: CreateVideoProcessorInputView failed");
return nullptr;
}
m_inputViews[index] = view;
return view.Get();
}
QOpenGLFunctions *m_gl;
DXOpenDevice m_open = nullptr;
DXCloseDevice m_close = nullptr;
DXRegisterObject m_register = nullptr;
DXUnregisterObject m_unregister = nullptr;
DXLockObjects m_dxLockObjects = nullptr; //!< Of the interop's objects, unlike GLVideoFrame::m_lock (the decoder's)
DXUnlockObjects m_dxUnlockObjects = nullptr;
ComPtr<ID3D11Device> m_device;
ComPtr<ID3D11DeviceContext> m_context;
ComPtr<ID3D11VideoDevice> m_videoDevice;
ComPtr<ID3D11VideoContext> m_videoContext;
HANDLE m_interopDevice;
ComPtr<ID3D11VideoProcessorEnumerator> m_enumerator;
ComPtr<ID3D11VideoProcessor> m_processor;
ComPtr<ID3D11Texture2D> m_output;
ComPtr<ID3D11Texture2D> m_shared; // A copy of m_output, shared with OpenGL
ComPtr<ID3D11VideoProcessorOutputView> m_outputView;
std::map<int, ComPtr<ID3D11VideoProcessorInputView>> m_inputViews;
int m_width;
int m_height;
int m_pastFrames;
bool m_canDeinterlace; //!< Else interlaced frames that are to be deinterlaced are shown as they are
GLuint m_glTexture;
HANDLE m_glObject;
bool m_locked;
bool m_acquired; //!< m_shared has the last frame acquired
UINT m_outputFrame;
ID3D11Texture2D *m_lastTexture;
};
} // namespace
std::unique_ptr<GLVideoInterop> createGLVideoInteropD3D11(QOpenGLContext *context)
{
if (context->isOpenGLES()) {
return nullptr;
}
std::unique_ptr<GLVideoInteropD3D11> interop(new GLVideoInteropD3D11());
if (!interop->init(context)) {
return nullptr;
}
return std::unique_ptr<GLVideoInterop>(interop.release());
}
#else
std::unique_ptr<GLVideoInterop> createGLVideoInteropD3D11(QOpenGLContext *)
{
return nullptr;
}
#endif

356
sdrgui/gui/glvideointeropdmabuf.cpp

@ -0,0 +1,356 @@
///////////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2026 Jon Beniston, M7RCE <jon@beniston.com> //
// Some code by AI //
// //
// This program is free software; you can redistribute it and/or modify //
// it under the terms of the GNU General Public License as published by //
// the Free Software Foundation as version 3 of the License, or //
// (at your option) any later version. //
// //
// This program is distributed in the hope that it will be useful, //
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
// GNU General Public License V3 for more details. //
// //
// You should have received a copy of the GNU General Public License //
// along with this program. If not, see <http://www.gnu.org/licenses/>. //
///////////////////////////////////////////////////////////////////////////////////
// DMA-BUF frames (Linux, VAAPI decoding, deinterlaced by deinterlace_vaapi, exported as DRM PRIME): each plane is
// imported as an EGLImage, bound to a texture, when its surface first comes, and kept for when it comes again. Needs
// an EGL context (Wayland, or X11 with QT_XCB_GL_INTEGRATION=xcb_egl); with GLX this isn't available, and frames are
// copied instead. libEGL is loaded when needed
#include "gui/glvideointerop.h"
#if defined(__linux__) && !defined(ANDROID)
#include <cstdint>
#include <memory>
#include <vector>
#include <QDebug>
#include <QLibrary>
#include <QOpenGLContext>
#include <QOpenGLFunctions>
namespace {
// The parts of EGL (egl.h, eglext.h) used
typedef void *EGLDisplay;
typedef void *EGLContext;
typedef void *EGLImageKHR;
typedef void *EGLClientBuffer;
typedef int32_t EGLint;
typedef unsigned int EGLenum;
typedef unsigned int EGLBoolean;
const EGLint eglNone = 0x3038;
const EGLint eglWidth = 0x3057;
const EGLint eglHeight = 0x3056;
const EGLenum eglLinuxDmaBuf = 0x3270;
const EGLint eglLinuxDrmFourcc = 0x3271;
const EGLint eglDmaBufPlane0Fd = 0x3272;
const EGLint eglDmaBufPlane0Offset = 0x3273;
const EGLint eglDmaBufPlane0Pitch = 0x3274;
const EGLint eglDmaBufPlane0ModifierLo = 0x3443;
const EGLint eglDmaBufPlane0ModifierHi = 0x3444;
// drm_fourcc.h
const uint64_t drmFormatModLinear = 0;
const uint64_t drmFormatModInvalid = 0x00ffffffffffffffULL; // Implicit: the driver's own layout
typedef EGLDisplay (*GetCurrentDisplay)();
typedef EGLContext (*GetCurrentContext)();
typedef void *(*GetProcAddress)(const char *);
typedef const char *(*QueryString)(EGLDisplay, EGLint);
typedef EGLImageKHR (*CreateImage)(EGLDisplay, EGLContext, EGLenum, EGLClientBuffer, const EGLint *);
typedef EGLBoolean (*DestroyImage)(EGLDisplay, EGLImageKHR);
typedef void (*ImageTargetTexture2D)(GLenum, void *);
const EGLint eglExtensions = 0x3055;
class GLVideoInteropDMABuf : public GLVideoInterop
{
public:
GLVideoInteropDMABuf() :
m_gl(nullptr),
m_display(nullptr),
m_pool(nullptr),
m_width(0),
m_height(0),
m_current(-1),
m_modifiers(false)
{
}
~GLVideoInteropDMABuf() override
{
releaseImages();
}
bool init(QOpenGLContext *context)
{
m_library.setFileNameAndVersion("EGL", 1);
if (!m_library.load()) {
return false;
}
GetCurrentDisplay getDisplay = reinterpret_cast<GetCurrentDisplay>(m_library.resolve("eglGetCurrentDisplay"));
GetCurrentContext getContext = reinterpret_cast<GetCurrentContext>(m_library.resolve("eglGetCurrentContext"));
GetProcAddress getProcAddress = reinterpret_cast<GetProcAddress>(m_library.resolve("eglGetProcAddress"));
QueryString queryString = reinterpret_cast<QueryString>(m_library.resolve("eglQueryString"));
// Only if Qt's context is EGL
if (!getDisplay || !getContext || !getProcAddress || !queryString || !getContext()) {
return false;
}
m_display = getDisplay();
const QByteArray extensions(queryString(m_display, eglExtensions));
if (!extensions.contains("EGL_EXT_image_dma_buf_import")) {
return false;
}
m_modifiers = extensions.contains("EGL_EXT_image_dma_buf_import_modifiers");
m_createImage = reinterpret_cast<CreateImage>(getProcAddress("eglCreateImageKHR"));
m_destroyImage = reinterpret_cast<DestroyImage>(getProcAddress("eglDestroyImageKHR"));
m_imageTargetTexture = reinterpret_cast<ImageTargetTexture2D>(getProcAddress("glEGLImageTargetTexture2DOES"));
if (!m_createImage || !m_destroyImage || !m_imageTargetTexture) {
return false;
}
m_gl = context->functions();
return true;
}
GLVideoFrame::Type type() const override { return GLVideoFrame::DMABuf; }
// The planes of each surface of the pool are imported the first time it comes, and kept, as a decoder (or
// deinterlacer) cycles through a few dozen. Without the pool, they're imported for each frame
bool acquire(const GLVideoFrame& frame, const GLVideoFrame *const *, int, bool, GLuint textures[2], Layout& layout) override
{
if (!frame.m_dmaBufPool || (frame.m_dmaBufPool != m_pool) || (frame.m_width != m_width) || (frame.m_height != m_height)
|| (m_surfaces.size() >= maxSurfaces))
{
releaseImages(); // Another pool's, or the previous frame's, or more than a pool would have
m_pool = frame.m_dmaBufPool;
m_poolOwner = frame.m_dmaBufPoolOwner;
m_width = frame.m_width;
m_height = frame.m_height;
}
m_current = -1;
for (int i = 0; m_pool && (i < (int) m_surfaces.size()); i++)
{
if (m_surfaces[i].m_id == frame.m_dmaBufSurface)
{
m_current = i;
break;
}
}
if (m_current < 0)
{
Surface surface;
if (!import(frame, surface)) {
return false;
}
m_surfaces.push_back(surface);
m_current = (int) m_surfaces.size() - 1;
}
textures[0] = m_surfaces[m_current].m_textures[0];
textures[1] = m_surfaces[m_current].m_textures[1];
layout = LayoutNV12;
return true;
}
// The images are kept, as drawing may not have finished with them
void release() override {}
bool reacquire(GLuint textures[2], Layout& layout) override
{
if ((m_current < 0) || (m_current >= (int) m_surfaces.size())) {
return false;
}
textures[0] = m_surfaces[m_current].m_textures[0];
textures[1] = m_surfaces[m_current].m_textures[1];
layout = LayoutNV12;
return true;
}
// The images, and the textures they are bound to, hold the decoder's surfaces
void releaseFrames() override
{
releaseImages();
}
void abandon() override
{
// The textures go with the context. The images aren't the context's (nor need it current), so are destroyed
for (Surface& surface : m_surfaces)
{
for (EGLImageKHR& image : surface.m_images)
{
if (image)
{
m_destroyImage(m_display, image);
image = nullptr;
}
}
}
m_gl = nullptr;
releaseImages();
}
private:
struct Surface
{
uint64_t m_id;
EGLImageKHR m_images[2];
GLuint m_textures[2];
};
static const size_t maxSurfaces = 64;
// Each plane as an EGLImage, bound to a texture of its own
bool import(const GLVideoFrame& frame, Surface& surface)
{
// A tiled or compressed layout can't be described without the modifiers extension: imported as linear, the
// picture would be scrambled, so it's copied instead
if (frame.m_dmaBufHasModifier && !m_modifiers)
{
for (int plane = 0; plane < 2; plane++)
{
const uint64_t modifier = frame.m_dmaBufModifier[plane];
if ((modifier != drmFormatModLinear) && (modifier != drmFormatModInvalid))
{
qWarning("GLVideoInteropDMABuf: plane %d has modifier 0x%llx, without EGL_EXT_image_dma_buf_import_modifiers",
plane, (unsigned long long) modifier);
return false;
}
}
}
surface.m_id = frame.m_dmaBufSurface;
surface.m_images[0] = surface.m_images[1] = nullptr;
m_gl->glGenTextures(2, surface.m_textures);
for (int plane = 0; plane < 2; plane++)
{
std::vector<EGLint> attributes = {
eglWidth, plane == 0 ? frame.m_width : (frame.m_width + 1) / 2,
eglHeight, plane == 0 ? frame.m_height : (frame.m_height + 1) / 2,
eglLinuxDrmFourcc, (EGLint) frame.m_dmaBufFourcc[plane],
eglDmaBufPlane0Fd, frame.m_dmaBufFd[plane],
eglDmaBufPlane0Offset, frame.m_dmaBufOffset[plane],
eglDmaBufPlane0Pitch, frame.m_dmaBufPitch[plane]
};
if (m_modifiers && frame.m_dmaBufHasModifier)
{
attributes.push_back(eglDmaBufPlane0ModifierLo);
attributes.push_back((EGLint) (frame.m_dmaBufModifier[plane] & 0xFFFFFFFF));
attributes.push_back(eglDmaBufPlane0ModifierHi);
attributes.push_back((EGLint) (frame.m_dmaBufModifier[plane] >> 32));
}
attributes.push_back(eglNone);
surface.m_images[plane] = m_createImage(m_display, nullptr, eglLinuxDmaBuf, nullptr, attributes.data());
if (!surface.m_images[plane])
{
qWarning("GLVideoInteropDMABuf: eglCreateImageKHR failed for plane %d", plane);
m_gl->glBindTexture(GL_TEXTURE_2D, 0);
destroy(surface);
return false;
}
m_gl->glBindTexture(GL_TEXTURE_2D, surface.m_textures[plane]);
m_imageTargetTexture(GL_TEXTURE_2D, surface.m_images[plane]);
m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
m_gl->glBindTexture(GL_TEXTURE_2D, 0);
return true;
}
// Deleting the textures and images doesn't free the memory while drawing still uses it
void destroy(Surface& surface)
{
m_gl->glDeleteTextures(2, surface.m_textures);
for (EGLImageKHR& image : surface.m_images)
{
if (image)
{
m_destroyImage(m_display, image);
image = nullptr;
}
}
}
void releaseImages()
{
if (m_gl)
{
for (Surface& surface : m_surfaces) {
destroy(surface);
}
}
m_surfaces.clear();
m_current = -1;
m_pool = nullptr;
m_poolOwner.reset();
}
QLibrary m_library;
QOpenGLFunctions *m_gl;
EGLDisplay m_display;
CreateImage m_createImage = nullptr;
DestroyImage m_destroyImage = nullptr;
ImageTargetTexture2D m_imageTargetTexture = nullptr;
std::vector<Surface> m_surfaces; //!< Of m_pool, imported
const void *m_pool; //!< GLVideoFrame::m_dmaBufPool of the surfaces
std::shared_ptr<void> m_poolOwner; //!< Keeps m_pool alive while its surfaces are kept
int m_width; //!< Of the surfaces
int m_height;
int m_current; //!< In m_surfaces, of the frame last acquired, -1 if none
bool m_modifiers;
};
} // namespace
std::unique_ptr<GLVideoInterop> createGLVideoInteropDMABuf(QOpenGLContext *context)
{
std::unique_ptr<GLVideoInteropDMABuf> interop(new GLVideoInteropDMABuf());
if (!interop->init(context)) {
return nullptr;
}
return std::unique_ptr<GLVideoInterop>(interop.release());
}
#else
std::unique_ptr<GLVideoInterop> createGLVideoInteropDMABuf(QOpenGLContext *)
{
return nullptr;
}
#endif

145
sdrgui/gui/glvideointeropiosurface.cpp

@ -0,0 +1,145 @@
///////////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2026 Jon Beniston, M7RCE <jon@beniston.com> //
// Some code by AI //
// //
// This program is free software; you can redistribute it and/or modify //
// it under the terms of the GNU General Public License as published by //
// the Free Software Foundation as version 3 of the License, or //
// (at your option) any later version. //
// //
// This program is distributed in the hope that it will be useful, //
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
// GNU General Public License V3 for more details. //
// //
// You should have received a copy of the GNU General Public License //
// along with this program. If not, see <http://www.gnu.org/licenses/>. //
///////////////////////////////////////////////////////////////////////////////////
// IOSurface frames (macOS, VideoToolbox decoding): each NV12 plane is bound to a rectangle texture, without copying
#include "gui/glvideointerop.h"
#if defined(__APPLE__)
#include <IOSurface/IOSurface.h>
#include <OpenGL/OpenGL.h>
#include <OpenGL/CGLIOSurface.h>
#include <OpenGL/gl3.h>
#include <QDebug>
#include <QOpenGLContext>
#include <QOpenGLFunctions>
namespace {
class GLVideoInteropIOSurface : public GLVideoInterop
{
public:
GLVideoInteropIOSurface() :
m_gl(nullptr),
m_textures{}
{
}
~GLVideoInteropIOSurface() override
{
if (m_textures[0]) {
m_gl->glDeleteTextures(2, m_textures);
}
}
bool init(QOpenGLContext *context)
{
m_gl = context->functions();
m_gl->glGenTextures(2, m_textures);
return true;
}
GLVideoFrame::Type type() const override { return GLVideoFrame::IOSurface; }
bool acquire(const GLVideoFrame& frame, const GLVideoFrame *const *, int, bool, GLuint textures[2], Layout& layout) override
{
IOSurfaceRef surface = static_cast<IOSurfaceRef>(frame.m_ioSurface);
CGLContextObj cgl = CGLGetCurrentContext();
if (!surface || !cgl || (IOSurfaceGetPlaneCount(surface) < 2)) {
return false;
}
for (int plane = 0; plane < 2; plane++)
{
m_gl->glBindTexture(GL_TEXTURE_RECTANGLE, m_textures[plane]);
const CGLError error = CGLTexImageIOSurface2D(cgl, GL_TEXTURE_RECTANGLE, plane == 0 ? GL_R8 : GL_RG8,
(GLsizei) IOSurfaceGetWidthOfPlane(surface, plane), (GLsizei) IOSurfaceGetHeightOfPlane(surface, plane),
plane == 0 ? GL_RED : GL_RG, GL_UNSIGNED_BYTE, surface, plane);
if (error != kCGLNoError)
{
qWarning("GLVideoInteropIOSurface: CGLTexImageIOSurface2D failed: %d", (int) error);
return false;
}
m_gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
m_gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
m_gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
m_gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
m_gl->glBindTexture(GL_TEXTURE_RECTANGLE, 0);
textures[0] = m_textures[0];
textures[1] = m_textures[1];
layout = LayoutNV12Rect;
return true;
}
// The textures hold the surface they were bound to
void releaseFrames() override
{
for (GLuint texture : m_textures)
{
m_gl->glBindTexture(GL_TEXTURE_RECTANGLE, texture);
m_gl->glTexImage2D(GL_TEXTURE_RECTANGLE, 0, GL_R8, 1, 1, 0, GL_RED, GL_UNSIGNED_BYTE, nullptr);
}
m_gl->glBindTexture(GL_TEXTURE_RECTANGLE, 0);
}
// The textures, and the surfaces bound to them, go with the context
void abandon() override
{
m_textures[0] = m_textures[1] = 0;
m_gl = nullptr;
}
private:
QOpenGLFunctions *m_gl;
GLuint m_textures[2];
};
} // namespace
std::unique_ptr<GLVideoInterop> createGLVideoInteropIOSurface(QOpenGLContext *context)
{
// Rectangle textures are sampled with GLSL 3.30, so a core profile, as SDRangel uses on macOS
if (context->format().profile() != QSurfaceFormat::CoreProfile) {
return nullptr;
}
std::unique_ptr<GLVideoInteropIOSurface> interop(new GLVideoInteropIOSurface());
if (!interop->init(context)) {
return nullptr;
}
return std::unique_ptr<GLVideoInterop>(interop.release());
}
#else
std::unique_ptr<GLVideoInterop> createGLVideoInteropIOSurface(QOpenGLContext *)
{
return nullptr;
}
#endif

764
sdrgui/gui/glvideorenderer.cpp

@ -0,0 +1,764 @@
///////////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2026 Jon Beniston, M7RCE <jon@beniston.com> //
// Some code by AI //
// //
// This program is free software; you can redistribute it and/or modify //
// it under the terms of the GNU General Public License as published by //
// the Free Software Foundation as version 3 of the License, or //
// (at your option) any later version. //
// //
// This program is distributed in the hope that it will be useful, //
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
// GNU General Public License V3 for more details. //
// //
// You should have received a copy of the GNU General Public License //
// along with this program. If not, see <http://www.gnu.org/licenses/>. //
///////////////////////////////////////////////////////////////////////////////////
#include <algorithm>
#include <cmath>
#include <cstring>
#include <QDebug>
#include <QGenericMatrix>
#include <QOpenGLBuffer>
#include <QOpenGLContext>
#include <QOpenGLShaderProgram>
#include <QOpenGLVertexArrayObject>
#include <QVector2D>
#include "gui/glvideorenderer.h"
#ifndef GL_UNPACK_ROW_LENGTH
#define GL_UNPACK_ROW_LENGTH 0x0CF2
#endif
#ifndef GL_RED
#define GL_RED 0x1903
#endif
#ifndef GL_RG
#define GL_RG 0x8227
#endif
#ifndef GL_R8
#define GL_R8 0x8229
#endif
#ifndef GL_RG8
#define GL_RG8 0x822B
#endif
#ifndef GL_TEXTURE_RECTANGLE
#define GL_TEXTURE_RECTANGLE 0x84F5
#endif
#ifndef GL_LUMINANCE
#define GL_LUMINANCE 0x1909
#endif
#ifndef GL_LUMINANCE_ALPHA
#define GL_LUMINANCE_ALPHA 0x190A
#endif
// A quad over the viewport: position, then texture coordinate, with the top of the image at the top
static const GLfloat quad[] = {
-1.0f, -1.0f, 0.0f, 1.0f,
1.0f, -1.0f, 1.0f, 1.0f,
-1.0f, 1.0f, 0.0f, 0.0f,
1.0f, 1.0f, 1.0f, 0.0f
};
GLVideoRenderer::GLVideoRenderer() :
m_initialized(false),
m_core(false),
m_gles(false),
m_rg(false),
m_programs{},
m_programFailed{},
m_vertices(nullptr),
m_vao(nullptr),
m_textures{},
m_textureWidth{},
m_textureHeight{},
m_textureFormat{},
m_imageTextures{},
m_imageKeys{-1, -1, -1},
m_imageDrawn{},
m_imageDraws(0),
m_subtitleTexture(0),
m_supported((1u << GLVideoFrame::Planar420) | (1u << GLVideoFrame::NV12)),
m_noDeinterlace(0),
m_deinterlacing(false),
m_lastDeinterlace(false)
{
}
GLVideoRenderer::~GLVideoRenderer()
{
// cleanup() should have been called with the context current. If not, GL objects are left to the context, which
// may not be current or may have gone, so the interops let go of the rest without calling OpenGL. Before the
// frames they may use go
if (m_initialized)
{
qWarning("GLVideoRenderer::~GLVideoRenderer: cleanup() wasn't called");
for (auto& interop : m_interops)
{
if (interop)
{
interop->abandon();
interop.reset();
}
}
}
for (QOpenGLShaderProgram *&p : m_programs)
{
delete p;
p = nullptr;
}
delete m_vertices;
delete m_vao;
}
void GLVideoRenderer::initialize()
{
if (m_initialized) {
return;
}
QOpenGLContext *context = QOpenGLContext::currentContext();
initializeOpenGLFunctions();
const QSurfaceFormat format = context->format();
m_gles = context->isOpenGLES();
m_core = m_gles ? (format.majorVersion() >= 3) : (format.profile() == QSurfaceFormat::CoreProfile);
m_rg = m_core || (!m_gles && (format.majorVersion() >= 3))
|| context->hasExtension(m_gles ? "GL_EXT_texture_rg" : "GL_ARB_texture_rg");
m_vao = new QOpenGLVertexArrayObject();
if (!m_vao->create())
{
delete m_vao;
m_vao = nullptr; // Compatibility contexts may not have them, and don't need them
}
if (m_vao) {
m_vao->bind();
}
m_vertices = new QOpenGLBuffer(QOpenGLBuffer::VertexBuffer);
m_vertices->create();
m_vertices->bind();
m_vertices->allocate(quad, sizeof(quad));
m_vertices->release();
if (m_vao) {
m_vao->release();
}
glGenTextures(3, m_textures);
glGenTextures(m_nbImages, m_imageTextures);
glGenTextures(1, &m_subtitleTexture);
// GPU frames: those this platform, driver and context can import
unsigned int supported = 0;
m_interops[GLVideoFrame::D3D11] = createGLVideoInteropD3D11(context);
m_interops[GLVideoFrame::IOSurface] = createGLVideoInteropIOSurface(context);
// Their UV planes are two channel (RG) textures, which the NV12 shader samples as such only with m_rg
if (m_rg)
{
m_interops[GLVideoFrame::CUDA] = createGLVideoInteropCUDA(context);
m_interops[GLVideoFrame::DMABuf] = createGLVideoInteropDMABuf(context);
}
for (int t = 0; t < GLVideoFrame::NbTypes; t++)
{
if (m_interops[t]) {
supported |= 1u << t;
}
}
supported |= (1u << GLVideoFrame::Planar420) | (1u << GLVideoFrame::NV12);
m_supported = supported;
m_initialized = true;
qDebug("GLVideoRenderer::initialize: OpenGL%s %d.%d %s, GPU frames:%s%s%s%s", m_gles ? " ES" : "",
format.majorVersion(), format.minorVersion(), m_core ? "core" : "compatibility",
m_interops[GLVideoFrame::D3D11] ? " D3D11" : "", m_interops[GLVideoFrame::CUDA] ? " CUDA" : "",
m_interops[GLVideoFrame::DMABuf] ? " DMA-BUF" : "", m_interops[GLVideoFrame::IOSurface] ? " IOSurface" : "");
}
void GLVideoRenderer::cleanup()
{
if (!m_initialized) {
return;
}
// The interops first, while the frames they may use are still alive
for (auto& interop : m_interops) {
interop.reset();
}
m_past.clear();
m_lastFrame.reset();
m_supported = (1u << GLVideoFrame::Planar420) | (1u << GLVideoFrame::NV12);
m_noDeinterlace = 0;
glDeleteTextures(3, m_textures);
glDeleteTextures(m_nbImages, m_imageTextures);
glDeleteTextures(1, &m_subtitleTexture);
// What was in them: the textures made for another context are allocated again
std::fill(std::begin(m_textures), std::end(m_textures), 0);
std::fill(std::begin(m_textureWidth), std::end(m_textureWidth), 0);
std::fill(std::begin(m_textureHeight), std::end(m_textureHeight), 0);
std::fill(std::begin(m_textureFormat), std::end(m_textureFormat), 0);
std::fill(std::begin(m_imageTextures), std::end(m_imageTextures), 0);
std::fill(std::begin(m_imageKeys), std::end(m_imageKeys), -1);
m_subtitleTexture = 0;
m_subtitle.reset();
for (QOpenGLShaderProgram *&p : m_programs)
{
delete p;
p = nullptr;
}
std::fill(std::begin(m_programFailed), std::end(m_programFailed), false); // Another context's may compile
delete m_vertices;
m_vertices = nullptr;
delete m_vao;
m_vao = nullptr;
m_initialized = false;
}
void GLVideoRenderer::releaseFrames()
{
m_past.clear();
m_lastFrame.reset();
m_noDeinterlace = 0; // The interops' devices are let go of too: the next may deinterlace
if (!m_initialized) {
return;
}
for (auto& interop : m_interops)
{
if (interop) {
interop->releaseFrames();
}
}
}
bool GLVideoRenderer::canShow(GLVideoFrame::Type type) const
{
unsigned int supported = m_supported.load();
if (m_deinterlacing.load()) {
supported &= ~m_noDeinterlace.load();
}
return (supported >> type) & 1u;
}
// Shaders: GLSL 3.30 core (or 3.00 ES), else 1.20 for compatibility contexts of older drivers. Null if it doesn't
// compile, which is only tried once (and warned of)
QOpenGLShaderProgram *GLVideoRenderer::program(Program which)
{
if (m_programs[which] || m_programFailed[which]) {
return m_programs[which];
}
QString header;
if (m_gles) {
header = m_core ? "#version 300 es\nprecision mediump float;\n#define IN in\n#define OUT out\n"
: "#version 100\nprecision mediump float;\n#define IN varying\n#define OUT varying\n";
} else {
header = m_core ? "#version 330 core\n#define IN in\n#define OUT out\n" : "#version 120\n#define IN varying\n#define OUT varying\n";
}
const bool modern = m_core;
const QString vertex = header + (modern ? "in vec2 position;\nin vec2 texCoord;\n" : "attribute vec2 position;\nattribute vec2 texCoord;\n")
+ "OUT vec2 vTexCoord;\n"
"void main() {\n"
" vTexCoord = texCoord;\n"
" gl_Position = vec4(position, 0.0, 1.0);\n"
"}\n";
QString fragment = header;
if (modern) {
fragment += "out vec4 fragColor;\n#define TEX texture\n";
} else {
fragment += "#define fragColor gl_FragColor\n#define TEX texture2D\n";
}
// YUV to RGB: remove the offsets and scale to 0..1 (luma) and -0.5..0.5 (chroma), then the matrix. Chroma is
// sampled at vTexCoord * cScale + cOffset, for where its samples are (see setYUVUniforms)
const QString yuv =
"uniform mat3 yuvMatrix;\n"
"uniform vec2 yRange;\n"
"uniform vec2 cRange;\n"
"uniform vec2 cScale;\n"
"uniform vec2 cOffset;\n"
"vec4 rgb(float y, vec2 c) {\n"
" vec3 v = vec3((y - yRange.x) * yRange.y, (c - vec2(cRange.x)) * cRange.y);\n"
" return vec4(clamp(yuvMatrix * v, 0.0, 1.0), 1.0);\n"
"}\n";
const QString uv = m_rg ? "rg" : "ra";
switch (which)
{
case ProgramPlanar:
fragment += "IN vec2 vTexCoord;\nuniform sampler2D tex0;\nuniform sampler2D tex1;\nuniform sampler2D tex2;\n" + yuv +
"void main() {\n"
" vec2 c = vTexCoord * cScale + cOffset;\n"
" fragColor = rgb(TEX(tex0, vTexCoord).r, vec2(TEX(tex1, c).r, TEX(tex2, c).r));\n"
"}\n";
break;
case ProgramNV12:
fragment += "IN vec2 vTexCoord;\nuniform sampler2D tex0;\nuniform sampler2D tex1;\n" + yuv +
"void main() {\n"
" fragColor = rgb(TEX(tex0, vTexCoord).r, TEX(tex1, vTexCoord * cScale + cOffset)." + uv + ");\n"
"}\n";
break;
case ProgramNV12Rect:
// Rectangle textures are sampled in pixels (macOS IOSurfaces), which may be larger than the picture
fragment += "IN vec2 vTexCoord;\nuniform sampler2DRect tex0;\nuniform sampler2DRect tex1;\nuniform vec2 size0;\n" + yuv +
"void main() {\n"
" fragColor = rgb(texture(tex0, vTexCoord * size0).r, texture(tex1, vTexCoord * cScale + cOffset).rg);\n"
"}\n";
break;
default:
fragment += "IN vec2 vTexCoord;\nuniform sampler2D tex0;\n"
"void main() {\n"
" fragColor = TEX(tex0, vTexCoord);\n"
"}\n";
break;
}
QOpenGLShaderProgram *p = new QOpenGLShaderProgram();
bool ok = p->addShaderFromSourceCode(QOpenGLShader::Vertex, vertex)
&& p->addShaderFromSourceCode(QOpenGLShader::Fragment, fragment);
if (ok)
{
p->bindAttributeLocation("position", 0);
p->bindAttributeLocation("texCoord", 1);
ok = p->link();
}
if (!ok)
{
qWarning() << "GLVideoRenderer::program: shader" << which << "failed:" << p->log();
delete p;
m_programFailed[which] = true;
return nullptr;
}
p->bind();
p->setUniformValue("tex0", 0);
p->setUniformValue("tex1", 1);
p->setUniformValue("tex2", 2);
p->release();
m_programs[which] = p;
return p;
}
// pixels: the chroma texture is sampled in pixels (a rectangle texture), else is (width + 1) / 2 x (height + 1) / 2
void GLVideoRenderer::setYUVUniforms(QOpenGLShaderProgram *p, const GLVideoFrame& frame, bool pixels)
{
// From the luma coefficients: R = Y + 2 (1 - Kr) Cr, B = Y + 2 (1 - Kb) Cb, G = (Y - Kr R - Kb B) / Kg
float kr, kb;
switch (frame.m_matrix)
{
case GLVideoFrame::BT709:
kr = 0.2126f;
kb = 0.0722f;
break;
case GLVideoFrame::BT2020:
kr = 0.2627f;
kb = 0.0593f;
break;
default:
kr = 0.299f;
kb = 0.114f;
break;
}
const float kg = 1.0f - kr - kb;
const float values[9] = {
1.0f, 0.0f, 2.0f * (1.0f - kr),
1.0f, -2.0f * kb * (1.0f - kb) / kg, -2.0f * kr * (1.0f - kr) / kg,
1.0f, 2.0f * (1.0f - kb), 0.0f
};
p->setUniformValue("yuvMatrix", QMatrix3x3(values));
// Chroma sample i is at luma sample 2i (left sited) or 2i + 0.5 (centre), so luma at u (0 to 1 across the
// picture) has chroma at u * width / 2 + 0.25 or u * width / 2 chroma texels (texel centres at i + 0.5).
// Likewise vertically, where only top left is co-sited
const float cw = pixels ? 1.0f : (float) ((frame.m_width + 1) / 2);
const float ch = pixels ? 1.0f : (float) ((frame.m_height + 1) / 2);
const bool left = frame.m_chromaLocation != GLVideoFrame::ChromaCenter;
const bool top = frame.m_chromaLocation == GLVideoFrame::ChromaTopLeft;
p->setUniformValue("cScale", QVector2D(frame.m_width / (2.0f * cw), frame.m_height / (2.0f * ch)));
p->setUniformValue("cOffset", QVector2D(left ? 0.25f / cw : 0.0f, top ? 0.25f / ch : 0.0f));
if (frame.m_fullRange)
{
p->setUniformValue("yRange", QVector2D(0.0f, 1.0f));
p->setUniformValue("cRange", QVector2D(0.5f, 1.0f));
}
else
{
p->setUniformValue("yRange", QVector2D(16.0f / 255.0f, 255.0f / 219.0f));
p->setUniformValue("cRange", QVector2D(128.0f / 255.0f, 255.0f / 224.0f));
}
}
// A plane in memory into one of the textures: one channel (Y, U, V) or two (interleaved UV)
void GLVideoRenderer::uploadPlane(int unit, int index, const uint8_t *data, int linesize, int width, int height, bool twoChannels)
{
glActiveTexture(GL_TEXTURE0 + unit);
glBindTexture(GL_TEXTURE_2D, m_textures[index]);
const GLenum format = twoChannels ? (m_rg ? GL_RG : GL_LUMINANCE_ALPHA) : (m_rg ? GL_RED : GL_LUMINANCE);
// OpenGL ES 2's internal format is the format (GL_EXT_texture_rg has no sized ones for glTexImage2D)
const GLint internal = m_gles && !m_core ? (GLint) format
: twoChannels ? (m_rg ? GL_RG8 : GL_LUMINANCE_ALPHA) : (m_rg ? GL_R8 : GL_LUMINANCE);
const int bytesPerPixel = twoChannels ? 2 : 1;
const bool rowLength = !m_gles || m_core;
glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
if (rowLength)
{
glPixelStorei(GL_UNPACK_ROW_LENGTH, linesize / bytesPerPixel);
}
else if (linesize != width * bytesPerPixel)
{
// OpenGL ES 2 can't skip the padding at the ends of rows, so the rows are packed together
const size_t row = (size_t) width * bytesPerPixel;
m_repack.resize(row * height);
for (int y = 0; y < height; y++) {
std::memcpy(m_repack.data() + row * y, data + (ptrdiff_t) linesize * y, row);
}
data = m_repack.data();
}
if ((m_textureWidth[index] != width) || (m_textureHeight[index] != height) || (m_textureFormat[index] != internal))
{
glTexImage2D(GL_TEXTURE_2D, 0, internal, width, height, 0, format, GL_UNSIGNED_BYTE, data);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
m_textureWidth[index] = width;
m_textureHeight[index] = height;
m_textureFormat[index] = internal;
}
else
{
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, format, GL_UNSIGNED_BYTE, data);
}
if (rowLength) {
glPixelStorei(GL_UNPACK_ROW_LENGTH, 0);
}
}
void GLVideoRenderer::drawQuad(QOpenGLShaderProgram *p)
{
if (m_vao) {
m_vao->bind();
}
m_vertices->bind();
p->enableAttributeArray(0);
p->enableAttributeArray(1);
p->setAttributeBuffer(0, GL_FLOAT, 0, 2, 4 * sizeof(GLfloat));
p->setAttributeBuffer(1, GL_FLOAT, 2 * sizeof(GLfloat), 2, 4 * sizeof(GLfloat));
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
p->disableAttributeArray(0);
p->disableAttributeArray(1);
m_vertices->release();
if (m_vao) {
m_vao->release();
}
}
QRect GLVideoRenderer::videoRect(const GLVideoFrame& frame, int width, int height, bool keepAspect)
{
if (!keepAspect || (frame.m_width <= 0) || (frame.m_height <= 0)) {
return QRect(0, 0, width, height);
}
const double aspect = frame.m_width * (frame.m_sampleAspectRatio > 0.0 ? frame.m_sampleAspectRatio : 1.0) / frame.m_height;
int w = width, h = (int) std::lround(width / aspect);
if (h > height)
{
h = height;
w = (int) std::lround(height * aspect);
}
return QRect((width - w) / 2, (height - h) / 2, w, h);
}
QRect GLVideoRenderer::subtitleRect(const GLVideoSubtitle& subtitle, const QRect& videoRect)
{
const double sx = videoRect.width() / (double) std::max(1, subtitle.m_canvasWidth);
const double sy = videoRect.height() / (double) std::max(1, subtitle.m_canvasHeight);
const int x0 = videoRect.x() + (int) std::lround(subtitle.m_x * sx);
const int y0 = videoRect.y() + (int) std::lround(subtitle.m_y * sy);
const int x1 = videoRect.x() + (int) std::lround((subtitle.m_x + subtitle.m_width) * sx);
const int y1 = videoRect.y() + (int) std::lround((subtitle.m_y + subtitle.m_height) * sy);
return QRect(x0, y0, x1 - x0, y1 - y0);
}
bool GLVideoRenderer::drawFrame(const GLVideoFramePtr& frame, int width, int height, bool keepAspect, bool deinterlace)
{
if (!m_initialized || !frame || (frame->m_width <= 0) || (frame->m_height <= 0)) {
return false;
}
m_deinterlacing = deinterlace;
const QRect r = videoRect(*frame, width, height, keepAspect);
glViewport(r.x(), height - r.y() - r.height(), r.width(), r.height());
glDisable(GL_BLEND);
QOpenGLShaderProgram *p = nullptr;
GLVideoInterop *interop = nullptr;
bool ok = true;
// Drawn again, as when the overlay changes: its textures still have it
const bool again = (m_lastFrame.lock() == frame) && (deinterlace == m_lastDeinterlace);
if ((frame->m_type == GLVideoFrame::Planar420) || (frame->m_type == GLVideoFrame::NV12))
{
const bool planar = frame->m_type == GLVideoFrame::Planar420;
p = program(planar ? ProgramPlanar : ProgramNV12);
if (!p) {
return false;
}
if (again)
{
for (int i = 0; i < (planar ? 3 : 2); i++)
{
glActiveTexture(GL_TEXTURE0 + i);
glBindTexture(GL_TEXTURE_2D, m_textures[i]);
}
}
else if (planar)
{
uploadPlane(0, 0, frame->m_data[0], frame->m_linesize[0], frame->m_width, frame->m_height, false);
uploadPlane(1, 1, frame->m_data[1], frame->m_linesize[1], (frame->m_width + 1) / 2, (frame->m_height + 1) / 2, false);
uploadPlane(2, 2, frame->m_data[2], frame->m_linesize[2], (frame->m_width + 1) / 2, (frame->m_height + 1) / 2, false);
}
else
{
uploadPlane(0, 0, frame->m_data[0], frame->m_linesize[0], frame->m_width, frame->m_height, false);
uploadPlane(1, 1, frame->m_data[1], frame->m_linesize[1], (frame->m_width + 1) / 2, (frame->m_height + 1) / 2, true);
}
p->bind();
setYUVUniforms(p, *frame, false);
}
else
{
interop = m_interops[frame->m_type].get();
// Not canShow(), as frames the GPU can't deinterlace are still shown, until the decoder deinterlaces them
if (!interop || !((m_supported.load() >> frame->m_type) & 1u)) {
return false;
}
// Previous frames of the same type, for deinterlacing
while (!m_past.empty() && (m_past.front()->m_type != frame->m_type)) {
m_past.clear();
}
GLuint textures[2] = {0, 0};
GLVideoInterop::Layout layout;
if (!again || !interop->reacquire(textures, layout))
{
// The frame itself isn't one of its previous frames, if it's drawn again
const int first = !m_past.empty() && (m_past.front() == frame) ? 1 : 0;
const GLVideoFrame *past[4];
const int nbPast = std::min((int) m_past.size() - first, std::min(4, interop->pastFrames()));
for (int i = 0; i < nbPast; i++) {
past[i] = m_past[first + i].get();
}
if (!interop->acquire(*frame, past, nbPast, deinterlace, textures, layout))
{
qWarning("GLVideoRenderer::drawFrame: GPU frames of type %d can't be shown: falling back", frame->m_type);
m_supported &= ~(1u << frame->m_type);
m_past.clear();
m_lastFrame.reset();
return false;
}
const bool deinterlaceFrame = deinterlace && frame->m_interlaced;
// Shown as it is: the decoder deinterlaces them in memory instead, while deinterlacing's on
if (deinterlaceFrame && !interop->canDeinterlace() && !((m_noDeinterlace.load() >> frame->m_type) & 1u))
{
qWarning("GLVideoRenderer::drawFrame: GPU frames of type %d can't be deinterlaced: falling back while deinterlacing", frame->m_type);
m_noDeinterlace |= 1u << frame->m_type;
}
// Previous frames are kept (holding the decoder's surfaces) only while they're used
if (!deinterlaceFrame || !interop->canDeinterlace() || (interop->pastFrames() <= 0))
{
m_past.clear();
}
else if (first == 0)
{
m_past.push_front(frame);
while ((int) m_past.size() > interop->pastFrames()) {
m_past.pop_back();
}
}
}
const GLenum target = layout == GLVideoInterop::LayoutNV12Rect ? GL_TEXTURE_RECTANGLE : GL_TEXTURE_2D;
glActiveTexture(GL_TEXTURE0);
glBindTexture(target, textures[0]);
if (layout != GLVideoInterop::LayoutRGBA)
{
glActiveTexture(GL_TEXTURE1);
glBindTexture(target, textures[1]);
}
p = program(layout == GLVideoInterop::LayoutRGBA ? ProgramRGBA : layout == GLVideoInterop::LayoutNV12 ? ProgramNV12 : ProgramNV12Rect);
// As if it couldn't be acquired: the decoder copies them to memory instead
if (!p)
{
qWarning("GLVideoRenderer::drawFrame: GPU frames of type %d can't be shown: falling back", frame->m_type);
interop->release();
m_supported &= ~(1u << frame->m_type);
m_past.clear();
m_lastFrame.reset();
return false;
}
p->bind();
if (layout != GLVideoInterop::LayoutRGBA) {
setYUVUniforms(p, *frame, layout == GLVideoInterop::LayoutNV12Rect);
}
if (layout == GLVideoInterop::LayoutNV12Rect) {
p->setUniformValue("size0", QVector2D(frame->m_width, frame->m_height));
}
}
m_lastFrame = frame;
m_lastDeinterlace = deinterlace;
drawQuad(p);
p->release();
if (interop) {
interop->release();
}
glActiveTexture(GL_TEXTURE0);
glViewport(0, 0, width, height);
if (frame->m_subtitle) {
drawSubtitle(frame->m_subtitle, r, width, height);
}
return ok;
}
// Uploaded once, and drawn with every frame it's attached to
void GLVideoRenderer::drawSubtitle(const GLVideoSubtitlePtr& subtitle, const QRect& videoRect, int width, int height)
{
if ((subtitle->m_width <= 0) || (subtitle->m_height <= 0)
|| (subtitle->m_rgba.size() < (size_t) subtitle->m_width * subtitle->m_height * 4)) {
return;
}
if (subtitle != m_subtitle)
{
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, m_subtitleTexture);
glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, subtitle->m_width, subtitle->m_height, 0, GL_RGBA, GL_UNSIGNED_BYTE, subtitle->m_rgba.data());
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
m_subtitle = subtitle;
}
drawTexture(m_subtitleTexture, subtitleRect(*subtitle, videoRect), width, height);
}
void GLVideoRenderer::drawImage(const QImage& image, const QRect& rect, int width, int height)
{
if (!m_initialized || image.isNull()) {
return;
}
// Its texture, if it's in one, else the least recently drawn's
const qint64 key = image.cacheKey();
int index = 0;
for (int i = 0; i < m_nbImages; i++)
{
if (m_imageKeys[i] == key)
{
index = i;
break;
}
if (m_imageDrawn[i] < m_imageDrawn[index]) {
index = i;
}
}
if (m_imageKeys[index] != key)
{
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, m_imageTextures[index]);
const QImage rgba = image.convertToFormat(QImage::Format_RGBA8888_Premultiplied);
glPixelStorei(GL_UNPACK_ALIGNMENT, 4);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, rgba.width(), rgba.height(), 0, GL_RGBA, GL_UNSIGNED_BYTE, rgba.constBits());
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
m_imageKeys[index] = key;
}
m_imageDrawn[index] = ++m_imageDraws;
drawTexture(m_imageTextures[index], rect, width, height);
}
// Premultiplied RGBA, blended
void GLVideoRenderer::drawTexture(GLuint texture, const QRect& rect, int width, int height)
{
QOpenGLShaderProgram *p = program(ProgramRGBA);
if (!p) {
return;
}
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, texture);
glViewport(rect.x(), height - rect.y() - rect.height(), rect.width(), rect.height());
glEnable(GL_BLEND);
glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
p->bind();
drawQuad(p);
p->release();
glDisable(GL_BLEND);
glViewport(0, 0, width, height);
}

109
sdrgui/gui/glvideorenderer.h

@ -0,0 +1,109 @@
///////////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2026 Jon Beniston, M7RCE <jon@beniston.com> //
// Some code by AI //
// //
// This program is free software; you can redistribute it and/or modify //
// it under the terms of the GNU General Public License as published by //
// the Free Software Foundation as version 3 of the License, or //
// (at your option) any later version. //
// //
// This program is distributed in the hope that it will be useful, //
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
// GNU General Public License V3 for more details. //
// //
// You should have received a copy of the GNU General Public License //
// along with this program. If not, see <http://www.gnu.org/licenses/>. //
///////////////////////////////////////////////////////////////////////////////////
#ifndef SDRGUI_GUI_GLVIDEORENDERER_H_
#define SDRGUI_GUI_GLVIDEORENDERER_H_
#include <atomic>
#include <deque>
#include <memory>
#include <vector>
#include <QImage>
#include <QOpenGLFunctions>
#include <QRect>
#include "gui/glvideoframe.h"
#include "gui/glvideointerop.h"
#include "export.h"
class QOpenGLShaderProgram;
class QOpenGLBuffer;
class QOpenGLVertexArrayObject;
// Draws video frames with OpenGL, in the current context: YUV 4:2:0 frames in memory are uploaded a plane at a time,
// and frames on the GPU (D3D11, CUDA, DMA-BUF, IOSurface) are imported without copying through the CPU, where the
// platform and driver allow. Either way, YUV is converted to RGB in a shader, so the CPU doesn't convert pixels.
// Separate from GLVideoWidget, so it can be tested with an offscreen context
class SDRGUI_API GLVideoRenderer : protected QOpenGLFunctions
{
public:
GLVideoRenderer();
~GLVideoRenderer();
void initialize(); //!< With the context current
void cleanup(); //!< With the context current, before it's destroyed
//! With the context current: let go of the frames held, for deinterlacing and by the GPU interops, so a decoder's
//! surfaces aren't held while nothing is shown
void releaseFrames();
//! If GPU frames of the type can be shown. From any thread. False once one has failed to be shown, or while
//! deinterlacing if the GPU can't deinterlace them, so the decoder deinterlaces them in memory instead
bool canShow(GLVideoFrame::Type type) const;
//! Draw the frame into the current framebuffer (width x height pixels), letterboxed to its aspect ratio if
//! keepAspect, deinterlacing GPU frames that need it if deinterlace. False if it couldn't be shown
bool drawFrame(const GLVideoFramePtr& frame, int width, int height, bool keepAspect, bool deinterlace);
//! Draw an image, blended, at rect (pixels from the top left) of the framebuffer of width x height pixels
void drawImage(const QImage& image, const QRect& rect, int width, int height);
//! Where the frame is drawn
static QRect videoRect(const GLVideoFrame& frame, int width, int height, bool keepAspect);
//! Where a subtitle is drawn, over the picture drawn at videoRect
static QRect subtitleRect(const GLVideoSubtitle& subtitle, const QRect& videoRect);
private:
enum Program { ProgramPlanar, ProgramNV12, ProgramNV12Rect, ProgramRGBA, NbPrograms };
QOpenGLShaderProgram *program(Program program);
void uploadPlane(int unit, int index, const uint8_t *data, int linesize, int width, int height, bool twoChannels);
void drawQuad(QOpenGLShaderProgram *program);
void setYUVUniforms(QOpenGLShaderProgram *program, const GLVideoFrame& frame, bool pixels);
void drawSubtitle(const GLVideoSubtitlePtr& subtitle, const QRect& videoRect, int width, int height);
void drawTexture(GLuint texture, const QRect& rect, int width, int height);
bool m_initialized;
bool m_core; //!< Core profile (or GLES 3), so GLSL 3.30 (300 es) and R8/RG8 textures
bool m_gles;
bool m_rg; //!< R8/RG8 textures, else luminance (alpha)
QOpenGLShaderProgram *m_programs[NbPrograms];
bool m_programFailed[NbPrograms]; //!< To compile or link, so it isn't tried again
QOpenGLBuffer *m_vertices;
QOpenGLVertexArrayObject *m_vao;
GLuint m_textures[3]; //!< Planes uploaded from memory
int m_textureWidth[3];
int m_textureHeight[3];
int m_textureFormat[3]; //!< Internal format, as NV12's UV plane is two channels
// Images drawn over or instead of the picture (a message, an overlay, a guide), each in a texture of its own, so
// those drawn together aren't uploaded again for each paint. The least recently drawn's is reused for another
static const int m_nbImages = 3;
GLuint m_imageTextures[m_nbImages];
qint64 m_imageKeys[m_nbImages]; //!< QImage::cacheKey() of the image in each, -1 if none
quint64 m_imageDrawn[m_nbImages]; //!< When each was last drawn, in m_imageDraws
quint64 m_imageDraws;
GLuint m_subtitleTexture;
GLVideoSubtitlePtr m_subtitle; //!< In m_subtitleTexture (held, so a new one can't have the same address)
std::unique_ptr<GLVideoInterop> m_interops[GLVideoFrame::NbTypes];
std::atomic<unsigned int> m_supported; //!< Bit per GLVideoFrame::Type
std::atomic<unsigned int> m_noDeinterlace; //!< Bit per GLVideoFrame::Type the GPU has been found not to deinterlace
std::atomic<bool> m_deinterlacing; //!< As frames were last drawn
std::deque<GLVideoFramePtr> m_past; //!< Previous GPU frames, for deinterlacing
//! The frame last drawn, which a repaint (as for an overlay) draws again without uploading or importing it again
std::weak_ptr<GLVideoFrame> m_lastFrame;
bool m_lastDeinterlace;
std::vector<uint8_t> m_repack; //!< Rows of a plane without padding, for OpenGL ES 2
};
#endif // SDRGUI_GUI_GLVIDEORENDERER_H_

275
sdrgui/gui/glvideowidget.cpp

@ -0,0 +1,275 @@
///////////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2026 Jon Beniston, M7RCE <jon@beniston.com> //
// Some code by AI //
// //
// This program is free software; you can redistribute it and/or modify //
// it under the terms of the GNU General Public License as published by //
// the Free Software Foundation as version 3 of the License, or //
// (at your option) any later version. //
// //
// This program is distributed in the hope that it will be useful, //
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
// GNU General Public License V3 for more details. //
// //
// You should have received a copy of the GNU General Public License //
// along with this program. If not, see <http://www.gnu.org/licenses/>. //
///////////////////////////////////////////////////////////////////////////////////
#include <algorithm>
#include <QFontMetrics>
#include <QOpenGLContext>
#include <QOpenGLFunctions>
#include <QPainter>
#include <QPainterPath>
#include "gui/glvideowidget.h"
GLVideoWidget::GLVideoWidget(QWidget *parent) :
QOpenGLWidget(parent),
m_hidden(false),
m_overlayHeight(0),
m_keepAspect(true),
m_deinterlace(true)
{
m_overlayTimer.setSingleShot(true);
connect(&m_overlayTimer, &QTimer::timeout, this, [this]() {
m_overlay.clear();
update();
});
}
GLVideoWidget::~GLVideoWidget()
{
cleanup();
}
void GLVideoWidget::initializeGL()
{
m_renderer.initialize();
// The context is destroyed when the widget is, or recreated when it's reparented: GL objects go first
connect(context(), &QOpenGLContext::aboutToBeDestroyed, this, &GLVideoWidget::cleanup, Qt::DirectConnection);
}
void GLVideoWidget::cleanup()
{
if (!context()) {
return;
}
makeCurrent();
m_current.reset(); // GPU frames may hold the decoder's surfaces
m_renderer.cleanup();
doneCurrent();
disconnect(context(), &QOpenGLContext::aboutToBeDestroyed, this, &GLVideoWidget::cleanup);
}
void GLVideoWidget::setFrame(GLVideoFramePtr frame)
{
{
QMutexLocker locker(&m_mutex);
// GPU frames hold the decoder's surfaces, so aren't kept while hidden. The next given after shown is shown
if (m_hidden && frame && frame->isGPU()) {
return;
}
m_pending = frame;
}
QMetaObject::invokeMethod(this, [this]() { update(); }, Qt::QueuedConnection);
}
void GLVideoWidget::clear()
{
{
QMutexLocker locker(&m_mutex);
m_pending.reset();
}
m_current.reset();
// The renderer's, as for deinterlacing, may hold the decoder's GPU surfaces too
if (context())
{
makeCurrent();
m_renderer.releaseFrames();
doneCurrent();
}
update();
}
// GPU frames, and the renderer's (for deinterlacing, and the interops' views of them), hold the decoder's surfaces,
// which a hidden widget has no use for. Frames in memory are kept, so a paused picture is shown again
void GLVideoWidget::hideEvent(QHideEvent *event)
{
{
QMutexLocker locker(&m_mutex);
m_hidden = true;
if (m_pending && m_pending->isGPU()) {
m_pending.reset();
}
}
if (m_current && m_current->isGPU()) {
m_current.reset();
}
if (context())
{
makeCurrent();
m_renderer.releaseFrames();
doneCurrent();
}
QOpenGLWidget::hideEvent(event);
}
void GLVideoWidget::showEvent(QShowEvent *event)
{
{
QMutexLocker locker(&m_mutex);
m_hidden = false;
}
QOpenGLWidget::showEvent(event);
}
void GLVideoWidget::setMessage(const QString& message)
{
if (message != m_message)
{
m_message = message;
m_messageImage = QImage();
update();
}
}
void GLVideoWidget::setOverlay(const QString& text, double seconds)
{
m_overlay = text;
m_overlayImage = QImage();
m_overlayTimer.start((int) (seconds * 1000.0));
update();
}
void GLVideoWidget::setGuide(const QImage& image)
{
m_guide = image;
update();
}
// As QOpenGLWidget sizes its framebuffer: rounded, so there's no edge left at fractional scales (125%, 150%)
QSize GLVideoWidget::pixelSize() const
{
return size() * devicePixelRatioF();
}
void GLVideoWidget::setKeepAspectRatio(bool keep)
{
m_keepAspect = keep;
update();
}
void GLVideoWidget::setDeinterlace(bool deinterlace)
{
m_deinterlace = deinterlace;
}
QSize GLVideoWidget::getFrameSize() const
{
return m_frameSize;
}
// White text, centred, on black, as large as the widget allows
QImage GLVideoWidget::messageImage(int width, int height)
{
if (!m_messageImage.isNull() && (m_messageImage.width() == width) && (m_messageImage.height() == height)) {
return m_messageImage;
}
QImage image(width, height, QImage::Format_RGBA8888_Premultiplied);
image.fill(Qt::black);
QPainter painter(&image);
painter.setRenderHint(QPainter::TextAntialiasing);
QFont font = painter.font();
font.setPixelSize(std::max(12, height / 20));
painter.setFont(font);
painter.setPen(Qt::white);
painter.drawText(image.rect().adjusted(width / 10, 0, -width / 10, 0), Qt::AlignCenter | Qt::TextWordWrap, m_message);
m_messageImage = image;
return m_messageImage;
}
// White text in a translucent rounded box, sized for the video
QImage GLVideoWidget::overlayImage(int videoHeight)
{
if (!m_overlayImage.isNull() && (m_overlayHeight == videoHeight)) {
return m_overlayImage;
}
QFont font = this->font();
font.setPixelSize(std::max(12, videoHeight / 16));
const QFontMetrics metrics(font);
const int padding = metrics.height() / 3;
const QRect text = metrics.boundingRect(QRect(0, 0, 4096, 4096), Qt::AlignCenter, m_overlay);
QImage image(text.width() + 2 * padding, text.height() + 2 * padding, QImage::Format_RGBA8888_Premultiplied);
image.fill(Qt::transparent);
QPainter painter(&image);
painter.setRenderHint(QPainter::Antialiasing);
painter.setRenderHint(QPainter::TextAntialiasing);
painter.setFont(font);
QPainterPath path;
path.addRoundedRect(image.rect(), padding, padding);
painter.fillPath(path, QColor(0, 0, 0, 160));
painter.setPen(Qt::white);
painter.drawText(image.rect(), Qt::AlignCenter, m_overlay);
m_overlayImage = image;
m_overlayHeight = videoHeight;
return m_overlayImage;
}
void GLVideoWidget::paintGL()
{
{
QMutexLocker locker(&m_mutex);
if (m_pending) {
m_current = std::move(m_pending);
}
}
const QSize pixels = pixelSize();
const int width = pixels.width();
const int height = pixels.height();
QOpenGLFunctions *gl = context()->functions();
gl->glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
gl->glClear(GL_COLOR_BUFFER_BIT);
if (!m_message.isEmpty())
{
m_renderer.drawImage(messageImage(width, height), QRect(0, 0, width, height), width, height);
}
else if (m_current)
{
m_frameSize = QSize(m_current->m_width, m_current->m_height);
m_renderer.drawFrame(m_current, width, height, m_keepAspect, m_deinterlace);
// The guide shows what the overlay would
if (!m_overlay.isEmpty() && m_guide.isNull())
{
const QRect video = GLVideoRenderer::videoRect(*m_current, width, height, m_keepAspect);
const QImage image = overlayImage(video.height());
const int margin = video.height() / 20;
const QRect rect(video.right() - margin - image.width(), video.bottom() - margin - image.height(), image.width(), image.height());
m_renderer.drawImage(image, rect, width, height);
}
}
if (!m_guide.isNull()) {
m_renderer.drawImage(m_guide, QRect(0, 0, width, height), width, height);
}
}

87
sdrgui/gui/glvideowidget.h

@ -0,0 +1,87 @@
///////////////////////////////////////////////////////////////////////////////////
// Copyright (C) 2026 Jon Beniston, M7RCE <jon@beniston.com> //
// Some code by AI //
// //
// This program is free software; you can redistribute it and/or modify //
// it under the terms of the GNU General Public License as published by //
// the Free Software Foundation as version 3 of the License, or //
// (at your option) any later version. //
// //
// This program is distributed in the hope that it will be useful, //
// but WITHOUT ANY WARRANTY; without even the implied warranty of //
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the //
// GNU General Public License V3 for more details. //
// //
// You should have received a copy of the GNU General Public License //
// along with this program. If not, see <http://www.gnu.org/licenses/>. //
///////////////////////////////////////////////////////////////////////////////////
#ifndef SDRGUI_GUI_GLVIDEOWIDGET_H_
#define SDRGUI_GUI_GLVIDEOWIDGET_H_
#include <QImage>
#include <QMutex>
#include <QOpenGLWidget>
#include <QTimer>
#include "gui/glvideoframe.h"
#include "gui/glvideorenderer.h"
#include "export.h"
// Displays video: frames of YUV in memory, or on the GPU (D3D11, CUDA, DMA-BUF, IOSurface) without copying them
// through the CPU, converted to RGB by the GPU, with the frame's aspect ratio. Also a message in place of video,
// text overlaid on it for a while, as a channel's name, and an image over the whole widget, as a programme guide.
// Frames can be given from any thread, as a decoder's: the latest is shown at the next paint
class SDRGUI_API GLVideoWidget : public QOpenGLWidget
{
Q_OBJECT
public:
explicit GLVideoWidget(QWidget *parent = nullptr);
~GLVideoWidget();
//! From any thread. The widget keeps the frame until a newer one has been shown, but not GPU frames while hidden
void setFrame(GLVideoFramePtr frame);
//! From any thread: if frames of the type can be shown. GPU types can't until the widget's OpenGL context
//! exists, or after one has failed to be shown, so the decoder should copy them to memory instead
bool canShow(GLVideoFrame::Type type) const { return m_renderer.canShow(type); }
void clear(); //!< No video, letting go of all frames given
void setMessage(const QString& message); //!< Shown in place of video, until set empty
void setOverlay(const QString& text, double seconds); //!< Over the video, bottom right, lines centred
//! Over everything, blended, as large as pixelSize() (scaled to fit if not), until set null
void setGuide(const QImage& image);
QSize pixelSize() const; //!< Of the widget, in device pixels
void setKeepAspectRatio(bool keep);
void setDeinterlace(bool deinterlace); //!< Of GPU frames the renderer deinterlaces (D3D11)
QSize getFrameSize() const; //!< Of the last frame, empty if none
protected:
void initializeGL() override;
void paintGL() override;
void hideEvent(QHideEvent *event) override;
void showEvent(QShowEvent *event) override;
private:
void cleanup();
QImage messageImage(int width, int height);
QImage overlayImage(int videoHeight);
GLVideoRenderer m_renderer;
mutable QMutex m_mutex; //!< For m_pending and m_hidden
GLVideoFramePtr m_pending; //!< Given, to be shown
GLVideoFramePtr m_current; //!< Being shown
bool m_hidden; //!< So GPU frames given aren't kept
QString m_message;
QImage m_messageImage;
QString m_overlay;
QImage m_overlayImage;
int m_overlayHeight; //!< Video height the overlay image was made for
QTimer m_overlayTimer;
QImage m_guide;
bool m_keepAspect;
bool m_deinterlace;
QSize m_frameSize;
};
#endif // SDRGUI_GUI_GLVIDEOWIDGET_H_
Loading…
Cancel
Save