From d9febf6281d194fe77144672699d341befd5ecdc Mon Sep 17 00:00:00 2001 From: Jon Beniston Date: Sun, 11 Oct 2026 12:08:49 +0100 Subject: [PATCH] Add GLVideoWidget for displaying frames of YUV data directly from GPU memory. --- sdrgui/CMakeLists.txt | 19 + sdrgui/gui/glvideoframe.h | 123 ++++ sdrgui/gui/glvideointerop.h | 71 +++ sdrgui/gui/glvideointeropcuda.cpp | 332 +++++++++++ sdrgui/gui/glvideointeropd3d11.cpp | 510 +++++++++++++++++ sdrgui/gui/glvideointeropdmabuf.cpp | 356 ++++++++++++ sdrgui/gui/glvideointeropiosurface.cpp | 145 +++++ sdrgui/gui/glvideorenderer.cpp | 764 +++++++++++++++++++++++++ sdrgui/gui/glvideorenderer.h | 109 ++++ sdrgui/gui/glvideowidget.cpp | 275 +++++++++ sdrgui/gui/glvideowidget.h | 87 +++ 11 files changed, 2791 insertions(+) create mode 100644 sdrgui/gui/glvideoframe.h create mode 100644 sdrgui/gui/glvideointerop.h create mode 100644 sdrgui/gui/glvideointeropcuda.cpp create mode 100644 sdrgui/gui/glvideointeropd3d11.cpp create mode 100644 sdrgui/gui/glvideointeropdmabuf.cpp create mode 100644 sdrgui/gui/glvideointeropiosurface.cpp create mode 100644 sdrgui/gui/glvideorenderer.cpp create mode 100644 sdrgui/gui/glvideorenderer.h create mode 100644 sdrgui/gui/glvideowidget.cpp create mode 100644 sdrgui/gui/glvideowidget.h diff --git a/sdrgui/CMakeLists.txt b/sdrgui/CMakeLists.txt index 93e8a5d9c..3e004195f 100644 --- a/sdrgui/CMakeLists.txt +++ b/sdrgui/CMakeLists.txt @@ -60,6 +60,12 @@ set(sdrgui_SOURCES gui/glshaderspectrogram.cpp gui/glshadertextured.cpp gui/glshadertvarray.cpp + gui/glvideorenderer.cpp + gui/glvideowidget.cpp + gui/glvideointeropd3d11.cpp + gui/glvideointeropcuda.cpp + gui/glvideointeropdmabuf.cpp + gui/glvideointeropiosurface.cpp gui/glspectrum.cpp gui/glspectrumgui.cpp gui/glspectrumview.cpp @@ -194,6 +200,10 @@ set(sdrgui_HEADERS gui/glshadersimple.h gui/glshaderspectrogram.h gui/glshadertvarray.h + gui/glvideoframe.h + gui/glvideointerop.h + gui/glvideorenderer.h + gui/glvideowidget.h gui/glshadertextured.h gui/glspectrum.h gui/glspectrumgui.h @@ -366,6 +376,15 @@ if(ANDROID) target_link_libraries(sdrgui GLESv3) endif() +# GPU video frames shown without copying them: D3D11 on Windows, IOSurface on macOS +if(WIN32 AND MSVC) + target_link_libraries(sdrgui d3d11) +endif() + +if(APPLE) + target_link_libraries(sdrgui "-framework IOSurface" "-framework OpenGL") +endif() + install(TARGETS sdrgui DESTINATION ${INSTALL_LIB_DIR}) if(WIN32 AND Qt6_FOUND) diff --git a/sdrgui/gui/glvideoframe.h b/sdrgui/gui/glvideoframe.h new file mode 100644 index 000000000..45744c9bf --- /dev/null +++ b/sdrgui/gui/glvideoframe.h @@ -0,0 +1,123 @@ +/////////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2026 Jon Beniston, M7RCE // +// Some code by AI // +// // +// This program is free software; you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation as version 3 of the License, or // +// (at your option) any later version. // +// // +// This program is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License V3 for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with this program. If not, see . // +/////////////////////////////////////////////////////////////////////////////////// + +#ifndef SDRGUI_GUI_GLVIDEOFRAME_H_ +#define SDRGUI_GUI_GLVIDEOFRAME_H_ + +#include +#include +#include +#include + +// A video frame for GLVideoWidget / GLVideoRenderer: YUV 4:2:0 in memory, or on the GPU as a native handle, which +// the renderer imports into OpenGL without copying it through the CPU. Whatever keeps the frame's memory valid (e.g. +// a decoder's frame reference) is held by a subclass, and released when the last pointer to the frame goes. No +// dependency on any decoder, so usable by any video plugin +// An image drawn over the picture, such as subtitles: premultiplied RGBA pixels (m_width x m_height), at (m_x, m_y) +// of a canvas (m_canvasWidth x m_canvasHeight, e.g. a DVB subtitle display) that is stretched over the picture +struct GLVideoSubtitle +{ + int m_x = 0; + int m_y = 0; + int m_width = 0; + int m_height = 0; + int m_canvasWidth = 720; + int m_canvasHeight = 576; + std::vector m_rgba; +}; + +typedef std::shared_ptr GLVideoSubtitlePtr; + +class GLVideoFrame +{ +public: + enum Type + { + Planar420, //!< Y, U, V planes in memory (e.g. AV_PIX_FMT_YUV420P) + NV12, //!< Y, interleaved UV planes in memory + D3D11, //!< NV12 texture array slice of a D3D11 device (Windows, D3D11VA) + DMABuf, //!< NV12 as DMA-BUF planes (Linux, VAAPI) + IOSurface, //!< NV12 IOSurface (macOS, VideoToolbox) + CUDA, //!< NV12 in CUDA device memory (NVIDIA) + NbTypes + }; + + enum Matrix { BT601, BT709, BT2020 }; //!< BT2020 non-constant luminance + + //! Where 4:2:0 chroma samples are, relative to luma: left is co-sited with even columns, between rows (MPEG-2, + //! H.264 and HEVC's default), centre between columns too (MPEG-1, JPEG), top left co-sited with even rows too + enum ChromaLocation { ChromaLeft, ChromaCenter, ChromaTopLeft }; + + virtual ~GLVideoFrame() {} + + Type m_type = Planar420; + int m_width = 0; + int m_height = 0; + double m_sampleAspectRatio = 1.0; //!< Pixel width / height: displayed width is m_width * this + Matrix m_matrix = BT601; + ChromaLocation m_chromaLocation = ChromaLeft; //!< Not used for D3D11 frames, as the video processor converts them + bool m_fullRange = false; //!< Else limited (16 to 235) + bool m_interlaced = false; //!< For deinterlacing by the renderer, which only D3D11 frames need + bool m_topFieldFirst = true; + double m_timestamp = 0.0; //!< Seconds, for information + GLVideoSubtitlePtr m_subtitle; //!< Drawn over the picture, if set. Frames share it while it's shown + + // Planar420 and NV12 in memory + const uint8_t *m_data[3] = {}; + int m_linesize[3] = {}; + + // D3D11: a slice of an NV12 texture array, of a device whose immediate context the decoder uses too, so it's + // used between m_lock() and m_unlock() + void *m_d3d11Device = nullptr; //!< ID3D11Device * + void *m_d3d11Texture = nullptr; //!< ID3D11Texture2D * + int m_d3d11Index = 0; + std::function m_lock; + std::function m_unlock; + + // DMABuf: Y (DRM_FORMAT_R8) and UV (DRM_FORMAT_GR88) planes + int m_dmaBufFd[2] = {-1, -1}; + int m_dmaBufOffset[2] = {}; + int m_dmaBufPitch[2] = {}; + uint32_t m_dmaBufFourcc[2] = {}; + uint64_t m_dmaBufModifier[2] = {}; + bool m_dmaBufHasModifier = false; + // Which surface (as a VASurfaceID) of which pool (as a decoder's frames context) the planes are, so the display + // can keep them imported for when the surface comes again, rather than import them for each frame (their file + // descriptors differ each time). And what keeps the pool alive (so its address can't be reused for another) while + // the display keeps them. m_dmaBufPool null if not known: then imported for each frame + uint64_t m_dmaBufSurface = 0; + const void *m_dmaBufPool = nullptr; + std::shared_ptr m_dmaBufPoolOwner; + + // IOSurface (IOSurfaceRef), NV12 (biplanar) + void *m_ioSurface = nullptr; + + // CUDA: Y and UV device pointers (CUdeviceptr), in a context (CUcontext), and what keeps the context alive (e.g. a + // reference to the decoder's device), for the display to hold while it has textures registered in the context, which + // would otherwise be destroyed with the decoder and its last frame + uint64_t m_cudaPlane[2] = {}; + int m_cudaPitch[2] = {}; + void *m_cudaContext = nullptr; + std::shared_ptr m_cudaContextOwner; + + bool isGPU() const { return (m_type != Planar420) && (m_type != NV12); } +}; + +typedef std::shared_ptr GLVideoFramePtr; + +#endif // SDRGUI_GUI_GLVIDEOFRAME_H_ diff --git a/sdrgui/gui/glvideointerop.h b/sdrgui/gui/glvideointerop.h new file mode 100644 index 000000000..0750afc8e --- /dev/null +++ b/sdrgui/gui/glvideointerop.h @@ -0,0 +1,71 @@ +/////////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2026 Jon Beniston, M7RCE // +// Some code by AI // +// // +// This program is free software; you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation as version 3 of the License, or // +// (at your option) any later version. // +// // +// This program is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License V3 for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with this program. If not, see . // +/////////////////////////////////////////////////////////////////////////////////// + +#ifndef SDRGUI_GUI_GLVIDEOINTEROP_H_ +#define SDRGUI_GUI_GLVIDEOINTEROP_H_ + +#include + +#include + +#include "gui/glvideoframe.h" + +class QOpenGLContext; + +// Internal to GLVideoRenderer: imports GPU frames of one type into OpenGL textures, in the renderer's context +class GLVideoInterop +{ +public: + //! How the textures acquired are to be sampled + enum Layout + { + LayoutNV12, //!< Unit 0 Y (R), unit 1 UV (RG) + LayoutNV12Rect, //!< As NV12, as rectangle textures (in pixels) + LayoutRGBA //!< Unit 0 RGBA, already converted (and deinterlaced) on the GPU + }; + + virtual ~GLVideoInterop() {} + virtual GLVideoFrame::Type type() const = 0; + //! Previous frames wanted, for deinterlacing + virtual int pastFrames() const { return 0; } + //! False if interlaced frames to be deinterlaced are shown as they are, as the GPU can't deinterlace them. Known + //! once a frame has been acquired + virtual bool canDeinterlace() const { return true; } + //! Make the frame available as textures, which are bound to units 0 and 1. past: the previous frames, most + //! recent first, of the same type. False if it can't be shown + virtual bool acquire(const GLVideoFrame& frame, const GLVideoFrame *const *past, int nbPast, bool deinterlace, + GLuint textures[2], Layout& layout) = 0; + //! The textures of the frame last acquired, again, as to draw it again, without importing (or deinterlacing) it + //! again. False if they aren't kept, so it's to be acquired + virtual bool reacquire(GLuint textures[2], Layout& layout) { (void) textures; (void) layout; return false; } + //! After drawing + virtual void release() {} + //! Let go of whatever is held of the frames acquired (as views of a decoder's surfaces), with nothing shown + virtual void releaseFrames() {} + //! Let go of everything without calling OpenGL or its platform's API (WGL, CUDA's OpenGL interop), as the context + //! isn't current or has gone: what's in the context is left to it. Then it can only be destroyed + virtual void abandon() = 0; +}; + +// Each returns null if the platform, driver or context doesn't support it +std::unique_ptr createGLVideoInteropD3D11(QOpenGLContext *context); +std::unique_ptr createGLVideoInteropCUDA(QOpenGLContext *context); +std::unique_ptr createGLVideoInteropDMABuf(QOpenGLContext *context); +std::unique_ptr createGLVideoInteropIOSurface(QOpenGLContext *context); + +#endif // SDRGUI_GUI_GLVIDEOINTEROP_H_ diff --git a/sdrgui/gui/glvideointeropcuda.cpp b/sdrgui/gui/glvideointeropcuda.cpp new file mode 100644 index 000000000..7217737f7 --- /dev/null +++ b/sdrgui/gui/glvideointeropcuda.cpp @@ -0,0 +1,332 @@ +/////////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2026 Jon Beniston, M7RCE // +// Some code by AI // +// // +// This program is free software; you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation as version 3 of the License, or // +// (at your option) any later version. // +// // +// This program is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License V3 for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with this program. If not, see . // +/////////////////////////////////////////////////////////////////////////////////// + +// CUDA frames (NVIDIA, NVDEC decoding, deinterlaced by bwdif_cuda): each NV12 plane is copied, within the GPU, into +// an OpenGL texture registered with CUDA. The CUDA driver is loaded when needed, so there's no dependency on it + +#include "gui/glvideointerop.h" + +#if defined(_WIN32) || (defined(__linux__) && !defined(ANDROID)) + +#include + +#include +#include +#include +#include + +#ifndef GL_RED +#define GL_RED 0x1903 +#endif +#ifndef GL_RG +#define GL_RG 0x8227 +#endif +#ifndef GL_R8 +#define GL_R8 0x8229 +#endif +#ifndef GL_RG8 +#define GL_RG8 0x822B +#endif + +namespace { + +// The parts of the CUDA driver API (cuda.h) used +typedef int CUresult; +typedef void *CUcontext; +typedef void *CUgraphicsResource; +typedef void *CUarray; +typedef void *CUstream; +typedef unsigned long long CUdeviceptr; + +enum { CU_MEMORYTYPE_DEVICE = 2, CU_MEMORYTYPE_ARRAY = 3 }; +enum { CU_GRAPHICS_REGISTER_FLAGS_WRITE_DISCARD = 2 }; + +struct CUDAMemcpy2D +{ + size_t srcXInBytes; + size_t srcY; + int srcMemoryType; + const void *srcHost; + CUdeviceptr srcDevice; + CUarray srcArray; + size_t srcPitch; + size_t dstXInBytes; + size_t dstY; + int dstMemoryType; + void *dstHost; + CUdeviceptr dstDevice; + CUarray dstArray; + size_t dstPitch; + size_t WidthInBytes; + size_t Height; +}; + +#if defined(_WIN32) +#define CUDAAPI __stdcall +#else +#define CUDAAPI +#endif + +typedef CUresult (CUDAAPI *CtxPushCurrent)(CUcontext); +typedef CUresult (CUDAAPI *CtxPopCurrent)(CUcontext *); +typedef CUresult (CUDAAPI *GraphicsGLRegisterImage)(CUgraphicsResource *, GLuint, GLenum, unsigned int); +typedef CUresult (CUDAAPI *GraphicsUnregisterResource)(CUgraphicsResource); +typedef CUresult (CUDAAPI *GraphicsMapResources)(unsigned int, CUgraphicsResource *, CUstream); +typedef CUresult (CUDAAPI *GraphicsUnmapResources)(unsigned int, CUgraphicsResource *, CUstream); +typedef CUresult (CUDAAPI *GraphicsSubResourceGetMappedArray)(CUarray *, CUgraphicsResource, unsigned int, unsigned int); +typedef CUresult (CUDAAPI *Memcpy2D)(const CUDAMemcpy2D *); + +class GLVideoInteropCUDA : public GLVideoInterop +{ +public: + GLVideoInteropCUDA() : + m_gl(nullptr), + m_context(nullptr), + m_width(0), + m_height(0), + m_textures{}, + m_resources{}, + m_acquired(false) + { + } + + ~GLVideoInteropCUDA() override + { + unregister(); + + if (m_textures[0]) { + m_gl->glDeleteTextures(2, m_textures); + } + } + + bool init(QOpenGLContext *context) + { +#if defined(_WIN32) + m_library.setFileName("nvcuda"); +#else + m_library.setFileName("cuda"); + m_library.setFileNameAndVersion("cuda", 1); +#endif + if (!m_library.load()) { + return false; + } + + m_push = reinterpret_cast(m_library.resolve("cuCtxPushCurrent_v2")); + m_pop = reinterpret_cast(m_library.resolve("cuCtxPopCurrent_v2")); + m_register = reinterpret_cast(m_library.resolve("cuGraphicsGLRegisterImage")); + m_unregister = reinterpret_cast(m_library.resolve("cuGraphicsUnregisterResource")); + m_map = reinterpret_cast(m_library.resolve("cuGraphicsMapResources")); + m_unmap = reinterpret_cast(m_library.resolve("cuGraphicsUnmapResources")); + m_mappedArray = reinterpret_cast(m_library.resolve("cuGraphicsSubResourceGetMappedArray")); + m_memcpy2D = reinterpret_cast(m_library.resolve("cuMemcpy2D_v2")); + + if (!m_push || !m_pop || !m_register || !m_unregister || !m_map || !m_unmap || !m_mappedArray || !m_memcpy2D) { + return false; + } + + m_gl = context->functions(); + m_gl->glGenTextures(2, m_textures); + return true; + } + + GLVideoFrame::Type type() const override { return GLVideoFrame::CUDA; } + + bool acquire(const GLVideoFrame& frame, const GLVideoFrame *const *, int, bool, GLuint textures[2], Layout& layout) override + { + CUcontext context = static_cast(frame.m_cudaContext); + m_acquired = false; + + if (!context || (m_push(context) != 0)) { + return false; + } + + bool ok = setup(context, frame.m_cudaContextOwner, frame.m_width, frame.m_height); + + if (ok) + { + ok = m_map(2, m_resources, nullptr) == 0; + + if (ok) + { + for (int plane = 0; (plane < 2) && ok; plane++) + { + CUarray array = nullptr; + ok = m_mappedArray(&array, m_resources[plane], 0, 0) == 0; + + if (ok) + { + CUDAMemcpy2D copy = {}; + copy.srcMemoryType = CU_MEMORYTYPE_DEVICE; + copy.srcDevice = frame.m_cudaPlane[plane]; + copy.srcPitch = frame.m_cudaPitch[plane]; + copy.dstMemoryType = CU_MEMORYTYPE_ARRAY; + copy.dstArray = array; + copy.WidthInBytes = plane == 0 ? frame.m_width : 2 * ((frame.m_width + 1) / 2); + copy.Height = plane == 0 ? frame.m_height : (frame.m_height + 1) / 2; + ok = m_memcpy2D(©) == 0; + } + } + + m_unmap(2, m_resources, nullptr); + } + } + + CUcontext popped; + m_pop(&popped); + + if (!ok) + { + qWarning("GLVideoInteropCUDA: copy to OpenGL failed"); + return false; + } + + m_acquired = true; + textures[0] = m_textures[0]; + textures[1] = m_textures[1]; + layout = LayoutNV12; + return true; + } + + // The textures keep the last frame copied + bool reacquire(GLuint textures[2], Layout& layout) override + { + if (!m_acquired) { + return false; + } + + textures[0] = m_textures[0]; + textures[1] = m_textures[1]; + layout = LayoutNV12; + return true; + } + + void abandon() override + { + // The registrations go with the CUDA context, the textures with the OpenGL context + m_resources[0] = m_resources[1] = nullptr; + m_textures[0] = m_textures[1] = 0; + unregister(); + } + +private: + // In the context they were registered in, which is kept alive until they are unregistered + void unregister() + { + if (m_context && (m_resources[0] || m_resources[1]) && (m_push(m_context) == 0)) + { + for (CUgraphicsResource& resource : m_resources) + { + if (resource) { + m_unregister(resource); + } + } + + CUcontext popped; + m_pop(&popped); + } + + m_resources[0] = m_resources[1] = nullptr; + m_acquired = false; + m_context = nullptr; + m_contextOwner.reset(); + m_width = 0; + m_height = 0; + } + + // Textures of the frame's size, registered with its context. In the context + bool setup(CUcontext context, const std::shared_ptr& contextOwner, int width, int height) + { + if ((width == m_width) && (height == m_height) && (context == m_context) && m_resources[0] && m_resources[1]) { + return true; + } + + unregister(); + m_context = context; + m_contextOwner = contextOwner; + + const int uvWidth = (width + 1) / 2, uvHeight = (height + 1) / 2; + + for (int plane = 0; plane < 2; plane++) + { + m_gl->glBindTexture(GL_TEXTURE_2D, m_textures[plane]); + m_gl->glTexImage2D(GL_TEXTURE_2D, 0, plane == 0 ? GL_R8 : GL_RG8, plane == 0 ? width : uvWidth, + plane == 0 ? height : uvHeight, 0, plane == 0 ? GL_RED : GL_RG, GL_UNSIGNED_BYTE, nullptr); + m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + + if (m_register(&m_resources[plane], m_textures[plane], GL_TEXTURE_2D, CU_GRAPHICS_REGISTER_FLAGS_WRITE_DISCARD) != 0) + { + qWarning("GLVideoInteropCUDA: cuGraphicsGLRegisterImage failed"); + m_resources[plane] = nullptr; + return false; + } + } + + m_gl->glBindTexture(GL_TEXTURE_2D, 0); + m_width = width; + m_height = height; + return true; + } + + QLibrary m_library; + CtxPushCurrent m_push = nullptr; + CtxPopCurrent m_pop = nullptr; + GraphicsGLRegisterImage m_register = nullptr; + GraphicsUnregisterResource m_unregister = nullptr; + GraphicsMapResources m_map = nullptr; + GraphicsUnmapResources m_unmap = nullptr; + GraphicsSubResourceGetMappedArray m_mappedArray = nullptr; + Memcpy2D m_memcpy2D = nullptr; + QOpenGLFunctions *m_gl; + CUcontext m_context; //!< Of the resources + std::shared_ptr m_contextOwner; //!< Keeps m_context alive while resources are registered in it + int m_width; + int m_height; + GLuint m_textures[2]; + CUgraphicsResource m_resources[2]; + bool m_acquired; //!< The textures have the last frame acquired +}; + +} // namespace + +std::unique_ptr createGLVideoInteropCUDA(QOpenGLContext *context) +{ + // Registering textures needs R8 and RG8, so desktop OpenGL 3 or core + if (context->isOpenGLES() || ((context->format().majorVersion() < 3) && !context->hasExtension("GL_ARB_texture_rg"))) { + return nullptr; + } + + std::unique_ptr interop(new GLVideoInteropCUDA()); + + if (!interop->init(context)) { + return nullptr; + } + + return std::unique_ptr(interop.release()); +} + +#else + +std::unique_ptr createGLVideoInteropCUDA(QOpenGLContext *) +{ + return nullptr; +} + +#endif diff --git a/sdrgui/gui/glvideointeropd3d11.cpp b/sdrgui/gui/glvideointeropd3d11.cpp new file mode 100644 index 000000000..58bc21687 --- /dev/null +++ b/sdrgui/gui/glvideointeropd3d11.cpp @@ -0,0 +1,510 @@ +/////////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2026 Jon Beniston, M7RCE // +// Some code by AI // +// // +// This program is free software; you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation as version 3 of the License, or // +// (at your option) any later version. // +// // +// This program is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License V3 for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with this program. If not, see . // +/////////////////////////////////////////////////////////////////////////////////// + +// D3D11 frames (Windows, D3D11VA decoding): the GPU's video processor deinterlaces and converts each NV12 frame to RGB +// into a texture of the same device, which is copied to one WGL_NV_DX_interop2 shares with OpenGL. Nothing goes +// through the CPU + +#include "gui/glvideointerop.h" + +#if defined(_WIN32) && !defined(__MINGW32__) + +#include + +#include +#include +#include + +#include +#include +#include + +using Microsoft::WRL::ComPtr; + +namespace { + +typedef HANDLE (WINAPI *DXOpenDevice)(void *dxDevice); +typedef BOOL (WINAPI *DXCloseDevice)(HANDLE device); +typedef HANDLE (WINAPI *DXRegisterObject)(HANDLE device, void *dxObject, GLuint name, GLenum type, GLenum access); +typedef BOOL (WINAPI *DXUnregisterObject)(HANDLE device, HANDLE object); +typedef BOOL (WINAPI *DXLockObjects)(HANDLE device, GLint count, HANDLE *objects); +typedef BOOL (WINAPI *DXUnlockObjects)(HANDLE device, GLint count, HANDLE *objects); + +const GLenum accessReadOnly = 0x0000; // WGL_ACCESS_READ_ONLY_NV + +class GLVideoInteropD3D11 : public GLVideoInterop +{ +public: + GLVideoInteropD3D11() : + m_gl(nullptr), + m_interopDevice(nullptr), + m_width(0), + m_height(0), + m_pastFrames(0), + m_canDeinterlace(false), + m_glTexture(0), + m_glObject(nullptr), + m_locked(false), + m_acquired(false), + m_outputFrame(0), + m_lastTexture(nullptr) + { + } + + ~GLVideoInteropD3D11() override + { + closeDevice(); + } + + bool init(QOpenGLContext *context) + { + m_gl = context->functions(); + m_open = reinterpret_cast(context->getProcAddress("wglDXOpenDeviceNV")); + m_close = reinterpret_cast(context->getProcAddress("wglDXCloseDeviceNV")); + m_register = reinterpret_cast(context->getProcAddress("wglDXRegisterObjectNV")); + m_unregister = reinterpret_cast(context->getProcAddress("wglDXUnregisterObjectNV")); + m_dxLockObjects = reinterpret_cast(context->getProcAddress("wglDXLockObjectsNV")); + m_dxUnlockObjects = reinterpret_cast(context->getProcAddress("wglDXUnlockObjectsNV")); + return m_open && m_close && m_register && m_unregister && m_dxLockObjects && m_dxUnlockObjects; + } + + GLVideoFrame::Type type() const override { return GLVideoFrame::D3D11; } + int pastFrames() const override { return m_pastFrames; } + bool canDeinterlace() const override { return m_canDeinterlace; } + + bool acquire(const GLVideoFrame& frame, const GLVideoFrame *const *past, int nbPast, bool deinterlace, + GLuint textures[2], Layout& layout) override + { + ID3D11Device *device = static_cast(frame.m_d3d11Device); + ID3D11Texture2D *texture = static_cast(frame.m_d3d11Texture); + m_acquired = false; + + if (!device || !texture || !openDevice(device) || !setupProcessor(frame.m_width, frame.m_height)) { + return false; + } + + // If the video processor can't deinterlace, the frame's shown as it is: the renderer then has the decoder + // deinterlace in memory, while deinterlacing's on + const bool deinterlaceFrame = deinterlace && frame.m_interlaced && m_canDeinterlace; + + if (texture != m_lastTexture) + { + m_inputViews.clear(); // A new decoder: its texture array + m_lastTexture = texture; + } + + if (frame.m_lock) { + frame.m_lock(); + } + + bool ok = false; + ID3D11VideoProcessorInputView *input = inputView(texture, frame.m_d3d11Index); + ID3D11VideoProcessorInputView *pastViews[4]; + int nbPastViews = 0; + + for (int i = 0; deinterlaceFrame && (i < nbPast) && (i < 4); i++) + { + ID3D11Texture2D *pastTexture = static_cast(past[i]->m_d3d11Texture); + + if (pastTexture == texture) + { + ID3D11VideoProcessorInputView *view = inputView(pastTexture, past[i]->m_d3d11Index); + + if (view) { + pastViews[nbPastViews++] = view; + } + } + } + + if (input) + { + const D3D11_VIDEO_FRAME_FORMAT format = !deinterlaceFrame ? D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE + : frame.m_topFieldFirst ? D3D11_VIDEO_FRAME_FORMAT_INTERLACED_TOP_FIELD_FIRST : D3D11_VIDEO_FRAME_FORMAT_INTERLACED_BOTTOM_FIELD_FIRST; + m_videoContext->VideoProcessorSetStreamFrameFormat(m_processor.Get(), 0, format); + // Only the picture, unscaled: the decoder's textures (and so the past frames', of the same array) are + // larger, aligned to its blocks (as 1080 lines in 1088), and the padding would otherwise be scaled in + const RECT rect = {0, 0, (LONG) frame.m_width, (LONG) frame.m_height}; + m_videoContext->VideoProcessorSetStreamSourceRect(m_processor.Get(), 0, TRUE, &rect); + m_videoContext->VideoProcessorSetStreamDestRect(m_processor.Get(), 0, TRUE, &rect); + m_videoContext->VideoProcessorSetOutputTargetRect(m_processor.Get(), TRUE, &rect); + // A frame out for each frame in, as bwdif's send_frame + m_videoContext->VideoProcessorSetStreamOutputRate(m_processor.Get(), 0, D3D11_VIDEO_PROCESSOR_OUTPUT_RATE_HALF, FALSE, nullptr); + m_videoContext->VideoProcessorSetStreamAutoProcessingMode(m_processor.Get(), 0, FALSE); + + D3D11_VIDEO_PROCESSOR_COLOR_SPACE in = {}; + // BT.2020 needs ID3D11VideoContext1's DXGI colour spaces: BT.709 is the nearest of these + in.YCbCr_Matrix = frame.m_matrix == GLVideoFrame::BT601 ? 0 : 1; + in.Nominal_Range = frame.m_fullRange ? D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_0_255 : D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_16_235; + m_videoContext->VideoProcessorSetStreamColorSpace(m_processor.Get(), 0, &in); + D3D11_VIDEO_PROCESSOR_COLOR_SPACE out = {}; + out.RGB_Range = 0; // 0 to 255 + m_videoContext->VideoProcessorSetOutputColorSpace(m_processor.Get(), &out); + + D3D11_VIDEO_PROCESSOR_STREAM stream = {}; + stream.Enable = TRUE; + stream.InputFrameOrField = 0; + stream.PastFrames = nbPastViews; + stream.ppPastSurfaces = nbPastViews > 0 ? pastViews : nullptr; + stream.pInputSurface = input; + const HRESULT hr = m_videoContext->VideoProcessorBlt(m_processor.Get(), m_outputView.Get(), m_outputFrame++, 1, &stream); + ok = SUCCEEDED(hr); + + if (ok) { + m_context->CopyResource(m_shared.Get(), m_output.Get()); + } + + m_context->Flush(); + + if (!ok) { + qWarning("GLVideoInteropD3D11: VideoProcessorBlt failed: 0x%08lx", (unsigned long) hr); + } + } + + if (frame.m_unlock) { + frame.m_unlock(); + } + + if (!ok) { + return false; + } + + if (!m_dxLockObjects(m_interopDevice, 1, &m_glObject)) + { + qWarning("GLVideoInteropD3D11: wglDXLockObjectsNV failed"); + return false; + } + + m_locked = true; + m_acquired = true; + textures[0] = m_glTexture; + textures[1] = 0; + layout = LayoutRGBA; + return true; + } + + // The shared texture keeps the last frame processed + bool reacquire(GLuint textures[2], Layout& layout) override + { + if (!m_acquired || !m_glObject || m_locked) { + return false; + } + + if (!m_dxLockObjects(m_interopDevice, 1, &m_glObject)) + { + qWarning("GLVideoInteropD3D11: wglDXLockObjectsNV failed"); + return false; + } + + m_locked = true; + textures[0] = m_glTexture; + textures[1] = 0; + layout = LayoutRGBA; + return true; + } + + void release() override + { + if (m_locked) + { + m_dxUnlockObjects(m_interopDevice, 1, &m_glObject); + m_locked = false; + } + } + + // The input views hold the decoder's texture array, and the device (and the interop's) the decoder's device. + // Opened again for the next frame + void releaseFrames() override + { + closeDevice(); + } + + void abandon() override + { + // The registration, OpenGL texture and interop device go with the context. Then only Direct3D's references + m_locked = false; + m_glObject = nullptr; + m_glTexture = 0; + m_interopDevice = nullptr; + closeDevice(); + } + +private: + bool openDevice(ID3D11Device *device) + { + if (device == m_device.Get()) { + return m_interopDevice != nullptr; + } + + closeDevice(); + m_device = device; + m_interopDevice = m_open(device); + + if (!m_interopDevice) + { + qWarning("GLVideoInteropD3D11: wglDXOpenDeviceNV failed"); + return false; + } + + device->GetImmediateContext(&m_context); + + if (FAILED(device->QueryInterface(IID_PPV_ARGS(&m_videoDevice))) || FAILED(m_context.As(&m_videoContext))) + { + qWarning("GLVideoInteropD3D11: no video device"); + return false; + } + + return true; + } + + void closeProcessor() + { + release(); + + if (m_glObject) + { + m_unregister(m_interopDevice, m_glObject); + m_glObject = nullptr; + } + + if (m_glTexture) + { + m_gl->glDeleteTextures(1, &m_glTexture); + m_glTexture = 0; + } + + m_inputViews.clear(); + m_acquired = false; + m_canDeinterlace = false; + m_outputView.Reset(); + m_output.Reset(); + m_shared.Reset(); + m_processor.Reset(); + m_enumerator.Reset(); + m_width = 0; + m_height = 0; + } + + void closeDevice() + { + closeProcessor(); + + if (m_interopDevice) + { + m_close(m_interopDevice); + m_interopDevice = nullptr; + } + + m_videoContext.Reset(); + m_videoDevice.Reset(); + m_context.Reset(); + m_device.Reset(); + m_lastTexture = nullptr; + } + + // A video processor that can deinterlace, preferably adaptively (with previous frames), and its output texture, + // shared with OpenGL + bool setupProcessor(int width, int height) + { + if ((width == m_width) && (height == m_height) && m_processor) { + return true; + } + + closeProcessor(); + D3D11_VIDEO_PROCESSOR_CONTENT_DESC desc = {}; + desc.InputFrameFormat = D3D11_VIDEO_FRAME_FORMAT_INTERLACED_TOP_FIELD_FIRST; + desc.InputFrameRate = {25, 1}; + desc.InputWidth = width; + desc.InputHeight = height; + desc.OutputFrameRate = {25, 1}; + desc.OutputWidth = width; + desc.OutputHeight = height; + desc.Usage = D3D11_VIDEO_USAGE_PLAYBACK_NORMAL; + + if (FAILED(m_videoDevice->CreateVideoProcessorEnumerator(&desc, &m_enumerator))) { + return false; + } + + D3D11_VIDEO_PROCESSOR_CAPS caps; + m_enumerator->GetVideoProcessorCaps(&caps); + UINT index = 0; + int best = -1; + + for (UINT i = 0; i < caps.RateConversionCapsCount; i++) + { + D3D11_VIDEO_PROCESSOR_RATE_CONVERSION_CAPS rate; + + if (FAILED(m_enumerator->GetVideoProcessorRateConversionCaps(i, &rate))) { + continue; + } + + const int score = (rate.ProcessorCaps & D3D11_VIDEO_PROCESSOR_PROCESSOR_CAPS_DEINTERLACE_ADAPTIVE) ? 2 + : (rate.ProcessorCaps & D3D11_VIDEO_PROCESSOR_PROCESSOR_CAPS_DEINTERLACE_BOB) ? 1 : 0; + + if (score > best) + { + best = score; + index = i; + m_pastFrames = (int) std::min(rate.PastFrames, 4); + } + } + + if (FAILED(m_videoDevice->CreateVideoProcessor(m_enumerator.Get(), index, &m_processor))) { + return false; + } + + UINT support = 0; + DXGI_FORMAT format = DXGI_FORMAT_B8G8R8A8_UNORM; + + if (FAILED(m_enumerator->CheckVideoProcessorFormat(format, &support)) || !(support & D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT)) { + format = DXGI_FORMAT_R8G8B8A8_UNORM; + } + + D3D11_TEXTURE2D_DESC texture = {}; + texture.Width = width; + texture.Height = height; + texture.MipLevels = 1; + texture.ArraySize = 1; + texture.Format = format; + texture.SampleDesc.Count = 1; + texture.Usage = D3D11_USAGE_DEFAULT; + texture.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE; + + if (FAILED(m_device->CreateTexture2D(&texture, nullptr, &m_output))) { + return false; + } + + D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC view = {}; + view.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D; + + if (FAILED(m_videoDevice->CreateVideoProcessorOutputView(m_output.Get(), m_enumerator.Get(), &view, &m_outputView))) { + return false; + } + + // The video processor can't write to a texture registered with OpenGL: NVIDIA's driver (2026) crashes later + // (in its worker thread), and OpenGL sees nothing. So its output is copied to another texture, that is + texture.BindFlags = D3D11_BIND_SHADER_RESOURCE; + + if (FAILED(m_device->CreateTexture2D(&texture, nullptr, &m_shared))) { + return false; + } + + m_gl->glGenTextures(1, &m_glTexture); + m_glObject = m_register(m_interopDevice, m_shared.Get(), m_glTexture, GL_TEXTURE_2D, accessReadOnly); + + if (!m_glObject) + { + qWarning("GLVideoInteropD3D11: wglDXRegisterObjectNV failed"); + return false; + } + + m_gl->glBindTexture(GL_TEXTURE_2D, m_glTexture); + m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + m_gl->glBindTexture(GL_TEXTURE_2D, 0); + m_width = width; + m_height = height; + m_canDeinterlace = best > 0; + qDebug("GLVideoInteropD3D11: %dx%d, deinterlacing %s, %d past frames", width, height, + best == 2 ? "adaptive" : best == 1 ? "bob" : "none", m_pastFrames); + return true; + } + + // Input views of the decoder's texture array slices, made once each + ID3D11VideoProcessorInputView *inputView(ID3D11Texture2D *texture, int index) + { + auto it = m_inputViews.find(index); + + if (it != m_inputViews.end()) { + return it->second.Get(); + } + + D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC desc = {}; + desc.FourCC = 0; + desc.ViewDimension = D3D11_VPIV_DIMENSION_TEXTURE2D; + desc.Texture2D.MipSlice = 0; + desc.Texture2D.ArraySlice = index; + ComPtr view; + + if (FAILED(m_videoDevice->CreateVideoProcessorInputView(texture, m_enumerator.Get(), &desc, &view))) + { + qWarning("GLVideoInteropD3D11: CreateVideoProcessorInputView failed"); + return nullptr; + } + + m_inputViews[index] = view; + return view.Get(); + } + + QOpenGLFunctions *m_gl; + DXOpenDevice m_open = nullptr; + DXCloseDevice m_close = nullptr; + DXRegisterObject m_register = nullptr; + DXUnregisterObject m_unregister = nullptr; + DXLockObjects m_dxLockObjects = nullptr; //!< Of the interop's objects, unlike GLVideoFrame::m_lock (the decoder's) + DXUnlockObjects m_dxUnlockObjects = nullptr; + + ComPtr m_device; + ComPtr m_context; + ComPtr m_videoDevice; + ComPtr m_videoContext; + HANDLE m_interopDevice; + ComPtr m_enumerator; + ComPtr m_processor; + ComPtr m_output; + ComPtr m_shared; // A copy of m_output, shared with OpenGL + ComPtr m_outputView; + std::map> m_inputViews; + int m_width; + int m_height; + int m_pastFrames; + bool m_canDeinterlace; //!< Else interlaced frames that are to be deinterlaced are shown as they are + GLuint m_glTexture; + HANDLE m_glObject; + bool m_locked; + bool m_acquired; //!< m_shared has the last frame acquired + UINT m_outputFrame; + ID3D11Texture2D *m_lastTexture; +}; + +} // namespace + +std::unique_ptr createGLVideoInteropD3D11(QOpenGLContext *context) +{ + if (context->isOpenGLES()) { + return nullptr; + } + + std::unique_ptr interop(new GLVideoInteropD3D11()); + + if (!interop->init(context)) { + return nullptr; + } + + return std::unique_ptr(interop.release()); +} + +#else + +std::unique_ptr createGLVideoInteropD3D11(QOpenGLContext *) +{ + return nullptr; +} + +#endif diff --git a/sdrgui/gui/glvideointeropdmabuf.cpp b/sdrgui/gui/glvideointeropdmabuf.cpp new file mode 100644 index 000000000..268b4cc24 --- /dev/null +++ b/sdrgui/gui/glvideointeropdmabuf.cpp @@ -0,0 +1,356 @@ +/////////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2026 Jon Beniston, M7RCE // +// Some code by AI // +// // +// This program is free software; you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation as version 3 of the License, or // +// (at your option) any later version. // +// // +// This program is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License V3 for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with this program. If not, see . // +/////////////////////////////////////////////////////////////////////////////////// + +// DMA-BUF frames (Linux, VAAPI decoding, deinterlaced by deinterlace_vaapi, exported as DRM PRIME): each plane is +// imported as an EGLImage, bound to a texture, when its surface first comes, and kept for when it comes again. Needs +// an EGL context (Wayland, or X11 with QT_XCB_GL_INTEGRATION=xcb_egl); with GLX this isn't available, and frames are +// copied instead. libEGL is loaded when needed + +#include "gui/glvideointerop.h" + +#if defined(__linux__) && !defined(ANDROID) + +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +// The parts of EGL (egl.h, eglext.h) used +typedef void *EGLDisplay; +typedef void *EGLContext; +typedef void *EGLImageKHR; +typedef void *EGLClientBuffer; +typedef int32_t EGLint; +typedef unsigned int EGLenum; +typedef unsigned int EGLBoolean; + +const EGLint eglNone = 0x3038; +const EGLint eglWidth = 0x3057; +const EGLint eglHeight = 0x3056; +const EGLenum eglLinuxDmaBuf = 0x3270; +const EGLint eglLinuxDrmFourcc = 0x3271; +const EGLint eglDmaBufPlane0Fd = 0x3272; +const EGLint eglDmaBufPlane0Offset = 0x3273; +const EGLint eglDmaBufPlane0Pitch = 0x3274; +const EGLint eglDmaBufPlane0ModifierLo = 0x3443; +const EGLint eglDmaBufPlane0ModifierHi = 0x3444; + +// drm_fourcc.h +const uint64_t drmFormatModLinear = 0; +const uint64_t drmFormatModInvalid = 0x00ffffffffffffffULL; // Implicit: the driver's own layout + +typedef EGLDisplay (*GetCurrentDisplay)(); +typedef EGLContext (*GetCurrentContext)(); +typedef void *(*GetProcAddress)(const char *); +typedef const char *(*QueryString)(EGLDisplay, EGLint); +typedef EGLImageKHR (*CreateImage)(EGLDisplay, EGLContext, EGLenum, EGLClientBuffer, const EGLint *); +typedef EGLBoolean (*DestroyImage)(EGLDisplay, EGLImageKHR); +typedef void (*ImageTargetTexture2D)(GLenum, void *); + +const EGLint eglExtensions = 0x3055; + +class GLVideoInteropDMABuf : public GLVideoInterop +{ +public: + GLVideoInteropDMABuf() : + m_gl(nullptr), + m_display(nullptr), + m_pool(nullptr), + m_width(0), + m_height(0), + m_current(-1), + m_modifiers(false) + { + } + + ~GLVideoInteropDMABuf() override + { + releaseImages(); + } + + bool init(QOpenGLContext *context) + { + m_library.setFileNameAndVersion("EGL", 1); + + if (!m_library.load()) { + return false; + } + + GetCurrentDisplay getDisplay = reinterpret_cast(m_library.resolve("eglGetCurrentDisplay")); + GetCurrentContext getContext = reinterpret_cast(m_library.resolve("eglGetCurrentContext")); + GetProcAddress getProcAddress = reinterpret_cast(m_library.resolve("eglGetProcAddress")); + QueryString queryString = reinterpret_cast(m_library.resolve("eglQueryString")); + + // Only if Qt's context is EGL + if (!getDisplay || !getContext || !getProcAddress || !queryString || !getContext()) { + return false; + } + + m_display = getDisplay(); + const QByteArray extensions(queryString(m_display, eglExtensions)); + + if (!extensions.contains("EGL_EXT_image_dma_buf_import")) { + return false; + } + + m_modifiers = extensions.contains("EGL_EXT_image_dma_buf_import_modifiers"); + m_createImage = reinterpret_cast(getProcAddress("eglCreateImageKHR")); + m_destroyImage = reinterpret_cast(getProcAddress("eglDestroyImageKHR")); + m_imageTargetTexture = reinterpret_cast(getProcAddress("glEGLImageTargetTexture2DOES")); + + if (!m_createImage || !m_destroyImage || !m_imageTargetTexture) { + return false; + } + + m_gl = context->functions(); + return true; + } + + GLVideoFrame::Type type() const override { return GLVideoFrame::DMABuf; } + + // The planes of each surface of the pool are imported the first time it comes, and kept, as a decoder (or + // deinterlacer) cycles through a few dozen. Without the pool, they're imported for each frame + bool acquire(const GLVideoFrame& frame, const GLVideoFrame *const *, int, bool, GLuint textures[2], Layout& layout) override + { + if (!frame.m_dmaBufPool || (frame.m_dmaBufPool != m_pool) || (frame.m_width != m_width) || (frame.m_height != m_height) + || (m_surfaces.size() >= maxSurfaces)) + { + releaseImages(); // Another pool's, or the previous frame's, or more than a pool would have + m_pool = frame.m_dmaBufPool; + m_poolOwner = frame.m_dmaBufPoolOwner; + m_width = frame.m_width; + m_height = frame.m_height; + } + + m_current = -1; + + for (int i = 0; m_pool && (i < (int) m_surfaces.size()); i++) + { + if (m_surfaces[i].m_id == frame.m_dmaBufSurface) + { + m_current = i; + break; + } + } + + if (m_current < 0) + { + Surface surface; + + if (!import(frame, surface)) { + return false; + } + + m_surfaces.push_back(surface); + m_current = (int) m_surfaces.size() - 1; + } + + textures[0] = m_surfaces[m_current].m_textures[0]; + textures[1] = m_surfaces[m_current].m_textures[1]; + layout = LayoutNV12; + return true; + } + + // The images are kept, as drawing may not have finished with them + void release() override {} + + bool reacquire(GLuint textures[2], Layout& layout) override + { + if ((m_current < 0) || (m_current >= (int) m_surfaces.size())) { + return false; + } + + textures[0] = m_surfaces[m_current].m_textures[0]; + textures[1] = m_surfaces[m_current].m_textures[1]; + layout = LayoutNV12; + return true; + } + + // The images, and the textures they are bound to, hold the decoder's surfaces + void releaseFrames() override + { + releaseImages(); + } + + void abandon() override + { + // The textures go with the context. The images aren't the context's (nor need it current), so are destroyed + for (Surface& surface : m_surfaces) + { + for (EGLImageKHR& image : surface.m_images) + { + if (image) + { + m_destroyImage(m_display, image); + image = nullptr; + } + } + } + + m_gl = nullptr; + releaseImages(); + } + +private: + struct Surface + { + uint64_t m_id; + EGLImageKHR m_images[2]; + GLuint m_textures[2]; + }; + + static const size_t maxSurfaces = 64; + + // Each plane as an EGLImage, bound to a texture of its own + bool import(const GLVideoFrame& frame, Surface& surface) + { + // A tiled or compressed layout can't be described without the modifiers extension: imported as linear, the + // picture would be scrambled, so it's copied instead + if (frame.m_dmaBufHasModifier && !m_modifiers) + { + for (int plane = 0; plane < 2; plane++) + { + const uint64_t modifier = frame.m_dmaBufModifier[plane]; + + if ((modifier != drmFormatModLinear) && (modifier != drmFormatModInvalid)) + { + qWarning("GLVideoInteropDMABuf: plane %d has modifier 0x%llx, without EGL_EXT_image_dma_buf_import_modifiers", + plane, (unsigned long long) modifier); + return false; + } + } + } + + surface.m_id = frame.m_dmaBufSurface; + surface.m_images[0] = surface.m_images[1] = nullptr; + m_gl->glGenTextures(2, surface.m_textures); + + for (int plane = 0; plane < 2; plane++) + { + std::vector attributes = { + eglWidth, plane == 0 ? frame.m_width : (frame.m_width + 1) / 2, + eglHeight, plane == 0 ? frame.m_height : (frame.m_height + 1) / 2, + eglLinuxDrmFourcc, (EGLint) frame.m_dmaBufFourcc[plane], + eglDmaBufPlane0Fd, frame.m_dmaBufFd[plane], + eglDmaBufPlane0Offset, frame.m_dmaBufOffset[plane], + eglDmaBufPlane0Pitch, frame.m_dmaBufPitch[plane] + }; + + if (m_modifiers && frame.m_dmaBufHasModifier) + { + attributes.push_back(eglDmaBufPlane0ModifierLo); + attributes.push_back((EGLint) (frame.m_dmaBufModifier[plane] & 0xFFFFFFFF)); + attributes.push_back(eglDmaBufPlane0ModifierHi); + attributes.push_back((EGLint) (frame.m_dmaBufModifier[plane] >> 32)); + } + + attributes.push_back(eglNone); + surface.m_images[plane] = m_createImage(m_display, nullptr, eglLinuxDmaBuf, nullptr, attributes.data()); + + if (!surface.m_images[plane]) + { + qWarning("GLVideoInteropDMABuf: eglCreateImageKHR failed for plane %d", plane); + m_gl->glBindTexture(GL_TEXTURE_2D, 0); + destroy(surface); + return false; + } + + m_gl->glBindTexture(GL_TEXTURE_2D, surface.m_textures[plane]); + m_imageTargetTexture(GL_TEXTURE_2D, surface.m_images[plane]); + m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + m_gl->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + } + + m_gl->glBindTexture(GL_TEXTURE_2D, 0); + return true; + } + + // Deleting the textures and images doesn't free the memory while drawing still uses it + void destroy(Surface& surface) + { + m_gl->glDeleteTextures(2, surface.m_textures); + + for (EGLImageKHR& image : surface.m_images) + { + if (image) + { + m_destroyImage(m_display, image); + image = nullptr; + } + } + } + + void releaseImages() + { + if (m_gl) + { + for (Surface& surface : m_surfaces) { + destroy(surface); + } + } + + m_surfaces.clear(); + m_current = -1; + m_pool = nullptr; + m_poolOwner.reset(); + } + + QLibrary m_library; + QOpenGLFunctions *m_gl; + EGLDisplay m_display; + CreateImage m_createImage = nullptr; + DestroyImage m_destroyImage = nullptr; + ImageTargetTexture2D m_imageTargetTexture = nullptr; + std::vector m_surfaces; //!< Of m_pool, imported + const void *m_pool; //!< GLVideoFrame::m_dmaBufPool of the surfaces + std::shared_ptr m_poolOwner; //!< Keeps m_pool alive while its surfaces are kept + int m_width; //!< Of the surfaces + int m_height; + int m_current; //!< In m_surfaces, of the frame last acquired, -1 if none + bool m_modifiers; +}; + +} // namespace + +std::unique_ptr createGLVideoInteropDMABuf(QOpenGLContext *context) +{ + std::unique_ptr interop(new GLVideoInteropDMABuf()); + + if (!interop->init(context)) { + return nullptr; + } + + return std::unique_ptr(interop.release()); +} + +#else + +std::unique_ptr createGLVideoInteropDMABuf(QOpenGLContext *) +{ + return nullptr; +} + +#endif diff --git a/sdrgui/gui/glvideointeropiosurface.cpp b/sdrgui/gui/glvideointeropiosurface.cpp new file mode 100644 index 000000000..791c8ca70 --- /dev/null +++ b/sdrgui/gui/glvideointeropiosurface.cpp @@ -0,0 +1,145 @@ +/////////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2026 Jon Beniston, M7RCE // +// Some code by AI // +// // +// This program is free software; you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation as version 3 of the License, or // +// (at your option) any later version. // +// // +// This program is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License V3 for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with this program. If not, see . // +/////////////////////////////////////////////////////////////////////////////////// + +// IOSurface frames (macOS, VideoToolbox decoding): each NV12 plane is bound to a rectangle texture, without copying + +#include "gui/glvideointerop.h" + +#if defined(__APPLE__) + +#include +#include +#include +#include + +#include +#include +#include + +namespace { + +class GLVideoInteropIOSurface : public GLVideoInterop +{ +public: + GLVideoInteropIOSurface() : + m_gl(nullptr), + m_textures{} + { + } + + ~GLVideoInteropIOSurface() override + { + if (m_textures[0]) { + m_gl->glDeleteTextures(2, m_textures); + } + } + + bool init(QOpenGLContext *context) + { + m_gl = context->functions(); + m_gl->glGenTextures(2, m_textures); + return true; + } + + GLVideoFrame::Type type() const override { return GLVideoFrame::IOSurface; } + + bool acquire(const GLVideoFrame& frame, const GLVideoFrame *const *, int, bool, GLuint textures[2], Layout& layout) override + { + IOSurfaceRef surface = static_cast(frame.m_ioSurface); + CGLContextObj cgl = CGLGetCurrentContext(); + + if (!surface || !cgl || (IOSurfaceGetPlaneCount(surface) < 2)) { + return false; + } + + for (int plane = 0; plane < 2; plane++) + { + m_gl->glBindTexture(GL_TEXTURE_RECTANGLE, m_textures[plane]); + const CGLError error = CGLTexImageIOSurface2D(cgl, GL_TEXTURE_RECTANGLE, plane == 0 ? GL_R8 : GL_RG8, + (GLsizei) IOSurfaceGetWidthOfPlane(surface, plane), (GLsizei) IOSurfaceGetHeightOfPlane(surface, plane), + plane == 0 ? GL_RED : GL_RG, GL_UNSIGNED_BYTE, surface, plane); + + if (error != kCGLNoError) + { + qWarning("GLVideoInteropIOSurface: CGLTexImageIOSurface2D failed: %d", (int) error); + return false; + } + + m_gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + m_gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + m_gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + m_gl->glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + } + + m_gl->glBindTexture(GL_TEXTURE_RECTANGLE, 0); + textures[0] = m_textures[0]; + textures[1] = m_textures[1]; + layout = LayoutNV12Rect; + return true; + } + + // The textures hold the surface they were bound to + void releaseFrames() override + { + for (GLuint texture : m_textures) + { + m_gl->glBindTexture(GL_TEXTURE_RECTANGLE, texture); + m_gl->glTexImage2D(GL_TEXTURE_RECTANGLE, 0, GL_R8, 1, 1, 0, GL_RED, GL_UNSIGNED_BYTE, nullptr); + } + + m_gl->glBindTexture(GL_TEXTURE_RECTANGLE, 0); + } + + // The textures, and the surfaces bound to them, go with the context + void abandon() override + { + m_textures[0] = m_textures[1] = 0; + m_gl = nullptr; + } + +private: + QOpenGLFunctions *m_gl; + GLuint m_textures[2]; +}; + +} // namespace + +std::unique_ptr createGLVideoInteropIOSurface(QOpenGLContext *context) +{ + // Rectangle textures are sampled with GLSL 3.30, so a core profile, as SDRangel uses on macOS + if (context->format().profile() != QSurfaceFormat::CoreProfile) { + return nullptr; + } + + std::unique_ptr interop(new GLVideoInteropIOSurface()); + + if (!interop->init(context)) { + return nullptr; + } + + return std::unique_ptr(interop.release()); +} + +#else + +std::unique_ptr createGLVideoInteropIOSurface(QOpenGLContext *) +{ + return nullptr; +} + +#endif diff --git a/sdrgui/gui/glvideorenderer.cpp b/sdrgui/gui/glvideorenderer.cpp new file mode 100644 index 000000000..44875dba1 --- /dev/null +++ b/sdrgui/gui/glvideorenderer.cpp @@ -0,0 +1,764 @@ +/////////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2026 Jon Beniston, M7RCE // +// Some code by AI // +// // +// This program is free software; you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation as version 3 of the License, or // +// (at your option) any later version. // +// // +// This program is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License V3 for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with this program. If not, see . // +/////////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "gui/glvideorenderer.h" + +#ifndef GL_UNPACK_ROW_LENGTH +#define GL_UNPACK_ROW_LENGTH 0x0CF2 +#endif +#ifndef GL_RED +#define GL_RED 0x1903 +#endif +#ifndef GL_RG +#define GL_RG 0x8227 +#endif +#ifndef GL_R8 +#define GL_R8 0x8229 +#endif +#ifndef GL_RG8 +#define GL_RG8 0x822B +#endif +#ifndef GL_TEXTURE_RECTANGLE +#define GL_TEXTURE_RECTANGLE 0x84F5 +#endif +#ifndef GL_LUMINANCE +#define GL_LUMINANCE 0x1909 +#endif +#ifndef GL_LUMINANCE_ALPHA +#define GL_LUMINANCE_ALPHA 0x190A +#endif + +// A quad over the viewport: position, then texture coordinate, with the top of the image at the top +static const GLfloat quad[] = { + -1.0f, -1.0f, 0.0f, 1.0f, + 1.0f, -1.0f, 1.0f, 1.0f, + -1.0f, 1.0f, 0.0f, 0.0f, + 1.0f, 1.0f, 1.0f, 0.0f +}; + +GLVideoRenderer::GLVideoRenderer() : + m_initialized(false), + m_core(false), + m_gles(false), + m_rg(false), + m_programs{}, + m_programFailed{}, + m_vertices(nullptr), + m_vao(nullptr), + m_textures{}, + m_textureWidth{}, + m_textureHeight{}, + m_textureFormat{}, + m_imageTextures{}, + m_imageKeys{-1, -1, -1}, + m_imageDrawn{}, + m_imageDraws(0), + m_subtitleTexture(0), + m_supported((1u << GLVideoFrame::Planar420) | (1u << GLVideoFrame::NV12)), + m_noDeinterlace(0), + m_deinterlacing(false), + m_lastDeinterlace(false) +{ +} + +GLVideoRenderer::~GLVideoRenderer() +{ + // cleanup() should have been called with the context current. If not, GL objects are left to the context, which + // may not be current or may have gone, so the interops let go of the rest without calling OpenGL. Before the + // frames they may use go + if (m_initialized) + { + qWarning("GLVideoRenderer::~GLVideoRenderer: cleanup() wasn't called"); + + for (auto& interop : m_interops) + { + if (interop) + { + interop->abandon(); + interop.reset(); + } + } + } + + for (QOpenGLShaderProgram *&p : m_programs) + { + delete p; + p = nullptr; + } + + delete m_vertices; + delete m_vao; +} + +void GLVideoRenderer::initialize() +{ + if (m_initialized) { + return; + } + + QOpenGLContext *context = QOpenGLContext::currentContext(); + initializeOpenGLFunctions(); + const QSurfaceFormat format = context->format(); + m_gles = context->isOpenGLES(); + m_core = m_gles ? (format.majorVersion() >= 3) : (format.profile() == QSurfaceFormat::CoreProfile); + m_rg = m_core || (!m_gles && (format.majorVersion() >= 3)) + || context->hasExtension(m_gles ? "GL_EXT_texture_rg" : "GL_ARB_texture_rg"); + + m_vao = new QOpenGLVertexArrayObject(); + + if (!m_vao->create()) + { + delete m_vao; + m_vao = nullptr; // Compatibility contexts may not have them, and don't need them + } + + if (m_vao) { + m_vao->bind(); + } + + m_vertices = new QOpenGLBuffer(QOpenGLBuffer::VertexBuffer); + m_vertices->create(); + m_vertices->bind(); + m_vertices->allocate(quad, sizeof(quad)); + m_vertices->release(); + + if (m_vao) { + m_vao->release(); + } + + glGenTextures(3, m_textures); + glGenTextures(m_nbImages, m_imageTextures); + glGenTextures(1, &m_subtitleTexture); + + // GPU frames: those this platform, driver and context can import + unsigned int supported = 0; + m_interops[GLVideoFrame::D3D11] = createGLVideoInteropD3D11(context); + m_interops[GLVideoFrame::IOSurface] = createGLVideoInteropIOSurface(context); + + // Their UV planes are two channel (RG) textures, which the NV12 shader samples as such only with m_rg + if (m_rg) + { + m_interops[GLVideoFrame::CUDA] = createGLVideoInteropCUDA(context); + m_interops[GLVideoFrame::DMABuf] = createGLVideoInteropDMABuf(context); + } + + for (int t = 0; t < GLVideoFrame::NbTypes; t++) + { + if (m_interops[t]) { + supported |= 1u << t; + } + } + + supported |= (1u << GLVideoFrame::Planar420) | (1u << GLVideoFrame::NV12); + m_supported = supported; + m_initialized = true; + qDebug("GLVideoRenderer::initialize: OpenGL%s %d.%d %s, GPU frames:%s%s%s%s", m_gles ? " ES" : "", + format.majorVersion(), format.minorVersion(), m_core ? "core" : "compatibility", + m_interops[GLVideoFrame::D3D11] ? " D3D11" : "", m_interops[GLVideoFrame::CUDA] ? " CUDA" : "", + m_interops[GLVideoFrame::DMABuf] ? " DMA-BUF" : "", m_interops[GLVideoFrame::IOSurface] ? " IOSurface" : ""); +} + +void GLVideoRenderer::cleanup() +{ + if (!m_initialized) { + return; + } + + // The interops first, while the frames they may use are still alive + for (auto& interop : m_interops) { + interop.reset(); + } + + m_past.clear(); + m_lastFrame.reset(); + + m_supported = (1u << GLVideoFrame::Planar420) | (1u << GLVideoFrame::NV12); + m_noDeinterlace = 0; + glDeleteTextures(3, m_textures); + glDeleteTextures(m_nbImages, m_imageTextures); + glDeleteTextures(1, &m_subtitleTexture); + // What was in them: the textures made for another context are allocated again + std::fill(std::begin(m_textures), std::end(m_textures), 0); + std::fill(std::begin(m_textureWidth), std::end(m_textureWidth), 0); + std::fill(std::begin(m_textureHeight), std::end(m_textureHeight), 0); + std::fill(std::begin(m_textureFormat), std::end(m_textureFormat), 0); + std::fill(std::begin(m_imageTextures), std::end(m_imageTextures), 0); + std::fill(std::begin(m_imageKeys), std::end(m_imageKeys), -1); + m_subtitleTexture = 0; + m_subtitle.reset(); + + for (QOpenGLShaderProgram *&p : m_programs) + { + delete p; + p = nullptr; + } + + std::fill(std::begin(m_programFailed), std::end(m_programFailed), false); // Another context's may compile + delete m_vertices; + m_vertices = nullptr; + delete m_vao; + m_vao = nullptr; + m_initialized = false; +} + +void GLVideoRenderer::releaseFrames() +{ + m_past.clear(); + m_lastFrame.reset(); + m_noDeinterlace = 0; // The interops' devices are let go of too: the next may deinterlace + + if (!m_initialized) { + return; + } + + for (auto& interop : m_interops) + { + if (interop) { + interop->releaseFrames(); + } + } +} + +bool GLVideoRenderer::canShow(GLVideoFrame::Type type) const +{ + unsigned int supported = m_supported.load(); + + if (m_deinterlacing.load()) { + supported &= ~m_noDeinterlace.load(); + } + + return (supported >> type) & 1u; +} + +// Shaders: GLSL 3.30 core (or 3.00 ES), else 1.20 for compatibility contexts of older drivers. Null if it doesn't +// compile, which is only tried once (and warned of) +QOpenGLShaderProgram *GLVideoRenderer::program(Program which) +{ + if (m_programs[which] || m_programFailed[which]) { + return m_programs[which]; + } + + QString header; + + if (m_gles) { + header = m_core ? "#version 300 es\nprecision mediump float;\n#define IN in\n#define OUT out\n" + : "#version 100\nprecision mediump float;\n#define IN varying\n#define OUT varying\n"; + } else { + header = m_core ? "#version 330 core\n#define IN in\n#define OUT out\n" : "#version 120\n#define IN varying\n#define OUT varying\n"; + } + + const bool modern = m_core; + const QString vertex = header + (modern ? "in vec2 position;\nin vec2 texCoord;\n" : "attribute vec2 position;\nattribute vec2 texCoord;\n") + + "OUT vec2 vTexCoord;\n" + "void main() {\n" + " vTexCoord = texCoord;\n" + " gl_Position = vec4(position, 0.0, 1.0);\n" + "}\n"; + + QString fragment = header; + + if (modern) { + fragment += "out vec4 fragColor;\n#define TEX texture\n"; + } else { + fragment += "#define fragColor gl_FragColor\n#define TEX texture2D\n"; + } + + // YUV to RGB: remove the offsets and scale to 0..1 (luma) and -0.5..0.5 (chroma), then the matrix. Chroma is + // sampled at vTexCoord * cScale + cOffset, for where its samples are (see setYUVUniforms) + const QString yuv = + "uniform mat3 yuvMatrix;\n" + "uniform vec2 yRange;\n" + "uniform vec2 cRange;\n" + "uniform vec2 cScale;\n" + "uniform vec2 cOffset;\n" + "vec4 rgb(float y, vec2 c) {\n" + " vec3 v = vec3((y - yRange.x) * yRange.y, (c - vec2(cRange.x)) * cRange.y);\n" + " return vec4(clamp(yuvMatrix * v, 0.0, 1.0), 1.0);\n" + "}\n"; + const QString uv = m_rg ? "rg" : "ra"; + + switch (which) + { + case ProgramPlanar: + fragment += "IN vec2 vTexCoord;\nuniform sampler2D tex0;\nuniform sampler2D tex1;\nuniform sampler2D tex2;\n" + yuv + + "void main() {\n" + " vec2 c = vTexCoord * cScale + cOffset;\n" + " fragColor = rgb(TEX(tex0, vTexCoord).r, vec2(TEX(tex1, c).r, TEX(tex2, c).r));\n" + "}\n"; + break; + case ProgramNV12: + fragment += "IN vec2 vTexCoord;\nuniform sampler2D tex0;\nuniform sampler2D tex1;\n" + yuv + + "void main() {\n" + " fragColor = rgb(TEX(tex0, vTexCoord).r, TEX(tex1, vTexCoord * cScale + cOffset)." + uv + ");\n" + "}\n"; + break; + case ProgramNV12Rect: + // Rectangle textures are sampled in pixels (macOS IOSurfaces), which may be larger than the picture + fragment += "IN vec2 vTexCoord;\nuniform sampler2DRect tex0;\nuniform sampler2DRect tex1;\nuniform vec2 size0;\n" + yuv + + "void main() {\n" + " fragColor = rgb(texture(tex0, vTexCoord * size0).r, texture(tex1, vTexCoord * cScale + cOffset).rg);\n" + "}\n"; + break; + default: + fragment += "IN vec2 vTexCoord;\nuniform sampler2D tex0;\n" + "void main() {\n" + " fragColor = TEX(tex0, vTexCoord);\n" + "}\n"; + break; + } + + QOpenGLShaderProgram *p = new QOpenGLShaderProgram(); + bool ok = p->addShaderFromSourceCode(QOpenGLShader::Vertex, vertex) + && p->addShaderFromSourceCode(QOpenGLShader::Fragment, fragment); + + if (ok) + { + p->bindAttributeLocation("position", 0); + p->bindAttributeLocation("texCoord", 1); + ok = p->link(); + } + + if (!ok) + { + qWarning() << "GLVideoRenderer::program: shader" << which << "failed:" << p->log(); + delete p; + m_programFailed[which] = true; + return nullptr; + } + + p->bind(); + p->setUniformValue("tex0", 0); + p->setUniformValue("tex1", 1); + p->setUniformValue("tex2", 2); + p->release(); + m_programs[which] = p; + return p; +} + +// pixels: the chroma texture is sampled in pixels (a rectangle texture), else is (width + 1) / 2 x (height + 1) / 2 +void GLVideoRenderer::setYUVUniforms(QOpenGLShaderProgram *p, const GLVideoFrame& frame, bool pixels) +{ + // From the luma coefficients: R = Y + 2 (1 - Kr) Cr, B = Y + 2 (1 - Kb) Cb, G = (Y - Kr R - Kb B) / Kg + float kr, kb; + + switch (frame.m_matrix) + { + case GLVideoFrame::BT709: + kr = 0.2126f; + kb = 0.0722f; + break; + case GLVideoFrame::BT2020: + kr = 0.2627f; + kb = 0.0593f; + break; + default: + kr = 0.299f; + kb = 0.114f; + break; + } + + const float kg = 1.0f - kr - kb; + const float values[9] = { + 1.0f, 0.0f, 2.0f * (1.0f - kr), + 1.0f, -2.0f * kb * (1.0f - kb) / kg, -2.0f * kr * (1.0f - kr) / kg, + 1.0f, 2.0f * (1.0f - kb), 0.0f + }; + p->setUniformValue("yuvMatrix", QMatrix3x3(values)); + + // Chroma sample i is at luma sample 2i (left sited) or 2i + 0.5 (centre), so luma at u (0 to 1 across the + // picture) has chroma at u * width / 2 + 0.25 or u * width / 2 chroma texels (texel centres at i + 0.5). + // Likewise vertically, where only top left is co-sited + const float cw = pixels ? 1.0f : (float) ((frame.m_width + 1) / 2); + const float ch = pixels ? 1.0f : (float) ((frame.m_height + 1) / 2); + const bool left = frame.m_chromaLocation != GLVideoFrame::ChromaCenter; + const bool top = frame.m_chromaLocation == GLVideoFrame::ChromaTopLeft; + p->setUniformValue("cScale", QVector2D(frame.m_width / (2.0f * cw), frame.m_height / (2.0f * ch))); + p->setUniformValue("cOffset", QVector2D(left ? 0.25f / cw : 0.0f, top ? 0.25f / ch : 0.0f)); + + if (frame.m_fullRange) + { + p->setUniformValue("yRange", QVector2D(0.0f, 1.0f)); + p->setUniformValue("cRange", QVector2D(0.5f, 1.0f)); + } + else + { + p->setUniformValue("yRange", QVector2D(16.0f / 255.0f, 255.0f / 219.0f)); + p->setUniformValue("cRange", QVector2D(128.0f / 255.0f, 255.0f / 224.0f)); + } +} + +// A plane in memory into one of the textures: one channel (Y, U, V) or two (interleaved UV) +void GLVideoRenderer::uploadPlane(int unit, int index, const uint8_t *data, int linesize, int width, int height, bool twoChannels) +{ + glActiveTexture(GL_TEXTURE0 + unit); + glBindTexture(GL_TEXTURE_2D, m_textures[index]); + const GLenum format = twoChannels ? (m_rg ? GL_RG : GL_LUMINANCE_ALPHA) : (m_rg ? GL_RED : GL_LUMINANCE); + // OpenGL ES 2's internal format is the format (GL_EXT_texture_rg has no sized ones for glTexImage2D) + const GLint internal = m_gles && !m_core ? (GLint) format + : twoChannels ? (m_rg ? GL_RG8 : GL_LUMINANCE_ALPHA) : (m_rg ? GL_R8 : GL_LUMINANCE); + const int bytesPerPixel = twoChannels ? 2 : 1; + const bool rowLength = !m_gles || m_core; + glPixelStorei(GL_UNPACK_ALIGNMENT, 1); + + if (rowLength) + { + glPixelStorei(GL_UNPACK_ROW_LENGTH, linesize / bytesPerPixel); + } + else if (linesize != width * bytesPerPixel) + { + // OpenGL ES 2 can't skip the padding at the ends of rows, so the rows are packed together + const size_t row = (size_t) width * bytesPerPixel; + m_repack.resize(row * height); + + for (int y = 0; y < height; y++) { + std::memcpy(m_repack.data() + row * y, data + (ptrdiff_t) linesize * y, row); + } + + data = m_repack.data(); + } + + if ((m_textureWidth[index] != width) || (m_textureHeight[index] != height) || (m_textureFormat[index] != internal)) + { + glTexImage2D(GL_TEXTURE_2D, 0, internal, width, height, 0, format, GL_UNSIGNED_BYTE, data); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + m_textureWidth[index] = width; + m_textureHeight[index] = height; + m_textureFormat[index] = internal; + } + else + { + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, format, GL_UNSIGNED_BYTE, data); + } + + if (rowLength) { + glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); + } +} + +void GLVideoRenderer::drawQuad(QOpenGLShaderProgram *p) +{ + if (m_vao) { + m_vao->bind(); + } + + m_vertices->bind(); + p->enableAttributeArray(0); + p->enableAttributeArray(1); + p->setAttributeBuffer(0, GL_FLOAT, 0, 2, 4 * sizeof(GLfloat)); + p->setAttributeBuffer(1, GL_FLOAT, 2 * sizeof(GLfloat), 2, 4 * sizeof(GLfloat)); + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); + p->disableAttributeArray(0); + p->disableAttributeArray(1); + m_vertices->release(); + + if (m_vao) { + m_vao->release(); + } +} + +QRect GLVideoRenderer::videoRect(const GLVideoFrame& frame, int width, int height, bool keepAspect) +{ + if (!keepAspect || (frame.m_width <= 0) || (frame.m_height <= 0)) { + return QRect(0, 0, width, height); + } + + const double aspect = frame.m_width * (frame.m_sampleAspectRatio > 0.0 ? frame.m_sampleAspectRatio : 1.0) / frame.m_height; + int w = width, h = (int) std::lround(width / aspect); + + if (h > height) + { + h = height; + w = (int) std::lround(height * aspect); + } + + return QRect((width - w) / 2, (height - h) / 2, w, h); +} + +QRect GLVideoRenderer::subtitleRect(const GLVideoSubtitle& subtitle, const QRect& videoRect) +{ + const double sx = videoRect.width() / (double) std::max(1, subtitle.m_canvasWidth); + const double sy = videoRect.height() / (double) std::max(1, subtitle.m_canvasHeight); + const int x0 = videoRect.x() + (int) std::lround(subtitle.m_x * sx); + const int y0 = videoRect.y() + (int) std::lround(subtitle.m_y * sy); + const int x1 = videoRect.x() + (int) std::lround((subtitle.m_x + subtitle.m_width) * sx); + const int y1 = videoRect.y() + (int) std::lround((subtitle.m_y + subtitle.m_height) * sy); + return QRect(x0, y0, x1 - x0, y1 - y0); +} + +bool GLVideoRenderer::drawFrame(const GLVideoFramePtr& frame, int width, int height, bool keepAspect, bool deinterlace) +{ + if (!m_initialized || !frame || (frame->m_width <= 0) || (frame->m_height <= 0)) { + return false; + } + + m_deinterlacing = deinterlace; + const QRect r = videoRect(*frame, width, height, keepAspect); + glViewport(r.x(), height - r.y() - r.height(), r.width(), r.height()); + glDisable(GL_BLEND); + QOpenGLShaderProgram *p = nullptr; + GLVideoInterop *interop = nullptr; + bool ok = true; + // Drawn again, as when the overlay changes: its textures still have it + const bool again = (m_lastFrame.lock() == frame) && (deinterlace == m_lastDeinterlace); + + if ((frame->m_type == GLVideoFrame::Planar420) || (frame->m_type == GLVideoFrame::NV12)) + { + const bool planar = frame->m_type == GLVideoFrame::Planar420; + p = program(planar ? ProgramPlanar : ProgramNV12); + + if (!p) { + return false; + } + + if (again) + { + for (int i = 0; i < (planar ? 3 : 2); i++) + { + glActiveTexture(GL_TEXTURE0 + i); + glBindTexture(GL_TEXTURE_2D, m_textures[i]); + } + } + else if (planar) + { + uploadPlane(0, 0, frame->m_data[0], frame->m_linesize[0], frame->m_width, frame->m_height, false); + uploadPlane(1, 1, frame->m_data[1], frame->m_linesize[1], (frame->m_width + 1) / 2, (frame->m_height + 1) / 2, false); + uploadPlane(2, 2, frame->m_data[2], frame->m_linesize[2], (frame->m_width + 1) / 2, (frame->m_height + 1) / 2, false); + } + else + { + uploadPlane(0, 0, frame->m_data[0], frame->m_linesize[0], frame->m_width, frame->m_height, false); + uploadPlane(1, 1, frame->m_data[1], frame->m_linesize[1], (frame->m_width + 1) / 2, (frame->m_height + 1) / 2, true); + } + + p->bind(); + setYUVUniforms(p, *frame, false); + } + else + { + interop = m_interops[frame->m_type].get(); + + // Not canShow(), as frames the GPU can't deinterlace are still shown, until the decoder deinterlaces them + if (!interop || !((m_supported.load() >> frame->m_type) & 1u)) { + return false; + } + + // Previous frames of the same type, for deinterlacing + while (!m_past.empty() && (m_past.front()->m_type != frame->m_type)) { + m_past.clear(); + } + + GLuint textures[2] = {0, 0}; + GLVideoInterop::Layout layout; + + if (!again || !interop->reacquire(textures, layout)) + { + // The frame itself isn't one of its previous frames, if it's drawn again + const int first = !m_past.empty() && (m_past.front() == frame) ? 1 : 0; + const GLVideoFrame *past[4]; + const int nbPast = std::min((int) m_past.size() - first, std::min(4, interop->pastFrames())); + + for (int i = 0; i < nbPast; i++) { + past[i] = m_past[first + i].get(); + } + + if (!interop->acquire(*frame, past, nbPast, deinterlace, textures, layout)) + { + qWarning("GLVideoRenderer::drawFrame: GPU frames of type %d can't be shown: falling back", frame->m_type); + m_supported &= ~(1u << frame->m_type); + m_past.clear(); + m_lastFrame.reset(); + return false; + } + + const bool deinterlaceFrame = deinterlace && frame->m_interlaced; + + // Shown as it is: the decoder deinterlaces them in memory instead, while deinterlacing's on + if (deinterlaceFrame && !interop->canDeinterlace() && !((m_noDeinterlace.load() >> frame->m_type) & 1u)) + { + qWarning("GLVideoRenderer::drawFrame: GPU frames of type %d can't be deinterlaced: falling back while deinterlacing", frame->m_type); + m_noDeinterlace |= 1u << frame->m_type; + } + + // Previous frames are kept (holding the decoder's surfaces) only while they're used + if (!deinterlaceFrame || !interop->canDeinterlace() || (interop->pastFrames() <= 0)) + { + m_past.clear(); + } + else if (first == 0) + { + m_past.push_front(frame); + + while ((int) m_past.size() > interop->pastFrames()) { + m_past.pop_back(); + } + } + } + + const GLenum target = layout == GLVideoInterop::LayoutNV12Rect ? GL_TEXTURE_RECTANGLE : GL_TEXTURE_2D; + glActiveTexture(GL_TEXTURE0); + glBindTexture(target, textures[0]); + + if (layout != GLVideoInterop::LayoutRGBA) + { + glActiveTexture(GL_TEXTURE1); + glBindTexture(target, textures[1]); + } + + p = program(layout == GLVideoInterop::LayoutRGBA ? ProgramRGBA : layout == GLVideoInterop::LayoutNV12 ? ProgramNV12 : ProgramNV12Rect); + + // As if it couldn't be acquired: the decoder copies them to memory instead + if (!p) + { + qWarning("GLVideoRenderer::drawFrame: GPU frames of type %d can't be shown: falling back", frame->m_type); + interop->release(); + m_supported &= ~(1u << frame->m_type); + m_past.clear(); + m_lastFrame.reset(); + return false; + } + + p->bind(); + + if (layout != GLVideoInterop::LayoutRGBA) { + setYUVUniforms(p, *frame, layout == GLVideoInterop::LayoutNV12Rect); + } + + if (layout == GLVideoInterop::LayoutNV12Rect) { + p->setUniformValue("size0", QVector2D(frame->m_width, frame->m_height)); + } + } + + m_lastFrame = frame; + m_lastDeinterlace = deinterlace; + drawQuad(p); + p->release(); + + if (interop) { + interop->release(); + } + + glActiveTexture(GL_TEXTURE0); + glViewport(0, 0, width, height); + + if (frame->m_subtitle) { + drawSubtitle(frame->m_subtitle, r, width, height); + } + + return ok; +} + +// Uploaded once, and drawn with every frame it's attached to +void GLVideoRenderer::drawSubtitle(const GLVideoSubtitlePtr& subtitle, const QRect& videoRect, int width, int height) +{ + if ((subtitle->m_width <= 0) || (subtitle->m_height <= 0) + || (subtitle->m_rgba.size() < (size_t) subtitle->m_width * subtitle->m_height * 4)) { + return; + } + + if (subtitle != m_subtitle) + { + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, m_subtitleTexture); + glPixelStorei(GL_UNPACK_ALIGNMENT, 4); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, subtitle->m_width, subtitle->m_height, 0, GL_RGBA, GL_UNSIGNED_BYTE, subtitle->m_rgba.data()); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + m_subtitle = subtitle; + } + + drawTexture(m_subtitleTexture, subtitleRect(*subtitle, videoRect), width, height); +} + +void GLVideoRenderer::drawImage(const QImage& image, const QRect& rect, int width, int height) +{ + if (!m_initialized || image.isNull()) { + return; + } + + // Its texture, if it's in one, else the least recently drawn's + const qint64 key = image.cacheKey(); + int index = 0; + + for (int i = 0; i < m_nbImages; i++) + { + if (m_imageKeys[i] == key) + { + index = i; + break; + } + + if (m_imageDrawn[i] < m_imageDrawn[index]) { + index = i; + } + } + + if (m_imageKeys[index] != key) + { + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, m_imageTextures[index]); + const QImage rgba = image.convertToFormat(QImage::Format_RGBA8888_Premultiplied); + glPixelStorei(GL_UNPACK_ALIGNMENT, 4); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, rgba.width(), rgba.height(), 0, GL_RGBA, GL_UNSIGNED_BYTE, rgba.constBits()); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + m_imageKeys[index] = key; + } + + m_imageDrawn[index] = ++m_imageDraws; + drawTexture(m_imageTextures[index], rect, width, height); +} + +// Premultiplied RGBA, blended +void GLVideoRenderer::drawTexture(GLuint texture, const QRect& rect, int width, int height) +{ + QOpenGLShaderProgram *p = program(ProgramRGBA); + + if (!p) { + return; + } + + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, texture); + glViewport(rect.x(), height - rect.y() - rect.height(), rect.width(), rect.height()); + glEnable(GL_BLEND); + glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA); + p->bind(); + drawQuad(p); + p->release(); + glDisable(GL_BLEND); + glViewport(0, 0, width, height); +} diff --git a/sdrgui/gui/glvideorenderer.h b/sdrgui/gui/glvideorenderer.h new file mode 100644 index 000000000..4618e0eac --- /dev/null +++ b/sdrgui/gui/glvideorenderer.h @@ -0,0 +1,109 @@ +/////////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2026 Jon Beniston, M7RCE // +// Some code by AI // +// // +// This program is free software; you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation as version 3 of the License, or // +// (at your option) any later version. // +// // +// This program is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License V3 for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with this program. If not, see . // +/////////////////////////////////////////////////////////////////////////////////// + +#ifndef SDRGUI_GUI_GLVIDEORENDERER_H_ +#define SDRGUI_GUI_GLVIDEORENDERER_H_ + +#include +#include +#include +#include + +#include +#include +#include + +#include "gui/glvideoframe.h" +#include "gui/glvideointerop.h" +#include "export.h" + +class QOpenGLShaderProgram; +class QOpenGLBuffer; +class QOpenGLVertexArrayObject; + +// Draws video frames with OpenGL, in the current context: YUV 4:2:0 frames in memory are uploaded a plane at a time, +// and frames on the GPU (D3D11, CUDA, DMA-BUF, IOSurface) are imported without copying through the CPU, where the +// platform and driver allow. Either way, YUV is converted to RGB in a shader, so the CPU doesn't convert pixels. +// Separate from GLVideoWidget, so it can be tested with an offscreen context +class SDRGUI_API GLVideoRenderer : protected QOpenGLFunctions +{ +public: + GLVideoRenderer(); + ~GLVideoRenderer(); + + void initialize(); //!< With the context current + void cleanup(); //!< With the context current, before it's destroyed + //! With the context current: let go of the frames held, for deinterlacing and by the GPU interops, so a decoder's + //! surfaces aren't held while nothing is shown + void releaseFrames(); + //! If GPU frames of the type can be shown. From any thread. False once one has failed to be shown, or while + //! deinterlacing if the GPU can't deinterlace them, so the decoder deinterlaces them in memory instead + bool canShow(GLVideoFrame::Type type) const; + //! Draw the frame into the current framebuffer (width x height pixels), letterboxed to its aspect ratio if + //! keepAspect, deinterlacing GPU frames that need it if deinterlace. False if it couldn't be shown + bool drawFrame(const GLVideoFramePtr& frame, int width, int height, bool keepAspect, bool deinterlace); + //! Draw an image, blended, at rect (pixels from the top left) of the framebuffer of width x height pixels + void drawImage(const QImage& image, const QRect& rect, int width, int height); + //! Where the frame is drawn + static QRect videoRect(const GLVideoFrame& frame, int width, int height, bool keepAspect); + //! Where a subtitle is drawn, over the picture drawn at videoRect + static QRect subtitleRect(const GLVideoSubtitle& subtitle, const QRect& videoRect); + +private: + enum Program { ProgramPlanar, ProgramNV12, ProgramNV12Rect, ProgramRGBA, NbPrograms }; + + QOpenGLShaderProgram *program(Program program); + void uploadPlane(int unit, int index, const uint8_t *data, int linesize, int width, int height, bool twoChannels); + void drawQuad(QOpenGLShaderProgram *program); + void setYUVUniforms(QOpenGLShaderProgram *program, const GLVideoFrame& frame, bool pixels); + void drawSubtitle(const GLVideoSubtitlePtr& subtitle, const QRect& videoRect, int width, int height); + void drawTexture(GLuint texture, const QRect& rect, int width, int height); + + bool m_initialized; + bool m_core; //!< Core profile (or GLES 3), so GLSL 3.30 (300 es) and R8/RG8 textures + bool m_gles; + bool m_rg; //!< R8/RG8 textures, else luminance (alpha) + QOpenGLShaderProgram *m_programs[NbPrograms]; + bool m_programFailed[NbPrograms]; //!< To compile or link, so it isn't tried again + QOpenGLBuffer *m_vertices; + QOpenGLVertexArrayObject *m_vao; + GLuint m_textures[3]; //!< Planes uploaded from memory + int m_textureWidth[3]; + int m_textureHeight[3]; + int m_textureFormat[3]; //!< Internal format, as NV12's UV plane is two channels + // Images drawn over or instead of the picture (a message, an overlay, a guide), each in a texture of its own, so + // those drawn together aren't uploaded again for each paint. The least recently drawn's is reused for another + static const int m_nbImages = 3; + GLuint m_imageTextures[m_nbImages]; + qint64 m_imageKeys[m_nbImages]; //!< QImage::cacheKey() of the image in each, -1 if none + quint64 m_imageDrawn[m_nbImages]; //!< When each was last drawn, in m_imageDraws + quint64 m_imageDraws; + GLuint m_subtitleTexture; + GLVideoSubtitlePtr m_subtitle; //!< In m_subtitleTexture (held, so a new one can't have the same address) + std::unique_ptr m_interops[GLVideoFrame::NbTypes]; + std::atomic m_supported; //!< Bit per GLVideoFrame::Type + std::atomic m_noDeinterlace; //!< Bit per GLVideoFrame::Type the GPU has been found not to deinterlace + std::atomic m_deinterlacing; //!< As frames were last drawn + std::deque m_past; //!< Previous GPU frames, for deinterlacing + //! The frame last drawn, which a repaint (as for an overlay) draws again without uploading or importing it again + std::weak_ptr m_lastFrame; + bool m_lastDeinterlace; + std::vector m_repack; //!< Rows of a plane without padding, for OpenGL ES 2 +}; + +#endif // SDRGUI_GUI_GLVIDEORENDERER_H_ diff --git a/sdrgui/gui/glvideowidget.cpp b/sdrgui/gui/glvideowidget.cpp new file mode 100644 index 000000000..d1dcc40a8 --- /dev/null +++ b/sdrgui/gui/glvideowidget.cpp @@ -0,0 +1,275 @@ +/////////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2026 Jon Beniston, M7RCE // +// Some code by AI // +// // +// This program is free software; you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation as version 3 of the License, or // +// (at your option) any later version. // +// // +// This program is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License V3 for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with this program. If not, see . // +/////////////////////////////////////////////////////////////////////////////////// + +#include + +#include +#include +#include +#include +#include + +#include "gui/glvideowidget.h" + +GLVideoWidget::GLVideoWidget(QWidget *parent) : + QOpenGLWidget(parent), + m_hidden(false), + m_overlayHeight(0), + m_keepAspect(true), + m_deinterlace(true) +{ + m_overlayTimer.setSingleShot(true); + connect(&m_overlayTimer, &QTimer::timeout, this, [this]() { + m_overlay.clear(); + update(); + }); +} + +GLVideoWidget::~GLVideoWidget() +{ + cleanup(); +} + +void GLVideoWidget::initializeGL() +{ + m_renderer.initialize(); + // The context is destroyed when the widget is, or recreated when it's reparented: GL objects go first + connect(context(), &QOpenGLContext::aboutToBeDestroyed, this, &GLVideoWidget::cleanup, Qt::DirectConnection); +} + +void GLVideoWidget::cleanup() +{ + if (!context()) { + return; + } + + makeCurrent(); + m_current.reset(); // GPU frames may hold the decoder's surfaces + m_renderer.cleanup(); + doneCurrent(); + disconnect(context(), &QOpenGLContext::aboutToBeDestroyed, this, &GLVideoWidget::cleanup); +} + +void GLVideoWidget::setFrame(GLVideoFramePtr frame) +{ + { + QMutexLocker locker(&m_mutex); + + // GPU frames hold the decoder's surfaces, so aren't kept while hidden. The next given after shown is shown + if (m_hidden && frame && frame->isGPU()) { + return; + } + + m_pending = frame; + } + + QMetaObject::invokeMethod(this, [this]() { update(); }, Qt::QueuedConnection); +} + +void GLVideoWidget::clear() +{ + { + QMutexLocker locker(&m_mutex); + m_pending.reset(); + } + + m_current.reset(); + + // The renderer's, as for deinterlacing, may hold the decoder's GPU surfaces too + if (context()) + { + makeCurrent(); + m_renderer.releaseFrames(); + doneCurrent(); + } + + update(); +} + +// GPU frames, and the renderer's (for deinterlacing, and the interops' views of them), hold the decoder's surfaces, +// which a hidden widget has no use for. Frames in memory are kept, so a paused picture is shown again +void GLVideoWidget::hideEvent(QHideEvent *event) +{ + { + QMutexLocker locker(&m_mutex); + m_hidden = true; + + if (m_pending && m_pending->isGPU()) { + m_pending.reset(); + } + } + + if (m_current && m_current->isGPU()) { + m_current.reset(); + } + + if (context()) + { + makeCurrent(); + m_renderer.releaseFrames(); + doneCurrent(); + } + + QOpenGLWidget::hideEvent(event); +} + +void GLVideoWidget::showEvent(QShowEvent *event) +{ + { + QMutexLocker locker(&m_mutex); + m_hidden = false; + } + + QOpenGLWidget::showEvent(event); +} + +void GLVideoWidget::setMessage(const QString& message) +{ + if (message != m_message) + { + m_message = message; + m_messageImage = QImage(); + update(); + } +} + +void GLVideoWidget::setOverlay(const QString& text, double seconds) +{ + m_overlay = text; + m_overlayImage = QImage(); + m_overlayTimer.start((int) (seconds * 1000.0)); + update(); +} + +void GLVideoWidget::setGuide(const QImage& image) +{ + m_guide = image; + update(); +} + +// As QOpenGLWidget sizes its framebuffer: rounded, so there's no edge left at fractional scales (125%, 150%) +QSize GLVideoWidget::pixelSize() const +{ + return size() * devicePixelRatioF(); +} + +void GLVideoWidget::setKeepAspectRatio(bool keep) +{ + m_keepAspect = keep; + update(); +} + +void GLVideoWidget::setDeinterlace(bool deinterlace) +{ + m_deinterlace = deinterlace; +} + +QSize GLVideoWidget::getFrameSize() const +{ + return m_frameSize; +} + +// White text, centred, on black, as large as the widget allows +QImage GLVideoWidget::messageImage(int width, int height) +{ + if (!m_messageImage.isNull() && (m_messageImage.width() == width) && (m_messageImage.height() == height)) { + return m_messageImage; + } + + QImage image(width, height, QImage::Format_RGBA8888_Premultiplied); + image.fill(Qt::black); + QPainter painter(&image); + painter.setRenderHint(QPainter::TextAntialiasing); + QFont font = painter.font(); + font.setPixelSize(std::max(12, height / 20)); + painter.setFont(font); + painter.setPen(Qt::white); + painter.drawText(image.rect().adjusted(width / 10, 0, -width / 10, 0), Qt::AlignCenter | Qt::TextWordWrap, m_message); + m_messageImage = image; + return m_messageImage; +} + +// White text in a translucent rounded box, sized for the video +QImage GLVideoWidget::overlayImage(int videoHeight) +{ + if (!m_overlayImage.isNull() && (m_overlayHeight == videoHeight)) { + return m_overlayImage; + } + + QFont font = this->font(); + font.setPixelSize(std::max(12, videoHeight / 16)); + const QFontMetrics metrics(font); + const int padding = metrics.height() / 3; + const QRect text = metrics.boundingRect(QRect(0, 0, 4096, 4096), Qt::AlignCenter, m_overlay); + QImage image(text.width() + 2 * padding, text.height() + 2 * padding, QImage::Format_RGBA8888_Premultiplied); + image.fill(Qt::transparent); + QPainter painter(&image); + painter.setRenderHint(QPainter::Antialiasing); + painter.setRenderHint(QPainter::TextAntialiasing); + painter.setFont(font); + QPainterPath path; + path.addRoundedRect(image.rect(), padding, padding); + painter.fillPath(path, QColor(0, 0, 0, 160)); + painter.setPen(Qt::white); + painter.drawText(image.rect(), Qt::AlignCenter, m_overlay); + m_overlayImage = image; + m_overlayHeight = videoHeight; + return m_overlayImage; +} + +void GLVideoWidget::paintGL() +{ + { + QMutexLocker locker(&m_mutex); + + if (m_pending) { + m_current = std::move(m_pending); + } + } + + const QSize pixels = pixelSize(); + const int width = pixels.width(); + const int height = pixels.height(); + QOpenGLFunctions *gl = context()->functions(); + gl->glClearColor(0.0f, 0.0f, 0.0f, 1.0f); + gl->glClear(GL_COLOR_BUFFER_BIT); + + if (!m_message.isEmpty()) + { + m_renderer.drawImage(messageImage(width, height), QRect(0, 0, width, height), width, height); + } + else if (m_current) + { + m_frameSize = QSize(m_current->m_width, m_current->m_height); + m_renderer.drawFrame(m_current, width, height, m_keepAspect, m_deinterlace); + + // The guide shows what the overlay would + if (!m_overlay.isEmpty() && m_guide.isNull()) + { + const QRect video = GLVideoRenderer::videoRect(*m_current, width, height, m_keepAspect); + const QImage image = overlayImage(video.height()); + const int margin = video.height() / 20; + const QRect rect(video.right() - margin - image.width(), video.bottom() - margin - image.height(), image.width(), image.height()); + m_renderer.drawImage(image, rect, width, height); + } + } + + if (!m_guide.isNull()) { + m_renderer.drawImage(m_guide, QRect(0, 0, width, height), width, height); + } +} diff --git a/sdrgui/gui/glvideowidget.h b/sdrgui/gui/glvideowidget.h new file mode 100644 index 000000000..f3d63ffee --- /dev/null +++ b/sdrgui/gui/glvideowidget.h @@ -0,0 +1,87 @@ +/////////////////////////////////////////////////////////////////////////////////// +// Copyright (C) 2026 Jon Beniston, M7RCE // +// Some code by AI // +// // +// This program is free software; you can redistribute it and/or modify // +// it under the terms of the GNU General Public License as published by // +// the Free Software Foundation as version 3 of the License, or // +// (at your option) any later version. // +// // +// This program is distributed in the hope that it will be useful, // +// but WITHOUT ANY WARRANTY; without even the implied warranty of // +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // +// GNU General Public License V3 for more details. // +// // +// You should have received a copy of the GNU General Public License // +// along with this program. If not, see . // +/////////////////////////////////////////////////////////////////////////////////// + +#ifndef SDRGUI_GUI_GLVIDEOWIDGET_H_ +#define SDRGUI_GUI_GLVIDEOWIDGET_H_ + +#include +#include +#include +#include + +#include "gui/glvideoframe.h" +#include "gui/glvideorenderer.h" +#include "export.h" + +// Displays video: frames of YUV in memory, or on the GPU (D3D11, CUDA, DMA-BUF, IOSurface) without copying them +// through the CPU, converted to RGB by the GPU, with the frame's aspect ratio. Also a message in place of video, +// text overlaid on it for a while, as a channel's name, and an image over the whole widget, as a programme guide. +// Frames can be given from any thread, as a decoder's: the latest is shown at the next paint +class SDRGUI_API GLVideoWidget : public QOpenGLWidget +{ + Q_OBJECT + +public: + explicit GLVideoWidget(QWidget *parent = nullptr); + ~GLVideoWidget(); + + //! From any thread. The widget keeps the frame until a newer one has been shown, but not GPU frames while hidden + void setFrame(GLVideoFramePtr frame); + //! From any thread: if frames of the type can be shown. GPU types can't until the widget's OpenGL context + //! exists, or after one has failed to be shown, so the decoder should copy them to memory instead + bool canShow(GLVideoFrame::Type type) const { return m_renderer.canShow(type); } + + void clear(); //!< No video, letting go of all frames given + void setMessage(const QString& message); //!< Shown in place of video, until set empty + void setOverlay(const QString& text, double seconds); //!< Over the video, bottom right, lines centred + //! Over everything, blended, as large as pixelSize() (scaled to fit if not), until set null + void setGuide(const QImage& image); + QSize pixelSize() const; //!< Of the widget, in device pixels + void setKeepAspectRatio(bool keep); + void setDeinterlace(bool deinterlace); //!< Of GPU frames the renderer deinterlaces (D3D11) + QSize getFrameSize() const; //!< Of the last frame, empty if none + +protected: + void initializeGL() override; + void paintGL() override; + void hideEvent(QHideEvent *event) override; + void showEvent(QShowEvent *event) override; + +private: + void cleanup(); + QImage messageImage(int width, int height); + QImage overlayImage(int videoHeight); + + GLVideoRenderer m_renderer; + mutable QMutex m_mutex; //!< For m_pending and m_hidden + GLVideoFramePtr m_pending; //!< Given, to be shown + GLVideoFramePtr m_current; //!< Being shown + bool m_hidden; //!< So GPU frames given aren't kept + QString m_message; + QImage m_messageImage; + QString m_overlay; + QImage m_overlayImage; + int m_overlayHeight; //!< Video height the overlay image was made for + QTimer m_overlayTimer; + QImage m_guide; + bool m_keepAspect; + bool m_deinterlace; + QSize m_frameSize; +}; + +#endif // SDRGUI_GUI_GLVIDEOWIDGET_H_