From 1ac99e058f413de7ae75652df2bee5ee6e0f2c1e Mon Sep 17 00:00:00 2001 From: OLC Date: Fri, 20 Feb 2026 08:19:52 +0800 Subject: [PATCH 01/12] =?UTF-8?q?feat:=20=E6=B7=BB=E5=8A=A0=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E7=BA=B9=E7=90=86=E6=94=AF=E6=8C=81=E5=8F=8A=E7=9B=B8?= =?UTF-8?q?=E5=85=B3=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- LuaSTG/LuaSTG/GameResource/ResourceManager.h | 2 + LuaSTG/LuaSTG/GameResource/ResourcePool.cpp | 38 ++ LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp | 94 ++++ data/example/video_example.lua | 26 + engine/graphics/CMakeLists.txt | 4 + engine/graphics/core/GraphicsDevice.hpp | 3 + engine/graphics/core/VideoDecoder.hpp | 42 ++ engine/graphics/d3d11/GraphicsDevice.hpp | 2 + engine/graphics/d3d11/VideoDecoder.cpp | 499 +++++++++++++++++++ engine/graphics/d3d11/VideoDecoder.hpp | 75 +++ engine/graphics/d3d11/VideoTexture.cpp | 102 ++++ engine/graphics/d3d11/VideoTexture.hpp | 56 +++ 12 files changed, 943 insertions(+) create mode 100644 data/example/video_example.lua create mode 100644 engine/graphics/core/VideoDecoder.hpp create mode 100644 engine/graphics/d3d11/VideoDecoder.cpp create mode 100644 engine/graphics/d3d11/VideoDecoder.hpp create mode 100644 engine/graphics/d3d11/VideoTexture.cpp create mode 100644 engine/graphics/d3d11/VideoTexture.hpp diff --git a/LuaSTG/LuaSTG/GameResource/ResourceManager.h b/LuaSTG/LuaSTG/GameResource/ResourceManager.h index 431926cde..6d98bd058 100644 --- a/LuaSTG/LuaSTG/GameResource/ResourceManager.h +++ b/LuaSTG/LuaSTG/GameResource/ResourceManager.h @@ -98,6 +98,8 @@ namespace luastg // 纹理 bool LoadTexture(const char* name, const char* path, bool mipmaps = true) noexcept; bool CreateTexture(const char* name, int width, int height) noexcept; + // 视频纹理 + bool LoadVideo(const char* name, const char* path) noexcept; // 渲染目标 bool CreateRenderTarget(const char* name, int width = 0, int height = 0, bool depth_buffer = false) noexcept; // 图片精灵 diff --git a/LuaSTG/LuaSTG/GameResource/ResourcePool.cpp b/LuaSTG/LuaSTG/GameResource/ResourcePool.cpp index 89a9c08d9..f6577b4da 100644 --- a/LuaSTG/LuaSTG/GameResource/ResourcePool.cpp +++ b/LuaSTG/LuaSTG/GameResource/ResourcePool.cpp @@ -227,6 +227,44 @@ namespace luastg return true; } + bool ResourcePool::LoadVideo(const char* name, const char* path) noexcept + { + if (m_TexturePool.find(std::string_view(name)) != m_TexturePool.end()) + { + if (ResourceMgr::GetResourceLoadingLog()) + { + spdlog::warn("[luastg] LoadVideo: 纹理 '{}' 已存在，加载操作已取消", name); + } + return true; + } + + core::SmartReference p_texture; + if (!LAPP.getGraphicsDevice()->createVideoTexture(path, p_texture.put())) + { + spdlog::error("[luastg] 从 '{}' 创建视频纹理 '{}' 失败", path, name); + return false; + } + + try + { + core::SmartReference tRes; + tRes.attach(new ResourceTextureImpl(name, p_texture.get())); + m_TexturePool.emplace(name, tRes); + } + catch (std::exception const& e) + { + spdlog::error("[luastg] LoadVideo: 创建视频纹理 '{}' 失败 ({})", name, e.what()); + return false; + } + + if (ResourceMgr::GetResourceLoadingLog()) + { + spdlog::info("[luastg] LoadVideo: 已从 '{}' 加载视频 '{}' ({})", path, name, getResourcePoolTypeName()); + } + + return true; + } + bool ResourcePool::CreateTexture(const char* name, int width, int height) noexcept { if (m_TexturePool.find(std::string_view(name)) != m_TexturePool.end()) diff --git a/LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp b/LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp index 3f9cf3545..1957cdb11 100644 --- a/LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp +++ b/LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp @@ -1,6 +1,8 @@ #include "LuaBinding/LuaWrapper.hpp" #include "lua/plus.hpp" #include "AppFrame.h" +#include "d3d11/VideoTexture.hpp" +#include "core/VideoDecoder.hpp" void luastg::binding::ResourceManager::Register(lua_State* L) noexcept { @@ -52,6 +54,18 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept return luaL_error(L, "can't load texture from file '%s'.", path); return 0; } + static int LoadVideo(lua_State* L) noexcept + { + const char* name = luaL_checkstring(L, 1); + const char* path = luaL_checkstring(L, 2); + + ResourcePool* pActivedPool = LRES.GetActivedPool(); + if (!pActivedPool) + return luaL_error(L, "can't load resource at this time."); + if (!pActivedPool->LoadVideo(name, path)) + return luaL_error(L, "can't load video from file '%s'.", path); + return 0; + } static int LoadSprite(lua_State* L) noexcept { const char* name = luaL_checkstring(L, 1); @@ -671,6 +685,78 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept LRES.CacheTTFFontString(luaL_checkstring(L, 1), str, len); return 0; } + + // 视频控制函数 + static core::IVideoDecoder* GetVideoDecoder(const char* name) noexcept { + auto tex = LRES.FindTexture(name); + if (!tex) return nullptr; + + auto texture2d = tex->GetTexture(); + if (!texture2d) return nullptr; + + // 尝试转换为VideoTexture + auto video_texture = dynamic_cast(texture2d); + if (!video_texture) return nullptr; + + return video_texture->getVideoDecoder(); + } + + static int VideoSeek(lua_State* L) noexcept { + const char* name = luaL_checkstring(L, 1); + double time = luaL_checknumber(L, 2); + auto decoder = GetVideoDecoder(name); + if (!decoder) + return luaL_error(L, "video texture '%s' not found.", name); + lua_pushboolean(L, decoder->seek(time)); + return 1; + } + + static int VideoSetLooping(lua_State* L) noexcept { + const char* name = luaL_checkstring(L, 1); + bool loop = lua_toboolean(L, 2) != 0; + auto decoder = GetVideoDecoder(name); + if (!decoder) + return luaL_error(L, "video texture '%s' not found.", name); + decoder->setLooping(loop); + return 0; + } + + static int VideoUpdate(lua_State* L) noexcept { + const char* name = luaL_checkstring(L, 1); + double time = luaL_checknumber(L, 2); + auto decoder = GetVideoDecoder(name); + if (!decoder) + return luaL_error(L, "video texture '%s' not found.", name); + lua_pushboolean(L, decoder->updateToTime(time)); + return 1; + } + + static int VideoGetInfo(lua_State* L) noexcept { + const char* name = luaL_checkstring(L, 1); + auto decoder = GetVideoDecoder(name); + if (!decoder) + return luaL_error(L, "video texture '%s' not found.", name); + + lua_createtable(L, 0, 5); + + lua_pushnumber(L, decoder->getDuration()); + lua_setfield(L, -2, "duration"); + + lua_pushnumber(L, decoder->getCurrentTime()); + lua_setfield(L, -2, "time"); + + lua_pushboolean(L, decoder->isLooping()); + lua_setfield(L, -2, "looping"); + + auto size = decoder->getVideoSize(); + lua_pushinteger(L, size.x); + lua_setfield(L, -2, "width"); + + lua_pushinteger(L, size.y); + lua_setfield(L, -2, "height"); + + return 1; + } }; luaL_Reg const lib[] = { @@ -678,6 +764,7 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept { "SetResourceStatus", &Wrapper::SetResourceStatus }, { "GetResourceStatus", &Wrapper::GetResourceStatus }, { "LoadTexture", &Wrapper::LoadTexture }, + { "LoadVideo", &Wrapper::LoadVideo }, { "LoadImage", &Wrapper::LoadSprite }, { "LoadAnimation", &Wrapper::LoadAnimation }, { "LoadPS", &Wrapper::LoadPS }, @@ -710,6 +797,13 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept { "SetFontState", &Wrapper::SetFontState }, { "CacheTTFString", &Wrapper::CacheTTFString }, + + // 视频控制函数 + { "VideoSeek", &Wrapper::VideoSeek }, + { "VideoSetLooping", &Wrapper::VideoSetLooping }, + { "VideoUpdate", &Wrapper::VideoUpdate }, + { "VideoGetInfo", &Wrapper::VideoGetInfo }, + { NULL, NULL }, }; diff --git a/data/example/video_example.lua b/data/example/video_example.lua new file mode 100644 index 000000000..9c0352454 --- /dev/null +++ b/data/example/video_example.lua @@ -0,0 +1,26 @@ +-- 视频播放示例 +-- 使用 StopWatch 提供绝对时间，手动调用 VideoUpdate 更新画面 + +-- 在资源池中加载视频 +-- 参数：名称，路径 +LoadVideo('video1', 'test_video.mp4') + +-- 创建计时器并设置循环播放 +local video_clock = lstg.StopWatch() +VideoSetLooping('video1', true) + +-- 像普通纹理一样创建精灵 +LoadImage('video_sprite', 'video1', 0, 0, 640, 480) + +-- 在 RenderFunc 中渲染 +function RenderFunc() + -- 以秒表绝对时间驱动视频更新 + VideoUpdate('video1', video_clock:GetElapsed()) + Render('video_sprite', 100, 100) +end + +-- 可用控制： +-- 1. VideoSetLooping(name, bool) -- 设置是否循环播放 +-- 2. VideoSeek(name, time) -- 控制时间，但是不更新画面，适合进度条等场景，完成后需要调用 VideoUpdate 来刷新画面 +-- 3. VideoUpdate(name, absolute_time) -- 根据绝对时间更新视频画面，适合与 StopWatch 结合使用，确保视频播放与游戏时间同步 +-- 4. VideoGetInfo(name) -- 获取视频信息，如帧率、总时长等，返回一个表格 diff --git a/engine/graphics/CMakeLists.txt b/engine/graphics/CMakeLists.txt index 19c29b5e6..a9f2a4f25 100644 --- a/engine/graphics/CMakeLists.txt +++ b/engine/graphics/CMakeLists.txt @@ -46,6 +46,10 @@ target_link_libraries(${lib_name} PUBLIC dxgi.lib d3d11.lib d2d1.lib + # media foundation (for video) + mfplat.lib + mfreadwrite.lib + mfuuid.lib Microsoft.Windows.ImplementationLibrary Microsoft::DirectXTexMini libqoi diff --git a/engine/graphics/core/GraphicsDevice.hpp b/engine/graphics/core/GraphicsDevice.hpp index f73c8dd07..3002a651e 100644 --- a/engine/graphics/core/GraphicsDevice.hpp +++ b/engine/graphics/core/GraphicsDevice.hpp @@ -3,6 +3,7 @@ #include "core/ImmutableString.hpp" #include "core/GraphicsBuffer.hpp" #include "core/Texture2D.hpp" +#include "core/VideoDecoder.hpp" #include "core/GraphicsSampler.hpp" #include "core/RenderTarget.hpp" #include "core/DepthStencilBuffer.hpp" @@ -108,6 +109,8 @@ namespace core { virtual bool createTextureFromFile(StringView path, bool mipmap, ITexture2D** out_texture) = 0; virtual bool createTextureFromImage(IImage* image, bool mipmap, ITexture2D** out_texture) = 0; virtual bool createTexture(Vector2U size, ITexture2D** out_texture) = 0; + virtual bool createVideoTexture(StringView path, ITexture2D** out_texture) = 0; + virtual bool createVideoDecoder(IVideoDecoder** out_decoder) = 0; virtual bool createSampler(const GraphicsSamplerInfo& info, IGraphicsSampler** out_sampler) = 0; diff --git a/engine/graphics/core/VideoDecoder.hpp b/engine/graphics/core/VideoDecoder.hpp new file mode 100644 index 000000000..c32b8df18 --- /dev/null +++ b/engine/graphics/core/VideoDecoder.hpp @@ -0,0 +1,42 @@ +#pragma once +#include "core/ReferenceCounted.hpp" +#include "core/Vector2.hpp" +#include "core/ImmutableString.hpp" + +namespace core { + CORE_INTERFACE IVideoDecoder : IReferenceCounted { + // 打开视频文件 + virtual bool open(StringView path) = 0; + + // 关闭视频 + virtual void close() = 0; + + // 获取状态 + virtual bool hasVideo() const noexcept = 0; + + // 视频信息 + virtual Vector2U getVideoSize() const noexcept = 0; + virtual double getDuration() const noexcept = 0; + virtual double getCurrentTime() const noexcept = 0; // 返回上次更新的帧时间 + + // 跳转到指定时间 + virtual bool seek(double time_in_seconds) = 0; + + // 循环播放设置 + virtual void setLooping(bool loop) = 0; + virtual bool isLooping() const noexcept = 0; + + // 手动更新到指定时间点 + // time_in_seconds: 目标时间（秒） + // 返回是否成功更新（失败可能是因为到达视频结尾且未循环） + virtual bool updateToTime(double time_in_seconds) = 0; + + // 获取用于渲染的纹理（返回 ID3D11Texture2D*） + virtual void* getNativeTexture() const noexcept = 0; + + // 获取 Shader Resource View（返回 ID3D11ShaderResourceView*） + virtual void* getNativeShaderResourceView() const noexcept = 0; + }; + + CORE_INTERFACE_ID(IVideoDecoder, "a4b5c6d7-e8f9-1234-5678-9abcdef01234") +} diff --git a/engine/graphics/d3d11/GraphicsDevice.hpp b/engine/graphics/d3d11/GraphicsDevice.hpp index 665374707..133c67f01 100644 --- a/engine/graphics/d3d11/GraphicsDevice.hpp +++ b/engine/graphics/d3d11/GraphicsDevice.hpp @@ -22,6 +22,8 @@ namespace core { bool createTextureFromFile(StringView path, bool mipmap, ITexture2D** out_texture) override; bool createTexture(Vector2U size, ITexture2D** out_texture) override; bool createTextureFromImage(IImage* image, bool mipmap, ITexture2D** out_texture) override; + bool createVideoTexture(StringView path, ITexture2D** out_texture) override; + bool createVideoDecoder(IVideoDecoder** out_decoder) override; bool createSampler(const GraphicsSamplerInfo& info, IGraphicsSampler** out_sampler) override; diff --git a/engine/graphics/d3d11/VideoDecoder.cpp b/engine/graphics/d3d11/VideoDecoder.cpp new file mode 100644 index 000000000..9bf79d82b --- /dev/null +++ b/engine/graphics/d3d11/VideoDecoder.cpp @@ -0,0 +1,499 @@ +#include "d3d11/VideoDecoder.hpp" +#include "core/Logger.hpp" +#include "utf8.hpp" +#include +#include + +#pragma comment(lib, "mfplat.lib") +#pragma comment(lib, "mfreadwrite.lib") +#pragma comment(lib, "mfuuid.lib") + +namespace { + class MFInitializer { + public: + static MFInitializer& getInstance() { + static MFInitializer instance; + return instance; + } + + bool isInitialized() const { return m_initialized; } + + private: + MFInitializer() { + HRESULT hr = MFStartup(MF_VERSION, MFSTARTUP_FULL); + m_initialized = SUCCEEDED(hr); + if (!m_initialized) { + core::Logger::error("[core] [VideoDecoder] Failed to initialize MediaFoundation, hr = {:#x}", (uint32_t)hr); + } + } + + ~MFInitializer() { + if (m_initialized) { + MFShutdown(); + } + } + + bool m_initialized{ false }; + }; +} + +namespace core { + VideoDecoder::VideoDecoder() = default; + + VideoDecoder::~VideoDecoder() { + if (m_initialized && m_device) { + m_device->removeEventListener(this); + } + close(); + } + + bool VideoDecoder::initialize(IGraphicsDevice* device) { + if (!device) { + Logger::error("[core] [VideoDecoder] Invalid device"); + return false; + } + + if (!MFInitializer::getInstance().isInitialized()) { + Logger::error("[core] [VideoDecoder] MediaFoundation not initialized"); + return false; + } + + m_device = device; + m_initialized = true; + m_device->addEventListener(this); + + return true; + } + + bool VideoDecoder::open(StringView path) { + if (!m_initialized) { + Logger::error("[core] [VideoDecoder] Not initialized"); + return false; + } + + close(); + + HRESULT hr = S_OK; + + std::wstring wide_path = utf8::to_wstring(path); + + win32::com_ptr attributes; + hr = MFCreateAttributes(attributes.put(), 1); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to create attributes, hr = {:#x}", (uint32_t)hr); + return false; + } + + hr = attributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE); + hr = attributes->SetUINT32(MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, TRUE); + + hr = MFCreateSourceReaderFromURL(wide_path.c_str(), attributes.get(), m_source_reader.put()); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to create source reader, hr = {:#x}", (uint32_t)hr); + return false; + } + + win32::com_ptr media_type; + hr = MFCreateMediaType(media_type.put()); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to create media type, hr = {:#x}", (uint32_t)hr); + return false; + } + + hr = media_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + + hr = media_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_ARGB32); + hr = m_source_reader->SetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, media_type.get()); + + if (FAILED(hr)) { + Logger::info("[core] [VideoDecoder] ARGB32 not supported, trying RGB32"); + hr = media_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); + hr = m_source_reader->SetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, media_type.get()); + + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to set media type (tried ARGB32 and RGB32), hr = {:#x}", (uint32_t)hr); + return false; + } + } + + hr = m_source_reader->GetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, m_media_type.put()); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to get media type, hr = {:#x}", (uint32_t)hr); + return false; + } + + GUID subtype = GUID_NULL; + hr = m_media_type->GetGUID(MF_MT_SUBTYPE, &subtype); + if (SUCCEEDED(hr)) { + if (subtype == MFVideoFormat_ARGB32) { + Logger::info("[core] [VideoDecoder] Using video format: ARGB32 (native BGRA)"); + } else if (subtype == MFVideoFormat_RGB32) { + Logger::info("[core] [VideoDecoder] Using video format: RGB32"); + } else { + Logger::warn("[core] [VideoDecoder] Using unknown video format, display may be incorrect"); + } + } + + UINT32 width = 0, height = 0; + hr = MFGetAttributeSize(m_media_type.get(), MF_MT_FRAME_SIZE, &width, &height); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to get frame size, hr = {:#x}", (uint32_t)hr); + return false; + } + + m_video_size = Vector2U{ width, height }; + m_target_size = m_video_size; + + PROPVARIANT var; + PropVariantInit(&var); + hr = m_source_reader->GetPresentationAttribute((DWORD)MF_SOURCE_READER_MEDIASOURCE, MF_PD_DURATION, &var); + if (SUCCEEDED(hr)) { + m_duration = var.hVal.QuadPart / 10000000.0; + PropVariantClear(&var); + } + + if (!createTexture()) { + close(); + return false; + } + + m_frame_pitch = m_target_size.x * 4; + + auto d3d_device = static_cast(m_device->getNativeDevice()); + if (d3d_device) { + d3d_device->GetImmediateContext(m_device_context.put()); + } + + m_current_time = 0.0; + m_last_requested_time = -1.0; + + bool first_frame_loaded = readFrameAtTime(0.0); + Logger::info("[core] [VideoDecoder] First frame loaded: {}", first_frame_loaded); + + Logger::info("[core] [VideoDecoder] Opened video: {}x{}, duration: {:.2f}s", + m_target_size.x, m_target_size.y, m_duration); + + return true; + } + + void VideoDecoder::close() { + m_source_reader.reset(); + m_media_type.reset(); + m_texture.reset(); + m_shader_resource_view.reset(); + m_device_context.reset(); + + m_video_size = Vector2U{}; + m_target_size = Vector2U{}; + m_duration = 0.0; + m_current_time = 0.0; + m_last_requested_time = -1.0; + m_frame_pitch = 0; + } + + bool VideoDecoder::seek(double time_in_seconds) { + if (!hasVideo()) { + return false; + } + + if (time_in_seconds < 0.0) { + time_in_seconds = 0.0; + } + if (time_in_seconds > m_duration) { + time_in_seconds = m_duration; + } + + PROPVARIANT var; + PropVariantInit(&var); + var.vt = VT_I8; + var.hVal.QuadPart = static_cast(time_in_seconds * 10000000.0); + + HRESULT hr = m_source_reader->SetCurrentPosition(GUID_NULL, var); + PropVariantClear(&var); + + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to seek, hr = {:#x}", (uint32_t)hr); + return false; + } + + m_current_time = time_in_seconds; + return true; + } + + bool VideoDecoder::updateToTime(double time_in_seconds) { + if (!hasVideo()) { + return false; + } + + if (m_duration <= 0.0) { + m_current_time = 0.0; + return false; + } + + if (time_in_seconds >= m_duration) { + if (m_looping) { + time_in_seconds = fmod(time_in_seconds, m_duration); + } else { + double const last_frame_time = std::max(0.0, m_duration - 1e-6); + bool const ok = readFrameAtTime(last_frame_time); + m_current_time = m_duration; + return ok; + } + } else if (time_in_seconds < 0.0) { + time_in_seconds = 0.0; + } + + constexpr double kTimeEpsilon = 1e-4; + constexpr double kBackwardTolerance = 1.0 / 120.0; + constexpr double kFrameTolerance = 1.0 / 24.0; + constexpr double kSeekThreshold = 0.25; + + bool const is_backward = (m_last_requested_time >= 0.0) && (time_in_seconds + kBackwardTolerance < m_last_requested_time); + bool const large_jump = (m_current_time + kSeekThreshold < time_in_seconds); + + m_last_requested_time = time_in_seconds; + + if (is_backward || large_jump) { + return readFrameAtTime(time_in_seconds); + } + + if (m_current_time + kFrameTolerance >= time_in_seconds) { + return true; + } + + for (int i = 0; i < 4; ++i) { + if (!readNextFrame()) { + return false; + } + if (m_current_time + kTimeEpsilon >= time_in_seconds) { + return true; + } + } + + return readFrameAtTime(time_in_seconds); + } + + bool VideoDecoder::readNextFrame() { + if (!hasVideo()) { + return false; + } + + HRESULT hr = S_OK; + win32::com_ptr sample; + DWORD stream_flags = 0; + LONGLONG timestamp = 0; + + hr = m_source_reader->ReadSample( + (DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, + 0, + nullptr, + &stream_flags, + ×tamp, + sample.put() + ); + + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to read sample, hr = {:#x}", (uint32_t)hr); + return false; + } + + if (stream_flags & MF_SOURCE_READERF_ENDOFSTREAM) { + m_current_time = m_duration; + return false; + } + + if (!sample) { + return false; + } + + m_current_time = timestamp / 10000000.0; + if (m_current_time > m_duration) { + m_current_time = m_duration; + } + + return updateTextureFromSample(sample.get()); + } + + bool VideoDecoder::readFrameAtTime(double time_in_seconds) { + if (!hasVideo()) { + return false; + } + + if (!seek(time_in_seconds)) { + return false; + } + + constexpr double kTimeEpsilon = 1e-4; + constexpr int kMaxDecodeAfterSeek = 360; + + for (int i = 0; i < kMaxDecodeAfterSeek; ++i) { + if (!readNextFrame()) { + return false; + } + if (m_current_time + kTimeEpsilon >= time_in_seconds) { + return true; + } + } + + return false; + } + + void VideoDecoder::onGraphicsDeviceCreate() { + if (hasVideo()) { + createTexture(); + } + } + + void VideoDecoder::onGraphicsDeviceDestroy() { + m_texture.reset(); + m_shader_resource_view.reset(); + m_device_context.reset(); + } + + bool VideoDecoder::createTexture() { + if (!m_device || m_target_size.x == 0 || m_target_size.y == 0) { + return false; + } + + auto d3d_device = static_cast(m_device->getNativeDevice()); + if (!d3d_device) { + return false; + } + + D3D11_TEXTURE2D_DESC desc = {}; + desc.Width = m_target_size.x; + desc.Height = m_target_size.y; + desc.MipLevels = 1; + desc.ArraySize = 1; + desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + desc.SampleDesc.Count = 1; + desc.SampleDesc.Quality = 0; + desc.Usage = D3D11_USAGE_DYNAMIC; + desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; + desc.MiscFlags = 0; + + HRESULT hr = d3d_device->CreateTexture2D(&desc, nullptr, m_texture.put()); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to create texture, hr = {:#x}", (uint32_t)hr); + return false; + } + + D3D11_SHADER_RESOURCE_VIEW_DESC srv_desc = {}; + srv_desc.Format = desc.Format; + srv_desc.ViewDimension = D3D11_SRV_DIMENSION_TEXTURE2D; + srv_desc.Texture2D.MipLevels = 1; + srv_desc.Texture2D.MostDetailedMip = 0; + + hr = d3d_device->CreateShaderResourceView(m_texture.get(), &srv_desc, m_shader_resource_view.put()); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to create shader resource view, hr = {:#x}", (uint32_t)hr); + m_texture.reset(); + return false; + } + + d3d_device->GetImmediateContext(m_device_context.put()); + + return true; + } + + bool VideoDecoder::updateTextureFromSample(IMFSample* sample) { + if (!m_texture || !sample || !m_device_context) { + return false; + } + + win32::com_ptr buffer; + HRESULT hr = sample->ConvertToContiguousBuffer(buffer.put()); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to get buffer, hr = {:#x}", (uint32_t)hr); + return false; + } + + win32::com_ptr buffer_2d; + LONG source_pitch = m_frame_pitch; + BYTE* src_data = nullptr; + bool using_2d_buffer = false; + + if (SUCCEEDED(buffer->QueryInterface(IID_PPV_ARGS(buffer_2d.put())))) { + hr = buffer_2d->Lock2D(&src_data, &source_pitch); + if (SUCCEEDED(hr)) { + using_2d_buffer = true; + } + } + + if (!using_2d_buffer) { + DWORD max_length = 0, current_length = 0; + hr = buffer->Lock(&src_data, &max_length, ¤t_length); + if (FAILED(hr)) { + return false; + } + } + + D3D11_MAPPED_SUBRESOURCE mapped; + hr = m_device_context->Map(m_texture.get(), 0, D3D11_MAP_WRITE_DISCARD, 0, &mapped); + if (FAILED(hr)) { + if (using_2d_buffer) { + buffer_2d->Unlock2D(); + } else { + buffer->Unlock(); + } + Logger::error("[core] [VideoDecoder] Failed to map texture, hr = {:#x}", (uint32_t)hr); + return false; + } + + uint8_t* dst = static_cast(mapped.pData); + const uint8_t* src = src_data; + const size_t copy_size = m_frame_pitch; + + GUID format = GUID_NULL; + bool force_opaque_alpha = false; + if (m_media_type && SUCCEEDED(m_media_type->GetGUID(MF_MT_SUBTYPE, &format))) { + if (format == MFVideoFormat_RGB32) { + force_opaque_alpha = true; + } + } + + static bool first_copy = true; + if (first_copy) { + Logger::info("[core] [VideoDecoder] Texture update: source_pitch={}, mapped.RowPitch={}, copy_size={}, height={}, force_alpha={}", + source_pitch, mapped.RowPitch, copy_size, m_target_size.y, force_opaque_alpha); + bool all_zero = true; + for (size_t i = 0; i < std::min(16, copy_size); ++i) { + if (src[i] != 0) { + all_zero = false; + break; + } + } + Logger::info("[core] [VideoDecoder] First 16 bytes all zero: {}", all_zero); + first_copy = false; + } + + if (!force_opaque_alpha) { + for (uint32_t y = 0; y < m_target_size.y; ++y) { + memcpy(dst, src, copy_size); + dst += mapped.RowPitch; + src += source_pitch; + } + } else { + for (uint32_t y = 0; y < m_target_size.y; ++y) { + memcpy(dst, src, copy_size); + for (uint32_t x = 0; x < m_target_size.x; ++x) { + dst[x * 4 + 3] = 0xFF; + } + dst += mapped.RowPitch; + src += source_pitch; + } + } + + m_device_context->Unmap(m_texture.get(), 0); + + if (using_2d_buffer) { + buffer_2d->Unlock2D(); + } else { + buffer->Unlock(); + } + + return true; + } + +} diff --git a/engine/graphics/d3d11/VideoDecoder.hpp b/engine/graphics/d3d11/VideoDecoder.hpp new file mode 100644 index 000000000..98940eb79 --- /dev/null +++ b/engine/graphics/d3d11/VideoDecoder.hpp @@ -0,0 +1,75 @@ +#pragma once +#include "core/VideoDecoder.hpp" +#include "core/GraphicsDevice.hpp" +#include "core/SmartReference.hpp" +#include "core/implement/ReferenceCounted.hpp" +#include "d3d11/pch.h" +#include +#include +#include +#include +#include + +namespace core { + class VideoDecoder final : + public implement::ReferenceCounted, + public IGraphicsDeviceEventListener { + public: + // IVideoDecoder + + bool open(StringView path) override; + void close() override; + + bool hasVideo() const noexcept override { return m_source_reader.get() != nullptr; } + + Vector2U getVideoSize() const noexcept override { return m_target_size; } + double getDuration() const noexcept override { return m_duration; } + double getCurrentTime() const noexcept override { return m_current_time; } + + bool seek(double time_in_seconds) override; + + void setLooping(bool loop) override { m_looping = loop; } + bool isLooping() const noexcept override { return m_looping; } + + bool updateToTime(double time_in_seconds) override; + void* getNativeTexture() const noexcept override { return m_texture.get(); } + void* getNativeShaderResourceView() const noexcept override { return m_shader_resource_view.get(); } + + // IGraphicsDeviceEventListener + + void onGraphicsDeviceCreate() override; + void onGraphicsDeviceDestroy() override; + + // VideoDecoder + + VideoDecoder(); + ~VideoDecoder(); + + bool initialize(IGraphicsDevice* device); + + private: + bool createTexture(); + bool updateTextureFromSample(IMFSample* sample); + bool readNextFrame(); + bool readFrameAtTime(double time_in_seconds); + + SmartReference m_device; + win32::com_ptr m_texture; + win32::com_ptr m_shader_resource_view; + win32::com_ptr m_device_context; + + win32::com_ptr m_source_reader; + win32::com_ptr m_media_type; + + Vector2U m_video_size{}; + Vector2U m_target_size{}; + double m_duration{ 0.0 }; + double m_current_time{ 0.0 }; // 当前帧时间 + double m_last_requested_time{ -1.0 }; + bool m_looping{ false }; + + uint32_t m_frame_pitch{ 0 }; + + bool m_initialized{ false }; + }; +} diff --git a/engine/graphics/d3d11/VideoTexture.cpp b/engine/graphics/d3d11/VideoTexture.cpp new file mode 100644 index 000000000..5fc1ace5d --- /dev/null +++ b/engine/graphics/d3d11/VideoTexture.cpp @@ -0,0 +1,102 @@ +#include "d3d11/VideoTexture.hpp" +#include "d3d11/VideoDecoder.hpp" +#include "d3d11/GraphicsDevice.hpp" +#include "core/Logger.hpp" +#include "core/SmartReference.hpp" + +namespace core { + VideoTexture::VideoTexture() = default; + + VideoTexture::~VideoTexture() { + if (m_initialized && static_cast(m_device)) { + m_device->removeEventListener(this); + } + } + + bool VideoTexture::initialize(IGraphicsDevice* device, StringView path) { + if (!device) { + Logger::error("[core] [VideoTexture] Invalid device"); + return false; + } + + m_device = device; + + // 创建视频解码器 + auto decoder = new VideoDecoder(); + if (!decoder->initialize(device)) { + Logger::error("[core] [VideoTexture] Failed to initialize video decoder"); + decoder->release(); + return false; + } + + // 打开视频文件 + if (!decoder->open(path)) { + Logger::error("[core] [VideoTexture] Failed to open video file: {}", path); + decoder->release(); + return false; + } + + m_decoder = decoder; + decoder->release(); + + m_initialized = true; + m_device->addEventListener(this); + + Logger::info("[core] [VideoTexture] Created video texture from: {}", path); + + return true; + } + + void* VideoTexture::getNativeResource() const noexcept { + if (m_decoder) { + return m_decoder->getNativeTexture(); // ID3D11Texture2D* + } + return nullptr; + } + + void* VideoTexture::getNativeView() const noexcept { + if (m_decoder) { + return m_decoder->getNativeShaderResourceView(); // ID3D11ShaderResourceView* + } + return nullptr; + } + + Vector2U VideoTexture::getSize() const noexcept { + if (m_decoder) { + return m_decoder->getVideoSize(); + } + return Vector2U{}; + } + + void VideoTexture::onGraphicsDeviceCreate() {} + + void VideoTexture::onGraphicsDeviceDestroy() {} + + // GraphicsDevice 扩展：视频功能 + + bool GraphicsDevice::createVideoTexture(StringView path, ITexture2D** out_texture) { + if (out_texture == nullptr) { + assert(false); return false; + } + SmartReference video_texture; + video_texture.attach(new VideoTexture); + if (!video_texture->initialize(this, path)) { + return false; + } + *out_texture = video_texture.detach(); + return true; + } + + bool GraphicsDevice::createVideoDecoder(IVideoDecoder** out_decoder) { + if (out_decoder == nullptr) { + assert(false); return false; + } + SmartReference decoder; + decoder.attach(new VideoDecoder); + if (!decoder->initialize(this)) { + return false; + } + *out_decoder = decoder.detach(); + return true; + } +} diff --git a/engine/graphics/d3d11/VideoTexture.hpp b/engine/graphics/d3d11/VideoTexture.hpp new file mode 100644 index 000000000..0cfb2741a --- /dev/null +++ b/engine/graphics/d3d11/VideoTexture.hpp @@ -0,0 +1,56 @@ +#pragma once +#include "core/Texture2D.hpp" +#include "core/VideoDecoder.hpp" +#include "core/GraphicsDevice.hpp" +#include "core/SmartReference.hpp" +#include "core/implement/ReferenceCounted.hpp" +#include "d3d11/pch.h" + +namespace core { + // VideoTexture 类：结合了 Texture2D 和 VideoDecoder 的功能 + // 可以像普通纹理一样使用，但内容来自视频 + class VideoTexture final : + public implement::ReferenceCounted, + public IGraphicsDeviceEventListener { + public: + // ITexture2D + + void* getNativeResource() const noexcept override; + void* getNativeView() const noexcept override; + + bool isDynamic() const noexcept override { return true; } + bool isPremultipliedAlpha() const noexcept override { return false; } + void setPremultipliedAlpha(bool v) override {} + Vector2U getSize() const noexcept override; + + bool setSize(Vector2U size) override { return false; } + bool update(RectU rect, void const* data, uint32_t row_pitch_in_bytes) override { return false; } + void setImage(IImage* image) override {} + + bool saveToFile(StringView path) override { return false; } + + void setSamplerState(IGraphicsSampler* sampler) override { m_sampler = sampler; } + IGraphicsSampler* getSamplerState() const noexcept override { return m_sampler.get(); } + + // IGraphicsDeviceEventListener + + void onGraphicsDeviceCreate() override; + void onGraphicsDeviceDestroy() override; + + // VideoTexture + + VideoTexture(); + ~VideoTexture(); + + bool initialize(IGraphicsDevice* device, StringView path); + + // 获取内部的视频解码器（用于控制播放等） + IVideoDecoder* getVideoDecoder() const noexcept { return m_decoder.get(); } + + private: + SmartReference m_device; + SmartReference m_decoder; + SmartReference m_sampler; + bool m_initialized{ false }; + }; +} From 00b9331cdd47e54ccfa9f703b5727b7346aff18c Mon Sep 17 00:00:00 2001 From: OLC Date: Sun, 22 Feb 2026 00:13:30 +0800 Subject: [PATCH 02/12] feat: enhance video texture support with memory stream handling and resource management --- LuaSTG/LuaSTG/GameResource/ResourceDebug.cpp | 10 +++--- engine/graphics/core/Texture2D.hpp | 1 + engine/graphics/d3d11/Texture2D.hpp | 1 + engine/graphics/d3d11/VideoDecoder.cpp | 32 ++++++++++++++++++-- engine/graphics/d3d11/VideoDecoder.hpp | 1 + engine/graphics/d3d11/VideoTexture.hpp | 1 + 6 files changed, 39 insertions(+), 7 deletions(-) diff --git a/LuaSTG/LuaSTG/GameResource/ResourceDebug.cpp b/LuaSTG/LuaSTG/GameResource/ResourceDebug.cpp index ecda732f2..74c034431 100644 --- a/LuaSTG/LuaSTG/GameResource/ResourceDebug.cpp +++ b/LuaSTG/LuaSTG/GameResource/ResourceDebug.cpp @@ -81,18 +81,20 @@ namespace luastg }; auto draw_texture = [](IResourceTexture* p_res, bool show_info, float scale) -> void { - auto const size = p_res->GetTexture()->getSize(); + auto* p_tex = p_res->GetTexture(); + auto const size = p_tex->getSize(); if (show_info) { ImGui::Text("Size: %u x %u", size.x, size.y); - ImGui::Text("RenderTarget: %s", p_res->IsRenderTarget() ? "Yes" : "Not"); - ImGui::Text("Dynamic: %s", p_res->IsRenderTarget() ? "Yes" : "Not"); + char const* type_str = p_tex->isVideoTexture() ? "Video" : (p_res->IsRenderTarget() ? "RenderTarget" : "Texture"); + ImGui::Text("Type: %s", type_str); + ImGui::Text("Dynamic: %s", p_tex->isDynamic() ? "Yes" : "Not"); unsigned long long mem_usage = size.x * size.y * 4; ImGui::Text("Adapter Memory Usage (Approximate): %s", bytes_count_to_string(mem_usage).c_str()); } ImGui::PushStyleVar(ImGuiStyleVar_ImageBorderSize, 1.0); ImGui::Image( - reinterpret_cast(p_res->GetTexture()->getNativeView()), + reinterpret_cast(p_tex->getNativeView()), ImVec2(scale * (float)size.x, scale * (float)size.y), ImVec2(0.0f, 0.0f), ImVec2(1.0f, 1.0f)); diff --git a/engine/graphics/core/Texture2D.hpp b/engine/graphics/core/Texture2D.hpp index 6daddee43..3cddc9da4 100644 --- a/engine/graphics/core/Texture2D.hpp +++ b/engine/graphics/core/Texture2D.hpp @@ -27,6 +27,7 @@ namespace core { virtual void* getNativeView() const noexcept = 0; virtual bool isDynamic() const noexcept = 0; + virtual bool isVideoTexture() const noexcept = 0; virtual bool isPremultipliedAlpha() const noexcept = 0; virtual void setPremultipliedAlpha(bool v) = 0; virtual Vector2U getSize() const noexcept = 0; diff --git a/engine/graphics/d3d11/Texture2D.hpp b/engine/graphics/d3d11/Texture2D.hpp index 96accdfc3..dd71e96f9 100644 --- a/engine/graphics/d3d11/Texture2D.hpp +++ b/engine/graphics/d3d11/Texture2D.hpp @@ -15,6 +15,7 @@ namespace core { void* getNativeView() const noexcept override { return m_view.get(); } bool isDynamic() const noexcept override { return m_dynamic; } + bool isVideoTexture() const noexcept override { return false; } bool isPremultipliedAlpha() const noexcept override { return m_pre_mul_alpha; } void setPremultipliedAlpha(bool const v) override { m_pre_mul_alpha = v; } Vector2U getSize() const noexcept override { return m_size; } diff --git a/engine/graphics/d3d11/VideoDecoder.cpp b/engine/graphics/d3d11/VideoDecoder.cpp index 9bf79d82b..08a20bfe7 100644 --- a/engine/graphics/d3d11/VideoDecoder.cpp +++ b/engine/graphics/d3d11/VideoDecoder.cpp @@ -1,12 +1,17 @@ #include "d3d11/VideoDecoder.hpp" +#include "core/FileSystem.hpp" #include "core/Logger.hpp" #include "utf8.hpp" #include #include +#include +#include +#include #pragma comment(lib, "mfplat.lib") #pragma comment(lib, "mfreadwrite.lib") #pragma comment(lib, "mfuuid.lib") +#pragma comment(lib, "shlwapi.lib") namespace { class MFInitializer { @@ -73,9 +78,29 @@ namespace core { close(); - HRESULT hr = S_OK; + SmartReference file_data; + if (!FileSystemManager::readFile(path, file_data.put())) { + Logger::error("[core] [VideoDecoder] Failed to read video file: {}", path); + return false; + } + + win32::com_ptr mem_stream; + mem_stream.attach(static_cast(SHCreateMemStream( + static_cast(file_data->data()), + static_cast(file_data->size())))); + if (!mem_stream) { + Logger::error("[core] [VideoDecoder] Failed to create memory stream"); + return false; + } + + win32::com_ptr byte_stream; + HRESULT hr = MFCreateMFByteStreamOnStream(mem_stream.get(), byte_stream.put()); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to create byte stream, hr = {:#x}", (uint32_t)hr); + return false; + } - std::wstring wide_path = utf8::to_wstring(path); + m_byte_stream = std::move(byte_stream); win32::com_ptr attributes; hr = MFCreateAttributes(attributes.put(), 1); @@ -87,7 +112,7 @@ namespace core { hr = attributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE); hr = attributes->SetUINT32(MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, TRUE); - hr = MFCreateSourceReaderFromURL(wide_path.c_str(), attributes.get(), m_source_reader.put()); + hr = MFCreateSourceReaderFromByteStream(m_byte_stream.get(), attributes.get(), m_source_reader.put()); if (FAILED(hr)) { Logger::error("[core] [VideoDecoder] Failed to create source reader, hr = {:#x}", (uint32_t)hr); return false; @@ -178,6 +203,7 @@ namespace core { void VideoDecoder::close() { m_source_reader.reset(); + m_byte_stream.reset(); m_media_type.reset(); m_texture.reset(); m_shader_resource_view.reset(); diff --git a/engine/graphics/d3d11/VideoDecoder.hpp b/engine/graphics/d3d11/VideoDecoder.hpp index 98940eb79..84544a5b4 100644 --- a/engine/graphics/d3d11/VideoDecoder.hpp +++ b/engine/graphics/d3d11/VideoDecoder.hpp @@ -59,6 +59,7 @@ namespace core { win32::com_ptr m_device_context; win32::com_ptr m_source_reader; + win32::com_ptr m_byte_stream; win32::com_ptr m_media_type; Vector2U m_video_size{}; diff --git a/engine/graphics/d3d11/VideoTexture.hpp b/engine/graphics/d3d11/VideoTexture.hpp index 0cfb2741a..120b26c10 100644 --- a/engine/graphics/d3d11/VideoTexture.hpp +++ b/engine/graphics/d3d11/VideoTexture.hpp @@ -19,6 +19,7 @@ namespace core { void* getNativeView() const noexcept override; bool isDynamic() const noexcept override { return true; } + bool isVideoTexture() const noexcept override { return true; } bool isPremultipliedAlpha() const noexcept override { return false; } void setPremultipliedAlpha(bool v) override {} Vector2U getSize() const noexcept override; From 6357b63dcb91f04fa7f44aefe766a794bc410816 Mon Sep 17 00:00:00 2001 From: OLC Date: Sun, 22 Feb 2026 00:36:48 +0800 Subject: [PATCH 03/12] feat: add D3D11 video decoding support and multithreading enhancements --- engine/graphics/d3d11/DeviceHelper.cpp | 4 +- engine/graphics/d3d11/VideoDecoder.cpp | 407 +++++++++++++++++++++++-- engine/graphics/d3d11/VideoDecoder.hpp | 9 + 3 files changed, 397 insertions(+), 23 deletions(-) diff --git a/engine/graphics/d3d11/DeviceHelper.cpp b/engine/graphics/d3d11/DeviceHelper.cpp index 090bc6919..ac9a02176 100644 --- a/engine/graphics/d3d11/DeviceHelper.cpp +++ b/engine/graphics/d3d11/DeviceHelper.cpp @@ -12,7 +12,7 @@ namespace d3d11 { ID3D11Device** const device, ID3D11DeviceContext** const device_context, D3D_FEATURE_LEVEL* const feature_level ) { const auto driver_type = (adapter != nullptr) ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE; - UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; + UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT; #if (!defined(NDEBUG) && defined(LUASTG_GRAPHICS_DEBUG_LAYER_ENABLE)) flags |= D3D11_CREATE_DEVICE_DEBUG; #endif @@ -49,7 +49,7 @@ namespace d3d11 { std::vector& results, const D3D_DRIVER_TYPE driver_type, ID3D11Device** const device, ID3D11DeviceContext** const device_context, D3D_FEATURE_LEVEL* const feature_level ) { - UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; + UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT; #if (!defined(NDEBUG) && defined(LUASTG_GRAPHICS_DEBUG_LAYER_ENABLE)) flags |= D3D11_CREATE_DEVICE_DEBUG; #endif diff --git a/engine/graphics/d3d11/VideoDecoder.cpp b/engine/graphics/d3d11/VideoDecoder.cpp index 08a20bfe7..27954a4b9 100644 --- a/engine/graphics/d3d11/VideoDecoder.cpp +++ b/engine/graphics/d3d11/VideoDecoder.cpp @@ -3,6 +3,7 @@ #include "core/Logger.hpp" #include "utf8.hpp" #include +#include #include #include #include @@ -102,20 +103,127 @@ namespace core { m_byte_stream = std::move(byte_stream); + auto d3d_device = static_cast(m_device->getNativeDevice()); + if (!d3d_device) { + Logger::error("[core] [VideoDecoder] Failed to get D3D11 device"); + return false; + } + + win32::com_ptr temp_context; + d3d_device->GetImmediateContext(temp_context.put()); + if (temp_context) { + win32::com_ptr multithread; + if (SUCCEEDED(temp_context->QueryInterface(IID_PPV_ARGS(multithread.put())))) { + multithread->SetMultithreadProtected(TRUE); + Logger::info("[core] [VideoDecoder] Enabled D3D11 multithread protection"); + } else { + Logger::warn("[core] [VideoDecoder] Failed to enable D3D11 multithread protection"); + } + } + + win32::com_ptr video_device; + hr = d3d_device->QueryInterface(IID_PPV_ARGS(video_device.put())); + if (SUCCEEDED(hr)) { + Logger::info("[core] [VideoDecoder] D3D11 device supports video decoding"); + } else { + Logger::warn("[core] [VideoDecoder] D3D11 device does not support video decoding (hr = {:#x})", (uint32_t)hr); + } + win32::com_ptr attributes; - hr = MFCreateAttributes(attributes.put(), 1); + hr = MFCreateAttributes(attributes.put(), 3); if (FAILED(hr)) { Logger::error("[core] [VideoDecoder] Failed to create attributes, hr = {:#x}", (uint32_t)hr); return false; } - hr = attributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE); - hr = attributes->SetUINT32(MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, TRUE); + win32::com_ptr dxgi_device_manager; + UINT dxgi_reset_token = 0; + bool use_hardware_accel = false; + + hr = MFCreateDXGIDeviceManager(&dxgi_reset_token, dxgi_device_manager.put()); + if (SUCCEEDED(hr)) { + hr = dxgi_device_manager->ResetDevice(d3d_device, dxgi_reset_token); + if (SUCCEEDED(hr)) { + if (video_device) { + hr = attributes->SetUnknown(MF_SOURCE_READER_D3D_MANAGER, dxgi_device_manager.get()); + if (SUCCEEDED(hr)) { + use_hardware_accel = true; + Logger::info("[core] [VideoDecoder] Attempting D3D11 hardware acceleration (IMFDXGIDeviceManager)"); + } else { + Logger::warn("[core] [VideoDecoder] Failed to set D3D manager attribute, hr = {:#x}", (uint32_t)hr); + } + } else { + Logger::warn("[core] [VideoDecoder] Skipping hardware acceleration - device does not support video"); + } + } else { + Logger::warn("[core] [VideoDecoder] Failed to reset DXGI device, hr = {:#x}", (uint32_t)hr); + } + } else { + Logger::warn("[core] [VideoDecoder] Failed to create DXGI device manager, hr = {:#x}", (uint32_t)hr); + } + + if (!use_hardware_accel) { + hr = attributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, TRUE); + if (FAILED(hr)) { + Logger::warn("[core] [VideoDecoder] Failed to set hardware transforms attribute, hr = {:#x}", (uint32_t)hr); + } + + hr = attributes->SetUINT32(MF_SOURCE_READER_ENABLE_VIDEO_PROCESSING, TRUE); + if (FAILED(hr)) { + Logger::warn("[core] [VideoDecoder] Failed to set video processing attribute, hr = {:#x}", (uint32_t)hr); + } + } hr = MFCreateSourceReaderFromByteStream(m_byte_stream.get(), attributes.get(), m_source_reader.put()); if (FAILED(hr)) { - Logger::error("[core] [VideoDecoder] Failed to create source reader, hr = {:#x}", (uint32_t)hr); - return false; + if (use_hardware_accel && hr == E_INVALIDARG) { + Logger::warn("[core] [VideoDecoder] Hardware acceleration failed (hr = {:#x}), retrying without D3D manager", (uint32_t)hr); + attributes->DeleteItem(MF_SOURCE_READER_D3D_MANAGER); + dxgi_device_manager.reset(); + use_hardware_accel = false; + + hr = MFCreateSourceReaderFromByteStream(m_byte_stream.get(), attributes.get(), m_source_reader.put()); + } + + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to create source reader, hr = {:#x}", (uint32_t)hr); + return false; + } else { + Logger::info("[core] [VideoDecoder] Falling back to software decoding"); + } + } else if (use_hardware_accel) { + Logger::info("[core] [VideoDecoder] Using D3D11 hardware acceleration (IMFDXGIDeviceManager)"); + } + + win32::com_ptr partial_media_type; + hr = m_source_reader->GetNativeMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, partial_media_type.put()); + if (FAILED(hr)) { + Logger::warn("[core] [VideoDecoder] Failed to get native media type, trying current type, hr = {:#x}", (uint32_t)hr); + hr = m_source_reader->GetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, partial_media_type.put()); + if (FAILED(hr)) { + Logger::warn("[core] [VideoDecoder] Failed to get current media type, hr = {:#x}", (uint32_t)hr); + } + } + + if (partial_media_type && use_hardware_accel) { + GUID native_subtype = GUID_NULL; + if (SUCCEEDED(partial_media_type->GetGUID(MF_MT_SUBTYPE, &native_subtype))) { + const char* format_name = "Unknown"; + if (native_subtype == MFVideoFormat_NV12) { + format_name = "NV12"; + } else if (native_subtype == MFVideoFormat_ARGB32) { + format_name = "ARGB32"; + } else if (native_subtype == MFVideoFormat_RGB32) { + format_name = "RGB32"; + } + Logger::info("[core] [VideoDecoder] Native hardware decoder format: {} ({:08x}-{:04x}-{:04x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x})", + format_name, + native_subtype.Data1, native_subtype.Data2, native_subtype.Data3, + native_subtype.Data4[0], native_subtype.Data4[1], + native_subtype.Data4[2], native_subtype.Data4[3], + native_subtype.Data4[4], native_subtype.Data4[5], + native_subtype.Data4[6], native_subtype.Data4[7]); + } } win32::com_ptr media_type; @@ -126,24 +234,89 @@ namespace core { } hr = media_type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to set major type, hr = {:#x}", (uint32_t)hr); + return false; + } + + if (use_hardware_accel && partial_media_type) { + GUID native_subtype = GUID_NULL; + if (SUCCEEDED(partial_media_type->GetGUID(MF_MT_SUBTYPE, &native_subtype))) { + constexpr GUID MFVideoFormat_H264 = { 0x34363248, 0x0000, 0x0010, { 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71 } }; + if (native_subtype == MFVideoFormat_H264 || native_subtype.Data1 == 0x34363248) { + Logger::info("[core] [VideoDecoder] Hardware decoder outputs compressed format, enumerating supported output formats"); + for (DWORD i = 0; ; ++i) { + win32::com_ptr output_type; + hr = m_source_reader->GetNativeMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, i, output_type.put()); + if (FAILED(hr)) { + break; + } + GUID output_subtype = GUID_NULL; + if (SUCCEEDED(output_type->GetGUID(MF_MT_SUBTYPE, &output_subtype))) { + const char* format_name = "Unknown"; + if (output_subtype == MFVideoFormat_NV12) format_name = "NV12"; + else if (output_subtype == MFVideoFormat_ARGB32) format_name = "ARGB32"; + else if (output_subtype == MFVideoFormat_RGB32) format_name = "RGB32"; + Logger::info("[core] [VideoDecoder] Output format {}: {}", i, format_name); + } + } + } + } + } + + if (partial_media_type) { + UINT32 width = 0, height = 0; + if (SUCCEEDED(MFGetAttributeSize(partial_media_type.get(), MF_MT_FRAME_SIZE, &width, &height))) { + hr = MFSetAttributeSize(media_type.get(), MF_MT_FRAME_SIZE, width, height); + if (FAILED(hr)) { + Logger::warn("[core] [VideoDecoder] Failed to set frame size, hr = {:#x}", (uint32_t)hr); + } + } + + MFVideoArea area = {}; + if (SUCCEEDED(partial_media_type->GetBlob(MF_MT_GEOMETRIC_APERTURE, (UINT8*)&area, sizeof(area), nullptr))) { + media_type->SetBlob(MF_MT_GEOMETRIC_APERTURE, (UINT8*)&area, sizeof(area)); + } + } hr = media_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_ARGB32); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to set ARGB32 subtype, hr = {:#x}", (uint32_t)hr); + return false; + } + hr = m_source_reader->SetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, media_type.get()); if (FAILED(hr)) { - Logger::info("[core] [VideoDecoder] ARGB32 not supported, trying RGB32"); + Logger::info("[core] [VideoDecoder] ARGB32 not supported (hr = {:#x}), trying RGB32", (uint32_t)hr); hr = media_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to set RGB32 subtype, hr = {:#x}", (uint32_t)hr); + return false; + } hr = m_source_reader->SetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, media_type.get()); if (FAILED(hr)) { - Logger::error("[core] [VideoDecoder] Failed to set media type (tried ARGB32 and RGB32), hr = {:#x}", (uint32_t)hr); - return false; + Logger::info("[core] [VideoDecoder] RGB32 not supported (hr = {:#x}), trying NV12 (hardware native)", (uint32_t)hr); + hr = media_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to set NV12 subtype, hr = {:#x}", (uint32_t)hr); + return false; + } + hr = m_source_reader->SetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, media_type.get()); + + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to set media type (tried ARGB32, RGB32, NV12), hr = {:#x}", (uint32_t)hr); + return false; + } + m_output_format_nv12 = true; + Logger::info("[core] [VideoDecoder] Using NV12 output, will convert to BGRA via Video Processor"); } } hr = m_source_reader->GetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, m_media_type.put()); if (FAILED(hr)) { - Logger::error("[core] [VideoDecoder] Failed to get media type, hr = {:#x}", (uint32_t)hr); + Logger::error("[core] [VideoDecoder] Failed to get current media type, hr = {:#x}", (uint32_t)hr); return false; } @@ -154,6 +327,8 @@ namespace core { Logger::info("[core] [VideoDecoder] Using video format: ARGB32 (native BGRA)"); } else if (subtype == MFVideoFormat_RGB32) { Logger::info("[core] [VideoDecoder] Using video format: RGB32"); + } else if (subtype == MFVideoFormat_NV12) { + Logger::info("[core] [VideoDecoder] Using video format: NV12 (hardware native, GPU conversion to BGRA)"); } else { Logger::warn("[core] [VideoDecoder] Using unknown video format, display may be incorrect"); } @@ -182,13 +357,14 @@ namespace core { return false; } - m_frame_pitch = m_target_size.x * 4; - - auto d3d_device = static_cast(m_device->getNativeDevice()); - if (d3d_device) { - d3d_device->GetImmediateContext(m_device_context.put()); + if (m_output_format_nv12 && !createVideoProcessor()) { + Logger::error("[core] [VideoDecoder] Failed to create Video Processor for NV12 conversion"); + close(); + return false; } + m_frame_pitch = m_target_size.x * 4; + m_current_time = 0.0; m_last_requested_time = -1.0; @@ -208,6 +384,11 @@ namespace core { m_texture.reset(); m_shader_resource_view.reset(); m_device_context.reset(); + m_video_processor.reset(); + m_video_processor_enum.reset(); + m_video_context.reset(); + m_video_device.reset(); + m_output_format_nv12 = false; m_video_size = Vector2U{}; m_target_size = Vector2U{}; @@ -367,13 +548,18 @@ namespace core { void VideoDecoder::onGraphicsDeviceCreate() { if (hasVideo()) { createTexture(); + if (m_output_format_nv12) { + createVideoProcessor(); + } } } void VideoDecoder::onGraphicsDeviceDestroy() { - m_texture.reset(); - m_shader_resource_view.reset(); - m_device_context.reset(); + m_video_processor.reset(); + m_video_processor_enum.reset(); + m_video_context.reset(); + m_video_device.reset(); + close(); } bool VideoDecoder::createTexture() { @@ -394,9 +580,15 @@ namespace core { desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM; desc.SampleDesc.Count = 1; desc.SampleDesc.Quality = 0; - desc.Usage = D3D11_USAGE_DYNAMIC; - desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; - desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; + if (m_output_format_nv12) { + desc.Usage = D3D11_USAGE_DEFAULT; + desc.BindFlags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE; + desc.CPUAccessFlags = 0; + } else { + desc.Usage = D3D11_USAGE_DYNAMIC; + desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + desc.CPUAccessFlags = D3D11_CPU_ACCESS_WRITE; + } desc.MiscFlags = 0; HRESULT hr = d3d_device->CreateTexture2D(&desc, nullptr, m_texture.put()); @@ -420,6 +612,150 @@ namespace core { d3d_device->GetImmediateContext(m_device_context.put()); + win32::com_ptr multithread; + if (SUCCEEDED(m_device_context->QueryInterface(IID_PPV_ARGS(multithread.put())))) { + multithread->SetMultithreadProtected(TRUE); + } + + return true; + } + + bool VideoDecoder::createVideoProcessor() { + if (!m_device || m_target_size.x == 0 || m_target_size.y == 0 || !m_texture) { + return false; + } + + auto d3d_device = static_cast(m_device->getNativeDevice()); + if (!d3d_device) { + return false; + } + + HRESULT hr = d3d_device->QueryInterface(IID_PPV_ARGS(m_video_device.put())); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to get ID3D11VideoDevice, hr = {:#x}", (uint32_t)hr); + return false; + } + + hr = m_device_context->QueryInterface(IID_PPV_ARGS(m_video_context.put())); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to get ID3D11VideoContext, hr = {:#x}", (uint32_t)hr); + return false; + } + + D3D11_VIDEO_PROCESSOR_CONTENT_DESC content_desc = {}; + content_desc.InputFrameFormat = D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE; + content_desc.InputFrameRate.Numerator = 30; + content_desc.InputFrameRate.Denominator = 1; + content_desc.InputWidth = m_target_size.x; + content_desc.InputHeight = m_target_size.y; + content_desc.OutputFrameRate.Numerator = 30; + content_desc.OutputFrameRate.Denominator = 1; + content_desc.OutputWidth = m_target_size.x; + content_desc.OutputHeight = m_target_size.y; + content_desc.Usage = D3D11_VIDEO_USAGE_PLAYBACK_NORMAL; + + hr = m_video_device->CreateVideoProcessorEnumerator(&content_desc, m_video_processor_enum.put()); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to create VideoProcessorEnumerator, hr = {:#x}", (uint32_t)hr); + return false; + } + + hr = m_video_device->CreateVideoProcessor(m_video_processor_enum.get(), 0, m_video_processor.put()); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to create VideoProcessor, hr = {:#x}", (uint32_t)hr); + return false; + } + + Logger::info("[core] [VideoDecoder] Video Processor created for NV12->BGRA conversion"); + return true; + } + + bool VideoDecoder::updateTextureFromNV12Sample(IMFSample* sample) { + if (!m_texture || !sample || !m_device_context || !m_video_device || !m_video_context + || !m_video_processor || !m_video_processor_enum) { + return false; + } + + DWORD buffer_count = 0; + HRESULT hr = sample->GetBufferCount(&buffer_count); + if (FAILED(hr) || buffer_count == 0) { + return false; + } + + win32::com_ptr buffer; + hr = sample->GetBufferByIndex(0, buffer.put()); + if (FAILED(hr)) { + return false; + } + + win32::com_ptr dxgi_buffer; + if (FAILED(buffer->QueryInterface(IID_PPV_ARGS(dxgi_buffer.put())))) { + Logger::error("[core] [VideoDecoder] NV12 sample does not have DXGI buffer"); + return false; + } + + win32::com_ptr nv12_texture; + hr = dxgi_buffer->GetResource(IID_PPV_ARGS(nv12_texture.put())); + if (FAILED(hr) || !nv12_texture) { + return false; + } + + D3D11_TEXTURE2D_DESC nv12_desc{}; + nv12_texture->GetDesc(&nv12_desc); + if (nv12_desc.Format != DXGI_FORMAT_NV12) { + Logger::error("[core] [VideoDecoder] Expected NV12 format, got {:x}", (uint32_t)nv12_desc.Format); + return false; + } + + UINT subresource_index = 0; + dxgi_buffer->GetSubresourceIndex(&subresource_index); + + D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC input_view_desc = {}; + input_view_desc.FourCC = 0; + input_view_desc.ViewDimension = D3D11_VPIV_DIMENSION_TEXTURE2D; + input_view_desc.Texture2D.ArraySlice = subresource_index / std::max(1u, nv12_desc.MipLevels); + input_view_desc.Texture2D.MipSlice = subresource_index % std::max(1u, nv12_desc.MipLevels); + + win32::com_ptr input_view; + hr = m_video_device->CreateVideoProcessorInputView( + nv12_texture.get(), m_video_processor_enum.get(), &input_view_desc, input_view.put()); + if (FAILED(hr)) { + input_view_desc.Texture2D.ArraySlice = 0; + input_view_desc.Texture2D.MipSlice = 0; + hr = m_video_device->CreateVideoProcessorInputView( + nv12_texture.get(), m_video_processor_enum.get(), &input_view_desc, input_view.put()); + } + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to create VideoProcessorInputView, hr = {:#x}", (uint32_t)hr); + return false; + } + + D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC output_view_desc = {}; + output_view_desc.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D; + output_view_desc.Texture2D.MipSlice = 0; + + win32::com_ptr output_view; + hr = m_video_device->CreateVideoProcessorOutputView( + m_texture.get(), m_video_processor_enum.get(), &output_view_desc, output_view.put()); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to create VideoProcessorOutputView, hr = {:#x}", (uint32_t)hr); + return false; + } + + D3D11_VIDEO_PROCESSOR_STREAM stream = {}; + stream.Enable = TRUE; + stream.OutputIndex = 0; + stream.InputFrameOrField = 0; + stream.PastFrames = 0; + stream.FutureFrames = 0; + stream.pInputSurface = input_view.get(); + + hr = m_video_context->VideoProcessorBlt(m_video_processor.get(), output_view.get(), 0, 1, &stream); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] VideoProcessorBlt failed, hr = {:#x}", (uint32_t)hr); + return false; + } + return true; } @@ -428,8 +764,37 @@ namespace core { return false; } + if (m_output_format_nv12 && m_video_processor) { + return updateTextureFromNV12Sample(sample); + } + + DWORD buffer_count = 0; + HRESULT hr = sample->GetBufferCount(&buffer_count); + if (SUCCEEDED(hr) && buffer_count > 0) { + win32::com_ptr buffer; + hr = sample->GetBufferByIndex(0, buffer.put()); + if (SUCCEEDED(hr)) { + win32::com_ptr dxgi_buffer; + if (SUCCEEDED(buffer->QueryInterface(IID_PPV_ARGS(dxgi_buffer.put())))) { + win32::com_ptr source_texture; + hr = dxgi_buffer->GetResource(IID_PPV_ARGS(source_texture.put())); + if (SUCCEEDED(hr) && source_texture) { + D3D11_TEXTURE2D_DESC src_desc{}; + source_texture->GetDesc(&src_desc); + D3D11_TEXTURE2D_DESC dst_desc{}; + m_texture->GetDesc(&dst_desc); + if (src_desc.Width == dst_desc.Width && src_desc.Height == dst_desc.Height + && src_desc.Format == dst_desc.Format) { + m_device_context->CopyResource(m_texture.get(), source_texture.get()); + return true; + } + } + } + } + } + win32::com_ptr buffer; - HRESULT hr = sample->ConvertToContiguousBuffer(buffer.put()); + hr = sample->ConvertToContiguousBuffer(buffer.put()); if (FAILED(hr)) { Logger::error("[core] [VideoDecoder] Failed to get buffer, hr = {:#x}", (uint32_t)hr); return false; diff --git a/engine/graphics/d3d11/VideoDecoder.hpp b/engine/graphics/d3d11/VideoDecoder.hpp index 84544a5b4..855a3c971 100644 --- a/engine/graphics/d3d11/VideoDecoder.hpp +++ b/engine/graphics/d3d11/VideoDecoder.hpp @@ -49,7 +49,9 @@ namespace core { private: bool createTexture(); + bool createVideoProcessor(); bool updateTextureFromSample(IMFSample* sample); + bool updateTextureFromNV12Sample(IMFSample* sample); bool readNextFrame(); bool readFrameAtTime(double time_in_seconds); @@ -62,6 +64,13 @@ namespace core { win32::com_ptr m_byte_stream; win32::com_ptr m_media_type; + win32::com_ptr m_video_device; + win32::com_ptr m_video_context; + win32::com_ptr m_video_processor; + win32::com_ptr m_video_processor_enum; + + bool m_output_format_nv12{ false }; + Vector2U m_video_size{}; Vector2U m_target_size{}; double m_duration{ 0.0 }; From 775f53c27f82c4b458a5ca56b2d2b2e0d3d732b2 Mon Sep 17 00:00:00 2001 From: OLC Date: Sun, 22 Feb 2026 09:45:08 +0800 Subject: [PATCH 04/12] refactor: remove video support flag from D3D11 device creation --- engine/graphics/d3d11/DeviceHelper.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/graphics/d3d11/DeviceHelper.cpp b/engine/graphics/d3d11/DeviceHelper.cpp index ac9a02176..090bc6919 100644 --- a/engine/graphics/d3d11/DeviceHelper.cpp +++ b/engine/graphics/d3d11/DeviceHelper.cpp @@ -12,7 +12,7 @@ namespace d3d11 { ID3D11Device** const device, ID3D11DeviceContext** const device_context, D3D_FEATURE_LEVEL* const feature_level ) { const auto driver_type = (adapter != nullptr) ? D3D_DRIVER_TYPE_UNKNOWN : D3D_DRIVER_TYPE_HARDWARE; - UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT; + UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; #if (!defined(NDEBUG) && defined(LUASTG_GRAPHICS_DEBUG_LAYER_ENABLE)) flags |= D3D11_CREATE_DEVICE_DEBUG; #endif @@ -49,7 +49,7 @@ namespace d3d11 { std::vector& results, const D3D_DRIVER_TYPE driver_type, ID3D11Device** const device, ID3D11DeviceContext** const device_context, D3D_FEATURE_LEVEL* const feature_level ) { - UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT; + UINT flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; #if (!defined(NDEBUG) && defined(LUASTG_GRAPHICS_DEBUG_LAYER_ENABLE)) flags |= D3D11_CREATE_DEVICE_DEBUG; #endif From e07f225db6e77919008b191a98a6fcc9017f97a2 Mon Sep 17 00:00:00 2001 From: OLC Date: Sun, 22 Feb 2026 23:39:06 +0800 Subject: [PATCH 05/12] feat: enhance video decoding with frame rate handling and dynamic tolerance adjustments --- engine/graphics/d3d11/VideoDecoder.cpp | 22 +++++++++++++++++----- engine/graphics/d3d11/VideoDecoder.hpp | 3 ++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/engine/graphics/d3d11/VideoDecoder.cpp b/engine/graphics/d3d11/VideoDecoder.cpp index 27954a4b9..966d0a123 100644 --- a/engine/graphics/d3d11/VideoDecoder.cpp +++ b/engine/graphics/d3d11/VideoDecoder.cpp @@ -352,6 +352,16 @@ namespace core { PropVariantClear(&var); } + UINT32 rate_num = 0, rate_den = 0; + hr = MFGetAttributeRatio(m_media_type.get(), MF_MT_FRAME_RATE, &rate_num, &rate_den); + if (SUCCEEDED(hr) && rate_den > 0 && rate_num > 0) { + m_frame_interval = static_cast(rate_den) / static_cast(rate_num); + Logger::info("[core] [VideoDecoder] Frame rate: {} / {} (= {:.2f} fps)", rate_num, rate_den, 1.0 / m_frame_interval); + } else { + m_frame_interval = 1.0 / 30.0; + Logger::warn("[core] [VideoDecoder] MF_MT_FRAME_RATE not available, assuming 30 fps"); + } + if (!createTexture()) { close(); return false; @@ -395,6 +405,7 @@ namespace core { m_duration = 0.0; m_current_time = 0.0; m_last_requested_time = -1.0; + m_frame_interval = 1.0 / 30.0; m_frame_pitch = 0; } @@ -452,11 +463,12 @@ namespace core { constexpr double kTimeEpsilon = 1e-4; constexpr double kBackwardTolerance = 1.0 / 120.0; - constexpr double kFrameTolerance = 1.0 / 24.0; - constexpr double kSeekThreshold = 0.25; + double const frame_tolerance = m_frame_interval * 0.5; + double const seek_threshold = (std::max)(0.1, 4.0 * m_frame_interval); + int const max_catch_up_frames = (std::min)(16, (std::max)(1, static_cast(0.2 / m_frame_interval))); bool const is_backward = (m_last_requested_time >= 0.0) && (time_in_seconds + kBackwardTolerance < m_last_requested_time); - bool const large_jump = (m_current_time + kSeekThreshold < time_in_seconds); + bool const large_jump = (m_current_time + seek_threshold < time_in_seconds); m_last_requested_time = time_in_seconds; @@ -464,11 +476,11 @@ namespace core { return readFrameAtTime(time_in_seconds); } - if (m_current_time + kFrameTolerance >= time_in_seconds) { + if (m_current_time + frame_tolerance >= time_in_seconds) { return true; } - for (int i = 0; i < 4; ++i) { + for (int i = 0; i < max_catch_up_frames; ++i) { if (!readNextFrame()) { return false; } diff --git a/engine/graphics/d3d11/VideoDecoder.hpp b/engine/graphics/d3d11/VideoDecoder.hpp index 855a3c971..f122fa1c9 100644 --- a/engine/graphics/d3d11/VideoDecoder.hpp +++ b/engine/graphics/d3d11/VideoDecoder.hpp @@ -74,8 +74,9 @@ namespace core { Vector2U m_video_size{}; Vector2U m_target_size{}; double m_duration{ 0.0 }; - double m_current_time{ 0.0 }; // 当前帧时间 + double m_current_time{ 0.0 }; // 当前帧 PTS（秒） double m_last_requested_time{ -1.0 }; + double m_frame_interval{ 1.0 / 30.0 }; // 每帧时长（秒），由 MF_MT_FRAME_RATE 解析，用于 tolerance/seek 阈值 bool m_looping{ false }; uint32_t m_frame_pitch{ 0 }; From 5c7f96c370f5d2b276bd52b15a9456da8134fe98 Mon Sep 17 00:00:00 2001 From: OLC Date: Mon, 23 Feb 2026 00:40:58 +0800 Subject: [PATCH 06/12] feat: add frame interval and fps calculation to video resource management --- LuaSTG/LuaSTG/GameResource/ResourceDebug.cpp | 4 +- LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp | 10 +++- data/example/video_example.lua | 6 +-- engine/configuration/core/Configuration.cpp | 6 +++ engine/configuration/core/Configuration.hpp | 2 + engine/graphics/core/VideoDecoder.hpp | 1 + engine/graphics/d3d11/VideoDecoder.cpp | 48 +++++++++++++++----- engine/graphics/d3d11/VideoDecoder.hpp | 8 ++-- 8 files changed, 65 insertions(+), 20 deletions(-) diff --git a/LuaSTG/LuaSTG/GameResource/ResourceDebug.cpp b/LuaSTG/LuaSTG/GameResource/ResourceDebug.cpp index 74c034431..a72cd4621 100644 --- a/LuaSTG/LuaSTG/GameResource/ResourceDebug.cpp +++ b/LuaSTG/LuaSTG/GameResource/ResourceDebug.cpp @@ -1,6 +1,7 @@ #include "GameResource/ResourceManager.h" #ifdef USING_DEAR_IMGUI #include "imgui.h" +#include "d3d11/VideoTexture.hpp" #endif static std::string bytes_count_to_string(unsigned long long size) @@ -89,7 +90,8 @@ namespace luastg char const* type_str = p_tex->isVideoTexture() ? "Video" : (p_res->IsRenderTarget() ? "RenderTarget" : "Texture"); ImGui::Text("Type: %s", type_str); ImGui::Text("Dynamic: %s", p_tex->isDynamic() ? "Yes" : "Not"); - unsigned long long mem_usage = size.x * size.y * 4; + unsigned long long display_mem = (unsigned long long)size.x * size.y * 4; + unsigned long long mem_usage = display_mem; ImGui::Text("Adapter Memory Usage (Approximate): %s", bytes_count_to_string(mem_usage).c_str()); } ImGui::PushStyleVar(ImGuiStyleVar_ImageBorderSize, 1.0); diff --git a/LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp b/LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp index 1957cdb11..7e52d5696 100644 --- a/LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp +++ b/LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp @@ -737,7 +737,7 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept if (!decoder) return luaL_error(L, "video texture '%s' not found.", name); - lua_createtable(L, 0, 5); + lua_createtable(L, 0, 7); lua_pushnumber(L, decoder->getDuration()); lua_setfield(L, -2, "duration"); @@ -755,6 +755,14 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept lua_pushinteger(L, size.y); lua_setfield(L, -2, "height"); + double fi = decoder->getFrameInterval(); + lua_pushnumber(L, fi); + lua_setfield(L, -2, "frame_interval"); + if (fi > 0.0) { + lua_pushnumber(L, 1.0 / fi); + lua_setfield(L, -2, "fps"); + } + return 1; } }; diff --git a/data/example/video_example.lua b/data/example/video_example.lua index 9c0352454..2b39173ee 100644 --- a/data/example/video_example.lua +++ b/data/example/video_example.lua @@ -21,6 +21,6 @@ end -- 可用控制： -- 1. VideoSetLooping(name, bool) -- 设置是否循环播放 --- 2. VideoSeek(name, time) -- 控制时间，但是不更新画面，适合进度条等场景，完成后需要调用 VideoUpdate 来刷新画面 --- 3. VideoUpdate(name, absolute_time) -- 根据绝对时间更新视频画面，适合与 StopWatch 结合使用，确保视频播放与游戏时间同步 --- 4. VideoGetInfo(name) -- 获取视频信息，如帧率、总时长等，返回一个表格 +-- 2. VideoSeek(name, time) -- 控制时间，不更新画面，适合进度条；完成后需 VideoUpdate 刷新 +-- 3. VideoUpdate(name, absolute_time) -- 按绝对时间更新画面；传入比上次小的时间可倒放/拖拽到该时刻 +-- 4. VideoGetInfo(name) -- 获取视频信息（时长、当前时间、尺寸等） diff --git a/engine/configuration/core/Configuration.cpp b/engine/configuration/core/Configuration.cpp index 6b0b77eab..13de3f3fc 100644 --- a/engine/configuration/core/Configuration.cpp +++ b/engine/configuration/core/Configuration.cpp @@ -467,6 +467,11 @@ namespace core { assert_type_is_boolean(allow_direct_composition, "/graphics_system/allow_direct_composition"sv); loader.graphics_system.setAllowDirectComposition(allow_direct_composition.get()); } + if (graphics_system.contains("disable_hardware_video_decode"sv)) { + auto const& v = graphics_system.at("disable_hardware_video_decode"sv); + assert_type_is_boolean(v, "/graphics_system/disable_hardware_video_decode"sv); + loader.graphics_system.setDisableHardwareVideoDecode(v.get()); + } // TODO: display } @@ -738,6 +743,7 @@ namespace { { .type = OptionType::boolean, .prefix = "--graphics_system.allow_exclusive_fullscreen="sv, .path = "/graphics_system/allow_exclusive_fullscreen"_json_pointer }, { .type = OptionType::boolean, .prefix = "--graphics_system.allow_modern_swap_chain="sv , .path = "/graphics_system/allow_modern_swap_chain"_json_pointer }, { .type = OptionType::boolean, .prefix = "--graphics_system.allow_direct_composition="sv , .path = "/graphics_system/allow_direct_composition"_json_pointer }, + { .type = OptionType::boolean, .prefix = "--graphics_system.disable_hardware_video_decode="sv, .path = "/graphics_system/disable_hardware_video_decode"_json_pointer }, // audio_system { .type = OptionType::string , .prefix = "--audio_system.preferred_endpoint_name="sv, .path = "/audio_system/preferred_endpoint_name"_json_pointer }, { .type = OptionType::number , .prefix = "--audio_system.sound_effect_volume="sv , .path = "/audio_system/sound_effect_volume"_json_pointer }, diff --git a/engine/configuration/core/Configuration.hpp b/engine/configuration/core/Configuration.hpp index b02bb8a58..8ec80a13d 100644 --- a/engine/configuration/core/Configuration.hpp +++ b/engine/configuration/core/Configuration.hpp @@ -153,6 +153,7 @@ namespace core { GetterSetterBoolean(GraphicsSystem, allow_exclusive_fullscreen, AllowExclusiveFullscreen); GetterSetterBoolean(GraphicsSystem, allow_modern_swap_chain, AllowModernSwapChain); GetterSetterBoolean(GraphicsSystem, allow_direct_composition, AllowDirectComposition); + GetterSetterBoolean(GraphicsSystem, disable_hardware_video_decode, DisableHardwareVideoDecode); private: std::string preferred_device_name; uint32_t width{ 640u }; @@ -163,6 +164,7 @@ namespace core { bool allow_exclusive_fullscreen{ true }; bool allow_modern_swap_chain{ true }; bool allow_direct_composition{ true }; + bool disable_hardware_video_decode{ false }; }; class AudioSystem { public: diff --git a/engine/graphics/core/VideoDecoder.hpp b/engine/graphics/core/VideoDecoder.hpp index c32b8df18..3f16e2ca6 100644 --- a/engine/graphics/core/VideoDecoder.hpp +++ b/engine/graphics/core/VideoDecoder.hpp @@ -18,6 +18,7 @@ namespace core { virtual Vector2U getVideoSize() const noexcept = 0; virtual double getDuration() const noexcept = 0; virtual double getCurrentTime() const noexcept = 0; // 返回上次更新的帧时间 + virtual double getFrameInterval() const noexcept = 0; // 每帧时长（秒），用于 fps = 1/getFrameInterval() // 跳转到指定时间 virtual bool seek(double time_in_seconds) = 0; diff --git a/engine/graphics/d3d11/VideoDecoder.cpp b/engine/graphics/d3d11/VideoDecoder.cpp index 966d0a123..3b715e88a 100644 --- a/engine/graphics/d3d11/VideoDecoder.cpp +++ b/engine/graphics/d3d11/VideoDecoder.cpp @@ -1,4 +1,5 @@ #include "d3d11/VideoDecoder.hpp" +#include "core/Configuration.hpp" #include "core/FileSystem.hpp" #include "core/Logger.hpp" #include "utf8.hpp" @@ -140,11 +141,12 @@ namespace core { UINT dxgi_reset_token = 0; bool use_hardware_accel = false; + bool const hw_decode_disabled = core::ConfigurationLoader::getInstance().getGraphicsSystem().isDisableHardwareVideoDecode(); hr = MFCreateDXGIDeviceManager(&dxgi_reset_token, dxgi_device_manager.put()); if (SUCCEEDED(hr)) { hr = dxgi_device_manager->ResetDevice(d3d_device, dxgi_reset_token); if (SUCCEEDED(hr)) { - if (video_device) { + if (video_device && !hw_decode_disabled) { hr = attributes->SetUnknown(MF_SOURCE_READER_D3D_MANAGER, dxgi_device_manager.get()); if (SUCCEEDED(hr)) { use_hardware_accel = true; @@ -152,6 +154,8 @@ namespace core { } else { Logger::warn("[core] [VideoDecoder] Failed to set D3D manager attribute, hr = {:#x}", (uint32_t)hr); } + } else if (hw_decode_disabled) { + Logger::info("[core] [VideoDecoder] Hardware video decode disabled by config"); } else { Logger::warn("[core] [VideoDecoder] Skipping hardware acceleration - device does not support video"); } @@ -353,13 +357,22 @@ namespace core { } UINT32 rate_num = 0, rate_den = 0; - hr = MFGetAttributeRatio(m_media_type.get(), MF_MT_FRAME_RATE, &rate_num, &rate_den); - if (SUCCEEDED(hr) && rate_den > 0 && rate_num > 0) { + for (DWORD i = 0; ; ++i) { + win32::com_ptr mt; + hr = m_source_reader->GetNativeMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, i, mt.put()); + if (FAILED(hr)) break; + if (SUCCEEDED(MFGetAttributeRatio(mt.get(), MF_MT_FRAME_RATE, &rate_num, &rate_den)) && rate_den > 0 && rate_num > 0) + break; + } + if (rate_den == 0 || rate_num == 0) { + hr = MFGetAttributeRatio(m_media_type.get(), MF_MT_FRAME_RATE, &rate_num, &rate_den); + } + if (rate_den > 0 && rate_num > 0) { m_frame_interval = static_cast(rate_den) / static_cast(rate_num); Logger::info("[core] [VideoDecoder] Frame rate: {} / {} (= {:.2f} fps)", rate_num, rate_den, 1.0 / m_frame_interval); } else { - m_frame_interval = 1.0 / 30.0; - Logger::warn("[core] [VideoDecoder] MF_MT_FRAME_RATE not available, assuming 30 fps"); + m_frame_interval = 1.0 / 60.0; + Logger::warn("[core] [VideoDecoder] MF_MT_FRAME_RATE not available, assuming 60 fps"); } if (!createTexture()) { @@ -458,13 +471,19 @@ namespace core { return ok; } } else if (time_in_seconds < 0.0) { - time_in_seconds = 0.0; + if (m_looping) { + double t = std::fmod(time_in_seconds, m_duration); + if (t < 0.0) t += m_duration; + time_in_seconds = t; + } else { + time_in_seconds = 0.0; + } } constexpr double kTimeEpsilon = 1e-4; constexpr double kBackwardTolerance = 1.0 / 120.0; double const frame_tolerance = m_frame_interval * 0.5; - double const seek_threshold = (std::max)(0.1, 4.0 * m_frame_interval); + double const seek_threshold = (std::max)(0.1, 16.0 * m_frame_interval); int const max_catch_up_frames = (std::min)(16, (std::max)(1, static_cast(0.2 / m_frame_interval))); bool const is_backward = (m_last_requested_time >= 0.0) && (time_in_seconds + kBackwardTolerance < m_last_requested_time); @@ -491,7 +510,7 @@ namespace core { return readFrameAtTime(time_in_seconds); } - + bool VideoDecoder::readNextFrame() { if (!hasVideo()) { return false; @@ -683,7 +702,12 @@ namespace core { } bool VideoDecoder::updateTextureFromNV12Sample(IMFSample* sample) { - if (!m_texture || !sample || !m_device_context || !m_video_device || !m_video_context + if (!m_texture || !sample) return false; + return updateTextureFromNV12SampleTo(sample, m_texture.get()); + } + + bool VideoDecoder::updateTextureFromNV12SampleTo(IMFSample* sample, ID3D11Texture2D* output_texture) { + if (!output_texture || !sample || !m_device_context || !m_video_device || !m_video_context || !m_video_processor || !m_video_processor_enum) { return false; } @@ -748,7 +772,7 @@ namespace core { win32::com_ptr output_view; hr = m_video_device->CreateVideoProcessorOutputView( - m_texture.get(), m_video_processor_enum.get(), &output_view_desc, output_view.put()); + output_texture, m_video_processor_enum.get(), &output_view_desc, output_view.put()); if (FAILED(hr)) { Logger::error("[core] [VideoDecoder] Failed to create VideoProcessorOutputView, hr = {:#x}", (uint32_t)hr); return false; @@ -775,9 +799,9 @@ namespace core { if (!m_texture || !sample || !m_device_context) { return false; } - + std::lock_guard lock(m_device_context_mutex); if (m_output_format_nv12 && m_video_processor) { - return updateTextureFromNV12Sample(sample); + return updateTextureFromNV12SampleTo(sample, m_texture.get()); } DWORD buffer_count = 0; diff --git a/engine/graphics/d3d11/VideoDecoder.hpp b/engine/graphics/d3d11/VideoDecoder.hpp index f122fa1c9..f8fb0db32 100644 --- a/engine/graphics/d3d11/VideoDecoder.hpp +++ b/engine/graphics/d3d11/VideoDecoder.hpp @@ -7,7 +7,6 @@ #include #include #include -#include #include namespace core { @@ -25,6 +24,7 @@ namespace core { Vector2U getVideoSize() const noexcept override { return m_target_size; } double getDuration() const noexcept override { return m_duration; } double getCurrentTime() const noexcept override { return m_current_time; } + double getFrameInterval() const noexcept override { return m_frame_interval; } bool seek(double time_in_seconds) override; @@ -52,9 +52,11 @@ namespace core { bool createVideoProcessor(); bool updateTextureFromSample(IMFSample* sample); bool updateTextureFromNV12Sample(IMFSample* sample); + bool updateTextureFromNV12SampleTo(IMFSample* sample, ID3D11Texture2D* output_texture); bool readNextFrame(); bool readFrameAtTime(double time_in_seconds); + std::mutex m_device_context_mutex; SmartReference m_device; win32::com_ptr m_texture; win32::com_ptr m_shader_resource_view; @@ -74,9 +76,9 @@ namespace core { Vector2U m_video_size{}; Vector2U m_target_size{}; double m_duration{ 0.0 }; - double m_current_time{ 0.0 }; // 当前帧 PTS（秒） + double m_current_time{ 0.0 }; double m_last_requested_time{ -1.0 }; - double m_frame_interval{ 1.0 / 30.0 }; // 每帧时长（秒），由 MF_MT_FRAME_RATE 解析，用于 tolerance/seek 阈值 + double m_frame_interval{ 1.0 / 30.0 }; bool m_looping{ false }; uint32_t m_frame_pitch{ 0 }; From 47d25f8e9851889f9a1c997f2a518454af3bfee9 Mon Sep 17 00:00:00 2001 From: OLC Date: Mon, 23 Feb 2026 04:21:15 +0800 Subject: [PATCH 07/12] feat: enhance video resource management with flexible loading options and stream handling --- LuaSTG/LuaSTG/GameResource/ResourceManager.h | 389 ++++++++++--------- LuaSTG/LuaSTG/GameResource/ResourcePool.cpp | 9 +- LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp | 154 +++++++- data/example/video_example.lua | 36 +- engine/graphics/core/GraphicsDevice.hpp | 1 + engine/graphics/core/VideoDecoder.hpp | 46 ++- engine/graphics/d3d11/GraphicsDevice.hpp | 1 + engine/graphics/d3d11/VideoDecoder.cpp | 212 ++++++++-- engine/graphics/d3d11/VideoDecoder.hpp | 23 +- engine/graphics/d3d11/VideoTexture.cpp | 18 +- engine/graphics/d3d11/VideoTexture.hpp | 16 +- 11 files changed, 648 insertions(+), 257 deletions(-) diff --git a/LuaSTG/LuaSTG/GameResource/ResourceManager.h b/LuaSTG/LuaSTG/GameResource/ResourceManager.h index 6d98bd058..0bb5d45b4 100644 --- a/LuaSTG/LuaSTG/GameResource/ResourceManager.h +++ b/LuaSTG/LuaSTG/GameResource/ResourceManager.h @@ -1,194 +1,195 @@ -#pragma once -#include "core/SmartReference.hpp" -#include "GameResource/ResourceTexture.hpp" -#include "GameResource/ResourceSprite.hpp" -#include "GameResource/ResourceAnimation.hpp" -#include "GameResource/ResourceMusic.hpp" -#include "GameResource/ResourceSoundEffect.hpp" -#include "GameResource/ResourceParticle.hpp" -#include "GameResource/ResourceFont.hpp" -#include "GameResource/ResourcePostEffectShader.hpp" -#include "GameResource/ResourceModel.hpp" -#include "lua.hpp" -#include "xxhash.h" - -namespace luastg -{ - class ResourceMgr; - - // 资源池类型 - enum class ResourcePoolType - { - None = 0, - Global, - Stage - }; - - // 资源池 - class ResourcePool - { - friend class ResourceMgr; - public: - struct dictionary_key_t - { - #if (SIZE_MAX == UINT32_MAX) - XXH32_hash_t const hash{}; - XXH32_hash_t const check{}; - #elif (SIZE_MAX == UINT64_MAX) - XXH64_hash_t const hash{}; - #else - static_assert(false, "unsupported platform"); - #endif - - inline dictionary_key_t(std::string_view key) noexcept - #if (SIZE_MAX == UINT32_MAX) - : hash(XXH32(key.data(), key.size(), 0)) - , check(XXH32(key.data(), key.size(), 0x65766F6C)) - #elif (SIZE_MAX == UINT64_MAX) - : hash(XXH3_64bits(key.data(), key.size())) - #else - : __invalid_member() - #endif - { - } - - inline bool operator==(dictionary_key_t const& right) const noexcept - { - #if (SIZE_MAX == UINT32_MAX) - return hash == right.hash && check == right.check; - #elif (SIZE_MAX == UINT64_MAX) - return hash == right.hash; - #else - static_assert(false, "unsupported platform"); - #endif - } - }; - struct dictionary_key_hash_t - { - inline size_t operator()(dictionary_key_t const& key) const noexcept - { - static_assert(sizeof(size_t) == sizeof(decltype(dictionary_key_t::hash))); - return key.hash; - } - }; - template - using dictionary_t = std::pmr::unordered_map; - private: - ResourceMgr* m_pMgr; - ResourcePoolType m_iType; - std::pmr::unsynchronized_pool_resource m_memory_resource; - dictionary_t> m_TexturePool; - dictionary_t> m_SpritePool; - dictionary_t> m_AnimationPool; - dictionary_t> m_MusicPool; - dictionary_t> m_SoundSpritePool; - dictionary_t> m_ParticlePool; - dictionary_t> m_SpriteFontPool; - dictionary_t> m_TTFFontPool; - dictionary_t> m_FXPool; - dictionary_t> m_ModelPool; - private: - const char* getResourcePoolTypeName(); - public: - void Clear() noexcept; - void RemoveResource(ResourceType t, const char* name) noexcept; - bool CheckResourceExists(ResourceType t, std::string_view name) const noexcept; - int ExportResourceList(lua_State* L, ResourceType t) const noexcept; - - // 纹理 - bool LoadTexture(const char* name, const char* path, bool mipmaps = true) noexcept; - bool CreateTexture(const char* name, int width, int height) noexcept; - // 视频纹理 - bool LoadVideo(const char* name, const char* path) noexcept; - // 渲染目标 - bool CreateRenderTarget(const char* name, int width = 0, int height = 0, bool depth_buffer = false) noexcept; - // 图片精灵 - bool CreateSprite(const char* name, const char* texname, - double x, double y, double w, double h, - double a, double b, bool rect = false) noexcept; - // 动画精灵 - bool CreateAnimation(const char* name, const char* texname, - double x, double y, double w, double h, int n, int m, int intv, - double a, double b, bool rect = false) noexcept; - bool CreateAnimation(const char* name, - std::vector> const& sprite_list, - int intv, - double a, double b, bool rect = false) noexcept; - // 音乐 - bool LoadMusic(const char* name, const char* path, double start, double end, bool once_decode) noexcept; - // 音效 - bool LoadSoundEffect(const char* name, const char* path) noexcept; - // 粒子特效(HGE) - bool LoadParticle(const char* name, const hgeParticleSystemInfo& info, const char* img_name, - double a, double b, bool rect = false, bool _nolog = false) noexcept; - bool LoadParticle(const char* name, const char* path, const char* img_name, - double a, double b, bool rect = false) noexcept; - // 装载纹理字体(HGE) - bool LoadSpriteFont(const char* name, const char* path, bool mipmaps = true) noexcept; - // 装载纹理字体(fancy2d) - bool LoadSpriteFont(const char* name, const char* path, const char* tex_path, bool mipmaps = true) noexcept; - // 加载矢量字体 - bool LoadTTFFont(const char* name, const char* path, float width, float height) noexcept; - bool LoadTrueTypeFont(const char* name, core::Graphics::TrueTypeFontInfo* fonts, size_t count) noexcept; - // 特效 - bool LoadFX(const char* name, const char* path) noexcept; - // 模型 - bool LoadModel(const char* name, const char* path) noexcept; - - core::SmartReference GetTexture(std::string_view name) noexcept; - core::SmartReference GetSprite(std::string_view name) noexcept; - core::SmartReference GetAnimation(std::string_view name) noexcept; - core::SmartReference GetMusic(std::string_view name) noexcept; - core::SmartReference GetSound(std::string_view name) noexcept; - core::SmartReference GetParticle(std::string_view name) noexcept; - core::SmartReference GetSpriteFont(std::string_view name) noexcept; - core::SmartReference GetTTFFont(std::string_view name) noexcept; - core::SmartReference GetFX(std::string_view name) noexcept; - core::SmartReference GetModel(std::string_view name) noexcept; - public: - ResourcePool(ResourceMgr* mgr, ResourcePoolType t); - ResourcePool& operator=(const ResourcePool&) = delete; - ResourcePool(const ResourcePool&) = delete; - }; - - // 资源管理器 - class ResourceMgr - { - private: - ResourcePoolType m_ActivedPool = ResourcePoolType::Global; - ResourcePool m_GlobalResourcePool; - ResourcePool m_StageResourcePool; - public: - ResourcePoolType GetActivedPoolType() noexcept; - void SetActivedPoolType(ResourcePoolType t) noexcept; - ResourcePool* GetActivedPool() noexcept; - ResourcePool* GetResourcePool(ResourcePoolType t) noexcept; - void ClearAllResource() noexcept; - - core::SmartReference FindTexture(const char* name) noexcept; - core::SmartReference FindSprite(const char* name) noexcept; - core::SmartReference FindAnimation(const char* name) noexcept; - core::SmartReference FindMusic(const char* name) noexcept; - core::SmartReference FindSound(const char* name) noexcept; - core::SmartReference FindParticle(const char* name) noexcept; - core::SmartReference FindSpriteFont(const char* name) noexcept; - core::SmartReference FindTTFFont(const char* name) noexcept; - core::SmartReference FindFX(const char* name) noexcept; - core::SmartReference FindModel(const char* name) noexcept; - - bool GetTextureSize(const char* name, core::Vector2U& out) noexcept; - void CacheTTFFontString(const char* name, const char* text, size_t len) noexcept; - void UpdateSound(); - private: - static bool g_ResourceLoadingLog; - float m_GlobalImageScaleFactor = 1.0f; - public: - static void SetResourceLoadingLog(bool b); - static bool GetResourceLoadingLog(); - float GetGlobalImageScaleFactor() const noexcept { return m_GlobalImageScaleFactor; } - void SetGlobalImageScaleFactor(float s) noexcept { m_GlobalImageScaleFactor = s; } - void ShowResourceManagerDebugWindow(bool* p_open = nullptr); - public: - ResourceMgr(); - }; -} +#pragma once +#include "core/SmartReference.hpp" +#include "core/VideoDecoder.hpp" +#include "GameResource/ResourceTexture.hpp" +#include "GameResource/ResourceSprite.hpp" +#include "GameResource/ResourceAnimation.hpp" +#include "GameResource/ResourceMusic.hpp" +#include "GameResource/ResourceSoundEffect.hpp" +#include "GameResource/ResourceParticle.hpp" +#include "GameResource/ResourceFont.hpp" +#include "GameResource/ResourcePostEffectShader.hpp" +#include "GameResource/ResourceModel.hpp" +#include "lua.hpp" +#include "xxhash.h" + +namespace luastg +{ + class ResourceMgr; + + // 资源池类型 + enum class ResourcePoolType + { + None = 0, + Global, + Stage + }; + + // 资源池 + class ResourcePool + { + friend class ResourceMgr; + public: + struct dictionary_key_t + { + #if (SIZE_MAX == UINT32_MAX) + XXH32_hash_t const hash{}; + XXH32_hash_t const check{}; + #elif (SIZE_MAX == UINT64_MAX) + XXH64_hash_t const hash{}; + #else + static_assert(false, "unsupported platform"); + #endif + + inline dictionary_key_t(std::string_view key) noexcept + #if (SIZE_MAX == UINT32_MAX) + : hash(XXH32(key.data(), key.size(), 0)) + , check(XXH32(key.data(), key.size(), 0x65766F6C)) + #elif (SIZE_MAX == UINT64_MAX) + : hash(XXH3_64bits(key.data(), key.size())) + #else + : __invalid_member() + #endif + { + } + + inline bool operator==(dictionary_key_t const& right) const noexcept + { + #if (SIZE_MAX == UINT32_MAX) + return hash == right.hash && check == right.check; + #elif (SIZE_MAX == UINT64_MAX) + return hash == right.hash; + #else + static_assert(false, "unsupported platform"); + #endif + } + }; + struct dictionary_key_hash_t + { + inline size_t operator()(dictionary_key_t const& key) const noexcept + { + static_assert(sizeof(size_t) == sizeof(decltype(dictionary_key_t::hash))); + return key.hash; + } + }; + template + using dictionary_t = std::pmr::unordered_map; + private: + ResourceMgr* m_pMgr; + ResourcePoolType m_iType; + std::pmr::unsynchronized_pool_resource m_memory_resource; + dictionary_t> m_TexturePool; + dictionary_t> m_SpritePool; + dictionary_t> m_AnimationPool; + dictionary_t> m_MusicPool; + dictionary_t> m_SoundSpritePool; + dictionary_t> m_ParticlePool; + dictionary_t> m_SpriteFontPool; + dictionary_t> m_TTFFontPool; + dictionary_t> m_FXPool; + dictionary_t> m_ModelPool; + private: + const char* getResourcePoolTypeName(); + public: + void Clear() noexcept; + void RemoveResource(ResourceType t, const char* name) noexcept; + bool CheckResourceExists(ResourceType t, std::string_view name) const noexcept; + int ExportResourceList(lua_State* L, ResourceType t) const noexcept; + + // 纹理 + bool LoadTexture(const char* name, const char* path, bool mipmaps = true) noexcept; + bool CreateTexture(const char* name, int width, int height) noexcept; + // 视频纹理（options 为 nullptr 时使用默认选项） + bool LoadVideo(const char* name, const char* path, core::VideoOpenOptions const* options = nullptr) noexcept; + // 渲染目标 + bool CreateRenderTarget(const char* name, int width = 0, int height = 0, bool depth_buffer = false) noexcept; + // 图片精灵 + bool CreateSprite(const char* name, const char* texname, + double x, double y, double w, double h, + double a, double b, bool rect = false) noexcept; + // 动画精灵 + bool CreateAnimation(const char* name, const char* texname, + double x, double y, double w, double h, int n, int m, int intv, + double a, double b, bool rect = false) noexcept; + bool CreateAnimation(const char* name, + std::vector> const& sprite_list, + int intv, + double a, double b, bool rect = false) noexcept; + // 音乐 + bool LoadMusic(const char* name, const char* path, double start, double end, bool once_decode) noexcept; + // 音效 + bool LoadSoundEffect(const char* name, const char* path) noexcept; + // 粒子特效(HGE) + bool LoadParticle(const char* name, const hgeParticleSystemInfo& info, const char* img_name, + double a, double b, bool rect = false, bool _nolog = false) noexcept; + bool LoadParticle(const char* name, const char* path, const char* img_name, + double a, double b, bool rect = false) noexcept; + // 装载纹理字体(HGE) + bool LoadSpriteFont(const char* name, const char* path, bool mipmaps = true) noexcept; + // 装载纹理字体(fancy2d) + bool LoadSpriteFont(const char* name, const char* path, const char* tex_path, bool mipmaps = true) noexcept; + // 加载矢量字体 + bool LoadTTFFont(const char* name, const char* path, float width, float height) noexcept; + bool LoadTrueTypeFont(const char* name, core::Graphics::TrueTypeFontInfo* fonts, size_t count) noexcept; + // 特效 + bool LoadFX(const char* name, const char* path) noexcept; + // 模型 + bool LoadModel(const char* name, const char* path) noexcept; + + core::SmartReference GetTexture(std::string_view name) noexcept; + core::SmartReference GetSprite(std::string_view name) noexcept; + core::SmartReference GetAnimation(std::string_view name) noexcept; + core::SmartReference GetMusic(std::string_view name) noexcept; + core::SmartReference GetSound(std::string_view name) noexcept; + core::SmartReference GetParticle(std::string_view name) noexcept; + core::SmartReference GetSpriteFont(std::string_view name) noexcept; + core::SmartReference GetTTFFont(std::string_view name) noexcept; + core::SmartReference GetFX(std::string_view name) noexcept; + core::SmartReference GetModel(std::string_view name) noexcept; + public: + ResourcePool(ResourceMgr* mgr, ResourcePoolType t); + ResourcePool& operator=(const ResourcePool&) = delete; + ResourcePool(const ResourcePool&) = delete; + }; + + // 资源管理器 + class ResourceMgr + { + private: + ResourcePoolType m_ActivedPool = ResourcePoolType::Global; + ResourcePool m_GlobalResourcePool; + ResourcePool m_StageResourcePool; + public: + ResourcePoolType GetActivedPoolType() noexcept; + void SetActivedPoolType(ResourcePoolType t) noexcept; + ResourcePool* GetActivedPool() noexcept; + ResourcePool* GetResourcePool(ResourcePoolType t) noexcept; + void ClearAllResource() noexcept; + + core::SmartReference FindTexture(const char* name) noexcept; + core::SmartReference FindSprite(const char* name) noexcept; + core::SmartReference FindAnimation(const char* name) noexcept; + core::SmartReference FindMusic(const char* name) noexcept; + core::SmartReference FindSound(const char* name) noexcept; + core::SmartReference FindParticle(const char* name) noexcept; + core::SmartReference FindSpriteFont(const char* name) noexcept; + core::SmartReference FindTTFFont(const char* name) noexcept; + core::SmartReference FindFX(const char* name) noexcept; + core::SmartReference FindModel(const char* name) noexcept; + + bool GetTextureSize(const char* name, core::Vector2U& out) noexcept; + void CacheTTFFontString(const char* name, const char* text, size_t len) noexcept; + void UpdateSound(); + private: + static bool g_ResourceLoadingLog; + float m_GlobalImageScaleFactor = 1.0f; + public: + static void SetResourceLoadingLog(bool b); + static bool GetResourceLoadingLog(); + float GetGlobalImageScaleFactor() const noexcept { return m_GlobalImageScaleFactor; } + void SetGlobalImageScaleFactor(float s) noexcept { m_GlobalImageScaleFactor = s; } + void ShowResourceManagerDebugWindow(bool* p_open = nullptr); + public: + ResourceMgr(); + }; +} diff --git a/LuaSTG/LuaSTG/GameResource/ResourcePool.cpp b/LuaSTG/LuaSTG/GameResource/ResourcePool.cpp index f6577b4da..ed0e9f48c 100644 --- a/LuaSTG/LuaSTG/GameResource/ResourcePool.cpp +++ b/LuaSTG/LuaSTG/GameResource/ResourcePool.cpp @@ -9,6 +9,8 @@ #include "GameResource/Implement/ResourcePostEffectShaderImpl.hpp" #include "GameResource/Implement/ResourceModelImpl.hpp" #include "core/FileSystem.hpp" +#include "core/AudioEngine.hpp" +#include "d3d11/VideoTexture.hpp" #include "AppFrame.h" #include "lua/plus.hpp" @@ -227,7 +229,7 @@ namespace luastg return true; } - bool ResourcePool::LoadVideo(const char* name, const char* path) noexcept + bool ResourcePool::LoadVideo(const char* name, const char* path, core::VideoOpenOptions const* options) noexcept { if (m_TexturePool.find(std::string_view(name)) != m_TexturePool.end()) { @@ -239,7 +241,10 @@ namespace luastg } core::SmartReference p_texture; - if (!LAPP.getGraphicsDevice()->createVideoTexture(path, p_texture.put())) + bool ok = options + ? LAPP.getGraphicsDevice()->createVideoTexture(path, *options, p_texture.put()) + : LAPP.getGraphicsDevice()->createVideoTexture(path, p_texture.put()); + if (!ok) { spdlog::error("[luastg] 从 '{}' 创建视频纹理 '{}' 失败", path, name); return false; diff --git a/LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp b/LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp index 7e52d5696..702dbf812 100644 --- a/LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp +++ b/LuaSTG/LuaSTG/LuaBinding/LW_ResourceMgr.cpp @@ -3,6 +3,10 @@ #include "AppFrame.h" #include "d3d11/VideoTexture.hpp" #include "core/VideoDecoder.hpp" +#include "core/AudioEngine.hpp" +#include +#include +#include void luastg::binding::ResourceManager::Register(lua_State* L) noexcept { @@ -54,6 +58,30 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept return luaL_error(L, "can't load texture from file '%s'.", path); return 0; } + static void ReadVideoOptions(lua_State* L, int index, core::VideoOpenOptions& opt) noexcept { + if (!lua_istable(L, index)) return; + lua_getfield(L, index, "video_stream"); + if (lua_isnumber(L, -1)) opt.video_stream_index = (uint32_t)lua_tointeger(L, -1); + lua_pop(L, 1); + lua_getfield(L, index, "width"); + if (lua_isnumber(L, -1)) opt.output_width = (uint32_t)lua_tointeger(L, -1); + lua_pop(L, 1); + lua_getfield(L, index, "height"); + if (lua_isnumber(L, -1)) opt.output_height = (uint32_t)lua_tointeger(L, -1); + lua_pop(L, 1); + lua_getfield(L, index, "premultiplied_alpha"); + if (lua_isboolean(L, -1)) opt.premultiplied_alpha = lua_toboolean(L, -1) != 0; + lua_pop(L, 1); + lua_getfield(L, index, "looping"); + if (lua_isboolean(L, -1)) opt.looping = lua_toboolean(L, -1) != 0; + lua_pop(L, 1); + lua_getfield(L, index, "loop_end"); + if (lua_isnumber(L, -1)) opt.loop_end = (double)lua_tonumber(L, -1); + lua_pop(L, 1); + lua_getfield(L, index, "loop_duration"); + if (lua_isnumber(L, -1)) opt.loop_duration = (double)lua_tonumber(L, -1); + lua_pop(L, 1); + } static int LoadVideo(lua_State* L) noexcept { const char* name = luaL_checkstring(L, 1); @@ -62,7 +90,10 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept ResourcePool* pActivedPool = LRES.GetActivedPool(); if (!pActivedPool) return luaL_error(L, "can't load resource at this time."); - if (!pActivedPool->LoadVideo(name, path)) + core::VideoOpenOptions opt; + if (lua_gettop(L) >= 3 && lua_istable(L, 3)) + ReadVideoOptions(L, 3, opt); + if (!pActivedPool->LoadVideo(name, path, lua_gettop(L) >= 3 && lua_istable(L, 3) ? &opt : nullptr)) return luaL_error(L, "can't load video from file '%s'.", path); return 0; } @@ -704,10 +735,14 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept static int VideoSeek(lua_State* L) noexcept { const char* name = luaL_checkstring(L, 1); double time = luaL_checknumber(L, 2); - auto decoder = GetVideoDecoder(name); - if (!decoder) + auto tex = LRES.FindTexture(name); + if (!tex) return luaL_error(L, "video texture '%s' not found.", name); - lua_pushboolean(L, decoder->seek(time)); + auto* vt = dynamic_cast(tex->GetTexture()); + if (!vt) + return luaL_error(L, "texture '%s' is not a video texture.", name); + bool ok = vt->getVideoDecoder()->seek(time); + lua_pushboolean(L, ok); return 1; } @@ -720,24 +755,37 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept decoder->setLooping(loop); return 0; } - - static int VideoUpdate(lua_State* L) noexcept { + static int VideoSetLoopRange(lua_State* L) noexcept { const char* name = luaL_checkstring(L, 1); - double time = luaL_checknumber(L, 2); auto decoder = GetVideoDecoder(name); if (!decoder) return luaL_error(L, "video texture '%s' not found.", name); - lua_pushboolean(L, decoder->updateToTime(time)); + double loop_end = luaL_checknumber(L, 2); + double loop_duration = luaL_checknumber(L, 3); + decoder->setLoopRange(loop_end, loop_duration); + return 0; + } + static int VideoUpdate(lua_State* L) noexcept { + const char* name = luaL_checkstring(L, 1); + double time = luaL_checknumber(L, 2); + auto tex = LRES.FindTexture(name); + if (!tex) + return luaL_error(L, "video texture '%s' not found.", name); + auto* vt = dynamic_cast(tex->GetTexture()); + if (!vt) + return luaL_error(L, "texture '%s' is not a video texture.", name); + auto decoder = vt->getVideoDecoder(); + bool ok = decoder->updateToTime(time); + lua_pushboolean(L, ok); return 1; } - static int VideoGetInfo(lua_State* L) noexcept { const char* name = luaL_checkstring(L, 1); auto decoder = GetVideoDecoder(name); if (!decoder) return luaL_error(L, "video texture '%s' not found.", name); - lua_createtable(L, 0, 7); + lua_createtable(L, 0, 8); lua_pushnumber(L, decoder->getDuration()); lua_setfield(L, -2, "duration"); @@ -747,6 +795,13 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept lua_pushboolean(L, decoder->isLooping()); lua_setfield(L, -2, "looping"); + + double loop_end = 0.0, loop_duration = 0.0; + decoder->getLoopRange(&loop_end, &loop_duration); + lua_pushnumber(L, loop_end); + lua_setfield(L, -2, "loop_end"); + lua_pushnumber(L, loop_duration); + lua_setfield(L, -2, "loop_duration"); auto size = decoder->getVideoSize(); lua_pushinteger(L, size.x); @@ -755,6 +810,9 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept lua_pushinteger(L, size.y); lua_setfield(L, -2, "height"); + lua_pushinteger(L, (lua_Integer)decoder->getVideoStreamIndex()); + lua_setfield(L, -2, "video_stream"); + double fi = decoder->getFrameInterval(); lua_pushnumber(L, fi); lua_setfield(L, -2, "frame_interval"); @@ -762,7 +820,77 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept lua_pushnumber(L, 1.0 / fi); lua_setfield(L, -2, "fps"); } - + return 1; + } + static int VideoGetVideoStreams(lua_State* L) noexcept { + const char* name = luaL_checkstring(L, 1); + auto decoder = GetVideoDecoder(name); + if (!decoder) + return luaL_error(L, "video texture '%s' not found.", name); + std::vector list; + auto cb = [](core::VideoStreamInfo const& info, void* ud) { + static_cast*>(ud)->push_back(info); + }; + decoder->getVideoStreams(cb, &list); + lua_createtable(L, (int)list.size(), 0); + for (size_t i = 0; i < list.size(); ++i) { + lua_createtable(L, 0, 5); + lua_pushinteger(L, (lua_Integer)list[i].index); + lua_setfield(L, -2, "index"); + lua_pushinteger(L, (lua_Integer)list[i].width); + lua_setfield(L, -2, "width"); + lua_pushinteger(L, (lua_Integer)list[i].height); + lua_setfield(L, -2, "height"); + lua_pushnumber(L, list[i].fps); + lua_setfield(L, -2, "fps"); + lua_pushnumber(L, list[i].duration_seconds); + lua_setfield(L, -2, "duration"); + lua_rawseti(L, -2, (int)i + 1); + } + return 1; + } + static int VideoGetAudioStreams(lua_State* L) noexcept { + const char* name = luaL_checkstring(L, 1); + auto decoder = GetVideoDecoder(name); + if (!decoder) + return luaL_error(L, "video texture '%s' not found.", name); + std::vector list; + auto cb = [](core::AudioStreamInfo const& info, void* ud) { + static_cast*>(ud)->push_back(info); + }; + decoder->getAudioStreams(cb, &list); + lua_createtable(L, (int)list.size(), 0); + for (size_t i = 0; i < list.size(); ++i) { + lua_createtable(L, 0, 4); + lua_pushinteger(L, (lua_Integer)list[i].index); + lua_setfield(L, -2, "index"); + lua_pushinteger(L, (lua_Integer)list[i].channels); + lua_setfield(L, -2, "channels"); + lua_pushinteger(L, (lua_Integer)list[i].sample_rate); + lua_setfield(L, -2, "sample_rate"); + lua_pushnumber(L, list[i].duration_seconds); + lua_setfield(L, -2, "duration"); + lua_rawseti(L, -2, (int)i + 1); + } + return 1; + } + static int VideoReopen(lua_State* L) noexcept { + const char* name = luaL_checkstring(L, 1); + auto tex = LRES.FindTexture(name); + if (!tex) + return luaL_error(L, "texture '%s' not found.", name); + auto* vt = dynamic_cast(tex->GetTexture()); + if (!vt) + return luaL_error(L, "texture '%s' is not a video texture.", name); + core::IVideoDecoder* dec = vt->getVideoDecoder(); + core::VideoOpenOptions opt = dec->getLastOpenOptions(); + if (lua_gettop(L) >= 2 && lua_istable(L, 2)) + ReadVideoOptions(L, 2, opt); + if (!dec->reopen(opt)) { + lua_pushboolean(L, false); + return 1; + } + lua_pushboolean(L, true); return 1; } }; @@ -809,8 +937,12 @@ void luastg::binding::ResourceManager::Register(lua_State* L) noexcept // 视频控制函数 { "VideoSeek", &Wrapper::VideoSeek }, { "VideoSetLooping", &Wrapper::VideoSetLooping }, + { "VideoSetLoopRange", &Wrapper::VideoSetLoopRange }, { "VideoUpdate", &Wrapper::VideoUpdate }, { "VideoGetInfo", &Wrapper::VideoGetInfo }, + { "VideoGetVideoStreams", &Wrapper::VideoGetVideoStreams }, + { "VideoGetAudioStreams", &Wrapper::VideoGetAudioStreams }, + { "VideoReopen", &Wrapper::VideoReopen }, { NULL, NULL }, }; diff --git a/data/example/video_example.lua b/data/example/video_example.lua index 2b39173ee..37c55ec58 100644 --- a/data/example/video_example.lua +++ b/data/example/video_example.lua @@ -2,25 +2,37 @@ -- 使用 StopWatch 提供绝对时间，手动调用 VideoUpdate 更新画面 -- 在资源池中加载视频 --- 参数：名称，路径 -LoadVideo('video1', 'test_video.mp4') +-- lstg.LoadVideo(name, path) 或 lstg.LoadVideo(name, path, options) +-- options 表可选：video_stream, width, height, premultiplied_alpha, looping, loop_end, loop_duration +lstg.LoadVideo('video1', 'test_video.mp4') +-- 示例：带选项加载（输出尺寸、预乘 Alpha、循环） +-- lstg.LoadVideo('video1', 'test_video.mp4', { width = 1280, height = 720, looping = true }) +-- 示例：带循环区间 [end-duration, end) +-- lstg.LoadVideo('video1', 'test_video.mp4', { looping = true, loop_end = 10, loop_duration = 5 }) --- 创建计时器并设置循环播放 +-- 枚举流（加载后从已打开的视频枚举，用于多轨选择） +-- local video_streams = lstg.VideoGetVideoStreams('video1') +-- local audio_streams = lstg.VideoGetAudioStreams('video1') + +-- 创建计时器；循环也可在 LoadVideo 的 options 里用 looping = true / loop_end / loop_duration 设置 local video_clock = lstg.StopWatch() -VideoSetLooping('video1', true) +lstg.VideoSetLooping('video1', true) -- 像普通纹理一样创建精灵 -LoadImage('video_sprite', 'video1', 0, 0, 640, 480) +lstg.LoadImage('video_sprite', 'video1', 0, 0, 640, 480) -- 在 RenderFunc 中渲染 function RenderFunc() - -- 以秒表绝对时间驱动视频更新 - VideoUpdate('video1', video_clock:GetElapsed()) - Render('video_sprite', 100, 100) + lstg.VideoUpdate('video1', video_clock:GetElapsed()) + lstg.Render('video_sprite', 100, 100) end -- 可用控制： --- 1. VideoSetLooping(name, bool) -- 设置是否循环播放 --- 2. VideoSeek(name, time) -- 控制时间，不更新画面，适合进度条；完成后需 VideoUpdate 刷新 --- 3. VideoUpdate(name, absolute_time) -- 按绝对时间更新画面；传入比上次小的时间可倒放/拖拽到该时刻 --- 4. VideoGetInfo(name) -- 获取视频信息（时长、当前时间、尺寸等） +-- lstg.VideoSetLooping(name, bool) +-- lstg.VideoSetLoopRange(name, loop_end, loop_duration) -- 区间 [end-duration, end)，超过 end 减 duration +-- lstg.VideoSeek(name, time) +-- lstg.VideoUpdate(name, absolute_time) +-- lstg.VideoGetInfo(name) -- duration, time, looping, loop_end, loop_duration, width, height, fps, video_stream +-- lstg.VideoGetVideoStreams(name) -- 视频流列表 { index, width, height, fps, duration } +-- lstg.VideoGetAudioStreams(name) -- 音频流列表 { index, channels, sample_rate, duration } +-- lstg.VideoReopen(name [, options]) diff --git a/engine/graphics/core/GraphicsDevice.hpp b/engine/graphics/core/GraphicsDevice.hpp index 3002a651e..34f15bf4c 100644 --- a/engine/graphics/core/GraphicsDevice.hpp +++ b/engine/graphics/core/GraphicsDevice.hpp @@ -110,6 +110,7 @@ namespace core { virtual bool createTextureFromImage(IImage* image, bool mipmap, ITexture2D** out_texture) = 0; virtual bool createTexture(Vector2U size, ITexture2D** out_texture) = 0; virtual bool createVideoTexture(StringView path, ITexture2D** out_texture) = 0; + virtual bool createVideoTexture(StringView path, VideoOpenOptions const& options, ITexture2D** out_texture) = 0; virtual bool createVideoDecoder(IVideoDecoder** out_decoder) = 0; virtual bool createSampler(const GraphicsSamplerInfo& info, IGraphicsSampler** out_sampler) = 0; diff --git a/engine/graphics/core/VideoDecoder.hpp b/engine/graphics/core/VideoDecoder.hpp index 3f16e2ca6..361a40e37 100644 --- a/engine/graphics/core/VideoDecoder.hpp +++ b/engine/graphics/core/VideoDecoder.hpp @@ -2,11 +2,40 @@ #include "core/ReferenceCounted.hpp" #include "core/Vector2.hpp" #include "core/ImmutableString.hpp" +#include +#include namespace core { + // 打开选项：流选择、输出尺寸、预乘 Alpha、循环等 + struct VideoOpenOptions { + uint32_t video_stream_index{ static_cast(-1) }; // -1 = 自动选择第一个视频流 + uint32_t output_width{ 0 }; // 0 = 使用视频原始宽度 + uint32_t output_height{ 0 }; // 0 = 使用视频原始高度 + bool premultiplied_alpha{ false }; + bool looping{ false }; + double loop_end{ 0.0 }; // 循环区间结束时间(秒)，仅当 loop_duration > 0 时生效 + double loop_duration{ 0.0 }; // 循环区间长度(秒)，> 0 时与 loop_end 一起设置循环区间 [loop_end-duration, loop_end) + }; + + // 流枚举结果（用于 getVideoStreams / getAudioStreams） + struct VideoStreamInfo { + uint32_t index{ 0 }; + uint32_t width{ 0 }; + uint32_t height{ 0 }; + double fps{ 0.0 }; + double duration_seconds{ 0.0 }; + }; + struct AudioStreamInfo { + uint32_t index{ 0 }; + uint32_t channels{ 0 }; + uint32_t sample_rate{ 0 }; + double duration_seconds{ 0.0 }; + }; + CORE_INTERFACE IVideoDecoder : IReferenceCounted { - // 打开视频文件 + // 打开视频文件（使用默认选项时可直接 open(path)） virtual bool open(StringView path) = 0; + virtual bool open(StringView path, VideoOpenOptions const& options) = 0; // 关闭视频 virtual void close() = 0; @@ -26,6 +55,9 @@ namespace core { // 循环播放设置 virtual void setLooping(bool loop) = 0; virtual bool isLooping() const noexcept = 0; + // 区间 [end_sec - duration_sec, end_sec)，超过 end 时减去 duration + virtual void setLoopRange(double end_sec, double duration_sec) = 0; + virtual void getLoopRange(double* end_sec, double* duration_sec) const noexcept = 0; // 手动更新到指定时间点 // time_in_seconds: 目标时间（秒） @@ -37,6 +69,18 @@ namespace core { // 获取 Shader Resource View（返回 ID3D11ShaderResourceView*） virtual void* getNativeShaderResourceView() const noexcept = 0; + + // 当前使用的视频流索引 + virtual uint32_t getVideoStreamIndex() const noexcept = 0; + + // 从当前已打开的 SourceReader 枚举流（仅在 hasVideo() 时有效，无需再次读文件） + virtual void getVideoStreams(void (*callback)(VideoStreamInfo const&, void*), void* userdata) const = 0; + virtual void getAudioStreams(void (*callback)(AudioStreamInfo const&, void*), void* userdata) const = 0; + + // 使用上次打开的路径重新打开，仅更新 options（用于切换视频流等） + virtual bool reopen(VideoOpenOptions const& options) = 0; + virtual VideoOpenOptions getLastOpenOptions() const noexcept = 0; + virtual std::string_view getLastOpenPath() const noexcept = 0; }; CORE_INTERFACE_ID(IVideoDecoder, "a4b5c6d7-e8f9-1234-5678-9abcdef01234") diff --git a/engine/graphics/d3d11/GraphicsDevice.hpp b/engine/graphics/d3d11/GraphicsDevice.hpp index 133c67f01..e364aff69 100644 --- a/engine/graphics/d3d11/GraphicsDevice.hpp +++ b/engine/graphics/d3d11/GraphicsDevice.hpp @@ -23,6 +23,7 @@ namespace core { bool createTexture(Vector2U size, ITexture2D** out_texture) override; bool createTextureFromImage(IImage* image, bool mipmap, ITexture2D** out_texture) override; bool createVideoTexture(StringView path, ITexture2D** out_texture) override; + bool createVideoTexture(StringView path, VideoOpenOptions const& options, ITexture2D** out_texture) override; bool createVideoDecoder(IVideoDecoder** out_decoder) override; bool createSampler(const GraphicsSamplerInfo& info, IGraphicsSampler** out_sampler) override; diff --git a/engine/graphics/d3d11/VideoDecoder.cpp b/engine/graphics/d3d11/VideoDecoder.cpp index 3b715e88a..45ecdba23 100644 --- a/engine/graphics/d3d11/VideoDecoder.cpp +++ b/engine/graphics/d3d11/VideoDecoder.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #pragma comment(lib, "mfplat.lib") #pragma comment(lib, "mfreadwrite.lib") @@ -73,6 +74,10 @@ namespace core { } bool VideoDecoder::open(StringView path) { + return open(path, VideoOpenOptions{}); + } + + bool VideoDecoder::open(StringView path, VideoOpenOptions const& options) { if (!m_initialized) { Logger::error("[core] [VideoDecoder] Not initialized"); return false; @@ -199,11 +204,34 @@ namespace core { Logger::info("[core] [VideoDecoder] Using D3D11 hardware acceleration (IMFDXGIDeviceManager)"); } + m_video_stream_index = options.video_stream_index; + if (m_video_stream_index == static_cast(-1)) { + for (DWORD i = 0; i < 16u; ++i) { + win32::com_ptr mt; + if (FAILED(m_source_reader->GetNativeMediaType(i, 0, mt.put()))) continue; + GUID major = GUID_NULL; + if (FAILED(mt->GetGUID(MF_MT_MAJOR_TYPE, &major)) || major != MFMediaType_Video) continue; + m_video_stream_index = i; + Logger::info("[core] [VideoDecoder] Auto-selected video stream index {}", (unsigned)i); + break; + } + if (m_video_stream_index == static_cast(-1)) { + Logger::error("[core] [VideoDecoder] No video stream found in file"); + return false; + } + } + + // 流选择：仅启用指定的视频流 + for (DWORD i = 0; i < 16u; ++i) { + BOOL const select = (i == m_video_stream_index); + m_source_reader->SetStreamSelection(i, select); + } + win32::com_ptr partial_media_type; - hr = m_source_reader->GetNativeMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, partial_media_type.put()); + hr = m_source_reader->GetNativeMediaType((DWORD)m_video_stream_index, 0, partial_media_type.put()); if (FAILED(hr)) { Logger::warn("[core] [VideoDecoder] Failed to get native media type, trying current type, hr = {:#x}", (uint32_t)hr); - hr = m_source_reader->GetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, partial_media_type.put()); + hr = m_source_reader->GetCurrentMediaType((DWORD)m_video_stream_index, partial_media_type.put()); if (FAILED(hr)) { Logger::warn("[core] [VideoDecoder] Failed to get current media type, hr = {:#x}", (uint32_t)hr); } @@ -246,12 +274,11 @@ namespace core { if (use_hardware_accel && partial_media_type) { GUID native_subtype = GUID_NULL; if (SUCCEEDED(partial_media_type->GetGUID(MF_MT_SUBTYPE, &native_subtype))) { - constexpr GUID MFVideoFormat_H264 = { 0x34363248, 0x0000, 0x0010, { 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71 } }; - if (native_subtype == MFVideoFormat_H264 || native_subtype.Data1 == 0x34363248) { + if (native_subtype == MFVideoFormat_H264) { Logger::info("[core] [VideoDecoder] Hardware decoder outputs compressed format, enumerating supported output formats"); for (DWORD i = 0; ; ++i) { win32::com_ptr output_type; - hr = m_source_reader->GetNativeMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, i, output_type.put()); + hr = m_source_reader->GetNativeMediaType((DWORD)m_video_stream_index, i, output_type.put()); if (FAILED(hr)) { break; } @@ -271,9 +298,19 @@ namespace core { if (partial_media_type) { UINT32 width = 0, height = 0; if (SUCCEEDED(MFGetAttributeSize(partial_media_type.get(), MF_MT_FRAME_SIZE, &width, &height))) { - hr = MFSetAttributeSize(media_type.get(), MF_MT_FRAME_SIZE, width, height); + UINT32 out_w = options.output_width > 0 ? options.output_width : width; + UINT32 out_h = options.output_height > 0 ? options.output_height : height; + if (out_w != width || out_h != height) { + Logger::info("[core] [VideoDecoder] Requesting output resolution: {}x{} (original: {}x{})", out_w, out_h, width, height); + } + hr = MFSetAttributeSize(media_type.get(), MF_MT_FRAME_SIZE, out_w, out_h); if (FAILED(hr)) { - Logger::warn("[core] [VideoDecoder] Failed to set frame size, hr = {:#x}", (uint32_t)hr); + Logger::warn("[core] [VideoDecoder] Failed to set output resolution {}x{}, using original {}x{}, hr = {:#x}", + out_w, out_h, width, height, (uint32_t)hr); + hr = MFSetAttributeSize(media_type.get(), MF_MT_FRAME_SIZE, width, height); + if (FAILED(hr)) { + Logger::error("[core] [VideoDecoder] Failed to set original frame size, hr = {:#x}", (uint32_t)hr); + } } } @@ -289,7 +326,7 @@ namespace core { return false; } - hr = m_source_reader->SetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, media_type.get()); + hr = m_source_reader->SetCurrentMediaType((DWORD)m_video_stream_index, nullptr, media_type.get()); if (FAILED(hr)) { Logger::info("[core] [VideoDecoder] ARGB32 not supported (hr = {:#x}), trying RGB32", (uint32_t)hr); @@ -298,7 +335,7 @@ namespace core { Logger::error("[core] [VideoDecoder] Failed to set RGB32 subtype, hr = {:#x}", (uint32_t)hr); return false; } - hr = m_source_reader->SetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, media_type.get()); + hr = m_source_reader->SetCurrentMediaType((DWORD)m_video_stream_index, nullptr, media_type.get()); if (FAILED(hr)) { Logger::info("[core] [VideoDecoder] RGB32 not supported (hr = {:#x}), trying NV12 (hardware native)", (uint32_t)hr); @@ -307,10 +344,14 @@ namespace core { Logger::error("[core] [VideoDecoder] Failed to set NV12 subtype, hr = {:#x}", (uint32_t)hr); return false; } - hr = m_source_reader->SetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, media_type.get()); + hr = m_source_reader->SetCurrentMediaType((DWORD)m_video_stream_index, nullptr, media_type.get()); if (FAILED(hr)) { - Logger::error("[core] [VideoDecoder] Failed to set media type (tried ARGB32, RGB32, NV12), hr = {:#x}", (uint32_t)hr); + UINT32 req_w = 0, req_h = 0; + MFGetAttributeSize(media_type.get(), MF_MT_FRAME_SIZE, &req_w, &req_h); + Logger::error("[core] [VideoDecoder] Failed to set media type (tried ARGB32, RGB32, NV12) at resolution {}x{}, hr = {:#x} (MF_E_INVALIDMEDIATYPE)", + req_w, req_h, (uint32_t)hr); + Logger::error("[core] [VideoDecoder] This may be due to unsupported output resolution or codec format"); return false; } m_output_format_nv12 = true; @@ -318,7 +359,7 @@ namespace core { } } - hr = m_source_reader->GetCurrentMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, m_media_type.put()); + hr = m_source_reader->GetCurrentMediaType((DWORD)m_video_stream_index, m_media_type.put()); if (FAILED(hr)) { Logger::error("[core] [VideoDecoder] Failed to get current media type, hr = {:#x}", (uint32_t)hr); return false; @@ -359,7 +400,7 @@ namespace core { UINT32 rate_num = 0, rate_den = 0; for (DWORD i = 0; ; ++i) { win32::com_ptr mt; - hr = m_source_reader->GetNativeMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, i, mt.put()); + hr = m_source_reader->GetNativeMediaType((DWORD)m_video_stream_index, i, mt.put()); if (FAILED(hr)) break; if (SUCCEEDED(MFGetAttributeRatio(mt.get(), MF_MT_FRAME_RATE, &rate_num, &rate_den)) && rate_den > 0 && rate_num > 0) break; @@ -396,9 +437,77 @@ namespace core { Logger::info("[core] [VideoDecoder] Opened video: {}x{}, duration: {:.2f}s", m_target_size.x, m_target_size.y, m_duration); - + + m_last_open_path.assign(path.data(), path.size()); + m_last_open_options = options; + + setLooping(options.looping); + if (options.loop_duration > 0.0) { + setLoopRange(options.loop_end, options.loop_duration); + } + return true; } + + void VideoDecoder::getVideoStreams(void (*callback)(VideoStreamInfo const&, void*), void* userdata) const { + if (!m_source_reader || !callback) return; + PROPVARIANT var; + PropVariantInit(&var); + double duration_sec = 0.0; + if (SUCCEEDED(m_source_reader->GetPresentationAttribute((DWORD)MF_SOURCE_READER_MEDIASOURCE, MF_PD_DURATION, &var))) { + duration_sec = var.hVal.QuadPart / 10000000.0; + PropVariantClear(&var); + } + for (DWORD si = 0; si < 16u; ++si) { + win32::com_ptr mt; + if (FAILED(m_source_reader->GetNativeMediaType(si, 0, mt.put()))) continue; + GUID major = GUID_NULL; + if (FAILED(mt->GetGUID(MF_MT_MAJOR_TYPE, &major)) || major != MFMediaType_Video) continue; + VideoStreamInfo info{}; + info.index = si; + info.duration_seconds = duration_sec; + UINT32 w = 0, h = 0; + if (SUCCEEDED(MFGetAttributeSize(mt.get(), MF_MT_FRAME_SIZE, &w, &h))) { + info.width = w; + info.height = h; + } + UINT32 num = 0, den = 0; + if (SUCCEEDED(MFGetAttributeRatio(mt.get(), MF_MT_FRAME_RATE, &num, &den)) && den > 0) + info.fps = static_cast(num) / static_cast(den); + callback(info, userdata); + } + } + + void VideoDecoder::getAudioStreams(void (*callback)(AudioStreamInfo const&, void*), void* userdata) const { + if (!m_source_reader || !callback) return; + PROPVARIANT var; + PropVariantInit(&var); + double duration_sec = 0.0; + if (SUCCEEDED(m_source_reader->GetPresentationAttribute((DWORD)MF_SOURCE_READER_MEDIASOURCE, MF_PD_DURATION, &var))) { + duration_sec = var.hVal.QuadPart / 10000000.0; + PropVariantClear(&var); + } + for (DWORD si = 0; si < 16u; ++si) { + win32::com_ptr mt; + if (FAILED(m_source_reader->GetNativeMediaType(si, 0, mt.put()))) continue; + GUID major = GUID_NULL; + if (FAILED(mt->GetGUID(MF_MT_MAJOR_TYPE, &major)) || major != MFMediaType_Audio) continue; + AudioStreamInfo info{}; + info.index = si; + info.duration_seconds = duration_sec; + info.channels = (UINT32)MFGetAttributeUINT32(mt.get(), MF_MT_AUDIO_NUM_CHANNELS, 0); + info.sample_rate = MFGetAttributeUINT32(mt.get(), MF_MT_AUDIO_SAMPLES_PER_SECOND, 0); + callback(info, userdata); + } + } + + bool VideoDecoder::reopen(VideoOpenOptions const& options) { + if (m_last_open_path.empty()) { + Logger::error("[core] [VideoDecoder] reopen: no previous path (open was never successful)"); + return false; + } + return open(StringView(m_last_open_path), options); + } void VideoDecoder::close() { m_source_reader.reset(); @@ -420,6 +529,29 @@ namespace core { m_last_requested_time = -1.0; m_frame_interval = 1.0 / 30.0; m_frame_pitch = 0; + m_video_stream_index = 0; + m_has_loop_range = false; + m_loop_start = 0.0; + m_loop_end = 0.0; + } + + void VideoDecoder::setLoopRange(double end_sec, double duration_sec) { + if (duration_sec > 0.0 && end_sec >= duration_sec) { + m_loop_start = end_sec - duration_sec; + double cap = m_duration > 0.0 ? m_duration : end_sec; + m_loop_end = (std::min)(end_sec, cap); + m_has_loop_range = (m_loop_end > m_loop_start); + if (!m_has_loop_range) { m_loop_start = 0.0; m_loop_end = m_duration; } + } else { + m_has_loop_range = false; + m_loop_start = 0.0; + m_loop_end = m_duration; + } + } + + void VideoDecoder::getLoopRange(double* end_sec, double* duration_sec) const noexcept { + if (end_sec) *end_sec = m_has_loop_range ? m_loop_end : m_duration; + if (duration_sec) *duration_sec = m_has_loop_range ? (m_loop_end - m_loop_start) : m_duration; } bool VideoDecoder::seek(double time_in_seconds) { @@ -461,20 +593,57 @@ namespace core { return false; } + auto const loop_begin = m_has_loop_range ? m_loop_start : 0.0; + auto const loop_end = m_has_loop_range ? m_loop_end : m_duration; + auto const loop_len = loop_end - loop_begin; + if (time_in_seconds >= m_duration) { if (m_looping) { - time_in_seconds = fmod(time_in_seconds, m_duration); + if (loop_end > m_duration && time_in_seconds < loop_end) { + constexpr double kEndEpsilon = 1e-6; + if (m_current_time < m_duration - kEndEpsilon) { + double const last_frame_time = std::max(0.0, m_duration - kEndEpsilon); + readFrameAtTime(last_frame_time); + m_current_time = m_duration; + } + return true; + } + if (loop_len > 0.0) + time_in_seconds = loop_begin + std::fmod(time_in_seconds - loop_begin, loop_len); + else + time_in_seconds = std::fmod(time_in_seconds, m_duration); } else { - double const last_frame_time = std::max(0.0, m_duration - 1e-6); + constexpr double kEndEpsilon = 1e-6; + if (m_current_time >= m_duration - kEndEpsilon) { + return true; + } + double const last_frame_time = std::max(0.0, m_duration - kEndEpsilon); bool const ok = readFrameAtTime(last_frame_time); m_current_time = m_duration; return ok; } + } else if (time_in_seconds >= loop_end && m_looping && loop_len > 0.0) { + time_in_seconds = loop_begin + std::fmod(time_in_seconds - loop_begin, loop_len); + } else if (time_in_seconds < loop_begin) { + if (m_looping && loop_len > 0.0) { + double t = time_in_seconds - loop_begin; + t = loop_begin + std::fmod(t, loop_len); + if (t < loop_begin) t += loop_len; + time_in_seconds = t; + } else { + time_in_seconds = loop_begin; + } } else if (time_in_seconds < 0.0) { if (m_looping) { - double t = std::fmod(time_in_seconds, m_duration); - if (t < 0.0) t += m_duration; - time_in_seconds = t; + if (loop_len > 0.0) { + double t = std::fmod(time_in_seconds - loop_begin, loop_len); + if (t < 0.0) t += loop_len; + time_in_seconds = loop_begin + t; + } else { + double t = std::fmod(time_in_seconds, m_duration); + if (t < 0.0) t += m_duration; + time_in_seconds = t; + } } else { time_in_seconds = 0.0; } @@ -522,7 +691,7 @@ namespace core { LONGLONG timestamp = 0; hr = m_source_reader->ReadSample( - (DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, + (DWORD)m_video_stream_index, 0, nullptr, &stream_flags, @@ -592,7 +761,7 @@ namespace core { m_video_device.reset(); close(); } - + bool VideoDecoder::createTexture() { if (!m_device || m_target_size.x == 0 || m_target_size.y == 0) { return false; @@ -922,5 +1091,4 @@ namespace core { return true; } - } diff --git a/engine/graphics/d3d11/VideoDecoder.hpp b/engine/graphics/d3d11/VideoDecoder.hpp index f8fb0db32..a4b358581 100644 --- a/engine/graphics/d3d11/VideoDecoder.hpp +++ b/engine/graphics/d3d11/VideoDecoder.hpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace core { class VideoDecoder final : @@ -17,6 +18,7 @@ namespace core { // IVideoDecoder bool open(StringView path) override; + bool open(StringView path, VideoOpenOptions const& options) override; void close() override; bool hasVideo() const noexcept override { return m_source_reader.get() != nullptr; } @@ -30,11 +32,20 @@ namespace core { void setLooping(bool loop) override { m_looping = loop; } bool isLooping() const noexcept override { return m_looping; } + void setLoopRange(double end_sec, double duration_sec) override; + void getLoopRange(double* end_sec, double* duration_sec) const noexcept override; bool updateToTime(double time_in_seconds) override; void* getNativeTexture() const noexcept override { return m_texture.get(); } void* getNativeShaderResourceView() const noexcept override { return m_shader_resource_view.get(); } - + + uint32_t getVideoStreamIndex() const noexcept override { return m_video_stream_index; } + void getVideoStreams(void (*callback)(VideoStreamInfo const&, void*), void* userdata) const override; + void getAudioStreams(void (*callback)(AudioStreamInfo const&, void*), void* userdata) const override; + bool reopen(VideoOpenOptions const& options) override; + VideoOpenOptions getLastOpenOptions() const noexcept override { return m_last_open_options; } + std::string_view getLastOpenPath() const noexcept override { return m_last_open_path; } + // IGraphicsDeviceEventListener void onGraphicsDeviceCreate() override; @@ -80,8 +91,16 @@ namespace core { double m_last_requested_time{ -1.0 }; double m_frame_interval{ 1.0 / 30.0 }; bool m_looping{ false }; - + bool m_has_loop_range{ false }; + double m_loop_start{ 0.0 }; + double m_loop_end{ 0.0 }; + uint32_t m_frame_pitch{ 0 }; + + uint32_t m_video_stream_index{ 0 }; + + std::string m_last_open_path; + VideoOpenOptions m_last_open_options; bool m_initialized{ false }; }; diff --git a/engine/graphics/d3d11/VideoTexture.cpp b/engine/graphics/d3d11/VideoTexture.cpp index 5fc1ace5d..554b4433b 100644 --- a/engine/graphics/d3d11/VideoTexture.cpp +++ b/engine/graphics/d3d11/VideoTexture.cpp @@ -14,6 +14,10 @@ namespace core { } bool VideoTexture::initialize(IGraphicsDevice* device, StringView path) { + return initialize(device, path, VideoOpenOptions{}); + } + + bool VideoTexture::initialize(IGraphicsDevice* device, StringView path, VideoOpenOptions const& options) { if (!device) { Logger::error("[core] [VideoTexture] Invalid device"); return false; @@ -21,7 +25,6 @@ namespace core { m_device = device; - // 创建视频解码器 auto decoder = new VideoDecoder(); if (!decoder->initialize(device)) { Logger::error("[core] [VideoTexture] Failed to initialize video decoder"); @@ -29,8 +32,7 @@ namespace core { return false; } - // 打开视频文件 - if (!decoder->open(path)) { + if (!decoder->open(path, options)) { Logger::error("[core] [VideoTexture] Failed to open video file: {}", path); decoder->release(); return false; @@ -38,14 +40,14 @@ namespace core { m_decoder = decoder; decoder->release(); - + m_premultiplied_alpha = options.premultiplied_alpha; m_initialized = true; m_device->addEventListener(this); Logger::info("[core] [VideoTexture] Created video texture from: {}", path); - return true; } + void* VideoTexture::getNativeResource() const noexcept { if (m_decoder) { @@ -75,12 +77,16 @@ namespace core { // GraphicsDevice 扩展：视频功能 bool GraphicsDevice::createVideoTexture(StringView path, ITexture2D** out_texture) { + return createVideoTexture(path, VideoOpenOptions{}, out_texture); + } + + bool GraphicsDevice::createVideoTexture(StringView path, VideoOpenOptions const& options, ITexture2D** out_texture) { if (out_texture == nullptr) { assert(false); return false; } SmartReference video_texture; video_texture.attach(new VideoTexture); - if (!video_texture->initialize(this, path)) { + if (!video_texture->initialize(this, path, options)) { return false; } *out_texture = video_texture.detach(); diff --git a/engine/graphics/d3d11/VideoTexture.hpp b/engine/graphics/d3d11/VideoTexture.hpp index 120b26c10..32ced87a5 100644 --- a/engine/graphics/d3d11/VideoTexture.hpp +++ b/engine/graphics/d3d11/VideoTexture.hpp @@ -20,15 +20,15 @@ namespace core { bool isDynamic() const noexcept override { return true; } bool isVideoTexture() const noexcept override { return true; } - bool isPremultipliedAlpha() const noexcept override { return false; } - void setPremultipliedAlpha(bool v) override {} + bool isPremultipliedAlpha() const noexcept override { return m_premultiplied_alpha; } + void setPremultipliedAlpha(bool v) override { m_premultiplied_alpha = v; } Vector2U getSize() const noexcept override; - bool setSize(Vector2U size) override { return false; } - bool update(RectU rect, void const* data, uint32_t row_pitch_in_bytes) override { return false; } - void setImage(IImage* image) override {} + bool setSize(Vector2U /* size */) override { return false; } + bool update(RectU /* rect */, void const* /* data */, uint32_t /* row_pitch_in_bytes */) override { return false; } + void setImage(IImage* /* image */) override {} - bool saveToFile(StringView path) override { return false; } + bool saveToFile(StringView /* path */) override { return false; } void setSamplerState(IGraphicsSampler* sampler) override { m_sampler = sampler; } IGraphicsSampler* getSamplerState() const noexcept override { return m_sampler.get(); } @@ -44,7 +44,8 @@ namespace core { ~VideoTexture(); bool initialize(IGraphicsDevice* device, StringView path); - + bool initialize(IGraphicsDevice* device, StringView path, VideoOpenOptions const& options); + // 获取内部的视频解码器（用于控制播放等） IVideoDecoder* getVideoDecoder() const noexcept { return m_decoder.get(); } @@ -52,6 +53,7 @@ namespace core { SmartReference m_device; SmartReference m_decoder; SmartReference m_sampler; + bool m_premultiplied_alpha{ false }; bool m_initialized{ false }; }; } From c8b8a1aaf78634a8ee4635ce5083ec3c3f2c4ea9 Mon Sep 17 00:00:00 2001 From: OLC Date: Mon, 23 Feb 2026 04:36:17 +0800 Subject: [PATCH 08/12] feat: improve video decoding by adding resolution retry logic and adjusting frame size handling --- engine/graphics/d3d11/VideoDecoder.cpp | 124 ++++++++++++++++++------- 1 file changed, 91 insertions(+), 33 deletions(-) diff --git a/engine/graphics/d3d11/VideoDecoder.cpp b/engine/graphics/d3d11/VideoDecoder.cpp index 45ecdba23..5d18e8ee4 100644 --- a/engine/graphics/d3d11/VideoDecoder.cpp +++ b/engine/graphics/d3d11/VideoDecoder.cpp @@ -345,17 +345,56 @@ namespace core { return false; } hr = m_source_reader->SetCurrentMediaType((DWORD)m_video_stream_index, nullptr, media_type.get()); - + bool succeeded_via_retry = false; if (FAILED(hr)) { UINT32 req_w = 0, req_h = 0; MFGetAttributeSize(media_type.get(), MF_MT_FRAME_SIZE, &req_w, &req_h); - Logger::error("[core] [VideoDecoder] Failed to set media type (tried ARGB32, RGB32, NV12) at resolution {}x{}, hr = {:#x} (MF_E_INVALIDMEDIATYPE)", - req_w, req_h, (uint32_t)hr); - Logger::error("[core] [VideoDecoder] This may be due to unsupported output resolution or codec format"); - return false; + UINT32 orig_w = 0, orig_h = 0; + bool can_retry_original = partial_media_type + && SUCCEEDED(MFGetAttributeSize(partial_media_type.get(), MF_MT_FRAME_SIZE, &orig_w, &orig_h)) + && (req_w != orig_w || req_h != orig_h); + if (can_retry_original) { + Logger::info("[core] [VideoDecoder] Scaled resolution {}x{} not supported by decoder, retrying with original {}x{}", + req_w, req_h, orig_w, orig_h); + hr = MFSetAttributeSize(media_type.get(), MF_MT_FRAME_SIZE, orig_w, orig_h); + if (SUCCEEDED(hr)) { + hr = media_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_ARGB32); + if (SUCCEEDED(hr)) { + hr = m_source_reader->SetCurrentMediaType((DWORD)m_video_stream_index, nullptr, media_type.get()); + } + if (FAILED(hr)) { + hr = media_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); + if (SUCCEEDED(hr)) { + hr = m_source_reader->SetCurrentMediaType((DWORD)m_video_stream_index, nullptr, media_type.get()); + } + } + if (FAILED(hr)) { + hr = media_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12); + if (SUCCEEDED(hr)) { + hr = m_source_reader->SetCurrentMediaType((DWORD)m_video_stream_index, nullptr, media_type.get()); + } + if (SUCCEEDED(hr)) { + m_output_format_nv12 = true; + Logger::info("[core] [VideoDecoder] Using NV12 output (original resolution), will convert to BGRA via Video Processor"); + } + } + if (SUCCEEDED(hr)) { + succeeded_via_retry = true; + } + } + } + if (FAILED(hr)) { + UINT32 err_w = 0, err_h = 0; + MFGetAttributeSize(media_type.get(), MF_MT_FRAME_SIZE, &err_w, &err_h); + Logger::error("[core] [VideoDecoder] Failed to set media type (tried ARGB32, RGB32, NV12) at resolution {}x{}, hr = {:#x} (MF_E_INVALIDMEDIATYPE)", + err_w, err_h, (uint32_t)hr); + Logger::error("[core] [VideoDecoder] This may be due to unsupported output resolution or codec format"); + return false; + } + } else if (!succeeded_via_retry) { + m_output_format_nv12 = true; + Logger::info("[core] [VideoDecoder] Using NV12 output, will convert to BGRA via Video Processor"); } - m_output_format_nv12 = true; - Logger::info("[core] [VideoDecoder] Using NV12 output, will convert to BGRA via Video Processor"); } } @@ -387,7 +426,10 @@ namespace core { } m_video_size = Vector2U{ width, height }; - m_target_size = m_video_size; + m_target_size = Vector2U{ + options.output_width > 0 ? options.output_width : width, + options.output_height > 0 ? options.output_height : height + }; PROPVARIANT var; PropVariantInit(&var); @@ -427,7 +469,7 @@ namespace core { return false; } - m_frame_pitch = m_target_size.x * 4; + m_frame_pitch = m_video_size.x * 4; m_current_time = 0.0; m_last_requested_time = -1.0; @@ -846,8 +888,8 @@ namespace core { content_desc.InputFrameFormat = D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE; content_desc.InputFrameRate.Numerator = 30; content_desc.InputFrameRate.Denominator = 1; - content_desc.InputWidth = m_target_size.x; - content_desc.InputHeight = m_target_size.y; + content_desc.InputWidth = m_video_size.x; + content_desc.InputHeight = m_video_size.y; content_desc.OutputFrameRate.Numerator = 30; content_desc.OutputFrameRate.Denominator = 1; content_desc.OutputWidth = m_target_size.x; @@ -1039,7 +1081,11 @@ namespace core { uint8_t* dst = static_cast(mapped.pData); const uint8_t* src = src_data; - const size_t copy_size = m_frame_pitch; + const uint32_t src_w = m_video_size.x; + const uint32_t src_h = m_video_size.y; + const uint32_t dst_w = m_target_size.x; + const uint32_t dst_h = m_target_size.y; + const bool need_scale = (src_w != dst_w || src_h != dst_h); GUID format = GUID_NULL; bool force_opaque_alpha = false; @@ -1051,33 +1097,45 @@ namespace core { static bool first_copy = true; if (first_copy) { - Logger::info("[core] [VideoDecoder] Texture update: source_pitch={}, mapped.RowPitch={}, copy_size={}, height={}, force_alpha={}", - source_pitch, mapped.RowPitch, copy_size, m_target_size.y, force_opaque_alpha); - bool all_zero = true; - for (size_t i = 0; i < std::min(16, copy_size); ++i) { - if (src[i] != 0) { - all_zero = false; - break; - } - } - Logger::info("[core] [VideoDecoder] First 16 bytes all zero: {}", all_zero); + Logger::info("[core] [VideoDecoder] Texture update: source={}x{} pitch={}, dest={}x{} pitch={}, force_alpha={}, scale={}", + src_w, src_h, source_pitch, dst_w, dst_h, mapped.RowPitch, force_opaque_alpha, need_scale); first_copy = false; } - if (!force_opaque_alpha) { - for (uint32_t y = 0; y < m_target_size.y; ++y) { - memcpy(dst, src, copy_size); - dst += mapped.RowPitch; - src += source_pitch; + if (!need_scale) { + const size_t copy_size = m_frame_pitch; + if (!force_opaque_alpha) { + for (uint32_t y = 0; y < dst_h; ++y) { + memcpy(dst, src, copy_size); + dst += mapped.RowPitch; + src += source_pitch; + } + } else { + for (uint32_t y = 0; y < dst_h; ++y) { + memcpy(dst, src, copy_size); + for (uint32_t x = 0; x < dst_w; ++x) { + dst[x * 4 + 3] = 0xFF; + } + dst += mapped.RowPitch; + src += source_pitch; + } } } else { - for (uint32_t y = 0; y < m_target_size.y; ++y) { - memcpy(dst, src, copy_size); - for (uint32_t x = 0; x < m_target_size.x; ++x) { - dst[x * 4 + 3] = 0xFF; + for (uint32_t y_dst = 0; y_dst < dst_h; ++y_dst) { + uint32_t y_src = (src_h > 1) ? (y_dst * src_h / dst_h) : 0; + if (y_src >= src_h) y_src = src_h - 1; + const uint8_t* row_src = src + static_cast(y_src) * source_pitch; + for (uint32_t x_dst = 0; x_dst < dst_w; ++x_dst) { + uint32_t x_src = (src_w > 1) ? (x_dst * src_w / dst_w) : 0; + if (x_src >= src_w) x_src = src_w - 1; + const uint8_t* px = row_src + static_cast(x_src) * 4; + dst[0] = px[0]; + dst[1] = px[1]; + dst[2] = px[2]; + dst[3] = force_opaque_alpha ? 0xFF : px[3]; + dst += 4; } - dst += mapped.RowPitch; - src += source_pitch; + dst += mapped.RowPitch - static_cast(dst_w) * 4; } } From 0994e4ba23321952ca7d8fdb1b9bb59cc69729cc Mon Sep 17 00:00:00 2001 From: OLC Date: Mon, 23 Feb 2026 04:39:32 +0800 Subject: [PATCH 09/12] docs: update video loading example to simplify options description --- data/example/video_example.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/example/video_example.lua b/data/example/video_example.lua index 37c55ec58..2fcef2b34 100644 --- a/data/example/video_example.lua +++ b/data/example/video_example.lua @@ -5,7 +5,7 @@ -- lstg.LoadVideo(name, path) 或 lstg.LoadVideo(name, path, options) -- options 表可选：video_stream, width, height, premultiplied_alpha, looping, loop_end, loop_duration lstg.LoadVideo('video1', 'test_video.mp4') --- 示例：带选项加载（输出尺寸、预乘 Alpha、循环） +-- 示例：带选项加载（输出尺寸、循环） -- lstg.LoadVideo('video1', 'test_video.mp4', { width = 1280, height = 720, looping = true }) -- 示例：带循环区间 [end-duration, end) -- lstg.LoadVideo('video1', 'test_video.mp4', { looping = true, loop_end = 10, loop_duration = 5 }) From e8596d6bfb08f02561aa43feeff268cbcb97b3e7 Mon Sep 17 00:00:00 2001 From: OLC Date: Mon, 23 Feb 2026 05:41:10 +0800 Subject: [PATCH 10/12] feat(video): add Lua bindings for video playback and stream management - Add Video.hpp and Video.cpp Lua binding classes for video resource management - Implement video information methods (getWidth, getHeight, getDuration, getCurrentTime, getFPS) - Implement playback control methods (seek, update, setLooping, setLoopRange, getLoopRange) - Add stream enumeration methods (getVideoStreams, getAudioStreams, getSubtitleStreams) - Add stream selection methods (selectVideoStream, selectAudioStream, selectSubtitleStream) - Implement getTexture method to retrieve associated Texture2D object - Register Video class in LuaWrapper for Lua API exposure - Add video_example_modern.lua example demonstrating video loading and playback - Update CMakeLists.txt to include new Video binding source files - Extend Texture2D and VideoTexture interfaces to support video decoder access --- LuaSTG/CMakeLists.txt | 2 + LuaSTG/LuaSTG/LuaBinding/LuaWrapper.cpp | 2 + LuaSTG/LuaSTG/LuaBinding/modern/Video.cpp | 412 ++++++++++++++++++++++ LuaSTG/LuaSTG/LuaBinding/modern/Video.hpp | 16 + data/example/video_example_modern.lua | 31 ++ engine/graphics/core/Texture2D.hpp | 5 + engine/graphics/d3d11/Texture2D.hpp | 2 + engine/graphics/d3d11/VideoTexture.hpp | 5 +- 8 files changed, 472 insertions(+), 3 deletions(-) create mode 100644 LuaSTG/LuaSTG/LuaBinding/modern/Video.cpp create mode 100644 LuaSTG/LuaSTG/LuaBinding/modern/Video.hpp create mode 100644 data/example/video_example_modern.lua diff --git a/LuaSTG/CMakeLists.txt b/LuaSTG/CMakeLists.txt index 031383156..6afccc10c 100644 --- a/LuaSTG/CMakeLists.txt +++ b/LuaSTG/CMakeLists.txt @@ -128,6 +128,8 @@ set(LUASTG_ENGINE_SOURCES LuaSTG/LuaBinding/modern/SwapChain.cpp LuaSTG/LuaBinding/modern/Texture2D.hpp LuaSTG/LuaBinding/modern/Texture2D.cpp + LuaSTG/LuaBinding/modern/Video.hpp + LuaSTG/LuaBinding/modern/Video.cpp LuaSTG/LuaBinding/modern/RenderTarget.hpp LuaSTG/LuaBinding/modern/RenderTarget.cpp LuaSTG/LuaBinding/modern/DepthStencilBuffer.hpp diff --git a/LuaSTG/LuaSTG/LuaBinding/LuaWrapper.cpp b/LuaSTG/LuaSTG/LuaBinding/LuaWrapper.cpp index 67c0e4b52..b668ca65f 100644 --- a/LuaSTG/LuaSTG/LuaBinding/LuaWrapper.cpp +++ b/LuaSTG/LuaSTG/LuaBinding/LuaWrapper.cpp @@ -7,6 +7,7 @@ #include "LuaBinding/modern/Window.hpp" #include "LuaBinding/modern/SwapChain.hpp" #include "LuaBinding/modern/Texture2D.hpp" +#include "LuaBinding/modern/Video.hpp" #include "LuaBinding/modern/RenderTarget.hpp" #include "LuaBinding/modern/DepthStencilBuffer.hpp" #include "LuaBinding/modern/Mesh.hpp" @@ -82,6 +83,7 @@ namespace luastg::binding Window_Windows11Extension::registerClass(L); SwapChain::registerClass(L); Texture2D::registerClass(L); + Video::registerClass(L); RenderTarget::registerClass(L); DepthStencilBuffer::registerClass(L); Mesh::registerClass(L); diff --git a/LuaSTG/LuaSTG/LuaBinding/modern/Video.cpp b/LuaSTG/LuaSTG/LuaBinding/modern/Video.cpp new file mode 100644 index 000000000..d2e9bfdcb --- /dev/null +++ b/LuaSTG/LuaSTG/LuaBinding/modern/Video.cpp @@ -0,0 +1,412 @@ +#include "Video.hpp" +#include "Texture2D.hpp" +#include "lua/plus.hpp" +#include "AppFrame.h" +#include "core/VideoDecoder.hpp" + +namespace luastg::binding { + std::string_view Video::class_name{ "lstg.Video" }; + + struct VideoBinding : Video { + // meta methods + + // NOLINTBEGIN(*-reserved-identifier) + + static int __gc(lua_State* vm) { + if (auto const self = as(vm, 1); self->data) { + self->data->release(); + self->data = nullptr; + } + return 0; + } + static int __tostring(lua_State* vm) { + lua::stack_t const ctx(vm); + [[maybe_unused]] auto const self = as(vm, 1); + ctx.push_value(class_name); + return 1; + } + static int __eq(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + if (is(vm, 2)) { + auto const other = as(vm, 2); + ctx.push_value(self->data == other->data); + } else { + ctx.push_value(false); + } + return 1; + } + + // NOLINTEND(*-reserved-identifier) + + // helper: get video decoder from texture + static core::IVideoDecoder* getDecoder(core::ITexture2D* texture) { + if (!texture || !texture->isVideoTexture()) { + return nullptr; + } + return texture->getVideoDecoder(); + } + + // method - video info + + static int getWidth(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + ctx.push_value(self->data->getSize().x); + return 1; + } + static int getHeight(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + ctx.push_value(self->data->getSize().y); + return 1; + } + static int getDuration(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + if (auto decoder = getDecoder(self->data)) { + ctx.push_value(decoder->getDuration()); + } else { + ctx.push_value(0.0); + } + return 1; + } + static int getCurrentTime(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + if (auto decoder = getDecoder(self->data)) { + ctx.push_value(decoder->getCurrentTime()); + } else { + ctx.push_value(0.0); + } + return 1; + } + static int getFPS(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + if (auto decoder = getDecoder(self->data)) { + auto interval = decoder->getFrameInterval(); + ctx.push_value(interval > 0.0 ? 1.0 / interval : 0.0); + } else { + ctx.push_value(0.0); + } + return 1; + } + static int isLooping(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + if (auto decoder = getDecoder(self->data)) { + ctx.push_value(decoder->isLooping()); + } else { + ctx.push_value(false); + } + return 1; + } + static int getTexture(lua_State* vm) { + auto const self = as(vm, 1); + auto const texture = Texture2D::create(vm); + texture->data = self->data; + if (texture->data) { + texture->data->retain(); + } + return 1; + } + + // method - playback control + + static int seek(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + auto const time = ctx.get_value(2); + if (auto decoder = getDecoder(self->data)) { + ctx.push_value(decoder->seek(time)); + } else { + ctx.push_value(false); + } + return 1; + } + static int update(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + auto const time = ctx.get_value(2); + if (auto decoder = getDecoder(self->data)) { + ctx.push_value(decoder->updateToTime(time)); + } else { + ctx.push_value(false); + } + return 1; + } + static int setLooping(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + auto const loop = ctx.get_value(2); + if (auto decoder = getDecoder(self->data)) { + decoder->setLooping(loop); + } + return 0; + } + static int setLoopRange(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + auto const loop_end = ctx.get_value(2); + auto const loop_duration = ctx.get_value(3); + if (auto decoder = getDecoder(self->data)) { + decoder->setLoopRange(loop_end, loop_duration); + } + return 0; + } + static int getLoopRange(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + if (auto decoder = getDecoder(self->data)) { + double loop_end = 0.0, loop_duration = 0.0; + decoder->getLoopRange(&loop_end, &loop_duration); + ctx.push_value(loop_end); + ctx.push_value(loop_duration); + return 2; + } + ctx.push_value(0.0); + ctx.push_value(0.0); + return 2; + } + + // method - stream info + + static int getVideoStreams(lua_State* vm) { + auto const self = as(vm, 1); + if (auto decoder = getDecoder(self->data)) { + std::vector list; + auto cb = [](core::VideoStreamInfo const& info, void* ud) { + static_cast*>(ud)->push_back(info); + }; + decoder->getVideoStreams(cb, &list); + lua_createtable(vm, (int)list.size(), 0); + for (size_t i = 0; i < list.size(); ++i) { + lua_createtable(vm, 0, 5); + lua_pushinteger(vm, (lua_Integer)list[i].index); + lua_setfield(vm, -2, "index"); + lua_pushinteger(vm, (lua_Integer)list[i].width); + lua_setfield(vm, -2, "width"); + lua_pushinteger(vm, (lua_Integer)list[i].height); + lua_setfield(vm, -2, "height"); + lua_pushnumber(vm, list[i].fps); + lua_setfield(vm, -2, "fps"); + lua_pushnumber(vm, list[i].duration_seconds); + lua_setfield(vm, -2, "duration"); + lua_rawseti(vm, -2, (int)i + 1); + } + } else { + lua_createtable(vm, 0, 0); + } + return 1; + } + static int getAudioStreams(lua_State* vm) { + auto const self = as(vm, 1); + if (auto decoder = getDecoder(self->data)) { + std::vector list; + auto cb = [](core::AudioStreamInfo const& info, void* ud) { + static_cast*>(ud)->push_back(info); + }; + decoder->getAudioStreams(cb, &list); + lua_createtable(vm, (int)list.size(), 0); + for (size_t i = 0; i < list.size(); ++i) { + lua_createtable(vm, 0, 4); + lua_pushinteger(vm, (lua_Integer)list[i].index); + lua_setfield(vm, -2, "index"); + lua_pushinteger(vm, (lua_Integer)list[i].channels); + lua_setfield(vm, -2, "channels"); + lua_pushinteger(vm, (lua_Integer)list[i].sample_rate); + lua_setfield(vm, -2, "sample_rate"); + lua_pushnumber(vm, list[i].duration_seconds); + lua_setfield(vm, -2, "duration"); + lua_rawseti(vm, -2, (int)i + 1); + } + } else { + lua_createtable(vm, 0, 0); + } + return 1; + } + static int getVideoStreamIndex(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + if (auto decoder = getDecoder(self->data)) { + ctx.push_value(decoder->getVideoStreamIndex()); + } else { + ctx.push_value(0); + } + return 1; + } + static int reopen(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const self = as(vm, 1); + if (auto decoder = getDecoder(self->data)) { + core::VideoOpenOptions opt = decoder->getLastOpenOptions(); + // 读取 options 表 + if (lua_gettop(vm) >= 2 && lua_istable(vm, 2)) { + lua_getfield(vm, 2, "video_stream"); + if (lua_isnumber(vm, -1)) { + opt.video_stream_index = static_cast(lua_tointeger(vm, -1)); + } + lua_pop(vm, 1); + + lua_getfield(vm, 2, "width"); + if (lua_isnumber(vm, -1)) { + opt.output_width = static_cast(lua_tointeger(vm, -1)); + } + lua_pop(vm, 1); + + lua_getfield(vm, 2, "height"); + if (lua_isnumber(vm, -1)) { + opt.output_height = static_cast(lua_tointeger(vm, -1)); + } + lua_pop(vm, 1); + + lua_getfield(vm, 2, "premultiplied_alpha"); + if (lua_isboolean(vm, -1)) { + opt.premultiplied_alpha = lua_toboolean(vm, -1); + } + lua_pop(vm, 1); + + lua_getfield(vm, 2, "looping"); + if (lua_isboolean(vm, -1)) { + opt.looping = lua_toboolean(vm, -1); + } + lua_pop(vm, 1); + + lua_getfield(vm, 2, "loop_end"); + if (lua_isnumber(vm, -1)) { + opt.loop_end = lua_tonumber(vm, -1); + } + lua_pop(vm, 1); + + lua_getfield(vm, 2, "loop_duration"); + if (lua_isnumber(vm, -1)) { + opt.loop_duration = lua_tonumber(vm, -1); + } + lua_pop(vm, 1); + } + ctx.push_value(decoder->reopen(opt)); + } else { + ctx.push_value(false); + } + return 1; + } + + // static method + + static int create(lua_State* vm) { + lua::stack_t const ctx(vm); + auto const path = ctx.get_value(1); + + core::VideoOpenOptions opt{}; + // 读取 options 表 + if (lua_gettop(vm) >= 2 && lua_istable(vm, 2)) { + lua_getfield(vm, 2, "video_stream"); + if (lua_isnumber(vm, -1)) { + opt.video_stream_index = static_cast(lua_tointeger(vm, -1)); + } + lua_pop(vm, 1); + + lua_getfield(vm, 2, "width"); + if (lua_isnumber(vm, -1)) { + opt.output_width = static_cast(lua_tointeger(vm, -1)); + } + lua_pop(vm, 1); + + lua_getfield(vm, 2, "height"); + if (lua_isnumber(vm, -1)) { + opt.output_height = static_cast(lua_tointeger(vm, -1)); + } + lua_pop(vm, 1); + + lua_getfield(vm, 2, "premultiplied_alpha"); + if (lua_isboolean(vm, -1)) { + opt.premultiplied_alpha = lua_toboolean(vm, -1); + } + lua_pop(vm, 1); + + lua_getfield(vm, 2, "looping"); + if (lua_isboolean(vm, -1)) { + opt.looping = lua_toboolean(vm, -1); + } + lua_pop(vm, 1); + + lua_getfield(vm, 2, "loop_end"); + if (lua_isnumber(vm, -1)) { + opt.loop_end = lua_tonumber(vm, -1); + } + lua_pop(vm, 1); + + lua_getfield(vm, 2, "loop_duration"); + if (lua_isnumber(vm, -1)) { + opt.loop_duration = lua_tonumber(vm, -1); + } + lua_pop(vm, 1); + } + + core::SmartReference texture; + if (!LAPP.getGraphicsDevice()->createVideoTexture(path, opt, texture.put())) { + auto const error_message = std::format( + "create Video from file '{}' failed", path); + return luaL_error(vm, error_message.c_str()); + } + auto const self = Video::create(vm); + self->data = texture.detach(); + return 1; + } + }; + + bool Video::is(lua_State* vm, int const index) { + lua::stack_t const ctx(vm); + return ctx.is_metatable(index, class_name); + } + Video* Video::as(lua_State* vm, int const index) { + lua::stack_t const ctx(vm); + return ctx.as_userdata