enne2/R36SHack
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

player: NV12 zero-copy SDL upload path for Rockchip MPP hardware decode

Browse Source 
- _Frame: add pixel_format ('BGRA'|'NV12'), uv_pixels, uv_pitch fields
- _create_appsink: accept NV12 caps when R36S_HW_DECODE=1 (NV12;BGRA fallback)
- render(): choose SDL_PIXELFORMAT_NV12 texture + SDL_UpdateNVTexture for NV12
  frames, avoiding any software colourspace conversion on the CPU
- _on_new_sample: detect format via VideoInfo.finfo.name, extract Y+UV planes
  separately from NV12 GStreamer buffers
- _destroy_texture: reset _texture_format to 'BGRA'
- deploy/arkos/MatHacks.sh: set R36S_HW_DECODE=1 to activate the path
- tests/test_player.py: add finfo mock, pixel_format to SimpleNamespace frame
This commit is contained in:
Matteo Benedetto
2026-03-23 22:58:08 +01:00
parent e23c57318f
commit 6e15fcab5a
3 changed files with 92 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/arkos/MatHacks.sh
+6
View File
@@ -16,6 +16,12 @@ export LD_LIBRARY_PATH="$ENV_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
export GST_PLUGIN_PATH="/usr/lib/aarch64-linux-gnu/gstreamer-1.0"
export LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libgomp.so.1"
# Enable Rockchip MPP hardware video decode (mppvideodec).
# NV12 frames are uploaded directly via SDL_UpdateNVTexture — no software
# colourspace conversion. Requires librockchip_mpp.so + libgstrockchipmpp.so
# installed by deploy/arkos/setup_hw_decode.sh.
export R36S_HW_DECODE="1"
if [ -f "/storage/.config/SDL-GameControllerDB/gamecontrollerdb.txt" ]; then
export SDL_GAMECONTROLLERCONFIG_FILE="/storage/.config/SDL-GameControllerDB/gamecontrollerdb.txt"
elif [ -f "/opt/inttools/gamecontrollerdb.txt" ]; then
src/r36s_dlna_browser/player/gstreamer_backend.py
+71 -10
View File
@@ -68,8 +68,8 @@ def _probe_hw_decoders(gst_module) -> list:
because mppvideodec outputs NV12 which requires an additional software
NV12→BGRA conversion step that outweighs the decode speedup.
Set R36S_HW_DECODE=1 (e.g. in MatHacks.sh) once a zero-copy NV12 SDL
texture upload path is implemented.
Set R36S_HW_DECODE=1 (e.g. in MatHacks.sh) to enable hardware decode with
zero-copy NV12→SDL upload via SDL_UpdateNVTexture.
Returns the list of element names whose rank was boosted.
"""
@@ -161,6 +161,10 @@ class _Frame:
height: int
pitch: int
pixels: bytes
pixel_format: str = "BGRA" # "BGRA" or "NV12"
# For NV12: pitch is the Y-plane stride; uv_pixels is the interleaved UV plane.
uv_pixels: bytes | None = None
uv_pitch: int = 0
class GStreamerBackend(PlayerBackend):
@@ -201,6 +205,7 @@ class GStreamerBackend(PlayerBackend):
self._texture = None
self._texture_renderer = None
self._texture_size = (0, 0)
self._texture_format: str = "BGRA"
self._resolution = ""
self._hw_decoders: list | None = None # None = not yet probed
@@ -225,11 +230,21 @@ class GStreamerBackend(PlayerBackend):
if frame is None:
return False
if self._texture is None or self._texture_renderer != renderer or self._texture_size != (frame.width, frame.height):
if (
self._texture is None
or self._texture_renderer != renderer
or self._texture_size != (frame.width, frame.height)
or self._texture_format != frame.pixel_format
):
self._destroy_texture()
sdl_fmt = (
sdl2.SDL_PIXELFORMAT_NV12
if frame.pixel_format == "NV12"
else sdl2.SDL_PIXELFORMAT_BGRA32
)
self._texture = sdl2.SDL_CreateTexture(
renderer,
sdl2.SDL_PIXELFORMAT_BGRA32,
sdl_fmt,
sdl2.SDL_TEXTUREACCESS_STREAMING,
frame.width,
frame.height,
@@ -239,10 +254,22 @@ class GStreamerBackend(PlayerBackend):
return False
self._texture_renderer = renderer
self._texture_size = (frame.width, frame.height)
self._texture_format = frame.pixel_format
if self._frame_dirty:
pixel_buffer = ctypes.create_string_buffer(frame.pixels)
result = sdl2.SDL_UpdateTexture(self._texture, None, pixel_buffer, frame.pitch)
if frame.pixel_format == "NV12" and frame.uv_pixels is not None:
# Zero-copy NV12 path: upload Y and UV planes separately.
# SDL_UpdateNVTexture avoids a full BGRA conversion on CPU.
y_buf = ctypes.create_string_buffer(frame.pixels)
uv_buf = ctypes.create_string_buffer(frame.uv_pixels)
result = sdl2.SDL_UpdateNVTexture(
self._texture, None,
y_buf, frame.pitch,
uv_buf, frame.uv_pitch,
)
else:
pixel_buffer = ctypes.create_string_buffer(frame.pixels)
result = sdl2.SDL_UpdateTexture(self._texture, None, pixel_buffer, frame.pitch)
if result != 0:
log.error("Could not upload SDL video texture: %s", sdl2.SDL_GetError())
return False
@@ -382,7 +409,14 @@ class GStreamerBackend(PlayerBackend):
sink.set_property("sync", True)
sink.set_property("max-buffers", 2)
sink.set_property("drop", True)
sink.set_property("caps", self._gst.Caps.from_string("video/x-raw,format=BGRA"))
# Accept NV12 when hardware decode is active (avoids a software colourspace
# conversion step); fall back to BGRA for the software-decode path.
hw_active = os.environ.get("R36S_HW_DECODE", "0") == "1"
if hw_active:
caps_str = "video/x-raw,format=NV12;video/x-raw,format=BGRA"
else:
caps_str = "video/x-raw,format=BGRA"
sink.set_property("caps", self._gst.Caps.from_string(caps_str))
sink.connect("new-sample", self._on_new_sample)
return sink
@@ -402,12 +436,38 @@ class GStreamerBackend(PlayerBackend):
width = int(info.width)
height = int(info.height)
pitch = int(info.stride[0]) if info.stride else width * 4
pixels = buffer.extract_dup(0, buffer.get_size())
resolution = f"{width}x{height}"
# Detect pixel format from caps to choose the right upload path.
fmt_str = "BGRA"
if info.finfo is not None:
try:
fmt_str = info.finfo.name.upper() # e.g. "NV12" or "BGRA"
except Exception:
pass
if fmt_str == "NV12":
# NV12: Y plane (stride[0]) followed immediately by interleaved UV plane (stride[1]).
y_size = int(info.stride[0]) * height
uv_size = int(info.stride[1]) * (height // 2)
raw = buffer.extract_dup(0, buffer.get_size())
pixels = raw[:y_size]
uv_pixels = raw[y_size:y_size + uv_size]
pitch = int(info.stride[0])
uv_pitch = int(info.stride[1])
frame = _Frame(
width=width, height=height,
pitch=pitch, pixels=pixels,
pixel_format="NV12",
uv_pixels=uv_pixels, uv_pitch=uv_pitch,
)
else:
pitch = int(info.stride[0]) if info.stride else width * 4
pixels = buffer.extract_dup(0, buffer.get_size())
frame = _Frame(width=width, height=height, pitch=pitch, pixels=pixels)
with self._frame_lock:
self._latest_frame = _Frame(width=width, height=height, pitch=pitch, pixels=pixels)
self._latest_frame = frame
self._frame_dirty = True
if resolution != self._resolution:
self._resolution = resolution
@@ -534,6 +594,7 @@ class GStreamerBackend(PlayerBackend):
self._texture = None
self._texture_renderer = None
self._texture_size = (0, 0)
self._texture_format = "BGRA"
def _clear_frames(self) -> None:
with self._frame_lock:
tests/test_player.py
+15 -4
View File
@@ -188,11 +188,17 @@ class FakeGst:
Caps = SimpleNamespace(from_string=lambda value: value)
class _FakeFinfo:
def __init__(self, name):
self.name = name
class FakeVideoInfoValue:
def __init__(self, width, height, stride):
def __init__(self, width, height, stride, fmt="BGRA"):
self.width = width
self.height = height
self.stride = [stride]
self.stride = [stride, stride]
self.finfo = _FakeFinfo(fmt)
class FakeVideoInfo:
@@ -201,7 +207,12 @@ class FakeVideoInfo:
structure = caps.get_structure(0)
width = structure.get_value("width")
height = structure.get_value("height")
return FakeVideoInfoValue(width, height, width * 4)
fmt = "BGRA"
try:
fmt = structure.get_value("format") or fmt
except Exception:
pass
return FakeVideoInfoValue(width, height, width * 4, fmt)
class FakeGstVideo:
@@ -341,7 +352,7 @@ class TestGStreamerBackend:
def test_render_uploads_latest_frame_and_clears_dirty_flag(self, monkeypatch):
backend, _pipeline, _sink = self._make_backend()
backend._latest_frame = SimpleNamespace(width=320, height=180, pitch=1280, pixels=b"\x00" * (320 * 180 * 4))
backend._latest_frame = SimpleNamespace(width=320, height=180, pitch=1280, pixels=b"\x00" * (320 * 180 * 4), pixel_format="BGRA", uv_pixels=None, uv_pitch=0)
backend._frame_dirty = True
calls = []