WebPlatformForEmbedded · philn · Mar 29, 2024 · Feb 26, 2024 · Mar 6, 2024 · Mar 6, 2024
diff --git a/LayoutTests/platform/wpe/media/video-punch-hole-expected.txt b/LayoutTests/platform/wpe/media/video-punch-hole-expected.txt
@@ -0,0 +1,21 @@
+
+(GraphicsLayer
+  (anchor 0.00 0.00)
+  (bounds 1288.00 807.00)
+  (children 1
+    (GraphicsLayer
+      (bounds 1288.00 807.00)
+      (contentsOpaque 1)
+      (children 1
+        (GraphicsLayer
+          (position 8.00 8.00)
+          (bounds 1280.00 720.00)
+        )
+      )
+    )
+  )
+)
+RUN(internals.enableGStreamerHolePunching(video))
+RUN(video.src = findMediaFile('video', '../../../media/content/test'))
+RUN(video.play())
+
diff --git a/LayoutTests/platform/wpe/media/video-punch-hole.html b/LayoutTests/platform/wpe/media/video-punch-hole.html
@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <script src="../../../media/media-file.js"></script>
+    <script src="../../../media/video-test.js"></script>
+    <script>
+    function dumpLayers()
+    {
+        if (window.testRunner) {
+            document.getElementById('outText').textContent = window.internals.layerTreeAsText(document);
+            testRunner.notifyDone();
+        }
+    }
+    window.addEventListener('load', async event => {
+        findMediaElement();
+        run("internals.enableGStreamerHolePunching(video)");
+        run("video.src = findMediaFile('video', '../../../media/content/test')");
+        run("video.play()");
+        setTimeout(dumpLayers, 500);
+    });
+    </script>
+</head>
+<body>
+    <video></video>
+    <pre id="outText"></pre>
+</body>
+</html>
diff --git a/Source/WebCore/html/HTMLVideoElement.h b/Source/WebCore/html/HTMLVideoElement.h
@@ -119,6 +119,11 @@ class HTMLVideoElement final : public HTMLMediaElement, public Supplementable<HT
     unsigned requestVideoFrameCallback(Ref<VideoFrameRequestCallback>&&);
     void cancelVideoFrameCallback(unsigned);
 
+#if USE(GSTREAMER)
+    void enableGStreamerHolePunching() { m_enableGStreamerHolePunching = true; }
+    bool isGStreamerHolePunchingEnabled() const final { return m_enableGStreamerHolePunching; }
+#endif
+
 private:
     HTMLVideoElement(const QualifiedName&, Document&, bool createdByParser);
 
@@ -175,6 +180,10 @@ class HTMLVideoElement final : public HTMLMediaElement, public Supplementable<HT
     Vector<UniqueRef<VideoFrameRequest>> m_videoFrameRequests;
     Vector<UniqueRef<VideoFrameRequest>> m_servicedVideoFrameRequests;
     unsigned m_nextVideoFrameRequestIndex { 0 };
+
+#if USE(GSTREAMER)
+    bool m_enableGStreamerHolePunching { false };
+#endif
 };
 
 } // namespace WebCore

diff --git a/Source/WebCore/platform/GStreamer.cmake b/Source/WebCore/platform/GStreamer.cmake
@@ -67,6 +67,14 @@ if (ENABLE_VIDEO OR ENABLE_WEB_AUDIO)
         platform/graphics/gstreamer/mse/WebKitMediaSourceGStreamer.cpp
 
         platform/gstreamer/GStreamerCodecUtilities.cpp
+        platform/gstreamer/GStreamerHolePunchQuirkBcmNexus.cpp
+        platform/gstreamer/GStreamerHolePunchQuirkWesteros.cpp
+        platform/gstreamer/GStreamerQuirkAmLogic.cpp
+        platform/gstreamer/GStreamerQuirkBcmNexus.cpp
+        platform/gstreamer/GStreamerQuirkBroadcom.cpp
+        platform/gstreamer/GStreamerQuirkRealtek.cpp
+        platform/gstreamer/GStreamerQuirkWesteros.cpp
+        platform/gstreamer/GStreamerQuirks.cpp
         platform/gstreamer/VideoEncoderPrivateGStreamer.cpp
 
         platform/mediarecorder/MediaRecorderPrivateGStreamer.cpp

@@ -27,6 +27,7 @@
 #include "AudioSourceProvider.h"
 #include "AudioUtilities.h"
 #include "GStreamerCommon.h"
+#include "GStreamerQuirks.h"
 #include "Logging.h"
 #include "WebKitAudioSinkGStreamer.h"
 #include "WebKitWebAudioSourceGStreamer.h"
@@ -124,34 +125,19 @@ AudioDestinationGStreamer::AudioDestinationGStreamer(AudioIOCallback& callback,
     m_src = GST_ELEMENT_CAST(g_object_new(WEBKIT_TYPE_WEB_AUDIO_SRC, "rate", sampleRate,
         "bus", m_renderBus.get(), "destination", this, "frames", AudioUtilities::renderQuantumSize, nullptr));
 
-#if PLATFORM(AMLOGIC)
-    // autoaudiosink changes child element state to READY internally in auto detection phase
-    // that causes resource acquisition in some cases interrupting any playback already running.
-    // On Amlogic we need to set direct-mode=false prop before changing state to READY
-    // but this is not possible with autoaudiosink.
-    GRefPtr<GstElement> audioSink = makeGStreamerElement("amlhalasink", nullptr);
-    ASSERT_WITH_MESSAGE(audioSink, "amlhalasink should be available in the system but it is not");
-    g_object_set(audioSink.get(), "direct-mode", FALSE, nullptr);
-#else
-    GRefPtr<GstElement> audioSink = createPlatformAudioSink("music"_s);
-#endif
+    auto& quirksManager = GStreamerQuirksManager::singleton();
+    GRefPtr<GstElement> audioSink = quirksManager.createWebAudioSink();
     m_audioSinkAvailable = audioSink;
     if (!audioSink) {
         GST_ERROR("Failed to create GStreamer audio sink element");
         return;
     }
 
-    // Probe platform early on for a working audio output device. This is not needed for the WebKit
-    // custom audio sink because it doesn't rely on autoaudiosink.
-    if (!WEBKIT_IS_AUDIO_SINK(audioSink.get())) {
+    // Probe platform early on for a working audio output device in autoaudiosink.
+    if (g_str_has_prefix(GST_OBJECT_NAME(audioSink.get()), "autoaudiosink")) {
         g_signal_connect(audioSink.get(), "child-added", G_CALLBACK(+[](GstChildProxy*, GObject* object, gchar*, gpointer) {
             if (GST_IS_AUDIO_BASE_SINK(object))
                 g_object_set(GST_AUDIO_BASE_SINK(object), "buffer-time", static_cast<gint64>(100000), nullptr);
-
-#if PLATFORM(REALTEK)
-            if (!g_strcmp0(G_OBJECT_TYPE_NAME(object), "GstRTKAudioSink"))
-                g_object_set(object, "media-tunnel", FALSE, "audio-service", TRUE, nullptr);
-#endif
         }), nullptr);
 
         // Autoaudiosink does the real sink detection in the GST_STATE_NULL->READY transition

@@ -24,6 +24,7 @@
 
 #include "AudioBus.h"
 #include "GStreamerCommon.h"
+#include "GStreamerQuirks.h"
 #include <gio/gio.h>
 #include <gst/app/gstappsink.h>
 #include <gst/audio/audio-info.h>
@@ -85,11 +86,11 @@ class AudioFileReader : public CanMakeWeakPtr<AudioFileReader> {
     bool m_errorOccurred { false };
 };
 
-#if PLATFORM(BCM_NEXUS) || PLATFORM(BROADCOM) || PLATFORM(REALTEK)
 int decodebinAutoplugSelectCallback(GstElement*, GstPad*, GstCaps*, GstElementFactory* factory, gpointer)
 {
     static int GST_AUTOPLUG_SELECT_SKIP;
     static int GST_AUTOPLUG_SELECT_TRY;
+    static Vector<String> pluginsToSkip;
     static std::once_flag onceFlag;
     std::call_once(onceFlag, [] {
         GEnumClass* enumClass = G_ENUM_CLASS(g_type_class_ref(g_type_from_name("GstAutoplugSelectResult")));
@@ -98,32 +99,17 @@ int decodebinAutoplugSelectCallback(GstElement*, GstPad*, GstCaps*, GstElementFa
         value = g_enum_get_value_by_name(enumClass, "GST_AUTOPLUG_SELECT_TRY");
         GST_AUTOPLUG_SELECT_TRY = value->value;
         g_type_class_unref(enumClass);
+
+        pluginsToSkip = GStreamerQuirksManager::singleton().disallowedWebAudioDecoders();
     });
 
-    const Vector<String> pluginsToSkip = {
-#if PLATFORM(BCM_NEXUS) || PLATFORM(BROADCOM)
-        "brcmaudfilter"_s,
-#endif
-#if PLATFORM(REALTEK)
-        "omxaacdec"_s,
-        "omxac3dec"_s,
-        "omxac4dec"_s,
-        "omxeac3dec"_s,
-        "omxflacdec"_s,
-        "omxlpcmdec"_s,
-        "omxmp3dec"_s,
-        "omxopusdec"_s,
-        "omxvorbisdec"_s,
-#endif
-    };
     auto factoryName = StringView::fromLatin1(gst_plugin_feature_get_name(GST_PLUGIN_FEATURE(factory)));
     for (const auto& pluginToSkip : pluginsToSkip) {
         if (pluginToSkip == factoryName)
             return GST_AUTOPLUG_SELECT_SKIP;
     }
     return GST_AUTOPLUG_SELECT_TRY;
 }
-#endif
 
 static void copyGstreamerBuffersToAudioChannel(const GRefPtr<GstBufferList>& buffers, AudioChannel* audioChannel)
 {
@@ -173,9 +159,7 @@ AudioFileReader::~AudioFileReader()
 
     if (m_decodebin) {
         g_signal_handlers_disconnect_matched(m_decodebin.get(), G_SIGNAL_MATCH_DATA, 0, 0, nullptr, nullptr, this);
-#if PLATFORM(BCM_NEXUS) || PLATFORM(BROADCOM) || PLATFORM(REALTEK)
         g_signal_handlers_disconnect_matched(m_decodebin.get(), G_SIGNAL_MATCH_FUNC, 0, 0, nullptr, reinterpret_cast<gpointer>(decodebinAutoplugSelectCallback), nullptr);
-#endif
         m_decodebin = nullptr;
     }
 
@@ -426,9 +410,7 @@ void AudioFileReader::decodeAudioForBusCreation()
     g_object_set(source, "stream", memoryStream.get(), nullptr);
 
     m_decodebin = makeGStreamerElement("decodebin", "decodebin");
-#if PLATFORM(BCM_NEXUS) || PLATFORM(BROADCOM) || PLATFORM(REALTEK)
     g_signal_connect(m_decodebin.get(), "autoplug-select", G_CALLBACK(decodebinAutoplugSelectCallback), nullptr);
-#endif
     g_signal_connect_swapped(m_decodebin.get(), "pad-added", G_CALLBACK(decodebinPadAddedCallback), this);
 
     gst_bin_add_many(GST_BIN(m_pipeline.get()), source, m_decodebin.get(), nullptr);

diff --git a/Source/WebCore/platform/graphics/MediaPlayer.cpp b/Source/WebCore/platform/graphics/MediaPlayer.cpp
@@ -1625,6 +1625,11 @@ void MediaPlayer::simulateAudioInterruption()
 
     m_private->simulateAudioInterruption();
 }
+
+bool MediaPlayer::isGStreamerHolePunchingEnabled()
+{
+    return client().isGStreamerHolePunchingEnabled();
+}
 #endif
 
 void MediaPlayer::beginSimulatedHDCPError()

diff --git a/Source/WebCore/platform/graphics/MediaPlayer.h b/Source/WebCore/platform/graphics/MediaPlayer.h
@@ -295,6 +295,8 @@ class MediaPlayerClient {
 
     virtual bool mediaPlayerShouldDisableHDR() const { return false; }
 
+    virtual bool isGStreamerHolePunchingEnabled() const { return false; }
+
 #if !RELEASE_LOG_DISABLED
     virtual const void* mediaPlayerLogIdentifier() { return nullptr; }
     virtual const Logger& mediaPlayerLogger() = 0;
@@ -620,6 +622,7 @@ class WEBCORE_EXPORT MediaPlayer : public MediaPlayerEnums, public ThreadSafeRef
 
 #if USE(GSTREAMER)
     void simulateAudioInterruption();
+    bool isGStreamerHolePunchingEnabled();
 #endif
 
     void beginSimulatedHDCPError();

@@ -27,6 +27,7 @@
 #include "DMABufVideoSinkGStreamer.h"
 #include "GLVideoSinkGStreamer.h"
 #include "GStreamerAudioMixer.h"
+#include "GStreamerQuirks.h"
 #include "GStreamerRegistryScanner.h"
 #include "GStreamerSinksWorkarounds.h"
 #include "GUniquePtrGStreamer.h"
@@ -307,19 +308,6 @@ bool ensureGStreamerInitialized()
             gst_mpegts_initialize();
 #endif
 
-#if PLATFORM(BCM_NEXUS)
-        {
-            auto registry = gst_registry_get();
-            GRefPtr<GstPluginFeature> brcmaudfilter = adoptGRef(gst_registry_lookup_feature(registry, "brcmaudfilter"));
-            GRefPtr<GstPluginFeature> mpegaudioparse = adoptGRef(gst_registry_lookup_feature(registry, "mpegaudioparse"));
-
-            if (brcmaudfilter && mpegaudioparse) {
-                GST_INFO("overriding mpegaudioparse rank with brcmaudfilter rank + 1");
-                gst_plugin_feature_set_rank(mpegaudioparse.get(), gst_plugin_feature_get_rank(brcmaudfilter.get()) + 1);
-            }
-        }
-#endif
-
         registerAppsinkWithWorkaroundsIfNeeded();
 #endif
     });
@@ -421,6 +409,10 @@ void registerWebKitGStreamerElements()
             if (auto vaapiPlugin = adoptGRef(gst_registry_find_plugin(registry, "vaapi")))
                 gst_registry_remove_plugin(registry, vaapiPlugin.get());
         }
+
+        // Make sure the quirks are created as early as possible.
+        [[maybe_unused]] auto& quirksManager = GStreamerQuirksManager::singleton();
+
         registryWasUpdated = true;
     });
 

@@ -19,6 +19,7 @@
 
 #include "config.h"
 #include "GStreamerRegistryScanner.h"
+#include "GStreamerQuirks.h"
 
 #if USE(GSTREAMER)
 #include "ContentType.h"
@@ -77,17 +78,12 @@ void GStreamerRegistryScanner::getSupportedDecodingTypes(HashSet<String, ASCIICa
 
 GStreamerRegistryScanner::ElementFactories::ElementFactories(OptionSet<ElementFactories::Type> types)
 {
-#if PLATFORM(BCM_NEXUS) || PLATFORM(BROADCOM)
+    auto& quirksManager = GStreamerQuirksManager::singleton();
+    auto audioVideoDecoderFactory = quirksManager.audioVideoDecoderFactoryListType();
     if (types.contains(Type::AudioDecoder))
-        audioDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_PARSER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO, GST_RANK_MARGINAL);
+        audioDecoderFactories = gst_element_factory_list_get_elements(audioVideoDecoderFactory | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO, GST_RANK_MARGINAL);
     if (types.contains(Type::VideoDecoder))
-        videoDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_PARSER | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, GST_RANK_MARGINAL);
-#else
-    if (types.contains(Type::AudioDecoder))
-        audioDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO, GST_RANK_MARGINAL);
-    if (types.contains(Type::VideoDecoder))
-        videoDecoderFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_DECODER | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, GST_RANK_MARGINAL);
-#endif
+        videoDecoderFactories = gst_element_factory_list_get_elements(audioVideoDecoderFactory | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, GST_RANK_MARGINAL);
     if (types.contains(Type::AudioParser))
         audioParserFactories = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_PARSER | GST_ELEMENT_FACTORY_TYPE_MEDIA_AUDIO, GST_RANK_NONE);
     if (types.contains(Type::VideoParser))
@@ -236,15 +232,16 @@ GStreamerRegistryScanner::RegistryLookupResult GStreamerRegistryScanner::Element
             auto* factory = reinterpret_cast<GstElementFactory*>(factories->data);
             auto metadata = String::fromLatin1(gst_element_factory_get_metadata(factory, GST_ELEMENT_METADATA_KLASS));
             auto components = metadata.split('/');
-            if (components.contains("Hardware"_s)
-#if PLATFORM(BCM_NEXUS) || PLATFORM(BROADCOM)
-                || g_str_has_prefix(GST_OBJECT_NAME(factory), "brcm")
-#elif PLATFORM(REALTEK)
-                || g_str_has_prefix(GST_OBJECT_NAME(factory), "omx")
-#elif USE(WESTEROS_SINK)
-                || g_str_has_prefix(GST_OBJECT_NAME(factory), "westeros")
-#endif
-                ) {
+            auto& quirksManager = GStreamerQuirksManager::singleton();
+            if (quirksManager.isEnabled()) {
+                auto isAccelerated = quirksManager.isHardwareAccelerated(factory);
+                if (isAccelerated && *isAccelerated) {
+                    isUsingHardware = true;
+                    selectedFactory = factory;
+                    break;
+                }
+            }
+            if (components.contains("Hardware"_s)) {
                 isUsingHardware = true;
                 selectedFactory = factory;
                 break;