[GStreamer][WebAudio] Garbled rendering when the internal audio mixer is enabled

https://bugs.webkit.org/show_bug.cgi?id=298235

Reviewed by Xabier Rodriguez-Calvar.

When the WEBKIT_GST_ENABLE_AUDIO_MIXER=1 environment variable is set, audio buffers are sent to a
separate pipeline using interaudiosink. That process-wide singleton pipeline is then in charge of
audio mixing and rendering to an actual audio device.

For WebAudio it wasn't working properly, leading to garbled rendering, due to the audiomixer sink
pad corresponding to the WebAudio source not using the same sample rate and also interaudiosrc using
a sample period-size too big. We now align these values between the WebAudio sink and its matching
interaudiosrc audiomixer branch.

The Rialto WebAudio sink configuration was also moved to the Rialto quirk and the version check
removed (I couldn't find the reasoning behind that).

* Source/WebCore/platform/audio/gstreamer/AudioDestinationGStreamer.cpp:
(WebCore::AudioDestinationGStreamer::AudioDestinationGStreamer):
* Source/WebCore/platform/graphics/gstreamer/GStreamerAudioMixer.cpp:
(WebCore::GStreamerAudioMixer::registerProducer):
(WebCore::GStreamerAudioMixer::configureSourcePeriodTime):
* Source/WebCore/platform/graphics/gstreamer/GStreamerAudioMixer.h:
* Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.cpp:
(WebCore::createPlatformAudioSink):
* Source/WebCore/platform/graphics/gstreamer/WebKitAudioSinkGStreamer.cpp:
(webKitAudioSinkConfigure):
(webKitAudioSinkChangeState):
(webKitAudioSinkConstructed):
(webkitAudioSinkNew):
* Source/WebCore/platform/graphics/gstreamer/WebKitAudioSinkGStreamer.h:
* Source/WebCore/platform/gstreamer/GStreamerQuirkRialto.cpp:
(WebCore::GStreamerQuirkRialto::createWebAudioSink):
* Source/WebCore/platform/gstreamer/GStreamerQuirks.cpp:
(WebCore::GStreamerQuirksManager::createWebAudioSink):

Canonical link: https://commits.webkit.org/299489@main

Author: philn

Source/WebCore/platform/audio/gstreamer/AudioDestinationGStreamer.cpp

@@ -165,24 +165,10 @@ AudioDestinationGStreamer::AudioDestinationGStreamer(const CreationOptions& opti
 
     gst_bin_add_many(GST_BIN_CAST(m_pipeline.get()), m_src.get(), audioConvert, audioResample, clockSync, audioSink.get(), nullptr);
 
-    // Link src pads from webkitAudioSrc to clocksync ! audioConvert ! audioResample ! [capsfilter !] autoaudiosink.
+    // Link src pads from webkitAudioSrc to clocksync ! audioConvert ! audioResample ! audiosink.
     gst_element_link_pads_full(m_src.get(), "src", clockSync, "sink", GST_PAD_LINK_CHECK_NOTHING);
     gst_element_link_pads_full(clockSync, "src", audioConvert, "sink", GST_PAD_LINK_CHECK_NOTHING);
     gst_element_link_pads_full(audioConvert, "src", audioResample, "sink", GST_PAD_LINK_CHECK_NOTHING);
-
-    if (!webkitGstCheckVersion(1, 20, 4)) {
-        // Force audio conversion to 'interleaved' format (by audioconvert element).
-        // 1) Some platform sinks don't support non-interleaved audio without special caps (rialtowebaudiosink).
-        // 2) Interaudio sink/src doesn't fully support non-interleaved audio (webkit audio sink)
-        // 3) audiomixer doesn't support non-interleaved audio in output pipeline (webkit audio sink)
-        GstElement* capsFilter = makeGStreamerElement("capsfilter"_s);
-        GRefPtr<GstCaps> caps = adoptGRef(gst_caps_new_simple("audio/x-raw", "layout", G_TYPE_STRING, "interleaved", nullptr));
-        g_object_set(capsFilter, "caps", caps.get(), nullptr);
-        gst_bin_add(GST_BIN_CAST(m_pipeline.get()), capsFilter);
-        gst_element_link_pads_full(audioResample, "src", capsFilter, "sink", GST_PAD_LINK_CHECK_NOTHING);
-        gst_element_link_pads_full(capsFilter, "src", audioSink.get(), "sink", GST_PAD_LINK_CHECK_NOTHING);
-        return;
-    }
     gst_element_link_pads_full(audioResample, "src", audioSink.get(), "sink", GST_PAD_LINK_CHECK_NOTHING);
 }
 

Source/WebCore/platform/graphics/gstreamer/GStreamerAudioMixer.cpp

@@ -86,9 +86,11 @@ void GStreamerAudioMixer::ensureState(GstStateChange stateChange)
     }
 }
 
-GRefPtr<GstPad> GStreamerAudioMixer::registerProducer(GstElement* interaudioSink)
+GRefPtr<GstPad> GStreamerAudioMixer::registerProducer(GstElement* interaudioSink, std::optional<int> forcedSampleRate)
 {
-    GstElement* src = makeGStreamerElement("interaudiosrc"_s);
+    auto name = StringView::fromLatin1(GST_ELEMENT_NAME(interaudioSink));
+    GstElement* src = makeGStreamerElement("interaudiosrc"_s, name.toStringWithoutCopying());
+
     g_object_set(src, "channel", GST_ELEMENT_NAME(interaudioSink), nullptr);
     g_object_set(interaudioSink, "channel", GST_ELEMENT_NAME(interaudioSink), nullptr);
 
@@ -98,6 +100,14 @@ GRefPtr<GstPad> GStreamerAudioMixer::registerProducer(GstElement* interaudioSink
     gst_bin_add_many(GST_BIN_CAST(bin), audioResample, audioConvert, nullptr);
     gst_element_link(audioConvert, audioResample);
 
+    if (forcedSampleRate) {
+        auto capsfilter = gst_element_factory_make("capsfilter", nullptr);
+        auto caps = adoptGRef(gst_caps_new_simple("audio/x-raw", "rate", G_TYPE_INT, *forcedSampleRate, nullptr));
+        g_object_set(capsfilter, "caps", caps.get(), nullptr);
+        gst_bin_add(GST_BIN_CAST(bin), capsfilter);
+        gst_element_link(audioResample, capsfilter);
+    }
+
     if (auto pad = adoptGRef(gst_bin_find_unlinked_pad(GST_BIN_CAST(bin), GST_PAD_SRC)))
         gst_element_add_pad(GST_ELEMENT_CAST(bin), gst_ghost_pad_new("src", pad.get()));
     if (auto pad = adoptGRef(gst_bin_find_unlinked_pad(GST_BIN_CAST(bin), GST_PAD_SINK)))
@@ -148,6 +158,15 @@ void GStreamerAudioMixer::unregisterProducer(const GRefPtr<GstPad>& mixerPad)
     GST_DEBUG_BIN_TO_DOT_FILE_WITH_TS(GST_BIN_CAST(m_pipeline.get()), GST_DEBUG_GRAPH_SHOW_ALL, "audio-mixer-after-producer-unregistration");
 }
 
+void GStreamerAudioMixer::configureSourcePeriodTime(StringView sourceName, uint64_t periodTime)
+{
+    auto src = adoptGRef(gst_bin_get_by_name(GST_BIN_CAST(m_pipeline.get()), sourceName.toStringWithoutCopying().ascii().data()));
+    if (!src) [[unlikely]]
+        return;
+
+    g_object_set(src.get(), "period-time", periodTime, nullptr);
+}
+
 #undef GST_CAT_DEFAULT
 
 } // namespace WebCore

Source/WebCore/platform/graphics/gstreamer/GStreamerAudioMixer.h

@@ -33,9 +33,11 @@ class GStreamerAudioMixer {
     static GStreamerAudioMixer& singleton();
 
     void ensureState(GstStateChange);
-    GRefPtr<GstPad> registerProducer(GstElement*);
+    GRefPtr<GstPad> registerProducer(GstElement*, std::optional<int> forcedSampleRate);
     void unregisterProducer(const GRefPtr<GstPad>&);
 
+    void configureSourcePeriodTime(StringView sourceName, uint64_t periodTime);
+
 private:
     GStreamerAudioMixer();
 

Source/WebCore/platform/graphics/gstreamer/GStreamerCommon.cpp

@@ -1026,14 +1026,12 @@ IGNORE_WARNINGS_END
 
 GstElement* /* (transfer floating) */ createPlatformAudioSink(const String& role)
 {
-    GstElement* audioSink = webkitAudioSinkNew();
+    GstElement* audioSink = webkitAudioSinkNew(role);
     if (!audioSink) {
         // This means the WebKit audio sink configuration failed. It can happen for the following reasons:
         // - audio mixing was not requested using the WEBKIT_GST_ENABLE_AUDIO_MIXER
         // - audio mixing was requested using the WEBKIT_GST_ENABLE_AUDIO_MIXER but the audio mixer
         //   runtime requirements are not fullfilled.
-        // - the sink was created for the WPE port, audio mixing was not requested and no
-        //   WPEBackend-FDO audio receiver has been registered at runtime.
         audioSink = createAutoAudioSink(role);
     }
 

Source/WebCore/platform/graphics/gstreamer/WebKitAudioSinkGStreamer.cpp

@@ -22,15 +22,21 @@
 
 #if USE(GSTREAMER)
 
+#include "AudioUtilities.h"
 #include "GStreamerAudioMixer.h"
 #include "GStreamerCommon.h"
 #include <wtf/glib/WTFGType.h>
 
+#if ENABLE(WEB_AUDIO)
+#include "AudioDestination.h"
+#endif
+
 using namespace WebCore;
 
 struct _WebKitAudioSinkPrivate {
     GRefPtr<GstElement> interAudioSink;
     GRefPtr<GstPad> mixerPad;
+    String role;
 };
 
 enum {
@@ -65,6 +71,36 @@ static bool webKitAudioSinkConfigure(WebKitAudioSink* sink)
         gst_bin_add(GST_BIN_CAST(sink), sink->priv->interAudioSink.get());
         auto targetPad = adoptGRef(gst_element_get_static_pad(sink->priv->interAudioSink.get(), "sink"));
         gst_element_add_pad(GST_ELEMENT_CAST(sink), webkitGstGhostPadFromStaticTemplate(&audioSinkTemplate, "sink"_s, targetPad.get()));
+
+        if (sink->priv->role != "webaudio"_s)
+            return true;
+
+        // Match the interaudiosrc period-time with the WebAudio renderQuantumSize applied to the
+        // sample rate, otherwise the samples created by the source will have clipping, leading to
+        // garbled rendering. For this to work the sample rate also needs to match between
+        // webkitaudiosink and the caps negotiated on the audiomixer sink pad (this is handled in
+        // webKitAudioSinkChangeState()).
+        gst_pad_add_probe(targetPad.get(), GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, reinterpret_cast<GstPadProbeCallback>(+[](GstPad* pad, GstPadProbeInfo* info, gpointer) -> GstPadProbeReturn {
+            auto event = GST_PAD_PROBE_INFO_EVENT(info);
+            if (GST_EVENT_TYPE(event) != GST_EVENT_CAPS)
+                return GST_PAD_PROBE_OK;
+
+            GstCaps* caps;
+            gst_event_parse_caps(event, &caps);
+
+            if (gst_caps_is_empty(caps) || gst_caps_is_any(caps)) [[unlikely]]
+                return GST_PAD_PROBE_OK;
+
+            auto structure = gst_caps_get_structure(caps, 0);
+            auto sampleRate = gstStructureGet<int>(structure, "rate");
+            if (!sampleRate) [[unlikely]]
+                return GST_PAD_PROBE_OK;
+
+            auto sink = adoptGRef(gst_pad_get_parent_element(pad));
+            uint64_t periodTime = gst_util_uint64_scale_ceil(AudioUtilities::renderQuantumSize, GST_SECOND, *sampleRate);
+            GStreamerAudioMixer::singleton().configureSourcePeriodTime(StringView::fromLatin1(GST_ELEMENT_NAME(sink.get())), periodTime);
+            return GST_PAD_PROBE_OK;
+        }), nullptr, nullptr);
         return true;
     }
     return false;
@@ -116,8 +152,14 @@ static GstStateChangeReturn webKitAudioSinkChangeState(GstElement* element, GstS
     GST_DEBUG_OBJECT(sink, "Handling %s transition", gst_state_change_get_name(stateChange));
 
     auto& mixer = GStreamerAudioMixer::singleton();
-    if (priv->interAudioSink && stateChange == GST_STATE_CHANGE_NULL_TO_READY)
-        priv->mixerPad = mixer.registerProducer(priv->interAudioSink.get());
+    if (priv->interAudioSink && stateChange == GST_STATE_CHANGE_NULL_TO_READY) {
+        std::optional<int> forcedSampleRate;
+#if ENABLE(WEB_AUDIO)
+        if (priv->role == "webaudio"_s)
+            forcedSampleRate = AudioDestination::hardwareSampleRate();
+#endif
+        priv->mixerPad = mixer.registerProducer(priv->interAudioSink.get(), forcedSampleRate);
+    }
 
     if (priv->mixerPad)
         mixer.ensureState(stateChange);
@@ -135,10 +177,10 @@ static GstStateChangeReturn webKitAudioSinkChangeState(GstElement* element, GstS
 static void webKitAudioSinkConstructed(GObject* object)
 {
     G_OBJECT_CLASS(webkit_audio_sink_parent_class)->constructed(object);
-IGNORE_WARNINGS_BEGIN("cast-align")
+    IGNORE_WARNINGS_BEGIN("cast-align");
     GST_OBJECT_FLAG_SET(GST_OBJECT_CAST(object), GST_ELEMENT_FLAG_SINK);
     gst_bin_set_suppressed_flags(GST_BIN_CAST(object), static_cast<GstElementFlags>(GST_ELEMENT_FLAG_SOURCE | GST_ELEMENT_FLAG_SINK));
-IGNORE_WARNINGS_END
+    IGNORE_WARNINGS_END;
 }
 
 static void webkit_audio_sink_class_init(WebKitAudioSinkClass* klass)
@@ -164,15 +206,18 @@ static void webkit_audio_sink_class_init(WebKitAudioSinkClass* klass)
     eklass->change_state = GST_DEBUG_FUNCPTR(webKitAudioSinkChangeState);
 }
 
-GstElement* /* (transfer floating) */ webkitAudioSinkNew()
+GstElement* /* (transfer floating) */ webkitAudioSinkNew(const String& role)
 {
-    auto* sink = GST_ELEMENT_CAST(g_object_new(WEBKIT_TYPE_AUDIO_SINK, nullptr));
-    if (!webKitAudioSinkConfigure(WEBKIT_AUDIO_SINK(sink))) {
-        gst_object_unref(sink);
+    auto element = GST_ELEMENT_CAST(g_object_new(WEBKIT_TYPE_AUDIO_SINK, nullptr));
+    auto audioSink = WEBKIT_AUDIO_SINK(element);
+
+    audioSink->priv->role = role;
+    if (!webKitAudioSinkConfigure(audioSink)) {
+        gst_object_unref(element);
         return nullptr;
     }
-    ASSERT(g_object_is_floating(sink));
-    return sink;
+    ASSERT(g_object_is_floating(element));
+    return element;
 }
 
 #undef GST_CAT_DEFAULT

Source/WebCore/platform/graphics/gstreamer/WebKitAudioSinkGStreamer.h

@@ -48,6 +48,6 @@ GType webkit_audio_sink_get_type(void);
 
 G_END_DECLS
 
-GstElement* webkitAudioSinkNew();
+GstElement* webkitAudioSinkNew(const String&);
 
 #endif // USE(GSTREAMER)

Source/WebCore/platform/gstreamer/GStreamerQuirkRialto.cpp

@@ -94,14 +94,29 @@ GstElement* GStreamerQuirkRialto::createAudioSink()
     return sink;
 }
 
-GstElement* GStreamerQuirkRialto::createWebAudioSink()
+GstElement* /* transfer floating */ GStreamerQuirkRialto::createWebAudioSink()
 {
-    if (GstElement* sink = webkitAudioSinkNew())
+    if (GstElement* sink = webkitAudioSinkNew("webaudio"_s))
         return sink;
 
     auto sink = makeGStreamerElement("rialtowebaudiosink"_s);
     RELEASE_ASSERT_WITH_MESSAGE(sink, "rialtowebaudiosink should be available in the system but it is not");
-    return sink;
+
+    // Force audio conversion to 'interleaved' format. The rialtowebaudiosink doesn't support
+    // non-interleaved audio without special caps, which seems like a bug in that sink's caps
+    // template and/or caps negotiation implementation.
+    auto bin = gst_bin_new(nullptr);
+    auto capsFilter = gst_element_factory_make("capsfilter", nullptr);
+    auto caps = adoptGRef(gst_caps_new_simple("audio/x-raw", "layout", G_TYPE_STRING, "interleaved", nullptr));
+    g_object_set(capsFilter, "caps", caps.get(), nullptr);
+
+    gst_bin_add_many(GST_BIN_CAST(bin), capsFilter, sink, nullptr);
+    gst_element_link(capsFilter, sink);
+
+    auto pad = adoptGRef(gst_element_get_static_pad(capsFilter, "sink"));
+    gst_element_add_pad(bin, gst_ghost_pad_new("sink", pad.get()));
+
+    return bin;
 }
 
 std::optional<bool> GStreamerQuirkRialto::isHardwareAccelerated(GstElementFactory* factory)

Source/WebCore/platform/gstreamer/GStreamerQuirks.cpp

@@ -168,7 +168,7 @@ GstElement* GStreamerQuirksManager::createWebAudioSink()
     }
 
     GST_DEBUG("Quirks didn't specify a WebAudioSink, falling back to default sink");
-    return createPlatformAudioSink("music"_s);
+    return createPlatformAudioSink("webaudio"_s);
 }
 
 GstElement* GStreamerQuirksManager::createHolePunchVideoSink(bool isLegacyPlaybin, const MediaPlayer* player)