refactor plugin to use OpenGL fbo rendering

CMakeLists.txt

@@ -15,18 +15,22 @@ find_package(GStreamer REQUIRED COMPONENTS gstreamer-audio gstreamer-gl gstreame
 find_package(GLIB2 REQUIRED)
 
 add_library(gstprojectm SHARED
-    src/caps.h
-    src/caps.c
+    src/gstprojectmcaps.h
+    src/gstprojectmcaps.c
     src/debug.h
     src/debug.c
-    src/config.h
-    src/enums.h
-    src/plugin.h
-    src/plugin.c
-    src/projectm.h
-    src/projectm.c
+    src/gstprojectmconfig.h
+    src/gstprojectm.h
+    src/gstprojectm.c
     src/gstglbaseaudiovisualizer.h
     src/gstglbaseaudiovisualizer.c
+    src/gstpmaudiovisualizer.h
+    src/gstpmaudiovisualizer.c
+    src/gstprojectmbase.h
+    src/gstprojectmbase.c
+    src/register.c
+    src/renderbuffer.h
+    src/renderbuffer.c
 )
 
 target_include_directories(gstprojectm

README.md

@@ -57,25 +57,50 @@ The documentation has been organized into distinct files, each dedicated to a sp
 - **[OSX](docs/OSX.md)**
 - **[Windows](docs/WINDOWS.md)**
 
-Once the plugin has been installed, you can use it something like this:
+Once the plugin has been installed, you can use it something like this to render to an OpenGL window:
 
 ```shell
-gst-launch pipewiresrc ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 ! video/x-raw,width=2048,height=1440,framerate=60/1 ! videoconvert ! xvimagesink sync=false
+gst-launch pipewiresrc ! queue ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! projectm preset=/usr/local/share/projectM/presets preset-duration=10 mesh-size=48,32 ! 'video/x-raw(memory:GLMemory),width=2048,height=1440,framerate=60/1' ! glimagesink sync=false
 ```
 
-Or to convert an audio file to video:
+To render from a live source in real-time to a gl window, an identity element can be used to provide a proper timestamp source for the pipeline. This example also includes a texture directory: 
+```shell
+gst-launch souphttpsrc location=http://your-radio-stream is-live=true ! queue ! decodebin ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! identity single-segment=true sync=true ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 mesh-size=48,32 is-live=true texture-dir=/usr/local/share/projectM/presets-milkdrop-texture-pack ! video/x-raw(memory:GLMemory),width=1920,height=1080,framerate=60/1 ! glimagesink sync=false
+```
+
+Or to convert an audio file to video using offline rendering:
 
 ```shell
+gst-launch-1.0 -e \
 filesrc location=input.mp3 ! decodebin name=dec \
     decodebin ! tee name=t \
       t. ! queue ! audioconvert ! audioresample ! \
             capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! avenc_aac bitrate=256000 ! queue ! mux. \
-      t. ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 ! \
-            identity sync=false ! videoconvert ! videorate ! video/x-raw,framerate=60/1,width=3840,height=2160 ! \
+      t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! \
+           projectm preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 is-live=false ! \
+            identity sync=false ! videoconvert ! videorate ! video/x-raw\(memory:GLMemory\),framerate=60/1,width=3840,height=2160 ! \
+            gldownload \
             x264enc bitrate=35000 key-int-max=300 speed-preset=veryslow ! video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \
   mp4mux name=mux ! filesink location=render.mp4;
 ```
 
+Or converting an audio file with the nVidia optimized encoder, directly from GL memory:
+```shell
+gst-launch-1.0 -e \
+  filesrc location=input.mp3 ! \
+    decodebin ! tee name=t \
+      t. ! queue ! audioconvert ! audioresample ! \
+            capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! \
+            avenc_aac bitrate=320000 ! queue ! mux. \
+      t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! projectm \
+            preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 is-live=false ! \
+            identity sync=false ! videoconvert ! videorate ! \
+            video/x-raw\(memory:GLMemory\),framerate=60/1,width=1920,height=1080 ! \
+            nvh264enc ! h264parse ! \
+            video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \
+    mp4mux name=mux ! filesink location=render.mp4;
+```
+
 Available options
 
 ```shell
@@ -193,21 +218,23 @@ If you have your own ProjectM preset files:
 Once the plugin has been installed, you can use it something like this:
 
 ```shell
-gst-launch pipewiresrc ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 ! video/x-raw,width=2048,height=1440,framerate=60/1 ! videoconvert ! xvimagesink sync=false
+gst-launch pipewiresrc ! queue ! audioconvert ! "audio/x-raw, format=S16LE, rate=44100, channels=2, layout=interleaved" ! projectm preset=/usr/local/share/projectM/presets preset-duration=5 mesh-size=48,32 ! 'video/x-raw(memory:GLMemory),width=2048,height=1440,framerate=60/1' ! glimagesink sync=false
 ```
 
 Or to convert an audio file to video:
 
 ```shell
 gst-launch-1.0 -e \
-  filesrc location=input.mp3  ! \
+  filesrc location=input.mp3 ! decodebin name=dec \
     decodebin ! tee name=t \
       t. ! queue ! audioconvert ! audioresample ! \
-            capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! avenc_aac bitrate=320000 ! queue ! mux. \
-      t. ! queue ! audioconvert ! projectm preset=/usr/local/share/projectM/presets texture-dir=/usr/local/share/projectM/textures preset-duration=6 mesh-size=1024,576 ! \
-            identity sync=false ! videoconvert ! videorate ! video/x-raw,framerate=60/1,width=3840,height=2160 ! \
-            x264enc bitrate=50000 key-int-max=200 speed-preset=veryslow ! video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \
-    mp4mux name=mux ! filesink location=output.mp4
+            capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! avenc_aac bitrate=256000 ! queue ! mux. \
+      t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! \
+           projectm preset=/usr/local/share/projectM/presets preset-duration=3 mesh-size=1024,576 is-live=false ! \
+            identity sync=false ! videoconvert ! videorate ! video/x-raw\(memory:GLMemory\),framerate=60/1,width=3840,height=2160 ! \
+            gldownload \
+            x264enc bitrate=35000 key-int-max=300 speed-preset=veryslow ! video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \
+  mp4mux name=mux ! filesink location=render.mp4;
 ```
 
 You may need to adjust some elements which may or may not be present in your GStreamer installation, such as x264enc, avenc_aac, etc.
@@ -220,6 +247,53 @@ gst-inspect projectm
 
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 
+## Technical Details
+
+### OpenGL Rendering and Buffer Handling
+
+- projectM output is rendered to OpenGL textures via **Frame Buffer Object (FBO)**.
+- **Textures are pooled** and reused across frames. 
+- Each rendered texture becomes a GStreamer video buffer pushed downstream. **All video buffers stay in GPU memory**.
+
+---
+
+### Timing and Synchronization
+
+The plugin synchronizes rendering to the GStreamer pipeline clock using **audio presentation timestamp (PTS) as the leading reference**.
+
+Pipeline caps control the desired video framerate for rendering. The render loop is **push-based** to conform with 
+GStreamer's pipeline timing concept, and to enable faster-than-real-time rendering.
+A **fixed number of audio samples is consumed per video frame**. 
+
+**Example:** `735 samples per frame at 44.1 kHz = ~60 FPS.`
+
+**Note:** Live pipelines are auto-detected by the plugin. For cases where auto-detection is not appropriate,
+the `is-live` property can be configured.
+
+**Live pipelines only:** Frames may be dropped or rendering FPS adjusted if frame rendering can't keep up with 
+pipeline caps FPS.
+
+Video frame PTS offset is derived from the **first audio buffer PTS** or **segment event** plus accumulated samples to align with audio timing.
+
+
+| Timing Source                    | Origin             | Applies to clock | Purpose                                                                                           |
+|----------------------------------|--------------------|------------------|---------------------------------------------------------------------------------------------------|
+| Audio Timestamps                 | Audio Input        | Always           | Determine video timing and sync.                                                                  |
+| Sample Rate / Pipeline FPS       | Audio Input / Caps | Always           | Defines how many audio samples are used per frame and target FPS.                                 |
+| Segment Info                     | Segment Event      | Always           | Tracks running time and playback position. Used for PTS offsets.                                  |
+| QoS Feedback                     | QoS Event          | Live             | Skips outdated frames to reduce latency.                                                          |
+| Render Frame Drop                | Render Loop        | Live             | Drop frames that cannot be rendered in time.                                                      |
+| Exponential Moving Average (EMA) | Render Loop        | Live             | Adjust plugin target FPS in case frame render time exceeds the real-time budget most of the time. |
+| Latency Event                    | Render Loop        | Live             | Inform upstream of latency changes in case of adaptive FPS changes (EMA).                         |
+
+
+---
+
+
+<p align="right">(<a href="#readme-top">back to top</a>)</p>
+
+---
+
 <!-- CONTRIBUTING -->
 
 ## Contributing
@@ -261,6 +335,8 @@ Blaquewithaq (Discord: SoFloppy#1289) - [@anomievision](https://twitter.com/anom
 
 Mischa (Discord: mish) - [@revmischa](https://github.com/revmischa)
 
+Michael [@mbaetgen-wup](https://github.com/mbaetgen-wup) - michael -at- widerup.com
+
 <p align="right">(<a href="#readme-top">back to top</a>)</p>
 
 <!----------------------------------------------------------------------->

convert.sh

@@ -154,13 +154,15 @@ gst-launch-1.0 -e \
       t. ! queue ! audioconvert ! audioresample ! \
             capsfilter caps="audio/x-raw, format=F32LE, channels=2, rate=44100" ! \
             avenc_aac bitrate=320000 ! queue ! mux. \
-      t. ! queue ! audioconvert ! projectm \
+      t. ! queue ! audioconvert ! capsfilter caps="audio/x-raw, format=S16LE, channels=2, rate=44100" ! \
+          projectm \
             preset=$PRESET_PATH \
             texture-dir=$TEXTURE_DIR \
             preset-duration=$PRESET_DURATION \
             mesh-size=${MESH_X},${MESH_Y} ! \
             identity sync=false ! videoconvert ! videorate ! \
-            video/x-raw,framerate=$FRAMERATE/1,width=$VIDEO_WIDTH,height=$VIDEO_HEIGHT ! \
+            video/x-raw\(memory:GLMemory\),framerate=$FRAMERATE/1,width=$VIDEO_WIDTH,height=$VIDEO_HEIGHT ! \
+            gldownload ! \
             x264enc bitrate=$(($BITRATE * 1000)) key-int-max=200 speed-preset=$SPEED_PRESET ! \
             video/x-h264,stream-format=avc,alignment=au ! queue ! mux. \
     mp4mux name=mux ! filesink location=$OUTPUT_FILE &

src/debug.c

@@ -7,7 +7,7 @@
 
 #include "debug.h"
 
-void gl_error_handler(GstGLContext *context, gpointer data) {
+void gl_error_handler(GstGLContext *context) {
   GLuint error = context->gl_vtable->GetError();
 
   switch (error) {
@@ -47,4 +47,4 @@ void gl_error_handler(GstGLContext *context, gpointer data) {
     g_error("OpenGL Error: Unknown error code - 0x%x\n", error);
     break;
   }
-}
+}

src/debug.h

@@ -1,7 +1,6 @@
 #ifndef __GST_PROJECTM_DEBUG_H__
 #define __GST_PROJECTM_DEBUG_H__
 
-#include <glib.h>
 #include <gst/gl/gl.h>
 
 G_BEGIN_DECLS
@@ -26,8 +25,8 @@ G_BEGIN_DECLS
  * @param context The OpenGL context.
  * @param data Unused.
  */
-void gl_error_handler(GstGLContext *context, gpointer data);
+void gl_error_handler(GstGLContext *context);
 
 G_END_DECLS
 
-#endif /* __GST_PROJECTM_DEBUG_H__ */
+#endif /* __GST_PROJECTM_DEBUG_H__ */