feat: implement playback speed display and timescale adjustments in audio playback

virajbhartiya · virajbhartiya · commit 5dba5c16dfe1 · 2025-05-05T01:56:03.000+05:30
diff --git a/apps/desktop/src/routes/editor/Player.tsx b/apps/desktop/src/routes/editor/Player.tsx
@@ -136,6 +136,12 @@ export function Player() {
         >
           Crop
         </EditorButton>
+        <Show when={currentSpeed()}>
+          <div class="flex items-center gap-1 px-3 py-1 bg-blue-500/20 rounded-lg text-gray-600 font-medium">
+            <IconLucideClock class="size-4 mr-1" />
+            {currentSpeed()}
+          </div>
+        </Show>
       </div>
       <PreviewCanvas />
       <div class="flex z-10 overflow-hidden flex-row gap-3 justify-between items-center p-5">
diff --git a/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx b/apps/desktop/src/routes/editor/Timeline/ClipTrack.tsx
@@ -42,13 +42,25 @@ export function ClipTrack(props: Pick<ComponentProps<"div">, "ref">) {
               .slice(0, i())
               .reduce((t, s) => t + (s.end - s.start) / s.timescale, 0);
 
+          const getSpeedClass = () => {
+            if (segment.timescale === 1) return "";
+            if (segment.timescale > 1)
+              return "bg-stripes-blue border-l-4 border-l-blue-500";
+            return "bg-stripes-green border-l-4 border-l-green-500";
+          };
+
+          const getSpeedText = () => {
+            if (segment.timescale === 1) return null;
+            return (1 / segment.timescale).toFixed(2) + "x";
+          };
+
           return (
             <SegmentRoot
               class={cx(
                 "overflow-hidden border border-transparent transition-colors duration-300 group",
                 "hover:border-gray-500",
                 "bg-gradient-to-r timeline-gradient-border from-[#2675DB] via-[#4FA0FF] to-[#2675DB] shadow-[inset_0_5px_10px_5px_rgba(255,255,255,0.2)]",
-                segment.timescale !== 1 && "bg-stripes-blue"
+                getSpeedClass()
               )}
               innerClass="ring-blue-300"
               segment={{
@@ -164,10 +176,9 @@ export function ClipTrack(props: Pick<ComponentProps<"div">, "ref">) {
                           <IconLucideClock class="size-3.5" />{" "}
                           {(segment.end - segment.start).toFixed(1)}s
                         </div>
-                        <Show when={segment.timescale !== 1}>
-                          <div class="flex gap-1 items-center text-gray-50 dark:text-gray-500 text-md bg-blue-500/20 px-2 py-0.5 rounded-full mt-1">
-                            <IconLucideClock class="size-3" />{" "}
-                            {(1 / segment.timescale).toFixed(2)}x
+                        <Show when={getSpeedText()}>
+                          <div class="flex gap-1 items-center text-gray-50 dark:text-gray-500 text-md bg-blue-500/30 px-2 py-1 rounded-full mt-1 font-medium shadow-sm">
+                            <IconLucideClock class="size-3" /> {getSpeedText()}
                           </div>
                         </Show>
                       </div>
diff --git a/crates/editor/src/playback.rs b/crates/editor/src/playback.rs
@@ -3,7 +3,7 @@ use std::{sync::Arc, time::Duration};
 use cap_media::data::{AudioInfo, FromSampleBytes};
 use cap_media::feeds::{AudioPlaybackBuffer, AudioSegment, AudioSegmentTrack};
 use cap_media::MediaError;
-use cap_project::{AudioConfiguration, ProjectConfiguration, XY};
+use cap_project::{AudioConfiguration, ProjectConfiguration, TimelineSegment, XY};
 use cap_rendering::{ProjectUniforms, RenderVideoConstants};
 use cpal::{
     traits::{DeviceTrait, HostTrait, StreamTrait},
@@ -68,46 +68,117 @@ impl Playback {
             }
             .spawn();
 
+            let mut current_segment_index: Option<usize> = None;
+            let mut segment_start_time = 0.0;
+            let mut segment_elapsed = 0.0;
+            let mut segment_timescale = 1.0;
+
             loop {
-                let time =
-                    (self.start_frame_number as f64 / fps as f64) + start.elapsed().as_secs_f64();
-                let frame_number = (time * fps as f64).floor() as u32;
+                let elapsed = start.elapsed().as_secs_f64();
+                let playhead_time = (self.start_frame_number as f64 / fps as f64) + elapsed;
+
+                let project = self.project.borrow().clone();
+                let mut time_in_timeline = 0.0;
+                let mut segment_found = false;
+
+                if let Some(timeline) = &project.timeline {
+                    for (i, segment) in timeline.segments.iter().enumerate() {
+                        let segment_duration = (segment.end - segment.start) / segment.timescale;
+
+                        if playhead_time >= time_in_timeline
+                            && playhead_time < time_in_timeline + segment_duration
+                        {
+                            if current_segment_index != Some(i) {
+                                current_segment_index = Some(i);
+                                segment_start_time = time_in_timeline;
+                                segment_elapsed = playhead_time - time_in_timeline;
+                                segment_timescale = segment.timescale;
+                            } else {
+                                segment_elapsed = playhead_time - segment_start_time;
+                            }
+                            segment_found = true;
+                            break;
+                        }
+
+                        time_in_timeline += segment_duration;
+                    }
+                }
+
+                if !segment_found {
+                    if playhead_time >= duration {
+                        break;
+                    }
+                    current_segment_index = Some(0);
+                    segment_start_time = 0.0;
+                    segment_elapsed = playhead_time;
+                    segment_timescale = 1.0;
+                }
+
+                let frame_number = (playhead_time * fps as f64).floor() as u32;
 
                 if frame_number as f64 >= fps as f64 * duration {
                     break;
                 };
 
-                let project = self.project.borrow().clone();
+                let segment_time = if let Some(segment_idx) = current_segment_index {
+                    if let Some(timeline) = &project.timeline {
+                        if segment_idx < timeline.segments.len() {
+                            let segment = &timeline.segments[segment_idx];
+                            segment.start + segment_elapsed * segment_timescale
+                        } else {
+                            elapsed
+                        }
+                    } else {
+                        elapsed
+                    }
+                } else {
+                    elapsed
+                };
 
-                if let Some((segment_time, segment_i)) = project.get_segment_time(time) {
-                    let segment = &self.segments[segment_i as usize];
-
-                    let data = tokio::select! {
-                        _ = stop_rx.changed() => { break; },
-                        data = segment.decoders.get_frames(segment_time as f32, !project.camera.hide) => { data }
-                    };
-
-                    if let Some(segment_frames) = data {
-                        let uniforms = ProjectUniforms::new(
-                            &self.render_constants,
-                            &project,
-                            frame_number,
-                            fps,
-                            resolution_base,
-                        );
-
-                        self.renderer
-                            .render_frame(segment_frames, uniforms, segment.cursor.clone())
-                            .await;
+                if let Some(segment_i) = current_segment_index {
+                    if segment_i < self.segments.len() {
+                        let segment = &self.segments[segment_i];
+
+                        let data = tokio::select! {
+                            _ = stop_rx.changed() => { break; },
+                            data = segment.decoders.get_frames(segment_time as f32, !project.camera.hide) => { data }
+                        };
+
+                        if let Some(segment_frames) = data {
+                            let uniforms = ProjectUniforms::new(
+                                &self.render_constants,
+                                &project,
+                                frame_number,
+                                fps,
+                                resolution_base,
+                            );
+
+                            self.renderer
+                                .render_frame(segment_frames, uniforms, segment.cursor.clone())
+                                .await;
+                        }
                     }
                 }
 
-                tokio::time::sleep_until(
-                    start
-                        + (frame_number - self.start_frame_number)
-                            * Duration::from_secs_f32(1.0 / fps as f32),
-                )
-                .await;
+                let next_frame = (frame_number + 1) as f64 / fps as f64;
+                let time_to_wait = if let Some(segment_idx) = current_segment_index {
+                    if let Some(timeline) = &project.timeline {
+                        if segment_idx < timeline.segments.len() {
+                            let segment = &timeline.segments[segment_idx];
+                            Duration::from_secs_f64(
+                                (next_frame - playhead_time) * segment.timescale,
+                            )
+                        } else {
+                            Duration::from_secs_f64(next_frame - playhead_time)
+                        }
+                    } else {
+                        Duration::from_secs_f64(next_frame - playhead_time)
+                    }
+                } else {
+                    Duration::from_secs_f64(next_frame - playhead_time)
+                };
+
+                tokio::time::sleep(time_to_wait).await;
 
                 event_tx.send(PlaybackEvent::Frame(frame_number)).ok();
             }
diff --git a/crates/media/src/feeds/audio.rs b/crates/media/src/feeds/audio.rs
@@ -325,6 +325,7 @@ pub struct AudioPlaybackBuffer<T: FromSampleBytes> {
     frame_buffer: AudioRenderer,
     resampler: AudioResampler,
     resampled_buffer: HeapRb<T>,
+    current_timescale: f64,
 }
 
 impl<T: FromSampleBytes> AudioPlaybackBuffer<T> {
@@ -349,11 +350,45 @@ impl<T: FromSampleBytes> AudioPlaybackBuffer<T> {
             frame_buffer,
             resampler,
             resampled_buffer,
+            current_timescale: 1.0,
         }
     }
 
     pub fn set_playhead(&mut self, playhead: f64, project: &ProjectConfiguration) {
-        self.resampler.reset();
+        let current_timescale = if let Some(timeline) = &project.timeline {
+            let mut accum_duration = 0.0;
+
+            for segment in timeline.segments.iter() {
+                let segment_duration = segment.duration();
+                if playhead < accum_duration + segment_duration {
+                    return self.set_playhead_with_timescale(playhead, project, segment.timescale);
+                }
+
+                accum_duration += segment_duration;
+            }
+
+            1.0
+        } else {
+            1.0
+        };
+
+        self.set_playhead_with_timescale(playhead, project, current_timescale);
+    }
+
+    fn set_playhead_with_timescale(
+        &mut self,
+        playhead: f64,
+        project: &ProjectConfiguration,
+        timescale: f64,
+    ) {
+        if self.current_timescale != timescale {
+            self.resampler.reset();
+            self.resampler.set_timescale(timescale).unwrap_or_else(|e| {
+                tracing::error!("Failed to set timescale: {}", e);
+            });
+            self.current_timescale = timescale;
+        }
+
         self.resampled_buffer.clear();
         self.frame_buffer.set_playhead(playhead, project);
     }
@@ -400,6 +435,7 @@ pub struct AudioResampler {
     pub output_frame: FFAudio,
     delay: Option<resampling::Delay>,
     output: AudioInfo,
+    timescale: f64,
 }
 
 impl AudioResampler {
@@ -422,11 +458,38 @@ impl AudioResampler {
             context,
             output_frame: FFAudio::empty(),
             delay: None,
+            timescale: 1.0,
         })
     }
 
     pub fn reset(&mut self) {
-        *self = Self::new(self.output).unwrap();
+        *self = Self::new(self.output.clone()).unwrap();
+    }
+
+    pub fn set_timescale(&mut self, timescale: f64) -> Result<(), MediaError> {
+        if self.timescale == timescale {
+            return Ok(());
+        }
+        let adjusted_rate = (AudioData::SAMPLE_RATE as f64 * timescale).round() as u32;
+
+        self.context = ffmpeg::software::resampler(
+            (
+                AudioData::SAMPLE_FORMAT,
+                ChannelLayout::STEREO,
+                adjusted_rate,
+            ),
+            (
+                self.output.sample_format,
+                self.output.channel_layout(),
+                self.output.sample_rate,
+            ),
+        )?;
+
+        self.timescale = timescale;
+        self.delay = None;
+        self.output_frame = FFAudio::empty();
+
+        Ok(())
     }
 
     fn current_frame_data(&self) -> &[u8] {
