Skip to content

Commit ca47e72

Browse files
clainclycopybara-github
claincly
authored and
copybara-github
committed
Fix clipping + speed adjust
The root problem is that, the start positions of the wrapped `ProgressiveMS` and `silenceMS` are different. This start position is set from the `ClippingMS` that wraps the `ProgressiveMS`. This start position is propagated up to the top level media source in `onChildSourceInfoRefreshed()`. For example, if the media is 6s long, and clipped from 3s, slowed down 2x. - `ClippingMS` sets start time to 3s, propagated up to the top level - Hence TrackSelection instruct the `SpeedChangingMS` to start from 3s. - `SpeedChangingMS` would scale this to 1.5s because of the slowdown - **This is an error!** Because the media should start from 3s. - In a subsequent `readDiscontinuity()` (from `EPII.updatePlaybackPositions()`), `ProgressiveMS` reports a discontinuity at 1.5s (the last seek position) - Next, `silenceMS` is seeked to this discontinuity and **this is another issue**: the clipping is from 3s, and thus the `clippingMS` seeks to 3s. This causes the `MergingMS` to throw "Unexpected child seekToUs result": seeking to 1.5s but actually seeked to 3. This CL solves this issue by setting correct, speed adjusted start position on the `SpeedChangingMS` PiperOrigin-RevId: 786239170
1 parent 8de7ca3 commit ca47e72

File tree

7 files changed

+267
-119
lines changed

7 files changed

+267
-119
lines changed

‎libraries/test_data/src/test/assets/media/mp4/sample_video_only.mp4‎

89 KB
Binary file not shown.

‎libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -382,6 +382,27 @@ public String toString() {
382382
834_166L, 867_533L, 900_900L, 934_266L, 967_633L))
383383
.build();
384384

385+
public static final AssetInfo MP4_VIDEO_ONLY_ASSET =
386+
new AssetInfo.Builder("asset:///media/mp4/sample_video_only.mp4")
387+
.setVideoFormat(
388+
new Format.Builder()
389+
.setSampleMimeType(VIDEO_H264)
390+
.setWidth(1080)
391+
.setHeight(720)
392+
.setFrameRate(29.97f)
393+
.setCodecs("avc1.64001F")
394+
.build())
395+
// This is slightly different from sample.mp4
396+
.setVideoDurationUs(1_001_000L)
397+
.setVideoFrameCount(30)
398+
.setVideoTimestampsUs(
399+
ImmutableList.of(
400+
0L, 33_366L, 66_733L, 100_100L, 133_466L, 166_833L, 200_200L, 233_566L, 266_933L,
401+
300_300L, 333_666L, 367_033L, 400_400L, 433_766L, 467_133L, 500_500L, 533_866L,
402+
567_233L, 600_600L, 633_966L, 667_333L, 700_700L, 734_066L, 767_433L, 800_800L,
403+
834_166L, 867_533L, 900_900L, 934_266L, 967_633L))
404+
.build();
405+
385406
public static final AssetInfo BT601_MOV_ASSET =
386407
new AssetInfo.Builder("asset:///media/mp4/bt601.mov")
387408
.setVideoFormat(

‎libraries/transformer/src/androidTest/java/androidx/media3/transformer/CompositionPlayerParameterizedPlaybackTest.java‎

Lines changed: 114 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,23 @@
1818
import static androidx.media3.common.util.Util.isRunningOnEmulator;
1919
import static androidx.media3.common.util.Util.usToMs;
2020
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET;
21+
import static androidx.media3.transformer.AndroidTestUtil.MP4_VIDEO_ONLY_ASSET;
2122
import static androidx.media3.transformer.AndroidTestUtil.PNG_ASSET;
2223
import static androidx.media3.transformer.AndroidTestUtil.WAV_ASSET;
2324
import static androidx.test.platform.app.InstrumentationRegistry.getInstrumentation;
2425
import static com.google.common.truth.Truth.assertThat;
2526
import static com.google.common.truth.TruthJUnit.assume;
2627

2728
import android.content.Context;
29+
import android.util.Pair;
2830
import android.view.SurfaceView;
31+
import androidx.media3.common.C;
32+
import androidx.media3.common.Effect;
2933
import androidx.media3.common.MediaItem;
3034
import androidx.media3.common.PlaybackException;
3135
import androidx.media3.common.VideoGraph;
36+
import androidx.media3.common.audio.AudioProcessor;
37+
import androidx.media3.common.audio.SpeedProvider;
3238
import androidx.media3.effect.GlEffect;
3339
import androidx.media3.effect.MultipleInputVideoGraph;
3440
import androidx.media3.effect.SingleInputVideoGraph;
@@ -51,6 +57,34 @@
5157
public class CompositionPlayerParameterizedPlaybackTest {
5258

5359
private static final long TEST_TIMEOUT_MS = isRunningOnEmulator() ? 30_000 : 20_000;
60+
private static final Pair<AudioProcessor, Effect> HALF_SPEED_CHANGE_EFFECTS =
61+
Effects.createExperimentalSpeedChangingEffect(
62+
new SpeedProvider() {
63+
@Override
64+
public float getSpeed(long timeUs) {
65+
return 0.5f;
66+
}
67+
68+
@Override
69+
public long getNextSpeedChangeTimeUs(long timeUs) {
70+
// Adjust speed for all timestamps.
71+
return C.TIME_UNSET;
72+
}
73+
});
74+
private static final Pair<AudioProcessor, Effect> TWICE_SPEED_CHANGE_EFFECTS =
75+
Effects.createExperimentalSpeedChangingEffect(
76+
new SpeedProvider() {
77+
@Override
78+
public float getSpeed(long timeUs) {
79+
return 2f;
80+
}
81+
82+
@Override
83+
public long getNextSpeedChangeTimeUs(long timeUs) {
84+
// Adjust speed for all timestamps.
85+
return C.TIME_UNSET;
86+
}
87+
});
5488
private static final Input IMAGE_INPUT =
5589
new Input(
5690
new EditedMediaItem.Builder(
@@ -60,7 +94,6 @@ public class CompositionPlayerParameterizedPlaybackTest {
6094
.build())
6195
.setDurationUs(500_000)
6296
.build(),
63-
// 200 ms at 30 fps (default frame rate)
6497
ImmutableList.of(
6598
0L, 33_333L, 66_667L, 100_000L, 133_333L, 166_667L, 200_000L, 233_333L, 266_667L,
6699
300_000L, 333_333L, 366_667L, 400_000L, 433_333L, 466_667L),
@@ -80,6 +113,71 @@ public class CompositionPlayerParameterizedPlaybackTest {
80113
.build(),
81114
MP4_ASSET.videoTimestampsUs,
82115
/* inputName= */ "Video_no_audio");
116+
117+
private static final MediaItem VIDEO_ONLY_CLIPPED =
118+
MediaItem.fromUri(MP4_VIDEO_ONLY_ASSET.uri)
119+
.buildUpon()
120+
.setClippingConfiguration(
121+
new MediaItem.ClippingConfiguration.Builder().setStartPositionMs(500).build())
122+
.build();
123+
private static final Input VIDEO_ONLY_CLIPPED_TWICE_SPEED =
124+
new Input(
125+
new EditedMediaItem.Builder(VIDEO_ONLY_CLIPPED)
126+
.setDurationUs(MP4_VIDEO_ONLY_ASSET.videoDurationUs)
127+
.setRemoveAudio(true)
128+
.setEffects(
129+
new Effects(
130+
/* audioProcessors= */ ImmutableList.of(),
131+
/* videoEffects= */ ImmutableList.of(TWICE_SPEED_CHANGE_EFFECTS.second)))
132+
.build(),
133+
/* expectedVideoTimestampsUs= */ ImmutableList.of(
134+
// The first timestamp is at clipping point, 500ms and speed up 2x to 250ms. The
135+
// last is at (967633 - 500_000) / 2
136+
250L,
137+
16933L,
138+
33616L,
139+
50300L,
140+
66983L,
141+
83666L,
142+
100350L,
143+
117033L,
144+
133716L,
145+
150400L,
146+
167083L,
147+
183766L,
148+
200450L,
149+
217133L,
150+
233816L),
151+
/* inputName= */ "Video_only_clippped_half_speed");
152+
private static final Input VIDEO_ONLY_CLIPPED_HALF_SPEED =
153+
new Input(
154+
new EditedMediaItem.Builder(VIDEO_ONLY_CLIPPED)
155+
.setDurationUs(MP4_VIDEO_ONLY_ASSET.videoDurationUs)
156+
.setRemoveAudio(true)
157+
.setEffects(
158+
new Effects(
159+
/* audioProcessors= */ ImmutableList.of(),
160+
/* videoEffects= */ ImmutableList.of(HALF_SPEED_CHANGE_EFFECTS.second)))
161+
.build(),
162+
/* expectedVideoTimestampsUs= */ ImmutableList.of(
163+
// The first timestamp is at clipping point, 500ms and slowed down 2x to 1000ms. The
164+
// last is at (967633 - 500_000) x 2
165+
1000L,
166+
67732L,
167+
134466L,
168+
201200L,
169+
267932L,
170+
334666L,
171+
401400L,
172+
468132L,
173+
534866L,
174+
601600L,
175+
668332L,
176+
735066L,
177+
801800L,
178+
868532L,
179+
935266L),
180+
/* inputName= */ "Video_only_clippped_half_speed");
83181
private static final Input AUDIO_INPUT =
84182
new Input(
85183
new EditedMediaItem.Builder(MediaItem.fromUri(WAV_ASSET.uri))
@@ -129,6 +227,20 @@ public static ImmutableList<TestConfig> params() {
129227
// TODO: b/412585977 - Enable once implicit gaps are implemented.
130228
// configs.add(new TestConfig(new InputSequence(AUDIO_INPUT,
131229
// VIDEO_INPUT).withForceVideoTrack()));
230+
configs.add(new TestConfig(new InputSequence(VIDEO_ONLY_CLIPPED_HALF_SPEED)));
231+
configs.add(new TestConfig(new InputSequence(VIDEO_ONLY_CLIPPED_TWICE_SPEED)));
232+
configs.add(
233+
new TestConfig(
234+
new InputSequence(VIDEO_ONLY_CLIPPED_TWICE_SPEED, VIDEO_ONLY_CLIPPED_TWICE_SPEED)));
235+
configs.add(
236+
new TestConfig(
237+
new InputSequence(VIDEO_ONLY_CLIPPED_TWICE_SPEED, VIDEO_ONLY_CLIPPED_HALF_SPEED)));
238+
configs.add(
239+
new TestConfig(
240+
new InputSequence(VIDEO_ONLY_CLIPPED_HALF_SPEED, VIDEO_ONLY_CLIPPED_TWICE_SPEED)));
241+
configs.add(
242+
new TestConfig(
243+
new InputSequence(VIDEO_ONLY_CLIPPED_HALF_SPEED, VIDEO_ONLY_CLIPPED_HALF_SPEED)));
132244

133245
// Multiple sequence.
134246
configs.add(
@@ -161,7 +273,6 @@ public static ImmutableList<TestConfig> params() {
161273
// new TestConfig(
162274
// new InputSequence(VIDEO_INPUT, VIDEO_INPUT),
163275
// new InputSequence(VIDEO_INPUT).withIsLooping()));
164-
165276
return configs.build();
166277
}
167278

@@ -355,7 +466,7 @@ public Input(
355466
String inputName) {
356467
this.editedMediaItem = editedMediaItem;
357468
this.expectedVideoTimestampsUs = expectedVideoTimestampsUs;
358-
this.durationUs = editedMediaItem.durationUs;
469+
this.durationUs = editedMediaItem.getPresentationDurationUs();
359470
this.inputName = inputName;
360471
}
361472
}

‎libraries/transformer/src/main/java/androidx/media3/transformer/CompositionPlayer.java‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -952,19 +952,17 @@ private static MediaSource createPrimarySequenceMediaSource(
952952
for (int i = 0; i < sequence.editedMediaItems.size(); i++) {
953953
EditedMediaItem editedMediaItem = sequence.editedMediaItems.get(i);
954954
checkArgument(editedMediaItem.durationUs != C.TIME_UNSET);
955-
long durationUs = editedMediaItem.getPresentationDurationUs();
956955

957956
MediaSource blankFramesAndSilenceGeneratedMediaSource =
958957
createMediaSourceWithBlankFramesAndSilence(
959958
mediaSourceFactory, editedMediaItem, shouldGenerateBlankFrames);
960959

961960
MediaSource itemMediaSource =
962961
wrapWithVideoEffectsBasedMediaSources(
963-
blankFramesAndSilenceGeneratedMediaSource,
964-
editedMediaItem.effects.videoEffects,
965-
durationUs);
962+
blankFramesAndSilenceGeneratedMediaSource, editedMediaItem.effects.videoEffects);
966963
mediaSourceBuilder.add(
967-
itemMediaSource, /* initialPlaceholderDurationMs= */ usToMs(durationUs));
964+
itemMediaSource,
965+
/* initialPlaceholderDurationMs= */ usToMs(editedMediaItem.getPresentationDurationUs()));
968966
}
969967
return mediaSourceBuilder.build();
970968
}
@@ -1072,15 +1070,13 @@ private static EditedMediaItem clipToDuration(EditedMediaItem editedMediaItem, l
10721070
}
10731071

10741072
private static MediaSource wrapWithVideoEffectsBasedMediaSources(
1075-
MediaSource mediaSource, ImmutableList<Effect> videoEffects, long durationUs) {
1073+
MediaSource mediaSource, ImmutableList<Effect> videoEffects) {
10761074
MediaSource newMediaSource = mediaSource;
10771075
for (Effect videoEffect : videoEffects) {
10781076
if (videoEffect instanceof InactiveTimestampAdjustment) {
10791077
newMediaSource =
10801078
new SpeedChangingMediaSource(
1081-
newMediaSource,
1082-
((InactiveTimestampAdjustment) videoEffect).speedProvider,
1083-
durationUs);
1079+
newMediaSource, ((InactiveTimestampAdjustment) videoEffect).speedProvider);
10841080
}
10851081
}
10861082
return newMediaSource;

‎libraries/transformer/src/main/java/androidx/media3/transformer/SequenceRenderersFactory.java‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -196,24 +196,23 @@ public Renderer createSecondaryRenderer(
196196
return null;
197197
}
198198

199+
/**
200+
* Returns the offset convert the renderers timestamp to the start of the {@link Composition}.
201+
*
202+
* @param timeline The {@link Timeline} associated with this renderer.
203+
* @param mediaPeriodId The {@link MediaSource.MediaPeriodId}.
204+
* @param offsetUs The offset added to timestamps of buffers to ensure monotonically increasing
205+
* timestamps, in microseconds. This is the constant offset between the current MediaPeriod
206+
* timestamps and the renderer timestamp.
207+
* <p>See <a
208+
* href="https://developer.android.com/reference/androidx/media3/exoplayer/Renderer#timestamps-and-offsets">this
209+
* corresponding topic on timestamps</a>.
210+
*/
199211
private static long getOffsetToCompositionTimeUs(
200-
EditedMediaItemSequence sequence, int mediaItemIndex, long offsetUs) {
201-
// Reverse engineer how timestamps and offsets are computed with a ConcatenatingMediaSource2
202-
// to compute an offset converting buffer timestamps to composition timestamps.
203-
// startPositionUs is not used because it is equal to offsetUs + clipping start time + seek
204-
// position when seeking from any MediaItem in the playlist to the first MediaItem.
205-
// The offset to convert the sample timestamps to composition time is negative because we need
206-
// to remove the large offset added by ExoPlayer to make sure the decoder doesn't received any
207-
// negative timestamps. We also need to remove the clipping start position.
208-
long offsetToCompositionTimeUs = -offsetUs;
209-
if (mediaItemIndex == 0) {
210-
offsetToCompositionTimeUs -=
211-
sequence.editedMediaItems.get(0).mediaItem.clippingConfiguration.startPositionUs;
212-
}
213-
for (int i = 0; i < mediaItemIndex; i++) {
214-
offsetToCompositionTimeUs += getEditedMediaItem(sequence, i).getPresentationDurationUs();
215-
}
216-
return offsetToCompositionTimeUs;
212+
Timeline timeline, MediaSource.MediaPeriodId mediaPeriodId, long offsetUs) {
213+
Timeline.Period period =
214+
timeline.getPeriodByUid(mediaPeriodId.periodUid, new Timeline.Period());
215+
return -offsetUs + period.positionInWindowUs;
217216
}
218217

219218
private static boolean isLastInSequence(
@@ -286,7 +285,7 @@ protected void onStreamChanged(
286285
checkStateNotNull(sequence);
287286
pendingEditedMediaItem = getEditedMediaItem(sequence, periodIndex);
288287
pendingOffsetToCompositionTimeUs =
289-
getOffsetToCompositionTimeUs(sequence, periodIndex, offsetUs);
288+
getOffsetToCompositionTimeUs(getTimeline(), mediaPeriodId, offsetUs);
290289
super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId);
291290
}
292291

@@ -391,7 +390,8 @@ protected void onStreamChanged(
391390
// The renderer has started processing this item, VideoGraph might still be processing the
392391
// previous one.
393392
currentEditedMediaItem = getEditedMediaItem(sequence, periodIndex);
394-
offsetToCompositionTimeUs = getOffsetToCompositionTimeUs(sequence, periodIndex, offsetUs);
393+
offsetToCompositionTimeUs =
394+
getOffsetToCompositionTimeUs(getTimeline(), mediaPeriodId, offsetUs);
395395
pendingEffects = checkNotNull(currentEditedMediaItem).effects.videoEffects;
396396
super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId);
397397
}
@@ -618,7 +618,7 @@ protected void onStreamChanged(
618618
int periodIndex = getTimeline().getIndexOfPeriod(mediaPeriodId.periodUid);
619619
currentEditedMediaItem = getEditedMediaItem(sequence, periodIndex);
620620
long offsetToCompositionTimeUs =
621-
getOffsetToCompositionTimeUs(sequence, periodIndex, offsetUs);
621+
getOffsetToCompositionTimeUs(getTimeline(), mediaPeriodId, offsetUs);
622622
videoSink.setBufferTimestampAdjustmentUs(offsetToCompositionTimeUs);
623623
timestampIterator = createTimestampIterator(/* positionUs= */ startPositionUs);
624624
videoEffects = checkNotNull(currentEditedMediaItem).effects.videoEffects;

0 commit comments

Comments
 (0)