NAVAND-552: predownload voice instructions

Author: dzinad

CHANGELOG.md

@@ -25,6 +25,9 @@ Mapbox welcomes participation and contributions from everyone.
   ```
 - Added guarantees that route progress with `RouteProgress#currentState == OFF_ROUTE` arrives earlier than `NavigationRerouteController#reroute` is called. [#6764](https://github.com/mapbox/mapbox-navigation-android/pull/6764)
 - Fixed a rare `java.lang.NullPointerException: Attempt to read from field 'SpeechAnnouncement PlayCallback.announcement' on a null object reference` crash in `PlayCallback.getAnnouncement`. [#6760](https://github.com/mapbox/mapbox-navigation-android/pull/6760)
+- Introduced `VoiceInstructionsDownloadTrigger` `MapboxSpeechAPI#generatePredownloaded` to use predownloaded voice instructions instead of downloading them on demand. Example usage can be found in the examples directory ( see `MapboxVoiceActivity`). [#6771](https://github.com/mapbox/mapbox-navigation-android/pull/6771)
+- Enabled voice instructions predownloading for those who use `MapboxAudioGuidance`. [#6771](https://github.com/mapbox/mapbox-navigation-android/pull/6771)
+- Fixed an issue where with low connectivity voice instruction might have been played too late for those who use `MapboxAudioGuidance`. If you use `MapboxSpeechAPI` directly, switch to voice instructions predownloading as described above if you encounter said issue. [#6771](https://github.com/mapbox/mapbox-navigation-android/pull/6771)
 
 ## Mapbox Navigation SDK 2.10.0-rc.1 - 16 December, 2022
 ### Changelog

examples/src/main/java/com/mapbox/navigation/examples/core/MapboxVoiceActivity.kt

@@ -60,6 +60,7 @@ import com.mapbox.navigation.ui.utils.internal.resource.ResourceLoadRequest
 import com.mapbox.navigation.ui.utils.internal.resource.ResourceLoaderFactory
 import com.mapbox.navigation.ui.voice.api.MapboxSpeechApi
 import com.mapbox.navigation.ui.voice.api.MapboxVoiceInstructionsPlayer
+import com.mapbox.navigation.ui.voice.api.VoiceInstructionsDownloadTrigger
 import com.mapbox.navigation.ui.voice.model.SpeechAnnouncement
 import com.mapbox.navigation.ui.voice.model.SpeechError
 import com.mapbox.navigation.ui.voice.model.SpeechValue
@@ -201,9 +202,13 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
         }
     }
 
+    private val voiceInstructionsDownloadTrigger by lazy {
+        VoiceInstructionsDownloadTrigger(speechApi)
+    }
+
     private val voiceInstructionsObserver =
         VoiceInstructionsObserver { voiceInstructions -> // The data obtained must be used to generate the synthesized speech mp3 file.
-            speechApi.generate(
+            speechApi.generatePredownloaded(
                 voiceInstructions,
                 speechCallback
             )
@@ -389,6 +394,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
         if (::mapboxNavigation.isInitialized) {
             mapboxNavigation.registerRoutesObserver(routesObserver)
             mapboxNavigation.registerLocationObserver(locationObserver)
+            mapboxNavigation.registerRouteProgressObserver(voiceInstructionsDownloadTrigger)
             mapboxNavigation.registerRouteProgressObserver(routeProgressObserver)
             mapboxNavigation.registerRouteProgressObserver(replayProgressObserver)
             mapboxNavigation.registerVoiceInstructionsObserver(voiceInstructionsObserver)
@@ -400,6 +406,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
         super.onStop()
         ResourceLoaderFactory.getInstance().unregisterObserver(resourceLoadObserver)
         mapboxNavigation.unregisterRoutesObserver(routesObserver)
+        mapboxNavigation.registerVoiceInstructionsTriggerObserver(voiceInstructionsDownloadTrigger)
         mapboxNavigation.unregisterLocationObserver(locationObserver)
         mapboxNavigation.unregisterRouteProgressObserver(routeProgressObserver)
         mapboxNavigation.unregisterRouteProgressObserver(replayProgressObserver)
@@ -413,6 +420,7 @@ class MapboxVoiceActivity : AppCompatActivity(), OnMapLongClickListener {
         mapboxReplayer.finish()
         mapboxNavigation.onDestroy()
         speechApi.cancel()
+        voiceInstructionsDownloadTrigger.destroy()
         voiceInstructionsPlayer.shutdown()
     }
 

libnavigation-core/api/current.txt

@@ -59,6 +59,7 @@ package com.mapbox.navigation.core {
     method @com.mapbox.navigation.base.ExperimentalPreviewMapboxNavigationAPI public void registerRoutesPreviewObserver(com.mapbox.navigation.core.preview.RoutesPreviewObserver observer);
     method public void registerTripSessionStateObserver(com.mapbox.navigation.core.trip.session.TripSessionStateObserver tripSessionStateObserver);
     method public void registerVoiceInstructionsObserver(com.mapbox.navigation.core.trip.session.VoiceInstructionsObserver voiceInstructionsObserver);
+    method public <T extends com.mapbox.navigation.core.directions.session.RoutesObserver & com.mapbox.navigation.core.trip.session.RouteProgressObserver> void registerVoiceInstructionsTriggerObserver(T observer);
     method public void requestAlternativeRoutes();
     method public void requestAlternativeRoutes(com.mapbox.navigation.core.routealternatives.NavigationRouteAlternativesRequestCallback? callback = null);
     method public void requestAlternativeRoutes(com.mapbox.navigation.core.routealternatives.RouteAlternativesRequestCallback callback);
@@ -102,6 +103,7 @@ package com.mapbox.navigation.core {
     method @com.mapbox.navigation.base.ExperimentalPreviewMapboxNavigationAPI public void unregisterRoutesPreviewObserver(com.mapbox.navigation.core.preview.RoutesPreviewObserver observer);
     method public void unregisterTripSessionStateObserver(com.mapbox.navigation.core.trip.session.TripSessionStateObserver tripSessionStateObserver);
     method public void unregisterVoiceInstructionsObserver(com.mapbox.navigation.core.trip.session.VoiceInstructionsObserver voiceInstructionsObserver);
+    method public <T extends com.mapbox.navigation.core.directions.session.RoutesObserver & com.mapbox.navigation.core.trip.session.RouteProgressObserver> void unregisterVoiceInstructionsTriggerObserver(T observer);
     property public final com.mapbox.navigator.Experimental experimental;
     property public final com.mapbox.navigation.core.trip.session.eh.GraphAccessor graphAccessor;
     property public final com.mapbox.navigation.core.history.MapboxHistoryRecorder historyRecorder;

libnavigation-core/src/main/java/com/mapbox/navigation/core/MapboxNavigation.kt

@@ -1283,6 +1283,26 @@ class MapboxNavigation @VisibleForTesting internal constructor(
         }
     }
 
+    /**
+     * Subscribes voice instructions trigger observer for all the necessary updates.
+     */
+    fun <T> registerVoiceInstructionsTriggerObserver(
+        observer: T
+    ) where T : RoutesObserver, T : RouteProgressObserver {
+        registerRoutesObserver(observer)
+        registerRouteProgressObserver(observer)
+    }
+
+    /**
+     * Unsubscribes voice instructions trigger observer from all the updates it was subsribed for.
+     */
+    fun <T> unregisterVoiceInstructionsTriggerObserver(
+        observer: T
+    ) where T : RoutesObserver, T : RouteProgressObserver {
+        unregisterRoutesObserver(observer)
+        unregisterRouteProgressObserver(observer)
+    }
+
     /**
      * Unregisters [RoutesObserver].
      */

libnavigation-core/src/test/java/com/mapbox/navigation/core/MapboxNavigationTest.kt

@@ -39,6 +39,7 @@ import com.mapbox.navigation.core.trip.session.NativeSetRouteValue
 import com.mapbox.navigation.core.trip.session.NavigationSession
 import com.mapbox.navigation.core.trip.session.OffRouteObserver
 import com.mapbox.navigation.core.trip.session.RoadObjectsOnRouteObserver
+import com.mapbox.navigation.core.trip.session.RouteProgressObserver
 import com.mapbox.navigation.core.trip.session.TripSessionState
 import com.mapbox.navigation.core.trip.session.TripSessionStateObserver
 import com.mapbox.navigation.core.trip.session.createSetRouteResult
@@ -2019,6 +2020,42 @@ internal class MapboxNavigationTest : MapboxNavigationBaseTest() {
         }
     }
 
+    @Test
+    fun registerVoiceInstructionsTriggerObserver() {
+        val observer = TestVoiceInstructionsTriggerObserver()
+        createMapboxNavigation()
+
+        mapboxNavigation.registerVoiceInstructionsTriggerObserver(observer)
+
+        verify(exactly = 1) {
+            directionsSession.registerRoutesObserver(observer)
+            tripSession.registerRouteProgressObserver(observer)
+        }
+    }
+
+    @Test
+    fun unregisterVoiceInstructionsTriggerObserver() {
+        val observer = TestVoiceInstructionsTriggerObserver()
+        createMapboxNavigation()
+        mapboxNavigation.registerVoiceInstructionsTriggerObserver(observer)
+
+        mapboxNavigation.unregisterVoiceInstructionsTriggerObserver(observer)
+
+        verify(exactly = 1) {
+            directionsSession.unregisterRoutesObserver(observer)
+            tripSession.unregisterRouteProgressObserver(observer)
+        }
+    }
+
+    private class TestVoiceInstructionsTriggerObserver : RoutesObserver, RouteProgressObserver {
+
+        override fun onRouteProgressChanged(routeProgress: RouteProgress) {
+        }
+
+        override fun onRoutesChanged(result: RoutesUpdatedResult) {
+        }
+    }
+
     private fun alternativeWithId(mockId: String): RouteAlternative {
         val mockedRoute = mockk<RouteInterface> {
             every { routeId } returns mockId

libnavigation-util/src/main/java/com/mapbox/navigation/utils/internal/Time.kt

@@ -1,12 +1,17 @@
 package com.mapbox.navigation.utils.internal
 
+import java.util.concurrent.TimeUnit
+
 interface Time {
     fun nanoTime(): Long
     fun millis(): Long
+    fun seconds(): Long
 
     object SystemImpl : Time {
         override fun nanoTime(): Long = System.nanoTime()
 
         override fun millis(): Long = System.currentTimeMillis()
+
+        override fun seconds(): Long = TimeUnit.MILLISECONDS.toSeconds(millis())
     }
 }

libnavigation-util/src/test/java/com/mapbox/navigation/utils/internal/TimeTest.kt

@@ -0,0 +1,29 @@
+package com.mapbox.navigation.utils.internal
+
+import org.junit.Assert.assertTrue
+import org.junit.Test
+import kotlin.math.abs
+
+class TimeTest {
+
+    @Test
+    fun seconds() {
+        val tolerance = 1
+        val diff = abs(System.currentTimeMillis() / 1000 - Time.SystemImpl.seconds())
+        assertTrue(diff < tolerance)
+    }
+
+    @Test
+    fun millis() {
+        val tolerance = 100
+        val diff = abs(System.currentTimeMillis() - Time.SystemImpl.millis())
+        assertTrue(diff < tolerance)
+    }
+
+    @Test
+    fun nanoTime() {
+        val tolerance = 100000000
+        val diff = abs(System.nanoTime() - Time.SystemImpl.nanoTime())
+        assertTrue(diff < tolerance)
+    }
+}

libnavui-voice/api/current.txt

@@ -64,6 +64,7 @@ package com.mapbox.navigation.ui.voice.api {
     method public void cancel();
     method public void clean(com.mapbox.navigation.ui.voice.model.SpeechAnnouncement announcement);
     method public void generate(com.mapbox.api.directions.v5.models.VoiceInstructions voiceInstruction, com.mapbox.navigation.ui.base.util.MapboxNavigationConsumer<com.mapbox.bindgen.Expected<com.mapbox.navigation.ui.voice.model.SpeechError,com.mapbox.navigation.ui.voice.model.SpeechValue>> consumer);
+    method public void generatePredownloaded(com.mapbox.api.directions.v5.models.VoiceInstructions voiceInstruction, com.mapbox.navigation.ui.base.util.MapboxNavigationConsumer<com.mapbox.bindgen.Expected<com.mapbox.navigation.ui.voice.model.SpeechError,com.mapbox.navigation.ui.voice.model.SpeechValue>> consumer);
   }
 
   @UiThread public final class MapboxVoiceInstructionsPlayer {
@@ -84,6 +85,20 @@ package com.mapbox.navigation.ui.voice.api {
     method @kotlin.jvm.Throws(exceptionClasses=IllegalArgumentException::class) public void volume(com.mapbox.navigation.ui.voice.model.SpeechVolume state) throws java.lang.IllegalArgumentException;
   }
 
+  public final class VoiceInstructionsDownloadTrigger implements com.mapbox.navigation.core.trip.session.RouteProgressObserver com.mapbox.navigation.core.directions.session.RoutesObserver {
+    ctor public VoiceInstructionsDownloadTrigger(com.mapbox.navigation.ui.voice.api.MapboxSpeechApi speechApi);
+    ctor public VoiceInstructionsDownloadTrigger(int observableTime, double timePercentageToTriggerAfter, com.mapbox.navigation.ui.voice.api.MapboxSpeechApi speechApi);
+    method public void destroy();
+    method public void onRouteProgressChanged(com.mapbox.navigation.base.trip.model.RouteProgress routeProgress);
+    method public void onRoutesChanged(com.mapbox.navigation.core.directions.session.RoutesUpdatedResult result);
+    field public static final com.mapbox.navigation.ui.voice.api.VoiceInstructionsDownloadTrigger.Companion Companion;
+    field public static final int DEFAULT_OBSERVABLE_TIME_SECONDS = 180; // 0xb4
+    field public static final double DEFAULT_TIME_PERCENTAGE_TO_TRIGGER_AFTER = 0.5;
+  }
+
+  public static final class VoiceInstructionsDownloadTrigger.Companion {
+  }
+
   public abstract sealed class VoiceInstructionsPlayerAttributes {
     method protected abstract kotlin.jvm.functions.Function1<android.media.AudioFocusRequest.Builder,kotlin.Unit> configureAudioFocusRequestBuilder(com.mapbox.navigation.ui.voice.model.AudioFocusOwner owner);
     method protected abstract kotlin.jvm.functions.Function1<android.media.MediaPlayer,kotlin.Unit> configureMediaPlayer();

libnavui-voice/src/main/java/com/mapbox/navigation/ui/voice/api/MapboxAudioGuidance.kt

@@ -43,6 +43,7 @@ internal constructor(
 
     private var dataStoreOwner: NavigationDataStoreOwner? = null
     private var configOwner: NavigationConfigOwner? = null
+    private var audioGuidanceVoice: MapboxAudioGuidanceVoice? = null
     private var mutedStateFlow = MutableStateFlow(false)
     private val internalStateFlow = MutableStateFlow(MapboxAudioGuidanceState())
     private val scope = CoroutineScope(SupervisorJob() + dispatcher)
@@ -74,6 +75,7 @@ internal constructor(
      */
     override fun onDetached(mapboxNavigation: MapboxNavigation) {
         mapboxVoiceInstructions.unregisterObservers(mapboxNavigation)
+        audioGuidanceVoice?.destroy()
         job?.cancel()
         job = null
     }
@@ -164,15 +166,24 @@ internal constructor(
         }
     }
 
-    private fun MapboxNavigation.audioGuidanceVoice(): Flow<MapboxAudioGuidanceVoice> =
-        combine(
+    private fun MapboxNavigation.audioGuidanceVoice(): Flow<MapboxAudioGuidanceVoice> {
+        var trigger: VoiceInstructionsDownloadTrigger? = null
+        return combine(
             mapboxVoiceInstructions.voiceLanguage(),
             configOwner!!.language(),
         ) { voiceLanguage, deviceLanguage -> voiceLanguage ?: deviceLanguage }
             .distinctUntilChanged()
             .map { language ->
-                audioGuidanceServices.mapboxAudioGuidanceVoice(this, language)
+                audioGuidanceVoice?.destroy()
+                trigger?.let { unregisterVoiceInstructionsTriggerObserver(it) }
+                audioGuidanceServices.mapboxAudioGuidanceVoice(this, language).also {
+                    audioGuidanceVoice = it
+                    trigger = VoiceInstructionsDownloadTrigger(it.mapboxSpeechApi).also {
+                        registerVoiceInstructionsTriggerObserver(it)
+                    }
+                }
             }
+    }
 
     private suspend fun restoreMutedState() {
         dataStoreOwner?.apply {

libnavui-voice/src/main/java/com/mapbox/navigation/ui/voice/api/MapboxSpeechApi.kt

@@ -4,6 +4,7 @@ import android.content.Context
 import com.mapbox.api.directions.v5.models.VoiceInstructions
 import com.mapbox.bindgen.Expected
 import com.mapbox.bindgen.ExpectedFactory
+import com.mapbox.navigation.core.trip.session.VoiceInstructionsObserver
 import com.mapbox.navigation.ui.base.util.MapboxNavigationConsumer
 import com.mapbox.navigation.ui.voice.model.SpeechAnnouncement
 import com.mapbox.navigation.ui.voice.model.SpeechError
@@ -40,6 +41,9 @@ class MapboxSpeechApi @JvmOverloads constructor(
      * Given [VoiceInstructions] the method will try to generate the
      * voice instruction [SpeechAnnouncement] including the synthesized speech mp3 file
      * from Mapbox's API Voice.
+     * NOTE: this method will try downloading an mp3 file from server. If you use voice instructions
+     * predownloading (see [VoiceInstructionsDownloadTrigger]), invoke [generatePredownloaded]
+     * instead of this method in your [VoiceInstructionsObserver].
      * @param voiceInstruction VoiceInstructions object representing [VoiceInstructions]
      * @param consumer is a [SpeechValue] including the announcement to be played when the
      * announcement is ready or a [SpeechError] including the error information and a fallback
@@ -51,7 +55,30 @@ class MapboxSpeechApi @JvmOverloads constructor(
         consumer: MapboxNavigationConsumer<Expected<SpeechError, SpeechValue>>
     ) {
         mainJobController.scope.launch {
-            retrieveVoiceFile(voiceInstruction, consumer)
+            retrieveVoiceFile(voiceInstruction, consumer, onlyCache = false)
+        }
+    }
+
+    /**
+     * Given [VoiceInstructions] the method will try to generate the
+     * voice instruction [SpeechAnnouncement] including the synthesized speech mp3 file
+     * from Mapbox's API Voice.
+     * NOTE: this method will NOT try downloading an mp3 file from server. It will either use
+     * an already predownloaded file or an onboard speech synthesizer. Only invoke this method
+     * if you use voice instructions predownloading (see [VoiceInstructionsDownloadTrigger]),
+     * otherwise invoke [generatePredownloaded] in your [VoiceInstructionsObserver].
+     * @param voiceInstruction VoiceInstructions object representing [VoiceInstructions]
+     * @param consumer is a [SpeechValue] including the announcement to be played when the
+     * announcement is ready or a [SpeechError] including the error information and a fallback
+     * with the raw announcement (without file) that can be played with a text-to-speech engine.
+     * @see [cancel]
+     */
+    fun generatePredownloaded(
+        voiceInstruction: VoiceInstructions,
+        consumer: MapboxNavigationConsumer<Expected<SpeechError, SpeechValue>>
+    ) {
+        mainJobController.scope.launch {
+            retrieveVoiceFile(voiceInstruction, consumer, onlyCache = true)
         }
     }
 
@@ -75,12 +102,23 @@ class MapboxSpeechApi @JvmOverloads constructor(
         voiceAPI.clean(announcement)
     }
 
+    internal fun predownload(instructions: List<VoiceInstructions>) {
+        mainJobController.scope.launch {
+            voiceAPI.predownload(instructions)
+        }
+    }
+
+    internal fun destroy() {
+        voiceAPI.destroy()
+    }
+
     @Throws(IllegalStateException::class)
     private suspend fun retrieveVoiceFile(
         voiceInstruction: VoiceInstructions,
-        consumer: MapboxNavigationConsumer<Expected<SpeechError, SpeechValue>>
+        consumer: MapboxNavigationConsumer<Expected<SpeechError, SpeechValue>>,
+        onlyCache: Boolean
     ) {
-        when (val result = voiceAPI.retrieveVoiceFile(voiceInstruction)) {
+        when (val result = voiceAPI.retrieveVoiceFile(voiceInstruction, onlyCache)) {
             is VoiceState.VoiceFile -> {
                 val announcement = voiceInstruction.announcement()
                 val ssmlAnnouncement = voiceInstruction.ssmlAnnouncement()