From 11646aa6f887a9a0aa81571255d1a63838c58cf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Mon, 6 May 2024 21:39:26 +0200 Subject: [PATCH 1/3] Update Cargo.toml --- livekit-protocol/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/livekit-protocol/Cargo.toml b/livekit-protocol/Cargo.toml index 077849e5f..98df43ffd 100644 --- a/livekit-protocol/Cargo.toml +++ b/livekit-protocol/Cargo.toml @@ -7,8 +7,8 @@ description = "Livekit protocol and utilities for the Rust SDK" repository = "https://github.com/livekit/rust-sdks" [dependencies] -livekit-runtime = { path = "../livekit-runtime", version = "0.3.0" } -tokio = { version = "1", default-features = false, features = [ "sync", "macros" ] } +livekit-runtime = { path = "../livekit-runtime", version = "0.3.0", features = [ "tokio" ] } +tokio = { version = "1", default-features = false, features = [ "sync", "macros", "time", "net" ] } futures-util = { version = "0.3", features = ["sink"] } parking_lot = "0.12" prost = "0.12" From 64281862c0c5312553e4340e0eaffc1c514ec0c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Fri, 24 May 2024 12:54:15 +0200 Subject: [PATCH 2/3] API for pre-encoded audio tracks --- examples/play_from_disk/Cargo.lock | 12 ++++++------ examples/play_from_disk/src/main.rs | 10 +++++++++- examples/save_to_disk/Cargo.toml | 2 ++ libwebrtc/src/audio_source.rs | 9 +++++---- libwebrtc/src/native/audio_source.rs | 2 ++ livekit-ffi/protocol/audio_frame.proto | 7 ++++--- livekit-ffi/src/conversion/audio_frame.rs | 1 + livekit-ffi/src/livekit.proto.rs | 14 ++++++++------ webrtc-sys/src/audio_track.cpp | 2 ++ webrtc-sys/src/audio_track.rs | 1 + 10 files changed, 40 insertions(+), 20 deletions(-) diff --git a/examples/play_from_disk/Cargo.lock b/examples/play_from_disk/Cargo.lock index 0b8ceba95..d0fe08422 100644 --- a/examples/play_from_disk/Cargo.lock +++ b/examples/play_from_disk/Cargo.lock @@ -756,7 +756,7 @@ checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libwebrtc" -version = "0.3.0" +version = "0.3.2" dependencies = [ "cxx", "jni", @@ -794,7 +794,7 @@ checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "livekit" -version = "0.3.0" +version = "0.3.2" dependencies = [ "futures-util", "lazy_static", @@ -813,7 +813,7 @@ dependencies = [ [[package]] name = "livekit-api" -version = "0.3.0" +version = "0.3.2" dependencies = [ "futures-util", "http", @@ -834,7 +834,7 @@ dependencies = [ [[package]] name = "livekit-protocol" -version = "0.3.0" +version = "0.3.2" dependencies = [ "futures-util", "livekit-runtime", @@ -1954,7 +1954,7 @@ dependencies = [ [[package]] name = "webrtc-sys" -version = "0.3.0" +version = "0.3.2" dependencies = [ "cc", "cxx", @@ -1966,7 +1966,7 @@ dependencies = [ [[package]] name = "webrtc-sys-build" -version = "0.3.0" +version = "0.3.2" dependencies = [ "fs2", "regex", diff --git a/examples/play_from_disk/src/main.rs b/examples/play_from_disk/src/main.rs index b96d4e5c7..7485a310d 100644 --- a/examples/play_from_disk/src/main.rs +++ b/examples/play_from_disk/src/main.rs @@ -131,7 +131,15 @@ async fn main() -> Result<(), Box> { header.num_channels as u32, ); - let track = LocalAudioTrack::create_audio_track("file", RtcAudioSource::Native(source.clone())); + let native_source = RtcAudioSource::Native(source.clone()); + // ...just to show an API and emphasis the source is *not* pre-encoded, + // otherwise set pre_encoded = true + native_source.set_audio_options(AudioSourceOptions{ + pre_encoded: false, + ..Default::default() + }); + + let track = LocalAudioTrack::create_audio_track("file", native_source); room.local_participant() .publish_track( diff --git a/examples/save_to_disk/Cargo.toml b/examples/save_to_disk/Cargo.toml index 22521f18c..56b5e20ed 100644 --- a/examples/save_to_disk/Cargo.toml +++ b/examples/save_to_disk/Cargo.toml @@ -11,3 +11,5 @@ livekit = { path = "../../livekit", version = "0.3.2" } bytes = "1.4.0" tokio-util = "0.7.8" futures = "0.3.28" + +[workspace] diff --git a/libwebrtc/src/audio_source.rs b/libwebrtc/src/audio_source.rs index 131c6d9eb..1364bc8b4 100644 --- a/libwebrtc/src/audio_source.rs +++ b/libwebrtc/src/audio_source.rs @@ -21,6 +21,7 @@ pub struct AudioSourceOptions { pub echo_cancellation: bool, pub noise_suppression: bool, pub auto_gain_control: bool, + pub pre_encoded: bool, } #[non_exhaustive] @@ -33,10 +34,10 @@ pub enum RtcAudioSource { impl RtcAudioSource { enum_dispatch!( [Native]; - fn set_audio_options(self: &Self, options: AudioSourceOptions) -> (); - fn audio_options(self: &Self) -> AudioSourceOptions; - fn sample_rate(self: &Self) -> u32; - fn num_channels(self: &Self) -> u32; + pub fn set_audio_options(self: &Self, options: AudioSourceOptions) -> (); + pub fn audio_options(self: &Self) -> AudioSourceOptions; + pub fn sample_rate(self: &Self) -> u32; + pub fn num_channels(self: &Self) -> u32; ); } diff --git a/libwebrtc/src/native/audio_source.rs b/libwebrtc/src/native/audio_source.rs index 62f357609..bacaed7ec 100644 --- a/libwebrtc/src/native/audio_source.rs +++ b/libwebrtc/src/native/audio_source.rs @@ -177,6 +177,7 @@ impl From for AudioSourceOptions { echo_cancellation: options.echo_cancellation, noise_suppression: options.noise_suppression, auto_gain_control: options.auto_gain_control, + pre_encoded: options.pre_encoded, } } } @@ -187,6 +188,7 @@ impl From for sys_at::ffi::AudioSourceOptions { echo_cancellation: options.echo_cancellation, noise_suppression: options.noise_suppression, auto_gain_control: options.auto_gain_control, + pre_encoded: options.pre_encoded, } } } diff --git a/livekit-ffi/protocol/audio_frame.proto b/livekit-ffi/protocol/audio_frame.proto index f61c74c38..86ed5b7f4 100644 --- a/livekit-ffi/protocol/audio_frame.proto +++ b/livekit-ffi/protocol/audio_frame.proto @@ -36,9 +36,9 @@ message NewAudioSourceRequest { } message NewAudioSourceResponse { OwnedAudioSource source = 1; } -// Push a frame to an AudioSource +// Push a frame to an AudioSource // The data provided must be available as long as the client receive the callback. -message CaptureAudioFrameRequest { +message CaptureAudioFrameRequest { uint64 source_handle = 1; AudioFrameBufferInfo buffer = 2; } @@ -104,7 +104,7 @@ message OwnedAudioStream { message AudioStreamEvent { uint64 stream_handle = 1; - oneof message { + oneof message { AudioFrameReceived frame_received = 2; AudioStreamEOS eos = 3; } @@ -124,6 +124,7 @@ message AudioSourceOptions { bool echo_cancellation = 1; bool noise_suppression = 2; bool auto_gain_control = 3; + bool pre_encoded = 4; } enum AudioSourceType { diff --git a/livekit-ffi/src/conversion/audio_frame.rs b/livekit-ffi/src/conversion/audio_frame.rs index c25ff415f..7c7387a30 100644 --- a/livekit-ffi/src/conversion/audio_frame.rs +++ b/livekit-ffi/src/conversion/audio_frame.rs @@ -25,6 +25,7 @@ impl From for AudioSourceOptions { echo_cancellation: opts.echo_cancellation, auto_gain_control: opts.auto_gain_control, noise_suppression: opts.noise_suppression, + pre_encoded: opts.pre_encoded, } } } diff --git a/livekit-ffi/src/livekit.proto.rs b/livekit-ffi/src/livekit.proto.rs index 65732f2e2..6b4df6c61 100644 --- a/livekit-ffi/src/livekit.proto.rs +++ b/livekit-ffi/src/livekit.proto.rs @@ -303,11 +303,11 @@ impl EncryptionState { /// # Safety /// The foreign language is responsable for disposing handles /// Forgetting to dispose the handle may lead to memory leaks -/// +/// /// Dropping a handle doesn't necessarily mean that the object is destroyed if it is still used /// on the FfiServer (Atomic reference counting) -/// -/// When refering to a handle without owning it, we just use a uint32 without this message. +/// +/// When refering to a handle without owning it, we just use a uint32 without this message. /// (the variable name is suffixed with "_handle") #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -1549,7 +1549,7 @@ pub struct NewVideoSourceRequest { #[prost(enumeration="VideoSourceType", tag="1")] pub r#type: i32, /// Used to determine which encodings to use + simulcast layers - /// Most of the time it corresponds to the source resolution + /// Most of the time it corresponds to the source resolution #[prost(message, optional, tag="2")] pub resolution: ::core::option::Option, } @@ -2716,7 +2716,7 @@ pub struct NewAudioSourceResponse { #[prost(message, optional, tag="1")] pub source: ::core::option::Option, } -/// Push a frame to an AudioSource +/// Push a frame to an AudioSource /// The data provided must be available as long as the client receive the callback. #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -2851,6 +2851,8 @@ pub struct AudioSourceOptions { pub noise_suppression: bool, #[prost(bool, tag="3")] pub auto_gain_control: bool, + #[prost(bool, tag="4")] + pub pre_encoded: bool, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -2955,7 +2957,7 @@ impl AudioSourceType { // that it receives from the server. // // Therefore, the ffi client is easier to implement if there is less handles to manage. -// +// // - We are mainly using FfiHandle on info messages (e.g: RoomInfo, TrackInfo, etc...) // For this reason, info are only sent once, at creation (We're not using them for updates, we can infer them from // events on the client implementation). diff --git a/webrtc-sys/src/audio_track.cpp b/webrtc-sys/src/audio_track.cpp index 951208d24..1b17bb469 100644 --- a/webrtc-sys/src/audio_track.cpp +++ b/webrtc-sys/src/audio_track.cpp @@ -39,6 +39,7 @@ inline cricket::AudioOptions to_native_audio_options( rtc_options.echo_cancellation = options.echo_cancellation; rtc_options.noise_suppression = options.noise_suppression; rtc_options.auto_gain_control = options.auto_gain_control; + rtc_options.pre_encoded = options.pre_encoded; return rtc_options; } @@ -48,6 +49,7 @@ inline AudioSourceOptions to_rust_audio_options( options.echo_cancellation = rtc_options.echo_cancellation.value_or(false); options.noise_suppression = rtc_options.noise_suppression.value_or(false); options.auto_gain_control = rtc_options.auto_gain_control.value_or(false); + options.pre_encoded = rtc_options.pre_encoded.value_or(false); return options; } diff --git a/webrtc-sys/src/audio_track.rs b/webrtc-sys/src/audio_track.rs index ee32e475d..d1cdd9437 100644 --- a/webrtc-sys/src/audio_track.rs +++ b/webrtc-sys/src/audio_track.rs @@ -23,6 +23,7 @@ pub mod ffi { pub echo_cancellation: bool, pub noise_suppression: bool, pub auto_gain_control: bool, + pub pre_encoded: bool, } extern "C++" { From cfec8328d3f12e86bd3b19e76c13ae16ab86ef67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kuba=20Podg=C3=B3rski?= Date: Sun, 2 Jun 2024 22:43:59 +0200 Subject: [PATCH 3/3] Update audio_device.cpp --- webrtc-sys/src/audio_device.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webrtc-sys/src/audio_device.cpp b/webrtc-sys/src/audio_device.cpp index 1d90c998d..75911917a 100644 --- a/webrtc-sys/src/audio_device.cpp +++ b/webrtc-sys/src/audio_device.cpp @@ -16,9 +16,9 @@ #include "livekit/audio_device.h" -const int kBytesPerSample = 2; const int kSampleRate = 48000; const int kChannels = 2; +const int kBytesPerSample = kChannels * sizeof(int16_t); const int kSamplesPer10Ms = kSampleRate / 100; namespace livekit {