Allow specifying sample rate per output - add output resamplers (#925)

CHANGELOG.md

@@ -6,13 +6,15 @@
 
 - Drop support for `SHADER_UNUSED_VERTEX_OUTPUT` `wgpu` feature.  ([#733](https://github.com/software-mansion/live-compositor/pull/733)) by [@jerzywilczek](https://github.com/jerzywilczek)
 - Rename component properties describing color. Remove `_rgba` suffix. ([#896](https://github.com/software-mansion/live-compositor/issues/896)) by [@BrtqKr](https://github.com/BrtqKr)
+- Replace the `LIVE_COMPOSITOR_OUTPUT_SAMPLE_RATE` configuration environment variable with `LIVE_COMPOSITOR_MIXING_SAMPLE_RATE`. The output sample rate is now determined using encoder options on `register_output`. Change the default output sample rate for AAC codec to 44100 Hz. ([#925](https://github.com/software-mansion/live-compositor/pull/925) by [@WojciechBarczynski](https://github.com/WojciechBarczynski)).
 
 ### ✨ New features
 
 - Add `loop` option for MP4 input. ([#699](https://github.com/software-mansion/live-compositor/pull/699) by [@WojciechBarczynski](https://github.com/WojciechBarczynski))
 - Add `LIVE_COMPOSITOR_LOG_FILE` environment variable to enable logging to file ([#853](https://github.com/software-mansion/live-compositor/pull/853) by [@wkozyra95](https://github.com/wkozyra95))
 - Add border, border radius and box shadow options to `Rescaler` and `View` components. ([#815](https://github.com/software-mansion/live-compositor/pull/815) by [@WojciechBarczynski](https://github.com/WojciechBarczynski), ([#839](https://github.com/software-mansion/live-compositor/pull/839), [#842](https://github.com/software-mansion/live-compositor/pull/842), [#858](https://github.com/software-mansion/live-compositor/pull/858) by [@wkozyra95](https://github.com/wkozyra95))
 - Extend supported color formats. ([#896](https://github.com/software-mansion/live-compositor/issues/896)) by [@BrtqKr](https://github.com/BrtqKr)
+- Allow specifying output sample rates per output in `register_output` requests. ([#925](https://github.com/software-mansion/live-compositor/pull/925) by [@WojciechBarczynski](https://github.com/WojciechBarczynski))
 
 ### 🐛 Bug fixes
 

compositor_api/src/types/from_register_output.rs

@@ -131,8 +131,12 @@ impl TryFrom<Mp4Output> for pipeline::RegisterOutputOptions<output::OutputOption
             },
         });
         let mp4_audio = audio.as_ref().map(|a| match &a.encoder {
-            Mp4AudioEncoderOptions::Aac { channels } => Mp4AudioTrack {
+            Mp4AudioEncoderOptions::Aac {
+                channels,
+                sample_rate,
+            } => Mp4AudioTrack {
                 channels: channels.clone().into(),
+                sample_rate: sample_rate.unwrap_or(44100),
             },
         });
 
@@ -198,6 +202,7 @@ impl TryFrom<WhipOutput> for pipeline::RegisterOutputOptions<output::OutputOptio
             WhipAudioEncoderOptions::Opus {
                 channels,
                 preset: _,
+                sample_rate: _,
             } => WhipAudioOptions {
                 codec: pipeline::AudioCodec::Opus,
                 channels: match channels {
@@ -292,37 +297,45 @@ fn maybe_video_options(
 impl From<Mp4AudioEncoderOptions> for pipeline::encoder::AudioEncoderOptions {
     fn from(value: Mp4AudioEncoderOptions) -> Self {
         match value {
-            Mp4AudioEncoderOptions::Aac { channels } => {
-                AudioEncoderOptions::Aac(AacEncoderOptions {
-                    channels: channels.into(),
-                })
-            }
+            Mp4AudioEncoderOptions::Aac {
+                channels,
+                sample_rate,
+            } => AudioEncoderOptions::Aac(AacEncoderOptions {
+                channels: channels.into(),
+                sample_rate: sample_rate.unwrap_or(44100),
+            }),
         }
     }
 }
 
 impl From<RtpAudioEncoderOptions> for pipeline::encoder::AudioEncoderOptions {
     fn from(value: RtpAudioEncoderOptions) -> Self {
         match value {
-            RtpAudioEncoderOptions::Opus { channels, preset } => {
-                AudioEncoderOptions::Opus(encoder::opus::OpusEncoderOptions {
-                    channels: channels.into(),
-                    preset: preset.unwrap_or(OpusEncoderPreset::Voip).into(),
-                })
-            }
+            RtpAudioEncoderOptions::Opus {
+                channels,
+                preset,
+                sample_rate,
+            } => AudioEncoderOptions::Opus(encoder::opus::OpusEncoderOptions {
+                channels: channels.into(),
+                preset: preset.unwrap_or(OpusEncoderPreset::Voip).into(),
+                sample_rate: sample_rate.unwrap_or(48000),
+            }),
         }
     }
 }
 
 impl From<WhipAudioEncoderOptions> for pipeline::encoder::AudioEncoderOptions {
     fn from(value: WhipAudioEncoderOptions) -> Self {
         match value {
-            WhipAudioEncoderOptions::Opus { channels, preset } => {
-                AudioEncoderOptions::Opus(encoder::opus::OpusEncoderOptions {
-                    channels: channels.into(),
-                    preset: preset.unwrap_or(OpusEncoderPreset::Voip).into(),
-                })
-            }
+            WhipAudioEncoderOptions::Opus {
+                channels,
+                preset,
+                sample_rate,
+            } => AudioEncoderOptions::Opus(encoder::opus::OpusEncoderOptions {
+                channels: channels.into(),
+                preset: preset.unwrap_or(OpusEncoderPreset::Voip).into(),
+                sample_rate: sample_rate.unwrap_or(48000),
+            }),
         }
     }
 }

compositor_api/src/types/register_output.rs

@@ -125,13 +125,20 @@ pub enum RtpAudioEncoderOptions {
 
         /// (**default="voip"**) Specifies preset for audio output encoder.
         preset: Option<OpusEncoderPreset>,
+
+        /// (**default=`48000`**) Sample rate. Allowed values: [8000, 16000, 24000, 48000].
+        sample_rate: Option<u32>,
     },
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)]
 #[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
 pub enum Mp4AudioEncoderOptions {
-    Aac { channels: AudioChannels },
+    Aac {
+        channels: AudioChannels,
+        /// (**default=`44100`**) Sample rate. Allowed values: [8000, 16000, 24000, 44100, 48000].
+        sample_rate: Option<u32>,
+    },
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)]
@@ -143,6 +150,9 @@ pub enum WhipAudioEncoderOptions {
 
         /// (**default="voip"**) Specifies preset for audio output encoder.
         preset: Option<OpusEncoderPreset>,
+
+        /// (**default=`48000`**) Sample rate. Allowed values: [8000, 16000, 24000, 48000].
+        sample_rate: Option<u32>,
     },
 }
 

compositor_pipeline/src/audio_mixer.rs

@@ -28,9 +28,9 @@ struct OutputInfo {
 pub(super) struct AudioMixer(Arc<Mutex<InternalAudioMixer>>);
 
 impl AudioMixer {
-    pub fn new(output_sample_rate: u32) -> Self {
+    pub fn new(mixing_sample_rate: u32) -> Self {
         Self(Arc::new(Mutex::new(InternalAudioMixer::new(
-            output_sample_rate,
+            mixing_sample_rate,
         ))))
     }
 
@@ -72,14 +72,14 @@ impl AudioMixer {
 #[derive(Debug)]
 pub(super) struct InternalAudioMixer {
     outputs: HashMap<OutputId, OutputInfo>,
-    output_sample_rate: u32,
+    mixing_sample_rate: u32,
 }
 
 impl InternalAudioMixer {
-    pub fn new(output_sample_rate: u32) -> Self {
+    pub fn new(mixing_sample_rate: u32) -> Self {
         Self {
             outputs: HashMap::new(),
-            output_sample_rate,
+            mixing_sample_rate,
         }
     }
 
@@ -102,9 +102,9 @@ impl InternalAudioMixer {
         let samples_count = expected_samples_count(
             samples_set.start_pts,
             samples_set.end_pts,
-            self.output_sample_rate,
+            self.mixing_sample_rate,
         );
-        let input_samples = prepare_input_samples(samples_set, self.output_sample_rate);
+        let input_samples = prepare_input_samples(samples_set, self.mixing_sample_rate);
 
         OutputSamplesSet(
             self.outputs

compositor_pipeline/src/audio_mixer/prepare_inputs.rs

@@ -15,7 +15,7 @@ pub(super) fn expected_samples_count(start: Duration, end: Duration, sample_rate
 }
 pub(super) fn prepare_input_samples(
     input_samples_set: InputSamplesSet,
-    output_sample_rate: u32,
+    mixing_sample_rate: u32,
 ) -> HashMap<InputId, Vec<(i16, i16)>> {
     input_samples_set
         .samples
@@ -25,7 +25,7 @@ pub(super) fn prepare_input_samples(
                 input_samples_set.start_pts,
                 input_samples_set.end_pts,
                 input_batch,
-                output_sample_rate,
+                mixing_sample_rate,
             );
 
             (input_id, samples)

compositor_pipeline/src/audio_mixer/types.rs

@@ -65,10 +65,10 @@ impl InputSamples {
     pub fn new(
         samples: Arc<Vec<(i16, i16)>>,
         start_pts: Duration,
-        output_sample_rate: u32,
+        mixing_sample_rate: u32,
     ) -> Self {
         let end_pts =
-            start_pts + Duration::from_secs_f64(samples.len() as f64 / output_sample_rate as f64);
+            start_pts + Duration::from_secs_f64(samples.len() as f64 / mixing_sample_rate as f64);
 
         Self {
             samples,

compositor_pipeline/src/error.rs

@@ -120,6 +120,9 @@ pub enum EncoderInitError {
 
     #[error("Internal FDK AAC encoder error: {0}")]
     AacError(fdk::AACENC_ERROR),
+
+    #[error(transparent)]
+    ResamplerError(#[from] rubato::ResamplerConstructionError),
 }
 
 #[derive(Debug, thiserror::Error)]

compositor_pipeline/src/pipeline.rs

@@ -127,7 +127,7 @@ pub struct Options {
     pub web_renderer: WebRendererInitOptions,
     pub force_gpu: bool,
     pub download_root: PathBuf,
-    pub output_sample_rate: u32,
+    pub mixing_sample_rate: u32,
     pub stun_servers: Arc<Vec<String>>,
     pub wgpu_features: WgpuFeatures,
     pub load_system_fonts: Option<bool>,
@@ -139,7 +139,7 @@ pub struct Options {
 
 #[derive(Clone)]
 pub struct PipelineCtx {
-    pub output_sample_rate: u32,
+    pub mixing_sample_rate: u32,
     pub output_framerate: Framerate,
     pub stun_servers: Arc<Vec<String>>,
     pub download_dir: Arc<PathBuf>,
@@ -154,7 +154,7 @@ pub struct PipelineCtx {
 impl std::fmt::Debug for PipelineCtx {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("PipelineCtx")
-            .field("output_sample_rate", &self.output_sample_rate)
+            .field("mixing_sample_rate", &self.mixing_sample_rate)
             .field("output_framerate", &self.output_framerate)
             .field("download_dir", &self.download_dir)
             .field("event_emitter", &self.event_emitter)
@@ -234,11 +234,11 @@ impl Pipeline {
             inputs: HashMap::new(),
             queue: Queue::new(opts.queue_options, &event_emitter),
             renderer,
-            audio_mixer: AudioMixer::new(opts.output_sample_rate),
+            audio_mixer: AudioMixer::new(opts.mixing_sample_rate),
             is_started: false,
             shutdown_whip_whep_sender,
             ctx: PipelineCtx {
-                output_sample_rate: opts.output_sample_rate,
+                mixing_sample_rate: opts.mixing_sample_rate,
                 output_framerate: opts.queue_options.output_framerate,
                 stun_servers,
                 download_dir: download_dir.into(),

compositor_pipeline/src/pipeline/decoder/audio.rs

@@ -37,7 +37,7 @@ trait AudioDecoderExt {
 
 pub fn start_audio_resampler_only_thread(
     input_sample_rate: u32,
-    output_sample_rate: u32,
+    mixing_sample_rate: u32,
     raw_samples_receiver: Receiver<PipelineEvent<DecodedSamples>>,
     samples_sender: Sender<PipelineEvent<InputSamples>>,
     input_id: InputId,
@@ -55,7 +55,7 @@ pub fn start_audio_resampler_only_thread(
 
             run_resampler_only_thread(
                 input_sample_rate,
-                output_sample_rate,
+                mixing_sample_rate,
                 raw_samples_receiver,
                 samples_sender,
                 decoder_init_result_sender,
@@ -72,12 +72,12 @@ pub fn start_audio_resampler_only_thread(
 
 fn run_resampler_only_thread(
     input_sample_rate: u32,
-    output_sample_rate: u32,
+    mixing_sample_rate: u32,
     raw_samples_receiver: Receiver<PipelineEvent<DecodedSamples>>,
     samples_sender: Sender<PipelineEvent<InputSamples>>,
     init_result_sender: Sender<Result<(), InputInitError>>,
 ) {
-    let mut resampler = match Resampler::new(input_sample_rate, output_sample_rate) {
+    let mut resampler = match Resampler::new(input_sample_rate, mixing_sample_rate) {
         Ok(resampler) => {
             if init_result_sender.send(Ok(())).is_err() {
                 error!("Failed to send rescaler init result.");
@@ -114,7 +114,7 @@ fn run_resampler_only_thread(
 
 pub fn start_audio_decoder_thread(
     opts: AudioDecoderOptions,
-    output_sample_rate: u32,
+    mixing_sample_rate: u32,
     chunks_receiver: Receiver<PipelineEvent<EncodedChunk>>,
     samples_sender: Sender<PipelineEvent<InputSamples>>,
     input_id: InputId,
@@ -139,7 +139,7 @@ pub fn start_audio_decoder_thread(
 
             run_decoding(
                 opts,
-                output_sample_rate,
+                mixing_sample_rate,
                 chunks_receiver,
                 sender,
                 init_result_sender,
@@ -163,7 +163,7 @@ pub fn start_audio_decoder_thread(
 /// - always ok for AAC (aac sample rate is unknown at register time, first chunk is need to determine it)
 fn run_decoding<F>(
     opts: AudioDecoderOptions,
-    output_sample_rate: u32,
+    mixing_sample_rate: u32,
     chunks_receiver: Receiver<PipelineEvent<EncodedChunk>>,
     samples_sender: F,
     init_result_sender: Sender<Result<(), InputInitError>>,
@@ -191,7 +191,7 @@ fn run_decoding<F>(
         AudioDecoderOptions::Opus(opus_decoder_opts) => {
             // Opus decoder initialization doesn't require input stream data,
             // so this can wait and send init result
-            match init_opus_decoder(opus_decoder_opts, output_sample_rate) {
+            match init_opus_decoder(opus_decoder_opts, mixing_sample_rate) {
                 Ok((mut decoder, mut resampler)) => {
                     send_result(Ok(()));
                     run_decoding_loop(
@@ -224,7 +224,7 @@ fn run_decoding<F>(
             let init_res = AacDecoder::new(aac_decoder_opts, &first_chunk)
                 .map(|decoder| {
                     let resampler =
-                        Resampler::new(decoder.decoded_sample_rate(), output_sample_rate)?;
+                        Resampler::new(decoder.decoded_sample_rate(), mixing_sample_rate)?;
                     Ok((decoder, resampler))
                 })
                 .and_then(|res| res);
@@ -277,9 +277,9 @@ fn run_decoding_loop<Decoder, F>(
 
 fn init_opus_decoder(
     opus_decoder_opts: OpusDecoderOptions,
-    output_sample_rate: u32,
+    mixing_sample_rate: u32,
 ) -> Result<(OpusDecoder, Resampler), InputInitError> {
-    let decoder = OpusDecoder::new(opus_decoder_opts, output_sample_rate)?;
-    let resampler = Resampler::new(decoder.decoded_sample_rate(), output_sample_rate)?;
+    let decoder = OpusDecoder::new(opus_decoder_opts, mixing_sample_rate)?;
+    let resampler = Resampler::new(decoder.decoded_sample_rate(), mixing_sample_rate)?;
     Ok((decoder, resampler))
 }

compositor_pipeline/src/pipeline/decoder/audio/opus.rs

@@ -18,10 +18,10 @@ pub(super) struct OpusDecoder {
 }
 
 impl OpusDecoder {
-    pub fn new(opts: OpusDecoderOptions, output_sample_rate: u32) -> Result<Self, InputInitError> {
+    pub fn new(opts: OpusDecoderOptions, mixing_sample_rate: u32) -> Result<Self, InputInitError> {
         const OPUS_SAMPLE_RATES: [u32; 5] = [8_000, 12_000, 16_000, 24_000, 48_000];
-        let decoded_sample_rate = if OPUS_SAMPLE_RATES.contains(&output_sample_rate) {
-            output_sample_rate
+        let decoded_sample_rate = if OPUS_SAMPLE_RATES.contains(&mixing_sample_rate) {
+            mixing_sample_rate
         } else {
             48_000
         };