Skip to content

Commit

Permalink
show mic input audio levels
Browse files Browse the repository at this point in the history
  • Loading branch information
Brendonovich committed Nov 14, 2024
1 parent fbdfbcf commit 2fca3bf
Show file tree
Hide file tree
Showing 13 changed files with 588 additions and 179 deletions.
12 changes: 12 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions apps/desktop/src-tauri/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ dotenvy_macro = "0.15.7"
global-hotkey = "0.6.3"
tauri-plugin-http = "2.0.0-rc.0"
rand = "0.8.5"
cpal.workspace = true
keyed_priority_queue = "0.4.2"

[target.'cfg(target_os = "macos")'.dependencies]
core-graphics = "0.24.0"
Expand Down
125 changes: 125 additions & 0 deletions apps/desktop/src-tauri/src/audio_meter.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
use cap_media::feeds::AudioInputSamples;
use cpal::{SampleFormat, StreamInstant};
use keyed_priority_queue::KeyedPriorityQueue;
use std::cmp::Ordering;
use std::collections::VecDeque;
use std::time::Duration;

pub const MAX_AMPLITUDE_F32: f64 = (u16::MAX / 2) as f64; // i16 max value
pub const ZERO_AMPLITUDE: u16 = 0;
pub const TERMINAL_WIDTH: f64 = 0.8;
pub const MIN_DB: f64 = -96.0;

// https://github.com/cgbur/meter/blob/master/src/time_window.rs
pub(crate) struct VolumeMeter {
pub(crate) keep_secs: f32,
keep_duration: Duration, // secs
maxes: KeyedPriorityQueue<StreamInstant, MinNonNan>,
times: VecDeque<StreamInstant>,
}

impl VolumeMeter {
pub fn new(keep_secs: f32) -> Self {
Self {
keep_duration: Duration::from_secs_f32(keep_secs),
keep_secs,
maxes: KeyedPriorityQueue::new(),
times: Default::default(),
}
}

pub fn push(&mut self, time: StreamInstant, value: f64) {
let value = MinNonNan(-value);
self.maxes.push(time, value);
self.times.push_back(time);

loop {
if let Some(time) = self
.times
.back()
.unwrap()
.duration_since(self.times.front().unwrap())
{
if time > self.keep_duration {
self.maxes.remove(self.times.front().unwrap());
self.times.pop_front();
} else {
break;
}
} else {
break;
}

if self.times.len() <= 1 {
break;
}
}
}

pub fn max(&self) -> f64 {
-self.maxes.peek().map(|(_, db)| db.0).unwrap_or(-MIN_DB)
}
}

#[derive(PartialEq)]
struct MinNonNan(f64);

impl Eq for MinNonNan {}

impl PartialOrd for MinNonNan {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
other.0.partial_cmp(&self.0)
}
}

impl Ord for MinNonNan {
fn cmp(&self, other: &MinNonNan) -> Ordering {
self.partial_cmp(other).unwrap()
}
}

use cpal::Sample;

pub fn db_fs(data: impl Iterator<Item = f64>) -> f64 {
let max = data
.map(|f| f.to_sample::<i16>().unsigned_abs())
.max()
.unwrap_or(ZERO_AMPLITUDE);

(20.0 * (max as f64 / MAX_AMPLITUDE_F32).log10()).clamp(MIN_DB, 0.0)
}

pub fn samples_to_f64(samples: &AudioInputSamples) -> impl Iterator<Item = f64> + use<'_> {
samples
.data
.chunks(samples.format.sample_size())
.map(|data| match samples.format {
SampleFormat::I8 => i8::from_ne_bytes([data[0]]) as f64 / i8::MAX as f64,
SampleFormat::U8 => u8::from_ne_bytes([data[0]]) as f64 / u8::MAX as f64,
SampleFormat::I16 => i16::from_ne_bytes([data[0], data[1]]) as f64 / i16::MAX as f64,
SampleFormat::U16 => u16::from_ne_bytes([data[0], data[1]]) as f64 / u16::MAX as f64,
SampleFormat::I32 => {
i32::from_ne_bytes([data[0], data[1], data[2], data[3]]) as f64 / i32::MAX as f64
}
SampleFormat::U32 => {
u32::from_ne_bytes([data[0], data[1], data[2], data[3]]) as f64 / u32::MAX as f64
}
SampleFormat::U64 => {
u64::from_ne_bytes([
data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7],
]) as f64
/ u64::MAX as f64
}
SampleFormat::I64 => {
i64::from_ne_bytes([
data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7],
]) as f64
/ i64::MAX as f64
}
SampleFormat::F32 => f32::from_ne_bytes([data[0], data[1], data[2], data[3]]) as f64,
SampleFormat::F64 => f64::from_ne_bytes([
data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7],
]),
_ => todo!(),
})
}
86 changes: 72 additions & 14 deletions apps/desktop/src-tauri/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ mod permissions;
mod platform;
mod recording;
// mod resource;
mod audio_meter;
mod cursor;
mod tray;
mod upload;
Expand All @@ -19,11 +20,12 @@ use audio::AppSounds;
use auth::{AuthStore, AuthenticationInvalid};
use cap_editor::{EditorInstance, FRAMES_WS_PATH};
use cap_editor::{EditorState, ProjectRecordings};
use cap_media::feeds::{AudioInputFeed, AudioInputSamplesSender};
use cap_media::sources::CaptureScreen;
use cap_media::{
feeds::{AudioFrameBuffer, CameraFeed, CameraFrameSender},
platform::Bounds,
sources::{AudioInputSource, ScreenCaptureTarget},
sources::ScreenCaptureTarget,
};
use cap_project::{
ProjectConfiguration, RecordingMeta, SharingMeta, TimelineConfiguration, TimelineSegment,
Expand Down Expand Up @@ -91,6 +93,10 @@ pub struct App {
#[serde(skip)]
camera_feed: Option<CameraFeed>,
#[serde(skip)]
audio_input_feed: Option<AudioInputFeed>,
#[serde(skip)]
audio_input_tx: AudioInputSamplesSender,
#[serde(skip)]
handle: AppHandle,
#[serde(skip)]
current_recording: Option<InProgressRecording>,
Expand Down Expand Up @@ -167,16 +173,16 @@ impl App {
_ => {}
}

match &new_options.camera_label {
match new_options.camera_label() {
Some(camera_label) => {
if self.camera_feed.is_none() {
self.camera_feed = CameraFeed::init(camera_label, self.camera_tx.clone())
if let Some(camera_feed) = self.camera_feed.as_mut() {
camera_feed
.switch_cameras(camera_label)
.await
.map_err(|error| eprintln!("{error}"))
.ok();
} else if let Some(camera_feed) = self.camera_feed.as_mut() {
camera_feed
.switch_cameras(camera_label)
} else {
self.camera_feed = CameraFeed::init(camera_label, self.camera_tx.clone())
.await
.map_err(|error| eprintln!("{error}"))
.ok();
Expand All @@ -187,6 +193,31 @@ impl App {
}
}

match new_options.audio_input_name() {
Some(audio_input_name) => {
if let Some(audio_input_feed) = self.audio_input_feed.as_mut() {
audio_input_feed
.switch_input(audio_input_name)
.await
.map_err(|error| eprintln!("{error}"))
.ok();
} else {
self.audio_input_feed = if let Ok(feed) = AudioInputFeed::init(audio_input_name)
.await
.map_err(|error| eprintln!("{error}"))
{
feed.add_sender(self.audio_input_tx.clone()).await.unwrap();
Some(feed)
} else {
None
};
}
}
None => {
self.audio_input_feed = None;
}
}

self.start_recording_options = new_options;

RecordingOptionsChanged.emit(&self.handle).ok();
Expand Down Expand Up @@ -242,6 +273,9 @@ pub struct NewNotification {
is_error: bool,
}

#[derive(Deserialize, specta::Type, Serialize, tauri_specta::Event, Debug, Clone)]
pub struct AudioInputLevelChange(f64);

type MutableState<'a, T> = State<'a, Arc<RwLock<T>>>;

#[tauri::command]
Expand Down Expand Up @@ -342,6 +376,7 @@ async fn start_recording(app: AppHandle, state: MutableState<'_, App>) -> Result
recording_dir,
&state.start_recording_options,
state.camera_feed.as_ref(),
state.audio_input_feed.as_ref(),
)
.await
{
Expand Down Expand Up @@ -1672,13 +1707,13 @@ async fn list_audio_devices() -> Result<Vec<String>, ()> {
}

// TODO: Check - is this necessary? `spawn_blocking` is quite a bit of overhead.
tokio::task::spawn_blocking(|| {
let devices = AudioInputSource::get_devices();
// tokio::task::spawn_blocking(|| {
let devices = AudioInputFeed::list_devices();

devices.keys().cloned().collect()
})
.await
.map_err(|_| ())
Ok(devices.keys().cloned().collect())
// })
// .await
// .map_err(|_| ())
}

#[tauri::command(async)]
Expand Down Expand Up @@ -2442,7 +2477,8 @@ pub async fn run() {
RequestNewScreenshot,
RequestOpenSettings,
NewNotification,
AuthenticationInvalid
AuthenticationInvalid,
AudioInputLevelChange
])
.typ::<ProjectConfiguration>()
.typ::<AuthStore>()
Expand All @@ -2461,6 +2497,8 @@ pub async fn run() {
let (camera_tx, camera_rx) = CameraFeed::create_channel();
let camera_ws_port = camera::create_camera_ws(camera_rx.clone()).await;

let (audio_input_tx, audio_input_rx) = AudioInputFeed::create_channel();

tauri::async_runtime::set(tokio::runtime::Handle::current());

#[allow(unused_mut)]
Expand Down Expand Up @@ -2509,11 +2547,31 @@ pub async fn run() {
CapWindow::Main.show(&app_handle).ok();
}

let mut time_window = audio_meter::VolumeMeter::new(0.2);
tokio::spawn({
let app_handle = app_handle.clone();
async move {
while let Ok(samples) = audio_input_rx.recv_async().await {
let floats = audio_meter::samples_to_f64(&samples);

let db = audio_meter::db_fs(floats);

time_window.push(samples.info.timestamp().capture, db);

let max = time_window.max();

AudioInputLevelChange(max).emit(&app_handle).ok();
}
}
});

app.manage(Arc::new(RwLock::new(App {
handle: app_handle.clone(),
camera_tx,
camera_ws_port,
camera_feed: None,
audio_input_tx,
audio_input_feed: None,
start_recording_options: RecordingOptions {
capture_target: ScreenCaptureTarget::Screen(CaptureScreen {
id: 1,
Expand Down
9 changes: 3 additions & 6 deletions apps/desktop/src-tauri/src/recording.rs
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ pub async fn start(
recording_dir: PathBuf,
recording_options: &RecordingOptions,
camera_feed: Option<&CameraFeed>,
audio_input_feed: Option<&AudioInputFeed>,
) -> Result<InProgressRecording, MediaError> {
let content_dir = recording_dir.join("content");
let cursors_dir = content_dir.join("cursors");
Expand Down Expand Up @@ -242,11 +243,7 @@ pub async fn start(
.sink("screen_capture_encoder", screen_encoder);
}

if let Some(mic_source) = recording_options
.audio_input_name
.as_ref()
.and_then(|name| AudioInputSource::init(name))
{
if let Some(mic_source) = audio_input_feed.map(AudioInputSource::init) {
let mic_config = mic_source.info();
audio_output_path = Some(content_dir.join("audio-input.mp3"));

Expand All @@ -263,7 +260,7 @@ pub async fn start(
.sink("microphone_encoder", mic_encoder);
}

if let Some(camera_source) = CameraSource::init(camera_feed) {
if let Some(camera_source) = camera_feed.map(CameraSource::init) {
let camera_config = camera_source.info();
let output_config = camera_config.scaled(1920, 30);
camera_output_path = Some(content_dir.join("camera.mp4"));
Expand Down
Loading

0 comments on commit 2fca3bf

Please sign in to comment.