Refactor silence removal functionality in audio processing

- Simplified the `removeSilences` method in `MediaClip` by removing unnecessary options, defaulting to a standard behavior.
- Updated the `removeSilences` method signature in `MediaTrack` to use the new `SilenceRemoveOptions` type for consistency.
- Added export for `media.types` to enhance type management in the media module.
- Removed outdated mock tests related to silence removal to streamline the test suite.

These changes enhance the clarity and maintainability of the silence removal logic in the audio processing workflow.

Author: k9p5

playground/main.ts

@@ -47,10 +47,7 @@ const audioSource = await core.AudioSource.from('/harvard.MP3');
 await audioTrack.add(
   await new core.AudioClip(audioSource)
 );
-await audioTrack.removeSilences({
-  minDuration: 300,
-  windowSize: 1,
-});
+await audioTrack.removeSilences();
 
 image.animate()
   .rotation(-16).to(14, 5).to(-7, 10).to(24, 7).to(-3, 9).to(19, 7).to(-14, 12).to(5, 9).to(-30, 13)

src/clips/media/index.ts

@@ -7,3 +7,4 @@
 
 export * from './media';
 export * from './media.interfaces';
+export * from './media.types';

src/clips/media/media.ts

@@ -6,7 +6,7 @@
  */
 
 import { Timestamp, Transcript } from '../../models';
-import { AudioSource, SilenceDetectionOptions } from '../../sources';
+import { AudioSource } from '../../sources';
 import { RangeDeserializer } from './media.deserializer';
 import { serializable } from '../../services';
 import { replaceKeyframes } from '../clip/clip.utils';
@@ -16,6 +16,7 @@ import { Clip } from '../clip';
 import type { CaptionPresetStrategy, CaptionTrack } from '../../tracks';
 import type { float, frame } from '../../types';
 import type { MediaClipProps } from './media.interfaces';
+import type { SilenceRemoveOptions } from './media.types';
 
 export class MediaClip<Props extends MediaClipProps = MediaClipProps> extends Clip<MediaClipProps> {
 	public source = new AudioSource();
@@ -319,7 +320,7 @@ export class MediaClip<Props extends MediaClipProps = MediaClipProps> extends Cl
 	 *
 	 * @param options - Options for silence detection
 	 */
-	public async removeSilences(options: SilenceDetectionOptions = {}): Promise<MediaClip<Props>[]> {
+	public async removeSilences(options: SilenceRemoveOptions = {}): Promise<MediaClip<Props>[]> {
 		const silences = (await this.source.silences(options))
 			.filter((silence) => inRange(silence, this.range))
 			.sort((a, b) => a.start.millis - b.start.millis);
@@ -328,6 +329,8 @@ export class MediaClip<Props extends MediaClipProps = MediaClipProps> extends Cl
 			return [this];
 		}
 
+		// default padding between clips
+		const padding = options.padding ?? 500;
 		const result: MediaClip<Props>[] = [this];
 
 		for (const silence of silences) {
@@ -336,17 +339,22 @@ export class MediaClip<Props extends MediaClipProps = MediaClipProps> extends Cl
 			if (!item) break;
 			if (!inRange(silence, item.range)) continue;
 
+			// start with padding
+			const start = new Timestamp(
+				Math.min(silence.start.millis + padding, silence.stop.millis)
+			);
+
 			if (silence.start.millis > item.range[0].millis && silence.stop.millis < item.range[1].millis) {
 				const copy = item.copy();
 
-				item.range[1] = silence.start;
+				item.range[1] = start;
 				copy.range[0] = silence.stop;
 
 				result.push(copy);
 			} else if (silence.start.millis <= item.range[0].millis) {
 				item.range[0] = silence.stop;
 			} else if (silence.stop.millis >= item.range[1].millis) {
-				item.range[1] = silence.start;
+				item.range[1] = start;
 			}
 		}
 

src/clips/media/media.types.ts

@@ -0,0 +1,10 @@
+import { SilenceDetectionOptions } from "../../sources";
+
+export type SilenceRemoveOptions = {
+  /**
+   * Adds padding in milliseconds after each detected non-silent segment.
+   * This helps prevent cutting off audio too abruptly.
+   * @default 500
+   */
+	padding?: number;
+} & SilenceDetectionOptions;

src/tracks/media/media.spec.ts

@@ -10,19 +10,6 @@ import { Composition } from '../../composition';
 import { MediaClip } from '../../clips';
 import { Timestamp } from '../../models';
 import { MediaTrack } from './media';
-import { AudioSource } from '../../sources';
-
-class MockMediaClip extends MediaClip {
-	constructor(range: [Timestamp, Timestamp], silences: { start: Timestamp, stop: Timestamp }[], element: HTMLMediaElement) {
-		super();
-		this.duration.millis = range[1].millis - range[0].millis;
-		this.range = range;
-		this.source = {
-			silences: async () => silences,
-		} as any as AudioSource;
-		this.element = element;
-	}
-}
 
 describe('The Media Track Object', () => {
 	let comp: Composition;
@@ -36,89 +23,6 @@ describe('The Media Track Object', () => {
 		track.on('update', updateMock);
 	});
 
-	it('ignores no silences', async () => {
-		const clip = new MockMediaClip([new Timestamp(10000), new Timestamp(20000)], [], new Audio());
-		clip.duration.frames = 30;
-		await track.add(clip);
-		await track.removeSilences();
-		expect(track.clips.length).toBe(1);
-	});
-
-	it('ignores not applicable silences', async () => {
-		const clip = new MockMediaClip([new Timestamp(10000), new Timestamp(20000)], [
-			{
-				start: new Timestamp(0),
-				stop: new Timestamp(500),
-			},
-			{
-				start: new Timestamp(30000),
-				stop: new Timestamp(30500),
-			},
-		], new Audio());
-		await track.add(clip);
-		expect(clip.source).toBeDefined();
-		await track.removeSilences();
-		expect(track.clips.length).toBe(1);
-		expect(track.clips.at(0)).toBe(clip);
-		expect(track.clips.at(0)?.state).toBe('ATTACHED');
-	});
-
-	it('removes silences', async () => {
-		const clip = new MockMediaClip([new Timestamp(10000), new Timestamp(20000)], [
-			{
-				start: new Timestamp(0),
-				stop: new Timestamp(10050),
-			},
-			{
-				start: new Timestamp(11000),
-				stop: new Timestamp(15000),
-			},
-			{
-				start: new Timestamp(19000),
-				stop: new Timestamp(30500),
-			},
-		], new Audio());
-		await track.add(clip);
-		expect(clip.source).toBeDefined();
-		await track.removeSilences();
-		expect(track.clips.length).toBe(2);
-		expect(track.clips.at(0)?.range[0].millis).toBe(10050);
-		expect(track.clips.at(0)?.range[1].millis).toBe(11000);
-		expect(track.clips.at(1)?.range[0].millis).toBe(15000);
-		expect(track.clips.at(1)?.range[1].millis).toBe(19000);
-		expect(track.clips.at(0)?.state).toBe('ATTACHED');
-		expect(track.clips.at(1)?.state).toBe('ATTACHED');
-	});
-
-	it('removes silences twice', async () => {
-		const clip = new MockMediaClip([new Timestamp(10000), new Timestamp(20000)], [
-			{
-				start: new Timestamp(0),
-				stop: new Timestamp(500),
-			},
-			{
-				start: new Timestamp(11000),
-				stop: new Timestamp(15000),
-			},
-			{
-				start: new Timestamp(19000),
-				stop: new Timestamp(30500),
-			},
-		], new Audio());
-		await track.add(clip);
-		expect(clip.source).toBeDefined();
-		await track.removeSilences();
-		await track.removeSilences();
-		expect(track.clips.length).toBe(2);
-		expect(track.clips.at(0)?.id).not.toBe(clip.id);
-		expect(track.clips.at(0)?.range[0].millis).toBe(10000);
-		expect(track.clips.at(0)?.range[1].millis).toBe(11000);
-		expect(track.clips.at(1)?.range[0].millis).toBe(15000);
-		expect(track.clips.at(1)?.range[1].millis).toBe(19000);
-		expect(track.clips.at(0)?.state).toBe('ATTACHED');
-		expect(track.clips.at(1)?.state).toBe('ATTACHED');
-	});
-
 	it('should propagate a seek call', async () => {
 		const clip = new MediaClip();
 		clip.element = new Audio();

src/tracks/media/media.ts

@@ -7,9 +7,8 @@
 
 import { Track } from '../track';
 import { Timestamp } from '../../models';
-import { SilenceDetectionOptions } from '../../sources';
 
-import type { MediaClip } from '../../clips';
+import type { MediaClip, SilenceRemoveOptions } from '../../clips';
 
 export class MediaTrack<Clip extends MediaClip> extends Track<MediaClip> {
 	public clips: Clip[] = [];
@@ -24,7 +23,7 @@ export class MediaTrack<Clip extends MediaClip> extends Track<MediaClip> {
 	 *
 	 * @param options - Options for silence detection
 	 */
-	public async removeSilences(options: SilenceDetectionOptions = {}) {
+	public async removeSilences(options: SilenceRemoveOptions = {}) {
 		const clips: MediaClip[] = [];
 
 		for (const clip of this.clips.map((clip) => clip.detach())) {