Experiments

This page aims to collect experiments and hacks that were made to the Remotion Recorder

Swear word detection

Here is a demonstration by Matt McGillivray of how swear words can be automatically censored and replaced with a sound effect.

AI audio enhancement

Another experiment by Matt McGillivray is this one where he uses the ai|coustics API to enhance the audio of his recording.

import {$} from 'bun';
import {WEBCAM_PREFIX} from './config/cameras';

const API_URL = 'https://api.ai-coustics.com/v1';
const API_KEY = process.env.AI_COUSTICS_API_KEY;

if (!API_KEY) {
  console.error('AI_COUSTICS_API_KEY environment variable is required');
  process.exit(1);
}

async function uploadAndEnhance(
  audioBuffer: ArrayBuffer,
  fileName: string,
  options: {
    loudness_target_level?: number;
    loudness_peak_limit?: number;
    enhancement_level?: number;
    transcode_kind?: string;
  } = {},
) {
  const {loudness_target_level = -14, loudness_peak_limit = -1, enhancement_level = 0.7, transcode_kind = 'MP3'} = options;

  const formData = new FormData();
  formData.append('loudness_target_level', loudness_target_level.toString());
  formData.append('loudness_peak_limit', loudness_peak_limit.toString());
  formData.append('enhancement_level', enhancement_level.toString());
  formData.append('transcode_kind', transcode_kind);
  formData.append('model_arch', 'FINCH');

  const audioBlob = new Blob([audioBuffer], {
    type: 'application/octet-stream',
  });
  formData.append('file', audioBlob, fileName);

  if (!API_KEY) {
    throw new Error('API_KEY is undefined');
  }

  try {
    const response = await fetch(`${API_URL}/media/enhance`, {
      method: 'POST',
      headers: {
        'X-API-Key': API_KEY,
      },
      body: formData,
    });

    if (response.status !== 201) {
      const responseText = await response.text();
      throw new Error(`API error: ${responseText}`);
    }

    const responseJson = await response.json();
    const generatedName = responseJson.generated_name;
    console.log(`Uploaded file's generated name: ${generatedName}`);
    return generatedName;
  } catch (error) {
    throw new Error(`Failed to enhance audio: ${error}`);
  }
}

async function downloadEnhancedMedia(generatedName: string, outputFilePath: string, maxRetries = 60, retryDelayMs = 5000) {
  const url = `${API_URL}/media/${generatedName}`;

  if (!API_KEY) {
    throw new Error('API_KEY is undefined');
  }

  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      const response = await fetch(url, {
        headers: {
          'X-API-Key': API_KEY,
        },
      });

      if (response.status === 200) {
        const arrayBuffer = await response.arrayBuffer();
        await Bun.write(outputFilePath, new Uint8Array(arrayBuffer));
        console.log(`✓ Downloaded enhanced audio to: ${outputFilePath}`);
        return;
      } else if (response.status === 202) {
        console.log(`⏳ Audio still processing... (attempt ${attempt}/${maxRetries})`);
        if (attempt < maxRetries) {
          await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
          continue;
        }
      } else {
        const responseText = await response.text();
        throw new Error(`API error: ${responseText}`);
      }
    } catch (error) {
      if (attempt === maxRetries) {
        throw new Error(`Failed to download after ${maxRetries} attempts: ${error}`);
      }
      console.log(`⚠️ Download attempt ${attempt} failed, retrying...`);
      await new Promise((resolve) => setTimeout(resolve, retryDelayMs));
    }
  }

  throw new Error(`Audio still processing after ${(maxRetries * retryDelayMs) / 1000} seconds`);
}

async function extractAudioForAPI(
  videoPath: string,
  options: {
    outputFormat?: 'mp3';
    bitrate?: number;
    sampleRate?: number;
  } = {},
) {
  const {outputFormat = 'mp3', bitrate = 128, sampleRate = 44100} = options;

  const fileName =
    videoPath
      .split('/')
      .pop()
      ?.replace(/\.[^/.]+$/, '') || 'audio';
  const outputDir = videoPath.replace(/\/[^/]+$/, '/audio');
  const outputPath = `${outputDir}/${fileName}.${outputFormat}`;

  await $`mkdir -p ${outputDir}`.quiet();

  try {
    await $`ffmpeg -hide_banner -i ${videoPath} -vn -acodec libmp3lame -ab ${bitrate}k -ar ${sampleRate} ${outputPath} -y`.quiet();

    const audioBuffer = await Bun.file(outputPath).arrayBuffer();
    return {audioBuffer, outputPath, fileName: `${fileName}.${outputFormat}`};
  } catch (error) {
    throw new Error(`Failed to extract audio from ${videoPath}: ${error}`);
  }
}

async function replaceAudioInVideo(originalVideoPath: string, enhancedAudioPath: string, outputVideoPath: string) {
  try {
    await $`ffmpeg -hide_banner -i ${originalVideoPath} -i ${enhancedAudioPath} -c:v copy -c:a libopus -map 0:v:0 -map 1:a:0 ${outputVideoPath} -y`;
    console.log(`✓ Replaced audio in video: ${outputVideoPath}`);
  } catch (error) {
    console.error('FFmpeg stderr:', error.stderr?.toString());
    console.error('FFmpeg stdout:', error.stdout?.toString());
    throw new Error(`Failed to replace audio in video: ${error}`);
  }
}

const id = process.argv[2];

if (!id) {
  console.error('Please provide a composition ID');
  console.error('Usage: bun enhanceAudio.ts <composition-id>');
  process.exit(1);
}

const files = await $`ls public/${id}`.quiet();
const webcamFiles = files.stdout
  .toString('utf8')
  .split('\n')
  .filter((f) => f.startsWith(WEBCAM_PREFIX));

if (webcamFiles.length === 0) {
  console.log(`No webcam files found in public/${id}`);
  process.exit(0);
}

console.log(`Found ${webcamFiles.length} webcam files to process`);

const rawDir = `public/${id}/raw`;
await $`mkdir -p ${rawDir}`.quiet();

for (const file of webcamFiles) {
  const videoPath = `public/${id}/${file}`;
  const rawVideoPath = `${rawDir}/${file}`;
  console.log(`Processing ${file}...`);

  try {
    await $`cp ${videoPath} ${rawVideoPath}`.quiet();
    console.log(`✓ Backed up original to raw/`);

    const {audioBuffer, outputPath, fileName} = await extractAudioForAPI(videoPath);
    console.log(`✓ Extracted audio: ${outputPath} (${audioBuffer.byteLength} bytes)`);

    console.log(`Enhancing audio with AI-coustics...`);
    const generatedName = await uploadAndEnhance(audioBuffer, fileName);
    console.log(`✓ Enhanced audio uploaded: ${generatedName}`);

    const enhancedOutputPath = outputPath.replace('.mp3', '_enhanced.mp3');
    await downloadEnhancedMedia(generatedName, enhancedOutputPath);

    await replaceAudioInVideo(rawVideoPath, enhancedOutputPath, videoPath);

    await $`rm ${outputPath}`.quiet();
    await $`rm ${enhancedOutputPath}`.quiet();
    console.log(`✓ Cleaned up temporary audio files`);
  } catch (error) {
    console.error(`✗ Failed to process ${file}:`, error);
  }
}

Code Hike integration

In a branch, we experimented with using Code snippets instead of videos as a source for the display.