Add audio, fix timing

src/audio.rs

@@ -0,0 +1,101 @@
+use anyhow::{bail, Result};
+use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
+use itertools::Itertools;
+use rubato::{FftFixedInOut, Resampler};
+
+pub struct Audio {
+    #[allow(unused)]
+    stream: cpal::Stream,
+    sampler: FftFixedInOut<f32>,
+    input_buffer: Vec<Vec<f32>>,
+    output_buffer: Vec<Vec<f32>>,
+    sample_sink: rtrb::Producer<f32>,
+}
+
+impl Audio {
+    pub fn init() -> Result<Self> {
+        let host = cpal::default_host();
+        let Some(device) = host.default_output_device() else {
+            bail!("No output device available");
+        };
+        let Some(config) = device
+            .supported_output_configs()?
+            .find(|c| c.channels() == 2 && c.sample_format().is_float())
+        else {
+            bail!("No suitable output config available");
+        };
+        let mut config = config.with_max_sample_rate().config();
+        let sampler = FftFixedInOut::new(41700, config.sample_rate.0 as usize, 834, 2)?;
+        config.buffer_size = cpal::BufferSize::Fixed(sampler.output_frames_max() as u32);
+
+        let input_buffer = sampler.input_buffer_allocate(true);
+        let output_buffer = sampler.output_buffer_allocate(true);
+        let (sample_sink, mut sample_source) =
+            rtrb::RingBuffer::new(sampler.output_frames_max() * 4);
+
+        let stream = device.build_output_stream(
+            &config,
+            move |data: &mut [f32], _| {
+                let requested = data.len();
+                let chunk = match sample_source.read_chunk(data.len()) {
+                    Ok(c) => c,
+                    Err(rtrb::chunks::ChunkError::TooFewSlots(n)) => {
+                        sample_source.read_chunk(n).unwrap()
+                    }
+                };
+                let len = chunk.len();
+                let (first, second) = chunk.as_slices();
+                data[0..first.len()].copy_from_slice(first);
+                data[first.len()..len].copy_from_slice(second);
+                for rest in &mut data[len..requested] {
+                    *rest = 0.0;
+                }
+                chunk.commit_all();
+            },
+            move |err| eprintln!("stream error: {err}"),
+            None,
+        )?;
+        stream.play()?;
+        Ok(Self {
+            stream,
+            sampler,
+            input_buffer,
+            output_buffer,
+            sample_sink,
+        })
+    }
+
+    pub fn update(&mut self, samples: &[f32]) {
+        for sample in samples.chunks_exact(2) {
+            for (channel, value) in self.input_buffer.iter_mut().zip(sample) {
+                channel.push(*value);
+            }
+            if self.input_buffer[0].len() >= self.sampler.input_frames_next() {
+                let (_, output_samples) = self
+                    .sampler
+                    .process_into_buffer(&self.input_buffer, &mut self.output_buffer, None)
+                    .unwrap();
+
+                let chunk = match self.sample_sink.write_chunk_uninit(output_samples * 2) {
+                    Ok(c) => c,
+                    Err(rtrb::chunks::ChunkError::TooFewSlots(n)) => {
+                        self.sample_sink.write_chunk_uninit(n).unwrap()
+                    }
+                };
+                let interleaved = self.output_buffer[0]
+                    .iter()
+                    .interleave(self.output_buffer[1].iter())
+                    .cloned();
+                chunk.fill_from_iter(interleaved);
+
+                for channel in &mut self.input_buffer {
+                    channel.clear();
+                }
+            }
+        }
+
+        while self.sample_sink.slots() < self.sampler.output_frames_max() * 2 {
+            std::hint::spin_loop();
+        }
+    }
+}

src/emulator.rs

@@ -1,29 +1,60 @@
 use std::{
     fs,
-    path::Path,
+    path::{Path, PathBuf},
     sync::mpsc::{self, TryRecvError},
 };
 
 use anyhow::Result;
 
-use crate::{renderer::GameRenderer, shrooms_vb_core::CoreVB};
+use crate::{audio::Audio, renderer::GameRenderer, shrooms_vb_core::CoreVB};
+
+pub struct EmulatorBuilder {
+    rom: Option<PathBuf>,
+    commands: mpsc::Receiver<EmulatorCommand>,
+}
+
+impl EmulatorBuilder {
+    pub fn new() -> (Self, EmulatorClient) {
+        let (queue, commands) = mpsc::channel();
+        let builder = Self {
+            rom: None,
+            commands,
+        };
+        let client = EmulatorClient { queue };
+        (builder, client)
+    }
+
+    pub fn with_rom(self, path: &Path) -> Self {
+        Self {
+            rom: Some(path.into()),
+            ..self
+        }
+    }
+
+    pub fn build(self) -> Result<Emulator> {
+        let mut emulator = Emulator::new(self.commands)?;
+        if let Some(path) = self.rom {
+            emulator.load_rom(&path)?;
+        }
+        Ok(emulator)
+    }
+}
 
 pub struct Emulator {
     sim: CoreVB,
+    audio: Audio,
     commands: mpsc::Receiver<EmulatorCommand>,
     renderer: Option<GameRenderer>,
 }
 
 impl Emulator {
-    pub fn new() -> (Self, EmulatorClient) {
-        let (sink, source) = mpsc::channel();
-        let emu = Emulator {
+    fn new(commands: mpsc::Receiver<EmulatorCommand>) -> Result<Self> {
+        Ok(Self {
             sim: CoreVB::new(),
-            commands: source,
+            audio: Audio::init()?,
+            commands,
             renderer: None,
-        };
-        let queue = EmulatorClient { queue: sink };
-        (emu, queue)
+        })
     }
 
     pub fn load_rom(&mut self, path: &Path) -> Result<()> {
@@ -34,6 +65,7 @@ impl Emulator {
 
     pub fn run(&mut self) {
         let mut eye_contents = vec![0u8; 384 * 224 * 2];
+        let mut audio_samples = vec![];
         loop {
             self.sim.emulate_frame();
             if let Some(renderer) = &mut self.renderer {
@@ -41,6 +73,11 @@ impl Emulator {
                     renderer.render(&eye_contents);
                 }
             }
+            self.sim.read_samples(&mut audio_samples);
+            if !audio_samples.is_empty() {
+                self.audio.update(&audio_samples);
+                audio_samples.clear();
+            }
             loop {
                 match self.commands.try_recv() {
                     Ok(command) => self.handle_command(command),

src/main.rs

@@ -3,10 +3,11 @@ use std::{path::PathBuf, thread};
 use anyhow::Result;
 use app::App;
 use clap::Parser;
-use emulator::Emulator;
+use emulator::EmulatorBuilder;
 use winit::event_loop::{ControlFlow, EventLoop};
 
 mod app;
+mod audio;
 mod emulator;
 mod renderer;
 mod shrooms_vb_core;
@@ -19,9 +20,10 @@ struct Args {
 fn main() -> Result<()> {
     let args = Args::parse();
 
-    let (mut emulator, client) = Emulator::new();
-    emulator.load_rom(&args.rom)?;
+    let (builder, client) = EmulatorBuilder::new();
+    let builder = builder.with_rom(&args.rom);
     thread::spawn(move || {
+        let mut emulator = builder.build().unwrap();
         emulator.run();
     });
 

src/shrooms_vb_core.rs

@@ -1,6 +1,7 @@
-use std::{ffi::c_void, ptr};
+use std::{ffi::c_void, ptr, slice};
 
 use anyhow::{anyhow, Result};
+use num_derive::{FromPrimitive, ToPrimitive};
 
 #[repr(C)]
 struct VB {
@@ -9,6 +10,20 @@ struct VB {
 
 #[allow(non_camel_case_types)]
 type c_int = i32;
+#[allow(non_camel_case_types)]
+type c_uint = u32;
+
+#[repr(u32)]
+#[derive(FromPrimitive, ToPrimitive)]
+enum VBDataType {
+    S8 = 0,
+    U8 = 1,
+    S16 = 2,
+    U16 = 3,
+    S32 = 4,
+    F32 = 5,
+}
+
 type OnFrame = extern "C" fn(sim: *mut VB) -> c_int;
 
 #[link(name = "vb")]
@@ -27,6 +42,13 @@ extern "C" {
         right_stride_x: c_int,
         right_stride_y: c_int,
     );
+    #[link_name = "vbGetSamples"]
+    fn vb_get_samples(
+        sim: *mut VB,
+        typ_: *mut VBDataType,
+        capacity: *mut c_uint,
+        position: *mut c_uint,
+    ) -> *mut c_void;
     #[link_name = "vbGetUserData"]
     fn vb_get_user_data(sim: *mut VB) -> *mut c_void;
     #[link_name = "vbInit"]
@@ -39,6 +61,13 @@ extern "C" {
     fn vb_set_keys(sim: *mut VB, keys: u16) -> u16;
     #[link_name = "vbSetFrameCallback"]
     fn vb_set_frame_callback(sim: *mut VB, on_frame: OnFrame);
+    #[link_name = "vbSetSamples"]
+    fn vb_set_samples(
+        sim: *mut VB,
+        samples: *mut c_void,
+        typ_: VBDataType,
+        capacity: c_uint,
+    ) -> c_int;
     #[link_name = "vbSetUserData"]
     fn vb_set_user_data(sim: *mut VB, tag: *mut c_void);
     #[link_name = "vbSizeOf"]
@@ -53,6 +82,9 @@ extern "C" fn on_frame(sim: *mut VB) -> i32 {
     1
 }
 
+const AUDIO_CAPACITY_SAMPLES: usize = 834 * 4;
+const AUDIO_CAPACITY_FLOATS: usize = AUDIO_CAPACITY_SAMPLES * 2;
+
 struct VBState {
     frame_seen: bool,
 }
@@ -79,6 +111,11 @@ impl CoreVB {
         unsafe { vb_set_user_data(sim, Box::into_raw(Box::new(state)).cast()) };
         unsafe { vb_set_frame_callback(sim, on_frame) };
 
+        // set up audio buffer
+        let audio_buffer = vec![0.0f32; AUDIO_CAPACITY_FLOATS];
+        let samples: *mut c_void = Box::into_raw(audio_buffer.into_boxed_slice()).cast();
+        unsafe { vb_set_samples(sim, samples, VBDataType::F32, AUDIO_CAPACITY_SAMPLES as u32) };
+
         CoreVB { sim }
     }
 
@@ -138,6 +175,25 @@ impl CoreVB {
         true
     }
 
+    pub fn read_samples(&mut self, samples: &mut Vec<f32>) {
+        let mut position = 0;
+        let ptr =
+            unsafe { vb_get_samples(self.sim, ptr::null_mut(), ptr::null_mut(), &mut position) };
+        // SAFETY: position is an offset in a buffer of (f32, f32). so, position * 2 is an offset in a buffer of f32.
+        let read_samples: &mut [f32] =
+            unsafe { slice::from_raw_parts_mut(ptr.cast(), position as usize * 2) };
+        samples.extend_from_slice(read_samples);
+
+        unsafe {
+            vb_set_samples(
+                self.sim,
+                ptr,
+                VBDataType::F32,
+                AUDIO_CAPACITY_SAMPLES as u32,
+            )
+        };
+    }
+
     pub fn set_keys(&mut self, keys: u16) {
         unsafe { vb_set_keys(self.sim, keys) };
     }
@@ -145,6 +201,14 @@ impl CoreVB {
 
 impl Drop for CoreVB {
     fn drop(&mut self) {
+        let ptr =
+            unsafe { vb_get_samples(self.sim, ptr::null_mut(), ptr::null_mut(), ptr::null_mut()) };
+        // SAFETY: the audio buffer originally came from a Vec<u32>
+        let floats: Vec<f32> = unsafe {
+            Vec::from_raw_parts(ptr.cast(), AUDIO_CAPACITY_FLOATS, AUDIO_CAPACITY_FLOATS)
+        };
+        drop(floats);
+
         // SAFETY: the *mut VB owns its userdata.
         // There is no way for the userdata to be null or otherwise invalid.
         let ptr: *mut VBState = unsafe { vb_get_user_data(self.sim).cast() };