Merge branch 'frame'

This commit is contained in:
Hailey Somerville 2023-12-29 20:17:29 +11:00
commit 5ea6a9ef6c
15 changed files with 112 additions and 81 deletions

4
Cargo.lock generated
View file

@ -235,9 +235,9 @@ checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1"
[[package]]
name = "bytemuck"
version = "1.13.1"
version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea"
checksum = "374d28ec25809ee0e23827c2ab573d729e293f281dfe393500e7ad618baa61c6"
dependencies = [
"bytemuck_derive",
]

View file

@ -11,7 +11,7 @@ bark-core = { path = "bark-core" }
bark-protocol = { path = "bark-protocol" }
bitflags = { version = "2.4", features = ["bytemuck"] }
bytemuck = { version = "1.13", features = ["derive"] }
bytemuck = { version = "1.14", features = ["derive", "must_cast"] }
derive_more = { version = "0.99" }
heapless = "0.7"
log = "0.4"

19
bark-core/src/audio.rs Normal file
View file

@ -0,0 +1,19 @@
use bytemuck::{Pod, Zeroable};
pub type Sample = f32;
#[derive(Pod, Zeroable, Copy, Clone, Debug)]
#[repr(C)]
pub struct Frame(pub Sample, pub Sample);
#[derive(Copy, Clone, Debug)]
#[repr(transparent)]
pub struct FrameCount(pub usize);
pub fn as_interleaved(frames: &[Frame]) -> &[Sample] {
bytemuck::must_cast_slice(frames)
}
pub fn as_interleaved_mut(frames: &mut [Frame]) -> &mut [Sample] {
bytemuck::must_cast_slice_mut(frames)
}

View file

@ -3,11 +3,13 @@ pub mod pcm;
use core::fmt::Display;
use bark_protocol::packet::Audio;
use thiserror::Error;
use bark_protocol::FRAMES_PER_PACKET;
use bark_protocol::packet::Audio;
use bark_protocol::types::{AudioPacketHeader, AudioPacketFormat};
use bark_protocol::SAMPLES_PER_PACKET;
use crate::audio::Frame;
#[derive(Debug, Error)]
pub enum NewDecoderError {
@ -19,8 +21,10 @@ pub enum NewDecoderError {
#[derive(Debug, Error)]
pub enum DecodeError {
#[error("wrong length: {length}, expected: {expected}")]
#[error("wrong byte length: {length}, expected: {expected}")]
WrongLength { length: usize, expected: usize },
#[error("wrong frame count: {frames}, expected: {expected}")]
WrongFrameCount { frames: usize, expected: usize },
#[error("opus codec error: {0}")]
Opus(#[from] ::opus::Error),
}
@ -29,7 +33,7 @@ pub struct Decoder {
decode: DecodeFormat,
}
pub type SampleBuffer = [f32; SAMPLES_PER_PACKET];
pub type FrameBuffer = [Frame; FRAMES_PER_PACKET];
impl Decoder {
pub fn new(header: &AudioPacketHeader) -> Result<Self, NewDecoderError> {
@ -47,14 +51,14 @@ impl Decoder {
&self.decode as &dyn Display
}
pub fn decode(&mut self, packet: Option<&Audio>, out: &mut SampleBuffer) -> Result<(), DecodeError> {
pub fn decode(&mut self, packet: Option<&Audio>, out: &mut FrameBuffer) -> Result<(), DecodeError> {
let bytes = packet.map(|packet| packet.buffer_bytes());
self.decode.decode_packet(bytes, out)
}
}
trait Decode: Display {
fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut SampleBuffer) -> Result<(), DecodeError>;
fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut FrameBuffer) -> Result<(), DecodeError>;
}
enum DecodeFormat {
@ -64,7 +68,7 @@ enum DecodeFormat {
}
impl Decode for DecodeFormat {
fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut SampleBuffer) -> Result<(), DecodeError> {
fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut FrameBuffer) -> Result<(), DecodeError> {
match self {
DecodeFormat::S16LE(dec) => dec.decode_packet(bytes, out),
DecodeFormat::F32LE(dec) => dec.decode_packet(bytes, out),

View file

@ -2,7 +2,9 @@ use core::fmt::{self, Display};
use bark_protocol::SAMPLE_RATE;
use super::{Decode, DecodeError, SampleBuffer};
use crate::audio;
use super::{Decode, DecodeError, FrameBuffer};
pub struct OpusDecoder {
opus: opus::Decoder,
@ -26,16 +28,16 @@ impl Display for OpusDecoder {
}
impl Decode for OpusDecoder {
fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut SampleBuffer) -> Result<(), DecodeError> {
let expected = out.len() / 2;
fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut FrameBuffer) -> Result<(), DecodeError> {
let expected = out.len();
let length = match bytes {
Some(bytes) => self.opus.decode_float(bytes, out, false)?,
None => self.opus.decode_float(&[], out, true)?,
let frames = match bytes {
Some(bytes) => self.opus.decode_float(bytes, audio::as_interleaved_mut(out), false)?,
None => self.opus.decode_float(&[], audio::as_interleaved_mut(out), true)?,
};
if expected != length {
return Err(DecodeError::WrongLength { length, expected });
if expected != frames {
return Err(DecodeError::WrongFrameCount { frames, expected });
}
Ok(())

View file

@ -1,6 +1,8 @@
use core::fmt::{self, Display};
use super::{Decode, DecodeError, SampleBuffer};
use crate::audio;
use super::{Decode, DecodeError, FrameBuffer};
pub struct S16LEDecoder;
@ -11,7 +13,7 @@ impl Display for S16LEDecoder {
}
impl Decode for S16LEDecoder {
fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut SampleBuffer) -> Result<(), DecodeError> {
fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut FrameBuffer) -> Result<(), DecodeError> {
decode_packed(bytes, out, |bytes| {
let input = i16::from_le_bytes(bytes);
let scale = i16::MAX as f32;
@ -29,26 +31,28 @@ impl Display for F32LEDecoder {
}
impl Decode for F32LEDecoder {
fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut SampleBuffer) -> Result<(), DecodeError> {
fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut FrameBuffer) -> Result<(), DecodeError> {
decode_packed(bytes, out, f32::from_le_bytes)
}
}
fn decode_packed<const N: usize>(
bytes: Option<&[u8]>,
out: &mut SampleBuffer,
out: &mut FrameBuffer,
func: impl Fn([u8; N]) -> f32,
) -> Result<(), DecodeError> {
let out_samples = audio::as_interleaved_mut(out);
let Some(bytes) = bytes else {
// PCM codecs have no packet loss correction
// just zero fill and return
out.fill(0.0);
out_samples.fill(0.0);
return Ok(());
};
check_length(bytes, out.len() * N)?;
check_length(bytes, out_samples.len() * N)?;
for (input, output) in bytes.chunks_exact(N).zip(out) {
for (input, output) in bytes.chunks_exact(N).zip(out_samples) {
// when array_chunks stabilises we can use that instead
// but for now use try_into to turn a &[u8] (guaranteed len == width)
// into a [u8; width]

View file

@ -6,6 +6,8 @@ use core::fmt::Display;
use bark_protocol::types::AudioPacketFormat;
use thiserror::Error;
use crate::audio::Frame;
#[derive(Debug, Error)]
pub enum NewEncoderError {
#[error("opus codec error: {0}")]
@ -22,5 +24,5 @@ pub enum EncodeError {
pub trait Encode: Display + Send {
fn header_format(&self) -> AudioPacketFormat;
fn encode_packet(&mut self, samples: &[f32], out: &mut [u8]) -> Result<usize, EncodeError>;
fn encode_packet(&mut self, frames: &[Frame], out: &mut [u8]) -> Result<usize, EncodeError>;
}

View file

@ -2,6 +2,8 @@ use core::fmt::{self, Display};
use bark_protocol::{types::AudioPacketFormat, SAMPLE_RATE};
use crate::audio::{Frame, self};
use super::{Encode, EncodeError, NewEncoderError};
pub struct OpusEncoder {
@ -35,7 +37,7 @@ impl Encode for OpusEncoder {
AudioPacketFormat::OPUS
}
fn encode_packet(&mut self, samples: &[f32], out: &mut [u8]) -> Result<usize, EncodeError> {
Ok(self.opus.encode_float(samples, out)?)
fn encode_packet(&mut self, samples: &[Frame], out: &mut [u8]) -> Result<usize, EncodeError> {
Ok(self.opus.encode_float(audio::as_interleaved(samples), out)?)
}
}

View file

@ -2,6 +2,8 @@ use core::fmt::{self, Display};
use bark_protocol::types::AudioPacketFormat;
use crate::audio::{Frame, self};
use super::{Encode, EncodeError};
pub struct S16LEEncoder;
@ -17,8 +19,8 @@ impl Encode for S16LEEncoder {
AudioPacketFormat::S16LE
}
fn encode_packet(&mut self, samples: &[f32], out: &mut [u8]) -> Result<usize, EncodeError> {
encode_packed(samples, out, |sample| {
fn encode_packet(&mut self, frames: &[Frame], out: &mut [u8]) -> Result<usize, EncodeError> {
encode_packed(frames, out, |sample| {
let scale = i16::MAX as f32;
let sample = sample.clamp(-1.0, 1.0) * scale;
i16::to_le_bytes(sample as i16)
@ -39,16 +41,17 @@ impl Encode for F32LEEncoder {
AudioPacketFormat::F32LE
}
fn encode_packet(&mut self, samples: &[f32], out: &mut [u8]) -> Result<usize, EncodeError> {
encode_packed(samples, out, f32::to_le_bytes)
fn encode_packet(&mut self, frames: &[Frame], out: &mut [u8]) -> Result<usize, EncodeError> {
encode_packed(frames, out, f32::to_le_bytes)
}
}
fn encode_packed<const N: usize>(
samples: &[f32],
frames: &[Frame],
out: &mut [u8],
func: impl Fn(f32) -> [u8; N],
) -> Result<usize, EncodeError> {
let samples = audio::as_interleaved(frames);
let out = check_length(out, samples.len() * N)?;
for (output, input) in out.chunks_exact_mut(N).zip(samples) {

View file

@ -1,3 +1,4 @@
pub mod audio;
pub mod consts;
pub mod decode;
pub mod encode;

View file

@ -2,7 +2,7 @@ use std::ffi::{c_void, c_int, CStr};
use std::fmt::Debug;
use std::ptr;
use bark_protocol::time::SampleDuration;
use crate::audio::{Frame, FrameCount};
use self::ffi::speex_resampler_strerror;
@ -46,10 +46,8 @@ pub struct Resampler {
unsafe impl Send for Resampler {}
pub struct ProcessResult {
/// per-channel
pub input_read: SampleDuration,
/// per-channel
pub output_written: SampleDuration,
pub input_read: FrameCount,
pub output_written: FrameCount,
}
impl Resampler {
@ -93,20 +91,16 @@ impl Resampler {
Ok(())
}
pub fn process_interleaved(&mut self, input: &[f32], output: &mut [f32])
pub fn process(&mut self, input: &[Frame], output: &mut [Frame])
-> Result<ProcessResult, SpeexError>
{
// speex API takes frame count:
let input_len = input.len() / usize::from(bark_protocol::CHANNELS);
let output_len = output.len() / usize::from(bark_protocol::CHANNELS);
// usize could technically be 64 bit, speex only takes u32 sizes,
// we don't want to panic or truncate, so let's just pick a reasonable
// length and cap input and output since the API allows us to.
// i'm going to say a reasonable length for a single call is 1<<20.
let max_reasonable_len = 1 << 20;
let input_len = std::cmp::min(input_len, max_reasonable_len);
let output_len = std::cmp::min(output_len, max_reasonable_len);
let input_len = std::cmp::min(input.len(), max_reasonable_len);
let output_len = std::cmp::min(output.len(), max_reasonable_len);
let mut input_len = u32::try_from(input_len).unwrap();
let mut output_len = u32::try_from(output_len).unwrap();
@ -114,9 +108,10 @@ impl Resampler {
let err = unsafe {
ffi::speex_resampler_process_interleaved_float(
self.ptr.0,
input.as_ptr(),
input.as_ptr().cast(),
// speex API takes frame count already:
&mut input_len,
output.as_mut_ptr(),
output.as_mut_ptr().cast(),
&mut output_len,
)
};
@ -126,8 +121,8 @@ impl Resampler {
}
Ok(ProcessResult {
input_read: SampleDuration::from_frame_count(u64::from(input_len)),
output_written: SampleDuration::from_frame_count(u64::from(output_len)),
input_read: FrameCount(usize::try_from(input_len).unwrap()),
output_written: FrameCount(usize::try_from(output_len).unwrap()),
})
}
}

View file

@ -1,7 +1,7 @@
use alsa::Direction;
use alsa::pcm::PCM;
use bark_protocol::{CHANNELS, time::Timestamp};
use bark_protocol::time::SampleDuration;
use bark_core::audio::{Frame, self};
use bark_protocol::time::{Timestamp, SampleDuration};
use nix::errno::Errno;
use thiserror::Error;
@ -24,7 +24,7 @@ impl Input {
Ok(Input { pcm })
}
pub fn read(&self, mut audio: &mut [f32]) -> Result<Timestamp, ReadAudioError> {
pub fn read(&self, mut audio: &mut [Frame]) -> Result<Timestamp, ReadAudioError> {
let now = Timestamp::from_micros_lossy(time::now());
let timestamp = now.saturating_sub(self.delay()?);
@ -36,7 +36,7 @@ impl Input {
Ok(timestamp)
}
fn read_partial(&self, audio: &mut [f32]) -> Result<usize, ReadAudioError> {
fn read_partial(&self, audio: &mut [Frame]) -> Result<usize, ReadAudioError> {
let io = unsafe {
// the checked versions of this function call
// snd_pcm_hw_params_current which mallocs under the hood
@ -45,10 +45,8 @@ impl Input {
loop {
// try to write audio
let err = match io.readi(audio) {
Ok(n) => {
return Ok(n * CHANNELS.0 as usize);
}
let err = match io.readi(audio::as_interleaved_mut(audio)) {
Ok(n) => { return Ok(n) }
Err(e) => e,
};

View file

@ -1,6 +1,6 @@
use alsa::Direction;
use alsa::pcm::PCM;
use bark_protocol::CHANNELS;
use bark_core::audio::{Frame, self};
use bark_protocol::time::SampleDuration;
use nix::errno::Errno;
use thiserror::Error;
@ -23,7 +23,7 @@ impl Output {
Ok(Output { pcm })
}
pub fn write(&self, mut audio: &[f32]) -> Result<(), WriteAudioError> {
pub fn write(&self, mut audio: &[Frame]) -> Result<(), WriteAudioError> {
while audio.len() > 0 {
let n = self.write_partial(audio)?;
audio = &audio[n..];
@ -32,7 +32,7 @@ impl Output {
Ok(())
}
fn write_partial(&self, audio: &[f32]) -> Result<usize, WriteAudioError> {
fn write_partial(&self, audio: &[Frame]) -> Result<usize, WriteAudioError> {
let io = unsafe {
// the checked versions of this function call
// snd_pcm_hw_params_current which mallocs under the hood
@ -41,10 +41,8 @@ impl Output {
loop {
// try to write audio
let err = match io.writei(audio) {
Ok(n) => {
return Ok(n * CHANNELS.0 as usize);
}
let err = match io.writei(audio::as_interleaved(audio)) {
Ok(n) => { return Ok(n) },
Err(e) => e,
};

View file

@ -2,14 +2,15 @@ use std::array;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use bark_core::decode::{Decoder, SampleBuffer};
use bark_core::audio::Frame;
use bark_core::decode::{Decoder, FrameBuffer};
use bytemuck::Zeroable;
use structopt::StructOpt;
use bark_core::receive::queue::PacketQueue;
use bark_core::receive::resample::Resampler;
use bark_protocol::{SampleRate, SAMPLES_PER_PACKET, FRAMES_PER_PACKET};
use bark_protocol::{SampleRate, FRAMES_PER_PACKET};
use bark_protocol::time::{Timestamp, SampleDuration, TimestampDelta, ClockDelta};
use bark_protocol::types::{SessionId, ReceiverId, TimePhase, AudioPacketHeader};
use bark_protocol::types::stats::receiver::{ReceiverStats, StreamStatus};
@ -163,12 +164,12 @@ impl Receiver {
}
}
pub fn write_audio(&mut self, buffer: &mut [f32], pts: Timestamp) -> SampleDuration {
pub fn write_audio(&mut self, buffer: &mut [Frame], pts: Timestamp) -> usize {
// get stream start timing information:
let Some(stream) = self.stream.as_mut() else {
// stream hasn't started, just fill buffer with silence and return
buffer[0..SAMPLES_PER_PACKET].fill(0f32);
return SampleDuration::ONE_PACKET;
buffer[0..FRAMES_PER_PACKET].fill(Frame::zeroed());
return FRAMES_PER_PACKET;
};
// get next packet from queue, or None if missing (packet loss)
@ -203,29 +204,29 @@ impl Receiver {
}
// decode packet
let mut decode_buffer: SampleBuffer = array::from_fn(|_| 0.0);
let mut decode_buffer: FrameBuffer = array::from_fn(|_| Frame::zeroed());
if let Some(decoder) = stream.decoder.as_mut() {
match decoder.decode(packet.as_ref(), &mut decode_buffer) {
Ok(()) => {}
Err(e) => {
log::warn!("error in decoder, skipping packet: {e}");
decode_buffer.fill(0.0);
decode_buffer.fill(Frame::zeroed());
}
}
}
// resample decoded audio
let resample = stream.resampler.process_interleaved(&decode_buffer, buffer)
let resample = stream.resampler.process(&decode_buffer, buffer)
.expect("resample error!");
assert_eq!(resample.input_read.as_buffer_offset(), decode_buffer.len());
assert_eq!(resample.input_read.0, decode_buffer.len());
// report stats and return
self.stats.set_buffer_length(
SampleDuration::from_frame_count(
(FRAMES_PER_PACKET * stream.queue.len()).try_into().unwrap()));
resample.output_written
resample.output_written.0
}
}
@ -381,14 +382,14 @@ pub fn run(opt: ReceiveOpt) -> Result<(), RunError> {
// this should be large enough for `write_audio` to process an
// entire packet with:
let mut buffer = [0f32; SAMPLES_PER_PACKET * 2];
let duration = state.recv.write_audio(&mut buffer, pts);
let mut buffer = [Frame::zeroed(); FRAMES_PER_PACKET * 2];
let count = state.recv.write_audio(&mut buffer, pts);
// drop lock before calling `Output::write` (blocking!)
drop(state);
// send audio to ALSA
match output.write(&buffer[0..duration.as_buffer_offset()]) {
match output.write(&buffer[0..count]) {
Ok(()) => {}
Err(e) => {
log::error!("error playing audio: {e}");

View file

@ -1,10 +1,12 @@
use std::sync::Arc;
use std::time::Duration;
use bark_core::audio::Frame;
use bark_core::encode::Encode;
use bark_core::encode::opus::OpusEncoder;
use bark_core::encode::pcm::{S16LEEncoder, F32LEEncoder};
use bark_protocol::SAMPLES_PER_PACKET;
use bark_protocol::FRAMES_PER_PACKET;
use bytemuck::Zeroable;
use structopt::StructOpt;
use bark_protocol::time::SampleDuration;
@ -92,10 +94,10 @@ pub fn run(opt: StreamOpt) -> Result<(), RunError> {
crate::thread::set_name("bark/audio");
loop {
let mut sample_buffer = [0f32; SAMPLES_PER_PACKET];
let mut audio_buffer = [Frame::zeroed(); FRAMES_PER_PACKET];
// read audio input
let timestamp = match input.read(&mut sample_buffer) {
let timestamp = match input.read(&mut audio_buffer) {
Ok(ts) => ts,
Err(e) => {
log::error!("error reading audio input: {e}");
@ -105,7 +107,7 @@ pub fn run(opt: StreamOpt) -> Result<(), RunError> {
// encode audio
let mut encode_buffer = [0; Audio::MAX_BUFFER_LENGTH];
let encoded_data = match encoder.encode_packet(&sample_buffer, &mut encode_buffer) {
let encoded_data = match encoder.encode_packet(&audio_buffer, &mut encode_buffer) {
Ok(size) => &encode_buffer[0..size],
Err(e) => {
log::error!("error encoding audio: {e}");