Audio Processing

🔊 Audio capture, playback, and sample processing More...

Files
file	audio.c
	🔊 Client audio management: capture thread, sample processing, and playback coordination
file	audio.h
	ascii-chat Client Audio Processing Management Interface

Functions
void	audio_process_received_samples (const float *samples, int num_samples)
	Process received audio samples from server.
int	audio_client_init (void)
	Initialize audio subsystem.
int	audio_start_thread (void)
	Start audio capture thread.
void	audio_stop_thread (void)
	Stop audio capture thread.
bool	audio_thread_exited (void)
	Check if audio capture thread has exited.
void	audio_cleanup (void)
	Cleanup audio subsystem.
client_audio_pipeline_t *	audio_get_pipeline (void)
	Get the audio pipeline (for advanced usage)
int	audio_decode_opus (const uint8_t *opus_data, size_t opus_len, float *output, int max_samples)
	Decode Opus packet using the audio pipeline.

Detailed Description

🔊 Audio capture, playback, and sample processing

Audio Processing

Overview

The client audio subsystem manages PortAudio initialization, audio capture from microphone, transmission to server, and playback of received audio samples with jitter buffering.

Implementation: src/client/audio.c, src/client/audio.h

Configuration

Audio Parameters:

• Sample rate: 44100 Hz
• Channels: 1 (mono)
• Format: 32-bit float
• Batch size: 256 samples (~5.8ms @ 44.1kHz)
• Ring buffer: 8192 samples (~185ms jitter buffer)

Audio Capture Thread

static void *audio_capture_thread_func(void *arg)
{
  (void)arg;
 
  float capture_buffer[AUDIO_BATCH_SIZE];
 
  while (!should_exit() && server_connection_is_active()) {
    // Capture audio samples from microphone
    int samples_captured = audio_capture_samples(capture_buffer, AUDIO_BATCH_SIZE);
 
    if (samples_captured > 0) {
      // Send to server
      send_packet_to_server(PACKET_TYPE_AUDIO, capture_buffer,
                           samples_captured * sizeof(float),
                           server_connection_get_client_id());
    }
 
    // Small sleep to avoid busy loop
    platform_sleep_usec(1000);  // 1ms
  }
 
  atomic_store(&g_audio_thread_exited, true);
  return NULL;
}

Audio Playback

Jitter Buffering

void audio_process_received_samples(const float *samples, int num_samples)
{
  // Write to ring buffer for jitter compensation
  audio_ringbuffer_write(&g_playback_ringbuffer, samples, num_samples);
 
  // PortAudio callback reads from ring buffer
}
 
static int audio_playback_callback(const void *input, void *output,
                                   unsigned long frameCount,
                                   const PaStreamCallbackTimeInfo *timeInfo,
                                   PaStreamCallbackFlags statusFlags,
                                   void *userData)
{
  (void)input; (void)timeInfo; (void)statusFlags; (void)userData;
 
  float *out = (float *)output;
 
  // Read from ring buffer
  int samples_read = audio_ringbuffer_read(&g_playback_ringbuffer, out, frameCount);
 
  // Fill remaining with silence if underrun
  if (samples_read < (int)frameCount) {
    memset(&out[samples_read], 0, (frameCount - samples_read) * sizeof(float));
  }
 
  return paContinue;
}

See also

src/client/audio.c

src/client/audio.h

lib/audio.h

Client Overview

Function Documentation

audio_cleanup()

void audio_cleanup

(

)

#include <audio.c>

Cleanup audio subsystem.

Cleanup audio subsystem

Stops audio threads and cleans up PortAudio resources. Called during client shutdown.

Definition at line 899 of file src/client/audio.c.

                     {
  if (!GET_OPTION(audio_enabled)) {
    return;
  }
 
  // Stop capture thread first (stops producing packets)
  audio_stop_thread();
 
  // Stop async sender thread (drains queue and exits)
  audio_sender_cleanup();
 
  // CRITICAL: Stop audio stream BEFORE destroying pipeline to prevent race condition
  // PortAudio may invoke the callback one more time after we request stop.
  // We need to clear the pipeline pointer first so the callback can't access freed memory.
  if (g_audio_context.initialized) {
    audio_stop_duplex(&g_audio_context);
  }
 
  // Clear the pipeline pointer from audio context BEFORE destroying pipeline
  // This prevents any lingering PortAudio callbacks from trying to access freed memory
  audio_set_pipeline(&g_audio_context, NULL);
 
  // CRITICAL: Sleep to allow CoreAudio threads to finish executing callbacks
  // On macOS, CoreAudio's internal threads may continue running after Pa_StopStream() returns.
  // The duplex_callback may still be in-flight on other threads. Even after we set the pipeline
  // pointer to NULL, a CoreAudio thread may have already cached the pointer before the assignment.
  // This sleep ensures all in-flight callbacks have fully completed before we destroy the pipeline.
  // 500ms is sufficient on macOS for CoreAudio's internal thread pool to completely wind down.
  platform_sleep_usec(500000); // 500ms - macOS CoreAudio needs time to shut down all threads
 
  // Destroy audio pipeline (handles Opus, AEC, etc.)
  if (g_audio_pipeline) {
    client_audio_pipeline_destroy(g_audio_pipeline);
    g_audio_pipeline = NULL;
    log_info("Audio pipeline destroyed");
  }
 
  // Close WAV dumpers
  if (g_wav_capture_raw) {
    wav_writer_close(g_wav_capture_raw);
    g_wav_capture_raw = NULL;
    log_info("Closed audio capture raw dump");
  }
  if (g_wav_capture_processed) {
    wav_writer_close(g_wav_capture_processed);
    g_wav_capture_processed = NULL;
    log_info("Closed audio capture processed dump");
  }
  if (g_wav_playback_received) {
    wav_writer_close(g_wav_playback_received);
    g_wav_playback_received = NULL;
    log_info("Closed audio playback received dump");
  }
 
  // Finally destroy the audio context
  if (g_audio_context.initialized) {
    audio_destroy(&g_audio_context);
  }
}

References audio_destroy(), audio_set_pipeline(), audio_stop_duplex(), audio_stop_thread(), client_audio_pipeline_destroy(), GET_OPTION, audio_context_t::initialized, log_info, platform_sleep_usec(), and wav_writer_close().

audio_client_init()

int audio_client_init

(

)

#include <audio.c>

Initialize audio subsystem.

Initialize audio subsystem

Sets up PortAudio context, creates the audio pipeline, and starts audio playback/capture if audio is enabled.

Returns

0 on success, negative on error

0 on success, negative on error

Definition at line 702 of file src/client/audio.c.

                        {
  if (!GET_OPTION(audio_enabled)) {
    return 0; // Audio disabled - not an error
  }
 
  // Initialize WAV dumper for received audio if debugging enabled
  if (wav_dump_enabled()) {
    g_wav_playback_received = wav_writer_open("/tmp/audio_playback_received.wav", AUDIO_SAMPLE_RATE, 1);
    if (g_wav_playback_received) {
      log_info("Audio debugging enabled: dumping received audio to /tmp/audio_playback_received.wav");
    }
  }
 
  // Initialize PortAudio context using library function
  if (audio_init(&g_audio_context) != ASCIICHAT_OK) {
    log_error("Failed to initialize audio system");
    // Clean up WAV writer if it was opened
    if (g_wav_playback_received) {
      wav_writer_close(g_wav_playback_received);
      g_wav_playback_received = NULL;
    }
    return -1;
  }
 
  // Create unified audio pipeline (handles AEC, AGC, noise suppression, Opus)
  client_audio_pipeline_config_t pipeline_config = client_audio_pipeline_default_config();
  pipeline_config.opus_bitrate = 128000; // 128 kbps AUDIO mode for music quality
 
  // Use FLAGS_MINIMAL but enable echo cancellation and jitter buffer
  // Noise suppression, AGC, VAD destroy music/non-voice audio, so keep them disabled
  // But AEC removes echo without destroying audio quality
  // Jitter buffer helps synchronize the AEC echo reference
  pipeline_config.flags.echo_cancel = true;     // ENABLE: removes echo
  pipeline_config.flags.jitter_buffer = true;   // ENABLE: needed for AEC sync
  pipeline_config.flags.noise_suppress = false; // DISABLED: destroys audio
  pipeline_config.flags.agc = false;            // DISABLED: destroys audio
  pipeline_config.flags.vad = false;            // DISABLED: destroys audio
  pipeline_config.flags.compressor = false;     // DISABLED: minimal processing
  pipeline_config.flags.noise_gate = false;     // DISABLED: minimal processing
  pipeline_config.flags.highpass = false;       // DISABLED: minimal processing
  pipeline_config.flags.lowpass = false;        // DISABLED: minimal processing
 
  // Set jitter buffer margin for smooth playback without excessive delay
  // 100ms is conservative - AEC3 will adapt to actual network delay automatically
  // We don't tune this; let the system adapt to its actual conditions
  pipeline_config.jitter_margin_ms = 100;
 
  g_audio_pipeline = client_audio_pipeline_create(&pipeline_config);
  if (!g_audio_pipeline) {
    log_error("Failed to create audio pipeline");
    audio_destroy(&g_audio_context);
    // Clean up WAV writer if it was opened
    if (g_wav_playback_received) {
      wav_writer_close(g_wav_playback_received);
      g_wav_playback_received = NULL;
    }
    return -1;
  }
 
  log_info("Audio pipeline created: %d Hz sample rate, %d bps bitrate", pipeline_config.sample_rate,
           pipeline_config.opus_bitrate);
 
  // Associate pipeline with audio context for echo cancellation
  // The audio output callback will feed playback samples directly to AEC3 from the speaker output,
  // ensuring proper timing synchronization (not from the decode path 50-100ms earlier)
  audio_set_pipeline(&g_audio_context, (void *)g_audio_pipeline);
 
  // Start full-duplex audio (simultaneous capture + playback for perfect AEC3 timing)
  if (audio_start_duplex(&g_audio_context) != ASCIICHAT_OK) {
    log_error("Failed to start full-duplex audio");
    client_audio_pipeline_destroy(g_audio_pipeline);
    g_audio_pipeline = NULL;
    audio_destroy(&g_audio_context);
    // Clean up WAV writer if it was opened
    if (g_wav_playback_received) {
      wav_writer_close(g_wav_playback_received);
      g_wav_playback_received = NULL;
    }
    return -1;
  }
 
  // Initialize async audio sender (decouples capture from network I/O)
  audio_sender_init();
 
  return 0;
}

References client_audio_pipeline_flags_t::agc, ASCIICHAT_OK, audio_destroy(), audio_init(), AUDIO_SAMPLE_RATE, audio_set_pipeline(), audio_start_duplex(), client_audio_pipeline_create(), client_audio_pipeline_default_config(), client_audio_pipeline_destroy(), client_audio_pipeline_flags_t::compressor, client_audio_pipeline_flags_t::echo_cancel, client_audio_pipeline_config_t::flags, GET_OPTION, client_audio_pipeline_flags_t::highpass, client_audio_pipeline_flags_t::jitter_buffer, client_audio_pipeline_config_t::jitter_margin_ms, log_error, log_info, client_audio_pipeline_flags_t::lowpass, client_audio_pipeline_flags_t::noise_gate, client_audio_pipeline_flags_t::noise_suppress, client_audio_pipeline_config_t::opus_bitrate, client_audio_pipeline_config_t::sample_rate, client_audio_pipeline_flags_t::vad, wav_dump_enabled(), wav_writer_close(), and wav_writer_open().

audio_decode_opus()

int audio_decode_opus	(	const uint8_t *	opus_data,
		size_t	opus_len,
		float *	output,
		int	max_samples
	)

#include <audio.c>

Decode Opus packet using the audio pipeline.

Parameters

opus_data	Opus packet data
opus_len	Opus packet length
output	Output buffer for decoded samples
max_samples	Maximum samples output buffer can hold

Returns

Number of decoded samples, or negative on error

Definition at line 979 of file src/client/audio.c.

                                                                                                 {
  if (!g_audio_pipeline || !output || max_samples <= 0) {
    return -1;
  }
 
  return client_audio_pipeline_playback(g_audio_pipeline, opus_data, (int)opus_len, output, max_samples);
}

References client_audio_pipeline_playback().

audio_get_pipeline()

client_audio_pipeline_t * audio_get_pipeline

(

void

)

#include <audio.c>

Get the audio pipeline (for advanced usage)

Returns

Pointer to the audio pipeline, or NULL if not initialized

Definition at line 965 of file src/client/audio.c.

                                                  {
  return g_audio_pipeline;
}

audio_process_received_samples()

void audio_process_received_samples	(	const float *	samples,
		int	num_samples
	)

#include <audio.c>

Process received audio samples from server.

Process received audio samples from server

Uses the audio pipeline for processing:

1. Input validation and size checking
2. Feed samples to pipeline (applies soft clipping)
3. Feed echo reference for AEC
4. Submit processed samples to PortAudio playback queue

Parameters

samples	Raw audio sample data from server
num_samples	Number of samples in the buffer
samples	Audio sample data from server
num_samples	Number of samples in buffer

Definition at line 368 of file src/client/audio.c.

                                                                           {
  // Validate parameters
  if (!samples || num_samples <= 0) {
    SET_ERRNO(ERROR_INVALID_PARAM, "Invalid audio samples: samples=%p, num_samples=%d", (void *)samples, num_samples);
    return;
  }
 
  if (!GET_OPTION(audio_enabled)) {
    log_warn_every(1000000, "Received audio samples but audio is disabled");
    return;
  }
 
  // Allow both single packets and batched packets
  if (num_samples > AUDIO_BATCH_SAMPLES) {
    log_warn("Audio packet too large: %d samples (max %d)", num_samples, AUDIO_BATCH_SAMPLES);
    return;
  }
 
  // Calculate RMS energy of received samples
  float sum_squares = 0.0f;
  for (int i = 0; i < num_samples; i++) {
    sum_squares += samples[i] * samples[i];
  }
  float received_rms = sqrtf(sum_squares / num_samples);
 
  // DUMP: Received audio from server (before playback processing)
  if (g_wav_playback_received) {
    wav_writer_write(g_wav_playback_received, samples, num_samples);
  }
 
  // Track samples for analysis
  if (GET_OPTION(audio_analysis_enabled)) {
    for (int i = 0; i < num_samples; i++) {
      audio_analysis_track_received_sample(samples[i]);
    }
  }
 
  // Copy samples to playback buffer (no processing needed - mixer already handled clipping)
  float audio_buffer[AUDIO_BATCH_SAMPLES];
  memcpy(audio_buffer, samples, (size_t)num_samples * sizeof(float));
 
  // DEBUG: Log what we're writing to playback buffer (with first 4 samples to verify audio integrity)
  static int recv_count = 0;
  recv_count++;
  if (recv_count <= 10 || recv_count % 50 == 0) {
    float peak = 0.0f;
    for (int i = 0; i < num_samples; i++) {
      float abs_val = fabsf(samples[i]);
      if (abs_val > peak)
        peak = abs_val;
    }
    log_info("CLIENT AUDIO RECV #%d: %d samples, RMS=%.6f, Peak=%.6f, first4=[%.4f,%.4f,%.4f,%.4f]", recv_count,
             num_samples, received_rms, peak, num_samples > 0 ? samples[0] : 0.0f, num_samples > 1 ? samples[1] : 0.0f,
             num_samples > 2 ? samples[2] : 0.0f, num_samples > 3 ? samples[3] : 0.0f);
  }
 
  // Submit to playback system (goes to jitter buffer and speakers)
  // NOTE: AEC3's AnalyzeRender is called in output_callback() when audio actually plays,
  // NOT here. The jitter buffer adds 50-100ms delay, so calling AnalyzeRender here
  // would give AEC3 the wrong timing and break echo cancellation.
  audio_write_samples(&g_audio_context, audio_buffer, num_samples);
 
  // Log latency after writing to playback buffer
  if (g_audio_context.playback_buffer) {
    size_t buffer_samples = audio_ring_buffer_available_read(g_audio_context.playback_buffer);
    float buffer_latency_ms = (float)buffer_samples / 48.0f;
    log_debug_every(500000, "LATENCY: Client playback buffer after recv: %.1fms (%zu samples)", buffer_latency_ms,
                    buffer_samples);
  }
 
#ifdef DEBUG_AUDIO
  log_debug("Processed %d received audio samples", num_samples);
#endif
}

References audio_analysis_track_received_sample(), AUDIO_BATCH_SAMPLES, audio_ring_buffer_available_read(), audio_write_samples(), ERROR_INVALID_PARAM, GET_OPTION, log_debug, log_debug_every, log_info, log_warn, log_warn_every, audio_context_t::playback_buffer, SET_ERRNO, and wav_writer_write().

audio_start_thread()

int audio_start_thread

(

)

#include <audio.c>

Start audio capture thread.

Start audio capture thread

Creates and starts the audio capture thread. Also sends stream start notification to server.

Returns

0 on success, negative on error

0 on success, negative on error

Definition at line 799 of file src/client/audio.c.

                         {
  log_info("audio_start_thread called: audio_enabled=%d", GET_OPTION(audio_enabled));
 
  if (!GET_OPTION(audio_enabled)) {
    log_info("Audio is disabled, skipping audio capture thread creation");
    return 0; // Audio disabled - not an error
  }
 
  // Check if thread is already running (not just created flag)
  if (g_audio_capture_thread_created && !atomic_load(&g_audio_capture_thread_exited)) {
    log_warn("Audio capture thread already running");
    return 0;
  }
 
  // If thread exited, allow recreation
  if (g_audio_capture_thread_created && atomic_load(&g_audio_capture_thread_exited)) {
    log_info("Previous audio capture thread exited, recreating");
    // Use timeout to prevent indefinite blocking
    int join_result = asciichat_thread_join_timeout(&g_audio_capture_thread, NULL, 5000);
    if (join_result != 0) {
      log_warn("Audio capture thread join timed out after 5s - thread may be deadlocked, "
               "forcing thread handle reset (stuck thread resources will not be cleaned up)");
      // Thread is stuck - we can't safely reuse the handle, but we can reset our tracking
      // This is a resource leak of the stuck thread but continuing is safer than hanging
    }
    g_audio_capture_thread_created = false;
  }
 
  // Notify server we're starting to send audio BEFORE spawning thread
  // IMPORTANT: Must send STREAM_START before thread starts sending packets to avoid protocol violation
  if (threaded_send_stream_start_packet(STREAM_TYPE_AUDIO) < 0) {
    log_error("Failed to send audio stream start packet");
    return -1; // Don't start thread if we can't notify server
  }
 
  // Start audio capture thread
  atomic_store(&g_audio_capture_thread_exited, false);
  if (thread_pool_spawn(g_client_worker_pool, audio_capture_thread_func, NULL, 4, "audio_capture") != ASCIICHAT_OK) {
    log_error("Failed to spawn audio capture thread in worker pool");
    LOG_ERRNO_IF_SET("Audio capture thread creation failed");
    return -1;
  }
 
  g_audio_capture_thread_created = true;
 
  return 0;
}

References ASCIICHAT_OK, asciichat_thread_join_timeout(), g_client_worker_pool, GET_OPTION, LOG_ERRNO_IF_SET, log_error, log_info, log_warn, STREAM_TYPE_AUDIO, thread_pool_spawn(), and threaded_send_stream_start_packet().

Referenced by protocol_start_connection().

audio_stop_thread()

void audio_stop_thread

(

)

#include <audio.c>

Stop audio capture thread.

Stop audio capture thread

Gracefully stops the audio capture thread and cleans up resources. Safe to call multiple times.

Definition at line 855 of file src/client/audio.c.

                         {
  if (!THREAD_IS_CREATED(g_audio_capture_thread_created)) {
    return;
  }
 
  // Note: We don't call signal_exit() here because that's for global shutdown only
  // The audio capture thread checks server_connection_is_active() to detect connection loss
 
  // Wait for thread to exit gracefully
  int wait_count = 0;
  while (wait_count < 20 && !atomic_load(&g_audio_capture_thread_exited)) {
    platform_sleep_usec(100000); // 100ms
    wait_count++;
  }
 
  if (!atomic_load(&g_audio_capture_thread_exited)) {
    log_warn("Audio capture thread not responding - will be joined by thread pool");
  }
 
  // Thread will be joined by thread_pool_stop_all() in protocol_stop_connection()
  g_audio_capture_thread_created = false;
 
  log_info("Audio capture thread stopped");
}

References log_info, log_warn, platform_sleep_usec(), and THREAD_IS_CREATED.

Referenced by audio_cleanup(), and protocol_stop_connection().

audio_thread_exited()

bool audio_thread_exited

(

)

#include <audio.c>

Check if audio capture thread has exited.

Check if audio capture thread has exited

Returns

true if thread has exited, false otherwise

true if thread exited, false otherwise

Definition at line 887 of file src/client/audio.c.

                           {
  return atomic_load(&g_audio_capture_thread_exited);
}