encoder.c File Reference

FFmpeg HEVC encoder for ASCII art frames. More...

Go to the source code of this file.

Data Structures
struct	h265_encoder

Typedefs
typedef struct h265_encoder	h265_encoder_t

Functions
h265_encoder_t *	h265_encoder_create (uint16_t initial_width, uint16_t initial_height)
void	h265_encoder_destroy (h265_encoder_t *encoder)
asciichat_error_t	h265_encode (h265_encoder_t *encoder, uint16_t width, uint16_t height, const uint8_t *ascii_data, uint8_t *output_buf, size_t *output_size)
void	h265_encoder_request_keyframe (h265_encoder_t *encoder)
asciichat_error_t	h265_encoder_flush (h265_encoder_t *encoder, uint8_t *output_buf, size_t *output_size)
void	h265_encoder_get_stats (h265_encoder_t *encoder, uint64_t *total_frames, uint64_t *keyframes, uint32_t *avg_bitrate)

Detailed Description

FFmpeg HEVC encoder for ASCII art frames.

Definition in file encoder.c.

Typedef Documentation

h265_encoder_t

typedef struct h265_encoder h265_encoder_t

Function Documentation

h265_encode()

asciichat_error_t h265_encode	(	h265_encoder_t *	encoder,
		uint16_t	width,
		uint16_t	height,
		const uint8_t *	ascii_data,
		uint8_t *	output_buf,
		size_t *	output_size
	)

Encode an ASCII art frame

The encoder automatically detects size changes and reinitializes when needed. The output packet includes frame dimensions so decoder can reconfigure.

Parameters

encoder	Encoder handle
width	Current frame width (characters)
height	Current frame height (characters)
ascii_data	Raw ASCII character grid (width × height bytes)
output_buf	Buffer for encoded output
output_size	Input: buffer size, Output: actual encoded size

Returns

Error code (ASCIICHAT_OK on success)

OUTPUT PACKET FORMAT: [flags: u8][width: u16][height: u16][x265_data...]

Definition at line 288 of file encoder.c.

                                                                        {
  log_info("[H265_ENCODE_1] Starting h265_encode %ux%u", width, height);
  if (!encoder || !ascii_data || !output_buf || !output_size) {
    return SET_ERRNO(ERROR_INTERNAL, "Invalid encoder arguments");
  }
 
  if (*output_size < 5) {
    return SET_ERRNO(ERROR_NETWORK_SIZE, "Output buffer too small (minimum 5 bytes)");
  }
 
  log_info("[H265_ENCODE_2] Reconfigure %ux%u -> %ux%u", encoder->current_width, encoder->current_height, width,
           height);
  asciichat_error_t result = h265_encoder_reconfigure(encoder, width, height);
  log_info("[H265_ENCODE_3] Reconfigure returned %d", result);
  if (result != ASCIICHAT_OK) {
    return result;
  }
 
  log_info("[H265_ENCODE_4] Converting ASCII to YUV420");
  h265_encoder_ascii_to_yuv420(ascii_data, width, height, encoder->yuv_buf);
  log_info("[H265_ENCODE_5] YUV420 conversion done");
 
  // Setup frame data pointers
  encoder->frame->data[0] = encoder->yuv_buf;
  encoder->frame->data[1] = encoder->yuv_buf + width * height;
  encoder->frame->data[2] = encoder->yuv_buf + width * height + (width / 2) * (height / 2);
  encoder->frame->linesize[0] = width;
  encoder->frame->linesize[1] = width / 2;
  encoder->frame->linesize[2] = width / 2;
  encoder->frame->width = width;
  encoder->frame->height = height;
  encoder->frame->format = AV_PIX_FMT_YUV420P;
 
  // Request keyframe if needed
  if (encoder->request_keyframe) {
    encoder->frame->pict_type = AV_PICTURE_TYPE_I;
    encoder->request_keyframe = false;
  }
 
  // Encode frame
  log_info("[H265_ENCODE_6] Sending frame to encoder: %ux%u format=%d", width, height, encoder->frame->format);
  int send_ret = avcodec_send_frame(encoder->codec_ctx, encoder->frame);
  log_info("[H265_ENCODE_7] avcodec_send_frame returned %d", send_ret);
  if (send_ret < 0) {
    log_error("H265_ENCODE: avcodec_send_frame failed: %d", send_ret);
    return SET_ERRNO(ERROR_MEDIA_DECODE, "Failed to send frame to encoder");
  }
 
  log_info("[H265_ENCODE_8] Receiving encoded packet...");
  int recv_ret = avcodec_receive_packet(encoder->codec_ctx, encoder->packet);
  if (recv_ret == AVERROR(EAGAIN)) {
    // EAGAIN is normal - encoder needs more frames before outputting a packet
    log_dev("H265_ENCODE: EAGAIN - encoder buffering frames (normal), waiting for next frame");
    *output_size = 0; // No output this time
    return ASCIICHAT_OK;
  }
  if (recv_ret < 0) {
    log_error("H265_ENCODE: avcodec_receive_packet failed: %d (EAGAIN=%d)", recv_ret, AVERROR(EAGAIN));
    return SET_ERRNO(ERROR_MEDIA_DECODE, "Failed to receive encoded packet");
  }
  log_debug("H265_ENCODE: Got packet: size=%d", (int)encoder->packet->size);
 
  uint8_t flags = 0;
  if (encoder->frame->pict_type == AV_PICTURE_TYPE_I) {
    flags |= H265_ENCODER_FLAG_KEYFRAME;
    encoder->keyframes++;
  }
 
  if (width != encoder->current_width || height != encoder->current_height) {
    flags |= H265_ENCODER_FLAG_SIZE_CHANGE;
  }
 
  size_t header_size = 5;
  size_t packet_size = encoder->packet->size;
  size_t required_size = header_size + packet_size;
 
  if (required_size > *output_size) {
    return SET_ERRNO(ERROR_NETWORK_SIZE, "Output buffer too small: need %zu, have %zu", required_size, *output_size);
  }
 
  // Write header: [flags][width:u16][height:u16]
  output_buf[0] = flags;
  output_buf[1] = (width >> 8) & 0xFF;
  output_buf[2] = width & 0xFF;
  output_buf[3] = (height >> 8) & 0xFF;
  output_buf[4] = height & 0xFF;
 
  // Write encoded data
  if (packet_size > 0) {
    memcpy(output_buf + header_size, encoder->packet->data, packet_size);
  }
 
  *output_size = required_size;
  encoder->total_frames++;
  log_dev("[H265_ENCODE_DONE] Encoding complete, output_size=%zu", required_size);
 
  return ASCIICHAT_OK;
}

References ASCIICHAT_OK, EAGAIN, ERROR_INTERNAL, ERROR_MEDIA_DECODE, ERROR_NETWORK_SIZE, H265_ENCODER_FLAG_KEYFRAME, H265_ENCODER_FLAG_SIZE_CHANGE, log_debug, log_dev, log_error, log_info, and SET_ERRNO.

h265_encoder_create()

h265_encoder_t * h265_encoder_create	(	uint16_t	initial_width,
		uint16_t	initial_height
	)

Create a new x265 encoder for ASCII frames

Parameters

initial_width	Initial frame width (characters)
initial_height	Initial frame height (characters)

Returns

Encoder handle, or NULL on error

Definition at line 55 of file encoder.c.

                                                                                     {
  if (initial_width == 0 || initial_height == 0) {
    SET_ERRNO(ERROR_MEDIA_INIT, "Frame dimensions must be non-zero");
    return NULL;
  }
 
  h265_encoder_t *enc = SAFE_CALLOC(1, sizeof(h265_encoder_t), h265_encoder_t *);
  if (!enc) {
    SET_ERRNO(ERROR_MEMORY, "Failed to allocate encoder structure");
    return NULL;
  }
 
  // Find best available HEVC encoder (try by ID first, then by name for specific implementations)
  const AVCodec *codec = avcodec_find_encoder(AV_CODEC_ID_HEVC);
 
  // If generic HEVC encoder not found, try specific implementations by name
  if (!codec) {
    codec = avcodec_find_encoder_by_name("libx265");
  }
  if (!codec) {
    codec = avcodec_find_encoder_by_name("hevc_vaapi");
  }
  if (!codec) {
    codec = avcodec_find_encoder_by_name("hevc_videotoolbox");
  }
  if (!codec) {
    codec = avcodec_find_encoder_by_name("hevc_nvenc");
  }
  if (!codec) {
    codec = avcodec_find_encoder_by_name("hevc_qsv");
  }
 
  if (!codec) {
    SET_ERRNO(ERROR_MEDIA_INIT, "HEVC encoder not found");
    SAFE_FREE(enc);
    return NULL;
  }
 
  enc->codec_ctx = avcodec_alloc_context3(codec);
  if (!enc->codec_ctx) {
    SET_ERRNO(ERROR_MEMORY, "Failed to allocate codec context");
    SAFE_FREE(enc);
    return NULL;
  }
 
  // Configure codec context
  enc->codec_ctx->width = initial_width;
  enc->codec_ctx->height = initial_height;
  enc->codec_ctx->pix_fmt = AV_PIX_FMT_YUV420P;
  enc->codec_ctx->time_base = (AVRational){1, 30};
  enc->codec_ctx->framerate = (AVRational){30, 1};
 
  // Set ultrafast preset for low-latency encoding
  if (av_opt_set(enc->codec_ctx->priv_data, "preset", "ultrafast", 0) < 0) {
    log_warn("Failed to set preset to ultrafast, using default");
  }
  if (av_opt_set(enc->codec_ctx->priv_data, "tune", "zerolatency", 0) < 0) {
    log_warn("Failed to set tune to zerolatency, using default");
  }
 
  enc->codec_ctx->thread_count = h265_thread_count();
 
  // Defer avcodec_open2 to first encode call. The initial resolution (800x600) is
  // usually wrong — the actual frame resolution is only known when the first capture
  // arrives. Opening the codec here would create an x265 thread pool (~1s) that gets
  // immediately discarded on reconfigure, wasting startup time.
  enc->codec_opened = false;
 
  // Allocate frame and packet
  enc->frame = av_frame_alloc();
  enc->packet = av_packet_alloc();
 
  if (!enc->frame || !enc->packet) {
    SET_ERRNO(ERROR_MEMORY, "Failed to allocate frame/packet");
    if (enc->frame)
      av_frame_free(&enc->frame);
    if (enc->packet)
      av_packet_free(&enc->packet);
    avcodec_free_context(&enc->codec_ctx);
    SAFE_FREE(enc);
    return NULL;
  }
 
  enc->current_width = initial_width;
  enc->current_height = initial_height;
 
  // Allocate YUV420P buffer (Y plane + U/V planes)
  size_t stride = initial_width;
  enc->yuv_buf_size = stride * initial_height * 3 / 2;
  enc->yuv_buf = SAFE_MALLOC(enc->yuv_buf_size, uint8_t *);
  if (!enc->yuv_buf) {
    SET_ERRNO(ERROR_MEMORY, "Failed to allocate YUV buffer");
    av_frame_free(&enc->frame);
    av_packet_free(&enc->packet);
    avcodec_free_context(&enc->codec_ctx);
    SAFE_FREE(enc);
    return NULL;
  }
 
  log_info("FFmpeg HEVC encoder created: %ux%u (codec: %s)", initial_width, initial_height, codec->name);
 
  // Register encoder for debugging
  NAMED_REGISTER_H265_ENCODER(enc, "h265_encoder", NULL);
 
  return enc;
}

References ERROR_MEDIA_INIT, ERROR_MEMORY, log_info, log_warn, NAMED_REGISTER_H265_ENCODER, SAFE_CALLOC, SAFE_FREE, SAFE_MALLOC, and SET_ERRNO.

h265_encoder_destroy()

void h265_encoder_destroy

(

h265_encoder_t *

encoder

)

Destroy an x265 encoder

Parameters

encoder

Encoder to destroy (may be NULL)

Definition at line 162 of file encoder.c.

                                                   {
  if (!encoder)
    return;
 
  if (encoder->frame) {
    av_frame_free(&encoder->frame);
  }
  if (encoder->packet) {
    av_packet_free(&encoder->packet);
  }
  if (encoder->codec_ctx) {
    // Capture x265 encoder cleanup output (final frame stats, etc.)
    LOG_IO("hevc", { avcodec_free_context(&encoder->codec_ctx); });
  }
  if (encoder->yuv_buf) {
    SAFE_FREE(encoder->yuv_buf);
  }
  SAFE_FREE(encoder);
}

References LOG_IO, and SAFE_FREE.

h265_encoder_flush()

asciichat_error_t h265_encoder_flush	(	h265_encoder_t *	encoder,
		uint8_t *	output_buf,
		size_t *	output_size
	)

Flush remaining buffered frames from encoder

Sends a NULL frame to the encoder to flush all buffered frames. This should be called before destroying the encoder or when you need to ensure all encoded data is retrieved.

Parameters

encoder	Encoder handle
output_buf	Buffer for encoded output
output_size	Input: buffer size, Output: actual encoded size

Returns

Error code (ASCIICHAT_OK on success, ASCIICHAT_OK with size=0 when no more data)

USAGE: // After final h265_encode() call, flush remaining frames while (size > 0) { size = max_size; h265_encoder_flush(encoder, buf, &size); if (size > 0) { // Send the flushed frame... } }

Definition at line 394 of file encoder.c.

                                                                                                        {
  if (!encoder || !output_buf || !output_size) {
    return SET_ERRNO(ERROR_INTERNAL, "Invalid encoder or output parameters for flush");
  }
 
  if (*output_size < 5) {
    return SET_ERRNO(ERROR_NETWORK_SIZE, "Output buffer too small (minimum 5 bytes)");
  }
 
  // Send NULL frame to signal end of stream (flushes encoder)
  log_debug("H265_FLUSH: Sending NULL frame to encoder to flush buffered data");
  int send_ret = avcodec_send_frame(encoder->codec_ctx, NULL);
  if (send_ret < 0) {
    log_error("H265_FLUSH: avcodec_send_frame(NULL) failed: %d", send_ret);
    return SET_ERRNO(ERROR_MEDIA_DECODE, "Failed to flush encoder");
  }
 
  // Try to receive a packet
  log_debug("H265_FLUSH: Attempting to receive flushed packet...");
  int recv_ret = avcodec_receive_packet(encoder->codec_ctx, encoder->packet);
  if (recv_ret == AVERROR(EAGAIN)) {
    log_dev("H265_FLUSH: EAGAIN - no more frames available");
    *output_size = 0;
    return ASCIICHAT_OK;
  }
  if (recv_ret == AVERROR_EOF) {
    log_debug("H265_FLUSH: EOF - encoder is flushed");
    *output_size = 0;
    return ASCIICHAT_OK;
  }
  if (recv_ret < 0) {
    log_error("H265_FLUSH: avcodec_receive_packet failed: %d", recv_ret);
    return SET_ERRNO(ERROR_MEDIA_DECODE, "Failed to receive flushed packet");
  }
 
  log_debug("H265_FLUSH: Got flushed packet: size=%d", (int)encoder->packet->size);
 
  // Check if packet size exceeds output buffer
  size_t header_size = 5;
  size_t packet_size = encoder->packet->size;
  size_t required_size = header_size + packet_size;
 
  if (required_size > *output_size) {
    return SET_ERRNO(ERROR_NETWORK_SIZE, "Output buffer too small: need %zu, have %zu", required_size, *output_size);
  }
 
  // Write header: [flags][width:u16][height:u16]
  uint8_t flags = 0;
  if (encoder->frame->pict_type == AV_PICTURE_TYPE_I) {
    flags |= H265_ENCODER_FLAG_KEYFRAME;
    encoder->keyframes++;
  }
 
  output_buf[0] = flags;
  output_buf[1] = (encoder->current_width >> 8) & 0xFF;
  output_buf[2] = encoder->current_width & 0xFF;
  output_buf[3] = (encoder->current_height >> 8) & 0xFF;
  output_buf[4] = encoder->current_height & 0xFF;
 
  // Write encoded data
  if (packet_size > 0) {
    memcpy(output_buf + header_size, encoder->packet->data, packet_size);
  }
 
  *output_size = required_size;
  encoder->total_frames++;
 
  return ASCIICHAT_OK;
}

References ASCIICHAT_OK, EAGAIN, ERROR_INTERNAL, ERROR_MEDIA_DECODE, ERROR_NETWORK_SIZE, H265_ENCODER_FLAG_KEYFRAME, log_debug, log_dev, log_error, and SET_ERRNO.

h265_encoder_get_stats()

void h265_encoder_get_stats	(	h265_encoder_t *	encoder,
		uint64_t *	total_frames,
		uint64_t *	keyframes,
		uint32_t *	avg_bitrate
	)

Get encoder statistics

Parameters

encoder	Encoder handle
total_frames	Output: total frames encoded
keyframes	Output: total keyframes encoded
avg_bitrate	Output: average bitrate in bits per second

Definition at line 464 of file encoder.c.

                                                   {
  if (!encoder)
    return;
 
  if (total_frames)
    *total_frames = encoder->total_frames;
  if (keyframes)
    *keyframes = encoder->keyframes;
  if (avg_bitrate)
    *avg_bitrate = encoder->avg_bitrate;
}

Referenced by h265_client_get_stats().

h265_encoder_request_keyframe()

void h265_encoder_request_keyframe

(

h265_encoder_t *

encoder

)

Request a keyframe on the next encode

Forces the encoder to produce an I-frame (keyframe) on the next call to h265_encoder_encode(). Useful for error recovery or stream synchronization.

Parameters

encoder

Encoder handle

Definition at line 388 of file encoder.c.

                                                            {
  if (encoder) {
    encoder->request_keyframe = true;
  }
}