Account for codec delay when calculating PCM delay

src/a2dp-aac.c

@@ -179,7 +179,7 @@ void *a2dp_aac_enc_thread(struct ba_transport_pcm *t_pcm) {
 	struct io_poll io = { .timeout = -1 };
 
 	HANDLE_AACENCODER handle;
-	AACENC_InfoStruct aacinf;
+	AACENC_InfoStruct info;
 	AACENC_ERROR err;
 
 	const a2dp_aac_t *configuration = &t->a2dp.configuration.aac;
@@ -293,7 +293,7 @@ void *a2dp_aac_enc_thread(struct ba_transport_pcm *t_pcm) {
 		error("Couldn't initialize AAC encoder: %s", aacenc_strerror(err));
 		goto fail_init;
 	}
-	if ((err = aacEncInfo(handle, &aacinf)) != AACENC_OK) {
+	if ((err = aacEncInfo(handle, &info)) != AACENC_OK) {
 		error("Couldn't get encoder info: %s", aacenc_strerror(err));
 		goto fail_init;
 	}
@@ -303,14 +303,17 @@ void *a2dp_aac_enc_thread(struct ba_transport_pcm *t_pcm) {
 	pthread_cleanup_push(PTHREAD_CLEANUP(ffb_free), &bt);
 	pthread_cleanup_push(PTHREAD_CLEANUP(ffb_free), &pcm);
 
-	const unsigned int aac_frame_size = aacinf.inputChannels * aacinf.frameLength;
+	const unsigned int aac_frame_size = info.inputChannels * info.frameLength;
 	const size_t sample_size = BA_TRANSPORT_PCM_FORMAT_BYTES(t_pcm->format);
 	if (ffb_init(&pcm, aac_frame_size, sample_size) == -1 ||
-			ffb_init_uint8_t(&bt, RTP_HEADER_LEN + aacinf.maxOutBufBytes) == -1) {
+			ffb_init_uint8_t(&bt, RTP_HEADER_LEN + info.maxOutBufBytes) == -1) {
 		error("Couldn't create data buffers: %s", strerror(errno));
 		goto fail_ffb;
 	}
 
+	/* Get the delay introduced by the encoder. */
+	t_pcm->codec_delay_dms = info.nDelay * 10000 / rate;
+
 	rtp_header_t *rtp_header;
 	/* initialize RTP header and get anchor for payload */
 	uint8_t *rtp_payload = rtp_a2dp_init(bt.data, &rtp_header, NULL, 0);
@@ -322,7 +325,7 @@ void *a2dp_aac_enc_thread(struct ba_transport_pcm *t_pcm) {
 	int in_bufferIdentifiers[] = { IN_AUDIO_DATA };
 	int out_bufferIdentifiers[] = { OUT_BITSTREAM_DATA };
 	int in_bufSizes[] = { pcm.nmemb * pcm.size };
-	int out_bufSizes[] = { aacinf.maxOutBufBytes };
+	int out_bufSizes[] = { info.maxOutBufBytes };
 	int in_bufElSizes[] = { pcm.size };
 	int out_bufElSizes[] = { bt.size };
 
@@ -363,7 +366,7 @@ void *a2dp_aac_enc_thread(struct ba_transport_pcm *t_pcm) {
 			continue;
 		}
 
-		while ((in_args.numInSamples = ffb_len_out(&pcm)) > (int)aacinf.inputChannels) {
+		while ((in_args.numInSamples = ffb_len_out(&pcm)) > (int)info.inputChannels) {
 
 			if ((err = aacEncEncode(handle, &in_buf, &out_buf, &in_args, &out_args)) != AACENC_OK)
 				error("AAC encoding error: %s", aacenc_strerror(err));
@@ -409,14 +412,14 @@ void *a2dp_aac_enc_thread(struct ba_transport_pcm *t_pcm) {
 
 			}
 
-			unsigned int pcm_frames = out_args.numInSamples / aacinf.inputChannels;
+			unsigned int pcm_frames = out_args.numInSamples / info.inputChannels;
 			/* keep data transfer at a constant bit rate */
 			asrsync_sync(&io.asrs, pcm_frames);
 			/* move forward RTP timestamp clock */
 			rtp_state_update(&rtp, pcm_frames);
 
 			/* update busy delay (encoding overhead) */
-			t_pcm->delay = asrsync_get_busy_usec(&io.asrs) / 100;
+			t_pcm->processing_delay_dms = asrsync_get_busy_usec(&io.asrs) / 100;
 
 			/* If the input buffer was not consumed, we have to append new data to
 			 * the existing one. Since we do not use ring buffer, we will simply
@@ -553,26 +556,29 @@ void *a2dp_aac_dec_thread(struct ba_transport_pcm *t_pcm) {
 
 		unsigned int data_len = ffb_len_out(&latm);
 		unsigned int valid = ffb_len_out(&latm);
-		CStreamInfo *aacinf;
+		CStreamInfo *info;
 
 		if ((err = aacDecoder_Fill(handle, (uint8_t **)&latm.data, &data_len, &valid)) != AAC_DEC_OK)
 			error("AAC buffer fill error: %s", aacdec_strerror(err));
 		else if ((err = aacDecoder_DecodeFrame(handle, pcm.tail, ffb_blen_in(&pcm), 0)) != AAC_DEC_OK)
 			error("AAC decode frame error: %s", aacdec_strerror(err));
-		else if ((aacinf = aacDecoder_GetStreamInfo(handle)) == NULL)
+		else if ((info = aacDecoder_GetStreamInfo(handle)) == NULL)
 			error("Couldn't get AAC stream info");
 		else {
 
-			if ((unsigned int)aacinf->numChannels != channels)
-				warn("AAC channels mismatch: %u != %u", aacinf->numChannels, channels);
+			if ((unsigned int)info->numChannels != channels)
+				warn("AAC channels mismatch: %u != %u", info->numChannels, channels);
 
-			const size_t samples = (size_t)aacinf->frameSize * channels;
+			const size_t samples = (size_t)info->frameSize * channels;
 			io_pcm_scale(t_pcm, pcm.data, samples);
 			if (io_pcm_write(t_pcm, pcm.data, samples) == -1)
 				error("PCM write error: %s", strerror(errno));
 
+			/* Update the delay introduced by the decoder. */
+			t_pcm->codec_delay_dms = info->outputDelay * 10000 / rate;
+
 			/* update local state with decoded PCM frames */
-			rtp_state_update(&rtp, aacinf->frameSize);
+			rtp_state_update(&rtp, info->frameSize);
 
 		}
 

src/a2dp-aptx-hd.c

@@ -220,7 +220,7 @@ void *a2dp_aptx_hd_enc_thread(struct ba_transport_pcm *t_pcm) {
 			rtp.ts_pcm_frames += pcm_frames;
 
 			/* update busy delay (encoding overhead) */
-			t_pcm->delay = asrsync_get_busy_usec(&io.asrs) / 100;
+			t_pcm->processing_delay_dms = asrsync_get_busy_usec(&io.asrs) / 100;
 
 			/* reinitialize output buffer */
 			ffb_rewind(&bt);

src/a2dp-aptx.c

@@ -201,7 +201,7 @@ void *a2dp_aptx_enc_thread(struct ba_transport_pcm *t_pcm) {
 			asrsync_sync(&io.asrs, pcm_samples / channels);
 
 			/* update busy delay (encoding overhead) */
-			t_pcm->delay = asrsync_get_busy_usec(&io.asrs) / 100;
+			t_pcm->processing_delay_dms = asrsync_get_busy_usec(&io.asrs) / 100;
 
 			/* reinitialize output buffer */
 			ffb_rewind(&bt);

src/a2dp-faststream.c

@@ -150,16 +150,21 @@ void *a2dp_fs_enc_thread(struct ba_transport_pcm *t_pcm) {
 	pthread_cleanup_push(PTHREAD_CLEANUP(ffb_free), &pcm);
 	pthread_cleanup_push(PTHREAD_CLEANUP(sbc_finish), &sbc);
 
-	const unsigned int channels = t_pcm->channels;
 	const size_t sbc_frame_len = sbc_get_frame_length(&sbc);
 	const size_t sbc_frame_samples = sbc_get_codesize(&sbc) / sizeof(int16_t);
+	const unsigned int channels = t_pcm->channels;
+	const unsigned int rate = t_pcm->rate;
 
 	if (ffb_init_int16_t(&pcm, sbc_frame_samples * 3) == -1 ||
 			ffb_init_uint8_t(&bt, t->mtu_write) == -1) {
 		error("Couldn't create data buffers: %s", strerror(ENOMEM));
 		goto fail_ffb;
 	}
 
+	const unsigned int sbc_delay_frames = 73;
+	/* Get the total delay introduced by the codec. */
+	t_pcm->codec_delay_dms = sbc_delay_frames * 10000 / rate;
+
 	debug_transport_pcm_thread_loop(t_pcm, "START");
 	for (ba_transport_pcm_state_set_running(t_pcm);;) {
 
@@ -223,7 +228,7 @@ void *a2dp_fs_enc_thread(struct ba_transport_pcm *t_pcm) {
 			asrsync_sync(&io.asrs, pcm_frames);
 
 			/* update busy delay (encoding overhead) */
-			t_pcm->delay = asrsync_get_busy_usec(&io.asrs) / 100;
+			t_pcm->processing_delay_dms = asrsync_get_busy_usec(&io.asrs) / 100;
 
 			/* If the input buffer was not consumed (due to codesize limit), we
 			 * have to append new data to the existing one. Since we do not use

src/a2dp-lc3plus.c

@@ -255,6 +255,13 @@ void *a2dp_lc3plus_enc_thread(struct ba_transport_pcm *t_pcm) {
 		goto fail_ffb;
 	}
 
+	/* Get the total delay introduced by the codec. The LC3plus library
+	 * reports total codec delay in case of both encoder and decoder API.
+	 * In order not to overestimate the delay, we are not going to report
+	 * delay in the decoder thread. */
+	const int lc3plus_delay_frames = lc3plus_enc_get_delay(handle);
+	t_pcm->codec_delay_dms = lc3plus_delay_frames * 10000 / rate;
+
 	rtp_header_t *rtp_header;
 	rtp_media_header_t *rtp_media_header;
 	/* initialize RTP headers and get anchor for payload */
@@ -377,7 +384,7 @@ void *a2dp_lc3plus_enc_thread(struct ba_transport_pcm *t_pcm) {
 			rtp_state_update(&rtp, pcm_frames);
 
 			/* update busy delay (encoding overhead) */
-			t_pcm->delay = asrsync_get_busy_usec(&io.asrs) / 100;
+			t_pcm->processing_delay_dms = asrsync_get_busy_usec(&io.asrs) / 100;
 
 			/* If the input buffer was not consumed (due to codesize limit), we
 			 * have to append new data to the existing one. Since we do not use

src/a2dp-ldac.c

@@ -174,6 +174,10 @@ void *a2dp_ldac_enc_thread(struct ba_transport_pcm *t_pcm) {
 		goto fail_ffb;
 	}
 
+	const unsigned int ldac_delay_frames = 128;
+	/* Get the total delay introduced by the codec. */
+	t_pcm->codec_delay_dms = ldac_delay_frames * 10000 / rate;
+
 	rtp_header_t *rtp_header;
 	rtp_media_header_t *rtp_media_header;
 	/* initialize RTP headers and get anchor for payload */
@@ -266,7 +270,7 @@ void *a2dp_ldac_enc_thread(struct ba_transport_pcm *t_pcm) {
 			rtp_state_update(&rtp, pcm_frames);
 
 			/* update busy delay (encoding overhead) */
-			t_pcm->delay = asrsync_get_busy_usec(&io.asrs) / 100;
+			t_pcm->processing_delay_dms = asrsync_get_busy_usec(&io.asrs) / 100;
 
 		}
 

src/a2dp-mpeg.c

@@ -234,6 +234,10 @@ void *a2dp_mp3_enc_thread(struct ba_transport_pcm *t_pcm) {
 		goto fail_ffb;
 	}
 
+	/* Get the total delay introduced by the codec. */
+	const int mpeg_delay_frames = lame_get_encoder_delay(handle);
+	t_pcm->codec_delay_dms = mpeg_delay_frames * 10000 / rate;
+
 	rtp_header_t *rtp_header;
 	rtp_mpeg_audio_header_t *rtp_mpeg_audio_header;
 	/* initialize RTP headers and get anchor for payload */
@@ -319,7 +323,7 @@ void *a2dp_mp3_enc_thread(struct ba_transport_pcm *t_pcm) {
 		rtp_state_update(&rtp, pcm_frames);
 
 		/* update busy delay (encoding overhead) */
-		t_pcm->delay = asrsync_get_busy_usec(&io.asrs) / 100;
+		t_pcm->processing_delay_dms = asrsync_get_busy_usec(&io.asrs) / 100;
 
 		/* If the input buffer was not consumed (due to frame alignment), we
 		 * have to append new data to the existing one. Since we do not use

src/a2dp-opus.c

@@ -174,6 +174,11 @@ void *a2dp_opus_enc_thread(struct ba_transport_pcm *t_pcm) {
 		goto fail_ffb;
 	}
 
+	int32_t opus_delay_frames = 0;
+	/* Get the delay introduced by the encoder. */
+	opus_encoder_ctl(opus, OPUS_GET_LOOKAHEAD(&opus_delay_frames));
+	t_pcm->codec_delay_dms = opus_delay_frames * 10000 / rate;
+
 	rtp_header_t *rtp_header;
 	rtp_media_header_t *rtp_media_header;
 	/* initialize RTP headers and get anchor for payload */
@@ -237,7 +242,7 @@ void *a2dp_opus_enc_thread(struct ba_transport_pcm *t_pcm) {
 			rtp_state_update(&rtp, opus_frame_pcm_frames);
 
 			/* update busy delay (encoding overhead) */
-			t_pcm->delay = asrsync_get_busy_usec(&io.asrs) / 100;
+			t_pcm->processing_delay_dms = asrsync_get_busy_usec(&io.asrs) / 100;
 
 			/* If the input buffer was not consumed (due to encoder frame
 			 * constraint), we have to append new data to the existing one.
@@ -300,6 +305,11 @@ void *a2dp_opus_dec_thread(struct ba_transport_pcm *t_pcm) {
 		goto fail_ffb;
 	}
 
+	int32_t opus_delay_frames = 0;
+	/* Get the delay introduced by the decoder. */
+	opus_decoder_ctl(opus, OPUS_GET_LOOKAHEAD(&opus_delay_frames));
+	t_pcm->codec_delay_dms = opus_delay_frames * 10000 / rate;
+
 	struct rtp_state rtp = { .synced = false };
 	/* RTP clock frequency equal to PCM sample rate */
 	rtp_state_init(&rtp, rate, rate);

src/a2dp-sbc.c

@@ -185,6 +185,10 @@ void *a2dp_sbc_enc_thread(struct ba_transport_pcm *t_pcm) {
 		goto fail_ffb;
 	}
 
+	const unsigned int sbc_delay_frames = 73;
+	/* Get the total delay introduced by the codec. */
+	t_pcm->codec_delay_dms = sbc_delay_frames * 10000 / rate;
+
 	rtp_header_t *rtp_header;
 	rtp_media_header_t *rtp_media_header;
 
@@ -269,7 +273,7 @@ void *a2dp_sbc_enc_thread(struct ba_transport_pcm *t_pcm) {
 			rtp_state_update(&rtp, pcm_frames);
 
 			/* update busy delay (encoding overhead) */
-			t_pcm->delay = asrsync_get_busy_usec(&io.asrs) / 100;
+			t_pcm->processing_delay_dms = asrsync_get_busy_usec(&io.asrs) / 100;
 
 			/* If the input buffer was not consumed (due to codesize limit), we
 			 * have to append new data to the existing one. Since we do not use

src/ba-transport-pcm.c

@@ -724,7 +724,9 @@ int ba_transport_pcm_get_hardware_volume(
 int ba_transport_pcm_get_delay(const struct ba_transport_pcm *pcm) {
 
 	const struct ba_transport *t = pcm->t;
-	int delay = pcm->delay + ba_transport_pcm_delay_adjustment_get(pcm);
+
+	int delay = pcm->codec_delay_dms + pcm->processing_delay_dms;
+	delay += ba_transport_pcm_delay_adjustment_get(pcm);
 
 	if (t->profile & BA_TRANSPORT_PROFILE_MASK_A2DP)
 		delay += t->a2dp.delay;

src/ba-transport-pcm.h

@@ -122,9 +122,13 @@ struct ba_transport_pcm {
 	/* PCM sample rate */
 	unsigned int rate;
 
-	/* Overall PCM delay in 1/10 of millisecond, caused by
-	 * audio encoding or decoding and data transfer. */
-	unsigned int delay;
+	/* Delay caused by the codec due to internal buffering. The delay is
+	 * expressed in 1/10 of millisecond. */
+	unsigned int codec_delay_dms;
+	/* Delay caused by data processing. This delay component depends on
+	 * the host computational power. It is used to compensate for the time
+	 * required to encode or decode audio. */
+	unsigned int processing_delay_dms;
 
 	/* guard delay adjustments access */
 	pthread_mutex_t delay_adjustments_mtx;

src/codec-lc3-swb.c

@@ -43,6 +43,13 @@ void lc3_swb_init(struct esco_lc3_swb *lc3_swb) {
 
 }
 
+/**
+ * Get LC3-SWB delay in number of samples. */
+ssize_t lc3_swb_get_delay(struct esco_lc3_swb *lc3_swb) {
+	(void)lc3_swb;
+	return lc3_delay_samples(7500, 32000);
+}
+
 /**
  * Encode single eSCO LC3-SWB frame. */
 ssize_t lc3_swb_encode(struct esco_lc3_swb *lc3_swb) {

src/codec-lc3-swb.h

@@ -74,6 +74,7 @@ struct esco_lc3_swb {
 
 void lc3_swb_init(struct esco_lc3_swb *lc3_swb);
 
+ssize_t lc3_swb_get_delay(struct esco_lc3_swb *lc3_swb);
 ssize_t lc3_swb_encode(struct esco_lc3_swb *lc3_swb);
 ssize_t lc3_swb_decode(struct esco_lc3_swb *lc3_swb);
 

src/sco-cvsd.c

@@ -79,7 +79,7 @@ void *sco_cvsd_enc_thread(struct ba_transport_pcm *t_pcm) {
 			/* keep data transfer at a constant bit rate */
 			asrsync_sync(&io.asrs, mtu_samples);
 			/* update busy delay (encoding overhead) */
-			t_pcm->delay = asrsync_get_busy_usec(&io.asrs) / 100;
+			t_pcm->processing_delay_dms = asrsync_get_busy_usec(&io.asrs) / 100;
 
 		}
 

src/sco-lc3-swb.c

@@ -41,6 +41,10 @@ void *sco_lc3_swb_enc_thread(struct ba_transport_pcm *t_pcm) {
 	struct esco_lc3_swb codec;
 	lc3_swb_init(&codec);
 
+	/* Get the total delay introduced by the codec. */
+	const ssize_t lc3_swb_delay_frames = lc3_swb_get_delay(&codec);
+	t_pcm->codec_delay_dms = lc3_swb_delay_frames * 10000 / t_pcm->rate;
+
 	debug_transport_pcm_thread_loop(t_pcm, "START");
 	for (ba_transport_pcm_state_set_running(t_pcm);;) {
 
@@ -81,7 +85,7 @@ void *sco_lc3_swb_enc_thread(struct ba_transport_pcm *t_pcm) {
 			/* keep data transfer at a constant bit rate */
 			asrsync_sync(&io.asrs, codec.frames * LC3_SWB_CODESAMPLES);
 			/* update busy delay (encoding overhead) */
-			t_pcm->delay = asrsync_get_busy_usec(&io.asrs) / 100;
+			t_pcm->processing_delay_dms = asrsync_get_busy_usec(&io.asrs) / 100;
 
 			/* Move unprocessed data to the front of our linear
 			 * buffer and clear the LC3-SWB frame counter. */

src/sco-msbc.c

@@ -43,6 +43,10 @@ void *sco_msbc_enc_thread(struct ba_transport_pcm *t_pcm) {
 		goto fail_msbc;
 	}
 
+	const unsigned int sbc_delay_frames = 73;
+	/* Get the total delay introduced by the codec. */
+	t_pcm->codec_delay_dms = sbc_delay_frames * 10000 / t_pcm->rate;
+
 	debug_transport_pcm_thread_loop(t_pcm, "START");
 	for (ba_transport_pcm_state_set_running(t_pcm);;) {
 
@@ -88,7 +92,7 @@ void *sco_msbc_enc_thread(struct ba_transport_pcm *t_pcm) {
 			/* keep data transfer at a constant bit rate */
 			asrsync_sync(&io.asrs, msbc.frames * MSBC_CODESAMPLES);
 			/* update busy delay (encoding overhead) */
-			t_pcm->delay = asrsync_get_busy_usec(&io.asrs) / 100;
+			t_pcm->processing_delay_dms = asrsync_get_busy_usec(&io.asrs) / 100;
 
 			/* Move unprocessed data to the front of our linear
 			 * buffer and clear the mSBC frame counter. */