audio: webrtc_aec: patches for a practical topology and some code fixes

app/webrtc_aec_overlay.conf

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# WebRTC AEC only Kconfig overlay (LLEXT, AECm fixed-point backend)
+
+CONFIG_COMP_WEBRTC_AEC=m
+CONFIG_WEBRTC_AEC_BACKEND_AECM=y
+CONFIG_WEBRTC_AEC_SUPPRESSION_LEVEL=1
+CONFIG_WEBRTC_AEC_SAMPLE_RATE_HZ=16000

src/audio/module_adapter/module_adapter.c

@@ -205,7 +205,7 @@ __cold static void module_adapter_mem_free(struct processing_module *mod)
 	 * on the module heap since we're freeing the heap itself too
 	 */
 #if CONFIG_IPC_MAJOR_4
-	sof_heap_free(alloc->heap, mod->priv.cfg.input_pins);
+	sof_ctx_free(alloc, mod->priv.cfg.input_pins);
 #endif
 	sof_ctx_free(alloc, mod->dev);
 	sof_ctx_free(alloc, mod);

src/audio/module_adapter/module_adapter_ipc4.c

@@ -201,7 +201,7 @@ __cold int module_adapter_init_data(struct comp_dev *dev,
 
 			dst->nb_input_pins = n_in;
 			dst->nb_output_pins = n_out;
-			dst->input_pins = sof_heap_alloc(dev->mod->priv.resources.alloc->heap,
+			dst->input_pins = sof_ctx_alloc(dev->mod->priv.resources.alloc,
 							 SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
 							 pinsz, 0);
 			if (!dst->input_pins)

src/audio/webrtc_aec/Kconfig

@@ -32,7 +32,6 @@ choice WEBRTC_AEC_BACKEND
 
 config WEBRTC_AEC_BACKEND_FLOAT
 	bool "Classic Float AEC (Multidelay Block Frequency Domain with hardware FPU)"
-	depends on COMP_WEBRTC_AEC = y
 	help
 	  Single-precision floating-point AEC using partitioned-block
 	  frequency domain adaptive filtering (MDF) accelerated by the

src/audio/webrtc_aec/llext/CMakeLists.txt

@@ -10,9 +10,15 @@ if(CONFIG_COMP_WEBRTC_AEC_STUB)
 else()
   include(${CMAKE_CURRENT_LIST_DIR}/../webrtc_aec.cmake)
 
+  if(CONFIG_WEBRTC_AEC_BACKEND_FLOAT)
+    set(_aec_backend_glue ../webrtc_aec-aec.c ../webrtc_aec-math.c)
+  else()
+    set(_aec_backend_glue ../webrtc_aec-aecm.c)
+  endif()
+
   sof_llext_build("webrtc_aec"
     SOURCES ../webrtc_aec.c
-            ../webrtc_aec-aecm.c
+            ${_aec_backend_glue}
             ../webrtc_aec-shims.c
     INCLUDES "${WEBRTC_AECM_INSTALL_DIR}/include"
              "${SOF_WEBRTC_APM_SRC_DIR}"

src/audio/webrtc_aec/webrtc_aec-math.c

@@ -61,6 +61,25 @@ float exp2f(float x)
 	return p * aec_ldexp2(n);
 }
 
+#ifdef LL_EXTENSION_BUILD
+/* --- sqrtf: not exported by the base firmware, bit estimate + 3 Newton steps --- */
+float sqrtf(float x)
+{
+	union { float f; uint32_t i; } u = { .f = x };
+	float y;
+
+	if (x <= 0.0f)
+		return 0.0f;
+
+	u.i = (u.i >> 1) + 0x1fc00000;
+	y = u.f;
+	y = 0.5f * (y + x / y);
+	y = 0.5f * (y + x / y);
+	y = 0.5f * (y + x / y);
+	return y;
+}
+#endif
+
 
 /* --- powf: x^y = 2^(y * log2(x)) --- */
 float powf(float x, float y)

src/audio/webrtc_aec/webrtc_aec-shims.c

@@ -73,7 +73,6 @@ void *webrtc_realloc(void *ptr, size_t size)
 		return NULL;
 	}
 
-	/* Not called by AEC3 */
-	k_panic();
+	/* Not called by AECm; k_panic() is not exported to LLEXT modules */
 	return NULL;
 }

src/audio/webrtc_aec/webrtc_aec.c

@@ -21,6 +21,7 @@
 #include <sof/audio/audio_stream.h>
 #include <ipc4/aec.h>
 #include <ipc4/header.h>
+#include <ipc4/module.h>
 #include <sof/math/numbers.h>
 #include <rtos/init.h>
 #include <rtos/string.h>
@@ -229,17 +230,17 @@ __cold static int webrtc_aec_prepare(struct processing_module *mod,
 	if (num_of_sources == 1) {
 		cd->mic_src = 0;
 		cd->ref_src = -1;
-	} else if (source_get_pipeline_id(sources[0]) == sink_get_pipeline_id(sinks[0]) &&
-	    source_get_pipeline_id(sources[1]) != sink_get_pipeline_id(sinks[0])) {
-		cd->mic_src = 0;
-		cd->ref_src = 1;
-	} else if (source_get_pipeline_id(sources[1]) == sink_get_pipeline_id(sinks[0]) &&
-		   source_get_pipeline_id(sources[0]) != sink_get_pipeline_id(sinks[0])) {
-		cd->mic_src = 1;
-		cd->ref_src = 0;
 	} else {
-		cd->mic_src = 0;
-		cd->ref_src = 1;
+#ifdef CONFIG_IPC_MAJOR_4
+		/* Bind order is arbitrary, input pin 0 is mic and pin 1 is reference */
+		cd->mic_src = IPC4_SINK_QUEUE_ID(source_get_id(sources[0])) == 0 ? 0 : 1;
+#else
+		/* The mic is the source in the same pipeline as the output */
+		cd->mic_src = source_get_pipeline_id(sources[1]) == sink_get_pipeline_id(sinks[0]) &&
+			      source_get_pipeline_id(sources[0]) != sink_get_pipeline_id(sinks[0]) ?
+			      1 : 0;
+#endif
+		cd->ref_src = 1 - cd->mic_src;
 	}
 
 	mic_fmt  = source_get_frm_fmt(sources[cd->mic_src]);
@@ -313,9 +314,14 @@ __cold static int webrtc_aec_prepare(struct processing_module *mod,
 
 #ifdef CONFIG_IPC_MAJOR_4
 	/* Apply reference format override from topology pin descriptor. */
-	if (cd->ref_src >= 0)
+	if (cd->ref_src >= 0) {
+		if (!mod->priv.cfg.input_pins || mod->priv.cfg.nb_input_pins < 2) {
+			comp_err(dev, "webrtc_aec: no reference pin format");
+			return -EINVAL;
+		}
 		ipc4_update_source_format(sources[cd->ref_src],
 					  &mod->priv.cfg.input_pins[1].audio_fmt);
+	}
 #endif
 
 	if (cd->backend->configure) {

src/audio/webrtc_aec/webrtc_aec.cmake

@@ -33,22 +33,35 @@ set(WEBRTC_AECM_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/webrtc-aecm-install"
 
 set(_aecm_src "${SOF_WEBRTC_APM_SRC_DIR}/webrtc")
 
-file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/webrtc-aecm-build.sh"
+if(CONFIG_WEBRTC_AEC_BACKEND_FLOAT)
+  set(_aec_backend_srcs "
+  $SRC/modules/audio_processing/aec/echo_cancellation.c
+  $SRC/modules/audio_processing/aec/aec_core.c
+  $SRC/modules/audio_processing/aec/aec_rdft.c
+  $SRC/modules/audio_processing/aec/aec_resampler.c")
+  set(_aec_backend_hdr "$SRC/modules/audio_processing/aec/include/echo_cancellation.h")
+else()
+  set(_aec_backend_srcs "
+  $SRC/modules/audio_processing/aecm/aecm_core.c
+  $SRC/modules/audio_processing/aecm/aecm_core_c.c
+  $SRC/modules/audio_processing/aecm/echo_control_mobile.c")
+  set(_aec_backend_hdr "$SRC/modules/audio_processing/aecm/include/echo_control_mobile.h")
+endif()
+
+file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/webrtc-aecm-build.sh.tmp"
 "#!/bin/sh
 set -e
 SRC=${_aecm_src}
 INST=${WEBRTC_AECM_INSTALL_DIR}
 OBJ=${CMAKE_CURRENT_BINARY_DIR}/webrtc-aecm-obj
 
+rm -rf \"$OBJ\" \"$INST/lib/libwebrtc_aecm.a\"
 mkdir -p \"$OBJ\" \"$INST/lib\" \"$INST/include\" \"$INST/include/webrtc\"
 
-CFLAGS=\"-O2 -fPIC -I${SOF_WEBRTC_APM_SRC_DIR} -I$SRC -I$SRC/common_audio/signal_processing/include -DWEBRTC_ARCH_32_BITS= -DWEBRTC_ARCH_LITTLE_ENDIAN= -Dmalloc=webrtc_malloc -Dcalloc=webrtc_calloc -Drealloc=webrtc_realloc -Dfree=webrtc_free\"
+CFLAGS=\"-O2 -fPIC -DNDEBUG -I${SOF_WEBRTC_APM_SRC_DIR} -I$SRC -I$SRC/common_audio/signal_processing/include -DWEBRTC_ARCH_32_BITS= -DWEBRTC_ARCH_LITTLE_ENDIAN= -Dmalloc=webrtc_malloc -Dcalloc=webrtc_calloc -Drealloc=webrtc_realloc -Dfree=webrtc_free\"
 
-# AECm core
-SRCS=\"
-  $SRC/modules/audio_processing/aecm/aecm_core.c
-  $SRC/modules/audio_processing/aecm/aecm_core_c.c
-  $SRC/modules/audio_processing/aecm/echo_control_mobile.c
+# AEC backend core
+SRCS=\"${_aec_backend_srcs}
   $SRC/common_audio/ring_buffer.c
   $SRC/modules/audio_processing/utility/delay_estimator.c
   $SRC/modules/audio_processing/utility/delay_estimator_wrapper.c
@@ -78,14 +91,19 @@ for f in \$SRCS; do
 done
 
 ${_aecm_ar} rcs \"$INST/lib/libwebrtc_aecm.a\" \"$OBJ\"/*.o
-cp $SRC/modules/audio_processing/aecm/include/echo_control_mobile.h \"$INST/include/\"
+cp ${_aec_backend_hdr} \"$INST/include/\"
 cp $SRC/typedefs.h \"$INST/include/webrtc/\" 2>/dev/null || true
 ")
 
+# Only touch the script when its content changes so that the library is
+# rebuilt on flag changes but not on every reconfigure.
+configure_file("${CMAKE_CURRENT_BINARY_DIR}/webrtc-aecm-build.sh.tmp"
+  "${CMAKE_CURRENT_BINARY_DIR}/webrtc-aecm-build.sh" COPYONLY)
+
 add_custom_command(
   OUTPUT "${WEBRTC_AECM_INSTALL_DIR}/lib/libwebrtc_aecm.a"
-         "${WEBRTC_AECM_INSTALL_DIR}/include/echo_control_mobile.h"
   COMMAND sh "${CMAKE_CURRENT_BINARY_DIR}/webrtc-aecm-build.sh"
+  DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/webrtc-aecm-build.sh"
   VERBATIM)
 
 add_custom_target(webrtc_aecm_ext

src/audio/webrtc_aec/webrtc_aec.toml

@@ -12,6 +12,7 @@
 	domain_types = "1"
 	load_type = LOAD_TYPE
 	module_type = "9"
+	init_config = "1"
 	auto_start = "0"
 	sched_caps = [1, 0x00008000]
 	REM # pin = [dir, type, sample rate, size, container, channel-cfg]

tools/topology/topology2/cavs-sdw.conf

@@ -64,6 +64,8 @@
 <deep-buffer-default.conf>
 <compr-default.conf>
 <siggen-host-copier-capture.conf>
+<webrtc-aec.conf>
+<input_pin_binding.conf>
 
 Define {
 	PLATFORM 		"none"
@@ -133,6 +135,7 @@ Define {
 	SDW_AMP_PIPELINE_SRC "generic"
 	SDW_JACK_PIPELINE_SRC "generic"
 	SDW_ECHO_REF_DAI "false"
+	SDW_SPK_AEC_CAPTURE_PCM "false"
 }
 
 # override defaults with platform-specific config
@@ -273,6 +276,10 @@ IncludeByKey.SDW_SPK_ECHO_REF {
 	"true" "platform/intel/sdw-amp-echo-ref.conf"
 }
 
+IncludeByKey.SDW_SPK_AEC_CAPTURE_PCM {
+	"true" "platform/intel/sdw-amp-aec.conf"
+}
+
 IncludeByKey.SDW_JACK_AUDIO_FEATURE_CAPTURE {
 	"true" "platform/intel/sdw-jack-audio-feature.conf"
 }

tools/topology/topology2/development/tplg-targets.cmake

@@ -479,6 +479,13 @@ PREPROCESS_PLUGINS=nhlt,NHLT_BIN=nhlt-sof-ptl-rt721-4ch-echoref.bin,DMIC0_ENHANC
 EFX_DMIC0_TDFB_PARAMS=line4_pass,EFX_DMIC0_DRC_PARAMS=dmic_default,\
 DEEPBUFFER_FW_DMA_MS=10,DEEP_BUF_SPK=true,SDW_JACK_ECHO_REF=true,SDW_SPK_ECHO_REF=true,SDW_ECHO_REF_DAI=true"
 
+# RT713 headset on link 3 and RT1320 amplifiers on links 1 and 2 with mono
+# 16 kHz WebRTC AEC of the SoundWire DMIC against the speaker echo reference
+"cavs-sdw\;sof-ptl-rt713-l3-rt1320-l12-aec\;PLATFORM=ptl,SDW_DMIC=1,NUM_SDW_AMP_LINKS=2,\
+SDW_AMP_FEEDBACK=false,SDW_SPK_STREAM=Playback-SmartAmp,SDW_DMIC_STREAM=Capture-SmartMic,\
+SDW_JACK_OUT_STREAM=Playback-SimpleJack,SDW_JACK_IN_STREAM=Capture-SimpleJack,\
+SDW_SPK_ECHO_REF=true,SDW_ECHO_REF_DAI=true,SDW_SPK_AEC_CAPTURE_PCM=true"
+
 # RT721 eval board with PCH-DMIC, sof_sdw_quirk_table with SOC_SDW_PCH_DMIC
 # Enable echo reference capture from jack and speaker output
 # Enable SSP2 BT

tools/topology/topology2/include/components/webrtc-aec.conf

@@ -142,6 +142,7 @@ Class.Widget."webrtc-aec" {
 	cpc			2000
 	scheduler_domain	"DP"
 	stack_bytes_requirement	16384
+	heap_bytes_requirement	262144
 	num_input_pins		2
 	num_output_pins		1
 }