LL userspace: enable cross-core DP pipelines

app/ll_userspace_overlay.conf

@@ -41,7 +41,6 @@ CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n
 CONFIG_SOF_BOOT_TEST_ALLOWED=n
 
 # misc features not yet compatible with user-space
-CONFIG_CROSS_CORE_STREAM=n
 CONFIG_INTEL_ADSP_MIC_PRIVACY=n
 CONFIG_XRUN_NOTIFICATIONS_ENABLE=n
 

src/audio/pipeline/pipeline-graph.c

@@ -285,9 +285,12 @@ static void buffer_set_comp(struct comp_buffer *buffer, struct comp_dev *comp,
  * lock via user_ll_lock_sched()/ll_block() while modifying pipeline
  * connections, which provides mutual exclusion with the LL thread. No
  * additional lock is taken here; instead assert that the lock is held.
+ * That's the lock of the pipeline core, whose LL thread walks the buffer
+ * lists of the pipeline modules. Not of the buffer core: cross-core
+ * buffers belong to the primary core.
  */
 #define PPL_LOCK_DECLARE
-#define PPL_LOCK(x) user_ll_assert_locked(x)
+#define PPL_LOCK(comp) user_ll_assert_locked(comp_ppl_core(comp))
 #define PPL_UNLOCK()
 #else
 /*
@@ -314,7 +317,7 @@ __cold int pipeline_connect(struct comp_dev *comp, struct comp_buffer *buffer,
 	else
 		comp_info(comp, "connect buffer %d as source", buf_get_id(buffer));
 
-	PPL_LOCK(buffer->core);
+	PPL_LOCK(comp);
 
 	comp_list = comp_buffer_list(comp, dir);
 	ret = buffer_attach(buffer, comp_list, dir);
@@ -343,7 +346,7 @@ __cold void pipeline_disconnect(struct comp_dev *comp, struct comp_buffer *buffe
 	else
 		comp_dbg(comp, "disconnect buffer %d as source", buf_get_id(buffer));
 
-	PPL_LOCK(buffer->core);
+	PPL_LOCK(comp);
 
 	comp_list = comp_buffer_list(comp, dir);
 	buffer_detach(buffer, comp_list, dir);

src/audio/pipeline/static_pipeline_loader.c

@@ -583,6 +583,9 @@ static int sof_static_topology_init_comps(const struct sof_static_topology *topo
 			.id = cdesc->id,
 			.pipeline_id = cdesc->pipeline_id,
 			.core = comp_core,
+#if CONFIG_IPC_MAJOR_4
+			.ppl_core = comp_core,
+#endif
 			.proc_domain = COMP_PROCESSING_DOMAIN_LL,
 			.frame_fmt = cdesc->caps.default_fmt,
 			.type = drv_type,

src/include/sof/audio/component.h

@@ -571,6 +571,7 @@ struct comp_ipc_config {
 #if CONFIG_IPC_MAJOR_4
 	bool ipc_extended_init;		/**< True if extended init is included in IPC payload */
 	uint32_t ipc_config_size;	/**< Size of config received by IPC */
+	uint32_t ppl_core;		/**< Core of the parent pipeline */
 #endif
 };
 

src/include/sof/audio/component_ext.h

@@ -18,8 +18,9 @@
 
 #include <sof/audio/component.h>
 #include <ipc/topology.h>
-/* TODO: the cross-core helpers below send IDC messages, which is kernel only */
+/* in user-space LL builds idc_send_msg() forwards to the remote user IPC thread */
 #include <rtos/idc.h>
+#include <sof/schedule/ll_schedule_domain.h>
 #include <sof/schedule/schedule.h>
 
 /** \addtogroup component_infra_api
@@ -299,9 +300,18 @@ static inline int comp_ipc4_get_attribute_remote(struct comp_dev *dev, uint32_t
 	if (type != COMP_ATTR_BASE_CONFIG)
 		return -EINVAL;
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	/* must be writable by the user-space thread on the remote core */
+	base_cfg = sof_heap_alloc(zephyr_ll_user_heap(), SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
+				  sizeof(*base_cfg), 0);
+	if (!base_cfg)
+		return -ENOMEM;
+	memset(base_cfg, 0, sizeof(*base_cfg));
+#else
 	base_cfg = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*base_cfg));
 	if (!base_cfg)
 		return -ENOMEM;
+#endif
 
 	payload.type = type;
 	payload.value = base_cfg;
@@ -312,7 +322,11 @@ static inline int comp_ipc4_get_attribute_remote(struct comp_dev *dev, uint32_t
 		memcpy_s(value, sizeof(struct ipc4_base_module_cfg),
 			 base_cfg, sizeof(struct ipc4_base_module_cfg));
 
+#ifdef CONFIG_SOF_USERSPACE_LL
+	sof_heap_free(zephyr_ll_user_heap(), base_cfg);
+#else
 	rfree(base_cfg);
+#endif
 	return ret;
 }
 #endif	/* CONFIG_IPC_MAJOR_4 */
@@ -501,6 +515,23 @@ static inline void comp_make_shared(struct comp_dev *dev)
 	dev->is_shared = true;
 }
 
+/**
+ * \brief Get the core of the parent pipeline of a component.
+ *
+ * Use this instead of dev->pipeline->core on other cores than the pipeline
+ * core: struct pipeline isn't coherent between cores, struct comp_dev is.
+ * \param[in] dev Component device.
+ * \return Pipeline core, or the component core if it has no pipeline.
+ */
+static inline uint32_t comp_ppl_core(const struct comp_dev *dev)
+{
+#if CONFIG_IPC_MAJOR_4
+	return dev->ipc_config.ppl_core;
+#else
+	return dev->ipc_config.core;
+#endif
+}
+
 /**
  * \brief Get the global component driver list.
  * \return Pointer to the component driver list.

src/include/sof/ipc/common.h

@@ -62,6 +62,33 @@ extern struct tr_ctx ipc_tr;
 #define IPC_DSP_CORE_COUNT 1
 #endif
 
+/** @brief Maximum payload size of a cross-core request between user threads */
+#define IPC_USER_XCORE_PAYLOAD_SIZE	128
+
+/**
+ * @brief Cross-core request slot of a user-space IPC thread.
+ *
+ * User-space threads can't use IDC. Instead, they pass IDC messages for
+ * another core to the user-space IPC thread of that core via this slot.
+ */
+struct ipc_user_xcore {
+	/** @brief Serializes senders to this core */
+	struct k_sem *lock;
+	/** @brief Thread, which has reserved the slot, or NULL */
+	struct k_thread *owner;
+	/** @brief Signalled by the target core when the request is done */
+	struct k_sem *done;
+	/** @brief IDC message header and extension */
+	uint32_t header;
+	uint32_t extension;
+	/** @brief Result from the target core */
+	int result;
+	/** @brief Copy of IPC4 bind / unbind message, referenced by the payload */
+	uint32_t ipc4_bind_data[2];
+	/** @brief Copy of the IDC message payload */
+	uint8_t payload[IPC_USER_XCORE_PAYLOAD_SIZE] __aligned(8);
+};
+
 struct ipc_user {
 	struct k_thread *thread[IPC_DSP_CORE_COUNT];
 	struct k_sem *sem;
@@ -96,6 +123,8 @@ struct ipc_user {
 	 * Size verified by BUILD_ASSERT in handler-user.c.
 	 */
 	uint8_t init_drv_data[160] __aligned(4);
+	/** @brief Cross-core request slots, one per target core */
+	struct ipc_user_xcore xcore[IPC_DSP_CORE_COUNT];
 };
 
 struct ipc {
@@ -346,6 +375,24 @@ extern bool ipc_enter_gdb;
  */
 int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core);
 
+/**
+ * @brief Reserve the cross-core request slots of the given cores.
+ *
+ * Reserve the slots before taking LL scheduler locks, if requests to the
+ * cores are sent while holding the locks. Otherwise a request in flight to
+ * such a core, that needs one of the locks, would deadlock with the sender.
+ *
+ * @param cores Bitmask of target cores, not including the current core
+ */
+void ipc_user_xcore_reserve(uint32_t cores);
+
+/**
+ * @brief Release the slots reserved with ipc_user_xcore_reserve().
+ *
+ * @param cores Bitmask of target cores, as passed to ipc_user_xcore_reserve()
+ */
+void ipc_user_xcore_release(uint32_t cores);
+
 /**
  * @brief Protocol-specific dispatch of a forwarded IPC command.
  *

src/include/sof/schedule/ll_schedule.h

@@ -58,6 +58,24 @@ void user_ll_lock_sched(int core);
  */
 void user_ll_unlock_sched(int core);
 
+/**
+ * @brief Release all holds of the calling thread on a per-core user
+ * Low-Latency scheduling lock.
+ *
+ * Must be paired with user_ll_relock_sched().
+ * @param[in] core Target core ID.
+ * @return Number of holds released, 0 if the calling thread doesn't hold
+ *	   the lock.
+ */
+unsigned int user_ll_unlock_sched_all(int core);
+
+/**
+ * @brief Re-take holds released by user_ll_unlock_sched_all().
+ * @param[in] core Target core ID.
+ * @param[in] depth Value returned by user_ll_unlock_sched_all().
+ */
+void user_ll_relock_sched(int core, unsigned int depth);
+
 #ifdef CONFIG_ASSERT
 /**
  * @brief Assert that the calling context holds the user Low-Latency lock for a core.

src/include/sof/schedule/ll_schedule_domain.h

@@ -229,6 +229,8 @@ void zephyr_ll_user_resources_init(void);
 
 /**
  * @brief Grant thread access to per-core Low-Latency mutexes.
+ *
+ * Kernel context only.
  * @param[in,out] thread Target thread.
  * @param[in] core Target core ID.
  */