diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index 415b2437c17d..b099d00ec97a 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -1024,8 +1024,12 @@ __cold int module_unbind(struct processing_module *mod, const struct bind_info * uint32_t module_get_deadline(struct processing_module *mod) { + /* a DP module can write mod: read the count once and bound it */ + size_t num_of_sinks = mod->num_of_sinks; uint32_t deadline; + num_of_sinks = MIN(num_of_sinks, ARRAY_SIZE(mod->sinks)); + /* LL modules have no deadline - it is always "now" */ if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL) return 0; @@ -1036,9 +1040,14 @@ uint32_t module_get_deadline(struct processing_module *mod) deadline = UINT32_MAX; /* calculate the shortest LFT for all sinks */ - for (size_t i = 0; i < mod->num_of_sinks; i++) { - uint32_t sink_lft = sink_get_last_feeding_time(mod->sinks[i]); + for (size_t i = 0; i < num_of_sinks; i++) { + struct sof_sink *sink = mod->sinks[i]; + uint32_t sink_lft; + + if (!sink) + continue; + sink_lft = sink_get_last_feeding_time(sink); deadline = MIN(deadline, sink_lft); } diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index 1dac74b600ef..c1da1b6182ef 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -86,13 +86,62 @@ __syscall void scheduler_dp_internal_free(struct task *task); /** * @brief Tick notification from the Low-Latency (LL) timer domain to the DP scheduler. * - * Invoked on each core after all LL tasks complete. Evaluates readiness for all - * queued DP tasks, updates their absolute deadlines, and signals worker threads. + * Evaluates readiness for all queued DP tasks, updates their absolute + * deadlines, and signals worker threads. Not available with + * CONFIG_SOF_USERSPACE_APPLICATION and CONFIG_SOF_USERSPACE_LL, see + * scheduler_dp_ll_post_tick(). */ __syscall void scheduler_dp_ll_tick(void); + +/** @brief scheduler_dp_kick flag: signal the DP thread to process data */ +#define SCHEDULER_DP_KICK_PROCESS BIT(0) +/** @brief scheduler_dp_kick flag: set the DP thread deadline */ +#define SCHEDULER_DP_KICK_DEADLINE BIT(1) + +/** @brief Maximum number of entries passed to one scheduler_dp_kick() call */ +#define SCHEDULER_DP_KICK_MAX 8 + +/** @brief Result of the DP readiness and deadline calculation for one task */ +struct scheduler_dp_kick { + const struct task *task; /**< DP task, only used to look up its record */ + uint32_t flags; /**< SCHEDULER_DP_KICK_* */ + uint32_t deadline; /**< deadline in cycles from now */ +}; + +/** + * @brief Apply DP readiness and deadline results to the DP threads. + * + * The DP thread kernel objects are looked up from the kernel-only DP task + * records, never from DP task memory. Entries for unknown tasks and for + * tasks of other cores are skipped. Only the LL user thread may call this, + * or the LL scheduler in kernel mode without CONFIG_SOF_USERSPACE_LL. + * + * @param[in] kicks Array of results. + * @param[in] count Number of entries, at most SCHEDULER_DP_KICK_MAX. + * @return Number of skipped entries. + */ +__syscall int scheduler_dp_kick(const struct scheduler_dp_kick *kicks, uint32_t count); #include #endif +/** + * @brief Run the DP scheduler after an LL tick. + * + * Called by the LL scheduler on each core after all LL tasks complete. With + * CONFIG_SOF_USERSPACE_APPLICATION and CONFIG_SOF_USERSPACE_LL, the DP task + * memory is writable by the DP threads, so the calculation runs in the calling + * LL user thread, and only its results are passed to the kernel with + * scheduler_dp_kick(). Otherwise this calls scheduler_dp_ll_tick(). + */ +void scheduler_dp_ll_post_tick(void); + +#if CONFIG_SOF_BOOT_TEST_STANDALONE && CONFIG_SOF_USERSPACE_APPLICATION +/* Test hooks: add and remove kernel-only DP task records without a DP task */ +int scheduler_dp_test_record_add(struct task *task, struct k_event *event, + struct k_thread *thread, uint16_t core); +void scheduler_dp_test_record_del(struct task *task); +#endif + /** * @brief Extract telemetry information about Data Processing scheduler tasks. * diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md index d8efa662aa2d..c3e84a799879 100644 --- a/src/include/sof/userspace/README.md +++ b/src/include/sof/userspace/README.md @@ -124,7 +124,7 @@ syscall by subsystem: | CPU | `cpu_get_id` | `sof/lib/cpu.h` | `zephyr/syscall/cpu.c` / `zephyr/lib/cpu.c` | | Module management | `mod_balloc_align`, `mod_alloc_ext`, `mod_free`, `mod_free_all`, `mod_data_blob_handler_new`, `mod_data_blob_set`, `mod_fast_get` | `sof/audio/module_adapter/module/generic.h` | scattered: `audio/module_adapter/module/generic.c` | | Library manager | `lib_manager_free_module` | `sof/lib_manager.h` | scattered: `kernel/library_manager/lib_manager.c` | -| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `schedule/zephyr_ll.c`, `schedule/zephyr_dp_schedule*.c` | +| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick`, `scheduler_dp_kick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `schedule/zephyr_ll.c`, `schedule/zephyr_dp_schedule*.c` | | Debug | `debug_stream_slot_send_record` | `user/debug_stream_slot.h` | scattered: `debug/debug_stream/debug_stream_slot.c` | | KPB | `kpb_notifier_init` | `sof/audio/kpb.h` | scattered: `audio/kpb.c` | @@ -141,9 +141,15 @@ from user-writable data such as `ipc_user->thread[]`. |-----------------|----------| | Any user thread, including DP module threads | `cpu_get_id`, all module management syscalls (`mod_*`), `ipc_msg_send`, `ipc_msg_list_remove`, `send_resource_notif` | | IPC user thread only | `ipc_compound_pre_start`, `ipc_compound_post_start`, `ipc_wait_for_compound_msg`, `lib_manager_free_module` | -| LL user thread only | `scheduler_dp_ll_tick` | +| LL user thread only | `scheduler_dp_ll_tick`, `scheduler_dp_kick` | | LL or IPC user thread | all other SOF syscalls | +With `CONFIG_SOF_USERSPACE_APPLICATION`, the DP task list and task memory are writable by DP +threads, so the kernel does not walk them. The DP readiness and deadline calculation runs in +the LL user thread (`scheduler_dp_ll_post_tick()`), which passes its results to +`scheduler_dp_kick()`; the kernel takes the DP thread and event objects from its own task +records. `scheduler_dp_ll_tick` is not built when `CONFIG_SOF_USERSPACE_LL` is also set. + DP modules only get the syscalls needed to implement the module interface. That includes sending notifications: modules queue messages from `mod_ipc_msg_init()` with `ipc_msg_send()` and free them with `mod_ipc_msg_free()`, and `dp_task_run()` reports diff --git a/src/schedule/README.md b/src/schedule/README.md index 0bef23248837..cebaab746835 100644 --- a/src/schedule/README.md +++ b/src/schedule/README.md @@ -107,7 +107,7 @@ The DP scheduler (`zephyr_dp_schedule.c`) manages asynchronous, compute-heavy ta ### Architecture - **Separate Threads**: Unlike LL which multiplexes tasks inside a single thread, **each DP task is assigned its own Zephyr thread**. -- **Wakeup Mechanism**: DP scheduling is evaluated at the end of each LL tick (`scheduler_dp_recalculate()`). +- **Wakeup Mechanism**: DP scheduling is evaluated at the end of each LL tick (`scheduler_dp_recalculate()`). With user-space LL and DP, this runs in the LL user thread and the kernel applies the results with `scheduler_dp_kick()`. - **Readiness**: It checks if a component has sufficient data across its sinks and sources. If so, it transitions to `RUNNING` and signals the individual DP thread via a Zephyr Event object. - **Deadlines**: Once ready, the DP thread computes its deadline absolute timestamp (`module_get_deadline()`) and calls `k_thread_absolute_deadline_set()`, submitting to the Zephyr kernel's EDF scheduler. diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index cb3148de20c7..a5ca0eec820f 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -227,6 +227,25 @@ static enum task_state scheduler_dp_ll_tick_dummy(void *data) * needed 1.2ms for processing - but the example would be too complicated) */ +#if CONFIG_SOF_USERSPACE_APPLICATION && CONFIG_SOF_USERSPACE_LL +/* Runs in the LL user thread, see scheduler_dp_ll_post_tick() documentation */ +void scheduler_dp_ll_post_tick(void) +{ + struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); + unsigned int lock_key; + + /* + * The LL thread is granted the DP lock when the first DP task of its + * core is created, so there is nothing to do and no access before. + */ + if (!dp_sch || list_is_empty(&dp_sch->tasks)) + return; + + lock_key = scheduler_dp_lock(cpu_get_id()); + scheduler_dp_recalculate(dp_sch); + scheduler_dp_unlock(lock_key); +} +#else void z_impl_scheduler_dp_ll_tick(void) { unsigned int lock_key; @@ -259,6 +278,12 @@ void z_vrfy_scheduler_dp_ll_tick(void) #include #endif +void scheduler_dp_ll_post_tick(void) +{ + scheduler_dp_ll_tick(); +} +#endif + #if CONFIG_SOF_USERSPACE_APPLICATION static int scheduler_dp_task_cancel(void *data, struct task *task) { @@ -392,16 +417,35 @@ __cold int scheduler_dp_init(void) void scheduler_get_task_info_dp(struct scheduler_props *scheduler_props, uint32_t *data_off_size) { - unsigned int lock_key; - scheduler_props->processing_domain = COMP_PROCESSING_DOMAIN_DP; +#if CONFIG_SOF_USERSPACE_APPLICATION + /* + * The task list links are in DP task memory, which DP threads can + * write. Count the kernel-only task records instead of walking it. + */ + struct task_props *task_props = (struct task_props *)(scheduler_props + 1); + unsigned int count = scheduler_dp_task_count(cpu_get_id()); + unsigned int i; + + scheduler_props->core_id = cpu_get_id(); + scheduler_props->task_count = count; + *data_off_size += sizeof(*scheduler_props); + + for (i = 0; i < count; i++) { + task_props[i].task_id = 0; + task_props[i].module_instance_count = 0; + *data_off_size += sizeof(*task_props); + } +#else #if CONFIG_SOF_USERSPACE_LL struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); #else struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); #endif + unsigned int lock_key; lock_key = scheduler_dp_lock(cpu_get_id()); scheduler_get_task_info(scheduler_props, data_off_size, &dp_sch->tasks); scheduler_dp_unlock(lock_key); +#endif } diff --git a/src/schedule/zephyr_dp_schedule.h b/src/schedule/zephyr_dp_schedule.h index 07d8ee4a4754..a08248171bb6 100644 --- a/src/schedule/zephyr_dp_schedule.h +++ b/src/schedule/zephyr_dp_schedule.h @@ -72,6 +72,15 @@ struct task_dp_pdata { */ void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch); +#if CONFIG_SOF_USERSPACE_APPLICATION +/** + * @brief Count the DP tasks of a core from the kernel-only DP task records. + * @param[in] core DSP core. + * @return Number of scheduled DP tasks on @p core. Kernel context only. + */ +unsigned int scheduler_dp_task_count(unsigned int core); +#endif + /** * @brief Worker thread entry point function for Data Processing tasks. * diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 6c9dafc1db28..1d2345d796c7 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -157,7 +157,7 @@ struct scheduler_dp_task_memory { * into the DP thread's memory domain, and is also writable by the LL and IPC * user threads. Freeing must therefore never use the pointers stored there: * scheduler_dp_internal_free() takes them from this record instead. The task - * pointer is only used as a lookup key. + * and module pointers are only used as lookup keys. */ struct scheduler_dp_task_kobj { struct list_item list; @@ -165,9 +165,12 @@ struct scheduler_dp_task_kobj { struct processing_module *mod; struct k_event *event; struct k_thread *thread; + struct ipc4_flat *flat; /* IPC parameter area in task memory */ + void *config_data; /* LARGE_CONFIG_GET reply buffer in task memory */ struct k_mem_domain *mdom; struct k_mem_partition mpart[SOF_DP_PART_TYPE_COUNT]; uint32_t comp_id; + uint16_t core; uint32_t gen; /* unique per record, see dp_task_kobj_take() */ }; @@ -200,6 +203,29 @@ static struct scheduler_dp_task_kobj *dp_task_kobj_find(const struct task *task) return NULL; } +/* Copy the record for @mod to @copy */ +static bool dp_task_kobj_peek_mod(const struct processing_module *mod, + struct scheduler_dp_task_kobj *copy) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + struct scheduler_dp_task_kobj *kobj; + struct list_item *item; + bool found = false; + + list_for_item(item, &dp_task_kobj_list) { + kobj = container_of(item, struct scheduler_dp_task_kobj, list); + if (kobj->mod == mod) { + *copy = *kobj; + found = true; + break; + } + } + + k_spin_unlock(&dp_task_kobj_lock, key); + + return found; +} + /* Copy the record for @task to @copy without unlinking it */ static bool dp_task_kobj_peek(const struct task *task, struct scheduler_dp_task_kobj *copy) { @@ -321,14 +347,41 @@ static void ipc_thread_unflatten_run(struct processing_module *pmod, struct ipc4 int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, union scheduler_dp_thread_ipc_param *param) { + const void *config_data; + struct k_event *event; + struct ipc4_flat *flat; + unsigned int core; + int ret; + if (!pmod) { tr_err(&dp_tr, "no thread module"); return -EINVAL; } - struct task_dp_pdata *pdata = pmod->dev->task->priv_data; - unsigned int core = pmod->dev->task->core; - int ret; + if (k_is_user_context()) { + struct task_dp_pdata *pdata = pmod->dev->task->priv_data; + + event = pdata->event; + flat = pdata->flat; + config_data = NULL; + core = pmod->dev->task->core; + } else { + /* + * The module and its task memory are writable by user threads, + * so the kernel takes the DP task objects from its own record. + */ + struct scheduler_dp_task_kobj kobj; + + if (!dp_task_kobj_peek_mod(pmod, &kobj)) { + tr_err(&dp_tr, "no DP task for module"); + return -EINVAL; + } + + event = kobj.event; + flat = kobj.flat; + config_data = kobj.config_data; + core = kobj.core; + } if (cmd == SOF_IPC4_MOD_INIT_INSTANCE) { /* Wait for the DP thread to start */ @@ -342,11 +395,11 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, unsigned int lock_key = scheduler_dp_lock(core); /* IPCs are serialised */ - pdata->flat->ret = -ENOSYS; + flat->ret = -ENOSYS; - ret = ipc_thread_flatten(cmd, param, pdata->flat); + ret = ipc_thread_flatten(cmd, param, flat); if (!ret) - k_event_post(pdata->event, DP_TASK_EVENT_IPC); + k_event_post(event, DP_TASK_EVENT_IPC); scheduler_dp_unlock(lock_key); @@ -360,14 +413,26 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, return ret; } - if (!pdata->flat->ret && cmd == SOF_IPC4_MOD_LARGE_CONFIG_GET) { - /* A single case of returning data */ - param->get_config.data_offset_size = pdata->flat->get_config.data_offset_size; + ret = flat->ret; + if (!ret && cmd == SOF_IPC4_MOD_LARGE_CONFIG_GET) { + /* + * A single case of returning data. The kernel only copies from + * the buffer in its record, which is NULL if the module had no + * get_configuration at init. User threads use the pointer set by + * the DP thread, they cannot read more through it than they can + * read anyway. + */ + if (k_is_user_context()) + config_data = flat->get_config.data; + if (!config_data) + return -EINVAL; + + param->get_config.data_offset_size = flat->get_config.data_offset_size; return memcpy_s(param->get_config.data, SOF_IPC_MSG_MAX_SIZE, - pdata->flat->get_config.data, param->get_config.fragment_size); + config_data, param->get_config.fragment_size); } - return pdata->flat->ret; + return ret; } /* Go through all DP tasks and recalculate their readiness and deadlines @@ -375,15 +440,25 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, */ void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch) { + struct scheduler_dp_kick kicks[SCHEDULER_DP_KICK_MAX]; struct list_item *tlist; struct task *curr_task; struct task_dp_pdata *pdata; + unsigned int n = 0; + /* + * The task list and task memory are writable by DP threads. This may + * run in the LL user thread, see scheduler_dp_ll_post_tick(), so the + * DP thread objects are only acted on by scheduler_dp_kick(), which + * takes them from the kernel-only task records. + */ list_for_item(tlist, &dp_sch->tasks) { curr_task = container_of(tlist, struct task, list); pdata = curr_task->priv_data; struct processing_module *mod = pdata->mod; bool trigger_task = false; + uint32_t flags = 0; + uint32_t deadline_cycles = 0; /* decrease number of LL ticks/cycles left till the module reaches its deadline */ if (mod->dp_startup_delay && pdata->ll_cycles_to_start) { @@ -400,7 +475,7 @@ void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch) /* trigger the task */ curr_task->state = SOF_TASK_STATE_RUNNING; trigger_task = true; - k_event_post(pdata->event, DP_TASK_EVENT_PROCESS); + flags |= SCHEDULER_DP_KICK_PROCESS; } if (curr_task->state == SOF_TASK_STATE_RUNNING) { @@ -418,20 +493,111 @@ void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch) /* round down to 1ms */ deadline = deadline / 1000; - /* calculate number of ticks */ - deadline = deadline * (CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC / 1000); + /* calculate number of ticks, scheduler_dp_kick() adds "NOW" */ + deadline_cycles = deadline * + (CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC / 1000); + flags |= SCHEDULER_DP_KICK_DEADLINE; + } + } - /* add to "NOW", overflows are OK */ - deadline = dp_sch->last_ll_tick_timestamp + deadline; + if (!flags) + continue; - /* set in Zephyr. Note that it may be in past, it does not matter, - * Zephyr still will schedule the thread with earlier deadline - * first - */ - k_thread_absolute_deadline_set(pdata->thread_id, deadline); - } + kicks[n].task = curr_task; + kicks[n].flags = flags; + kicks[n].deadline = deadline_cycles; + if (++n == ARRAY_SIZE(kicks)) { + scheduler_dp_kick(kicks, n); + n = 0; + } + } + + if (n) + scheduler_dp_kick(kicks, n); +} + +unsigned int scheduler_dp_task_count(unsigned int core) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + struct scheduler_dp_task_kobj *kobj; + struct list_item *item; + unsigned int count = 0; + + /* + * Task memory is only freed after its record is unlinked, so it can be + * read here. Only the state value is used, no pointers from it: a + * forged state only changes whether that task is counted. + */ + list_for_item(item, &dp_task_kobj_list) { + kobj = container_of(item, struct scheduler_dp_task_kobj, list); + if (kobj->core == core && task_is_active(kobj->task)) + count++; + } + + k_spin_unlock(&dp_task_kobj_lock, key); + + return count; +} + +#if CONFIG_SOF_BOOT_TEST_STANDALONE +/* Add a DP task record for zephyr/test/userspace/test_dp_kick.c */ +int scheduler_dp_test_record_add(struct task *task, struct k_event *event, + struct k_thread *thread, uint16_t core) +{ + struct scheduler_dp_task_kobj *kobj = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + sizeof(*kobj)); + + if (!kobj) + return -ENOMEM; + + kobj->task = task; + kobj->event = event; + kobj->thread = thread; + kobj->core = core; + dp_task_kobj_add(kobj); + + return 0; +} + +void scheduler_dp_test_record_del(struct task *task) +{ + rfree(dp_task_kobj_take(task, NULL)); +} +#endif + +int z_impl_scheduler_dp_kick(const struct scheduler_dp_kick *kicks, uint32_t count) +{ + struct scheduler_dp_task_kobj *kobj; + unsigned int core = cpu_get_id(); + uint32_t now = k_cycle_get_32(); + k_spinlock_key_t key; + int skipped = 0; + uint32_t i; + + /* the record cannot be freed while dp_task_kobj_lock is held */ + key = k_spin_lock(&dp_task_kobj_lock); + + for (i = 0; i < count; i++) { + kobj = dp_task_kobj_find(kicks[i].task); + if (!kobj || kobj->core != core) { + skipped++; + continue; } + + /* + * Overflows are OK. The deadline may be in the past, Zephyr + * still schedules the thread with the earliest deadline first. + */ + if (kicks[i].flags & SCHEDULER_DP_KICK_DEADLINE) + k_thread_absolute_deadline_set(kobj->thread, now + kicks[i].deadline); + + if (kicks[i].flags & SCHEDULER_DP_KICK_PROCESS) + k_event_post(kobj->event, DP_TASK_EVENT_PROCESS); } + + k_spin_unlock(&dp_task_kobj_lock, key); + + return skipped; } /* Thread function called in component context, on target core */ @@ -684,6 +850,7 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, kobj->task = ptask; kobj->mod = mod; + kobj->core = core; comp_id = mod->dev->ipc_config.id; kobj->comp_id = comp_id; @@ -696,8 +863,10 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, } } pdata->ipc_config_data = ipc_config_data; + kobj->config_data = ipc_config_data; - pdata->flat = &task_memory->flat; + kobj->flat = &task_memory->flat; + pdata->flat = kobj->flat; kobj->event = k_object_alloc(K_OBJ_EVENT); if (!kobj->event) { @@ -926,4 +1095,19 @@ void z_vrfy_scheduler_dp_internal_free(struct task *task) K_OOPS(!scheduler_dp_internal_free_kobj(task, &kobj)); } #include + +int z_vrfy_scheduler_dp_kick(const struct scheduler_dp_kick *kicks, uint32_t count) +{ + struct scheduler_dp_kick kkicks[SCHEDULER_DP_KICK_MAX]; + + /* without user-space LL, only the kernel LL scheduler kicks DP tasks */ + K_OOPS(!IS_ENABLED(CONFIG_SOF_USERSPACE_LL)); + SOF_SYSCALL_CALLER_CHECK(SOF_SYSUSER_LL); + + K_OOPS(K_SYSCALL_VERIFY(count <= ARRAY_SIZE(kkicks))); + K_OOPS(k_usermode_from_copy(kkicks, kicks, count * sizeof(kkicks[0]))); + + return z_impl_scheduler_dp_kick(kkicks, count); +} +#include #endif /* CONFIG_USERSPACE */ diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index b83a4ae0f115..eb1eb4e222d4 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -327,7 +327,7 @@ static void zephyr_ll_run(void *data) zephyr_ll_unlock(sch, &flags); #ifdef CONFIG_ZEPHYR_DP_SCHEDULER - scheduler_dp_ll_tick(); + scheduler_dp_ll_post_tick(); #endif } diff --git a/zephyr/test/CMakeLists.txt b/zephyr/test/CMakeLists.txt index e64b61314da4..54dccaa5986f 100644 --- a/zephyr/test/CMakeLists.txt +++ b/zephyr/test/CMakeLists.txt @@ -37,3 +37,8 @@ if(CONFIG_SOF_BOOT_TEST_STANDALONE AND CONFIG_SOF_USERSPACE_SYSCALL_CALLER_CHECK CONFIG_IPC_MAJOR_4) zephyr_library_sources(userspace/test_syscall_caller.c) endif() + +if(CONFIG_SOF_BOOT_TEST_STANDALONE AND CONFIG_SOF_USERSPACE_SYSCALL_CALLER_CHECK AND + CONFIG_SOF_USERSPACE_APPLICATION AND CONFIG_ZEPHYR_DP_SCHEDULER) + zephyr_library_sources(userspace/test_dp_kick.c) +endif() diff --git a/zephyr/test/userspace/test_dp_kick.c b/zephyr/test/userspace/test_dp_kick.c new file mode 100644 index 000000000000..926c673b7272 --- /dev/null +++ b/zephyr/test/userspace/test_dp_kick.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2026 Intel Corporation. + */ + +/* + * Tests for scheduler_dp_kick(): the DP thread objects are taken from the + * kernel-only DP task records, never from the caller, and only the LL user + * thread may call it. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +LOG_MODULE_DECLARE(sof_boot_test, LOG_LEVEL_DBG); + +#define USER_STACKSIZE 2048 +#define TEST_DEADLINE 1000 + +/* registry slot used for the test thread, not used by any LL thread here */ +#define TEST_LL_ROLE_CORE (CONFIG_CORE_COUNT - 1) + +static struct k_thread user_thread; +static K_THREAD_STACK_DEFINE(user_stack, USER_STACKSIZE); + +/* stand-in DP thread, never started, only its deadline is checked */ +static struct k_thread dp_thread; +static K_THREAD_STACK_DEFINE(dp_stack, 512); +static struct k_event dp_event; + +/* DP tasks, only their addresses are used as record keys */ +static struct task task_here; +static struct task task_other_core; +static struct task task_unknown; + +static struct scheduler_dp_kick kernel_only_kicks[1]; + +enum kick_test_case { + /* kick the task of this core */ + KICK_THIS_CORE, + /* kick the task of another core */ + KICK_OTHER_CORE, + /* kick a task without a record */ + KICK_UNKNOWN_TASK, + /* pass more entries than allowed */ + KICK_TOO_MANY, + /* pass a kernel address as the array */ + KICK_KERNEL_ARRAY, +}; + +static void dp_thread_fn(void *p1, void *p2, void *p3) +{ +} + +static void user_function(void *p1, void *p2, void *p3) +{ + enum kick_test_case test_case = (enum kick_test_case)(uintptr_t)p1; + struct scheduler_dp_kick kicks[SCHEDULER_DP_KICK_MAX + 1] = { + [0] = { + .flags = SCHEDULER_DP_KICK_PROCESS | SCHEDULER_DP_KICK_DEADLINE, + .deadline = TEST_DEADLINE, + }, + }; + + zassert_true(k_is_user_context(), "isn't user"); + + switch (test_case) { + case KICK_THIS_CORE: + kicks[0].task = &task_here; + zassert_equal(scheduler_dp_kick(kicks, 1), 0); + return; + case KICK_OTHER_CORE: + kicks[0].task = &task_other_core; + zassert_equal(scheduler_dp_kick(kicks, 1), 1); + return; + case KICK_UNKNOWN_TASK: + kicks[0].task = &task_unknown; + zassert_equal(scheduler_dp_kick(kicks, 1), 1); + return; + case KICK_TOO_MANY: + (void)scheduler_dp_kick(kicks, ARRAY_SIZE(kicks)); + break; + case KICK_KERNEL_ARRAY: + (void)scheduler_dp_kick(kernel_only_kicks, 1); + break; + default: + zassert_unreachable("unknown test case"); + } + + zassert_unreachable("syscall security check did not fault"); +} + +static void run_case(enum kick_test_case test_case, bool register_ll, bool expect_fault) +{ + int ret; + + k_thread_create(&user_thread, user_stack, USER_STACKSIZE, + user_function, (void *)(uintptr_t)test_case, NULL, NULL, + -1, K_USER, K_FOREVER); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), &user_thread); + /* records are checked against the core the kick runs on */ + zassert_equal(k_thread_cpu_pin(&user_thread, cpu_get_id()), 0); + + if (register_ll) + sof_sysuser_thread_set(SOF_SYSUSER_LL, TEST_LL_ROLE_CORE, &user_thread); + if (expect_fault) + sof_boot_test_set_fault_valid(&user_thread); + + k_thread_start(&user_thread); + ret = k_thread_join(&user_thread, K_FOREVER); + + sof_sysuser_thread_set(SOF_SYSUSER_LL, TEST_LL_ROLE_CORE, NULL); + sof_boot_test_set_fault_valid(NULL); + + zassert_equal(ret, 0, "user thread join failed: %d", ret); +} + +static void *dp_kick_setup(void) +{ + unsigned int other_core = (cpu_get_id() + 1) % CONFIG_CORE_COUNT; + + k_event_init(&dp_event); + k_thread_create(&dp_thread, dp_stack, K_THREAD_STACK_SIZEOF(dp_stack), + dp_thread_fn, NULL, NULL, NULL, 0, 0, K_FOREVER); + + zassert_equal(scheduler_dp_test_record_add(&task_here, &dp_event, &dp_thread, + cpu_get_id()), 0); + zassert_equal(scheduler_dp_test_record_add(&task_other_core, &dp_event, &dp_thread, + other_core), 0); + + return NULL; +} + +static void dp_kick_teardown(void *fixture) +{ + scheduler_dp_test_record_del(&task_here); + scheduler_dp_test_record_del(&task_other_core); + k_thread_abort(&dp_thread); +} + +static void dp_kick_before(void *fixture) +{ + k_event_clear(&dp_event, UINT32_MAX); +} + +ZTEST(userspace_dp_kick, ll_thread_kicks_own_core_task) +{ + uint32_t before = k_cycle_get_32(); + uint32_t after; + uint32_t start; + + run_case(KICK_THIS_CORE, true, false); + after = k_cycle_get_32(); + + zassert_equal(k_event_test(&dp_event, UINT32_MAX), DP_TASK_EVENT_PROCESS); + + /* the deadline is relative to the time of the call */ + start = (uint32_t)dp_thread.base.prio_deadline - TEST_DEADLINE; + zassert_true(start - before <= after - before, "unexpected deadline"); +} + +ZTEST(userspace_dp_kick, other_core_task_skipped) +{ + if (CONFIG_CORE_COUNT < 2) + ztest_test_skip(); + + run_case(KICK_OTHER_CORE, true, false); + zassert_equal(k_event_test(&dp_event, UINT32_MAX), 0); +} + +ZTEST(userspace_dp_kick, unknown_task_skipped) +{ + run_case(KICK_UNKNOWN_TASK, true, false); + zassert_equal(k_event_test(&dp_event, UINT32_MAX), 0); +} + +ZTEST(userspace_dp_kick, unregistered_thread_rejected) +{ + run_case(KICK_THIS_CORE, false, true); + zassert_equal(k_event_test(&dp_event, UINT32_MAX), 0); +} + +ZTEST(userspace_dp_kick, too_many_entries_rejected) +{ + run_case(KICK_TOO_MANY, true, true); +} + +ZTEST(userspace_dp_kick, kernel_array_rejected) +{ + run_case(KICK_KERNEL_ARRAY, true, true); +} + +ZTEST_SUITE(userspace_dp_kick, sof_boot_test_standalone_predicate, dp_kick_setup, + dp_kick_before, NULL, dp_kick_teardown); + +/** + * SOF main has booted up and IPC handling is stopped. + * Run test suites with ztest_run_all. + */ +static int run_tests(void) +{ + ztest_run_test_suite(userspace_dp_kick, false, 1, 1, NULL); + return 0; +} + +SYS_INIT(run_tests, APPLICATION, 99); diff --git a/zephyr/test/userspace/test_syscall_caller.c b/zephyr/test/userspace/test_syscall_caller.c index 416ea518dc2d..4db957cf2636 100644 --- a/zephyr/test/userspace/test_syscall_caller.c +++ b/zephyr/test/userspace/test_syscall_caller.c @@ -103,7 +103,10 @@ static void user_function(void *p1, void *p2, void *p3) zassert_equal(ipc_wait_for_compound_msg(), 0); return; case CALLER_IPC_LL_SYSCALL: -#if CONFIG_ZEPHYR_DP_SCHEDULER +#if CONFIG_ZEPHYR_DP_SCHEDULER && CONFIG_SOF_USERSPACE_APPLICATION + (void)scheduler_dp_kick(NULL, 0); + break; +#elif CONFIG_ZEPHYR_DP_SCHEDULER scheduler_dp_ll_tick(); break; #else