diff --git a/app/overlays/ptl/ll_userspace_overlay.conf b/app/overlays/ptl/ll_userspace_overlay.conf index 53b881320e6f..90082d26e16f 100644 --- a/app/overlays/ptl/ll_userspace_overlay.conf +++ b/app/overlays/ptl/ll_userspace_overlay.conf @@ -11,24 +11,19 @@ CONFIG_SOF_USERSPACE_LL=y # make the drivers work in user-space CONFIG_SOF_USERSPACE_INTERFACE_DMA=y CONFIG_DAI_USERSPACE=y +CONFIG_MAX_THREAD_BYTES=4 # Temporary settings that are needed currently to enable user-space LL # -------------------------------------------------------------------- # problem with DSP panics due to illegal instruction hit in user-space if cold # store execution is enabled. Disable it for now until rootcause is found. -CONFIG_COLD_STORE_EXECUTE_DRAM=n CONFIG_COLD_STORE_EXECUTE_DEBUG=n # telemetry not yet user-space compatible CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n -# disable loadable modules (hits privilege issues in user-space now) -CONFIG_LLEXT_STORAGE_WRITABLE=n -CONFIG_LLEXT_EXPERIMENTAL=n -CONFIG_MODULES=n - # some of current boot tests interfere with user-space setup CONFIG_SOF_BOOT_TEST_ALLOWED=n @@ -36,4 +31,3 @@ CONFIG_SOF_BOOT_TEST_ALLOWED=n CONFIG_CROSS_CORE_STREAM=n CONFIG_INTEL_ADSP_MIC_PRIVACY=n CONFIG_XRUN_NOTIFICATIONS_ENABLE=n -CONFIG_ZEPHYR_DP_SCHEDULER=n diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index 8a3d44133d4b..37d2cc07e16c 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -166,7 +166,7 @@ static void comp_buffer_free(struct sof_audio_buffer *audio_buffer) if (alloc && alloc->vreg) { vregion_free(alloc->vreg, buffer); if (!vregion_put(alloc->vreg)) - rfree(alloc); + sof_heap_free(alloc->heap, alloc); } else { sof_heap_free(alloc ? alloc->heap : NULL, buffer); } diff --git a/src/audio/module_adapter/module_adapter.c b/src/audio/module_adapter/module_adapter.c index 995501bfbe0a..0ac5de8edf1e 100644 --- a/src/audio/module_adapter/module_adapter.c +++ b/src/audio/module_adapter/module_adapter.c @@ -51,16 +51,6 @@ struct comp_dev *module_adapter_new(const struct comp_driver *drv, return module_adapter_new_ext(drv, config, spec, NULL, NULL, NULL); } -static struct vregion *module_adapter_dp_heap_new(const struct comp_ipc_config *config, - size_t *heap_size) -{ - /* src-lite with 8 channels has been seen allocating 14k in one go */ - /* FIXME: the size will be derived from configuration */ - const size_t buf_size = 28 * 1024; - - return vregion_create(buf_size); -} - static struct processing_module *module_adapter_mem_alloc(const struct comp_driver *drv, const struct comp_ipc_config *config) { @@ -77,11 +67,15 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv */ uint32_t flags = config->proc_domain == COMP_PROCESSING_DOMAIN_DP ? SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT : SOF_MEM_FLAG_USER; - size_t heap_size; + size_t vreg_size; + uintptr_t vreg_start; if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP && IS_ENABLED(CONFIG_SOF_VREGIONS) && IS_ENABLED(CONFIG_USERSPACE) && !IS_ENABLED(CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP)) { - mod_vreg = module_adapter_dp_heap_new(config, &heap_size); + /* src-lite with 8 channels has been seen allocating 14k in one go */ + /* FIXME: the size will be derived from configuration */ + vreg_size = 28 * 1024; + mod_vreg = vregion_create_map(&vreg_start, &vreg_size); if (!mod_vreg) { comp_cl_err(drv, "Failed to allocate DP module heap / vregion"); return NULL; @@ -98,7 +92,8 @@ static struct processing_module *module_adapter_mem_alloc(const struct comp_driv #else mod_heap = drv->user_heap; #endif - heap_size = 0; + vreg_size = 0; + vreg_start = 0; mod_vreg = NULL; } @@ -199,6 +194,7 @@ static void module_adapter_mem_free(struct processing_module *mod) * * Note: Use the ext version if you need to set the module's private data before calling * the create method. + * Note 2: ATM runs in privileged / kernel mode for DP modules */ struct comp_dev *module_adapter_new_ext(const struct comp_driver *drv, const struct comp_ipc_config *config, diff --git a/src/audio/mux/mux_ipc4.c b/src/audio/mux/mux_ipc4.c index f5ead3a93368..8f33e44ea1ae 100644 --- a/src/audio/mux/mux_ipc4.c +++ b/src/audio/mux/mux_ipc4.c @@ -148,7 +148,7 @@ int mux_params(struct processing_module *mod) int ret; cfg = comp_get_data_blob(cd->model_handler, &blob_size, NULL); - if (!cfg || blob_size > MUX_BLOB_MAX_SIZE) { + if (!cfg || blob_size != sizeof(*cfg)) { comp_err(mod->dev, "illegal blob size %zu", blob_size); return -EINVAL; } diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index 670df08e9383..b292b7095cd3 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include diff --git a/src/include/ipc4/handler.h b/src/include/ipc4/handler.h index d6f839458d54..cb54ecc08db0 100644 --- a/src/include/ipc4/handler.h +++ b/src/include/ipc4/handler.h @@ -16,6 +16,14 @@ struct ipc4_message_request; */ int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); +/** + * \brief Load a dynamically loadable module. + * @param[in] drv Component driver. + * @param[in] mi SOF_IPC4_MOD_INIT_INSTANCE data + */ +int ipc4_user_module_load(const struct comp_driver *drv, + const struct ipc4_module_init_instance *mi); + /** * @brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context. * @param[in] ipc4 IPC4 message request. diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index a910c6d42c92..328a5e85327f 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -57,9 +57,9 @@ extern struct tr_ctx ipc_tr; #define IPC_TASK_POWERDOWN BIT(3) struct ipc_user { - struct k_thread *thread; + struct k_thread *thread[CONFIG_CORE_COUNT]; struct k_sem *sem; - struct k_event *event; + struct k_event *event[CONFIG_CORE_COUNT]; /** @brief Copy of IPC4 message primary word forwarded to user thread */ uint32_t ipc_msg_pri; /** @brief Copy of IPC4 message extension word forwarded to user thread */ @@ -73,9 +73,10 @@ struct ipc_user { /** @brief Reply TX data pointer from user thread (e.g. LARGE_CONFIG_GET result) */ void *reply_tx_data; struct ipc *ipc; - struct k_thread *audio_thread; + struct k_thread *audio_thread[CONFIG_CORE_COUNT]; /** @brief Original kernel driver pointer for restoring dev->drv after create */ const struct comp_driver *init_drv; + bool init_needed[CONFIG_CORE_COUNT]; /** * @brief User-accessible copy of comp_driver + tr_ctx for create(). * @@ -326,7 +327,7 @@ extern bool ipc_enter_gdb; * @param extension Extension message word * @return Result code from user thread processing */ -int ipc_user_forward_cmd(uint32_t primary, uint32_t extension); +int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core); /** * @brief Protocol-specific dispatch of a forwarded IPC command. @@ -338,6 +339,19 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension); * @return Result code to report back to the host */ int ipc_user_thread_dispatch(struct ipc_user *ipc_user); + +/** + * @brief Initialize IPC and LL scheduler threads on a booting secondary core. + * + * @param core Secondary core ID + * @return 0 or a negative error code + */ +int ipc_user_init_secondary(unsigned int core); #endif +/** + * \brief get pointer to the userspace IPC thread for core + */ +struct k_thread *ipc_thread_user(unsigned int core); + #endif /* __SOF_DRIVERS_IPC_H__ */ diff --git a/src/include/sof/lib/vregion.h b/src/include/sof/lib/vregion.h index 5c066c90dbc8..559c47ed3431 100644 --- a/src/include/sof/lib/vregion.h +++ b/src/include/sof/lib/vregion.h @@ -6,6 +6,8 @@ #define __SOF_LIB_VREGION_H__ #include +#include +#include #ifdef __cplusplus extern "C" { @@ -40,6 +42,13 @@ enum vregion_mem_type { */ struct vregion *vregion_create(size_t memsize); +/** + * @brief Create and map a new virtual region instance. + * + * Like above, but usable by userspace and can add domain access. + */ +__syscall struct vregion *vregion_create_map(uintptr_t *vreg_start, size_t *vreg_size); + /** * @brief Switch virtual region allocations to interim mode. * @@ -49,7 +58,7 @@ struct vregion *vregion_create(size_t memsize); * * @param[in] vr Pointer to the virtual region instance. */ -void vregion_set_interim(struct vregion *vr); +__syscall void vregion_set_interim(struct vregion *vr); /** * @brief Increment virtual region's user count. @@ -60,7 +69,7 @@ void vregion_set_interim(struct vregion *vr); * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance. */ -struct vregion *vregion_get(struct vregion *vr); +__syscall struct vregion *vregion_get(struct vregion *vr); /** * @brief Decrement virtual region's user count or destroy it. @@ -71,39 +80,45 @@ struct vregion *vregion_get(struct vregion *vr); * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance or NULL if it has been destroyed. */ -struct vregion *vregion_put(struct vregion *vr); +__syscall struct vregion *vregion_put(struct vregion *vr); /** - * @brief Allocate memory from the specified virtual region. + * @brief Allocate aligned memory from the specified virtual region. + * + * Allocate aligned memory from the specified virtual region using the + * current allocation mode (lifetime or interim). * * @param[in] vr Pointer to the virtual region instance. * @param[in] size Size of memory to allocate in bytes. + * @param[in] alignment Alignment of memory to allocate in bytes. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc(struct vregion *vr, size_t size); +__syscall void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); /** - * @brief like vregion_alloc() but allocates coherent memory + * @brief like vregion_alloc_align() but allocates coherent memory */ -void *vregion_alloc_coherent(struct vregion *vr, size_t size); +__syscall void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); /** - * @brief Allocate aligned memory from the specified virtual region. - * - * Allocate aligned memory from the specified virtual region using the - * current allocation mode (lifetime or interim). + * @brief Allocate memory from the specified virtual region. * * @param[in] vr Pointer to the virtual region instance. * @param[in] size Size of memory to allocate in bytes. - * @param[in] alignment Alignment of memory to allocate in bytes. * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment); +static inline void *vregion_alloc(struct vregion *vr, size_t size) +{ + return vregion_alloc_align(vr, size, 0); +} /** - * @brief like vregion_alloc_align() but allocates coherent memory + * @brief like vregion_alloc() but allocates coherent memory */ -void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment); +static inline void *vregion_alloc_coherent(struct vregion *vr, size_t size) +{ + return vregion_alloc_coherent_align(vr, size, 0); +} /** * @brief Free memory allocated from the specified virtual region. @@ -113,7 +128,7 @@ void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t align * @param[in] vr Pointer to the virtual region instance. * @param[in] ptr Pointer to the memory to free. */ -void vregion_free(struct vregion *vr, void *ptr); +__syscall void vregion_free(struct vregion *vr, void *ptr); /** * @brief Log virtual region memory usage. @@ -131,6 +146,10 @@ void vregion_info(struct vregion *vr); */ void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start); +bool vregion_verify(struct vregion *vr); + +#include + #else /* CONFIG_SOF_VREGIONS */ struct vregion { @@ -141,6 +160,10 @@ static inline struct vregion *vregion_create(size_t memsize) { return NULL; } +static inline struct vregion *vregion_create_map(uintptr_t *vreg_start, size_t *vreg_size) +{ + return NULL; +} static inline void vregion_set_interim(struct vregion *vr) {} static inline struct vregion *vregion_get(struct vregion *vr) { @@ -174,6 +197,7 @@ static inline void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t if (size) *size = 0; } +static inline bool vregion_verify(struct vregion *vr) {return false;} #endif /* CONFIG_SOF_VREGIONS */ diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index 019695be69ef..ceb47e6abe6c 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -220,6 +220,25 @@ void lib_manager_get_instance_bss_address(uint32_t instance_id, */ int lib_manager_load_library(uint32_t dma_id, uint32_t lib_id, uint32_t type); +struct userspace_context; +/* + * \brief Allocate the module and start the agent if needed + */ +int lib_manager_mod_create_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec, void **adapter_priv, + struct userspace_context **userspace, + const struct module_interface **ops); + +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +__syscall int lib_manager_free_module(const uint32_t component_id); + +#include +#else +int z_impl_lib_manager_free_module(const uint32_t component_id); +#define lib_manager_free_module z_impl_lib_manager_free_module +#endif + /* * \brief Initialize message * diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index 1b55405d0fb1..e61b6c5670e7 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -78,7 +78,12 @@ int scheduler_dp_task_init(struct task **task, uint16_t core, size_t stack_size, uint32_t options); -void scheduler_dp_ll_tick(void); + +#if defined(__ZEPHYR__) && CONFIG_SOF_FULL_ZEPHYR_APPLICATION +__syscall void scheduler_dp_internal_free(struct task *task); +__syscall void scheduler_dp_ll_tick(unsigned int core); +#include +#endif /** * \brief Extract information about scheduler's tasks diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index 03991debb0cb..cfd0043995db 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -330,6 +330,7 @@ struct ll_schedule_domain *zephyr_domain_init(int clk); struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain); struct k_thread *zephyr_domain_thread_tid_for_core(int core); struct k_mem_domain *zephyr_ll_mem_domain(void); +struct k_thread *zephyr_ll_domain_thread(void); #endif /* CONFIG_SOF_USERSPACE_LL */ #ifdef CONFIG_SOF_FULL_ZEPHYR_APPLICATION __syscall int zephyr_ll_task_sem_alloc(struct task *task); diff --git a/src/include/sof/schedule/schedule.h b/src/include/sof/schedule/schedule.h index 32b09f24f473..f28f472b121f 100644 --- a/src/include/sof/schedule/schedule.h +++ b/src/include/sof/schedule/schedule.h @@ -186,14 +186,8 @@ struct schedulers **arch_user_schedulers_get(void); struct schedulers **arch_user_schedulers_get_for_core(int core); -/** - * Retrieves scheduler's data. - * @param type SOF_SCHEDULE_ type. - * @return Pointer to scheduler's data. - */ -static inline void *scheduler_get_data(uint16_t type) +static inline void *scheduler_list_get_data(struct schedulers *schedulers, uint16_t type) { - struct schedulers *schedulers = *arch_schedulers_get(); struct schedule_data *sch; struct list_item *slist; @@ -209,6 +203,26 @@ static inline void *scheduler_get_data(uint16_t type) return NULL; } +/** + * Retrieves scheduler's data. + * @param type SOF_SCHEDULE_ type. + * @return Pointer to scheduler's data. + */ +static inline void *scheduler_get_data(uint16_t type) +{ + return scheduler_list_get_data(*arch_schedulers_get(), type); +} + +/** + * Retrieves userspace scheduler's data. + * @param type SOF_SCHEDULE_ type. + * @return Pointer to scheduler's data. + */ +static inline void *scheduler_get_user_data(uint16_t type) +{ + return scheduler_list_get_data(*arch_user_schedulers_get(), type); +} + /** See scheduler_ops::schedule_task_running */ static inline int schedule_task_running(struct task *task) { diff --git a/src/init/init.c b/src/init/init.c index 5990cfebc2dc..ba650dfc5256 100644 --- a/src/init/init.c +++ b/src/init/init.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -110,6 +111,7 @@ static inline int secondary_core_restore(void) { return 0; }; __cold int secondary_core_init(struct sof *sof) { + unsigned int core = cpu_get_id(); int err; struct ll_schedule_domain *dma_domain; @@ -134,6 +136,12 @@ __cold int secondary_core_init(struct sof *sof) if (dma_domain) scheduler_init_ll(dma_domain); +#if CONFIG_SOF_USERSPACE_LL + err = ipc_user_init_secondary(core); + if (err < 0) + return err; +#endif + #if CONFIG_ZEPHYR_DP_SCHEDULER err = scheduler_dp_init(); if (err < 0) @@ -152,7 +160,7 @@ __cold int secondary_core_init(struct sof *sof) return err; #endif #if CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL - err = core_kcps_adjust(cpu_get_id(), SECONDARY_CORE_BASE_CPS_USAGE); + err = core_kcps_adjust(core, SECONDARY_CORE_BASE_CPS_USAGE); if (err < 0) return err; #endif diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index c5e2727ab3cb..dd1a1ee6ee01 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -318,7 +318,10 @@ void ipc_schedule_process(struct ipc *ipc) #define IPC_USER_EVENT_CMD BIT(0) #define IPC_USER_EVENT_STOP BIT(1) -static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); +SOF_DEFINE_REG_UUID(sec_core_init); + +static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); /** * @brief Forward an IPC command to the user-space thread. @@ -332,7 +335,7 @@ static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SI * @param extension Extension message word * @return Result from user thread processing */ -int ipc_user_forward_cmd(uint32_t primary, uint32_t extension) +int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core) { struct ipc *ipc = ipc_get(); struct ipc_user *pdata = ipc->ipc_user_pdata; @@ -346,13 +349,22 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension) pdata->ipc_msg_ext = extension; pdata->ipc = ipc; + /* + * Forwarding the first IPC to this core, wait for its userspace IPC + * thread to start + */ + if (pdata->init_needed[core]) { + pdata->init_needed[core] = false; + k_sem_take(pdata->sem, K_FOREVER); + } + /* Prevent host completion until user thread finishes */ key = k_spin_lock(&ipc->lock); ipc->task_mask |= IPC_TASK_IN_THREAD; k_spin_unlock(&ipc->lock, key); /* Wake the user thread */ - k_event_set(pdata->event, IPC_USER_EVENT_CMD); + k_event_set(pdata->event[core], IPC_USER_EVENT_CMD); /* Wait for user thread to complete */ ret = k_sem_take(pdata->sem, K_MSEC(100)); @@ -389,8 +401,8 @@ __weak int ipc_user_thread_dispatch(struct ipc_user *ipc_user) static void ipc_user_thread_fn(void *p1, void *p2, void *p3) { struct ipc_user *ipc_user = p1; + unsigned int core = POINTER_TO_UINT(p2); - ARG_UNUSED(p2); ARG_UNUSED(p3); __ASSERT(k_is_user_context(), "expected user context"); @@ -400,7 +412,7 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) LOG_INF("IPC user-space thread started"); for (;;) { - uint32_t mask = k_event_wait_safe(ipc_user->event, + uint32_t mask = k_event_wait_safe(ipc_user->event[core], IPC_USER_EVENT_CMD | IPC_USER_EVENT_STOP, false, K_FOREVER); @@ -418,7 +430,7 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) } } -__cold static int ipc_user_init_thread(struct ipc_user *ipc_user) +__cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int core) { char thread_name[] = "ll_user0"; int ret; @@ -426,47 +438,93 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user) assert_can_be_cold(); /* Allocate kernel objects for the user-space thread */ - ipc_user->event = k_object_alloc(K_OBJ_EVENT); - if (!ipc_user->event) { + ipc_user->event[core] = k_object_alloc(K_OBJ_EVENT); + if (!ipc_user->event[core]) { LOG_ERR("user IPC event alloc failed"); return -ENOMEM; } - k_event_init(ipc_user->event); + k_event_init(ipc_user->event[core]); - ipc_user->thread = k_object_alloc(K_OBJ_THREAD); - if (!ipc_user->thread) { + ipc_user->thread[core] = k_object_alloc(K_OBJ_THREAD); + if (!ipc_user->thread[core]) { LOG_ERR("user IPC thread alloc failed"); ret = -ENOMEM; goto e_event; } - k_thread_create(ipc_user->thread, ipc_user_stack, + k_thread_create(ipc_user->thread[core], ipc_user_stack[core], CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, - ipc_user_thread_fn, ipc_user, NULL, NULL, + ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, -1, K_USER, K_FOREVER); - k_thread_cpu_pin(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID); - k_thread_name_set(ipc_user->thread, thread_name); + k_thread_cpu_pin(ipc_user->thread[core], core); + thread_name[sizeof(thread_name) - 2] = '0' + core; + k_thread_name_set(ipc_user->thread[core], thread_name); /* * Each userspace IPC thread must be able to wait on its private event * and signal completion on the primary core semaphore */ - k_thread_access_grant(ipc_user->thread, ipc_user->sem, ipc_user->event); - user_grant_dai_access_all(ipc_user->thread); - user_grant_dma_access_all(ipc_user->thread); - k_mem_domain_add_thread(zephyr_ll_mem_domain(), ipc_user->thread); - user_ll_grant_access(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID); - pipeline_posn_grant_access(ipc_user->thread); + k_thread_access_grant(ipc_user->thread[core], ipc_user->sem, ipc_user->event[core]); + user_grant_dai_access_all(ipc_user->thread[core]); + user_grant_dma_access_all(ipc_user->thread[core]); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), ipc_user->thread[core]); + user_ll_grant_access(ipc_user->thread[core], core); + pipeline_posn_grant_access(ipc_user->thread[core]); return 0; e_event: - k_object_free(ipc_user->event); + k_object_free(ipc_user->event[core]); return ret; } +__cold int ipc_user_init_secondary(unsigned int core) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *ipc_user = ipc->ipc_user_pdata; + int ret = ipc_user_init_thread(ipc_user, core); + + if (ret < 0) + return ret; + + assert_can_be_cold(); + + k_thread_start(ipc_user->thread[core]); + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("user LL task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, core, 0); + + ipc_user->audio_thread[core] = scheduler_init_context(task); + if (!ipc_user->audio_thread[core]) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + k_thread_access_grant(ipc_user->thread[core], ipc_user->audio_thread[core]); + ipc_user->init_needed[core] = true; + + /* Wait for user thread startup — consumes the initial k_sem_give from thread */ + return 0; +} + +struct k_thread *ipc_thread_user(unsigned int core) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *ipc_user = ipc->ipc_user_pdata; + + return ipc_user->thread[core]; +} + +/* Primary core only */ __cold static void ipc_user_init(void) { struct ipc *ipc = ipc_get(); @@ -480,6 +538,8 @@ __cold static void ipc_user_init(void) sof_panic(SOF_IPC_PANIC_IPC); } + assert_can_be_cold(); + ipc_user->sem = k_object_alloc(K_OBJ_SEM); if (!ipc_user->sem) { LOG_ERR("user IPC sem alloc failed"); @@ -490,15 +550,54 @@ __cold static void ipc_user_init(void) if (ret < 0) LOG_WRN("ipc context partition add failed: %d", ret); + /* + * Grant user-space access to .cold (execute) and .coldrodata (read) + * sections in IMR. The prepare path walks component code that may + * reference __cold functions and __cold_rodata data. + */ +#ifdef CONFIG_COLD_STORE_EXECUTE_DRAM + extern char __cold_start[], __cold_end[]; + extern char __coldrodata_start[]; + extern char _imr_end[]; + + if (&__cold_end[0] > &__cold_start[0]) { + struct k_mem_partition cold_part = { + .start = (uintptr_t)__cold_start, + .size = ALIGN_UP((uintptr_t)__cold_end - (uintptr_t)__cold_start, + CONFIG_MMU_PAGE_SIZE), + .attr = K_MEM_PARTITION_P_RX_U_RX, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); + if (ret < 0) + LOG_WRN("cold text partition add failed: %d", ret); + } + + if (&_imr_end[0] > &__coldrodata_start[0]) { + struct k_mem_partition cold_part = { + .start = (uintptr_t)__coldrodata_start, + .size = ALIGN_UP((uintptr_t)_imr_end - (uintptr_t)__coldrodata_start, + CONFIG_MMU_PAGE_SIZE), + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); + if (ret < 0) + LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", + cold_part.size, cold_part.start, ret); + } +#endif + k_sem_init(ipc_user->sem, 0, 1); - ret = ipc_user_init_thread(ipc_user); + ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); if (ret < 0) { LOG_ERR("user IPC thread initialization failed"); sof_panic(SOF_IPC_PANIC_IPC); } - ret = user_access_to_mailbox(zephyr_ll_mem_domain(), ipc_user->thread); + ret = user_access_to_mailbox(zephyr_ll_mem_domain(), + ipc_user->thread[PLATFORM_PRIMARY_CORE_ID]); if (ret < 0) { LOG_ERR("ipc user: mailbox access grant failed: %d", ret); sof_panic(SOF_IPC_PANIC_IPC); @@ -507,22 +606,32 @@ __cold static void ipc_user_init(void) /* Store references in ipc struct so kernel handler can forward commands */ ipc->ipc_user_pdata = ipc_user; - k_thread_start(ipc_user->thread); - struct task *task = zephyr_ll_task_alloc(); + if (!task) { + LOG_ERR("task allocation failed"); + k_panic(); + } + schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, - 0, NULL, NULL, cpu_get_id(), 0); - ipc_user->audio_thread = scheduler_init_context(task); + 0, NULL, NULL, PLATFORM_PRIMARY_CORE_ID, 0); + ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = scheduler_init_context(task); + if (!ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID]) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } /* Grant ipc_user thread permission on the audio thread object. * Needed so user-space dai_common_new() can call * k_thread_access_grant(audio_thread, dai_mutex) from user context. */ - k_thread_access_grant(ipc_user->thread, ipc_user->audio_thread); + k_thread_access_grant(ipc_user->thread[PLATFORM_PRIMARY_CORE_ID], + ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID]); + + k_thread_start(ipc_user->thread[PLATFORM_PRIMARY_CORE_ID]); /* Wait for user thread startup — consumes the initial k_sem_give from thread */ - k_sem_take(ipc->ipc_user_pdata->sem, K_FOREVER); + k_sem_take(ipc_user->sem, K_FOREVER); } #else static void ipc_user_init(void) diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 12f1234aef7a..2d7d8b71db15 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -90,7 +90,31 @@ static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data( /* * Global IPC Operations. */ -#ifndef CONFIG_SOF_USERSPACE_LL +#ifdef CONFIG_SOF_USERSPACE_LL +/* + * Determine the target core for an IPC4 module message. + * Falls back to current core when no component is bound yet. + */ +static unsigned int ipc4_user_target_core_module(struct ipc4_message_request *ipc4) +{ + /* + * Also works for struct ipc4_module_large_config, struct ipc4_module_bind_unbind, + * struct ipc4_module_delete_instance + */ + struct ipc4_module_config *config = (struct ipc4_module_config *)ipc4; + uint32_t module_id = config->primary.r.module_id; + + if (module_id) { + uint32_t instance_id = config->primary.r.instance_id; + struct comp_dev *dev = ipc4_get_comp_dev(IPC4_COMP_ID(module_id, instance_id)); + + if (dev) + return dev->ipc_config.core; + } + + return cpu_get_id(); +} +#else __cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) { struct ipc *ipc = ipc_get(); @@ -99,9 +123,7 @@ __cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) return ipc_pipeline_new(ipc, (ipc_pipe_new *)ipc4); } -#endif -#ifndef CONFIG_SOF_USERSPACE_LL __cold static int ipc4_delete_pipeline(struct ipc4_message_request *ipc4) { struct ipc4_pipeline_delete *pipe; @@ -675,6 +697,7 @@ static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply) { + struct ipc *ipc = ipc_get(); uint32_t type; int ret; @@ -698,21 +721,57 @@ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, /* pipeline settings */ case SOF_IPC4_GLB_CREATE_PIPELINE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + { + const struct ipc4_pipeline_create *create = + (const struct ipc4_pipeline_create *)ipc4; + + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + create->extension.r.core_id); + } #else ret = ipc4_new_pipeline(ipc4); #endif break; case SOF_IPC4_GLB_DELETE_PIPELINE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + { + const struct ipc4_pipeline_delete *del = (const struct ipc4_pipeline_delete *)ipc4; + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, + del->primary.r.instance_id, + IPC_COMP_ALL); + + if (!ppl) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); + } #else ret = ipc4_delete_pipeline(ipc4); #endif break; case SOF_IPC4_GLB_SET_PIPELINE_STATE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + { + struct ipc4_pipeline_set_state state = { + .primary.dat = ipc4->primary.dat, + .extension.dat = ipc4->extension.dat, + }; + int id = ipc4_pipeline_id_get(ipc4, &state, NULL, NULL); + if (id < 0) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, id, + IPC_COMP_ALL); + + if (!ppl) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); + } #else ret = ipc4_set_pipeline_state(ipc4); #endif @@ -1490,6 +1549,7 @@ __cold static int ipc4_delete_module_instance(struct ipc4_message_request *ipc4) __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply) { + const struct ipc4_module_init_instance *mi; uint32_t type; int ret; @@ -1500,66 +1560,58 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, switch (type) { case SOF_IPC4_MOD_INIT_INSTANCE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= - sizeof(((struct ipc_user *)0)->init_drv_data), - "ipc_user.init_drv_data too small for driver copy"); - - /* User-space init: kernel does driver lookup only (requires - * access to IMR manifest and driver list in kernel memory). - * Component creation (drv->ops.create) runs in user thread - * so untrusted module code does not execute in kernel context. - * Cross-core creation stays fully in kernel. - */ - const struct ipc4_module_init_instance *mi = - (const struct ipc4_module_init_instance *)ipc4; + mi = (const struct ipc4_module_init_instance *)ipc4; - if (!cpu_is_me(mi->extension.r.core_id)) { - ret = ipc4_init_module_instance(ipc4); - } else { - struct ipc *ipc = ipc_get(); + if (cpu_is_me(mi->extension.r.core_id) && !mi->extension.r.proc_domain && + IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { +#ifdef CONFIG_SOF_USERSPACE_LL + BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= + sizeof(((struct ipc_user *)0)->init_drv_data), + "ipc_user.init_drv_data too small for driver copy"); + + /* User-space init: kernel does driver lookup only (requires + * access to IMR manifest and driver list in kernel memory). + * Component creation (drv->ops.create) runs in user thread + * so untrusted module code does not execute in kernel context. + */ uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, mi->primary.r.instance_id); - const struct comp_driver *drv = ipc4_get_comp_drv( - IPC4_MOD_ID(comp_id)); - struct ipc_user *pdata = ipc->ipc_user_pdata; + const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); if (!drv) { ret = IPC4_MOD_NOT_INITIALIZED; break; } - /* Copy comp_driver and tr_ctx into user-accessible ipc_user buffer - * originals are in kernel .rodata/.data and not readable from user mode. - */ - struct comp_driver *drv_copy = (struct comp_driver *)pdata->init_drv_data; - struct tr_ctx *tctx_copy = - (struct tr_ctx *)(pdata->init_drv_data + - sizeof(struct comp_driver)); - - ret = memcpy_s(drv_copy, sizeof(*drv_copy), drv, sizeof(*drv)); - if (!ret && drv->tctx) { - ret = memcpy_s(tctx_copy, sizeof(*tctx_copy), - drv->tctx, sizeof(*drv->tctx)); - drv_copy->tctx = tctx_copy; - } + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); - if (ret < 0) - break; + if (ctx && drv->type == SOF_COMP_MODULE_ADAPTER) { + int err = ipc4_user_module_load(drv, mi); - pdata->init_drv = drv; - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); - } - } -#else - ret = ipc4_init_module_instance(ipc4); + if (err < 0) { + ret = IPC4_MOD_NOT_INITIALIZED; + break; + } + } + + ipc_get()->ipc_user_pdata->init_drv = drv; + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + mi->extension.r.core_id); #endif + } else { + /* + * DP module creation starts running in kernel mode and + * switches to userspace later via a scheduler_dp_thread_ipc() + * call in module_init(). + */ + ret = ipc4_init_module_instance(ipc4); + } break; case SOF_IPC4_MOD_CONFIG_GET: #ifdef CONFIG_SOF_USERSPACE_LL /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); if (!ret) { struct ipc *ipc = ipc_get(); struct ipc_user *pdata = ipc->ipc_user_pdata; @@ -1573,7 +1625,8 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, case SOF_IPC4_MOD_CONFIG_SET: #ifdef CONFIG_SOF_USERSPACE_LL /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); #else ret = ipc4_set_get_config_module_instance(ipc4, true); #endif @@ -1586,8 +1639,8 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, if (config->primary.r.module_id) { /* Module case: forward to user thread */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, - ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); if (!ret) { struct ipc *ipc = ipc_get(); struct ipc_user *pdata = ipc->ipc_user_pdata; @@ -1615,8 +1668,8 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, (const struct ipc4_module_large_config *)ipc4; if (config->primary.r.module_id) { - ret = ipc_user_forward_cmd(ipc4->primary.dat, - ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); } else { /* Base firmware: keep in kernel (IMR access) */ ret = ipc4_set_large_config_module_instance(ipc4); @@ -1628,21 +1681,24 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, break; case SOF_IPC4_MOD_BIND: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); #else ret = ipc4_bind_module_instance(ipc4); #endif break; case SOF_IPC4_MOD_UNBIND: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); #else ret = ipc4_unbind_module_instance(ipc4); #endif break; case SOF_IPC4_MOD_DELETE_INSTANCE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); #else ret = ipc4_delete_module_instance(ipc4); #endif @@ -1712,36 +1768,28 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user) * module code does not execute with kernel privileges. * * init_drv = original kernel pointer - * init_drv_data = user-accessible copy */ - const struct comp_driver *orig_drv = ipc_user->init_drv; - const struct comp_driver *drv_copy = - (const struct comp_driver *)ipc_user->init_drv_data; - struct comp_dev *dev; + const struct comp_driver *drv = ipc_user->init_drv; ipc_user->init_drv = NULL; - if (!orig_drv) { + if (!drv) { result = IPC4_MOD_NOT_INITIALIZED; break; } - dev = comp_new_ipc4_user(&msg, drv_copy); + struct comp_dev *dev = comp_new_ipc4_user(&msg, drv); + if (!dev) { result = IPC4_MOD_NOT_INITIALIZED; break; } - /* Restore original kernel driver pointer. comp_init() - * set dev->drv to the copy; runtime code expects the - * canonical kernel address. - */ - dev->drv = orig_drv; - result = ipc4_add_comp_dev(dev); if (result != IPC4_SUCCESS) break; comp_update_ibs_obs_cpc(dev); + result = 0; break; } diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 467a40a8d66d..ae6a5d7abd9c 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -111,42 +111,65 @@ __cold static inline unsigned char *ipc4_get_comp_new_data(void) } #endif -/* Only called from ipc4_init_module_instance(), which is __cold */ -__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) +__cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, + const struct ipc4_module_init_instance *module_init) { - struct comp_ipc_config ipc_config; - const struct comp_driver *drv; - struct comp_dev *dev; - uint32_t comp_id; - unsigned char *data; + uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, + module_init->primary.r.instance_id); assert_can_be_cold(); - comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, - module_init->primary.r.instance_id); - if (ipc4_get_comp_dev(comp_id)) { tr_err(&ipc_tr, "comp 0x%x exists", comp_id); - return NULL; + return -EEXIST; } if (module_init->extension.r.core_id >= CONFIG_CORE_COUNT) { tr_err(&ipc_tr, "ipc: comp->core = %u", (uint32_t)module_init->extension.r.core_id); - return NULL; + return -EINVAL; } - memset(&ipc_config, 0, sizeof(ipc_config)); - ipc_config.id = comp_id; - ipc_config.pipeline_id = module_init->extension.r.ppl_instance_id; - ipc_config.core = module_init->extension.r.core_id; - ipc_config.ipc_config_size = module_init->extension.r.param_block_size * sizeof(uint32_t); - ipc_config.ipc_extended_init = module_init->extension.r.extended_init; - if (ipc_config.ipc_config_size > MAILBOX_HOSTBOX_SIZE) { + memset(ipc_config, 0, sizeof(*ipc_config)); + ipc_config->id = comp_id; + ipc_config->pipeline_id = module_init->extension.r.ppl_instance_id; + ipc_config->core = module_init->extension.r.core_id; + ipc_config->ipc_config_size = module_init->extension.r.param_block_size * sizeof(uint32_t); + ipc_config->ipc_extended_init = module_init->extension.r.extended_init; + if (ipc_config->ipc_config_size > MAILBOX_HOSTBOX_SIZE) { tr_err(&ipc_tr, "IPC payload size %u too big for the message window", - ipc_config.ipc_config_size); - return NULL; + ipc_config->ipc_config_size); + return -ENOSPC; } + if (!module_init->extension.r.proc_domain) { + ipc_config->proc_domain = COMP_PROCESSING_DOMAIN_LL; + } else if (IS_ENABLED(CONFIG_ZEPHYR_DP_SCHEDULER)) { + ipc_config->proc_domain = COMP_PROCESSING_DOMAIN_DP; + } else { + tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); + return -EINVAL; + } + + return 0; +} + +/* Only called from ipc4_init_module_instance(), which is __cold */ +__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) +{ + struct comp_ipc_config ipc_config; + const struct comp_driver *drv; + struct comp_dev *dev; + unsigned char *data; + uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, + module_init->primary.r.instance_id); + + assert_can_be_cold(); + + int ret = ipc4_comp_new_config(&ipc_config, module_init); + + if (ret < 0) + return NULL; + /* Reject a module naming a non-existent parent pipeline: otherwise * dev->pipeline stays NULL and a later init path (e.g. the copier) * dereferences it. IPC4_INVALID_PIPELINE_ID is exempt - it marks base FW @@ -160,6 +183,7 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo (uint32_t)ipc_config.pipeline_id); return NULL; } + #ifdef CONFIG_DCACHE_LINE_SIZE if (!IS_ENABLED(CONFIG_LIBRARY)) sys_cache_data_invd_range((__sparse_force void __sparse_cache *) @@ -177,19 +201,6 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo if (!drv) return NULL; -#if CONFIG_ZEPHYR_DP_SCHEDULER - if (module_init->extension.r.proc_domain) - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; - else - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#else /* CONFIG_ZEPHYR_DP_SCHEDULER */ - if (module_init->extension.r.proc_domain) { - tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); - return NULL; - } - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#endif /* CONFIG_ZEPHYR_DP_SCHEDULER */ - if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { .data = data, @@ -225,6 +236,39 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo } #ifdef CONFIG_SOF_USERSPACE_LL + +int ipc4_user_module_load(const struct comp_driver *drv, + const struct ipc4_module_init_instance *mi) +{ + /* + * move a part to the kernel thread: + * the userspace IPC handling thread would call comp_new_ipc4_user() to + * then call library manager .create method lib_manager_module_create(). + * That one calls lib_manager_mod_create_priv(), then + * lib_manager_allocate_module() and eventually + * llext_manager_allocate_module() for LLEXT modules. + */ + struct comp_ipc_config ipc_config; + int ret = ipc4_comp_new_config(&ipc_config, mi); + + if (ret < 0) + return ret; + + const struct ipc_config_process spec = { + .data = ipc4_get_comp_new_data(), + .size = ipc_config.ipc_config_size, + }; + +#if CONFIG_DCACHE_LINE_SIZE && !CONFIG_LIBRARY + sys_cache_data_invd_range((__sparse_force void __sparse_cache *)spec.data, spec.size); +#endif + + struct userspace_context *userspace = NULL; + const struct module_interface *ops = NULL; + + return lib_manager_mod_create_priv(drv, &ipc_config, &spec, NULL, &userspace, &ops); +} + /** * comp_new_ipc4_user - Create component in user-space IPC thread context. * @@ -242,43 +286,19 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, struct ipc4_module_init_instance module_init; struct comp_ipc_config ipc_config; struct comp_dev *dev; - uint32_t comp_id; unsigned char *data; - int ret; assert_can_be_cold(); - ret = memcpy_s(&module_init, sizeof(module_init), ipc4, sizeof(*ipc4)); - if (ret < 0) - return NULL; - - comp_id = IPC4_COMP_ID(module_init.primary.r.module_id, - module_init.primary.r.instance_id); + int ret = memcpy_s(&module_init, sizeof(module_init), ipc4, sizeof(*ipc4)); - if (ipc4_get_comp_dev(comp_id)) { - tr_err(&ipc_tr, "comp 0x%x exists", comp_id); + if (ret < 0) return NULL; - } - if (module_init.extension.r.core_id >= CONFIG_CORE_COUNT) { - tr_err(&ipc_tr, "ipc: comp->core = %u", - (uint32_t)module_init.extension.r.core_id); + ret = ipc4_comp_new_config(&ipc_config, &module_init); + if (ret < 0) return NULL; - } - memset(&ipc_config, 0, sizeof(ipc_config)); - ipc_config.id = comp_id; - ipc_config.pipeline_id = module_init.extension.r.ppl_instance_id; - ipc_config.core = module_init.extension.r.core_id; - ipc_config.ipc_config_size = - module_init.extension.r.param_block_size * sizeof(uint32_t); - ipc_config.ipc_extended_init = module_init.extension.r.extended_init; - if (ipc_config.ipc_config_size > MAILBOX_HOSTBOX_SIZE) { - tr_err(&ipc_tr, - "IPC payload size %u too big for the message window", - ipc_config.ipc_config_size); - return NULL; - } #ifdef CONFIG_DCACHE_LINE_SIZE if (!IS_ENABLED(CONFIG_LIBRARY)) sys_cache_data_invd_range( @@ -288,19 +308,6 @@ __cold struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4, #endif data = ipc4_get_comp_new_data(); -#if CONFIG_ZEPHYR_DP_SCHEDULER - if (module_init.extension.r.proc_domain) - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_DP; - else - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#else - if (module_init.extension.r.proc_domain) { - tr_err(&ipc_tr, "ipc: DP scheduling is disabled, cannot create comp 0x%x", comp_id); - return NULL; - } - ipc_config.proc_domain = COMP_PROCESSING_DOMAIN_LL; -#endif - if (drv->type == SOF_COMP_MODULE_ADAPTER) { const struct ipc_config_process spec = { .data = data, diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index a672f51133d2..62233cb431a5 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -415,7 +415,7 @@ static uintptr_t lib_manager_allocate_module(const struct sof_man_fw_desc *const * * Function is responsible to free module resources in HP memory. */ -static int lib_manager_free_module(const uint32_t component_id) +int z_impl_lib_manager_free_module(const uint32_t component_id) { const struct sof_man_module *mod; const uint32_t module_id = IPC4_MOD_ID(component_id); @@ -465,7 +465,7 @@ static uintptr_t lib_manager_allocate_module(const struct sof_man_fw_desc *const return 0; } -static int lib_manager_free_module(const uint32_t component_id) +static int z_impl_lib_manager_free_module(const uint32_t component_id) { /* Since we cannot allocate the freeing is not considered to be an error */ tr_warn(&lib_manager_tr, "Dynamic module freeing is not supported"); @@ -646,34 +646,36 @@ static enum buildinfo_mod_type lib_manager_get_module_type(const struct sof_man_ } } -/* - * \brief Load module code, allocate its instance and create a module adapter component. - * \param[in] drv - component driver pointer. - * \param[in] config - component ipc descriptor pointer. - * \param[in] spec - passdowned data from driver. - * - * \return: a pointer to newly created module adapter component on success. NULL on error. - */ -static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, - const struct comp_ipc_config *config, - const void *spec) +/* Error path resource freeing */ +static void lib_manager_mod_free_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + struct userspace_context *userspace) +{ +#if CONFIG_SOF_USERSPACE_PROXY + if (userspace) + userspace_proxy_destroy(drv, userspace); +#endif /* CONFIG_SOF_USERSPACE_PROXY */ + lib_manager_free_module(config->id); +} + +int lib_manager_mod_create_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec, void **adapter_priv, + struct userspace_context **userspace, + const struct module_interface **ops) { const struct sof_man_fw_desc *const desc = lib_manager_get_library_manifest(config->id); const struct ipc_config_process *args = (const struct ipc_config_process *)spec; const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(config->id); - struct userspace_context *userspace = NULL; - const struct module_interface *ops; const struct sof_man_module *mod; system_agent_start_fn agent; - void *adapter_priv = NULL; const void **agent_iface; - struct comp_dev *dev; int ret; #ifdef CONFIG_SOF_USERSPACE_PROXY if (drv->user_heap && config->proc_domain != COMP_PROCESSING_DOMAIN_DP) { tr_err(&lib_manager_tr, "Userspace supports only DP modules."); - return NULL; + return -EOPNOTSUPP; } #endif @@ -681,12 +683,12 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, if (!desc) { tr_err(&lib_manager_tr, "Error: Couldn't find loadable module with id %u.", config->id); - return NULL; + return -ENOENT; } if (entry_index >= desc->header.num_module_entries) { tr_err(&lib_manager_tr, "Entry index %u out of bounds.", entry_index); - return NULL; + return -EINVAL; } mod = (const struct sof_man_module *) @@ -697,53 +699,96 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, if (!module_entry_point) { tr_err(&lib_manager_tr, "lib_manager_allocate_module() failed!"); - return NULL; + return -ENOENT; } switch (lib_manager_get_module_type(desc, mod)) { case MOD_TYPE_LLEXT: agent = NULL; - ops = (const struct module_interface *)module_entry_point; + *ops = (const struct module_interface *)module_entry_point; agent_iface = NULL; break; case MOD_TYPE_LMDK: agent = &native_system_agent_start; - agent_iface = (const void **)&ops; + agent_iface = (const void **)ops; break; #if CONFIG_INTEL_MODULES case MOD_TYPE_IADK: agent = &system_agent_start; - ops = &processing_module_adapter_interface; - agent_iface = (const void **)&adapter_priv; + *ops = &processing_module_adapter_interface; + agent_iface = (const void **)adapter_priv; break; #endif case MOD_TYPE_INVALID: + default: + ret = -EINVAL; goto err; } if (agent || IS_ENABLED(CONFIG_SOF_USERSPACE_PROXY)) { /* At this point module resources are allocated and it is moved to L2 memory. */ ret = lib_manager_start_agent(drv, config, mod, args, module_entry_point, agent, - agent_iface, &userspace, &ops); + agent_iface, userspace, ops); if (ret) goto err; } - if (comp_set_adapter_ops(drv, ops) < 0) + ret = comp_set_adapter_ops(drv, *ops); + if (ret < 0) goto err; - dev = module_adapter_new_ext(drv, config, spec, adapter_priv, userspace, NULL); + return 0; + +err: + lib_manager_mod_free_priv(drv, config, *userspace); + return ret; +} + +#ifdef CONFIG_USERSPACE +#include + +static int z_vrfy_lib_manager_free_module(const uint32_t component_id) +{ + return z_impl_lib_manager_free_module(component_id); +} +#include + +#endif /* CONFIG_USERSPACE */ + +/* + * \brief Load module code, allocate its instance and create a module adapter component. + * \param[in] drv - component driver pointer. + * \param[in] config - component ipc descriptor pointer. + * \param[in] spec - passdowned data from driver. + * + * \return: a pointer to newly created module adapter component on success. NULL on error. + */ +static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec) +{ + struct userspace_context *userspace = NULL; + const struct module_interface *ops = NULL; + void *adapter_priv = NULL; + struct comp_dev *dev; + + if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP || + !IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { + int ret = lib_manager_mod_create_priv(drv, config, spec, &adapter_priv, + &userspace, &ops); + + if (ret < 0) + return NULL; + } + + dev = module_adapter_new_ext(drv, config, spec, adapter_priv, userspace, ops); if (!dev) goto err; return dev; err: -#if CONFIG_SOF_USERSPACE_PROXY - if (userspace) - userspace_proxy_destroy(drv, userspace); -#endif /* CONFIG_SOF_USERSPACE_PROXY */ - lib_manager_free_module(config->id); + lib_manager_mod_free_priv(drv, config, userspace); return NULL; } diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index bcbfd290ceb0..30ff461ace7c 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -334,12 +334,10 @@ static int llext_manager_load_module(struct lib_manager_module *mctx) mctx->mapped = true; #ifdef CONFIG_SOF_USERSPACE_LL - if (!mctx->domain_dp) { - ret = llext_manager_add_mod_domain(mctx, zephyr_ll_mem_domain()); - if (ret < 0) { - tr_err(&lib_manager_tr, "failed to add domain: %d", ret); - goto e_data; - } + ret = llext_manager_add_mod_domain(mctx, zephyr_ll_mem_domain()); + if (ret < 0) { + tr_err(&lib_manager_tr, "failed to add domain: %d", ret); + goto e_data; } #endif @@ -426,8 +424,7 @@ static int llext_manager_unload_module(struct lib_manager_module *mctx) mctx->mapped = false; #ifdef CONFIG_SOF_USERSPACE_LL - if (!mctx->domain_dp) - llext_manager_rm_mod_domain(mctx, zephyr_ll_mem_domain()); + llext_manager_rm_mod_domain(mctx, zephyr_ll_mem_domain()); #endif return err; diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index b37f91285854..cc1b76600dee 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -525,6 +525,18 @@ struct k_thread *zephyr_domain_thread_tid_for_core(int core) return ll_thread_tid[core]; } +struct k_thread *zephyr_ll_domain_thread(void) +{ + struct ll_schedule_domain *ll_domain = zephyr_ll_domain(); + + if (!ll_domain) + return NULL; + + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(ll_domain); + + return zephyr_domain->domain_thread[cpu_get_id()].ll_thread; +} + #endif /* CONFIG_SOF_USERSPACE_LL */ #if CONFIG_CROSS_CORE_STREAM diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index fa8bbd285791..070d3f0ad763 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -224,10 +224,10 @@ static enum task_state scheduler_dp_ll_tick_dummy(void *data) * needed 1.2ms for processing - but the example would be too complicated) */ -void scheduler_dp_ll_tick(void) +void z_impl_scheduler_dp_ll_tick(unsigned int core) { unsigned int lock_key; - struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); + struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); if (!dp_sch) return; @@ -235,11 +235,20 @@ void scheduler_dp_ll_tick(void) /* remember current timestamp as "NOW" */ dp_sch->last_ll_tick_timestamp = k_cycle_get_32(); - lock_key = scheduler_dp_lock(cpu_get_id()); + lock_key = scheduler_dp_lock(core); scheduler_dp_recalculate(dp_sch); scheduler_dp_unlock(lock_key); } +#ifdef CONFIG_USERSPACE +#include +void z_vrfy_scheduler_dp_ll_tick(unsigned int core) +{ + z_impl_scheduler_dp_ll_tick(core); +} +#include +#endif + #if CONFIG_SOF_USERSPACE_APPLICATION static int scheduler_dp_task_cancel(void *data, struct task *task) { @@ -376,8 +385,7 @@ void scheduler_get_task_info_dp(struct scheduler_props *scheduler_props, uint32_ unsigned int lock_key; scheduler_props->processing_domain = COMP_PROCESSING_DOMAIN_DP; - struct scheduler_dp_data *dp_sch = - (struct scheduler_dp_data *)scheduler_get_data(SOF_SCHEDULE_DP); + struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); lock_key = scheduler_dp_lock(cpu_get_id()); scheduler_get_task_info(scheduler_props, data_off_size, &dp_sch->tasks); diff --git a/src/schedule/zephyr_dp_schedule.h b/src/schedule/zephyr_dp_schedule.h index 694bb541f87e..2119e44c9dfc 100644 --- a/src/schedule/zephyr_dp_schedule.h +++ b/src/schedule/zephyr_dp_schedule.h @@ -57,7 +57,3 @@ void dp_thread_fn(void *p1, void *p2, void *p3); unsigned int scheduler_dp_lock(uint16_t core); void scheduler_dp_unlock(unsigned int key); void scheduler_dp_grant(k_tid_t thread_id, uint16_t core); -int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, - const struct task_ops *ops, struct processing_module *mod, - uint16_t core, size_t stack_size, uint32_t options); -void scheduler_dp_internal_free(struct task *task); diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index daf9070ae4dc..8abac1b7e1bf 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -8,7 +8,9 @@ #include #include +#include #include +#include #include #include #include @@ -155,6 +157,7 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, const union scheduler_dp_thread_ipc_param *param) { struct task_dp_pdata *pdata = pmod->dev->task->priv_data; + unsigned int core = pmod->dev->task->core; int ret; if (!pmod) { @@ -164,14 +167,14 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, if (cmd == SOF_IPC4_MOD_INIT_INSTANCE) { /* Wait for the DP thread to start */ - ret = k_sem_take(&dp_sync[pmod->dev->task->core], DP_THREAD_IPC_TIMEOUT); + ret = k_sem_take(&dp_sync[core], DP_THREAD_IPC_TIMEOUT); if (ret < 0) { tr_err(&dp_tr, "Failed waiting for DP thread to start: %d", ret); return ret; } } - unsigned int lock_key = scheduler_dp_lock(pmod->dev->task->core); + unsigned int lock_key = scheduler_dp_lock(core); /* IPCs are serialised */ pdata->flat->ret = -ENOSYS; @@ -184,7 +187,7 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, if (!ret) { /* Wait for completion */ - ret = k_sem_take(&dp_sync[cpu_get_id()], DP_THREAD_IPC_TIMEOUT); + ret = k_sem_take(&dp_sync[core], DP_THREAD_IPC_TIMEOUT); if (ret < 0) tr_err(&dp_tr, "Failed waiting for DP thread: %d", ret); else @@ -400,7 +403,7 @@ struct scheduler_dp_task_memory { struct ipc4_flat flat; }; -void scheduler_dp_internal_free(struct task *task) +void z_impl_scheduler_dp_internal_free(struct task *task) { struct task_dp_pdata *pdata = task->priv_data; @@ -425,7 +428,7 @@ static void scheduler_dp_thread_name_set(k_tid_t thread_id, struct processing_mo #define scheduler_dp_thread_name_set(x, y) #endif -/* Called only in IPC context */ +/* Called only in IPC context in kernel mode (this can change) */ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, const struct task_ops *ops, struct processing_module *mod, uint16_t core, size_t stack_size, uint32_t options) @@ -437,6 +440,7 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, /* must be called on the same core the task will be bound to */ assert(cpu_get_id() == core); + assert(!k_is_user_context()); /* * allocate memory @@ -526,6 +530,13 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, k_thread_access_grant(pdata->thread_id, pdata->event, &dp_sync[core]); scheduler_dp_grant(pdata->thread_id, core); +#if CONFIG_SOF_USERSPACE_LL + struct k_thread *thread_ipc = ipc_thread_user(core); + + k_thread_access_grant(thread_ipc, pdata->event, pdata->thread_id, p_stack, &dp_sync[core]); + scheduler_dp_grant(thread_ipc, core); + scheduler_dp_grant(zephyr_ll_domain_thread(), core); +#endif struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), SOF_MEM_FLAG_COHERENT); @@ -615,3 +626,46 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, mod_free(mod, task_memory); return ret; } + +#ifdef CONFIG_USERSPACE +#include + +static void scheduler_dp_mod_vrfy(struct processing_module *mod) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod->dev, sizeof(*mod->dev))); + K_OOPS(K_SYSCALL_MEMORY_READ(mod->dev->drv, sizeof(*mod->dev->drv))); + + struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; + + assert(alloc); + if (alloc->heap) { + size_t h_size = 0; + uintptr_t h_start; + + mod_heap_info(mod, &h_size, &h_start); + if (h_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); + } + if (alloc->vreg) { + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(alloc->vreg, &vr_size, &vr_start); + K_OOPS(K_SYSCALL_MEMORY_WRITE(vr_start, vr_size)); + } +} + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); + + struct task_dp_pdata *pdata = task->priv_data; + + K_OOPS(K_SYSCALL_OBJ(pdata->event, K_OBJ_EVENT)); + K_OOPS(K_SYSCALL_OBJ_INIT(pdata->thread, K_OBJ_THREAD)); + scheduler_dp_mod_vrfy(pdata->mod); + return z_impl_scheduler_dp_internal_free(task); +} +#include +#endif diff --git a/src/schedule/zephyr_dp_schedule_thread.c b/src/schedule/zephyr_dp_schedule_thread.c index f2c2434bc2b5..43c4b43e35e2 100644 --- a/src/schedule/zephyr_dp_schedule_thread.c +++ b/src/schedule/zephyr_dp_schedule_thread.c @@ -340,7 +340,7 @@ int scheduler_dp_task_init(struct task **task, return ret; } -void scheduler_dp_internal_free(struct task *task) +void z_impl_scheduler_dp_internal_free(struct task *task) { struct task_dp_pdata *pdata = task->priv_data; @@ -354,3 +354,17 @@ void scheduler_dp_internal_free(struct task *task) /* task is the first member in task_memory above */ sof_heap_free(pdata->mod->dev->drv->user_heap, task); } + +#ifdef CONFIG_USERSPACE +#include + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + /* + * With the thread DP scheduler variant scheduler_dp_internal_free() is + * never called from the userspace context + */ + K_OOPS(true); +} +#include +#endif diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index e7a72e6db028..74414c7820ff 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -326,7 +326,7 @@ static void zephyr_ll_run(void *data) zephyr_ll_unlock(sch, &flags); #ifdef CONFIG_ZEPHYR_DP_SCHEDULER - scheduler_dp_ll_tick(); + scheduler_dp_ll_tick(sch->core); #endif } @@ -889,9 +889,13 @@ void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, uint32_t *data_off_size) { uint32_t flags; +#if CONFIG_SOF_USERSPACE_LL + struct zephyr_ll *ll_sch = scheduler_get_user_data(SOF_SCHEDULE_LL_TIMER); +#else + struct zephyr_ll *ll_sch = scheduler_get_data(SOF_SCHEDULE_LL_TIMER); +#endif scheduler_props->processing_domain = COMP_PROCESSING_DOMAIN_LL; - struct zephyr_ll *ll_sch = scheduler_get_data(SOF_SCHEDULE_LL_TIMER); zephyr_ll_lock(ll_sch, &flags); scheduler_get_task_info(scheduler_props, data_off_size, &ll_sch->tasks); @@ -901,7 +905,11 @@ void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, /* Return a pointer to the LL scheduler timer domain */ struct ll_schedule_domain *zephyr_ll_domain(void) { +#if CONFIG_SOF_USERSPACE_LL + struct zephyr_ll *ll_sch = scheduler_get_user_data(SOF_SCHEDULE_LL_TIMER); +#else struct zephyr_ll *ll_sch = scheduler_get_data(SOF_SCHEDULE_LL_TIMER); +#endif return ll_sch ? ll_sch->ll_domain : NULL; } diff --git a/uuid-registry.txt b/uuid-registry.txt index 04c4a3881b21..b235311b312c 100644 --- a/uuid-registry.txt +++ b/uuid-registry.txt @@ -148,6 +148,7 @@ d7f6712d-131c-45a7-82ed6aa9dc2291ea pm_runtime 9302adf5-88be-4234-a0a7dca538ef81f4 sai 3dee06de-f25a-4e10-ae1fabc9573873ea schedule 70d223ef-2b91-4aac-b444d89a0db2793a sdma +bdcb1461-34f5-4047-b9cc70fdf8dfb234 sec_core_init 55a88ed5-3d18-46ca-88f10ee6eae9930f selector 32fe92c1-1e17-4fc2-9758c7f3542e980a selector4 cf90d851-68a2-4987-a2de85aed0c8531c sgen_mt8186 diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index e0e7e8bfb302..e493fd21e050 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -629,7 +629,11 @@ zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/schedule/ll_schedule_domain.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/handler.h) zephyr_syscall_header(include/rtos/alloc.h) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc.c) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/vregion.h) +zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_VREGION syscall/vregion.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib_manager.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/schedule/dp_schedule.h) zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/dai.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/user/debug_stream_slot.h) diff --git a/zephyr/Kconfig b/zephyr/Kconfig index ab88efcb9a6a..2d03a6d6f7ae 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -36,11 +36,20 @@ config SOF_USERSPACE_INTERFACE_ALLOC Allow user-space threads to use sof_heap_alloc/sof_heap_free as Zephyr system calls. +config SOF_USERSPACE_INTERFACE_VREGION + bool "Enable SOF vregion interface to userspace threads" + depends on USERSPACE + depends on SOF_VREGIONS + help + Allow user-space threads to use vregion_alloc/vregion_free + and their variants as Zephyr system calls. + config SOF_USERSPACE_LL bool "Run Low-Latency pipelines in userspace threads" depends on USERSPACE select SOF_USERSPACE_INTERFACE_ALLOC select SOF_USERSPACE_INTERFACE_DMA + select SOF_USERSPACE_INTERFACE_VREGION if SOF_VREGIONS help Run Low-Latency (LL) pipelines in userspace threads. This adds memory protection between operating system resources and diff --git a/zephyr/lib/userspace_helper.c b/zephyr/lib/userspace_helper.c index ea9db4181b9d..e205fc256308 100644 --- a/zephyr/lib/userspace_helper.c +++ b/zephyr/lib/userspace_helper.c @@ -110,7 +110,7 @@ int user_access_to_mailbox(struct k_mem_domain *domain, k_tid_t thread_id) if (ret < 0) return ret; -#if defined(CONFIG_SOF_USERSPACE_LL) && defined(CONFIG_IPC_MAJOR_4) +#ifdef CONFIG_IPC_MAJOR_4 /* HOSTBOX partitions for IPC4 module init parameter block reads. * comp_new_ipc4() accesses MAILBOX_HOSTBOX_BASE directly to get * the module configuration data sent by the host. @@ -141,9 +141,7 @@ int user_access_to_mailbox(struct k_mem_domain *domain, k_tid_t thread_id) ret = k_mem_domain_add_partition(domain, &hostbox_partition); if (ret < 0) return ret; -#endif /* CONFIG_IPC_MAJOR_4 */ - -#ifndef CONFIG_IPC_MAJOR_4 +#else /* CONFIG_IPC_MAJOR_4 */ /* * Next mailbox_stream (not available in IPC4). Stream access is cached, * so different mapping this time. diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c index 9c8c94c23f97..e8cfb2347246 100644 --- a/zephyr/lib/vregion.c +++ b/zephyr/lib/vregion.c @@ -8,11 +8,13 @@ #include #include #include +#include #include #include #include #include #include +#include LOG_MODULE_REGISTER(vregion, CONFIG_SOF_LOG_LEVEL); @@ -87,6 +89,7 @@ struct vregion { unsigned int pages; /* size of whole region in pages */ struct k_mutex lock; /* protect vregion heaps and use-count */ unsigned int use_count; + struct k_mem_domain *domain; /* current allocation mode */ enum vregion_mem_type type; /* LIFETIME at creation, switch to INTERIM */ @@ -144,6 +147,7 @@ struct vregion *vregion_create(size_t memsize) vr->base = vregion_base; vr->size = total_size; vr->pages = pages; + vr->domain = NULL; /* lifetime linear allocator starts at the beginning of the vregion memory */ vr->lifetime.base = vregion_base; @@ -161,12 +165,12 @@ struct vregion *vregion_create(size_t memsize) /* log the new vregion */ LOG_INF("new at base %p size %#zx pages %u metadata at %p", - (void *)vr->base, total_size, pages, (void *)vr); + (void *)vregion_base, total_size, pages, (void *)vr); return vr; } -struct vregion *vregion_get(struct vregion *vr) +struct vregion *z_impl_vregion_get(struct vregion *vr) { if (!vr) return NULL; @@ -178,13 +182,79 @@ struct vregion *vregion_get(struct vregion *vr) return vr; } +struct vregion *z_impl_vregion_create_map(uintptr_t *vreg_start, size_t *vreg_size) +{ + if (!vreg_start || !vreg_size || !*vreg_size) + return NULL; + + struct vregion *vr = vregion_create(*vreg_size); + + if (!vr) + return NULL; + +#if CONFIG_USERSPACE && CONFIG_SOF_USERSPACE_LL + vregion_mem_info(vr, vreg_size, vreg_start); + + /* + * In the userspace LL case allocations are also performed by the + * userspace IPC thread, which is also the one, executing this syscall + */ + struct k_mem_domain *domain = zephyr_ll_mem_domain(); + struct k_mem_partition part = { + .start = *vreg_start, + .size = *vreg_size, + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + int ret = k_mem_domain_add_partition(domain, &part); + + if (ret < 0) { + vregion_put(vr); + return NULL; + } + + part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start); + part.attr = K_MEM_PARTITION_P_RW_U_RW; + + ret = k_mem_domain_add_partition(domain, &part); + if (ret < 0) { + vregion_put(vr); + return NULL; + } + + vr->domain = domain; +#endif + + return vr; +} + +static void vregion_unmap(struct vregion *vr) +{ +#if CONFIG_USERSPACE && CONFIG_SOF_USERSPACE_LL + if (!vr->domain) + return; + + struct k_mem_partition part = { + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + .start = (uintptr_t)vr->base, + .size = vr->size, + }; + + k_mem_domain_remove_partition(vr->domain, &part); + + part.start = (uintptr_t)sys_cache_uncached_ptr_get((void *)part.start); + part.attr = K_MEM_PARTITION_P_RW_U_RW; + + k_mem_domain_remove_partition(vr->domain, &part); +#endif +} + /** * @brief Decrement virtual region's user count or destroy it. * * @param[in] vr Pointer to the virtual region instance to release. * @return struct vregion* Pointer to the virtual region instance or NULL if it has been destroyed. */ -struct vregion *vregion_put(struct vregion *vr) +struct vregion *z_impl_vregion_put(struct vregion *vr) { unsigned int use_count; @@ -204,6 +274,8 @@ struct vregion *vregion_put(struct vregion *vr) LOG_DBG("destroy %p size %#zx pages %u", (void *)vr->base, vr->size, vr->pages); LOG_DBG(" lifetime used %zu free count %d", vr->lifetime.used, vr->lifetime.free_count); vpage_free(vr->base); + + vregion_unmap(vr); rfree(vr); return NULL; @@ -259,7 +331,7 @@ static void interim_heap_init(struct vregion *vr) vr->lifetime.used = (uint8_t *)vr->lifetime.ptr - (uint8_t *)vr->lifetime.base; } -void vregion_set_interim(struct vregion *vr) +void z_impl_vregion_set_interim(struct vregion *vr) { if (!vr) return; @@ -365,7 +437,7 @@ static void lifetime_free(struct vlinear_heap *heap, void *ptr) * @param vr Pointer to the virtual region instance. * @param ptr Pointer to the memory to free. */ -void vregion_free(struct vregion *vr, void *ptr) +void z_impl_vregion_free(struct vregion *vr, void *ptr) { if (!vr || !ptr) return; @@ -390,7 +462,7 @@ void vregion_free(struct vregion *vr, void *ptr) k_mutex_unlock(&vr->lock); } -EXPORT_SYMBOL(vregion_free); +EXPORT_SYMBOL(z_impl_vregion_free); /** * @brief Allocate memory from the virtual region. @@ -401,7 +473,8 @@ EXPORT_SYMBOL(vregion_free); * * @return void* Pointer to the allocated memory, or NULL on failure. */ -void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) +void *z_impl_vregion_alloc_align(struct vregion *vr, + size_t size, size_t alignment) { void *p; @@ -429,41 +502,15 @@ void *vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) return p; } -EXPORT_SYMBOL(vregion_alloc_align); +EXPORT_SYMBOL(z_impl_vregion_alloc_align); -/** - * @brief Allocate memory from the virtual region. - * @param[in] vr Pointer to the virtual region instance. - * @param[in] size Size of the allocation. - * @return void* Pointer to the allocated memory, or NULL on failure. - */ -void *vregion_alloc(struct vregion *vr, size_t size) -{ - return vregion_alloc_align(vr, size, 0); -} -EXPORT_SYMBOL(vregion_alloc); - -void *vregion_alloc_coherent(struct vregion *vr, size_t size) -{ - size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); - - void *p = vregion_alloc_align(vr, size, CONFIG_DCACHE_LINE_SIZE); - - if (!p) - return NULL; - - sys_cache_data_invd_range(p, size); - - return sys_cache_uncached_ptr_get(p); -} - -void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment) +void *z_impl_vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t alignment) { if (alignment < CONFIG_DCACHE_LINE_SIZE) alignment = CONFIG_DCACHE_LINE_SIZE; size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); - void *p = vregion_alloc_align(vr, size, alignment); + void *p = z_impl_vregion_alloc_align(vr, size, alignment); if (!p) return NULL; @@ -472,6 +519,7 @@ void *vregion_alloc_coherent_align(struct vregion *vr, size_t size, size_t align return sys_cache_uncached_ptr_get(p); } +EXPORT_SYMBOL(z_impl_vregion_alloc_coherent_align); /** * @brief Log virtual region memory usage. @@ -488,7 +536,6 @@ void vregion_info(struct vregion *vr) LOG_INF("lifetime used %#zx free count %d", vr->lifetime.used, vr->lifetime.free_count); } -EXPORT_SYMBOL(vregion_info); void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start) { @@ -498,3 +545,25 @@ void vregion_mem_info(struct vregion *vr, size_t *size, uintptr_t *start) if (start) *start = (uintptr_t)vr->base; } + +#if CONFIG_SOF_VREGIONS && CONFIG_USERSPACE +#include + +bool vregion_verify(struct vregion *vr) +{ + if (!vr) + return false; + + /* vregion instances must not be accessible to the userspace. */ + K_OOPS(!K_SYSCALL_MEMORY_READ(vr, sizeof(*vr))); + + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(vr, &vr_size, &vr_start); + if (vr_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE((void *)vr_start, vr_size)); + + return true; +} +#endif diff --git a/zephyr/syscall/vregion.c b/zephyr/syscall/vregion.c new file mode 100644 index 000000000000..dd02f854cf82 --- /dev/null +++ b/zephyr/syscall/vregion.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. + +#include +#include +#include + +static inline void *z_vrfy_vregion_alloc_align(struct vregion *vr, size_t size, size_t alignment) +{ + if (vregion_verify(vr)) + return z_impl_vregion_alloc_align(vr, size, alignment); + + return NULL; +} +#include + +static inline void *z_vrfy_vregion_alloc_coherent_align(struct vregion *vr, + size_t size, size_t alignment) +{ + if (vregion_verify(vr)) + return z_impl_vregion_alloc_coherent_align(vr, size, alignment); + + return NULL; +} +#include + +static inline void z_vrfy_vregion_free(struct vregion *vr, void *ptr) +{ + if (vregion_verify(vr)) + z_impl_vregion_free(vr, ptr); +} +#include + +struct vregion *z_vrfy_vregion_get(struct vregion *vr) +{ + if (vregion_verify(vr)) + return z_impl_vregion_get(vr); + return NULL; +} +#include + +struct vregion *z_vrfy_vregion_put(struct vregion *vr) +{ + if (vregion_verify(vr)) + return z_impl_vregion_put(vr); + return NULL; +} +#include + +struct vregion *z_vrfy_vregion_create_map(uintptr_t *vreg_start, size_t *vreg_size) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_start, sizeof(*vreg_start))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(vreg_size, sizeof(*vreg_size))); + return z_impl_vregion_create_map(vreg_start, vreg_size); +} +#include + +void z_vrfy_vregion_set_interim(struct vregion *vr) +{ + if (vregion_verify(vr)) + z_impl_vregion_set_interim(vr); +} +#include