diff --git a/src/audio/chain_dma.c b/src/audio/chain_dma.c index e7207e636ee8..2752318c4d06 100644 --- a/src/audio/chain_dma.c +++ b/src/audio/chain_dma.c @@ -15,7 +15,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -61,13 +63,13 @@ struct chain_dma_data { /* local host DMA config */ struct sof_dma *dma_host; - struct dma_chan_data *chan_host; + int chan_host_index; struct dma_config z_config_host; struct dma_block_config dma_block_cfg_host; /* local link DMA config */ struct sof_dma *dma_link; - struct dma_chan_data *chan_link; + int chan_link_index; struct dma_config z_config_link; struct dma_block_config dma_block_cfg_link; @@ -79,18 +81,18 @@ static int chain_host_start(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - if (!cd->chan_host || !cd->chan_host->dma) { + if (cd->chan_host_index < 0 || !cd->dma_host) { comp_err(dev, "incomplete initialization detected, aborting host %p", - cd->chan_host); + cd->dma_host); return -ENODEV; } - err = dma_start(cd->chan_host->dma->z_dev, cd->chan_host->index); + err = sof_dma_start(cd->dma_host, cd->chan_host_index); if (err < 0) return err; comp_info(dev, "dma_start() host chan_index = %u", - cd->chan_host->index); + cd->chan_host_index); return 0; } @@ -99,12 +101,12 @@ static int chain_link_start(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - err = dma_start(cd->chan_link->dma->z_dev, cd->chan_link->index); + err = sof_dma_start(cd->dma_link, cd->chan_link_index); if (err < 0) return err; comp_info(dev, "dma_start() link chan_index = %u", - cd->chan_link->index); + cd->chan_link_index); return 0; } @@ -113,12 +115,12 @@ static int chain_link_stop(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - err = dma_stop(cd->chan_link->dma->z_dev, cd->chan_link->index); + err = sof_dma_stop(cd->dma_link, cd->chan_link_index); if (err < 0) return err; comp_info(dev, "dma_stop() link chan_index = %u", - cd->chan_link->index); + cd->chan_link_index); return 0; } @@ -128,12 +130,12 @@ static int chain_host_stop(struct comp_dev *dev) struct chain_dma_data *cd = comp_get_drvdata(dev); int err; - err = dma_stop(cd->chan_host->dma->z_dev, cd->chan_host->index); + err = sof_dma_stop(cd->dma_host, cd->chan_host_index); if (err < 0) return err; comp_info(dev, "dma_stop() host chan_index = %u", - cd->chan_host->index); + cd->chan_host_index); return 0; } @@ -171,7 +173,7 @@ static enum task_state chain_task_run(void *data) /* Link DMA can return -EPIPE and current status if xrun occurs, then it is not critical * and flow shall continue. Other error values will be treated as critical. */ - ret = dma_get_status(cd->chan_link->dma->z_dev, cd->chan_link->index, &stat); + ret = sof_dma_get_status(cd->dma_link, cd->chan_link_index, &stat); switch (ret) { case 0: #if CONFIG_XRUN_NOTIFICATIONS_ENABLE @@ -195,7 +197,7 @@ static enum task_state chain_task_run(void *data) link_read_pos = stat.read_position; /* Host DMA does not report xruns. All error values will be treated as critical. */ - ret = dma_get_status(cd->chan_host->dma->z_dev, cd->chan_host->index, &stat); + ret = sof_dma_get_status(cd->dma_host, cd->chan_host_index, &stat); if (ret < 0) { tr_err(&chain_dma_tr, "dma_get_status() error, ret = %d", ret); return SOF_TASK_STATE_COMPLETED; @@ -213,14 +215,14 @@ static enum task_state chain_task_run(void *data) */ const size_t increment = MIN(host_free_bytes, link_avail_bytes); - ret = dma_reload(cd->chan_host->dma->z_dev, cd->chan_host->index, 0, 0, increment); + ret = sof_dma_reload(cd->dma_host, cd->chan_host_index, increment); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() host error, ret = %d", ret); return SOF_TASK_STATE_COMPLETED; } - ret = dma_reload(cd->chan_link->dma->z_dev, cd->chan_link->index, 0, 0, increment); + ret = sof_dma_reload(cd->dma_link, cd->chan_link_index, increment); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() link error, ret = %d", ret); @@ -236,9 +238,8 @@ static enum task_state chain_task_run(void *data) const size_t half_buff_size = buff_size / 2; if (!cd->first_data_received && host_avail_bytes > half_buff_size) { - ret = dma_reload(cd->chan_link->dma->z_dev, - cd->chan_link->index, 0, 0, - MIN(host_avail_bytes, link_free_bytes)); + ret = sof_dma_reload(cd->dma_link, cd->chan_link_index, + MIN(host_avail_bytes, link_free_bytes)); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() link error, ret = %d", ret); @@ -252,8 +253,8 @@ static enum task_state chain_task_run(void *data) host_read_pos, buff_size); - ret = dma_reload(cd->chan_host->dma->z_dev, cd->chan_host->index, - 0, 0, transferred); + ret = sof_dma_reload(cd->dma_host, cd->chan_host_index, + transferred); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() host error, ret = %d", ret); @@ -262,8 +263,8 @@ static enum task_state chain_task_run(void *data) if (host_avail_bytes >= half_buff_size && link_free_bytes >= half_buff_size) { - ret = dma_reload(cd->chan_link->dma->z_dev, cd->chan_link->index, - 0, 0, half_buff_size); + ret = sof_dma_reload(cd->dma_link, cd->chan_link_index, + half_buff_size); if (ret < 0) { tr_err(&chain_dma_tr, "dma_reload() link error, ret = %d", ret); @@ -373,9 +374,9 @@ __cold static void chain_release(struct comp_dev *dev) assert_can_be_cold(); - dma_release_channel(cd->chan_host->dma->z_dev, cd->chan_host->index); + sof_dma_release_channel(cd->dma_host, cd->chan_host_index); sof_dma_put(cd->dma_host); - dma_release_channel(cd->chan_link->dma->z_dev, cd->chan_link->index); + sof_dma_release_channel(cd->dma_link, cd->chan_link_index); sof_dma_put(cd->dma_link); if (cd->dma_buffer) { @@ -463,16 +464,16 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) /* get host DMA channel */ channel = cd->host_connector_node_id.f.v_index; - channel = dma_request_channel(cd->dma_host->z_dev, &channel); + channel = sof_dma_request_channel(cd->dma_host, channel); if (channel < 0) { comp_err(dev, "host dma_request_channel() failed for %u", cd->host_connector_node_id.f.v_index); return channel; } - cd->chan_host = &cd->dma_host->chan[channel]; + cd->chan_host_index = channel; - err = dma_config(cd->dma_host->z_dev, cd->chan_host->index, dma_cfg_host); + err = sof_dma_config(cd->dma_host, cd->chan_host_index, dma_cfg_host); if (err < 0) { comp_err(dev, "host dma_config() failed for %d", channel); goto error_host; @@ -480,7 +481,7 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) /* get link DMA channel */ channel = cd->link_connector_node_id.f.v_index; - channel = dma_request_channel(cd->dma_link->z_dev, &channel); + channel = sof_dma_request_channel(cd->dma_link, channel); if (channel < 0) { comp_err(dev, "link dma_request_channel() failed for %u", cd->link_connector_node_id.f.v_index); @@ -488,9 +489,9 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) goto error_host; } - cd->chan_link = &cd->dma_link->chan[channel]; + cd->chan_link_index = channel; - err = dma_config(cd->dma_link->z_dev, cd->chan_link->index, dma_cfg_link); + err = sof_dma_config(cd->dma_link, cd->chan_link_index, dma_cfg_link); if (err < 0) { comp_err(dev, "link dma_config() failed for %d", channel); goto error_link; @@ -498,11 +499,9 @@ __cold static int chain_init(struct comp_dev *dev, void *addr, size_t length) return 0; error_link: - dma_release_channel(cd->dma_link->z_dev, cd->chan_link->index); - cd->chan_link = NULL; + sof_dma_release_channel(cd->dma_link, cd->chan_link_index); error_host: - dma_release_channel(cd->dma_host->z_dev, cd->chan_host->index); - cd->chan_host = NULL; + sof_dma_release_channel(cd->dma_host, cd->chan_host_index); return err; } @@ -510,6 +509,7 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin uint32_t fifo_size) { struct chain_dma_data *cd = comp_get_drvdata(dev); + struct mod_alloc_ctx *alloc_ctx = NULL; uint32_t addr_align; size_t buff_size; void *buff_addr; @@ -559,8 +559,8 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin } /* retrieve DMA buffer address alignment */ - ret = dma_get_attribute(cd->dma_host->z_dev, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, - &addr_align); + ret = sof_dma_get_attribute(cd->dma_host, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, + &addr_align); if (ret < 0) { comp_err(dev, "could not get dma buffer address alignment, err = %d", ret); @@ -588,8 +588,14 @@ __cold static int chain_task_init(struct comp_dev *dev, uint8_t host_dma_id, uin } fifo_size = ALIGN_UP_INTERNAL(fifo_size, addr_align); + +#ifdef CONFIG_SOF_USERSPACE_LL + alloc_ctx = ipc_get()->ll_alloc; +#endif + /* allocate not shared buffer */ - cd->dma_buffer = buffer_alloc(NULL, fifo_size, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, + cd->dma_buffer = buffer_alloc(alloc_ctx, fifo_size, + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, addr_align, BUFFER_USAGE_NOT_SHARED); if (!cd->dma_buffer) { @@ -631,6 +637,70 @@ static int chain_task_trigger(struct comp_dev *dev, int cmd) } } +/* + * comp_dev and private data allocation helpers. For user-space LL both + * objects must live on the user heap so the (unprivileged) user LL thread + * can access them; otherwise the normal component/rmalloc paths are used. + */ +#ifdef CONFIG_SOF_USERSPACE_LL +__cold static struct comp_dev *chain_dev_alloc(const struct comp_driver *drv) +{ + struct comp_dev *dev; + + dev = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*dev), 0); + if (!dev) + return NULL; + + memset(dev, 0, sizeof(*dev)); + comp_init(drv, dev, sizeof(*dev)); + + return dev; +} + +__cold static struct chain_dma_data *chain_cd_alloc(void) +{ + struct chain_dma_data *cd; + + cd = sof_heap_alloc(sof_sys_user_heap_get(), SOF_MEM_FLAG_USER, sizeof(*cd), 0); + if (cd) + memset(cd, 0, sizeof(*cd)); + + return cd; +} + +__cold static void chain_dev_free(struct comp_dev *dev) +{ + sof_heap_free(sof_sys_user_heap_get(), dev); +} + +__cold static void chain_cd_free(struct chain_dma_data *cd) +{ + sof_heap_free(sof_sys_user_heap_get(), cd); +} +#else +__cold static struct comp_dev *chain_dev_alloc(const struct comp_driver *drv) +{ + return comp_alloc(drv, sizeof(struct comp_dev)); +} + +__cold static struct chain_dma_data *chain_cd_alloc(void) +{ + return rzalloc(SOF_MEM_FLAG_USER, sizeof(struct chain_dma_data)); +} + +__cold static void chain_dev_free(struct comp_dev *dev) +{ + comp_free_device(dev); +} + +__cold static void chain_cd_free(struct chain_dma_data *cd) +{ + rfree(cd); +} +#endif + __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv, const struct comp_ipc_config *ipc_config, const void *ipc_specific_config) @@ -649,17 +719,19 @@ __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv, if (host_dma_id >= max_chain_number) return NULL; - dev = comp_alloc(drv, sizeof(*dev)); + dev = chain_dev_alloc(drv); if (!dev) return NULL; - cd = rzalloc(SOF_MEM_FLAG_USER, sizeof(*cd)); + cd = chain_cd_alloc(); if (!cd) goto error; cd->first_data_received = false; cd->cs = scs ? 2 : 4; cd->chain_task.state = SOF_TASK_STATE_INIT; + cd->chan_host_index = -EINVAL; + cd->chan_link_index = -EINVAL; comp_set_drvdata(dev, cd); @@ -667,9 +739,9 @@ __cold static struct comp_dev *chain_task_create(const struct comp_driver *drv, if (!ret) return dev; - rfree(cd); + chain_cd_free(cd); error: - comp_free_device(dev); + chain_dev_free(dev); return NULL; } @@ -680,8 +752,8 @@ __cold static void chain_task_free(struct comp_dev *dev) assert_can_be_cold(); chain_release(dev); - rfree(cd); - comp_free_device(dev); + chain_cd_free(cd); + chain_dev_free(dev); } static const struct comp_driver comp_chain_dma = { diff --git a/src/audio/module_adapter/Kconfig b/src/audio/module_adapter/Kconfig index fe8425ae5467..d80e42583b60 100644 --- a/src/audio/module_adapter/Kconfig +++ b/src/audio/module_adapter/Kconfig @@ -13,6 +13,17 @@ menu "Processing modules" containers to allocate at once is selected by this config option. + config MODULE_MEMORY_API_DEBUG + bool "Turn on memory API thread safety checks" + default y if DEBUG + help + The Module Memory API structures are not protected + by locks. This is because the initialization, + allocation, and freeing of resources should always + be done in the same thread. This option adds an + assert to make sure no other thread makes such + operations. + config CADENCE_CODEC bool "Cadence codec" help diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index ee1b2df92829..e1bd0813b097 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -12,7 +12,6 @@ */ #include -#include #include #include #include @@ -26,6 +25,16 @@ #include #endif +/* The __ZEPHYR__ condition is to keep cmocka tests working */ +#if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) +#define MEM_API_CHECK_THREAD(res) do { \ + if ((res)->rsrc_mngr != k_current_get()) \ + LOG_WRN("mngr %p != cur %p", (res)->rsrc_mngr, k_current_get()); \ +} while (0) +#else +#define MEM_API_CHECK_THREAD(res) +#endif + LOG_MODULE_DECLARE(module_adapter, CONFIG_SOF_LOG_LEVEL); int module_load_config(struct comp_dev *dev, const void *cfg, size_t size) @@ -79,7 +88,6 @@ void mod_resource_init(struct processing_module *mod) struct module_resources *res = &mod->priv.resources; /* Init memory list */ - k_mutex_init(&res->lock); list_init(&res->objpool.list); res->objpool.heap = res->alloc->heap; res->objpool.vreg = res->alloc->vreg; @@ -114,6 +122,9 @@ int module_init(struct processing_module *mod) return -EIO; } +#if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) + mod->priv.resources.rsrc_mngr = k_current_get(); +#endif /* Now we can proceed with module specific initialization */ #if CONFIG_SOF_USERSPACE_APPLICATION if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) @@ -179,18 +190,15 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t struct module_resources *res = &mod->priv.resources; struct module_resource *container; - k_mutex_lock(&res->lock, K_FOREVER); + MEM_API_CHECK_THREAD(res); container = container_get(mod); - if (!container) { - k_mutex_unlock(&res->lock); + if (!container) return NULL; - } if (!size) { comp_err(mod->dev, "requested allocation of 0 bytes."); container_put(mod, container); - k_mutex_unlock(&res->lock); return NULL; } @@ -202,7 +210,6 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t comp_err(mod->dev, "Failed to alloc %zu bytes %zu alignment for comp %#x.", size, alignment, dev_comp_id(mod->dev)); container_put(mod, container); - k_mutex_unlock(&res->lock); return NULL; } /* Store reference to allocated memory */ @@ -214,7 +221,6 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t if (res->heap_usage > res->heap_high_water_mark) res->heap_high_water_mark = res->heap_usage; - k_mutex_unlock(&res->lock); return ptr; } EXPORT_SYMBOL(z_impl_mod_balloc_align); @@ -235,18 +241,15 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t struct module_resources *res = &mod->priv.resources; struct module_resource *container; - k_mutex_lock(&res->lock, K_FOREVER); + MEM_API_CHECK_THREAD(res); container = container_get(mod); - if (!container) { - k_mutex_unlock(&res->lock); + if (!container) return NULL; - } if (!size) { comp_err(mod->dev, "requested allocation of 0 bytes."); container_put(mod, container); - k_mutex_unlock(&res->lock); return NULL; } @@ -257,7 +260,6 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t comp_err(mod->dev, "Failed to alloc %zu bytes %zu alignment for comp %#x.", size, alignment, dev_comp_id(mod->dev)); container_put(mod, container); - k_mutex_unlock(&res->lock); return NULL; } /* Store reference to allocated memory */ @@ -269,7 +271,6 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t if (res->heap_usage > res->heap_high_water_mark) res->heap_high_water_mark = res->heap_usage; - k_mutex_unlock(&res->lock); return ptr; } EXPORT_SYMBOL(z_impl_mod_alloc_ext); @@ -284,22 +285,19 @@ EXPORT_SYMBOL(z_impl_mod_alloc_ext); #if CONFIG_COMP_BLOB struct comp_data_blob_handler *z_impl_mod_data_blob_handler_new(struct processing_module *mod) { - struct module_resources *res = &mod->priv.resources; + struct module_resources * __maybe_unused res = &mod->priv.resources; struct comp_data_blob_handler *bhp; struct module_resource *container; - k_mutex_lock(&res->lock, K_FOREVER); + MEM_API_CHECK_THREAD(res); container = container_get(mod); - if (!container) { - k_mutex_unlock(&res->lock); + if (!container) return NULL; - } bhp = comp_data_blob_handler_new_ext(mod->dev, false, NULL, NULL); if (!bhp) { container_put(mod, container); - k_mutex_unlock(&res->lock); return NULL; } @@ -307,7 +305,6 @@ struct comp_data_blob_handler *z_impl_mod_data_blob_handler_new(struct processin container->size = 0; container->type = MOD_RES_BLOB_HANDLER; - k_mutex_unlock(&res->lock); return bhp; } EXPORT_SYMBOL(z_impl_mod_data_blob_handler_new); @@ -328,18 +325,15 @@ const void *z_impl_mod_fast_get(struct processing_module *mod, const void * cons struct module_resource *container; const void *ptr; - k_mutex_lock(&res->lock, K_FOREVER); + MEM_API_CHECK_THREAD(res); container = container_get(mod); - if (!container) { - k_mutex_unlock(&res->lock); + if (!container) return NULL; - } ptr = fast_get(res->alloc, dram_ptr, size); if (!ptr) { container_put(mod, container); - k_mutex_unlock(&res->lock); return NULL; } @@ -347,7 +341,6 @@ const void *z_impl_mod_fast_get(struct processing_module *mod, const void * cons container->size = 0; container->type = MOD_RES_FAST_GET; - k_mutex_unlock(&res->lock); return ptr; } EXPORT_SYMBOL(z_impl_mod_fast_get); @@ -419,17 +412,14 @@ int z_impl_mod_free(struct processing_module *mod, const void *ptr) { struct module_resources *res = &mod->priv.resources; + MEM_API_CHECK_THREAD(res); if (!ptr) return 0; /* Find which container holds this memory */ struct mod_res_cb_arg cb_arg = {mod, ptr}; - - k_mutex_lock(&res->lock, K_FOREVER); int ret = objpool_iterate(&res->objpool, mod_res_free, &cb_arg); - k_mutex_unlock(&res->lock); - if (ret < 0) comp_err(mod->dev, "error: could not find memory pointed by %p", ptr); @@ -738,13 +728,13 @@ void mod_free_all(struct processing_module *mod) { struct module_resources *res = &mod->priv.resources; + MEM_API_CHECK_THREAD(res); + /* Free all contents found in used containers */ struct mod_res_cb_arg cb_arg = {mod, NULL}; - k_mutex_lock(&res->lock, K_FOREVER); objpool_iterate(&res->objpool, mod_res_free, &cb_arg); objpool_prune(&res->objpool); - k_mutex_unlock(&res->lock); /* Make sure resource lists and accounting are reset */ mod_resource_init(mod); diff --git a/src/audio/pipeline/pipeline-graph.c b/src/audio/pipeline/pipeline-graph.c index adcb00a80719..fa8d24d146b9 100644 --- a/src/audio/pipeline/pipeline-graph.c +++ b/src/audio/pipeline/pipeline-graph.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -44,10 +45,20 @@ DECLARE_TR_CTX(pipe_tr, SOF_UUID(pipe_uuid), LOG_LEVEL_INFO); /* lookup table to determine busy/free pipeline metadata objects */ struct pipeline_posn { bool posn_offset[PPL_POSN_OFFSETS]; /**< available offsets */ +#ifndef CONFIG_SOF_USERSPACE_LL struct k_spinlock lock; /**< lock mechanism */ +#endif }; /* the pipeline position lookup table */ -static SHARED_DATA struct pipeline_posn pipeline_posn_shared; +static APP_SYSUSER_BSS SHARED_DATA struct pipeline_posn pipeline_posn_shared; + +#ifdef CONFIG_SOF_USERSPACE_LL +/* Mutex pointer in user-accessible partition so user-space threads + * can read the pointer for syscalls. Kept outside the SHARED_DATA + * struct to avoid kernel object tracking issues. + */ +static APP_SYSUSER_BSS struct k_mutex *pipeline_posn_lock; +#endif /** * \brief Retrieves pipeline position structure. @@ -55,7 +66,11 @@ static SHARED_DATA struct pipeline_posn pipeline_posn_shared; */ static inline struct pipeline_posn *pipeline_posn_get(void) { +#ifdef CONFIG_SOF_USERSPACE_LL + return &pipeline_posn_shared; +#else return sof_get()->pipeline_posn; +#endif } /** @@ -68,9 +83,14 @@ static inline int pipeline_posn_offset_get(uint32_t *posn_offset) struct pipeline_posn *pipeline_posn = pipeline_posn_get(); int ret = -EINVAL; uint32_t i; + +#ifdef CONFIG_SOF_USERSPACE_LL + k_mutex_lock(pipeline_posn_lock, K_FOREVER); +#else k_spinlock_key_t key; key = k_spin_lock(&pipeline_posn->lock); +#endif for (i = 0; i < PPL_POSN_OFFSETS; ++i) { if (!pipeline_posn->posn_offset[i]) { @@ -81,8 +101,11 @@ static inline int pipeline_posn_offset_get(uint32_t *posn_offset) } } - +#ifdef CONFIG_SOF_USERSPACE_LL + k_mutex_unlock(pipeline_posn_lock); +#else k_spin_unlock(&pipeline_posn->lock, key); +#endif return ret; } @@ -95,20 +118,41 @@ static inline void pipeline_posn_offset_put(uint32_t posn_offset) { struct pipeline_posn *pipeline_posn = pipeline_posn_get(); int i = posn_offset / sizeof(struct sof_ipc_stream_posn); + +#ifdef CONFIG_SOF_USERSPACE_LL + k_mutex_lock(pipeline_posn_lock, K_FOREVER); + pipeline_posn->posn_offset[i] = false; + k_mutex_unlock(pipeline_posn_lock); +#else k_spinlock_key_t key; key = k_spin_lock(&pipeline_posn->lock); - pipeline_posn->posn_offset[i] = false; - k_spin_unlock(&pipeline_posn->lock, key); +#endif } void pipeline_posn_init(struct sof *sof) { sof->pipeline_posn = &pipeline_posn_shared; +#ifdef CONFIG_SOF_USERSPACE_LL + pipeline_posn_lock = k_object_alloc(K_OBJ_MUTEX); + if (!pipeline_posn_lock) { + pipe_cl_err("pipeline posn mutex alloc failed"); + k_panic(); + } + k_mutex_init(pipeline_posn_lock); +#else k_spinlock_init(&sof->pipeline_posn->lock); +#endif +} + +#ifdef CONFIG_SOF_USERSPACE_LL +void pipeline_posn_grant_access(struct k_thread *thread) +{ + k_thread_access_grant(thread, pipeline_posn_lock); } +#endif /* create new pipeline - returns pipeline id or negative error */ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_t priority, @@ -140,12 +184,17 @@ struct pipeline *pipeline_new(struct k_heap *heap, uint32_t pipeline_id, uint32_ p->pipeline_id = pipeline_id; p->status = COMP_STATE_INIT; p->trigger.cmd = COMP_TRIGGER_NO_ACTION; + +#ifdef CONFIG_SOF_USERSPACE_LL + LOG_WRN("pipeline trace settings cannot be copied"); +#else ret = memcpy_s(&p->tctx, sizeof(struct tr_ctx), &pipe_tr, sizeof(struct tr_ctx)); if (ret < 0) { pipe_err(p, "failed to copy trace settings"); goto free; } +#endif ret = pipeline_posn_offset_get(&p->posn_offset); if (ret < 0) { @@ -523,6 +572,10 @@ struct comp_dev *pipeline_get_dai_comp(uint32_t pipeline_id, int dir) */ struct comp_dev *pipeline_get_dai_comp_latency(uint32_t pipeline_id, uint32_t *latency) { +#ifdef CONFIG_SOF_USERSPACE_LL + LOG_WRN("latency cannot be computed in user-space pipelines!"); + *latency = 0; +#else struct ipc_comp_dev *ipc_sink; struct ipc_comp_dev *ipc_source; struct comp_dev *source; @@ -590,7 +643,7 @@ struct comp_dev *pipeline_get_dai_comp_latency(uint32_t pipeline_id, uint32_t *l /* Get a next sink component */ ipc_sink = ipc_get_ppl_sink_comp(ipc, source->pipeline->pipeline_id); } - +#endif return NULL; } EXPORT_SYMBOL(pipeline_get_dai_comp_latency); diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 6740593a7cf1..a5416f370cd1 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -13,7 +13,6 @@ #ifndef __SOF_AUDIO_MODULE_GENERIC__ #define __SOF_AUDIO_MODULE_GENERIC__ -#include #include #include #include @@ -21,6 +20,10 @@ #include #include "module_interface.h" +/* The __ZEPHYR__ condition is to keep cmocka tests working */ +#if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) +#include +#endif #include /* @@ -125,11 +128,13 @@ struct module_param { * when the module unloads. */ struct module_resources { - struct k_mutex lock; struct objpool_head objpool; size_t heap_usage; size_t heap_high_water_mark; struct mod_alloc_ctx *alloc; +#if CONFIG_MODULE_MEMORY_API_DEBUG && defined(__ZEPHYR__) + k_tid_t rsrc_mngr; +#endif }; enum mod_resource_type { diff --git a/src/include/sof/audio/pipeline.h b/src/include/sof/audio/pipeline.h index 913a569c208c..ff456fbceb7d 100644 --- a/src/include/sof/audio/pipeline.h +++ b/src/include/sof/audio/pipeline.h @@ -206,6 +206,14 @@ int pipeline_complete(struct pipeline *p, struct comp_dev *source, */ void pipeline_posn_init(struct sof *sof); +#ifdef CONFIG_SOF_USERSPACE_LL +/** + * \brief Grants user-space thread access to pipeline position mutex. + * \param[in] thread Thread to grant access to. + */ +void pipeline_posn_grant_access(struct k_thread *thread); +#endif + /** * \brief Resets the pipeline and free runtime resources. * \param[in] p pipeline. diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index f356566ca0aa..b4b58e6923e2 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -330,6 +330,16 @@ struct ll_schedule_domain *zephyr_domain_init(int clk); struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain); struct k_mem_domain *zephyr_ll_mem_domain(void); #endif /* CONFIG_SOF_USERSPACE_LL */ +#ifdef CONFIG_SOF_FULL_ZEPHYR_APPLICATION +__syscall int zephyr_ll_task_sem_alloc(struct task *task); +__syscall int zephyr_ll_task_sem_free(struct task *task); +#include +#else +int z_impl_zephyr_ll_task_sem_alloc(struct task *task); +int z_impl_zephyr_ll_task_sem_free(struct task *task); +#define zephyr_ll_task_sem_alloc z_impl_zephyr_ll_task_sem_alloc +#define zephyr_ll_task_sem_free z_impl_zephyr_ll_task_sem_free +#endif /* CONFIG_SOF_FULL_ZEPHYR_APPLICATION */ #endif /* __ZEPHYR__ */ struct ll_schedule_domain *dma_multi_chan_domain_init(struct dma *dma_array, diff --git a/src/init/init.c b/src/init/init.c index 7976e2eb673e..5990cfebc2dc 100644 --- a/src/init/init.c +++ b/src/init/init.c @@ -32,6 +32,7 @@ #include #include #include +#include #include #if CONFIG_IPC_MAJOR_4 #include @@ -232,6 +233,11 @@ __cold static int primary_core_init(int argc, char *argv[], struct sof *sof) zephyr_ll_user_resources_init(); #endif + /* init pipeline position offsets - must be before platform_init() + * which calls ipc_init() -> ipc_user_init() that needs the posn mutex. + */ + pipeline_posn_init(sof); + /* init the platform */ if (platform_init(sof) < 0) sof_panic(SOF_IPC_PANIC_PLATFORM); diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index afc8fe45de05..95b4bf28485d 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -466,6 +466,7 @@ __cold static void ipc_user_init(void) sof_panic(SOF_IPC_PANIC_IPC); } user_ll_grant_access(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID); + pipeline_posn_grant_access(&ipc_user_thread); k_mem_domain_add_thread(zephyr_ll_mem_domain(), &ipc_user_thread); k_thread_cpu_pin(&ipc_user_thread, PLATFORM_PRIMARY_CORE_ID); diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 85c7f80552c6..b1aa50031fd1 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -41,7 +41,7 @@ struct zephyr_ll { struct zephyr_ll_pdata { bool run; bool freeing; - struct k_sem sem; + struct k_sem *sem; }; #if CONFIG_SOF_USERSPACE_LL @@ -136,7 +136,7 @@ static void zephyr_ll_task_done(struct zephyr_ll *sch, * zephyr_ll_task_free() is trying to free this task. Complete * it and signal the semaphore to let the function proceed */ - k_sem_give(&pdata->sem); + k_sem_give(pdata->sem); tr_info(&ll_tr, "task complete %p %pU", task, task->uid); tr_info(&ll_tr, "num_tasks %d total_num_tasks %ld", @@ -448,6 +448,96 @@ static int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t return zephyr_ll_task_schedule_common(sch, task, start, period, after, false); } +static struct list_item zephyr_ll_task_sem_list = LIST_INIT(zephyr_ll_task_sem_list); + +struct zephyr_ll_task_sem { + struct task *task; + struct k_sem *sem; + struct list_item list; +}; + +int z_impl_zephyr_ll_task_sem_alloc(struct task *task) +{ + struct zephyr_ll_pdata *pdata = task->priv_data; + struct zephyr_ll_task_sem *ts = rmalloc(SOF_MEM_FLAG_COHERENT, sizeof(*ts)); + + if (!ts) + return -ENOMEM; + + ts->sem = k_object_alloc(K_OBJ_SEM); + if (!ts->sem) { + rfree(ts); + return -ENOMEM; + } + + k_sem_init(ts->sem, 0, 1); + + ts->task = task; + pdata->sem = ts->sem; + /* List is protected by IPC serialization */ + list_item_append(&ts->list, &zephyr_ll_task_sem_list); + + return 0; +} + +int z_impl_zephyr_ll_task_sem_free(struct task *task) +{ + struct zephyr_ll_pdata *pdata = task->priv_data; + struct list_item *list; + struct zephyr_ll_task_sem *ts; + bool found = false; + + /* List is protected by IPC serialization */ + list_for_item(list, &zephyr_ll_task_sem_list) { + ts = container_of(list, struct zephyr_ll_task_sem, list); + if (ts->task == task) { + found = true; + break; + } + } + + if (!found) + return -ENOENT; + + if (pdata->sem != ts->sem) + return -EINVAL; + + list_item_del(list); + k_object_free(ts->sem); + rfree(ts); + + return 0; +} + +#ifdef CONFIG_USERSPACE +#include +static inline int z_vrfy_zephyr_ll_task_sem_alloc(struct task *task) +{ + if (!task) + return -EINVAL; + K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); + if (!task->priv_data) + return -EINVAL; + K_OOPS(K_SYSCALL_MEMORY_WRITE(task->priv_data, sizeof(struct zephyr_ll_pdata))); + + return z_impl_zephyr_ll_task_sem_alloc(task); +} +#include + +static inline int z_vrfy_zephyr_ll_task_sem_free(struct task *task) +{ + if (!task) + return -EINVAL; + K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); + if (!task->priv_data) + return -EINVAL; + K_OOPS(K_SYSCALL_MEMORY_WRITE(task->priv_data, sizeof(struct zephyr_ll_pdata))); + + return z_impl_zephyr_ll_task_sem_free(task); +} +#include +#endif + /* * This is synchronous - after this returns the object can be destroyed! * Assertion: under Zephyr this is always called from a thread context! @@ -505,10 +595,11 @@ static int zephyr_ll_task_free(void *data, struct task *task) if (must_wait) /* Wait for up to 100 periods */ - k_sem_take(&pdata->sem, K_USEC(LL_TIMER_PERIOD_US * 100)); + k_sem_take(pdata->sem, K_USEC(LL_TIMER_PERIOD_US * 100)); /* Protect against racing with schedule_task() */ zephyr_ll_lock(sch, &flags); + zephyr_ll_task_sem_free(task); task->priv_data = NULL; sof_heap_free(sch->heap, pdata); zephyr_ll_unlock(sch, &flags); @@ -573,6 +664,7 @@ static void zephyr_ll_scheduler_free(void *data, uint32_t flags) struct k_thread *zephyr_ll_init_context(void *data, struct task *task) { struct zephyr_ll *sch = data; + struct zephyr_ll_pdata *pdata = task->priv_data; int ret; /* @@ -587,7 +679,7 @@ struct k_thread *zephyr_ll_init_context(void *data, struct task *task) } assert(!k_is_user_context()); - k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain), sch->lock); + k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain), sch->lock, pdata->sem); tr_dbg(&ll_tr, "granting access to lock %p for thread %p", sch->lock, zephyr_domain_thread_tid(sch->ll_domain)); @@ -698,10 +790,15 @@ int zephyr_ll_task_init(struct task *task, memset(pdata, 0, sizeof(*pdata)); - k_sem_init(&pdata->sem, 0, 1); - task->priv_data = pdata; + ret = zephyr_ll_task_sem_alloc(task); + if (ret < 0) { + sof_heap_free(heap, pdata); + task->priv_data = NULL; + return ret; + } + return 0; } EXPORT_SYMBOL(zephyr_ll_task_init); diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 4b61a9517d46..e0e7e8bfb302 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -620,9 +620,12 @@ zephyr_library_sources_ifdef(CONFIG_SHELL sof_shell.c ) +zephyr_syscall_header(include/sof/lib/cpu.h) +zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/cpu.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/audio/module_adapter/module/generic.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/fast-get.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/ipc/ipc_reply.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/schedule/ll_schedule_domain.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/handler.h) zephyr_syscall_header(include/rtos/alloc.h) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc.c) diff --git a/zephyr/include/sof/lib/cpu.h b/zephyr/include/sof/lib/cpu.h index c23405e85121..a47b93898973 100644 --- a/zephyr/include/sof/lib/cpu.h +++ b/zephyr/include/sof/lib/cpu.h @@ -40,13 +40,32 @@ void cpu_notify_state_exit(enum pm_state state); #endif /* CONFIG_PM */ +/* + * cpu_get_id() is exposed as a Zephyr system call so that user-mode + * threads (e.g. user-space LL pipelines) can query the current core + * id. The underlying arch_proc_id() reads a privileged special + * register (e.g. Xtensa PRID) which would fault if executed directly + * from user mode. In supervisor context the generated wrapper inlines + * the z_impl_cpu_get_id() body, so there is no overhead there. + */ +#if defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +__syscall int cpu_get_id(void); +#endif + /* let the compiler optimise when in single core mode */ #if CONFIG_MULTICORE && CONFIG_SMP +#if defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +static inline int z_impl_cpu_get_id(void) +{ + return arch_proc_id(); +} +#else static inline int cpu_get_id(void) { return arch_proc_id(); } +#endif static inline bool cpu_is_primary(int id) { @@ -73,7 +92,11 @@ int cpu_restore_secondary_cores(void); int cpu_secondary_cores_prepare_d0ix(void); #else +#if defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +static inline int z_impl_cpu_get_id(void) { return 0; }; +#else static inline int cpu_get_id(void) { return 0; }; +#endif static inline bool cpu_is_primary(int id) { return 1; }; @@ -93,6 +116,10 @@ static inline int cpu_secondary_cores_prepare_d0ix(void) { return 0; }; #endif /* CONFIG_MULTICORE && CONFIG_SMP */ +#if defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +#include +#endif + #endif #endif /* __SOF_LIB_CPU_H__ */ diff --git a/zephyr/syscall/cpu.c b/zephyr/syscall/cpu.c new file mode 100644 index 000000000000..ee1990db8fd4 --- /dev/null +++ b/zephyr/syscall/cpu.c @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. + +#include +#include +#include + +/** + * \brief Userspace verification wrapper for cpu_get_id(). + * + * The system call takes no arguments and passes no pointers, so no + * access validation is required; the call is simply forwarded to the + * implementation running in supervisor context. + * + * @return Id of the DSP core executing the call. + */ +static inline int z_vrfy_cpu_get_id(void) +{ + return z_impl_cpu_get_id(); +} +#include diff --git a/zephyr/wrapper.c b/zephyr/wrapper.c index 9bbb43f8a798..afdc5b54a2e9 100644 --- a/zephyr/wrapper.c +++ b/zephyr/wrapper.c @@ -177,9 +177,6 @@ int task_main_start(struct sof *sof) /* init default audio components */ sys_comp_init(sof); - /* init pipeline position offsets */ - pipeline_posn_init(sof); - return 0; }