diff --git a/src/audio/buffers/audio_buffer.c b/src/audio/buffers/audio_buffer.c index 8ab0a711e7b1..1e2620423b0c 100644 --- a/src/audio/buffers/audio_buffer.c +++ b/src/audio/buffers/audio_buffer.c @@ -10,6 +10,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include @@ -182,6 +186,38 @@ int audio_buffer_source_set_alignment_constants(struct sof_source *source, return 0; } +uint32_t audio_buffer_sink_get_lft(struct sof_sink *sink) +{ + struct sof_audio_buffer *buffer = sof_audio_buffer_from_sink(sink); + /* get number of ms in the buffer */ + size_t bytes_per_sec = sink_get_frame_bytes(&buffer->_sink_api) * + sink_get_rate(&buffer->_sink_api); + size_t bytes_per_ms = bytes_per_sec / 1000; + + /* round up for frequencies like 44100 */ + if (bytes_per_ms * 1000 != bytes_per_sec) + bytes_per_ms++; + uint32_t us_in_buffer = + 1000 * source_get_data_available(&buffer->_source_api) / bytes_per_ms; + + return us_in_buffer; + + /* + * TODO, Currently there's no DP to DP connection + * >>> the code below is never accessible and won't work because of cache incoherence <<< + * + * to make DP to DP connection possible: + * + * 1) module data must be ALWAYS located in non cached memory alias, allowing + * cross core access to params like period (needed below) and calling + * module_get_deadline for the next module, regardless of cores the modules are + * running on + * 2) comp_buffer must be removed from all pipeline code, replaced with a generic abstract + * class audio_buffer - allowing using comp_buffer and ring_buffer without current + * "hybrid buffer" solution + */ +} + void audio_buffer_init(struct sof_audio_buffer *buffer, uint32_t buffer_type, bool is_shared, const struct source_ops *source_ops, const struct sink_ops *sink_ops, const struct audio_buffer_ops *audio_buffer_ops, diff --git a/src/audio/buffers/comp_buffer.c b/src/audio/buffers/comp_buffer.c index b4f53e7cf7f1..8cc99a73319b 100644 --- a/src/audio/buffers/comp_buffer.c +++ b/src/audio/buffers/comp_buffer.c @@ -178,6 +178,7 @@ APP_TASK_DATA static const struct sink_ops comp_buffer_sink_ops = { .audio_set_ipc_params = audio_buffer_sink_set_ipc_params, .on_audio_format_set = audio_buffer_sink_on_audio_format_set, .set_alignment_constants = audio_buffer_sink_set_alignment_constants, + .get_lft = audio_buffer_sink_get_lft, }; static const struct audio_buffer_ops audio_buffer_ops = { @@ -185,7 +186,7 @@ static const struct audio_buffer_ops audio_buffer_ops = { .reset = comp_buffer_reset, .audio_set_ipc_params = comp_buffer_set_ipc_params, .on_audio_format_set = comp_buffer_format_set, - .set_alignment_constants = comp_buffer_set_alignment_constants + .set_alignment_constants = comp_buffer_set_alignment_constants, }; static struct comp_buffer *buffer_alloc_struct(void *stream_addr, size_t size, diff --git a/src/audio/buffers/ring_buffer.c b/src/audio/buffers/ring_buffer.c index a71a27022e0a..572fe6eeed79 100644 --- a/src/audio/buffers/ring_buffer.c +++ b/src/audio/buffers/ring_buffer.c @@ -273,11 +273,12 @@ static const struct sink_ops ring_buffer_sink_ops = { .audio_set_ipc_params = audio_buffer_sink_set_ipc_params, .on_audio_format_set = audio_buffer_sink_on_audio_format_set, .set_alignment_constants = audio_buffer_sink_set_alignment_constants, + .get_lft = audio_buffer_sink_get_lft, }; static const struct audio_buffer_ops audio_buffer_ops = { .free = ring_buffer_free, - .reset = ring_buffer_reset + .reset = ring_buffer_reset, }; struct ring_buffer *ring_buffer_create(struct comp_dev *dev, size_t min_available, diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index f77377a54cea..0131bc872e25 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -796,3 +796,27 @@ void module_update_buffer_position(struct input_stream_buffer *input_buffers, output_buffers->size += audio_stream_frame_bytes(sink) * frames; } EXPORT_SYMBOL(module_update_buffer_position); + +uint32_t module_get_deadline(struct processing_module *mod) +{ + uint32_t deadline; + + /* LL modules have no deadline - it is always "now" */ + if (mod->dev->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_LL) + return 0; + + /* startup condition - set deadline to "unknown" */ + if (mod->dp_startup_delay) + return UINT32_MAX / 2; + + deadline = UINT32_MAX; + /* calculate the shortest LFT for all sinks */ + for (size_t i = 0; i < mod->num_of_sinks; i++) { + uint32_t sink_lft = sink_get_last_feeding_time(mod->sinks[i]); + + deadline = MIN(deadline, sink_lft); + } + + return deadline; +} +EXPORT_SYMBOL(module_get_deadline); diff --git a/src/include/module/audio/sink_api.h b/src/include/module/audio/sink_api.h index 527fd1dc11c6..920087d6f8b3 100644 --- a/src/include/module/audio/sink_api.h +++ b/src/include/module/audio/sink_api.h @@ -77,6 +77,12 @@ struct sink_ops { */ int (*commit_buffer)(struct sof_sink *sink, size_t commit_size); + /** + * get latest feeding time for this sink, result is a number of microseconds since "NOW" + * where "now" means a start of last LL cycle, as described in zephyr_dp_schedule.c + */ + uint32_t (*get_lft)(struct sof_sink *sink); + /** * OPTIONAL: Notification to the sink implementation about changes in audio format * @@ -348,4 +354,9 @@ static inline struct processing_module *sink_get_bound_module(struct sof_sink *s return sink->bound_module; } +static inline uint32_t sink_get_last_feeding_time(struct sof_sink *sink) +{ + return sink->ops->get_lft(sink); +} + #endif /* __MODULE_AUDIO_SINK_API_H__ */ diff --git a/src/include/sof/audio/audio_buffer.h b/src/include/sof/audio/audio_buffer.h index 5156971d515f..e19937be156f 100644 --- a/src/include/sof/audio/audio_buffer.h +++ b/src/include/sof/audio/audio_buffer.h @@ -337,5 +337,6 @@ int audio_buffer_sink_on_audio_format_set(struct sof_sink *sink); int audio_buffer_sink_set_alignment_constants(struct sof_sink *sink, const uint32_t byte_align, const uint32_t frame_align_req); +uint32_t audio_buffer_sink_get_lft(struct sof_sink *sink); #endif /* __SOF_AUDIO_BUFFER__ */ diff --git a/src/include/sof/audio/module_adapter/module/generic.h b/src/include/sof/audio/module_adapter/module/generic.h index 82430a017204..7138da3a697a 100644 --- a/src/include/sof/audio/module_adapter/module/generic.h +++ b/src/include/sof/audio/module_adapter/module/generic.h @@ -407,4 +407,29 @@ void module_adapter_set_params(struct processing_module *mod, struct sof_ipc_str int module_adapter_set_state(struct processing_module *mod, struct comp_dev *dev, int cmd); int module_adapter_sink_src_prepare(struct comp_dev *dev); + +/** + * Get a deadline based on current LFT reported by all sinks + * it returns a value >= UINT32_MAX / 2 in case the deadline cannot be calculated: + * - if a module is in a dealayed start + * - if there's no sink - i.e. DP module is a pure data consumer (like key phrare detector) + * + * @return a deadline the module must finish processing since NOW [in us] + */ +uint32_t module_get_deadline(struct processing_module *mod); + +/** + * Get a Longest Processing Time estimation for the module, in us + * It is either + * - a value taken from the manifest or estimated from module period (TODO) + * - or a value taken from IPC call (TODO) + * - a worst case - module period + * note that module period may be calculated reasonably late, i.e. in prepare() method + */ +static inline uint32_t module_get_lpt(struct processing_module *mod) +{ + /* return worst case of LPT - a module period. See zephyr_dp_schedule.c for description */ + return mod->dev->period; +} + #endif /* __SOF_AUDIO_MODULE_GENERIC__ */ diff --git a/src/include/sof/lib/notifier.h b/src/include/sof/lib/notifier.h index 060906655cb8..ddc82dbe76b2 100644 --- a/src/include/sof/lib/notifier.h +++ b/src/include/sof/lib/notifier.h @@ -32,6 +32,7 @@ enum notify_id { NOTIFIER_ID_BUFFER_FREE, /* struct buffer_cb_free* */ NOTIFIER_ID_DMA_COPY, /* struct dma_cb_data* */ NOTIFIER_ID_LL_POST_RUN, /* NULL */ + NOTIFIER_ID_LL_PRE_RUN, /* NULL */ NOTIFIER_ID_DMA_IRQ, /* struct dma_chan_data * */ NOTIFIER_ID_DAI_TRIGGER, /* struct dai_group * */ NOTIFIER_ID_MIC_PRIVACY_STATE_CHANGE, /* struct mic_privacy_settings * */ diff --git a/src/schedule/ll_schedule.c b/src/schedule/ll_schedule.c index 10615045d4d2..491dc01233ab 100644 --- a/src/schedule/ll_schedule.c +++ b/src/schedule/ll_schedule.c @@ -313,6 +313,9 @@ static void schedule_ll_tasks_run(void *data) perf_cnt_init(&sch->pcd); + notifier_event(sch, NOTIFIER_ID_LL_PRE_RUN, + NOTIFIER_TARGET_CORE_LOCAL, NULL, 0); + /* run tasks if there are any pending */ if (schedule_ll_is_pending(sch)) schedule_ll_tasks_execute(sch); diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index c4f3aaedeb37..1d87bdc84da0 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -33,6 +33,10 @@ DECLARE_TR_CTX(dp_tr, SOF_UUID(dp_sched_uuid), LOG_LEVEL_INFO); struct scheduler_dp_data { struct list_item tasks; /* list of active dp tasks */ struct task ll_tick_src; /* LL task - source of DP tick */ + uint32_t last_ll_tick_timestamp;/* a timestamp as k_cycle_get_32 of last LL tick, + * "NOW" for DP deadline calculation + */ + }; struct task_dp_pdata { @@ -258,31 +262,31 @@ static enum task_state scheduler_dp_ll_tick_dummy(void *data) * Now - pipeline is in stable state, CPU used almost in 100% (it would be 100% if DP3 * needed 1.2ms for processing - but the example would be too complicated) */ -void scheduler_dp_ll_tick(void *receiver_data, enum notify_id event_type, void *caller_data) + +/* Go through all DP tasks and recalculate their readness and dedlines + * NOT REENTRANT, should be called with scheduler_dp_lock() + */ +static void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch, bool is_ll_post_run) { - (void)receiver_data; - (void)event_type; - (void)caller_data; struct list_item *tlist; struct task *curr_task; struct task_dp_pdata *pdata; - unsigned int lock_key; - struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); - lock_key = scheduler_dp_lock(cpu_get_id()); list_for_item(tlist, &dp_sch->tasks) { curr_task = container_of(tlist, struct task, list); pdata = curr_task->priv_data; struct processing_module *mod = pdata->mod; + bool trigger_task = false; /* decrease number of LL ticks/cycles left till the module reaches its deadline */ - if (pdata->ll_cycles_to_start) { + if (mod->dp_startup_delay && is_ll_post_run && pdata->ll_cycles_to_start) { pdata->ll_cycles_to_start--; if (!pdata->ll_cycles_to_start) - /* deadline reached, clear startup delay flag. + /* delayed start complete, clear startup delay flag. * see dp_startup_delay comment for details */ mod->dp_startup_delay = false; + } if (curr_task->state == SOF_TASK_STATE_QUEUED) { @@ -293,16 +297,66 @@ void scheduler_dp_ll_tick(void *receiver_data, enum notify_id event_type, void * mod->sinks, mod->num_of_sinks); if (mod_ready) { - /* set a deadline for given num of ticks, starting now */ - k_thread_deadline_set(pdata->thread_id, - pdata->deadline_clock_ticks); - /* trigger the task */ curr_task->state = SOF_TASK_STATE_RUNNING; + if (mod->dp_startup_delay && !pdata->ll_cycles_to_start) { + /* first time run - use delayed start */ + pdata->ll_cycles_to_start = + module_get_lpt(pdata->mod) / LL_TIMER_PERIOD_US; + + /* in case LPT < LL cycle - delay at least cycle */ + if (!pdata->ll_cycles_to_start) + pdata->ll_cycles_to_start = 1; + } + trigger_task = true; k_sem_give(pdata->sem); } } + if (curr_task->state == SOF_TASK_STATE_RUNNING) { + /* (re) calculate deadline for all running tasks */ + /* get module deadline in us*/ + uint32_t deadline = module_get_deadline(mod); + + /* if a deadline cannot be calculated, use a fixed value relative to its + * first start + */ + if (deadline >= UINT32_MAX / 2 && trigger_task) + deadline = module_get_lpt(mod); + + if (deadline < UINT32_MAX) { + /* round down to 1ms */ + deadline = deadline / 1000; + + /* calculate number of ticks */ + deadline = deadline * (CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC / 1000); + + /* add to "NOW", overflows are OK */ + deadline = dp_sch->last_ll_tick_timestamp + deadline; + + /* set in Zephyr. Note that it may be in past, it does not matter, + * Zephyr still will schedule the thread with earlier deadline + * first + */ + k_thread_absolute_deadline_set(pdata->thread_id, deadline); + } + } } +} + +void scheduler_dp_ll_tick(void *receiver_data, enum notify_id event_type, void *caller_data) +{ + (void)receiver_data; + (void)event_type; + (void)caller_data; + unsigned int lock_key; + struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); + + if (event_type == NOTIFIER_ID_LL_PRE_RUN) + /* remember current timestamp as "NOW" */ + dp_sch->last_ll_tick_timestamp = k_cycle_get_32(); + + lock_key = scheduler_dp_lock(cpu_get_id()); + scheduler_dp_recalculate(dp_sch, event_type == NOTIFIER_ID_LL_POST_RUN); scheduler_dp_unlock(lock_key); } @@ -374,6 +428,7 @@ static void dp_thread_fn(void *p1, void *p2, void *p3) unsigned int lock_key; enum task_state state; bool task_stop; + struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); do { /* @@ -415,6 +470,11 @@ static void dp_thread_fn(void *p1, void *p2, void *p3) /* if true exit the while loop, terminate the thread */ task_stop = task->state == SOF_TASK_STATE_COMPLETED || task->state == SOF_TASK_STATE_CANCEL; + /* recalculate all DP tasks readiness and deadlines + * TODO: it should be for all tasks, for all cores + * currently its limited to current core only + */ + scheduler_dp_recalculate(dp_sch, false); scheduler_dp_unlock(lock_key); } while (!task_stop); @@ -430,7 +490,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta struct scheduler_dp_data *dp_sch = (struct scheduler_dp_data *)data; struct task_dp_pdata *pdata = task->priv_data; unsigned int lock_key; - uint64_t deadline_clock_ticks; int ret; lock_key = scheduler_dp_lock(cpu_get_id()); @@ -483,14 +542,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta task->state = SOF_TASK_STATE_QUEUED; list_item_prepend(&task->list, &dp_sch->tasks); - deadline_clock_ticks = period * CONFIG_SYS_CLOCK_TICKS_PER_SEC; - /* period/deadline is in us - convert to seconds in next step - * or it always will be zero because of integer calculation - */ - deadline_clock_ticks /= 1000000; - - pdata->deadline_clock_ticks = deadline_clock_ticks; - pdata->ll_cycles_to_start = period / LL_TIMER_PERIOD_US; pdata->mod->dp_startup_delay = true; scheduler_dp_unlock(lock_key); @@ -532,6 +583,7 @@ int scheduler_dp_init(void) if (ret) return ret; + notifier_register(NULL, NULL, NOTIFIER_ID_LL_PRE_RUN, scheduler_dp_ll_tick, 0); notifier_register(NULL, NULL, NOTIFIER_ID_LL_POST_RUN, scheduler_dp_ll_tick, 0); return 0; diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 851fd9a96a98..007e7fbd545a 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -176,6 +176,9 @@ static void zephyr_ll_run(void *data) struct list_item *list, *tmp, task_head = LIST_INIT(task_head); uint32_t flags; + notifier_event(sch, NOTIFIER_ID_LL_PRE_RUN, + NOTIFIER_TARGET_CORE_LOCAL, NULL, 0); + zephyr_ll_lock(sch, &flags); /*