From 93ba653952920ef66a139fb4ffe431148fca20e2 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Tue, 10 Nov 2026 16:07:58 +0000 Subject: [PATCH 1/9] telemetry: headers: add extended TEL2 and PMW2 window definitions Define the Memory Window 2 (TEL2 v2.0) self-describing directory header, cache-line-aligned hot counter zone, and cold metadata sections (memory, vregion, pipelines, gateways, and Zephyr threads), along with the Memory Window 3 (PMW2) per-module cold resource and hot counter structures and telemetry notification helper prototypes. Signed-off-by: Liam Girdwood --- .../sof/debug/telemetry/performance_monitor.h | 46 +++ src/include/sof/debug/telemetry/telemetry.h | 365 +++++++++++++++++- 2 files changed, 410 insertions(+), 1 deletion(-) diff --git a/src/include/sof/debug/telemetry/performance_monitor.h b/src/include/sof/debug/telemetry/performance_monitor.h index 68a1592ff40f..fe081c8bd4ef 100644 --- a/src/include/sof/debug/telemetry/performance_monitor.h +++ b/src/include/sof/debug/telemetry/performance_monitor.h @@ -23,6 +23,52 @@ (sys_cache_uncached_ptr_get((__sparse_force void __sparse_cache *) \ (WIN3_MBASE + WIN3_OFFSET)))) +struct comp_dev; + +/* Memory Window 3 extended partition constants (8,192 B window) */ +#define SOF_TEL_MW3_MAGIC 0x32574D50U /* "PMW2" */ +#define SOF_TEL_MW3_MOD_HOT_OFFSET 0x1080U +#define SOF_TEL_MW3_MOD_COLD_OFFSET 0x1200U +#define SOF_TEL_MW3_IO_PERF_OFFSET 0x1800U +#define SOF_TEL_MW3_HDR_OFFSET 0x1FC0U +#define SOF_TEL_MW3_MAX_IO_ITEMS 128U + +/* Dense hot per-component counter (4 B -> 16 components per 64B cache line) */ +struct sof_tel_mod_hot { + uint16_t cpc_violation_count; + uint16_t xrun_count; +} __packed; + +/* Cold per-component module resource record (16 B -> 4 components per 64B cache line) */ +struct sof_tel_mod_cold { + uint32_t resource_id; + uint32_t heap_usage; + uint32_t heap_high_water_mark; + uint32_t configured_cpc; +} __packed; + +/* MW3 partition footer directory at 0x1FC0 (64 B = 1 cache line) */ +struct sof_tel_mw3_hdr { + uint32_t magic; /* SOF_TEL_MW3_MAGIC ("PMW2") */ + uint16_t version; + uint16_t max_comp_entries; + uint16_t max_io_entries; + uint16_t mod_hot_offset; + uint16_t mod_cold_offset; + uint16_t io_perf_offset; + uint8_t rsvd[48]; +} __packed; + +#if IS_ENABLED(CONFIG_SOF_TELEMETRY_MODULE_RESOURCES) +void perf_data_update_module_resources(struct comp_dev *dev); +void perf_data_inc_cpc_violation(struct comp_dev *dev); +void perf_data_inc_xrun(struct comp_dev *dev); +#else +static inline void perf_data_update_module_resources(struct comp_dev *dev) { (void)dev; } +static inline void perf_data_inc_cpc_violation(struct comp_dev *dev) { (void)dev; } +static inline void perf_data_inc_xrun(struct comp_dev *dev) { (void)dev; } +#endif + #if IS_ENABLED(CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS) || defined(__DOXYGEN__) /** * \brief Initializer for component performance data record. diff --git a/src/include/sof/debug/telemetry/telemetry.h b/src/include/sof/debug/telemetry/telemetry.h index a3694735cb4f..109916414781 100644 --- a/src/include/sof/debug/telemetry/telemetry.h +++ b/src/include/sof/debug/telemetry/telemetry.h @@ -13,10 +13,20 @@ #define __SOF_TELEMETRY_H__ #include +#include +#include +#include #ifdef __ZEPHYR__ +#include +#include #include #endif +struct pipeline; +struct dai_data; +struct host_data; +struct comp_dev; + #ifndef CONFIG_INTEL_ADSP_DEBUG_SLOT_MANAGER /** Slot index in memory window 2 (Debug Window) allocated for telemetry structures */ #define SOF_DW_TELEMETRY_SLOT 1 @@ -27,6 +37,48 @@ /** Measurement period in runs over which rolling average is computed */ #define SOF_AVG_PERF_MEAS_PERIOD 16 +/* Extended telemetry constants */ +#define SOF_TEL_EXT_MAGIC 0x324C4554U /* "TEL2" */ +#define SOF_TEL_EXT_VERSION 1 +#define SOF_TEL_EXT_HDR_OFFSET 0x0100U +#define SOF_TEL_MAX_CORES 5 +#define SOF_TEL_MAX_SECTIONS 6 +#define SOF_TEL_MAX_HEAPS 6 +#define SOF_TEL_MAX_VMH_BUNDLES 10 +#define SOF_TEL_MAX_VPAGE_ELEMS 16 +#define SOF_TEL_MAX_VREGIONS 16 +#define SOF_TEL_MAX_PIPELINES 16 +#define SOF_TEL_MAX_GATEWAYS 16 +#define SOF_TEL_MAX_THREADS 32 + +enum sof_tel_section_id { + SOF_TEL_SEC_HOT_ZONE = 1, /* 64B-aligned hot counters (sched, ppl, dai) */ + SOF_TEL_SEC_MEMORY = 2, /* Heaps, VMH, physical SRAM pages, fast_get */ + SOF_TEL_SEC_VREGION = 3, /* vpage allocator + active vregions */ + SOF_TEL_SEC_PIPELINES = 4, /* Cold pipeline & scheduler topology */ + SOF_TEL_SEC_DAIS = 5, /* Cold DAI & Host DMA gateway descriptors */ + SOF_TEL_SEC_THREADS = 6, /* Per-core Zephyr thread CPU & stack usage */ +}; + +enum sof_tel_heap_type { + SOF_TEL_HEAP_NONE = 0, + SOF_TEL_HEAP_SYS = 1, + SOF_TEL_HEAP_SHARED = 2, + SOF_TEL_HEAP_LL_USER = 3, + SOF_TEL_HEAP_L3_IMR = 4, + SOF_TEL_HEAP_DRV_USER = 5, +}; + +enum sof_tel_gw_type { + SOF_TEL_GW_NONE = 0, + SOF_TEL_GW_SSP = 1, + SOF_TEL_GW_DMIC = 2, + SOF_TEL_GW_ALH = 3, + SOF_TEL_GW_HDA = 4, + SOF_TEL_GW_HOST_DMA = 5, + SOF_TEL_GW_OTHER = 6, +}; + /** * \brief Per-core system tick and cycle execution metrics. * \ingroup telemetry_perf_api @@ -47,13 +99,234 @@ struct system_tick_info { uint32_t rsvd[2]; /**< Reserved for padding and future metrics */ } __packed; +/* Section descriptor in TEL2 directory header (8 bytes) */ +struct sof_tel_sec_desc { + uint8_t id; /* enum sof_tel_section_id */ + uint8_t elem_size; /* Size of each item in bytes (or 0 if composite) */ + uint16_t offset; /* 64B-aligned byte offset from start of MW2 Slot 1 (subtract 4 for /sys/kernel/debug/sof/exception) */ + uint16_t max_count; /* Allocated capacity in items */ + uint16_t active_mask; /* Bitmask or count of active entries */ +} __packed; + +/* Self-describing directory header at slot offset 0x0100 (64 bytes = 1 cache line) */ +struct sof_tel_ext_hdr { + uint32_t magic; /* SOF_TEL_EXT_MAGIC ("TEL2") */ + uint8_t version; /* SOF_TEL_EXT_VERSION */ + uint8_t max_cores; /* CONFIG_MAX_CORE_COUNT */ + uint16_t num_secs; /* Number of enabled sections */ + uint32_t seq; /* Odd while updating cold/bg snapshot, even when consistent */ + uint32_t timestamp_us; /* DSP uptime in us at last snapshot update */ + struct sof_tel_sec_desc secs[SOF_TEL_MAX_SECTIONS]; +} __packed; + +/* Hot Cache Line 0 at slot offset 0x0140 (64 bytes = 1 cache line) */ +struct sof_tel_hot_line0 { + uint16_t ll_overrun_count[SOF_TEL_MAX_CORES]; /* 10 B */ + uint16_t dp_deadline_misses[SOF_TEL_MAX_CORES]; /* 10 B */ + uint16_t active_ll_tasks[SOF_TEL_MAX_CORES]; /* 10 B */ + uint16_t total_xrun_events; /* 2 B */ + uint16_t ppl_xrun_count[SOF_TEL_MAX_PIPELINES]; /* 32 B */ +} __packed; + +/* Hot Gateway entry at slot offset 0x0180 (8 bytes -> 8 gateways per 64B cache line) */ +struct sof_tel_gw_hot { + uint32_t total_kb_or_frames; /* Rolling KB (bytes >> 10) processed */ + uint16_t dma_buf_avail_bytes; /* Live DMA buffer fill level (clamped to 64KB) */ + uint16_t xrun_or_skip_count; /* dd->xrun count or hd->n_skipped */ +} __packed; + +/* Physical SRAM & EBB summary (24 bytes) */ +struct sof_tel_sram_info { + uint16_t hp_sram_free_pages; + uint16_t hp_sram_total_pages; + uint16_t lp_sram_free_pages; + uint16_t lp_sram_total_pages; + uint32_t ebb_power_mask_lo; + uint32_t ebb_power_mask_hi; + uint32_t lpsram_ebb_mask; + uint32_t rsvd; +} __packed; + +/* Zephyr k_heap telemetry entry (20 bytes) */ +struct sof_tel_heap_info { + uint16_t heap_type; /* enum sof_tel_heap_type */ + uint16_t alloc_fail_count; + uint32_t total_bytes; + uint32_t allocated_bytes; + uint32_t free_bytes; + uint32_t max_allocated_bytes; +} __packed; + +/* Virtual Memory Heap (vmh_heap) block bundle entry (8 bytes) */ +struct sof_tel_vmh_bundle { + uint8_t blk_sz_shift; + uint8_t rsvd; + uint16_t total_blocks; + uint16_t used_blocks; + uint16_t max_blocks_or_oom; +} __packed; + +/* fast_get DRAM-to-SRAM cache pool summary (16 bytes) */ +struct sof_tel_fast_get_info { + uint16_t active_entries; + uint16_t fail_count; + uint32_t total_sram_bytes; + uint32_t peak_sram_bytes; + uint32_t total_get_calls; +} __packed; + +/* Memory & Heaps Section at slot offset 0x0240 (256 bytes = 4 cache lines) */ +struct sof_tel_mem_section { + struct sof_tel_sram_info sram; /* 24 B */ + struct sof_tel_heap_info heaps[SOF_TEL_MAX_HEAPS]; /* 120 B */ + uint32_t vmh_region_size; /* 4 B */ + uint16_t vmh_bundle_count; /* 2 B */ + uint16_t vmh_oom_total; /* 2 B */ + struct sof_tel_vmh_bundle vmh_bundles[SOF_TEL_MAX_VMH_BUNDLES]; /* 80 B */ + struct sof_tel_fast_get_info fast_get; /* 16 B */ + uint8_t pad[8]; /* 8 B */ +} __packed; + +/* vpage allocation element (4 bytes) */ +struct sof_tel_valloc_elem { + uint16_t pages; + uint16_t vpage; +} __packed; + +/* Active vregion instance telemetry (24 bytes) */ +struct sof_tel_vregion_info { + uint32_t base_va; + uint16_t total_pages; + uint8_t use_count; + uint8_t type_and_flags; + uint32_t lifetime_size_bytes; + uint32_t lifetime_used_bytes; + uint32_t interim_alloc_bytes; + uint16_t interim_free_kb; + uint16_t lifetime_free_count; +} __packed; + +/* Virtual Pages & vregion Section at slot offset 0x0340 (512 bytes = 8 cache lines) */ +struct sof_tel_vregion_section { + uint16_t total_pages; /* 2 B */ + uint16_t free_pages; /* 2 B */ + uint16_t min_free_pages; /* 2 B */ + uint16_t num_elems_in_use; /* 2 B */ + uint32_t alloc_fail_count; /* 4 B */ + uint32_t region_base_va; /* 4 B */ + struct sof_tel_valloc_elem velems[SOF_TEL_MAX_VPAGE_ELEMS]; /* 64 B */ + uint16_t active_vregion_count; /* 2 B */ + uint16_t total_vregions_created; /* 2 B */ + uint8_t pad[44]; /* 44 B */ + struct sof_tel_vregion_info vregions[SOF_TEL_MAX_VREGIONS]; /* 384 B */ +} __packed; + +/* Per-core scheduler summary (12 bytes) */ +struct sof_tel_core_sched { + uint8_t core_id; + uint8_t ll_task_count; + uint8_t dp_task_count; + uint8_t active_ppl_count; + uint32_t active_kcps_budget; + uint32_t rsvd; +} __packed; + +/* Cold pipeline descriptor (20 bytes) */ +struct sof_tel_ppl_info { + uint16_t pipeline_id; + uint8_t core; + uint8_t priority; + uint8_t status; + uint8_t comp_count; + uint16_t attributes; + uint32_t period_us; + uint32_t sched_comp_id; + int32_t last_xrun_bytes; +} __packed; + +/* Pipelines & Schedulers Section at slot offset 0x0540 (384 bytes = 6 cache lines) */ +struct sof_tel_ppl_section { + struct sof_tel_core_sched core_sched[SOF_TEL_MAX_CORES]; /* 60 B */ + uint16_t active_ppl_count; /* 2 B */ + uint16_t total_comp_count; /* 2 B */ + struct sof_tel_ppl_info pipelines[SOF_TEL_MAX_PIPELINES]; /* 320 B */ +} __packed; + +/* Cold DAI / Host DMA gateway descriptor (16 bytes) */ +struct sof_tel_gw_cold { + uint8_t gw_type; /* enum sof_tel_gw_type */ + uint8_t index; /* dai_index or stream_tag */ + uint8_t direction; /* PLAYBACK=0, CAPTURE=1 */ + int8_t dma_chan; /* chan_index */ + uint32_t comp_id; /* Owning Copier component ID */ + uint8_t state; /* COMP_STATE_* */ + uint8_t group_id; /* DAI group ID */ + uint16_t period_bytes; + uint32_t dma_buf_size; +} __packed; + +/* Cold DAIs & Host Gateways Section at slot offset 0x06C0 (256 bytes = 4 cache lines) */ +struct sof_tel_dai_section { + struct sof_tel_gw_cold gateways[SOF_TEL_MAX_GATEWAYS]; /* 256 B */ +} __packed; + +#define SOF_TEL_THD_FLG_IDLE BIT(0) +#define SOF_TEL_THD_FLG_USER BIT(1) +#define SOF_TEL_THD_FLG_DP BIT(2) +#define SOF_TEL_THD_FLG_LL BIT(3) + +/* Per-thread telemetry entry (24 bytes) */ +struct sof_tel_thread_info { + uint32_t thread_id; + uint8_t core; + int8_t prio; + uint8_t state; + uint8_t cpu_load; /* 0..255 scaled (0..100%) */ + uint8_t stack_usage; /* 0..255 scaled high-water mark (0..100%) */ + uint8_t flags; /* bit0=idle, bit1=user, bit2=dp, bit3=ll */ + uint16_t stack_size_chunks; /* stack size in 64-byte chunks */ + uint32_t total_exec_kcycles; /* execution_cycles >> 10 */ + char name[8]; /* 8-char thread name prefix */ +} __packed; + +/* Zephyr RTOS Threads Section at slot offset 0x07C0 (832 bytes = 13 cache lines) */ +struct sof_tel_thread_section { + uint8_t core_load[8]; /* 8 B */ + uint16_t thread_count; /* 2 B */ + uint16_t dropped_threads; /* 2 B */ + uint32_t sample_seq; /* 4 B */ + uint32_t rsvd[4]; /* 16 B */ + struct sof_tel_thread_info threads[SOF_TEL_MAX_THREADS];/* 768 B */ + uint8_t pad[32]; /* 32 B */ +} __packed; + /** - * \brief Layout of telemetry data published in shared Memory Window. + * \brief Layout of telemetry data published in shared Memory Window 2 Slot 1. * \ingroup telemetry_perf_api + * + * Legacy separator_1 + system_tick_info[] sits at 0x0000..0x00FF, + * followed at 64-byte-aligned offset 0x0100 by the TEL2 directory, + * Hot Counter Zone (0x0140..0x023F), and Cold Metadata Sections (0x0240..0x0AFF). */ struct telemetry_wnd_data { uint32_t separator_1; /**< Header boundary delimiter */ struct system_tick_info system_tick_info[CONFIG_MAX_CORE_COUNT]; /**< Per-core execution metrics */ +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW + uint8_t pad_to_0x100[SOF_TEL_EXT_HDR_OFFSET - sizeof(uint32_t) - + sizeof(struct system_tick_info) * CONFIG_MAX_CORE_COUNT]; + /* 0x0100: Self-describing directory header (64 B / 1 CL) */ + struct sof_tel_ext_hdr ext_hdr; + /* 0x0140: Hot Counter Zone (256 B / 4 CLs) */ + struct sof_tel_hot_line0 hot_line0; + struct sof_tel_gw_hot gw_hot[SOF_TEL_MAX_GATEWAYS]; + uint8_t hot_rsvd[64]; + /* 0x0240: Cold Metadata & Background Snapshot Zone */ + struct sof_tel_mem_section mem_sec; + struct sof_tel_vregion_section vreg_sec; + struct sof_tel_ppl_section ppl_sec; + struct sof_tel_dai_section dai_sec; + struct sof_tel_thread_section thd_sec; +#endif } __packed; /** @@ -85,6 +358,96 @@ void telemetry_update(uint32_t begin_ccount, uint32_t current_ccount); struct system_tick_info *telemetry_get_systick_info_ptr(void); #endif +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW +struct telemetry_wnd_data *telemetry_get_wnd_ptr(void); +void telemetry_refresh_cold_sections(void); +#else +static inline struct telemetry_wnd_data *telemetry_get_wnd_ptr(void) { return NULL; } +static inline void telemetry_refresh_cold_sections(void) {} +#endif + +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +void telemetry_collect_mem_stats(struct sof_tel_mem_section *mem); +void telemetry_collect_fast_get_stats(struct sof_tel_fast_get_info *fg); +void telemetry_notify_mem_alloc_fail(uint16_t heap_type); +#else +static inline void telemetry_collect_mem_stats(struct sof_tel_mem_section *mem) { (void)mem; } +static inline void telemetry_collect_fast_get_stats(struct sof_tel_fast_get_info *fg) { (void)fg; } +static inline void telemetry_notify_mem_alloc_fail(uint16_t heap_type) { (void)heap_type; } +#endif + +#ifdef CONFIG_SOF_TELEMETRY_VREGION +void telemetry_collect_vpage_stats(struct sof_tel_vregion_section *vsec); +void telemetry_collect_vregion_stats(struct sof_tel_vregion_section *vsec); +void telemetry_notify_vregion_change(void); +#else +static inline void telemetry_collect_vpage_stats(struct sof_tel_vregion_section *vsec) +{ + (void)vsec; +} +static inline void telemetry_collect_vregion_stats(struct sof_tel_vregion_section *vsec) +{ + (void)vsec; +} +static inline void telemetry_notify_vregion_change(void) {} +#endif + +#ifdef CONFIG_SOF_TELEMETRY_PIPELINES +void telemetry_notify_pipeline_change(void); +void telemetry_notify_pipeline_xrun(struct pipeline *p, int32_t bytes); +void telemetry_notify_dp_deadline_miss(int core); +void telemetry_notify_ll_task_count(int core, unsigned int n_tasks); +#else +static inline void telemetry_notify_pipeline_change(void) {} +static inline void telemetry_notify_pipeline_xrun(struct pipeline *p, int32_t bytes) +{ + (void)p; + (void)bytes; +} +static inline void telemetry_notify_dp_deadline_miss(int core) { (void)core; } +static inline void telemetry_notify_ll_task_count(int core, unsigned int n_tasks) +{ + (void)core; + (void)n_tasks; +} +#endif + +#ifdef CONFIG_SOF_TELEMETRY_DAIS +int telemetry_register_dai(struct dai_data *dd, struct comp_dev *dev); +void telemetry_unregister_dai(struct dai_data *dd); +void telemetry_update_dai_hot(struct dai_data *dd, struct comp_dev *dev); +int telemetry_register_host(struct host_data *hd, struct comp_dev *dev); +void telemetry_unregister_host(struct host_data *hd); +void telemetry_update_host_hot(struct host_data *hd, struct comp_dev *dev); +void telemetry_notify_host_skip(struct host_data *hd); +#else +static inline int telemetry_register_dai(struct dai_data *dd, struct comp_dev *dev) +{ + (void)dd; + (void)dev; + return -1; +} +static inline void telemetry_unregister_dai(struct dai_data *dd) { (void)dd; } +static inline void telemetry_update_dai_hot(struct dai_data *dd, struct comp_dev *dev) +{ + (void)dd; + (void)dev; +} +static inline int telemetry_register_host(struct host_data *hd, struct comp_dev *dev) +{ + (void)hd; + (void)dev; + return -1; +} +static inline void telemetry_unregister_host(struct host_data *hd) { (void)hd; } +static inline void telemetry_update_host_hot(struct host_data *hd, struct comp_dev *dev) +{ + (void)hd; + (void)dev; +} +static inline void telemetry_notify_host_skip(struct host_data *hd) { (void)hd; } +#endif + #ifdef CONFIG_TIMING_FUNCTIONS /** Timestamp counter accessor using Zephyr timing functions */ #define telemetry_timestamp timing_counter_get From fc11e699bf28f18354533de80d75bd3c48376fc5 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Tue, 10 Nov 2026 16:07:58 +0000 Subject: [PATCH 2/9] zephyr: lib: add heap, VMH, and fast-get memory telemetry Implement memory telemetry collection and allocation failure tracking across SOF system/shared/LL-user/L3 heaps, physical SRAM/LPSRAM page tables, Virtual Memory Heap (VMH) block bundles, and the fast_get DRAM-to-SRAM cache pool for the TEL2 memory telemetry section. Signed-off-by: Liam Girdwood --- zephyr/include/sof/lib/regions_mm.h | 4 + zephyr/lib/alloc.c | 168 +++++++++++++++++++++++++++- zephyr/lib/fast-get.c | 42 +++++++ zephyr/lib/regions_mm.c | 51 +++++++++ 4 files changed, 264 insertions(+), 1 deletion(-) diff --git a/zephyr/include/sof/lib/regions_mm.h b/zephyr/include/sof/lib/regions_mm.h index 259091ab4df3..fbbea048a8c8 100644 --- a/zephyr/include/sof/lib/regions_mm.h +++ b/zephyr/include/sof/lib/regions_mm.h @@ -133,6 +133,10 @@ void vmh_log_stats(struct vmh_heap *heap); */ void vmh_log_alloc_stats(struct vmh_heap *heap); #endif +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +struct sof_tel_mem_section; +void vmh_fill_telemetry_stats(struct vmh_heap *heap, struct sof_tel_mem_section *mem); +#endif /** * @brief Checks if ptr is in range of given memory range. * diff --git a/zephyr/lib/alloc.c b/zephyr/lib/alloc.c index 89bd49286831..827ea2a49461 100644 --- a/zephyr/lib/alloc.c +++ b/zephyr/lib/alloc.c @@ -13,6 +13,7 @@ #include #include #include +#include #include /* for zephyr_ll_user_heap() */ #include #include @@ -152,6 +153,37 @@ extern char _end[], _heap_sentry[]; static struct k_heap sof_heap; +#if !defined(CONFIG_SOF_NATIVE_SIM_HOST_HEAP) && defined(CONFIG_SOF_TELEMETRY_MEMORY) +#define TEL_HEAP_SYS_IDX 0 +#define TEL_HEAP_SHARED_IDX 1 +#define TEL_HEAP_LL_USER_IDX 2 +#define TEL_HEAP_L3_IMR_IDX 3 +#define TEL_HEAP_OTHER_IDX 4 + +static uint16_t heap_fail_counts[SOF_TEL_MAX_HEAPS]; + +void telemetry_notify_mem_alloc_fail(uint16_t heap_type) +{ + switch (heap_type) { + case SOF_TEL_HEAP_SYS: + heap_fail_counts[TEL_HEAP_SYS_IDX]++; + break; + case SOF_TEL_HEAP_SHARED: + heap_fail_counts[TEL_HEAP_SHARED_IDX]++; + break; + case SOF_TEL_HEAP_LL_USER: + heap_fail_counts[TEL_HEAP_LL_USER_IDX]++; + break; + case SOF_TEL_HEAP_L3_IMR: + heap_fail_counts[TEL_HEAP_L3_IMR_IDX]++; + break; + default: + heap_fail_counts[TEL_HEAP_OTHER_IDX]++; + break; + } +} +#endif + #if !defined(CONFIG_SOF_NATIVE_SIM_HOST_HEAP) /** * Checks whether pointer is from a given heap memory. @@ -256,17 +288,23 @@ static void *l3_heap_alloc_aligned(struct k_heap *h, size_t min_align, size_t by #endif if (!cpu_is_primary(arch_proc_id())) { tr_err(&zephyr_tr, "L3_HEAP available only for primary core!"); + telemetry_notify_mem_alloc_fail(SOF_TEL_HEAP_L3_IMR); return NULL; } /* not initialized, e.g. if no IMR space left after the image */ - if (!h->heap.heap) + if (!h->heap.heap) { + telemetry_notify_mem_alloc_fail(SOF_TEL_HEAP_L3_IMR); return NULL; + } key = k_spin_lock(&h->lock); ret = sys_heap_aligned_alloc(&h->heap, min_align, bytes); k_spin_unlock(&h->lock, key); + if (!ret) + telemetry_notify_mem_alloc_fail(SOF_TEL_HEAP_L3_IMR); + #if CONFIG_SYS_HEAP_RUNTIME_STATS && CONFIG_IPC_MAJOR_4 sys_heap_runtime_stats_get(&h->heap, &stats); tr_info(&zephyr_tr, "heap allocated: %u free: %u max allocated: %u", @@ -437,6 +475,19 @@ static void *heap_alloc_aligned(struct k_heap *h, size_t min_align, size_t bytes ret = sys_heap_aligned_alloc(&h->heap, min_align, bytes); k_spin_unlock(&h->lock, key); +#ifdef CONFIG_SOF_TELEMETRY_MEMORY + if (!ret) { + if (h == &sof_heap) + heap_fail_counts[TEL_HEAP_SYS_IDX]++; +#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP + else if (h == &shared_buffer_heap) + heap_fail_counts[TEL_HEAP_SHARED_IDX]++; +#endif + else + heap_fail_counts[TEL_HEAP_OTHER_IDX]++; + } +#endif + #if CONFIG_SYS_HEAP_RUNTIME_STATS && CONFIG_IPC_MAJOR_4 sys_heap_runtime_stats_get(&h->heap, &stats); tr_info(&zephyr_tr, "heap allocated: %u free: %u max allocated: %u", @@ -910,4 +961,119 @@ __weak void *_sbrk_r(struct _reent *ptr, ptrdiff_t incr) return NULL; } +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +static void fill_one_heap_telemetry(struct sof_tel_heap_info *dst, struct k_heap *h, + uint16_t type, uint16_t fail_cnt) +{ + dst->heap_type = type; + dst->alloc_fail_count = fail_cnt; + if (!h || !h->heap.heap) { + dst->total_bytes = 0; + dst->allocated_bytes = 0; + dst->free_bytes = 0; + dst->max_allocated_bytes = 0; + return; + } + + dst->total_bytes = (uint32_t)h->heap.init_bytes; +#if CONFIG_SYS_HEAP_RUNTIME_STATS + struct sys_memory_stats stats = { 0 }; + int ret = sys_heap_runtime_stats_get(&h->heap, &stats); + + if (ret == 0) { + dst->allocated_bytes = (uint32_t)stats.allocated_bytes; + dst->free_bytes = (uint32_t)stats.free_bytes; + dst->max_allocated_bytes = (uint32_t)stats.max_allocated_bytes; + if (dst->total_bytes == 0) + dst->total_bytes = dst->allocated_bytes + dst->free_bytes; + } +#endif +} + +void telemetry_collect_mem_stats(struct sof_tel_mem_section *mem) +{ + uint32_t free_heap_pages; + + if (!mem) + return; + + fill_one_heap_telemetry(&mem->heaps[TEL_HEAP_SYS_IDX], &sof_heap, + SOF_TEL_HEAP_SYS, heap_fail_counts[TEL_HEAP_SYS_IDX]); +#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP + fill_one_heap_telemetry(&mem->heaps[TEL_HEAP_SHARED_IDX], &shared_buffer_heap, + SOF_TEL_HEAP_SHARED, heap_fail_counts[TEL_HEAP_SHARED_IDX]); +#endif +#ifdef CONFIG_SOF_USERSPACE_LL + fill_one_heap_telemetry(&mem->heaps[TEL_HEAP_LL_USER_IDX], zephyr_ll_user_heap(), + SOF_TEL_HEAP_LL_USER, heap_fail_counts[TEL_HEAP_LL_USER_IDX]); +#endif +#if CONFIG_L3_HEAP + fill_one_heap_telemetry(&mem->heaps[TEL_HEAP_L3_IMR_IDX], &l3_heap, + SOF_TEL_HEAP_L3_IMR, heap_fail_counts[TEL_HEAP_L3_IMR_IDX]); +#endif + + free_heap_pages = (mem->heaps[TEL_HEAP_SYS_IDX].free_bytes + + mem->heaps[TEL_HEAP_SHARED_IDX].free_bytes + + mem->heaps[TEL_HEAP_LL_USER_IDX].free_bytes) / HOST_PAGE_SIZE; + +#if defined(CONFIG_SOC_SERIES_INTEL_ADSP_ACE) + uint32_t hp_banks = ace_hpsram_get_bank_count(); + uint32_t lp_banks = ace_lpsram_get_bank_count(); + uint64_t hp_mask = 0; + uint32_t lp_mask = 0; + uint32_t active_hp_banks = 0; + uint32_t active_lp_banks = 0; + uint32_t unpowered_hp_pages = 0; + uint32_t unpowered_lp_pages = 0; + + for (uint32_t i = 0; i < MIN(hp_banks, 64U); i++) { + if (HPSRAM_REGS(i)->HSxPGISTS == 0) { + hp_mask |= BIT64(i); + active_hp_banks++; + } + } + for (uint32_t i = 0; i < MIN(lp_banks, 32U); i++) { + if (LPSRAM_REGS(i)->USxPGISTS == 0) { + lp_mask |= BIT(i); + active_lp_banks++; + } + } + mem->sram.hp_sram_total_pages = (uint16_t)(L2_SRAM_SIZE / HOST_PAGE_SIZE); + if (hp_banks && hp_banks >= active_hp_banks) + unpowered_hp_pages = + (mem->sram.hp_sram_total_pages * (hp_banks - active_hp_banks)) / hp_banks; + mem->sram.hp_sram_free_pages = + (uint16_t)MIN((uint32_t)mem->sram.hp_sram_total_pages, + unpowered_hp_pages + free_heap_pages); + + mem->sram.lp_sram_total_pages = (uint16_t)(LP_SRAM_SIZE / HOST_PAGE_SIZE); + if (lp_banks && lp_banks >= active_lp_banks) + unpowered_lp_pages = + (mem->sram.lp_sram_total_pages * (lp_banks - active_lp_banks)) / lp_banks; + mem->sram.lp_sram_free_pages = (uint16_t)unpowered_lp_pages; + mem->sram.ebb_power_mask_lo = (uint32_t)(hp_mask & 0xFFFFFFFFU); + mem->sram.ebb_power_mask_hi = (uint32_t)(hp_mask >> 32); + mem->sram.lpsram_ebb_mask = lp_mask; +#elif defined(PLATFORM_HPSRAM_EBB_COUNT) + mem->sram.hp_sram_total_pages = + (uint16_t)((SRAM_BANK_SIZE * PLATFORM_HPSRAM_EBB_COUNT) / HOST_PAGE_SIZE); + mem->sram.hp_sram_free_pages = + (uint16_t)MIN((uint32_t)mem->sram.hp_sram_total_pages, free_heap_pages); + mem->sram.ebb_power_mask_lo = HPSRAM_REGS(0)->HSxPGCTL; +#if PLATFORM_HPSRAM_EBB_COUNT > 32 + mem->sram.ebb_power_mask_hi = HPSRAM_REGS(1)->HSxPGCTL; +#endif +#endif + +#if CONFIG_VIRTUAL_HEAP + if (virtual_buffers_heap) + vmh_fill_telemetry_stats(virtual_buffers_heap, mem); +#endif + +#if defined(CONFIG_FAST_GET) && defined(CONFIG_COLD_STORE_EXECUTE_DRAM) + telemetry_collect_fast_get_stats(&mem->fast_get); +#endif +} +#endif + SYS_INIT(heap_init, PRE_KERNEL_1, CONFIG_KERNEL_INIT_PRIORITY_OBJECTS); diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c index 24d6af4750b5..df9c0b1619a1 100644 --- a/zephyr/lib/fast-get.c +++ b/zephyr/lib/fast-get.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -37,6 +38,14 @@ static struct sof_fast_get_data fast_get_data = { .pool.list = LIST_INIT(fast_get_data.pool.list), }; +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +static uint32_t fast_get_total_calls; +static uint16_t fast_get_active_entries; +static uint16_t fast_get_fail_count; +static uint32_t fast_get_live_bytes; +static uint32_t fast_get_peak_bytes; +#endif + LOG_MODULE_REGISTER(fast_get, CONFIG_SOF_LOG_LEVEL); struct fast_get_find { @@ -270,7 +279,18 @@ const void *fast_get(struct mod_alloc_ctx *alloc, const void *dram_ptr, size_t s entry->size = size; entry->sram_ptr = ret; entry->refcount = 1; +#ifdef CONFIG_SOF_TELEMETRY_MEMORY + fast_get_active_entries++; + fast_get_live_bytes += (uint32_t)size; + if (fast_get_live_bytes > fast_get_peak_bytes) + fast_get_peak_bytes = fast_get_live_bytes; +#endif out: +#ifdef CONFIG_SOF_TELEMETRY_MEMORY + fast_get_total_calls++; + if (!ret) + fast_get_fail_count++; +#endif k_spin_unlock(&data->lock, key); LOG_DBG("get %p, %p, size %u, refcnt %u", dram_ptr, ret, size, entry ? entry->refcount : 0); @@ -278,6 +298,20 @@ const void *fast_get(struct mod_alloc_ctx *alloc, const void *dram_ptr, size_t s } EXPORT_SYMBOL(fast_get); +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +void telemetry_collect_fast_get_stats(struct sof_tel_fast_get_info *fg) +{ + if (!fg) + return; + + fg->active_entries = fast_get_active_entries; + fg->fail_count = fast_get_fail_count; + fg->total_sram_bytes = fast_get_live_bytes; + fg->peak_sram_bytes = fast_get_peak_bytes; + fg->total_get_calls = fast_get_total_calls; +} +#endif + static bool fast_put_find_entry(void *data, void *arg) { struct sof_fast_get_entry *entry = data; @@ -312,6 +346,14 @@ void fast_put(struct mod_alloc_ctx *alloc, struct k_mem_domain *mdom, const void entry->refcount--; if (!entry->refcount) { +#ifdef CONFIG_SOF_TELEMETRY_MEMORY + if (fast_get_active_entries > 0) + fast_get_active_entries--; + if (fast_get_live_bytes >= (uint32_t)entry->size) + fast_get_live_bytes -= (uint32_t)entry->size; + else + fast_get_live_bytes = 0; +#endif LOG_DBG("freeing buffer %p", sram_ptr); if (alloc && alloc->vreg && entry->size <= FAST_GET_MAX_COPY_SIZE) vregion_free(alloc->vreg, entry->sram_ptr); diff --git a/zephyr/lib/regions_mm.c b/zephyr/lib/regions_mm.c index 24d642cd0903..be0c32b679ac 100644 --- a/zephyr/lib/regions_mm.c +++ b/zephyr/lib/regions_mm.c @@ -10,6 +10,9 @@ #if defined(CONFIG_MM_DRV) #include #include +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +#include +#endif LOG_MODULE_DECLARE(mem_allocator, CONFIG_SOF_LOG_LEVEL); @@ -797,4 +800,52 @@ void vmh_get_default_heap_config(const struct sys_mm_drv_region *region, } } +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +#ifndef CONFIG_SYS_MEM_BLOCKS_RUNTIME_STATS +static uint16_t vmh_max_used_blocks[SOF_TEL_MAX_VMH_BUNDLES]; +#endif + +void vmh_fill_telemetry_stats(struct vmh_heap *heap, struct sof_tel_mem_section *mem) +{ + uint16_t bundle_cnt = 0; + uint16_t oom_total = 0; + + if (!heap || !mem) + return; + + mem->vmh_region_size = heap->virtual_region ? (uint32_t)heap->virtual_region->size : 0; + + for (int idx = 0; idx < MAX_MEMORY_ALLOCATORS_COUNT && + bundle_cnt < SOF_TEL_MAX_VMH_BUNDLES; idx++) { + struct sys_mem_blocks *blk = heap->physical_blocks_allocators[idx]; + uint32_t used = 0; + + if (!blk) + continue; + + if (blk->bitmap && blk->bitmap->bundles) { + for (size_t k = 0; k < blk->bitmap->num_bundles; k++) + used += POPCOUNT(blk->bitmap->bundles[k]); + } + + mem->vmh_bundles[bundle_cnt].blk_sz_shift = blk->info.blk_sz_shift; + mem->vmh_bundles[bundle_cnt].total_blocks = (uint16_t)blk->info.num_blocks; + mem->vmh_bundles[bundle_cnt].used_blocks = (uint16_t)used; +#ifdef CONFIG_SYS_MEM_BLOCKS_RUNTIME_STATS + mem->vmh_bundles[bundle_cnt].max_blocks_or_oom = + (uint16_t)blk->info.max_used_blocks; + oom_total += (uint16_t)heap->out_of_blocks[idx]; +#else + if ((uint16_t)used > vmh_max_used_blocks[bundle_cnt]) + vmh_max_used_blocks[bundle_cnt] = (uint16_t)used; + mem->vmh_bundles[bundle_cnt].max_blocks_or_oom = vmh_max_used_blocks[bundle_cnt]; +#endif + bundle_cnt++; + } + + mem->vmh_bundle_count = bundle_cnt; + mem->vmh_oom_total = oom_total; +} +#endif + #endif /* if defined (CONFIG_MM_DRV) */ From 65bcac8670853386cadf386fe2bc73dde9cbc0a5 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Tue, 10 Nov 2026 16:07:58 +0000 Subject: [PATCH 3/9] zephyr: vregion: add vpage and virtual region telemetry Track virtual page allocator utilization, fragmentation, minimum free pages, and allocation failures in vpage.c, and snapshot active virtual region lifetime and interim heap metrics in vregion.c for the TEL2 vregion telemetry section. Signed-off-by: Liam Girdwood --- zephyr/lib/vpage.c | 43 ++++++++++++++++- zephyr/lib/vregion.c | 111 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 152 insertions(+), 2 deletions(-) diff --git a/zephyr/lib/vpage.c b/zephyr/lib/vpage.c index 8a7e54261006..67f64f439b45 100644 --- a/zephyr/lib/vpage.c +++ b/zephyr/lib/vpage.c @@ -12,6 +12,7 @@ #include #include #include +#include #include LOG_MODULE_REGISTER(vpage, CONFIG_SOF_LOG_LEVEL); @@ -63,6 +64,10 @@ struct vpage_context { /* uncache persistent across all cores */ static struct vpage_context vpage_ctx; +#ifdef CONFIG_SOF_TELEMETRY_VREGION +static uint16_t vpage_min_free_pages = UINT16_MAX; +static uint32_t vpage_alloc_fail_count; +#endif /** * @brief Allocate and map virtual memory pages @@ -159,6 +164,12 @@ __cold void *vpage_alloc(unsigned int pages) k_mutex_lock(&vpage_ctx.lock, K_FOREVER); ret = vpages_alloc_and_map(pages, &ptr); +#ifdef CONFIG_SOF_TELEMETRY_VREGION + if (ret < 0) + vpage_alloc_fail_count++; + else if (vpage_ctx.free_pages < vpage_min_free_pages) + vpage_min_free_pages = (uint16_t)vpage_ctx.free_pages; +#endif k_mutex_unlock(&vpage_ctx.lock); if (ret < 0) LOG_ERR("vpage_alloc failed %d for %d pages, total %d free %d", @@ -166,6 +177,7 @@ __cold void *vpage_alloc(unsigned int pages) else LOG_INF("vpage_alloc ptr %p pages %u free %u/%u", ptr, pages, vpage_ctx.free_pages, vpage_ctx.total_pages); + telemetry_notify_vregion_change(); return ptr; } @@ -258,10 +270,39 @@ void vpage_free(void *ptr) ret = vpages_free_and_unmap((uintptr_t *)ptr); k_mutex_unlock(&vpage_ctx.lock); - if (!ret) + if (!ret) { LOG_INF("vptr %p free/total pages %d/%d", ptr, vpage_ctx.free_pages, vpage_ctx.total_pages); + telemetry_notify_vregion_change(); + } +} + +#ifdef CONFIG_SOF_TELEMETRY_VREGION +void telemetry_collect_vpage_stats(struct sof_tel_vregion_section *vsec) +{ + if (!vsec || !vpage_ctx.virtual_region) + return; + + vsec->total_pages = (uint16_t)vpage_ctx.total_pages; + vsec->free_pages = (uint16_t)vpage_ctx.free_pages; + vsec->min_free_pages = (vpage_min_free_pages == UINT16_MAX) ? + (uint16_t)vpage_ctx.free_pages : + vpage_min_free_pages; + vsec->num_elems_in_use = (uint16_t)vpage_ctx.num_elems_in_use; + vsec->alloc_fail_count = vpage_alloc_fail_count; + vsec->region_base_va = (uint32_t)POINTER_TO_UINT(vpage_ctx.virtual_region->addr); + + for (int i = 0; i < SOF_TEL_MAX_VPAGE_ELEMS; i++) { + if (i < VPAGE_MAX_ALLOCS) { + vsec->velems[i].pages = vpage_ctx.velems[i].pages; + vsec->velems[i].vpage = vpage_ctx.velems[i].vpage; + } else { + vsec->velems[i].pages = 0; + vsec->velems[i].vpage = 0; + } + } } +#endif /** * @brief Initialize virtual page allocator diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c index 846f6d24356e..f245af699d35 100644 --- a/zephyr/lib/vregion.c +++ b/zephyr/lib/vregion.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -117,6 +118,81 @@ struct vregion { static sys_dlist_t vregion_list = SYS_DLIST_STATIC_INIT(&vregion_list); static K_MUTEX_DEFINE(vregion_list_lock); +#ifdef CONFIG_SOF_TELEMETRY_VREGION +static struct vregion *tel_vregions[SOF_TEL_MAX_VREGIONS]; +static struct sof_tel_vregion_info tel_vregion_snap[SOF_TEL_MAX_VREGIONS]; +static uint16_t tel_vregions_created; + +static void vregion_tel_fill_info(struct vregion *vr, struct sof_tel_vregion_info *info) +{ + memset(info, 0, sizeof(*info)); + info->base_va = (uint32_t)POINTER_TO_UINT(vr->base); + info->total_pages = (uint16_t)vr->pages; + info->use_count = (uint8_t)MIN(vr->use_count, UINT8_MAX); + info->type_and_flags = (uint8_t)vr->type; + info->lifetime_size_bytes = (uint32_t)vr->lifetime.size; + info->lifetime_used_bytes = (uint32_t)MIN(vr->lifetime.used, vr->lifetime.size); + info->lifetime_free_count = (uint16_t)MIN(vr->lifetime.free_count, UINT16_MAX); +#if CONFIG_SYS_HEAP_RUNTIME_STATS + if (vr->type == VREGION_MEM_TYPE_INTERIM && vr->interim.heap.heap.heap) { + struct sys_memory_stats stats = { 0 }; + int ret = sys_heap_runtime_stats_get(&vr->interim.heap.heap, &stats); + + if (ret == 0) { + info->interim_alloc_bytes = (uint32_t)stats.allocated_bytes; + info->interim_free_kb = (uint16_t)(stats.free_bytes >> 10); + } + } +#endif +} + +/* Must be called while vr->lock is held (or during creation before vr is published) */ +static void vregion_tel_sync_locked(struct vregion *vr) +{ + struct sof_tel_vregion_info info; + + vregion_tel_fill_info(vr, &info); + for (int i = 0; i < SOF_TEL_MAX_VREGIONS; i++) { + if (tel_vregions[i] == vr) { + tel_vregion_snap[i] = info; + break; + } + } +} + +static void vregion_tel_register(struct vregion *vr) +{ + struct sof_tel_vregion_info info; + + vregion_tel_fill_info(vr, &info); + tel_vregions_created++; + for (int i = 0; i < SOF_TEL_MAX_VREGIONS; i++) { + if (!tel_vregions[i]) { + tel_vregions[i] = vr; + tel_vregion_snap[i] = info; + break; + } + } + telemetry_notify_vregion_change(); +} + +static void vregion_tel_unregister(struct vregion *vr) +{ + for (int i = 0; i < SOF_TEL_MAX_VREGIONS; i++) { + if (tel_vregions[i] == vr) { + tel_vregions[i] = NULL; + memset(&tel_vregion_snap[i], 0, sizeof(tel_vregion_snap[i])); + break; + } + } + telemetry_notify_vregion_change(); +} +#else +static inline void vregion_tel_sync_locked(struct vregion *vr) { (void)vr; } +static inline void vregion_tel_register(struct vregion *vr) { (void)vr; } +static inline void vregion_tel_unregister(struct vregion *vr) { (void)vr; } +#endif + /** * @brief Create a new virtual region instance. * @@ -197,6 +273,8 @@ __cold struct vregion *vregion_create(size_t memsize) LOG_INF("new at base %p size %#zx pages %u metadata at %p", (void *)vregion_base, total_size, pages, (void *)vr); + vregion_tel_register(vr); + return vr; } @@ -207,6 +285,7 @@ struct vregion *z_impl_vregion_get(struct vregion *vr) k_mutex_lock(&vr->lock, K_FOREVER); vr->use_count++; + vregion_tel_sync_locked(vr); k_mutex_unlock(&vr->lock); return vr; @@ -299,6 +378,7 @@ static void vregion_destroy(struct vregion *vr) /* log the vregion being destroyed */ LOG_DBG("destroy %p size %#zx pages %u", (void *)vr->base, vr->size, vr->pages); LOG_DBG(" lifetime used %zu free count %d", vr->lifetime.used, vr->lifetime.free_count); + vregion_tel_unregister(vr); vpage_free(vr->base); vregion_unmap(vr); @@ -319,6 +399,8 @@ static int vregion_put_locked(struct vregion *vr) if (!--vr->use_count) sys_dlist_remove(&vr->node); + else + vregion_tel_sync_locked(vr); return vr->use_count; } @@ -414,6 +496,7 @@ void z_impl_vregion_set_interim(struct vregion *vr) k_mutex_lock(&vr->lock, K_FOREVER); interim_heap_init(vr); + vregion_tel_sync_locked(vr); k_mutex_unlock(&vr->lock); } @@ -535,6 +618,7 @@ void z_impl_vregion_free(struct vregion *vr, void *ptr) else LOG_ERR("error: vregion free invalid pointer %p", ptr); + vregion_tel_sync_locked(vr); k_mutex_unlock(&vr->lock); } EXPORT_SYMBOL(z_impl_vregion_free); @@ -573,6 +657,7 @@ void *z_impl_vregion_alloc_align(struct vregion *vr, p = NULL; } + vregion_tel_sync_locked(vr); k_mutex_unlock(&vr->lock); #if CONFIG_SYS_HEAP_RUNTIME_STATS @@ -703,8 +788,10 @@ bool vregion_pinned_get(struct vregion *vr) k_mutex_lock(&vr->lock, K_FOREVER); /* do not revive a vregion whose last reference is already gone */ ok = vr->use_count && vr->use_count < UINT_MAX; - if (ok) + if (ok) { vr->use_count++; + vregion_tel_sync_locked(vr); + } k_mutex_unlock(&vr->lock); return ok; @@ -724,3 +811,25 @@ int vregion_pinned_put(struct vregion *vr) return ret; } #endif + +#ifdef CONFIG_SOF_TELEMETRY_VREGION +void telemetry_collect_vregion_stats(struct sof_tel_vregion_section *vsec) +{ + uint16_t active_cnt = 0; + + if (!vsec) + return; + + for (int i = 0; i < SOF_TEL_MAX_VREGIONS; i++) { + if (!tel_vregions[i]) { + memset(&vsec->vregions[i], 0, sizeof(vsec->vregions[i])); + continue; + } + active_cnt++; + vsec->vregions[i] = tel_vregion_snap[i]; + } + + vsec->active_vregion_count = active_cnt; + vsec->total_vregions_created = tel_vregions_created; +} +#endif From d8d8ca7a63b0dd7ecfbb8e4da303e1373724f138 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Tue, 10 Nov 2026 16:07:58 +0000 Subject: [PATCH 4/9] telemetry: implement extended TEL2 and PMW2 telemetry windows Implement the extended Memory Window 2 (TEL2) and Memory Window 3 (PMW2) telemetry sections, seqlock publication protocol, uncached hot-counter staging, periodic background cold-section and Zephyr thread snapshots, and active LL tick cycle accounting in telemetry.c and performance_monitor.c. Signed-off-by: Liam Girdwood --- src/kernel/debug/telemetry/Kconfig | 87 ++ .../debug/telemetry/performance_monitor.c | 177 ++- src/kernel/debug/telemetry/telemetry.c | 1118 ++++++++++++++++- 3 files changed, 1367 insertions(+), 15 deletions(-) diff --git a/src/kernel/debug/telemetry/Kconfig b/src/kernel/debug/telemetry/Kconfig index f380a90e1808..a0c30fc9f3b3 100644 --- a/src/kernel/debug/telemetry/Kconfig +++ b/src/kernel/debug/telemetry/Kconfig @@ -27,3 +27,90 @@ config SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS Disabled by default and enabled with IPC. Measurements can be extracted also by IPC. Interfaces measured: IPC, IDC, DMIC, I2S, SNDW, HDA, USB, GPIO, I2c, I3C, UART, SPI, CSI_2, DTF. +if SOF_TELEMETRY + +config SOF_TELEMETRY_EXTENDED_WINDOW + bool "Extended telemetry directory in Memory Window 2 Slot 1" + default y + help + Places a 64-byte-aligned self-describing TEL2 directory header and + cache-line-concentrated hot counter zone after system_tick_info in + the telemetry debug window slot to expose internal SOF resources. + +config SOF_TELEMETRY_HOT_CACHE_STAGING + bool "Stage LL hot-path telemetry updates in L1 D-cache lines" + depends on SOF_TELEMETRY_EXTENDED_WINDOW + default y + help + Accumulates LL hot-path counter increments in 64-byte-aligned L1 D$ + lines and flushes dirty cache lines to the uncached memory window + once at the end of the LL tick. + +config SOF_TELEMETRY_MEMORY + bool "Memory, k_heap, virtual heap, and SRAM page telemetry" + depends on SOF_TELEMETRY_EXTENDED_WINDOW + select SYS_HEAP_RUNTIME_STATS + select SYS_MEM_BLOCKS_RUNTIME_STATS if SYS_MEM_BLOCKS + default y + help + Exports live usage, free bytes, high-water marks, and allocation + failure counters for sof_heap, shared_buffer_heap, ll_user_heap, + l3_heap, virtual_buffers_heap (vmh), and fast_get in MW2 Slot 1. + +config SOF_TELEMETRY_VREGION + bool "Virtual page (vpage) and virtual region (vregion) telemetry" + depends on SOF_TELEMETRY_EXTENDED_WINDOW && SOF_VREGIONS + default y + help + Tracks vpage_ctx page usage/fragmentation and per-vregion lifetime + linear allocator and interim k_heap usage in MW2 Slot 1. + +config SOF_TELEMETRY_PIPELINES + bool "Pipeline and LL/DP scheduler telemetry" + depends on SOF_TELEMETRY_EXTENDED_WINDOW + default y + help + Exports active pipeline states, periods, priorities, XRUN counts, + and per-core LL/DP task counts and deadline misses in MW2 Slot 1. + +config SOF_TELEMETRY_DAIS + bool "DAI and Host DMA gateway telemetry" + depends on SOF_TELEMETRY_EXTENDED_WINDOW + default y + help + Exports active DAI and Host DMA gateway state, DMA channel, buffer + fill levels, XRUN/skip counts, and total bytes processed in MW2 Slot 1. + +config SOF_TELEMETRY_THREADS + bool "Compact Zephyr thread CPU and stack telemetry in MW2 Slot 1" + depends on SOF_TELEMETRY_EXTENDED_WINDOW + select THREAD_MONITOR + select THREAD_NAME + select THREAD_STACK_INFO + select INIT_STACKS + select THREAD_RUNTIME_STATS + default y + help + Periodically snapshots per-thread CPU load (%) and stack high-water + mark (%) into MW2 Slot 1 without requiring an extra 4 KB debug_stream slot. + +config SOF_TELEMETRY_MODULE_RESOURCES + bool "Per-module heap usage, CPC violation, and I/O telemetry in MW3" + depends on SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + default y + help + Partitions Memory Window 3 to include per-component module_resources + (heap_usage, heap_high_water_mark), peak CPC vs budget, and relocating + io_perf_data_items from .bss into MW3. + +config SOF_TELEMETRY_MAX_COMP_ENTRIES + int "Maximum component entries in Memory Window 3" + depends on SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + range 1 88 if SOF_TELEMETRY_MODULE_RESOURCES + range 1 170 + default 88 if SOF_TELEMETRY_MODULE_RESOURCES + default 170 + help + Maximum number of component performance entries in Memory Window 3. + +endif # SOF_TELEMETRY diff --git a/src/kernel/debug/telemetry/performance_monitor.c b/src/kernel/debug/telemetry/performance_monitor.c index 49f5427b3982..65adaba88bdb 100644 --- a/src/kernel/debug/telemetry/performance_monitor.c +++ b/src/kernel/debug/telemetry/performance_monitor.c @@ -5,6 +5,7 @@ // Author: Tobiasz Dryjanski #include +#include #include #include #include @@ -28,8 +29,41 @@ LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES +#define PERF_DATA_MAX_WINDOW_BYTES SOF_TEL_MW3_MOD_HOT_OFFSET +#else +#define PERF_DATA_MAX_WINDOW_BYTES CONFIG_MEMORY_WIN_3_SIZE +#endif + +#ifdef CONFIG_SOF_TELEMETRY_MAX_COMP_ENTRIES +#define PERFORMANCE_DATA_ENTRIES_COUNT CONFIG_SOF_TELEMETRY_MAX_COMP_ENTRIES +#else #define PERFORMANCE_DATA_ENTRIES_COUNT \ - (CONFIG_MEMORY_WIN_3_SIZE / sizeof(struct perf_data_item_comp)) + (PERF_DATA_MAX_WINDOW_BYTES / sizeof(struct perf_data_item_comp)) +#endif + +BUILD_ASSERT(PERFORMANCE_DATA_ENTRIES_COUNT * sizeof(struct perf_data_item_comp) <= + PERF_DATA_MAX_WINDOW_BYTES, + "PERFORMANCE_DATA_ENTRIES_COUNT exceeds Memory Window 3 perf_data partition"); +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES +BUILD_ASSERT(SOF_TEL_MW3_MOD_HOT_OFFSET + + PERFORMANCE_DATA_ENTRIES_COUNT * sizeof(struct sof_tel_mod_hot) <= + SOF_TEL_MW3_MOD_COLD_OFFSET, + "mod_hot_data exceeds its Memory Window 3 partition"); +BUILD_ASSERT(SOF_TEL_MW3_MOD_COLD_OFFSET + + PERFORMANCE_DATA_ENTRIES_COUNT * sizeof(struct sof_tel_mod_cold) <= + SOF_TEL_MW3_IO_PERF_OFFSET, + "mod_cold_data exceeds its Memory Window 3 partition"); +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS +BUILD_ASSERT(SOF_TEL_MW3_IO_PERF_OFFSET + + SOF_TEL_MW3_MAX_IO_ITEMS * sizeof(struct io_perf_data_item) <= + SOF_TEL_MW3_HDR_OFFSET, + "io_perf_data exceeds its Memory Window 3 partition"); +#endif +BUILD_ASSERT(SOF_TEL_MW3_HDR_OFFSET + sizeof(struct sof_tel_mw3_hdr) <= + CONFIG_MEMORY_WIN_3_SIZE, + "sof_tel_mw3_hdr exceeds Memory Window 3 size"); +#endif SYS_BITARRAY_DEFINE_STATIC(performance_data_bit_array, PERFORMANCE_DATA_ENTRIES_COUNT); @@ -50,6 +84,11 @@ struct perf_bitmap performance_data_bitmap; struct perf_data_item_comp *perf_data; +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES +static struct sof_tel_mod_hot *mod_hot_data; +static struct sof_tel_mod_cold *mod_cold_data; +#endif + /* Note that ref. FW used one state per core, all set together to the same state * by one IPC but only for active cores. It may work slightly different in case * where we enable a core while perf meas is started. @@ -157,6 +196,14 @@ int perf_data_free(struct perf_data_item_comp * const item) if (ret < 0) return ret; +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES + if (mod_hot_data && idx >= 0 && idx < PERFORMANCE_DATA_ENTRIES_COUNT) { + memset(item, 0, sizeof(*item)); + memset(&mod_hot_data[idx], 0, sizeof(mod_hot_data[idx])); + memset(&mod_cold_data[idx], 0, sizeof(mod_cold_data[idx])); + } +#endif + return 0; } @@ -169,6 +216,13 @@ void perf_data_item_comp_reset(struct perf_data_item_comp *perf) perf->restricted_peak_cycles = 0; perf->item.peak_kcps = 0; perf->item.avg_kcps = 0; + +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES + int idx = (int)(perf - perf_data); + + if (mod_hot_data && idx >= 0 && idx < PERFORMANCE_DATA_ENTRIES_COUNT) + memset(&mod_hot_data[idx], 0, sizeof(mod_hot_data[idx])); +#endif } __cold void perf_data_item_comp_init(struct perf_data_item_comp *perf, uint32_t resource_id, @@ -180,7 +234,88 @@ __cold void perf_data_item_comp_init(struct perf_data_item_comp *perf, uint32_t perf->item.resource_id = resource_id; perf->item.is_removed = false; perf->item.power_mode = power_mode; + +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES + int idx = (int)(perf - perf_data); + + if (mod_cold_data && idx >= 0 && idx < PERFORMANCE_DATA_ENTRIES_COUNT) { + mod_cold_data[idx].resource_id = resource_id; + mod_cold_data[idx].heap_usage = 0; + mod_cold_data[idx].heap_high_water_mark = 0; + mod_cold_data[idx].configured_cpc = 0; + } +#endif +} + +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES +void perf_data_update_module_resources(struct comp_dev *dev) +{ + struct perf_data_item_comp *item; +#if CONFIG_COMP_MODULE_ADAPTER + struct processing_module *mod; + size_t hwm = 0; + size_t usage; +#endif + int idx; + + if (!dev || !mod_cold_data || !perf_data) + return; + + item = dev->perf_data.perf_data_item; + if (!item || item->item.is_removed) + return; + + idx = (int)(item - perf_data); + if (idx < 0 || idx >= PERFORMANCE_DATA_ENTRIES_COUNT) + return; + + mod_cold_data[idx].resource_id = dev->ipc_config.id; + mod_cold_data[idx].configured_cpc = dev->cpc; +#if CONFIG_COMP_MODULE_ADAPTER + mod = dev->mod; + if (mod) { + usage = module_adapter_heap_usage(mod, &hwm); + mod_cold_data[idx].heap_usage = (uint32_t)usage; + mod_cold_data[idx].heap_high_water_mark = + MAX(mod_cold_data[idx].heap_high_water_mark, (uint32_t)hwm); + } +#endif +} + +void perf_data_inc_cpc_violation(struct comp_dev *dev) +{ + struct perf_data_item_comp *item; + int idx; + + if (!dev || !mod_hot_data || !perf_data) + return; + + item = dev->perf_data.perf_data_item; + if (!item || item->item.is_removed) + return; + + idx = (int)(item - perf_data); + if (idx >= 0 && idx < PERFORMANCE_DATA_ENTRIES_COUNT) + mod_hot_data[idx].cpc_violation_count++; +} + +void perf_data_inc_xrun(struct comp_dev *dev) +{ + struct perf_data_item_comp *item; + int idx; + + if (!dev || !mod_hot_data || !perf_data) + return; + + item = dev->perf_data.perf_data_item; + if (!item || item->item.is_removed) + return; + + idx = (int)(item - perf_data); + if (idx >= 0 && idx < PERFORMANCE_DATA_ENTRIES_COUNT) + mod_hot_data[idx].xrun_count++; } +#endif __cold int free_performance_data(struct perf_data_item_comp *item) { @@ -432,21 +567,44 @@ __cold int performance_monitor_init(void) perf_bitmap_init(&performance_data_bitmap, &performance_data_bit_array, PERFORMANCE_DATA_ENTRIES_COUNT); +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES + uint8_t *pmw_base = (uint8_t *)ADSP_PMW; + struct sof_tel_mw3_hdr *hdr = (struct sof_tel_mw3_hdr *)(pmw_base + SOF_TEL_MW3_HDR_OFFSET); + + memset(perf_data, 0, sizeof(*perf_data) * PERFORMANCE_DATA_ENTRIES_COUNT); + mod_hot_data = (struct sof_tel_mod_hot *)(pmw_base + SOF_TEL_MW3_MOD_HOT_OFFSET); + mod_cold_data = (struct sof_tel_mod_cold *)(pmw_base + SOF_TEL_MW3_MOD_COLD_OFFSET); + memset(mod_hot_data, 0, sizeof(*mod_hot_data) * PERFORMANCE_DATA_ENTRIES_COUNT); + memset(mod_cold_data, 0, sizeof(*mod_cold_data) * PERFORMANCE_DATA_ENTRIES_COUNT); + memset(hdr, 0, sizeof(*hdr)); + hdr->magic = SOF_TEL_MW3_MAGIC; + hdr->version = 1; + hdr->max_comp_entries = PERFORMANCE_DATA_ENTRIES_COUNT; + hdr->max_io_entries = SOF_TEL_MW3_MAX_IO_ITEMS; + hdr->mod_hot_offset = SOF_TEL_MW3_MOD_HOT_OFFSET; + hdr->mod_cold_offset = SOF_TEL_MW3_MOD_COLD_OFFSET; + hdr->io_perf_offset = SOF_TEL_MW3_IO_PERF_OFFSET; +#endif + return 0; } -/* init performance monitor using Zephyr */ -SYS_INIT(performance_monitor_init, APPLICATION, CONFIG_APPLICATION_INIT_PRIORITY); +/* init performance monitor before sof_init (POST_KERNEL, 99) creates components */ +SYS_INIT(performance_monitor_init, PRE_KERNEL_2, 0); #ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS -SYS_BITARRAY_DEFINE_STATIC(io_performance_data_bit_array, PERFORMANCE_DATA_ENTRIES_COUNT); - +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES +#define IO_PERFORMANCE_MAX_ENTRIES SOF_TEL_MW3_MAX_IO_ITEMS +#else #define IO_PERFORMANCE_ALLOC_BYTES 0x1000 #define IO_PERFORMANCE_MAX_ENTRIES (IO_PERFORMANCE_ALLOC_BYTES / sizeof(struct io_perf_data_item)) +static struct io_perf_data_item io_perf_data_items[IO_PERFORMANCE_MAX_ENTRIES]; +#endif + +SYS_BITARRAY_DEFINE_STATIC(io_performance_data_bit_array, IO_PERFORMANCE_MAX_ENTRIES); static struct io_perf_monitor_ctx perf_monitor_ctx; -static struct io_perf_data_item io_perf_data_items[IO_PERFORMANCE_MAX_ENTRIES]; __cold int io_perf_monitor_init(void) { @@ -458,8 +616,15 @@ __cold int io_perf_monitor_init(void) k_spinlock_init(&self->lock); k_spinlock_key_t key = k_spin_lock(&self->lock); +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES + self->io_perf_data = (struct io_perf_data_item *)((uint8_t *)ADSP_PMW + + SOF_TEL_MW3_IO_PERF_OFFSET); + memset(self->io_perf_data, 0, sizeof(*self->io_perf_data) * IO_PERFORMANCE_MAX_ENTRIES); + self->state = IPC4_PERF_MEASUREMENTS_STARTED; +#else self->io_perf_data = (struct io_perf_data_item *)io_perf_data_items; self->state = IPC4_PERF_MEASUREMENTS_DISABLED; +#endif ret = perf_bitmap_init(&self->io_performance_data_bitmap, &io_performance_data_bit_array, IO_PERFORMANCE_MAX_ENTRIES); diff --git a/src/kernel/debug/telemetry/telemetry.c b/src/kernel/debug/telemetry/telemetry.c index 676265592a9b..eec009f61738 100644 --- a/src/kernel/debug/telemetry/telemetry.c +++ b/src/kernel/debug/telemetry/telemetry.c @@ -5,12 +5,25 @@ // Author: Tobiasz Dryjanski #include +#include +#include #include #include +#include #include +#include +#include #include +#include +#include +#include #include +#include +#include +#include + +#include "../../../audio/copier/host_copier.h" #include #include @@ -23,22 +36,1050 @@ #include #include #include +#include LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); -/* Systic variables, one set per core */ +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW +BUILD_ASSERT(sizeof(struct sof_tel_ext_hdr) == 64, "sof_tel_ext_hdr must be 64B"); +BUILD_ASSERT(sizeof(struct sof_tel_hot_line0) == 64, "sof_tel_hot_line0 must be 64B"); +BUILD_ASSERT(sizeof(struct sof_tel_gw_hot) == 8, "sof_tel_gw_hot must be 8B"); +BUILD_ASSERT(sizeof(struct sof_tel_mem_section) == 256, "sof_tel_mem_section must be 256B"); +BUILD_ASSERT(sizeof(struct sof_tel_vregion_section) == 512, "sof_tel_vregion_section must be 512B"); +BUILD_ASSERT(sizeof(struct sof_tel_ppl_section) == 384, "sof_tel_ppl_section must be 384B"); +BUILD_ASSERT(sizeof(struct sof_tel_dai_section) == 256, "sof_tel_dai_section must be 256B"); +BUILD_ASSERT(sizeof(struct sof_tel_thread_section) == 832, "sof_tel_thread_section must be 832B"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, ext_hdr) == 0x0100, "ext_hdr offset != 0x0100"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, hot_line0) == 0x0140, "hot_line0 offset != 0x0140"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, gw_hot) == 0x0180, "gw_hot offset != 0x0180"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, mem_sec) == 0x0240, "mem_sec offset != 0x0240"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, vreg_sec) == 0x0340, "vreg_sec offset != 0x0340"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, ppl_sec) == 0x0540, "ppl_sec offset != 0x0540"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, dai_sec) == 0x06C0, "dai_sec offset != 0x06C0"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, thd_sec) == 0x07C0, "thd_sec offset != 0x07C0"); +BUILD_ASSERT(sizeof(struct telemetry_wnd_data) <= 4096, "telemetry_wnd_data exceeds 4KB slot"); +#endif + +/* Systick variables, one set per core */ static int telemetry_systick_counter[CONFIG_MAX_CORE_COUNT]; #ifdef CONFIG_INTEL_ADSP_DEBUG_SLOT_MANAGER static struct telemetry_wnd_data *wnd_data; #endif #ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS -static int telemetry_prev_ccount[CONFIG_MAX_CORE_COUNT]; static int telemetry_perf_period_sum[CONFIG_MAX_CORE_COUNT]; static int telemetry_perf_period_cnt[CONFIG_MAX_CORE_COUNT]; static struct telemetry_perf_queue telemetry_perf_queue[CONFIG_MAX_CORE_COUNT]; #endif +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW +#define TELEMETRY_BG_REFRESH_PERIOD K_MSEC(1000) + +static struct k_work_delayable telemetry_bg_dwork; +static bool telemetry_wnd_initialized; +static bool telemetry_ext_ready; + +struct telemetry_shared_state { +#ifdef CONFIG_SOF_TELEMETRY_HOT_CACHE_STAGING + struct sof_tel_hot_line0 hot_line0_cache __aligned(64); + struct sof_tel_gw_hot gw_hot_cache[SOF_TEL_MAX_GATEWAYS] __aligned(64); + bool hot_line0_dirty; + bool gw_line1_dirty; + bool gw_line2_dirty; +#endif + uint32_t total_tick_counter; + uint16_t ll_task_count[SOF_TEL_MAX_CORES]; +} __aligned(64); + +static struct telemetry_shared_state tel_state_storage __aligned(64); + +static inline struct telemetry_shared_state *tel_state(void) +{ + return (struct telemetry_shared_state *)cache_to_uncache( + (__sparse_force void __sparse_cache *)&tel_state_storage); +} + +#ifdef CONFIG_SOF_TELEMETRY_PIPELINES +struct tel_ppl_slot { + bool in_use; + uint16_t pipeline_id; + uint16_t xrun_count; + int32_t last_xrun_bytes; +}; + +static struct tel_ppl_slot tel_ppl_slots[SOF_TEL_MAX_PIPELINES]; +#endif + +#ifdef CONFIG_SOF_TELEMETRY_DAIS +struct tel_gw_slot { + bool in_use; + bool is_host; + bool prev_xrun; + const void *key; + struct comp_dev *dev; + struct sof_tel_gw_cold cold; +}; + +static struct tel_gw_slot tel_gw_slots[SOF_TEL_MAX_GATEWAYS]; +#endif + +struct telemetry_wnd_data *telemetry_get_wnd_ptr(void) +{ +#ifdef CONFIG_INTEL_ADSP_DEBUG_SLOT_MANAGER + return wnd_data; +#else + return (struct telemetry_wnd_data *)ADSP_DW->slots[SOF_DW_TELEMETRY_SLOT]; +#endif +} + +static inline struct sof_tel_hot_line0 *telemetry_get_hot_line0(void) +{ +#ifdef CONFIG_SOF_TELEMETRY_HOT_CACHE_STAGING + struct telemetry_shared_state *st = tel_state(); + + st->hot_line0_dirty = true; + return &st->hot_line0_cache; +#else + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + + return wnd ? &wnd->hot_line0 : NULL; +#endif +} + +static inline struct sof_tel_gw_hot *telemetry_get_gw_hot(int slot) +{ +#ifdef CONFIG_SOF_TELEMETRY_HOT_CACHE_STAGING + struct telemetry_shared_state *st = tel_state(); + + if (slot < 8) + st->gw_line1_dirty = true; + else + st->gw_line2_dirty = true; + return &st->gw_hot_cache[slot]; +#else + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + + return wnd ? &wnd->gw_hot[slot] : NULL; +#endif +} + +static inline void telemetry_flush_hot_staging(struct telemetry_wnd_data *wnd) +{ +#ifdef CONFIG_SOF_TELEMETRY_HOT_CACHE_STAGING + struct telemetry_shared_state *st = tel_state(); + + if (!wnd) + return; + + if (st->hot_line0_dirty) { + wnd->hot_line0 = st->hot_line0_cache; + st->hot_line0_dirty = false; + } + if (st->gw_line1_dirty) { + memcpy(&wnd->gw_hot[0], &st->gw_hot_cache[0], 64); + st->gw_line1_dirty = false; + } + if (st->gw_line2_dirty) { + memcpy(&wnd->gw_hot[8], &st->gw_hot_cache[8], 64); + st->gw_line2_dirty = false; + } +#else + (void)wnd; +#endif +} + +static inline void telemetry_seq_begin(struct telemetry_wnd_data *wnd) +{ + wnd->ext_hdr.seq |= 1U; + compiler_barrier(); +} + +static inline void telemetry_seq_end(struct telemetry_wnd_data *wnd) +{ + compiler_barrier(); + wnd->ext_hdr.timestamp_us = (uint32_t)k_cyc_to_us_floor64(k_cycle_get_64()); + compiler_barrier(); + wnd->ext_hdr.seq = (wnd->ext_hdr.seq + 1U) & ~1U; +} + +#ifdef CONFIG_SOF_TELEMETRY_PIPELINES +static void telemetry_update_pipelines_section(struct telemetry_wnd_data *wnd) +{ + struct ipc *ipc = ipc_get(); + struct sof_tel_ppl_section *psec = &wnd->ppl_sec; + struct sof_tel_hot_line0 *hot0 = telemetry_get_hot_line0(); + struct tel_ppl_slot new_slots[SOF_TEL_MAX_PIPELINES]; + struct list_item *clist; + uint16_t ppl_cnt = 0; + uint16_t comp_cnt = 0; + + memset(new_slots, 0, sizeof(new_slots)); + memset(psec->core_sched, 0, sizeof(psec->core_sched)); + for (int c = 0; c < MIN(CONFIG_MAX_CORE_COUNT, SOF_TEL_MAX_CORES); c++) { + uint16_t ll_tasks = tel_state()->ll_task_count[c]; + + psec->core_sched[c].core_id = (uint8_t)c; + psec->core_sched[c].ll_task_count = ll_tasks; + if (hot0) + hot0->active_ll_tasks[c] = ll_tasks; +#if CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL + if (cpu_is_core_enabled(c)) + psec->core_sched[c].active_kcps_budget = (uint32_t)core_kcps_get(c); +#endif + } + + if (!ipc) { + memset(tel_ppl_slots, 0, sizeof(tel_ppl_slots)); + if (hot0) + memset(hot0->ppl_xrun_count, 0, sizeof(hot0->ppl_xrun_count)); + psec->active_ppl_count = 0; + psec->total_comp_count = 0; + wnd->ext_hdr.secs[3].active_mask = 0; + return; + } + + /* First pass: collect pipelines */ + list_for_item(clist, &ipc->comp_list) { + struct ipc_comp_dev *icd = container_of(clist, struct ipc_comp_dev, list); + + if (icd->type == COMP_TYPE_COMPONENT) { + comp_cnt++; + if (icd->cd && icd->core < SOF_TEL_MAX_CORES) { + if (icd->cd->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) + psec->core_sched[icd->core].dp_task_count++; + } + continue; + } + + if (icd->type != COMP_TYPE_PIPELINE || !icd->pipeline) + continue; + + if (ppl_cnt >= SOF_TEL_MAX_PIPELINES) + continue; + + struct pipeline *p = icd->pipeline; + struct sof_tel_ppl_info *pinfo = &psec->pipelines[ppl_cnt]; + uint16_t pid = (uint16_t)p->pipeline_id; + uint16_t prev_xruns = 0; + int32_t prev_xrun_bytes = 0; + + for (int s = 0; s < SOF_TEL_MAX_PIPELINES; s++) { + if (tel_ppl_slots[s].in_use && tel_ppl_slots[s].pipeline_id == pid) { + prev_xruns = tel_ppl_slots[s].xrun_count; + prev_xrun_bytes = tel_ppl_slots[s].last_xrun_bytes; + break; + } + } + + new_slots[ppl_cnt].in_use = true; + new_slots[ppl_cnt].pipeline_id = pid; + new_slots[ppl_cnt].xrun_count = prev_xruns; + new_slots[ppl_cnt].last_xrun_bytes = p->xrun_bytes ? p->xrun_bytes : + prev_xrun_bytes; + + pinfo->pipeline_id = pid; + pinfo->core = (uint8_t)p->core; + pinfo->priority = (uint8_t)p->priority; + pinfo->status = (uint8_t)p->status; + pinfo->comp_count = 0; + pinfo->attributes = (uint16_t)p->attributes; + pinfo->period_us = p->period; + pinfo->sched_comp_id = p->sched_id; + pinfo->last_xrun_bytes = new_slots[ppl_cnt].last_xrun_bytes; + + if (p->core < SOF_TEL_MAX_CORES) + psec->core_sched[p->core].active_ppl_count++; + ppl_cnt++; + } + + /* Clear unused pipeline slots */ + for (uint16_t i = ppl_cnt; i < SOF_TEL_MAX_PIPELINES; i++) + memset(&psec->pipelines[i], 0, sizeof(psec->pipelines[i])); + + /* Second pass: count components per pipeline */ + list_for_item(clist, &ipc->comp_list) { + struct ipc_comp_dev *icd = container_of(clist, struct ipc_comp_dev, list); + + if (icd->type != COMP_TYPE_COMPONENT || !icd->cd) + continue; + + uint32_t pid = dev_comp_pipe_id(icd->cd); + + for (uint16_t i = 0; i < ppl_cnt; i++) { + if (psec->pipelines[i].pipeline_id == (uint16_t)pid) { + psec->pipelines[i].comp_count++; + break; + } + } + } + + memcpy(tel_ppl_slots, new_slots, sizeof(tel_ppl_slots)); + if (hot0) { + for (uint16_t i = 0; i < SOF_TEL_MAX_PIPELINES; i++) + hot0->ppl_xrun_count[i] = tel_ppl_slots[i].in_use ? + tel_ppl_slots[i].xrun_count : 0; + } + + for (int c = 0; c < MIN(CONFIG_MAX_CORE_COUNT, SOF_TEL_MAX_CORES); c++) { + if (psec->core_sched[c].ll_task_count == 0) { + wnd->system_tick_info[c].last_time_elapsed = 0; + wnd->system_tick_info[c].avg_utilization = 0; +#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + telemetry_perf_period_sum[c] = 0; + telemetry_perf_period_cnt[c] = 0; + memset(&telemetry_perf_queue[c], 0, sizeof(telemetry_perf_queue[c])); +#endif + } + } + + psec->active_ppl_count = ppl_cnt; + psec->total_comp_count = comp_cnt; + wnd->ext_hdr.secs[3].active_mask = ppl_cnt; +} + +void telemetry_notify_ll_task_count(int core, unsigned int n_tasks) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + uint16_t count = (uint16_t)MIN(n_tasks, (unsigned int)UINT16_MAX); + + if (core < 0 || core >= MIN(CONFIG_MAX_CORE_COUNT, SOF_TEL_MAX_CORES)) + return; + + tel_state()->ll_task_count[core] = count; + if (telemetry_ext_ready && wnd) { + struct sof_tel_hot_line0 *hot0 = telemetry_get_hot_line0(); + + telemetry_seq_begin(wnd); + if (hot0) + hot0->active_ll_tasks[core] = count; + wnd->ppl_sec.core_sched[core].ll_task_count = count; + if (count == 0) { + wnd->system_tick_info[core].last_time_elapsed = 0; + wnd->system_tick_info[core].avg_utilization = 0; +#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + telemetry_perf_period_sum[core] = 0; + telemetry_perf_period_cnt[core] = 0; + memset(&telemetry_perf_queue[core], 0, + sizeof(telemetry_perf_queue[core])); +#endif + } + telemetry_flush_hot_staging(wnd); + telemetry_seq_end(wnd); + } +} + +void telemetry_notify_pipeline_change(void) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + + if (!telemetry_ext_ready || !wnd) + return; + + telemetry_seq_begin(wnd); + telemetry_update_pipelines_section(wnd); + telemetry_flush_hot_staging(wnd); + telemetry_seq_end(wnd); +} + +void telemetry_notify_pipeline_xrun(struct pipeline *p, int32_t bytes) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + uint16_t pid; + int slot = -1; + int free_slot = -1; + + if (!telemetry_ext_ready || !wnd || !p) + return; + + pid = (uint16_t)p->pipeline_id; + struct sof_tel_hot_line0 *hot0 = telemetry_get_hot_line0(); + + if (hot0) + hot0->total_xrun_events++; + + for (int i = 0; i < SOF_TEL_MAX_PIPELINES; i++) { + if (tel_ppl_slots[i].in_use && tel_ppl_slots[i].pipeline_id == pid) { + slot = i; + break; + } + if (!tel_ppl_slots[i].in_use && free_slot < 0) + free_slot = i; + } + + if (slot < 0 && free_slot >= 0) { + slot = free_slot; + tel_ppl_slots[slot].in_use = true; + tel_ppl_slots[slot].pipeline_id = pid; + tel_ppl_slots[slot].xrun_count = 0; + } + + if (slot >= 0) { + tel_ppl_slots[slot].xrun_count++; + tel_ppl_slots[slot].last_xrun_bytes = bytes; + if (hot0) + hot0->ppl_xrun_count[slot] = tel_ppl_slots[slot].xrun_count; + telemetry_seq_begin(wnd); + wnd->ppl_sec.pipelines[slot].pipeline_id = pid; + wnd->ppl_sec.pipelines[slot].last_xrun_bytes = bytes; + telemetry_seq_end(wnd); + } +} + +void telemetry_notify_dp_deadline_miss(int core) +{ + if (!telemetry_ext_ready || core < 0 || core >= SOF_TEL_MAX_CORES) + return; + + struct sof_tel_hot_line0 *hot0 = telemetry_get_hot_line0(); + + if (hot0) + hot0->dp_deadline_misses[core]++; +} +#endif /* CONFIG_SOF_TELEMETRY_PIPELINES */ + +#ifdef CONFIG_SOF_TELEMETRY_DAIS +static uint8_t map_sof_dai_type(uint32_t type) +{ + switch (type) { + case SOF_DAI_INTEL_SSP: + return SOF_TEL_GW_SSP; + case SOF_DAI_INTEL_DMIC: + return SOF_TEL_GW_DMIC; + case SOF_DAI_INTEL_ALH: + return SOF_TEL_GW_ALH; + case SOF_DAI_INTEL_HDA: + return SOF_TEL_GW_HDA; + default: + return SOF_TEL_GW_OTHER; + } +} + +static void telemetry_fill_dai_cold(struct sof_tel_gw_cold *cold, + struct dai_data *dd, struct comp_dev *dev) +{ + memset(cold, 0, sizeof(*cold)); + cold->gw_type = map_sof_dai_type(dd->ipc_config.type); + cold->index = (uint8_t)dd->ipc_config.dai_index; + cold->direction = dev ? (uint8_t)dev->direction : (uint8_t)dd->ipc_config.direction; + cold->dma_chan = (int8_t)dd->chan_index; + cold->comp_id = dev ? (uint32_t)dev_comp_id(dev) : 0; + cold->state = dev ? (uint8_t)dev->state : 0; + cold->group_id = dd->group ? (uint8_t)dd->group->group_id : 0; + cold->period_bytes = (uint16_t)MIN(dd->period_bytes, UINT16_MAX); + cold->dma_buf_size = dd->dma_buffer ? + (uint32_t)audio_stream_get_size(&dd->dma_buffer->stream) : + dd->ipc_config.dma_buffer_size; +} + +static void telemetry_fill_host_cold(struct sof_tel_gw_cold *cold, + struct host_data *hd, struct comp_dev *dev) +{ + memset(cold, 0, sizeof(*cold)); + cold->gw_type = SOF_TEL_GW_HOST_DMA; + cold->index = (uint8_t)MAX(hd->chan_index, 0); + cold->direction = dev ? (uint8_t)dev->direction : 0; + cold->dma_chan = (int8_t)hd->chan_index; + cold->comp_id = dev ? (uint32_t)dev_comp_id(dev) : 0; + cold->state = dev ? (uint8_t)dev->state : 0; + cold->group_id = 0; + cold->period_bytes = (uint16_t)MIN(hd->period_bytes, UINT16_MAX); + cold->dma_buf_size = hd->dma_buffer ? + (uint32_t)audio_stream_get_size(&hd->dma_buffer->stream) : + hd->host_size; +} + +static void telemetry_sync_gw_cold_slot(struct telemetry_wnd_data *wnd, int slot) +{ + uint16_t mask = 0; + + if (!telemetry_ext_ready || !wnd || slot < 0 || slot >= SOF_TEL_MAX_GATEWAYS) + return; + + struct sof_tel_gw_cold *cold = &wnd->dai_sec.gateways[slot]; + struct tel_gw_slot *s = &tel_gw_slots[slot]; + + if (s->in_use) { + if (s->dev) + s->cold.state = (uint8_t)s->dev->state; + *cold = s->cold; + } else { + memset(cold, 0, sizeof(*cold)); + } + + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use) + mask |= BIT(i); + } + wnd->ext_hdr.secs[4].active_mask = mask; +} + +int telemetry_register_dai(struct dai_data *dd, struct comp_dev *dev) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + struct sof_tel_gw_cold cold; + int free_slot = -1; + int slot = -1; + + if (!dd) + return -EINVAL; + + telemetry_fill_dai_cold(&cold, dd, dev); + + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use && !tel_gw_slots[i].is_host && + tel_gw_slots[i].key == dd) { + slot = i; + break; + } + if (!tel_gw_slots[i].in_use && free_slot < 0) + free_slot = i; + } + + if (slot < 0) { + if (free_slot < 0) + return -ENOMEM; + slot = free_slot; + tel_gw_slots[slot].in_use = true; + tel_gw_slots[slot].is_host = false; + tel_gw_slots[slot].prev_xrun = false; + tel_gw_slots[slot].key = dd; + } + + tel_gw_slots[slot].dev = dev; + tel_gw_slots[slot].cold = cold; + if (telemetry_ext_ready && wnd) { + telemetry_seq_begin(wnd); + telemetry_sync_gw_cold_slot(wnd, slot); + telemetry_seq_end(wnd); + } + return slot; +} + +void telemetry_unregister_dai(struct dai_data *dd) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use && !tel_gw_slots[i].is_host && + tel_gw_slots[i].key == dd) { + struct sof_tel_gw_hot *hot = telemetry_get_gw_hot(i); + + tel_gw_slots[i].in_use = false; + tel_gw_slots[i].prev_xrun = false; + tel_gw_slots[i].key = NULL; + tel_gw_slots[i].dev = NULL; + memset(&tel_gw_slots[i].cold, 0, sizeof(tel_gw_slots[i].cold)); + if (hot) + memset(hot, 0, sizeof(*hot)); + if (telemetry_ext_ready && wnd) { + telemetry_seq_begin(wnd); + telemetry_sync_gw_cold_slot(wnd, i); + telemetry_flush_hot_staging(wnd); + telemetry_seq_end(wnd); + } + break; + } + } +} + +void telemetry_update_dai_hot(struct dai_data *dd, struct comp_dev *dev) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + struct sof_tel_gw_cold cold; + uint32_t avail = 0; + bool has_avail = false; + + if (!dd) + return; + + if (dd->dma_buffer) { + avail = audio_stream_get_avail_bytes(&dd->dma_buffer->stream); + has_avail = true; + } + + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use && !tel_gw_slots[i].is_host && + tel_gw_slots[i].key == dd) { + struct comp_dev *cdev = dev ? dev : tel_gw_slots[i].dev; + struct sof_tel_gw_hot *hot = telemetry_get_gw_hot(i); + + if (dev) + tel_gw_slots[i].dev = dev; + telemetry_fill_dai_cold(&cold, dd, cdev); + if (!dev && !dd->dma_buffer) + cold.state = COMP_STATE_READY; + else if (dd->dma_buffer && cold.state == COMP_STATE_READY) + cold.state = COMP_STATE_PREPARE; + + if (memcmp(&tel_gw_slots[i].cold, &cold, sizeof(cold)) != 0) { + tel_gw_slots[i].cold = cold; + if (telemetry_ext_ready && wnd) { + telemetry_seq_begin(wnd); + wnd->dai_sec.gateways[i] = cold; + telemetry_seq_end(wnd); + } + } + + if (!hot) + break; + + hot->total_kb_or_frames = (uint32_t)(dd->total_data_processed >> 10); + if (has_avail) + hot->dma_buf_avail_bytes = (uint16_t)MIN(avail, UINT16_MAX); + else if (!dd->dma_buffer) + hot->dma_buf_avail_bytes = 0; + if (dd->xrun && !tel_gw_slots[i].prev_xrun) + hot->xrun_or_skip_count++; + tel_gw_slots[i].prev_xrun = dd->xrun; + break; + } + } +} + +int telemetry_register_host(struct host_data *hd, struct comp_dev *dev) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + struct sof_tel_gw_cold cold; + int free_slot = -1; + int slot = -1; + + if (!hd) + return -EINVAL; + + telemetry_fill_host_cold(&cold, hd, dev); + + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use && tel_gw_slots[i].is_host && + tel_gw_slots[i].key == hd) { + slot = i; + break; + } + if (!tel_gw_slots[i].in_use && free_slot < 0) + free_slot = i; + } + + if (slot < 0) { + if (free_slot < 0) + return -ENOMEM; + slot = free_slot; + tel_gw_slots[slot].in_use = true; + tel_gw_slots[slot].is_host = true; + tel_gw_slots[slot].prev_xrun = false; + tel_gw_slots[slot].key = hd; + } + + tel_gw_slots[slot].dev = dev; + tel_gw_slots[slot].cold = cold; + if (telemetry_ext_ready && wnd) { + telemetry_seq_begin(wnd); + telemetry_sync_gw_cold_slot(wnd, slot); + telemetry_seq_end(wnd); + } + return slot; +} + +void telemetry_unregister_host(struct host_data *hd) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use && tel_gw_slots[i].is_host && + tel_gw_slots[i].key == hd) { + struct sof_tel_gw_hot *hot = telemetry_get_gw_hot(i); + + tel_gw_slots[i].in_use = false; + tel_gw_slots[i].prev_xrun = false; + tel_gw_slots[i].key = NULL; + tel_gw_slots[i].dev = NULL; + memset(&tel_gw_slots[i].cold, 0, sizeof(tel_gw_slots[i].cold)); + if (hot) + memset(hot, 0, sizeof(*hot)); + if (telemetry_ext_ready && wnd) { + telemetry_seq_begin(wnd); + telemetry_sync_gw_cold_slot(wnd, i); + telemetry_flush_hot_staging(wnd); + telemetry_seq_end(wnd); + } + break; + } + } +} + +void telemetry_update_host_hot(struct host_data *hd, struct comp_dev *dev) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + struct sof_tel_gw_cold cold; + uint32_t avail = 0; + bool has_avail = false; + + if (!hd) + return; + + if (hd->dma_buffer) { + avail = audio_stream_get_avail_bytes(&hd->dma_buffer->stream); + has_avail = true; + } + + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use && tel_gw_slots[i].is_host && + tel_gw_slots[i].key == hd) { + struct comp_dev *cdev = dev ? dev : tel_gw_slots[i].dev; + struct sof_tel_gw_hot *hot = telemetry_get_gw_hot(i); + + if (dev) + tel_gw_slots[i].dev = dev; + telemetry_fill_host_cold(&cold, hd, cdev); + if (!dev && !hd->dma_buffer) + cold.state = COMP_STATE_READY; + else if (hd->dma_buffer && cold.state == COMP_STATE_READY) + cold.state = COMP_STATE_PREPARE; + + if (memcmp(&tel_gw_slots[i].cold, &cold, sizeof(cold)) != 0) { + tel_gw_slots[i].cold = cold; + if (telemetry_ext_ready && wnd) { + telemetry_seq_begin(wnd); + wnd->dai_sec.gateways[i] = cold; + telemetry_seq_end(wnd); + } + } + + if (!hot) + break; + + hot->total_kb_or_frames = (uint32_t)(hd->total_data_processed >> 10); + if (has_avail) + hot->dma_buf_avail_bytes = (uint16_t)MIN(avail, UINT16_MAX); + else if (!hd->dma_buffer) + hot->dma_buf_avail_bytes = 0; + break; + } + } +} + +void telemetry_notify_host_skip(struct host_data *hd) +{ + if (!hd) + return; + + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use && tel_gw_slots[i].is_host && + tel_gw_slots[i].key == hd) { + struct sof_tel_gw_hot *hot = telemetry_get_gw_hot(i); + + if (hot) + hot->xrun_or_skip_count++; + break; + } + } +} +#endif /* CONFIG_SOF_TELEMETRY_DAIS */ + +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +static struct sof_tel_mem_section tel_mem_staging; +#endif + +#ifdef CONFIG_SOF_TELEMETRY_VREGION +static struct sof_tel_vregion_section tel_vreg_staging; + +void telemetry_notify_vregion_change(void) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + + if (!telemetry_ext_ready || !wnd || k_is_in_isr()) + return; + + memset(&tel_vreg_staging, 0, sizeof(tel_vreg_staging)); + telemetry_collect_vpage_stats(&tel_vreg_staging); + telemetry_collect_vregion_stats(&tel_vreg_staging); + + telemetry_seq_begin(wnd); + wnd->vreg_sec = tel_vreg_staging; + wnd->ext_hdr.secs[2].active_mask = tel_vreg_staging.active_vregion_count; + telemetry_seq_end(wnd); +} +#endif + +#ifdef CONFIG_SOF_TELEMETRY_THREADS +extern int z_impl_k_thread_stack_space_get(const struct k_thread *thread, size_t *unused_ptr); + +static struct sof_tel_thread_section tel_thd_staging; +static uint32_t tel_thd_sample_seq; + +#ifdef CONFIG_THREAD_RUNTIME_STATS +static uint64_t tel_prev_core_active[SOF_TEL_MAX_CORES]; +static uint64_t tel_prev_core_all[SOF_TEL_MAX_CORES]; +static uint32_t tel_last_core_delta_all[SOF_TEL_MAX_CORES]; +static struct { + const struct k_thread *tid; + uint64_t cycles; + bool valid; +} tel_prev_thread_cycles[SOF_TEL_MAX_THREADS]; +#endif + +struct tel_thread_collect_ctx { + struct sof_tel_thread_section *tsec; +#ifdef CONFIG_THREAD_RUNTIME_STATS + const struct k_thread *threads[SOF_TEL_MAX_THREADS]; + uint64_t exec_cycles[SOF_TEL_MAX_THREADS]; + bool has_stats[SOF_TEL_MAX_THREADS]; +#endif +}; + +static void telemetry_thread_cb(const struct k_thread *cthread, void *user_data) +{ + struct tel_thread_collect_ctx *ctx = user_data; + struct sof_tel_thread_section *tsec = ctx->tsec; + struct k_thread *thread = (struct k_thread *)cthread; + uint16_t idx = tsec->thread_count; + + if (idx >= SOF_TEL_MAX_THREADS) { + tsec->dropped_threads++; + return; + } + + struct sof_tel_thread_info *ti = &tsec->threads[idx]; + const char *name = k_thread_name_get((k_tid_t)thread); + + memset(ti, 0, sizeof(*ti)); + ti->thread_id = (uint32_t)POINTER_TO_UINT(thread); +#ifdef CONFIG_SMP + ti->core = thread->base.cpu; + if (thread->base.is_idle) + ti->flags |= SOF_TEL_THD_FLG_IDLE; +#endif + ti->prio = thread->base.prio; + ti->state = thread->base.thread_state; +#ifdef CONFIG_USERSPACE + if (thread->base.user_options & K_USER) + ti->flags |= SOF_TEL_THD_FLG_USER; +#endif + if (name) { + if (strncmp(name, "ll_thread", 9) == 0) + ti->flags |= SOF_TEL_THD_FLG_LL; + else if (strncmp(name, "DP_", 3) == 0 || strncmp(name, "edf_work", 8) == 0) + ti->flags |= SOF_TEL_THD_FLG_DP; + else if (strncmp(name, "idle", 4) == 0) + ti->flags |= SOF_TEL_THD_FLG_IDLE; + } + +#ifdef CONFIG_THREAD_STACK_INFO + if (thread->stack_info.size > 0) { + size_t unused = 0; + + ti->stack_size_chunks = (uint16_t)(thread->stack_info.size >> 6); + if (z_impl_k_thread_stack_space_get(thread, &unused) == 0 && + unused <= thread->stack_info.size) { + ti->stack_usage = (uint8_t)((255ULL * (thread->stack_info.size - unused)) / + thread->stack_info.size); + } + } +#endif + +#ifdef CONFIG_THREAD_RUNTIME_STATS + k_thread_runtime_stats_t rt_stats; + + ctx->threads[idx] = cthread; + if (k_thread_runtime_stats_get(thread, &rt_stats) == 0) { + ctx->exec_cycles[idx] = rt_stats.execution_cycles; + ctx->has_stats[idx] = true; + ti->total_exec_kcycles = (uint32_t)(rt_stats.execution_cycles >> 10); + } +#endif + + if (name && name[0] != '\0') + strncpy(ti->name, name, sizeof(ti->name)); + + tsec->thread_count++; +} + +static void telemetry_collect_threads_section(struct sof_tel_thread_section *tsec) +{ + struct tel_thread_collect_ctx ctx = { .tsec = tsec }; + + memset(tsec, 0, sizeof(*tsec)); + +#ifdef CONFIG_THREAD_RUNTIME_STATS + for (int c = 0; c < MIN(CONFIG_MAX_CORE_COUNT, SOF_TEL_MAX_CORES); c++) { + k_thread_runtime_stats_t core_stats; + + if (!cpu_is_core_enabled(c)) { + tsec->core_load[c] = 0; + tel_last_core_delta_all[c] = 0; + continue; + } + if (k_thread_runtime_stats_cpu_get(c, &core_stats) == 0) { + uint32_t act = (uint32_t)(core_stats.total_cycles - tel_prev_core_active[c]); + uint32_t all = (uint32_t)(core_stats.execution_cycles - tel_prev_core_all[c]); + + tel_prev_core_active[c] = core_stats.total_cycles; + tel_prev_core_all[c] = core_stats.execution_cycles; + tel_last_core_delta_all[c] = all; + tsec->core_load[c] = all ? (uint8_t)MIN(255ULL, (255ULL * act) / all) : 0; + } + } +#endif + + k_thread_foreach_unlocked(telemetry_thread_cb, &ctx); + tsec->sample_seq = ++tel_thd_sample_seq; + +#ifdef CONFIG_THREAD_RUNTIME_STATS + for (uint16_t idx = 0; idx < tsec->thread_count; idx++) { + struct sof_tel_thread_info *ti = &tsec->threads[idx]; + uint32_t delta_cyc = 0; + + if (!ctx.has_stats[idx]) + continue; + + for (int i = 0; i < SOF_TEL_MAX_THREADS; i++) { + if (tel_prev_thread_cycles[i].valid && + tel_prev_thread_cycles[i].tid == ctx.threads[idx]) { + delta_cyc = (uint32_t)(ctx.exec_cycles[idx] - + tel_prev_thread_cycles[i].cycles); + break; + } + } + + uint8_t c = ti->core < SOF_TEL_MAX_CORES ? ti->core : 0; + + if (tel_last_core_delta_all[c] > 0 && delta_cyc <= tel_last_core_delta_all[c]) { + ti->cpu_load = (uint8_t)((255ULL * delta_cyc) / + tel_last_core_delta_all[c]); + } + } + + for (uint16_t idx = 0; idx < tsec->thread_count; idx++) { + tel_prev_thread_cycles[idx].tid = ctx.threads[idx]; + tel_prev_thread_cycles[idx].cycles = ctx.exec_cycles[idx]; + tel_prev_thread_cycles[idx].valid = ctx.has_stats[idx]; + } + for (uint16_t idx = tsec->thread_count; idx < SOF_TEL_MAX_THREADS; idx++) { + tel_prev_thread_cycles[idx].tid = NULL; + tel_prev_thread_cycles[idx].cycles = 0; + tel_prev_thread_cycles[idx].valid = false; + } +#endif +} +#endif /* CONFIG_SOF_TELEMETRY_THREADS */ + +void telemetry_refresh_cold_sections(void) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + + if (!telemetry_ext_ready || !wnd || k_is_in_isr()) + return; + +#ifdef CONFIG_SOF_TELEMETRY_MEMORY + memset(&tel_mem_staging, 0, sizeof(tel_mem_staging)); + telemetry_collect_mem_stats(&tel_mem_staging); +#endif + +#ifdef CONFIG_SOF_TELEMETRY_VREGION + memset(&tel_vreg_staging, 0, sizeof(tel_vreg_staging)); + telemetry_collect_vpage_stats(&tel_vreg_staging); + telemetry_collect_vregion_stats(&tel_vreg_staging); +#endif + +#ifdef CONFIG_SOF_TELEMETRY_THREADS + telemetry_collect_threads_section(&tel_thd_staging); +#endif + + telemetry_seq_begin(wnd); + +#ifdef CONFIG_SOF_TELEMETRY_MEMORY + wnd->mem_sec = tel_mem_staging; + wnd->ext_hdr.secs[1].active_mask = SOF_TEL_MAX_HEAPS; +#endif + +#ifdef CONFIG_SOF_TELEMETRY_VREGION + wnd->vreg_sec = tel_vreg_staging; + wnd->ext_hdr.secs[2].active_mask = tel_vreg_staging.active_vregion_count; +#endif + +#ifdef CONFIG_SOF_TELEMETRY_PIPELINES + telemetry_update_pipelines_section(wnd); +#endif + +#ifdef CONFIG_SOF_TELEMETRY_DAIS + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) + telemetry_sync_gw_cold_slot(wnd, i); +#endif + +#ifdef CONFIG_SOF_TELEMETRY_THREADS + wnd->thd_sec = tel_thd_staging; + wnd->ext_hdr.secs[5].active_mask = tel_thd_staging.thread_count; +#endif + + telemetry_flush_hot_staging(wnd); + telemetry_seq_end(wnd); +} + +static void telemetry_bg_work_handler(struct k_work *work) +{ + (void)work; + telemetry_refresh_cold_sections(); + if (telemetry_ext_ready) + k_work_reschedule(&telemetry_bg_dwork, TELEMETRY_BG_REFRESH_PERIOD); +} + +static void telemetry_ext_init(struct telemetry_wnd_data *wnd) +{ + struct sof_tel_ext_hdr *hdr = &wnd->ext_hdr; + + memset((uint8_t *)wnd + offsetof(struct telemetry_wnd_data, pad_to_0x100), 0, + sizeof(*wnd) - offsetof(struct telemetry_wnd_data, pad_to_0x100)); + + hdr->magic = SOF_TEL_EXT_MAGIC; + hdr->version = SOF_TEL_EXT_VERSION; + hdr->max_cores = CONFIG_MAX_CORE_COUNT; + hdr->num_secs = SOF_TEL_MAX_SECTIONS; + hdr->seq = 0; + + /* Section 0: Hot Counter Zone (0x0140, 256 B) */ + hdr->secs[0].id = SOF_TEL_SEC_HOT_ZONE; + hdr->secs[0].elem_size = 64; + hdr->secs[0].offset = (uint16_t)offsetof(struct telemetry_wnd_data, hot_line0); + hdr->secs[0].max_count = 4; + hdr->secs[0].active_mask = 3; + + /* Section 1: Memory & Heaps (0x0240, 256 B = 4 cache lines) */ + hdr->secs[1].id = SOF_TEL_SEC_MEMORY; + hdr->secs[1].elem_size = 64; + hdr->secs[1].offset = (uint16_t)offsetof(struct telemetry_wnd_data, mem_sec); + hdr->secs[1].max_count = IS_ENABLED(CONFIG_SOF_TELEMETRY_MEMORY) ? 4 : 0; + + /* Section 2: vpage & vregion (0x0340, 512 B) */ + hdr->secs[2].id = SOF_TEL_SEC_VREGION; + hdr->secs[2].elem_size = sizeof(struct sof_tel_vregion_info); + hdr->secs[2].offset = (uint16_t)offsetof(struct telemetry_wnd_data, vreg_sec); + hdr->secs[2].max_count = IS_ENABLED(CONFIG_SOF_TELEMETRY_VREGION) ? + SOF_TEL_MAX_VREGIONS : 0; + + /* Section 3: Pipelines & Schedulers (0x0540, 384 B) */ + hdr->secs[3].id = SOF_TEL_SEC_PIPELINES; + hdr->secs[3].elem_size = sizeof(struct sof_tel_ppl_info); + hdr->secs[3].offset = (uint16_t)offsetof(struct telemetry_wnd_data, ppl_sec); + hdr->secs[3].max_count = IS_ENABLED(CONFIG_SOF_TELEMETRY_PIPELINES) ? + SOF_TEL_MAX_PIPELINES : 0; + + /* Section 4: DAIs & Host Gateways (0x06C0, 256 B) */ + hdr->secs[4].id = SOF_TEL_SEC_DAIS; + hdr->secs[4].elem_size = sizeof(struct sof_tel_gw_cold); + hdr->secs[4].offset = (uint16_t)offsetof(struct telemetry_wnd_data, dai_sec); + hdr->secs[4].max_count = IS_ENABLED(CONFIG_SOF_TELEMETRY_DAIS) ? + SOF_TEL_MAX_GATEWAYS : 0; + + /* Section 5: Zephyr Threads (0x07C0, 832 B) */ + hdr->secs[5].id = SOF_TEL_SEC_THREADS; + hdr->secs[5].elem_size = sizeof(struct sof_tel_thread_info); + hdr->secs[5].offset = (uint16_t)offsetof(struct telemetry_wnd_data, thd_sec); + hdr->secs[5].max_count = IS_ENABLED(CONFIG_SOF_TELEMETRY_THREADS) ? + SOF_TEL_MAX_THREADS : 0; + + k_work_init_delayable(&telemetry_bg_dwork, telemetry_bg_work_handler); + telemetry_wnd_initialized = true; +} +#endif /* CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW */ + #ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS static void telemetry_perf_queue_append(struct telemetry_perf_queue *q, size_t element) { @@ -130,12 +1171,17 @@ __cold int telemetry_init(void) systick_info[i].peak_utilization_8k = 0; } +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW + telemetry_ext_init(wnd_data); +#endif + return 0; } void telemetry_update(uint32_t begin_stamp, uint32_t current_stamp) { int prid = cpu_get_id(); + uint32_t elapsed = current_stamp - begin_stamp; #ifdef CONFIG_INTEL_ADSP_DEBUG_SLOT_MANAGER struct system_tick_info *systick_info = telemetry_get_systick_info_ptr(); @@ -151,16 +1197,14 @@ void telemetry_update(uint32_t begin_stamp, uint32_t current_stamp) ++telemetry_systick_counter[prid]; systick_info[prid].count = telemetry_systick_counter[prid]; - systick_info[prid].last_time_elapsed = current_stamp - begin_stamp; + systick_info[prid].last_time_elapsed = elapsed; systick_info[prid].max_time_elapsed = - MAX(current_stamp - begin_stamp, - systick_info[prid].max_time_elapsed); + MAX(elapsed, systick_info[prid].max_time_elapsed); systick_info[prid].last_ccount = current_stamp; #ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS - const size_t measured_systick = begin_stamp - telemetry_prev_ccount[prid]; + const size_t measured_systick = elapsed; - telemetry_prev_ccount[prid] = begin_stamp; if (telemetry_systick_counter[prid] > 2) { telemetry_perf_period_sum[prid] += measured_systick; telemetry_perf_period_cnt[prid] = @@ -194,8 +1238,64 @@ void telemetry_update(uint32_t begin_stamp, uint32_t current_stamp) systick_info[prid].peak_utilization_8k = 0; } #endif + +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW + if (telemetry_ext_ready) { +#ifdef CONFIG_TIMING_FUNCTIONS + bool ll_overrun = timing_cycles_to_ns(elapsed) > + (uint64_t)LL_TIMER_PERIOD_US * NSEC_PER_USEC; +#else + bool ll_overrun = elapsed > (uint32_t)k_us_to_cyc_ceil32(LL_TIMER_PERIOD_US); +#endif + uint32_t ticks; + + if (ll_overrun && prid < SOF_TEL_MAX_CORES) { + struct sof_tel_hot_line0 *hot0 = telemetry_get_hot_line0(); + + if (hot0) + hot0->ll_overrun_count[prid]++; + } + +#ifdef CONFIG_SOF_TELEMETRY_PIPELINES + if (prid < SOF_TEL_MAX_CORES && + tel_state()->ll_task_count[prid] == 0) { + systick_info[prid].last_time_elapsed = 0; + systick_info[prid].avg_utilization = 0; +#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + telemetry_perf_period_sum[prid] = 0; + telemetry_perf_period_cnt[prid] = 0; + memset(&telemetry_perf_queue[prid], 0, + sizeof(telemetry_perf_queue[prid])); +#endif + } +#endif + + telemetry_flush_hot_staging(wnd_data); + ticks = ++tel_state()->total_tick_counter; + + if ((ticks & 0x3ff) == 0) + k_work_reschedule(&telemetry_bg_dwork, K_NO_WAIT); + } +#endif } -/* init telemetry using Zephyr*/ -SYS_INIT(telemetry_init, APPLICATION, CONFIG_APPLICATION_INIT_PRIORITY); +/* + * Register the telemetry debug window slot before notify_host_boot_complete + * (POST_KERNEL, 0) sends IPC4_FW_READY, then refresh all cold sections once + * SOF and Zephyr application init (APPLICATION, 99) have completed. + */ +static int telemetry_late_init(void) +{ +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW + if (!telemetry_wnd_initialized || !telemetry_get_wnd_ptr()) + return 0; + + telemetry_ext_ready = true; + telemetry_refresh_cold_sections(); + k_work_reschedule(&telemetry_bg_dwork, TELEMETRY_BG_REFRESH_PERIOD); +#endif + return 0; +} +SYS_INIT(telemetry_init, PRE_KERNEL_2, 0); +SYS_INIT(telemetry_late_init, APPLICATION, 99); From e5ebe5d96cabf69236d9d6a0acf7205b372f75b6 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Tue, 10 Nov 2026 16:07:58 +0000 Subject: [PATCH 5/9] ipc: unlink freed components/pipelines first and notify telemetry Unlink components and pipelines from ipc->comp_list and clear their pointers before freeing them so concurrent telemetry readers never observe a freed object, clear freed component performance data pointers, and notify the telemetry subsystem when pipelines and components are created or freed across IPC3 and IPC4. Signed-off-by: Liam Girdwood --- src/ipc/ipc-helper.c | 24 +++++++++++++++--------- src/ipc/ipc4/helper.c | 19 +++++++++++++++---- src/kernel/ipc/ipc3/helper.c | 15 +++++++++++---- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/src/ipc/ipc-helper.c b/src/ipc/ipc-helper.c index 3480f113a4a2..ce78ce0e92cf 100644 --- a/src/ipc/ipc-helper.c +++ b/src/ipc/ipc-helper.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -317,6 +318,7 @@ __cold int ipc_comp_free(struct ipc *ipc, uint32_t comp_id) #ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS free_performance_data(icd->cd->perf_data.perf_data_item); + icd->cd->perf_data.perf_data_item = NULL; #endif if (!icd->cd->bsource_list.next || !icd->cd->bsink_list.next) { @@ -362,29 +364,33 @@ __cold int ipc_comp_free(struct ipc *ipc, uint32_t comp_id) /* * A completed pipeline stores raw comp_dev pointers in its - * source_comp/sink_comp/sched_comp fields. + * source_comp/sink_comp/sched_comp fields. Unlink icd before + * freeing cd so concurrent readers of ipc->comp_list never + * observe a freed component. */ + struct comp_dev *cd = icd->cd; + list_for_item(clist, &ipc->comp_list) { struct ipc_comp_dev *ppl_icd = container_of(clist, struct ipc_comp_dev, list); - if (ppl_icd->type != COMP_TYPE_PIPELINE) + if (ppl_icd->type != COMP_TYPE_PIPELINE || !ppl_icd->pipeline) continue; - if (ppl_icd->pipeline->source_comp == icd->cd) + if (ppl_icd->pipeline->source_comp == cd) ppl_icd->pipeline->source_comp = NULL; - if (ppl_icd->pipeline->sink_comp == icd->cd) + if (ppl_icd->pipeline->sink_comp == cd) ppl_icd->pipeline->sink_comp = NULL; - if (ppl_icd->pipeline->sched_comp == icd->cd) + if (ppl_icd->pipeline->sched_comp == cd) ppl_icd->pipeline->sched_comp = NULL; } - /* free component and remove from list */ - comp_free(icd->cd); - icd->cd = NULL; - list_item_del(&icd->list); + + /* free component and container */ + comp_free(cd); sof_heap_free(sof_sys_user_heap_get(), icd); + telemetry_notify_pipeline_change(); return 0; } diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index d5a0e095b3f4..74a0ab6ba0dd 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -524,6 +525,7 @@ __cold static int ipc4_create_pipeline(struct ipc4_pipeline_create *pipe_desc, /* add new pipeline to the list */ list_item_append(&ipc_pipe->list, &ipc->comp_list); + telemetry_notify_pipeline_change(); return IPC4_SUCCESS; } @@ -678,16 +680,24 @@ __cold int ipc_pipeline_free(struct ipc *ipc, uint32_t comp_id) return ret; } - /* free buffer, delete all tasks and remove from list */ - ret = pipeline_free(ipc_pipe->pipeline); + /* Unlink from ipc->comp_list before freeing pipeline so concurrent + * readers never observe a freed pipeline pointer. + */ + struct pipeline *pipe = ipc_pipe->pipeline; + + ipc_pipe->pipeline = NULL; + list_item_del(&ipc_pipe->list); + + /* free buffer, delete all tasks and free pipeline */ + ret = pipeline_free(pipe); if (ret < 0) { tr_err(&ipc_tr, "pipeline_free() failed"); + sof_heap_free(sof_sys_user_heap_get(), ipc_pipe); return IPC4_INVALID_RESOURCE_STATE; } - ipc_pipe->pipeline = NULL; - list_item_del(&ipc_pipe->list); sof_heap_free(sof_sys_user_heap_get(), ipc_pipe); + telemetry_notify_pipeline_change(); return IPC4_SUCCESS; } @@ -1547,6 +1557,7 @@ __cold int ipc4_add_comp_dev(struct comp_dev *dev) tr_dbg(&ipc_tr, "add comp 0x%x", icd->id); /* add new component to the list */ list_item_append(&icd->list, &ipc->comp_list); + telemetry_notify_pipeline_change(); return IPC4_SUCCESS; }; diff --git a/src/kernel/ipc/ipc3/helper.c b/src/kernel/ipc/ipc3/helper.c index 2fb7b52e1f4a..9a67259a87f2 100644 --- a/src/kernel/ipc/ipc3/helper.c +++ b/src/kernel/ipc/ipc3/helper.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -434,6 +435,7 @@ int ipc_pipeline_new(struct ipc *ipc, ipc_pipe_new *_pipe_desc) /* add new pipeline to the list */ list_item_append(&ipc_pipe->list, &ipc->comp_list); + telemetry_notify_pipeline_change(); return 0; } @@ -443,6 +445,7 @@ int ipc_pipeline_free(struct ipc *ipc, uint32_t comp_id) struct ipc_comp_dev *ipc_pipe; struct ipc_comp_dev *icd; struct list_item *clist; + struct pipeline *pipe; int ret; /* check whether pipeline exists */ @@ -467,22 +470,25 @@ int ipc_pipeline_free(struct ipc *ipc, uint32_t comp_id) * (or in fuzz/error paths) the dangling pointer would be a * use-after-free on any subsequent IPC referencing that component. */ + pipe = ipc_pipe->pipeline; list_for_item(clist, &ipc->comp_list) { icd = container_of(clist, struct ipc_comp_dev, list); if (icd->type == COMP_TYPE_COMPONENT && - icd->cd && icd->cd->pipeline == ipc_pipe->pipeline) + icd->cd && icd->cd->pipeline == pipe) icd->cd->pipeline = NULL; } + ipc_pipe->pipeline = NULL; + list_item_del(&ipc_pipe->list); /* free buffer and remove from list */ - ret = pipeline_free(ipc_pipe->pipeline); + ret = pipeline_free(pipe); if (ret < 0) { tr_err(&ipc_tr, "pipeline_free() failed"); + rfree(ipc_pipe); return ret; } - ipc_pipe->pipeline = NULL; - list_item_del(&ipc_pipe->list); rfree(ipc_pipe); + telemetry_notify_pipeline_change(); return 0; } @@ -723,6 +729,7 @@ int ipc_comp_new(struct ipc *ipc, ipc_comp *_comp) /* add new component to the list */ list_item_append(&icd->list, &ipc->comp_list); + telemetry_notify_pipeline_change(); return 0; } From 493d841a8dfce3d6d8e3e741d7a6451309f9dda8 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Tue, 10 Nov 2026 16:07:58 +0000 Subject: [PATCH 6/9] schedule: report active LL task counts and DP deadline misses to telemetry Notify telemetry when the per-core active Low-Latency (LL) task count changes or when an LL tick overruns, and track Data Processing (DP) task deadline misses across kernel, K_USER, and application worker threads via scheduler_dp_check_deadline(). Signed-off-by: Liam Girdwood --- src/include/sof/schedule/dp_schedule.h | 14 ++++++++++++++ src/schedule/zephyr_dp_schedule.c | 17 +++++++++++++++++ src/schedule/zephyr_dp_schedule_application.c | 18 ++++++++++++++++++ src/schedule/zephyr_dp_schedule_thread.c | 17 +++++++++++++++++ src/schedule/zephyr_ll.c | 5 ++++- 5 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index 1dac74b600ef..288caa3396ee 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -90,7 +90,21 @@ __syscall void scheduler_dp_internal_free(struct task *task); * queued DP tasks, updates their absolute deadlines, and signals worker threads. */ __syscall void scheduler_dp_ll_tick(void); + +/** + * @brief Check whether a DP task finished after its deadline and report a telemetry miss. + * + * @param[in] core DSP core ID of the DP task. + * @param[in] deadline_clock_ticks Absolute hardware cycle deadline of the run. + */ +__syscall void scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks); #include +#else +static inline void scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks) +{ + (void)core; + (void)deadline_clock_ticks; +} #endif /** diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 314f8df5bba0..0946045506aa 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -256,6 +257,22 @@ void z_vrfy_scheduler_dp_ll_tick(void) #include #endif +void z_impl_scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks) +{ + if (deadline_clock_ticks && + (int32_t)(k_cycle_get_32() - deadline_clock_ticks) > 0) + telemetry_notify_dp_deadline_miss(core); +} + +#ifdef CONFIG_USERSPACE +void z_vrfy_scheduler_dp_check_deadline(uint16_t core, uint32_t deadline_clock_ticks) +{ + K_OOPS(K_SYSCALL_VERIFY(core < CONFIG_MP_MAX_NUM_CPUS)); + z_impl_scheduler_dp_check_deadline(core, deadline_clock_ticks); +} +#include +#endif + #if CONFIG_SOF_USERSPACE_APPLICATION static int scheduler_dp_task_cancel(void *data, struct task *task) { diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 5e69d0943419..6d44802a55f8 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -408,6 +409,15 @@ void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch) /* get module deadline in us */ uint32_t deadline = module_get_deadline(mod); + if (!trigger_task && !mod->dp_startup_delay && + pdata->deadline_clock_ticks && + (deadline == 0 || + (int32_t)(dp_sch->last_ll_tick_timestamp - + pdata->deadline_clock_ticks) > 0)) { + telemetry_notify_dp_deadline_miss(curr_task->core); + pdata->deadline_clock_ticks = 0; + } + /* if a deadline cannot be calculated, use a fixed value relative to its * first start */ @@ -423,6 +433,8 @@ void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch) /* add to "NOW", overflows are OK */ deadline = dp_sch->last_ll_tick_timestamp + deadline; + if (trigger_task) + pdata->deadline_clock_ticks = deadline ? deadline : 1U; /* set in Zephyr. Note that it may be in past, it does not matter, * Zephyr still will schedule the thread with earlier deadline @@ -495,6 +507,11 @@ void dp_thread_fn(void *p1, void *p2, void *p3) * if not, set the state returned by run procedure */ if (ready && task->state == SOF_TASK_STATE_RUNNING) { + if (!pmod->dp_startup_delay && + task_pdata->deadline_clock_ticks) + scheduler_dp_check_deadline(task->core, + task_pdata->deadline_clock_ticks); + task_pdata->deadline_clock_ticks = 0; task->state = state; switch (state) { case SOF_TASK_STATE_RESCHEDULE: @@ -512,6 +529,7 @@ void dp_thread_fn(void *p1, void *p2, void *p3) k_oops(); } } else { + task_pdata->deadline_clock_ticks = 0; task->state = SOF_TASK_STATE_QUEUED; } } else { diff --git a/src/schedule/zephyr_dp_schedule_thread.c b/src/schedule/zephyr_dp_schedule_thread.c index fa8583832a03..8e33ee1ab2ce 100644 --- a/src/schedule/zephyr_dp_schedule_thread.c +++ b/src/schedule/zephyr_dp_schedule_thread.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -82,6 +83,15 @@ static void scheduler_dp_recalculate_thread(struct scheduler_dp_data *dp_sch, bo /* get module deadline in us*/ uint32_t deadline = module_get_deadline(mod); + if (!trigger_task && is_ll_post_run && !mod->dp_startup_delay && + pdata->deadline_clock_ticks && + (deadline == 0 || + (int32_t)(dp_sch->last_ll_tick_timestamp - + pdata->deadline_clock_ticks) > 0)) { + telemetry_notify_dp_deadline_miss(curr_task->core); + pdata->deadline_clock_ticks = 0; + } + /* if a deadline cannot be calculated, use a fixed value relative to its * first start */ @@ -97,6 +107,8 @@ static void scheduler_dp_recalculate_thread(struct scheduler_dp_data *dp_sch, bo /* add to "NOW", overflows are OK */ deadline = dp_sch->last_ll_tick_timestamp + deadline; + if (trigger_task) + pdata->deadline_clock_ticks = deadline ? deadline : 1U; /* set in Zephyr. Note that it may be in past, it does not matter, * Zephyr still will schedule the thread with earlier deadline @@ -156,6 +168,11 @@ void dp_thread_fn(void *p1, void *p2, void *p3) * if not, set the state returned by run procedure */ if (task->state == SOF_TASK_STATE_RUNNING) { + if (!task_pdata->mod->dp_startup_delay && + task_pdata->deadline_clock_ticks) + scheduler_dp_check_deadline(task->core, + task_pdata->deadline_clock_ticks); + task_pdata->deadline_clock_ticks = 0; task->state = state; switch (state) { case SOF_TASK_STATE_RESCHEDULE: diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 3edebbca5b20..cd5b7715b5fe 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -151,7 +151,9 @@ static void zephyr_ll_task_done(struct zephyr_ll *sch, * important to decrement the task counter last before aborting the * thread. */ - domain_unregister(sch->ll_domain, task, --sch->n_tasks); + sch->n_tasks--; + telemetry_notify_ll_task_count(sch->core, sch->n_tasks); + domain_unregister(sch->ll_domain, task, sch->n_tasks); } /* The caller must hold the lock and possibly disable interrupts */ @@ -416,6 +418,7 @@ static int zephyr_ll_task_schedule_common(struct zephyr_ll *sch, struct task *ta zephyr_ll_task_insert_after_unlocked(task, reference); sch->n_tasks++; + telemetry_notify_ll_task_count(sch->core, sch->n_tasks); zephyr_ll_unlock(sch, &flags); From 50608e37b052baa494e20f06b757b8479f65c793 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Tue, 10 Nov 2026 16:07:58 +0000 Subject: [PATCH 7/9] pipeline: notify telemetry on pipeline state changes and XRUNs Notify the telemetry subsystem on pipeline schedule, trigger, and reset transitions and record pipeline and component XRUN events in the TEL2 and PMW2 telemetry windows. Signed-off-by: Liam Girdwood --- src/audio/pipeline/pipeline-schedule.c | 9 ++++++++- src/audio/pipeline/pipeline-stream.c | 7 ++++++- src/audio/pipeline/pipeline-xrun.c | 5 +++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/audio/pipeline/pipeline-schedule.c b/src/audio/pipeline/pipeline-schedule.c index 3893cd227c9b..63b9fd979088 100644 --- a/src/audio/pipeline/pipeline-schedule.c +++ b/src/audio/pipeline/pipeline-schedule.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +108,7 @@ static enum task_state pipeline_task_cmd(struct pipeline *p, } p->trigger.aborted = false; + telemetry_notify_pipeline_change(); return ret; } @@ -144,8 +146,10 @@ static enum task_state pipeline_task_cmd(struct pipeline *p, } } else if (p->trigger.cmd != cmd) { /* PRE stage completed */ - if (p->trigger.delay) + if (p->trigger.delay) { + telemetry_notify_pipeline_change(); return SOF_TASK_STATE_RESCHEDULE; + } /* No delay: the final stage has already run too */ err = SOF_TASK_STATE_RESCHEDULE; } else if (p->status == COMP_STATE_PAUSED) { @@ -163,6 +167,7 @@ static enum task_state pipeline_task_cmd(struct pipeline *p, } p->trigger.cmd = COMP_TRIGGER_NO_ACTION; + telemetry_notify_pipeline_change(); ipc_msg_reply(reply); @@ -240,6 +245,7 @@ static enum task_state ipc3_pipeline_task(void *arg) err = pipeline_xrun_recover(p); if (err < 0) { pipe_err(p, "xrun recovery failed! pipeline is stopped."); + telemetry_notify_pipeline_change(); /* failed - host will stop this pipeline */ return SOF_TASK_STATE_COMPLETED; } @@ -332,6 +338,7 @@ static enum task_state ipc4_pipeline_copy_task(void *arg) err = pipeline_xrun_recover(p); if (err < 0) { pipe_err(p, "xrun recovery failed! pipeline is stopped."); + telemetry_notify_pipeline_change(); /* failed - host will stop this pipeline */ return SOF_TASK_STATE_COMPLETED; } diff --git a/src/audio/pipeline/pipeline-stream.c b/src/audio/pipeline/pipeline-stream.c index 9b015f26cb4d..9028247dc9cb 100644 --- a/src/audio/pipeline/pipeline-stream.c +++ b/src/audio/pipeline/pipeline-stream.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -401,7 +402,10 @@ int pipeline_trigger(struct pipeline *p, struct comp_dev *host, int cmd) if (p->status == COMP_STATE_PAUSED || p->xrun_bytes) { /* The task isn't running, trigger inline */ ret = pipeline_trigger_run(p, host, cmd); - return ret < 0 ? ret : 0; + if (ret < 0) + return ret; + telemetry_notify_pipeline_change(); + return 0; } COMPILER_FALLTHROUGH; @@ -435,6 +439,7 @@ int pipeline_trigger(struct pipeline *p, struct comp_dev *host, int cmd) return PPL_STATUS_SCHEDULED; } + telemetry_notify_pipeline_change(); return 0; } diff --git a/src/audio/pipeline/pipeline-xrun.c b/src/audio/pipeline/pipeline-xrun.c index fde01cbe699e..09ad9be84457 100644 --- a/src/audio/pipeline/pipeline-xrun.c +++ b/src/audio/pipeline/pipeline-xrun.c @@ -7,6 +7,8 @@ #include #include +#include +#include #include #include #include @@ -163,6 +165,9 @@ void pipeline_xrun(struct pipeline *p, struct comp_dev *dev, if (dev->state != COMP_STATE_ACTIVE) return; + telemetry_notify_pipeline_xrun(p, bytes); + perf_data_inc_xrun(dev); + /* notify all pipeline comps we are in XRUN, and stop copying */ ret = pipeline_trigger(p, p->source_comp, COMP_TRIGGER_XRUN); if (ret < 0) From f72e6b724a543f799f5847c276b14f755b8919e1 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Tue, 10 Nov 2026 16:07:58 +0000 Subject: [PATCH 8/9] audio: module_adapter: update per-module telemetry resource metrics Refresh per-module cold resource metrics (private heap usage, heap high-water mark, and configured CPC) in Memory Window 3 during component state transitions and module adapter lifecycle operations. Signed-off-by: Liam Girdwood --- src/audio/component.c | 13 +++++++++++-- src/audio/module_adapter/module/generic.c | 6 ++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/audio/component.c b/src/audio/component.c index 02797941d6b6..e2e670670126 100644 --- a/src/audio/component.c +++ b/src/audio/component.c @@ -640,13 +640,18 @@ void comp_init_performance_data(struct comp_dev *dev) { struct perf_data_item_comp *item = dev->perf_data.perf_data_item; - if (item) + if (item) { perf_data_item_comp_init(item, dev->ipc_config.id, 0); + perf_data_update_module_resources(dev); + } } /* returns true if budget violation occurred */ static bool update_peak_of_measured_cpc(struct comp_dev *dev, size_t measured_cpc) { + if (dev->cpc && measured_cpc > dev->cpc) + perf_data_inc_cpc_violation(dev); + if (measured_cpc <= dev->perf_data.peak_of_measured_cpc) return false; dev->perf_data.peak_of_measured_cpc = measured_cpc; @@ -657,7 +662,8 @@ bool comp_update_performance_data(struct comp_dev *dev, uint32_t cycles_used) { struct perf_data_item_comp *item = dev->perf_data.perf_data_item; - if (perf_meas_get_state() == IPC4_PERF_MEASUREMENTS_STARTED) { + if (perf_meas_get_state() == IPC4_PERF_MEASUREMENTS_STARTED || + IS_ENABLED(CONFIG_SOF_TELEMETRY_MODULE_RESOURCES)) { /* we divide by ibs so we need to check if its set */ if (item && dev->ibs != 0) { item->total_iteration_count++; @@ -720,6 +726,9 @@ void comp_update_ibs_obs_cpc(struct comp_dev *dev) dev->obs = dev_cfg.obs; dev->ibs = dev_cfg.ibs; dev->cpc = dev_cfg.cpc; +#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + perf_data_update_module_resources(dev); +#endif #else /* set neutral values */ dev->ll_chunk_size = 0; diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index 415b2437c17d..6724e983faa0 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -231,6 +232,7 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t res->heap_high_water_mark = res->heap_usage; k_mutex_unlock(&mod_res_lock); + perf_data_update_module_resources(mod->dev); return ptr; } EXPORT_SYMBOL(z_impl_mod_balloc_align); @@ -284,6 +286,7 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t res->heap_high_water_mark = res->heap_usage; k_mutex_unlock(&mod_res_lock); + perf_data_update_module_resources(mod->dev); return ptr; } EXPORT_SYMBOL(z_impl_mod_alloc_ext); @@ -486,6 +489,8 @@ int z_impl_mod_free(struct processing_module *mod, const void *ptr) if (ret < 0) comp_err(mod->dev, "error: could not find memory pointed by %p", ptr); + else + perf_data_update_module_resources(mod->dev); return ret; } @@ -791,6 +796,7 @@ __cold void z_impl_mod_free_all(struct processing_module *mod) /* Make sure resource lists and accounting are reset */ mod_resource_init(mod); + perf_data_update_module_resources(mod->dev); } __cold int module_free(struct processing_module *mod) From 5cdfda19202fe112e182228d530be75ab967938a Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Tue, 10 Nov 2026 16:07:58 +0000 Subject: [PATCH 9/9] audio: dai/host: wire DAI and Host DMA gateway telemetry Register and unregister DAI and Host DMA gateways with the TEL2 telemetry window and update live gateway buffer fill levels, processed byte counts, and XRUN/skip counters. Signed-off-by: Liam Girdwood --- src/audio/dai-zephyr.c | 17 ++++++++++++++++- src/audio/host-zephyr.c | 11 +++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/audio/dai-zephyr.c b/src/audio/dai-zephyr.c index f6a0b66d2956..40d0a9f4902a 100644 --- a/src/audio/dai-zephyr.c +++ b/src/audio/dai-zephyr.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -357,6 +358,7 @@ dai_dma_cb(struct dai_data *dd, struct comp_dev *dev, uint32_t bytes, /* fill buffer with silence */ buffer_zero(dd->dma_buffer); + telemetry_update_dai_hot(dd, dev); return dma_status; } @@ -502,6 +504,7 @@ dai_dma_cb(struct dai_data *dd, struct comp_dev *dev, uint32_t bytes, /* Increment performance counters */ io_perf_monitor_update_data(dd->io_perf_dai_byte_count, bytes); #endif + telemetry_update_dai_hot(dd, dev); return dma_status; } @@ -532,6 +535,7 @@ dai_dma_multi_endpoint_cb(struct dai_data *dd, struct comp_dev *dev, uint32_t fr /* fill buffer with silence */ buffer_zero(dd->dma_buffer); + telemetry_update_dai_hot(dd, dev); return dma_status; } @@ -564,6 +568,7 @@ dai_dma_multi_endpoint_cb(struct dai_data *dd, struct comp_dev *dev, uint32_t fr /* update host position (in bytes offset) for drivers */ dd->total_data_processed += bytes; + telemetry_update_dai_hot(dd, dev); return dma_status; } @@ -656,6 +661,7 @@ __cold int dai_common_new(struct dai_data *dd, struct comp_dev *dev, io_perf_monitor_init_data(&dd->io_perf_dai_byte_count, &init_data); } #endif + telemetry_register_dai(dd, dev); return 0; } @@ -790,6 +796,7 @@ __cold void dai_common_free(struct dai_data *dd) #ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS io_perf_monitor_release_slot(dd->io_perf_dai_byte_count); #endif + telemetry_unregister_dai(dd); if (dd->group) dai_group_put(dd->group); @@ -1293,6 +1300,7 @@ int dai_common_params(struct dai_data *dd, struct comp_dev *dev, goto out; } + telemetry_register_dai(dd, dev); out: /* * Make sure to free all allocated items, all functions @@ -1395,12 +1403,15 @@ int dai_common_prepare(struct dai_data *dd, struct comp_dev *dev) if (dd->xrun) { /* after prepare, we have recovered from xrun */ dd->xrun = 0; + telemetry_update_dai_hot(dd, dev); return 0; } ret = dai_dma_config(dd); if (ret < 0) comp_set_state(dev, COMP_TRIGGER_RESET); + else + telemetry_update_dai_hot(dd, dev); return ret; } @@ -1456,6 +1467,7 @@ void dai_common_reset(struct dai_data *dd, struct comp_dev *dev) dd->wallclock = 0; dd->total_data_processed = 0; dd->xrun = 0; + telemetry_update_dai_hot(dd, NULL); } static int dai_reset(struct comp_dev *dev) @@ -1559,6 +1571,7 @@ static int dai_comp_trigger_internal(struct dai_data *dd, struct comp_dev *dev, case COMP_TRIGGER_XRUN: comp_info(dev, "XRUN"); dd->xrun = 1; + telemetry_update_dai_hot(dd, dev); COMPILER_FALLTHROUGH; case COMP_TRIGGER_STOP: @@ -1611,8 +1624,10 @@ static int dai_comp_trigger_internal(struct dai_data *dd, struct comp_dev *dev, * See issue #8920 for details. */ if (!ret) - return comp_set_state(dev, cmd); + ret = comp_set_state(dev, cmd); #endif /* CONFIG_IPC_MAJOR_3 */ + if (!ret) + telemetry_update_dai_hot(dd, dev); return ret; } diff --git a/src/audio/host-zephyr.c b/src/audio/host-zephyr.c index 0884fd834d8b..30449f3c0edf 100644 --- a/src/audio/host-zephyr.c +++ b/src/audio/host-zephyr.c @@ -25,6 +25,7 @@ #include #include #include +#include #include /* zephyr_ll_user_heap() */ #include #include @@ -291,6 +292,7 @@ void host_common_update(struct host_data *hd, struct comp_dev *dev, uint32_t byt #endif hd->total_data_processed += bytes; + telemetry_update_host_hot(hd, dev); /* new local period, update host buffer position blks * local_pos is queried by the ops.position() API @@ -557,6 +559,8 @@ static uint32_t host_get_copy_bytes_normal(struct host_data *hd, struct comp_dev avail_samples, free_samples); } hd->n_skipped++; + telemetry_notify_host_skip(hd); + telemetry_update_host_hot(hd, dev); } } @@ -738,6 +742,9 @@ int host_common_trigger(struct host_data *hd, struct comp_dev *dev, int cmd) break; } + if (!ret) + telemetry_update_host_hot(hd, dev); + return ret; } @@ -866,6 +873,7 @@ __cold void host_common_free(struct host_data *hd) hd->io_perf_host_byte_count = NULL; } #endif + telemetry_unregister_host(hd); /* release DMA channel if not already done by reset */ if (hd->chan_index >= 0) { @@ -1197,6 +1205,7 @@ int host_common_params(struct host_data *hd, struct comp_dev *dev, io_perf_monitor_init_data(&hd->io_perf_host_byte_count, &init_data); } #endif + telemetry_register_host(hd, dev); return 0; @@ -1232,6 +1241,7 @@ static int host_params(struct comp_dev *dev, int host_common_prepare(struct host_data *hd) { buffer_zero(hd->dma_buffer); + telemetry_update_host_hot(hd, NULL); return 0; } @@ -1299,6 +1309,7 @@ void host_common_reset(struct host_data *hd, uint16_t state) hd->copy_type = COMP_COPY_NORMAL; hd->source = NULL; hd->sink = NULL; + telemetry_update_host_hot(hd, NULL); } #if CONFIG_IPC_MAJOR_3