--- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -6,6 +6,10 @@ #include #include #include +#include +#ifdef CONFIG_SCHED_CACHE +int sds_window_cpu = -1; /* SDS-WITNESS */ +#endif #include "sched.h" DEFINE_MUTEX(sched_domains_mutex); @@ -828,6 +832,16 @@ dirty_sched_domain_sysctl(cpu); destroy_sched_domains(tmp); + /* + * SDS-WITNESS: widen the window between entering the RCU removal + * phase (destroy_sched_domains) and publishing the new per-CPU + * pointers (update_top_cache_domain). Verification only. + */ + WRITE_ONCE(sds_window_cpu, cpu); + pr_info("SDS-WINDOW: cpu=%d\n", cpu); + mdelay(100); + pr_info("SDS-STORE: cpu=%d\n", cpu); + update_top_cache_domain(cpu); } @@ -3502,3 +3516,63 @@ partition_sched_domains_locked(ndoms_new, doms_new, dattr_new); sched_domains_mutex_unlock(); } + +/* ---------------- SDS-WITNESS: verification instrumentation ---------------- */ +/* + * Reacts to the first window of a detach/build pass (cpu 0), reports a + * quiescent state after the grace period has started, then loads the old + * per-CPU domain pointer, holds the RCU read-side section across the + * callback, and dereferences the domain. With the mdelay(100) in + * cpu_attach_domain() the window is wide enough to load the old pointer + * after call_rcu() and before the per-CPU store. + */ +#ifdef CONFIG_SCHED_CACHE +#include + +#define SDS_WITNESS_CPU 0 +#define SDS_WITNESS_HOLD_MS 1500 + +static int sds_witness_fn(void *unused) +{ + struct sched_domain *sd; + struct sched_domain_shared *sds; + + while (!kthread_should_stop()) { + if (READ_ONCE(sds_window_cpu) != SDS_WITNESS_CPU) { + msleep(1); + continue; + } + WRITE_ONCE(sds_window_cpu, -1); + + /* QS after the GP started, before entering the section. */ + msleep(5); + + rcu_read_lock(); + sd = rcu_dereference_all(per_cpu(sd_llc, SDS_WITNESS_CPU)); + if (sd) { + pr_info("SDS-LOAD: cpu=%d sd=%px\n", + SDS_WITNESS_CPU, (void *)sd); + msleep(SDS_WITNESS_HOLD_MS); + sds = rcu_dereference_all(sd->shared); + pr_info("SDS-WITNESS: cpu=%d sd=%px flags=%x shared=%px nr_busy=%d\n", + SDS_WITNESS_CPU, (void *)sd, + (unsigned int)READ_ONCE(sd->flags), (void *)sds, + sds ? atomic_read(&sds->nr_busy_cpus) : -1); + } else { + pr_info("SDS-LOAD: cpu=%d NULL\n", SDS_WITNESS_CPU); + } + rcu_read_unlock(); + msleep(10); + } + + return 0; +} + +static int __init sds_witness_init(void) +{ + pr_info("SDS-WITNESS: starting, target cpu %d\n", SDS_WITNESS_CPU); + kthread_run(sds_witness_fn, NULL, "sds_witness"); + return 0; +} +late_initcall(sds_witness_init); +#endif