* [PATCH 1/3] blkcg: allow printing rwstat samples without policy data
2026-11-10 10:31 [PATCH 0/3] blkcg: decouple cgroup v1 I/O accounting from throttling Zhe Liu
@ 2026-11-10 10:31 ` Zhe Liu
2026-11-10 10:32 ` [PATCH 2/3] blkcg: move cgroup v1 I/O statistics out of the throttle policy Zhe Liu
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Zhe Liu @ 2026-11-10 10:31 UTC (permalink / raw)
To: axboe, tj, josef; +Cc: yukuai, linux-block, linux-kernel, cgroups, Zhe Liu
The existing rwstat formatter takes a blkg_policy_data pointer, although
it only uses it to obtain the associated blkg. This unnecessarily ties
statistics formatting to a policy.
Introduce blkg_print_rwstat() to print rwstat samples directly from a
blkg, allowing cgroup v1 I/O statistics to be reported independently of
the throttle policy.
Keep __blkg_prfill_rwstat() as a wrapper for existing policy users,
including BFQ. The output format and total calculation remain unchanged.
Signed-off-by: Zhe Liu <liuzhe1@kylinos.cn>
---
block/blk-cgroup-rwstat.c | 18 ++++++++++++------
block/blk-cgroup-rwstat.h | 2 ++
2 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/block/blk-cgroup-rwstat.c b/block/blk-cgroup-rwstat.c
index 2e9bf081cce2..604d8414dec6 100644
--- a/block/blk-cgroup-rwstat.c
+++ b/block/blk-cgroup-rwstat.c
@@ -26,15 +26,15 @@ void blkg_rwstat_exit(struct blkg_rwstat *rwstat)
EXPORT_SYMBOL_GPL(blkg_rwstat_exit);
/**
- * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
+ * blkg_print_rwstat - print a blkg_rwstat sample
* @sf: seq_file to print to
- * @pd: policy private data of interest
+ * @blkg: block cgroup and queue association
* @rwstat: rwstat to print
*
- * Print @rwstat to @sf for the device assocaited with @pd.
+ * Print @rwstat to @sf for the device associated with @blkg.
*/
-u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
- const struct blkg_rwstat_sample *rwstat)
+u64 blkg_print_rwstat(struct seq_file *sf, struct blkcg_gq *blkg,
+ const struct blkg_rwstat_sample *rwstat)
{
static const char *rwstr[] = {
[BLKG_RWSTAT_READ] = "Read",
@@ -43,7 +43,7 @@ u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
[BLKG_RWSTAT_ASYNC] = "Async",
[BLKG_RWSTAT_DISCARD] = "Discard",
};
- const char *dname = blkg_dev_name(pd->blkg);
+ const char *dname = blkg_dev_name(blkg);
u64 v;
int i;
@@ -60,6 +60,12 @@ u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
seq_printf(sf, "%s Total %llu\n", dname, v);
return v;
}
+
+u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
+ const struct blkg_rwstat_sample *rwstat)
+{
+ return blkg_print_rwstat(sf, pd->blkg, rwstat);
+}
EXPORT_SYMBOL_GPL(__blkg_prfill_rwstat);
/**
diff --git a/block/blk-cgroup-rwstat.h b/block/blk-cgroup-rwstat.h
index 703a16fe1404..014ac2a73306 100644
--- a/block/blk-cgroup-rwstat.h
+++ b/block/blk-cgroup-rwstat.h
@@ -41,6 +41,8 @@ static inline u64 blkg_rwstat_read_counter(struct blkg_rwstat *rwstat,
int blkg_rwstat_init(struct blkg_rwstat *rwstat, gfp_t gfp);
void blkg_rwstat_exit(struct blkg_rwstat *rwstat);
+u64 blkg_print_rwstat(struct seq_file *sf, struct blkcg_gq *blkg,
+ const struct blkg_rwstat_sample *rwstat);
u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
const struct blkg_rwstat_sample *rwstat);
u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
--
2.25.1
^ permalink raw reply [flat|nested] 5+ messages in thread* [PATCH 2/3] blkcg: move cgroup v1 I/O statistics out of the throttle policy
2026-11-10 10:31 [PATCH 0/3] blkcg: decouple cgroup v1 I/O accounting from throttling Zhe Liu
2026-11-10 10:31 ` [PATCH 1/3] blkcg: allow printing rwstat samples without policy data Zhe Liu
@ 2026-11-10 10:32 ` Zhe Liu
2026-11-10 10:32 ` [PATCH 3/3] blkcg: account cgroup v1 I/O before checking throttle activation Zhe Liu
2026-11-10 16:06 ` [PATCH 0/3] blkcg: decouple cgroup v1 I/O accounting from throttling Tejun Heo
3 siblings, 0 replies; 5+ messages in thread
From: Zhe Liu @ 2026-11-10 10:32 UTC (permalink / raw)
To: axboe, tj, josef; +Cc: yukuai, linux-block, linux-kernel, cgroups, Zhe Liu
Cgroup v1 I/O statistics are currently stored in throtl_grp, making
their allocation and lifetime dependent on the throttle policy. With
lazy throttle initialization, these counters may not be available until
an I/O limit is configured.
Move the byte and I/O counters into an optional blkg-owned statistics
object, independent of throttle activation. Allocate the counters for
v1 and root blkgs to support late controller binding, while avoiding
unnecessary allocations for non-root v2 blkgs.
Move cgroup v1 statistics reporting, reset and offline propagation into
the blkcg core. Add recursive aggregation for the optional statistics
objects and preserve accumulated statistics when child blkgs are
destroyed. Keep the existing userspace interfaces unchanged.
Keep the accounting call site in blk_should_throtl(), routed through
the new blkcg_account_v1_bio(). Moving accounting before the throttle
activation check is handled in a separate patch.
Signed-off-by: Zhe Liu <liuzhe1@kylinos.cn>
---
block/blk-cgroup.c | 188 ++++++++++++++++++++++++++++++++++++++++++-
block/blk-cgroup.h | 15 ++++
block/blk-throttle.c | 78 ------------------
block/blk-throttle.h | 13 +--
4 files changed, 204 insertions(+), 90 deletions(-)
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 5b7a32293dc6..52093b06688a 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -32,6 +32,7 @@
#include <linux/part_stat.h>
#include "blk.h"
#include "blk-cgroup.h"
+#include "blk-cgroup-rwstat.h"
#include "blk-ioprio.h"
#include "blk-throttle.h"
@@ -66,6 +67,90 @@ static DEFINE_RAW_SPINLOCK(blkg_stat_lock);
#define BLKG_DESTROY_BATCH_SIZE 64
+#ifdef CONFIG_BLK_DEV_THROTTLING
+struct blkg_v1_stats {
+ struct blkg_rwstat bytes;
+ struct blkg_rwstat ios;
+};
+
+static int blkg_v1_stats_init(struct blkcg_gq *blkg, struct blkcg *blkcg,
+ gfp_t gfp)
+{
+ struct blkg_v1_stats *stats;
+ int ret;
+
+ /*
+ * Keep stats for the root blkcg to support late v1 binding.
+ * Non-root v2 blkgs do not need v1 statistics.
+ */
+ if (blkcg != &blkcg_root && cgroup_on_dfl(blkcg->css.cgroup))
+ return 0;
+
+ stats = kzalloc_obj(*stats, gfp);
+ if (!stats)
+ return -ENOMEM;
+ ret = blkg_rwstat_init(&stats->bytes, gfp);
+ if (ret)
+ goto out_free;
+ ret = blkg_rwstat_init(&stats->ios, gfp);
+ if (ret)
+ goto out_exit_bytes;
+ blkg->v1_stats = stats;
+ return 0;
+
+out_exit_bytes:
+ blkg_rwstat_exit(&stats->bytes);
+out_free:
+ kfree(stats);
+ return ret;
+}
+
+static void blkg_v1_stats_exit(struct blkcg_gq *blkg)
+{
+ struct blkg_v1_stats *stats = blkg->v1_stats;
+
+ if (!stats)
+ return;
+ blkg_rwstat_exit(&stats->ios);
+ blkg_rwstat_exit(&stats->bytes);
+ kfree(stats);
+}
+
+static void blkg_v1_stats_reset(struct blkcg_gq *blkg)
+{
+ struct blkg_v1_stats *stats = blkg->v1_stats;
+
+ if (!stats)
+ return;
+ blkg_rwstat_reset(&stats->bytes);
+ blkg_rwstat_reset(&stats->ios);
+}
+
+static void blkg_v1_stats_offline(struct blkcg_gq *blkg)
+{
+ struct blkg_v1_stats *stats = blkg->v1_stats;
+ struct blkg_v1_stats *parent;
+
+ if (!stats || !blkg->parent)
+ return;
+
+ parent = blkg->parent->v1_stats;
+ blkg_rwstat_add_aux(&parent->bytes, &stats->bytes);
+ blkg_rwstat_add_aux(&parent->ios, &stats->ios);
+ blkg_v1_stats_reset(blkg);
+}
+#else
+static inline int blkg_v1_stats_init(struct blkcg_gq *blkg,
+ struct blkcg *blkcg, gfp_t gfp)
+{
+ return 0;
+}
+
+static inline void blkg_v1_stats_exit(struct blkcg_gq *blkg) { }
+static inline void blkg_v1_stats_reset(struct blkcg_gq *blkg) { }
+static inline void blkg_v1_stats_offline(struct blkcg_gq *blkg) { }
+#endif
+
/*
* Lockless lists for tracking IO stats update
*
@@ -150,6 +235,7 @@ static void blkg_free_workfn(struct work_struct *work)
*/
css_put(&blkg->blkcg->css);
blk_put_queue(q);
+ blkg_v1_stats_exit(blkg);
free_percpu(blkg->iostat_cpu);
percpu_ref_exit(&blkg->refcnt);
kfree(blkg);
@@ -329,8 +415,10 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,
blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask);
if (!blkg->iostat_cpu)
goto out_exit_refcnt;
- if (!blk_get_queue(disk->queue))
+ if (blkg_v1_stats_init(blkg, blkcg, gfp_mask))
goto out_free_iostat;
+ if (!blk_get_queue(disk->queue))
+ goto out_exit_stats;
/* blkg holds a reference to blkcg */
if (!css_tryget_online(&blkcg->css))
goto out_put_queue;
@@ -377,6 +465,8 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct gendisk *disk,
css_put(&blkcg->css);
out_put_queue:
blk_put_queue(disk->queue);
+out_exit_stats:
+ blkg_v1_stats_exit(blkg);
out_free_iostat:
free_percpu(blkg->iostat_cpu);
out_exit_refcnt:
@@ -555,6 +645,7 @@ static void blkg_destroy(struct blkcg_gq *blkg)
}
}
+ blkg_v1_stats_offline(blkg);
blkg->online = false;
radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
@@ -678,6 +769,7 @@ static int blkcg_reset_stats(struct cgroup_subsys_state *css,
*/
hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
blkg_clear_stat(blkg);
+ blkg_v1_stats_reset(blkg);
for (i = 0; i < BLKCG_MAX_POLS; i++) {
struct blkcg_policy *pol = blkcg_policy[i];
@@ -1231,7 +1323,101 @@ static struct cftype blkcg_files[] = {
{ } /* terminate */
};
+#ifdef CONFIG_BLK_DEV_THROTTLING
+void __blkcg_account_v1_bio(struct bio *bio)
+{
+ struct blkg_v1_stats *stats = bio->bi_blkg->v1_stats;
+
+ if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
+ bio_set_flag(bio, BIO_CGROUP_ACCT);
+ blkg_rwstat_add(&stats->bytes, bio->bi_opf,
+ bio->bi_iter.bi_size);
+ }
+ blkg_rwstat_add(&stats->ios, bio->bi_opf, 1);
+}
+
+static void blkg_v1_stat_recursive_sum(struct blkcg_gq *blkg, int off,
+ struct blkg_rwstat_sample *sum)
+{
+ struct cgroup_subsys_state *pos_css;
+ struct blkcg_gq *pos_blkg;
+ int i;
+
+ WARN_ON_ONCE(!rcu_read_lock_held());
+ memset(sum, 0, sizeof(*sum));
+ blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
+ struct blkg_rwstat *rwstat;
+
+ /* Tolerate racing with blkg offlining. */
+ if (!data_race(pos_blkg->online) || !pos_blkg->v1_stats)
+ continue;
+ rwstat = (void *)pos_blkg->v1_stats + off;
+ for (i = 0; i < BLKG_RWSTAT_NR; i++)
+ sum->cnt[i] += blkg_rwstat_read_counter(rwstat, i);
+ }
+}
+
+static int blkcg_print_v1_stats(struct seq_file *sf, bool recursive)
+{
+ struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
+ int off = seq_cft(sf)->private;
+ struct blkcg_gq *blkg;
+ u64 total = 0;
+
+ rcu_read_lock();
+ hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
+ struct gendisk *disk = READ_ONCE(blkg->q->disk);
+ struct blkg_rwstat_sample sum;
+
+ if (!blkg->v1_stats || !disk)
+ continue;
+
+ if (recursive)
+ blkg_v1_stat_recursive_sum(blkg, off, &sum);
+ else
+ blkg_rwstat_read((void *)blkg->v1_stats + off, &sum);
+ total += blkg_print_rwstat(sf, blkg, &sum);
+ }
+ rcu_read_unlock();
+
+ seq_printf(sf, "Total %llu\n", total);
+ return 0;
+}
+
+static int blkcg_print_v1_rwstat(struct seq_file *sf, void *v)
+{
+ return blkcg_print_v1_stats(sf, false);
+}
+
+static int blkcg_print_v1_rwstat_recursive(struct seq_file *sf, void *v)
+{
+ return blkcg_print_v1_stats(sf, true);
+}
+#endif
+
static struct cftype blkcg_legacy_files[] = {
+#ifdef CONFIG_BLK_DEV_THROTTLING
+ {
+ .name = "throttle.io_service_bytes",
+ .private = offsetof(struct blkg_v1_stats, bytes),
+ .seq_show = blkcg_print_v1_rwstat,
+ },
+ {
+ .name = "throttle.io_service_bytes_recursive",
+ .private = offsetof(struct blkg_v1_stats, bytes),
+ .seq_show = blkcg_print_v1_rwstat_recursive,
+ },
+ {
+ .name = "throttle.io_serviced",
+ .private = offsetof(struct blkg_v1_stats, ios),
+ .seq_show = blkcg_print_v1_rwstat,
+ },
+ {
+ .name = "throttle.io_serviced_recursive",
+ .private = offsetof(struct blkg_v1_stats, ios),
+ .seq_show = blkcg_print_v1_rwstat_recursive,
+ },
+#endif
{
.name = "reset_stats",
.write_u64 = blkcg_reset_stats,
diff --git a/block/blk-cgroup.h b/block/blk-cgroup.h
index 1fd60cb4157c..bd528d52ab4c 100644
--- a/block/blk-cgroup.h
+++ b/block/blk-cgroup.h
@@ -23,6 +23,7 @@
struct blkcg_gq;
struct blkg_policy_data;
+struct blkg_v1_stats;
/* percpu_counter batch for blkg_[rw]stats, per-cpu drift doesn't matter */
@@ -71,6 +72,9 @@ struct blkcg_gq {
struct blkg_iostat_set __percpu *iostat_cpu;
struct blkg_iostat_set iostat;
+#ifdef CONFIG_BLK_DEV_THROTTLING
+ struct blkg_v1_stats *v1_stats;
+#endif
struct blkg_policy_data *pd[BLKCG_MAX_POLS];
#ifdef CONFIG_BLK_CGROUP_PUNT_BIO
@@ -520,4 +524,15 @@ static inline bool blk_cgroup_mergeable(struct request *rq, struct bio *bio) { r
#endif /* CONFIG_BLK_CGROUP */
+#ifdef CONFIG_BLK_DEV_THROTTLING
+void __blkcg_account_v1_bio(struct bio *bio);
+static inline void blkcg_account_v1_bio(struct bio *bio)
+{
+ if (!cgroup_subsys_on_dfl(io_cgrp_subsys))
+ __blkcg_account_v1_bio(bio);
+}
+#else
+static inline void blkcg_account_v1_bio(struct bio *bio) { }
+#endif
+
#endif /* _BLK_CGROUP_PRIVATE_H */
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index c5e26551956a..8f0632868780 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -268,12 +268,6 @@ static struct blkg_policy_data *throtl_pd_alloc(struct gendisk *disk,
if (!tg)
return NULL;
- if (blkg_rwstat_init(&tg->stat_bytes, gfp))
- goto err_free_tg;
-
- if (blkg_rwstat_init(&tg->stat_ios, gfp))
- goto err_exit_stat_bytes;
-
throtl_service_queue_init(&tg->service_queue);
for (rw = READ; rw <= WRITE; rw++) {
@@ -288,12 +282,6 @@ static struct blkg_policy_data *throtl_pd_alloc(struct gendisk *disk,
tg->iops[WRITE] = UINT_MAX;
return &tg->pd;
-
-err_exit_stat_bytes:
- blkg_rwstat_exit(&tg->stat_bytes);
-err_free_tg:
- kfree(tg);
- return NULL;
}
static void throtl_pd_init(struct blkg_policy_data *pd)
@@ -359,8 +347,6 @@ static void tg_release(struct rcu_head *rcu)
container_of(rcu, struct blkg_policy_data, rcu_head);
struct throtl_grp *tg = pd_to_tg(pd);
- blkg_rwstat_exit(&tg->stat_bytes);
- blkg_rwstat_exit(&tg->stat_ios);
kfree(tg);
}
@@ -1413,32 +1399,6 @@ static ssize_t tg_set_conf_uint(struct kernfs_open_file *of,
return tg_set_conf(of, buf, nbytes, off, false);
}
-static int tg_print_rwstat(struct seq_file *sf, void *v)
-{
- blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
- blkg_prfill_rwstat, &blkcg_policy_throtl,
- seq_cft(sf)->private, true);
- return 0;
-}
-
-static u64 tg_prfill_rwstat_recursive(struct seq_file *sf,
- struct blkg_policy_data *pd, int off)
-{
- struct blkg_rwstat_sample sum;
-
- blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_throtl, off,
- &sum);
- return __blkg_prfill_rwstat(sf, pd, &sum);
-}
-
-static int tg_print_rwstat_recursive(struct seq_file *sf, void *v)
-{
- blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
- tg_prfill_rwstat_recursive, &blkcg_policy_throtl,
- seq_cft(sf)->private, true);
- return 0;
-}
-
static struct cftype throtl_legacy_files[] = {
{
.name = "throttle.read_bps_device",
@@ -1464,26 +1424,6 @@ static struct cftype throtl_legacy_files[] = {
.seq_show = tg_print_conf_uint,
.write = tg_set_conf_uint,
},
- {
- .name = "throttle.io_service_bytes",
- .private = offsetof(struct throtl_grp, stat_bytes),
- .seq_show = tg_print_rwstat,
- },
- {
- .name = "throttle.io_service_bytes_recursive",
- .private = offsetof(struct throtl_grp, stat_bytes),
- .seq_show = tg_print_rwstat_recursive,
- },
- {
- .name = "throttle.io_serviced",
- .private = offsetof(struct throtl_grp, stat_ios),
- .seq_show = tg_print_rwstat,
- },
- {
- .name = "throttle.io_serviced_recursive",
- .private = offsetof(struct throtl_grp, stat_ios),
- .seq_show = tg_print_rwstat_recursive,
- },
{ } /* terminate */
};
@@ -1672,25 +1612,8 @@ static void tg_flush_bios(struct throtl_grp *tg)
static void throtl_pd_offline(struct blkg_policy_data *pd)
{
struct throtl_grp *tg = pd_to_tg(pd);
- struct throtl_grp *parent = blkg_to_tg(tg_to_blkg(tg)->parent);
tg_flush_bios(tg);
-
- /* the recursive files of the ancestors go on counting what it did */
- if (parent) {
- blkg_rwstat_add_aux(&parent->stat_bytes, &tg->stat_bytes);
- blkg_rwstat_add_aux(&parent->stat_ios, &tg->stat_ios);
- blkg_rwstat_reset(&tg->stat_bytes);
- blkg_rwstat_reset(&tg->stat_ios);
- }
-}
-
-static void throtl_pd_reset_stats(struct blkg_policy_data *pd)
-{
- struct throtl_grp *tg = pd_to_tg(pd);
-
- blkg_rwstat_reset(&tg->stat_bytes);
- blkg_rwstat_reset(&tg->stat_ios);
}
struct blkcg_policy blkcg_policy_throtl = {
@@ -1702,7 +1625,6 @@ struct blkcg_policy blkcg_policy_throtl = {
.pd_online_fn = throtl_pd_online,
.pd_offline_fn = throtl_pd_offline,
.pd_free_fn = throtl_pd_free,
- .pd_reset_stats_fn = throtl_pd_reset_stats,
};
static void tg_cancel_writeback_bios(struct throtl_grp *tg,
diff --git a/block/blk-throttle.h b/block/blk-throttle.h
index 1b5775771e07..02c08daf3dc3 100644
--- a/block/blk-throttle.h
+++ b/block/blk-throttle.h
@@ -2,7 +2,7 @@
#ifndef BLK_THROTTLE_H
#define BLK_THROTTLE_H
-#include "blk-cgroup-rwstat.h"
+#include "blk-cgroup.h"
/*
* To implement hierarchical throttling, throtl_grps form a tree and bios
@@ -124,8 +124,6 @@ struct throtl_grp {
unsigned long slice_start[2];
unsigned long slice_end[2];
- struct blkg_rwstat stat_bytes;
- struct blkg_rwstat stat_ios;
};
extern struct blkcg_policy blkcg_policy_throtl;
@@ -172,14 +170,7 @@ static inline bool blk_should_throtl(struct bio *bio)
return false;
tg = blkg_to_tg(bio->bi_blkg);
- if (!cgroup_subsys_on_dfl(io_cgrp_subsys)) {
- if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
- bio_set_flag(bio, BIO_CGROUP_ACCT);
- blkg_rwstat_add(&tg->stat_bytes, bio->bi_opf,
- bio->bi_iter.bi_size);
- }
- blkg_rwstat_add(&tg->stat_ios, bio->bi_opf, 1);
- }
+ blkcg_account_v1_bio(bio);
/* iops limit is always counted */
if (tg->has_rules_iops[rw])
--
2.25.1
^ permalink raw reply [flat|nested] 5+ messages in thread