* [PATCH v4 1/2] blk-mq: move data->rq_flags initialization code
2023-06-29 12:13 [PATCH v4 0/2] blk-mq: fix start_time_ns and alloc_time_ns for pre-allocated rq chengming.zhou
@ 2023-06-29 12:13 ` chengming.zhou
2023-06-29 12:13 ` [PATCH v4 2/2] blk-mq: fix start_time_ns and alloc_time_ns for pre-allocated rq chengming.zhou
1 sibling, 0 replies; 3+ messages in thread
From: chengming.zhou @ 2023-06-29 12:13 UTC (permalink / raw)
To: axboe, ming.lei, hch, tj; +Cc: linux-block, linux-kernel, Chengming Zhou
From: Chengming Zhou <zhouchengming@bytedance.com>
The blk_mq_rq_ctx_init() has some data->rq_flags initialization:
```
if (data->flags & BLK_MQ_REQ_PM)
data->rq_flags |= RQF_PM;
if (blk_queue_io_stat(q))
data->rq_flags |= RQF_IO_STAT;
rq->rq_flags = data->rq_flags;
```
We will need this data->rq_flags to tell if we need start_time_ns, so
we need to put these initialization in the callers of blk_mq_rq_ctx_init().
Now we basically have two callers, the 1st is general __blk_mq_alloc_requests(),
the 2nd is the special blk_mq_alloc_request_hctx(). So change the 2nd caller
to reuse the 1st __blk_mq_alloc_requests() to avoid code duplication.
This is safe because blk_mq_alloc_request_hctx() always has BLK_MQ_REQ_NOWAIT
flag, it won't go into the normal retry path when doesn't have free tag.
But it indeed make the general __blk_mq_alloc_requests() more complex.
Signed-off-by: Chengming Zhou <zhouchengming@bytedance.com>
---
block/blk-mq.c | 46 ++++++++++++++++++----------------------------
1 file changed, 18 insertions(+), 28 deletions(-)
diff --git a/block/blk-mq.c b/block/blk-mq.c
index decb6ab2d508..c50ef953759f 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -349,11 +349,6 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
rq->mq_ctx = ctx;
rq->mq_hctx = hctx;
rq->cmd_flags = data->cmd_flags;
-
- if (data->flags & BLK_MQ_REQ_PM)
- data->rq_flags |= RQF_PM;
- if (blk_queue_io_stat(q))
- data->rq_flags |= RQF_IO_STAT;
rq->rq_flags = data->rq_flags;
if (data->rq_flags & RQF_SCHED_TAGS) {
@@ -447,6 +442,15 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
if (data->cmd_flags & REQ_NOWAIT)
data->flags |= BLK_MQ_REQ_NOWAIT;
+ if (data->flags & BLK_MQ_REQ_RESERVED)
+ data->rq_flags |= RQF_RESV;
+
+ if (data->flags & BLK_MQ_REQ_PM)
+ data->rq_flags |= RQF_PM;
+
+ if (blk_queue_io_stat(q))
+ data->rq_flags |= RQF_IO_STAT;
+
if (q->elevator) {
/*
* All requests use scheduler tags when an I/O scheduler is
@@ -471,14 +475,15 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
}
retry:
- data->ctx = blk_mq_get_ctx(q);
- data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx);
+ /* See blk_mq_alloc_request_hctx() for details */
+ if (!data->ctx) {
+ data->ctx = blk_mq_get_ctx(q);
+ data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx);
+ }
+
if (!(data->rq_flags & RQF_SCHED_TAGS))
blk_mq_tag_busy(data->hctx);
- if (data->flags & BLK_MQ_REQ_RESERVED)
- data->rq_flags |= RQF_RESV;
-
/*
* Try batched alloc if we want more than 1 tag.
*/
@@ -505,6 +510,7 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
* is going away.
*/
msleep(3);
+ data->ctx = NULL;
goto retry;
}
@@ -613,16 +619,10 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
.cmd_flags = opf,
.nr_tags = 1,
};
- u64 alloc_time_ns = 0;
struct request *rq;
unsigned int cpu;
- unsigned int tag;
int ret;
- /* alloc_time includes depth and tag waits */
- if (blk_queue_rq_alloc_time(q))
- alloc_time_ns = ktime_get_ns();
-
/*
* If the tag allocator sleeps we could get an allocation for a
* different hardware context. No need to complicate the low level
@@ -653,20 +653,10 @@ struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
goto out_queue_exit;
data.ctx = __blk_mq_get_ctx(q, cpu);
- if (q->elevator)
- data.rq_flags |= RQF_SCHED_TAGS;
- else
- blk_mq_tag_busy(data.hctx);
-
- if (flags & BLK_MQ_REQ_RESERVED)
- data.rq_flags |= RQF_RESV;
-
ret = -EWOULDBLOCK;
- tag = blk_mq_get_tag(&data);
- if (tag == BLK_MQ_NO_TAG)
+ rq = __blk_mq_alloc_requests(&data);
+ if (!rq)
goto out_queue_exit;
- rq = blk_mq_rq_ctx_init(&data, blk_mq_tags_from_data(&data), tag,
- alloc_time_ns);
rq->__data_len = 0;
rq->__sector = (sector_t) -1;
rq->bio = rq->biotail = NULL;
--
2.39.2
^ permalink raw reply [flat|nested] 3+ messages in thread* [PATCH v4 2/2] blk-mq: fix start_time_ns and alloc_time_ns for pre-allocated rq
2023-06-29 12:13 [PATCH v4 0/2] blk-mq: fix start_time_ns and alloc_time_ns for pre-allocated rq chengming.zhou
2023-06-29 12:13 ` [PATCH v4 1/2] blk-mq: move data->rq_flags initialization code chengming.zhou
@ 2023-06-29 12:13 ` chengming.zhou
1 sibling, 0 replies; 3+ messages in thread
From: chengming.zhou @ 2023-06-29 12:13 UTC (permalink / raw)
To: axboe, ming.lei, hch, tj; +Cc: linux-block, linux-kernel, Chengming Zhou
From: Chengming Zhou <zhouchengming@bytedance.com>
The iocost rely on rq start_time_ns and alloc_time_ns to tell saturation
state of the block device. Most of the time request is allocated after
rq_qos_throttle() and its alloc_time_ns or start_time_ns won't be affected.
But for plug batched allocation introduced by the commit 47c122e35d7e
("block: pre-allocate requests if plug is started and is a batch"), we can
rq_qos_throttle() after the allocation of the request. This is what the
blk_mq_get_cached_request() does.
In this case, the cached request alloc_time_ns or start_time_ns is much
ahead if blocked in any qos ->throttle().
This patch fix it by setting alloc_time_ns and start_time_ns to now
when the pre-allocated rq is actually used. And we skip setting the
alloc_time_ns and start_time_ns during pre-allocation, so just pass 0
in __blk_mq_alloc_requests_batch().
Signed-off-by: Chengming Zhou <zhouchengming@bytedance.com>
---
block/blk-mq.c | 41 +++++++++++++++++++++++++++++++----------
block/blk-mq.h | 1 +
include/linux/blk-mq.h | 6 +++---
3 files changed, 35 insertions(+), 13 deletions(-)
diff --git a/block/blk-mq.c b/block/blk-mq.c
index c50ef953759f..2849395ca4c6 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -337,6 +337,24 @@ void blk_rq_init(struct request_queue *q, struct request *rq)
}
EXPORT_SYMBOL(blk_rq_init);
+/* Set alloc and start time when pre-allocated rq is actually used */
+static inline void blk_mq_rq_time_init(struct request *rq)
+{
+ if (blk_mq_need_time_stamp(rq->rq_flags)) {
+ u64 now = ktime_get_ns();
+
+#ifdef CONFIG_BLK_RQ_ALLOC_TIME
+ /*
+ * The alloc time is only used by iocost for now,
+ * only possible when blk_mq_need_time_stamp().
+ */
+ if (blk_queue_rq_alloc_time(rq->q))
+ rq->alloc_time_ns = now;
+#endif
+ rq->start_time_ns = now;
+ }
+}
+
static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
struct blk_mq_tags *tags, unsigned int tag, u64 alloc_time_ns)
{
@@ -360,14 +378,11 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
}
rq->timeout = 0;
- if (blk_mq_need_time_stamp(rq))
- rq->start_time_ns = ktime_get_ns();
- else
- rq->start_time_ns = 0;
rq->part = NULL;
#ifdef CONFIG_BLK_RQ_ALLOC_TIME
rq->alloc_time_ns = alloc_time_ns;
#endif
+ rq->start_time_ns = data->start_time_ns;
rq->io_start_time_ns = 0;
rq->stats_sectors = 0;
rq->nr_phys_segments = 0;
@@ -397,8 +412,7 @@ static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
}
static inline struct request *
-__blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data,
- u64 alloc_time_ns)
+__blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data)
{
unsigned int tag, tag_offset;
struct blk_mq_tags *tags;
@@ -417,7 +431,7 @@ __blk_mq_alloc_requests_batch(struct blk_mq_alloc_data *data,
tag = tag_offset + i;
prefetch(tags->static_rqs[tag]);
tag_mask &= ~(1UL << i);
- rq = blk_mq_rq_ctx_init(data, tags, tag, alloc_time_ns);
+ rq = blk_mq_rq_ctx_init(data, tags, tag, 0);
rq_list_add(data->cached_rq, rq);
nr++;
}
@@ -488,9 +502,11 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
* Try batched alloc if we want more than 1 tag.
*/
if (data->nr_tags > 1) {
- rq = __blk_mq_alloc_requests_batch(data, alloc_time_ns);
- if (rq)
+ rq = __blk_mq_alloc_requests_batch(data);
+ if (rq) {
+ blk_mq_rq_time_init(rq);
return rq;
+ }
data->nr_tags = 1;
}
@@ -514,6 +530,9 @@ static struct request *__blk_mq_alloc_requests(struct blk_mq_alloc_data *data)
goto retry;
}
+ if (blk_mq_need_time_stamp(data->rq_flags))
+ data->start_time_ns = ktime_get_ns();
+
return blk_mq_rq_ctx_init(data, blk_mq_tags_from_data(data), tag,
alloc_time_ns);
}
@@ -570,6 +589,7 @@ static struct request *blk_mq_alloc_cached_request(struct request_queue *q,
return NULL;
plug->cached_rq = rq_list_next(rq);
+ blk_mq_rq_time_init(rq);
}
rq->cmd_flags = opf;
@@ -1004,7 +1024,7 @@ static inline void __blk_mq_end_request_acct(struct request *rq, u64 now)
inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
{
- if (blk_mq_need_time_stamp(rq))
+ if (blk_mq_need_time_stamp(rq->rq_flags))
__blk_mq_end_request_acct(rq, ktime_get_ns());
if (rq->end_io) {
@@ -2891,6 +2911,7 @@ static inline struct request *blk_mq_get_cached_request(struct request_queue *q,
plug->cached_rq = rq_list_next(rq);
rq_qos_throttle(q, *bio);
+ blk_mq_rq_time_init(rq);
rq->cmd_flags = (*bio)->bi_opf;
INIT_LIST_HEAD(&rq->queuelist);
return rq;
diff --git a/block/blk-mq.h b/block/blk-mq.h
index 1743857e0b01..29ed4d05110e 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -150,6 +150,7 @@ struct blk_mq_alloc_data {
unsigned int shallow_depth;
blk_opf_t cmd_flags;
req_flags_t rq_flags;
+ u64 start_time_ns;
/* allocate multiple requests/tags in one go */
unsigned int nr_tags;
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index f401067ac03a..e8366e9c3388 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -834,9 +834,9 @@ void blk_mq_end_request_batch(struct io_comp_batch *ib);
* Only need start/end time stamping if we have iostat or
* blk stats enabled, or using an IO scheduler.
*/
-static inline bool blk_mq_need_time_stamp(struct request *rq)
+static inline bool blk_mq_need_time_stamp(req_flags_t rq_flags)
{
- return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_USE_SCHED));
+ return (rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_USE_SCHED));
}
static inline bool blk_mq_is_reserved_rq(struct request *rq)
@@ -860,7 +860,7 @@ static inline bool blk_mq_add_to_batch(struct request *req,
iob->complete = complete;
else if (iob->complete != complete)
return false;
- iob->need_ts |= blk_mq_need_time_stamp(req);
+ iob->need_ts |= blk_mq_need_time_stamp(req->rq_flags);
rq_list_add(&iob->req_list, req);
return true;
}
--
2.39.2
^ permalink raw reply [flat|nested] 3+ messages in thread