From 428d2843010a655fd5c6bc2a3d3089b5ff4c8ce4 Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Mon, 31 Aug 2026 19:01:41 +0800 Subject: [PATCH 1/4] block/mq-deadline: reject zero prio_aging_expire A prio_aging_expire of zero does not disable I/O priority in mq-deadline. Instead the priority aging path in dd_dispatch_prio_aged_requests() is invoked with "now - 0 == now", which causes best-effort and idle requests to be dispatched ahead of pending real-time requests -- a classic priority inversion, not the "priority disabled" behavior users may expect when writing zero. Reject zero (and negative) values in the sysfs store with -EINVAL so that a misconfiguration is reported rather than silently accepted. Signed-off-by: Ye Bin --- block/mq-deadline.c | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 5f643c0ce2a8..a4358625cfc8 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -31,7 +31,10 @@ static const int read_expire = HZ / 2; /* max time before a read is submitted. static const int write_expire = 5 * HZ; /* ditto for writes, these limits are SOFT! */ /* * Time after which to dispatch lower priority requests even if higher - * priority requests are pending. + * priority requests are pending. Must be > 0: a value of zero would make + * the priority aging path dispatch best-effort and idle requests ahead of + * pending real-time requests through "now - 0 == now", a classic priority + * inversion. */ static const int prio_aging_expire = 10 * HZ; static const int writes_starved = 2; /* max times reads can starve a write */ @@ -770,7 +773,6 @@ static ssize_t __FUNC(struct elevator_queue *e, const char *page, size_t count) STORE_FUNCTION(__FUNC, __PTR, MIN, MAX, msecs_to_jiffies) STORE_JIFFIES(deadline_read_expire_store, &dd->fifo_expire[DD_READ], 0, INT_MAX); STORE_JIFFIES(deadline_write_expire_store, &dd->fifo_expire[DD_WRITE], 0, INT_MAX); -STORE_JIFFIES(deadline_prio_aging_expire_store, &dd->prio_aging_expire, 0, INT_MAX); STORE_INT(deadline_writes_starved_store, &dd->writes_starved, INT_MIN, INT_MAX); STORE_INT(deadline_front_merges_store, &dd->front_merges, 0, 1); STORE_INT(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX); @@ -778,6 +780,28 @@ STORE_INT(deadline_fifo_batch_store, &dd->fifo_batch, 0, INT_MAX); #undef STORE_INT #undef STORE_JIFFIES +/* + * prio_aging_expire must be positive: a value of zero would make the + * priority aging path dispatch best-effort and idle requests ahead of + * pending real-time requests through "now - 0 == now", a classic priority + * inversion. Reject zero and negative values instead of clamping, so that + * a misconfiguration is reported rather than silently accepted. + */ +static ssize_t deadline_prio_aging_expire_store(struct elevator_queue *e, + const char *page, size_t count) +{ + struct deadline_data *dd = e->elevator_data; + int val, ret; + + ret = kstrtoint(page, 0, &val); + if (ret < 0) + return ret; + if (val <= 0) + return -EINVAL; + dd->prio_aging_expire = msecs_to_jiffies(val); + return count; +} + #define DD_ATTR(name) \ __ATTR(name, 0644, deadline_##name##_show, deadline_##name##_store) From 70aa523bd6097878c432330a1eb65cad94dc27e2 Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Mon, 31 Aug 2026 19:01:42 +0800 Subject: [PATCH 2/4] block/mq-deadline: add prio_enable switch for I/O priority control Since mq-deadline introduced support for I/O priorities, processes without an explicit I/O priority are bound to their scheduling priority. This forces applications to plan their I/O priorities, but many applications do not care about I/O priorities and have no way to disable the distinction. Add a per-queue prio_enable boolean to control whether I/O priority (RT/BE/IDLE) support is active. When disabled, every request is filed in the best-effort bucket and the priority aging path is bypassed. To avoid priority inversion while toggling the switch at runtime, the sysfs store follows the same sequence as elevator_switch(): freeze the queue, quiesce, flip the flag, then unquiesce and unfreeze. This drains in-flight I/O so that requests already queued in RT or IDLE buckets complete before the mode changes. Since requests may now be filed in a bucket that does not match their ioprio, update dd_start_request(), dd_request_merged() and dd_merged_requests() to look up the per-priority bucket from rq->elv.priv[0] instead of recomputing it from the request ioprio. Remove the now-unused dd_rq_ioclass() helper. Signed-off-by: Ye Bin --- block/mq-deadline.c | 109 +++++++++++++++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 27 deletions(-) diff --git a/block/mq-deadline.c b/block/mq-deadline.c index a4358625cfc8..276ce249d326 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -86,6 +86,7 @@ struct deadline_data { * run time data */ + struct request_queue *q; /* associated request queue */ struct list_head dispatch; struct dd_per_prio per_prio[DD_PRIO_COUNT]; @@ -102,6 +103,7 @@ struct deadline_data { int writes_starved; int front_merges; int prio_aging_expire; + bool prio_enable; spinlock_t lock; }; @@ -120,15 +122,6 @@ deadline_rb_root(struct dd_per_prio *per_prio, struct request *rq) return &per_prio->sort_list[rq_data_dir(rq)]; } -/* - * Returns the I/O priority class (IOPRIO_CLASS_*) that has been assigned to a - * request. - */ -static u8 dd_rq_ioclass(struct request *rq) -{ - return IOPRIO_PRIO_CLASS(req_get_ioprio(rq)); -} - /* * Return the first request for which blk_rq_pos() >= @pos. */ @@ -187,10 +180,7 @@ static void deadline_remove_request(struct request_queue *q, static void dd_request_merged(struct request_queue *q, struct request *req, enum elv_merge type) { - struct deadline_data *dd = q->elevator->elevator_data; - const u8 ioprio_class = dd_rq_ioclass(req); - const enum dd_prio prio = ioprio_class_to_prio[ioprio_class]; - struct dd_per_prio *per_prio = &dd->per_prio[prio]; + struct dd_per_prio *per_prio = req->elv.priv[0]; /* * if the merge was a front merge, we need to reposition request @@ -208,12 +198,11 @@ static void dd_merged_requests(struct request_queue *q, struct request *req, struct request *next) { struct deadline_data *dd = q->elevator->elevator_data; - const u8 ioprio_class = dd_rq_ioclass(next); - const enum dd_prio prio = ioprio_class_to_prio[ioprio_class]; + struct dd_per_prio *per_prio = next->elv.priv[0]; lockdep_assert_held(&dd->lock); - dd->per_prio[prio].stats.merged++; + per_prio->stats.merged++; /* * if next expires before rq, assign its expire time to rq @@ -230,7 +219,7 @@ static void dd_merged_requests(struct request_queue *q, struct request *req, /* * kill knowledge of next, this one is a goner */ - deadline_remove_request(q, &dd->per_prio[prio], next); + deadline_remove_request(q, per_prio, next); } /* @@ -305,15 +294,13 @@ static bool started_after(struct deadline_data *dd, struct request *rq, return time_after(start_time, latest_start); } -static struct request *dd_start_request(struct deadline_data *dd, - enum dd_data_dir data_dir, +static struct request *dd_start_request(enum dd_data_dir data_dir, struct request *rq) { - u8 ioprio_class = dd_rq_ioclass(rq); - enum dd_prio prio = ioprio_class_to_prio[ioprio_class]; + struct dd_per_prio *per_prio = rq->elv.priv[0]; - dd->per_prio[prio].latest_pos[data_dir] = blk_rq_pos(rq); - dd->per_prio[prio].stats.dispatched++; + per_prio->latest_pos[data_dir] = blk_rq_pos(rq); + per_prio->stats.dispatched++; rq->rq_flags |= RQF_STARTED; return rq; } @@ -410,7 +397,7 @@ static struct request *__dd_dispatch_request(struct deadline_data *dd, */ dd->batching++; deadline_move_request(per_prio, rq); - return dd_start_request(dd, data_dir, rq); + return dd_start_request(data_dir, rq); } /* @@ -426,6 +413,16 @@ static struct request *dd_dispatch_prio_aged_requests(struct deadline_data *dd, lockdep_assert_held(&dd->lock); + /* + * When I/O priority is disabled every request is filed in the + * DD_BE_PRIO bucket, so the priority aging path must be bypassed to + * avoid dispatching best-effort (or idle) requests ahead of pending + * real-time requests through "now - prio_aging_expire", which would + * cause priority inversion. + */ + if (!dd->prio_enable) + return NULL; + prio_cnt = !!dd_queued(dd, DD_RT_PRIO) + !!dd_queued(dd, DD_BE_PRIO) + !!dd_queued(dd, DD_IDLE_PRIO); if (prio_cnt < 2) @@ -461,7 +458,17 @@ static struct request *dd_dispatch_request(struct blk_mq_hw_ctx *hctx) if (!list_empty(&dd->dispatch)) { rq = list_first_entry(&dd->dispatch, struct request, queuelist); list_del_init(&rq->queuelist); - dd_start_request(dd, rq_data_dir(rq), rq); + dd_start_request(rq_data_dir(rq), rq); + goto unlock; + } + + /* + * When I/O priority is disabled every request is filed in the + * best-effort bucket, so skip the priority aging path and the + * multi-priority loop and dispatch directly from that single bucket. + */ + if (!dd->prio_enable) { + rq = __dd_dispatch_request(dd, &dd->per_prio[DD_BE_PRIO], now); goto unlock; } @@ -536,6 +543,7 @@ static int dd_init_sched(struct request_queue *q, struct elevator_queue *eq) return -ENOMEM; eq->elevator_data = dd; + dd->q = q; INIT_LIST_HEAD(&dd->dispatch); for (prio = 0; prio <= DD_PRIO_MAX; prio++) { @@ -553,6 +561,7 @@ static int dd_init_sched(struct request_queue *q, struct elevator_queue *eq) dd->last_dir = DD_WRITE; dd->fifo_batch = fifo_batch; dd->prio_aging_expire = prio_aging_expire; + dd->prio_enable = true; spin_lock_init(&dd->lock); /* We dispatch from request queue wide instead of hw queue */ @@ -573,7 +582,8 @@ static int dd_request_merge(struct request_queue *q, struct request **rq, { struct deadline_data *dd = q->elevator->elevator_data; const u8 ioprio_class = IOPRIO_PRIO_CLASS(bio->bi_ioprio); - const enum dd_prio prio = ioprio_class_to_prio[ioprio_class]; + const enum dd_prio prio = !dd->prio_enable ? DD_BE_PRIO : + ioprio_class_to_prio[ioprio_class]; struct dd_per_prio *per_prio = &dd->per_prio[prio]; sector_t sector = bio_end_sector(bio); struct request *__rq; @@ -633,7 +643,15 @@ static void dd_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq, lockdep_assert_held(&dd->lock); - prio = ioprio_class_to_prio[ioprio_class]; + /* + * When I/O priority is disabled, file every request in the best-effort + * bucket so that the dispatch path no longer distinguishes between RT, + * BE and IDLE classes. + */ + if (!dd->prio_enable) + prio = DD_BE_PRIO; + else + prio = ioprio_class_to_prio[ioprio_class]; per_prio = &dd->per_prio[prio]; if (!rq->elv.priv[0]) per_prio->stats.inserted++; @@ -748,6 +766,7 @@ SHOW_JIFFIES(deadline_prio_aging_expire_show, dd->prio_aging_expire); SHOW_INT(deadline_writes_starved_show, dd->writes_starved); SHOW_INT(deadline_front_merges_show, dd->front_merges); SHOW_INT(deadline_fifo_batch_show, dd->fifo_batch); +SHOW_INT(deadline_prio_enable_show, dd->prio_enable); #undef SHOW_INT #undef SHOW_JIFFIES @@ -802,6 +821,41 @@ static ssize_t deadline_prio_aging_expire_store(struct elevator_queue *e, return count; } +/* + * Writing zero to prio_enable disables I/O priority: all requests are + * treated as best-effort. To avoid priority inversion while the mode is + * being switched, first drain all in-flight I/O by following the same + * sequence used by elevator_switch(): freeze the queue so that new + * upper-layer I/O is blocked and all outstanding requests complete, + * quiesce the queue so that no dispatch is in progress, then flip + * prio_enable to false. New I/O queued after the switch lands in the + * best-effort bucket. + */ +static ssize_t deadline_prio_enable_store(struct elevator_queue *e, + const char *page, size_t count) +{ + struct deadline_data *dd = e->elevator_data; + int val, ret; + + ret = kstrtoint(page, 0, &val); + if (ret < 0) + return ret; + if (val < 0) + val = 0; + + if (!!val != dd->prio_enable) { + unsigned int memflags; + + memflags = blk_mq_freeze_queue(dd->q); + blk_mq_quiesce_queue(dd->q); + dd->prio_enable = !!val; + blk_mq_unquiesce_queue(dd->q); + blk_mq_unfreeze_queue(dd->q, memflags); + } + + return count; +} + #define DD_ATTR(name) \ __ATTR(name, 0644, deadline_##name##_show, deadline_##name##_store) @@ -812,6 +866,7 @@ static const struct elv_fs_entry deadline_attrs[] = { DD_ATTR(front_merges), DD_ATTR(fifo_batch), DD_ATTR(prio_aging_expire), + DD_ATTR(prio_enable), __ATTR_NULL }; From fb1d82701138d5616fbae6da2cda42400b699cc6 Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Mon, 31 Aug 2026 19:01:43 +0800 Subject: [PATCH 3/4] block/mq-deadline: add module parameter for prio_enable Allow the default of prio_enable to be overridden at load time: - built-in: mq_deadline.prio_enable=0 on the kernel command line - module: modprobe mq_deadline prio_enable=0 A value of zero disables I/O priority from boot/load: every request is filed in the best-effort bucket and the priority aging path is bypassed, so systems that do not want RT/BE/IDLE distinction can opt out without writing to sysfs after every queue creation. Signed-off-by: Ye Bin --- block/mq-deadline.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 276ce249d326..b30924e0b297 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -37,6 +37,17 @@ static const int write_expire = 5 * HZ; /* ditto for writes, these limits are SO * inversion. */ static const int prio_aging_expire = 10 * HZ; + +/* + * Whether to enable I/O priority support (RT/BE/IDLE distinction). + * When false every request is filed in the best-effort bucket and the + * priority aging path is bypassed, so systems that do not want RT/BE/IDLE + * distinction can opt out. + */ +static bool prio_enable = true; +module_param(prio_enable, bool, 0644); +MODULE_PARM_DESC(prio_enable, + "Enable I/O priority (RT/BE/IDLE); 0 = best-effort only."); static const int writes_starved = 2; /* max times reads can starve a write */ static const int fifo_batch = 16; /* # of sequential requests treated as one by the above parameters. For throughput. */ @@ -561,7 +572,7 @@ static int dd_init_sched(struct request_queue *q, struct elevator_queue *eq) dd->last_dir = DD_WRITE; dd->fifo_batch = fifo_batch; dd->prio_aging_expire = prio_aging_expire; - dd->prio_enable = true; + dd->prio_enable = prio_enable; spin_lock_init(&dd->lock); /* We dispatch from request queue wide instead of hw queue */ From 805dfff9152a16112a96950ef5d76b8254653f87 Mon Sep 17 00:00:00 2001 From: Ye Bin Date: Mon, 31 Aug 2026 19:01:44 +0800 Subject: [PATCH 4/4] docs: block: document prio_enable and prio_aging_expire in deadline-iosched The mq-deadline scheduler exposes two sysfs tunables, prio_enable and prio_aging_expire, that control its I/O priority (RT/BE/IDLE) support, but neither was described in the deadline-iosched documentation. Add sections covering: - prio_enable: enables/disables RT/BE/IDLE distinction; when disabled all requests fall into the best-effort bucket and the priority aging path is bypassed. Switching the value drains in-flight I/O (queue freeze and quiesce) to avoid priority inversion during the transition. Also available as a module parameter. - prio_aging_expire: the time after which a waiting best-effort or idle request may be dispatched despite pending real-time requests, to prevent indefinite starvation. Defaults to 10000 ms; only effective when prio_enable is on and at least two priority buckets are populated. Zero and negative values are rejected with -EINVAL to avoid the "now - 0 == now" priority inversion. Signed-off-by: Ye Bin --- Documentation/block/deadline-iosched.rst | 32 ++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Documentation/block/deadline-iosched.rst b/Documentation/block/deadline-iosched.rst index 9f5c5a4c370e..ffa88cbf2951 100644 --- a/Documentation/block/deadline-iosched.rst +++ b/Documentation/block/deadline-iosched.rst @@ -69,4 +69,36 @@ that comes at basically 0 cost we leave that on. We simply disable the rbtree front sector lookup when the io scheduler merge function is called. +prio_enable (bool) +---------------------- + +Whether to enable I/O priority support that distinguishes real-time (RT), +best-effort (BE) and idle requests. When enabled (the default), requests are +filed into separate per-priority buckets and dispatched in priority order: lower +priority requests are deferred while any higher priority requests are pending, +subject to the prio_aging_expire aging mechanism described below. When disabled, +every request is filed in the best-effort bucket, the priority aging path is +bypassed, and the scheduler dispatches from that single bucket. This lets +systems that do not want RT/BE/IDLE distinction opt out of the extra overhead. +Switching the value drains all in-flight I/O (queue freeze and quiesce) to avoid +priority inversion during the transition. This parameter can also be set at +module load time via the prio_enable module parameter. + + +prio_aging_expire (in ms) +------------------------------ + +To prevent lower priority requests from being starved indefinitely by a steady +stream of higher priority requests, the deadline scheduler ages pending +requests. prio_aging_expire is the time after which a best-effort or idle +request that has been waiting longer than this threshold may be dispatched even +though real-time requests are still pending. The default is 10000 ms (10 s). + +This parameter only takes effect when prio_enable is enabled and there are +requests queued in at least two distinct priority buckets. The value must be +positive: zero or negative values are rejected with -EINVAL, since a value of +zero would dispatch best-effort and idle requests ahead of pending real-time +requests through "now - 0 == now", a classic priority inversion. + + Nov 11 2002, Jens Axboe