block: inherit CPU completion on bio->rq and rq->rq merges

Somewhat incomplete, as we do allow merges of requests and bios
that have different completion CPUs given. This is done on the
assumption that a larger IO is still more beneficial than CPU
locality.

Signed-off-by: Jens Axboe <jens.axboe@oracle.com>

diff --git a/block/blk-core.c b/block/blk-core.c
index 5484838f46e76a6b90ea88290c7086097a16ef81..b9a252cae4df224332b220cb75ad34ee5c83641c 100644 (file)
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -1134,6 +1134,8 @@ static int __make_request(struct request_queue *q, struct bio *bio)
                req->biotail = bio;
                req->nr_sectors = req->hard_nr_sectors += nr_sectors;
                req->ioprio = ioprio_best(req->ioprio, prio);
+               if (!blk_rq_cpu_valid(req))
+                       req->cpu = bio->bi_comp_cpu;
                drive_stat_acct(req, 0);
                if (!attempt_back_merge(q, req))
                        elv_merged_request(q, req, el_ret);
@@ -1161,6 +1163,8 @@ static int __make_request(struct request_queue *q, struct bio *bio)
                req->sector = req->hard_sector = bio->bi_sector;
                req->nr_sectors = req->hard_nr_sectors += nr_sectors;
                req->ioprio = ioprio_best(req->ioprio, prio);
+               if (!blk_rq_cpu_valid(req))
+                       req->cpu = bio->bi_comp_cpu;
                drive_stat_acct(req, 0);
                if (!attempt_front_merge(q, req))
                        elv_merged_request(q, req, el_ret);

diff --git a/block/blk-merge.c b/block/blk-merge.c
index c77196d55899729ec6ddceccc80ef44b69f24f42..908d3e11ac523e032f3f73a91932eb49f1a5994a 100644 (file)
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -400,6 +400,8 @@ static int attempt_merge(struct request_queue *q, struct request *req,
        }
 
        req->ioprio = ioprio_best(req->ioprio, next->ioprio);
+       if (blk_rq_cpu_valid(next))
+               req->cpu = next->cpu;
 
        __blk_put_request(q, next);
        return 1;

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 93204bf7b297dc0630f41271f0602f98186dac28..12df8efeef1960288b3d1846be828d5d361eddca 100644 (file)
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -545,6 +545,7 @@ enum {
 #define blk_pm_request(rq)     \
        (blk_pm_suspend_request(rq) || blk_pm_resume_request(rq))
 
+#define blk_rq_cpu_valid(rq)   ((rq)->cpu != -1)
 #define blk_sorted_rq(rq)      ((rq)->cmd_flags & REQ_SORTED)
 #define blk_barrier_rq(rq)     ((rq)->cmd_flags & REQ_HARDBARRIER)
 #define blk_fua_rq(rq)         ((rq)->cmd_flags & REQ_FUA)