1*4882a593Smuzhiyun // SPDX-License-Identifier: GPL-2.0
2*4882a593Smuzhiyun /*
3*4882a593Smuzhiyun * CPU <-> hardware queue mapping helpers
4*4882a593Smuzhiyun *
5*4882a593Smuzhiyun * Copyright (C) 2013-2014 Jens Axboe
6*4882a593Smuzhiyun */
7*4882a593Smuzhiyun #include <linux/kernel.h>
8*4882a593Smuzhiyun #include <linux/threads.h>
9*4882a593Smuzhiyun #include <linux/module.h>
10*4882a593Smuzhiyun #include <linux/mm.h>
11*4882a593Smuzhiyun #include <linux/smp.h>
12*4882a593Smuzhiyun #include <linux/cpu.h>
13*4882a593Smuzhiyun
14*4882a593Smuzhiyun #include <linux/blk-mq.h>
15*4882a593Smuzhiyun #include "blk.h"
16*4882a593Smuzhiyun #include "blk-mq.h"
17*4882a593Smuzhiyun
queue_index(struct blk_mq_queue_map * qmap,unsigned int nr_queues,const int q)18*4882a593Smuzhiyun static int queue_index(struct blk_mq_queue_map *qmap,
19*4882a593Smuzhiyun unsigned int nr_queues, const int q)
20*4882a593Smuzhiyun {
21*4882a593Smuzhiyun return qmap->queue_offset + (q % nr_queues);
22*4882a593Smuzhiyun }
23*4882a593Smuzhiyun
get_first_sibling(unsigned int cpu)24*4882a593Smuzhiyun static int get_first_sibling(unsigned int cpu)
25*4882a593Smuzhiyun {
26*4882a593Smuzhiyun unsigned int ret;
27*4882a593Smuzhiyun
28*4882a593Smuzhiyun ret = cpumask_first(topology_sibling_cpumask(cpu));
29*4882a593Smuzhiyun if (ret < nr_cpu_ids)
30*4882a593Smuzhiyun return ret;
31*4882a593Smuzhiyun
32*4882a593Smuzhiyun return cpu;
33*4882a593Smuzhiyun }
34*4882a593Smuzhiyun
blk_mq_map_queues(struct blk_mq_queue_map * qmap)35*4882a593Smuzhiyun int blk_mq_map_queues(struct blk_mq_queue_map *qmap)
36*4882a593Smuzhiyun {
37*4882a593Smuzhiyun unsigned int *map = qmap->mq_map;
38*4882a593Smuzhiyun unsigned int nr_queues = qmap->nr_queues;
39*4882a593Smuzhiyun unsigned int cpu, first_sibling, q = 0;
40*4882a593Smuzhiyun
41*4882a593Smuzhiyun for_each_possible_cpu(cpu)
42*4882a593Smuzhiyun map[cpu] = -1;
43*4882a593Smuzhiyun
44*4882a593Smuzhiyun /*
45*4882a593Smuzhiyun * Spread queues among present CPUs first for minimizing
46*4882a593Smuzhiyun * count of dead queues which are mapped by all un-present CPUs
47*4882a593Smuzhiyun */
48*4882a593Smuzhiyun for_each_present_cpu(cpu) {
49*4882a593Smuzhiyun if (q >= nr_queues)
50*4882a593Smuzhiyun break;
51*4882a593Smuzhiyun map[cpu] = queue_index(qmap, nr_queues, q++);
52*4882a593Smuzhiyun }
53*4882a593Smuzhiyun
54*4882a593Smuzhiyun for_each_possible_cpu(cpu) {
55*4882a593Smuzhiyun if (map[cpu] != -1)
56*4882a593Smuzhiyun continue;
57*4882a593Smuzhiyun /*
58*4882a593Smuzhiyun * First do sequential mapping between CPUs and queues.
59*4882a593Smuzhiyun * In case we still have CPUs to map, and we have some number of
60*4882a593Smuzhiyun * threads per cores then map sibling threads to the same queue
61*4882a593Smuzhiyun * for performance optimizations.
62*4882a593Smuzhiyun */
63*4882a593Smuzhiyun if (q < nr_queues) {
64*4882a593Smuzhiyun map[cpu] = queue_index(qmap, nr_queues, q++);
65*4882a593Smuzhiyun } else {
66*4882a593Smuzhiyun first_sibling = get_first_sibling(cpu);
67*4882a593Smuzhiyun if (first_sibling == cpu)
68*4882a593Smuzhiyun map[cpu] = queue_index(qmap, nr_queues, q++);
69*4882a593Smuzhiyun else
70*4882a593Smuzhiyun map[cpu] = map[first_sibling];
71*4882a593Smuzhiyun }
72*4882a593Smuzhiyun }
73*4882a593Smuzhiyun
74*4882a593Smuzhiyun return 0;
75*4882a593Smuzhiyun }
76*4882a593Smuzhiyun EXPORT_SYMBOL_GPL(blk_mq_map_queues);
77*4882a593Smuzhiyun
78*4882a593Smuzhiyun /**
79*4882a593Smuzhiyun * blk_mq_hw_queue_to_node - Look up the memory node for a hardware queue index
80*4882a593Smuzhiyun * @qmap: CPU to hardware queue map.
81*4882a593Smuzhiyun * @index: hardware queue index.
82*4882a593Smuzhiyun *
83*4882a593Smuzhiyun * We have no quick way of doing reverse lookups. This is only used at
84*4882a593Smuzhiyun * queue init time, so runtime isn't important.
85*4882a593Smuzhiyun */
blk_mq_hw_queue_to_node(struct blk_mq_queue_map * qmap,unsigned int index)86*4882a593Smuzhiyun int blk_mq_hw_queue_to_node(struct blk_mq_queue_map *qmap, unsigned int index)
87*4882a593Smuzhiyun {
88*4882a593Smuzhiyun int i;
89*4882a593Smuzhiyun
90*4882a593Smuzhiyun for_each_possible_cpu(i) {
91*4882a593Smuzhiyun if (index == qmap->mq_map[i])
92*4882a593Smuzhiyun return cpu_to_node(i);
93*4882a593Smuzhiyun }
94*4882a593Smuzhiyun
95*4882a593Smuzhiyun return NUMA_NO_NODE;
96*4882a593Smuzhiyun }
97