1*4882a593Smuzhiyun // SPDX-License-Identifier: GPL-2.0-only
2*4882a593Smuzhiyun /*
3*4882a593Smuzhiyun * mm/percpu-km.c - kernel memory based chunk allocation
4*4882a593Smuzhiyun *
5*4882a593Smuzhiyun * Copyright (C) 2010 SUSE Linux Products GmbH
6*4882a593Smuzhiyun * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
7*4882a593Smuzhiyun *
8*4882a593Smuzhiyun * Chunks are allocated as a contiguous kernel memory using gfp
9*4882a593Smuzhiyun * allocation. This is to be used on nommu architectures.
10*4882a593Smuzhiyun *
11*4882a593Smuzhiyun * To use percpu-km,
12*4882a593Smuzhiyun *
13*4882a593Smuzhiyun * - define CONFIG_NEED_PER_CPU_KM from the arch Kconfig.
14*4882a593Smuzhiyun *
15*4882a593Smuzhiyun * - CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK must not be defined. It's
16*4882a593Smuzhiyun * not compatible with PER_CPU_KM. EMBED_FIRST_CHUNK should work
17*4882a593Smuzhiyun * fine.
18*4882a593Smuzhiyun *
19*4882a593Smuzhiyun * - NUMA is not supported. When setting up the first chunk,
20*4882a593Smuzhiyun * @cpu_distance_fn should be NULL or report all CPUs to be nearer
21*4882a593Smuzhiyun * than or at LOCAL_DISTANCE.
22*4882a593Smuzhiyun *
23*4882a593Smuzhiyun * - It's best if the chunk size is power of two multiple of
24*4882a593Smuzhiyun * PAGE_SIZE. Because each chunk is allocated as a contiguous
25*4882a593Smuzhiyun * kernel memory block using alloc_pages(), memory will be wasted if
26*4882a593Smuzhiyun * chunk size is not aligned. percpu-km code will whine about it.
27*4882a593Smuzhiyun */
28*4882a593Smuzhiyun
29*4882a593Smuzhiyun #if defined(CONFIG_SMP) && defined(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK)
30*4882a593Smuzhiyun #error "contiguous percpu allocation is incompatible with paged first chunk"
31*4882a593Smuzhiyun #endif
32*4882a593Smuzhiyun
33*4882a593Smuzhiyun #include <linux/log2.h>
34*4882a593Smuzhiyun
pcpu_populate_chunk(struct pcpu_chunk * chunk,int page_start,int page_end,gfp_t gfp)35*4882a593Smuzhiyun static int pcpu_populate_chunk(struct pcpu_chunk *chunk,
36*4882a593Smuzhiyun int page_start, int page_end, gfp_t gfp)
37*4882a593Smuzhiyun {
38*4882a593Smuzhiyun return 0;
39*4882a593Smuzhiyun }
40*4882a593Smuzhiyun
pcpu_depopulate_chunk(struct pcpu_chunk * chunk,int page_start,int page_end)41*4882a593Smuzhiyun static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk,
42*4882a593Smuzhiyun int page_start, int page_end)
43*4882a593Smuzhiyun {
44*4882a593Smuzhiyun /* nada */
45*4882a593Smuzhiyun }
46*4882a593Smuzhiyun
pcpu_create_chunk(enum pcpu_chunk_type type,gfp_t gfp)47*4882a593Smuzhiyun static struct pcpu_chunk *pcpu_create_chunk(enum pcpu_chunk_type type,
48*4882a593Smuzhiyun gfp_t gfp)
49*4882a593Smuzhiyun {
50*4882a593Smuzhiyun const int nr_pages = pcpu_group_sizes[0] >> PAGE_SHIFT;
51*4882a593Smuzhiyun struct pcpu_chunk *chunk;
52*4882a593Smuzhiyun struct page *pages;
53*4882a593Smuzhiyun unsigned long flags;
54*4882a593Smuzhiyun int i;
55*4882a593Smuzhiyun
56*4882a593Smuzhiyun chunk = pcpu_alloc_chunk(type, gfp);
57*4882a593Smuzhiyun if (!chunk)
58*4882a593Smuzhiyun return NULL;
59*4882a593Smuzhiyun
60*4882a593Smuzhiyun pages = alloc_pages(gfp, order_base_2(nr_pages));
61*4882a593Smuzhiyun if (!pages) {
62*4882a593Smuzhiyun pcpu_free_chunk(chunk);
63*4882a593Smuzhiyun return NULL;
64*4882a593Smuzhiyun }
65*4882a593Smuzhiyun
66*4882a593Smuzhiyun for (i = 0; i < nr_pages; i++)
67*4882a593Smuzhiyun pcpu_set_page_chunk(nth_page(pages, i), chunk);
68*4882a593Smuzhiyun
69*4882a593Smuzhiyun chunk->data = pages;
70*4882a593Smuzhiyun chunk->base_addr = page_address(pages);
71*4882a593Smuzhiyun
72*4882a593Smuzhiyun spin_lock_irqsave(&pcpu_lock, flags);
73*4882a593Smuzhiyun pcpu_chunk_populated(chunk, 0, nr_pages);
74*4882a593Smuzhiyun spin_unlock_irqrestore(&pcpu_lock, flags);
75*4882a593Smuzhiyun
76*4882a593Smuzhiyun pcpu_stats_chunk_alloc();
77*4882a593Smuzhiyun trace_percpu_create_chunk(chunk->base_addr);
78*4882a593Smuzhiyun
79*4882a593Smuzhiyun return chunk;
80*4882a593Smuzhiyun }
81*4882a593Smuzhiyun
pcpu_destroy_chunk(struct pcpu_chunk * chunk)82*4882a593Smuzhiyun static void pcpu_destroy_chunk(struct pcpu_chunk *chunk)
83*4882a593Smuzhiyun {
84*4882a593Smuzhiyun const int nr_pages = pcpu_group_sizes[0] >> PAGE_SHIFT;
85*4882a593Smuzhiyun
86*4882a593Smuzhiyun if (!chunk)
87*4882a593Smuzhiyun return;
88*4882a593Smuzhiyun
89*4882a593Smuzhiyun pcpu_stats_chunk_dealloc();
90*4882a593Smuzhiyun trace_percpu_destroy_chunk(chunk->base_addr);
91*4882a593Smuzhiyun
92*4882a593Smuzhiyun if (chunk->data)
93*4882a593Smuzhiyun __free_pages(chunk->data, order_base_2(nr_pages));
94*4882a593Smuzhiyun pcpu_free_chunk(chunk);
95*4882a593Smuzhiyun }
96*4882a593Smuzhiyun
pcpu_addr_to_page(void * addr)97*4882a593Smuzhiyun static struct page *pcpu_addr_to_page(void *addr)
98*4882a593Smuzhiyun {
99*4882a593Smuzhiyun return virt_to_page(addr);
100*4882a593Smuzhiyun }
101*4882a593Smuzhiyun
pcpu_verify_alloc_info(const struct pcpu_alloc_info * ai)102*4882a593Smuzhiyun static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai)
103*4882a593Smuzhiyun {
104*4882a593Smuzhiyun size_t nr_pages, alloc_pages;
105*4882a593Smuzhiyun
106*4882a593Smuzhiyun /* all units must be in a single group */
107*4882a593Smuzhiyun if (ai->nr_groups != 1) {
108*4882a593Smuzhiyun pr_crit("can't handle more than one group\n");
109*4882a593Smuzhiyun return -EINVAL;
110*4882a593Smuzhiyun }
111*4882a593Smuzhiyun
112*4882a593Smuzhiyun nr_pages = (ai->groups[0].nr_units * ai->unit_size) >> PAGE_SHIFT;
113*4882a593Smuzhiyun alloc_pages = roundup_pow_of_two(nr_pages);
114*4882a593Smuzhiyun
115*4882a593Smuzhiyun if (alloc_pages > nr_pages)
116*4882a593Smuzhiyun pr_warn("wasting %zu pages per chunk\n",
117*4882a593Smuzhiyun alloc_pages - nr_pages);
118*4882a593Smuzhiyun
119*4882a593Smuzhiyun return 0;
120*4882a593Smuzhiyun }
121