1*4882a593Smuzhiyun // SPDX-License-Identifier: GPL-2.0-only
2*4882a593Smuzhiyun /*
3*4882a593Smuzhiyun * Copyright (C) Gao Xiang <xiang@kernel.org>
4*4882a593Smuzhiyun *
5*4882a593Smuzhiyun * For low-latency decompression algorithms (e.g. lz4), reserve consecutive
6*4882a593Smuzhiyun * per-CPU virtual memory (in pages) in advance to store such inplace I/O
7*4882a593Smuzhiyun * data if inplace decompression is failed (due to unmet inplace margin for
8*4882a593Smuzhiyun * example).
9*4882a593Smuzhiyun */
10*4882a593Smuzhiyun #include "internal.h"
11*4882a593Smuzhiyun
12*4882a593Smuzhiyun struct erofs_pcpubuf {
13*4882a593Smuzhiyun raw_spinlock_t lock;
14*4882a593Smuzhiyun void *ptr;
15*4882a593Smuzhiyun struct page **pages;
16*4882a593Smuzhiyun unsigned int nrpages;
17*4882a593Smuzhiyun };
18*4882a593Smuzhiyun
19*4882a593Smuzhiyun static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb);
20*4882a593Smuzhiyun
erofs_get_pcpubuf(unsigned int requiredpages)21*4882a593Smuzhiyun void *erofs_get_pcpubuf(unsigned int requiredpages)
22*4882a593Smuzhiyun __acquires(pcb->lock)
23*4882a593Smuzhiyun {
24*4882a593Smuzhiyun struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb);
25*4882a593Smuzhiyun
26*4882a593Smuzhiyun raw_spin_lock(&pcb->lock);
27*4882a593Smuzhiyun /* check if the per-CPU buffer is too small */
28*4882a593Smuzhiyun if (requiredpages > pcb->nrpages) {
29*4882a593Smuzhiyun raw_spin_unlock(&pcb->lock);
30*4882a593Smuzhiyun put_cpu_var(erofs_pcb);
31*4882a593Smuzhiyun /* (for sparse checker) pretend pcb->lock is still taken */
32*4882a593Smuzhiyun __acquire(pcb->lock);
33*4882a593Smuzhiyun return NULL;
34*4882a593Smuzhiyun }
35*4882a593Smuzhiyun return pcb->ptr;
36*4882a593Smuzhiyun }
37*4882a593Smuzhiyun
erofs_put_pcpubuf(void * ptr)38*4882a593Smuzhiyun void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock)
39*4882a593Smuzhiyun {
40*4882a593Smuzhiyun struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id());
41*4882a593Smuzhiyun
42*4882a593Smuzhiyun DBG_BUGON(pcb->ptr != ptr);
43*4882a593Smuzhiyun raw_spin_unlock(&pcb->lock);
44*4882a593Smuzhiyun put_cpu_var(erofs_pcb);
45*4882a593Smuzhiyun }
46*4882a593Smuzhiyun
47*4882a593Smuzhiyun /* the next step: support per-CPU page buffers hotplug */
erofs_pcpubuf_growsize(unsigned int nrpages)48*4882a593Smuzhiyun int erofs_pcpubuf_growsize(unsigned int nrpages)
49*4882a593Smuzhiyun {
50*4882a593Smuzhiyun static DEFINE_MUTEX(pcb_resize_mutex);
51*4882a593Smuzhiyun static unsigned int pcb_nrpages;
52*4882a593Smuzhiyun LIST_HEAD(pagepool);
53*4882a593Smuzhiyun int delta, cpu, ret, i;
54*4882a593Smuzhiyun
55*4882a593Smuzhiyun mutex_lock(&pcb_resize_mutex);
56*4882a593Smuzhiyun delta = nrpages - pcb_nrpages;
57*4882a593Smuzhiyun ret = 0;
58*4882a593Smuzhiyun /* avoid shrinking pcpubuf, since no idea how many fses rely on */
59*4882a593Smuzhiyun if (delta <= 0)
60*4882a593Smuzhiyun goto out;
61*4882a593Smuzhiyun
62*4882a593Smuzhiyun for_each_possible_cpu(cpu) {
63*4882a593Smuzhiyun struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
64*4882a593Smuzhiyun struct page **pages, **oldpages;
65*4882a593Smuzhiyun void *ptr, *old_ptr;
66*4882a593Smuzhiyun
67*4882a593Smuzhiyun pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL);
68*4882a593Smuzhiyun if (!pages) {
69*4882a593Smuzhiyun ret = -ENOMEM;
70*4882a593Smuzhiyun break;
71*4882a593Smuzhiyun }
72*4882a593Smuzhiyun
73*4882a593Smuzhiyun for (i = 0; i < nrpages; ++i) {
74*4882a593Smuzhiyun pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL);
75*4882a593Smuzhiyun if (!pages[i]) {
76*4882a593Smuzhiyun ret = -ENOMEM;
77*4882a593Smuzhiyun oldpages = pages;
78*4882a593Smuzhiyun goto free_pagearray;
79*4882a593Smuzhiyun }
80*4882a593Smuzhiyun }
81*4882a593Smuzhiyun ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL);
82*4882a593Smuzhiyun if (!ptr) {
83*4882a593Smuzhiyun ret = -ENOMEM;
84*4882a593Smuzhiyun oldpages = pages;
85*4882a593Smuzhiyun goto free_pagearray;
86*4882a593Smuzhiyun }
87*4882a593Smuzhiyun raw_spin_lock(&pcb->lock);
88*4882a593Smuzhiyun old_ptr = pcb->ptr;
89*4882a593Smuzhiyun pcb->ptr = ptr;
90*4882a593Smuzhiyun oldpages = pcb->pages;
91*4882a593Smuzhiyun pcb->pages = pages;
92*4882a593Smuzhiyun i = pcb->nrpages;
93*4882a593Smuzhiyun pcb->nrpages = nrpages;
94*4882a593Smuzhiyun raw_spin_unlock(&pcb->lock);
95*4882a593Smuzhiyun
96*4882a593Smuzhiyun if (!oldpages) {
97*4882a593Smuzhiyun DBG_BUGON(old_ptr);
98*4882a593Smuzhiyun continue;
99*4882a593Smuzhiyun }
100*4882a593Smuzhiyun
101*4882a593Smuzhiyun if (old_ptr)
102*4882a593Smuzhiyun vunmap(old_ptr);
103*4882a593Smuzhiyun free_pagearray:
104*4882a593Smuzhiyun while (i)
105*4882a593Smuzhiyun list_add(&oldpages[--i]->lru, &pagepool);
106*4882a593Smuzhiyun kfree(oldpages);
107*4882a593Smuzhiyun if (ret)
108*4882a593Smuzhiyun break;
109*4882a593Smuzhiyun }
110*4882a593Smuzhiyun pcb_nrpages = nrpages;
111*4882a593Smuzhiyun put_pages_list(&pagepool);
112*4882a593Smuzhiyun out:
113*4882a593Smuzhiyun mutex_unlock(&pcb_resize_mutex);
114*4882a593Smuzhiyun return ret;
115*4882a593Smuzhiyun }
116*4882a593Smuzhiyun
erofs_pcpubuf_init(void)117*4882a593Smuzhiyun void erofs_pcpubuf_init(void)
118*4882a593Smuzhiyun {
119*4882a593Smuzhiyun int cpu;
120*4882a593Smuzhiyun
121*4882a593Smuzhiyun for_each_possible_cpu(cpu) {
122*4882a593Smuzhiyun struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
123*4882a593Smuzhiyun
124*4882a593Smuzhiyun raw_spin_lock_init(&pcb->lock);
125*4882a593Smuzhiyun }
126*4882a593Smuzhiyun }
127*4882a593Smuzhiyun
erofs_pcpubuf_exit(void)128*4882a593Smuzhiyun void erofs_pcpubuf_exit(void)
129*4882a593Smuzhiyun {
130*4882a593Smuzhiyun int cpu, i;
131*4882a593Smuzhiyun
132*4882a593Smuzhiyun for_each_possible_cpu(cpu) {
133*4882a593Smuzhiyun struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
134*4882a593Smuzhiyun
135*4882a593Smuzhiyun if (pcb->ptr) {
136*4882a593Smuzhiyun vunmap(pcb->ptr);
137*4882a593Smuzhiyun pcb->ptr = NULL;
138*4882a593Smuzhiyun }
139*4882a593Smuzhiyun if (!pcb->pages)
140*4882a593Smuzhiyun continue;
141*4882a593Smuzhiyun
142*4882a593Smuzhiyun for (i = 0; i < pcb->nrpages; ++i)
143*4882a593Smuzhiyun if (pcb->pages[i])
144*4882a593Smuzhiyun put_page(pcb->pages[i]);
145*4882a593Smuzhiyun kfree(pcb->pages);
146*4882a593Smuzhiyun pcb->pages = NULL;
147*4882a593Smuzhiyun }
148*4882a593Smuzhiyun }
149