1*4882a593Smuzhiyun // SPDX-License-Identifier: GPL-2.0
2*4882a593Smuzhiyun #include <linux/slab.h>
3*4882a593Smuzhiyun #include <linux/spinlock.h>
4*4882a593Smuzhiyun #include <linux/once.h>
5*4882a593Smuzhiyun #include <linux/random.h>
6*4882a593Smuzhiyun
7*4882a593Smuzhiyun struct once_work {
8*4882a593Smuzhiyun struct work_struct work;
9*4882a593Smuzhiyun struct static_key_true *key;
10*4882a593Smuzhiyun };
11*4882a593Smuzhiyun
once_deferred(struct work_struct * w)12*4882a593Smuzhiyun static void once_deferred(struct work_struct *w)
13*4882a593Smuzhiyun {
14*4882a593Smuzhiyun struct once_work *work;
15*4882a593Smuzhiyun
16*4882a593Smuzhiyun work = container_of(w, struct once_work, work);
17*4882a593Smuzhiyun BUG_ON(!static_key_enabled(work->key));
18*4882a593Smuzhiyun static_branch_disable(work->key);
19*4882a593Smuzhiyun kfree(work);
20*4882a593Smuzhiyun }
21*4882a593Smuzhiyun
once_disable_jump(struct static_key_true * key)22*4882a593Smuzhiyun static void once_disable_jump(struct static_key_true *key)
23*4882a593Smuzhiyun {
24*4882a593Smuzhiyun struct once_work *w;
25*4882a593Smuzhiyun
26*4882a593Smuzhiyun w = kmalloc(sizeof(*w), GFP_ATOMIC);
27*4882a593Smuzhiyun if (!w)
28*4882a593Smuzhiyun return;
29*4882a593Smuzhiyun
30*4882a593Smuzhiyun INIT_WORK(&w->work, once_deferred);
31*4882a593Smuzhiyun w->key = key;
32*4882a593Smuzhiyun schedule_work(&w->work);
33*4882a593Smuzhiyun }
34*4882a593Smuzhiyun
35*4882a593Smuzhiyun static DEFINE_SPINLOCK(once_lock);
36*4882a593Smuzhiyun
__do_once_start(bool * done,unsigned long * flags)37*4882a593Smuzhiyun bool __do_once_start(bool *done, unsigned long *flags)
38*4882a593Smuzhiyun __acquires(once_lock)
39*4882a593Smuzhiyun {
40*4882a593Smuzhiyun spin_lock_irqsave(&once_lock, *flags);
41*4882a593Smuzhiyun if (*done) {
42*4882a593Smuzhiyun spin_unlock_irqrestore(&once_lock, *flags);
43*4882a593Smuzhiyun /* Keep sparse happy by restoring an even lock count on
44*4882a593Smuzhiyun * this lock. In case we return here, we don't call into
45*4882a593Smuzhiyun * __do_once_done but return early in the DO_ONCE() macro.
46*4882a593Smuzhiyun */
47*4882a593Smuzhiyun __acquire(once_lock);
48*4882a593Smuzhiyun return false;
49*4882a593Smuzhiyun }
50*4882a593Smuzhiyun
51*4882a593Smuzhiyun return true;
52*4882a593Smuzhiyun }
53*4882a593Smuzhiyun EXPORT_SYMBOL(__do_once_start);
54*4882a593Smuzhiyun
__do_once_done(bool * done,struct static_key_true * once_key,unsigned long * flags)55*4882a593Smuzhiyun void __do_once_done(bool *done, struct static_key_true *once_key,
56*4882a593Smuzhiyun unsigned long *flags)
57*4882a593Smuzhiyun __releases(once_lock)
58*4882a593Smuzhiyun {
59*4882a593Smuzhiyun *done = true;
60*4882a593Smuzhiyun spin_unlock_irqrestore(&once_lock, *flags);
61*4882a593Smuzhiyun once_disable_jump(once_key);
62*4882a593Smuzhiyun }
63*4882a593Smuzhiyun EXPORT_SYMBOL(__do_once_done);
64*4882a593Smuzhiyun
65*4882a593Smuzhiyun static DEFINE_MUTEX(once_mutex);
66*4882a593Smuzhiyun
__do_once_slow_start(bool * done)67*4882a593Smuzhiyun bool __do_once_slow_start(bool *done)
68*4882a593Smuzhiyun __acquires(once_mutex)
69*4882a593Smuzhiyun {
70*4882a593Smuzhiyun mutex_lock(&once_mutex);
71*4882a593Smuzhiyun if (*done) {
72*4882a593Smuzhiyun mutex_unlock(&once_mutex);
73*4882a593Smuzhiyun /* Keep sparse happy by restoring an even lock count on
74*4882a593Smuzhiyun * this mutex. In case we return here, we don't call into
75*4882a593Smuzhiyun * __do_once_done but return early in the DO_ONCE_SLOW() macro.
76*4882a593Smuzhiyun */
77*4882a593Smuzhiyun __acquire(once_mutex);
78*4882a593Smuzhiyun return false;
79*4882a593Smuzhiyun }
80*4882a593Smuzhiyun
81*4882a593Smuzhiyun return true;
82*4882a593Smuzhiyun }
83*4882a593Smuzhiyun EXPORT_SYMBOL(__do_once_slow_start);
84*4882a593Smuzhiyun
__do_once_slow_done(bool * done,struct static_key_true * once_key,struct module * mod)85*4882a593Smuzhiyun void __do_once_slow_done(bool *done, struct static_key_true *once_key,
86*4882a593Smuzhiyun struct module *mod)
87*4882a593Smuzhiyun __releases(once_mutex)
88*4882a593Smuzhiyun {
89*4882a593Smuzhiyun *done = true;
90*4882a593Smuzhiyun mutex_unlock(&once_mutex);
91*4882a593Smuzhiyun once_disable_jump(once_key);
92*4882a593Smuzhiyun }
93*4882a593Smuzhiyun EXPORT_SYMBOL(__do_once_slow_done);
94