1*4882a593Smuzhiyun /* SPDX-License-Identifier: GPL-2.0-only */
2*4882a593Smuzhiyun /*
3*4882a593Smuzhiyun * padata.h - header for the padata parallelization interface
4*4882a593Smuzhiyun *
5*4882a593Smuzhiyun * Copyright (C) 2008, 2009 secunet Security Networks AG
6*4882a593Smuzhiyun * Copyright (C) 2008, 2009 Steffen Klassert <steffen.klassert@secunet.com>
7*4882a593Smuzhiyun *
8*4882a593Smuzhiyun * Copyright (c) 2020 Oracle and/or its affiliates.
9*4882a593Smuzhiyun * Author: Daniel Jordan <daniel.m.jordan@oracle.com>
10*4882a593Smuzhiyun */
11*4882a593Smuzhiyun
12*4882a593Smuzhiyun #ifndef PADATA_H
13*4882a593Smuzhiyun #define PADATA_H
14*4882a593Smuzhiyun
15*4882a593Smuzhiyun #include <linux/compiler_types.h>
16*4882a593Smuzhiyun #include <linux/workqueue.h>
17*4882a593Smuzhiyun #include <linux/spinlock.h>
18*4882a593Smuzhiyun #include <linux/list.h>
19*4882a593Smuzhiyun #include <linux/kobject.h>
20*4882a593Smuzhiyun
21*4882a593Smuzhiyun #define PADATA_CPU_SERIAL 0x01
22*4882a593Smuzhiyun #define PADATA_CPU_PARALLEL 0x02
23*4882a593Smuzhiyun
24*4882a593Smuzhiyun /**
25*4882a593Smuzhiyun * struct padata_priv - Represents one job
26*4882a593Smuzhiyun *
27*4882a593Smuzhiyun * @list: List entry, to attach to the padata lists.
28*4882a593Smuzhiyun * @pd: Pointer to the internal control structure.
29*4882a593Smuzhiyun * @cb_cpu: Callback cpu for serializatioon.
30*4882a593Smuzhiyun * @seq_nr: Sequence number of the parallelized data object.
31*4882a593Smuzhiyun * @info: Used to pass information from the parallel to the serial function.
32*4882a593Smuzhiyun * @parallel: Parallel execution function.
33*4882a593Smuzhiyun * @serial: Serial complete function.
34*4882a593Smuzhiyun */
35*4882a593Smuzhiyun struct padata_priv {
36*4882a593Smuzhiyun struct list_head list;
37*4882a593Smuzhiyun struct parallel_data *pd;
38*4882a593Smuzhiyun int cb_cpu;
39*4882a593Smuzhiyun unsigned int seq_nr;
40*4882a593Smuzhiyun int info;
41*4882a593Smuzhiyun void (*parallel)(struct padata_priv *padata);
42*4882a593Smuzhiyun void (*serial)(struct padata_priv *padata);
43*4882a593Smuzhiyun };
44*4882a593Smuzhiyun
45*4882a593Smuzhiyun /**
46*4882a593Smuzhiyun * struct padata_list - one per work type per CPU
47*4882a593Smuzhiyun *
48*4882a593Smuzhiyun * @list: List head.
49*4882a593Smuzhiyun * @lock: List lock.
50*4882a593Smuzhiyun */
51*4882a593Smuzhiyun struct padata_list {
52*4882a593Smuzhiyun struct list_head list;
53*4882a593Smuzhiyun spinlock_t lock;
54*4882a593Smuzhiyun };
55*4882a593Smuzhiyun
56*4882a593Smuzhiyun /**
57*4882a593Smuzhiyun * struct padata_serial_queue - The percpu padata serial queue
58*4882a593Smuzhiyun *
59*4882a593Smuzhiyun * @serial: List to wait for serialization after reordering.
60*4882a593Smuzhiyun * @work: work struct for serialization.
61*4882a593Smuzhiyun * @pd: Backpointer to the internal control structure.
62*4882a593Smuzhiyun */
63*4882a593Smuzhiyun struct padata_serial_queue {
64*4882a593Smuzhiyun struct padata_list serial;
65*4882a593Smuzhiyun struct work_struct work;
66*4882a593Smuzhiyun struct parallel_data *pd;
67*4882a593Smuzhiyun };
68*4882a593Smuzhiyun
69*4882a593Smuzhiyun /**
70*4882a593Smuzhiyun * struct padata_cpumask - The cpumasks for the parallel/serial workers
71*4882a593Smuzhiyun *
72*4882a593Smuzhiyun * @pcpu: cpumask for the parallel workers.
73*4882a593Smuzhiyun * @cbcpu: cpumask for the serial (callback) workers.
74*4882a593Smuzhiyun */
75*4882a593Smuzhiyun struct padata_cpumask {
76*4882a593Smuzhiyun cpumask_var_t pcpu;
77*4882a593Smuzhiyun cpumask_var_t cbcpu;
78*4882a593Smuzhiyun };
79*4882a593Smuzhiyun
80*4882a593Smuzhiyun /**
81*4882a593Smuzhiyun * struct parallel_data - Internal control structure, covers everything
82*4882a593Smuzhiyun * that depends on the cpumask in use.
83*4882a593Smuzhiyun *
84*4882a593Smuzhiyun * @ps: padata_shell object.
85*4882a593Smuzhiyun * @reorder_list: percpu reorder lists
86*4882a593Smuzhiyun * @squeue: percpu padata queues used for serialuzation.
87*4882a593Smuzhiyun * @refcnt: Number of objects holding a reference on this parallel_data.
88*4882a593Smuzhiyun * @seq_nr: Sequence number of the parallelized data object.
89*4882a593Smuzhiyun * @processed: Number of already processed objects.
90*4882a593Smuzhiyun * @cpu: Next CPU to be processed.
91*4882a593Smuzhiyun * @cpumask: The cpumasks in use for parallel and serial workers.
92*4882a593Smuzhiyun * @reorder_work: work struct for reordering.
93*4882a593Smuzhiyun * @lock: Reorder lock.
94*4882a593Smuzhiyun */
95*4882a593Smuzhiyun struct parallel_data {
96*4882a593Smuzhiyun struct padata_shell *ps;
97*4882a593Smuzhiyun struct padata_list __percpu *reorder_list;
98*4882a593Smuzhiyun struct padata_serial_queue __percpu *squeue;
99*4882a593Smuzhiyun atomic_t refcnt;
100*4882a593Smuzhiyun unsigned int seq_nr;
101*4882a593Smuzhiyun unsigned int processed;
102*4882a593Smuzhiyun int cpu;
103*4882a593Smuzhiyun struct padata_cpumask cpumask;
104*4882a593Smuzhiyun struct work_struct reorder_work;
105*4882a593Smuzhiyun spinlock_t ____cacheline_aligned lock;
106*4882a593Smuzhiyun };
107*4882a593Smuzhiyun
108*4882a593Smuzhiyun /**
109*4882a593Smuzhiyun * struct padata_shell - Wrapper around struct parallel_data, its
110*4882a593Smuzhiyun * purpose is to allow the underlying control structure to be replaced
111*4882a593Smuzhiyun * on the fly using RCU.
112*4882a593Smuzhiyun *
113*4882a593Smuzhiyun * @pinst: padat instance.
114*4882a593Smuzhiyun * @pd: Actual parallel_data structure which may be substituted on the fly.
115*4882a593Smuzhiyun * @opd: Pointer to old pd to be freed by padata_replace.
116*4882a593Smuzhiyun * @list: List entry in padata_instance list.
117*4882a593Smuzhiyun */
118*4882a593Smuzhiyun struct padata_shell {
119*4882a593Smuzhiyun struct padata_instance *pinst;
120*4882a593Smuzhiyun struct parallel_data __rcu *pd;
121*4882a593Smuzhiyun struct parallel_data *opd;
122*4882a593Smuzhiyun struct list_head list;
123*4882a593Smuzhiyun };
124*4882a593Smuzhiyun
125*4882a593Smuzhiyun /**
126*4882a593Smuzhiyun * struct padata_mt_job - represents one multithreaded job
127*4882a593Smuzhiyun *
128*4882a593Smuzhiyun * @thread_fn: Called for each chunk of work that a padata thread does.
129*4882a593Smuzhiyun * @fn_arg: The thread function argument.
130*4882a593Smuzhiyun * @start: The start of the job (units are job-specific).
131*4882a593Smuzhiyun * @size: size of this node's work (units are job-specific).
132*4882a593Smuzhiyun * @align: Ranges passed to the thread function fall on this boundary, with the
133*4882a593Smuzhiyun * possible exceptions of the beginning and end of the job.
134*4882a593Smuzhiyun * @min_chunk: The minimum chunk size in job-specific units. This allows
135*4882a593Smuzhiyun * the client to communicate the minimum amount of work that's
136*4882a593Smuzhiyun * appropriate for one worker thread to do at once.
137*4882a593Smuzhiyun * @max_threads: Max threads to use for the job, actual number may be less
138*4882a593Smuzhiyun * depending on task size and minimum chunk size.
139*4882a593Smuzhiyun */
140*4882a593Smuzhiyun struct padata_mt_job {
141*4882a593Smuzhiyun void (*thread_fn)(unsigned long start, unsigned long end, void *arg);
142*4882a593Smuzhiyun void *fn_arg;
143*4882a593Smuzhiyun unsigned long start;
144*4882a593Smuzhiyun unsigned long size;
145*4882a593Smuzhiyun unsigned long align;
146*4882a593Smuzhiyun unsigned long min_chunk;
147*4882a593Smuzhiyun int max_threads;
148*4882a593Smuzhiyun };
149*4882a593Smuzhiyun
150*4882a593Smuzhiyun /**
151*4882a593Smuzhiyun * struct padata_instance - The overall control structure.
152*4882a593Smuzhiyun *
153*4882a593Smuzhiyun * @cpu_online_node: Linkage for CPU online callback.
154*4882a593Smuzhiyun * @cpu_dead_node: Linkage for CPU offline callback.
155*4882a593Smuzhiyun * @parallel_wq: The workqueue used for parallel work.
156*4882a593Smuzhiyun * @serial_wq: The workqueue used for serial work.
157*4882a593Smuzhiyun * @pslist: List of padata_shell objects attached to this instance.
158*4882a593Smuzhiyun * @cpumask: User supplied cpumasks for parallel and serial works.
159*4882a593Smuzhiyun * @kobj: padata instance kernel object.
160*4882a593Smuzhiyun * @lock: padata instance lock.
161*4882a593Smuzhiyun * @flags: padata flags.
162*4882a593Smuzhiyun */
163*4882a593Smuzhiyun struct padata_instance {
164*4882a593Smuzhiyun struct hlist_node cpu_online_node;
165*4882a593Smuzhiyun struct hlist_node cpu_dead_node;
166*4882a593Smuzhiyun struct workqueue_struct *parallel_wq;
167*4882a593Smuzhiyun struct workqueue_struct *serial_wq;
168*4882a593Smuzhiyun struct list_head pslist;
169*4882a593Smuzhiyun struct padata_cpumask cpumask;
170*4882a593Smuzhiyun struct kobject kobj;
171*4882a593Smuzhiyun struct mutex lock;
172*4882a593Smuzhiyun u8 flags;
173*4882a593Smuzhiyun #define PADATA_INIT 1
174*4882a593Smuzhiyun #define PADATA_RESET 2
175*4882a593Smuzhiyun #define PADATA_INVALID 4
176*4882a593Smuzhiyun };
177*4882a593Smuzhiyun
178*4882a593Smuzhiyun #ifdef CONFIG_PADATA
179*4882a593Smuzhiyun extern void __init padata_init(void);
180*4882a593Smuzhiyun #else
padata_init(void)181*4882a593Smuzhiyun static inline void __init padata_init(void) {}
182*4882a593Smuzhiyun #endif
183*4882a593Smuzhiyun
184*4882a593Smuzhiyun extern struct padata_instance *padata_alloc(const char *name);
185*4882a593Smuzhiyun extern void padata_free(struct padata_instance *pinst);
186*4882a593Smuzhiyun extern struct padata_shell *padata_alloc_shell(struct padata_instance *pinst);
187*4882a593Smuzhiyun extern void padata_free_shell(struct padata_shell *ps);
188*4882a593Smuzhiyun extern int padata_do_parallel(struct padata_shell *ps,
189*4882a593Smuzhiyun struct padata_priv *padata, int *cb_cpu);
190*4882a593Smuzhiyun extern void padata_do_serial(struct padata_priv *padata);
191*4882a593Smuzhiyun extern void __init padata_do_multithreaded(struct padata_mt_job *job);
192*4882a593Smuzhiyun extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,
193*4882a593Smuzhiyun cpumask_var_t cpumask);
194*4882a593Smuzhiyun #endif
195