24 	struct percpu_counter *fbc = addr;  in percpu_counter_fixup_free()  local
28 		percpu_counter_destroy(fbc);  in percpu_counter_fixup_free()
29 		debug_object_free(fbc, &percpu_counter_debug_descr);  in percpu_counter_fixup_free()
41 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)  in debug_percpu_counter_activate()  argument
43 	debug_object_init(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_activate()
44 	debug_object_activate(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_activate()
47 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)  in debug_percpu_counter_deactivate()  argument
49 	debug_object_deactivate(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_deactivate()
50 	debug_object_free(fbc, &percpu_counter_debug_descr);  in debug_percpu_counter_deactivate()
54 static inline void debug_percpu_counter_activate(struct percpu_counter *fbc)  in debug_percpu_counter_activate()  argument
56 static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc)  in debug_percpu_counter_deactivate()  argument
60 void percpu_counter_set(struct percpu_counter *fbc, s64 amount)  in percpu_counter_set()  argument
65 	raw_spin_lock_irqsave(&fbc->lock, flags);  in percpu_counter_set()
67 		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);  in percpu_counter_set()
70 	fbc->count = amount;  in percpu_counter_set()
71 	raw_spin_unlock_irqrestore(&fbc->lock, flags);  in percpu_counter_set()
82 void percpu_counter_add_batch(struct percpu_counter *fbc, s64 amount, s32 batch)  in percpu_counter_add_batch()  argument
87 	count = __this_cpu_read(*fbc->counters) + amount;  in percpu_counter_add_batch()
90 		raw_spin_lock_irqsave(&fbc->lock, flags);  in percpu_counter_add_batch()
91 		fbc->count += count;  in percpu_counter_add_batch()
92 		__this_cpu_sub(*fbc->counters, count - amount);  in percpu_counter_add_batch()
93 		raw_spin_unlock_irqrestore(&fbc->lock, flags);  in percpu_counter_add_batch()
95 		this_cpu_add(*fbc->counters, amount);  in percpu_counter_add_batch()
107 void percpu_counter_sync(struct percpu_counter *fbc)  in percpu_counter_sync()  argument
112 	raw_spin_lock_irqsave(&fbc->lock, flags);  in percpu_counter_sync()
113 	count = __this_cpu_read(*fbc->counters);  in percpu_counter_sync()
114 	fbc->count += count;  in percpu_counter_sync()
115 	__this_cpu_sub(*fbc->counters, count);  in percpu_counter_sync()
116 	raw_spin_unlock_irqrestore(&fbc->lock, flags);  in percpu_counter_sync()
124 s64 __percpu_counter_sum(struct percpu_counter *fbc)  in __percpu_counter_sum()  argument
130 	raw_spin_lock_irqsave(&fbc->lock, flags);  in __percpu_counter_sum()
131 	ret = fbc->count;  in __percpu_counter_sum()
133 		s32 *pcount = per_cpu_ptr(fbc->counters, cpu);  in __percpu_counter_sum()
136 	raw_spin_unlock_irqrestore(&fbc->lock, flags);  in __percpu_counter_sum()
141 int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, gfp_t gfp,  in __percpu_counter_init()  argument
146 	raw_spin_lock_init(&fbc->lock);  in __percpu_counter_init()
147 	lockdep_set_class(&fbc->lock, key);  in __percpu_counter_init()
148 	fbc->count = amount;  in __percpu_counter_init()
149 	fbc->counters = alloc_percpu_gfp(s32, gfp);  in __percpu_counter_init()
150 	if (!fbc->counters)  in __percpu_counter_init()
153 	debug_percpu_counter_activate(fbc);  in __percpu_counter_init()
156 	INIT_LIST_HEAD(&fbc->list);  in __percpu_counter_init()
158 	list_add(&fbc->list, &percpu_counters);  in __percpu_counter_init()
165 void percpu_counter_destroy(struct percpu_counter *fbc)  in percpu_counter_destroy()  argument
169 	if (!fbc->counters)  in percpu_counter_destroy()
172 	debug_percpu_counter_deactivate(fbc);  in percpu_counter_destroy()
176 	list_del(&fbc->list);  in percpu_counter_destroy()
179 	free_percpu(fbc->counters);  in percpu_counter_destroy()
180 	fbc->counters = NULL;  in percpu_counter_destroy()
198 	struct percpu_counter *fbc;  in percpu_counter_cpu_dead()  local
203 	list_for_each_entry(fbc, &percpu_counters, list) {  in percpu_counter_cpu_dead()
206 		raw_spin_lock(&fbc->lock);  in percpu_counter_cpu_dead()
207 		pcount = per_cpu_ptr(fbc->counters, cpu);  in percpu_counter_cpu_dead()
208 		fbc->count += *pcount;  in percpu_counter_cpu_dead()
210 		raw_spin_unlock(&fbc->lock);  in percpu_counter_cpu_dead()
221 int __percpu_counter_compare(struct percpu_counter *fbc, s64 rhs, s32 batch)  in __percpu_counter_compare()  argument
225 	count = percpu_counter_read(fbc);  in __percpu_counter_compare()
234 	count = percpu_counter_sum(fbc);  in __percpu_counter_compare()