1 // SPDX-License-Identifier: GPL-2.0
2
3 #define pr_fmt(fmt) "kcsan: " fmt
4
5 #include <linux/atomic.h>
6 #include <linux/bug.h>
7 #include <linux/delay.h>
8 #include <linux/export.h>
9 #include <linux/init.h>
10 #include <linux/kernel.h>
11 #include <linux/list.h>
12 #include <linux/moduleparam.h>
13 #include <linux/percpu.h>
14 #include <linux/preempt.h>
15 #include <linux/sched.h>
16 #include <linux/uaccess.h>
17
18 #include "atomic.h"
19 #include "encoding.h"
20 #include "kcsan.h"
21
22 static bool kcsan_early_enable = IS_ENABLED(CONFIG_KCSAN_EARLY_ENABLE);
23 unsigned int kcsan_udelay_task = CONFIG_KCSAN_UDELAY_TASK;
24 unsigned int kcsan_udelay_interrupt = CONFIG_KCSAN_UDELAY_INTERRUPT;
25 static long kcsan_skip_watch = CONFIG_KCSAN_SKIP_WATCH;
26 static bool kcsan_interrupt_watcher = IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER);
27
28 #ifdef MODULE_PARAM_PREFIX
29 #undef MODULE_PARAM_PREFIX
30 #endif
31 #define MODULE_PARAM_PREFIX "kcsan."
32 module_param_named(early_enable, kcsan_early_enable, bool, 0);
33 module_param_named(udelay_task, kcsan_udelay_task, uint, 0644);
34 module_param_named(udelay_interrupt, kcsan_udelay_interrupt, uint, 0644);
35 module_param_named(skip_watch, kcsan_skip_watch, long, 0644);
36 module_param_named(interrupt_watcher, kcsan_interrupt_watcher, bool, 0444);
37
38 bool kcsan_enabled;
39
40 /* Per-CPU kcsan_ctx for interrupts */
41 static DEFINE_PER_CPU(struct kcsan_ctx, kcsan_cpu_ctx) = {
42 .disable_count = 0,
43 .atomic_next = 0,
44 .atomic_nest_count = 0,
45 .in_flat_atomic = false,
46 .access_mask = 0,
47 .scoped_accesses = {LIST_POISON1, NULL},
48 };
49
50 /*
51 * Helper macros to index into adjacent slots, starting from address slot
52 * itself, followed by the right and left slots.
53 *
54 * The purpose is 2-fold:
55 *
56 * 1. if during insertion the address slot is already occupied, check if
57 * any adjacent slots are free;
58 * 2. accesses that straddle a slot boundary due to size that exceeds a
59 * slot's range may check adjacent slots if any watchpoint matches.
60 *
61 * Note that accesses with very large size may still miss a watchpoint; however,
62 * given this should be rare, this is a reasonable trade-off to make, since this
63 * will avoid:
64 *
65 * 1. excessive contention between watchpoint checks and setup;
66 * 2. larger number of simultaneous watchpoints without sacrificing
67 * performance.
68 *
69 * Example: SLOT_IDX values for KCSAN_CHECK_ADJACENT=1, where i is [0, 1, 2]:
70 *
71 * slot=0: [ 1, 2, 0]
72 * slot=9: [10, 11, 9]
73 * slot=63: [64, 65, 63]
74 */
75 #define SLOT_IDX(slot, i) (slot + ((i + KCSAN_CHECK_ADJACENT) % NUM_SLOTS))
76
77 /*
78 * SLOT_IDX_FAST is used in the fast-path. Not first checking the address's primary
79 * slot (middle) is fine if we assume that races occur rarely. The set of
80 * indices {SLOT_IDX(slot, i) | i in [0, NUM_SLOTS)} is equivalent to
81 * {SLOT_IDX_FAST(slot, i) | i in [0, NUM_SLOTS)}.
82 */
83 #define SLOT_IDX_FAST(slot, i) (slot + i)
84
85 /*
86 * Watchpoints, with each entry encoded as defined in encoding.h: in order to be
87 * able to safely update and access a watchpoint without introducing locking
88 * overhead, we encode each watchpoint as a single atomic long. The initial
89 * zero-initialized state matches INVALID_WATCHPOINT.
90 *
91 * Add NUM_SLOTS-1 entries to account for overflow; this helps avoid having to
92 * use more complicated SLOT_IDX_FAST calculation with modulo in the fast-path.
93 */
94 static atomic_long_t watchpoints[CONFIG_KCSAN_NUM_WATCHPOINTS + NUM_SLOTS-1];
95
96 /*
97 * Instructions to skip watching counter, used in should_watch(). We use a
98 * per-CPU counter to avoid excessive contention.
99 */
100 static DEFINE_PER_CPU(long, kcsan_skip);
101
102 /* For kcsan_prandom_u32_max(). */
103 static DEFINE_PER_CPU(u32, kcsan_rand_state);
104
find_watchpoint(unsigned long addr,size_t size,bool expect_write,long * encoded_watchpoint)105 static __always_inline atomic_long_t *find_watchpoint(unsigned long addr,
106 size_t size,
107 bool expect_write,
108 long *encoded_watchpoint)
109 {
110 const int slot = watchpoint_slot(addr);
111 const unsigned long addr_masked = addr & WATCHPOINT_ADDR_MASK;
112 atomic_long_t *watchpoint;
113 unsigned long wp_addr_masked;
114 size_t wp_size;
115 bool is_write;
116 int i;
117
118 BUILD_BUG_ON(CONFIG_KCSAN_NUM_WATCHPOINTS < NUM_SLOTS);
119
120 for (i = 0; i < NUM_SLOTS; ++i) {
121 watchpoint = &watchpoints[SLOT_IDX_FAST(slot, i)];
122 *encoded_watchpoint = atomic_long_read(watchpoint);
123 if (!decode_watchpoint(*encoded_watchpoint, &wp_addr_masked,
124 &wp_size, &is_write))
125 continue;
126
127 if (expect_write && !is_write)
128 continue;
129
130 /* Check if the watchpoint matches the access. */
131 if (matching_access(wp_addr_masked, wp_size, addr_masked, size))
132 return watchpoint;
133 }
134
135 return NULL;
136 }
137
138 static inline atomic_long_t *
insert_watchpoint(unsigned long addr,size_t size,bool is_write)139 insert_watchpoint(unsigned long addr, size_t size, bool is_write)
140 {
141 const int slot = watchpoint_slot(addr);
142 const long encoded_watchpoint = encode_watchpoint(addr, size, is_write);
143 atomic_long_t *watchpoint;
144 int i;
145
146 /* Check slot index logic, ensuring we stay within array bounds. */
147 BUILD_BUG_ON(SLOT_IDX(0, 0) != KCSAN_CHECK_ADJACENT);
148 BUILD_BUG_ON(SLOT_IDX(0, KCSAN_CHECK_ADJACENT+1) != 0);
149 BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS-1, KCSAN_CHECK_ADJACENT) != ARRAY_SIZE(watchpoints)-1);
150 BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS-1, KCSAN_CHECK_ADJACENT+1) != ARRAY_SIZE(watchpoints) - NUM_SLOTS);
151
152 for (i = 0; i < NUM_SLOTS; ++i) {
153 long expect_val = INVALID_WATCHPOINT;
154
155 /* Try to acquire this slot. */
156 watchpoint = &watchpoints[SLOT_IDX(slot, i)];
157 if (atomic_long_try_cmpxchg_relaxed(watchpoint, &expect_val, encoded_watchpoint))
158 return watchpoint;
159 }
160
161 return NULL;
162 }
163
164 /*
165 * Return true if watchpoint was successfully consumed, false otherwise.
166 *
167 * This may return false if:
168 *
169 * 1. another thread already consumed the watchpoint;
170 * 2. the thread that set up the watchpoint already removed it;
171 * 3. the watchpoint was removed and then re-used.
172 */
173 static __always_inline bool
try_consume_watchpoint(atomic_long_t * watchpoint,long encoded_watchpoint)174 try_consume_watchpoint(atomic_long_t *watchpoint, long encoded_watchpoint)
175 {
176 return atomic_long_try_cmpxchg_relaxed(watchpoint, &encoded_watchpoint, CONSUMED_WATCHPOINT);
177 }
178
179 /* Return true if watchpoint was not touched, false if already consumed. */
consume_watchpoint(atomic_long_t * watchpoint)180 static inline bool consume_watchpoint(atomic_long_t *watchpoint)
181 {
182 return atomic_long_xchg_relaxed(watchpoint, CONSUMED_WATCHPOINT) != CONSUMED_WATCHPOINT;
183 }
184
185 /* Remove the watchpoint -- its slot may be reused after. */
remove_watchpoint(atomic_long_t * watchpoint)186 static inline void remove_watchpoint(atomic_long_t *watchpoint)
187 {
188 atomic_long_set(watchpoint, INVALID_WATCHPOINT);
189 }
190
get_ctx(void)191 static __always_inline struct kcsan_ctx *get_ctx(void)
192 {
193 /*
194 * In interrupts, use raw_cpu_ptr to avoid unnecessary checks, that would
195 * also result in calls that generate warnings in uaccess regions.
196 */
197 return in_task() ? ¤t->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx);
198 }
199
200 /* Check scoped accesses; never inline because this is a slow-path! */
kcsan_check_scoped_accesses(void)201 static noinline void kcsan_check_scoped_accesses(void)
202 {
203 struct kcsan_ctx *ctx = get_ctx();
204 struct list_head *prev_save = ctx->scoped_accesses.prev;
205 struct kcsan_scoped_access *scoped_access;
206
207 ctx->scoped_accesses.prev = NULL; /* Avoid recursion. */
208 list_for_each_entry(scoped_access, &ctx->scoped_accesses, list)
209 __kcsan_check_access(scoped_access->ptr, scoped_access->size, scoped_access->type);
210 ctx->scoped_accesses.prev = prev_save;
211 }
212
213 /* Rules for generic atomic accesses. Called from fast-path. */
214 static __always_inline bool
is_atomic(const volatile void * ptr,size_t size,int type,struct kcsan_ctx * ctx)215 is_atomic(const volatile void *ptr, size_t size, int type, struct kcsan_ctx *ctx)
216 {
217 if (type & KCSAN_ACCESS_ATOMIC)
218 return true;
219
220 /*
221 * Unless explicitly declared atomic, never consider an assertion access
222 * as atomic. This allows using them also in atomic regions, such as
223 * seqlocks, without implicitly changing their semantics.
224 */
225 if (type & KCSAN_ACCESS_ASSERT)
226 return false;
227
228 if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC) &&
229 (type & KCSAN_ACCESS_WRITE) && size <= sizeof(long) &&
230 !(type & KCSAN_ACCESS_COMPOUND) && IS_ALIGNED((unsigned long)ptr, size))
231 return true; /* Assume aligned writes up to word size are atomic. */
232
233 if (ctx->atomic_next > 0) {
234 /*
235 * Because we do not have separate contexts for nested
236 * interrupts, in case atomic_next is set, we simply assume that
237 * the outer interrupt set atomic_next. In the worst case, we
238 * will conservatively consider operations as atomic. This is a
239 * reasonable trade-off to make, since this case should be
240 * extremely rare; however, even if extremely rare, it could
241 * lead to false positives otherwise.
242 */
243 if ((hardirq_count() >> HARDIRQ_SHIFT) < 2)
244 --ctx->atomic_next; /* in task, or outer interrupt */
245 return true;
246 }
247
248 return ctx->atomic_nest_count > 0 || ctx->in_flat_atomic;
249 }
250
251 static __always_inline bool
should_watch(const volatile void * ptr,size_t size,int type,struct kcsan_ctx * ctx)252 should_watch(const volatile void *ptr, size_t size, int type, struct kcsan_ctx *ctx)
253 {
254 /*
255 * Never set up watchpoints when memory operations are atomic.
256 *
257 * Need to check this first, before kcsan_skip check below: (1) atomics
258 * should not count towards skipped instructions, and (2) to actually
259 * decrement kcsan_atomic_next for consecutive instruction stream.
260 */
261 if (is_atomic(ptr, size, type, ctx))
262 return false;
263
264 if (this_cpu_dec_return(kcsan_skip) >= 0)
265 return false;
266
267 /*
268 * NOTE: If we get here, kcsan_skip must always be reset in slow path
269 * via reset_kcsan_skip() to avoid underflow.
270 */
271
272 /* this operation should be watched */
273 return true;
274 }
275
276 /*
277 * Returns a pseudo-random number in interval [0, ep_ro). Simple linear
278 * congruential generator, using constants from "Numerical Recipes".
279 */
kcsan_prandom_u32_max(u32 ep_ro)280 static u32 kcsan_prandom_u32_max(u32 ep_ro)
281 {
282 u32 state = this_cpu_read(kcsan_rand_state);
283
284 state = 1664525 * state + 1013904223;
285 this_cpu_write(kcsan_rand_state, state);
286
287 return state % ep_ro;
288 }
289
reset_kcsan_skip(void)290 static inline void reset_kcsan_skip(void)
291 {
292 long skip_count = kcsan_skip_watch -
293 (IS_ENABLED(CONFIG_KCSAN_SKIP_WATCH_RANDOMIZE) ?
294 kcsan_prandom_u32_max(kcsan_skip_watch) :
295 0);
296 this_cpu_write(kcsan_skip, skip_count);
297 }
298
kcsan_is_enabled(void)299 static __always_inline bool kcsan_is_enabled(void)
300 {
301 return READ_ONCE(kcsan_enabled) && get_ctx()->disable_count == 0;
302 }
303
304 /* Introduce delay depending on context and configuration. */
delay_access(int type)305 static void delay_access(int type)
306 {
307 unsigned int delay = in_task() ? kcsan_udelay_task : kcsan_udelay_interrupt;
308 /* For certain access types, skew the random delay to be longer. */
309 unsigned int skew_delay_order =
310 (type & (KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_ASSERT)) ? 1 : 0;
311
312 delay -= IS_ENABLED(CONFIG_KCSAN_DELAY_RANDOMIZE) ?
313 kcsan_prandom_u32_max(delay >> skew_delay_order) :
314 0;
315 udelay(delay);
316 }
317
kcsan_save_irqtrace(struct task_struct * task)318 void kcsan_save_irqtrace(struct task_struct *task)
319 {
320 #ifdef CONFIG_TRACE_IRQFLAGS
321 task->kcsan_save_irqtrace = task->irqtrace;
322 #endif
323 }
324
kcsan_restore_irqtrace(struct task_struct * task)325 void kcsan_restore_irqtrace(struct task_struct *task)
326 {
327 #ifdef CONFIG_TRACE_IRQFLAGS
328 task->irqtrace = task->kcsan_save_irqtrace;
329 #endif
330 }
331
332 /*
333 * Pull everything together: check_access() below contains the performance
334 * critical operations; the fast-path (including check_access) functions should
335 * all be inlinable by the instrumentation functions.
336 *
337 * The slow-path (kcsan_found_watchpoint, kcsan_setup_watchpoint) are
338 * non-inlinable -- note that, we prefix these with "kcsan_" to ensure they can
339 * be filtered from the stacktrace, as well as give them unique names for the
340 * UACCESS whitelist of objtool. Each function uses user_access_save/restore(),
341 * since they do not access any user memory, but instrumentation is still
342 * emitted in UACCESS regions.
343 */
344
kcsan_found_watchpoint(const volatile void * ptr,size_t size,int type,atomic_long_t * watchpoint,long encoded_watchpoint)345 static noinline void kcsan_found_watchpoint(const volatile void *ptr,
346 size_t size,
347 int type,
348 atomic_long_t *watchpoint,
349 long encoded_watchpoint)
350 {
351 unsigned long flags;
352 bool consumed;
353
354 if (!kcsan_is_enabled())
355 return;
356
357 /*
358 * The access_mask check relies on value-change comparison. To avoid
359 * reporting a race where e.g. the writer set up the watchpoint, but the
360 * reader has access_mask!=0, we have to ignore the found watchpoint.
361 */
362 if (get_ctx()->access_mask != 0)
363 return;
364
365 /*
366 * Consume the watchpoint as soon as possible, to minimize the chances
367 * of !consumed. Consuming the watchpoint must always be guarded by
368 * kcsan_is_enabled() check, as otherwise we might erroneously
369 * triggering reports when disabled.
370 */
371 consumed = try_consume_watchpoint(watchpoint, encoded_watchpoint);
372
373 /* keep this after try_consume_watchpoint */
374 flags = user_access_save();
375
376 if (consumed) {
377 kcsan_save_irqtrace(current);
378 kcsan_report(ptr, size, type, KCSAN_VALUE_CHANGE_MAYBE,
379 KCSAN_REPORT_CONSUMED_WATCHPOINT,
380 watchpoint - watchpoints);
381 kcsan_restore_irqtrace(current);
382 } else {
383 /*
384 * The other thread may not print any diagnostics, as it has
385 * already removed the watchpoint, or another thread consumed
386 * the watchpoint before this thread.
387 */
388 atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_REPORT_RACES]);
389 }
390
391 if ((type & KCSAN_ACCESS_ASSERT) != 0)
392 atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_ASSERT_FAILURES]);
393 else
394 atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_DATA_RACES]);
395
396 user_access_restore(flags);
397 }
398
399 static noinline void
kcsan_setup_watchpoint(const volatile void * ptr,size_t size,int type)400 kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type)
401 {
402 const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0;
403 const bool is_assert = (type & KCSAN_ACCESS_ASSERT) != 0;
404 atomic_long_t *watchpoint;
405 union {
406 u8 _1;
407 u16 _2;
408 u32 _4;
409 u64 _8;
410 } expect_value;
411 unsigned long access_mask;
412 enum kcsan_value_change value_change = KCSAN_VALUE_CHANGE_MAYBE;
413 unsigned long ua_flags = user_access_save();
414 unsigned long irq_flags = 0;
415
416 /*
417 * Always reset kcsan_skip counter in slow-path to avoid underflow; see
418 * should_watch().
419 */
420 reset_kcsan_skip();
421
422 if (!kcsan_is_enabled())
423 goto out;
424
425 /*
426 * Special atomic rules: unlikely to be true, so we check them here in
427 * the slow-path, and not in the fast-path in is_atomic(). Call after
428 * kcsan_is_enabled(), as we may access memory that is not yet
429 * initialized during early boot.
430 */
431 if (!is_assert && kcsan_is_atomic_special(ptr))
432 goto out;
433
434 if (!check_encodable((unsigned long)ptr, size)) {
435 atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_UNENCODABLE_ACCESSES]);
436 goto out;
437 }
438
439 /*
440 * Save and restore the IRQ state trace touched by KCSAN, since KCSAN's
441 * runtime is entered for every memory access, and potentially useful
442 * information is lost if dirtied by KCSAN.
443 */
444 kcsan_save_irqtrace(current);
445 if (!kcsan_interrupt_watcher)
446 local_irq_save(irq_flags);
447
448 watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write);
449 if (watchpoint == NULL) {
450 /*
451 * Out of capacity: the size of 'watchpoints', and the frequency
452 * with which should_watch() returns true should be tweaked so
453 * that this case happens very rarely.
454 */
455 atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_NO_CAPACITY]);
456 goto out_unlock;
457 }
458
459 atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_SETUP_WATCHPOINTS]);
460 atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_USED_WATCHPOINTS]);
461
462 /*
463 * Read the current value, to later check and infer a race if the data
464 * was modified via a non-instrumented access, e.g. from a device.
465 */
466 expect_value._8 = 0;
467 switch (size) {
468 case 1:
469 expect_value._1 = READ_ONCE(*(const u8 *)ptr);
470 break;
471 case 2:
472 expect_value._2 = READ_ONCE(*(const u16 *)ptr);
473 break;
474 case 4:
475 expect_value._4 = READ_ONCE(*(const u32 *)ptr);
476 break;
477 case 8:
478 expect_value._8 = READ_ONCE(*(const u64 *)ptr);
479 break;
480 default:
481 break; /* ignore; we do not diff the values */
482 }
483
484 if (IS_ENABLED(CONFIG_KCSAN_DEBUG)) {
485 kcsan_disable_current();
486 pr_err("watching %s, size: %zu, addr: %px [slot: %d, encoded: %lx]\n",
487 is_write ? "write" : "read", size, ptr,
488 watchpoint_slot((unsigned long)ptr),
489 encode_watchpoint((unsigned long)ptr, size, is_write));
490 kcsan_enable_current();
491 }
492
493 /*
494 * Delay this thread, to increase probability of observing a racy
495 * conflicting access.
496 */
497 delay_access(type);
498
499 /*
500 * Re-read value, and check if it is as expected; if not, we infer a
501 * racy access.
502 */
503 access_mask = get_ctx()->access_mask;
504 switch (size) {
505 case 1:
506 expect_value._1 ^= READ_ONCE(*(const u8 *)ptr);
507 if (access_mask)
508 expect_value._1 &= (u8)access_mask;
509 break;
510 case 2:
511 expect_value._2 ^= READ_ONCE(*(const u16 *)ptr);
512 if (access_mask)
513 expect_value._2 &= (u16)access_mask;
514 break;
515 case 4:
516 expect_value._4 ^= READ_ONCE(*(const u32 *)ptr);
517 if (access_mask)
518 expect_value._4 &= (u32)access_mask;
519 break;
520 case 8:
521 expect_value._8 ^= READ_ONCE(*(const u64 *)ptr);
522 if (access_mask)
523 expect_value._8 &= (u64)access_mask;
524 break;
525 default:
526 break; /* ignore; we do not diff the values */
527 }
528
529 /* Were we able to observe a value-change? */
530 if (expect_value._8 != 0)
531 value_change = KCSAN_VALUE_CHANGE_TRUE;
532
533 /* Check if this access raced with another. */
534 if (!consume_watchpoint(watchpoint)) {
535 /*
536 * Depending on the access type, map a value_change of MAYBE to
537 * TRUE (always report) or FALSE (never report).
538 */
539 if (value_change == KCSAN_VALUE_CHANGE_MAYBE) {
540 if (access_mask != 0) {
541 /*
542 * For access with access_mask, we require a
543 * value-change, as it is likely that races on
544 * ~access_mask bits are expected.
545 */
546 value_change = KCSAN_VALUE_CHANGE_FALSE;
547 } else if (size > 8 || is_assert) {
548 /* Always assume a value-change. */
549 value_change = KCSAN_VALUE_CHANGE_TRUE;
550 }
551 }
552
553 /*
554 * No need to increment 'data_races' counter, as the racing
555 * thread already did.
556 *
557 * Count 'assert_failures' for each failed ASSERT access,
558 * therefore both this thread and the racing thread may
559 * increment this counter.
560 */
561 if (is_assert && value_change == KCSAN_VALUE_CHANGE_TRUE)
562 atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_ASSERT_FAILURES]);
563
564 kcsan_report(ptr, size, type, value_change, KCSAN_REPORT_RACE_SIGNAL,
565 watchpoint - watchpoints);
566 } else if (value_change == KCSAN_VALUE_CHANGE_TRUE) {
567 /* Inferring a race, since the value should not have changed. */
568
569 atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN]);
570 if (is_assert)
571 atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_ASSERT_FAILURES]);
572
573 if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN) || is_assert)
574 kcsan_report(ptr, size, type, KCSAN_VALUE_CHANGE_TRUE,
575 KCSAN_REPORT_RACE_UNKNOWN_ORIGIN,
576 watchpoint - watchpoints);
577 }
578
579 /*
580 * Remove watchpoint; must be after reporting, since the slot may be
581 * reused after this point.
582 */
583 remove_watchpoint(watchpoint);
584 atomic_long_dec(&kcsan_counters[KCSAN_COUNTER_USED_WATCHPOINTS]);
585 out_unlock:
586 if (!kcsan_interrupt_watcher)
587 local_irq_restore(irq_flags);
588 kcsan_restore_irqtrace(current);
589 out:
590 user_access_restore(ua_flags);
591 }
592
check_access(const volatile void * ptr,size_t size,int type)593 static __always_inline void check_access(const volatile void *ptr, size_t size,
594 int type)
595 {
596 const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0;
597 atomic_long_t *watchpoint;
598 long encoded_watchpoint;
599
600 /*
601 * Do nothing for 0 sized check; this comparison will be optimized out
602 * for constant sized instrumentation (__tsan_{read,write}N).
603 */
604 if (unlikely(size == 0))
605 return;
606
607 /*
608 * Avoid user_access_save in fast-path: find_watchpoint is safe without
609 * user_access_save, as the address that ptr points to is only used to
610 * check if a watchpoint exists; ptr is never dereferenced.
611 */
612 watchpoint = find_watchpoint((unsigned long)ptr, size, !is_write,
613 &encoded_watchpoint);
614 /*
615 * It is safe to check kcsan_is_enabled() after find_watchpoint in the
616 * slow-path, as long as no state changes that cause a race to be
617 * detected and reported have occurred until kcsan_is_enabled() is
618 * checked.
619 */
620
621 if (unlikely(watchpoint != NULL))
622 kcsan_found_watchpoint(ptr, size, type, watchpoint,
623 encoded_watchpoint);
624 else {
625 struct kcsan_ctx *ctx = get_ctx(); /* Call only once in fast-path. */
626
627 if (unlikely(should_watch(ptr, size, type, ctx)))
628 kcsan_setup_watchpoint(ptr, size, type);
629 else if (unlikely(ctx->scoped_accesses.prev))
630 kcsan_check_scoped_accesses();
631 }
632 }
633
634 /* === Public interface ===================================================== */
635
kcsan_init(void)636 void __init kcsan_init(void)
637 {
638 int cpu;
639
640 BUG_ON(!in_task());
641
642 for_each_possible_cpu(cpu)
643 per_cpu(kcsan_rand_state, cpu) = (u32)get_cycles();
644
645 /*
646 * We are in the init task, and no other tasks should be running;
647 * WRITE_ONCE without memory barrier is sufficient.
648 */
649 if (kcsan_early_enable) {
650 pr_info("enabled early\n");
651 WRITE_ONCE(kcsan_enabled, true);
652 }
653 }
654
655 /* === Exported interface =================================================== */
656
kcsan_disable_current(void)657 void kcsan_disable_current(void)
658 {
659 ++get_ctx()->disable_count;
660 }
661 EXPORT_SYMBOL(kcsan_disable_current);
662
kcsan_enable_current(void)663 void kcsan_enable_current(void)
664 {
665 if (get_ctx()->disable_count-- == 0) {
666 /*
667 * Warn if kcsan_enable_current() calls are unbalanced with
668 * kcsan_disable_current() calls, which causes disable_count to
669 * become negative and should not happen.
670 */
671 kcsan_disable_current(); /* restore to 0, KCSAN still enabled */
672 kcsan_disable_current(); /* disable to generate warning */
673 WARN(1, "Unbalanced %s()", __func__);
674 kcsan_enable_current();
675 }
676 }
677 EXPORT_SYMBOL(kcsan_enable_current);
678
kcsan_enable_current_nowarn(void)679 void kcsan_enable_current_nowarn(void)
680 {
681 if (get_ctx()->disable_count-- == 0)
682 kcsan_disable_current();
683 }
684 EXPORT_SYMBOL(kcsan_enable_current_nowarn);
685
kcsan_nestable_atomic_begin(void)686 void kcsan_nestable_atomic_begin(void)
687 {
688 /*
689 * Do *not* check and warn if we are in a flat atomic region: nestable
690 * and flat atomic regions are independent from each other.
691 * See include/linux/kcsan.h: struct kcsan_ctx comments for more
692 * comments.
693 */
694
695 ++get_ctx()->atomic_nest_count;
696 }
697 EXPORT_SYMBOL(kcsan_nestable_atomic_begin);
698
kcsan_nestable_atomic_end(void)699 void kcsan_nestable_atomic_end(void)
700 {
701 if (get_ctx()->atomic_nest_count-- == 0) {
702 /*
703 * Warn if kcsan_nestable_atomic_end() calls are unbalanced with
704 * kcsan_nestable_atomic_begin() calls, which causes
705 * atomic_nest_count to become negative and should not happen.
706 */
707 kcsan_nestable_atomic_begin(); /* restore to 0 */
708 kcsan_disable_current(); /* disable to generate warning */
709 WARN(1, "Unbalanced %s()", __func__);
710 kcsan_enable_current();
711 }
712 }
713 EXPORT_SYMBOL(kcsan_nestable_atomic_end);
714
kcsan_flat_atomic_begin(void)715 void kcsan_flat_atomic_begin(void)
716 {
717 get_ctx()->in_flat_atomic = true;
718 }
719 EXPORT_SYMBOL(kcsan_flat_atomic_begin);
720
kcsan_flat_atomic_end(void)721 void kcsan_flat_atomic_end(void)
722 {
723 get_ctx()->in_flat_atomic = false;
724 }
725 EXPORT_SYMBOL(kcsan_flat_atomic_end);
726
kcsan_atomic_next(int n)727 void kcsan_atomic_next(int n)
728 {
729 get_ctx()->atomic_next = n;
730 }
731 EXPORT_SYMBOL(kcsan_atomic_next);
732
kcsan_set_access_mask(unsigned long mask)733 void kcsan_set_access_mask(unsigned long mask)
734 {
735 get_ctx()->access_mask = mask;
736 }
737 EXPORT_SYMBOL(kcsan_set_access_mask);
738
739 struct kcsan_scoped_access *
kcsan_begin_scoped_access(const volatile void * ptr,size_t size,int type,struct kcsan_scoped_access * sa)740 kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type,
741 struct kcsan_scoped_access *sa)
742 {
743 struct kcsan_ctx *ctx = get_ctx();
744
745 __kcsan_check_access(ptr, size, type);
746
747 ctx->disable_count++; /* Disable KCSAN, in case list debugging is on. */
748
749 INIT_LIST_HEAD(&sa->list);
750 sa->ptr = ptr;
751 sa->size = size;
752 sa->type = type;
753
754 if (!ctx->scoped_accesses.prev) /* Lazy initialize list head. */
755 INIT_LIST_HEAD(&ctx->scoped_accesses);
756 list_add(&sa->list, &ctx->scoped_accesses);
757
758 ctx->disable_count--;
759 return sa;
760 }
761 EXPORT_SYMBOL(kcsan_begin_scoped_access);
762
kcsan_end_scoped_access(struct kcsan_scoped_access * sa)763 void kcsan_end_scoped_access(struct kcsan_scoped_access *sa)
764 {
765 struct kcsan_ctx *ctx = get_ctx();
766
767 if (WARN(!ctx->scoped_accesses.prev, "Unbalanced %s()?", __func__))
768 return;
769
770 ctx->disable_count++; /* Disable KCSAN, in case list debugging is on. */
771
772 list_del(&sa->list);
773 if (list_empty(&ctx->scoped_accesses))
774 /*
775 * Ensure we do not enter kcsan_check_scoped_accesses()
776 * slow-path if unnecessary, and avoids requiring list_empty()
777 * in the fast-path (to avoid a READ_ONCE() and potential
778 * uaccess warning).
779 */
780 ctx->scoped_accesses.prev = NULL;
781
782 ctx->disable_count--;
783
784 __kcsan_check_access(sa->ptr, sa->size, sa->type);
785 }
786 EXPORT_SYMBOL(kcsan_end_scoped_access);
787
__kcsan_check_access(const volatile void * ptr,size_t size,int type)788 void __kcsan_check_access(const volatile void *ptr, size_t size, int type)
789 {
790 check_access(ptr, size, type);
791 }
792 EXPORT_SYMBOL(__kcsan_check_access);
793
794 /*
795 * KCSAN uses the same instrumentation that is emitted by supported compilers
796 * for ThreadSanitizer (TSAN).
797 *
798 * When enabled, the compiler emits instrumentation calls (the functions
799 * prefixed with "__tsan" below) for all loads and stores that it generated;
800 * inline asm is not instrumented.
801 *
802 * Note that, not all supported compiler versions distinguish aligned/unaligned
803 * accesses, but e.g. recent versions of Clang do. We simply alias the unaligned
804 * version to the generic version, which can handle both.
805 */
806
807 #define DEFINE_TSAN_READ_WRITE(size) \
808 void __tsan_read##size(void *ptr); \
809 void __tsan_read##size(void *ptr) \
810 { \
811 check_access(ptr, size, 0); \
812 } \
813 EXPORT_SYMBOL(__tsan_read##size); \
814 void __tsan_unaligned_read##size(void *ptr) \
815 __alias(__tsan_read##size); \
816 EXPORT_SYMBOL(__tsan_unaligned_read##size); \
817 void __tsan_write##size(void *ptr); \
818 void __tsan_write##size(void *ptr) \
819 { \
820 check_access(ptr, size, KCSAN_ACCESS_WRITE); \
821 } \
822 EXPORT_SYMBOL(__tsan_write##size); \
823 void __tsan_unaligned_write##size(void *ptr) \
824 __alias(__tsan_write##size); \
825 EXPORT_SYMBOL(__tsan_unaligned_write##size); \
826 void __tsan_read_write##size(void *ptr); \
827 void __tsan_read_write##size(void *ptr) \
828 { \
829 check_access(ptr, size, \
830 KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE); \
831 } \
832 EXPORT_SYMBOL(__tsan_read_write##size); \
833 void __tsan_unaligned_read_write##size(void *ptr) \
834 __alias(__tsan_read_write##size); \
835 EXPORT_SYMBOL(__tsan_unaligned_read_write##size)
836
837 DEFINE_TSAN_READ_WRITE(1);
838 DEFINE_TSAN_READ_WRITE(2);
839 DEFINE_TSAN_READ_WRITE(4);
840 DEFINE_TSAN_READ_WRITE(8);
841 DEFINE_TSAN_READ_WRITE(16);
842
843 void __tsan_read_range(void *ptr, size_t size);
__tsan_read_range(void * ptr,size_t size)844 void __tsan_read_range(void *ptr, size_t size)
845 {
846 check_access(ptr, size, 0);
847 }
848 EXPORT_SYMBOL(__tsan_read_range);
849
850 void __tsan_write_range(void *ptr, size_t size);
__tsan_write_range(void * ptr,size_t size)851 void __tsan_write_range(void *ptr, size_t size)
852 {
853 check_access(ptr, size, KCSAN_ACCESS_WRITE);
854 }
855 EXPORT_SYMBOL(__tsan_write_range);
856
857 /*
858 * Use of explicit volatile is generally disallowed [1], however, volatile is
859 * still used in various concurrent context, whether in low-level
860 * synchronization primitives or for legacy reasons.
861 * [1] https://lwn.net/Articles/233479/
862 *
863 * We only consider volatile accesses atomic if they are aligned and would pass
864 * the size-check of compiletime_assert_rwonce_type().
865 */
866 #define DEFINE_TSAN_VOLATILE_READ_WRITE(size) \
867 void __tsan_volatile_read##size(void *ptr); \
868 void __tsan_volatile_read##size(void *ptr) \
869 { \
870 const bool is_atomic = size <= sizeof(long long) && \
871 IS_ALIGNED((unsigned long)ptr, size); \
872 if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) && is_atomic) \
873 return; \
874 check_access(ptr, size, is_atomic ? KCSAN_ACCESS_ATOMIC : 0); \
875 } \
876 EXPORT_SYMBOL(__tsan_volatile_read##size); \
877 void __tsan_unaligned_volatile_read##size(void *ptr) \
878 __alias(__tsan_volatile_read##size); \
879 EXPORT_SYMBOL(__tsan_unaligned_volatile_read##size); \
880 void __tsan_volatile_write##size(void *ptr); \
881 void __tsan_volatile_write##size(void *ptr) \
882 { \
883 const bool is_atomic = size <= sizeof(long long) && \
884 IS_ALIGNED((unsigned long)ptr, size); \
885 if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) && is_atomic) \
886 return; \
887 check_access(ptr, size, \
888 KCSAN_ACCESS_WRITE | \
889 (is_atomic ? KCSAN_ACCESS_ATOMIC : 0)); \
890 } \
891 EXPORT_SYMBOL(__tsan_volatile_write##size); \
892 void __tsan_unaligned_volatile_write##size(void *ptr) \
893 __alias(__tsan_volatile_write##size); \
894 EXPORT_SYMBOL(__tsan_unaligned_volatile_write##size)
895
896 DEFINE_TSAN_VOLATILE_READ_WRITE(1);
897 DEFINE_TSAN_VOLATILE_READ_WRITE(2);
898 DEFINE_TSAN_VOLATILE_READ_WRITE(4);
899 DEFINE_TSAN_VOLATILE_READ_WRITE(8);
900 DEFINE_TSAN_VOLATILE_READ_WRITE(16);
901
902 /*
903 * The below are not required by KCSAN, but can still be emitted by the
904 * compiler.
905 */
906 void __tsan_func_entry(void *call_pc);
__tsan_func_entry(void * call_pc)907 void __tsan_func_entry(void *call_pc)
908 {
909 }
910 EXPORT_SYMBOL(__tsan_func_entry);
911 void __tsan_func_exit(void);
__tsan_func_exit(void)912 void __tsan_func_exit(void)
913 {
914 }
915 EXPORT_SYMBOL(__tsan_func_exit);
916 void __tsan_init(void);
__tsan_init(void)917 void __tsan_init(void)
918 {
919 }
920 EXPORT_SYMBOL(__tsan_init);
921
922 /*
923 * Instrumentation for atomic builtins (__atomic_*, __sync_*).
924 *
925 * Normal kernel code _should not_ be using them directly, but some
926 * architectures may implement some or all atomics using the compilers'
927 * builtins.
928 *
929 * Note: If an architecture decides to fully implement atomics using the
930 * builtins, because they are implicitly instrumented by KCSAN (and KASAN,
931 * etc.), implementing the ARCH_ATOMIC interface (to get instrumentation via
932 * atomic-instrumented) is no longer necessary.
933 *
934 * TSAN instrumentation replaces atomic accesses with calls to any of the below
935 * functions, whose job is to also execute the operation itself.
936 */
937
938 #define DEFINE_TSAN_ATOMIC_LOAD_STORE(bits) \
939 u##bits __tsan_atomic##bits##_load(const u##bits *ptr, int memorder); \
940 u##bits __tsan_atomic##bits##_load(const u##bits *ptr, int memorder) \
941 { \
942 if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
943 check_access(ptr, bits / BITS_PER_BYTE, KCSAN_ACCESS_ATOMIC); \
944 } \
945 return __atomic_load_n(ptr, memorder); \
946 } \
947 EXPORT_SYMBOL(__tsan_atomic##bits##_load); \
948 void __tsan_atomic##bits##_store(u##bits *ptr, u##bits v, int memorder); \
949 void __tsan_atomic##bits##_store(u##bits *ptr, u##bits v, int memorder) \
950 { \
951 if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
952 check_access(ptr, bits / BITS_PER_BYTE, \
953 KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC); \
954 } \
955 __atomic_store_n(ptr, v, memorder); \
956 } \
957 EXPORT_SYMBOL(__tsan_atomic##bits##_store)
958
959 #define DEFINE_TSAN_ATOMIC_RMW(op, bits, suffix) \
960 u##bits __tsan_atomic##bits##_##op(u##bits *ptr, u##bits v, int memorder); \
961 u##bits __tsan_atomic##bits##_##op(u##bits *ptr, u##bits v, int memorder) \
962 { \
963 if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
964 check_access(ptr, bits / BITS_PER_BYTE, \
965 KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | \
966 KCSAN_ACCESS_ATOMIC); \
967 } \
968 return __atomic_##op##suffix(ptr, v, memorder); \
969 } \
970 EXPORT_SYMBOL(__tsan_atomic##bits##_##op)
971
972 /*
973 * Note: CAS operations are always classified as write, even in case they
974 * fail. We cannot perform check_access() after a write, as it might lead to
975 * false positives, in cases such as:
976 *
977 * T0: __atomic_compare_exchange_n(&p->flag, &old, 1, ...)
978 *
979 * T1: if (__atomic_load_n(&p->flag, ...)) {
980 * modify *p;
981 * p->flag = 0;
982 * }
983 *
984 * The only downside is that, if there are 3 threads, with one CAS that
985 * succeeds, another CAS that fails, and an unmarked racing operation, we may
986 * point at the wrong CAS as the source of the race. However, if we assume that
987 * all CAS can succeed in some other execution, the data race is still valid.
988 */
989 #define DEFINE_TSAN_ATOMIC_CMPXCHG(bits, strength, weak) \
990 int __tsan_atomic##bits##_compare_exchange_##strength(u##bits *ptr, u##bits *exp, \
991 u##bits val, int mo, int fail_mo); \
992 int __tsan_atomic##bits##_compare_exchange_##strength(u##bits *ptr, u##bits *exp, \
993 u##bits val, int mo, int fail_mo) \
994 { \
995 if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
996 check_access(ptr, bits / BITS_PER_BYTE, \
997 KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | \
998 KCSAN_ACCESS_ATOMIC); \
999 } \
1000 return __atomic_compare_exchange_n(ptr, exp, val, weak, mo, fail_mo); \
1001 } \
1002 EXPORT_SYMBOL(__tsan_atomic##bits##_compare_exchange_##strength)
1003
1004 #define DEFINE_TSAN_ATOMIC_CMPXCHG_VAL(bits) \
1005 u##bits __tsan_atomic##bits##_compare_exchange_val(u##bits *ptr, u##bits exp, u##bits val, \
1006 int mo, int fail_mo); \
1007 u##bits __tsan_atomic##bits##_compare_exchange_val(u##bits *ptr, u##bits exp, u##bits val, \
1008 int mo, int fail_mo) \
1009 { \
1010 if (!IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS)) { \
1011 check_access(ptr, bits / BITS_PER_BYTE, \
1012 KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | \
1013 KCSAN_ACCESS_ATOMIC); \
1014 } \
1015 __atomic_compare_exchange_n(ptr, &exp, val, 0, mo, fail_mo); \
1016 return exp; \
1017 } \
1018 EXPORT_SYMBOL(__tsan_atomic##bits##_compare_exchange_val)
1019
1020 #define DEFINE_TSAN_ATOMIC_OPS(bits) \
1021 DEFINE_TSAN_ATOMIC_LOAD_STORE(bits); \
1022 DEFINE_TSAN_ATOMIC_RMW(exchange, bits, _n); \
1023 DEFINE_TSAN_ATOMIC_RMW(fetch_add, bits, ); \
1024 DEFINE_TSAN_ATOMIC_RMW(fetch_sub, bits, ); \
1025 DEFINE_TSAN_ATOMIC_RMW(fetch_and, bits, ); \
1026 DEFINE_TSAN_ATOMIC_RMW(fetch_or, bits, ); \
1027 DEFINE_TSAN_ATOMIC_RMW(fetch_xor, bits, ); \
1028 DEFINE_TSAN_ATOMIC_RMW(fetch_nand, bits, ); \
1029 DEFINE_TSAN_ATOMIC_CMPXCHG(bits, strong, 0); \
1030 DEFINE_TSAN_ATOMIC_CMPXCHG(bits, weak, 1); \
1031 DEFINE_TSAN_ATOMIC_CMPXCHG_VAL(bits)
1032
1033 DEFINE_TSAN_ATOMIC_OPS(8);
1034 DEFINE_TSAN_ATOMIC_OPS(16);
1035 DEFINE_TSAN_ATOMIC_OPS(32);
1036 DEFINE_TSAN_ATOMIC_OPS(64);
1037
1038 void __tsan_atomic_thread_fence(int memorder);
__tsan_atomic_thread_fence(int memorder)1039 void __tsan_atomic_thread_fence(int memorder)
1040 {
1041 __atomic_thread_fence(memorder);
1042 }
1043 EXPORT_SYMBOL(__tsan_atomic_thread_fence);
1044
1045 void __tsan_atomic_signal_fence(int memorder);
__tsan_atomic_signal_fence(int memorder)1046 void __tsan_atomic_signal_fence(int memorder) { }
1047 EXPORT_SYMBOL(__tsan_atomic_signal_fence);
1048