1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
3  * Copyright (C) 2017-2022 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
10  *   - Initialization and readiness waiting.
11  *   - Fast key erasure RNG, the "crng".
12  *   - Entropy accumulation and extraction routines.
13  *   - Entropy collection routines.
14  *   - Userspace reader/writer interfaces.
15  *   - Sysctl interface.
78  * crng_init is protected by base_crng->lock, and only increases
79  * its value (from empty->early->ready).
87 /* Various types of waiters for crng_init->CRNG_READY transition. */
99 MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression");
127  *          -ERESTARTSYS if the function was interrupted by a signal.
148  *	    -EALREADY if pool is already initialised (callback not called)
153 	int ret = -EALREADY;  in register_random_ready_notifier()
203  * RNG described at <https://blog.cr.yp.to/20170723-random.html>.
216  * functions may be higher performance for one-off random integers,
262 	 * because the per-cpu crngs are initialized to ULONG_MAX, so this  in crng_reseed()
347 	 * then re-check once locked later. In the case where we're really not  in crng_make_state()
368 	 * If the base_crng is old enough, we reseed, which in turn bumps the  in crng_make_state()
378 	 * If our per-cpu crng is older than the base_crng, then it means  in crng_make_state()
381 	 * for our per-cpu crng. This brings us up to date with base_crng.  in crng_make_state()
383 	if (unlikely(crng->generation != READ_ONCE(base_crng.generation))) {  in crng_make_state()
386 				      crng->key, sizeof(crng->key));  in crng_make_state()
387 		crng->generation = base_crng.generation;  in crng_make_state()
392 	 * Finally, when we've made it this far, our per-cpu crng has an up  in crng_make_state()
398 	crng_fast_key_erasure(crng->key, chacha_state, random_data, random_data_len);  in crng_make_state()
413 	len -= first_block_len;  in _get_random_bytes()
427 		len -= CHACHA_BLOCK_SIZE;  in _get_random_bytes()
497 	return ret ? ret : -EFAULT;  in get_random_bytes_user()
545 	if (batch->position >= ARRAY_SIZE(batch->entropy) ||			\
546 	    next_gen != batch->generation) {					\
547 		_get_random_bytes(batch->entropy, sizeof(batch->entropy));	\
548 		batch->position = 0;						\
549 		batch->generation = next_gen;					\
552 	ret = batch->entropy[batch->position];					\
553 	batch->entropy[batch->position] = 0;					\
554 	++batch->position;							\
572 	 * the per-cpu crng and all batches, so that we serve fresh  in DEFINE_BATCHED_ENTROPY()
575 	per_cpu_ptr(&crngs, cpu)->generation = ULONG_MAX;  in DEFINE_BATCHED_ENTROPY()
576 	per_cpu_ptr(&batched_entropy_u32, cpu)->position = UINT_MAX;  in DEFINE_BATCHED_ENTROPY()
577 	per_cpu_ptr(&batched_entropy_u64, cpu)->position = UINT_MAX;  in DEFINE_BATCHED_ENTROPY()
583  * This function will use the architecture-specific hardware random
602 		left -= block_len;  in get_random_bytes_arch()
605 	return len - left;  in get_random_bytes_arch()
630 	POOL_READY_BITS = POOL_BITS, /* When crng_init->CRNG_READY */
631 	POOL_EARLY_BITS = POOL_READY_BITS / 2 /* When crng_init->CRNG_EARLY */
666  * This is an HKDF-like construction for using the hashed collected entropy
667  * as a PRF key, that's then expanded block-by-block.
703 		len -= i;  in extract_entropy()
766  * read-out of the RTC. This does *not* credit any actual entropy to
788  * layer request events, on a per-disk_devt basis, as input to the
789  * entropy pool. Note that high-speed solid state drives with very low
836 			arch_bytes -= sizeof(entropy);  in random_init()
854  * Add device- or boot-specific data to the input pool to help
874  * Interface for in-kernel drivers of true hardware RNGs.
925  * This is [Half]SipHash-1-x, starting from an empty key. Because
926  * the key is fixed, it assumes that its inputs are non-malicious,
928  * four-word SipHash state, while v represents a two-word input.
958 	per_cpu_ptr(&irq_randomness, cpu)->count = 0;  in random_online_cpu()
987 	memcpy(pool, fast_pool->pool, sizeof(pool));  in mix_interrupt_randomness()
988 	count = fast_pool->count;  in mix_interrupt_randomness()
989 	fast_pool->count = 0;  in mix_interrupt_randomness()
990 	fast_pool->last = jiffies;  in mix_interrupt_randomness()
1007 	fast_mix(fast_pool->pool, entropy,  in add_interrupt_randomness()
1009 	new_count = ++fast_pool->count;  in add_interrupt_randomness()
1014 	if (new_count < 1024 && !time_is_before_jiffies(fast_pool->last + HZ))  in add_interrupt_randomness()
1017 	fast_pool->count |= MIX_INFLIGHT;  in add_interrupt_randomness()
1018 	if (!timer_pending(&fast_pool->mix)) {  in add_interrupt_randomness()
1019 		fast_pool->mix.expires = jiffies;  in add_interrupt_randomness()
1020 		add_timer_on(&fast_pool->mix, raw_smp_processor_id());  in add_interrupt_randomness()
1049 		fast_mix(this_cpu_ptr(&irq_randomness)->pool, entropy, num);  in add_timer_randomness()
1062 	 * We take into account the first, second and third-order deltas  in add_timer_randomness()
1065 	delta = now - READ_ONCE(state->last_time);  in add_timer_randomness()
1066 	WRITE_ONCE(state->last_time, now);  in add_timer_randomness()
1068 	delta2 = delta - READ_ONCE(state->last_delta);  in add_timer_randomness()
1069 	WRITE_ONCE(state->last_delta, delta);  in add_timer_randomness()
1071 	delta3 = delta2 - READ_ONCE(state->last_delta2);  in add_timer_randomness()
1072 	WRITE_ONCE(state->last_delta2, delta2);  in add_timer_randomness()
1075 		delta = -delta;  in add_timer_randomness()
1077 		delta2 = -delta2;  in add_timer_randomness()
1079 		delta3 = -delta3;  in add_timer_randomness()
1099 		this_cpu_ptr(&irq_randomness)->count += max(1u, bits * 64) - 1;  in add_timer_randomness()
1122 	if (!disk || !disk->random)  in add_disk_randomness()
1125 	add_timer_randomness(disk->random, 0x100 + disk_devt(disk));  in add_disk_randomness()
1139 		state->last_time = INITIAL_JIFFIES;  in rand_initialize_disk()
1140 		disk->random = state;  in rand_initialize_disk()
1151  * Note that we don't re-arm the timer in the timer itself - we are
1156  * So the re-arming always happens in the entropy loop itself.
1176 	/* Slow counter - or none. Don't even bother */  in try_to_generate_entropy()
1230 		return -EINVAL;  in SYSCALL_DEFINE3()
1237 		return -EINVAL;  in SYSCALL_DEFINE3()
1241 			return -EAGAIN;  in SYSCALL_DEFINE3()
1284 	return ret ? ret : -EFAULT;  in write_pool_user()
1300 			--maxwarn;  in urandom_read_iter()
1302 				  current->comm, iov_iter_count(iter));  in urandom_read_iter()
1314 	    ((kiocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO)) ||  in random_read_iter()
1315 	     (kiocb->ki_filp->f_flags & O_NONBLOCK)))  in random_read_iter()
1316 		return -EAGAIN;  in random_read_iter()
1333 			return -EFAULT;  in random_ioctl()
1337 			return -EPERM;  in random_ioctl()
1339 			return -EFAULT;  in random_ioctl()
1341 			return -EINVAL;  in random_ioctl()
1351 			return -EPERM;  in random_ioctl()
1353 			return -EFAULT;  in random_ioctl()
1355 			return -EINVAL;  in random_ioctl()
1357 			return -EFAULT;  in random_ioctl()
1366 			return -EFAULT;  in random_ioctl()
1374 			return -EPERM;  in random_ioctl()
1378 			return -EPERM;  in random_ioctl()
1380 			return -ENODATA;  in random_ioctl()
1384 		return -EINVAL;  in random_ioctl()
1425  * - boot_id - a UUID representing the current boot.
1427  * - uuid - a random UUID, different each time the file is read.
1429  * - poolsize - the number of bits of entropy that the input pool can
1432  * - entropy_avail - the number of bits of entropy currently in the
1435  * - write_wakeup_threshold - the amount of entropy in the input pool
1441  * - urandom_min_reseed_secs - fixed to the value CRNG_RESEED_INTERVAL.
1458  * UUID. The difference is in whether table->data is NULL; if it is,
1472 		return -EPERM;  in proc_do_uuid()
1474 	uuid = table->data;  in proc_do_uuid()
1546  * Add back two functions that were being used by out-of-tree drivers.
1557 	int err = -EALREADY;  in add_random_ready_callback()
1562 	owner = rdy->owner;  in add_random_ready_callback()
1564 		return -ENOENT;  in add_random_ready_callback()
1572 	list_add(&rdy->list, &random_ready_list);  in add_random_ready_callback()
1590 	if (!list_empty(&rdy->list)) {  in del_random_ready_callback()
1591 		list_del_init(&rdy->list);  in del_random_ready_callback()
1592 		owner = rdy->owner;  in del_random_ready_callback()
1607 		struct module *owner = rdy->owner;  in process_oldschool_random_ready_list()
1609 		list_del_init(&rdy->list);  in process_oldschool_random_ready_list()
1610 		rdy->func(rdy);  in process_oldschool_random_ready_list()