159 static inline unsigned long metadata_to_pageaddr(const struct kfence_metadata *meta)  in metadata_to_pageaddr()  argument
161 	unsigned long offset = (meta - kfence_metadata + 1) * PAGE_SIZE * 2;  in metadata_to_pageaddr()
167 	if (KFENCE_WARN_ON(meta < kfence_metadata ||  in metadata_to_pageaddr()
168 			   meta >= kfence_metadata + CONFIG_KFENCE_NUM_OBJECTS))  in metadata_to_pageaddr()
175 	if (KFENCE_WARN_ON(ALIGN_DOWN(meta->addr, PAGE_SIZE) != pageaddr))  in metadata_to_pageaddr()
185 static noinline void metadata_update_state(struct kfence_metadata *meta,  in metadata_update_state()  argument
189 		next == KFENCE_OBJECT_FREED ? &meta->free_track : &meta->alloc_track;  in metadata_update_state()
191 	lockdep_assert_held(&meta->lock);  in metadata_update_state()
205 	WRITE_ONCE(meta->state, next);  in metadata_update_state()
228 static __always_inline void for_each_canary(const struct kfence_metadata *meta, bool (*fn)(u8 *))  in for_each_canary()  argument
230 	const unsigned long pageaddr = ALIGN_DOWN(meta->addr, PAGE_SIZE);  in for_each_canary()
233 	lockdep_assert_held(&meta->lock);  in for_each_canary()
245 	for (addr = pageaddr; addr < meta->addr; addr++) {  in for_each_canary()
251 	for (addr = meta->addr + meta->size; addr < pageaddr + PAGE_SIZE; addr++) {  in for_each_canary()
259 	struct kfence_metadata *meta = NULL;  in kfence_guarded_alloc()  local
267 		meta = list_entry(kfence_freelist.next, struct kfence_metadata, list);  in kfence_guarded_alloc()
268 		list_del_init(&meta->list);  in kfence_guarded_alloc()
271 	if (!meta)  in kfence_guarded_alloc()
274 	if (unlikely(!raw_spin_trylock_irqsave(&meta->lock, flags))) {  in kfence_guarded_alloc()
286 		list_add_tail(&meta->list, &kfence_freelist);  in kfence_guarded_alloc()
292 	meta->addr = metadata_to_pageaddr(meta);  in kfence_guarded_alloc()
294 	if (meta->state == KFENCE_OBJECT_FREED)  in kfence_guarded_alloc()
295 		kfence_unprotect(meta->addr);  in kfence_guarded_alloc()
307 		meta->addr += PAGE_SIZE - size;  in kfence_guarded_alloc()
308 		meta->addr = ALIGN_DOWN(meta->addr, cache->align);  in kfence_guarded_alloc()
311 	addr = (void *)meta->addr;  in kfence_guarded_alloc()
314 	metadata_update_state(meta, KFENCE_OBJECT_ALLOCATED);  in kfence_guarded_alloc()
316 	WRITE_ONCE(meta->cache, cache);  in kfence_guarded_alloc()
317 	meta->size = size;  in kfence_guarded_alloc()
318 	for_each_canary(meta, set_canary_byte);  in kfence_guarded_alloc()
321 	page = virt_to_page(meta->addr);  in kfence_guarded_alloc()
328 	raw_spin_unlock_irqrestore(&meta->lock, flags);  in kfence_guarded_alloc()
343 		kfence_protect(meta->addr); /* Random "faults" by protecting the object. */  in kfence_guarded_alloc()
351 static void kfence_guarded_free(void *addr, struct kfence_metadata *meta, bool zombie)  in kfence_guarded_free()  argument
356 	raw_spin_lock_irqsave(&meta->lock, flags);  in kfence_guarded_free()
358 	if (meta->state != KFENCE_OBJECT_ALLOCATED || meta->addr != (unsigned long)addr) {  in kfence_guarded_free()
361 		kfence_report_error((unsigned long)addr, false, NULL, meta,  in kfence_guarded_free()
363 		raw_spin_unlock_irqrestore(&meta->lock, flags);  in kfence_guarded_free()
376 	if (meta->unprotected_page) {  in kfence_guarded_free()
377 		memzero_explicit((void *)ALIGN_DOWN(meta->unprotected_page, PAGE_SIZE), PAGE_SIZE);  in kfence_guarded_free()
378 		kfence_protect(meta->unprotected_page);  in kfence_guarded_free()
379 		meta->unprotected_page = 0;  in kfence_guarded_free()
383 	for_each_canary(meta, check_canary_byte);  in kfence_guarded_free()
390 	if (!zombie && unlikely(slab_want_init_on_free(meta->cache)))  in kfence_guarded_free()
391 		memzero_explicit(addr, meta->size);  in kfence_guarded_free()
394 	metadata_update_state(meta, KFENCE_OBJECT_FREED);  in kfence_guarded_free()
396 	raw_spin_unlock_irqrestore(&meta->lock, flags);  in kfence_guarded_free()
405 		KFENCE_WARN_ON(!list_empty(&meta->list));  in kfence_guarded_free()
406 		list_add_tail(&meta->list, &kfence_freelist);  in kfence_guarded_free()
419 	struct kfence_metadata *meta = container_of(h, struct kfence_metadata, rcu_head);  in rcu_guarded_free()  local
421 	kfence_guarded_free((void *)meta->addr, meta, false);  in rcu_guarded_free()
471 		struct kfence_metadata *meta = &kfence_metadata[i];  in kfence_init_pool()  local
474 		INIT_LIST_HEAD(&meta->list);  in kfence_init_pool()
475 		raw_spin_lock_init(&meta->lock);  in kfence_init_pool()
476 		meta->state = KFENCE_OBJECT_UNUSED;  in kfence_init_pool()
477 		meta->addr = addr; /* Initialize for validation in metadata_to_pageaddr(). */  in kfence_init_pool()
478 		list_add_tail(&meta->list, &kfence_freelist);  in kfence_init_pool()
550 	struct kfence_metadata *meta = &kfence_metadata[(long)v - 1];  in show_object()  local
553 	raw_spin_lock_irqsave(&meta->lock, flags);  in show_object()
554 	kfence_print_object(seq, meta);  in show_object()
555 	raw_spin_unlock_irqrestore(&meta->lock, flags);  in show_object()
679 	struct kfence_metadata *meta;  in kfence_shutdown_cache()  local
685 		meta = &kfence_metadata[i];  in kfence_shutdown_cache()
694 		if (READ_ONCE(meta->cache) != s ||  in kfence_shutdown_cache()
695 		    READ_ONCE(meta->state) != KFENCE_OBJECT_ALLOCATED)  in kfence_shutdown_cache()
698 		raw_spin_lock_irqsave(&meta->lock, flags);  in kfence_shutdown_cache()
699 		in_use = meta->cache == s && meta->state == KFENCE_OBJECT_ALLOCATED;  in kfence_shutdown_cache()
700 		raw_spin_unlock_irqrestore(&meta->lock, flags);  in kfence_shutdown_cache()
717 			kfence_guarded_free((void *)meta->addr, meta, /*zombie=*/true);  in kfence_shutdown_cache()
722 		meta = &kfence_metadata[i];  in kfence_shutdown_cache()
725 		if (READ_ONCE(meta->cache) != s || READ_ONCE(meta->state) != KFENCE_OBJECT_FREED)  in kfence_shutdown_cache()
728 		raw_spin_lock_irqsave(&meta->lock, flags);  in kfence_shutdown_cache()
729 		if (meta->cache == s && meta->state == KFENCE_OBJECT_FREED)  in kfence_shutdown_cache()
730 			meta->cache = NULL;  in kfence_shutdown_cache()
731 		raw_spin_unlock_irqrestore(&meta->lock, flags);  in kfence_shutdown_cache()
782 	const struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr);  in kfence_ksize()  local
788 	return meta ? meta->size : 0;  in kfence_ksize()
793 	const struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr);  in kfence_object_start()  local
799 	return meta ? (void *)meta->addr : NULL;  in kfence_object_start()
804 	struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr);  in __kfence_free()  local
812 	if (unlikely(meta->cache && (meta->cache->flags & SLAB_TYPESAFE_BY_RCU)))  in __kfence_free()
813 		call_rcu(&meta->rcu_head, rcu_guarded_free);  in __kfence_free()
815 		kfence_guarded_free(addr, meta, false);  in __kfence_free()
835 		struct kfence_metadata *meta;  in kfence_handle_page_fault()  local
838 		meta = addr_to_metadata(addr - PAGE_SIZE);  in kfence_handle_page_fault()
839 		if (meta && READ_ONCE(meta->state) == KFENCE_OBJECT_ALLOCATED) {  in kfence_handle_page_fault()
840 			to_report = meta;  in kfence_handle_page_fault()
842 			distance = addr - data_race(meta->addr + meta->size);  in kfence_handle_page_fault()
845 		meta = addr_to_metadata(addr + PAGE_SIZE);  in kfence_handle_page_fault()
846 		if (meta && READ_ONCE(meta->state) == KFENCE_OBJECT_ALLOCATED) {  in kfence_handle_page_fault()
848 			if (!to_report || distance > data_race(meta->addr) - addr)  in kfence_handle_page_fault()
849 				to_report = meta;  in kfence_handle_page_fault()