1 // SPDX-License-Identifier: GPL-2.0-only
13  * 1) one per-CPU thread takes a per-page pthread_mutex in a random
15  *    area_src), and increments a per-page counter in the same page,
18  * 2) another per-CPU thread handles the userfaults generated by
22  * 3) one last per-CPU thread transfers the memory in the background
32  * per-CPU threads 1 by triggering userfaults inside
81 /* Whether to test uffd write-protection */
91 static int uffd = -1;
110  * to avoid non alignment faults on non-x86 archs.
116 				  sizeof(unsigned long long) - 1) &	\
118 						  -  1)))
129     "# 10MiB-~6GiB 999 bounces anonymous test, "
213 			   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);  in anon_allocate_area()
269 	 * MADV_DONTEED won't work. So exercise -EEXIST on a alias  in hugetlb_alias_mapping()
271 	 * this way we'll exercise the -EEXEC at the fs level.  in hugetlb_alias_mapping()
383 			if (close(pipefd[i]))  in uffd_test_ctx_clear()
384 				err("close pipefd");  in uffd_test_ctx_clear()
395 	if (uffd != -1) {  in uffd_test_ctx_clear()
396 		if (close(uffd))  in uffd_test_ctx_clear()
397 			err("close uffd");  in uffd_test_ctx_clear()
398 		uffd = -1;  in uffd_test_ctx_clear()
414 	uffd_test_ops->allocate_area((void **)&area_src);  in uffd_test_ctx_init_ext()
415 	uffd_test_ops->allocate_area((void **)&area_dst);  in uffd_test_ctx_init_ext()
430 		 * zero, so leave a placeholder below always non-zero  in uffd_test_ctx_init_ext()
455 	uffd_test_ops->release_pages(area_dst);  in uffd_test_ctx_init_ext()
484 	prms.range.start = start;  in wp_range()
485 	prms.range.len = len;  in wp_range()
486 	/* Undo write-protect, do wakeup after that */  in wp_range()
498 	req.range.start = start;  in continue_range()
499 	req.range.len = len;  in continue_range()
509 	 * error (-EEXIST) on purpose, to verify doing so doesn't cause a BUG.  in continue_range()
513 	if (ret >= 0 || req.mapped != -EEXIST)  in continue_range()
529 		seed = (unsigned int) time(NULL) - bounces;  in locking_thread()
537 		page_nr = -bounces;  in locking_thread()
572 	uffd_test_ops->alias_mapping(&uffdio_copy->dst,  in retry_copy_page()
573 				     uffdio_copy->len,  in retry_copy_page()
577 		if (uffdio_copy->copy != -EEXIST)  in retry_copy_page()
579 			    (int64_t)uffdio_copy->copy);  in retry_copy_page()
582 		    (int64_t)uffdio_copy->copy);  in retry_copy_page()
602 		if (uffdio_copy.copy != -EEXIST)  in __copy_page()
649 	if (msg->event != UFFD_EVENT_PAGEFAULT)  in uffd_handle_page_fault()
650 		err("unexpected msg event %u", msg->event);  in uffd_handle_page_fault()
652 	if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {  in uffd_handle_page_fault()
654 		wp_range(uffd, msg->arg.pagefault.address, page_size, false);  in uffd_handle_page_fault()
655 		stats->wp_faults++;  in uffd_handle_page_fault()
656 	} else if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) {  in uffd_handle_page_fault()
663 		 * To prove we can modify the original range for testing  in uffd_handle_page_fault()
664 		 * purposes, we're going to bit flip this range before  in uffd_handle_page_fault()
668 		 * area_dst (non-UFFD-registered) and area_dst_alias  in uffd_handle_page_fault()
669 		 * (UFFD-registered).  in uffd_handle_page_fault()
673 				   ((char *)msg->arg.pagefault.address -  in uffd_handle_page_fault()
677 		continue_range(uffd, msg->arg.pagefault.address, page_size);  in uffd_handle_page_fault()
678 		stats->minor_faults++;  in uffd_handle_page_fault()
681 		if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)  in uffd_handle_page_fault()
684 		offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_handle_page_fault()
685 		offset &= ~(page_size-1);  in uffd_handle_page_fault()
688 			stats->missing_faults++;  in uffd_handle_page_fault()
695 	unsigned long cpu = stats->cpu;  in uffd_poll_thread()
708 		ret = poll(pollfd, 2, -1);  in uffd_poll_thread()
728 			close(uffd);  in uffd_poll_thread()
733 			uffd_reg.range.start = msg.arg.remove.start;  in uffd_poll_thread()
734 			uffd_reg.range.len = msg.arg.remove.end -  in uffd_poll_thread()
736 			if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range))  in uffd_poll_thread()
781 	 * If we need to test uffd-wp, set it up now.  Then we'll have  in background_thread()
783 	 * can be write-protected for testing  in background_thread()
836 	 * area_src (but they're guaranteed to get -EEXIST from  in stress()
840 	uffd_test_ops->release_pages(area_src);  in stress()
878  * For non-cooperative userfaultfd test we fork() a process that will
893  * test robustness use case - we release monitored area, fork a process
919 		lastnr = (unsigned long)-1;  in faulting_process()
990 	uffd_test_ops->release_pages(area_dst);  in faulting_process()
1003 	uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,  in retry_uffdio_zeropage()
1004 				     uffdio_zeropage->range.len,  in retry_uffdio_zeropage()
1007 		if (uffdio_zeropage->zeropage != -EEXIST)  in retry_uffdio_zeropage()
1009 			    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
1012 		    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
1023 	has_zeropage = uffd_test_ops->expected_ioctls & (1 << _UFFDIO_ZEROPAGE);  in __uffdio_zeropage()
1027 	uffdio_zeropage.range.start = (unsigned long) area_dst + offset;  in __uffdio_zeropage()
1028 	uffdio_zeropage.range.len = page_size;  in __uffdio_zeropage()
1036 		else if (res != -EINVAL)  in __uffdio_zeropage()
1037 			err("UFFDIO_ZEROPAGE not -EINVAL");  in __uffdio_zeropage()
1071 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_zeropage_test()
1072 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_zeropage_test()
1079 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_zeropage_test()
1110 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_events_test()
1111 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_events_test()
1118 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_events_test()
1164 	uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_sig_test()
1165 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_sig_test()
1172 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_sig_test()
1179 	uffd_test_ops->release_pages(area_dst);  in userfaultfd_sig_test()
1240 	uffdio_register.range.start = (unsigned long)area_dst_alias;  in userfaultfd_minor_test()
1241 	uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_minor_test()
1246 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_minor_test()
1252 	 * After registering with UFFD, populate the non-UFFD-registered side of  in userfaultfd_minor_test()
1256 		memset(area_dst + (p * page_size), p % ((uint8_t)-1),  in userfaultfd_minor_test()
1264 	 * Read each of the pages back using the UFFD-registered mapping. We  in userfaultfd_minor_test()
1266 	 * fault. uffd_poll_thread will resolve the fault by bit-flipping the  in userfaultfd_minor_test()
1274 		expected_byte = ~((uint8_t)(p % ((uint8_t)-1)));  in userfaultfd_minor_test()
1311 	while (bounces--) {  in userfaultfd_stress()
1332 		uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_stress()
1333 		uffdio_register.range.len = nr_pages * page_size;  in userfaultfd_stress()
1339 		expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_stress()
1345 			uffdio_register.range.start = (unsigned long)  in userfaultfd_stress()
1359 		 * return -EEXIST). The problem comes at the next  in userfaultfd_stress()
1364 		 * area_src would lead to -EEXIST failure during the  in userfaultfd_stress()
1375 		uffd_test_ops->release_pages(area_dst);  in userfaultfd_stress()
1389 		if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range))  in userfaultfd_stress()
1392 			uffdio_register.range.start = (unsigned long) area_dst;  in userfaultfd_stress()
1394 				  &uffdio_register.range))  in userfaultfd_stress()
1423  * Copied from mlock2-tests.c
1451 		/* Only enable write-protect test for anonymous test */  in set_test_type()