1 // SPDX-License-Identifier: GPL-2.0-only
11  * There are three threads running per CPU:
13  * 1) one per-CPU thread takes a per-page pthread_mutex in a random
15  *    area_src), and increments a per-page counter in the same page,
18  * 2) another per-CPU thread handles the userfaults generated by
22  * 3) one last per-CPU thread transfers the memory in the background
24  *    2). Each cpu thread takes cares of transferring a portion of the
32  * per-CPU threads 1 by triggering userfaults inside
81 /* Whether to test uffd write-protection */
91 static int uffd = -1;
99 	int cpu;  member
110  * to avoid non alignment faults on non-x86 archs.
116 				  sizeof(unsigned long long) - 1) &	\
118 						  -  1)))
129     "# 10MiB-~6GiB 999 bounces anonymous test, "
164 		uffd_stats[i].cpu = i;  in uffd_stats_reset()
213 			   MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);  in anon_allocate_area()
269 	 * MADV_DONTEED won't work. So exercise -EEXIST on a alias  in hugetlb_alias_mapping()
271 	 * this way we'll exercise the -EEXEC at the fs level.  in hugetlb_alias_mapping()
395 	if (uffd != -1) {  in uffd_test_ctx_clear()
398 		uffd = -1;  in uffd_test_ctx_clear()
410 	unsigned long nr, cpu;  in uffd_test_ctx_init_ext()  local
414 	uffd_test_ops->allocate_area((void **)&area_src);  in uffd_test_ctx_init_ext()
415 	uffd_test_ops->allocate_area((void **)&area_dst);  in uffd_test_ctx_init_ext()
423 	for (nr = 0; nr < nr_pages; nr++) {  in uffd_test_ctx_init_ext()
424 		*area_mutex(area_src, nr) =  in uffd_test_ctx_init_ext()
426 		count_verify[nr] = *area_count(area_src, nr) = 1;  in uffd_test_ctx_init_ext()
430 		 * zero, so leave a placeholder below always non-zero  in uffd_test_ctx_init_ext()
434 		*(area_count(area_src, nr) + 1) = 1;  in uffd_test_ctx_init_ext()
455 	uffd_test_ops->release_pages(area_dst);  in uffd_test_ctx_init_ext()
460 	for (cpu = 0; cpu < nr_cpus; cpu++)  in uffd_test_ctx_init_ext()
461 		if (pipe2(&pipefd[cpu * 2], O_CLOEXEC | O_NONBLOCK))  in uffd_test_ctx_init_ext()
486 	/* Undo write-protect, do wakeup after that */  in wp_range()
509 	 * error (-EEXIST) on purpose, to verify doing so doesn't cause a BUG.  in continue_range()
513 	if (ret >= 0 || req.mapped != -EEXIST)  in continue_range()
520 	unsigned long cpu = (unsigned long) arg;  in locking_thread()  local
529 		seed = (unsigned int) time(NULL) - bounces;  in locking_thread()
531 			seed += cpu;  in locking_thread()
537 		page_nr = -bounces;  in locking_thread()
539 			page_nr += cpu * nr_pages_per_cpu;  in locking_thread()
572 	uffd_test_ops->alias_mapping(&uffdio_copy->dst,  in retry_copy_page()
573 				     uffdio_copy->len,  in retry_copy_page()
577 		if (uffdio_copy->copy != -EEXIST)  in retry_copy_page()
579 			    (int64_t)uffdio_copy->copy);  in retry_copy_page()
582 		    (int64_t)uffdio_copy->copy);  in retry_copy_page()
602 		if (uffdio_copy.copy != -EEXIST)  in __copy_page()
649 	if (msg->event != UFFD_EVENT_PAGEFAULT)  in uffd_handle_page_fault()
650 		err("unexpected msg event %u", msg->event);  in uffd_handle_page_fault()
652 	if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {  in uffd_handle_page_fault()
654 		wp_range(uffd, msg->arg.pagefault.address, page_size, false);  in uffd_handle_page_fault()
655 		stats->wp_faults++;  in uffd_handle_page_fault()
656 	} else if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) {  in uffd_handle_page_fault()
668 		 * area_dst (non-UFFD-registered) and area_dst_alias  in uffd_handle_page_fault()
669 		 * (UFFD-registered).  in uffd_handle_page_fault()
673 				   ((char *)msg->arg.pagefault.address -  in uffd_handle_page_fault()
677 		continue_range(uffd, msg->arg.pagefault.address, page_size);  in uffd_handle_page_fault()
678 		stats->minor_faults++;  in uffd_handle_page_fault()
681 		if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)  in uffd_handle_page_fault()
684 		offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;  in uffd_handle_page_fault()
685 		offset &= ~(page_size-1);  in uffd_handle_page_fault()
688 			stats->missing_faults++;  in uffd_handle_page_fault()
695 	unsigned long cpu = stats->cpu;  in uffd_poll_thread()  local
704 	pollfd[1].fd = pipefd[cpu*2];  in uffd_poll_thread()
708 		ret = poll(pollfd, 2, -1);  in uffd_poll_thread()
734 			uffd_reg.range.len = msg.arg.remove.end -  in uffd_poll_thread()
769 	unsigned long cpu = (unsigned long) arg;  in background_thread()  local
772 	start_nr = cpu * nr_pages_per_cpu;  in background_thread()
773 	end_nr = (cpu+1) * nr_pages_per_cpu;  in background_thread()
781 	 * If we need to test uffd-wp, set it up now.  Then we'll have  in background_thread()
783 	 * can be write-protected for testing  in background_thread()
801 	unsigned long cpu;  in stress()  local
807 	for (cpu = 0; cpu < nr_cpus; cpu++) {  in stress()
808 		if (pthread_create(&locking_threads[cpu], &attr,  in stress()
809 				   locking_thread, (void *)cpu))  in stress()
812 			if (pthread_create(&uffd_threads[cpu], &attr,  in stress()
814 					   (void *)&uffd_stats[cpu]))  in stress()
817 			if (pthread_create(&uffd_threads[cpu], &attr,  in stress()
819 					   (void *)&uffd_stats[cpu]))  in stress()
823 		if (pthread_create(&background_threads[cpu], &attr,  in stress()
824 				   background_thread, (void *)cpu))  in stress()
827 	for (cpu = 0; cpu < nr_cpus; cpu++)  in stress()
828 		if (pthread_join(background_threads[cpu], NULL))  in stress()
836 	 * area_src (but they're guaranteed to get -EEXIST from  in stress()
840 	uffd_test_ops->release_pages(area_src);  in stress()
843 	for (cpu = 0; cpu < nr_cpus; cpu++)  in stress()
844 		if (pthread_join(locking_threads[cpu], NULL))  in stress()
847 	for (cpu = 0; cpu < nr_cpus; cpu++) {  in stress()
850 			if (write(pipefd[cpu*2+1], &c, 1) != 1)  in stress()
852 			if (pthread_join(uffd_threads[cpu],  in stress()
853 					 (void *)&uffd_stats[cpu]))  in stress()
856 			if (pthread_cancel(uffd_threads[cpu]))  in stress()
858 			if (pthread_join(uffd_threads[cpu], NULL))  in stress()
878  * For non-cooperative userfaultfd test we fork() a process that will
893  * test robustness use case - we release monitored area, fork a process
900 	unsigned long nr;  in faulting_process()  local
919 		lastnr = (unsigned long)-1;  in faulting_process()
922 	for (nr = 0; nr < split_nr_pages; nr++) {  in faulting_process()
924 		unsigned long offset = nr * page_size;  in faulting_process()
928 				if (steps == 1 && nr == lastnr)  in faulting_process()
931 				lastnr = nr;  in faulting_process()
953 		count = *area_count(area_dst, nr);  in faulting_process()
954 		if (count != count_verify[nr])  in faulting_process()
955 			err("nr %lu memory corruption %llu %llu\n",  in faulting_process()
956 			    nr, count, count_verify[nr]);  in faulting_process()
961 		*area_count(area_dst, nr) = count;  in faulting_process()
977 	for (; nr < nr_pages; nr++) {  in faulting_process()
978 		count = *area_count(area_dst, nr);  in faulting_process()
979 		if (count != count_verify[nr]) {  in faulting_process()
980 			err("nr %lu memory corruption %llu %llu\n",  in faulting_process()
981 			    nr, count, count_verify[nr]);  in faulting_process()
987 		*area_count(area_dst, nr) = count;  in faulting_process()
990 	uffd_test_ops->release_pages(area_dst);  in faulting_process()
992 	for (nr = 0; nr < nr_pages; nr++)  in faulting_process()
993 		if (my_bcmp(area_dst + nr * page_size, zeropage, page_size))  in faulting_process()
994 			err("nr %lu is not zero", nr);  in faulting_process()
1003 	uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,  in retry_uffdio_zeropage()
1004 				     uffdio_zeropage->range.len,  in retry_uffdio_zeropage()
1007 		if (uffdio_zeropage->zeropage != -EEXIST)  in retry_uffdio_zeropage()
1009 			    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
1012 		    (int64_t)uffdio_zeropage->zeropage);  in retry_uffdio_zeropage()
1023 	has_zeropage = uffd_test_ops->expected_ioctls & (1 << _UFFDIO_ZEROPAGE);  in __uffdio_zeropage()
1036 		else if (res != -EINVAL)  in __uffdio_zeropage()
1037 			err("UFFDIO_ZEROPAGE not -EINVAL");  in __uffdio_zeropage()
1079 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_zeropage_test()
1118 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_events_test()
1172 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_sig_test()
1179 	uffd_test_ops->release_pages(area_dst);  in userfaultfd_sig_test()
1246 	expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_minor_test()
1252 	 * After registering with UFFD, populate the non-UFFD-registered side of  in userfaultfd_minor_test()
1256 		memset(area_dst + (p * page_size), p % ((uint8_t)-1),  in userfaultfd_minor_test()
1264 	 * Read each of the pages back using the UFFD-registered mapping. We  in userfaultfd_minor_test()
1266 	 * fault. uffd_poll_thread will resolve the fault by bit-flipping the  in userfaultfd_minor_test()
1274 		expected_byte = ~((uint8_t)(p % ((uint8_t)-1)));  in userfaultfd_minor_test()
1295 	unsigned long nr;  in userfaultfd_stress()  local
1311 	while (bounces--) {  in userfaultfd_stress()
1339 		expected_ioctls = uffd_test_ops->expected_ioctls;  in userfaultfd_stress()
1359 		 * return -EEXIST). The problem comes at the next  in userfaultfd_stress()
1364 		 * area_src would lead to -EEXIST failure during the  in userfaultfd_stress()
1375 		uffd_test_ops->release_pages(area_dst);  in userfaultfd_stress()
1400 			for (nr = 0; nr < nr_pages; nr++)  in userfaultfd_stress()
1401 				if (*area_count(area_dst, nr) != count_verify[nr])  in userfaultfd_stress()
1403 					    *area_count(area_src, nr),  in userfaultfd_stress()
1404 					    count_verify[nr], nr);  in userfaultfd_stress()
1423  * Copied from mlock2-tests.c
1451 		/* Only enable write-protect test for anonymous test */  in set_test_type()