1*4882a593Smuzhiyun /*
2*4882a593Smuzhiyun * Stress test for transparent huge pages, memory compaction and migration.
3*4882a593Smuzhiyun *
4*4882a593Smuzhiyun * Authors: Konstantin Khlebnikov <koct9i@gmail.com>
5*4882a593Smuzhiyun *
6*4882a593Smuzhiyun * This is free and unencumbered software released into the public domain.
7*4882a593Smuzhiyun */
8*4882a593Smuzhiyun
9*4882a593Smuzhiyun #include <stdlib.h>
10*4882a593Smuzhiyun #include <stdio.h>
11*4882a593Smuzhiyun #include <stdint.h>
12*4882a593Smuzhiyun #include <err.h>
13*4882a593Smuzhiyun #include <time.h>
14*4882a593Smuzhiyun #include <unistd.h>
15*4882a593Smuzhiyun #include <fcntl.h>
16*4882a593Smuzhiyun #include <string.h>
17*4882a593Smuzhiyun #include <sys/mman.h>
18*4882a593Smuzhiyun
19*4882a593Smuzhiyun #define PAGE_SHIFT 12
20*4882a593Smuzhiyun #define HPAGE_SHIFT 21
21*4882a593Smuzhiyun
22*4882a593Smuzhiyun #define PAGE_SIZE (1 << PAGE_SHIFT)
23*4882a593Smuzhiyun #define HPAGE_SIZE (1 << HPAGE_SHIFT)
24*4882a593Smuzhiyun
25*4882a593Smuzhiyun #define PAGEMAP_PRESENT(ent) (((ent) & (1ull << 63)) != 0)
26*4882a593Smuzhiyun #define PAGEMAP_PFN(ent) ((ent) & ((1ull << 55) - 1))
27*4882a593Smuzhiyun
28*4882a593Smuzhiyun int pagemap_fd;
29*4882a593Smuzhiyun
allocate_transhuge(void * ptr)30*4882a593Smuzhiyun int64_t allocate_transhuge(void *ptr)
31*4882a593Smuzhiyun {
32*4882a593Smuzhiyun uint64_t ent[2];
33*4882a593Smuzhiyun
34*4882a593Smuzhiyun /* drop pmd */
35*4882a593Smuzhiyun if (mmap(ptr, HPAGE_SIZE, PROT_READ | PROT_WRITE,
36*4882a593Smuzhiyun MAP_FIXED | MAP_ANONYMOUS |
37*4882a593Smuzhiyun MAP_NORESERVE | MAP_PRIVATE, -1, 0) != ptr)
38*4882a593Smuzhiyun errx(2, "mmap transhuge");
39*4882a593Smuzhiyun
40*4882a593Smuzhiyun if (madvise(ptr, HPAGE_SIZE, MADV_HUGEPAGE))
41*4882a593Smuzhiyun err(2, "MADV_HUGEPAGE");
42*4882a593Smuzhiyun
43*4882a593Smuzhiyun /* allocate transparent huge page */
44*4882a593Smuzhiyun *(volatile void **)ptr = ptr;
45*4882a593Smuzhiyun
46*4882a593Smuzhiyun if (pread(pagemap_fd, ent, sizeof(ent),
47*4882a593Smuzhiyun (uintptr_t)ptr >> (PAGE_SHIFT - 3)) != sizeof(ent))
48*4882a593Smuzhiyun err(2, "read pagemap");
49*4882a593Smuzhiyun
50*4882a593Smuzhiyun if (PAGEMAP_PRESENT(ent[0]) && PAGEMAP_PRESENT(ent[1]) &&
51*4882a593Smuzhiyun PAGEMAP_PFN(ent[0]) + 1 == PAGEMAP_PFN(ent[1]) &&
52*4882a593Smuzhiyun !(PAGEMAP_PFN(ent[0]) & ((1 << (HPAGE_SHIFT - PAGE_SHIFT)) - 1)))
53*4882a593Smuzhiyun return PAGEMAP_PFN(ent[0]);
54*4882a593Smuzhiyun
55*4882a593Smuzhiyun return -1;
56*4882a593Smuzhiyun }
57*4882a593Smuzhiyun
main(int argc,char ** argv)58*4882a593Smuzhiyun int main(int argc, char **argv)
59*4882a593Smuzhiyun {
60*4882a593Smuzhiyun size_t ram, len;
61*4882a593Smuzhiyun void *ptr, *p;
62*4882a593Smuzhiyun struct timespec a, b;
63*4882a593Smuzhiyun double s;
64*4882a593Smuzhiyun uint8_t *map;
65*4882a593Smuzhiyun size_t map_len;
66*4882a593Smuzhiyun
67*4882a593Smuzhiyun ram = sysconf(_SC_PHYS_PAGES);
68*4882a593Smuzhiyun if (ram > SIZE_MAX / sysconf(_SC_PAGESIZE) / 4)
69*4882a593Smuzhiyun ram = SIZE_MAX / 4;
70*4882a593Smuzhiyun else
71*4882a593Smuzhiyun ram *= sysconf(_SC_PAGESIZE);
72*4882a593Smuzhiyun
73*4882a593Smuzhiyun if (argc == 1)
74*4882a593Smuzhiyun len = ram;
75*4882a593Smuzhiyun else if (!strcmp(argv[1], "-h"))
76*4882a593Smuzhiyun errx(1, "usage: %s [size in MiB]", argv[0]);
77*4882a593Smuzhiyun else
78*4882a593Smuzhiyun len = atoll(argv[1]) << 20;
79*4882a593Smuzhiyun
80*4882a593Smuzhiyun warnx("allocate %zd transhuge pages, using %zd MiB virtual memory"
81*4882a593Smuzhiyun " and %zd MiB of ram", len >> HPAGE_SHIFT, len >> 20,
82*4882a593Smuzhiyun len >> (20 + HPAGE_SHIFT - PAGE_SHIFT - 1));
83*4882a593Smuzhiyun
84*4882a593Smuzhiyun pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
85*4882a593Smuzhiyun if (pagemap_fd < 0)
86*4882a593Smuzhiyun err(2, "open pagemap");
87*4882a593Smuzhiyun
88*4882a593Smuzhiyun len -= len % HPAGE_SIZE;
89*4882a593Smuzhiyun ptr = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE,
90*4882a593Smuzhiyun MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0);
91*4882a593Smuzhiyun if (ptr == MAP_FAILED)
92*4882a593Smuzhiyun err(2, "initial mmap");
93*4882a593Smuzhiyun ptr += HPAGE_SIZE - (uintptr_t)ptr % HPAGE_SIZE;
94*4882a593Smuzhiyun
95*4882a593Smuzhiyun if (madvise(ptr, len, MADV_HUGEPAGE))
96*4882a593Smuzhiyun err(2, "MADV_HUGEPAGE");
97*4882a593Smuzhiyun
98*4882a593Smuzhiyun map_len = ram >> (HPAGE_SHIFT - 1);
99*4882a593Smuzhiyun map = malloc(map_len);
100*4882a593Smuzhiyun if (!map)
101*4882a593Smuzhiyun errx(2, "map malloc");
102*4882a593Smuzhiyun
103*4882a593Smuzhiyun while (1) {
104*4882a593Smuzhiyun int nr_succeed = 0, nr_failed = 0, nr_pages = 0;
105*4882a593Smuzhiyun
106*4882a593Smuzhiyun memset(map, 0, map_len);
107*4882a593Smuzhiyun
108*4882a593Smuzhiyun clock_gettime(CLOCK_MONOTONIC, &a);
109*4882a593Smuzhiyun for (p = ptr; p < ptr + len; p += HPAGE_SIZE) {
110*4882a593Smuzhiyun int64_t pfn;
111*4882a593Smuzhiyun
112*4882a593Smuzhiyun pfn = allocate_transhuge(p);
113*4882a593Smuzhiyun
114*4882a593Smuzhiyun if (pfn < 0) {
115*4882a593Smuzhiyun nr_failed++;
116*4882a593Smuzhiyun } else {
117*4882a593Smuzhiyun size_t idx = pfn >> (HPAGE_SHIFT - PAGE_SHIFT);
118*4882a593Smuzhiyun
119*4882a593Smuzhiyun nr_succeed++;
120*4882a593Smuzhiyun if (idx >= map_len) {
121*4882a593Smuzhiyun map = realloc(map, idx + 1);
122*4882a593Smuzhiyun if (!map)
123*4882a593Smuzhiyun errx(2, "map realloc");
124*4882a593Smuzhiyun memset(map + map_len, 0, idx + 1 - map_len);
125*4882a593Smuzhiyun map_len = idx + 1;
126*4882a593Smuzhiyun }
127*4882a593Smuzhiyun if (!map[idx])
128*4882a593Smuzhiyun nr_pages++;
129*4882a593Smuzhiyun map[idx] = 1;
130*4882a593Smuzhiyun }
131*4882a593Smuzhiyun
132*4882a593Smuzhiyun /* split transhuge page, keep last page */
133*4882a593Smuzhiyun if (madvise(p, HPAGE_SIZE - PAGE_SIZE, MADV_DONTNEED))
134*4882a593Smuzhiyun err(2, "MADV_DONTNEED");
135*4882a593Smuzhiyun }
136*4882a593Smuzhiyun clock_gettime(CLOCK_MONOTONIC, &b);
137*4882a593Smuzhiyun s = b.tv_sec - a.tv_sec + (b.tv_nsec - a.tv_nsec) / 1000000000.;
138*4882a593Smuzhiyun
139*4882a593Smuzhiyun warnx("%.3f s/loop, %.3f ms/page, %10.3f MiB/s\t"
140*4882a593Smuzhiyun "%4d succeed, %4d failed, %4d different pages",
141*4882a593Smuzhiyun s, s * 1000 / (len >> HPAGE_SHIFT), len / s / (1 << 20),
142*4882a593Smuzhiyun nr_succeed, nr_failed, nr_pages);
143*4882a593Smuzhiyun }
144*4882a593Smuzhiyun }
145