1*4882a593Smuzhiyun /* SPDX-License-Identifier: GPL-2.0-or-later */
2*4882a593Smuzhiyun /*
3*4882a593Smuzhiyun * Copyright 2013 Red Hat Inc.
4*4882a593Smuzhiyun *
5*4882a593Smuzhiyun * Authors: Jérôme Glisse <jglisse@redhat.com>
6*4882a593Smuzhiyun *
7*4882a593Smuzhiyun * See Documentation/vm/hmm.rst for reasons and overview of what HMM is.
8*4882a593Smuzhiyun */
9*4882a593Smuzhiyun #ifndef LINUX_HMM_H
10*4882a593Smuzhiyun #define LINUX_HMM_H
11*4882a593Smuzhiyun
12*4882a593Smuzhiyun #include <linux/kconfig.h>
13*4882a593Smuzhiyun #include <linux/pgtable.h>
14*4882a593Smuzhiyun
15*4882a593Smuzhiyun #include <linux/device.h>
16*4882a593Smuzhiyun #include <linux/migrate.h>
17*4882a593Smuzhiyun #include <linux/memremap.h>
18*4882a593Smuzhiyun #include <linux/completion.h>
19*4882a593Smuzhiyun #include <linux/mmu_notifier.h>
20*4882a593Smuzhiyun
21*4882a593Smuzhiyun /*
22*4882a593Smuzhiyun * On output:
23*4882a593Smuzhiyun * 0 - The page is faultable and a future call with
24*4882a593Smuzhiyun * HMM_PFN_REQ_FAULT could succeed.
25*4882a593Smuzhiyun * HMM_PFN_VALID - the pfn field points to a valid PFN. This PFN is at
26*4882a593Smuzhiyun * least readable. If dev_private_owner is !NULL then this could
27*4882a593Smuzhiyun * point at a DEVICE_PRIVATE page.
28*4882a593Smuzhiyun * HMM_PFN_WRITE - if the page memory can be written to (requires HMM_PFN_VALID)
29*4882a593Smuzhiyun * HMM_PFN_ERROR - accessing the pfn is impossible and the device should
30*4882a593Smuzhiyun * fail. ie poisoned memory, special pages, no vma, etc
31*4882a593Smuzhiyun *
32*4882a593Smuzhiyun * On input:
33*4882a593Smuzhiyun * 0 - Return the current state of the page, do not fault it.
34*4882a593Smuzhiyun * HMM_PFN_REQ_FAULT - The output must have HMM_PFN_VALID or hmm_range_fault()
35*4882a593Smuzhiyun * will fail
36*4882a593Smuzhiyun * HMM_PFN_REQ_WRITE - The output must have HMM_PFN_WRITE or hmm_range_fault()
37*4882a593Smuzhiyun * will fail. Must be combined with HMM_PFN_REQ_FAULT.
38*4882a593Smuzhiyun */
39*4882a593Smuzhiyun enum hmm_pfn_flags {
40*4882a593Smuzhiyun /* Output fields and flags */
41*4882a593Smuzhiyun HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1),
42*4882a593Smuzhiyun HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2),
43*4882a593Smuzhiyun HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3),
44*4882a593Smuzhiyun HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8),
45*4882a593Smuzhiyun
46*4882a593Smuzhiyun /* Input flags */
47*4882a593Smuzhiyun HMM_PFN_REQ_FAULT = HMM_PFN_VALID,
48*4882a593Smuzhiyun HMM_PFN_REQ_WRITE = HMM_PFN_WRITE,
49*4882a593Smuzhiyun
50*4882a593Smuzhiyun HMM_PFN_FLAGS = 0xFFUL << HMM_PFN_ORDER_SHIFT,
51*4882a593Smuzhiyun };
52*4882a593Smuzhiyun
53*4882a593Smuzhiyun /*
54*4882a593Smuzhiyun * hmm_pfn_to_page() - return struct page pointed to by a device entry
55*4882a593Smuzhiyun *
56*4882a593Smuzhiyun * This must be called under the caller 'user_lock' after a successful
57*4882a593Smuzhiyun * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID
58*4882a593Smuzhiyun * already.
59*4882a593Smuzhiyun */
hmm_pfn_to_page(unsigned long hmm_pfn)60*4882a593Smuzhiyun static inline struct page *hmm_pfn_to_page(unsigned long hmm_pfn)
61*4882a593Smuzhiyun {
62*4882a593Smuzhiyun return pfn_to_page(hmm_pfn & ~HMM_PFN_FLAGS);
63*4882a593Smuzhiyun }
64*4882a593Smuzhiyun
65*4882a593Smuzhiyun /*
66*4882a593Smuzhiyun * hmm_pfn_to_map_order() - return the CPU mapping size order
67*4882a593Smuzhiyun *
68*4882a593Smuzhiyun * This is optionally useful to optimize processing of the pfn result
69*4882a593Smuzhiyun * array. It indicates that the page starts at the order aligned VA and is
70*4882a593Smuzhiyun * 1<<order bytes long. Every pfn within an high order page will have the
71*4882a593Smuzhiyun * same pfn flags, both access protections and the map_order. The caller must
72*4882a593Smuzhiyun * be careful with edge cases as the start and end VA of the given page may
73*4882a593Smuzhiyun * extend past the range used with hmm_range_fault().
74*4882a593Smuzhiyun *
75*4882a593Smuzhiyun * This must be called under the caller 'user_lock' after a successful
76*4882a593Smuzhiyun * mmu_interval_read_begin(). The caller must have tested for HMM_PFN_VALID
77*4882a593Smuzhiyun * already.
78*4882a593Smuzhiyun */
hmm_pfn_to_map_order(unsigned long hmm_pfn)79*4882a593Smuzhiyun static inline unsigned int hmm_pfn_to_map_order(unsigned long hmm_pfn)
80*4882a593Smuzhiyun {
81*4882a593Smuzhiyun return (hmm_pfn >> HMM_PFN_ORDER_SHIFT) & 0x1F;
82*4882a593Smuzhiyun }
83*4882a593Smuzhiyun
84*4882a593Smuzhiyun /*
85*4882a593Smuzhiyun * struct hmm_range - track invalidation lock on virtual address range
86*4882a593Smuzhiyun *
87*4882a593Smuzhiyun * @notifier: a mmu_interval_notifier that includes the start/end
88*4882a593Smuzhiyun * @notifier_seq: result of mmu_interval_read_begin()
89*4882a593Smuzhiyun * @start: range virtual start address (inclusive)
90*4882a593Smuzhiyun * @end: range virtual end address (exclusive)
91*4882a593Smuzhiyun * @hmm_pfns: array of pfns (big enough for the range)
92*4882a593Smuzhiyun * @default_flags: default flags for the range (write, read, ... see hmm doc)
93*4882a593Smuzhiyun * @pfn_flags_mask: allows to mask pfn flags so that only default_flags matter
94*4882a593Smuzhiyun * @dev_private_owner: owner of device private pages
95*4882a593Smuzhiyun */
96*4882a593Smuzhiyun struct hmm_range {
97*4882a593Smuzhiyun struct mmu_interval_notifier *notifier;
98*4882a593Smuzhiyun unsigned long notifier_seq;
99*4882a593Smuzhiyun unsigned long start;
100*4882a593Smuzhiyun unsigned long end;
101*4882a593Smuzhiyun unsigned long *hmm_pfns;
102*4882a593Smuzhiyun unsigned long default_flags;
103*4882a593Smuzhiyun unsigned long pfn_flags_mask;
104*4882a593Smuzhiyun void *dev_private_owner;
105*4882a593Smuzhiyun };
106*4882a593Smuzhiyun
107*4882a593Smuzhiyun /*
108*4882a593Smuzhiyun * Please see Documentation/vm/hmm.rst for how to use the range API.
109*4882a593Smuzhiyun */
110*4882a593Smuzhiyun int hmm_range_fault(struct hmm_range *range);
111*4882a593Smuzhiyun
112*4882a593Smuzhiyun /*
113*4882a593Smuzhiyun * HMM_RANGE_DEFAULT_TIMEOUT - default timeout (ms) when waiting for a range
114*4882a593Smuzhiyun *
115*4882a593Smuzhiyun * When waiting for mmu notifiers we need some kind of time out otherwise we
116*4882a593Smuzhiyun * could potentialy wait for ever, 1000ms ie 1s sounds like a long time to
117*4882a593Smuzhiyun * wait already.
118*4882a593Smuzhiyun */
119*4882a593Smuzhiyun #define HMM_RANGE_DEFAULT_TIMEOUT 1000
120*4882a593Smuzhiyun
121*4882a593Smuzhiyun #endif /* LINUX_HMM_H */
122