1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Fault injection for both 32 and 64bit guests.
4 *
5 * Copyright (C) 2012,2013 - ARM Ltd
6 * Author: Marc Zyngier <marc.zyngier@arm.com>
7 *
8 * Based on arch/arm/kvm/emulate.c
9 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
10 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
11 */
12
13 #include <hyp/adjust_pc.h>
14 #include <linux/kvm_host.h>
15 #include <asm/kvm_emulate.h>
16
17 #if !defined (__KVM_NVHE_HYPERVISOR__) && !defined (__KVM_VHE_HYPERVISOR__)
18 #error Hypervisor code only!
19 #endif
20
__vcpu_read_sys_reg(const struct kvm_vcpu * vcpu,int reg)21 static inline u64 __vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
22 {
23 u64 val;
24
25 if (__vcpu_read_sys_reg_from_cpu(reg, &val))
26 return val;
27
28 return __vcpu_sys_reg(vcpu, reg);
29 }
30
__vcpu_write_sys_reg(struct kvm_vcpu * vcpu,u64 val,int reg)31 static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
32 {
33 if (__vcpu_write_sys_reg_to_cpu(val, reg))
34 return;
35
36 __vcpu_sys_reg(vcpu, reg) = val;
37 }
38
__vcpu_write_spsr(struct kvm_vcpu * vcpu,u64 val)39 static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, u64 val)
40 {
41 if (has_vhe())
42 write_sysreg_el1(val, SYS_SPSR);
43 else
44 __vcpu_sys_reg(vcpu, SPSR_EL1) = val;
45 }
46
__vcpu_write_spsr_abt(struct kvm_vcpu * vcpu,u64 val)47 static void __vcpu_write_spsr_abt(struct kvm_vcpu *vcpu, u64 val)
48 {
49 if (has_vhe())
50 write_sysreg(val, spsr_abt);
51 else
52 vcpu->arch.ctxt.spsr_abt = val;
53 }
54
__vcpu_write_spsr_und(struct kvm_vcpu * vcpu,u64 val)55 static void __vcpu_write_spsr_und(struct kvm_vcpu *vcpu, u64 val)
56 {
57 if (has_vhe())
58 write_sysreg(val, spsr_und);
59 else
60 vcpu->arch.ctxt.spsr_und = val;
61 }
62
63 /*
64 * This performs the exception entry at a given EL (@target_mode), stashing PC
65 * and PSTATE into ELR and SPSR respectively, and compute the new PC/PSTATE.
66 * The EL passed to this function *must* be a non-secure, privileged mode with
67 * bit 0 being set (PSTATE.SP == 1).
68 *
69 * When an exception is taken, most PSTATE fields are left unchanged in the
70 * handler. However, some are explicitly overridden (e.g. M[4:0]). Luckily all
71 * of the inherited bits have the same position in the AArch64/AArch32 SPSR_ELx
72 * layouts, so we don't need to shuffle these for exceptions from AArch32 EL0.
73 *
74 * For the SPSR_ELx layout for AArch64, see ARM DDI 0487E.a page C5-429.
75 * For the SPSR_ELx layout for AArch32, see ARM DDI 0487E.a page C5-426.
76 *
77 * Here we manipulate the fields in order of the AArch64 SPSR_ELx layout, from
78 * MSB to LSB.
79 */
enter_exception64(struct kvm_vcpu * vcpu,unsigned long target_mode,enum exception_type type)80 static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode,
81 enum exception_type type)
82 {
83 unsigned long sctlr, vbar, old, new, mode;
84 u64 exc_offset;
85
86 mode = *vcpu_cpsr(vcpu) & (PSR_MODE_MASK | PSR_MODE32_BIT);
87
88 if (mode == target_mode)
89 exc_offset = CURRENT_EL_SP_ELx_VECTOR;
90 else if ((mode | PSR_MODE_THREAD_BIT) == target_mode)
91 exc_offset = CURRENT_EL_SP_EL0_VECTOR;
92 else if (!(mode & PSR_MODE32_BIT))
93 exc_offset = LOWER_EL_AArch64_VECTOR;
94 else
95 exc_offset = LOWER_EL_AArch32_VECTOR;
96
97 switch (target_mode) {
98 case PSR_MODE_EL1h:
99 vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL1);
100 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
101 __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1);
102 break;
103 default:
104 /* Don't do that */
105 BUG();
106 }
107
108 *vcpu_pc(vcpu) = vbar + exc_offset + type;
109
110 old = *vcpu_cpsr(vcpu);
111 new = 0;
112
113 new |= (old & PSR_N_BIT);
114 new |= (old & PSR_Z_BIT);
115 new |= (old & PSR_C_BIT);
116 new |= (old & PSR_V_BIT);
117
118 // TODO: TCO (if/when ARMv8.5-MemTag is exposed to guests)
119
120 new |= (old & PSR_DIT_BIT);
121
122 // PSTATE.UAO is set to zero upon any exception to AArch64
123 // See ARM DDI 0487E.a, page D5-2579.
124
125 // PSTATE.PAN is unchanged unless SCTLR_ELx.SPAN == 0b0
126 // SCTLR_ELx.SPAN is RES1 when ARMv8.1-PAN is not implemented
127 // See ARM DDI 0487E.a, page D5-2578.
128 new |= (old & PSR_PAN_BIT);
129 if (!(sctlr & SCTLR_EL1_SPAN))
130 new |= PSR_PAN_BIT;
131
132 // PSTATE.SS is set to zero upon any exception to AArch64
133 // See ARM DDI 0487E.a, page D2-2452.
134
135 // PSTATE.IL is set to zero upon any exception to AArch64
136 // See ARM DDI 0487E.a, page D1-2306.
137
138 // PSTATE.SSBS is set to SCTLR_ELx.DSSBS upon any exception to AArch64
139 // See ARM DDI 0487E.a, page D13-3258
140 if (sctlr & SCTLR_ELx_DSSBS)
141 new |= PSR_SSBS_BIT;
142
143 // PSTATE.BTYPE is set to zero upon any exception to AArch64
144 // See ARM DDI 0487E.a, pages D1-2293 to D1-2294.
145
146 new |= PSR_D_BIT;
147 new |= PSR_A_BIT;
148 new |= PSR_I_BIT;
149 new |= PSR_F_BIT;
150
151 new |= target_mode;
152
153 *vcpu_cpsr(vcpu) = new;
154 __vcpu_write_spsr(vcpu, old);
155 }
156
157 /*
158 * When an exception is taken, most CPSR fields are left unchanged in the
159 * handler. However, some are explicitly overridden (e.g. M[4:0]).
160 *
161 * The SPSR/SPSR_ELx layouts differ, and the below is intended to work with
162 * either format. Note: SPSR.J bit doesn't exist in SPSR_ELx, but this bit was
163 * obsoleted by the ARMv7 virtualization extensions and is RES0.
164 *
165 * For the SPSR layout seen from AArch32, see:
166 * - ARM DDI 0406C.d, page B1-1148
167 * - ARM DDI 0487E.a, page G8-6264
168 *
169 * For the SPSR_ELx layout for AArch32 seen from AArch64, see:
170 * - ARM DDI 0487E.a, page C5-426
171 *
172 * Here we manipulate the fields in order of the AArch32 SPSR_ELx layout, from
173 * MSB to LSB.
174 */
get_except32_cpsr(struct kvm_vcpu * vcpu,u32 mode)175 static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode)
176 {
177 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
178 unsigned long old, new;
179
180 old = *vcpu_cpsr(vcpu);
181 new = 0;
182
183 new |= (old & PSR_AA32_N_BIT);
184 new |= (old & PSR_AA32_Z_BIT);
185 new |= (old & PSR_AA32_C_BIT);
186 new |= (old & PSR_AA32_V_BIT);
187 new |= (old & PSR_AA32_Q_BIT);
188
189 // CPSR.IT[7:0] are set to zero upon any exception
190 // See ARM DDI 0487E.a, section G1.12.3
191 // See ARM DDI 0406C.d, section B1.8.3
192
193 new |= (old & PSR_AA32_DIT_BIT);
194
195 // CPSR.SSBS is set to SCTLR.DSSBS upon any exception
196 // See ARM DDI 0487E.a, page G8-6244
197 if (sctlr & BIT(31))
198 new |= PSR_AA32_SSBS_BIT;
199
200 // CPSR.PAN is unchanged unless SCTLR.SPAN == 0b0
201 // SCTLR.SPAN is RES1 when ARMv8.1-PAN is not implemented
202 // See ARM DDI 0487E.a, page G8-6246
203 new |= (old & PSR_AA32_PAN_BIT);
204 if (!(sctlr & BIT(23)))
205 new |= PSR_AA32_PAN_BIT;
206
207 // SS does not exist in AArch32, so ignore
208
209 // CPSR.IL is set to zero upon any exception
210 // See ARM DDI 0487E.a, page G1-5527
211
212 new |= (old & PSR_AA32_GE_MASK);
213
214 // CPSR.IT[7:0] are set to zero upon any exception
215 // See prior comment above
216
217 // CPSR.E is set to SCTLR.EE upon any exception
218 // See ARM DDI 0487E.a, page G8-6245
219 // See ARM DDI 0406C.d, page B4-1701
220 if (sctlr & BIT(25))
221 new |= PSR_AA32_E_BIT;
222
223 // CPSR.A is unchanged upon an exception to Undefined, Supervisor
224 // CPSR.A is set upon an exception to other modes
225 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
226 // See ARM DDI 0406C.d, page B1-1182
227 new |= (old & PSR_AA32_A_BIT);
228 if (mode != PSR_AA32_MODE_UND && mode != PSR_AA32_MODE_SVC)
229 new |= PSR_AA32_A_BIT;
230
231 // CPSR.I is set upon any exception
232 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
233 // See ARM DDI 0406C.d, page B1-1182
234 new |= PSR_AA32_I_BIT;
235
236 // CPSR.F is set upon an exception to FIQ
237 // CPSR.F is unchanged upon an exception to other modes
238 // See ARM DDI 0487E.a, pages G1-5515 to G1-5516
239 // See ARM DDI 0406C.d, page B1-1182
240 new |= (old & PSR_AA32_F_BIT);
241 if (mode == PSR_AA32_MODE_FIQ)
242 new |= PSR_AA32_F_BIT;
243
244 // CPSR.T is set to SCTLR.TE upon any exception
245 // See ARM DDI 0487E.a, page G8-5514
246 // See ARM DDI 0406C.d, page B1-1181
247 if (sctlr & BIT(30))
248 new |= PSR_AA32_T_BIT;
249
250 new |= mode;
251
252 return new;
253 }
254
255 /*
256 * Table taken from ARMv8 ARM DDI0487B-B, table G1-10.
257 */
258 static const u8 return_offsets[8][2] = {
259 [0] = { 0, 0 }, /* Reset, unused */
260 [1] = { 4, 2 }, /* Undefined */
261 [2] = { 0, 0 }, /* SVC, unused */
262 [3] = { 4, 4 }, /* Prefetch abort */
263 [4] = { 8, 8 }, /* Data abort */
264 [5] = { 0, 0 }, /* HVC, unused */
265 [6] = { 4, 4 }, /* IRQ, unused */
266 [7] = { 4, 4 }, /* FIQ, unused */
267 };
268
enter_exception32(struct kvm_vcpu * vcpu,u32 mode,u32 vect_offset)269 static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset)
270 {
271 unsigned long spsr = *vcpu_cpsr(vcpu);
272 bool is_thumb = (spsr & PSR_AA32_T_BIT);
273 u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
274 u32 return_address;
275
276 *vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode);
277 return_address = *vcpu_pc(vcpu);
278 return_address += return_offsets[vect_offset >> 2][is_thumb];
279
280 /* KVM only enters the ABT and UND modes, so only deal with those */
281 switch(mode) {
282 case PSR_AA32_MODE_ABT:
283 __vcpu_write_spsr_abt(vcpu, host_spsr_to_spsr32(spsr));
284 vcpu_gp_regs(vcpu)->compat_lr_abt = return_address;
285 break;
286
287 case PSR_AA32_MODE_UND:
288 __vcpu_write_spsr_und(vcpu, host_spsr_to_spsr32(spsr));
289 vcpu_gp_regs(vcpu)->compat_lr_und = return_address;
290 break;
291 }
292
293 /* Branch to exception vector */
294 if (sctlr & (1 << 13))
295 vect_offset += 0xffff0000;
296 else /* always have security exceptions */
297 vect_offset += __vcpu_read_sys_reg(vcpu, VBAR_EL1);
298
299 *vcpu_pc(vcpu) = vect_offset;
300 }
301
kvm_inject_exception(struct kvm_vcpu * vcpu)302 void kvm_inject_exception(struct kvm_vcpu *vcpu)
303 {
304 if (vcpu_el1_is_32bit(vcpu)) {
305 switch (vcpu->arch.flags & KVM_ARM64_EXCEPT_MASK) {
306 case KVM_ARM64_EXCEPT_AA32_UND:
307 enter_exception32(vcpu, PSR_AA32_MODE_UND, 4);
308 break;
309 case KVM_ARM64_EXCEPT_AA32_IABT:
310 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 12);
311 break;
312 case KVM_ARM64_EXCEPT_AA32_DABT:
313 enter_exception32(vcpu, PSR_AA32_MODE_ABT, 16);
314 break;
315 default:
316 /* Err... */
317 break;
318 }
319 } else {
320 switch (vcpu->arch.flags & KVM_ARM64_EXCEPT_MASK) {
321 case (KVM_ARM64_EXCEPT_AA64_ELx_SYNC |
322 KVM_ARM64_EXCEPT_AA64_EL1):
323 enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync);
324 break;
325 default:
326 /*
327 * Only EL1_SYNC makes sense so far, EL2_{SYNC,IRQ}
328 * will be implemented at some point. Everything
329 * else gets silently ignored.
330 */
331 break;
332 }
333 }
334 }
335