1 // SPDX-License-Identifier: GPL-2.0-only
2 #define pr_fmt(fmt) "SMP alternatives: " fmt
21 #include <asm/text-patching.h>
31 #include <asm/asm-prototypes.h>
37 #define MAX_PATCH_LEN (255-1)
46 __setup("debug-alternative", debug_alt);
55 __setup("noreplace-smp", setup_noreplace_smp);
72 		for (j = 0; j < (len) - 1; j++)				\
81  * add to the array the offset that is equal to the sum of all sizes of
213 		 * the "k8_nops" than with the SDM-recommended NOPs.  in arch_init_ideal_nops()
268 		len -= noplen;  in add_nops()
279  * Are we looking at a near JMP with a 1 or 4-byte displacement.
293 	if (a->replacementlen != 5)  in recompute_jump()
299 	next_rip = repl_insn + a->replacementlen;  in recompute_jump()
302 	n_dspl = tgt_rip - orig_insn;  in recompute_jump()
306 	if (tgt_rip - orig_insn >= 0) {  in recompute_jump()
307 		if (n_dspl - 2 <= 127)  in recompute_jump()
311 	/* negative offset */  in recompute_jump()
313 		if (((n_dspl - 2) & 0xff) == (n_dspl - 2))  in recompute_jump()
320 	n_dspl -= 2;  in recompute_jump()
330 	n_dspl -= 5;  in recompute_jump()
344  * optimize_nops_range() - Optimize a sequence of single byte NOPs (0x90)
348  * @off: offset within @instr where the first NOP has been detected
364 	nnops = i - off;  in optimize_nops_range()
388 	 * Jump over the non-NOP insns and optimize single-byte NOPs into bigger  in optimize_nops()
411  * before SMP is initialized to avoid SMP problems with self modifying code.
426 	DPRINTK("alt table %px, -> %px", start, end);  in apply_alternatives()
439 		u16 feature = a->cpuid & ~ALTINSTR_FLAG_INV;  in apply_alternatives()
441 		instr = (u8 *)&a->instr_offset + a->instr_offset;  in apply_alternatives()
442 		replacement = (u8 *)&a->repl_offset + a->repl_offset;  in apply_alternatives()
443 		BUG_ON(a->instrlen > sizeof(insn_buff));  in apply_alternatives()
448 		 * - feature is present  in apply_alternatives()
449 		 * - feature not present but ALTINSTR_FLAG_INV is set to mean,  in apply_alternatives()
452 		if (!boot_cpu_has(feature) == !(a->cpuid & ALTINSTR_FLAG_INV))  in apply_alternatives()
456 			(a->cpuid & ALTINSTR_FLAG_INV) ? "!" : "",  in apply_alternatives()
459 			instr, instr, a->instrlen,  in apply_alternatives()
460 			replacement, a->replacementlen);  in apply_alternatives()
462 		DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);  in apply_alternatives()
463 		DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement);  in apply_alternatives()
465 		memcpy(insn_buff, replacement, a->replacementlen);  in apply_alternatives()
466 		insn_buff_sz = a->replacementlen;  in apply_alternatives()
469 		 * 0xe8 is a relative jump; fix the offset.  in apply_alternatives()
472 		 * accessing uninitialized bytes for zero-length replacements.  in apply_alternatives()
474 		if (a->replacementlen == 5 && *insn_buff == 0xe8) {  in apply_alternatives()
475 			*(s32 *)(insn_buff + 1) += replacement - instr;  in apply_alternatives()
476 			DPRINTK("Fix CALL offset: 0x%x, CALL 0x%lx",  in apply_alternatives()
481 		if (a->replacementlen && is_jmp(replacement[0]))  in apply_alternatives()
484 		for (; insn_buff_sz < a->instrlen; insn_buff_sz++)  in apply_alternatives()
492 		optimize_nops(instr, a->instrlen);  in apply_alternatives()
517 		return -1;  in emit_indirect()
522 		reg -= 8;  in emit_indirect()
556 	target = addr + insn->length + insn->immediate.value;  in patch_retpoline()
557 	reg = target - __x86_indirect_thunk_array;  in patch_retpoline()
560 		return -1;  in patch_retpoline()
567 		return -1;  in patch_retpoline()
569 	op = insn->opcode.bytes[0];  in patch_retpoline()
584 	/* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */  in patch_retpoline()
585 	if (op == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80) {  in patch_retpoline()
586 		cc = insn->opcode.bytes[1] & 0xf;  in patch_retpoline()
590 		bytes[i++] = insn->length - 2; /* sizeof(Jcc.d8) == 2 */  in patch_retpoline()
610 	for (; i < insn->length;)  in patch_retpoline()
617  * Generated by 'objtool --retpoline'.
667  * Rewrite the compiler generated return thunk tail-calls.
682 		return -1;  in patch_return()
686 	for (; i < insn->length;)  in patch_return()
713 			      "missing return thunk: %pS-%pS: %*ph",  in apply_returns()
796 	struct smp_alt_module *smp;  in alternatives_smp_module_add()  local
806 	smp = kzalloc(sizeof(*smp), GFP_KERNEL);  in alternatives_smp_module_add()
807 	if (NULL == smp)  in alternatives_smp_module_add()
808 		/* we'll run the (safe but slow) SMP code then ... */  in alternatives_smp_module_add()
811 	smp->mod	= mod;  in alternatives_smp_module_add()
812 	smp->name	= name;  in alternatives_smp_module_add()
813 	smp->locks	= locks;  in alternatives_smp_module_add()
814 	smp->locks_end	= locks_end;  in alternatives_smp_module_add()
815 	smp->text	= text;  in alternatives_smp_module_add()
816 	smp->text_end	= text_end;  in alternatives_smp_module_add()
817 	DPRINTK("locks %p -> %p, text %p -> %p, name %s\n",  in alternatives_smp_module_add()
818 		smp->locks, smp->locks_end,  in alternatives_smp_module_add()
819 		smp->text, smp->text_end, smp->name);  in alternatives_smp_module_add()
821 	list_add_tail(&smp->next, &smp_alt_modules);  in alternatives_smp_module_add()
834 		if (mod != item->mod)  in alternatives_smp_module_del()
836 		list_del(&item->next);  in alternatives_smp_module_del()
853 		pr_info("switching to SMP code\n");  in alternatives_enable_smp()
858 			alternatives_smp_lock(mod->locks, mod->locks_end,  in alternatives_enable_smp()
859 					      mod->text, mod->text_end);  in alternatives_enable_smp()
866  * Return 1 if the address range is reserved for SMP-alternatives.
879 		if (mod->text > text_end || mod->text_end < text_start)  in alternatives_text_reserved()
881 		for (poff = mod->locks; poff < mod->locks_end; poff++) {  in alternatives_text_reserved()
903 		BUG_ON(p->len > MAX_PATCH_LEN);  in apply_paravirt()
905 		memcpy(insn_buff, p->instr, p->len);  in apply_paravirt()
906 		used = pv_ops.init.patch(p->type, insn_buff, (unsigned long)p->instr, p->len);  in apply_paravirt()
908 		BUG_ON(used > p->len);  in apply_paravirt()
911 		add_nops(insn_buff + used, p->len - used);  in apply_paravirt()
912 		text_poke_early(p->instr, insn_buff, p->len);  in apply_paravirt()
920  * Self-test for the INT3 based CALL emulation code.
940 	struct pt_regs *regs = args->regs;  in int3_exception_notify()
948 	if (regs->ip - INT3_INSN_SIZE != int3_selftest_ip)  in int3_exception_notify()
959 		.priority	= INT_MAX-1, /* last */  in int3_selftest()
966 	 * Basically: int3_magic(&val); but really complicated :-)  in int3_selftest()
1030 		free_init_pages("SMP alternatives",  in alternative_instructions()
1043  * text_poke_early - Update instructions on a live kernel at boot time
1062 		 * Modules text is marked initially as non-executable, so the  in text_poke_early()
1063 		 * code cannot be running and speculative code-fetches are  in text_poke_early()
1088  * temporary page-table mappings that are required for these write operations to
1181 	 * flush_tlb_mm_range(), which is intended for non-global PTEs.  in __text_poke()
1186 	 * The lock is not really needed, but this allows to avoid open-coding.  in __text_poke()
1226 	 * Loading the previous page-table hierarchy requires a serializing  in __text_poke()
1228 	 * Xen-PV is assumed to serialize execution in a similar manner.  in __text_poke()
1252  * text_poke - Update instructions on a live kernel
1275  * text_poke_kgdb - Update instructions on a live kernel by kgdb
1327 	if (!arch_atomic_inc_not_zero(&desc->refs))  in try_get_desc()
1338 	arch_atomic_dec(&desc->refs);  in put_desc()
1343 	return _stext + tp->rel_addr;  in text_poke_addr()
1351 		return -1;  in patch_cmp()
1369 	 * bp_desc with non-zero refcount:  in poke_int3_handler()
1384 	ip = (void *) regs->ip - INT3_INSN_SIZE;  in poke_int3_handler()
1389 	if (unlikely(desc->nr_entries > 1)) {  in poke_int3_handler()
1390 		tp = __inline_bsearch(ip, desc->vec, desc->nr_entries,  in poke_int3_handler()
1396 		tp = desc->vec;  in poke_int3_handler()
1401 	ip += tp->len;  in poke_int3_handler()
1403 	switch (tp->opcode) {  in poke_int3_handler()
1416 		int3_emulate_call(regs, (long)ip + tp->disp);  in poke_int3_handler()
1421 		int3_emulate_jmp(regs, (long)ip + tp->disp);  in poke_int3_handler()
1440  * text_poke_bp_batch() -- update instructions on live kernel on SMP
1444  * Modify multi-byte instruction by using int3 breakpoint on SMP.
1449  *	- For each entry in the vector:
1450  *		- add a int3 trap to the address that will be patched
1451  *	- sync cores
1452  *	- For each entry in the vector:
1453  *		- update all but the first byte of the patched range
1454  *	- sync cores
1455  *	- For each entry in the vector:
1456  *		- replace the first byte (int3) by the first byte of
1458  *	- sync cores
1473 	 * ensure reading a non-zero refcount provides up to date bp_desc data.  in text_poke_bp_batch()
1500 		if (len - INT3_INSN_SIZE > 0) {  in text_poke_bp_batch()
1503 			       len - INT3_INSN_SIZE);  in text_poke_bp_batch()
1506 				  len - INT3_INSN_SIZE);  in text_poke_bp_batch()
1514 		 *   - write INT3 byte  in text_poke_bp_batch()
1515 		 *   - IPI-SYNC  in text_poke_bp_batch()
1516 		 *   - write instruction tail  in text_poke_bp_batch()
1521 		 *   - emit RECORD_TEXT_POKE with the new instruction  in text_poke_bp_batch()
1522 		 *   - IPI-SYNC  in text_poke_bp_batch()
1523 		 *   - write first byte  in text_poke_bp_batch()
1524 		 *   - IPI-SYNC  in text_poke_bp_batch()
1575 	memcpy((void *)tp->text, opcode, len);  in text_poke_loc_init()
1582 	tp->rel_addr = addr - (void *)_stext;  in text_poke_loc_init()
1583 	tp->len = len;  in text_poke_loc_init()
1584 	tp->opcode = insn.opcode.bytes[0];  in text_poke_loc_init()
1586 	switch (tp->opcode) {  in text_poke_loc_init()
1595 			BUG_ON(tp->text[i] != INT3_INSN_OPCODE);  in text_poke_loc_init()
1603 	switch (tp->opcode) {  in text_poke_loc_init()
1611 		tp->disp = insn.immediate.value;  in text_poke_loc_init()
1616 		case 2: /* NOP2 -- emulate as JMP8+0 */  in text_poke_loc_init()
1618 			tp->opcode = JMP8_INSN_OPCODE;  in text_poke_loc_init()
1619 			tp->disp = 0;  in text_poke_loc_init()
1622 		case 5: /* NOP5 -- emulate as JMP32+0 */  in text_poke_loc_init()
1624 			tp->opcode = JMP32_INSN_OPCODE;  in text_poke_loc_init()
1625 			tp->disp = 0;  in text_poke_loc_init()
1649 	tp = &tp_vec[tp_vec_nr - 1];  in tp_order_fail()
1685  * text_poke_bp() -- update instructions on live kernel on SMP