1*4882a593Smuzhiyun/* SPDX-License-Identifier: GPL-2.0-only */ 2*4882a593Smuzhiyun/* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */ 3*4882a593Smuzhiyun 4*4882a593Smuzhiyun#include <linux/linkage.h> 5*4882a593Smuzhiyun#include <asm/copy_mc_test.h> 6*4882a593Smuzhiyun#include <asm/export.h> 7*4882a593Smuzhiyun#include <asm/asm.h> 8*4882a593Smuzhiyun 9*4882a593Smuzhiyun#ifndef CONFIG_UML 10*4882a593Smuzhiyun 11*4882a593Smuzhiyun#ifdef CONFIG_X86_MCE 12*4882a593SmuzhiyunCOPY_MC_TEST_CTL 13*4882a593Smuzhiyun 14*4882a593Smuzhiyun/* 15*4882a593Smuzhiyun * copy_mc_fragile - copy memory with indication if an exception / fault happened 16*4882a593Smuzhiyun * 17*4882a593Smuzhiyun * The 'fragile' version is opted into by platform quirks and takes 18*4882a593Smuzhiyun * pains to avoid unrecoverable corner cases like 'fast-string' 19*4882a593Smuzhiyun * instruction sequences, and consuming poison across a cacheline 20*4882a593Smuzhiyun * boundary. The non-fragile version is equivalent to memcpy() 21*4882a593Smuzhiyun * regardless of CPU machine-check-recovery capability. 22*4882a593Smuzhiyun */ 23*4882a593SmuzhiyunSYM_FUNC_START(copy_mc_fragile) 24*4882a593Smuzhiyun cmpl $8, %edx 25*4882a593Smuzhiyun /* Less than 8 bytes? Go to byte copy loop */ 26*4882a593Smuzhiyun jb .L_no_whole_words 27*4882a593Smuzhiyun 28*4882a593Smuzhiyun /* Check for bad alignment of source */ 29*4882a593Smuzhiyun testl $7, %esi 30*4882a593Smuzhiyun /* Already aligned */ 31*4882a593Smuzhiyun jz .L_8byte_aligned 32*4882a593Smuzhiyun 33*4882a593Smuzhiyun /* Copy one byte at a time until source is 8-byte aligned */ 34*4882a593Smuzhiyun movl %esi, %ecx 35*4882a593Smuzhiyun andl $7, %ecx 36*4882a593Smuzhiyun subl $8, %ecx 37*4882a593Smuzhiyun negl %ecx 38*4882a593Smuzhiyun subl %ecx, %edx 39*4882a593Smuzhiyun.L_read_leading_bytes: 40*4882a593Smuzhiyun movb (%rsi), %al 41*4882a593Smuzhiyun COPY_MC_TEST_SRC %rsi 1 .E_leading_bytes 42*4882a593Smuzhiyun COPY_MC_TEST_DST %rdi 1 .E_leading_bytes 43*4882a593Smuzhiyun.L_write_leading_bytes: 44*4882a593Smuzhiyun movb %al, (%rdi) 45*4882a593Smuzhiyun incq %rsi 46*4882a593Smuzhiyun incq %rdi 47*4882a593Smuzhiyun decl %ecx 48*4882a593Smuzhiyun jnz .L_read_leading_bytes 49*4882a593Smuzhiyun 50*4882a593Smuzhiyun.L_8byte_aligned: 51*4882a593Smuzhiyun movl %edx, %ecx 52*4882a593Smuzhiyun andl $7, %edx 53*4882a593Smuzhiyun shrl $3, %ecx 54*4882a593Smuzhiyun jz .L_no_whole_words 55*4882a593Smuzhiyun 56*4882a593Smuzhiyun.L_read_words: 57*4882a593Smuzhiyun movq (%rsi), %r8 58*4882a593Smuzhiyun COPY_MC_TEST_SRC %rsi 8 .E_read_words 59*4882a593Smuzhiyun COPY_MC_TEST_DST %rdi 8 .E_write_words 60*4882a593Smuzhiyun.L_write_words: 61*4882a593Smuzhiyun movq %r8, (%rdi) 62*4882a593Smuzhiyun addq $8, %rsi 63*4882a593Smuzhiyun addq $8, %rdi 64*4882a593Smuzhiyun decl %ecx 65*4882a593Smuzhiyun jnz .L_read_words 66*4882a593Smuzhiyun 67*4882a593Smuzhiyun /* Any trailing bytes? */ 68*4882a593Smuzhiyun.L_no_whole_words: 69*4882a593Smuzhiyun andl %edx, %edx 70*4882a593Smuzhiyun jz .L_done_memcpy_trap 71*4882a593Smuzhiyun 72*4882a593Smuzhiyun /* Copy trailing bytes */ 73*4882a593Smuzhiyun movl %edx, %ecx 74*4882a593Smuzhiyun.L_read_trailing_bytes: 75*4882a593Smuzhiyun movb (%rsi), %al 76*4882a593Smuzhiyun COPY_MC_TEST_SRC %rsi 1 .E_trailing_bytes 77*4882a593Smuzhiyun COPY_MC_TEST_DST %rdi 1 .E_trailing_bytes 78*4882a593Smuzhiyun.L_write_trailing_bytes: 79*4882a593Smuzhiyun movb %al, (%rdi) 80*4882a593Smuzhiyun incq %rsi 81*4882a593Smuzhiyun incq %rdi 82*4882a593Smuzhiyun decl %ecx 83*4882a593Smuzhiyun jnz .L_read_trailing_bytes 84*4882a593Smuzhiyun 85*4882a593Smuzhiyun /* Copy successful. Return zero */ 86*4882a593Smuzhiyun.L_done_memcpy_trap: 87*4882a593Smuzhiyun xorl %eax, %eax 88*4882a593Smuzhiyun.L_done: 89*4882a593Smuzhiyun RET 90*4882a593SmuzhiyunSYM_FUNC_END(copy_mc_fragile) 91*4882a593SmuzhiyunEXPORT_SYMBOL_GPL(copy_mc_fragile) 92*4882a593Smuzhiyun 93*4882a593Smuzhiyun .section .fixup, "ax" 94*4882a593Smuzhiyun /* 95*4882a593Smuzhiyun * Return number of bytes not copied for any failure. Note that 96*4882a593Smuzhiyun * there is no "tail" handling since the source buffer is 8-byte 97*4882a593Smuzhiyun * aligned and poison is cacheline aligned. 98*4882a593Smuzhiyun */ 99*4882a593Smuzhiyun.E_read_words: 100*4882a593Smuzhiyun shll $3, %ecx 101*4882a593Smuzhiyun.E_leading_bytes: 102*4882a593Smuzhiyun addl %edx, %ecx 103*4882a593Smuzhiyun.E_trailing_bytes: 104*4882a593Smuzhiyun mov %ecx, %eax 105*4882a593Smuzhiyun jmp .L_done 106*4882a593Smuzhiyun 107*4882a593Smuzhiyun /* 108*4882a593Smuzhiyun * For write fault handling, given the destination is unaligned, 109*4882a593Smuzhiyun * we handle faults on multi-byte writes with a byte-by-byte 110*4882a593Smuzhiyun * copy up to the write-protected page. 111*4882a593Smuzhiyun */ 112*4882a593Smuzhiyun.E_write_words: 113*4882a593Smuzhiyun shll $3, %ecx 114*4882a593Smuzhiyun addl %edx, %ecx 115*4882a593Smuzhiyun movl %ecx, %edx 116*4882a593Smuzhiyun jmp copy_mc_fragile_handle_tail 117*4882a593Smuzhiyun 118*4882a593Smuzhiyun .previous 119*4882a593Smuzhiyun 120*4882a593Smuzhiyun _ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes) 121*4882a593Smuzhiyun _ASM_EXTABLE_FAULT(.L_read_words, .E_read_words) 122*4882a593Smuzhiyun _ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes) 123*4882a593Smuzhiyun _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes) 124*4882a593Smuzhiyun _ASM_EXTABLE(.L_write_words, .E_write_words) 125*4882a593Smuzhiyun _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes) 126*4882a593Smuzhiyun#endif /* CONFIG_X86_MCE */ 127*4882a593Smuzhiyun 128*4882a593Smuzhiyun/* 129*4882a593Smuzhiyun * copy_mc_enhanced_fast_string - memory copy with exception handling 130*4882a593Smuzhiyun * 131*4882a593Smuzhiyun * Fast string copy + fault / exception handling. If the CPU does 132*4882a593Smuzhiyun * support machine check exception recovery, but does not support 133*4882a593Smuzhiyun * recovering from fast-string exceptions then this CPU needs to be 134*4882a593Smuzhiyun * added to the copy_mc_fragile_key set of quirks. Otherwise, absent any 135*4882a593Smuzhiyun * machine check recovery support this version should be no slower than 136*4882a593Smuzhiyun * standard memcpy. 137*4882a593Smuzhiyun */ 138*4882a593SmuzhiyunSYM_FUNC_START(copy_mc_enhanced_fast_string) 139*4882a593Smuzhiyun movq %rdi, %rax 140*4882a593Smuzhiyun movq %rdx, %rcx 141*4882a593Smuzhiyun.L_copy: 142*4882a593Smuzhiyun rep movsb 143*4882a593Smuzhiyun /* Copy successful. Return zero */ 144*4882a593Smuzhiyun xorl %eax, %eax 145*4882a593Smuzhiyun RET 146*4882a593SmuzhiyunSYM_FUNC_END(copy_mc_enhanced_fast_string) 147*4882a593Smuzhiyun 148*4882a593Smuzhiyun .section .fixup, "ax" 149*4882a593Smuzhiyun.E_copy: 150*4882a593Smuzhiyun /* 151*4882a593Smuzhiyun * On fault %rcx is updated such that the copy instruction could 152*4882a593Smuzhiyun * optionally be restarted at the fault position, i.e. it 153*4882a593Smuzhiyun * contains 'bytes remaining'. A non-zero return indicates error 154*4882a593Smuzhiyun * to copy_mc_generic() users, or indicate short transfers to 155*4882a593Smuzhiyun * user-copy routines. 156*4882a593Smuzhiyun */ 157*4882a593Smuzhiyun movq %rcx, %rax 158*4882a593Smuzhiyun RET 159*4882a593Smuzhiyun 160*4882a593Smuzhiyun .previous 161*4882a593Smuzhiyun 162*4882a593Smuzhiyun _ASM_EXTABLE_FAULT(.L_copy, .E_copy) 163*4882a593Smuzhiyun#endif /* !CONFIG_UML */ 164