1*4882a593Smuzhiyun // SPDX-License-Identifier: GPL-2.0
2*4882a593Smuzhiyun /* Copyright(c) 2016-2020 Intel Corporation. All rights reserved. */
3*4882a593Smuzhiyun
4*4882a593Smuzhiyun #include <linux/jump_label.h>
5*4882a593Smuzhiyun #include <linux/uaccess.h>
6*4882a593Smuzhiyun #include <linux/export.h>
7*4882a593Smuzhiyun #include <linux/string.h>
8*4882a593Smuzhiyun #include <linux/types.h>
9*4882a593Smuzhiyun
10*4882a593Smuzhiyun #include <asm/mce.h>
11*4882a593Smuzhiyun
12*4882a593Smuzhiyun #ifdef CONFIG_X86_MCE
13*4882a593Smuzhiyun /*
14*4882a593Smuzhiyun * See COPY_MC_TEST for self-test of the copy_mc_fragile()
15*4882a593Smuzhiyun * implementation.
16*4882a593Smuzhiyun */
17*4882a593Smuzhiyun static DEFINE_STATIC_KEY_FALSE(copy_mc_fragile_key);
18*4882a593Smuzhiyun
enable_copy_mc_fragile(void)19*4882a593Smuzhiyun void enable_copy_mc_fragile(void)
20*4882a593Smuzhiyun {
21*4882a593Smuzhiyun static_branch_inc(©_mc_fragile_key);
22*4882a593Smuzhiyun }
23*4882a593Smuzhiyun #define copy_mc_fragile_enabled (static_branch_unlikely(©_mc_fragile_key))
24*4882a593Smuzhiyun
25*4882a593Smuzhiyun /*
26*4882a593Smuzhiyun * Similar to copy_user_handle_tail, probe for the write fault point, or
27*4882a593Smuzhiyun * source exception point.
28*4882a593Smuzhiyun */
29*4882a593Smuzhiyun __visible notrace unsigned long
copy_mc_fragile_handle_tail(char * to,char * from,unsigned len)30*4882a593Smuzhiyun copy_mc_fragile_handle_tail(char *to, char *from, unsigned len)
31*4882a593Smuzhiyun {
32*4882a593Smuzhiyun for (; len; --len, to++, from++)
33*4882a593Smuzhiyun if (copy_mc_fragile(to, from, 1))
34*4882a593Smuzhiyun break;
35*4882a593Smuzhiyun return len;
36*4882a593Smuzhiyun }
37*4882a593Smuzhiyun #else
38*4882a593Smuzhiyun /*
39*4882a593Smuzhiyun * No point in doing careful copying, or consulting a static key when
40*4882a593Smuzhiyun * there is no #MC handler in the CONFIG_X86_MCE=n case.
41*4882a593Smuzhiyun */
enable_copy_mc_fragile(void)42*4882a593Smuzhiyun void enable_copy_mc_fragile(void)
43*4882a593Smuzhiyun {
44*4882a593Smuzhiyun }
45*4882a593Smuzhiyun #define copy_mc_fragile_enabled (0)
46*4882a593Smuzhiyun #endif
47*4882a593Smuzhiyun
48*4882a593Smuzhiyun unsigned long copy_mc_enhanced_fast_string(void *dst, const void *src, unsigned len);
49*4882a593Smuzhiyun
50*4882a593Smuzhiyun /**
51*4882a593Smuzhiyun * copy_mc_to_kernel - memory copy that handles source exceptions
52*4882a593Smuzhiyun *
53*4882a593Smuzhiyun * @dst: destination address
54*4882a593Smuzhiyun * @src: source address
55*4882a593Smuzhiyun * @len: number of bytes to copy
56*4882a593Smuzhiyun *
57*4882a593Smuzhiyun * Call into the 'fragile' version on systems that benefit from avoiding
58*4882a593Smuzhiyun * corner case poison consumption scenarios, For example, accessing
59*4882a593Smuzhiyun * poison across 2 cachelines with a single instruction. Almost all
60*4882a593Smuzhiyun * other uses case can use copy_mc_enhanced_fast_string() for a fast
61*4882a593Smuzhiyun * recoverable copy, or fallback to plain memcpy.
62*4882a593Smuzhiyun *
63*4882a593Smuzhiyun * Return 0 for success, or number of bytes not copied if there was an
64*4882a593Smuzhiyun * exception.
65*4882a593Smuzhiyun */
copy_mc_to_kernel(void * dst,const void * src,unsigned len)66*4882a593Smuzhiyun unsigned long __must_check copy_mc_to_kernel(void *dst, const void *src, unsigned len)
67*4882a593Smuzhiyun {
68*4882a593Smuzhiyun if (copy_mc_fragile_enabled)
69*4882a593Smuzhiyun return copy_mc_fragile(dst, src, len);
70*4882a593Smuzhiyun if (static_cpu_has(X86_FEATURE_ERMS))
71*4882a593Smuzhiyun return copy_mc_enhanced_fast_string(dst, src, len);
72*4882a593Smuzhiyun memcpy(dst, src, len);
73*4882a593Smuzhiyun return 0;
74*4882a593Smuzhiyun }
75*4882a593Smuzhiyun EXPORT_SYMBOL_GPL(copy_mc_to_kernel);
76*4882a593Smuzhiyun
copy_mc_to_user(void * dst,const void * src,unsigned len)77*4882a593Smuzhiyun unsigned long __must_check copy_mc_to_user(void *dst, const void *src, unsigned len)
78*4882a593Smuzhiyun {
79*4882a593Smuzhiyun unsigned long ret;
80*4882a593Smuzhiyun
81*4882a593Smuzhiyun if (copy_mc_fragile_enabled) {
82*4882a593Smuzhiyun __uaccess_begin();
83*4882a593Smuzhiyun ret = copy_mc_fragile(dst, src, len);
84*4882a593Smuzhiyun __uaccess_end();
85*4882a593Smuzhiyun return ret;
86*4882a593Smuzhiyun }
87*4882a593Smuzhiyun
88*4882a593Smuzhiyun if (static_cpu_has(X86_FEATURE_ERMS)) {
89*4882a593Smuzhiyun __uaccess_begin();
90*4882a593Smuzhiyun ret = copy_mc_enhanced_fast_string(dst, src, len);
91*4882a593Smuzhiyun __uaccess_end();
92*4882a593Smuzhiyun return ret;
93*4882a593Smuzhiyun }
94*4882a593Smuzhiyun
95*4882a593Smuzhiyun return copy_user_generic(dst, src, len);
96*4882a593Smuzhiyun }
97