1 /* SPDX-License-Identifier: BSL-1.0 */ 2 /* Copyright (c) 2017 Arm Limited. All rights reserved. 3 4 Boost Software License - Version 1.0 - August 17th, 2003 5 6 Permission is hereby granted, free of charge, to any person or organization 7 obtaining a copy of the software and accompanying documentation covered by 8 this license (the "Software") to use, reproduce, display, distribute, 9 execute, and transmit the Software, and to prepare derivative works of the 10 Software, and to permit third-parties to whom the Software is furnished to do 11 so, all subject to the following: 12 13 The copyright notices in the Software and this entire statement, including 14 the above license grant, this restriction and the following disclaimer, must 15 be included in all copies of the Software, in whole or in part, and all 16 derivative works of the Software, unless such copies or derivative works are 17 solely in the form of machine-executable object code generated by a source 18 language processor. 19 20 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT 23 SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE FOR 24 ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, 25 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 26 DEALINGS IN THE SOFTWARE. */ 27 28 #ifdef __HAVE_LOAD_NO_SPECULATE 29 #define load_no_speculate(__ptr, __low, __high) \ 30 (__extension__ ({ \ 31 __typeof__ ((__ptr)) __ptr_once = (__ptr); \ 32 __builtin_load_no_speculate (__ptr_once, __low, __high, \ 33 0, __ptr_once); \ 34 })) 35 36 #define load_no_speculate_fail(__ptr, __low, __high, __failval) \ 37 (__extension__ ({ \ 38 __typeof__ ((__ptr)) __ptr_once = (__ptr); \ 39 __builtin_load_no_speculate (__ptr_once, __low, __high, \ 40 __failval, __ptr_once); \ 41 })) 42 43 #define load_no_speculate_cmp(__ptr, __low, __high, __failval, __cmpptr) \ 44 (__builtin_load_no_speculate (__ptr, __low, __high, __failval, __cmpptr)) 45 46 #else 47 #ifdef __aarch64__ 48 49 #define __load_no_speculate1(__ptr, __low, __high, __failval, \ 50 __cmpptr, __w, __sz) \ 51 (__extension__ ({ \ 52 __typeof__ (0 + (*(__ptr))) __nln_val; \ 53 /* This typecasting is required to ensure correct handling of upper \ 54 bits of failval, to ensure a clean return from the CSEL below. */ \ 55 __typeof__(*(__ptr)) __fv \ 56 = (__typeof__(*(__ptr)))(unsigned long long) (__failval); \ 57 /* If __high is explicitly NULL, we must not emit the \ 58 upper-bound comparison. We need to cast __high to an \ 59 unsigned long long before handing it to __builtin_constant_p to \ 60 ensure that clang/llvm correctly detects NULL as a constant if it \ 61 is defined as (void*) 0. */ \ 62 if (__builtin_constant_p ((unsigned long long)__high) \ 63 && __high == ((void *)0)) \ 64 { \ 65 __asm__ volatile ( \ 66 "cmp\t%[__c], %[__l]\n\t" \ 67 "bcc\t.ns%=\n\t" \ 68 "ldr" __sz "\t%" __w "[__v], %[__p]\n" \ 69 ".ns%=:\n\t" \ 70 "csel\t%" __w "[__v], %" __w "[__v], %" __w "[__f], cs\n\t" \ 71 "hint\t#0x14 // CSDB" \ 72 /* The value we have loaded, or failval if the condition check \ 73 fails. */ \ 74 : [__v] "=&r" (__nln_val) \ 75 /* The pointer we wish to use for comparisons, and the low and \ 76 high bounds to use in that comparison. Note that this need \ 77 not be the same as the pointer from which we will load. */ \ 78 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 79 /* The memory location from which we will load. */ \ 80 [__p] "m" (*(__ptr)), \ 81 /* The value to return if the condition check fails. */ \ 82 [__f] "rZ" (__fv) \ 83 /* We always clobber the condition codes. */ \ 84 : "cc"); \ 85 } \ 86 else \ 87 { \ 88 __asm__ volatile ( \ 89 "cmp\t%[__c], %[__l]\n\t" \ 90 "ccmp\t%[__c], %[__h], 2, cs\n\t" \ 91 "bcs\t.ns%=\n\t" \ 92 "ldr" __sz "\t%" __w "[__v], %[__p]\n" \ 93 ".ns%=:\n\t" \ 94 "csel\t%" __w "[__v], %" __w "[__v], %" __w "[__f], cc\n\t" \ 95 "hint\t#0x14 // CSDB" \ 96 /* The value we have loaded, or failval if the condition check \ 97 fails. */ \ 98 : [__v] "=&r" (__nln_val) \ 99 /* The pointer we wish to use for comparisons, and the low and \ 100 high bounds to use in that comparison. Note that this need \ 101 not be the same as the pointer from which we will load. */ \ 102 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 103 [__h] "r" (__high), \ 104 /* The memory location from which we will load. */ \ 105 [__p] "m" (*(__ptr)), \ 106 /* The value to return if the condition check fails. */ \ 107 [__f] "rZ" (__fv) \ 108 /* We always clobber the condition codes. */ \ 109 : "cc"); \ 110 } \ 111 (__typeof__ (*(__ptr))) __nln_val; \ 112 })) 113 114 #define __load_no_speculate(__ptr, __low, __high, __failval, __cmpptr) \ 115 (__extension__ ({ \ 116 __typeof__ (0 + *(__ptr)) __nl_val; \ 117 \ 118 switch (sizeof(*(__ptr))) { \ 119 case 1: \ 120 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 121 __failval, __cmpptr, "w", "b"); \ 122 break; \ 123 case 2: \ 124 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 125 __failval, __cmpptr, "w", "h"); \ 126 break; \ 127 case 4: \ 128 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 129 __failval, __cmpptr, "w", ""); \ 130 break; \ 131 case 8: \ 132 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 133 __failval, __cmpptr, "x", ""); \ 134 break; \ 135 default: \ 136 { \ 137 char __static_assert_no_speculate_load_size_too_big \ 138 [sizeof (__nl_val) > 8 ? -1 : 1]; \ 139 break; \ 140 } \ 141 } \ 142 \ 143 (__typeof__ (*(__ptr))) __nl_val; \ 144 })) 145 146 #define load_no_speculate(__ptr, __low, __high) \ 147 (__extension__ ({ \ 148 __typeof__ ((__ptr)) __ptr_once = (__ptr); \ 149 __load_no_speculate (__ptr_once, __low, __high, 0, __ptr_once); \ 150 })) 151 152 #define load_no_speculate_fail(__ptr, __low, __high, __failval) \ 153 (__extension__ ({ \ 154 __typeof__ ((__ptr)) __ptr_once = (__ptr); \ 155 __load_no_speculate (__ptr_once, __low, __high, \ 156 __failval, __ptr_once); \ 157 })) 158 159 #define load_no_speculate_cmp(__ptr, __low, __high, __failval, __cmpptr) \ 160 (__load_no_speculate (__ptr, __low, __high, __failval, __cmpptr)) 161 162 /* AArch32 support for ARM and Thumb-2. Thumb-1 is not supported. */ 163 #elif defined (__ARM_32BIT_STATE) && (defined (__thumb2__) || !defined (__thumb__)) 164 #ifdef __thumb2__ 165 /* Thumb2 case. */ 166 167 #define __load_no_speculate1(__ptr, __low, __high, __failval, \ 168 __cmpptr, __sz) \ 169 (__extension__ ({ \ 170 __typeof__ (0 + *(__ptr)) __nln_val; \ 171 __typeof__(*(__ptr)) __fv \ 172 = (__typeof__(*(__ptr)))(unsigned long) (__failval); \ 173 /* If __high is explicitly NULL, we must not emit the \ 174 upper-bound comparison. We need to cast __high to an \ 175 unsigned long before handing it to __builtin_constant_p to \ 176 ensure that clang/llvm correctly detects NULL as a constant if it \ 177 is defined as (void*) 0. */ \ 178 if (__builtin_constant_p ((unsigned long)__high) \ 179 && __high == ((void *)0)) \ 180 { \ 181 __asm__ volatile ( \ 182 ".syntax unified\n\t" \ 183 "cmp\t%[__c], %[__l]\n\t" \ 184 "bcc\t.ns%=\n\t" \ 185 "ldr" __sz "\t%[__v], %[__p]\n" \ 186 ".ns%=:\n\t" \ 187 "it\tcc\n\t" \ 188 "movcc\t%[__v], %[__f]\n\t" \ 189 ".inst.n 0xf3af\t@ CSDB\n\t" \ 190 ".inst.n 0x8014\t@ CSDB" \ 191 /* The value we have loaded, or failval if the condition check \ 192 fails. */ \ 193 : [__v] "=&l" (__nln_val) \ 194 /* The pointer we wish to use for comparisons, and the low and \ 195 high bounds to use in that comparison. Note that this need \ 196 not be the same as the pointer from which we will load. */ \ 197 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 198 /* The memory location from which we will load. */ \ 199 [__p] "m" (*(__ptr)), \ 200 /* The value to return if the condition check fails. */ \ 201 [__f] "r" (__fv) \ 202 /* We always clobber the condition codes. */ \ 203 : "cc"); \ 204 } \ 205 else \ 206 { \ 207 __asm__ volatile ( \ 208 ".syntax unified\n\t" \ 209 "cmp\t%[__c], %[__l]\n\t" \ 210 "it\tcs\n\t" \ 211 "cmpcs\t%[__h], %[__c]\n\t" \ 212 "bls\t.ns%=\n\t" \ 213 "ldr" __sz "\t%[__v], %[__p]\n" \ 214 ".ns%=:\n\t" \ 215 "it\tls\n\t" \ 216 "movls\t%[__v], %[__f]\n\t" \ 217 ".inst.n 0xf3af\t@ CSDB\n\t" \ 218 ".inst.n 0x8014\t@ CSDB" \ 219 /* The value we have loaded, or failval if the condition check \ 220 fails. */ \ 221 : [__v] "=&l" (__nln_val) \ 222 /* The pointer we wish to use for comparisons, and the low and \ 223 high bounds to use in that comparison. Note that this need \ 224 not be the same as the pointer from which we will load. */ \ 225 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 226 [__h] "r" (__high), \ 227 /* The memory location from which we will load. */ \ 228 [__p] "m" (*(__ptr)), \ 229 /* The value to return if the condition check fails. */ \ 230 [__f] "r" (__fv) \ 231 /* We always clobber the condition codes. */ \ 232 : "cc"); \ 233 } \ 234 (__typeof__ (*(__ptr))) __nln_val; \ 235 })) \ 236 \ 237 /* Double-word version. */ 238 #define __load_no_speculate2(__ptr, __low, __high, __failval, \ 239 __cmpptr) \ 240 (__extension__ ({ \ 241 __typeof__ (0 + *(__ptr)) __nln_val; \ 242 __typeof__(*(__ptr)) __fv \ 243 = (__typeof__(*(__ptr)))(unsigned long) (__failval); \ 244 /* If __high is explicitly NULL, we must not emit the \ 245 upper-bound comparison. We need to cast __high to an \ 246 unsigned long before handing it to __builtin_constant_p to \ 247 ensure that clang/llvm correctly detects NULL as a constant if it \ 248 is defined as (void*) 0. */ \ 249 __typeof__ (__ptr) __tmp_ptr; \ 250 if (__builtin_constant_p ((unsigned long)__high) \ 251 && __high == ((void *)0)) \ 252 { \ 253 __asm__ volatile ( \ 254 ".syntax unified\n\t" \ 255 "cmp\t%[__c], %[__l]\n\t" \ 256 "bcc\t.ns%=\n\t" \ 257 "ldr\t%Q[__v], [%[__p]]\n\t" \ 258 "ldr\t%R[__v], [%[__p], #4]\n" \ 259 ".ns%=:\n\t" \ 260 "it\tcc\n\t" \ 261 "movcc\t%Q[__v], %Q[__f]\n\t" \ 262 "it\tcc\n\t" \ 263 "movcc\t%R[__v], %R[__f]\n\t" \ 264 ".inst.n 0xf3af\t@ CSDB\n\t" \ 265 ".inst.n 0x8014\t@ CSDB" \ 266 /* The value we have loaded, or failval if the condition check \ 267 fails. */ \ 268 : [__v] "=&l" (__nln_val) \ 269 /* The pointer we wish to use for comparisons, and the low and \ 270 high bounds to use in that comparison. Note that this need \ 271 not be the same as the pointer from which we will load. */ \ 272 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 273 /* The memory location from which we will load. */ \ 274 [__p] "r" (__ptr), \ 275 /* The value to return if the condition check fails. */ \ 276 [__f] "r" (__fv) \ 277 /* We always clobber the condition codes. */ \ 278 : "cc"); \ 279 } \ 280 else \ 281 { \ 282 __asm__ volatile ( \ 283 ".syntax unified\n\t" \ 284 "cmp\t%[__c], %[__l]\n\t" \ 285 "it\tcs\n\t" \ 286 "cmpcs\t%[__h], %[__c]\n\t" \ 287 "bls\t.ns%=\n\t" \ 288 "ldr\t%Q[__v], [%[__p]]\n\t" \ 289 "ldr\t%R[__v], [%[__p], #4]\n" \ 290 ".ns%=:\n\t" \ 291 "it\tls\n\t" \ 292 "movls\t%Q[__v], %Q[__f]\n\t" \ 293 "it\tls\n\t" \ 294 "movls\t%R[__v], %R[__f]\n\t" \ 295 ".inst.n 0xf3af\t@ CSDB\n\t" \ 296 ".inst.n 0x8014\t@ CSDB" \ 297 /* The value we have loaded, or failval if the condition check \ 298 fails. */ \ 299 : [__v] "=&l" (__nln_val) \ 300 /* The pointer we wish to use for comparisons, and the low and \ 301 high bounds to use in that comparison. Note that this need \ 302 not be the same as the pointer from which we will load. */ \ 303 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 304 [__h] "r" (__high), \ 305 /* The memory location from which we will load. */ \ 306 [__p] "r" (__ptr), \ 307 /* The value to return if the condition check fails. */ \ 308 [__f] "r" (__fv) \ 309 /* We always clobber the condition codes. */ \ 310 : "cc"); \ 311 } \ 312 (__typeof__ (*(__ptr))) __nln_val; \ 313 })) 314 315 #else 316 /* ARM case. */ 317 318 #define __load_no_speculate1(__ptr, __low, __high, __failval, \ 319 __cmpptr, __sz) \ 320 (__extension__ ({ \ 321 __typeof__ (0 + *(__ptr)) __nln_val; \ 322 __typeof__(*(__ptr)) __fv \ 323 = (__typeof__(*(__ptr)))(unsigned long) (__failval); \ 324 /* If __high is explicitly NULL, we must not emit the \ 325 upper-bound comparison. We need to cast __high to an \ 326 unsigned long before handing it to __builtin_constant_p to \ 327 ensure that clang/llvm correctly detects NULL as a constant if it \ 328 is defined as (void*) 0. */ \ 329 if (__builtin_constant_p ((unsigned long)__high) \ 330 && __high == ((void *)0)) \ 331 { \ 332 __asm__ volatile ( \ 333 ".syntax unified\n\t" \ 334 "cmp\t%[__c], %[__l]\n\t" \ 335 "ldr" __sz "cs\t%[__v], %[__p]\n\t" \ 336 "movcc\t%[__v], %[__f]\n\t" \ 337 ".inst 0xe320f014\t@ CSDB" \ 338 /* The value we have loaded, or failval if the condition check \ 339 fails. */ \ 340 : [__v] "=&r" (__nln_val) \ 341 /* The pointer we wish to use for comparisons, and the low and \ 342 high bounds to use in that comparison. Note that this need \ 343 not be the same as the pointer from which we will load. */ \ 344 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 345 /* The memory location from which we will load. */ \ 346 [__p] "m" (*(__ptr)), \ 347 /* The value to return if the condition check fails. */ \ 348 [__f] "rKI" (__fv) \ 349 /* We always clobber the condition codes. */ \ 350 : "cc"); \ 351 } \ 352 else \ 353 { \ 354 __asm__ volatile ( \ 355 ".syntax unified\n\t" \ 356 "cmp\t%[__c], %[__l]\n\t" \ 357 "cmpcs\t%[__h], %[__c]\n\t" \ 358 "ldr" __sz "hi\t%[__v], %[__p]\n\t" \ 359 "movls\t%[__v], %[__f]\n\t" \ 360 ".inst 0xe320f014\t@ CSDB" \ 361 /* The value we have loaded, or failval if the condition check \ 362 fails. */ \ 363 : [__v] "=&r" (__nln_val) \ 364 /* The pointer we wish to use for comparisons, and the low and \ 365 high bounds to use in that comparison. Note that this need \ 366 not be the same as the pointer from which we will load. */ \ 367 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 368 [__h] "r" (__high), \ 369 /* The memory location from which we will load. */ \ 370 [__p] "m" (*(__ptr)), \ 371 /* The value to return if the condition check fails. */ \ 372 [__f] "rKI" (__fv) \ 373 /* We always clobber the condition codes. */ \ 374 : "cc"); \ 375 } \ 376 (__typeof__ (*(__ptr))) __nln_val; \ 377 })) 378 379 /* Double-word version. */ 380 #define __load_no_speculate2(__ptr, __low, __high, __failval, \ 381 __cmpptr) \ 382 (__extension__ ({ \ 383 __typeof__ (0 + *(__ptr)) __nln_val; \ 384 __typeof__(*(__ptr)) __fv \ 385 = (__typeof__(*(__ptr)))(unsigned long) (__failval); \ 386 /* If __high is explicitly NULL, we must not emit the \ 387 upper-bound comparison. We need to cast __high to an \ 388 unsigned long before handing it to __builtin_constant_p to \ 389 ensure that clang/llvm correctly detects NULL as a constant if it \ 390 is defined as (void*) 0. */ \ 391 if (__builtin_constant_p ((unsigned long)__high) \ 392 && __high == ((void *)0)) \ 393 { \ 394 __asm__ volatile ( \ 395 ".syntax unified\n\t" \ 396 "cmp\t%[__c], %[__l]\n\t" \ 397 "ldrcs\t%Q[__v], [%[__p]]\n\t" \ 398 "ldrcs\t%R[__v], [%[__p], #4]\n\t" \ 399 "movcc\t%Q[__v], %Q[__f]\n\t" \ 400 "movcc\t%R[__v], %R[__f]\n\t" \ 401 ".inst 0xe320f014\t@ CSDB" \ 402 /* The value we have loaded, or failval if the condition check \ 403 fails. */ \ 404 : [__v] "=&r" (__nln_val) \ 405 /* The pointer we wish to use for comparisons, and the low and \ 406 high bounds to use in that comparison. Note that this need \ 407 not be the same as the pointer from which we will load. */ \ 408 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 409 /* The memory location from which we will load. */ \ 410 [__p] "r" (__ptr), \ 411 /* The value to return if the condition check fails. */ \ 412 [__f] "r" (__fv) \ 413 /* We always clobber the condition codes. */ \ 414 : "cc"); \ 415 } \ 416 else \ 417 { \ 418 __asm__ volatile ( \ 419 ".syntax unified\n\t" \ 420 "cmp\t%[__c], %[__l]\n\t" \ 421 "cmpcs\t%[__h], %[__c]\n\t" \ 422 "ldrhi\t%Q[__v], [%[__p]]\n\t" \ 423 "ldrhi\t%R[__v], [%[__p], #4]\n\t" \ 424 "movls\t%Q[__v], %Q[__f]\n\t" \ 425 "movls\t%R[__v], %R[__f]\n\t" \ 426 ".inst 0xe320f014\t@ CSDB" \ 427 /* The value we have loaded, or failval if the condition check \ 428 fails. */ \ 429 : [__v] "=&r" (__nln_val) \ 430 /* The pointer we wish to use for comparisons, and the low and \ 431 high bounds to use in that comparison. Note that this need \ 432 not be the same as the pointer from which we will load. */ \ 433 : [__c] "r" (__cmpptr), [__l] "r" (__low), \ 434 [__h] "r" (__high), \ 435 /* The memory location from which we will load. */ \ 436 [__p] "r" (__ptr), \ 437 /* The value to return if the condition check fails. */ \ 438 [__f] "r" (__fv) \ 439 /* We always clobber the condition codes. */ \ 440 : "cc"); \ 441 } \ 442 (__typeof__ (*(__ptr))) __nln_val; \ 443 })) 444 445 #endif // __thumb2__ 446 447 /* Common to ARM and Thumb2. */ 448 449 #define __load_no_speculate(__ptr, __low, __high, __failval, __cmpptr) \ 450 (__extension__ ({ \ 451 __typeof__ (0 + *(__ptr)) __nl_val; \ 452 \ 453 switch (sizeof(*(__ptr))) { \ 454 case 1: \ 455 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 456 __failval, __cmpptr, "b"); \ 457 break; \ 458 case 2: \ 459 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 460 __failval, __cmpptr, "h"); \ 461 break; \ 462 case 4: \ 463 __nl_val = __load_no_speculate1 (__ptr, __low, __high, \ 464 __failval, __cmpptr, ""); \ 465 break; \ 466 case 8: \ 467 __nl_val = __load_no_speculate2 (__ptr, __low, __high, \ 468 __failval, __cmpptr); \ 469 break; \ 470 default: \ 471 { \ 472 char __static_assert_no_speculate_load_size_too_big \ 473 [sizeof (__nl_val) > 8 ? -1 : 1]; \ 474 break; \ 475 } \ 476 } \ 477 \ 478 (__typeof__ (*(__ptr))) __nl_val; \ 479 })) 480 481 #define load_no_speculate(__ptr, __low, __high) \ 482 (__extension__ ({ \ 483 __typeof__ ((__ptr)) __ptr_once = (__ptr); \ 484 __load_no_speculate (__ptr_once, __low, __high, 0, __ptr_once); \ 485 })) 486 487 #define load_no_speculate_fail(__ptr, __low, __high, __failval) \ 488 (__extension__ ({ \ 489 __typeof__ ((__ptr)) __ptr_once = (__ptr); \ 490 __load_no_speculate (__ptr_once, __low, __high, \ 491 __failval, __ptr_once); \ 492 })) 493 494 #define load_no_speculate_cmp(__ptr, __low, __high, __failval, __cmpptr) \ 495 (__load_no_speculate (__ptr, __low, __high, __failval, __cmpptr)) 496 497 #else 498 #error "No fallback provided for load_no_speculate" 499 #endif 500 501 #endif 502