1 /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 /* 3 * rseq-ppc.h 4 * 5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers (at) efficios.com> 6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng (at) gmail.com> 7 */ 8 9 #define RSEQ_SIG 0x53053053 10 11 #define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc") 12 #define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc") 13 #define rseq_smp_rmb() rseq_smp_lwsync() 14 #define rseq_smp_wmb() rseq_smp_lwsync() 15 16 #define rseq_smp_load_acquire(p) \ 17 __extension__ ({ \ 18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \ 19 rseq_smp_lwsync(); \ 20 ____p1; \ 21 }) 22 23 #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync() 24 25 #define rseq_smp_store_release(p, v) \ 26 do { \ 27 rseq_smp_lwsync(); \ 28 RSEQ_WRITE_ONCE(*p, v); \ 29 } while (0) 30 31 #ifdef RSEQ_SKIP_FASTPATH 32 #include "rseq-skip.h" 33 #else /* !RSEQ_SKIP_FASTPATH */ 34 35 /* 36 * The __rseq_table section can be used by debuggers to better handle 37 * single-stepping through the restartable critical sections. 38 */ 39 40 #ifdef __PPC64__ 41 42 #define STORE_WORD "std " 43 #define LOAD_WORD "ld " 44 #define LOADX_WORD "ldx " 45 #define CMP_WORD "cmpd " 46 47 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 48 start_ip, post_commit_offset, abort_ip) \ 49 ".pushsection __rseq_table, \"aw\"\n\t" \ 50 ".balign 32\n\t" \ 51 __rseq_str(label) ":\n\t" \ 52 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 53 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \ 54 ".popsection\n\t" 55 56 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 57 RSEQ_INJECT_ASM(1) \ 58 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \ 59 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \ 60 "rldicr %%r17, %%r17, 32, 31\n\t" \ 61 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \ 62 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \ 63 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ 64 __rseq_str(label) ":\n\t" 65 66 #else /* #ifdef __PPC64__ */ 67 68 #define STORE_WORD "stw " 69 #define LOAD_WORD "lwz " 70 #define LOADX_WORD "lwzx " 71 #define CMP_WORD "cmpw " 72 73 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ 74 start_ip, post_commit_offset, abort_ip) \ 75 ".pushsection __rseq_table, \"aw\"\n\t" \ 76 ".balign 32\n\t" \ 77 __rseq_str(label) ":\n\t" \ 78 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \ 79 /* 32-bit only supported on BE */ \ 80 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \ 81 ".popsection\n\t" 82 83 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \ 84 RSEQ_INJECT_ASM(1) \ 85 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \ 86 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \ 87 "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \ 88 __rseq_str(label) ":\n\t" 89 90 #endif /* #ifdef __PPC64__ */ 91 92 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \ 93 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ 94 (post_commit_ip - start_ip), abort_ip) 95 96 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ 97 RSEQ_INJECT_ASM(2) \ 98 "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \ 99 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \ 100 "bne- cr7, " __rseq_str(label) "\n\t" 101 102 #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \ 103 ".pushsection __rseq_failure, \"ax\"\n\t" \ 104 ".long " __rseq_str(RSEQ_SIG) "\n\t" \ 105 __rseq_str(label) ":\n\t" \ 106 "b %l[" __rseq_str(abort_label) "]\n\t" \ 107 ".popsection\n\t" 108 109 /* 110 * RSEQ_ASM_OPs: asm operations for rseq 111 * RSEQ_ASM_OP_R_*: has hard-code registers in it 112 * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7) 113 */ 114 #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ 115 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ 116 CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \ 117 "bne- cr7, " __rseq_str(label) "\n\t" 118 119 #define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \ 120 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ 121 CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \ 122 "beq- cr7, " __rseq_str(label) "\n\t" 123 124 #define RSEQ_ASM_OP_STORE(value, var) \ 125 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" 126 127 /* Load @var to r17 */ 128 #define RSEQ_ASM_OP_R_LOAD(var) \ 129 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" 130 131 /* Store r17 to @var */ 132 #define RSEQ_ASM_OP_R_STORE(var) \ 133 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" 134 135 /* Add @count to r17 */ 136 #define RSEQ_ASM_OP_R_ADD(count) \ 137 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t" 138 139 /* Load (r17 + voffp) to r17 */ 140 #define RSEQ_ASM_OP_R_LOADX(voffp) \ 141 LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t" 142 143 /* TODO: implement a faster memcpy. */ 144 #define RSEQ_ASM_OP_R_MEMCPY() \ 145 "cmpdi %%r19, 0\n\t" \ 146 "beq 333f\n\t" \ 147 "addi %%r20, %%r20, -1\n\t" \ 148 "addi %%r21, %%r21, -1\n\t" \ 149 "222:\n\t" \ 150 "lbzu %%r18, 1(%%r20)\n\t" \ 151 "stbu %%r18, 1(%%r21)\n\t" \ 152 "addi %%r19, %%r19, -1\n\t" \ 153 "cmpdi %%r19, 0\n\t" \ 154 "bne 222b\n\t" \ 155 "333:\n\t" \ 156 157 #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \ 158 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \ 159 __rseq_str(post_commit_label) ":\n\t" 160 161 #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \ 162 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \ 163 __rseq_str(post_commit_label) ":\n\t" 164 165 static inline __attribute__((always_inline)) 166 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 167 { 168 RSEQ_INJECT_C(9) 169 170 __asm__ __volatile__ goto ( 171 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 172 /* Start rseq by storing table entry pointer into rseq_cs. */ 173 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 174 /* cmp cpuid */ 175 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 176 RSEQ_INJECT_ASM(3) 177 /* cmp @v equal to @expect */ 178 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 179 RSEQ_INJECT_ASM(4) 180 #ifdef RSEQ_COMPARE_TWICE 181 /* cmp cpuid */ 182 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 183 /* cmp @v equal to @expect */ 184 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 185 #endif 186 /* final store */ 187 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 188 RSEQ_INJECT_ASM(5) 189 RSEQ_ASM_DEFINE_ABORT(4, abort) 190 : /* gcc asm goto does not allow outputs */ 191 : [cpu_id] "r" (cpu), 192 [current_cpu_id] "m" (__rseq_abi.cpu_id), 193 [rseq_cs] "m" (__rseq_abi.rseq_cs), 194 [v] "m" (*v), 195 [expect] "r" (expect), 196 [newv] "r" (newv) 197 RSEQ_INJECT_INPUT 198 : "memory", "cc", "r17" 199 RSEQ_INJECT_CLOBBER 200 : abort, cmpfail 201 #ifdef RSEQ_COMPARE_TWICE 202 , error1, error2 203 #endif 204 ); 205 return 0; 206 abort: 207 RSEQ_INJECT_FAILED 208 return -1; 209 cmpfail: 210 return 1; 211 #ifdef RSEQ_COMPARE_TWICE 212 error1: 213 rseq_bug("cpu_id comparison failed"); 214 error2: 215 rseq_bug("expected value comparison failed"); 216 #endif 217 } 218 219 static inline __attribute__((always_inline)) 220 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 221 off_t voffp, intptr_t *load, int cpu) 222 { 223 RSEQ_INJECT_C(9) 224 225 __asm__ __volatile__ goto ( 226 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 227 /* Start rseq by storing table entry pointer into rseq_cs. */ 228 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 229 /* cmp cpuid */ 230 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 231 RSEQ_INJECT_ASM(3) 232 /* cmp @v not equal to @expectnot */ 233 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 234 RSEQ_INJECT_ASM(4) 235 #ifdef RSEQ_COMPARE_TWICE 236 /* cmp cpuid */ 237 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 238 /* cmp @v not equal to @expectnot */ 239 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 240 #endif 241 /* load the value of @v */ 242 RSEQ_ASM_OP_R_LOAD(v) 243 /* store it in @load */ 244 RSEQ_ASM_OP_R_STORE(load) 245 /* dereference voffp(v) */ 246 RSEQ_ASM_OP_R_LOADX(voffp) 247 /* final store the value at voffp(v) */ 248 RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 249 RSEQ_INJECT_ASM(5) 250 RSEQ_ASM_DEFINE_ABORT(4, abort) 251 : /* gcc asm goto does not allow outputs */ 252 : [cpu_id] "r" (cpu), 253 [current_cpu_id] "m" (__rseq_abi.cpu_id), 254 [rseq_cs] "m" (__rseq_abi.rseq_cs), 255 /* final store input */ 256 [v] "m" (*v), 257 [expectnot] "r" (expectnot), 258 [voffp] "b" (voffp), 259 [load] "m" (*load) 260 RSEQ_INJECT_INPUT 261 : "memory", "cc", "r17" 262 RSEQ_INJECT_CLOBBER 263 : abort, cmpfail 264 #ifdef RSEQ_COMPARE_TWICE 265 , error1, error2 266 #endif 267 ); 268 return 0; 269 abort: 270 RSEQ_INJECT_FAILED 271 return -1; 272 cmpfail: 273 return 1; 274 #ifdef RSEQ_COMPARE_TWICE 275 error1: 276 rseq_bug("cpu_id comparison failed"); 277 error2: 278 rseq_bug("expected value comparison failed"); 279 #endif 280 } 281 282 static inline __attribute__((always_inline)) 283 int rseq_addv(intptr_t *v, intptr_t count, int cpu) 284 { 285 RSEQ_INJECT_C(9) 286 287 __asm__ __volatile__ goto ( 288 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 289 /* Start rseq by storing table entry pointer into rseq_cs. */ 290 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 291 /* cmp cpuid */ 292 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 293 RSEQ_INJECT_ASM(3) 294 #ifdef RSEQ_COMPARE_TWICE 295 /* cmp cpuid */ 296 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 297 #endif 298 /* load the value of @v */ 299 RSEQ_ASM_OP_R_LOAD(v) 300 /* add @count to it */ 301 RSEQ_ASM_OP_R_ADD(count) 302 /* final store */ 303 RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 304 RSEQ_INJECT_ASM(4) 305 RSEQ_ASM_DEFINE_ABORT(4, abort) 306 : /* gcc asm goto does not allow outputs */ 307 : [cpu_id] "r" (cpu), 308 [current_cpu_id] "m" (__rseq_abi.cpu_id), 309 [rseq_cs] "m" (__rseq_abi.rseq_cs), 310 /* final store input */ 311 [v] "m" (*v), 312 [count] "r" (count) 313 RSEQ_INJECT_INPUT 314 : "memory", "cc", "r17" 315 RSEQ_INJECT_CLOBBER 316 : abort 317 #ifdef RSEQ_COMPARE_TWICE 318 , error1 319 #endif 320 ); 321 return 0; 322 abort: 323 RSEQ_INJECT_FAILED 324 return -1; 325 #ifdef RSEQ_COMPARE_TWICE 326 error1: 327 rseq_bug("cpu_id comparison failed"); 328 #endif 329 } 330 331 static inline __attribute__((always_inline)) 332 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 333 intptr_t *v2, intptr_t newv2, 334 intptr_t newv, int cpu) 335 { 336 RSEQ_INJECT_C(9) 337 338 __asm__ __volatile__ goto ( 339 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 340 /* Start rseq by storing table entry pointer into rseq_cs. */ 341 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 342 /* cmp cpuid */ 343 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 344 RSEQ_INJECT_ASM(3) 345 /* cmp @v equal to @expect */ 346 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 347 RSEQ_INJECT_ASM(4) 348 #ifdef RSEQ_COMPARE_TWICE 349 /* cmp cpuid */ 350 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 351 /* cmp @v equal to @expect */ 352 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 353 #endif 354 /* try store */ 355 RSEQ_ASM_OP_STORE(newv2, v2) 356 RSEQ_INJECT_ASM(5) 357 /* final store */ 358 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 359 RSEQ_INJECT_ASM(6) 360 RSEQ_ASM_DEFINE_ABORT(4, abort) 361 : /* gcc asm goto does not allow outputs */ 362 : [cpu_id] "r" (cpu), 363 [current_cpu_id] "m" (__rseq_abi.cpu_id), 364 [rseq_cs] "m" (__rseq_abi.rseq_cs), 365 /* try store input */ 366 [v2] "m" (*v2), 367 [newv2] "r" (newv2), 368 /* final store input */ 369 [v] "m" (*v), 370 [expect] "r" (expect), 371 [newv] "r" (newv) 372 RSEQ_INJECT_INPUT 373 : "memory", "cc", "r17" 374 RSEQ_INJECT_CLOBBER 375 : abort, cmpfail 376 #ifdef RSEQ_COMPARE_TWICE 377 , error1, error2 378 #endif 379 ); 380 return 0; 381 abort: 382 RSEQ_INJECT_FAILED 383 return -1; 384 cmpfail: 385 return 1; 386 #ifdef RSEQ_COMPARE_TWICE 387 error1: 388 rseq_bug("cpu_id comparison failed"); 389 error2: 390 rseq_bug("expected value comparison failed"); 391 #endif 392 } 393 394 static inline __attribute__((always_inline)) 395 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 396 intptr_t *v2, intptr_t newv2, 397 intptr_t newv, int cpu) 398 { 399 RSEQ_INJECT_C(9) 400 401 __asm__ __volatile__ goto ( 402 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 403 /* Start rseq by storing table entry pointer into rseq_cs. */ 404 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 405 /* cmp cpuid */ 406 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 407 RSEQ_INJECT_ASM(3) 408 /* cmp @v equal to @expect */ 409 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 410 RSEQ_INJECT_ASM(4) 411 #ifdef RSEQ_COMPARE_TWICE 412 /* cmp cpuid */ 413 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 414 /* cmp @v equal to @expect */ 415 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 416 #endif 417 /* try store */ 418 RSEQ_ASM_OP_STORE(newv2, v2) 419 RSEQ_INJECT_ASM(5) 420 /* for 'release' */ 421 "lwsync\n\t" 422 /* final store */ 423 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 424 RSEQ_INJECT_ASM(6) 425 RSEQ_ASM_DEFINE_ABORT(4, abort) 426 : /* gcc asm goto does not allow outputs */ 427 : [cpu_id] "r" (cpu), 428 [current_cpu_id] "m" (__rseq_abi.cpu_id), 429 [rseq_cs] "m" (__rseq_abi.rseq_cs), 430 /* try store input */ 431 [v2] "m" (*v2), 432 [newv2] "r" (newv2), 433 /* final store input */ 434 [v] "m" (*v), 435 [expect] "r" (expect), 436 [newv] "r" (newv) 437 RSEQ_INJECT_INPUT 438 : "memory", "cc", "r17" 439 RSEQ_INJECT_CLOBBER 440 : abort, cmpfail 441 #ifdef RSEQ_COMPARE_TWICE 442 , error1, error2 443 #endif 444 ); 445 return 0; 446 abort: 447 RSEQ_INJECT_FAILED 448 return -1; 449 cmpfail: 450 return 1; 451 #ifdef RSEQ_COMPARE_TWICE 452 error1: 453 rseq_bug("cpu_id comparison failed"); 454 error2: 455 rseq_bug("expected value comparison failed"); 456 #endif 457 } 458 459 static inline __attribute__((always_inline)) 460 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 461 intptr_t *v2, intptr_t expect2, 462 intptr_t newv, int cpu) 463 { 464 RSEQ_INJECT_C(9) 465 466 __asm__ __volatile__ goto ( 467 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 468 /* Start rseq by storing table entry pointer into rseq_cs. */ 469 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 470 /* cmp cpuid */ 471 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 472 RSEQ_INJECT_ASM(3) 473 /* cmp @v equal to @expect */ 474 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 475 RSEQ_INJECT_ASM(4) 476 /* cmp @v2 equal to @expct2 */ 477 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 478 RSEQ_INJECT_ASM(5) 479 #ifdef RSEQ_COMPARE_TWICE 480 /* cmp cpuid */ 481 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 482 /* cmp @v equal to @expect */ 483 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 484 /* cmp @v2 equal to @expct2 */ 485 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 486 #endif 487 /* final store */ 488 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 489 RSEQ_INJECT_ASM(6) 490 RSEQ_ASM_DEFINE_ABORT(4, abort) 491 : /* gcc asm goto does not allow outputs */ 492 : [cpu_id] "r" (cpu), 493 [current_cpu_id] "m" (__rseq_abi.cpu_id), 494 [rseq_cs] "m" (__rseq_abi.rseq_cs), 495 /* cmp2 input */ 496 [v2] "m" (*v2), 497 [expect2] "r" (expect2), 498 /* final store input */ 499 [v] "m" (*v), 500 [expect] "r" (expect), 501 [newv] "r" (newv) 502 RSEQ_INJECT_INPUT 503 : "memory", "cc", "r17" 504 RSEQ_INJECT_CLOBBER 505 : abort, cmpfail 506 #ifdef RSEQ_COMPARE_TWICE 507 , error1, error2, error3 508 #endif 509 ); 510 return 0; 511 abort: 512 RSEQ_INJECT_FAILED 513 return -1; 514 cmpfail: 515 return 1; 516 #ifdef RSEQ_COMPARE_TWICE 517 error1: 518 rseq_bug("cpu_id comparison failed"); 519 error2: 520 rseq_bug("1st expected value comparison failed"); 521 error3: 522 rseq_bug("2nd expected value comparison failed"); 523 #endif 524 } 525 526 static inline __attribute__((always_inline)) 527 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 528 void *dst, void *src, size_t len, 529 intptr_t newv, int cpu) 530 { 531 RSEQ_INJECT_C(9) 532 533 __asm__ __volatile__ goto ( 534 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 535 /* setup for mempcy */ 536 "mr %%r19, %[len]\n\t" 537 "mr %%r20, %[src]\n\t" 538 "mr %%r21, %[dst]\n\t" 539 /* Start rseq by storing table entry pointer into rseq_cs. */ 540 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 541 /* cmp cpuid */ 542 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 543 RSEQ_INJECT_ASM(3) 544 /* cmp @v equal to @expect */ 545 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 546 RSEQ_INJECT_ASM(4) 547 #ifdef RSEQ_COMPARE_TWICE 548 /* cmp cpuid */ 549 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 550 /* cmp @v equal to @expect */ 551 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 552 #endif 553 /* try memcpy */ 554 RSEQ_ASM_OP_R_MEMCPY() 555 RSEQ_INJECT_ASM(5) 556 /* final store */ 557 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 558 RSEQ_INJECT_ASM(6) 559 /* teardown */ 560 RSEQ_ASM_DEFINE_ABORT(4, abort) 561 : /* gcc asm goto does not allow outputs */ 562 : [cpu_id] "r" (cpu), 563 [current_cpu_id] "m" (__rseq_abi.cpu_id), 564 [rseq_cs] "m" (__rseq_abi.rseq_cs), 565 /* final store input */ 566 [v] "m" (*v), 567 [expect] "r" (expect), 568 [newv] "r" (newv), 569 /* try memcpy input */ 570 [dst] "r" (dst), 571 [src] "r" (src), 572 [len] "r" (len) 573 RSEQ_INJECT_INPUT 574 : "memory", "cc", "r17", "r18", "r19", "r20", "r21" 575 RSEQ_INJECT_CLOBBER 576 : abort, cmpfail 577 #ifdef RSEQ_COMPARE_TWICE 578 , error1, error2 579 #endif 580 ); 581 return 0; 582 abort: 583 RSEQ_INJECT_FAILED 584 return -1; 585 cmpfail: 586 return 1; 587 #ifdef RSEQ_COMPARE_TWICE 588 error1: 589 rseq_bug("cpu_id comparison failed"); 590 error2: 591 rseq_bug("expected value comparison failed"); 592 #endif 593 } 594 595 static inline __attribute__((always_inline)) 596 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 597 void *dst, void *src, size_t len, 598 intptr_t newv, int cpu) 599 { 600 RSEQ_INJECT_C(9) 601 602 __asm__ __volatile__ goto ( 603 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 604 /* setup for mempcy */ 605 "mr %%r19, %[len]\n\t" 606 "mr %%r20, %[src]\n\t" 607 "mr %%r21, %[dst]\n\t" 608 /* Start rseq by storing table entry pointer into rseq_cs. */ 609 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 610 /* cmp cpuid */ 611 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 612 RSEQ_INJECT_ASM(3) 613 /* cmp @v equal to @expect */ 614 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 615 RSEQ_INJECT_ASM(4) 616 #ifdef RSEQ_COMPARE_TWICE 617 /* cmp cpuid */ 618 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 619 /* cmp @v equal to @expect */ 620 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 621 #endif 622 /* try memcpy */ 623 RSEQ_ASM_OP_R_MEMCPY() 624 RSEQ_INJECT_ASM(5) 625 /* for 'release' */ 626 "lwsync\n\t" 627 /* final store */ 628 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 629 RSEQ_INJECT_ASM(6) 630 /* teardown */ 631 RSEQ_ASM_DEFINE_ABORT(4, abort) 632 : /* gcc asm goto does not allow outputs */ 633 : [cpu_id] "r" (cpu), 634 [current_cpu_id] "m" (__rseq_abi.cpu_id), 635 [rseq_cs] "m" (__rseq_abi.rseq_cs), 636 /* final store input */ 637 [v] "m" (*v), 638 [expect] "r" (expect), 639 [newv] "r" (newv), 640 /* try memcpy input */ 641 [dst] "r" (dst), 642 [src] "r" (src), 643 [len] "r" (len) 644 RSEQ_INJECT_INPUT 645 : "memory", "cc", "r17", "r18", "r19", "r20", "r21" 646 RSEQ_INJECT_CLOBBER 647 : abort, cmpfail 648 #ifdef RSEQ_COMPARE_TWICE 649 , error1, error2 650 #endif 651 ); 652 return 0; 653 abort: 654 RSEQ_INJECT_FAILED 655 return -1; 656 cmpfail: 657 return 1; 658 #ifdef RSEQ_COMPARE_TWICE 659 error1: 660 rseq_bug("cpu_id comparison failed"); 661 error2: 662 rseq_bug("expected value comparison failed"); 663 #endif 664 } 665 666 #undef STORE_WORD 667 #undef LOAD_WORD 668 #undef LOADX_WORD 669 #undef CMP_WORD 670 671 #endif /* !RSEQ_SKIP_FASTPATH */ 672
