diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs index 66d6cd1fe0a7..4600e1b55754 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs @@ -1713,7 +1713,7 @@ impl MachInstEmit for Inst { /* Emit this: again: ldaxr{,b,h} x/w27, [x25] - cmp x27, x/w26 uxt{b,h} + cmp x27, x/w26 uxt{b,h,w} b.ne out stlxr{,b,h} w24, x/w28, [x25] cbnz x24, again @@ -1744,15 +1744,17 @@ impl MachInstEmit for Inst { // ldaxr x27, [x25] sink.put4(enc_ldaxr(ty, x27wr, x25)); - // The top 32-bits are zero-extended by the ldaxr so we don't - // have to use UXTW, just the x-form of the register. + // The loaded value in x27 is zero-extended by the ldaxr, but the + // upper bits of the expected value in x26 are unspecified for + // narrow types, so compare against its zero-extension. let (bit21, extend_op) = match ty { I8 => (0b1, 0b000000), I16 => (0b1, 0b001000), + I32 => (0b1, 0b010000), _ => (0b0, 0b000000), }; let bits_31_21 = 0b111_01011_000 | bit21; - // cmp x27, x26 (== subs xzr, x27, x26) + // cmp x27, x/w26 uxt{b,h,w} (== subs xzr, x27, x/w26 uxt{b,h,w}) sink.put4(enc_arith_rrr(bits_31_21, extend_op, xzrwr, x27, x26)); // b.ne out diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs index 04d7e77deff5..18b059d85c66 100644 --- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs +++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs @@ -7821,7 +7821,7 @@ fn test_aarch64_binemit() { oldval: writable_xreg(27), scratch: writable_xreg(24), }, - "3BFF5F887F031AEB610000543CFF188898FFFFB5", + "3BFF5F887F433AEB610000543CFF188898FFFFB5", "atomic_cas_loop_32 addr=x25, expect=x26, replacement=x28, oldval=x27, scratch=x24", )); diff --git a/cranelift/filetests/filetests/isa/aarch64/atomic-cas.clif b/cranelift/filetests/filetests/isa/aarch64/atomic-cas.clif index dd0e2994563a..86c420e7eeb3 100644 --- a/cranelift/filetests/filetests/isa/aarch64/atomic-cas.clif +++ b/cranelift/filetests/filetests/isa/aarch64/atomic-cas.clif @@ -41,7 +41,7 @@ block0(v0: i64, v1: i32, v2: i32): ; mov x26, x1 ; mov x28, x2 ; ldaxr w27, [x25] ; trap: heap_oob -; cmp x27, x26 +; cmp x27, w26, uxtw ; b.ne #0x34 ; stlxr w24, w28, [x25] ; trap: heap_oob ; cbnz x24, #0x20 @@ -53,3 +53,51 @@ block0(v0: i64, v1: i32, v2: i32): ; ldp x29, x30, [sp], #0x10 ; ret +function %atomic_cas_i32_ireduce(i64, i64, i32) -> i32 { +block0(v0: i64, v1: i64, v2: i32): + v3 = ireduce.i32 v1 + v4 = atomic_cas.i32 v0, v3, v2 + return v4 +} + +; VCode: +; stp fp, lr, [sp, #-16]! +; mov fp, sp +; str x28, [sp, #-16]! +; stp x26, x27, [sp, #-16]! +; stp x24, x25, [sp, #-16]! +; block0: +; mov x25, x0 +; mov x26, x1 +; mov x28, x2 +; atomic_cas_loop_32 addr=x25, expect=x26, replacement=x28, oldval=x27, scratch=x24 +; mov x0, x27 +; ldp x24, x25, [sp], #16 +; ldp x26, x27, [sp], #16 +; ldr x28, [sp], #16 +; ldp fp, lr, [sp], #16 +; ret +; +; Disassembled: +; block0: ; offset 0x0 +; stp x29, x30, [sp, #-0x10]! +; mov x29, sp +; str x28, [sp, #-0x10]! +; stp x26, x27, [sp, #-0x10]! +; stp x24, x25, [sp, #-0x10]! +; block1: ; offset 0x14 +; mov x25, x0 +; mov x26, x1 +; mov x28, x2 +; ldaxr w27, [x25] ; trap: heap_oob +; cmp x27, w26, uxtw +; b.ne #0x34 +; stlxr w24, w28, [x25] ; trap: heap_oob +; cbnz x24, #0x20 +; mov x0, x27 +; ldp x24, x25, [sp], #0x10 +; ldp x26, x27, [sp], #0x10 +; ldr x28, [sp], #0x10 +; ldp x29, x30, [sp], #0x10 +; ret + diff --git a/cranelift/filetests/filetests/runtests/atomic-cas-i32-upper-bits.clif b/cranelift/filetests/filetests/runtests/atomic-cas-i32-upper-bits.clif new file mode 100644 index 000000000000..44ec6a93dca6 --- /dev/null +++ b/cranelift/filetests/filetests/runtests/atomic-cas-i32-upper-bits.clif @@ -0,0 +1,30 @@ +test interpret +test run +target aarch64 +target aarch64 has_lse +target x86_64 +target s390x +target riscv64 has_a +target riscv64 has_c has_zcb + +; The upper 32 bits of the register holding an `i32` value are unspecified on +; 64-bit backends. An `i32` atomic_cas must only compare the low 32 bits of the +; expected value with the value in memory. + +function %atomic_cas_i32_ireduce(i32, i64, i32) -> i32, i32 { + ss0 = explicit_slot 4 + +block0(v0: i32, v1: i64, v2: i32): + v3 = stack_addr.i64 ss0 + store.i32 notrap v0, v3 + v4 = ireduce.i32 v1 + v5 = atomic_cas.i32 v3, v4, v2 + v6 = load.i32 notrap v3 + return v5, v6 +} +; run: %atomic_cas_i32_ireduce(5, 5, 7) == [5, 7] +; run: %atomic_cas_i32_ireduce(5, 0x1_00000005, 7) == [5, 7] +; run: %atomic_cas_i32_ireduce(5, 0xFFFFFFFF_00000005, 7) == [5, 7] +; run: %atomic_cas_i32_ireduce(0xC0FFEEEE, 0xDECAFFFF_C0FFEEEE, 1) == [0xC0FFEEEE, 1] +; run: %atomic_cas_i32_ireduce(5, 0x1_00000006, 7) == [5, 5] +; run: %atomic_cas_i32_ireduce(5, 0x5_00000000, 7) == [5, 5]