cranelift: aarch64: zero-extend expected value in i32 atomic_cas loop

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -1713,7 +1713,7 @@ impl MachInstEmit for Inst {
                 /* Emit this:
                     again:
                      ldaxr{,b,h} x/w27, [x25]
-                     cmp         x27, x/w26 uxt{b,h}
+                     cmp         x27, x/w26 uxt{b,h,w}
                      b.ne        out
                      stlxr{,b,h} w24, x/w28, [x25]
                      cbnz        x24, again
@@ -1744,15 +1744,17 @@ impl MachInstEmit for Inst {
                 // ldaxr x27, [x25]
                 sink.put4(enc_ldaxr(ty, x27wr, x25));
 
-                // The top 32-bits are zero-extended by the ldaxr so we don't
-                // have to use UXTW, just the x-form of the register.
+                // The loaded value in x27 is zero-extended by the ldaxr, but the
+                // upper bits of the expected value in x26 are unspecified for
+                // narrow types, so compare against its zero-extension.
                 let (bit21, extend_op) = match ty {
                     I8 => (0b1, 0b000000),
                     I16 => (0b1, 0b001000),
+                    I32 => (0b1, 0b010000),
                     _ => (0b0, 0b000000),
                 };
                 let bits_31_21 = 0b111_01011_000 | bit21;
-                // cmp x27, x26 (== subs xzr, x27, x26)
+                // cmp x27, x/w26 uxt{b,h,w} (== subs xzr, x27, x/w26 uxt{b,h,w})
                 sink.put4(enc_arith_rrr(bits_31_21, extend_op, xzrwr, x27, x26));
 
                 // b.ne out

cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs

@@ -7821,7 +7821,7 @@ fn test_aarch64_binemit() {
             oldval: writable_xreg(27),
             scratch: writable_xreg(24),
         },
-        "3BFF5F887F031AEB610000543CFF188898FFFFB5",
+        "3BFF5F887F433AEB610000543CFF188898FFFFB5",
         "atomic_cas_loop_32 addr=x25, expect=x26, replacement=x28, oldval=x27, scratch=x24",
     ));
 

cranelift/filetests/filetests/isa/aarch64/atomic-cas.clif

@@ -41,7 +41,7 @@ block0(v0: i64, v1: i32, v2: i32):
 ;   mov x26, x1
 ;   mov x28, x2
 ;   ldaxr w27, [x25] ; trap: heap_oob
-;   cmp x27, x26
+;   cmp x27, w26, uxtw
 ;   b.ne #0x34
 ;   stlxr w24, w28, [x25] ; trap: heap_oob
 ;   cbnz x24, #0x20
@@ -53,3 +53,51 @@ block0(v0: i64, v1: i32, v2: i32):
 ;   ldp x29, x30, [sp], #0x10
 ;   ret
 
+function %atomic_cas_i32_ireduce(i64, i64, i32) -> i32 {
+block0(v0: i64, v1: i64, v2: i32):
+    v3 = ireduce.i32 v1
+    v4 = atomic_cas.i32 v0, v3, v2
+    return v4
+}
+
+; VCode:
+;   stp fp, lr, [sp, #-16]!
+;   mov fp, sp
+;   str x28, [sp, #-16]!
+;   stp x26, x27, [sp, #-16]!
+;   stp x24, x25, [sp, #-16]!
+; block0:
+;   mov x25, x0
+;   mov x26, x1
+;   mov x28, x2
+;   atomic_cas_loop_32 addr=x25, expect=x26, replacement=x28, oldval=x27, scratch=x24
+;   mov x0, x27
+;   ldp x24, x25, [sp], #16
+;   ldp x26, x27, [sp], #16
+;   ldr x28, [sp], #16
+;   ldp fp, lr, [sp], #16
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   stp x29, x30, [sp, #-0x10]!
+;   mov x29, sp
+;   str x28, [sp, #-0x10]!
+;   stp x26, x27, [sp, #-0x10]!
+;   stp x24, x25, [sp, #-0x10]!
+; block1: ; offset 0x14
+;   mov x25, x0
+;   mov x26, x1
+;   mov x28, x2
+;   ldaxr w27, [x25] ; trap: heap_oob
+;   cmp x27, w26, uxtw
+;   b.ne #0x34
+;   stlxr w24, w28, [x25] ; trap: heap_oob
+;   cbnz x24, #0x20
+;   mov x0, x27
+;   ldp x24, x25, [sp], #0x10
+;   ldp x26, x27, [sp], #0x10
+;   ldr x28, [sp], #0x10
+;   ldp x29, x30, [sp], #0x10
+;   ret
+

cranelift/filetests/filetests/runtests/atomic-cas-i32-upper-bits.clif

@@ -0,0 +1,30 @@
+test interpret
+test run
+target aarch64
+target aarch64 has_lse
+target x86_64
+target s390x
+target riscv64 has_a
+target riscv64 has_c has_zcb
+
+; The upper 32 bits of the register holding an `i32` value are unspecified on
+; 64-bit backends. An `i32` atomic_cas must only compare the low 32 bits of the
+; expected value with the value in memory.
+
+function %atomic_cas_i32_ireduce(i32, i64, i32) -> i32, i32 {
+    ss0 = explicit_slot 4
+
+block0(v0: i32, v1: i64, v2: i32):
+    v3 = stack_addr.i64 ss0
+    store.i32 notrap v0, v3
+    v4 = ireduce.i32 v1
+    v5 = atomic_cas.i32 v3, v4, v2
+    v6 = load.i32 notrap v3
+    return v5, v6
+}
+; run: %atomic_cas_i32_ireduce(5, 5, 7) == [5, 7]
+; run: %atomic_cas_i32_ireduce(5, 0x1_00000005, 7) == [5, 7]
+; run: %atomic_cas_i32_ireduce(5, 0xFFFFFFFF_00000005, 7) == [5, 7]
+; run: %atomic_cas_i32_ireduce(0xC0FFEEEE, 0xDECAFFFF_C0FFEEEE, 1) == [0xC0FFEEEE, 1]
+; run: %atomic_cas_i32_ireduce(5, 0x1_00000006, 7) == [5, 5]
+; run: %atomic_cas_i32_ireduce(5, 0x5_00000000, 7) == [5, 5]