src/cmd/compile/internal/ssa/gen/MIPS64.rules | 3 ++- src/cmd/compile/internal/ssa/gen/RISCV64.rules | 2 +- src/cmd/compile/internal/ssa/rewriteMIPS64.go | 24 ++++++++++++++++++++++-- src/cmd/compile/internal/ssa/rewriteRISCV64.go | 24 ++++++++++++++++++++++-- src/runtime/internal/atomic/atomic_test.go | 30 ++++++++++++++++++++++++++++++ diff --git a/src/cmd/compile/internal/ssa/gen/MIPS64.rules b/src/cmd/compile/internal/ssa/gen/MIPS64.rules index 292ff2fc7966092fac267f78a39a1d939e221c12..0d6d30fa4c93c9eeeaffca21574a12a1b6fcb335 100644 --- a/src/cmd/compile/internal/ssa/gen/MIPS64.rules +++ b/src/cmd/compile/internal/ssa/gen/MIPS64.rules @@ -392,7 +392,8 @@ (AtomicExchange(32|64) ...) => (LoweredAtomicExchange(32|64) ...) (AtomicAdd(32|64) ...) => (LoweredAtomicAdd(32|64) ...) -(AtomicCompareAndSwap(32|64) ...) => (LoweredAtomicCas(32|64) ...) +(AtomicCompareAndSwap32 ptr old new mem) => (LoweredAtomicCas32 ptr (SignExt32to64 old) new mem) +(AtomicCompareAndSwap64 ...) => (LoweredAtomicCas64 ...) // checks (NilCheck ...) => (LoweredNilCheck ...) diff --git a/src/cmd/compile/internal/ssa/gen/RISCV64.rules b/src/cmd/compile/internal/ssa/gen/RISCV64.rules index 7aea622c5e81139212726c98f7c72db9ca9be3c3..acef3df3893b9a48eb9ff5bb5819217ed272f41b 100644 --- a/src/cmd/compile/internal/ssa/gen/RISCV64.rules +++ b/src/cmd/compile/internal/ssa/gen/RISCV64.rules @@ -568,7 +568,7 @@ (SLLI [3] (ANDI [3] ptr)))) mem) (AtomicAnd32 ...) => (LoweredAtomicAnd32 ...) -(AtomicCompareAndSwap32 ...) => (LoweredAtomicCas32 ...) +(AtomicCompareAndSwap32 ptr old new mem) => (LoweredAtomicCas32 ptr (SignExt32to64 old) new mem) (AtomicCompareAndSwap64 ...) => (LoweredAtomicCas64 ...) (AtomicExchange32 ...) => (LoweredAtomicExchange32 ...) diff --git a/src/cmd/compile/internal/ssa/rewriteMIPS64.go b/src/cmd/compile/internal/ssa/rewriteMIPS64.go index 1fbd556b5cbcd7d32cb9ad6424d5c5d2a14580e7..6a0fd3ad6ee20352bb2eb4b72fc125de7cc8646c 100644 --- a/src/cmd/compile/internal/ssa/rewriteMIPS64.go +++ b/src/cmd/compile/internal/ssa/rewriteMIPS64.go @@ -52,8 +52,7 @@ case OpAtomicAdd64: v.Op = OpMIPS64LoweredAtomicAdd64 return true case OpAtomicCompareAndSwap32: - v.Op = OpMIPS64LoweredAtomicCas32 - return true + return rewriteValueMIPS64_OpAtomicCompareAndSwap32(v) case OpAtomicCompareAndSwap64: v.Op = OpMIPS64LoweredAtomicCas64 return true @@ -694,6 +693,27 @@ base := v_0 v.reset(OpMIPS64MOVVaddr) v.Aux = symToAux(sym) v.AddArg(base) + return true + } +} +func rewriteValueMIPS64_OpAtomicCompareAndSwap32(v *Value) bool { + v_3 := v.Args[3] + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (AtomicCompareAndSwap32 ptr old new mem) + // result: (LoweredAtomicCas32 ptr (SignExt32to64 old) new mem) + for { + ptr := v_0 + old := v_1 + new := v_2 + mem := v_3 + v.reset(OpMIPS64LoweredAtomicCas32) + v0 := b.NewValue0(v.Pos, OpSignExt32to64, typ.Int64) + v0.AddArg(old) + v.AddArg4(ptr, v0, new, mem) return true } } diff --git a/src/cmd/compile/internal/ssa/rewriteRISCV64.go b/src/cmd/compile/internal/ssa/rewriteRISCV64.go index 6828d97ff8f7df0e26d9360b776110f5aeeb8c7b..b277979061ecc8620b32de15b78e68b8dc6dfaf4 100644 --- a/src/cmd/compile/internal/ssa/rewriteRISCV64.go +++ b/src/cmd/compile/internal/ssa/rewriteRISCV64.go @@ -61,8 +61,7 @@ return true case OpAtomicAnd8: return rewriteValueRISCV64_OpAtomicAnd8(v) case OpAtomicCompareAndSwap32: - v.Op = OpRISCV64LoweredAtomicCas32 - return true + return rewriteValueRISCV64_OpAtomicCompareAndSwap32(v) case OpAtomicCompareAndSwap64: v.Op = OpRISCV64LoweredAtomicCas64 return true @@ -762,6 +761,27 @@ v5.AddArg(v6) v2.AddArg2(v3, v5) v1.AddArg(v2) v.AddArg3(v0, v1, mem) + return true + } +} +func rewriteValueRISCV64_OpAtomicCompareAndSwap32(v *Value) bool { + v_3 := v.Args[3] + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (AtomicCompareAndSwap32 ptr old new mem) + // result: (LoweredAtomicCas32 ptr (SignExt32to64 old) new mem) + for { + ptr := v_0 + old := v_1 + new := v_2 + mem := v_3 + v.reset(OpRISCV64LoweredAtomicCas32) + v0 := b.NewValue0(v.Pos, OpSignExt32to64, typ.Int64) + v0.AddArg(old) + v.AddArg4(ptr, v0, new, mem) return true } } diff --git a/src/runtime/internal/atomic/atomic_test.go b/src/runtime/internal/atomic/atomic_test.go index 2ae60b8507ef5831e2d5345514224696c0b85623..2427bfd211ee9cd84e858783d3a2c63473b7c2fe 100644 --- a/src/runtime/internal/atomic/atomic_test.go +++ b/src/runtime/internal/atomic/atomic_test.go @@ -345,6 +345,36 @@ } } } +func TestCasRel(t *testing.T) { + const _magic = 0x5a5aa5a5 + var x struct { + before uint32 + i uint32 + after uint32 + o uint32 + n uint32 + } + + x.before = _magic + x.after = _magic + for j := 0; j < 32; j += 1 { + x.i = (1 << j) + 0 + x.o = (1 << j) + 0 + x.n = (1 << j) + 1 + if !atomic.CasRel(&x.i, x.o, x.n) { + t.Fatalf("should have swapped %#x %#x", x.o, x.n) + } + + if x.i != x.n { + t.Fatalf("wrong x.i after swap: x.i=%#x x.n=%#x", x.i, x.n) + } + + if x.before != _magic || x.after != _magic { + t.Fatalf("wrong magic: %#x _ %#x != %#x _ %#x", x.before, x.after, _magic, _magic) + } + } +} + func TestStorepNoWB(t *testing.T) { var p [2]*int for i := range p {