src/runtime/os3_solaris.go | 3 ++- src/runtime/os_aix.go | 3 ++- src/runtime/os_js.go | 3 ++- src/runtime/os_openbsd_syscall2.go | 5 +++-- src/runtime/os_plan9.go | 2 +- src/runtime/os_windows.go | 2 +- src/runtime/proc.go | 48 ++++++++++++++++++++++++++++-------------------- src/runtime/runtime2.go | 9 ++++++++- src/runtime/stubs2.go | 9 ++++++--- src/runtime/sys_darwin.go | 3 ++- src/runtime/sys_dragonfly_amd64.s | 2 +- src/runtime/sys_freebsd_386.s | 2 +- src/runtime/sys_freebsd_amd64.s | 2 +- src/runtime/sys_freebsd_arm.s | 2 +- src/runtime/sys_freebsd_arm64.s | 2 +- src/runtime/sys_linux_386.s | 2 +- src/runtime/sys_linux_amd64.s | 2 +- src/runtime/sys_linux_arm.s | 2 +- src/runtime/sys_linux_arm64.s | 2 +- src/runtime/sys_linux_mips64x.s | 2 +- src/runtime/sys_linux_mipsx.s | 2 +- src/runtime/sys_linux_ppc64x.s | 2 +- src/runtime/sys_linux_riscv64.s | 2 +- src/runtime/sys_linux_s390x.s | 2 +- src/runtime/sys_netbsd_386.s | 2 +- src/runtime/sys_netbsd_amd64.s | 2 +- src/runtime/sys_netbsd_arm.s | 2 +- src/runtime/sys_netbsd_arm64.s | 2 +- src/runtime/sys_openbsd2.go | 3 ++- src/runtime/sys_openbsd_mips64.s | 2 +- diff --git a/src/runtime/os3_solaris.go b/src/runtime/os3_solaris.go index 4aba0ff64ba3da16d3ac78b21e2fe6940a9454fe..3ccce776ca35d64c7bb58762ec248fda784d056b 100644 --- a/src/runtime/os3_solaris.go +++ b/src/runtime/os3_solaris.go @@ -7,6 +7,7 @@ import ( "internal/abi" "internal/goarch" + "runtime/internal/atomic" "unsafe" ) @@ -184,7 +185,7 @@ throw("newosproc") } } -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on Solaris because we let // libc clean up threads. throw("exitThread") diff --git a/src/runtime/os_aix.go b/src/runtime/os_aix.go index 292ff94795677a65c8c661867a7f118e742e1d96..41352b3a5a1f19109f31adb128e9a2ce977d46a7 100644 --- a/src/runtime/os_aix.go +++ b/src/runtime/os_aix.go @@ -8,6 +8,7 @@ package runtime import ( "internal/abi" + "runtime/internal/atomic" "unsafe" ) @@ -232,7 +233,7 @@ } } -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on AIX because we let // libc clean up threads. throw("exitThread") diff --git a/src/runtime/os_js.go b/src/runtime/os_js.go index 9ed916705bc517e906f1f1618fc0e58ae3492277..6f358d36d6f1621d15c4da2b9687aef0085bc2dc 100644 --- a/src/runtime/os_js.go +++ b/src/runtime/os_js.go @@ -7,6 +7,7 @@ package runtime import ( + "runtime/internal/atomic" "unsafe" ) @@ -35,7 +36,7 @@ func usleep_no_g(usec uint32) { usleep(usec) } -func exitThread(wait *uint32) +func exitThread(wait *atomic.Uint32) type mOS struct{} diff --git a/src/runtime/os_openbsd_syscall2.go b/src/runtime/os_openbsd_syscall2.go index 99542fb2de161786d8974b17fb96d3476205b99e..810d599508886f8ba555ee0ee3635c0053492524 100644 --- a/src/runtime/os_openbsd_syscall2.go +++ b/src/runtime/os_openbsd_syscall2.go @@ -7,6 +7,7 @@ package runtime import ( + "runtime/internal/atomic" "unsafe" ) @@ -48,11 +49,11 @@ // return value is only set on linux to be used in osinit() func madvise(addr unsafe.Pointer, n uintptr, flags int32) int32 -// exitThread terminates the current thread, writing *wait = 0 when +// exitThread terminates the current thread, writing *wait = freeMStack when // the stack is safe to reclaim. // //go:noescape -func exitThread(wait *uint32) +func exitThread(wait *atomic.Uint32) //go:noescape func obsdsigprocmask(how int32, new sigset) sigset diff --git a/src/runtime/os_plan9.go b/src/runtime/os_plan9.go index 975d460a7da924ede62917c0b6800d826cfdf4c5..2bb6ec0fcee7e8a2f2ce22e742afd998fa2f938a 100644 --- a/src/runtime/os_plan9.go +++ b/src/runtime/os_plan9.go @@ -458,7 +458,7 @@ tstart_plan9(mp) } } -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on Plan 9 because we let // the OS clean up threads. throw("exitThread") diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index c76add78025a26bb72455dc4a8728f449ade88ed..b49e9baec5880302871fcc34770c0a85e39be6d8 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -939,7 +939,7 @@ // Check os_linux.go for an implementation that might actually work. throw("bad newosproc0") } -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { // We should never reach exitThread on Windows because we let // the OS clean up threads. throw("exitThread") diff --git a/src/runtime/proc.go b/src/runtime/proc.go index b997a467bab447b58edd9475504a6ab0de8a5614..cae15bc8e2c32d6df7f53f2bded6fa742e845c76 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -1508,19 +1508,18 @@ } } throw("m not found in allm") found: - if !osStack { - // Delay reaping m until it's done with the stack. - // - // If this is using an OS stack, the OS will free it - // so there's no need for reaping. - atomic.Store(&m.freeWait, 1) - // Put m on the free list, though it will not be reaped until - // freeWait is 0. Note that the free list must not be linked - // through alllink because some functions walk allm without - // locking, so may be using alllink. - m.freelink = sched.freem - sched.freem = m - } + // Delay reaping m until it's done with the stack. + // + // Put mp on the free list, though it will not be reaped while freeWait + // is freeMWait. mp is no longer reachable via allm, so even if it is + // on an OS stack, we must keep a reference to mp alive so that the GC + // doesn't free mp while we are still using it. + // + // Note that the free list must not be linked through alllink because + // some functions walk allm without locking, so may be using alllink. + m.freeWait.Store(freeMWait) + m.freelink = sched.freem + sched.freem = m unlock(&sched.lock) atomic.Xadd64(&ncgocall, int64(m.ncgocall)) @@ -1550,6 +1549,9 @@ // longer take any locks. mdestroy(m) if osStack { + // No more uses of mp, so it is safe to drop the reference. + m.freeWait.Store(freeMRef) + // Return from mstart and let the system thread // library free the g0 stack and terminate the thread. return @@ -1721,19 +1723,25 @@ if sched.freem != nil { lock(&sched.lock) var newList *m for freem := sched.freem; freem != nil; { - if freem.freeWait != 0 { + wait := freem.freeWait.Load() + if wait == freeMWait { next := freem.freelink freem.freelink = newList newList = freem freem = next continue } - // stackfree must be on the system stack, but allocm is - // reachable off the system stack transitively from - // startm. - systemstack(func() { - stackfree(freem.g0.stack) - }) + // Free the stack if needed. For freeMRef, there is + // nothing to do except drop freem from the sched.freem + // list. + if wait == freeMStack { + // stackfree must be on the system stack, but allocm is + // reachable off the system stack transitively from + // startm. + systemstack(func() { + stackfree(freem.g0.stack) + }) + } freem = freem.freelink } sched.freem = newList diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index b40045e4a5b36e399192068c52e6881807062401..9cd4777cf8de2d7e8ebdc038064a6fad639a9893 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -510,6 +510,13 @@ tlsSlots = 6 tlsSize = tlsSlots * goarch.PtrSize ) +// Values for m.freeWait. +const ( + freeMStack = 0 // M done, free stack and reference. + freeMRef = 1 // M done, free reference. + freeMWait = 2 // M still in use. +) + type m struct { g0 *g // goroutine with scheduling stack morebuf gobuf // gobuf arg to morestack @@ -540,7 +547,7 @@ blocked bool // m is blocked on a note newSigstack bool // minit on C thread called sigaltstack printlock int8 incgo bool // m is executing a cgo call - freeWait uint32 // if == 0, safe to free g0 and delete m (atomic) + freeWait atomic.Uint32 // Whether it is safe to free g0 and delete m (one of freeMRef, freeMStack, freeMWait) fastrand uint64 needextram bool traceback uint8 diff --git a/src/runtime/stubs2.go b/src/runtime/stubs2.go index 9aa965454d991a4fda9dd84c786d65f5d9f94f8d..0d211e2db98e44609b381c374375c1fae0a37d78 100644 --- a/src/runtime/stubs2.go +++ b/src/runtime/stubs2.go @@ -6,7 +6,10 @@ //go:build !aix && !darwin && !js && !openbsd && !plan9 && !solaris && !windows package runtime -import "unsafe" +import ( + "runtime/internal/atomic" + "unsafe" +) // read calls the read system call. // It returns a non-negative number of bytes written or a negative errno value. @@ -33,8 +36,8 @@ // return value is only set on linux to be used in osinit() func madvise(addr unsafe.Pointer, n uintptr, flags int32) int32 -// exitThread terminates the current thread, writing *wait = 0 when +// exitThread terminates the current thread, writing *wait = freeMStack when // the stack is safe to reclaim. // //go:noescape -func exitThread(wait *uint32) +func exitThread(wait *atomic.Uint32) diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index 58b3a9171c79740c98b786eb0af32dbed618d723..c90cc78968caca7e79888e1e0632fdf8a2112404 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -6,6 +6,7 @@ package runtime import ( "internal/abi" + "runtime/internal/atomic" "unsafe" ) @@ -473,7 +474,7 @@ } func pthread_cond_signal_trampoline() // Not used on Darwin, but must be defined. -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { } //go:nosplit diff --git a/src/runtime/sys_dragonfly_amd64.s b/src/runtime/sys_dragonfly_amd64.s index d57bc2a7a4ee8591a5e2c58cde77772668f6aaa0..3af0928828a7af1bd08cc46519311b16eb91e699 100644 --- a/src/runtime/sys_dragonfly_amd64.s +++ b/src/runtime/sys_dragonfly_amd64.s @@ -65,7 +65,7 @@ SYSCALL MOVL $0xf1, 0xf1 // crash RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVQ wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_freebsd_386.s b/src/runtime/sys_freebsd_386.s index 97e6d9ab36634ced64091e5831d2d4589405824d..d4c4cc7fdbbd076ec65a18eafae84d5a6d5d85f6 100644 --- a/src/runtime/sys_freebsd_386.s +++ b/src/runtime/sys_freebsd_386.s @@ -63,7 +63,7 @@ GLOBL exitStack<>(SB),RODATA,$8 DATA exitStack<>+0x00(SB)/4, $0 DATA exitStack<>+0x04(SB)/4, $0 -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVL wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_freebsd_amd64.s b/src/runtime/sys_freebsd_amd64.s index 165e97c60d735881ea3797df151f5c3aa79f9409..57ae0399a5f4b9f950f85d42c98deb8bc5ae01d0 100644 --- a/src/runtime/sys_freebsd_amd64.s +++ b/src/runtime/sys_freebsd_amd64.s @@ -60,7 +60,7 @@ SYSCALL MOVL $0xf1, 0xf1 // crash RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVQ wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_freebsd_arm.s b/src/runtime/sys_freebsd_arm.s index b12e47c576daeb8a3b3556f405e5c5fbea0d4007..8546eba8058e2285e7d7d7d98c65a72dd853376f 100644 --- a/src/runtime/sys_freebsd_arm.s +++ b/src/runtime/sys_freebsd_arm.s @@ -86,7 +86,7 @@ MOVW.CS $0, R8 // crash on syscall failure MOVW.CS R8, (R8) RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVW wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_freebsd_arm64.s b/src/runtime/sys_freebsd_arm64.s index 1aa09e87ca1dc3fc304ff01f8925525a9c72dcf5..7dab3028de41e6a41c27f23ebc27505c60fcc02e 100644 --- a/src/runtime/sys_freebsd_arm64.s +++ b/src/runtime/sys_freebsd_arm64.s @@ -98,7 +98,7 @@ SVC MOVD $0, R0 MOVD R0, (R0) -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVD wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_linux_386.s b/src/runtime/sys_linux_386.s index 6df812234c3f9cf4591a83375a2dc39b4745662f..ddc5534352526120f859da9a6fa71765a1bca029 100644 --- a/src/runtime/sys_linux_386.s +++ b/src/runtime/sys_linux_386.s @@ -78,7 +78,7 @@ INVOKE_SYSCALL INT $3 // not reached RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVL wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index f0e58e11db36ec56ed0bf2de730f198a64cec0a3..5431dea4fc178c0c4dde9bb9f32c0cf066dddff6 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -60,7 +60,7 @@ MOVL $SYS_exit_group, AX SYSCALL RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVQ wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s index ca443b699fdcf3b2cf6b3866fadad8a008ff5c57..e41afdb066a3bb34bebddf77d532ded6c5745304 100644 --- a/src/runtime/sys_linux_arm.s +++ b/src/runtime/sys_linux_arm.s @@ -131,7 +131,7 @@ MOVW $1234, R0 MOVW $1003, R1 MOVW R0, (R1) // fail hard -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-4 MOVW wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_linux_arm64.s b/src/runtime/sys_linux_arm64.s index 1276c077d701f2c8bf894d780a79d8938d05268e..47e17182527fd9713aa48c63c17e8e7c2bc7e4b0 100644 --- a/src/runtime/sys_linux_arm64.s +++ b/src/runtime/sys_linux_arm64.s @@ -59,7 +59,7 @@ MOVD $SYS_exit_group, R8 SVC RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVD wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_linux_mips64x.s b/src/runtime/sys_linux_mips64x.s index 0df25979939357b4b464e0aaa44586903ef7be44..315a6e48354f46ca04e3ba66d06295a4f8871ac9 100644 --- a/src/runtime/sys_linux_mips64x.s +++ b/src/runtime/sys_linux_mips64x.s @@ -56,7 +56,7 @@ MOVV $SYS_exit_group, R2 SYSCALL RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVV wait+0(FP), R1 // We're done using the stack. diff --git a/src/runtime/sys_linux_mipsx.s b/src/runtime/sys_linux_mipsx.s index 2207e9ab98f702fb0a2b43ddf6d2a6d19a1d668d..991b7ba60e252f5a2e514d986de38fa25b1f5080 100644 --- a/src/runtime/sys_linux_mipsx.s +++ b/src/runtime/sys_linux_mipsx.s @@ -56,7 +56,7 @@ SYSCALL UNDEF RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVW wait+0(FP), R1 // We're done using the stack. diff --git a/src/runtime/sys_linux_ppc64x.s b/src/runtime/sys_linux_ppc64x.s index dc3d89fae741c7ff808cf38bc253e2287ef91d4c..01d6c855979a340db5ff90a436c9e4d19d16355e 100644 --- a/src/runtime/sys_linux_ppc64x.s +++ b/src/runtime/sys_linux_ppc64x.s @@ -55,7 +55,7 @@ MOVW code+0(FP), R3 SYSCALL $SYS_exit_group RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVD wait+0(FP), R1 // We're done using the stack. diff --git a/src/runtime/sys_linux_riscv64.s b/src/runtime/sys_linux_riscv64.s index a3da46d1362fbcd615ff4378cbc229b302d6e644..8b1b26b03b5b34aec2810c0acac8921df36b2e72 100644 --- a/src/runtime/sys_linux_riscv64.s +++ b/src/runtime/sys_linux_riscv64.s @@ -61,7 +61,7 @@ MOV $SYS_exit_group, A7 ECALL RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOV wait+0(FP), A0 // We're done using the stack. diff --git a/src/runtime/sys_linux_s390x.s b/src/runtime/sys_linux_s390x.s index 886add8b543b3c927db0e362f535a1c90ec6d1cb..ca080b422517f0476158eb2ba91907540f969316 100644 --- a/src/runtime/sys_linux_s390x.s +++ b/src/runtime/sys_linux_s390x.s @@ -52,7 +52,7 @@ MOVW $SYS_exit_group, R1 SYSCALL RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT|NOFRAME,$0-8 MOVD wait+0(FP), R1 // We're done using the stack. diff --git a/src/runtime/sys_netbsd_386.s b/src/runtime/sys_netbsd_386.s index 8a33894892f1f1e0ca8b1b9612b470984aee89a9..6c5386bcf1270b059ad5d8851890a8d9531ef42c 100644 --- a/src/runtime/sys_netbsd_386.s +++ b/src/runtime/sys_netbsd_386.s @@ -53,7 +53,7 @@ INT $0x80 MOVL $0xf1, 0xf1 // crash RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVL wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_netbsd_amd64.s b/src/runtime/sys_netbsd_amd64.s index 02f5b4ba3b71c386ee2beb37db3e8c332e3aff34..c1cd95df1493d12cbffe1409c1d16ea994c8dc31 100644 --- a/src/runtime/sys_netbsd_amd64.s +++ b/src/runtime/sys_netbsd_amd64.s @@ -122,7 +122,7 @@ SYSCALL MOVL $0xf1, 0xf1 // crash RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVQ wait+0(FP), AX // We're done using the stack. diff --git a/src/runtime/sys_netbsd_arm.s b/src/runtime/sys_netbsd_arm.s index 3a763b2a6a7730035488b5323ff32023c129f50e..2422b0282e99de34991f022d043dfc63bc90d478 100644 --- a/src/runtime/sys_netbsd_arm.s +++ b/src/runtime/sys_netbsd_arm.s @@ -56,7 +56,7 @@ MOVW.CS $0, R8 // crash on syscall failure MOVW.CS R8, (R8) RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-4 MOVW wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_netbsd_arm64.s b/src/runtime/sys_netbsd_arm64.s index 8a0496e80722e349bfc59ffe405be2419b99cc02..6d2c31631d0e7c28952ce82eba199479c9313fe9 100644 --- a/src/runtime/sys_netbsd_arm64.s +++ b/src/runtime/sys_netbsd_arm64.s @@ -114,7 +114,7 @@ SVC $SYS_exit MOVD $0, R0 // If we're still running, MOVD R0, (R0) // crash -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0-8 MOVD wait+0(FP), R0 // We're done using the stack. diff --git a/src/runtime/sys_openbsd2.go b/src/runtime/sys_openbsd2.go index 4d50b4f6b136f23d5c615acadf1ad28748e130cc..cba35cfee3e73db7f5b3c749961c08d1fa96bdfc 100644 --- a/src/runtime/sys_openbsd2.go +++ b/src/runtime/sys_openbsd2.go @@ -8,6 +8,7 @@ package runtime import ( "internal/abi" + "runtime/internal/atomic" "unsafe" ) @@ -250,7 +251,7 @@ } func sigaltstack_trampoline() // Not used on OpenBSD, but must be defined. -func exitThread(wait *uint32) { +func exitThread(wait *atomic.Uint32) { } //go:nosplit diff --git a/src/runtime/sys_openbsd_mips64.s b/src/runtime/sys_openbsd_mips64.s index f8ae8e7c3014b7eb8a0ff28a5d517354ded709fc..bc392e4c54d436bdb01f317dfdc51ac95b30d363 100644 --- a/src/runtime/sys_openbsd_mips64.s +++ b/src/runtime/sys_openbsd_mips64.s @@ -24,7 +24,7 @@ MOVV $0, R2 // crash on syscall failure MOVV R2, (R2) RET -// func exitThread(wait *uint32) +// func exitThread(wait *atomic.Uint32) TEXT runtime·exitThread(SB),NOSPLIT,$0 MOVV wait+0(FP), R4 // arg 1 - notdead MOVV $302, R2 // sys___threxit