src/runtime/proc.c | 32 ++++++++++++++++++++++++++++++++ src/runtime/runtime.h | 2 ++ src/runtime/stack.c | 7 +------ diff --git a/src/runtime/proc.c b/src/runtime/proc.c index 91e3fe16d69ce1fb1b7c2efb7d8c70a4b20be8da..8462c4b1d61b7913415d3056c5be5c2740137e85 100644 --- a/src/runtime/proc.c +++ b/src/runtime/proc.c @@ -402,6 +402,7 @@ } static void badcasgstatus(void); static void helpcasgstatus(void); +static void badgstatusrunnable(void); // If asked to move to or from a Gscanstatus this will throw. Use the castogscanstatus // and casfromgscanstatus instead. @@ -423,6 +424,10 @@ // loop if gp->atomicstatus is in a scan state giving // GC time to finish and change the state to oldval. while(!runtime·cas(&gp->atomicstatus, oldval, newval)) { + if(oldval == Gwaiting && gp->atomicstatus == Grunnable) { + fn = badgstatusrunnable; + runtime·onM(&fn); + } // Help GC if needed. if(gp->preemptscan && !gp->gcworkdone && (oldval == Grunning || oldval == Gsyscall)) { gp->preemptscan = false; @@ -431,6 +436,33 @@ fn = helpcasgstatus; runtime·onM(&fn); } } +} + +static void +badgstatusrunnable(void) +{ + runtime·throw("casgstatus: waiting for Gwaiting but is Grunnable"); +} + +// casgstatus(gp, oldstatus, Gcopystack), assuming oldstatus is Gwaiting or Grunnable. +// Returns old status. Cannot call casgstatus directly, because we are racing with an +// async wakeup that might come in from netpoll. If we see Gwaiting from the readgstatus, +// it might have become Grunnable by the time we get to the cas. If we called casgstatus, +// it would loop waiting for the status to go back to Gwaiting, which it never will. +#pragma textflag NOSPLIT +uint32 +runtime·casgcopystack(G *gp) +{ + uint32 oldstatus; + + for(;;) { + oldstatus = runtime·readgstatus(gp) & ~Gscan; + if(oldstatus != Gwaiting && oldstatus != Grunnable) + runtime·throw("copystack: bad status, not Gwaiting or Grunnable"); + if(runtime·cas(&gp->atomicstatus, oldstatus, Gcopystack)) + break; + } + return oldstatus; } static void diff --git a/src/runtime/runtime.h b/src/runtime/runtime.h index 977c4547dfbacf55f2627968e9c4d3fd39c72016..177a1287eca0c41c9c1821d40980bcee1818d4e3 100644 --- a/src/runtime/runtime.h +++ b/src/runtime/runtime.h @@ -666,6 +666,8 @@ }; uint32 runtime·readgstatus(G*); void runtime·casgstatus(G*, uint32, uint32); +void runtime·casgstatus(G*, uint32, uint32); +uint32 runtime·casgcopystack(G*); void runtime·quiesce(G*); bool runtime·stopg(G*); void runtime·restartg(G*); diff --git a/src/runtime/stack.c b/src/runtime/stack.c index 072bc242bc2a5d3cad6bd004a2d2011b17fd5f92..cb9557243b5e921a6348134c9e547c72beaf29d9 100644 --- a/src/runtime/stack.c +++ b/src/runtime/stack.c @@ -637,12 +637,7 @@ *p++ = 0xfb; } runtime·memmove((byte*)new.hi - used, (byte*)old.hi - used, used); - oldstatus = runtime·readgstatus(gp); - oldstatus &= ~Gscan; - if(oldstatus == Gwaiting || oldstatus == Grunnable) - runtime·casgstatus(gp, oldstatus, Gcopystack); // oldstatus is Gwaiting or Grunnable - else - runtime·throw("copystack: bad status, not Gwaiting or Grunnable"); + oldstatus = runtime·casgcopystack(gp); // cas from Gwaiting or Grunnable to Gcopystack, return old status // Swap out old stack for new one gp->stack = new;