runtime: run syscall/js finalizers on wasm without a manual GC (#5545)

* runtime: run syscall/js finalizers on wasm without a manual GC

* runtime: address review feedback on finalizer idle GC

* runtime: clear a finished task's args pointer so its arguments are collectable

* runtime: skip the finalizer scan with a per-block registration bit

* runtime: guard the finalizer registration bitmap with gcLock

* testdata: cover finalizer invariants on every scheduler

* main_test: limit the finalizer scheduler variants to linux and darwin

* testdata: wait for the finalizer queue to drain before asserting

* testdata: make the finalizer counters atomic and wait for a known drain count

* runtime: add finalizer bookkeeping asserts under runtime_asserts

* runtime: address finalizer GC review feedback

* testdata: strengthen blocked stack finalizer test

* runtime: fix finalizer cleanup edge cases

* runtime: decouple wasm export scheduling from finalizers

* runtime: avoid redundant wakeups for re-entrant wasm exports

* runtime: simplify finalizer comments

src/internal/task/task_asyncify.go

@@ -25,6 +25,10 @@ type state struct {
 	stackState
 
 	launched bool
+
+	// finishing marks a goroutine that paused after it completed.
+	// Resume uses this per task flag to clear the stack.
+	finishing bool
 }
 
 // stackState is the saved state of a stack while unwound.
@@ -42,6 +46,9 @@ type stackState struct {
 	// overwritten. It can be checked from time to time to see whether a stack
 	// overflow happened in the past.
 	canaryPtr *uintptr
+
+	// top marks the end of the stack buffer so it can be cleared after completion.
+	top unsafe.Pointer
 }
 
 // start creates and starts a new goroutine with the given function and arguments.
@@ -78,6 +85,22 @@ func (s *state) initialize(fn uintptr, args unsafe.Pointer, stackSize uintptr) {
 	// Calculate stack base addresses.
 	s.asyncifysp = unsafe.Add(stack, unsafe.Sizeof(uintptr(0)))
 	s.csp = unsafe.Add(stack, stackSize)
+	s.top = unsafe.Add(stack, stackSize)
+}
+
+//go:linkname memzero runtime.memzero
+func memzero(ptr unsafe.Pointer, size uintptr)
+
+// MarkFinishing marks the current goroutine for stack cleanup after it returns to the scheduler.
+func MarkFinishing() {
+	currentTask.state.finishing = true
+}
+
+// clearStack removes stale pointers from a finished asyncify stack.
+// The GC can then collect the stack and referenced objects.
+func (t *Task) clearStack() {
+	base := unsafe.Pointer(t.state.canaryPtr)
+	memzero(base, uintptr(t.state.top)-uintptr(base))
 }
 
 // currentTask is the current running task, or nil if currently in the scheduler.
@@ -123,6 +146,12 @@ func (t *Task) Resume() {
 	if uintptr(t.state.asyncifysp) > uintptr(t.state.csp) {
 		runtimeFatal("stack overflow")
 	}
+	if t.state.finishing {
+		// The task is complete. Clear stale stack pointers and release its argument bundle.
+		t.state.finishing = false
+		t.clearStack()
+		t.state.args = nil
+	}
 }
 
 //go:linkname saveStackPointer runtime.saveStackPointer

src/internal/task/task_finishing_tasks.go

@@ -0,0 +1,6 @@
+//go:build scheduler.tasks
+
+package task
+
+// MarkFinishing does nothing for scheduler.tasks because it does not use asyncify heap stacks.
+func MarkFinishing() {}

src/runtime/gc_finalizer.go

@@ -33,10 +33,15 @@ type finalizerEntry struct {
 	fn interface{}
 }
 
+// finalizerGCThreshold starts pressure GC when registrations indicate external memory pressure.
+// Larger tables use a proportional threshold. Zero disables this trigger.
+const finalizerGCThreshold = 32
+
 var (
 	finalizers        *finalizerEntry // registered finalizers; a GC root that keeps fn values alive
 	finalizerPending  *finalizerEntry // finalizers whose object died, waiting to run
 	numFinalizers     uintptr         // number of registered finalizers; fast-path gate for scanFinalizers
+	finalizersSinceGC uintptr         // tracks registration pressure for the scheduler trigger
 	finalizersQueued  bool            // set when scanFinalizers queued at least one finalizer to run
 	finalizerFutex    task.Futex      // wakes the finalizerRunner goroutine after a GC queues work
 	finalizerDraining bool            // guards against re-entrant inline draining (scheduler.none)
@@ -50,6 +55,72 @@ var (
 	finalizerRunnerStarted bool
 )
 
+const finalizerGCDivisor = 2
+
+// finalizerGCTrigger scales the threshold so scan work stays proportional to registrations.
+// It uses finalizerGCThreshold as the minimum.
+func finalizerGCTrigger() uintptr {
+	if finalizerGCThreshold == 0 {
+		return 0
+	}
+	if proportional := numFinalizers / finalizerGCDivisor; proportional > finalizerGCThreshold {
+		return proportional
+	}
+	return finalizerGCThreshold
+}
+
+// finalizerBits records finalizer registrations by heap block for fast lookup.
+// Hold gcLock for every access because heap growth can replace the slice.
+var finalizerBits []byte
+
+// finalizerBitsShortfall returns the required bitmap size or zero.
+// Call it with gcLock held and release the lock before allocation.
+func finalizerBitsShortfall() uintptr {
+	need := (uintptr(endBlock) + 7) / 8
+	if uintptr(len(finalizerBits)) >= need {
+		return 0
+	}
+	return need
+}
+
+// adoptFinalizerBits installs a wider bitmap while gcLock is held.
+// It accepts a stale size because the heap can grow during allocation.
+func adoptFinalizerBits(buf []byte) {
+	if len(buf) <= len(finalizerBits) {
+		return
+	}
+	copy(buf, finalizerBits)
+	finalizerBits = buf
+}
+
+func finalizerBitIndex(addr uintptr) uintptr { return uintptr(blockFromAddr(addr)) }
+
+func finalizerBitGet(addr uintptr) bool {
+	i := finalizerBitIndex(addr)
+	if i/8 >= uintptr(len(finalizerBits)) {
+		// Return true when the bitmap does not cover the address.
+		// A false result could register a second finalizer for the object.
+		return true
+	}
+	return finalizerBits[i/8]&(1<<(i%8)) != 0
+}
+
+func finalizerBitSet(addr uintptr) {
+	i := finalizerBitIndex(addr)
+	if i/8 >= uintptr(len(finalizerBits)) {
+		return
+	}
+	finalizerBits[i/8] |= 1 << (i % 8)
+}
+
+func finalizerBitClear(addr uintptr) {
+	i := finalizerBitIndex(addr)
+	if i/8 >= uintptr(len(finalizerBits)) {
+		return
+	}
+	finalizerBits[i/8] &^= 1 << (i % 8)
+}
+
 // The object address is stored bitwise-NOT so it never looks like a live heap
 // pointer to the conservative scanner. Otherwise the entry would pin every
 // finalizable object forever and the object could never be detected as dead.
@@ -58,6 +129,32 @@ var (
 func encodeFinalizerPtr(addr uintptr) uintptr { return ^addr }
 func decodeFinalizerPtr(enc uintptr) uintptr  { return ^enc }
 
+// finalizerRegistered checks the table when gcAsserts validates the bitmap.
+func finalizerRegistered(enc uintptr) bool {
+	for n := finalizers; n != nil; n = n.next {
+		if n.obj == enc {
+			return true
+		}
+	}
+	return false
+}
+
+// assertFinalizerTable checks that the table, count, and bitmap agree.
+// A missing bit can prevent removal or allow two finalizers for one object.
+func assertFinalizerTable() {
+	var count uintptr
+	for n := finalizers; n != nil; n = n.next {
+		count++
+		addr := decodeFinalizerPtr(n.obj)
+		if isOnHeap(addr) && !finalizerBitGet(addr) {
+			runtimeFatal("gc: registered finalizer without its bitmap bit")
+		}
+	}
+	if count != numFinalizers {
+		runtimeFatal("gc: numFinalizers does not match the finalizer table")
+	}
+}
+
 // registerFinalizer records fn as the finalizer for the object at addr. A nil fn
 // removes any registration for the object. Growing the table (allocating a node)
 // is the only allocation and it happens here, on the caller, never during GC.
@@ -67,13 +164,32 @@ func registerFinalizer(addr uintptr, fn interface{}) {
 	enc := encodeFinalizerPtr(addr)
 
 	if fn == nil {
-		// Clear: remove every registration for this object.
+		// Hold gcLock while checking the bit because another core can update the bitmap.
+		// A clear bit avoids a scan of the finalizer table.
 		gcLock.Lock()
+		tracked := isOnHeap(addr)
+		if tracked && !finalizerBitGet(addr) {
+			// Taking this shortcut on a stale bit would silently skip the
+			// removal, so check the answer against the table it stands in for.
+			if gcAsserts && finalizerRegistered(enc) {
+				runtimeFatal("gc: finalizer bit clear but the object is registered")
+			}
+			gcLock.Unlock()
+			return
+		}
+		if tracked {
+			finalizerBitClear(addr)
+		}
 		prev := &finalizers
 		for n := *prev; n != nil; n = *prev {
 			if n.obj == enc {
 				*prev = n.next
 				numFinalizers--
+				// Clearing offsets net registration pressure. Saturate at zero
+				// because the cleared entry may predate the last collection.
+				if finalizersSinceGC != 0 {
+					finalizersSinceGC--
+				}
 			} else {
 				prev = &n.next
 			}
@@ -82,11 +198,25 @@ func registerFinalizer(addr uintptr, fn interface{}) {
 		return
 	}
 
-	// Register or replace. The allocation happens before gcLock is taken,
-	// because alloc acquires gcLock itself.
+	// Allocate before taking gcLock because allocation also takes this lock.
+	// Release gcLock only when the bitmap must grow.
 	entry := &finalizerEntry{obj: enc, fn: fn}
 	gcLock.Lock()
-	for n := finalizers; n != nil; n = n.next {
+	if shortfall := finalizerBitsShortfall(); shortfall != 0 {
+		gcLock.Unlock()
+		wider := make([]byte, shortfall)
+		gcLock.Lock()
+		adoptFinalizerBits(wider)
+	}
+	tracked := isOnHeap(addr)
+	// Skipping the scan on a stale bit would add a second entry for an object
+	// that already has one, and its finalizer would then run twice.
+	if gcAsserts && tracked && !finalizerBitGet(addr) && finalizerRegistered(enc) {
+		runtimeFatal("gc: finalizer bit clear but the object is registered")
+	}
+	// Scan only if the bitmap can contain a registration for this object.
+	// Always scan addresses that the bitmap does not cover.
+	for n := finalizers; (!tracked || finalizerBitGet(addr)) && n != nil; n = n.next {
 		if n.obj == enc {
 			// Replace the finalizer for an already-registered object, so it
 			// still runs only once (Go SetFinalizer replace semantics).
@@ -105,7 +235,11 @@ func registerFinalizer(addr uintptr, fn interface{}) {
 	}
 	entry.next = finalizers
 	finalizers = entry
+	if tracked {
+		finalizerBitSet(addr)
+	}
 	numFinalizers++
+	finalizersSinceGC++
 	// A finalizer is registered, so make sure the runner exists. The flag is
 	// serialized by gcLock; the spawn itself allocates, so it must run after the
 	// lock is released.
@@ -121,6 +255,9 @@ func registerFinalizer(addr uintptr, fn interface{}) {
 // current GC cycle and queues their finalizers. It must be called under gcLock,
 // after marking is complete and before sweep frees anything.
 func scanFinalizers() {
+	// Reset pressure at the start of every collection, even if no finalizer runs.
+	finalizersSinceGC = 0
+
 	// Nothing registered and nothing waiting to run: fast path.
 	if numFinalizers == 0 && finalizerPending == nil {
 		return
@@ -146,6 +283,8 @@ func scanFinalizers() {
 		// and into the pending queue (alloc-free), so its finalizer runs once.
 		*prev = n.next
 		numFinalizers--
+		// Clear the bit so a later object at this address starts clean.
+		finalizerBitClear(addr)
 		n.next = finalizerPending
 		finalizerPending = n
 		finalizersQueued = true
@@ -155,18 +294,37 @@ func scanFinalizers() {
 	// found above and any queued by an earlier cycle that the runner has not
 	// drained yet. Otherwise the next GC would not mark them (their only
 	// reference is the encoded, scanner-invisible pending entry) and sweep would
-	// free them out from under a finalizer that hasn't run — a use-after-free.
+	// free them out from under a finalizer that hasn't run, a use-after-free.
 	// Walking the pending list is safe: scanFinalizers and dequeueFinalizer are
 	// both serialized under gcLock.
 	var resurrected bool
 	for n := finalizerPending; n != nil; n = n.next {
-		markRoot(0, decodeFinalizerPtr(n.obj))
+		addr := decodeFinalizerPtr(n.obj)
+		if gcAsserts && !isOnHeap(addr) {
+			runtimeFatal("gc: pending finalizer for an object off the heap")
+		}
+		markRoot(0, addr)
 		resurrected = true
 	}
 	if resurrected {
 		// Re-scan so objects reachable only from resurrected objects also
 		// survive this sweep.
 		finishMark()
+		if gcAsserts {
+			// Verify that every pending object survived resurrection.
+			// Otherwise callFinalizer can use memory that sweep freed.
+			for n := finalizerPending; n != nil; n = n.next {
+				// Inside the collection, so the resurrected object is expected
+				// to carry the mark state rather than plain head.
+				if blockFromAddr(decodeFinalizerPtr(n.obj)).findHead().state() != blockStateMark {
+					runtimeFatal("gc: pending finalizer object was not resurrected")
+				}
+			}
+		}
+	}
+
+	if gcAsserts {
+		assertFinalizerTable()
 	}
 }
 
@@ -219,12 +377,40 @@ func dequeueFinalizer() (*finalizerEntry, unsafe.Pointer) {
 	var objPtr unsafe.Pointer
 	if n != nil {
 		finalizerPending = n.next
-		objPtr = unsafe.Pointer(decodeFinalizerPtr(n.obj))
+		addr := decodeFinalizerPtr(n.obj)
+		if gcAsserts {
+			if !isOnHeap(addr) {
+				runtimeFatal("gc: dequeued finalizer for an object off the heap")
+			}
+			// A pending object must remain allocated until its finalizer runs.
+			// Check the block state because this runs after marks become heads.
+			if blockFromAddr(addr).state() == blockStateFree {
+				runtimeFatal("gc: dequeued finalizer for a freed object")
+			}
+		}
+		objPtr = unsafe.Pointer(addr)
 	}
 	gcLock.Unlock()
 	return n, objPtr
 }
 
+// finalizerPressureGC runs a GC when registrations indicate external memory pressure.
+// It wakes the finalizer runner when the GC queues work.
+func finalizerPressureGC() bool {
+	trigger := finalizerGCTrigger()
+	if trigger == 0 || finalizersSinceGC < trigger {
+		return false
+	}
+	gcLock.Lock()
+	runGC()
+	gcLock.Unlock()
+	if finalizersQueued {
+		finalizersQueued = false
+		wakeFinalizer()
+	}
+	return true
+}
+
 // wakeFinalizer is called after a GC (with gcLock already released) that queued
 // finalizers. On schedulers with goroutines it wakes the finalizerRunner; on
 // scheduler.none it drains inline.

src/runtime/gc_finalizer_sched.go

@@ -1,8 +1,10 @@
-//go:build (gc.conservative || gc.precise) && !scheduler.none
+//go:build (gc.conservative || gc.precise) && (scheduler.tasks || scheduler.asyncify)
 
 package runtime
 
-// The go statement lives in this scheduler-gated file, not inline in
-// registerFinalizer, so scheduler.none builds never reference internal/task.start
-// and the runner is DCE'd when SetFinalizer is unused.
-func spawnFinalizerRunner() { go finalizerRunner() }
+// Keep this setup in a file for these schedulers so unused finalizer code can be removed.
+// Cooperative schedulers also install the idle GC hook.
+func spawnFinalizerRunner() {
+	finalizerIdleGC = finalizerPressureGC
+	go finalizerRunner()
+}

src/runtime/gc_finalizer_sched_other.go

@@ -0,0 +1,7 @@
+//go:build (gc.conservative || gc.precise) && !scheduler.none && !scheduler.tasks && !scheduler.asyncify
+
+package runtime
+
+// spawnFinalizerRunner is the fallback for noncooperative schedulers.
+// These schedulers run finalizers but do not install the idle GC hook.
+func spawnFinalizerRunner() { go finalizerRunner() }

src/runtime/runtime_wasmentry.go

@@ -7,7 +7,6 @@ package runtime
 // compiler for //go:wasmexport support.
 
 import (
-	"internal/task"
 	"unsafe"
 )
 
@@ -85,19 +84,3 @@ func wasmExportRun(done *bool) {
 		runtimePanic("//go:wasmexport function did not finish")
 	}
 }
-
-// Called from the goroutine wrapper for the //go:wasmexport function. It just
-// signals to the runtime that the //go:wasmexport call has finished, and can
-// switch back to the wasmExportRun function.
-//
-// This function is not called when the scheduler is disabled.
-func wasmExportExit() {
-	// Signal to the scheduler that it should return, since this call to a
-	// //go:wasmexport function has exited.
-	schedulerExit = true
-
-	task.Pause()
-
-	// TODO: we could cache the allocated stack so we don't have to keep
-	// allocating a new stack on every //go:wasmexport call.
-}

src/runtime/scheduler_cooperative.go

@@ -39,8 +39,13 @@ var (
 	runqueue           task.Queue
 	sleepQueue         *task.Task
 	sleepQueueBaseTime timeUnit
+	deadlockedTasks    task.Queue
 )
 
+// finalizerIdleGC runs pressure GC at safe points and enables asyncify stack cleanup.
+// The first finalizer installs it so unused code can be removed.
+var finalizerIdleGC func() bool
+
 // deadlock is called when a goroutine cannot proceed any more, but is in theory
 // not exited (so deferred calls won't run). This can happen for example in code
 // like this, that blocks forever:
@@ -49,12 +54,41 @@ var (
 //
 //go:noinline
 func deadlock() {
-	// call yield without requesting a wakeup
+	// Keep permanently blocked tasks reachable so their suspended stacks remain
+	// GC roots, but never put them back on the runnable queue.
+	deadlockedTasks.Push(task.Current())
 	task.Pause()
 	runtimeFatal("unreachable")
 }
 
+// exitGoroutine ends an asyncify task that returned from its function.
+// Unlike deadlock, this task will not resume.
+func exitGoroutine() {
+	if finalizerIdleGC != nil {
+		task.MarkFinishing()
+	}
+	task.Pause()
+	runtimeFatal("unreachable")
+}
+
+// wasmExportExit stops the scheduler after a //go:wasmexport function returns.
+// It is not used when the scheduler is disabled.
+func wasmExportExit() {
+	schedulerExit = true
+	if finalizerIdleGC != nil {
+		task.MarkFinishing()
+	}
+
+	task.Pause()
+
+	// TODO: we could cache the allocated stack so we don't have to keep
+	// allocating a new stack on every //go:wasmexport call.
+}
+
 func goexit() {
+	if finalizerIdleGC != nil {
+		task.MarkFinishing()
+	}
 	task.Exit()
 }
 
@@ -183,6 +217,11 @@ func scheduler(returnAtDeadlock bool) {
 
 		t := runqueue.Pop()
 		if t == nil {
+			// Run the pressure GC only when the scheduler is idle at the top level.
+			// This batches completed work and avoids collections during allocation.
+			if task.Current() == nil && finalizerIdleGC != nil && finalizerIdleGC() {
+				continue
+			}
 			if sleepQueue == nil && timerQueue == nil {
 				if returnAtDeadlock {
 					return
@@ -236,6 +275,16 @@ func scheduler(returnAtDeadlock bool) {
 		// //go:wasmexport function returned.
 		if GOARCH == "wasm" && schedulerExit {
 			schedulerExit = false // reset the signal
+			if task.Current() == nil {
+				if finalizerIdleGC != nil {
+					finalizerIdleGC()
+				}
+				// Return from an export at the top level before unrelated goroutines run.
+				// A nested export returns to its active outer scheduler.
+				if asyncScheduler && (!runqueue.Empty() || sleepQueue != nil || timerQueue != nil) {
+					sleepTicks(0)
+				}
+			}
 			return
 		}
 	}