diff --git a/compileopts/finalizer_coverage_test.go b/compileopts/finalizer_coverage_test.go new file mode 100644 index 000000000..bf12e0c5e --- /dev/null +++ b/compileopts/finalizer_coverage_test.go @@ -0,0 +1,65 @@ +package compileopts + +import ( + "go/build/constraint" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestFinalizerRunnerSchedulerCoverage verifies that each scheduler selects one runner file. +// It uses validSchedulerOptions so new schedulers are included. +func TestFinalizerRunnerSchedulerCoverage(t *testing.T) { + files := []string{ + "gc_finalizer_sched.go", + "gc_finalizer_sched_none.go", + "gc_finalizer_sched_other.go", + } + exprs := make([]constraint.Expr, len(files)) + for i, name := range files { + exprs[i] = readBuildConstraint(t, filepath.Join("..", "src", "runtime", name)) + } + + for _, sched := range validSchedulerOptions { + // The finalizer table exists under block GCs. + // gc.conservative satisfies the GC condition in every constraint. + tags := map[string]bool{ + "gc.conservative": true, + "scheduler." + sched: true, + } + var matched []string + for i, expr := range exprs { + if expr.Eval(func(tag string) bool { return tags[tag] }) { + matched = append(matched, files[i]) + } + } + if len(matched) != 1 { + t.Errorf("scheduler.%s: spawnFinalizerRunner defined in %d files %v, want exactly 1", + sched, len(matched), matched) + } + } +} + +func readBuildConstraint(t *testing.T, path string) constraint.Expr { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if constraint.IsGoBuild(line) { + expr, err := constraint.Parse(line) + if err != nil { + t.Fatalf("%s: %v", path, err) + } + return expr + } + if line != "" && !strings.HasPrefix(line, "//") { + break // reached code before any //go:build line + } + } + t.Fatalf("%s: no //go:build line found", path) + return nil +} diff --git a/compiler/goroutine.go b/compiler/goroutine.go index 26d489a3a..8bc7da53c 100644 --- a/compiler/goroutine.go +++ b/compiler/goroutine.go @@ -309,10 +309,10 @@ func (c *compilerContext) createGoroutineStartWrapper(fnType llvm.Type, fn llvm. } defer b.Dispose() - var deadlock llvm.Value - var deadlockType llvm.Type + var exitGoroutine llvm.Value + var exitGoroutineType llvm.Type if c.Scheduler == "asyncify" { - deadlockType, deadlock = c.getFunction(c.program.ImportedPackage("runtime").Members["deadlock"].(*ssa.Function)) + exitGoroutineType, exitGoroutine = c.getFunction(c.program.ImportedPackage("runtime").Members["exitGoroutine"].(*ssa.Function)) } if !fn.IsAFunction().IsNil() { @@ -377,7 +377,7 @@ func (c *compilerContext) createGoroutineStartWrapper(fnType llvm.Type, fn llvm. b.CreateCall(fnType, fn, params, "") if c.Scheduler == "asyncify" { - b.CreateCall(deadlockType, deadlock, []llvm.Value{ + b.CreateCall(exitGoroutineType, exitGoroutine, []llvm.Value{ llvm.Undef(c.dataPtrType), }, "") } @@ -528,14 +528,13 @@ func (c *compilerContext) createGoroutineStartWrapper(fnType llvm.Type, fn llvm. b.CreateCall(fnType, fnPtr, params, "") if c.Scheduler == "asyncify" { - b.CreateCall(deadlockType, deadlock, []llvm.Value{ + b.CreateCall(exitGoroutineType, exitGoroutine, []llvm.Value{ llvm.Undef(c.dataPtrType), }, "") } } if c.Scheduler == "asyncify" { - // The goroutine was terminated via deadlock. b.CreateUnreachable() } else { // Finish the function. Every basic block must end in a terminator, and diff --git a/compiler/testdata/goroutine-wasm-asyncify.ll b/compiler/testdata/goroutine-wasm-asyncify.ll index 0c3f2f707..062ec1a42 100644 --- a/compiler/testdata/goroutine-wasm-asyncify.ll +++ b/compiler/testdata/goroutine-wasm-asyncify.ll @@ -22,14 +22,14 @@ entry: declare void @main.regularFunction(i32, ptr) #0 -declare void @runtime.deadlock(ptr) #0 +declare void @runtime.exitGoroutine(ptr) #0 ; Function Attrs: nounwind define linkonce_odr void @"main.regularFunction$gowrapper"(ptr %0) unnamed_addr #2 { entry: %unpack.int = ptrtoint ptr %0 to i32 call void @main.regularFunction(i32 %unpack.int, ptr undef) #11 - call void @runtime.deadlock(ptr undef) #11 + call void @runtime.exitGoroutine(ptr undef) #11 unreachable } @@ -53,7 +53,7 @@ define linkonce_odr void @"main.inlineFunctionGoroutine$1$gowrapper"(ptr %0) unn entry: %unpack.int = ptrtoint ptr %0 to i32 call void @"main.inlineFunctionGoroutine$1"(i32 %unpack.int, ptr undef) - call void @runtime.deadlock(ptr undef) #11 + call void @runtime.exitGoroutine(ptr undef) #11 unreachable } @@ -96,7 +96,7 @@ entry: %2 = getelementptr inbounds nuw i8, ptr %0, i32 4 %3 = load ptr, ptr %2, align 4 call void @"main.closureFunctionGoroutine$1"(i32 %1, ptr %3) - call void @runtime.deadlock(ptr undef) #11 + call void @runtime.exitGoroutine(ptr undef) #11 unreachable } @@ -130,7 +130,7 @@ entry: %4 = getelementptr inbounds nuw i8, ptr %0, i32 8 %5 = load ptr, ptr %4, align 4 call void %5(i32 %1, ptr %3) #11 - call void @runtime.deadlock(ptr undef) #11 + call void @runtime.exitGoroutine(ptr undef) #11 unreachable } @@ -193,7 +193,7 @@ entry: %6 = getelementptr inbounds nuw i8, ptr %0, i32 12 %7 = load ptr, ptr %6, align 4 call void @"interface:{Print:func:{basic:string}{}}.Print$invoke"(ptr %1, ptr %3, i32 %5, ptr %7, ptr undef) #11 - call void @runtime.deadlock(ptr undef) #11 + call void @runtime.exitGoroutine(ptr undef) #11 unreachable } diff --git a/compiler/testdata/large.ll b/compiler/testdata/large.ll index 4be9d9e4f..cc1f8556b 100644 --- a/compiler/testdata/large.ll +++ b/compiler/testdata/large.ll @@ -203,13 +203,13 @@ entry: ret void } -declare void @runtime.deadlock(ptr) #0 +declare void @runtime.exitGoroutine(ptr) #0 ; Function Attrs: nounwind define linkonce_odr void @"main.readLargeValue$gowrapper"(ptr %0) unnamed_addr #6 { entry: %1 = call i8 @main.readLargeValue(ptr %0, ptr undef) - call void @runtime.deadlock(ptr undef) #9 + call void @runtime.exitGoroutine(ptr undef) #9 unreachable } diff --git a/main_test.go b/main_test.go index 99a28ac63..17b2b5a57 100644 --- a/main_test.go +++ b/main_test.go @@ -60,6 +60,10 @@ func TestBuild(t *testing.T) { "channel.go", "embed/", "finalizer.go", + "finalizerbits.go", + "finalizeridle.go", + "finalizerinvariants.go", + "finalizerlarge.go", "float.go", "gc.go", "generics.go", @@ -116,7 +120,25 @@ func TestBuild(t *testing.T) { t.Run("Host", func(t *testing.T) { t.Parallel() - runPlatTests(optionsFromTarget("", sema), tests, t) + hostOptions := optionsFromTarget("", sema) + runPlatTests(hostOptions, tests, t) + + // scheduler.threads needs threadID, which exists only on Linux and Darwin. + // scheduler.none does not link on Windows. + switch runtime.GOOS { + case "darwin", "linux": + for _, scheduler := range []string{"threads", "none"} { + scheduler := scheduler + t.Run("finalizerinvariants.go-gc-conservative-scheduler-"+scheduler, func(t *testing.T) { + t.Parallel() + options := compileopts.Options(hostOptions) + options.GC = "conservative" + options.Scheduler = scheduler + options.Tags = append(append([]string(nil), hostOptions.Tags...), "runtime_asserts") + runTest("finalizerinvariants.go", options, t, nil, nil) + }) + } + } }) // Test a few build options. @@ -379,22 +401,38 @@ func runPlatTests(options compileopts.Options, tests []string, t *testing.T) { continue } } - if name == "finalizer.go" && options.Target != "wasm" { - // runtime.SetFinalizer is implemented for the block GC, but the - // test asserts deterministic collection of a dropped object, which - // only holds on the GOOS=js wasm target. The host default GC is - // boehm (SetFinalizer is a no-op there); conservative stack scanning - // on the emulated targets can pin the object; and the wasip2 - // component entry lays out the stack differently, so collection is - // not deterministic on those. The feature still works on all of - // them, it just can't be golden-tested for firing. - continue + if options.Target != "wasm" { + switch name { + case "finalizer.go", "finalizerbits.go", "finalizeridle.go", "finalizerlarge.go": + // These tests require deterministic finalization on target wasm. + // finalizerinvariants.go covers other block GC targets. + continue + } + } + if options.Target == "" && options.GC == "" { + switch name { + case "finalizerinvariants.go": + // Skip the default host GC because it does not implement finalizers. + // Explicit conservative GC variants cover this test. + continue + } + } + if options.Target == "simavr" { + switch name { + case "finalizerinvariants.go": + // Skip because runtime.GC does not return. See the gc.go exclusion above. + continue + } } name := name // redefine to avoid race condition t.Run(name, func(t *testing.T) { t.Parallel() - runTest(name, options, t, nil, nil) + testOptions := compileopts.Options(options) + if name == "finalizerinvariants.go" || name == "finalizerlarge.go" { + testOptions.Tags = append(append([]string(nil), options.Tags...), "runtime_asserts") + } + runTest(name, testOptions, t, nil, nil) }) } if !strings.HasPrefix(spec.Emulator, "simavr ") { @@ -950,6 +988,30 @@ func TestWasmExportJS(t *testing.T) { } } +func TestWasmExportFinalizersJS(t *testing.T) { + t.Parallel() + + tmpdir := t.TempDir() + options := optionsFromTarget("wasm", sema) + options.BuildMode = "c-shared" + buildConfig, err := builder.NewConfig(&options) + if err != nil { + t.Fatal(err) + } + result, err := builder.Build("testdata/wasmexport-finalizer.go", ".wasm", tmpdir, buildConfig) + if err != nil { + t.Fatal("failed to build binary:", err) + } + + output := &bytes.Buffer{} + cmd := exec.Command("node", "testdata/wasmexport-finalizer.js", result.Binary) + cmd.Stdout = output + cmd.Stderr = output + if err := cmd.Run(); err != nil { + t.Fatalf("failed to run node: %v\n%s", err, output) + } +} + // Test whether Go.run() (in wasm_exec.js) normally returns and returns the // right exit code. func TestWasmExit(t *testing.T) { diff --git a/src/internal/task/task_asyncify.go b/src/internal/task/task_asyncify.go index d7d9a6de4..4d78e1937 100644 --- a/src/internal/task/task_asyncify.go +++ b/src/internal/task/task_asyncify.go @@ -25,6 +25,10 @@ type state struct { stackState launched bool + + // finishing marks a goroutine that paused after it completed. + // Resume uses this per task flag to clear the stack. + finishing bool } // stackState is the saved state of a stack while unwound. @@ -42,6 +46,9 @@ type stackState struct { // overwritten. It can be checked from time to time to see whether a stack // overflow happened in the past. canaryPtr *uintptr + + // top marks the end of the stack buffer so it can be cleared after completion. + top unsafe.Pointer } // start creates and starts a new goroutine with the given function and arguments. @@ -78,6 +85,22 @@ func (s *state) initialize(fn uintptr, args unsafe.Pointer, stackSize uintptr) { // Calculate stack base addresses. s.asyncifysp = unsafe.Add(stack, unsafe.Sizeof(uintptr(0))) s.csp = unsafe.Add(stack, stackSize) + s.top = unsafe.Add(stack, stackSize) +} + +//go:linkname memzero runtime.memzero +func memzero(ptr unsafe.Pointer, size uintptr) + +// MarkFinishing marks the current goroutine for stack cleanup after it returns to the scheduler. +func MarkFinishing() { + currentTask.state.finishing = true +} + +// clearStack removes stale pointers from a finished asyncify stack. +// The GC can then collect the stack and referenced objects. +func (t *Task) clearStack() { + base := unsafe.Pointer(t.state.canaryPtr) + memzero(base, uintptr(t.state.top)-uintptr(base)) } // currentTask is the current running task, or nil if currently in the scheduler. @@ -123,6 +146,12 @@ func (t *Task) Resume() { if uintptr(t.state.asyncifysp) > uintptr(t.state.csp) { runtimeFatal("stack overflow") } + if t.state.finishing { + // The task is complete. Clear stale stack pointers and release its argument bundle. + t.state.finishing = false + t.clearStack() + t.state.args = nil + } } //go:linkname saveStackPointer runtime.saveStackPointer diff --git a/src/internal/task/task_finishing_tasks.go b/src/internal/task/task_finishing_tasks.go new file mode 100644 index 000000000..0c6f03261 --- /dev/null +++ b/src/internal/task/task_finishing_tasks.go @@ -0,0 +1,6 @@ +//go:build scheduler.tasks + +package task + +// MarkFinishing does nothing for scheduler.tasks because it does not use asyncify heap stacks. +func MarkFinishing() {} diff --git a/src/runtime/gc_finalizer.go b/src/runtime/gc_finalizer.go index 13dd6f2c7..df3a4c6c6 100644 --- a/src/runtime/gc_finalizer.go +++ b/src/runtime/gc_finalizer.go @@ -33,10 +33,15 @@ type finalizerEntry struct { fn interface{} } +// finalizerGCThreshold starts pressure GC when registrations indicate external memory pressure. +// Larger tables use a proportional threshold. Zero disables this trigger. +const finalizerGCThreshold = 32 + var ( finalizers *finalizerEntry // registered finalizers; a GC root that keeps fn values alive finalizerPending *finalizerEntry // finalizers whose object died, waiting to run numFinalizers uintptr // number of registered finalizers; fast-path gate for scanFinalizers + finalizersSinceGC uintptr // tracks registration pressure for the scheduler trigger finalizersQueued bool // set when scanFinalizers queued at least one finalizer to run finalizerFutex task.Futex // wakes the finalizerRunner goroutine after a GC queues work finalizerDraining bool // guards against re-entrant inline draining (scheduler.none) @@ -50,6 +55,72 @@ var ( finalizerRunnerStarted bool ) +const finalizerGCDivisor = 2 + +// finalizerGCTrigger scales the threshold so scan work stays proportional to registrations. +// It uses finalizerGCThreshold as the minimum. +func finalizerGCTrigger() uintptr { + if finalizerGCThreshold == 0 { + return 0 + } + if proportional := numFinalizers / finalizerGCDivisor; proportional > finalizerGCThreshold { + return proportional + } + return finalizerGCThreshold +} + +// finalizerBits records finalizer registrations by heap block for fast lookup. +// Hold gcLock for every access because heap growth can replace the slice. +var finalizerBits []byte + +// finalizerBitsShortfall returns the required bitmap size or zero. +// Call it with gcLock held and release the lock before allocation. +func finalizerBitsShortfall() uintptr { + need := (uintptr(endBlock) + 7) / 8 + if uintptr(len(finalizerBits)) >= need { + return 0 + } + return need +} + +// adoptFinalizerBits installs a wider bitmap while gcLock is held. +// It accepts a stale size because the heap can grow during allocation. +func adoptFinalizerBits(buf []byte) { + if len(buf) <= len(finalizerBits) { + return + } + copy(buf, finalizerBits) + finalizerBits = buf +} + +func finalizerBitIndex(addr uintptr) uintptr { return uintptr(blockFromAddr(addr)) } + +func finalizerBitGet(addr uintptr) bool { + i := finalizerBitIndex(addr) + if i/8 >= uintptr(len(finalizerBits)) { + // Return true when the bitmap does not cover the address. + // A false result could register a second finalizer for the object. + return true + } + return finalizerBits[i/8]&(1<<(i%8)) != 0 +} + +func finalizerBitSet(addr uintptr) { + i := finalizerBitIndex(addr) + if i/8 >= uintptr(len(finalizerBits)) { + return + } + finalizerBits[i/8] |= 1 << (i % 8) +} + +func finalizerBitClear(addr uintptr) { + i := finalizerBitIndex(addr) + if i/8 >= uintptr(len(finalizerBits)) { + return + } + finalizerBits[i/8] &^= 1 << (i % 8) +} + // The object address is stored bitwise-NOT so it never looks like a live heap // pointer to the conservative scanner. Otherwise the entry would pin every // finalizable object forever and the object could never be detected as dead. @@ -58,6 +129,32 @@ var ( func encodeFinalizerPtr(addr uintptr) uintptr { return ^addr } func decodeFinalizerPtr(enc uintptr) uintptr { return ^enc } +// finalizerRegistered checks the table when gcAsserts validates the bitmap. +func finalizerRegistered(enc uintptr) bool { + for n := finalizers; n != nil; n = n.next { + if n.obj == enc { + return true + } + } + return false +} + +// assertFinalizerTable checks that the table, count, and bitmap agree. +// A missing bit can prevent removal or allow two finalizers for one object. +func assertFinalizerTable() { + var count uintptr + for n := finalizers; n != nil; n = n.next { + count++ + addr := decodeFinalizerPtr(n.obj) + if isOnHeap(addr) && !finalizerBitGet(addr) { + runtimeFatal("gc: registered finalizer without its bitmap bit") + } + } + if count != numFinalizers { + runtimeFatal("gc: numFinalizers does not match the finalizer table") + } +} + // registerFinalizer records fn as the finalizer for the object at addr. A nil fn // removes any registration for the object. Growing the table (allocating a node) // is the only allocation and it happens here, on the caller, never during GC. @@ -67,13 +164,32 @@ func registerFinalizer(addr uintptr, fn interface{}) { enc := encodeFinalizerPtr(addr) if fn == nil { - // Clear: remove every registration for this object. + // Hold gcLock while checking the bit because another core can update the bitmap. + // A clear bit avoids a scan of the finalizer table. gcLock.Lock() + tracked := isOnHeap(addr) + if tracked && !finalizerBitGet(addr) { + // Taking this shortcut on a stale bit would silently skip the + // removal, so check the answer against the table it stands in for. + if gcAsserts && finalizerRegistered(enc) { + runtimeFatal("gc: finalizer bit clear but the object is registered") + } + gcLock.Unlock() + return + } + if tracked { + finalizerBitClear(addr) + } prev := &finalizers for n := *prev; n != nil; n = *prev { if n.obj == enc { *prev = n.next numFinalizers-- + // Clearing offsets net registration pressure. Saturate at zero + // because the cleared entry may predate the last collection. + if finalizersSinceGC != 0 { + finalizersSinceGC-- + } } else { prev = &n.next } @@ -82,11 +198,25 @@ func registerFinalizer(addr uintptr, fn interface{}) { return } - // Register or replace. The allocation happens before gcLock is taken, - // because alloc acquires gcLock itself. + // Allocate before taking gcLock because allocation also takes this lock. + // Release gcLock only when the bitmap must grow. entry := &finalizerEntry{obj: enc, fn: fn} gcLock.Lock() - for n := finalizers; n != nil; n = n.next { + if shortfall := finalizerBitsShortfall(); shortfall != 0 { + gcLock.Unlock() + wider := make([]byte, shortfall) + gcLock.Lock() + adoptFinalizerBits(wider) + } + tracked := isOnHeap(addr) + // Skipping the scan on a stale bit would add a second entry for an object + // that already has one, and its finalizer would then run twice. + if gcAsserts && tracked && !finalizerBitGet(addr) && finalizerRegistered(enc) { + runtimeFatal("gc: finalizer bit clear but the object is registered") + } + // Scan only if the bitmap can contain a registration for this object. + // Always scan addresses that the bitmap does not cover. + for n := finalizers; (!tracked || finalizerBitGet(addr)) && n != nil; n = n.next { if n.obj == enc { // Replace the finalizer for an already-registered object, so it // still runs only once (Go SetFinalizer replace semantics). @@ -105,7 +235,11 @@ func registerFinalizer(addr uintptr, fn interface{}) { } entry.next = finalizers finalizers = entry + if tracked { + finalizerBitSet(addr) + } numFinalizers++ + finalizersSinceGC++ // A finalizer is registered, so make sure the runner exists. The flag is // serialized by gcLock; the spawn itself allocates, so it must run after the // lock is released. @@ -121,6 +255,9 @@ func registerFinalizer(addr uintptr, fn interface{}) { // current GC cycle and queues their finalizers. It must be called under gcLock, // after marking is complete and before sweep frees anything. func scanFinalizers() { + // Reset pressure at the start of every collection, even if no finalizer runs. + finalizersSinceGC = 0 + // Nothing registered and nothing waiting to run: fast path. if numFinalizers == 0 && finalizerPending == nil { return @@ -146,6 +283,8 @@ func scanFinalizers() { // and into the pending queue (alloc-free), so its finalizer runs once. *prev = n.next numFinalizers-- + // Clear the bit so a later object at this address starts clean. + finalizerBitClear(addr) n.next = finalizerPending finalizerPending = n finalizersQueued = true @@ -155,18 +294,37 @@ func scanFinalizers() { // found above and any queued by an earlier cycle that the runner has not // drained yet. Otherwise the next GC would not mark them (their only // reference is the encoded, scanner-invisible pending entry) and sweep would - // free them out from under a finalizer that hasn't run — a use-after-free. + // free them out from under a finalizer that hasn't run, a use-after-free. // Walking the pending list is safe: scanFinalizers and dequeueFinalizer are // both serialized under gcLock. var resurrected bool for n := finalizerPending; n != nil; n = n.next { - markRoot(0, decodeFinalizerPtr(n.obj)) + addr := decodeFinalizerPtr(n.obj) + if gcAsserts && !isOnHeap(addr) { + runtimeFatal("gc: pending finalizer for an object off the heap") + } + markRoot(0, addr) resurrected = true } if resurrected { // Re-scan so objects reachable only from resurrected objects also // survive this sweep. finishMark() + if gcAsserts { + // Verify that every pending object survived resurrection. + // Otherwise callFinalizer can use memory that sweep freed. + for n := finalizerPending; n != nil; n = n.next { + // Inside the collection, so the resurrected object is expected + // to carry the mark state rather than plain head. + if blockFromAddr(decodeFinalizerPtr(n.obj)).findHead().state() != blockStateMark { + runtimeFatal("gc: pending finalizer object was not resurrected") + } + } + } + } + + if gcAsserts { + assertFinalizerTable() } } @@ -219,12 +377,40 @@ func dequeueFinalizer() (*finalizerEntry, unsafe.Pointer) { var objPtr unsafe.Pointer if n != nil { finalizerPending = n.next - objPtr = unsafe.Pointer(decodeFinalizerPtr(n.obj)) + addr := decodeFinalizerPtr(n.obj) + if gcAsserts { + if !isOnHeap(addr) { + runtimeFatal("gc: dequeued finalizer for an object off the heap") + } + // A pending object must remain allocated until its finalizer runs. + // Check the block state because this runs after marks become heads. + if blockFromAddr(addr).state() == blockStateFree { + runtimeFatal("gc: dequeued finalizer for a freed object") + } + } + objPtr = unsafe.Pointer(addr) } gcLock.Unlock() return n, objPtr } +// finalizerPressureGC runs a GC when registrations indicate external memory pressure. +// It wakes the finalizer runner when the GC queues work. +func finalizerPressureGC() bool { + trigger := finalizerGCTrigger() + if trigger == 0 || finalizersSinceGC < trigger { + return false + } + gcLock.Lock() + runGC() + gcLock.Unlock() + if finalizersQueued { + finalizersQueued = false + wakeFinalizer() + } + return true +} + // wakeFinalizer is called after a GC (with gcLock already released) that queued // finalizers. On schedulers with goroutines it wakes the finalizerRunner; on // scheduler.none it drains inline. diff --git a/src/runtime/gc_finalizer_sched.go b/src/runtime/gc_finalizer_sched.go index d983decc7..0a4783eb8 100644 --- a/src/runtime/gc_finalizer_sched.go +++ b/src/runtime/gc_finalizer_sched.go @@ -1,8 +1,10 @@ -//go:build (gc.conservative || gc.precise) && !scheduler.none +//go:build (gc.conservative || gc.precise) && (scheduler.tasks || scheduler.asyncify) package runtime -// The go statement lives in this scheduler-gated file, not inline in -// registerFinalizer, so scheduler.none builds never reference internal/task.start -// and the runner is DCE'd when SetFinalizer is unused. -func spawnFinalizerRunner() { go finalizerRunner() } +// Keep this setup in a file for these schedulers so unused finalizer code can be removed. +// Cooperative schedulers also install the idle GC hook. +func spawnFinalizerRunner() { + finalizerIdleGC = finalizerPressureGC + go finalizerRunner() +} diff --git a/src/runtime/gc_finalizer_sched_other.go b/src/runtime/gc_finalizer_sched_other.go new file mode 100644 index 000000000..f874ec46b --- /dev/null +++ b/src/runtime/gc_finalizer_sched_other.go @@ -0,0 +1,7 @@ +//go:build (gc.conservative || gc.precise) && !scheduler.none && !scheduler.tasks && !scheduler.asyncify + +package runtime + +// spawnFinalizerRunner is the fallback for noncooperative schedulers. +// These schedulers run finalizers but do not install the idle GC hook. +func spawnFinalizerRunner() { go finalizerRunner() } diff --git a/src/runtime/runtime_wasmentry.go b/src/runtime/runtime_wasmentry.go index 59cacb3b0..b621f3883 100644 --- a/src/runtime/runtime_wasmentry.go +++ b/src/runtime/runtime_wasmentry.go @@ -7,7 +7,6 @@ package runtime // compiler for //go:wasmexport support. import ( - "internal/task" "unsafe" ) @@ -85,19 +84,3 @@ func wasmExportRun(done *bool) { runtimePanic("//go:wasmexport function did not finish") } } - -// Called from the goroutine wrapper for the //go:wasmexport function. It just -// signals to the runtime that the //go:wasmexport call has finished, and can -// switch back to the wasmExportRun function. -// -// This function is not called when the scheduler is disabled. -func wasmExportExit() { - // Signal to the scheduler that it should return, since this call to a - // //go:wasmexport function has exited. - schedulerExit = true - - task.Pause() - - // TODO: we could cache the allocated stack so we don't have to keep - // allocating a new stack on every //go:wasmexport call. -} diff --git a/src/runtime/scheduler_cooperative.go b/src/runtime/scheduler_cooperative.go index 6f8d6b0da..72d9e175c 100644 --- a/src/runtime/scheduler_cooperative.go +++ b/src/runtime/scheduler_cooperative.go @@ -39,8 +39,13 @@ var ( runqueue task.Queue sleepQueue *task.Task sleepQueueBaseTime timeUnit + deadlockedTasks task.Queue ) +// finalizerIdleGC runs pressure GC at safe points and enables asyncify stack cleanup. +// The first finalizer installs it so unused code can be removed. +var finalizerIdleGC func() bool + // deadlock is called when a goroutine cannot proceed any more, but is in theory // not exited (so deferred calls won't run). This can happen for example in code // like this, that blocks forever: @@ -49,12 +54,41 @@ var ( // //go:noinline func deadlock() { - // call yield without requesting a wakeup + // Keep permanently blocked tasks reachable so their suspended stacks remain + // GC roots, but never put them back on the runnable queue. + deadlockedTasks.Push(task.Current()) task.Pause() runtimeFatal("unreachable") } +// exitGoroutine ends an asyncify task that returned from its function. +// Unlike deadlock, this task will not resume. +func exitGoroutine() { + if finalizerIdleGC != nil { + task.MarkFinishing() + } + task.Pause() + runtimeFatal("unreachable") +} + +// wasmExportExit stops the scheduler after a //go:wasmexport function returns. +// It is not used when the scheduler is disabled. +func wasmExportExit() { + schedulerExit = true + if finalizerIdleGC != nil { + task.MarkFinishing() + } + + task.Pause() + + // TODO: we could cache the allocated stack so we don't have to keep + // allocating a new stack on every //go:wasmexport call. +} + func goexit() { + if finalizerIdleGC != nil { + task.MarkFinishing() + } task.Exit() } @@ -183,6 +217,11 @@ func scheduler(returnAtDeadlock bool) { t := runqueue.Pop() if t == nil { + // Run the pressure GC only when the scheduler is idle at the top level. + // This batches completed work and avoids collections during allocation. + if task.Current() == nil && finalizerIdleGC != nil && finalizerIdleGC() { + continue + } if sleepQueue == nil && timerQueue == nil { if returnAtDeadlock { return @@ -236,6 +275,16 @@ func scheduler(returnAtDeadlock bool) { // //go:wasmexport function returned. if GOARCH == "wasm" && schedulerExit { schedulerExit = false // reset the signal + if task.Current() == nil { + if finalizerIdleGC != nil { + finalizerIdleGC() + } + // Return from an export at the top level before unrelated goroutines run. + // A nested export returns to its active outer scheduler. + if asyncScheduler && (!runqueue.Empty() || sleepQueue != nil || timerQueue != nil) { + sleepTicks(0) + } + } return } } diff --git a/testdata/finalizerbits.go b/testdata/finalizerbits.go new file mode 100644 index 000000000..60ffadc86 --- /dev/null +++ b/testdata/finalizerbits.go @@ -0,0 +1,177 @@ +package main + +// Test finalizer registration, replacement, removal, and reused heap addresses. +// The wasm target provides deterministic finalization for these tests. + +import "runtime" + +type box struct{ x int } + +const batch = 32 + +var ( + reregisteredRan int + replacedOldRan int + replacedNewRan int + churnRan int + reuseFirstRan int + reuseSecondRan int + keptRan int + droppedRan int + sink int +) + +// scrubStack removes stale pointers from the helper frame so collection is deterministic. +// Call it at the same call depth as the allocation helper. +// +//go:noinline +func scrubStack(depth int) int { + if depth <= 0 { + return sink + } + var buf [64]int + for i := range buf { + buf[i] = depth + i + } + sink += buf[depth&63] + return scrubStack(depth-1) + buf[0] +} + +//go:noinline +func allocClearThenRegister() { + p := &box{x: 1} + runtime.SetFinalizer(p, func(*box) { panic("cleared finalizer ran") }) + runtime.SetFinalizer(p, nil) + runtime.SetFinalizer(p, func(*box) { reregisteredRan++ }) +} + +func testClearThenRegister() { + allocClearThenRegister() + for i := 0; i < 200 && reregisteredRan == 0; i++ { + sink += scrubStack(40) + runtime.GC() + runtime.Gosched() + } + if reregisteredRan != 1 { + panic("finalizerbits: re-registered finalizer did not run exactly once") + } +} + +//go:noinline +func allocRegisterTwice() { + for i := 0; i < batch; i++ { + p := &box{x: i} + runtime.SetFinalizer(p, func(*box) { replacedOldRan++ }) + runtime.SetFinalizer(p, func(*box) { replacedNewRan++ }) + } +} + +func testRegisterTwiceLeavesOne() { + allocRegisterTwice() + for i := 0; i < 200 && replacedNewRan < batch; i++ { + sink += scrubStack(40) + runtime.GC() + runtime.Gosched() + } + if replacedOldRan != 0 { + panic("finalizerbits: replaced finalizer still ran") + } + if replacedNewRan != batch { + panic("finalizerbits: replacement did not run exactly once per object") + } +} + +//go:noinline +func allocChurn() { + p := &box{x: 3} + for i := 0; i < 64; i++ { + runtime.SetFinalizer(p, func(*box) { churnRan++ }) + runtime.SetFinalizer(p, nil) + } +} + +func testChurnLeavesNothing() { + allocChurn() + for i := 0; i < 200; i++ { + sink += scrubStack(40) + runtime.GC() + runtime.Gosched() + } + if churnRan != 0 { + panic("finalizerbits: churned register/clear left a live registration") + } +} + +//go:noinline +func allocFirstRound() { + for i := 0; i < batch; i++ { + p := &box{x: i} + runtime.SetFinalizer(p, func(*box) { reuseFirstRan++ }) + } +} + +//go:noinline +func allocSecondRound() { + for i := 0; i < batch; i++ { + p := &box{x: i} + runtime.SetFinalizer(p, func(*box) { reuseSecondRan++ }) + } +} + +func testAddressReuse() { + allocFirstRound() + for i := 0; i < 200 && reuseFirstRan < batch; i++ { + sink += scrubStack(40) + runtime.GC() + runtime.Gosched() + } + if reuseFirstRan != batch { + panic("finalizerbits: first round did not run every finalizer") + } + allocSecondRound() + for i := 0; i < 200 && reuseSecondRan < batch; i++ { + sink += scrubStack(40) + runtime.GC() + runtime.Gosched() + } + if reuseSecondRan != batch { + panic("finalizerbits: second round into reused memory lost finalizers") + } +} + +//go:noinline +func allocMixedBatch() { + for i := 0; i < batch; i++ { + p := &box{x: i} + if i%2 == 0 { + runtime.SetFinalizer(p, func(*box) { droppedRan++ }) + runtime.SetFinalizer(p, nil) + } else { + runtime.SetFinalizer(p, func(*box) { keptRan++ }) + } + } +} + +func testMixedBatch() { + allocMixedBatch() + for i := 0; i < 200 && keptRan < batch/2; i++ { + sink += scrubStack(40) + runtime.GC() + runtime.Gosched() + } + if droppedRan != 0 { + panic("finalizerbits: a cleared finalizer inside the batch ran") + } + if keptRan != batch/2 { + panic("finalizerbits: kept finalizers did not all run exactly once") + } +} + +func main() { + testClearThenRegister() + testRegisterTwiceLeavesOne() + testChurnLeavesNothing() + testAddressReuse() + testMixedBatch() + println("ok") +} diff --git a/testdata/finalizerbits.txt b/testdata/finalizerbits.txt new file mode 100644 index 000000000..9766475a4 --- /dev/null +++ b/testdata/finalizerbits.txt @@ -0,0 +1 @@ +ok diff --git a/testdata/finalizeridle.go b/testdata/finalizeridle.go new file mode 100644 index 000000000..f533ec77b --- /dev/null +++ b/testdata/finalizeridle.go @@ -0,0 +1,182 @@ +package main + +// Test idle finalizer collection and the lifetime of blocked and completed asyncify stacks. +// The wasm target provides deterministic finalization for these tests. + +import ( + "runtime" + "sync/atomic" + "time" +) + +// batch must exceed the finalizer registration threshold to trigger idle collection. +const batch = 64 + +var ( + ranDropped int + ranOnStack int + ranInArgs int + sink int + + blockedRan [3]atomic.Int32 + controlRan atomic.Int32 +) + +type blockedObject struct{ x int } + +//go:noinline +func blockOperation(kind int, ready chan<- struct{}, ch chan struct{}) { + ready <- struct{}{} + switch kind { + case 0: + select {} + case 1: + ch <- struct{}{} + case 2: + <-ch + } +} + +//go:noinline +func holdWhileBlocked(kind int, ready chan<- struct{}, ch chan struct{}) { + p := &blockedObject{x: kind} + runtime.SetFinalizer(p, func(*blockedObject) { blockedRan[kind].Add(1) }) + blockOperation(kind, ready, ch) + // blockOperation can return, so p remains live on this suspended stack. + runtime.KeepAlive(p) +} + +//go:noinline +func dropProgressControl() { + p := &blockedObject{x: 8} + runtime.SetFinalizer(p, func(*blockedObject) { controlRan.Add(1) }) +} + +// testPermanentlyBlockedStacks checks that blocked task stacks remain GC roots. +// A control finalizer confirms that GC and finalizer processing made progress. +func testPermanentlyBlockedStacks() { + ready := make(chan struct{}, 3) + go holdWhileBlocked(0, ready, nil) // select{} + go holdWhileBlocked(1, ready, nil) // nil channel send + go holdWhileBlocked(2, ready, nil) // nil channel receive + <-ready + <-ready + <-ready + + dropProgressControl() + for i := 0; i < 100 && controlRan.Load() == 0; i++ { + sink += scrubStack(40) + runtime.GC() + runtime.Gosched() + } + if controlRan.Load() != 1 { + panic("control finalizer did not prove GC progress") + } + // Collect once more so temporary scheduler roots cannot hide an unrooted + // blocked task during the control collection. + runtime.GC() + runtime.Gosched() + for i, name := range [...]string{"select{}", "nil-channel send", "nil-channel receive"} { + if blockedRan[i].Load() != 0 { + panic(name + " stack-held object was finalized") + } + } +} + +// scrubStack removes stale pointers from the helper frame so collection is deterministic. +// Call it at the same call depth as the allocation helper. +// +//go:noinline +func scrubStack(depth int) int { + if depth <= 0 { + return sink + } + var buf [64]int + for i := range buf { + buf[i] = depth + i + } + sink += buf[depth&63] + return scrubStack(depth-1) + buf[0] +} + +// registerAndDrop creates unreachable objects with finalizers that do not capture them. +// This allows the idle GC to collect the objects. +// +//go:noinline +func registerAndDrop() { + for i := 0; i < batch; i++ { + p := new([2]int) + runtime.SetFinalizer(p, func(*[2]int) { ranDropped++ }) + } +} + +func testIdleCollect() { + registerAndDrop() + for i := 0; i < 500 && ranDropped < batch; i++ { + sink += scrubStack(40) + time.Sleep(time.Millisecond) + } + if ranDropped != batch { + panic("idle collection did not run every finalizer") + } +} + +func testFinishedGoroutineStacks() { + done := make(chan struct{}) + for i := 0; i < batch; i++ { + go func() { + p := new([2]int) + runtime.SetFinalizer(p, func(*[2]int) { ranOnStack++ }) + // p stays on this goroutine's stack until it returns just below. + done <- struct{}{} + }() + } + for i := 0; i < batch; i++ { + <-done + } + for i := 0; i < 500 && ranOnStack < batch; i++ { + sink += scrubStack(40) + time.Sleep(time.Millisecond) + } + if ranOnStack != batch { + panic("finished goroutine stack still pinned finalized objects") + } +} + +// launchArgGoroutine passes an object through the task argument bundle. +// The caller returns so scrubStack can remove its transient pointer. +// +//go:noinline +func launchArgGoroutine(done chan struct{}) { + p := new([2]int) + runtime.SetFinalizer(p, func(*[2]int) { ranInArgs++ }) + go func(q *[2]int) { + sink += q[0] + done <- struct{}{} + }(p) +} + +func testFinishedGoroutineArgs() { + done := make(chan struct{}) + for i := 0; i < batch; i++ { + launchArgGoroutine(done) + } + for i := 0; i < batch; i++ { + <-done + } + for i := 0; i < 500 && ranInArgs < batch; i++ { + sink += scrubStack(40) + time.Sleep(time.Millisecond) + } + if ranInArgs != batch { + panic("finished goroutine args still pinned finalized objects") + } +} + +func main() { + testPermanentlyBlockedStacks() + testIdleCollect() + testFinishedGoroutineStacks() + testFinishedGoroutineArgs() + println("ok") +} diff --git a/testdata/finalizeridle.txt b/testdata/finalizeridle.txt new file mode 100644 index 000000000..9766475a4 --- /dev/null +++ b/testdata/finalizeridle.txt @@ -0,0 +1 @@ +ok diff --git a/testdata/finalizerinvariants.go b/testdata/finalizerinvariants.go new file mode 100644 index 000000000..9939f27ea --- /dev/null +++ b/testdata/finalizerinvariants.go @@ -0,0 +1,87 @@ +package main + +// Test finalizer invariants that do not require unreachable objects to be collected. +// runtime_asserts checks the finalizer table, count, and bitmap. + +import ( + "runtime" + "sync/atomic" +) + +type obj struct{ x int } + +const batch = 8 + +// Finalizers may run concurrently with main under the threads and cores +// schedulers, so all observations shared with a finalizer are atomic. +var ( + clearedRan atomic.Int32 + replacedRan atomic.Int32 + reachedRan atomic.Int32 + ranTwice atomic.Int32 + seen [batch]atomic.Int32 + reachable []*obj +) + +//go:noinline +func dropCleared() { + p := &obj{} + runtime.SetFinalizer(p, func(*obj) { clearedRan.Add(1) }) + runtime.SetFinalizer(p, nil) +} + +//go:noinline +func dropReplaced(id int) { + p := &obj{} + runtime.SetFinalizer(p, func(*obj) { replacedRan.Add(1) }) + runtime.SetFinalizer(p, func(*obj) { + if seen[id].Add(1) > 1 { + ranTwice.Add(1) + } + }) +} + +//go:noinline +func keepReachable(id int) { + p := &obj{x: id} + runtime.SetFinalizer(p, func(*obj) { reachedRan.Add(1) }) + reachable = append(reachable, p) +} + +func main() { + for i := 0; i < batch; i++ { + dropCleared() + dropReplaced(i) + keepReachable(i) + } + + // Run GC to check bookkeeping and give finalizers bounded opportunities to run. + // The checks do not require finalization of an unreachable object. + for i := 0; i < 8; i++ { + runtime.GC() + runtime.Gosched() + } + + // Keep the reachable objects live across every collection above. + total := 0 + for _, p := range reachable { + total += p.x + } + if total != batch*(batch-1)/2 { + println("FAIL: reachable set corrupted:", total) + return + } + + switch { + case clearedRan.Load() != 0: + println("FAIL: cleared finalizer ran:", clearedRan.Load()) + case replacedRan.Load() != 0: + println("FAIL: replaced finalizer ran:", replacedRan.Load()) + case reachedRan.Load() != 0: + println("FAIL: reachable object was finalized:", reachedRan.Load()) + case ranTwice.Load() != 0: + println("FAIL: finalizer ran more than once:", ranTwice.Load()) + default: + println("ok") + } +} diff --git a/testdata/finalizerinvariants.txt b/testdata/finalizerinvariants.txt new file mode 100644 index 000000000..9766475a4 --- /dev/null +++ b/testdata/finalizerinvariants.txt @@ -0,0 +1 @@ +ok diff --git a/testdata/finalizerlarge.go b/testdata/finalizerlarge.go new file mode 100644 index 000000000..4ba8ddc04 --- /dev/null +++ b/testdata/finalizerlarge.go @@ -0,0 +1,29 @@ +package main + +import "runtime" + +type largeFinalizerObject struct { + data [128]byte +} + +var largeFinalizerRan bool + +//go:noinline +func registerLargeFinalizer() { + p := new(largeFinalizerObject) + runtime.SetFinalizer(p, func(*largeFinalizerObject) { + largeFinalizerRan = true + }) +} + +func main() { + registerLargeFinalizer() + for i := 0; i < 100 && !largeFinalizerRan; i++ { + runtime.GC() + runtime.Gosched() + } + if !largeFinalizerRan { + panic("large object finalizer did not run") + } + println("ok") +} diff --git a/testdata/finalizerlarge.txt b/testdata/finalizerlarge.txt new file mode 100644 index 000000000..9766475a4 --- /dev/null +++ b/testdata/finalizerlarge.txt @@ -0,0 +1 @@ +ok diff --git a/testdata/wasmexport-finalizer.go b/testdata/wasmexport-finalizer.go new file mode 100644 index 000000000..99a0fdd41 --- /dev/null +++ b/testdata/wasmexport-finalizer.go @@ -0,0 +1,51 @@ +package main + +import ( + "runtime" + "syscall/js" +) + +//go:wasmimport tester finalizerRan +func finalizerRan() + +//go:wasmimport tester backgroundRan +func backgroundRan() + +//go:wasmimport tester callNestedExport +func callNestedExport() + +var nestedCallback js.Func + +//go:wasmexport launchBackground +func launchBackground() { + go func() { + backgroundRan() + }() +} + +//go:wasmexport installNestedCallback +func installNestedCallback() { + nestedCallback = js.FuncOf(func(js.Value, []js.Value) any { + callNestedExport() + return nil + }) + js.Global().Set("nestedExportCallback", nestedCallback) +} + +//go:noinline +func registerFinalizersImpl() { + for i := 0; i < 32; i++ { + p := new([2]int) + runtime.SetFinalizer(p, func(*[2]int) { finalizerRan() }) + } +} + +//go:wasmexport registerFinalizers +func registerFinalizers() { + registerFinalizersImpl() + go func() { + backgroundRan() + }() +} + +func main() {} diff --git a/testdata/wasmexport-finalizer.js b/testdata/wasmexport-finalizer.js new file mode 100644 index 000000000..44305b9de --- /dev/null +++ b/testdata/wasmexport-finalizer.js @@ -0,0 +1,74 @@ +const fs = require('fs'); + +require('../targets/wasm_exec.js'); + +let finalized = 0; +let backgroundRuns = 0; +let instance; +const go = new Go(); +const sleepTicks = go.importObject.gojs['runtime.sleepTicks']; +let zeroRearms = 0; +go.importObject.gojs['runtime.sleepTicks'] = timeout => { + if (Number(timeout) === 0) { + zeroRearms++; + } + return sleepTicks(timeout); +}; +go.importObject.tester = { + finalizerRan: () => { + finalized++; + }, + backgroundRan: () => { + backgroundRuns++; + }, + callNestedExport: () => { + instance.exports.launchBackground(); + }, +}; + +WebAssembly.instantiate(fs.readFileSync(process.argv[2]), go.importObject).then(async result => { + instance = result.instance; + await go.run(instance); + + const topLevelRearms = zeroRearms; + instance.exports.launchBackground(); + if (backgroundRuns !== 0) { + throw new Error('wasm export ran a background goroutine before returning'); + } + if (zeroRearms !== topLevelRearms + 1) { + throw new Error('top-level wasm export did not schedule exactly one wakeup'); + } + for (let i = 0; i < 500 && backgroundRuns === 0; i++) { + await new Promise(resolve => setTimeout(resolve, 1)); + } + if (backgroundRuns !== 1) { + throw new Error('wasm scheduler did not resume after an export without finalizers'); + } + + instance.exports.installNestedCallback(); + const nestedRearms = zeroRearms; + global.nestedExportCallback(); + if (zeroRearms !== nestedRearms) { + throw new Error('re-entrant wasm export scheduled a redundant wakeup'); + } + if (backgroundRuns !== 2) { + throw new Error('outer scheduler did not drain work from a re-entrant wasm export'); + } + + instance.exports.registerFinalizers(); + if (backgroundRuns !== 2) { + throw new Error('wasm export ran an unrelated goroutine before returning'); + } + for (let i = 0; i < 500 && (finalized === 0 || backgroundRuns < 3); i++) { + await new Promise(resolve => setTimeout(resolve, 1)); + } + if (finalized === 0) { + throw new Error('no wasm-export finalizer ran after returning to JavaScript'); + } + if (backgroundRuns !== 3) { + throw new Error('wasm scheduler did not resume after a finalizer export'); + } +}).catch(err => { + console.error(err); + process.exit(1); +});