From 829fe514cca4fd7fd149b0d062f164b79407c3d7 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Fri, 7 Aug 2026 10:32:00 -0700 Subject: [PATCH] runtime: precisely scan Wasm globals --- src/runtime/gc_custom.go | 1 - src/runtime/gc_globals_blocks.go | 22 +++ src/runtime/gc_globals_custom.go | 18 +++ src/runtime/gc_globals_range.go | 40 +++++ src/runtime/gc_stack_portable.go | 2 +- transform/gc.go | 178 ++++++++++++++++++++++- transform/gc_test.go | 7 + transform/testdata/gc-globals-avr.ll | 12 ++ transform/testdata/gc-globals-avr.out.ll | 28 ++++ transform/testdata/gc-stackslots.ll | 11 ++ transform/testdata/gc-stackslots.out.ll | 31 ++++ 11 files changed, 345 insertions(+), 5 deletions(-) create mode 100644 src/runtime/gc_globals_blocks.go create mode 100644 src/runtime/gc_globals_custom.go create mode 100644 src/runtime/gc_globals_range.go create mode 100644 transform/testdata/gc-globals-avr.ll create mode 100644 transform/testdata/gc-globals-avr.out.ll diff --git a/src/runtime/gc_custom.go b/src/runtime/gc_custom.go index 0125f1688..12f1b2e12 100644 --- a/src/runtime/gc_custom.go +++ b/src/runtime/gc_custom.go @@ -48,7 +48,6 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer func free(ptr unsafe.Pointer) // markRoots is called with the start and end addresses to scan for references. -// It is currently only called with the top and bottom of the stack. func markRoots(start, end uintptr) // GC is called to explicitly run garbage collection. diff --git a/src/runtime/gc_globals_blocks.go b/src/runtime/gc_globals_blocks.go new file mode 100644 index 000000000..b9cb94cb8 --- /dev/null +++ b/src/runtime/gc_globals_blocks.go @@ -0,0 +1,22 @@ +//go:build (gc.conservative || gc.precise) && tinygo.wasm + +package runtime + +import "unsafe" + +func markGlobals() { + for i := uintptr(0); i < gcGlobalRootCount(); i++ { + addr := gcGlobalRoot(i) + size := gcGlobalRootSize(i) + for offset := uintptr(0); offset < size; offset += unsafe.Sizeof(uintptr(0)) { + slot := unsafe.Add(addr, offset) + markRoot(uintptr(slot), *(*uintptr)(slot)) + } + } +} + +// These functions are generated by the compiler from the pointer layouts of +// mutable globals. Each range contains only pointer slots. +func gcGlobalRootCount() uintptr +func gcGlobalRoot(index uintptr) unsafe.Pointer +func gcGlobalRootSize(index uintptr) uintptr diff --git a/src/runtime/gc_globals_custom.go b/src/runtime/gc_globals_custom.go new file mode 100644 index 000000000..bf612e6ac --- /dev/null +++ b/src/runtime/gc_globals_custom.go @@ -0,0 +1,18 @@ +//go:build gc.custom && tinygo.wasm + +package runtime + +import "unsafe" + +func markGlobals() { + for i := uintptr(0); i < gcGlobalRootCount(); i++ { + start := uintptr(gcGlobalRoot(i)) + markRoots(start, start+gcGlobalRootSize(i)) + } +} + +// These functions are generated by the compiler from the pointer layouts of +// mutable globals. Each range contains only pointer slots. +func gcGlobalRootCount() uintptr +func gcGlobalRoot(index uintptr) unsafe.Pointer +func gcGlobalRootSize(index uintptr) uintptr diff --git a/src/runtime/gc_globals_range.go b/src/runtime/gc_globals_range.go new file mode 100644 index 000000000..5702e2b1c --- /dev/null +++ b/src/runtime/gc_globals_range.go @@ -0,0 +1,40 @@ +//go:build gc.boehm && tinygo.wasm + +package runtime + +import "unsafe" + +func markGlobals() { + rangeCount := gcGlobalRootCount() + if rangeCount == 0 { + return + } + + var rootCount uintptr + for i := uintptr(0); i < rangeCount; i++ { + rootCount += gcGlobalRootSize(i) / unsafe.Sizeof(uintptr(0)) + } + + // markRoots only accepts a range, so copy all global pointers into + // contiguous scratch space for marking. + roots := unsafe.Slice((*uintptr)(gcGlobalRootValues()), rootCount) + var rootIndex uintptr + for i := uintptr(0); i < rangeCount; i++ { + addr := gcGlobalRoot(i) + size := gcGlobalRootSize(i) + for offset := uintptr(0); offset < size; offset += unsafe.Sizeof(uintptr(0)) { + roots[rootIndex] = *(*uintptr)(unsafe.Add(addr, offset)) + rootIndex++ + } + } + + start := uintptr(unsafe.Pointer(&roots[0])) + markRoots(start, start+rootCount*unsafe.Sizeof(roots[0])) +} + +// These functions are generated by the compiler from the pointer layouts of +// mutable globals. Each range contains only pointer slots. +func gcGlobalRootCount() uintptr +func gcGlobalRoot(index uintptr) unsafe.Pointer +func gcGlobalRootSize(index uintptr) uintptr +func gcGlobalRootValues() unsafe.Pointer diff --git a/src/runtime/gc_stack_portable.go b/src/runtime/gc_stack_portable.go index 04162bb07..fdf0a7cae 100644 --- a/src/runtime/gc_stack_portable.go +++ b/src/runtime/gc_stack_portable.go @@ -10,7 +10,7 @@ import ( func gcMarkReachable() { markStack() - findGlobals(markRoots) + markGlobals() } //go:extern runtime.stackChainStart diff --git a/transform/gc.go b/transform/gc.go index abbe3cb7b..c45e65158 100644 --- a/transform/gc.go +++ b/transform/gc.go @@ -1,6 +1,8 @@ package transform import ( + "strings" + "tinygo.org/x/go-llvm" ) @@ -12,6 +14,8 @@ const shiftExcludeArgMem = 2 // MakeGCStackSlots converts all calls to runtime.trackPointer to explicit // stores to stack slots that are scannable by the GC. func MakeGCStackSlots(mod llvm.Module) bool { + hasGlobalRoots := makeGCGlobalRoots(mod) + // Check whether there are allocations at all. alloc := mod.NamedFunction("runtime.alloc") if alloc.IsNil() { @@ -26,12 +30,12 @@ func MakeGCStackSlots(mod llvm.Module) bool { stackChainStart.SetInitializer(llvm.ConstNull(stackChainStart.GlobalValueType())) stackChainStart.SetGlobalConstant(true) } - return false + return hasGlobalRoots } trackPointer := mod.NamedFunction("runtime.trackPointer") if trackPointer.IsNil() || trackPointer.FirstUse().IsNil() { - return false // nothing to do + return hasGlobalRoots } ctx := mod.Context() @@ -107,7 +111,7 @@ func MakeGCStackSlots(mod llvm.Module) bool { for _, use := range getUses(trackPointer) { use.EraseFromParentAsInstruction() } - return false + return hasGlobalRoots } stackChainStart.SetLinkage(llvm.InternalLinkage) stackChainStartType := stackChainStart.GlobalValueType() @@ -285,6 +289,174 @@ func MakeGCStackSlots(mod llvm.Module) bool { return true } +func makeGCGlobalRoots(mod llvm.Module) bool { + rootCount := mod.NamedFunction("runtime.gcGlobalRootCount") + rootAt := mod.NamedFunction("runtime.gcGlobalRoot") + rootSize := mod.NamedFunction("runtime.gcGlobalRootSize") + rootValues := mod.NamedFunction("runtime.gcGlobalRootValues") + if rootCount.IsNil() || rootAt.IsNil() || rootSize.IsNil() || + !rootCount.FirstBasicBlock().IsNil() || + !rootAt.FirstBasicBlock().IsNil() || + !rootSize.FirstBasicBlock().IsNil() { + return false + } + if !rootValues.IsNil() && !rootValues.FirstBasicBlock().IsNil() { + return false + } + + ctx := mod.Context() + uintptrType := rootCount.GlobalValueType().ReturnType() + targetData := llvm.NewTargetData(mod.DataLayout()) + defer targetData.Dispose() + var roots []gcGlobalRootRange + for global := mod.FirstGlobal(); !global.IsNil(); global = llvm.NextGlobal(global) { + if strings.HasPrefix(global.Name(), "llvm.") || + global.IsGlobalConstant() || + global.Initializer().IsNil() || + !gcTypeHasPointers(global.GlobalValueType()) { + continue + } + roots = appendGCGlobalRootRanges(roots, global, global.GlobalValueType(), targetData, ctx.Int8Type(), uintptrType) + } + + ptrType := rootAt.GlobalValueType().ReturnType() + rootType := ctx.StructType([]llvm.Type{ptrType, uintptrType}, false) + rootInitializers := make([]llvm.Value, len(roots)) + for i, root := range roots { + rootInitializers[i] = llvm.ConstNamedStruct(rootType, []llvm.Value{ + root.address, + llvm.ConstInt(uintptrType, root.size, false), + }) + } + rootArrayType := llvm.ArrayType(rootType, len(roots)) + rootArray := llvm.AddGlobal(mod, rootArrayType, "runtime.gcGlobalRoots") + rootArray.SetInitializer(llvm.ConstArray(rootType, rootInitializers)) + rootArray.SetGlobalConstant(true) + rootArray.SetLinkage(llvm.InternalLinkage) + + builder := ctx.NewBuilder() + defer builder.Dispose() + + entry := ctx.AddBasicBlock(rootCount, "entry") + builder.SetInsertPointAtEnd(entry) + builder.CreateRet(llvm.ConstInt(rootCount.GlobalValueType().ReturnType(), uint64(len(roots)), false)) + + entry = ctx.AddBasicBlock(rootAt, "entry") + builder.SetInsertPointAtEnd(entry) + index := rootAt.FirstParam() + root := builder.CreateInBoundsGEP(rootArrayType, rootArray, []llvm.Value{ + llvm.ConstInt(ctx.Int32Type(), 0, false), + index, + }, "") + addr := builder.CreateStructGEP(rootType, root, 0, "") + builder.CreateRet(builder.CreateLoad(ptrType, addr, "")) + + entry = ctx.AddBasicBlock(rootSize, "entry") + builder.SetInsertPointAtEnd(entry) + index = rootSize.FirstParam() + root = builder.CreateInBoundsGEP(rootArrayType, rootArray, []llvm.Value{ + llvm.ConstInt(ctx.Int32Type(), 0, false), + index, + }, "") + size := builder.CreateStructGEP(rootType, root, 1, "") + builder.CreateRet(builder.CreateLoad(uintptrType, size, "")) + + if !rootValues.IsNil() { + pointerSize := uint64(targetData.PointerSize()) + var rootValueCount uint64 + for _, root := range roots { + rootValueCount += root.size / pointerSize + } + rootValueArray := llvm.AddGlobal(mod, llvm.ArrayType(uintptrType, int(rootValueCount)), "runtime.gcGlobalRootValueArray") + rootValueArray.SetInitializer(llvm.ConstNull(rootValueArray.GlobalValueType())) + rootValueArray.SetLinkage(llvm.InternalLinkage) + + entry = ctx.AddBasicBlock(rootValues, "entry") + builder.SetInsertPointAtEnd(entry) + builder.CreateRet(rootValueArray) + } + + return true +} + +// gcGlobalRootRange is a contiguous range of pointer slots. +// It never includes padding or non-pointer fields. +type gcGlobalRootRange struct { + address llvm.Value + size uint64 +} + +func appendGCGlobalRootRanges(roots []gcGlobalRootRange, global llvm.Value, typ llvm.Type, targetData llvm.TargetData, i8Type, uintptrType llvm.Type) []gcGlobalRootRange { + var offsets []uint64 + offsets = appendGCGlobalRootOffsets(offsets, typ, targetData, 0) + if len(offsets) == 0 { + return roots + } + + pointerSize := uint64(targetData.PointerSize()) + rangeStart := offsets[0] + rangeEnd := rangeStart + pointerSize + for _, offset := range offsets[1:] { + if offset == rangeEnd { + rangeEnd += pointerSize + continue + } + roots = appendGCGlobalRootRange(roots, global, rangeStart, rangeEnd-rangeStart, i8Type, uintptrType) + rangeStart = offset + rangeEnd = offset + pointerSize + } + return appendGCGlobalRootRange(roots, global, rangeStart, rangeEnd-rangeStart, i8Type, uintptrType) +} + +func appendGCGlobalRootRange(roots []gcGlobalRootRange, global llvm.Value, offset, size uint64, i8Type, uintptrType llvm.Type) []gcGlobalRootRange { + address := global + if offset != 0 { + address = llvm.ConstGEP(i8Type, global, []llvm.Value{ + llvm.ConstInt(uintptrType, offset, false), + }) + } + return append(roots, gcGlobalRootRange{address: address, size: size}) +} + +func appendGCGlobalRootOffsets(offsets []uint64, typ llvm.Type, targetData llvm.TargetData, baseOffset uint64) []uint64 { + switch typ.TypeKind() { + case llvm.PointerTypeKind: + return append(offsets, baseOffset) + case llvm.StructTypeKind: + for i, fieldType := range typ.StructElementTypes() { + if gcTypeHasPointers(fieldType) { + fieldOffset := targetData.ElementOffset(typ, i) + offsets = appendGCGlobalRootOffsets(offsets, fieldType, targetData, baseOffset+fieldOffset) + } + } + case llvm.ArrayTypeKind: + elemType := typ.ElementType() + if gcTypeHasPointers(elemType) { + elemSize := targetData.TypeAllocSize(elemType) + for i := 0; i < typ.ArrayLength(); i++ { + offsets = appendGCGlobalRootOffsets(offsets, elemType, targetData, baseOffset+uint64(i)*elemSize) + } + } + } + return offsets +} + +func gcTypeHasPointers(typ llvm.Type) bool { + switch typ.TypeKind() { + case llvm.PointerTypeKind: + return true + case llvm.StructTypeKind: + for _, field := range typ.StructElementTypes() { + if gcTypeHasPointers(field) { + return true + } + } + case llvm.ArrayTypeKind: + return typ.ArrayLength() != 0 && gcTypeHasPointers(typ.ElementType()) + } + return false +} + // markParentFunctions traverses all parent function calls (recursively) and // adds them to the set of marked functions. It only considers function calls: // any other uses of such a function is ignored. diff --git a/transform/gc_test.go b/transform/gc_test.go index c071c5f4d..194d41316 100644 --- a/transform/gc_test.go +++ b/transform/gc_test.go @@ -13,3 +13,10 @@ func TestMakeGCStackSlots(t *testing.T) { transform.MakeGCStackSlots(mod) }) } + +func TestMakeGCGlobalRootsAVR(t *testing.T) { + t.Parallel() + testTransform(t, "testdata/gc-globals-avr", func(mod llvm.Module) { + transform.MakeGCStackSlots(mod) + }) +} diff --git a/transform/testdata/gc-globals-avr.ll b/transform/testdata/gc-globals-avr.ll new file mode 100644 index 000000000..bfdd832a9 --- /dev/null +++ b/transform/testdata/gc-globals-avr.ll @@ -0,0 +1,12 @@ +target datalayout = "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8:16-a:8" +target triple = "avr-unknown-unknown" + +%entry = type { i8, ptr } + +@table = global [4 x %entry] zeroinitializer + +declare i16 @runtime.gcGlobalRootCount() + +declare ptr @runtime.gcGlobalRoot(i16) + +declare i16 @runtime.gcGlobalRootSize(i16) diff --git a/transform/testdata/gc-globals-avr.out.ll b/transform/testdata/gc-globals-avr.out.ll new file mode 100644 index 000000000..604896ad8 --- /dev/null +++ b/transform/testdata/gc-globals-avr.out.ll @@ -0,0 +1,28 @@ +target datalayout = "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8:16-a:8" +target triple = "avr-unknown-unknown" + +%entry = type { i8, ptr } + +@table = global [4 x %entry] zeroinitializer +@runtime.gcGlobalRoots = internal constant [4 x { ptr, i16 }] [{ ptr, i16 } { ptr getelementptr (i8, ptr @table, i16 1), i16 2 }, { ptr, i16 } { ptr getelementptr (i8, ptr @table, i16 4), i16 2 }, { ptr, i16 } { ptr getelementptr (i8, ptr @table, i16 7), i16 2 }, { ptr, i16 } { ptr getelementptr (i8, ptr @table, i16 10), i16 2 }] + +define i16 @runtime.gcGlobalRootCount() addrspace(1) { +entry: + ret i16 4 +} + +define ptr @runtime.gcGlobalRoot(i16 %0) addrspace(1) { +entry: + %1 = getelementptr inbounds [4 x { ptr, i16 }], ptr @runtime.gcGlobalRoots, i32 0, i16 %0 + %2 = getelementptr inbounds nuw { ptr, i16 }, ptr %1, i32 0, i32 0 + %3 = load ptr, ptr %2, align 1 + ret ptr %3 +} + +define i16 @runtime.gcGlobalRootSize(i16 %0) addrspace(1) { +entry: + %1 = getelementptr inbounds [4 x { ptr, i16 }], ptr @runtime.gcGlobalRoots, i32 0, i16 %0 + %2 = getelementptr inbounds nuw { ptr, i16 }, ptr %1, i32 0, i32 1 + %3 = load i16, ptr %2, align 1 + ret i16 %3 +} diff --git a/transform/testdata/gc-stackslots.ll b/transform/testdata/gc-stackslots.ll index 8da5110cb..912051901 100644 --- a/transform/testdata/gc-stackslots.ll +++ b/transform/testdata/gc-stackslots.ll @@ -5,11 +5,22 @@ target triple = "wasm32-unknown-unknown-wasm" @someGlobal = global i8 3 @ptrGlobal = global ptr null @arrGlobal = global [8 x i8] zeroinitializer +@structGlobal = global {ptr, i32, [2 x ptr]} zeroinitializer +@ptrArrayGlobal = global [8 x ptr] zeroinitializer +@constantPtrGlobal = constant ptr @someGlobal declare void @runtime.trackPointer(ptr nocapture readonly) declare noalias nonnull ptr @runtime.alloc(i32, ptr) +declare i32 @runtime.gcGlobalRootCount() + +declare ptr @runtime.gcGlobalRoot(i32) + +declare i32 @runtime.gcGlobalRootSize(i32) + +declare ptr @runtime.gcGlobalRootValues() + ; Generic function that returns a pointer (that must be tracked). define ptr @getPointer() { ret ptr @someGlobal diff --git a/transform/testdata/gc-stackslots.out.ll b/transform/testdata/gc-stackslots.out.ll index 28094c395..5fdad3f6a 100644 --- a/transform/testdata/gc-stackslots.out.ll +++ b/transform/testdata/gc-stackslots.out.ll @@ -5,11 +5,42 @@ target triple = "wasm32-unknown-unknown-wasm" @someGlobal = global i8 3 @ptrGlobal = global ptr null @arrGlobal = global [8 x i8] zeroinitializer +@structGlobal = global { ptr, i32, [2 x ptr] } zeroinitializer +@ptrArrayGlobal = global [8 x ptr] zeroinitializer +@constantPtrGlobal = constant ptr @someGlobal +@runtime.gcGlobalRoots = internal constant [4 x { ptr, i32 }] [{ ptr, i32 } { ptr @ptrGlobal, i32 4 }, { ptr, i32 } { ptr @structGlobal, i32 4 }, { ptr, i32 } { ptr getelementptr (i8, ptr @structGlobal, i32 8), i32 8 }, { ptr, i32 } { ptr @ptrArrayGlobal, i32 32 }] +@runtime.gcGlobalRootValueArray = internal global [12 x i32] zeroinitializer declare void @runtime.trackPointer(ptr nocapture readonly) declare noalias nonnull ptr @runtime.alloc(i32, ptr) +define i32 @runtime.gcGlobalRootCount() { +entry: + ret i32 4 +} + +define ptr @runtime.gcGlobalRoot(i32 %0) { +entry: + %1 = getelementptr inbounds [4 x { ptr, i32 }], ptr @runtime.gcGlobalRoots, i32 0, i32 %0 + %2 = getelementptr inbounds nuw { ptr, i32 }, ptr %1, i32 0, i32 0 + %3 = load ptr, ptr %2, align 4 + ret ptr %3 +} + +define i32 @runtime.gcGlobalRootSize(i32 %0) { +entry: + %1 = getelementptr inbounds [4 x { ptr, i32 }], ptr @runtime.gcGlobalRoots, i32 0, i32 %0 + %2 = getelementptr inbounds nuw { ptr, i32 }, ptr %1, i32 0, i32 1 + %3 = load i32, ptr %2, align 4 + ret i32 %3 +} + +define ptr @runtime.gcGlobalRootValues() { +entry: + ret ptr @runtime.gcGlobalRootValueArray +} + define ptr @getPointer() { ret ptr @someGlobal }