runtime: precisely scan Wasm globals

src/runtime/gc_custom.go

@@ -48,7 +48,6 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer
 func free(ptr unsafe.Pointer)
 
 // markRoots is called with the start and end addresses to scan for references.
-// It is currently only called with the top and bottom of the stack.
 func markRoots(start, end uintptr)
 
 // GC is called to explicitly run garbage collection.

src/runtime/gc_globals_blocks.go

@@ -0,0 +1,22 @@
+//go:build (gc.conservative || gc.precise) && tinygo.wasm
+
+package runtime
+
+import "unsafe"
+
+func markGlobals() {
+	for i := uintptr(0); i < gcGlobalRootCount(); i++ {
+		addr := gcGlobalRoot(i)
+		size := gcGlobalRootSize(i)
+		for offset := uintptr(0); offset < size; offset += unsafe.Sizeof(uintptr(0)) {
+			slot := unsafe.Add(addr, offset)
+			markRoot(uintptr(slot), *(*uintptr)(slot))
+		}
+	}
+}
+
+// These functions are generated by the compiler from the pointer layouts of
+// mutable globals. Each range contains only pointer slots.
+func gcGlobalRootCount() uintptr
+func gcGlobalRoot(index uintptr) unsafe.Pointer
+func gcGlobalRootSize(index uintptr) uintptr

src/runtime/gc_globals_custom.go

@@ -0,0 +1,18 @@
+//go:build gc.custom && tinygo.wasm
+
+package runtime
+
+import "unsafe"
+
+func markGlobals() {
+	for i := uintptr(0); i < gcGlobalRootCount(); i++ {
+		start := uintptr(gcGlobalRoot(i))
+		markRoots(start, start+gcGlobalRootSize(i))
+	}
+}
+
+// These functions are generated by the compiler from the pointer layouts of
+// mutable globals. Each range contains only pointer slots.
+func gcGlobalRootCount() uintptr
+func gcGlobalRoot(index uintptr) unsafe.Pointer
+func gcGlobalRootSize(index uintptr) uintptr

src/runtime/gc_globals_range.go

@@ -0,0 +1,40 @@
+//go:build gc.boehm && tinygo.wasm
+
+package runtime
+
+import "unsafe"
+
+func markGlobals() {
+	rangeCount := gcGlobalRootCount()
+	if rangeCount == 0 {
+		return
+	}
+
+	var rootCount uintptr
+	for i := uintptr(0); i < rangeCount; i++ {
+		rootCount += gcGlobalRootSize(i) / unsafe.Sizeof(uintptr(0))
+	}
+
+	// markRoots only accepts a range, so copy all global pointers into
+	// contiguous scratch space for marking.
+	roots := unsafe.Slice((*uintptr)(gcGlobalRootValues()), rootCount)
+	var rootIndex uintptr
+	for i := uintptr(0); i < rangeCount; i++ {
+		addr := gcGlobalRoot(i)
+		size := gcGlobalRootSize(i)
+		for offset := uintptr(0); offset < size; offset += unsafe.Sizeof(uintptr(0)) {
+			roots[rootIndex] = *(*uintptr)(unsafe.Add(addr, offset))
+			rootIndex++
+		}
+	}
+
+	start := uintptr(unsafe.Pointer(&roots[0]))
+	markRoots(start, start+rootCount*unsafe.Sizeof(roots[0]))
+}
+
+// These functions are generated by the compiler from the pointer layouts of
+// mutable globals. Each range contains only pointer slots.
+func gcGlobalRootCount() uintptr
+func gcGlobalRoot(index uintptr) unsafe.Pointer
+func gcGlobalRootSize(index uintptr) uintptr
+func gcGlobalRootValues() unsafe.Pointer

src/runtime/gc_stack_portable.go

@@ -10,7 +10,7 @@ import (
 
 func gcMarkReachable() {
 	markStack()
-	findGlobals(markRoots)
+	markGlobals()
 }
 
 //go:extern runtime.stackChainStart

transform/gc.go

@@ -1,6 +1,8 @@
 package transform
 
 import (
+	"strings"
+
 	"tinygo.org/x/go-llvm"
 )
 
@@ -12,6 +14,8 @@ const shiftExcludeArgMem = 2
 // MakeGCStackSlots converts all calls to runtime.trackPointer to explicit
 // stores to stack slots that are scannable by the GC.
 func MakeGCStackSlots(mod llvm.Module) bool {
+	hasGlobalRoots := makeGCGlobalRoots(mod)
+
 	// Check whether there are allocations at all.
 	alloc := mod.NamedFunction("runtime.alloc")
 	if alloc.IsNil() {
@@ -26,12 +30,12 @@ func MakeGCStackSlots(mod llvm.Module) bool {
 			stackChainStart.SetInitializer(llvm.ConstNull(stackChainStart.GlobalValueType()))
 			stackChainStart.SetGlobalConstant(true)
 		}
-		return false
+		return hasGlobalRoots
 	}
 
 	trackPointer := mod.NamedFunction("runtime.trackPointer")
 	if trackPointer.IsNil() || trackPointer.FirstUse().IsNil() {
-		return false // nothing to do
+		return hasGlobalRoots
 	}
 
 	ctx := mod.Context()
@@ -107,7 +111,7 @@ func MakeGCStackSlots(mod llvm.Module) bool {
 		for _, use := range getUses(trackPointer) {
 			use.EraseFromParentAsInstruction()
 		}
-		return false
+		return hasGlobalRoots
 	}
 	stackChainStart.SetLinkage(llvm.InternalLinkage)
 	stackChainStartType := stackChainStart.GlobalValueType()
@@ -285,6 +289,174 @@ func MakeGCStackSlots(mod llvm.Module) bool {
 	return true
 }
 
+func makeGCGlobalRoots(mod llvm.Module) bool {
+	rootCount := mod.NamedFunction("runtime.gcGlobalRootCount")
+	rootAt := mod.NamedFunction("runtime.gcGlobalRoot")
+	rootSize := mod.NamedFunction("runtime.gcGlobalRootSize")
+	rootValues := mod.NamedFunction("runtime.gcGlobalRootValues")
+	if rootCount.IsNil() || rootAt.IsNil() || rootSize.IsNil() ||
+		!rootCount.FirstBasicBlock().IsNil() ||
+		!rootAt.FirstBasicBlock().IsNil() ||
+		!rootSize.FirstBasicBlock().IsNil() {
+		return false
+	}
+	if !rootValues.IsNil() && !rootValues.FirstBasicBlock().IsNil() {
+		return false
+	}
+
+	ctx := mod.Context()
+	uintptrType := rootCount.GlobalValueType().ReturnType()
+	targetData := llvm.NewTargetData(mod.DataLayout())
+	defer targetData.Dispose()
+	var roots []gcGlobalRootRange
+	for global := mod.FirstGlobal(); !global.IsNil(); global = llvm.NextGlobal(global) {
+		if strings.HasPrefix(global.Name(), "llvm.") ||
+			global.IsGlobalConstant() ||
+			global.Initializer().IsNil() ||
+			!gcTypeHasPointers(global.GlobalValueType()) {
+			continue
+		}
+		roots = appendGCGlobalRootRanges(roots, global, global.GlobalValueType(), targetData, ctx.Int8Type(), uintptrType)
+	}
+
+	ptrType := rootAt.GlobalValueType().ReturnType()
+	rootType := ctx.StructType([]llvm.Type{ptrType, uintptrType}, false)
+	rootInitializers := make([]llvm.Value, len(roots))
+	for i, root := range roots {
+		rootInitializers[i] = llvm.ConstNamedStruct(rootType, []llvm.Value{
+			root.address,
+			llvm.ConstInt(uintptrType, root.size, false),
+		})
+	}
+	rootArrayType := llvm.ArrayType(rootType, len(roots))
+	rootArray := llvm.AddGlobal(mod, rootArrayType, "runtime.gcGlobalRoots")
+	rootArray.SetInitializer(llvm.ConstArray(rootType, rootInitializers))
+	rootArray.SetGlobalConstant(true)
+	rootArray.SetLinkage(llvm.InternalLinkage)
+
+	builder := ctx.NewBuilder()
+	defer builder.Dispose()
+
+	entry := ctx.AddBasicBlock(rootCount, "entry")
+	builder.SetInsertPointAtEnd(entry)
+	builder.CreateRet(llvm.ConstInt(rootCount.GlobalValueType().ReturnType(), uint64(len(roots)), false))
+
+	entry = ctx.AddBasicBlock(rootAt, "entry")
+	builder.SetInsertPointAtEnd(entry)
+	index := rootAt.FirstParam()
+	root := builder.CreateInBoundsGEP(rootArrayType, rootArray, []llvm.Value{
+		llvm.ConstInt(ctx.Int32Type(), 0, false),
+		index,
+	}, "")
+	addr := builder.CreateStructGEP(rootType, root, 0, "")
+	builder.CreateRet(builder.CreateLoad(ptrType, addr, ""))
+
+	entry = ctx.AddBasicBlock(rootSize, "entry")
+	builder.SetInsertPointAtEnd(entry)
+	index = rootSize.FirstParam()
+	root = builder.CreateInBoundsGEP(rootArrayType, rootArray, []llvm.Value{
+		llvm.ConstInt(ctx.Int32Type(), 0, false),
+		index,
+	}, "")
+	size := builder.CreateStructGEP(rootType, root, 1, "")
+	builder.CreateRet(builder.CreateLoad(uintptrType, size, ""))
+
+	if !rootValues.IsNil() {
+		pointerSize := uint64(targetData.PointerSize())
+		var rootValueCount uint64
+		for _, root := range roots {
+			rootValueCount += root.size / pointerSize
+		}
+		rootValueArray := llvm.AddGlobal(mod, llvm.ArrayType(uintptrType, int(rootValueCount)), "runtime.gcGlobalRootValueArray")
+		rootValueArray.SetInitializer(llvm.ConstNull(rootValueArray.GlobalValueType()))
+		rootValueArray.SetLinkage(llvm.InternalLinkage)
+
+		entry = ctx.AddBasicBlock(rootValues, "entry")
+		builder.SetInsertPointAtEnd(entry)
+		builder.CreateRet(rootValueArray)
+	}
+
+	return true
+}
+
+// gcGlobalRootRange is a contiguous range of pointer slots.
+// It never includes padding or non-pointer fields.
+type gcGlobalRootRange struct {
+	address llvm.Value
+	size    uint64
+}
+
+func appendGCGlobalRootRanges(roots []gcGlobalRootRange, global llvm.Value, typ llvm.Type, targetData llvm.TargetData, i8Type, uintptrType llvm.Type) []gcGlobalRootRange {
+	var offsets []uint64
+	offsets = appendGCGlobalRootOffsets(offsets, typ, targetData, 0)
+	if len(offsets) == 0 {
+		return roots
+	}
+
+	pointerSize := uint64(targetData.PointerSize())
+	rangeStart := offsets[0]
+	rangeEnd := rangeStart + pointerSize
+	for _, offset := range offsets[1:] {
+		if offset == rangeEnd {
+			rangeEnd += pointerSize
+			continue
+		}
+		roots = appendGCGlobalRootRange(roots, global, rangeStart, rangeEnd-rangeStart, i8Type, uintptrType)
+		rangeStart = offset
+		rangeEnd = offset + pointerSize
+	}
+	return appendGCGlobalRootRange(roots, global, rangeStart, rangeEnd-rangeStart, i8Type, uintptrType)
+}
+
+func appendGCGlobalRootRange(roots []gcGlobalRootRange, global llvm.Value, offset, size uint64, i8Type, uintptrType llvm.Type) []gcGlobalRootRange {
+	address := global
+	if offset != 0 {
+		address = llvm.ConstGEP(i8Type, global, []llvm.Value{
+			llvm.ConstInt(uintptrType, offset, false),
+		})
+	}
+	return append(roots, gcGlobalRootRange{address: address, size: size})
+}
+
+func appendGCGlobalRootOffsets(offsets []uint64, typ llvm.Type, targetData llvm.TargetData, baseOffset uint64) []uint64 {
+	switch typ.TypeKind() {
+	case llvm.PointerTypeKind:
+		return append(offsets, baseOffset)
+	case llvm.StructTypeKind:
+		for i, fieldType := range typ.StructElementTypes() {
+			if gcTypeHasPointers(fieldType) {
+				fieldOffset := targetData.ElementOffset(typ, i)
+				offsets = appendGCGlobalRootOffsets(offsets, fieldType, targetData, baseOffset+fieldOffset)
+			}
+		}
+	case llvm.ArrayTypeKind:
+		elemType := typ.ElementType()
+		if gcTypeHasPointers(elemType) {
+			elemSize := targetData.TypeAllocSize(elemType)
+			for i := 0; i < typ.ArrayLength(); i++ {
+				offsets = appendGCGlobalRootOffsets(offsets, elemType, targetData, baseOffset+uint64(i)*elemSize)
+			}
+		}
+	}
+	return offsets
+}
+
+func gcTypeHasPointers(typ llvm.Type) bool {
+	switch typ.TypeKind() {
+	case llvm.PointerTypeKind:
+		return true
+	case llvm.StructTypeKind:
+		for _, field := range typ.StructElementTypes() {
+			if gcTypeHasPointers(field) {
+				return true
+			}
+		}
+	case llvm.ArrayTypeKind:
+		return typ.ArrayLength() != 0 && gcTypeHasPointers(typ.ElementType())
+	}
+	return false
+}
+
 // markParentFunctions traverses all parent function calls (recursively) and
 // adds them to the set of marked functions. It only considers function calls:
 // any other uses of such a function is ignored.

transform/gc_test.go

@@ -13,3 +13,10 @@ func TestMakeGCStackSlots(t *testing.T) {
 		transform.MakeGCStackSlots(mod)
 	})
 }
+
+func TestMakeGCGlobalRootsAVR(t *testing.T) {
+	t.Parallel()
+	testTransform(t, "testdata/gc-globals-avr", func(mod llvm.Module) {
+		transform.MakeGCStackSlots(mod)
+	})
+}

transform/testdata/gc-globals-avr.ll

@@ -0,0 +1,12 @@
+target datalayout = "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8:16-a:8"
+target triple = "avr-unknown-unknown"
+
+%entry = type { i8, ptr }
+
+@table = global [4 x %entry] zeroinitializer
+
+declare i16 @runtime.gcGlobalRootCount()
+
+declare ptr @runtime.gcGlobalRoot(i16)
+
+declare i16 @runtime.gcGlobalRootSize(i16)

transform/testdata/gc-globals-avr.out.ll

@@ -0,0 +1,28 @@
+target datalayout = "e-P1-p:16:8-i8:8-i16:8-i32:8-i64:8-f32:8-f64:8-n8:16-a:8"
+target triple = "avr-unknown-unknown"
+
+%entry = type { i8, ptr }
+
+@table = global [4 x %entry] zeroinitializer
+@runtime.gcGlobalRoots = internal constant [4 x { ptr, i16 }] [{ ptr, i16 } { ptr getelementptr (i8, ptr @table, i16 1), i16 2 }, { ptr, i16 } { ptr getelementptr (i8, ptr @table, i16 4), i16 2 }, { ptr, i16 } { ptr getelementptr (i8, ptr @table, i16 7), i16 2 }, { ptr, i16 } { ptr getelementptr (i8, ptr @table, i16 10), i16 2 }]
+
+define i16 @runtime.gcGlobalRootCount() addrspace(1) {
+entry:
+  ret i16 4
+}
+
+define ptr @runtime.gcGlobalRoot(i16 %0) addrspace(1) {
+entry:
+  %1 = getelementptr inbounds [4 x { ptr, i16 }], ptr @runtime.gcGlobalRoots, i32 0, i16 %0
+  %2 = getelementptr inbounds nuw { ptr, i16 }, ptr %1, i32 0, i32 0
+  %3 = load ptr, ptr %2, align 1
+  ret ptr %3
+}
+
+define i16 @runtime.gcGlobalRootSize(i16 %0) addrspace(1) {
+entry:
+  %1 = getelementptr inbounds [4 x { ptr, i16 }], ptr @runtime.gcGlobalRoots, i32 0, i16 %0
+  %2 = getelementptr inbounds nuw { ptr, i16 }, ptr %1, i32 0, i32 1
+  %3 = load i16, ptr %2, align 1
+  ret i16 %3
+}

transform/testdata/gc-stackslots.ll

@@ -5,11 +5,22 @@ target triple = "wasm32-unknown-unknown-wasm"
 @someGlobal = global i8 3
 @ptrGlobal = global ptr null
 @arrGlobal = global [8 x i8] zeroinitializer
+@structGlobal = global {ptr, i32, [2 x ptr]} zeroinitializer
+@ptrArrayGlobal = global [8 x ptr] zeroinitializer
+@constantPtrGlobal = constant ptr @someGlobal
 
 declare void @runtime.trackPointer(ptr nocapture readonly)
 
 declare noalias nonnull ptr @runtime.alloc(i32, ptr)
 
+declare i32 @runtime.gcGlobalRootCount()
+
+declare ptr @runtime.gcGlobalRoot(i32)
+
+declare i32 @runtime.gcGlobalRootSize(i32)
+
+declare ptr @runtime.gcGlobalRootValues()
+
 ; Generic function that returns a pointer (that must be tracked).
 define ptr @getPointer() {
     ret ptr @someGlobal

transform/testdata/gc-stackslots.out.ll

@@ -5,11 +5,42 @@ target triple = "wasm32-unknown-unknown-wasm"
 @someGlobal = global i8 3
 @ptrGlobal = global ptr null
 @arrGlobal = global [8 x i8] zeroinitializer
+@structGlobal = global { ptr, i32, [2 x ptr] } zeroinitializer
+@ptrArrayGlobal = global [8 x ptr] zeroinitializer
+@constantPtrGlobal = constant ptr @someGlobal
+@runtime.gcGlobalRoots = internal constant [4 x { ptr, i32 }] [{ ptr, i32 } { ptr @ptrGlobal, i32 4 }, { ptr, i32 } { ptr @structGlobal, i32 4 }, { ptr, i32 } { ptr getelementptr (i8, ptr @structGlobal, i32 8), i32 8 }, { ptr, i32 } { ptr @ptrArrayGlobal, i32 32 }]
+@runtime.gcGlobalRootValueArray = internal global [12 x i32] zeroinitializer
 
 declare void @runtime.trackPointer(ptr nocapture readonly)
 
 declare noalias nonnull ptr @runtime.alloc(i32, ptr)
 
+define i32 @runtime.gcGlobalRootCount() {
+entry:
+  ret i32 4
+}
+
+define ptr @runtime.gcGlobalRoot(i32 %0) {
+entry:
+  %1 = getelementptr inbounds [4 x { ptr, i32 }], ptr @runtime.gcGlobalRoots, i32 0, i32 %0
+  %2 = getelementptr inbounds nuw { ptr, i32 }, ptr %1, i32 0, i32 0
+  %3 = load ptr, ptr %2, align 4
+  ret ptr %3
+}
+
+define i32 @runtime.gcGlobalRootSize(i32 %0) {
+entry:
+  %1 = getelementptr inbounds [4 x { ptr, i32 }], ptr @runtime.gcGlobalRoots, i32 0, i32 %0
+  %2 = getelementptr inbounds nuw { ptr, i32 }, ptr %1, i32 0, i32 1
+  %3 = load i32, ptr %2, align 4
+  ret i32 %3
+}
+
+define ptr @runtime.gcGlobalRootValues() {
+entry:
+  ret ptr @runtime.gcGlobalRootValueArray
+}
+
 define ptr @getPointer() {
   ret ptr @someGlobal
 }