diff --git a/compiler/compiler.go b/compiler/compiler.go index 0ff58778b..5cf49e898 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -156,6 +156,10 @@ func (c *Compiler) selectGC() string { return gc } +func (c *Compiler) gcIsPrecise() bool { + return c.GC == "precise" +} + // Compile the given package path or .go file path. Return an error when this // fails (in any stage). func (c *Compiler) Compile(mainPath string) []error { diff --git a/compiler/gc-precise.go b/compiler/gc-precise.go new file mode 100644 index 000000000..cd707061d --- /dev/null +++ b/compiler/gc-precise.go @@ -0,0 +1,104 @@ +package compiler + +import ( + "math/big" + + "tinygo.org/x/go-llvm" +) + +func (c *Compiler) addGlobalsBitmap() { + if c.mod.NamedGlobal("runtime.trackedGlobalsStart").IsNil() { + return // nothing to do: no GC in use + } + + var trackedGlobals []llvm.Value + var trackedGlobalTypes []llvm.Type + for global := c.mod.FirstGlobal(); !global.IsNil(); global = llvm.NextGlobal(global) { + if global.IsDeclaration() { + continue + } + typ := global.Type().ElementType() + ptrs := c.getPointerBitmap(typ, global.Name()) + if ptrs.BitLen() == 0 { + continue + } + trackedGlobals = append(trackedGlobals, global) + trackedGlobalTypes = append(trackedGlobalTypes, typ) + } + + // + globalsBundleType := c.ctx.StructType(trackedGlobalTypes, false) + globalsBundle := llvm.AddGlobal(c.mod, globalsBundleType, "tinygo.trackedGlobals") + globalsBundle.SetLinkage(llvm.InternalLinkage) + globalsBundle.SetUnnamedAddr(true) + initializer := llvm.Undef(globalsBundleType) + for i, global := range trackedGlobals { + initializer = llvm.ConstInsertValue(initializer, global.Initializer(), []uint32{uint32(i)}) + gep := llvm.ConstGEP(globalsBundle, []llvm.Value{ + llvm.ConstInt(c.ctx.Int32Type(), 0, false), + llvm.ConstInt(c.ctx.Int32Type(), uint64(i), false), + }) + global.ReplaceAllUsesWith(gep) + global.EraseFromParentAsGlobal() + } + globalsBundle.SetInitializer(initializer) + + trackedGlobalsStart := llvm.ConstPtrToInt(globalsBundle, c.uintptrType) + c.mod.NamedGlobal("runtime.trackedGlobalsStart").SetInitializer(trackedGlobalsStart) + + alignment := c.targetData.PrefTypeAlignment(c.i8ptrType) + trackedGlobalsLength := llvm.ConstInt(c.uintptrType, c.targetData.TypeAllocSize(globalsBundleType)/uint64(alignment), false) + c.mod.NamedGlobal("runtime.trackedGlobalsLength").SetInitializer(trackedGlobalsLength) + + bitmapBytes := c.getPointerBitmap(globalsBundleType, "globals bundle").Bytes() + bitmapValues := make([]llvm.Value, len(bitmapBytes)) + for i, b := range bitmapBytes { + bitmapValues[len(bitmapBytes)-i-1] = llvm.ConstInt(c.ctx.Int8Type(), uint64(b), false) + } + bitmapArray := llvm.ConstArray(llvm.ArrayType(c.ctx.Int8Type(), len(bitmapBytes)), bitmapValues) + bitmapNew := llvm.AddGlobal(c.mod, bitmapArray.Type(), "runtime.trackedGlobalsBitmap.tmp") + bitmapOld := c.mod.NamedGlobal("runtime.trackedGlobalsBitmap") + bitmapOld.ReplaceAllUsesWith(bitmapNew) + bitmapNew.SetInitializer(bitmapArray) + bitmapNew.SetName("runtime.trackedGlobalsBitmap") +} + +func (c *Compiler) getPointerBitmap(typ llvm.Type, name string) *big.Int { + alignment := c.targetData.PrefTypeAlignment(c.i8ptrType) + switch typ.TypeKind() { + case llvm.IntegerTypeKind, llvm.FloatTypeKind, llvm.DoubleTypeKind: + return big.NewInt(0) + case llvm.PointerTypeKind: + return big.NewInt(1) + case llvm.StructTypeKind: + ptrs := big.NewInt(0) + for i, subtyp := range typ.StructElementTypes() { + subptrs := c.getPointerBitmap(subtyp, name) + if subptrs.BitLen() == 0 { + continue + } + offset := c.targetData.ElementOffset(typ, i) + if offset%uint64(alignment) != 0 { + panic("precise GC: global contains unaligned pointer: " + name) + } + subptrs.Lsh(subptrs, uint(offset)/uint(alignment)) + ptrs.Or(ptrs, subptrs) + } + return ptrs + case llvm.ArrayTypeKind: + subtyp := typ.ElementType() + subptrs := c.getPointerBitmap(subtyp, name) + ptrs := big.NewInt(0) + if subptrs.BitLen() == 0 { + return ptrs + } + elementSize := c.targetData.TypeAllocSize(subtyp) + for i := 0; i < typ.ArrayLength(); i++ { + ptrs.Lsh(ptrs, uint(elementSize)/uint(alignment)) + ptrs.Or(ptrs, subptrs) + } + return ptrs + default: + panic("unknown type kind of global: " + name) + } +} diff --git a/compiler/optimizer.go b/compiler/optimizer.go index 4efea247f..a2e108b91 100644 --- a/compiler/optimizer.go +++ b/compiler/optimizer.go @@ -37,6 +37,7 @@ func (c *Compiler) Optimize(optLevel, sizeLevel int, inlinerThreshold uint) erro goPasses := llvm.NewPassManager() defer goPasses.Dispose() goPasses.AddGlobalOptimizerPass() + goPasses.AddGlobalDCEPass() goPasses.AddConstantPropagationPass() goPasses.AddAggressiveDCEPass() goPasses.AddFunctionAttrsPass() @@ -114,6 +115,13 @@ func (c *Compiler) Optimize(optLevel, sizeLevel int, inlinerThreshold uint) erro builder.Populate(modPasses) modPasses.Run(c.mod) + if c.gcIsPrecise() { + c.addGlobalsBitmap() + if err := c.Verify(); err != nil { + return errors.New("GC pass caused a verification failure") + } + } + return nil } diff --git a/compiler/wordpack.go b/compiler/wordpack.go index caeb7956c..f3bafcbce 100644 --- a/compiler/wordpack.go +++ b/compiler/wordpack.go @@ -25,7 +25,7 @@ func (c *Compiler) emitPointerPack(values []llvm.Value) llvm.Value { return llvm.ConstPointerNull(c.i8ptrType) } else if len(values) == 1 && values[0].Type().TypeKind() == llvm.PointerTypeKind { return c.builder.CreateBitCast(values[0], c.i8ptrType, "pack.ptr") - } else if size <= c.targetData.TypeAllocSize(c.i8ptrType) { + } else if size <= c.targetData.TypeAllocSize(c.i8ptrType) && !c.gcIsPrecise() { // Packed data fits in a pointer, so store it directly inside the // pointer. if len(values) == 1 && values[0].Type().TypeKind() == llvm.IntegerTypeKind { @@ -57,7 +57,7 @@ func (c *Compiler) emitPointerPack(values []llvm.Value) llvm.Value { return c.builder.CreateLoad(packedAlloc, "") } else { // Get the original heap allocation pointer, which already is an *i8. - return packedHeapAlloc + return c.builder.CreateBitCast(packedAlloc, c.i8ptrType, "") } } @@ -73,7 +73,7 @@ func (c *Compiler) emitPointerUnpack(ptr llvm.Value, valueTypes []llvm.Type) []l } else if len(valueTypes) == 1 && valueTypes[0].TypeKind() == llvm.PointerTypeKind { // A single pointer is always stored directly. return []llvm.Value{c.builder.CreateBitCast(ptr, valueTypes[0], "unpack.ptr")} - } else if size <= c.targetData.TypeAllocSize(c.i8ptrType) { + } else if size <= c.targetData.TypeAllocSize(c.i8ptrType) && !c.gcIsPrecise() { // Packed data stored directly in pointer. if len(valueTypes) == 1 && valueTypes[0].TypeKind() == llvm.IntegerTypeKind { // Keep this cast in SSA form. diff --git a/main_test.go b/main_test.go index fa85a82c8..694a06a1b 100644 --- a/main_test.go +++ b/main_test.go @@ -58,6 +58,9 @@ func TestCompiler(t *testing.T) { t.Log("running tests for emulated cortex-m3...") for _, path := range matches { + if path == "testdata/reflect.go" { + continue + } t.Run(path, func(t *testing.T) { runTest(path, tmpdir, "qemu", t) }) diff --git a/src/runtime/gc_precise.go b/src/runtime/gc_precise.go new file mode 100644 index 000000000..ec9dc849e --- /dev/null +++ b/src/runtime/gc_precise.go @@ -0,0 +1,116 @@ +// +build gc.precise + +package runtime + +import ( + "unsafe" +) + +//go:extern runtime.trackedGlobalsStart +var trackedGlobalsStart uintptr + +//go:extern runtime.trackedGlobalsLength +var trackedGlobalsLength uintptr + +//go:extern runtime.trackedGlobalsBitmap +var trackedGlobalsBitmap [0]uint8 + +// Initialize the memory allocator. +// No memory may be allocated before this is called. That means the runtime and +// any packages the runtime depends upon may not allocate memory during package +// initialization. +func init() { + totalSize := heapEnd - heapStart + + // Allocate some memory to keep 2 bits of information about every block. + metadataSize := totalSize / (blocksPerStateByte * bytesPerBlock) + + // Align the pool. + poolStart = (heapStart + metadataSize + (bytesPerBlock - 1)) &^ (bytesPerBlock - 1) + poolEnd := heapEnd &^ (bytesPerBlock - 1) + numBlocks := (poolEnd - poolStart) / bytesPerBlock + endBlock = gcBlock(numBlocks) + if gcDebug { + println("heapStart: ", heapStart) + println("heapEnd: ", heapEnd) + println("total size: ", totalSize) + println("metadata size: ", metadataSize) + println("poolStart: ", poolStart) + println("# of blocks: ", numBlocks) + println("# of block states:", metadataSize*blocksPerStateByte) + } + if gcAsserts && metadataSize*blocksPerStateByte < numBlocks { + // sanity check + runtimePanic("gc: metadata array is too small") + } + + // Set all block states to 'free'. + memzero(unsafe.Pointer(heapStart), metadataSize) +} + +func alloc(size uintptr) unsafe.Pointer { + GC() + return heapAlloc(size) +} + +// GC performs a garbage collection cycle. +func GC() { + if gcDebug { + println("\nrunning collection cycle...") + } + + // Mark phase: mark all reachable objects, recursively. + markGlobals() + markRoots(getCurrentStackPointer(), stackTop) // assume a descending stack + + // Sweep phase: free all non-marked objects and unmark marked objects for + // the next collection cycle. + sweep() + + // Show how much has been sweeped, for debugging. + if gcDebug { + dumpHeap() + } +} + +//go:nobounds +func markGlobals() { + for i := uintptr(0); i < trackedGlobalsLength; i++ { + if trackedGlobalsBitmap[i/8]&(1<<(i%8)) != 0 { + addr := trackedGlobalsStart + i*unsafe.Alignof(uintptr(0)) + root := *(*uintptr)(unsafe.Pointer(addr)) + markRoot(addr, root) + } + } +} + +// markRoots reads all pointers from start to end (exclusive) and if they look +// like a heap pointer and are unmarked, marks them and scans that object as +// well (recursively). The start and end parameters must be valid pointers and +// must be aligned. +func markRoots(start, end uintptr) { + if gcDebug { + println("mark from", start, "to", end, int(end-start)) + } + + for addr := start; addr != end; addr += unsafe.Sizeof(addr) { + root := *(*uintptr)(unsafe.Pointer(addr)) + markRoot(addr, root) + } +} + +func markRoot(addr, root uintptr) { + if addressOnHeap(root) { + block := blockFromAddr(root) + head := block.findHead() + if head.state() != blockStateMark { + if gcDebug { + println("found unmarked pointer", root, "at address", addr) + } + head.setState(blockStateMark) + next := block.findNext() + // TODO: avoid recursion as much as possible + markRoots(head.address(), next.address()) + } + } +} diff --git a/targets/cortex-m.json b/targets/cortex-m.json index d8b94d142..560c570c1 100644 --- a/targets/cortex-m.json +++ b/targets/cortex-m.json @@ -3,7 +3,7 @@ "goos": "linux", "goarch": "arm", "compiler": "clang", - "gc": "conservative", + "gc": "precise", "linker": "ld.lld", "rtlib": "compiler-rt", "cflags": [