runtime: precisely scan globals on all platforms

builder/build.go

@@ -1293,6 +1293,9 @@ func makeGlobalsModule(ctx llvm.Context, globals map[string]map[string]string, m
 			global := llvm.AddGlobal(mod, stringType, globalName)
 			global.SetInitializer(initializer)
 			global.SetAlignment(targetData.PrefTypeAlignment(stringType))
+			// Keep external linkage for module resolution. Hidden visibility permits internalization.
+			// See https://llvm.org/docs/LangRef.html#visibility-styles.
+			global.SetVisibility(llvm.HiddenVisibility)
 		}
 	}
 

builder/testdata/binary-size.txt

@@ -1,4 +1,4 @@
 target      package               code rodata data  bss
-hifive1b    examples/echo         4321    323    0 2268
+hifive1b    examples/echo         4405    323    0 2268
-microbit    examples/serial       2842    382    8 2264
+microbit    examples/serial       2922    382    8 2264
-wioterminal examples/pininterrupt 8039   1669  132 7496
+wioterminal examples/pininterrupt 8251   1717  148 7496

src/runtime/gc_globals.go

@@ -1,15 +0,0 @@
-//go:build (baremetal || tinygo.wasm) && !uefi
-
-package runtime
-
-// This file implements findGlobals for all systems where the start and end of
-// the globals section can be found through linker-defined symbols.
-
-// findGlobals finds all globals (which are reachable by definition) and calls
-// the callback for them.
-//
-// This implementation marks all globals conservatively and assumes it can use
-// linker-defined symbols for the start and end of the .data section.
-func findGlobals(found func(start, end uintptr)) {
-	found(globalsStart, globalsEnd)
-}

src/runtime/gc_globals_blocks.go

@@ -1,4 +1,4 @@
-//go:build (gc.conservative || gc.precise) && tinygo.wasm
+//go:build gc.conservative || gc.precise
 
 package runtime
 

src/runtime/gc_globals_boehm.go

@@ -1,4 +1,4 @@
-//go:build gc.boehm && tinygo.wasm
+//go:build gc.boehm
 
 package runtime
 

src/runtime/gc_globals_custom.go

@@ -1,4 +1,4 @@
-//go:build gc.custom && tinygo.wasm
+//go:build gc.custom
 
 package runtime
 

src/runtime/gc_globals_none.go

@@ -0,0 +1,6 @@
+//go:build gc.leaking || gc.none
+
+package runtime
+
+func markGlobals() {
+}

src/runtime/gc_stack_cores.go

@@ -26,7 +26,7 @@ func gcMarkReachable() {
 		}
 
 		// Scan globals.
-		findGlobals(markRoots)
+		markGlobals()
 
 		// Nothing more to do: the other cores haven't started yet.
 		return
@@ -57,7 +57,7 @@ func gcMarkReachable() {
 	}
 
 	// Scan globals.
-	findGlobals(markRoots)
+	markGlobals()
 
 	// Signal each core in turn that they can scan the stack.
 	for i := uint32(0); i < numCPU; i++ {

src/runtime/gc_stack_raw.go

@@ -12,7 +12,7 @@ var gcScanState atomic.Uint32
 
 func gcMarkReachable() {
 	markStack()
-	findGlobals(markRoots)
+	markGlobals()
 }
 
 // markStack marks all root pointers found on the stack.

src/runtime/gc_stack_threads.go

@@ -13,7 +13,7 @@ func gcMarkReachable() {
 //
 //go:linkname gcScanGlobals internal/task.gcScanGlobals
 func gcScanGlobals() {
-	findGlobals(markRoots)
+	markGlobals()
 }
 
 // Function called from assembly with all registers pushed, to actually scan the

src/runtime/os_darwin.go

@@ -31,97 +31,6 @@ const (
 	sig_SIGSEGV = 11
 )
 
-// https://opensource.apple.com/source/xnu/xnu-7195.141.2/EXTERNAL_HEADERS/mach-o/loader.h.auto.html
-type machHeader struct {
-	magic      uint32
-	cputype    uint32
-	cpusubtype uint32
-	filetype   uint32
-	ncmds      uint32
-	sizeofcmds uint32
-	flags      uint32
-	reserved   uint32
-}
-
-// Struct for the LC_SEGMENT_64 load command.
-type segmentLoadCommand struct {
-	cmd      uint32 // LC_SEGMENT_64
-	cmdsize  uint32
-	segname  [16]byte
-	vmaddr   uintptr
-	vmsize   uintptr
-	fileoff  uintptr
-	filesize uintptr
-	maxprot  uint32
-	initprot uint32
-	nsects   uint32
-	flags    uint32
-}
-
-// MachO header of the currently running process.
-//
-//go:extern _mh_execute_header
-var libc_mh_execute_header machHeader
-
-// Find global variables in .data/.bss sections.
-// The MachO linker doesn't seem to provide symbols for the start and end of the
-// data section. There is get_etext, get_edata, and get_end, but these are
-// undocumented and don't work with ASLR (which is enabled by default).
-// Therefore, read the MachO header directly.
-func findGlobals(found func(start, end uintptr)) {
-	// Here is a useful blog post to understand the MachO file format:
-	// https://h3adsh0tzz.com/2020/01/macho-file-format/
-
-	const (
-		MH_MAGIC_64   = 0xfeedfacf
-		LC_SEGMENT_64 = 0x19
-		VM_PROT_WRITE = 0x02
-	)
-
-	// Sanity check that we're actually looking at a MachO header.
-	if gcAsserts && libc_mh_execute_header.magic != MH_MAGIC_64 {
-		runtimeFatal("gc: unexpected MachO header")
-	}
-
-	// Iterate through the load commands.
-	// Because we're only interested in LC_SEGMENT_64 load commands, cast the
-	// pointer to that struct in advance.
-	var offset uintptr
-	var hasOffset bool
-	cmd := (*segmentLoadCommand)(unsafe.Pointer(uintptr(unsafe.Pointer(&libc_mh_execute_header)) + unsafe.Sizeof(machHeader{})))
-	for i := libc_mh_execute_header.ncmds; i != 0; i-- {
-		if cmd.cmd == LC_SEGMENT_64 {
-			if cmd.fileoff == 0 && cmd.nsects != 0 {
-				// Detect ASLR offset by checking fileoff and nsects. This
-				// locates the __TEXT segment. This matches getsectiondata:
-				// https://opensource.apple.com/source/cctools/cctools-973.0.1/libmacho/getsecbyname.c.auto.html
-				offset = uintptr(unsafe.Pointer(&libc_mh_execute_header)) - cmd.vmaddr
-				hasOffset = true
-			}
-			if cmd.maxprot&VM_PROT_WRITE != 0 {
-				// Found a writable segment, which may contain Go globals.
-				if gcAsserts && !hasOffset {
-					// No ASLR offset detected. Did the __TEXT segment come
-					// after the __DATA segment?
-					// Note that when ASLR is disabled (for example, when
-					// running inside lldb), the offset is zero. That's why we
-					// need a separate hasOffset for this assert.
-					runtimeFatal("gc: did not detect ASLR offset")
-				}
-				// Scan this segment for GC roots.
-				// This could be improved by only reading the memory areas
-				// covered by sections. That would reduce the amount of memory
-				// scanned a little bit (up to a single VM page).
-				found(offset+cmd.vmaddr, offset+cmd.vmaddr+cmd.vmsize)
-			}
-		}
-
-		// Move on to the next load command (which may or may not be a
-		// LC_SEGMENT_64).
-		cmd = (*segmentLoadCommand)(unsafe.Add(unsafe.Pointer(cmd), cmd.cmdsize))
-	}
-}
-
 func hardwareRand() (n uint64, ok bool) {
 	n |= uint64(libc_arc4random())
 	n |= uint64(libc_arc4random()) << 32

src/runtime/os_linux.go

@@ -34,101 +34,11 @@ const (
 	sig_SIGSEGV = linux_SIGSEGV
 )
 
-// For the definition of the various header structs, see:
-// https://refspecs.linuxfoundation.org/elf/elf.pdf
-// Also useful:
-// https://en.wikipedia.org/wiki/Executable_and_Linkable_Format
-type elfHeader struct {
-	ident_magic      uint32
-	ident_class      uint8
-	ident_data       uint8
-	ident_version    uint8
-	ident_osabi      uint8
-	ident_abiversion uint8
-	_                [7]byte // reserved
-	filetype         uint16
-	machine          uint16
-	version          uint32
-	entry            uintptr
-	phoff            uintptr
-	shoff            uintptr
-	flags            uint32
-	ehsize           uint16
-	phentsize        uint16
-	phnum            uint16
-	shentsize        uint16
-	shnum            uint16
-	shstrndx         uint16
-}
-
-type elfProgramHeader64 struct {
-	_type  uint32
-	flags  uint32
-	offset uintptr
-	vaddr  uintptr
-	paddr  uintptr
-	filesz uintptr
-	memsz  uintptr
-	align  uintptr
-}
-
-type elfProgramHeader32 struct {
-	_type  uint32
-	offset uintptr
-	vaddr  uintptr
-	paddr  uintptr
-	filesz uintptr
-	memsz  uintptr
-	flags  uint32
-	align  uintptr
-}
-
-// ELF header of the currently running process.
-//
-//go:extern __ehdr_start
-var ehdr_start elfHeader
-
 // int *__errno_location(void);
 //
 //export __errno_location
 func libc_errno_location() *int32
 
-// findGlobals finds globals in the .data/.bss sections.
-// It parses the ELF program header to find writable segments.
-func findGlobals(found func(start, end uintptr)) {
-	// Relevant constants from the ELF specification.
-	// See: https://refspecs.linuxfoundation.org/elf/elf.pdf
-	const (
-		PT_LOAD = 1
-		PF_W    = 0x2 // program flag: write access
-	)
-
-	headerPtr := unsafe.Pointer(uintptr(unsafe.Pointer(&ehdr_start)) + ehdr_start.phoff)
-	for i := 0; i < int(ehdr_start.phnum); i++ {
-		// Look for a writable segment and scan its contents.
-		// There is a little bit of duplication here, which is unfortunate. But
-		// the alternative would be to put elfProgramHeader in separate files
-		// which is IMHO a lot uglier. If only the ELF spec was consistent
-		// between 32-bit and 64-bit...
-		if TargetBits == 64 {
-			header := (*elfProgramHeader64)(headerPtr)
-			if header._type == PT_LOAD && header.flags&PF_W != 0 {
-				start := header.vaddr
-				end := start + header.memsz
-				found(start, end)
-			}
-		} else {
-			header := (*elfProgramHeader32)(headerPtr)
-			if header._type == PT_LOAD && header.flags&PF_W != 0 {
-				start := header.vaddr
-				end := start + header.memsz
-				found(start, end)
-			}
-		}
-		headerPtr = unsafe.Add(headerPtr, ehdr_start.phentsize)
-	}
-}
-
 //export getpagesize
 func libc_getpagesize() int
 

src/runtime/os_windows.go

@@ -6,36 +6,6 @@ const GOOS = "windows"
 
 const zeroSizeAllocPtr uintptr = 16 // part of the first protected page
 
-//export GetModuleHandleExA
-func _GetModuleHandleExA(dwFlags uint32, lpModuleName unsafe.Pointer, phModule **exeHeader) bool
-
-// Mark global variables.
-// Unfortunately, the linker doesn't provide symbols for the start and end of
-// the data/bss sections. Therefore these addresses need to be determined at
-// runtime. This might seem complex and it kind of is, but it only compiles to
-// around 160 bytes of amd64 instructions.
-// Most of this function is based on the documentation in
-// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format.
-func findGlobals(found func(start, end uintptr)) {
-	// Constants used in this function.
-	const (
-		// https://docs.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-getmodulehandleexa
-		GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT = 0x00000002
-	)
-
-	if module == nil {
-		// Obtain a handle to the currently executing image. What we're getting
-		// here is really just __ImageBase, but it's probably better to obtain
-		// it using GetModuleHandle to account for ASLR etc.
-		result := _GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, nil, &module)
-		if gcAsserts && (!result || module.signature != 0x5A4D) { // 0x4D5A is "MZ"
-			runtimeFatal("cannot get module handle")
-		}
-	}
-
-	findGlobalsForPE(found)
-}
-
 type systeminfo struct {
 	anon0                       [4]byte
 	dwpagesize                  uint32

src/runtime/os_windows_pe.go

@@ -1,63 +0,0 @@
-//go:build windows || uefi
-
-package runtime
-
-import "unsafe"
-
-// MS-DOS stub with PE header offset:
-// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#ms-dos-stub-image-only
-type exeHeader struct {
-	signature uint16
-	_         [58]byte // skip DOS header
-	peHeader  uint32   // at offset 0x3C
-}
-
-// COFF file header:
-// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#file-headers
-type peHeader struct {
-	magic                uint32
-	machine              uint16
-	numberOfSections     uint16
-	timeDateStamp        uint32
-	pointerToSymbolTable uint32
-	numberOfSymbols      uint32
-	sizeOfOptionalHeader uint16
-	characteristics      uint16
-}
-
-// COFF section header:
-// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format#section-table-section-headers
-type peSection struct {
-	name                 [8]byte
-	virtualSize          uint32
-	virtualAddress       uint32
-	sizeOfRawData        uint32
-	pointerToRawData     uint32
-	pointerToRelocations uint32
-	pointerToLinenumbers uint32
-	numberOfRelocations  uint16
-	numberOfLinenumbers  uint16
-	characteristics      uint32
-}
-
-var module *exeHeader
-
-func findGlobalsForPE(found func(start, end uintptr)) {
-	// https://docs.microsoft.com/en-us/windows/win32/debug/pe-format
-	const imageSCNMemWrite = 0x80000000
-
-	pe := (*peHeader)(unsafe.Add(unsafe.Pointer(module), module.peHeader))
-	if gcAsserts && pe.magic != 0x00004550 { // 0x4550 is "PE"
-		runtimeFatal("cannot find PE header")
-	}
-
-	section := (*peSection)(unsafe.Pointer(uintptr(unsafe.Pointer(pe)) + uintptr(pe.sizeOfOptionalHeader) + unsafe.Sizeof(peHeader{})))
-	for i := 0; i < int(pe.numberOfSections); i++ {
-		if section.characteristics&imageSCNMemWrite != 0 {
-			start := uintptr(unsafe.Pointer(module)) + uintptr(section.virtualAddress)
-			end := uintptr(unsafe.Pointer(module)) + uintptr(section.virtualAddress) + uintptr(section.virtualSize)
-			found(start, end)
-		}
-		section = (*peSection)(unsafe.Add(unsafe.Pointer(section), unsafe.Sizeof(peSection{})))
-	}
-}

src/runtime/runtime_nintendoswitch.go

@@ -257,31 +257,6 @@ func getHeapEnd() uintptr {
 	return heapEnd
 }
 
-//go:extern __data_start
-var dataStartSymbol [0]byte
-
-//go:extern __data_end
-var dataEndSymbol [0]byte
-
-//go:extern __bss_start
-var bssStartSymbol [0]byte
-
-//go:extern __bss_end
-var bssEndSymbol [0]byte
-
-// Find global variables.
-// The linker script provides __*_start and __*_end symbols that can be used to
-// scan the given sections. They are already aligned so don't need to be
-// manually aligned here.
-func findGlobals(found func(start, end uintptr)) {
-	dataStart := uintptr(unsafe.Pointer(&dataStartSymbol))
-	dataEnd := uintptr(unsafe.Pointer(&dataEndSymbol))
-	found(dataStart, dataEnd)
-	bssStart := uintptr(unsafe.Pointer(&bssStartSymbol))
-	bssEnd := uintptr(unsafe.Pointer(&bssEndSymbol))
-	found(bssStart, bssEnd)
-}
-
 // getContextPtr returns the hblauncher context
 // this is externally linked by gonx
 func getContextPtr() uintptr {

transform/optimizer.go

@@ -58,7 +58,9 @@ func Optimize(mod llvm.Module, config *compileopts.Config) []error {
 			// LLVM 17 doesn't have the no-verify-fixpoint flag.
 			optPasses = "globaldce,globalopt,ipsccp,instcombine,adce,function-attrs"
 		}
+		blockGlobalAllocPromotion(mod)
 		err := mod.RunPasses(optPasses, llvm.TargetMachine{}, po)
+		removeGlobalAllocPromotionMarker(mod)
 		if err != nil {
 			return []error{fmt.Errorf("could not build pass pipeline: %w", err)}
 		}
@@ -80,7 +82,9 @@ func Optimize(mod llvm.Module, config *compileopts.Config) []error {
 		// After interfaces are lowered, there are many more opportunities for
 		// interprocedural optimizations. To get them to work, function
 		// attributes have to be updated first.
+		blockGlobalAllocPromotion(mod)
 		err = mod.RunPasses(optPasses, llvm.TargetMachine{}, po)
+		removeGlobalAllocPromotionMarker(mod)
 		if err != nil {
 			return []error{fmt.Errorf("could not build pass pipeline: %w", err)}
 		}
@@ -162,7 +166,9 @@ func Optimize(mod llvm.Module, config *compileopts.Config) []error {
 	po := llvm.NewPassBuilderOptions()
 	defer po.Dispose()
 	passes := fmt.Sprintf("thinlto-pre-link<%s>", optLevel)
+	blockGlobalAllocPromotion(mod)
 	err := mod.RunPasses(passes, llvm.TargetMachine{}, po)
+	removeGlobalAllocPromotionMarker(mod)
 	if err != nil {
 		return []error{fmt.Errorf("could not build pass pipeline: %w", err)}
 	}
@@ -177,6 +183,64 @@ func Optimize(mod llvm.Module, config *compileopts.Config) []error {
 	return nil
 }
 
+func blockGlobalAllocPromotion(mod llvm.Module) {
+	ctx := mod.Context()
+	ptrType := llvm.PointerType(ctx.Int8Type(), 0)
+	marker := llvm.AddFunction(mod, "tinygo.gc.alloc.marker", llvm.FunctionType(ctx.VoidType(), []llvm.Type{ptrType}, false))
+
+	builder := ctx.NewBuilder()
+	defer builder.Dispose()
+	var marked bool
+	for _, name := range []string{"runtime.alloc", "runtime.alloc_noheap"} {
+		alloc := mod.NamedFunction(name)
+		if alloc.IsNil() {
+			continue
+		}
+		for _, call := range getUses(alloc) {
+			if call.IsACallInst().IsNil() || call.CalledValue() != alloc {
+				continue
+			}
+			if isPointerFreeAllocation(call) {
+				continue
+			}
+
+			// GlobalOpt may otherwise turn this allocation into an untyped
+			// global, hiding its pointer fields from makeGCGlobalRoots.
+			next := llvm.NextInstruction(call)
+			if next.IsNil() {
+				continue
+			}
+			builder.SetInsertPointBefore(next)
+			builder.CreateCall(marker.GlobalValueType(), marker, []llvm.Value{call}, "")
+			marked = true
+		}
+	}
+	if !marked {
+		marker.EraseFromParentAsFunction()
+	}
+}
+
+func isPointerFreeAllocation(call llvm.Value) bool {
+	const noPointerLayout = 3
+
+	layout := call.Operand(1)
+	return !layout.IsAConstantExpr().IsNil() &&
+		layout.Opcode() == llvm.IntToPtr &&
+		!layout.Operand(0).IsAConstantInt().IsNil() &&
+		layout.Operand(0).ZExtValue() == noPointerLayout
+}
+
+func removeGlobalAllocPromotionMarker(mod llvm.Module) {
+	marker := mod.NamedFunction("tinygo.gc.alloc.marker")
+	if marker.IsNil() {
+		return
+	}
+	for _, call := range getUses(marker) {
+		call.EraseFromParentAsInstruction()
+	}
+	marker.EraseFromParentAsFunction()
+}
+
 // functionsUsedInTransform is a list of function symbols that may be used
 // during TinyGo optimization passes so they have to be marked as external
 // linkage until all TinyGo passes have finished.

transform/optimizer_internal_test.go

@@ -0,0 +1,33 @@
+package transform
+
+import (
+	"testing"
+
+	"tinygo.org/x/go-llvm"
+)
+
+func TestBlockGlobalAllocPromotionUses(t *testing.T) {
+	ctx := llvm.NewContext()
+	defer ctx.Dispose()
+	buf, err := llvm.NewMemoryBufferFromFile("testdata/optimizer-alloc-uses.ll")
+	if err != nil {
+		t.Fatal(err)
+	}
+	mod, err := ctx.ParseIR(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer mod.Dispose()
+
+	blockGlobalAllocPromotion(mod)
+	if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil {
+		t.Fatal(err)
+	}
+	marker := mod.NamedFunction("tinygo.gc.alloc.marker")
+	if marker.IsNil() {
+		t.Fatal("allocation marker was not created")
+	}
+	if uses := getUses(marker); len(uses) != 1 {
+		t.Fatalf("got %d marker uses, want 1", len(uses))
+	}
+}

transform/testdata/optimizer-alloc-uses.ll

@@ -0,0 +1,20 @@
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown-wasm"
+
+@allocFunction = constant ptr @runtime.alloc
+
+declare ptr @runtime.alloc(i32, ptr)
+
+declare void @use(ptr)
+
+define void @passAllocator() {
+entry:
+  call void @use(ptr @runtime.alloc)
+  ret void
+}
+
+define ptr @allocate() {
+entry:
+  %allocation = call ptr @runtime.alloc(i32 4, ptr inttoptr (i32 5 to ptr))
+  ret ptr %allocation
+}