diff --git a/builder/sizes_test.go b/builder/sizes_test.go index c1d44f443..7349d5757 100644 --- a/builder/sizes_test.go +++ b/builder/sizes_test.go @@ -42,9 +42,9 @@ func TestBinarySize(t *testing.T) { // This is a small number of very diverse targets that we want to test. tests := []sizeTest{ // microcontrollers - {"hifive1b", "examples/echo", 3896, 280, 0, 2268}, - {"microbit", "examples/serial", 2860, 360, 8, 2272}, - {"wioterminal", "examples/pininterrupt", 7361, 1491, 116, 6912}, + {"hifive1b", "examples/echo", 3756, 280, 0, 2268}, + {"microbit", "examples/serial", 2756, 340, 8, 2272}, + {"wioterminal", "examples/pininterrupt", 7297, 1491, 116, 6912}, // TODO: also check wasm. Right now this is difficult, because // wasm binaries are run through wasm-opt and therefore the diff --git a/src/runtime/gc_blocks.go b/src/runtime/gc_blocks.go index 408656b25..2ee7c9d2f 100644 --- a/src/runtime/gc_blocks.go +++ b/src/runtime/gc_blocks.go @@ -46,11 +46,11 @@ const ( bytesPerBlock = wordsPerBlock * unsafe.Sizeof(heapStart) stateBits = 2 // how many bits a block state takes (see blockState type) blocksPerStateByte = 8 / stateBits - markStackSize = 8 * unsafe.Sizeof((*int)(nil)) // number of to-be-marked blocks to queue before forcing a rescan ) var ( metadataStart unsafe.Pointer // pointer to the start of the heap metadata + scanList *objHeader // scanList is a singly linked list of heap objects that have been marked but not scanned nextAlloc gcBlock // the next block that should be tried by the allocator endBlock gcBlock // the block just past the end of the available space gcTotalAlloc uint64 // total number of bytes allocated @@ -225,6 +225,15 @@ func (b gcBlock) unmark() { } } +// objHeader is a structure prepended to every heap object to hold metadata. +type objHeader struct { + // next is the next object to scan after this. + next *objHeader + + // layout holds the layout bitmap used to find pointers in the object. + layout gcLayout +} + func isOnHeap(ptr uintptr) bool { return ptr >= heapStart && ptr < uintptr(metadataStart) } @@ -315,13 +324,10 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer { runtimePanicAt(returnAddress(0), "heap alloc in interrupt") } - // Round the size up to a multiple of blocks. + // Round the size up to a multiple of blocks, adding space for the header. rawSize := size + size += align(unsafe.Sizeof(objHeader{})) size += bytesPerBlock - 1 - if preciseHeap { - // Add space for the layout. - size += align(unsafe.Sizeof(layout)) - } if size < rawSize { // The size overflowed. runtimePanicAt(returnAddress(0), "out of memory") @@ -414,20 +420,18 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer { i.setState(blockStateTail) } + // Create the object header. + pointer := thisAlloc.pointer() + header := (*objHeader)(pointer) + header.layout = parseGCLayout(layout) + // We've claimed this allocation, now we can unlock the heap. gcLock.Unlock() // Return a pointer to this allocation. - pointer := thisAlloc.pointer() - if preciseHeap { - // Store the object layout at the start of the object. - // TODO: this wastes a little bit of space on systems with - // larger-than-pointer alignment requirements. - *(*unsafe.Pointer)(pointer) = layout - add := align(unsafe.Sizeof(layout)) - pointer = unsafe.Add(pointer, add) - size -= add - } + add := align(unsafe.Sizeof(objHeader{})) + pointer = unsafe.Add(pointer, add) + size -= add memzero(pointer, size) return pointer } @@ -562,42 +566,33 @@ func markCurrentGoroutineStack(sp uintptr) { markRoot(0, sp) } -// stackOverflow is a flag which is set when the GC scans too deep while marking. -// After it is set, all marked allocations must be re-scanned. -var stackOverflow bool - -// startMark starts the marking process on a root and all of its children. -func startMark(root gcBlock) { - var stack [markStackSize]gcBlock - stack[0] = root - root.setState(blockStateMark) - stackLen := 1 - for stackLen > 0 { - // Pop a block off of the stack. - stackLen-- - block := stack[stackLen] - if gcDebug { - println("stack popped, remaining stack:", stackLen) +// finishMark finishes the marking process by scanning all heap objects on scanList. +func finishMark() { + for { + // Remove an object from the scan list. + obj := scanList + if obj == nil { + return } + scanList = obj.next - // Scan all pointers inside the block. - scanner := newGCObjectScanner(block) + // Create a scanner with the object layout. + scanner := obj.layout.scanner() if scanner.pointerFree() { // This object doesn't contain any pointers. // This is a fast path for objects like make([]int, 4096). continue } - start, end := block.address(), block.findNext().address() - if preciseHeap { - // The first word of the object is just the pointer layout value. - // Skip it. - start += align(unsafe.Sizeof(uintptr(0))) - } + + // Scan all pointers in the object. + start := uintptr(unsafe.Pointer(obj)) + align(unsafe.Sizeof(objHeader{})) + end := blockFromAddr(uintptr(unsafe.Pointer(obj))).findNext().address() + for addr := start; addr != end; addr += unsafe.Alignof(addr) { // Load the word. word := *(*uintptr)(unsafe.Pointer(addr)) - if !scanner.nextIsPointer(word, root.address(), addr) { + if !scanner.nextIsPointer(word, uintptr(unsafe.Pointer(obj)), addr) { // Not a heap pointer. continue } @@ -628,58 +623,46 @@ func startMark(root gcBlock) { } referencedBlock.setState(blockStateMark) - if stackLen == len(stack) { - // The stack is full. - // It is necessary to rescan all marked blocks once we are done. - stackOverflow = true - if gcDebug { - println("gc stack overflowed") - } - continue - } - - // Push the pointer onto the stack to be scanned later. - stack[stackLen] = referencedBlock - stackLen++ - } - } -} - -// finishMark finishes the marking process by processing all stack overflows. -func finishMark() { - for stackOverflow { - // Re-mark all blocks. - stackOverflow = false - for block := gcBlock(0); block < endBlock; block++ { - if block.state() != blockStateMark { - // Block is not marked, so we do not need to rescan it. - continue - } - - // Re-mark the block. - startMark(block) + // Add the object to the scan list. + header := (*objHeader)(referencedBlock.pointer()) + header.next = scanList + scanList = header } } } // mark a GC root at the address addr. func markRoot(addr, root uintptr) { - if isOnHeap(root) { - block := blockFromAddr(root) - if block.state() == blockStateFree { - // The to-be-marked object doesn't actually exist. - // This could either be a dangling pointer (oops!) but most likely - // just a false positive. - return - } - head := block.findHead() - if head.state() != blockStateMark { - if gcDebug { - println("found unmarked pointer", root, "at address", addr) - } - startMark(head) - } + // Find the heap block corresponding to the root. + if !isOnHeap(root) { + // This is not a heap pointer. + return } + block := blockFromAddr(root) + + // Find the head of the corresponding object. + if block.state() == blockStateFree { + // The to-be-marked object doesn't actually exist. + // This could either be a dangling pointer (oops!) but most likely + // just a false positive. + return + } + head := block.findHead() + + // Mark the object. + if head.state() == blockStateMark { + // This object is already marked. + return + } + if gcDebug { + println("found unmarked pointer", root, "at address", addr) + } + head.setState(blockStateMark) + + // Add the object to the scan list. + header := (*objHeader)(head.pointer()) + header.next = scanList + scanList = header } // Sweep goes through all memory and frees unmarked memory. diff --git a/src/runtime/gc_conservative.go b/src/runtime/gc_conservative.go index 90e5cb098..f7b874370 100644 --- a/src/runtime/gc_conservative.go +++ b/src/runtime/gc_conservative.go @@ -6,15 +6,27 @@ package runtime -const preciseHeap = false +import "unsafe" -type gcObjectScanner struct { +// gcLayout tracks pointer locations in a heap object. +// The conservative GC treats all locations as potential pointers, so this doesn't need to store anything. +type gcLayout struct { } -func newGCObjectScanner(block gcBlock) gcObjectScanner { +// parseGCLayout stores the layout information passed to alloc into a gcLayout value. +// The conservative GC discards this information. +func parseGCLayout(layout unsafe.Pointer) gcLayout { + return gcLayout{} +} + +// scanner creates a gcObjectScanner with this layout. +func (l gcLayout) scanner() gcObjectScanner { return gcObjectScanner{} } +type gcObjectScanner struct { +} + func (scanner *gcObjectScanner) pointerFree() bool { // We don't know whether this object contains pointers, so conservatively // return false. diff --git a/src/runtime/gc_precise.go b/src/runtime/gc_precise.go index aa716585c..96f813f58 100644 --- a/src/runtime/gc_precise.go +++ b/src/runtime/gc_precise.go @@ -59,19 +59,19 @@ import "unsafe" const preciseHeap = true -type gcObjectScanner struct { - index uintptr - size uintptr - bitmap uintptr - bitmapAddr unsafe.Pointer +// parseGCLayout stores the layout information passed to alloc into a gcLayout value. +func parseGCLayout(layout unsafe.Pointer) gcLayout { + return gcLayout{layout: uintptr(layout)} } -func newGCObjectScanner(block gcBlock) gcObjectScanner { - if gcAsserts && block != block.findHead() { - runtimePanic("gc: object scanner must start at head") - } - scanner := gcObjectScanner{} - layout := *(*uintptr)(unsafe.Pointer(block.address())) +// gcLayout tracks pointer locations in a heap object. +type gcLayout struct { + layout uintptr +} + +// scanner creates a gcObjectScanner with this layout. +func (l gcLayout) scanner() (scanner gcObjectScanner) { + layout := l.layout if layout == 0 { // Unknown layout. Assume all words in the object could be pointers. // This layout value below corresponds to a slice of pointers like: @@ -104,7 +104,14 @@ func newGCObjectScanner(block gcBlock) gcObjectScanner { scanner.size = *(*uintptr)(layoutAddr) scanner.bitmapAddr = unsafe.Add(layoutAddr, unsafe.Sizeof(uintptr(0))) } - return scanner + return +} + +type gcObjectScanner struct { + index uintptr + size uintptr + bitmap uintptr + bitmapAddr unsafe.Pointer } func (scanner *gcObjectScanner) pointerFree() bool {