runtime: require explicit GC layouts

This commit is contained in:
Jake Bailey
2026-08-07 10:21:19 -07:00
committed by Ron Evans
parent 1a4cb2032e
commit a7360d5ad3
28 changed files with 279 additions and 108 deletions
+9 -4
View File
@@ -17,10 +17,15 @@ const (
sizeShift = sizeBits + 1
NoPtrs = Layout((0 << sizeShift) | (1 << 1) | 1)
Pointer = Layout((1 << sizeShift) | ((unsafe.Sizeof(unsafe.Pointer(nil)) / ptrAlign) << 1) | 1)
String = Layout((1 << sizeShift) | ((unsafe.Sizeof("") / ptrAlign) << 1) | 1)
Slice = Layout((1 << sizeShift) | ((unsafe.Sizeof([]byte{}) / ptrAlign) << 1) | 1)
NoPtrs = Layout((0 << sizeShift) | (1 << 1) | 1)
Pointer = Layout((1 << sizeShift) | ((unsafe.Sizeof(unsafe.Pointer(nil)) / ptrAlign) << 1) | 1)
PointerPair = Layout((3 << sizeShift) | ((2 * unsafe.Sizeof(unsafe.Pointer(nil)) / ptrAlign) << 1) | 1)
String = Layout((1 << sizeShift) | ((unsafe.Sizeof("") / ptrAlign) << 1) | 1)
Slice = Layout((1 << sizeShift) | ((unsafe.Sizeof([]byte{}) / ptrAlign) << 1) | 1)
// Conservative is reserved for stack storage, which does not have an
// ordinary Go object layout.
Conservative = Layout(2)
)
func (l Layout) AsPtr() unsafe.Pointer { return unsafe.Pointer(l) }
+27 -4
View File
@@ -166,6 +166,11 @@ type RawType struct {
meta uint8 // metadata byte, contains kind and flags (see constants above)
}
type basicType struct {
RawType
ptrTo *RawType
}
// All types that have an element type: named, chan, slice, array, map (but not
// pointer because it doesn't have ptrTo).
type elemType struct {
@@ -200,6 +205,7 @@ type arrayType struct {
elem *RawType
arrayLen uintptr
slicePtr *RawType
layout unsafe.Pointer
}
type mapType struct {
@@ -208,6 +214,7 @@ type mapType struct {
ptrTo *RawType
elem *RawType
key *RawType
typeInfo unsafe.Pointer
}
// namedType is the type descriptor for named types. The numMethod field uses
@@ -243,6 +250,7 @@ type structType struct {
pkgpath *byte
size uint32
numField uint16
layout unsafe.Pointer
fields [1]structField // the remaining fields are all of type structField
// methods methodSet follows after fields, only when numMethod & numMethodHasMethodSet != 0
}
@@ -298,6 +306,8 @@ func pointerTo(t *RawType) *RawType {
}
switch t.Kind() {
case Bool, Int, Int8, Int16, Int32, Int64, Uint, Uint8, Uint16, Uint32, Uint64, Uintptr, Complex64, Complex128, Float32, Float64, String, UnsafePointer:
return (*basicType)(unsafe.Pointer(t)).ptrTo
case Pointer:
if tag := t.ptrtag(); tag < 3 {
return (*RawType)(unsafe.Add(unsafe.Pointer(t), 1))
@@ -306,6 +316,8 @@ func pointerTo(t *RawType) *RawType {
// TODO(dgryski): This is blocking https://github.com/tinygo-org/tinygo/issues/3131
// We need to be able to create types that match existing types to prevent typecode equality.
panic("reflect: cannot make *****T type")
case Interface, Func:
return (*interfaceType)(unsafe.Pointer(t)).ptrTo
case Struct:
return (*structType)(unsafe.Pointer(t)).ptrTo
default:
@@ -729,6 +741,7 @@ func (t *RawType) Align() int {
}
func (r *RawType) gcLayout() unsafe.Pointer {
r = r.underlying()
kind := r.Kind()
if kind < String {
@@ -736,16 +749,26 @@ func (r *RawType) gcLayout() unsafe.Pointer {
}
switch kind {
case Pointer, UnsafePointer, Chan, Map:
return gclayout.Pointer.AsPtr()
case String:
return gclayout.String.AsPtr()
case UnsafePointer, Chan, Pointer, Map:
return gclayout.Pointer.AsPtr()
case Interface, Func:
return gclayout.PointerPair.AsPtr()
case Slice:
return gclayout.Slice.AsPtr()
case Array:
return (*arrayType)(unsafe.Pointer(r)).layout
case Struct:
return (*structType)(unsafe.Pointer(r)).layout
default:
panic("reflect: invalid GC layout kind")
}
}
// Unknown (for now); let the conservative pointer scanning handle it
return nil
func (r *RawType) hashmapTypeInfo() unsafe.Pointer {
r = r.underlying()
return (*mapType)(unsafe.Pointer(r)).typeInfo
}
// FieldAlign returns the alignment if this type is used in a struct field. It
+14 -12
View File
@@ -1,6 +1,7 @@
package reflectlite
import (
"internal/gclayout"
"math"
"unsafe"
)
@@ -1644,7 +1645,7 @@ func makeInt(flags valueFlags, bits uint64, t *RawType) Value {
ptr := unsafe.Pointer(&v.value)
if size > unsafe.Sizeof(uintptr(0)) {
ptr = alloc(size, nil)
ptr = alloc(size, gclayout.NoPtrs.AsPtr())
v.value = ptr
}
@@ -1671,7 +1672,7 @@ func makeFloat(flags valueFlags, f float64, t *RawType) Value {
ptr := unsafe.Pointer(&v.value)
if size > unsafe.Sizeof(uintptr(0)) {
ptr = alloc(size, nil)
ptr = alloc(size, gclayout.NoPtrs.AsPtr())
v.value = ptr
}
@@ -1703,7 +1704,7 @@ func makeComplex(flags valueFlags, f complex128, t *RawType) Value {
ptr := unsafe.Pointer(&v.value)
if size > unsafe.Sizeof(uintptr(0)) {
ptr = alloc(size, nil)
ptr = alloc(size, gclayout.NoPtrs.AsPtr())
v.value = ptr
}
@@ -1834,7 +1835,7 @@ func Zero(typ Type) Value {
return Value{
typecode: typ.(*RawType),
value: alloc(size, nil),
value: alloc(size, typ.(*RawType).gcLayout()),
flags: valueFlagExported | valueFlagRO,
}
}
@@ -1844,7 +1845,7 @@ func Zero(typ Type) Value {
func New(typ Type) Value {
return Value{
typecode: pointerTo(typ.(*RawType)),
value: alloc(typ.Size(), nil),
value: alloc(typ.Size(), typ.(*RawType).gcLayout()),
flags: valueFlagExported,
}
}
@@ -2203,13 +2204,13 @@ func (v Value) FieldByNameFunc(match func(string) bool) Value {
}
//go:linkname hashmapMake runtime.hashmapMake
func hashmapMake(keySize, valueSize uintptr, sizeHint uintptr, alg uint8) unsafe.Pointer
func hashmapMake(keySize, valueSize uintptr, sizeHint uintptr, typeInfo unsafe.Pointer, alg uint8) unsafe.Pointer
//go:linkname hashmapMakeReflect runtime.hashmapMakeReflect
func hashmapMakeReflect(keySize, valueSize, sizeHint uintptr, keyType unsafe.Pointer) unsafe.Pointer
func hashmapMakeReflect(keySize, valueSize, sizeHint uintptr, typeInfo, keyType unsafe.Pointer) unsafe.Pointer
//go:linkname chanMake runtime.chanMake
func chanMake(elementSize uintptr, bufSize uintptr) unsafe.Pointer
func chanMake(elementSize uintptr, bufSize uintptr, elementLayout unsafe.Pointer) unsafe.Pointer
// MakeMapWithSize creates a new map with the specified type and initial space
// for approximately n elements.
@@ -2231,18 +2232,19 @@ func MakeMapWithSize(typ Type, n int) Value {
key := typ.Key().(*RawType)
val := typ.Elem().(*RawType)
typeInfo := typ.(*RawType).hashmapTypeInfo()
var m unsafe.Pointer
if key.Kind() == String {
m = hashmapMake(key.Size(), val.Size(), uintptr(n), hashmapAlgorithmString)
m = hashmapMake(key.Size(), val.Size(), uintptr(n), typeInfo, hashmapAlgorithmString)
} else if key.isBinary() {
m = hashmapMake(key.Size(), val.Size(), uintptr(n), hashmapAlgorithmBinary)
m = hashmapMake(key.Size(), val.Size(), uintptr(n), typeInfo, hashmapAlgorithmBinary)
} else {
// Composite key type (struct with strings, floats, etc.).
// Use runtime-generated hash/equal closures that walk the
// type structure, matching the compiler-generated functions.
m = hashmapMakeReflect(key.Size(), val.Size(), uintptr(n), unsafe.Pointer(key))
m = hashmapMakeReflect(key.Size(), val.Size(), uintptr(n), typeInfo, unsafe.Pointer(key))
}
return Value{
@@ -2269,7 +2271,7 @@ func MakeChan(typ Type, size int) Value {
panic("reflect.MakeChan: unidirectional channel type")
}
elem := typ.Elem().(*RawType)
ch := chanMake(elem.Size(), uintptr(size))
ch := chanMake(elem.Size(), uintptr(size), elem.gcLayout())
return Value{
typecode: typ.(*RawType),
value: ch,
+2 -1
View File
@@ -3,6 +3,7 @@
package task
import (
"internal/gclayout"
"unsafe"
)
@@ -73,7 +74,7 @@ func (s *state) initialize(fn uintptr, args unsafe.Pointer, stackSize uintptr) {
s.args = args
// Create a stack.
stack := runtime_alloc(stackSize, nil)
stack := runtime_alloc(stackSize, gclayout.Conservative.AsPtr())
// Set up the stack canary, a random number that should be checked when
// switching from the task back to the scheduler. The stack canary pointer
+2 -1
View File
@@ -3,6 +3,7 @@
package task
import (
"internal/gclayout"
"unsafe"
)
@@ -36,7 +37,7 @@ func taskExit() {
// initialize the state and prepare to call the specified function with the specified argument bundle.
func (s *state) initialize(fn uintptr, args unsafe.Pointer, stackSize uintptr) {
// Create a stack.
stack := runtime_alloc(stackSize, nil)
stack := runtime_alloc(stackSize, gclayout.Conservative.AsPtr())
// Set up the stack canary, a random number that should be checked when
// switching from the task back to the scheduler. The stack canary pointer
+30
View File
@@ -994,6 +994,20 @@ func TestTypeAssertPanic(t *testing.T) {
})
}
type tinyMakeChanElement struct {
ptr *int
text string
}
var tinyMakeChanChurn []*int
//go:noinline
func fillTinyMakeChan(ch chan tinyMakeChanElement) {
value := new(int)
*value = 42
ch <- tinyMakeChanElement{ptr: value, text: "hello"}
}
func TestTinyMakeChan(t *testing.T) {
// Value.Send and Value.Recv are not implemented yet, so the channel is
// exercised through Interface(): that proves MakeChan returns a working
@@ -1027,6 +1041,22 @@ func TestTinyMakeChan(t *testing.T) {
}
})
t.Run("buffered pointers survive GC", func(t *testing.T) {
v := MakeChan(TypeOf(make(chan tinyMakeChanElement)), 1)
ch := v.Interface().(chan tinyMakeChanElement)
fillTinyMakeChan(ch)
runtime.GC()
tinyMakeChanChurn = make([]*int, 128)
for i := range tinyMakeChanChurn {
tinyMakeChanChurn[i] = new(int)
}
got := <-ch
if *got.ptr != 42 || got.text != "hello" {
t.Errorf("<-ch=%v, want {42 hello}", got)
}
})
t.Run("unbuffered", func(t *testing.T) {
v := MakeChan(TypeOf(make(chan string)), 0)
if got, want := v.Cap(), 0; got != want {
+6 -3
View File
@@ -2,7 +2,10 @@
package runtime
import "unsafe"
import (
"internal/gclayout"
"unsafe"
)
// The below functions override the default allocator of wasi-libc. This ensures
// code linked from other languages can allocate memory without colliding with
@@ -21,7 +24,7 @@ func libc_malloc(size uintptr) unsafe.Pointer {
if size == 0 {
return nil
}
ptr := alloc(size, nil)
ptr := alloc(size, gclayout.NoPtrs.AsPtr())
allocs[(*byte)(ptr)] = size
return ptr
}
@@ -54,7 +57,7 @@ func libc_realloc(oldPtr unsafe.Pointer, size uintptr) unsafe.Pointer {
// It's hard to optimize this to expand the current buffer with our GC, but
// it is theoretically possible. For now, just always allocate fresh.
// TODO: we could skip this if the new allocation is smaller than the old.
ptr := alloc(size, nil)
ptr := alloc(size, gclayout.NoPtrs.AsPtr())
if oldPtr != nil {
if oldSize, ok := allocs[(*byte)(oldPtr)]; ok {
+2 -1
View File
@@ -3,6 +3,7 @@
package runtime
import (
"internal/gclayout"
"sync/atomic"
"unsafe"
)
@@ -11,7 +12,7 @@ import (
func libc_malloc(size uintptr) unsafe.Pointer {
// Note: this zeroes the returned buffer which is not necessary.
// The same goes for bytealg.MakeNoZero.
return alloc(size, nil)
return alloc(size, gclayout.NoPtrs.AsPtr())
}
//export calloc
+2 -2
View File
@@ -137,11 +137,11 @@ type chanSelectState struct {
value unsafe.Pointer
}
func chanMake(elementSize uintptr, bufSize uintptr) *channel {
func chanMake(elementSize uintptr, bufSize uintptr, elementLayout unsafe.Pointer) *channel {
return &channel{
elementSize: elementSize,
bufCap: bufSize,
buf: alloc(elementSize*bufSize, nil),
buf: alloc(elementSize*bufSize, elementLayout),
}
}
+3 -2
View File
@@ -31,6 +31,7 @@ package runtime
// Moss.
import (
"internal/gclayout"
"internal/reflectlite"
"internal/task"
"runtime/interrupt"
@@ -501,7 +502,7 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer {
func realloc(ptr unsafe.Pointer, size uintptr) unsafe.Pointer {
if ptr == nil {
return alloc(size, nil)
return alloc(size, gclayout.NoPtrs.AsPtr())
}
// Find the first block of the original allocation.
@@ -526,7 +527,7 @@ func realloc(ptr unsafe.Pointer, size uintptr) unsafe.Pointer {
}
// Create a new allocation and copy the old data.
newAlloc := alloc(size, nil)
newAlloc := alloc(size, gclayout.NoPtrs.AsPtr())
memcpy(newAlloc, ptr, oldSize)
free(ptr)
+2 -1
View File
@@ -7,6 +7,7 @@ package runtime
// may be the only memory allocator possible.
import (
"internal/gclayout"
"internal/task"
"sync/atomic"
"unsafe"
@@ -69,7 +70,7 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer {
}
func realloc(ptr unsafe.Pointer, size uintptr) unsafe.Pointer {
newAlloc := alloc(size, nil)
newAlloc := alloc(size, gclayout.NoPtrs.AsPtr())
if ptr == nil {
return newAlloc
}
+5 -4
View File
@@ -55,7 +55,10 @@
package runtime
import "unsafe"
import (
"internal/gclayout"
"unsafe"
)
const sizeFieldBits = 4 + (unsafe.Sizeof(uintptr(0)) / 4)
@@ -76,9 +79,7 @@ func (layout gcLayout) pointerFree() bool {
// The length is rounded down to a multiple of the element size.
func (layout gcLayout) scan(start, len uintptr) {
switch {
case layout == 0:
// This is an unknown layout.
// Scan conservatively.
case layout == gcLayout(gclayout.Conservative):
// NOTE: This is *NOT* equivalent to a slice of pointers on AVR.
scanConservative(start, len)
+27 -12
View File
@@ -14,6 +14,7 @@ import (
// The underlying hashmap structure for Go.
type hashmap struct {
buckets unsafe.Pointer // pointer to array of buckets
typeInfo *hashmapTypeInfo
seed uintptr
count uintptr
keySize uintptr
@@ -26,6 +27,17 @@ type hashmap struct {
keyHash func(key unsafe.Pointer, size, seed uintptr) uint32
}
type hashmapTypeInfo struct {
keyLayout unsafe.Pointer
valueLayout unsafe.Pointer
bucketLayout unsafe.Pointer
}
//go:inline
func hashmapType(m *hashmap) *hashmapTypeInfo {
return m.typeInfo
}
const (
hashmapMaxKeySize = 128
hashmapMaxValueSize = 128
@@ -113,7 +125,7 @@ func hashmapTopHash(hash uint32) uint8 {
}
// Create a new hashmap with the given keySize and valueSize.
func hashmapMake(keySize, valueSize uintptr, sizeHint uintptr, alg uint8) *hashmap {
func hashmapMake(keySize, valueSize uintptr, sizeHint uintptr, typeInfo unsafe.Pointer, alg uint8) *hashmap {
bucketBits := uint8(0)
for hashmapHasSpaceToGrow(bucketBits) && hashmapOverLoadFactor(sizeHint, bucketBits) {
bucketBits++
@@ -132,13 +144,14 @@ func hashmapMake(keySize, valueSize uintptr, sizeHint uintptr, alg uint8) *hashm
}
bucketBufSize := hashmapBucketHeaderSize + keySlotSize*8 + valueSlotSize*8
buckets := alloc(bucketBufSize*(1<<bucketBits), nil)
buckets := alloc(bucketBufSize*(1<<bucketBits), (*hashmapTypeInfo)(typeInfo).bucketLayout)
keyHash := hashmapKeyHashAlg(tinygo.HashmapAlgorithm(alg))
keyEqual := hashmapKeyEqualAlg(tinygo.HashmapAlgorithm(alg))
return &hashmap{
buckets: buckets,
typeInfo: (*hashmapTypeInfo)(typeInfo),
seed: uintptr(fastrand()),
keySize: keySize,
valueSize: valueSize,
@@ -325,7 +338,7 @@ func hashmapSet(m *hashmap, key unsafe.Pointer, value unsafe.Pointer, hash uint3
//go:inline
func hashmapStoreKey(m *hashmap, slotKey, key unsafe.Pointer) {
if m.flags&hashmapFlagIndirectKey != 0 {
p := alloc(m.keySize, nil)
p := alloc(m.keySize, hashmapType(m).keyLayout)
memcpy(p, key, m.keySize)
*(*unsafe.Pointer)(slotKey) = p
} else {
@@ -343,7 +356,7 @@ func hashmapStoreValue(m *hashmap, slotValue, value unsafe.Pointer) {
p := *(*unsafe.Pointer)(slotValue)
if p == nil {
// First insert: allocate backing storage.
p = alloc(m.valueSize, nil)
p = alloc(m.valueSize, hashmapType(m).valueLayout)
*(*unsafe.Pointer)(slotValue) = p
}
memcpy(p, value, m.valueSize)
@@ -356,7 +369,7 @@ func hashmapStoreValue(m *hashmap, slotValue, value unsafe.Pointer) {
// value into the bucket, and returns a pointer to this bucket.
func hashmapInsertIntoNewBucket(m *hashmap, key, value unsafe.Pointer, tophash uint8) *hashmapBucket {
bucketBufSize := hashmapBucketSize(m)
bucketBuf := alloc(bucketBufSize, nil)
bucketBuf := alloc(bucketBufSize, hashmapType(m).bucketLayout)
bucket := (*hashmapBucket)(bucketBuf)
// Insert into the first slot, which is empty as it has just been allocated.
@@ -392,13 +405,13 @@ func hashmapCopy(m *hashmap, sizeBits uint8) hashmap {
n.bucketBits = sizeBits
numBuckets := uintptr(1) << n.bucketBits
bucketBufSize := hashmapBucketSize(m)
n.buckets = alloc(bucketBufSize*numBuckets, nil)
n.buckets = alloc(bucketBufSize*numBuckets, hashmapType(m).bucketLayout)
// use a hashmap iterator to go through the old map
var it hashmapIterator
var key = alloc(m.keySize, nil)
var value = alloc(m.valueSize, nil)
var key = alloc(m.keySize, hashmapType(m).keyLayout)
var value = alloc(m.valueSize, hashmapType(m).valueLayout)
for hashmapNext(m, &it, key, value) {
h := n.keyHash(key, uintptr(n.keySize), n.seed)
@@ -624,6 +637,7 @@ func hashmapGenericDelete(m *hashmap, key unsafe.Pointer) {
// equal functions. This avoids the interface/reflection path for composite
// key types like structs containing strings.
func hashmapMakeGeneric(keySize, valueSize uintptr, sizeHint uintptr,
typeInfo unsafe.Pointer,
keyHash func(key unsafe.Pointer, size, seed uintptr) uint32,
keyEqual func(x, y unsafe.Pointer, n uintptr) bool) *hashmap {
bucketBits := uint8(0)
@@ -644,10 +658,11 @@ func hashmapMakeGeneric(keySize, valueSize uintptr, sizeHint uintptr,
}
bucketBufSize := hashmapBucketHeaderSize + keySlotSize*8 + valueSlotSize*8
buckets := alloc(bucketBufSize*(1<<bucketBits), nil)
buckets := alloc(bucketBufSize*(1<<bucketBits), (*hashmapTypeInfo)(typeInfo).bucketLayout)
return &hashmap{
buckets: buckets,
typeInfo: (*hashmapTypeInfo)(typeInfo),
seed: uintptr(fastrand()),
keySize: keySize,
valueSize: valueSize,
@@ -663,12 +678,12 @@ func hashmapMakeGeneric(keySize, valueSize uintptr, sizeHint uintptr,
// hashmapMakeReflect creates a hashmap for reflect.MakeMapWithSize using
// closures that reconstruct interface{} values from raw key bytes,
// delegating to hashmapInterfaceHash for hashing and == for equality.
func hashmapMakeReflect(keySize, valueSize, sizeHint uintptr, keyType unsafe.Pointer) *hashmap {
func hashmapMakeReflect(keySize, valueSize, sizeHint uintptr, typeInfo, keyType unsafe.Pointer) *hashmap {
t := (*reflectlite.RawType)(keyType)
if t.Kind() == reflectlite.Interface {
// Interface keys are already stored as interface values in the
// bucket; use the existing interface hash/equal directly.
return hashmapMakeGeneric(keySize, valueSize, sizeHint,
return hashmapMakeGeneric(keySize, valueSize, sizeHint, typeInfo,
hashmapInterfacePtrHash, hashmapInterfaceEqual)
}
keyHash := func(key unsafe.Pointer, size, seed uintptr) uint32 {
@@ -677,7 +692,7 @@ func hashmapMakeReflect(keySize, valueSize, sizeHint uintptr, keyType unsafe.Poi
keyEqual := func(x, y unsafe.Pointer, n uintptr) bool {
return rawToInterface(t, x) == rawToInterface(t, y)
}
return hashmapMakeGeneric(keySize, valueSize, sizeHint, keyHash, keyEqual)
return hashmapMakeGeneric(keySize, valueSize, sizeHint, typeInfo, keyHash, keyEqual)
}
// rawToInterface reconstructs an interface{} from raw bytes at ptr.