runtime: map every goroutine to a new OS thread

This is not a scheduler in the runtime, instead every goroutine is
mapped to a single OS thread - meaning 1:1 scheduling.

While this may not perform well (or at all) for large numbers of
threads, it greatly simplifies many things in the runtime. For example,
blocking syscalls can be called directly instead of having to use epoll
or similar. Also, we don't need to do anything special to call C code -
the default stack is all we need.

src/runtime/gc_blocks.go

@@ -467,8 +467,7 @@ func runGC() (freeBytes uintptr) {
 	}
 
 	// Mark phase: mark all reachable objects, recursively.
-	markStack()
-	findGlobals(markRoots)
+	gcMarkReachable()
 
 	if baremetal && hasScheduler {
 		// Channel operations in interrupts may move task pointers around while we are marking.
@@ -502,6 +501,10 @@ func runGC() (freeBytes uintptr) {
 		finishMark()
 	}
 
+	// If we're using threads, resume all other threads before starting the
+	// sweep.
+	gcResumeWorld()
+
 	// Sweep phase: free all non-marked objects and unmark marked objects for
 	// the next collection cycle.
 	freeBytes = sweep()

src/runtime/gc_boehm.go

@@ -1,5 +1,21 @@
 //go:build gc.boehm
 
+// This is the Boehm-Demers-Weiser conservative garbage collector, integrated
+// into TinyGo.
+//
+// Note that we use a special way of dealing with threads:
+//   * All calls to the bdwgc library are serialized using locks.
+//   * When the bdwgc library wants to push GC roots, all other threads that are
+//     running are stopped.
+//   * After returning from a bdwgc library call, the caller checks whether
+//     other threads were stopped (meaning a GC cycle happened) and resumes the
+//     world.
+// This is not exactly the most efficient way to do this. We can likely speed
+// things up by using bdwgc-native wrappers for starting/stopping threads (and
+// also to resume the world while sweeping). Also, thread local allocation might
+// help. But we don't do any of these right now, it is left as a possible future
+// improvement.
+
 package runtime
 
 import (
@@ -16,6 +32,10 @@ var zeroSizedAlloc uint8
 
 var gcLock task.PMutex
 
+// Normally false, set to true during a GC scan when all other threads get
+// paused.
+var needsResumeWorld bool
+
 func initHeap() {
 	libgc_init()
 
@@ -25,13 +45,16 @@ func initHeap() {
 var gcCallbackPtr = reflectlite.ValueOf(gcCallback).UnsafePointer()
 
 func gcCallback() {
-	// Mark the system stack and (if we're on a goroutine stack) also the
-	// current goroutine stack.
-	markStack()
+	// Mark globals and all stacks, and stop the world if we're using threading.
+	gcMarkReachable()
 
-	findGlobals(func(start, end uintptr) {
-		libgc_push_all(start, end)
-	})
+	if needsResumeWorld {
+		// Should never happen, check for it anyway.
+		runtimePanic("gc: world already stopped")
+	}
+
+	// Note that we need to resume the world after finishing the GC call.
+	needsResumeWorld = true
 }
 
 func markRoots(start, end uintptr) {
@@ -57,6 +80,7 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer {
 	}
 
 	gcLock.Lock()
+	needsResumeWorld = false
 	var ptr unsafe.Pointer
 	if layout == gclayout.NoPtrs {
 		// This object is entirely pointer free, for example make([]int, ...).
@@ -73,6 +97,9 @@ func alloc(size uintptr, layout unsafe.Pointer) unsafe.Pointer {
 		// Memory returned from libgc_malloc has already been zeroed, so nothing
 		// to do here.
 	}
+	if needsResumeWorld {
+		gcResumeWorld()
+	}
 	gcLock.Unlock()
 	if ptr == nil {
 		runtimePanic("gc: out of memory")
@@ -86,7 +113,13 @@ func free(ptr unsafe.Pointer) {
 }
 
 func GC() {
+	gcLock.Lock()
+	needsResumeWorld = false
 	libgc_gcollect()
+	if needsResumeWorld {
+		gcResumeWorld()
+	}
+	gcLock.Unlock()
 }
 
 // This should be stack-allocated, but we don't currently have a good way of

src/runtime/gc_leaking.go

@@ -73,6 +73,10 @@ func free(ptr unsafe.Pointer) {
 	// Memory is never freed.
 }
 
+func markRoots(start, end uintptr) {
+	runtimePanic("unreachable: markRoots")
+}
+
 // ReadMemStats populates m with memory statistics.
 //
 // The returned memory statistics are up to date as of the

src/runtime/gc_none.go

@@ -28,6 +28,10 @@ func GC() {
 	// Unimplemented.
 }
 
+func markRoots(start, end uintptr) {
+	runtimePanic("unreachable: markRoots")
+}
+
 func SetFinalizer(obj interface{}, finalizer interface{}) {
 	// Unimplemented.
 }

src/runtime/gc_stack_portable.go

@@ -8,6 +8,11 @@ import (
 	"unsafe"
 )
 
+func gcMarkReachable() {
+	markStack()
+	findGlobals(markRoots)
+}
+
 //go:extern runtime.stackChainStart
 var stackChainStart *stackChainObject
 
@@ -60,3 +65,7 @@ func trackPointer(ptr, alloca unsafe.Pointer)
 func swapStackChain(dst **stackChainObject) {
 	*dst, stackChainStart = stackChainStart, *dst
 }
+
+func gcResumeWorld() {
+	// Nothing to do here (single threaded).
+}

src/runtime/gc_stack_raw.go

@@ -1,9 +1,14 @@
-//go:build (gc.conservative || gc.precise || gc.boehm) && !tinygo.wasm
+//go:build (gc.conservative || gc.precise || gc.boehm) && !tinygo.wasm && !scheduler.threads
 
 package runtime
 
 import "internal/task"
 
+func gcMarkReachable() {
+	markStack()
+	findGlobals(markRoots)
+}
+
 // markStack marks all root pointers found on the stack.
 //
 // This implementation is conservative and relies on the stack top (provided by
@@ -36,3 +41,7 @@ func scanstack(sp uintptr) {
 		markCurrentGoroutineStack(sp)
 	}
 }
+
+func gcResumeWorld() {
+	// Nothing to do here (single threaded).
+}

src/runtime/gc_stack_threads.go

@@ -0,0 +1,29 @@
+//go:build scheduler.threads
+
+package runtime
+
+import "internal/task"
+
+func gcMarkReachable() {
+	task.GCStopWorldAndScan()
+}
+
+// Scan globals inside the stop-the-world phase. Called from the STW
+// implementation in the internal/task package.
+//
+//go:linkname gcScanGlobals internal/task.gcScanGlobals
+func gcScanGlobals() {
+	findGlobals(markRoots)
+}
+
+// Function called from assembly with all registers pushed, to actually scan the
+// stack.
+//
+//go:export tinygo_scanstack
+func scanstack(sp uintptr) {
+	markRoots(sp, task.StackTop())
+}
+
+func gcResumeWorld() {
+	task.GCResumeWorld()
+}

src/runtime/runtime_unix.go

@@ -73,6 +73,7 @@ type timespec struct {
 	tv_nsec int64 // unsigned 64-bit integer on all time64 platforms
 }
 
+// Highest address of the stack of the main thread.
 var stackTop uintptr
 
 // Entry point for Go. Initialize all packages and call main.main().

src/runtime/scheduler_threads.go

@@ -0,0 +1,124 @@
+//go:build scheduler.threads
+
+package runtime
+
+import "internal/task"
+
+const hasScheduler = false // not using the cooperative scheduler
+
+// We use threads, so yes there is parallelism.
+const hasParallelism = true
+
+var (
+	timerQueueLock    task.PMutex
+	timerQueueStarted bool
+	timerFutex        task.Futex
+)
+
+// Because we just use OS threads, we don't need to do anything special here. We
+// can just initialize everything and run main.main on the main thread.
+func run() {
+	initHeap()
+	task.Init(stackTop)
+	initAll()
+	callMain()
+}
+
+// Pause the current task for a given time.
+//
+//go:linkname sleep time.Sleep
+func sleep(duration int64) {
+	if duration <= 0 {
+		return
+	}
+
+	sleepTicks(nanosecondsToTicks(duration))
+}
+
+func deadlock() {
+	// TODO: exit the thread via pthread_exit.
+	task.Pause()
+}
+
+func scheduleTask(t *task.Task) {
+	t.Resume()
+}
+
+func Gosched() {
+	// Each goroutine runs in a thread, so there's not much we can do here.
+	// There is sched_yield but it's only really intended for realtime
+	// operation, so is probably best not to use.
+}
+
+// Separate goroutine (thread) that runs timer callbacks when they expire.
+func timerRunner() {
+	for {
+		timerQueueLock.Lock()
+
+		if timerQueue == nil {
+			// No timer in the queue, so wait until one becomes available.
+			val := timerFutex.Load()
+			timerQueueLock.Unlock()
+			timerFutex.Wait(val)
+			continue
+		}
+
+		now := ticks()
+		if now < timerQueue.whenTicks() {
+			// There is a timer in the queue, but we need to wait until it
+			// expires.
+			// Using a futex, so that the wait is exited early when adding a new
+			// (sooner-to-expire) timer.
+			val := timerFutex.Load()
+			timerQueueLock.Unlock()
+			timeout := ticksToNanoseconds(timerQueue.whenTicks() - now)
+			timerFutex.WaitUntil(val, uint64(timeout))
+			continue
+		}
+
+		// Pop timer from queue.
+		tn := timerQueue
+		timerQueue = tn.next
+		tn.next = nil
+
+		timerQueueLock.Unlock()
+
+		// Run the callback stored in this timer node.
+		delay := ticksToNanoseconds(now - tn.whenTicks())
+		tn.callback(tn, delay)
+	}
+}
+
+func addTimer(tim *timerNode) {
+	timerQueueLock.Lock()
+
+	if !timerQueueStarted {
+		timerQueueStarted = true
+		go timerRunner()
+	}
+
+	timerQueueAdd(tim)
+
+	timerFutex.Add(1)
+	timerFutex.Wake()
+
+	timerQueueLock.Unlock()
+}
+
+func removeTimer(tim *timer) bool {
+	timerQueueLock.Lock()
+	removed := timerQueueRemove(tim)
+	timerQueueLock.Unlock()
+	return removed
+}
+
+func schedulerRunQueue() *task.Queue {
+	// This function is not actually used, it is only called when hasScheduler
+	// is true. So we can just return nil here.
+	return nil
+}
+
+func runqueueForGC() *task.Queue {
+	// There is only a runqueue when using the cooperative scheduler.
+	return nil
+}