all: add support for multicore scheduler

This commit adds support for a scheduler that runs a scheduler on all
available cores. It is meant to be used on baremetal systems with a
fixed number of cores, such as the RP2040.

The initial implementation adds support for multicore scheduling to the
riscv-qemu target as a convenient testing target. This means that this
new multicore scheduler is tested in CI, including a bunch of standard
library tests (`make tinygo-test-baremetal`). This should ensure the new
scheduler is reasonably well tested before trying to use it on
harder-to-debug targets like the RP2040.

src/internal/task/atomic-cooperative.go

@@ -1,4 +1,4 @@
-//go:build !scheduler.threads
+//go:build tinygo.unicore
 
 package task
 

src/internal/task/atomic-preemptive.go

@@ -1,4 +1,4 @@
-//go:build scheduler.threads
+//go:build !tinygo.unicore
 
 package task
 

src/internal/task/futex-cooperative.go

@@ -1,4 +1,4 @@
-//go:build !scheduler.threads
+//go:build tinygo.unicore
 
 package task
 

src/internal/task/futex-cores.go

@@ -0,0 +1,64 @@
+//go:build scheduler.cores
+
+package task
+
+import "runtime/interrupt"
+
+// A futex is a way for userspace to wait with the pointer as the key, and for
+// another thread to wake one or all waiting threads keyed on the same pointer.
+//
+// A futex does not change the underlying value, it only reads it before to prevent
+// lost wake-ups.
+type Futex struct {
+	Uint32
+
+	waiters Stack
+}
+
+// Atomically check for cmp to still be equal to the futex value and if so, go
+// to sleep. Return true if we were definitely awoken by a call to Wake or
+// WakeAll, and false if we can't be sure of that.
+func (f *Futex) Wait(cmp uint32) (awoken bool) {
+	mask := lockFutex()
+
+	if f.Uint32.Load() != cmp {
+		unlockFutex(mask)
+		return false
+	}
+
+	// Push the current goroutine onto the waiter stack.
+	f.waiters.Push(Current())
+
+	unlockFutex(mask)
+
+	// Pause until this task is awoken by Wake/WakeAll.
+	Pause()
+
+	// We were awoken by a call to Wake or WakeAll. There is no chance for
+	// spurious wakeups.
+	return true
+}
+
+// Wake a single waiter.
+func (f *Futex) Wake() {
+	mask := lockFutex()
+	if t := f.waiters.Pop(); t != nil {
+		scheduleTask(t)
+	}
+	unlockFutex(mask)
+}
+
+// Wake all waiters.
+func (f *Futex) WakeAll() {
+	mask := lockFutex()
+	for t := f.waiters.Pop(); t != nil; t = f.waiters.Pop() {
+		scheduleTask(t)
+	}
+	unlockFutex(mask)
+}
+
+//go:linkname lockFutex runtime.lockFutex
+func lockFutex() interrupt.State
+
+//go:linkname unlockFutex runtime.unlockFutex
+func unlockFutex(interrupt.State)

src/internal/task/mutex-cooperative.go

@@ -1,4 +1,4 @@
-//go:build !scheduler.threads
+//go:build tinygo.unicore
 
 package task
 

src/internal/task/mutex-preemptive.go

@@ -1,4 +1,4 @@
-//go:build scheduler.threads
+//go:build !tinygo.unicore
 
 package task
 

src/internal/task/pmutex-cooperative.go

@@ -1,4 +1,4 @@
-//go:build !scheduler.threads
+//go:build tinygo.unicore
 
 package task
 

src/internal/task/pmutex-preemptive.go

@@ -1,4 +1,4 @@
-//go:build scheduler.threads
+//go:build !tinygo.unicore
 
 package task
 

src/internal/task/queue.go

@@ -12,9 +12,9 @@ type Queue struct {
 
 // Push a task onto the queue.
 func (q *Queue) Push(t *Task) {
-	i := interrupt.Disable()
+	mask := lockAtomics()
 	if asserts && t.Next != nil {
-		interrupt.Restore(i)
+		unlockAtomics(mask)
 		panic("runtime: pushing a task to a queue with a non-nil Next pointer")
 	}
 	if q.tail != nil {
@@ -25,15 +25,15 @@ func (q *Queue) Push(t *Task) {
 	if q.head == nil {
 		q.head = t
 	}
-	interrupt.Restore(i)
+	unlockAtomics(mask)
 }
 
 // Pop a task off of the queue.
 func (q *Queue) Pop() *Task {
-	i := interrupt.Disable()
+	mask := lockAtomics()
 	t := q.head
 	if t == nil {
-		interrupt.Restore(i)
+		unlockAtomics(mask)
 		return nil
 	}
 	q.head = t.Next
@@ -41,13 +41,13 @@ func (q *Queue) Pop() *Task {
 		q.tail = nil
 	}
 	t.Next = nil
-	interrupt.Restore(i)
+	unlockAtomics(mask)
 	return t
 }
 
 // Append pops the contents of another queue and pushes them onto the end of this queue.
 func (q *Queue) Append(other *Queue) {
-	i := interrupt.Disable()
+	mask := lockAtomics()
 	if q.head == nil {
 		q.head = other.head
 	} else {
@@ -55,14 +55,14 @@ func (q *Queue) Append(other *Queue) {
 	}
 	q.tail = other.tail
 	other.head, other.tail = nil, nil
-	interrupt.Restore(i)
+	unlockAtomics(mask)
 }
 
 // Empty checks if the queue is empty.
 func (q *Queue) Empty() bool {
-	i := interrupt.Disable()
+	mask := lockAtomics()
 	empty := q.head == nil
-	interrupt.Restore(i)
+	unlockAtomics(mask)
 	return empty
 }
 
@@ -75,24 +75,24 @@ type Stack struct {
 
 // Push a task onto the stack.
 func (s *Stack) Push(t *Task) {
-	i := interrupt.Disable()
+	mask := lockAtomics()
 	if asserts && t.Next != nil {
-		interrupt.Restore(i)
+		unlockAtomics(mask)
 		panic("runtime: pushing a task to a stack with a non-nil Next pointer")
 	}
 	s.top, t.Next = t, s.top
-	interrupt.Restore(i)
+	unlockAtomics(mask)
 }
 
 // Pop a task off of the stack.
 func (s *Stack) Pop() *Task {
-	i := interrupt.Disable()
+	mask := lockAtomics()
 	t := s.top
 	if t != nil {
 		s.top = t.Next
 		t.Next = nil
 	}
-	interrupt.Restore(i)
+	unlockAtomics(mask)
 	return t
 }
 
@@ -112,13 +112,26 @@ func (t *Task) tail() *Task {
 // Queue moves the contents of the stack into a queue.
 // Elements can be popped from the queue in the same order that they would be popped from the stack.
 func (s *Stack) Queue() Queue {
-	i := interrupt.Disable()
+	mask := lockAtomics()
 	head := s.top
 	s.top = nil
 	q := Queue{
 		head: head,
 		tail: head.tail(),
 	}
-	interrupt.Restore(i)
+	unlockAtomics(mask)
 	return q
 }
+
+// Use runtime.lockAtomics and runtime.unlockAtomics so that Queue and Stack
+// work correctly even on multicore systems. These functions are normally used
+// to implement atomic operations, but the same spinlock can also be used for
+// Queue/Stack operations which are very fast.
+// These functions are just plain old interrupt disable/restore on non-multicore
+// systems.
+
+//go:linkname lockAtomics runtime.lockAtomics
+func lockAtomics() interrupt.State
+
+//go:linkname unlockAtomics runtime.unlockAtomics
+func unlockAtomics(mask interrupt.State)

src/internal/task/task.go

@@ -24,11 +24,28 @@ type Task struct {
 	// This is needed for some crypto packages.
 	FipsIndicator uint8
 
+	// State of the goroutine: running, paused, or must-resume-next-pause.
+	// This extra field doesn't increase memory usage on 32-bit CPUs and above,
+	// since it falls into the padding of the FipsIndicator bit above.
+	RunState uint8
+
 	// DeferFrame stores a pointer to the (stack allocated) defer frame of the
 	// goroutine that is used for the recover builtin.
 	DeferFrame unsafe.Pointer
 }
 
+const (
+	// Initial state: the goroutine state is saved on the stack.
+	RunStatePaused = iota
+
+	// The goroutine is running right now.
+	RunStateRunning
+
+	// The goroutine is running, but already marked as "can resume".
+	// The next call to Pause() won't actually pause the goroutine.
+	RunStateResuming
+)
+
 // DataUint32 returns the Data field as a uint32. The value is only valid after
 // setting it through SetDataUint32 or by storing to it using DataAtomicUint32.
 func (t *Task) DataUint32() uint32 {

src/internal/task/task_stack.go

@@ -1,9 +1,8 @@
-//go:build scheduler.tasks
+//go:build scheduler.tasks || scheduler.cores
 
 package task
 
 import (
-	"runtime/interrupt"
 	"unsafe"
 )
 
@@ -32,44 +31,12 @@ type state struct {
 	canaryPtr *uintptr
 }
 
-// currentTask is the current running task, or nil if currently in the scheduler.
-var currentTask *Task
-
-// Current returns the current active task.
-func Current() *Task {
-	return currentTask
-}
-
-// Pause suspends the current task and returns to the scheduler.
-// This function may only be called when running on a goroutine stack, not when running on the system stack or in an interrupt.
-func Pause() {
-	// Check whether the canary (the lowest address of the stack) is still
-	// valid. If it is not, a stack overflow has occurred.
-	if *currentTask.state.canaryPtr != stackCanary {
-		runtimePanic("goroutine stack overflow")
-	}
-	if interrupt.In() {
-		runtimePanic("blocked inside interrupt")
-	}
-	currentTask.state.pause()
-}
-
 //export tinygo_task_exit
 func taskExit() {
 	// TODO: explicitly free the stack after switching back to the scheduler.
 	Pause()
 }
 
-// Resume the task until it pauses or completes.
-// This may only be called from the scheduler.
-func (t *Task) Resume() {
-	currentTask = t
-	t.gcData.swap()
-	t.state.resume()
-	t.gcData.swap()
-	currentTask = nil
-}
-
 // initialize the state and prepare to call the specified function with the specified argument bundle.
 func (s *state) initialize(fn uintptr, args unsafe.Pointer, stackSize uintptr) {
 	// Create a stack.

src/internal/task/task_stack_multicore.go

@@ -0,0 +1,53 @@
+//go:build scheduler.cores
+
+package task
+
+import "runtime/interrupt"
+
+// Current returns the current active task.
+//
+//go:linkname Current runtime.currentTask
+func Current() *Task
+
+// Pause suspends the current task and returns to the scheduler.
+// This function may only be called when running on a goroutine stack, not when running on the system stack or in an interrupt.
+func Pause() {
+	lockScheduler()
+	PauseLocked()
+}
+
+// PauseLocked is the same as Pause, but must be called with the scheduler lock
+// already taken.
+func PauseLocked() {
+	// Check whether the canary (the lowest address of the stack) is still
+	// valid. If it is not, a stack overflow has occurred.
+	current := Current()
+	if *current.state.canaryPtr != stackCanary {
+		runtimePanic("goroutine stack overflow")
+	}
+	if interrupt.In() {
+		runtimePanic("blocked inside interrupt")
+	}
+	if current.RunState == RunStateResuming {
+		// Another core already marked this goroutine as ready to resume.
+		current.RunState = RunStateRunning
+		unlockScheduler()
+		return
+	}
+	current.RunState = RunStatePaused
+	current.state.pause()
+}
+
+// Resume the task until it pauses or completes.
+// This may only be called from the scheduler.
+func (t *Task) Resume() {
+	t.gcData.swap()
+	t.state.resume()
+	t.gcData.swap()
+}
+
+//go:linkname lockScheduler runtime.lockScheduler
+func lockScheduler()
+
+//go:linkname unlockScheduler runtime.unlockScheduler
+func unlockScheduler()

src/internal/task/task_stack_tinygoriscv.go

@@ -1,10 +1,16 @@
-//go:build scheduler.tasks && tinygo.riscv
+//go:build (scheduler.tasks || scheduler.cores) && tinygo.riscv
 
 package task
 
 import "unsafe"
 
-var systemStack uintptr
+// Returns a pointer where the system stack can be stored.
+// This is a layering violation! We should probably refactor this so that we
+// don't need such gymnastics to store the system stack pointer. (It should
+// probably be moved to the runtime).
+//
+//go:linkname runtime_systemStackPtr runtime.systemStackPtr
+func runtime_systemStackPtr() *uintptr
 
 // calleeSavedRegs is the list of registers that must be saved and restored when
 // switching between tasks. Also see scheduler_riscv.S that relies on the
@@ -50,17 +56,18 @@ func (s *state) archInit(r *calleeSavedRegs, fn uintptr, args unsafe.Pointer) {
 }
 
 func (s *state) resume() {
-	swapTask(s.sp, &systemStack)
+	swapTask(s.sp, runtime_systemStackPtr())
 }
 
 func (s *state) pause() {
-	newStack := systemStack
-	systemStack = 0
+	systemStackPtr := runtime_systemStackPtr()
+	newStack := *systemStackPtr
+	*systemStackPtr = 0
 	swapTask(newStack, &s.sp)
 }
 
 // SystemStack returns the system stack pointer when called from a task stack.
 // When called from the system stack, it returns 0.
 func SystemStack() uintptr {
-	return systemStack
+	return *runtime_systemStackPtr()
 }

src/internal/task/task_stack_unicore.go

@@ -0,0 +1,37 @@
+//go:build scheduler.tasks
+
+package task
+
+import "runtime/interrupt"
+
+// currentTask is the current running task, or nil if currently in the scheduler.
+var currentTask *Task
+
+// Current returns the current active task.
+func Current() *Task {
+	return currentTask
+}
+
+// Pause suspends the current task and returns to the scheduler.
+// This function may only be called when running on a goroutine stack, not when running on the system stack or in an interrupt.
+func Pause() {
+	// Check whether the canary (the lowest address of the stack) is still
+	// valid. If it is not, a stack overflow has occurred.
+	if *currentTask.state.canaryPtr != stackCanary {
+		runtimePanic("goroutine stack overflow")
+	}
+	if interrupt.In() {
+		runtimePanic("blocked inside interrupt")
+	}
+	currentTask.state.pause()
+}
+
+// Resume the task until it pauses or completes.
+// This may only be called from the scheduler.
+func (t *Task) Resume() {
+	currentTask = t
+	t.gcData.swap()
+	t.state.resume()
+	t.gcData.swap()
+	currentTask = nil
+}