all: add support for multicore scheduler

This commit adds support for a scheduler that runs a scheduler on all
available cores. It is meant to be used on baremetal systems with a
fixed number of cores, such as the RP2040.

The initial implementation adds support for multicore scheduling to the
riscv-qemu target as a convenient testing target. This means that this
new multicore scheduler is tested in CI, including a bunch of standard
library tests (`make tinygo-test-baremetal`). This should ensure the new
scheduler is reasonably well tested before trying to use it on
harder-to-debug targets like the RP2040.

src/runtime/atomics_critical.go

@@ -6,7 +6,6 @@
 package runtime
 
 import (
-	"runtime/interrupt"
 	_ "unsafe"
 )
 
@@ -23,27 +22,27 @@ import (
 func __atomic_load_2(ptr *uint16, ordering uintptr) uint16 {
 	// The LLVM docs for this say that there is a val argument after the pointer.
 	// That is a typo, and the GCC docs omit it.
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	val := *ptr
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 	return val
 }
 
 //export __atomic_store_2
 func __atomic_store_2(ptr *uint16, val uint16, ordering uintptr) {
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	*ptr = val
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 }
 
 //go:inline
 func doAtomicCAS16(ptr *uint16, expected, desired uint16) uint16 {
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	old := *ptr
 	if old == expected {
 		*ptr = desired
 	}
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 	return old
 }
 
@@ -61,10 +60,10 @@ func __atomic_compare_exchange_2(ptr, expected *uint16, desired uint16, successO
 
 //go:inline
 func doAtomicSwap16(ptr *uint16, new uint16) uint16 {
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	old := *ptr
 	*ptr = new
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 	return old
 }
 
@@ -80,11 +79,11 @@ func __atomic_exchange_2(ptr *uint16, new uint16, ordering uintptr) uint16 {
 
 //go:inline
 func doAtomicAdd16(ptr *uint16, value uint16) (old, new uint16) {
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	old = *ptr
 	new = old + value
 	*ptr = new
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 	return old, new
 }
 
@@ -112,27 +111,27 @@ func __atomic_add_fetch_2(ptr *uint16, value uint16, ordering uintptr) uint16 {
 func __atomic_load_4(ptr *uint32, ordering uintptr) uint32 {
 	// The LLVM docs for this say that there is a val argument after the pointer.
 	// That is a typo, and the GCC docs omit it.
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	val := *ptr
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 	return val
 }
 
 //export __atomic_store_4
 func __atomic_store_4(ptr *uint32, val uint32, ordering uintptr) {
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	*ptr = val
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 }
 
 //go:inline
 func doAtomicCAS32(ptr *uint32, expected, desired uint32) uint32 {
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	old := *ptr
 	if old == expected {
 		*ptr = desired
 	}
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 	return old
 }
 
@@ -150,10 +149,10 @@ func __atomic_compare_exchange_4(ptr, expected *uint32, desired uint32, successO
 
 //go:inline
 func doAtomicSwap32(ptr *uint32, new uint32) uint32 {
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	old := *ptr
 	*ptr = new
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 	return old
 }
 
@@ -169,11 +168,11 @@ func __atomic_exchange_4(ptr *uint32, new uint32, ordering uintptr) uint32 {
 
 //go:inline
 func doAtomicAdd32(ptr *uint32, value uint32) (old, new uint32) {
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	old = *ptr
 	new = old + value
 	*ptr = new
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 	return old, new
 }
 
@@ -201,27 +200,27 @@ func __atomic_add_fetch_4(ptr *uint32, value uint32, ordering uintptr) uint32 {
 func __atomic_load_8(ptr *uint64, ordering uintptr) uint64 {
 	// The LLVM docs for this say that there is a val argument after the pointer.
 	// That is a typo, and the GCC docs omit it.
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	val := *ptr
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 	return val
 }
 
 //export __atomic_store_8
 func __atomic_store_8(ptr *uint64, val uint64, ordering uintptr) {
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	*ptr = val
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 }
 
 //go:inline
 func doAtomicCAS64(ptr *uint64, expected, desired uint64) uint64 {
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	old := *ptr
 	if old == expected {
 		*ptr = desired
 	}
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 	return old
 }
 
@@ -239,10 +238,10 @@ func __atomic_compare_exchange_8(ptr, expected *uint64, desired uint64, successO
 
 //go:inline
 func doAtomicSwap64(ptr *uint64, new uint64) uint64 {
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	old := *ptr
 	*ptr = new
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 	return old
 }
 
@@ -258,11 +257,11 @@ func __atomic_exchange_8(ptr *uint64, new uint64, ordering uintptr) uint64 {
 
 //go:inline
 func doAtomicAdd64(ptr *uint64, value uint64) (old, new uint64) {
-	mask := interrupt.Disable()
+	mask := lockAtomics()
 	old = *ptr
 	new = old + value
 	*ptr = new
-	interrupt.Restore(mask)
+	unlockAtomics(mask)
 	return old, new
 }
 

src/runtime/gc_stack_cores.go

@@ -0,0 +1,101 @@
+//go:build scheduler.cores
+
+package runtime
+
+import (
+	"internal/task"
+	"sync/atomic"
+)
+
+// Normally 0. During a GC scan it has various purposes for signalling between
+// the core running the GC and the other cores in the system.
+var gcScanState atomic.Uint32
+
+// Start GC scan by pausing the world (all other cores) and scanning their
+// stacks. It doesn't resume the world.
+func gcMarkReachable() {
+	core := currentCPU()
+
+	// Interrupt all other cores.
+	gcScanState.Store(1)
+	for i := uint32(0); i < numCPU; i++ {
+		if i == core {
+			continue
+		}
+		gcPauseCore(i)
+	}
+
+	// Scan the stack(s) of the current core.
+	scanCurrentStack()
+	if !task.OnSystemStack() {
+		// Mark system stack.
+		markRoots(task.SystemStack(), coreStackTop(core))
+	}
+
+	// Scan globals.
+	findGlobals(markRoots)
+
+	// Busy-wait until all the other cores are ready. They certainly should be,
+	// after the scanning we did above.
+	for gcScanState.Load() != numCPU {
+		spinLoopHint()
+	}
+	gcScanState.Store(0)
+
+	// Signal each core in turn that they can scan the stack.
+	for i := uint32(0); i < numCPU; i++ {
+		if i == core {
+			continue
+		}
+
+		// Wake up the core to scan the stack.
+		gcSignalCore(i)
+
+		// Busy-wait until this core finished scanning.
+		for gcScanState.Load() == 0 {
+			spinLoopHint()
+		}
+		gcScanState.Store(0)
+	}
+
+	// All the stack are now scanned.
+}
+
+//go:export tinygo_scanCurrentStack
+func scanCurrentStack()
+
+//go:export tinygo_scanstack
+func scanstack(sp uintptr) {
+	// Mark the current stack.
+	// This function is called by scanCurrentStack, after pushing all registers
+	// onto the stack.
+	if task.OnSystemStack() {
+		// This is the system stack.
+		// Scan all words on the stack.
+		markRoots(sp, coreStackTop(currentCPU()))
+	} else {
+		// This is a goroutine stack.
+		markCurrentGoroutineStack(sp)
+	}
+}
+
+// Resume the world after a call to gcMarkReachable.
+func gcResumeWorld() {
+	// Signal each core that they can resume.
+	hartID := currentCPU()
+	for i := uint32(0); i < numCPU; i++ {
+		if i == hartID {
+			continue
+		}
+
+		// Signal the core.
+		gcSignalCore(i)
+	}
+
+	// Busy-wait until the core acknowledges the signal (and is going to return
+	// from the interrupt handler).
+	for gcScanState.Load() != numCPU-1 {
+		spinLoopHint()
+	}
+	gcScanState.Store(0)
+}

src/runtime/gc_stack_raw.go

@@ -1,8 +1,14 @@
-//go:build (gc.conservative || gc.precise || gc.boehm) && !tinygo.wasm && !scheduler.threads
+//go:build (gc.conservative || gc.precise || gc.boehm) && !tinygo.wasm && !scheduler.threads && !scheduler.cores
 
 package runtime
 
-import "internal/task"
+import (
+	"internal/task"
+	"sync/atomic"
+)
+
+// Unused.
+var gcScanState atomic.Uint32
 
 func gcMarkReachable() {
 	markStack()

src/runtime/panic.go

@@ -99,7 +99,8 @@ func runtimePanicAt(addr unsafe.Pointer, msg string) {
 	} else {
 		printstring("panic: runtime error: ")
 	}
-	println(msg)
+	printstring(msg)
+	printnl()
 	abort()
 }
 

src/runtime/print.go

@@ -1,7 +1,6 @@
 package runtime
 
 import (
-	"internal/task"
 	"unsafe"
 )
 
@@ -9,18 +8,6 @@ type stringer interface {
 	String() string
 }
 
-// Lock to make sure print calls do not interleave.
-// This is a no-op lock on systems that do not have parallelism.
-var printLock task.PMutex
-
-func printlock() {
-	printLock.Lock()
-}
-
-func printunlock() {
-	printLock.Unlock()
-}
-
 //go:nobounds
 func printstring(s string) {
 	for i := 0; i < len(s); i++ {

src/runtime/runtime_tinygoriscv_qemu.go

@@ -4,26 +4,82 @@ package runtime
 
 import (
 	"device/riscv"
+	"internal/task"
+	"math/bits"
+	"runtime/interrupt"
 	"runtime/volatile"
+	"sync/atomic"
 	"unsafe"
 )
 
 // This file implements the VirtIO RISC-V interface implemented in QEMU, which
 // is an interface designed for emulation.
 
+const numCPU = 4
+
 //export main
 func main() {
-	preinit()
-
 	// Set the interrupt address.
 	// Note that this address must be aligned specially, otherwise the MODE bits
 	// of MTVEC won't be zero.
 	riscv.MTVEC.Set(uintptr(unsafe.Pointer(&handleInterruptASM)))
 
+	// Enable software interrupts. We'll need them to wake up other cores.
+	riscv.MIE.SetBits(riscv.MIE_MSIE)
+
+	// If we're not hart 0, wait until we get the signal everything has been set
+	// up.
+	if hartID := riscv.MHARTID.Get(); hartID != 0 {
+		// Wait until we get the signal this hart is ready to start.
+		// Note that interrupts are disabled, which means that the interrupt
+		// isn't actually taken. But we can still wait for it using wfi.
+		// If the cores scheduler is not used, we'll stay in this state forever.
+		for riscv.MIP.Get()&riscv.MIP_MSIP == 0 {
+			riscv.Asm("wfi")
+		}
+
+		// Clear the software interrupt.
+		aclintMSWI.MSIP[hartID].Set(0)
+
+		// Now that we've cleared the software interrupt, we can enable
+		// interrupts as was already done on hart 0.
+		riscv.MSTATUS.SetBits(riscv.MSTATUS_MIE)
+
+		// Also enable timer interrupts, for sleepTicksMulticore.
+		riscv.MIE.SetBits(riscv.MIE_MTIE)
+
+		// Now start running the scheduler on this core.
+		schedulerLock.Lock()
+		scheduler(false)
+
+		// The scheduler exited, which means main returned and the program
+		// should exit immediately.
+		// Signal hart 0 to exit.
+		exitCodePlusOne.Store(0 + 1) // exit code 0
+		aclintMSWI.MSIP[0].Set(1)
+
+		// Unlock the scheduler to be sure. Shouldn't be needed.
+		schedulerLock.Unlock()
+
+		// Wait until hart 0 actually exits.
+		for {
+			riscv.Asm("wfi")
+		}
+	}
+
 	// Enable global interrupts now that they've been set up.
 	// This is currently only for timer interrupts.
 	riscv.MSTATUS.SetBits(riscv.MSTATUS_MIE)
 
+	// Set all MTIMECMP registers to a value that clears the MTIP bit in MIP.
+	// If we don't do this, the wfi instruction won't work as expected.
+	for i := 0; i < numCPU; i++ {
+		aclintMTIMECMP[i].Set(0xffff_ffff_ffff_ffff)
+	}
+
+	// Enable timer interrupts on hart 0.
+	riscv.MIE.SetBits(riscv.MIE_MTIE)
+
 	run()
 	exit(0)
 }
@@ -37,13 +93,32 @@ func handleInterrupt() {
 	code := uint(cause &^ (1 << 31))
 	if cause&(1<<31) != 0 {
 		// Topmost bit is set, which means that it is an interrupt.
+		hartID := currentCPU()
 		switch code {
+		case riscv.MachineSoftwareInterrupt:
+			if exitCodePlusOne.Load() != 0 {
+				exitNow(exitCodePlusOne.Load() - 1)
+			}
+			if gcScanState.Load() != 0 {
+				// The GC needs to run.
+				gcInterruptHandler(hartID)
+			}
+			checkpoint := &schedulerWaitCheckpoints[hartID]
+			if checkpoint.Saved() {
+				aclintMSWI.MSIP[hartID].Set(0)
+				riscv.MCAUSE.Set(0)
+				checkpoint.Jump()
+			}
 		case riscv.MachineTimerInterrupt:
-			// Signal timeout.
-			timerWakeup.Set(1)
-			// Disable the timer, to avoid triggering the interrupt right after
-			// this interrupt returns.
-			riscv.MIE.ClearBits(riscv.MIE_MTIE)
+			if sleepCheckpoint.Saved() {
+				// Set MTIMECMP to a high value so that MTIP goes low.
+				aclintMTIMECMP[hartID].Set(0xffff_ffff_ffff_ffff)
+				riscv.MCAUSE.Set(0)
+				sleepCheckpoint.Jump()
+			}
+		default:
+			runtimePanic("unknown interrupt")
+			abort()
 		}
 	} else {
 		// Topmost bit is clear, so it is an exception of some sort.
@@ -57,6 +132,79 @@ func handleInterrupt() {
 	riscv.MCAUSE.Set(0)
 }
 
+// The GC interrupted this core for the stop-the-world phase.
+// This function handles that, and only returns after the stop-the-world phase
+// ended.
+func gcInterruptHandler(hartID uint32) {
+	// *only* enable the MSIE interrupt
+	savedMIE := riscv.MIE.Get()
+	riscv.MIE.Set(riscv.MIE_MSIE)
+
+	// Disable this interrupt (to be enabled again soon).
+	aclintMSWI.MSIP[hartID].Set(0)
+
+	// Let the GC know we're ready.
+	gcScanState.Add(1)
+
+	// Wait until we get a signal to start scanning.
+	for riscv.MIP.Get()&riscv.MIP_MSIP == 0 {
+		riscv.Asm("wfi")
+	}
+	aclintMSWI.MSIP[hartID].Set(0)
+
+	// Scan the stack(s) of this core.
+	scanCurrentStack()
+	if !task.OnSystemStack() {
+		// Mark system stack.
+		markRoots(task.SystemStack(), coreStackTop(hartID))
+	}
+
+	// Signal we've finished scanning.
+	gcScanState.Store(1)
+
+	// Wait until we get a signal that the stop-the-world phase has ended.
+	for riscv.MIP.Get()&riscv.MIP_MSIP == 0 {
+		riscv.Asm("wfi")
+	}
+	aclintMSWI.MSIP[hartID].Set(0)
+
+	// Restore MIE bits.
+	riscv.MIE.Set(savedMIE)
+
+	// Signal we received the signal and are going to exit the interrupt.
+	gcScanState.Add(1)
+}
+
+//go:extern _stack_top
+var stack0TopSymbol [0]byte
+
+//go:extern _stack1_top
+var stack1TopSymbol [0]byte
+
+//go:extern _stack2_top
+var stack2TopSymbol [0]byte
+
+//go:extern _stack3_top
+var stack3TopSymbol [0]byte
+
+// Returns the stack top (highest address) of the system stack of the given
+// core.
+func coreStackTop(core uint32) uintptr {
+	switch core {
+	case 0:
+		return uintptr(unsafe.Pointer(&stack0TopSymbol))
+	case 1:
+		return uintptr(unsafe.Pointer(&stack1TopSymbol))
+	case 2:
+		return uintptr(unsafe.Pointer(&stack2TopSymbol))
+	case 3:
+		return uintptr(unsafe.Pointer(&stack3TopSymbol))
+	default:
+		runtimePanic("unexpected core")
+		return 0
+	}
+}
+
 // One tick is 100ns by default in QEMU.
 // (This is not a standard, just the default used by QEMU).
 func ticksToNanoseconds(ticks timeUnit) int64 {
@@ -67,22 +215,79 @@ func nanosecondsToTicks(ns int64) timeUnit {
 	return timeUnit(ns / 100) // one tick is 100ns
 }
 
-var timerWakeup volatile.Register8
+var sleepCheckpoint interrupt.Checkpoint
 
 func sleepTicks(d timeUnit) {
-	// Enable the timer.
-	target := uint64(ticks() + d)
-	aclintMTIMECMP.Set(target)
-	riscv.MIE.SetBits(riscv.MIE_MTIE)
+	hartID := currentCPU()
+	if sleepCheckpoint.Save() {
+		// Configure timeout.
+		target := uint64(ticks() + d)
+		aclintMTIMECMP[hartID].Set(target)
 
-	// Wait until it fires.
-	for {
-		if timerWakeup.Get() != 0 {
-			timerWakeup.Set(0)
-			// Disable timer.
-			break
+		// Wait for the interrupt to happen.
+		for {
+			riscv.Asm("wfi")
+		}
+	}
+
+	// We got awoken.
+}
+
+// Currently sleeping core, or 0xff.
+// Must only be accessed with the scheduler lock held.
+var sleepingCore uint8 = 0xff
+
+// Return whether another core is sleeping.
+// May only be called with the scheduler lock held.
+func hasSleepingCore() bool {
+	return sleepingCore != 0xff
+}
+
+// Almost identical to sleepTicks, except that it will unlock/lock the scheduler
+// while sleeping and is interruptible by interruptSleepTicksMulticore.
+// This may only be called with the scheduler lock held.
+func sleepTicksMulticore(d timeUnit) {
+	// Disable interrupts while configuring sleep.
+	// This is needed because unlocking the scheduler and setting the timer
+	// interrupt need to happen atomically.
+	riscv.MSTATUS.ClearBits(riscv.MSTATUS_MIE)
+
+	hartID := currentCPU()
+	if sleepCheckpoint.Save() {
+		sleepingCore = uint8(hartID)
+
+		// Configure timeout.
+		target := uint64(ticks() + d)
+		aclintMTIMECMP[hartID].Set(target)
+
+		// Unlock, now that the timeout has been set (so that
+		// interruptSleepTicksMulticore will see the correct wakeup time).
+		schedulerLock.Unlock()
+
+		// Sleep has been configured, interrupts may happen again.
+		riscv.MSTATUS.SetBits(riscv.MSTATUS_MIE)
+
+		// Wait for the interrupt to happen.
+		for {
+			riscv.Asm("wfi")
+		}
+	}
+	// We got awoken.
+
+	// Lock again, after we finished sleeping.
+	schedulerLock.Lock()
+	sleepingCore = 0xff
+}
+
+// Interrupt an ongoing call to sleepTicksMulticore on another core.
+// This may only be called with the scheduler lock held.
+func interruptSleepTicksMulticore(wakeup timeUnit) {
+	if sleepingCore != 0xff {
+		// Immediately exit the sleep.
+		old := aclintMTIMECMP[sleepingCore].Get()
+		if uint64(wakeup) < old {
+			aclintMTIMECMP[sleepingCore].Set(uint64(wakeup))
 		}
-		riscv.Asm("wfi")
 	}
 }
 
@@ -98,7 +303,7 @@ func ticks() timeUnit {
 			return timeUnit(lowBits) | (timeUnit(highBits) << 32)
 		}
 		// Retry, because there was a rollover in the low bits (happening every
-		// 429 days).
+		// ~7 days).
 		highBits = newHighBits
 	}
 }
@@ -120,7 +325,10 @@ var (
 		low  volatile.Register32
 		high volatile.Register32
 	})(unsafe.Pointer(uintptr(0x0200_bff8)))
-	aclintMTIMECMP = (*volatile.Register64)(unsafe.Pointer(uintptr(0x0200_4000)))
+	aclintMTIMECMP = (*[4095]volatile.Register64)(unsafe.Pointer(uintptr(0x0200_4000)))
+	aclintMSWI     = (*struct {
+		MSIP [4095]volatile.Register32
+	})(unsafe.Pointer(uintptr(0x0200_0000)))
 )
 
 func putchar(c byte) {
@@ -137,17 +345,166 @@ func buffered() int {
 	return 0
 }
 
+// Define the various spinlocks needed by the runtime.
+var (
+	schedulerLock spinLock
+	futexLock     spinLock
+	atomicsLock   spinLock
+	printLock     spinLock
+)
+
+type spinLock struct {
+	atomic.Uint32
+}
+
+func (l *spinLock) Lock() {
+	// Try to replace 0 with 1. Once we succeed, the lock has been acquired.
+	for !l.Uint32.CompareAndSwap(0, 1) {
+		spinLoopHint()
+	}
+}
+
+func (l *spinLock) Unlock() {
+	// Safety check: the spinlock should have been locked.
+	if schedulerAsserts && l.Uint32.Load() != 1 {
+		runtimePanic("unlock of unlocked spinlock")
+	}
+
+	// Unlock the lock. Simply write 0, because we already know it is locked.
+	l.Uint32.Store(0)
+}
+
+// Hint to the CPU that this core is just waiting, and the core can go into a
+// lower energy state.
+func spinLoopHint() {
+	// This is a no-op in QEMU TCG (but added here for completeness):
+	// https://github.com/qemu/qemu/blob/v9.2.3/target/riscv/insn_trans/trans_rvi.c.inc#L856
+	riscv.Asm("pause")
+}
+
+func currentCPU() uint32 {
+	return uint32(riscv.MHARTID.Get())
+}
+
+func startSecondaryCores() {
+	// Start all the other cores besides hart 0.
+	for hart := 1; hart < numCPU; hart++ {
+		// Signal the given hart it is ready to start using a software
+		// interrupt.
+		aclintMSWI.MSIP[hart].Set(1)
+	}
+}
+
+// Bitset of harts that are currently sleeping in schedulerUnlockAndWait.
+// This supports up to 8 harts.
+// This variable may only be accessed with the scheduler lock held.
+var sleepingHarts uint8
+
+// Checkpoints for cores waiting for runnable tasks.
+var schedulerWaitCheckpoints [numCPU]interrupt.Checkpoint
+
+// Put the scheduler to sleep, since there are no tasks to run.
+// This will unlock the scheduler lock, and must be called with the scheduler
+// lock held.
+func schedulerUnlockAndWait() {
+	hartID := currentCPU()
+
+	// Mark the current hart as sleeping.
+	sleepingHarts |= uint8(1 << hartID)
+
+	// If this is the last core awake and is going to sleep, the scheduler is
+	// deadlocked.
+	// We can do this check since this is not baremetal: there won't be any
+	// external interrupts that might unblock a goroutine.
+	if sleepingHarts == (1<<numCPU)-1 {
+		runtimePanic("all cores are sleeping - deadlock!")
+	}
+
+	// Need to disable interrupts while saving the checkpoint, otherwise if the
+	// software interrupt happens earlier for another reason (e.g. a GC cycle)
+	// it will see an incomplete checkpoint and the schedulerLock might not be
+	// unlocked yet. That will lead to an invalid state.
+	riscv.MSTATUS.ClearBits(riscv.MSTATUS_MIE)
+	if schedulerWaitCheckpoints[hartID].Save() {
+		schedulerLock.Unlock()
+		riscv.MSTATUS.SetBits(riscv.MSTATUS_MIE)
+
+		// Wait until we get awoken :)
+		for {
+			riscv.Asm("wfi")
+		}
+	}
+
+	// We got awoken again. We need to lock the scheduler again before
+	// returning.
+	schedulerLock.Lock()
+}
+
+// Wake another core, if one is sleeping. Must be called with the scheduler lock
+// held.
+func schedulerWake() {
+	// Look up the lowest-numbered hart that is sleeping.
+	// Returns 8 if there are no sleeping harts.
+	hart := bits.TrailingZeros8(sleepingHarts)
+
+	if hart < 8 {
+		// There is a sleeping hart. Wake it.
+		sleepingHarts &^= 1 << hart  // clear the bit
+		aclintMSWI.MSIP[hart].Set(1) // send software interrupt
+	}
+}
+
+// Pause the given core by sending it an interrupt.
+func gcPauseCore(core uint32) {
+	aclintMSWI.MSIP[core].Set(1) // send software interrupt
+}
+
+// Signal the given core that it can resume one step.
+// This is called twice after gcPauseCore: the first time to scan the stack of
+// the core, and the second time to end the stop-the-world phase.
+func gcSignalCore(core uint32) {
+	aclintMSWI.MSIP[core].Set(1) // send software interrupt
+}
+
 func abort() {
 	exit(1)
 }
 
+// Zero in the default state, when non-zero it indicates the exit code plus one.
+// So exit(0) will result in 1, exit(1) in 2, etc.
+var exitCodePlusOne atomic.Uint32
+
 func exit(code int) {
+	// Check for invalid values, to be sure.
+	if code < 0 {
+		code = 255
+	}
+
+	// If we're not on hart 0, we can't exit QEMU.
+	// Therefore, send an interrupt to hart 0 instead to request an exit.
+	if currentCPU() != 0 {
+		// Signal hart 0 to exit.
+		exitCodePlusOne.Store(uint32(code) + 1)
+		aclintMSWI.MSIP[0].Set(1)
+
+		// Wait for the interrupt to happen. This should happen immediately.
+		for {
+			riscv.Asm("wfi")
+		}
+	}
+
+	exitNow(uint32(code))
+}
+
+// Send an exit signal to the test finisher pseudo-device, without checking
+// whether we are on hart 0.
+func exitNow(code uint32) {
 	// Make sure the QEMU process exits.
 	if code == 0 {
 		testFinisher.Set(0x5555) // FINISHER_PASS
 	} else {
 		// Exit code is stored in the upper 16 bits of the 32 bit value.
-		testFinisher.Set(uint32(code)<<16 | 0x3333) // FINISHER_FAIL
+		testFinisher.Set(code<<16 | 0x3333) // FINISHER_FAIL
 	}
 
 	// Lock up forever (as a fallback).
@@ -162,10 +519,6 @@ func exit(code int) {
 func handleException(code uint) {
 	// For a list of exception codes, see:
 	// https://content.riscv.org/wp-content/uploads/2019/08/riscv-privileged-20190608-1.pdf#page=49
-	print("fatal error: exception with mcause=")
-	print(code)
-	print(" pc=")
-	print(riscv.MEPC.Get())
-	println()
+	print("fatal error: exception with mcause=", code, " pc=", riscv.MEPC.Get(), " hart=", uint(riscv.MHARTID.Get()), "\r\n")
 	abort()
 }

src/runtime/scheduler_cooperative.go

@@ -252,3 +252,19 @@ func run() {
 	}()
 	scheduler(false)
 }
+
+func lockAtomics() interrupt.State {
+	return interrupt.Disable()
+}
+
+func unlockAtomics(mask interrupt.State) {
+	interrupt.Restore(mask)
+}
+
+func printlock() {
+	// nothing to do
+}
+
+func printunlock() {
+	// nothing to do
+}

src/runtime/scheduler_cores.go

@@ -0,0 +1,317 @@
+//go:build scheduler.cores
+
+package runtime
+
+import (
+	"internal/task"
+	"runtime/interrupt"
+	"sync/atomic"
+)
+
+const hasScheduler = true
+
+const hasParallelism = true
+
+var mainExited atomic.Uint32
+
+// Which task is running on a given core (or nil if there is no task running on
+// the core).
+var cpuTasks [numCPU]*task.Task
+
+var (
+	sleepQueue *task.Task
+	runqueue   task.Queue
+)
+
+func deadlock() {
+	// Call yield without requesting a wakeup.
+	task.Pause()
+	trap()
+}
+
+// Mark the given task as ready to resume.
+// This is allowed even if the task isn't paused yet, but will pause soon.
+func scheduleTask(t *task.Task) {
+	schedulerLock.Lock()
+	switch t.RunState {
+	case task.RunStatePaused:
+		// Paused, state is saved on the stack.
+		// Add it to the runqueue...
+		runqueue.Push(t)
+		// ...and wake up a sleeping core, if there is one.
+		// (If all cores are already busy, this is a no-op).
+		schedulerWake()
+	case task.RunStateRunning:
+		// Not yet paused (probably going to pause very soon), so let the
+		// Pause() function know it can resume immediately.
+		t.RunState = task.RunStateResuming
+	default:
+		if schedulerAsserts {
+			runtimePanic("scheduler: unknown run state")
+		}
+	}
+	schedulerLock.Unlock()
+}
+
+func addSleepTask(t *task.Task, wakeup timeUnit) {
+	// Save the timestamp when the task should be woken up.
+	t.Data = uint64(wakeup)
+
+	// If another core is currently using the timer, make sure it wakes up at
+	// the right time.
+	interruptSleepTicksMulticore(wakeup)
+
+	// Find the position where we should insert this task in the queue.
+	q := &sleepQueue
+	for {
+		if *q == nil {
+			// Found the end of the time queue. Insert it here, at the end.
+			break
+		}
+		if timeUnit((*q).Data) > timeUnit(t.Data) {
+			// Found a task in the queue that has a timeout before the
+			// to-be-sleeping task. Insert our task right before.
+			break
+		}
+		q = &(*q).Next
+	}
+
+	// Insert the task into the queue (this could be at the end, if *q is nil).
+	t.Next = *q
+	*q = t
+}
+
+func Gosched() {
+	schedulerLock.Lock()
+	runqueue.Push(task.Current())
+	task.PauseLocked()
+}
+
+func addTimer(tn *timerNode) {
+	schedulerLock.Lock()
+	timerQueueAdd(tn)
+	interruptSleepTicksMulticore(tn.whenTicks())
+	schedulerLock.Unlock()
+}
+
+func removeTimer(t *timer) *timerNode {
+	schedulerLock.Lock()
+	n := timerQueueRemove(t)
+	schedulerLock.Unlock()
+	return n
+}
+
+func schedulerRunQueue() *task.Queue {
+	return &runqueue
+}
+
+// Pause the current task for a given time.
+//
+//go:linkname sleep time.Sleep
+func sleep(duration int64) {
+	if duration <= 0 {
+		return
+	}
+
+	wakeup := ticks() + nanosecondsToTicks(duration)
+
+	// While the scheduler is locked:
+	// - add this task to the sleep queue
+	// - switch to the scheduler (only allowed while locked)
+	// - let the scheduler handle it from there
+	schedulerLock.Lock()
+	addSleepTask(task.Current(), wakeup)
+	task.PauseLocked()
+}
+
+// This function is called on the first core in the system. It will wake up the
+// other cores when ready.
+func run() {
+	initHeap()
+
+	go func() {
+		// Package initializers are currently run single-threaded.
+		// This might help with registering interrupts and such.
+		initAll()
+
+		// After package initializers have finished, start all the other cores.
+		startSecondaryCores()
+
+		// Run main.main.
+		callMain()
+
+		// main.main has exited, so the program should exit.
+		mainExited.Store(1)
+	}()
+
+	// The scheduler must always be entered while the scheduler lock is taken.
+	schedulerLock.Lock()
+	scheduler(false)
+	schedulerLock.Unlock()
+}
+
+func scheduler(_ bool) {
+	for mainExited.Load() == 0 {
+		// Check for ready-to-run tasks.
+		if runnable := runqueue.Pop(); runnable != nil {
+			// Resume it now.
+			setCurrentTask(runnable)
+			runnable.RunState = task.RunStateRunning
+			schedulerLock.Unlock() // unlock before resuming, Pause() will lock again
+			runnable.Resume()
+			setCurrentTask(nil)
+
+			continue
+		}
+
+		var now timeUnit
+		if sleepQueue != nil || timerQueue != nil {
+			now = ticks()
+
+			// Check whether the first task in the sleep queue is ready to run.
+			if sleepingTask := sleepQueue; sleepingTask != nil && now >= timeUnit(sleepingTask.Data) {
+				// It is, pop it from the queue.
+				sleepQueue = sleepQueue.Next
+				sleepingTask.Next = nil
+
+				// Run it now.
+				setCurrentTask(sleepingTask)
+				sleepingTask.RunState = task.RunStateRunning
+				schedulerLock.Unlock() // unlock before resuming, Pause() will lock again
+				sleepingTask.Resume()
+				setCurrentTask(nil)
+				continue
+			}
+
+			// Check whether a timer has expired that needs to be run.
+			if timerQueue != nil && now >= timerQueue.whenTicks() {
+				delay := ticksToNanoseconds(now - timerQueue.whenTicks())
+				// Pop timer from queue.
+				tn := timerQueue
+				timerQueue = tn.next
+				tn.next = nil
+
+				// Run the callback stored in this timer node.
+				schedulerLock.Unlock()
+				tn.callback(tn, delay)
+				schedulerLock.Lock()
+				continue
+			}
+		}
+
+		// At this point, there are no runnable tasks anymore.
+		// If another core is using the clock, let it handle the sleep queue.
+		if hasSleepingCore() {
+			schedulerUnlockAndWait()
+			continue
+		}
+
+		// The timer is free to use, so check whether there are any future
+		// tasks/timers that we can wait for.
+		var timeLeft timeUnit
+		if sleepingTask := sleepQueue; sleepingTask != nil {
+			// We already checked that there is no ready-to-run sleeping task
+			// (using the same 'now' value), so timeLeft will always be
+			// positive.
+			timeLeft = timeUnit(sleepingTask.Data) - now
+		}
+		if timerQueue != nil {
+			// If the timer queue needs to run earlier, reduce the time we are
+			// going to sleep.
+			// Like with sleepQueue, we already know there is no timer ready to
+			// run since we already checked above.
+			timeLeftForTimer := timerQueue.whenTicks() - now
+			if sleepQueue == nil || timeLeftForTimer < timeLeft {
+				timeLeft = timeLeftForTimer
+			}
+		}
+
+		if timeLeft > 0 {
+			// Sleep for a bit until the next task or timer is ready to run.
+			sleepTicksMulticore(timeLeft)
+			continue
+		}
+
+		// No runnable tasks and no sleeping tasks or timers. There's nothing to
+		// do.
+		// Wait until something happens (like an interrupt).
+		schedulerUnlockAndWait()
+	}
+}
+
+func currentTask() *task.Task {
+	return cpuTasks[currentCPU()]
+}
+
+func setCurrentTask(task *task.Task) {
+	cpuTasks[currentCPU()] = task
+}
+
+func lockScheduler() {
+	schedulerLock.Lock()
+}
+
+func unlockScheduler() {
+	schedulerLock.Unlock()
+}
+
+func lockFutex() interrupt.State {
+	mask := interrupt.Disable()
+	futexLock.Lock()
+	return mask
+}
+
+func unlockFutex(state interrupt.State) {
+	futexLock.Unlock()
+	interrupt.Restore(state)
+}
+
+// Use a single spinlock for atomics. This works fine, since atomics are very
+// short sequences of instructions.
+func lockAtomics() interrupt.State {
+	mask := interrupt.Disable()
+	atomicsLock.Lock()
+	return mask
+}
+
+func unlockAtomics(mask interrupt.State) {
+	atomicsLock.Unlock()
+	interrupt.Restore(mask)
+}
+
+var systemStack [numCPU]uintptr
+
+// Implementation detail of the internal/task package.
+// It needs to store the system stack pointer somewhere, and needs to know how
+// many cores there are to do so. But it doesn't know the number of cores. Hence
+// why this is implemented in the runtime.
+func systemStackPtr() *uintptr {
+	return &systemStack[currentCPU()]
+}
+
+// Color the 'print' and 'println' output according to the current CPU.
+// This may be helpful for debugging, but should be disabled otherwise.
+const cpuColoredPrint = false
+
+func printlock() {
+	printLock.Lock()
+	if cpuColoredPrint {
+		switch currentCPU() {
+		case 1:
+			printstring("\x1b[32m") // green
+		case 2:
+			printstring("\x1b[33m") // yellow
+		case 3:
+			printstring("\x1b[34m") // blue
+		}
+	}
+}
+
+func printunlock() {
+	if cpuColoredPrint {
+		if currentCPU() != 0 {
+			printstring("\x1b[0m") // reset colored output
+		}
+	}
+	printLock.Unlock()
+}

src/runtime/scheduler_none.go

@@ -2,7 +2,10 @@
 
 package runtime
 
-import "internal/task"
+import (
+	"internal/task"
+	"runtime/interrupt"
+)
 
 const hasScheduler = false
 
@@ -73,3 +76,19 @@ func scheduler(returnAtDeadlock bool) {
 	// this code should be unreachable.
 	runtimePanic("unreachable: scheduler must not be called with the 'none' scheduler")
 }
+
+func lockAtomics() interrupt.State {
+	return interrupt.Disable()
+}
+
+func unlockAtomics(mask interrupt.State) {
+	interrupt.Restore(mask)
+}
+
+func printlock() {
+	// nothing to do
+}
+
+func printunlock() {
+	// nothing to do
+}

src/runtime/scheduler_tasks.go

@@ -0,0 +1,13 @@
+//go:build scheduler.tasks
+
+package runtime
+
+var systemStack uintptr
+
+// Implementation detail of the internal/task package.
+// It needs to store the system stack pointer somewhere, and needs to know how
+// many cores there are to do so. But it doesn't know the number of cores. Hence
+// why this is implemented in the runtime.
+func systemStackPtr() *uintptr {
+	return &systemStack
+}

src/runtime/scheduler_threads.go

@@ -2,7 +2,10 @@
 
 package runtime
 
-import "internal/task"
+import (
+	"internal/task"
+	"runtime/interrupt"
+)
 
 const hasScheduler = false // not using the cooperative scheduler
 
@@ -127,3 +130,30 @@ func runqueueForGC() *task.Queue {
 	// There is only a runqueue when using the cooperative scheduler.
 	return nil
 }
+
+// Lock to make sure print calls do not interleave.
+var printLock task.Mutex
+
+func printlock() {
+	printLock.Lock()
+}
+
+func printunlock() {
+	printLock.Unlock()
+}
+
+// The atomics lock isn't used as a lock for actual atomics. It is used inside
+// internal/task.Stack and internal/task.Queue to make sure their operations are
+// actually atomic. (This might not actually be needed, since the use in
+// sync.Cond doesn't need atomicity).
+
+var atomicsLock task.Mutex
+
+func lockAtomics() interrupt.State {
+	atomicsLock.Lock()
+	return 0
+}
+
+func unlockAtomics(mask interrupt.State) {
+	atomicsLock.Unlock()
+}