mirror of
https://github.com/tinygo-org/tinygo.git
synced 2026-08-10 14:03:39 +00:00
all: add support for multicore scheduler
This commit adds support for a scheduler that runs a scheduler on all available cores. It is meant to be used on baremetal systems with a fixed number of cores, such as the RP2040. The initial implementation adds support for multicore scheduling to the riscv-qemu target as a convenient testing target. This means that this new multicore scheduler is tested in CI, including a bunch of standard library tests (`make tinygo-test-baremetal`). This should ensure the new scheduler is reasonably well tested before trying to use it on harder-to-debug targets like the RP2040.
This commit is contained in:
Ayke van Laethem
committed by
Ron Evans
Ron Evans
parent
0c7c2926f9
commit
60f8a62978
@@ -4,26 +4,82 @@ package runtime
|
||||
|
||||
import (
|
||||
"device/riscv"
|
||||
"internal/task"
|
||||
"math/bits"
|
||||
"runtime/interrupt"
|
||||
"runtime/volatile"
|
||||
"sync/atomic"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// This file implements the VirtIO RISC-V interface implemented in QEMU, which
|
||||
// is an interface designed for emulation.
|
||||
|
||||
const numCPU = 4
|
||||
|
||||
//export main
|
||||
func main() {
|
||||
preinit()
|
||||
|
||||
// Set the interrupt address.
|
||||
// Note that this address must be aligned specially, otherwise the MODE bits
|
||||
// of MTVEC won't be zero.
|
||||
riscv.MTVEC.Set(uintptr(unsafe.Pointer(&handleInterruptASM)))
|
||||
|
||||
// Enable software interrupts. We'll need them to wake up other cores.
|
||||
riscv.MIE.SetBits(riscv.MIE_MSIE)
|
||||
|
||||
// If we're not hart 0, wait until we get the signal everything has been set
|
||||
// up.
|
||||
if hartID := riscv.MHARTID.Get(); hartID != 0 {
|
||||
// Wait until we get the signal this hart is ready to start.
|
||||
// Note that interrupts are disabled, which means that the interrupt
|
||||
// isn't actually taken. But we can still wait for it using wfi.
|
||||
// If the cores scheduler is not used, we'll stay in this state forever.
|
||||
for riscv.MIP.Get()&riscv.MIP_MSIP == 0 {
|
||||
riscv.Asm("wfi")
|
||||
}
|
||||
|
||||
// Clear the software interrupt.
|
||||
aclintMSWI.MSIP[hartID].Set(0)
|
||||
|
||||
// Now that we've cleared the software interrupt, we can enable
|
||||
// interrupts as was already done on hart 0.
|
||||
riscv.MSTATUS.SetBits(riscv.MSTATUS_MIE)
|
||||
|
||||
// Also enable timer interrupts, for sleepTicksMulticore.
|
||||
riscv.MIE.SetBits(riscv.MIE_MTIE)
|
||||
|
||||
// Now start running the scheduler on this core.
|
||||
schedulerLock.Lock()
|
||||
scheduler(false)
|
||||
|
||||
// The scheduler exited, which means main returned and the program
|
||||
// should exit immediately.
|
||||
// Signal hart 0 to exit.
|
||||
exitCodePlusOne.Store(0 + 1) // exit code 0
|
||||
aclintMSWI.MSIP[0].Set(1)
|
||||
|
||||
// Unlock the scheduler to be sure. Shouldn't be needed.
|
||||
schedulerLock.Unlock()
|
||||
|
||||
// Wait until hart 0 actually exits.
|
||||
for {
|
||||
riscv.Asm("wfi")
|
||||
}
|
||||
}
|
||||
|
||||
// Enable global interrupts now that they've been set up.
|
||||
// This is currently only for timer interrupts.
|
||||
riscv.MSTATUS.SetBits(riscv.MSTATUS_MIE)
|
||||
|
||||
// Set all MTIMECMP registers to a value that clears the MTIP bit in MIP.
|
||||
// If we don't do this, the wfi instruction won't work as expected.
|
||||
for i := 0; i < numCPU; i++ {
|
||||
aclintMTIMECMP[i].Set(0xffff_ffff_ffff_ffff)
|
||||
}
|
||||
|
||||
// Enable timer interrupts on hart 0.
|
||||
riscv.MIE.SetBits(riscv.MIE_MTIE)
|
||||
|
||||
run()
|
||||
exit(0)
|
||||
}
|
||||
@@ -37,13 +93,32 @@ func handleInterrupt() {
|
||||
code := uint(cause &^ (1 << 31))
|
||||
if cause&(1<<31) != 0 {
|
||||
// Topmost bit is set, which means that it is an interrupt.
|
||||
hartID := currentCPU()
|
||||
switch code {
|
||||
case riscv.MachineSoftwareInterrupt:
|
||||
if exitCodePlusOne.Load() != 0 {
|
||||
exitNow(exitCodePlusOne.Load() - 1)
|
||||
}
|
||||
if gcScanState.Load() != 0 {
|
||||
// The GC needs to run.
|
||||
gcInterruptHandler(hartID)
|
||||
}
|
||||
checkpoint := &schedulerWaitCheckpoints[hartID]
|
||||
if checkpoint.Saved() {
|
||||
aclintMSWI.MSIP[hartID].Set(0)
|
||||
riscv.MCAUSE.Set(0)
|
||||
checkpoint.Jump()
|
||||
}
|
||||
case riscv.MachineTimerInterrupt:
|
||||
// Signal timeout.
|
||||
timerWakeup.Set(1)
|
||||
// Disable the timer, to avoid triggering the interrupt right after
|
||||
// this interrupt returns.
|
||||
riscv.MIE.ClearBits(riscv.MIE_MTIE)
|
||||
if sleepCheckpoint.Saved() {
|
||||
// Set MTIMECMP to a high value so that MTIP goes low.
|
||||
aclintMTIMECMP[hartID].Set(0xffff_ffff_ffff_ffff)
|
||||
riscv.MCAUSE.Set(0)
|
||||
sleepCheckpoint.Jump()
|
||||
}
|
||||
default:
|
||||
runtimePanic("unknown interrupt")
|
||||
abort()
|
||||
}
|
||||
} else {
|
||||
// Topmost bit is clear, so it is an exception of some sort.
|
||||
@@ -57,6 +132,79 @@ func handleInterrupt() {
|
||||
riscv.MCAUSE.Set(0)
|
||||
}
|
||||
|
||||
// The GC interrupted this core for the stop-the-world phase.
|
||||
// This function handles that, and only returns after the stop-the-world phase
|
||||
// ended.
|
||||
func gcInterruptHandler(hartID uint32) {
|
||||
// *only* enable the MSIE interrupt
|
||||
savedMIE := riscv.MIE.Get()
|
||||
riscv.MIE.Set(riscv.MIE_MSIE)
|
||||
|
||||
// Disable this interrupt (to be enabled again soon).
|
||||
aclintMSWI.MSIP[hartID].Set(0)
|
||||
|
||||
// Let the GC know we're ready.
|
||||
gcScanState.Add(1)
|
||||
|
||||
// Wait until we get a signal to start scanning.
|
||||
for riscv.MIP.Get()&riscv.MIP_MSIP == 0 {
|
||||
riscv.Asm("wfi")
|
||||
}
|
||||
aclintMSWI.MSIP[hartID].Set(0)
|
||||
|
||||
// Scan the stack(s) of this core.
|
||||
scanCurrentStack()
|
||||
if !task.OnSystemStack() {
|
||||
// Mark system stack.
|
||||
markRoots(task.SystemStack(), coreStackTop(hartID))
|
||||
}
|
||||
|
||||
// Signal we've finished scanning.
|
||||
gcScanState.Store(1)
|
||||
|
||||
// Wait until we get a signal that the stop-the-world phase has ended.
|
||||
for riscv.MIP.Get()&riscv.MIP_MSIP == 0 {
|
||||
riscv.Asm("wfi")
|
||||
}
|
||||
aclintMSWI.MSIP[hartID].Set(0)
|
||||
|
||||
// Restore MIE bits.
|
||||
riscv.MIE.Set(savedMIE)
|
||||
|
||||
// Signal we received the signal and are going to exit the interrupt.
|
||||
gcScanState.Add(1)
|
||||
}
|
||||
|
||||
//go:extern _stack_top
|
||||
var stack0TopSymbol [0]byte
|
||||
|
||||
//go:extern _stack1_top
|
||||
var stack1TopSymbol [0]byte
|
||||
|
||||
//go:extern _stack2_top
|
||||
var stack2TopSymbol [0]byte
|
||||
|
||||
//go:extern _stack3_top
|
||||
var stack3TopSymbol [0]byte
|
||||
|
||||
// Returns the stack top (highest address) of the system stack of the given
|
||||
// core.
|
||||
func coreStackTop(core uint32) uintptr {
|
||||
switch core {
|
||||
case 0:
|
||||
return uintptr(unsafe.Pointer(&stack0TopSymbol))
|
||||
case 1:
|
||||
return uintptr(unsafe.Pointer(&stack1TopSymbol))
|
||||
case 2:
|
||||
return uintptr(unsafe.Pointer(&stack2TopSymbol))
|
||||
case 3:
|
||||
return uintptr(unsafe.Pointer(&stack3TopSymbol))
|
||||
default:
|
||||
runtimePanic("unexpected core")
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// One tick is 100ns by default in QEMU.
|
||||
// (This is not a standard, just the default used by QEMU).
|
||||
func ticksToNanoseconds(ticks timeUnit) int64 {
|
||||
@@ -67,22 +215,79 @@ func nanosecondsToTicks(ns int64) timeUnit {
|
||||
return timeUnit(ns / 100) // one tick is 100ns
|
||||
}
|
||||
|
||||
var timerWakeup volatile.Register8
|
||||
var sleepCheckpoint interrupt.Checkpoint
|
||||
|
||||
func sleepTicks(d timeUnit) {
|
||||
// Enable the timer.
|
||||
target := uint64(ticks() + d)
|
||||
aclintMTIMECMP.Set(target)
|
||||
riscv.MIE.SetBits(riscv.MIE_MTIE)
|
||||
hartID := currentCPU()
|
||||
if sleepCheckpoint.Save() {
|
||||
// Configure timeout.
|
||||
target := uint64(ticks() + d)
|
||||
aclintMTIMECMP[hartID].Set(target)
|
||||
|
||||
// Wait until it fires.
|
||||
for {
|
||||
if timerWakeup.Get() != 0 {
|
||||
timerWakeup.Set(0)
|
||||
// Disable timer.
|
||||
break
|
||||
// Wait for the interrupt to happen.
|
||||
for {
|
||||
riscv.Asm("wfi")
|
||||
}
|
||||
}
|
||||
|
||||
// We got awoken.
|
||||
}
|
||||
|
||||
// Currently sleeping core, or 0xff.
|
||||
// Must only be accessed with the scheduler lock held.
|
||||
var sleepingCore uint8 = 0xff
|
||||
|
||||
// Return whether another core is sleeping.
|
||||
// May only be called with the scheduler lock held.
|
||||
func hasSleepingCore() bool {
|
||||
return sleepingCore != 0xff
|
||||
}
|
||||
|
||||
// Almost identical to sleepTicks, except that it will unlock/lock the scheduler
|
||||
// while sleeping and is interruptible by interruptSleepTicksMulticore.
|
||||
// This may only be called with the scheduler lock held.
|
||||
func sleepTicksMulticore(d timeUnit) {
|
||||
// Disable interrupts while configuring sleep.
|
||||
// This is needed because unlocking the scheduler and setting the timer
|
||||
// interrupt need to happen atomically.
|
||||
riscv.MSTATUS.ClearBits(riscv.MSTATUS_MIE)
|
||||
|
||||
hartID := currentCPU()
|
||||
if sleepCheckpoint.Save() {
|
||||
sleepingCore = uint8(hartID)
|
||||
|
||||
// Configure timeout.
|
||||
target := uint64(ticks() + d)
|
||||
aclintMTIMECMP[hartID].Set(target)
|
||||
|
||||
// Unlock, now that the timeout has been set (so that
|
||||
// interruptSleepTicksMulticore will see the correct wakeup time).
|
||||
schedulerLock.Unlock()
|
||||
|
||||
// Sleep has been configured, interrupts may happen again.
|
||||
riscv.MSTATUS.SetBits(riscv.MSTATUS_MIE)
|
||||
|
||||
// Wait for the interrupt to happen.
|
||||
for {
|
||||
riscv.Asm("wfi")
|
||||
}
|
||||
}
|
||||
// We got awoken.
|
||||
|
||||
// Lock again, after we finished sleeping.
|
||||
schedulerLock.Lock()
|
||||
sleepingCore = 0xff
|
||||
}
|
||||
|
||||
// Interrupt an ongoing call to sleepTicksMulticore on another core.
|
||||
// This may only be called with the scheduler lock held.
|
||||
func interruptSleepTicksMulticore(wakeup timeUnit) {
|
||||
if sleepingCore != 0xff {
|
||||
// Immediately exit the sleep.
|
||||
old := aclintMTIMECMP[sleepingCore].Get()
|
||||
if uint64(wakeup) < old {
|
||||
aclintMTIMECMP[sleepingCore].Set(uint64(wakeup))
|
||||
}
|
||||
riscv.Asm("wfi")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,7 +303,7 @@ func ticks() timeUnit {
|
||||
return timeUnit(lowBits) | (timeUnit(highBits) << 32)
|
||||
}
|
||||
// Retry, because there was a rollover in the low bits (happening every
|
||||
// 429 days).
|
||||
// ~7 days).
|
||||
highBits = newHighBits
|
||||
}
|
||||
}
|
||||
@@ -120,7 +325,10 @@ var (
|
||||
low volatile.Register32
|
||||
high volatile.Register32
|
||||
})(unsafe.Pointer(uintptr(0x0200_bff8)))
|
||||
aclintMTIMECMP = (*volatile.Register64)(unsafe.Pointer(uintptr(0x0200_4000)))
|
||||
aclintMTIMECMP = (*[4095]volatile.Register64)(unsafe.Pointer(uintptr(0x0200_4000)))
|
||||
aclintMSWI = (*struct {
|
||||
MSIP [4095]volatile.Register32
|
||||
})(unsafe.Pointer(uintptr(0x0200_0000)))
|
||||
)
|
||||
|
||||
func putchar(c byte) {
|
||||
@@ -137,17 +345,166 @@ func buffered() int {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Define the various spinlocks needed by the runtime.
|
||||
var (
|
||||
schedulerLock spinLock
|
||||
futexLock spinLock
|
||||
atomicsLock spinLock
|
||||
printLock spinLock
|
||||
)
|
||||
|
||||
type spinLock struct {
|
||||
atomic.Uint32
|
||||
}
|
||||
|
||||
func (l *spinLock) Lock() {
|
||||
// Try to replace 0 with 1. Once we succeed, the lock has been acquired.
|
||||
for !l.Uint32.CompareAndSwap(0, 1) {
|
||||
spinLoopHint()
|
||||
}
|
||||
}
|
||||
|
||||
func (l *spinLock) Unlock() {
|
||||
// Safety check: the spinlock should have been locked.
|
||||
if schedulerAsserts && l.Uint32.Load() != 1 {
|
||||
runtimePanic("unlock of unlocked spinlock")
|
||||
}
|
||||
|
||||
// Unlock the lock. Simply write 0, because we already know it is locked.
|
||||
l.Uint32.Store(0)
|
||||
}
|
||||
|
||||
// Hint to the CPU that this core is just waiting, and the core can go into a
|
||||
// lower energy state.
|
||||
func spinLoopHint() {
|
||||
// This is a no-op in QEMU TCG (but added here for completeness):
|
||||
// https://github.com/qemu/qemu/blob/v9.2.3/target/riscv/insn_trans/trans_rvi.c.inc#L856
|
||||
riscv.Asm("pause")
|
||||
}
|
||||
|
||||
func currentCPU() uint32 {
|
||||
return uint32(riscv.MHARTID.Get())
|
||||
}
|
||||
|
||||
func startSecondaryCores() {
|
||||
// Start all the other cores besides hart 0.
|
||||
for hart := 1; hart < numCPU; hart++ {
|
||||
// Signal the given hart it is ready to start using a software
|
||||
// interrupt.
|
||||
aclintMSWI.MSIP[hart].Set(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Bitset of harts that are currently sleeping in schedulerUnlockAndWait.
|
||||
// This supports up to 8 harts.
|
||||
// This variable may only be accessed with the scheduler lock held.
|
||||
var sleepingHarts uint8
|
||||
|
||||
// Checkpoints for cores waiting for runnable tasks.
|
||||
var schedulerWaitCheckpoints [numCPU]interrupt.Checkpoint
|
||||
|
||||
// Put the scheduler to sleep, since there are no tasks to run.
|
||||
// This will unlock the scheduler lock, and must be called with the scheduler
|
||||
// lock held.
|
||||
func schedulerUnlockAndWait() {
|
||||
hartID := currentCPU()
|
||||
|
||||
// Mark the current hart as sleeping.
|
||||
sleepingHarts |= uint8(1 << hartID)
|
||||
|
||||
// If this is the last core awake and is going to sleep, the scheduler is
|
||||
// deadlocked.
|
||||
// We can do this check since this is not baremetal: there won't be any
|
||||
// external interrupts that might unblock a goroutine.
|
||||
if sleepingHarts == (1<<numCPU)-1 {
|
||||
runtimePanic("all cores are sleeping - deadlock!")
|
||||
}
|
||||
|
||||
// Need to disable interrupts while saving the checkpoint, otherwise if the
|
||||
// software interrupt happens earlier for another reason (e.g. a GC cycle)
|
||||
// it will see an incomplete checkpoint and the schedulerLock might not be
|
||||
// unlocked yet. That will lead to an invalid state.
|
||||
riscv.MSTATUS.ClearBits(riscv.MSTATUS_MIE)
|
||||
if schedulerWaitCheckpoints[hartID].Save() {
|
||||
schedulerLock.Unlock()
|
||||
riscv.MSTATUS.SetBits(riscv.MSTATUS_MIE)
|
||||
|
||||
// Wait until we get awoken :)
|
||||
for {
|
||||
riscv.Asm("wfi")
|
||||
}
|
||||
}
|
||||
|
||||
// We got awoken again. We need to lock the scheduler again before
|
||||
// returning.
|
||||
schedulerLock.Lock()
|
||||
}
|
||||
|
||||
// Wake another core, if one is sleeping. Must be called with the scheduler lock
|
||||
// held.
|
||||
func schedulerWake() {
|
||||
// Look up the lowest-numbered hart that is sleeping.
|
||||
// Returns 8 if there are no sleeping harts.
|
||||
hart := bits.TrailingZeros8(sleepingHarts)
|
||||
|
||||
if hart < 8 {
|
||||
// There is a sleeping hart. Wake it.
|
||||
sleepingHarts &^= 1 << hart // clear the bit
|
||||
aclintMSWI.MSIP[hart].Set(1) // send software interrupt
|
||||
}
|
||||
}
|
||||
|
||||
// Pause the given core by sending it an interrupt.
|
||||
func gcPauseCore(core uint32) {
|
||||
aclintMSWI.MSIP[core].Set(1) // send software interrupt
|
||||
}
|
||||
|
||||
// Signal the given core that it can resume one step.
|
||||
// This is called twice after gcPauseCore: the first time to scan the stack of
|
||||
// the core, and the second time to end the stop-the-world phase.
|
||||
func gcSignalCore(core uint32) {
|
||||
aclintMSWI.MSIP[core].Set(1) // send software interrupt
|
||||
}
|
||||
|
||||
func abort() {
|
||||
exit(1)
|
||||
}
|
||||
|
||||
// Zero in the default state, when non-zero it indicates the exit code plus one.
|
||||
// So exit(0) will result in 1, exit(1) in 2, etc.
|
||||
var exitCodePlusOne atomic.Uint32
|
||||
|
||||
func exit(code int) {
|
||||
// Check for invalid values, to be sure.
|
||||
if code < 0 {
|
||||
code = 255
|
||||
}
|
||||
|
||||
// If we're not on hart 0, we can't exit QEMU.
|
||||
// Therefore, send an interrupt to hart 0 instead to request an exit.
|
||||
if currentCPU() != 0 {
|
||||
// Signal hart 0 to exit.
|
||||
exitCodePlusOne.Store(uint32(code) + 1)
|
||||
aclintMSWI.MSIP[0].Set(1)
|
||||
|
||||
// Wait for the interrupt to happen. This should happen immediately.
|
||||
for {
|
||||
riscv.Asm("wfi")
|
||||
}
|
||||
}
|
||||
|
||||
exitNow(uint32(code))
|
||||
}
|
||||
|
||||
// Send an exit signal to the test finisher pseudo-device, without checking
|
||||
// whether we are on hart 0.
|
||||
func exitNow(code uint32) {
|
||||
// Make sure the QEMU process exits.
|
||||
if code == 0 {
|
||||
testFinisher.Set(0x5555) // FINISHER_PASS
|
||||
} else {
|
||||
// Exit code is stored in the upper 16 bits of the 32 bit value.
|
||||
testFinisher.Set(uint32(code)<<16 | 0x3333) // FINISHER_FAIL
|
||||
testFinisher.Set(code<<16 | 0x3333) // FINISHER_FAIL
|
||||
}
|
||||
|
||||
// Lock up forever (as a fallback).
|
||||
@@ -162,10 +519,6 @@ func exit(code int) {
|
||||
func handleException(code uint) {
|
||||
// For a list of exception codes, see:
|
||||
// https://content.riscv.org/wp-content/uploads/2019/08/riscv-privileged-20190608-1.pdf#page=49
|
||||
print("fatal error: exception with mcause=")
|
||||
print(code)
|
||||
print(" pc=")
|
||||
print(riscv.MEPC.Get())
|
||||
println()
|
||||
print("fatal error: exception with mcause=", code, " pc=", riscv.MEPC.Get(), " hart=", uint(riscv.MHARTID.Get()), "\r\n")
|
||||
abort()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user