From 64caab1adeea14b2561d7a4f26920ef751fd05c0 Mon Sep 17 00:00:00 2001 From: Michael Smith Date: Thu, 7 Aug 2025 18:12:53 -0400 Subject: [PATCH] feat: enable multi-core scheduler for rp2350 --- src/runtime/runtime_rp2.go | 367 ++++++++++++++++++++++++++++++++++ src/runtime/runtime_rp2040.go | 361 +-------------------------------- src/runtime/runtime_rp2350.go | 88 +------- targets/rp2350.json | 4 +- 4 files changed, 382 insertions(+), 438 deletions(-) create mode 100644 src/runtime/runtime_rp2.go diff --git a/src/runtime/runtime_rp2.go b/src/runtime/runtime_rp2.go new file mode 100644 index 000000000..93e488d02 --- /dev/null +++ b/src/runtime/runtime_rp2.go @@ -0,0 +1,367 @@ +//go:build rp2040 || rp2350 + +package runtime + +import ( + "device/arm" + "device/rp" + "internal/task" + "machine" + "machine/usb/cdc" + "runtime/interrupt" + "runtime/volatile" + "unsafe" +) + +const numCPU = 2 + +// machineTicks is provided by package machine. +func machineTicks() uint64 + +// machineLightSleep is provided by package machine. +func machineLightSleep(uint64) + +// ticks returns the number of ticks (microseconds) elapsed since power up. +func ticks() timeUnit { + t := machineTicks() + return timeUnit(t) +} + +func ticksToNanoseconds(ticks timeUnit) int64 { + return int64(ticks) * 1000 +} + +func nanosecondsToTicks(ns int64) timeUnit { + return timeUnit(ns / 1000) +} + +func sleepTicks(d timeUnit) { + if hasScheduler { + // With scheduler, sleepTicks may return early if an interrupt or + // event fires - so scheduler can schedule any go routines now + // eligible to run + machineLightSleep(uint64(d)) + return + } + + // Busy loop + sleepUntil := ticks() + d + for ticks() < sleepUntil { + } +} + +// Currently sleeping core, or 0xff. +// Must only be accessed with the scheduler lock held. +var sleepingCore uint8 = 0xff + +// Return whether another core is sleeping. +// May only be called with the scheduler lock held. +func hasSleepingCore() bool { + return sleepingCore != 0xff +} + +// Almost identical to sleepTicks, except that it will unlock/lock the scheduler +// while sleeping and is interruptible by interruptSleepTicksMulticore. +// This may only be called with the scheduler lock held. +func sleepTicksMulticore(d timeUnit) { + sleepingCore = uint8(currentCPU()) + + // Note: interruptSleepTicksMulticore will be able to interrupt this, since + // it executes the "sev" instruction which would make sleepTicks return + // immediately without sleeping. Even if it happens while configuring the + // sleep operation. + + schedulerLock.Unlock() + sleepTicks(d) + schedulerLock.Lock() + + sleepingCore = 0xff +} + +// Interrupt an ongoing call to sleepTicksMulticore on another core. +func interruptSleepTicksMulticore(wakeup timeUnit) { + arm.Asm("sev") +} + +// Number of cores that are currently in schedulerUnlockAndWait. +// It is possible for both cores to be sleeping, if the program is waiting for +// an interrupt (or is deadlocked). +var waitingCore uint8 + +// Put the scheduler to sleep, since there are no tasks to run. +// This will unlock the scheduler lock, and must be called with the scheduler +// lock held. +func schedulerUnlockAndWait() { + waitingCore++ + schedulerLock.Unlock() + arm.Asm("wfe") + schedulerLock.Lock() + waitingCore-- +} + +// Wake another core, if one is sleeping. Must be called with the scheduler lock +// held. +func schedulerWake() { + if waitingCore != 0 { + arm.Asm("sev") + } +} + +// Return the current core number: 0 or 1. +func currentCPU() uint32 { + return rp.SIO.CPUID.Get() +} + +// Start the secondary cores for this chip. +// On the RP2040/RP2350, there is only one other core to start. +func startSecondaryCores() { + // Start the second core of the RP2040/RP2350. + // See sections 2.8.2 and 5.3 in the datasheets for RP2040 and RP2350 respectively. + seq := 0 + for { + cmd := core1StartSequence[seq] + if cmd == 0 { + multicore_fifo_drain() + arm.Asm("sev") + } + multicore_fifo_push_blocking(cmd) + response := multicore_fifo_pop_blocking() + if cmd != response { + seq = 0 + continue + } + seq = seq + 1 + if seq >= len(core1StartSequence) { + break + } + } + + // Enable the FIFO interrupt for the GC stop the world phase. + // We can only do this after we don't need the FIFO anymore for starting the + // second core. + intr := interrupt.New(sioIrqFifoProc0, func(intr interrupt.Interrupt) { + switch rp.SIO.FIFO_RD.Get() { + case 1: + gcInterruptHandler(0) + } + }) + intr.Enable() + intr.SetPriority(0xff) +} + +var core1StartSequence = [...]uint32{ + 0, 0, 1, + uint32(uintptr(unsafe.Pointer(&__isr_vector))), + uint32(uintptr(unsafe.Pointer(&stack1TopSymbol))), + uint32(exportedFuncPtr(runCore1)), +} + +//go:extern __isr_vector +var __isr_vector [0]uint32 + +//go:extern _stack1_top +var stack1TopSymbol [0]uint32 + +// The function that is started on the second core. +// +//export tinygo_runCore1 +func runCore1() { + // Clear sticky bit that seems to have been set while starting this core. + rp.SIO.FIFO_ST.Set(rp.SIO_FIFO_ST_ROE) + + // Enable the FIFO interrupt, mainly used for the stop-the-world phase of + // the GC. + // Use the lowest possible priority (highest priority value), so that other + // interrupts can still happen while the GC is running. + intr := interrupt.New(sioIrqFifoProc1, func(intr interrupt.Interrupt) { + switch rp.SIO.FIFO_RD.Get() { + case 1: + gcInterruptHandler(1) + } + }) + intr.Enable() + intr.SetPriority(0xff) + + // Now start running the scheduler on this core. + schedulerLock.Lock() + scheduler(false) + schedulerLock.Unlock() + + // The main function returned. + exit(0) +} + +// The below multicore_fifo_* functions have been translated from the Raspberry +// Pi Pico SDK. + +func multicore_fifo_rvalid() bool { + return rp.SIO.FIFO_ST.Get()&rp.SIO_FIFO_ST_VLD != 0 +} + +func multicore_fifo_wready() bool { + return rp.SIO.FIFO_ST.Get()&rp.SIO_FIFO_ST_RDY != 0 +} + +func multicore_fifo_drain() { + for multicore_fifo_rvalid() { + rp.SIO.FIFO_RD.Get() + } +} + +func multicore_fifo_push_blocking(data uint32) { + for !multicore_fifo_wready() { + } + rp.SIO.FIFO_WR.Set(data) + arm.Asm("sev") +} + +func multicore_fifo_pop_blocking() uint32 { + for !multicore_fifo_rvalid() { + arm.Asm("wfe") + } + + return rp.SIO.FIFO_RD.Get() +} + +// Value used to communicate between the GC core and the other (paused) cores. +var gcSignalWait volatile.Register8 + +// The GC interrupted this core for the stop-the-world phase. +// This function handles that, and only returns after the stop-the-world phase +// ended. +func gcInterruptHandler(hartID uint32) { + // Let the GC know we're ready. + gcScanState.Add(1) + arm.Asm("sev") + + // Wait until we get a signal to start scanning. + for gcSignalWait.Get() == 0 { + arm.Asm("wfe") + } + gcSignalWait.Set(0) + + // Scan the stack(s) of this core. + scanCurrentStack() + if !task.OnSystemStack() { + // Mark system stack. + markRoots(task.SystemStack(), coreStackTop(hartID)) + } + + // Signal we've finished scanning. + gcScanState.Store(1) + arm.Asm("sev") + + // Wait until we get a signal that the stop-the-world phase has ended. + for gcSignalWait.Get() == 0 { + arm.Asm("wfe") + } + gcSignalWait.Set(0) + + // Signal we received the signal and are going to exit the interrupt. + gcScanState.Add(1) + arm.Asm("sev") +} + +// Pause the given core by sending it an interrupt. +func gcPauseCore(core uint32) { + rp.SIO.FIFO_WR.Set(1) +} + +// Signal the given core that it can resume one step. +// This is called twice after gcPauseCore: the first time to scan the stack of +// the core, and the second time to end the stop-the-world phase. +func gcSignalCore(core uint32) { + gcSignalWait.Set(1) + arm.Asm("sev") +} + +// Returns the stack top (highest address) of the system stack of the given +// core. +func coreStackTop(core uint32) uintptr { + switch core { + case 0: + return uintptr(unsafe.Pointer(&stackTopSymbol)) + case 1: + return uintptr(unsafe.Pointer(&stack1TopSymbol)) + default: + runtimePanic("unexpected core") + return 0 + } +} + +// These spinlocks are needed by the runtime. +var ( + printLock = spinLock{id: 0} + schedulerLock = spinLock{id: 1} + atomicsLock = spinLock{id: 2} + futexLock = spinLock{id: 3} +) + +// A hardware spinlock, one of the 32 spinlocks defined in the SIO peripheral. +type spinLock struct { + id uint8 +} + +// Return the spinlock register: rp.SIO.SPINLOCKx +func (l *spinLock) spinlock() *volatile.Register32 { + return (*volatile.Register32)(unsafe.Add(unsafe.Pointer(&rp.SIO.SPINLOCK0), l.id*4)) +} + +func (l *spinLock) Lock() { + // Wait for the lock to be available. + spinlock := l.spinlock() + for spinlock.Get() == 0 { + // TODO: use wfe and send an event when unlocking so the CPU can go to + // sleep while waiting for the lock. + // Unfortunately when doing that, time.Sleep() seems to hang somewhere. + // This needs some debugging to figure out. + } +} + +func (l *spinLock) Unlock() { + l.spinlock().Set(0) +} + +// Wait until a signal is received, indicating that it can resume from the +// spinloop. +func spinLoopWait() { + arm.Asm("wfe") +} + +func waitForEvents() { + arm.Asm("wfe") +} + +func putchar(c byte) { + machine.Serial.WriteByte(c) +} + +func getchar() byte { + for machine.Serial.Buffered() == 0 { + Gosched() + } + v, _ := machine.Serial.ReadByte() + return v +} + +func buffered() int { + return machine.Serial.Buffered() +} + +// machineInit is provided by package machine. +func machineInit() + +func init() { + machineInit() + + cdc.EnableUSBCDC() + machine.USBDev.Configure(machine.UARTConfig{}) + machine.InitSerial() +} + +//export Reset_Handler +func main() { + preinit() + run() + exit(0) +} diff --git a/src/runtime/runtime_rp2040.go b/src/runtime/runtime_rp2040.go index ae32bf3a4..2ca3605e0 100644 --- a/src/runtime/runtime_rp2040.go +++ b/src/runtime/runtime_rp2040.go @@ -3,365 +3,10 @@ package runtime import ( - "device/arm" "device/rp" - "internal/task" - "machine" - "machine/usb/cdc" - "runtime/interrupt" - "runtime/volatile" - "unsafe" ) -const numCPU = 2 - -// machineTicks is provided by package machine. -func machineTicks() uint64 - -// machineLightSleep is provided by package machine. -func machineLightSleep(uint64) - -// ticks returns the number of ticks (microseconds) elapsed since power up. -func ticks() timeUnit { - t := machineTicks() - return timeUnit(t) -} - -func ticksToNanoseconds(ticks timeUnit) int64 { - return int64(ticks) * 1000 -} - -func nanosecondsToTicks(ns int64) timeUnit { - return timeUnit(ns / 1000) -} - -func sleepTicks(d timeUnit) { - if hasScheduler { - // With scheduler, sleepTicks may return early if an interrupt or - // event fires - so scheduler can schedule any go routines now - // eligible to run - machineLightSleep(uint64(d)) - return - } - - // Busy loop - sleepUntil := ticks() + d - for ticks() < sleepUntil { - } -} - -// Currently sleeping core, or 0xff. -// Must only be accessed with the scheduler lock held. -var sleepingCore uint8 = 0xff - -// Return whether another core is sleeping. -// May only be called with the scheduler lock held. -func hasSleepingCore() bool { - return sleepingCore != 0xff -} - -// Almost identical to sleepTicks, except that it will unlock/lock the scheduler -// while sleeping and is interruptible by interruptSleepTicksMulticore. -// This may only be called with the scheduler lock held. -func sleepTicksMulticore(d timeUnit) { - sleepingCore = uint8(currentCPU()) - - // Note: interruptSleepTicksMulticore will be able to interrupt this, since - // it executes the "sev" instruction which would make sleepTicks return - // immediately without sleeping. Even if it happens while configuring the - // sleep operation. - - schedulerLock.Unlock() - sleepTicks(d) - schedulerLock.Lock() - - sleepingCore = 0xff -} - -// Interrupt an ongoing call to sleepTicksMulticore on another core. -func interruptSleepTicksMulticore(wakeup timeUnit) { - arm.Asm("sev") -} - -// Number of cores that are currently in schedulerUnlockAndWait. -// It is possible for both cores to be sleeping, if the program is waiting for -// an interrupt (or is deadlocked). -var waitingCore uint8 - -// Put the scheduler to sleep, since there are no tasks to run. -// This will unlock the scheduler lock, and must be called with the scheduler -// lock held. -func schedulerUnlockAndWait() { - waitingCore++ - schedulerLock.Unlock() - arm.Asm("wfe") - schedulerLock.Lock() - waitingCore-- -} - -// Wake another core, if one is sleeping. Must be called with the scheduler lock -// held. -func schedulerWake() { - if waitingCore != 0 { - arm.Asm("sev") - } -} - -// Return the current core number: 0 or 1. -func currentCPU() uint32 { - return rp.SIO.CPUID.Get() -} - -// Start the secondary cores for this chip. -// On the RP2040, there is only one other core to start. -func startSecondaryCores() { - // Start the second core of the RP2040. - // See section 2.8.2 in the datasheet. - seq := 0 - for { - cmd := core1StartSequence[seq] - if cmd == 0 { - multicore_fifo_drain() - arm.Asm("sev") - } - multicore_fifo_push_blocking(cmd) - response := multicore_fifo_pop_blocking() - if cmd != response { - seq = 0 - continue - } - seq = seq + 1 - if seq >= len(core1StartSequence) { - break - } - } - - // Enable the FIFO interrupt for the GC stop the world phase. - // We can only do this after we don't need the FIFO anymore for starting the - // second core. - intr := interrupt.New(rp.IRQ_SIO_IRQ_PROC0, func(intr interrupt.Interrupt) { - switch rp.SIO.FIFO_RD.Get() { - case 1: - gcInterruptHandler(0) - } - }) - intr.Enable() - intr.SetPriority(0xff) -} - -var core1StartSequence = [...]uint32{ - 0, 0, 1, - uint32(uintptr(unsafe.Pointer(&__isr_vector))), - uint32(uintptr(unsafe.Pointer(&stack1TopSymbol))), - uint32(exportedFuncPtr(runCore1)), -} - -//go:extern __isr_vector -var __isr_vector [0]uint32 - -//go:extern _stack1_top -var stack1TopSymbol [0]uint32 - -// The function that is started on the second core. -// -//export tinygo_runCore1 -func runCore1() { - // Clear sticky bit that seems to have been set while starting this core. - rp.SIO.FIFO_ST.Set(rp.SIO_FIFO_ST_ROE) - - // Enable the FIFO interrupt, mainly used for the stop-the-world phase of - // the GC. - // Use the lowest possible priority (highest priority value), so that other - // interrupts can still happen while the GC is running. - intr := interrupt.New(rp.IRQ_SIO_IRQ_PROC1, func(intr interrupt.Interrupt) { - switch rp.SIO.FIFO_RD.Get() { - case 1: - gcInterruptHandler(1) - } - }) - intr.Enable() - intr.SetPriority(0xff) - - // Now start running the scheduler on this core. - schedulerLock.Lock() - scheduler(false) - schedulerLock.Unlock() - - // The main function returned. - exit(0) -} - -// The below multicore_fifo_* functions have been translated from the Raspberry -// Pi Pico SDK. - -func multicore_fifo_rvalid() bool { - return rp.SIO.FIFO_ST.Get()&rp.SIO_FIFO_ST_VLD != 0 -} - -func multicore_fifo_wready() bool { - return rp.SIO.FIFO_ST.Get()&rp.SIO_FIFO_ST_RDY != 0 -} - -func multicore_fifo_drain() { - for multicore_fifo_rvalid() { - rp.SIO.FIFO_RD.Get() - } -} - -func multicore_fifo_push_blocking(data uint32) { - for !multicore_fifo_wready() { - } - rp.SIO.FIFO_WR.Set(data) - arm.Asm("sev") -} - -func multicore_fifo_pop_blocking() uint32 { - for !multicore_fifo_rvalid() { - arm.Asm("wfe") - } - - return rp.SIO.FIFO_RD.Get() -} - -// Value used to communicate between the GC core and the other (paused) cores. -var gcSignalWait volatile.Register8 - -// The GC interrupted this core for the stop-the-world phase. -// This function handles that, and only returns after the stop-the-world phase -// ended. -func gcInterruptHandler(hartID uint32) { - // Let the GC know we're ready. - gcScanState.Add(1) - arm.Asm("sev") - - // Wait until we get a signal to start scanning. - for gcSignalWait.Get() == 0 { - arm.Asm("wfe") - } - gcSignalWait.Set(0) - - // Scan the stack(s) of this core. - scanCurrentStack() - if !task.OnSystemStack() { - // Mark system stack. - markRoots(task.SystemStack(), coreStackTop(hartID)) - } - - // Signal we've finished scanning. - gcScanState.Store(1) - arm.Asm("sev") - - // Wait until we get a signal that the stop-the-world phase has ended. - for gcSignalWait.Get() == 0 { - arm.Asm("wfe") - } - gcSignalWait.Set(0) - - // Signal we received the signal and are going to exit the interrupt. - gcScanState.Add(1) - arm.Asm("sev") -} - -// Pause the given core by sending it an interrupt. -func gcPauseCore(core uint32) { - rp.SIO.FIFO_WR.Set(1) -} - -// Signal the given core that it can resume one step. -// This is called twice after gcPauseCore: the first time to scan the stack of -// the core, and the second time to end the stop-the-world phase. -func gcSignalCore(core uint32) { - gcSignalWait.Set(1) - arm.Asm("sev") -} - -// Returns the stack top (highest address) of the system stack of the given -// core. -func coreStackTop(core uint32) uintptr { - switch core { - case 0: - return uintptr(unsafe.Pointer(&stackTopSymbol)) - case 1: - return uintptr(unsafe.Pointer(&stack1TopSymbol)) - default: - runtimePanic("unexpected core") - return 0 - } -} - -// These spinlocks are needed by the runtime. -var ( - printLock = spinLock{id: 0} - schedulerLock = spinLock{id: 1} - atomicsLock = spinLock{id: 2} - futexLock = spinLock{id: 3} +const ( + sioIrqFifoProc0 = rp.IRQ_SIO_IRQ_PROC0 + sioIrqFifoProc1 = rp.IRQ_SIO_IRQ_PROC1 ) - -// A hardware spinlock, one of the 32 spinlocks defined in the SIO peripheral. -type spinLock struct { - id uint8 -} - -// Return the spinlock register: rp.SIO.SPINLOCKx -func (l *spinLock) spinlock() *volatile.Register32 { - return (*volatile.Register32)(unsafe.Add(unsafe.Pointer(&rp.SIO.SPINLOCK0), l.id*4)) -} - -func (l *spinLock) Lock() { - // Wait for the lock to be available. - spinlock := l.spinlock() - for spinlock.Get() == 0 { - // TODO: use wfe and send an event when unlocking so the CPU can go to - // sleep while waiting for the lock. - // Unfortunately when doing that, time.Sleep() seems to hang somewhere. - // This needs some debugging to figure out. - } -} - -func (l *spinLock) Unlock() { - l.spinlock().Set(0) -} - -// Wait until a signal is received, indicating that it can resume from the -// spinloop. -func spinLoopWait() { - arm.Asm("wfe") -} - -func waitForEvents() { - arm.Asm("wfe") -} - -func putchar(c byte) { - machine.Serial.WriteByte(c) -} - -func getchar() byte { - for machine.Serial.Buffered() == 0 { - Gosched() - } - v, _ := machine.Serial.ReadByte() - return v -} - -func buffered() int { - return machine.Serial.Buffered() -} - -// machineInit is provided by package machine. -func machineInit() - -func init() { - machineInit() - - cdc.EnableUSBCDC() - machine.USBDev.Configure(machine.UARTConfig{}) - machine.InitSerial() -} - -//export Reset_Handler -func main() { - preinit() - run() - exit(0) -} diff --git a/src/runtime/runtime_rp2350.go b/src/runtime/runtime_rp2350.go index 500e4c9f2..91af23212 100644 --- a/src/runtime/runtime_rp2350.go +++ b/src/runtime/runtime_rp2350.go @@ -3,84 +3,14 @@ package runtime import ( - "device/arm" - "machine" - "machine/usb/cdc" + "device/rp" ) -// machineTicks is provided by package machine. -func machineTicks() uint64 - -// machineLightSleep is provided by package machine. -func machineLightSleep(uint64) - -// ticks returns the number of ticks (microseconds) elapsed since power up. -func ticks() timeUnit { - t := machineTicks() - return timeUnit(t) -} - -func ticksToNanoseconds(ticks timeUnit) int64 { - return int64(ticks) * 1000 -} - -func nanosecondsToTicks(ns int64) timeUnit { - return timeUnit(ns / 1000) -} - -func sleepTicks(d timeUnit) { - if d <= 0 { - return - } - - if hasScheduler { - // With scheduler, sleepTicks may return early if an interrupt or - // event fires - so scheduler can schedule any go routines now - // eligible to run - machineLightSleep(uint64(d)) - return - } - - // Busy loop - sleepUntil := ticks() + d - for ticks() < sleepUntil { - } -} - -func waitForEvents() { - arm.Asm("wfe") -} - -func putchar(c byte) { - machine.Serial.WriteByte(c) -} - -func getchar() byte { - for machine.Serial.Buffered() == 0 { - Gosched() - } - v, _ := machine.Serial.ReadByte() - return v -} - -func buffered() int { - return machine.Serial.Buffered() -} - -// machineInit is provided by package machine. -func machineInit() - -func init() { - machineInit() - - cdc.EnableUSBCDC() - machine.USBDev.Configure(machine.UARTConfig{}) - machine.InitSerial() -} - -//export Reset_Handler -func main() { - preinit() - run() - exit(0) -} +const ( + // On RP2040 each core has a different IRQ number: SIO_IRQ_PROC0 and SIO_IRQ_PROC1. + // On RP2350 both cores share the same irq number (SIO_IRQ_PROC) just with a + // different SIO interrupt output routed to that IRQ input on each core. + // https://www.raspberrypi.com/documentation/pico-sdk/high_level.html#group_pico_multicore_1ga1413ebfa65114c6f408f4675897ac5ee + sioIrqFifoProc0 = rp.IRQ_SIO_IRQ_FIFO + sioIrqFifoProc1 = rp.IRQ_SIO_IRQ_FIFO +) diff --git a/targets/rp2350.json b/targets/rp2350.json index 0487aa14d..ebffcbdec 100644 --- a/targets/rp2350.json +++ b/targets/rp2350.json @@ -1,6 +1,7 @@ { "inherits": ["cortex-m33"], "build-tags": ["rp2350", "rp"], + "scheduler": "cores", "flash-1200-bps-reset": "true", "flash-method": "msd", "serial": "usb", @@ -13,7 +14,8 @@ "targets/rp2350_embedded_block.s" ], "ldflags": [ - "--defsym=__flash_size=2M" + "--defsym=__flash_size=2M", + "--defsym=__num_stacks=2" ], "linkerscript": "targets/rp2350.ld", "openocd-interface": "picoprobe",