diff --git a/src/internal/task/task_stack_cortexm.c b/src/internal/task/task_stack_cortexm.c index a14513d29..3c7e95797 100644 --- a/src/internal/task/task_stack_cortexm.c +++ b/src/internal/task/task_stack_cortexm.c @@ -1,4 +1,4 @@ -//go:build scheduler.tasks && cortexm +//go:build (scheduler.tasks || scheduler.cores) && cortexm #include uintptr_t SystemStack() { diff --git a/src/internal/task/task_stack_cortexm.go b/src/internal/task/task_stack_cortexm.go index 226a088c8..653dc06e1 100644 --- a/src/internal/task/task_stack_cortexm.go +++ b/src/internal/task/task_stack_cortexm.go @@ -1,4 +1,4 @@ -//go:build scheduler.tasks && cortexm +//go:build (scheduler.tasks || scheduler.cores) && cortexm package task diff --git a/src/runtime/gc_stack_cores.go b/src/runtime/gc_stack_cores.go index faabfb0c6..b14a54fef 100644 --- a/src/runtime/gc_stack_cores.go +++ b/src/runtime/gc_stack_cores.go @@ -56,7 +56,7 @@ func gcMarkReachable() { // Busy-wait until all the other cores are ready. They certainly should be, // after the scanning we did above. for gcScanState.Load() != numCPU { - spinLoopHint() + spinLoopWait() } gcScanState.Store(0) @@ -71,7 +71,7 @@ func gcMarkReachable() { // Busy-wait until this core finished scanning. for gcScanState.Load() == 0 { - spinLoopHint() + spinLoopWait() } gcScanState.Store(0) } @@ -118,7 +118,7 @@ func gcResumeWorld() { // Busy-wait until the core acknowledges the signal (and is going to return // from the interrupt handler). for gcScanState.Load() != numCPU-1 { - spinLoopHint() + spinLoopWait() } gcScanState.Store(0) } diff --git a/src/runtime/runtime_rp2040.go b/src/runtime/runtime_rp2040.go index fb1d1ddbd..ae32bf3a4 100644 --- a/src/runtime/runtime_rp2040.go +++ b/src/runtime/runtime_rp2040.go @@ -4,10 +4,17 @@ package runtime import ( "device/arm" + "device/rp" + "internal/task" "machine" "machine/usb/cdc" + "runtime/interrupt" + "runtime/volatile" + "unsafe" ) +const numCPU = 2 + // machineTicks is provided by package machine. func machineTicks() uint64 @@ -43,6 +50,284 @@ func sleepTicks(d timeUnit) { } } +// Currently sleeping core, or 0xff. +// Must only be accessed with the scheduler lock held. +var sleepingCore uint8 = 0xff + +// Return whether another core is sleeping. +// May only be called with the scheduler lock held. +func hasSleepingCore() bool { + return sleepingCore != 0xff +} + +// Almost identical to sleepTicks, except that it will unlock/lock the scheduler +// while sleeping and is interruptible by interruptSleepTicksMulticore. +// This may only be called with the scheduler lock held. +func sleepTicksMulticore(d timeUnit) { + sleepingCore = uint8(currentCPU()) + + // Note: interruptSleepTicksMulticore will be able to interrupt this, since + // it executes the "sev" instruction which would make sleepTicks return + // immediately without sleeping. Even if it happens while configuring the + // sleep operation. + + schedulerLock.Unlock() + sleepTicks(d) + schedulerLock.Lock() + + sleepingCore = 0xff +} + +// Interrupt an ongoing call to sleepTicksMulticore on another core. +func interruptSleepTicksMulticore(wakeup timeUnit) { + arm.Asm("sev") +} + +// Number of cores that are currently in schedulerUnlockAndWait. +// It is possible for both cores to be sleeping, if the program is waiting for +// an interrupt (or is deadlocked). +var waitingCore uint8 + +// Put the scheduler to sleep, since there are no tasks to run. +// This will unlock the scheduler lock, and must be called with the scheduler +// lock held. +func schedulerUnlockAndWait() { + waitingCore++ + schedulerLock.Unlock() + arm.Asm("wfe") + schedulerLock.Lock() + waitingCore-- +} + +// Wake another core, if one is sleeping. Must be called with the scheduler lock +// held. +func schedulerWake() { + if waitingCore != 0 { + arm.Asm("sev") + } +} + +// Return the current core number: 0 or 1. +func currentCPU() uint32 { + return rp.SIO.CPUID.Get() +} + +// Start the secondary cores for this chip. +// On the RP2040, there is only one other core to start. +func startSecondaryCores() { + // Start the second core of the RP2040. + // See section 2.8.2 in the datasheet. + seq := 0 + for { + cmd := core1StartSequence[seq] + if cmd == 0 { + multicore_fifo_drain() + arm.Asm("sev") + } + multicore_fifo_push_blocking(cmd) + response := multicore_fifo_pop_blocking() + if cmd != response { + seq = 0 + continue + } + seq = seq + 1 + if seq >= len(core1StartSequence) { + break + } + } + + // Enable the FIFO interrupt for the GC stop the world phase. + // We can only do this after we don't need the FIFO anymore for starting the + // second core. + intr := interrupt.New(rp.IRQ_SIO_IRQ_PROC0, func(intr interrupt.Interrupt) { + switch rp.SIO.FIFO_RD.Get() { + case 1: + gcInterruptHandler(0) + } + }) + intr.Enable() + intr.SetPriority(0xff) +} + +var core1StartSequence = [...]uint32{ + 0, 0, 1, + uint32(uintptr(unsafe.Pointer(&__isr_vector))), + uint32(uintptr(unsafe.Pointer(&stack1TopSymbol))), + uint32(exportedFuncPtr(runCore1)), +} + +//go:extern __isr_vector +var __isr_vector [0]uint32 + +//go:extern _stack1_top +var stack1TopSymbol [0]uint32 + +// The function that is started on the second core. +// +//export tinygo_runCore1 +func runCore1() { + // Clear sticky bit that seems to have been set while starting this core. + rp.SIO.FIFO_ST.Set(rp.SIO_FIFO_ST_ROE) + + // Enable the FIFO interrupt, mainly used for the stop-the-world phase of + // the GC. + // Use the lowest possible priority (highest priority value), so that other + // interrupts can still happen while the GC is running. + intr := interrupt.New(rp.IRQ_SIO_IRQ_PROC1, func(intr interrupt.Interrupt) { + switch rp.SIO.FIFO_RD.Get() { + case 1: + gcInterruptHandler(1) + } + }) + intr.Enable() + intr.SetPriority(0xff) + + // Now start running the scheduler on this core. + schedulerLock.Lock() + scheduler(false) + schedulerLock.Unlock() + + // The main function returned. + exit(0) +} + +// The below multicore_fifo_* functions have been translated from the Raspberry +// Pi Pico SDK. + +func multicore_fifo_rvalid() bool { + return rp.SIO.FIFO_ST.Get()&rp.SIO_FIFO_ST_VLD != 0 +} + +func multicore_fifo_wready() bool { + return rp.SIO.FIFO_ST.Get()&rp.SIO_FIFO_ST_RDY != 0 +} + +func multicore_fifo_drain() { + for multicore_fifo_rvalid() { + rp.SIO.FIFO_RD.Get() + } +} + +func multicore_fifo_push_blocking(data uint32) { + for !multicore_fifo_wready() { + } + rp.SIO.FIFO_WR.Set(data) + arm.Asm("sev") +} + +func multicore_fifo_pop_blocking() uint32 { + for !multicore_fifo_rvalid() { + arm.Asm("wfe") + } + + return rp.SIO.FIFO_RD.Get() +} + +// Value used to communicate between the GC core and the other (paused) cores. +var gcSignalWait volatile.Register8 + +// The GC interrupted this core for the stop-the-world phase. +// This function handles that, and only returns after the stop-the-world phase +// ended. +func gcInterruptHandler(hartID uint32) { + // Let the GC know we're ready. + gcScanState.Add(1) + arm.Asm("sev") + + // Wait until we get a signal to start scanning. + for gcSignalWait.Get() == 0 { + arm.Asm("wfe") + } + gcSignalWait.Set(0) + + // Scan the stack(s) of this core. + scanCurrentStack() + if !task.OnSystemStack() { + // Mark system stack. + markRoots(task.SystemStack(), coreStackTop(hartID)) + } + + // Signal we've finished scanning. + gcScanState.Store(1) + arm.Asm("sev") + + // Wait until we get a signal that the stop-the-world phase has ended. + for gcSignalWait.Get() == 0 { + arm.Asm("wfe") + } + gcSignalWait.Set(0) + + // Signal we received the signal and are going to exit the interrupt. + gcScanState.Add(1) + arm.Asm("sev") +} + +// Pause the given core by sending it an interrupt. +func gcPauseCore(core uint32) { + rp.SIO.FIFO_WR.Set(1) +} + +// Signal the given core that it can resume one step. +// This is called twice after gcPauseCore: the first time to scan the stack of +// the core, and the second time to end the stop-the-world phase. +func gcSignalCore(core uint32) { + gcSignalWait.Set(1) + arm.Asm("sev") +} + +// Returns the stack top (highest address) of the system stack of the given +// core. +func coreStackTop(core uint32) uintptr { + switch core { + case 0: + return uintptr(unsafe.Pointer(&stackTopSymbol)) + case 1: + return uintptr(unsafe.Pointer(&stack1TopSymbol)) + default: + runtimePanic("unexpected core") + return 0 + } +} + +// These spinlocks are needed by the runtime. +var ( + printLock = spinLock{id: 0} + schedulerLock = spinLock{id: 1} + atomicsLock = spinLock{id: 2} + futexLock = spinLock{id: 3} +) + +// A hardware spinlock, one of the 32 spinlocks defined in the SIO peripheral. +type spinLock struct { + id uint8 +} + +// Return the spinlock register: rp.SIO.SPINLOCKx +func (l *spinLock) spinlock() *volatile.Register32 { + return (*volatile.Register32)(unsafe.Add(unsafe.Pointer(&rp.SIO.SPINLOCK0), l.id*4)) +} + +func (l *spinLock) Lock() { + // Wait for the lock to be available. + spinlock := l.spinlock() + for spinlock.Get() == 0 { + // TODO: use wfe and send an event when unlocking so the CPU can go to + // sleep while waiting for the lock. + // Unfortunately when doing that, time.Sleep() seems to hang somewhere. + // This needs some debugging to figure out. + } +} + +func (l *spinLock) Unlock() { + l.spinlock().Set(0) +} + +// Wait until a signal is received, indicating that it can resume from the +// spinloop. +func spinLoopWait() { + arm.Asm("wfe") +} + func waitForEvents() { arm.Asm("wfe") } diff --git a/src/runtime/runtime_tinygoriscv_qemu.go b/src/runtime/runtime_tinygoriscv_qemu.go index 09d67b624..1ee0d22d8 100644 --- a/src/runtime/runtime_tinygoriscv_qemu.go +++ b/src/runtime/runtime_tinygoriscv_qemu.go @@ -360,7 +360,7 @@ type spinLock struct { func (l *spinLock) Lock() { // Try to replace 0 with 1. Once we succeed, the lock has been acquired. for !l.Uint32.CompareAndSwap(0, 1) { - spinLoopHint() + spinLoopWait() } } @@ -376,7 +376,7 @@ func (l *spinLock) Unlock() { // Hint to the CPU that this core is just waiting, and the core can go into a // lower energy state. -func spinLoopHint() { +func spinLoopWait() { // This is a no-op in QEMU TCG (but added here for completeness): // https://github.com/qemu/qemu/blob/v9.2.3/target/riscv/insn_trans/trans_rvi.c.inc#L856 riscv.Asm("pause") diff --git a/targets/arm.ld b/targets/arm.ld index cdf5b1dd4..c5f30ea6b 100644 --- a/targets/arm.ld +++ b/targets/arm.ld @@ -32,6 +32,14 @@ SECTIONS _stack_top = .; } >RAM + /* Stack for second core (core 1), if there is one. */ + .stack1 (NOLOAD) : + { + . = ALIGN(4); + . += DEFINED(__num_stacks) && __num_stacks >= 2 ? _stack_size : 0; + _stack1_top = .; + } >RAM + /* Start address (in flash) of .data, used by startup code. */ _sidata = LOADADDR(.data); diff --git a/targets/rp2040.json b/targets/rp2040.json index 342410368..3f9fea459 100644 --- a/targets/rp2040.json +++ b/targets/rp2040.json @@ -1,6 +1,7 @@ { "inherits": ["cortex-m0plus"], "build-tags": ["rp2040", "rp"], + "scheduler": "cores", "flash-1200-bps-reset": "true", "flash-method": "msd", "serial": "usb", @@ -12,6 +13,9 @@ "extra-files": [ "src/device/rp/rp2040.s" ], + "ldflags": [ + "--defsym=__num_stacks=2" + ], "linkerscript": "targets/rp2040.ld", "openocd-interface": "picoprobe", "openocd-transport": "swd",