From ea2f0e444dfc9739d240dd306b253a4023f8df1e Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Tue, 1 Apr 2025 09:16:35 +0200 Subject: [PATCH] WIP stashed changes --- src/internal/task/task.go | 8 + src/internal/task/task_cores.c | 50 ----- src/internal/task/task_cores.go | 266 ------------------------ src/internal/task/task_stack.go | 40 ++-- src/internal/task/task_stack_cortexm.S | 2 +- src/internal/task/task_stack_cortexm.go | 2 +- src/runtime/gc_stack_raw.go | 2 +- src/runtime/print.go | 4 + src/runtime/runtime_nrf.go | 30 +++ src/runtime/runtime_rp2040.go | 53 +++-- src/runtime/runtime_unix.go | 6 + src/runtime/scheduler_cores.go | 191 ++++++++++++++++- testdata/channel.go | 1 + testdata/goroutines.go | 20 +- 14 files changed, 311 insertions(+), 364 deletions(-) delete mode 100644 src/internal/task/task_cores.c delete mode 100644 src/internal/task/task_cores.go diff --git a/src/internal/task/task.go b/src/internal/task/task.go index aa563f638..3f2a36cda 100644 --- a/src/internal/task/task.go +++ b/src/internal/task/task.go @@ -21,11 +21,19 @@ type Task struct { // state is the underlying running state of the task. state state + RunState uint8 + // DeferFrame stores a pointer to the (stack allocated) defer frame of the // goroutine that is used for the recover builtin. DeferFrame unsafe.Pointer } +const ( + RunStatePaused = iota + RunStateRunning + RunStateResuming +) + // DataUint32 returns the Data field as a uint32. The value is only valid after // setting it through SetDataUint32 or by storing to it using DataAtomicUint32. func (t *Task) DataUint32() uint32 { diff --git a/src/internal/task/task_cores.c b/src/internal/task/task_cores.c deleted file mode 100644 index ae3f203bb..000000000 --- a/src/internal/task/task_cores.c +++ /dev/null @@ -1,50 +0,0 @@ -//go:build scheduler.cores - -#include - -__attribute__((naked)) -void tinygo_cores_startTask(void) { - asm volatile( - "bl tinygo_schedulerUnlock\n\t" - - "ldr r0, =tinygo_exitTask\n\t" - "mov lr, r0\n\t" - - "pop {r0, pc}\n\t" - ); -} - -void tinygo_switchTask(uintptr_t *oldStack, uintptr_t newStack) { -#if defined(__thumb__) - register uintptr_t *oldStackReg asm("r0"); - oldStackReg = oldStack; - register uintptr_t newStackReg asm("r1"); - newStackReg = newStack; - asm volatile( - // Push PC to switch back to. - // Note: adding 1 to set the Thumb bit. - "ldr r2, =1f+1\n\t" - "push {r2}\n\t" - - // Save stack pointer in oldStack for the switch back. - "mov r2, sp\n\t" - "str r2, [%[oldStack]]\n\t" - - // Switch to the new stack. - "mov sp, %[newStack]\n\t" - - // Return into the new stack. - "pop {pc}\n\t" - - // address where we should resume - "1:" - - : [oldStack]"+r"(oldStackReg), - [newStack]"+r"(newStackReg) - : - : "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12", "lr", "cc", "memory" - ); -#else - #error unknown architecture -#endif -} diff --git a/src/internal/task/task_cores.go b/src/internal/task/task_cores.go deleted file mode 100644 index c4f4b558c..000000000 --- a/src/internal/task/task_cores.go +++ /dev/null @@ -1,266 +0,0 @@ -//go:build scheduler.cores - -package task - -import ( - "runtime/interrupt" - "unsafe" -) - -import "C" // dummy import, to make sure task_cores.c is included in the build - -type runState uint8 - -const ( - runStateRunning runState = iota - runStateResuming - runStatePaused -) - -type state struct { - // Which state the task is currently in. - // The state is protected by the scheduler lock, and must only be - // read/modified with that lock held. - runState runState - - // The stack pointer while the task is switched away. - sp unsafe.Pointer - - // canaryPtr points to the top word of the stack (the lowest address). - // This is used to detect stack overflows. - // When initializing the goroutine, the stackCanary constant is stored there. - // If the stack overflowed, the word will likely no longer equal stackCanary. - canaryPtr *uintptr -} - -var ( - runQueue *Task - sleepQueue *Task -) - -//go:linkname runtimeCurrentTask runtime.currentTask -func runtimeCurrentTask() *Task - -// Current returns the current task, or nil if we're in the scheduler. -func Current() *Task { - return runtimeCurrentTask() -} - -func Init(mainTask *Task, canaryPtr *uintptr) { - // The topmost word of the default stack is used as a stack canary. - *canaryPtr = stackCanary - mainTask.state.canaryPtr = canaryPtr -} - -func Pause() { - // Check whether the canary (the lowest address of the stack) is still - // valid. If it is not, a stack overflow has occurred. - if *Current().state.canaryPtr != stackCanary { - runtimePanic("goroutine stack overflow") - } - if interrupt.In() { - runtimePanic("blocked inside interrupt") - } - - // Note: Pause() must be called with the scheduler lock locked! - schedulerLock() - pauseLocked() - schedulerUnlock() -} - -var schedulerIsRunning bool - -func pauseLocked() { - t := Current() - for { - if t.state.runState == runStateResuming { - t.state.runState = runStateRunning - return - } - - // Make sure only one core is calling sleepTicks etc. - if schedulerIsRunning { - schedulerUnlock() - waitForEvents() - schedulerLock() - continue - } - - if runnable := runQueue; runnable != nil { - // Resume it now. - runQueue = runQueue.Next - runnable.Next = nil - if t == runnable { - // We're actually the task that's supposed to be resumed, so we - // are ready! - } else { - // It's not us that's ready, so switch to this other task. - setCurrentTask(runnable) - t.state.runState = runStatePaused - - // Switch away! - switchTask(&t.state.sp, runnable.state.sp) - - // We got back from the switch, so another task resumed us. - t.state.runState = runStateRunning - } - return - } - - // Check whether there's a sleeping task that is ready to run. - if sleepingTask := sleepQueue; sleepingTask != nil { - now := runtimeTicks() - if now >= sleepingTask.Data { - // This task is done sleeping. - // Resume it now. - sleepQueue = sleepQueue.Next - sleepingTask.Next = nil - if t == sleepingTask { - // We're actually the task that's sleeping, so we are ready! - } else { - // It's not us that's ready, so switch to this other task. - setCurrentTask(sleepingTask) - t.state.runState = runStatePaused - - // Switch away! - switchTask(&t.state.sp, sleepingTask.state.sp) - - // We got back from the switch, so another task resumed us. - t.state.runState = runStateRunning - } - return - } else { - // Sleep for a bit until the next task is ready to run. - schedulerIsRunning = true - schedulerUnlock() - delay := sleepingTask.Data - now - runtimeSleepTicks(delay) - schedulerLock() - schedulerIsRunning = false - continue - } - } - } -} - -func start(fn uintptr, args unsafe.Pointer, stackSize uintptr) { - t := &Task{} - stack := runtime_alloc(stackSize, nil) - stackTop := unsafe.Add(stack, stackSize-16) - topRegs := unsafe.Slice((*uintptr)(stackTop), 4) - topRegs[0] = uintptr(unsafe.Pointer(&startTask)) - topRegs[1] = uintptr(args) - topRegs[2] = fn - t.state.sp = stackTop - - canaryPtr := (*uintptr)(stack) - *canaryPtr = stackCanary - t.state.canaryPtr = canaryPtr - - schedulerLock() - addToRunqueue(t) - schedulerUnlock() -} - -func GCScan() { - panic("todo: task.GCScan") -} - -func StackTop() uintptr { - println("todo: task.StackTop") - for { - } -} - -func Sleep(wakeup uint64) { - schedulerLock() - addSleepTask(Current(), wakeup) - pauseLocked() - schedulerUnlock() -} - -func Resume(t *Task) { - schedulerLock() - switch t.state.runState { - case runStatePaused: - // Paused, state is saved on the stack. - addToRunqueue(t) - case runStateRunning: - // Going to pause soon, so let the Pause() function know it can resume - // immediately. - t.state.runState = runStateResuming - default: - println("unknown run state??") - for { - } - } - schedulerUnlock() -} - -// May only be called with the scheduler lock held! -func addToRunqueue(t *Task) { - t.Next = runQueue - runQueue = t -} - -func addSleepTask(t *Task, wakeup uint64) { - // Save the timestamp when the task should be woken up. - t.Data = wakeup - - // Find the position where we should insert this task in the queue. - q := &sleepQueue - for { - if *q == nil { - // Found the end of the time queue. Insert it here, at the end. - break - } - if (*q).Data > t.Data { - // Found a task in the queue that has a timeout before the - // to-be-sleeping task. Insert our task right before. - break - } - q = &(*q).Next - } - - // Insert the task into the queue (this could be at the end, if *q is nil). - t.Next = *q - *q = t -} - -//go:linkname schedulerLock runtime.schedulerLock -func schedulerLock() - -//go:linkname schedulerUnlock runtime.schedulerUnlock -func schedulerUnlock() - -//go:linkname runtimeTicks runtime.runtimeTicks -func runtimeTicks() uint64 - -//go:linkname runtimeSleepTicks runtime.runtimeSleepTicks -func runtimeSleepTicks(duration uint64) - -// startTask is a small wrapper function that sets up the first (and only) -// argument to the new goroutine and makes sure it is exited when the goroutine -// finishes. -// -//go:extern tinygo_cores_startTask -var startTask [0]uint8 - -//export tinygo_exitTask -func exitTask() { - Pause() -} - -//export tinygo_schedulerUnlock -func tinygo_schedulerUnlock() { - schedulerUnlock() -} - -//export tinygo_switchTask -func switchTask(oldStack *unsafe.Pointer, newStack unsafe.Pointer) - -//go:linkname waitForEvents runtime.waitForEvents -func waitForEvents() - -//go:linkname setCurrentTask runtime.setCurrentTask -func setCurrentTask(task *Task) diff --git a/src/internal/task/task_stack.go b/src/internal/task/task_stack.go index dc846107a..cbba8e380 100644 --- a/src/internal/task/task_stack.go +++ b/src/internal/task/task_stack.go @@ -1,4 +1,4 @@ -//go:build scheduler.tasks +//go:build scheduler.tasks || scheduler.cores package task @@ -25,40 +25,54 @@ type state struct { } // currentTask is the current running task, or nil if currently in the scheduler. -var currentTask *Task +//var currentTask *Task // Current returns the current active task. -func Current() *Task { - return currentTask +//func Current() *Task { +// return currentTask +//} + +//go:linkname Current runtime.currentTask +func Current() *Task + +//go:linkname schedulerLock runtime.schedulerLock +func schedulerLock() + +func Pause() { + schedulerLock() + PauseLocked() } // Pause suspends the current task and returns to the scheduler. // This function may only be called when running on a goroutine stack, not when running on the system stack or in an interrupt. -func Pause() { +func PauseLocked() { // Check whether the canary (the lowest address of the stack) is still // valid. If it is not, a stack overflow has occurred. - if *currentTask.state.canaryPtr != stackCanary { + if *Current().state.canaryPtr != stackCanary { runtimePanic("goroutine stack overflow") } if interrupt.In() { runtimePanic("blocked inside interrupt") } - currentTask.state.pause() + current := Current() + current.RunState = RunStatePaused + current.state.pause() } -//export tinygo_pause -func pause() { +//export tinygo_task_exit +func taskExit() { + println("-- exiting task") Pause() } // Resume the task until it pauses or completes. // This may only be called from the scheduler. func (t *Task) Resume() { - currentTask = t - t.gcData.swap() + //currentTask = t + //t.gcData.swap() t.state.resume() - t.gcData.swap() - currentTask = nil + //t.gcData.swap() + //currentTask = nil } // initialize the state and prepare to call the specified function with the specified argument bundle. diff --git a/src/internal/task/task_stack_cortexm.S b/src/internal/task/task_stack_cortexm.S index dfe713552..8e6520106 100644 --- a/src/internal/task/task_stack_cortexm.S +++ b/src/internal/task/task_stack_cortexm.S @@ -28,7 +28,7 @@ tinygo_startTask: blx r4 // After return, exit this goroutine. This is a tail call. - bl tinygo_pause + bl tinygo_task_exit .cfi_endproc .size tinygo_startTask, .-tinygo_startTask diff --git a/src/internal/task/task_stack_cortexm.go b/src/internal/task/task_stack_cortexm.go index 226a088c8..653dc06e1 100644 --- a/src/internal/task/task_stack_cortexm.go +++ b/src/internal/task/task_stack_cortexm.go @@ -1,4 +1,4 @@ -//go:build scheduler.tasks && cortexm +//go:build (scheduler.tasks || scheduler.cores) && cortexm package task diff --git a/src/runtime/gc_stack_raw.go b/src/runtime/gc_stack_raw.go index bdc3154fa..f4483e58c 100644 --- a/src/runtime/gc_stack_raw.go +++ b/src/runtime/gc_stack_raw.go @@ -1,4 +1,4 @@ -//go:build (gc.conservative || gc.precise) && !tinygo.wasm && !scheduler.threads +//go:build (gc.conservative || gc.precise) && !tinygo.wasm && !scheduler.threads && !scheduler.cores package runtime diff --git a/src/runtime/print.go b/src/runtime/print.go index a4de46025..ef5c8a7dc 100644 --- a/src/runtime/print.go +++ b/src/runtime/print.go @@ -13,11 +13,15 @@ type stringer interface { // This is a no-op lock on systems that do not have parallelism. var printLock task.PMutex +//var printLocked interrupt.State + func printlock() { + //printLocked = serialLock() printLock.Lock() } func printunlock() { + //serialUnlock(printLocked) printLock.Unlock() } diff --git a/src/runtime/runtime_nrf.go b/src/runtime/runtime_nrf.go index 729c6bb20..30a6cda9c 100644 --- a/src/runtime/runtime_nrf.go +++ b/src/runtime/runtime_nrf.go @@ -15,6 +15,8 @@ type timeUnit int64 //go:linkname systemInit SystemInit func systemInit() +const numCPU = 1 + //export Reset_Handler func main() { if nrf.FPUPresent { @@ -145,3 +147,31 @@ func rtc_sleep(ticks uint32) { waitForEvents() } } + +func atomicLockImpl() interrupt.State { + mask := interrupt.Disable() + return mask +} + +func atomicUnlockImpl(mask interrupt.State) { + interrupt.Restore(mask) +} + +func futexLock() interrupt.State { + mask := interrupt.Disable() + return mask +} + +func futexUnlock(mask interrupt.State) { + interrupt.Restore(mask) +} + +func schedulerLock() { +} + +func schedulerUnlock() { +} + +func currentCPU() uint32 { + return 0 +} diff --git a/src/runtime/runtime_rp2040.go b/src/runtime/runtime_rp2040.go index c55f0b787..3c9e8935a 100644 --- a/src/runtime/runtime_rp2040.go +++ b/src/runtime/runtime_rp2040.go @@ -5,6 +5,7 @@ package runtime import ( "device/arm" "device/rp" + "internal/task" "machine" "machine/usb/cdc" "reflect" @@ -56,9 +57,9 @@ func waitForEvents() { } func putchar(c byte) { - mask := serialLock() + //mask := serialLock() machine.Serial.WriteByte(c) - serialUnlock(mask) + //serialUnlock(mask) } func getchar() byte { @@ -137,7 +138,7 @@ var core1StartSequence = [...]uint32{ uint32(uintptr(reflect.ValueOf(runCore1).Pointer())), } -func startOtherCores() { +func startSecondaryCores() { // Start the second core of the RP2040. // See section 2.8.2 in the datasheet. seq := 0 @@ -160,21 +161,30 @@ func startOtherCores() { } } +var core1Task task.Task + func runCore1() { + //until := ticks() + nanosecondsToTicks(1900e6) + //for ticks() < until { + //} + println("starting core 1") + + runSecondary(1, &core1Task) + // Just blink a LED to show that this core is running. // TODO: use a real scheduler. - led := machine.GP0 - led.Configure(machine.PinConfig{Mode: machine.PinOutput}) - const cycles = 7000_000 - for { - for i := 0; i < cycles; i++ { - led.Low() - } + //led := machine.GP16 + //led.Configure(machine.PinConfig{Mode: machine.PinOutput}) + //const cycles = 7000_000 + //for { + // for i := 0; i < cycles; i++ { + // led.Low() + // } - for i := 0; i < cycles; i++ { - led.High() - } - } + // for i := 0; i < cycles; i++ { + // led.High() + // } + //} } func currentCPU() uint32 { @@ -223,27 +233,36 @@ func futexUnlock(mask interrupt.State) { } var schedulerLockMasks [numCPU]interrupt.State +var schedulerLocked = false // WARNING: doesn't check for deadlocks! func schedulerLock() { //schedulerLockMasks[currentCPU()] = interrupt.Disable() for rp.SIO.SPINLOCK2.Get() == 0 { } + schedulerLocked = true } func schedulerUnlock() { + if !schedulerLocked { + println("!!! not locked at unlock") + for { + } + } + schedulerLocked = false rp.SIO.SPINLOCK2.Set(0) //interrupt.Restore(schedulerLockMasks[currentCPU()]) } func serialLock() interrupt.State { - mask := interrupt.Disable() + //mask := interrupt.Disable() for rp.SIO.SPINLOCK3.Get() == 0 { } - return mask + //return mask + return 0 } func serialUnlock(mask interrupt.State) { rp.SIO.SPINLOCK3.Set(0) - interrupt.Restore(mask) + //interrupt.Restore(mask) } diff --git a/src/runtime/runtime_unix.go b/src/runtime/runtime_unix.go index 17e004b2e..fd71d7c48 100644 --- a/src/runtime/runtime_unix.go +++ b/src/runtime/runtime_unix.go @@ -11,6 +11,12 @@ import ( "unsafe" ) +const numCPU = 1 + +func currentCPU() int { + return 0 +} + //export write func libc_write(fd int32, buf unsafe.Pointer, count uint) int diff --git a/src/runtime/scheduler_cores.go b/src/runtime/scheduler_cores.go index 9216cebac..4a664acf8 100644 --- a/src/runtime/scheduler_cores.go +++ b/src/runtime/scheduler_cores.go @@ -3,17 +3,21 @@ package runtime import ( + "device/arm" "internal/task" - "unsafe" ) const hasScheduler = true const hasParallelism = true +const coresVerbose = false + var ( - mainTask task.Task - cpuTasks [numCPU]*task.Task + mainTask task.Task + cpuTasks [numCPU]*task.Task + sleepQueue *task.Task + runQueue *task.Task ) func deadlock() { @@ -23,7 +27,60 @@ func deadlock() { } func scheduleTask(t *task.Task) { - task.Resume(t) + schedulerLock() + switch t.RunState { + case task.RunStatePaused: + // Paused, state is saved on the stack. + + if coresVerbose { + println("## schedule: add to runQueue") + } + addToRunQueue(t) + arm.Asm("sev") + case task.RunStateRunning: + // Not yet paused (probably going to pause very soon), so let the + // Pause() function know it can resume immediately. + t.RunState = task.RunStateResuming + if coresVerbose { + println("## schedule: mark as resuming") + } + default: + println("Unknown run state??") + for { + } + } + schedulerUnlock() +} + +// Add task to runQueue. +// Scheduler lock must be held when calling this function. +func addToRunQueue(t *task.Task) { + t.Next = runQueue + runQueue = t +} + +func addSleepTask(t *task.Task, wakeup timeUnit) { + // Save the timestamp when the task should be woken up. + t.Data = uint64(wakeup) + + // Find the position where we should insert this task in the queue. + q := &sleepQueue + for { + if *q == nil { + // Found the end of the time queue. Insert it here, at the end. + break + } + if timeUnit((*q).Data) > timeUnit(t.Data) { + // Found a task in the queue that has a timeout before the + // to-be-sleeping task. Insert our task right before. + break + } + q = &(*q).Next + } + + // Insert the task into the queue (this could be at the end, if *q is nil). + t.Next = *q + *q = t } func Gosched() { @@ -61,17 +118,126 @@ func sleep(duration int64) { } wakeup := ticks() + nanosecondsToTicks(duration) - task.Sleep(uint64(wakeup)) + + schedulerLock() + addSleepTask(task.Current(), wakeup) + task.PauseLocked() } func run() { initHeap() cpuTasks[0] = &mainTask - task.Init(&mainTask, (*uintptr)(unsafe.Pointer(&stackTopSymbol))) - initAll() - startOtherCores() - callMain() - mainExited = true + initAll() // TODO: move into main goroutine! + + until := ticks() + nanosecondsToTicks(200e6) + for ticks() < until { + } + println("\n\n=====") + + go func() { + //initAll() + startSecondaryCores() + callMain() + mainExited = true + }() + schedulerLock() + scheduler() +} + +func runSecondary(core uint32, t *task.Task) { + println("-- runSecondary") + cpuTasks[core] = t + println("-- locking for 2nd core") + schedulerLock() + println("-- locked!") + scheduler() +} + +var schedulerIsRunning = false + +func scheduler() { + if coresVerbose { + println("** scheduler on core:", currentCPU()) + } + for { + //until := ticks() + nanosecondsToTicks(100e6) + //for ticks() < until { + //} + + // Check for ready-to-run tasks. + if runnable := runQueue; runnable != nil { + if coresVerbose { + println("** scheduler", currentCPU(), "run runnable") + } + // Pop off the run queue. + runQueue = runnable.Next + runnable.Next = nil + + // Resume it now. + setCurrentTask(runnable) + schedulerUnlock() + runnable.Resume() + if coresVerbose { + println("** scheduler", currentCPU(), " returned (from runqueue resume)") + } + setCurrentTask(nil) + + continue + } + + // If another core is using the clock, let it handle the sleep queue. + if schedulerIsRunning { + if coresVerbose { + println("** scheduler", currentCPU(), "wait for other core") + } + schedulerUnlock() + waitForEvents() + schedulerLock() + continue + } + + if sleepingTask := sleepQueue; sleepingTask != nil { + now := ticks() + if now >= timeUnit(sleepingTask.Data) { + if coresVerbose { + println("** scheduler", currentCPU(), "run sleeping") + } + // This task is done sleeping. + // Resume it now. + sleepQueue = sleepQueue.Next + sleepingTask.Next = nil + + setCurrentTask(sleepingTask) + schedulerUnlock() + sleepingTask.Resume() + if coresVerbose { + println("** scheduler", currentCPU(), " returned (from sleepQueue resume)") + } + setCurrentTask(nil) + continue + } + + delay := timeUnit(sleepingTask.Data) - now + if coresVerbose { + println("** scheduler", currentCPU(), "sleep", ticksToNanoseconds(delay)/1e6) + } + + // Sleep for a bit until the next task is ready to run. + schedulerIsRunning = true + schedulerUnlock() + sleepTicks(delay) + schedulerLock() + schedulerIsRunning = false + continue + } + + if coresVerbose { + println("** scheduler", currentCPU(), "wait for events") + } + schedulerUnlock() + waitForEvents() + schedulerLock() + } } func currentTask() *task.Task { @@ -89,3 +255,8 @@ func runtimeTicks() uint64 { func runtimeSleepTicks(delay uint64) { sleepTicks(timeUnit(delay)) } + +//export tinygo_schedulerUnlock +func tinygo_schedulerUnlock() { + schedulerUnlock() +} diff --git a/testdata/channel.go b/testdata/channel.go index 9c0fee5b7..a247c808f 100644 --- a/testdata/channel.go +++ b/testdata/channel.go @@ -12,6 +12,7 @@ var wg sync.WaitGroup type intchan chan int func main() { + time.Sleep(time.Second * 2) ch := make(chan int, 2) ch <- 1 println("len, cap of channel:", len(ch), cap(ch), ch == nil) diff --git a/testdata/goroutines.go b/testdata/goroutines.go index cf19cc3ca..bed2ecb1a 100644 --- a/testdata/goroutines.go +++ b/testdata/goroutines.go @@ -5,19 +5,27 @@ import ( "time" ) -func init() { - println("init") - go println("goroutine in init") - time.Sleep(1 * time.Millisecond) -} +//func init() { +// println("\n----") +// println("init") +// go println("goroutine in init") +// time.Sleep(1 * time.Millisecond) +//} func main() { + //for i := 0; i < 2; i++ { + // println("...") + // time.Sleep(time.Second) + //} + println("main 1") go sub() time.Sleep(1 * time.Millisecond) println("main 2") time.Sleep(2 * time.Millisecond) println("main 3") + //time.Sleep(2 * time.Millisecond) + //println("main 4") // Await a blocking call. println("wait:") @@ -101,6 +109,8 @@ func sub() { println("sub 1") time.Sleep(2 * time.Millisecond) println("sub 2") + //time.Sleep(2 * time.Millisecond) + //println("sub 3") } func wait() {