WIP stashed changes

src/internal/task/task.go

@@ -21,11 +21,19 @@ type Task struct {
 	// state is the underlying running state of the task.
 	state state
 
+	RunState uint8
+
 	// DeferFrame stores a pointer to the (stack allocated) defer frame of the
 	// goroutine that is used for the recover builtin.
 	DeferFrame unsafe.Pointer
 }
 
+const (
+	RunStatePaused = iota
+	RunStateRunning
+	RunStateResuming
+)
+
 // DataUint32 returns the Data field as a uint32. The value is only valid after
 // setting it through SetDataUint32 or by storing to it using DataAtomicUint32.
 func (t *Task) DataUint32() uint32 {

src/internal/task/task_cores.c

@@ -1,50 +0,0 @@
-//go:build scheduler.cores
-
-#include <inttypes.h>
-
-__attribute__((naked))
-void tinygo_cores_startTask(void) {
-    asm volatile(
-        "bl tinygo_schedulerUnlock\n\t"
-
-        "ldr r0, =tinygo_exitTask\n\t"
-        "mov lr, r0\n\t"
-
-        "pop {r0, pc}\n\t"
-    );
-}
-
-void tinygo_switchTask(uintptr_t *oldStack, uintptr_t newStack) {
-#if defined(__thumb__)
-    register uintptr_t *oldStackReg asm("r0");
-    oldStackReg = oldStack;
-    register uintptr_t newStackReg asm("r1");
-    newStackReg = newStack;
-    asm volatile(
-        // Push PC to switch back to.
-        // Note: adding 1 to set the Thumb bit.
-        "ldr r2, =1f+1\n\t"
-        "push {r2}\n\t"
-
-        // Save stack pointer in oldStack for the switch back.
-        "mov r2, sp\n\t"
-        "str r2, [%[oldStack]]\n\t"
-
-        // Switch to the new stack.
-        "mov sp, %[newStack]\n\t"
-
-        // Return into the new stack.
-        "pop {pc}\n\t"
-
-        // address where we should resume
-        "1:"
-
-        : [oldStack]"+r"(oldStackReg),
-          [newStack]"+r"(newStackReg)
-        :
-        : "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", "r11", "r12", "lr", "cc", "memory"
-    );
-#else
-    #error unknown architecture
-#endif
-}

src/internal/task/task_cores.go

@@ -1,266 +0,0 @@
-//go:build scheduler.cores
-
-package task
-
-import (
-	"runtime/interrupt"
-	"unsafe"
-)
-
-import "C" // dummy import, to make sure task_cores.c is included in the build
-
-type runState uint8
-
-const (
-	runStateRunning runState = iota
-	runStateResuming
-	runStatePaused
-)
-
-type state struct {
-	// Which state the task is currently in.
-	// The state is protected by the scheduler lock, and must only be
-	// read/modified with that lock held.
-	runState runState
-
-	// The stack pointer while the task is switched away.
-	sp unsafe.Pointer
-
-	// canaryPtr points to the top word of the stack (the lowest address).
-	// This is used to detect stack overflows.
-	// When initializing the goroutine, the stackCanary constant is stored there.
-	// If the stack overflowed, the word will likely no longer equal stackCanary.
-	canaryPtr *uintptr
-}
-
-var (
-	runQueue   *Task
-	sleepQueue *Task
-)
-
-//go:linkname runtimeCurrentTask runtime.currentTask
-func runtimeCurrentTask() *Task
-
-// Current returns the current task, or nil if we're in the scheduler.
-func Current() *Task {
-	return runtimeCurrentTask()
-}
-
-func Init(mainTask *Task, canaryPtr *uintptr) {
-	// The topmost word of the default stack is used as a stack canary.
-	*canaryPtr = stackCanary
-	mainTask.state.canaryPtr = canaryPtr
-}
-
-func Pause() {
-	// Check whether the canary (the lowest address of the stack) is still
-	// valid. If it is not, a stack overflow has occurred.
-	if *Current().state.canaryPtr != stackCanary {
-		runtimePanic("goroutine stack overflow")
-	}
-	if interrupt.In() {
-		runtimePanic("blocked inside interrupt")
-	}
-
-	// Note: Pause() must be called with the scheduler lock locked!
-	schedulerLock()
-	pauseLocked()
-	schedulerUnlock()
-}
-
-var schedulerIsRunning bool
-
-func pauseLocked() {
-	t := Current()
-	for {
-		if t.state.runState == runStateResuming {
-			t.state.runState = runStateRunning
-			return
-		}
-
-		// Make sure only one core is calling sleepTicks etc.
-		if schedulerIsRunning {
-			schedulerUnlock()
-			waitForEvents()
-			schedulerLock()
-			continue
-		}
-
-		if runnable := runQueue; runnable != nil {
-			// Resume it now.
-			runQueue = runQueue.Next
-			runnable.Next = nil
-			if t == runnable {
-				// We're actually the task that's supposed to be resumed, so we
-				// are ready!
-			} else {
-				// It's not us that's ready, so switch to this other task.
-				setCurrentTask(runnable)
-				t.state.runState = runStatePaused
-
-				// Switch away!
-				switchTask(&t.state.sp, runnable.state.sp)
-
-				// We got back from the switch, so another task resumed us.
-				t.state.runState = runStateRunning
-			}
-			return
-		}
-
-		// Check whether there's a sleeping task that is ready to run.
-		if sleepingTask := sleepQueue; sleepingTask != nil {
-			now := runtimeTicks()
-			if now >= sleepingTask.Data {
-				// This task is done sleeping.
-				// Resume it now.
-				sleepQueue = sleepQueue.Next
-				sleepingTask.Next = nil
-				if t == sleepingTask {
-					// We're actually the task that's sleeping, so we are ready!
-				} else {
-					// It's not us that's ready, so switch to this other task.
-					setCurrentTask(sleepingTask)
-					t.state.runState = runStatePaused
-
-					// Switch away!
-					switchTask(&t.state.sp, sleepingTask.state.sp)
-
-					// We got back from the switch, so another task resumed us.
-					t.state.runState = runStateRunning
-				}
-				return
-			} else {
-				// Sleep for a bit until the next task is ready to run.
-				schedulerIsRunning = true
-				schedulerUnlock()
-				delay := sleepingTask.Data - now
-				runtimeSleepTicks(delay)
-				schedulerLock()
-				schedulerIsRunning = false
-				continue
-			}
-		}
-	}
-}
-
-func start(fn uintptr, args unsafe.Pointer, stackSize uintptr) {
-	t := &Task{}
-	stack := runtime_alloc(stackSize, nil)
-	stackTop := unsafe.Add(stack, stackSize-16)
-	topRegs := unsafe.Slice((*uintptr)(stackTop), 4)
-	topRegs[0] = uintptr(unsafe.Pointer(&startTask))
-	topRegs[1] = uintptr(args)
-	topRegs[2] = fn
-	t.state.sp = stackTop
-
-	canaryPtr := (*uintptr)(stack)
-	*canaryPtr = stackCanary
-	t.state.canaryPtr = canaryPtr
-
-	schedulerLock()
-	addToRunqueue(t)
-	schedulerUnlock()
-}
-
-func GCScan() {
-	panic("todo: task.GCScan")
-}
-
-func StackTop() uintptr {
-	println("todo: task.StackTop")
-	for {
-	}
-}
-
-func Sleep(wakeup uint64) {
-	schedulerLock()
-	addSleepTask(Current(), wakeup)
-	pauseLocked()
-	schedulerUnlock()
-}
-
-func Resume(t *Task) {
-	schedulerLock()
-	switch t.state.runState {
-	case runStatePaused:
-		// Paused, state is saved on the stack.
-		addToRunqueue(t)
-	case runStateRunning:
-		// Going to pause soon, so let the Pause() function know it can resume
-		// immediately.
-		t.state.runState = runStateResuming
-	default:
-		println("unknown run state??")
-		for {
-		}
-	}
-	schedulerUnlock()
-}
-
-// May only be called with the scheduler lock held!
-func addToRunqueue(t *Task) {
-	t.Next = runQueue
-	runQueue = t
-}
-
-func addSleepTask(t *Task, wakeup uint64) {
-	// Save the timestamp when the task should be woken up.
-	t.Data = wakeup
-
-	// Find the position where we should insert this task in the queue.
-	q := &sleepQueue
-	for {
-		if *q == nil {
-			// Found the end of the time queue. Insert it here, at the end.
-			break
-		}
-		if (*q).Data > t.Data {
-			// Found a task in the queue that has a timeout before the
-			// to-be-sleeping task. Insert our task right before.
-			break
-		}
-		q = &(*q).Next
-	}
-
-	// Insert the task into the queue (this could be at the end, if *q is nil).
-	t.Next = *q
-	*q = t
-}
-
-//go:linkname schedulerLock runtime.schedulerLock
-func schedulerLock()
-
-//go:linkname schedulerUnlock runtime.schedulerUnlock
-func schedulerUnlock()
-
-//go:linkname runtimeTicks runtime.runtimeTicks
-func runtimeTicks() uint64
-
-//go:linkname runtimeSleepTicks runtime.runtimeSleepTicks
-func runtimeSleepTicks(duration uint64)
-
-// startTask is a small wrapper function that sets up the first (and only)
-// argument to the new goroutine and makes sure it is exited when the goroutine
-// finishes.
-//
-//go:extern tinygo_cores_startTask
-var startTask [0]uint8
-
-//export tinygo_exitTask
-func exitTask() {
-	Pause()
-}
-
-//export tinygo_schedulerUnlock
-func tinygo_schedulerUnlock() {
-	schedulerUnlock()
-}
-
-//export tinygo_switchTask
-func switchTask(oldStack *unsafe.Pointer, newStack unsafe.Pointer)
-
-//go:linkname waitForEvents runtime.waitForEvents
-func waitForEvents()
-
-//go:linkname setCurrentTask runtime.setCurrentTask
-func setCurrentTask(task *Task)

src/internal/task/task_stack.go

@@ -1,4 +1,4 @@
-//go:build scheduler.tasks
+//go:build scheduler.tasks || scheduler.cores
 
 package task
 
@@ -25,40 +25,54 @@ type state struct {
 }
 
 // currentTask is the current running task, or nil if currently in the scheduler.
-var currentTask *Task
+//var currentTask *Task
 
 // Current returns the current active task.
-func Current() *Task {
+//func Current() *Task {
-	return currentTask
+//	return currentTask
+//}
+
+//go:linkname Current runtime.currentTask
+func Current() *Task
+
+//go:linkname schedulerLock runtime.schedulerLock
+func schedulerLock()
+
+func Pause() {
+	schedulerLock()
+	PauseLocked()
 }
 
 // Pause suspends the current task and returns to the scheduler.
 // This function may only be called when running on a goroutine stack, not when running on the system stack or in an interrupt.
-func Pause() {
+func PauseLocked() {
 	// Check whether the canary (the lowest address of the stack) is still
 	// valid. If it is not, a stack overflow has occurred.
-	if *currentTask.state.canaryPtr != stackCanary {
+	if *Current().state.canaryPtr != stackCanary {
 		runtimePanic("goroutine stack overflow")
 	}
 	if interrupt.In() {
 		runtimePanic("blocked inside interrupt")
 	}
-	currentTask.state.pause()
+	current := Current()
+	current.RunState = RunStatePaused
+	current.state.pause()
 }
 
-//export tinygo_pause
+//export tinygo_task_exit
-func pause() {
+func taskExit() {
+	println("-- exiting task")
 	Pause()
 }
 
 // Resume the task until it pauses or completes.
 // This may only be called from the scheduler.
 func (t *Task) Resume() {
-	currentTask = t
+	//currentTask = t
-	t.gcData.swap()
+	//t.gcData.swap()
 	t.state.resume()
-	t.gcData.swap()
+	//t.gcData.swap()
-	currentTask = nil
+	//currentTask = nil
 }
 
 // initialize the state and prepare to call the specified function with the specified argument bundle.

src/internal/task/task_stack_cortexm.S

@@ -28,7 +28,7 @@ tinygo_startTask:
     blx   r4
 
     // After return, exit this goroutine. This is a tail call.
-    bl    	tinygo_pause
+    bl    	tinygo_task_exit
     .cfi_endproc
 .size tinygo_startTask, .-tinygo_startTask
 

src/internal/task/task_stack_cortexm.go

@@ -1,4 +1,4 @@
-//go:build scheduler.tasks && cortexm
+//go:build (scheduler.tasks || scheduler.cores) && cortexm
 
 package task
 

src/runtime/gc_stack_raw.go

@@ -1,4 +1,4 @@
-//go:build (gc.conservative || gc.precise) && !tinygo.wasm && !scheduler.threads
+//go:build (gc.conservative || gc.precise) && !tinygo.wasm && !scheduler.threads && !scheduler.cores
 
 package runtime
 

src/runtime/print.go

@@ -13,11 +13,15 @@ type stringer interface {
 // This is a no-op lock on systems that do not have parallelism.
 var printLock task.PMutex
 
+//var printLocked interrupt.State
+
 func printlock() {
+	//printLocked = serialLock()
 	printLock.Lock()
 }
 
 func printunlock() {
+	//serialUnlock(printLocked)
 	printLock.Unlock()
 }
 

src/runtime/runtime_nrf.go

@@ -15,6 +15,8 @@ type timeUnit int64
 //go:linkname systemInit SystemInit
 func systemInit()
 
+const numCPU = 1
+
 //export Reset_Handler
 func main() {
 	if nrf.FPUPresent {
@@ -145,3 +147,31 @@ func rtc_sleep(ticks uint32) {
 		waitForEvents()
 	}
 }
+
+func atomicLockImpl() interrupt.State {
+	mask := interrupt.Disable()
+	return mask
+}
+
+func atomicUnlockImpl(mask interrupt.State) {
+	interrupt.Restore(mask)
+}
+
+func futexLock() interrupt.State {
+	mask := interrupt.Disable()
+	return mask
+}
+
+func futexUnlock(mask interrupt.State) {
+	interrupt.Restore(mask)
+}
+
+func schedulerLock() {
+}
+
+func schedulerUnlock() {
+}
+
+func currentCPU() uint32 {
+	return 0
+}

src/runtime/runtime_rp2040.go

@@ -5,6 +5,7 @@ package runtime
 import (
 	"device/arm"
 	"device/rp"
+	"internal/task"
 	"machine"
 	"machine/usb/cdc"
 	"reflect"
@@ -56,9 +57,9 @@ func waitForEvents() {
 }
 
 func putchar(c byte) {
-	mask := serialLock()
+	//mask := serialLock()
 	machine.Serial.WriteByte(c)
-	serialUnlock(mask)
+	//serialUnlock(mask)
 }
 
 func getchar() byte {
@@ -137,7 +138,7 @@ var core1StartSequence = [...]uint32{
 	uint32(uintptr(reflect.ValueOf(runCore1).Pointer())),
 }
 
-func startOtherCores() {
+func startSecondaryCores() {
 	// Start the second core of the RP2040.
 	// See section 2.8.2 in the datasheet.
 	seq := 0
@@ -160,21 +161,30 @@ func startOtherCores() {
 	}
 }
 
+var core1Task task.Task
+
 func runCore1() {
+	//until := ticks() + nanosecondsToTicks(1900e6)
+	//for ticks() < until {
+	//}
+	println("starting core 1")
+
+	runSecondary(1, &core1Task)
+
 	// Just blink a LED to show that this core is running.
 	// TODO: use a real scheduler.
-	led := machine.GP0
+	//led := machine.GP16
-	led.Configure(machine.PinConfig{Mode: machine.PinOutput})
+	//led.Configure(machine.PinConfig{Mode: machine.PinOutput})
-	const cycles = 7000_000
+	//const cycles = 7000_000
-	for {
+	//for {
-		for i := 0; i < cycles; i++ {
+	//	for i := 0; i < cycles; i++ {
-			led.Low()
+	//		led.Low()
-		}
+	//	}
 
-		for i := 0; i < cycles; i++ {
+	//	for i := 0; i < cycles; i++ {
-			led.High()
+	//		led.High()
-		}
+	//	}
-	}
+	//}
 }
 
 func currentCPU() uint32 {
@@ -223,27 +233,36 @@ func futexUnlock(mask interrupt.State) {
 }
 
 var schedulerLockMasks [numCPU]interrupt.State
+var schedulerLocked = false
 
 // WARNING: doesn't check for deadlocks!
 func schedulerLock() {
 	//schedulerLockMasks[currentCPU()] = interrupt.Disable()
 	for rp.SIO.SPINLOCK2.Get() == 0 {
 	}
+	schedulerLocked = true
 }
 
 func schedulerUnlock() {
+	if !schedulerLocked {
+		println("!!! not locked at unlock")
+		for {
+		}
+	}
+	schedulerLocked = false
 	rp.SIO.SPINLOCK2.Set(0)
 	//interrupt.Restore(schedulerLockMasks[currentCPU()])
 }
 
 func serialLock() interrupt.State {
-	mask := interrupt.Disable()
+	//mask := interrupt.Disable()
 	for rp.SIO.SPINLOCK3.Get() == 0 {
 	}
-	return mask
+	//return mask
+	return 0
 }
 
 func serialUnlock(mask interrupt.State) {
 	rp.SIO.SPINLOCK3.Set(0)
-	interrupt.Restore(mask)
+	//interrupt.Restore(mask)
 }

src/runtime/runtime_unix.go

@@ -11,6 +11,12 @@ import (
 	"unsafe"
 )
 
+const numCPU = 1
+
+func currentCPU() int {
+	return 0
+}
+
 //export write
 func libc_write(fd int32, buf unsafe.Pointer, count uint) int
 

src/runtime/scheduler_cores.go

@@ -3,17 +3,21 @@
 package runtime
 
 import (
+	"device/arm"
 	"internal/task"
-	"unsafe"
 )
 
 const hasScheduler = true
 
 const hasParallelism = true
 
+const coresVerbose = false
+
 var (
-	mainTask task.Task
+	mainTask   task.Task
-	cpuTasks [numCPU]*task.Task
+	cpuTasks   [numCPU]*task.Task
+	sleepQueue *task.Task
+	runQueue   *task.Task
 )
 
 func deadlock() {
@@ -23,7 +27,60 @@ func deadlock() {
 }
 
 func scheduleTask(t *task.Task) {
-	task.Resume(t)
+	schedulerLock()
+	switch t.RunState {
+	case task.RunStatePaused:
+		// Paused, state is saved on the stack.
+
+		if coresVerbose {
+			println("## schedule: add to runQueue")
+		}
+		addToRunQueue(t)
+		arm.Asm("sev")
+	case task.RunStateRunning:
+		// Not yet paused (probably going to pause very soon), so let the
+		// Pause() function know it can resume immediately.
+		t.RunState = task.RunStateResuming
+		if coresVerbose {
+			println("## schedule: mark as resuming")
+		}
+	default:
+		println("Unknown run state??")
+		for {
+		}
+	}
+	schedulerUnlock()
+}
+
+// Add task to runQueue.
+// Scheduler lock must be held when calling this function.
+func addToRunQueue(t *task.Task) {
+	t.Next = runQueue
+	runQueue = t
+}
+
+func addSleepTask(t *task.Task, wakeup timeUnit) {
+	// Save the timestamp when the task should be woken up.
+	t.Data = uint64(wakeup)
+
+	// Find the position where we should insert this task in the queue.
+	q := &sleepQueue
+	for {
+		if *q == nil {
+			// Found the end of the time queue. Insert it here, at the end.
+			break
+		}
+		if timeUnit((*q).Data) > timeUnit(t.Data) {
+			// Found a task in the queue that has a timeout before the
+			// to-be-sleeping task. Insert our task right before.
+			break
+		}
+		q = &(*q).Next
+	}
+
+	// Insert the task into the queue (this could be at the end, if *q is nil).
+	t.Next = *q
+	*q = t
 }
 
 func Gosched() {
@@ -61,17 +118,126 @@ func sleep(duration int64) {
 	}
 
 	wakeup := ticks() + nanosecondsToTicks(duration)
-	task.Sleep(uint64(wakeup))
+
+	schedulerLock()
+	addSleepTask(task.Current(), wakeup)
+	task.PauseLocked()
 }
 
 func run() {
 	initHeap()
 	cpuTasks[0] = &mainTask
-	task.Init(&mainTask, (*uintptr)(unsafe.Pointer(&stackTopSymbol)))
+	initAll() // TODO: move into main goroutine!
-	initAll()
+
-	startOtherCores()
+	until := ticks() + nanosecondsToTicks(200e6)
-	callMain()
+	for ticks() < until {
-	mainExited = true
+	}
+	println("\n\n=====")
+
+	go func() {
+		//initAll()
+		startSecondaryCores()
+		callMain()
+		mainExited = true
+	}()
+	schedulerLock()
+	scheduler()
+}
+
+func runSecondary(core uint32, t *task.Task) {
+	println("-- runSecondary")
+	cpuTasks[core] = t
+	println("-- locking for 2nd core")
+	schedulerLock()
+	println("--   locked!")
+	scheduler()
+}
+
+var schedulerIsRunning = false
+
+func scheduler() {
+	if coresVerbose {
+		println("** scheduler on core:", currentCPU())
+	}
+	for {
+		//until := ticks() + nanosecondsToTicks(100e6)
+		//for ticks() < until {
+		//}
+
+		// Check for ready-to-run tasks.
+		if runnable := runQueue; runnable != nil {
+			if coresVerbose {
+				println("** scheduler", currentCPU(), "run runnable")
+			}
+			// Pop off the run queue.
+			runQueue = runnable.Next
+			runnable.Next = nil
+
+			// Resume it now.
+			setCurrentTask(runnable)
+			schedulerUnlock()
+			runnable.Resume()
+			if coresVerbose {
+				println("** scheduler", currentCPU(), "  returned (from runqueue resume)")
+			}
+			setCurrentTask(nil)
+
+			continue
+		}
+
+		// If another core is using the clock, let it handle the sleep queue.
+		if schedulerIsRunning {
+			if coresVerbose {
+				println("** scheduler", currentCPU(), "wait for other core")
+			}
+			schedulerUnlock()
+			waitForEvents()
+			schedulerLock()
+			continue
+		}
+
+		if sleepingTask := sleepQueue; sleepingTask != nil {
+			now := ticks()
+			if now >= timeUnit(sleepingTask.Data) {
+				if coresVerbose {
+					println("** scheduler", currentCPU(), "run sleeping")
+				}
+				// This task is done sleeping.
+				// Resume it now.
+				sleepQueue = sleepQueue.Next
+				sleepingTask.Next = nil
+
+				setCurrentTask(sleepingTask)
+				schedulerUnlock()
+				sleepingTask.Resume()
+				if coresVerbose {
+					println("** scheduler", currentCPU(), "  returned (from sleepQueue resume)")
+				}
+				setCurrentTask(nil)
+				continue
+			}
+
+			delay := timeUnit(sleepingTask.Data) - now
+			if coresVerbose {
+				println("** scheduler", currentCPU(), "sleep", ticksToNanoseconds(delay)/1e6)
+			}
+
+			// Sleep for a bit until the next task is ready to run.
+			schedulerIsRunning = true
+			schedulerUnlock()
+			sleepTicks(delay)
+			schedulerLock()
+			schedulerIsRunning = false
+			continue
+		}
+
+		if coresVerbose {
+			println("** scheduler", currentCPU(), "wait for events")
+		}
+		schedulerUnlock()
+		waitForEvents()
+		schedulerLock()
+	}
 }
 
 func currentTask() *task.Task {
@@ -89,3 +255,8 @@ func runtimeTicks() uint64 {
 func runtimeSleepTicks(delay uint64) {
 	sleepTicks(timeUnit(delay))
 }
+
+//export tinygo_schedulerUnlock
+func tinygo_schedulerUnlock() {
+	schedulerUnlock()
+}

testdata/channel.go

@@ -12,6 +12,7 @@ var wg sync.WaitGroup
 type intchan chan int
 
 func main() {
+	time.Sleep(time.Second * 2)
 	ch := make(chan int, 2)
 	ch <- 1
 	println("len, cap of channel:", len(ch), cap(ch), ch == nil)

testdata/goroutines.go

@@ -5,19 +5,27 @@ import (
 	"time"
 )
 
-func init() {
+//func init() {
-	println("init")
+//	println("\n----")
-	go println("goroutine in init")
+//	println("init")
-	time.Sleep(1 * time.Millisecond)
+//	go println("goroutine in init")
-}
+//	time.Sleep(1 * time.Millisecond)
+//}
 
 func main() {
+	//for i := 0; i < 2; i++ {
+	//	println("...")
+	//	time.Sleep(time.Second)
+	//}
+
 	println("main 1")
 	go sub()
 	time.Sleep(1 * time.Millisecond)
 	println("main 2")
 	time.Sleep(2 * time.Millisecond)
 	println("main 3")
+	//time.Sleep(2 * time.Millisecond)
+	//println("main 4")
 
 	// Await a blocking call.
 	println("wait:")
@@ -101,6 +109,8 @@ func sub() {
 	println("sub 1")
 	time.Sleep(2 * time.Millisecond)
 	println("sub 2")
+	//time.Sleep(2 * time.Millisecond)
+	//println("sub 3")
 }
 
 func wait() {