refactor coroutine lowering and tasks

transform/coroutines.go

@@ -0,0 +1,929 @@
+package transform
+
+// This file lowers asynchronous functions and goroutine starts when using the coroutines scheduler.
+// This is accomplished by inserting LLVM intrinsics which are used in order to save the states of functions.
+
+import (
+	"errors"
+	"github.com/tinygo-org/tinygo/compiler/llvmutil"
+	"strconv"
+	"tinygo.org/x/go-llvm"
+)
+
+// LowerCoroutines turns async functions into coroutines.
+// This must be run with the coroutines scheduler.
+//
+// Before this pass, goroutine starts are expressed as a call to an intrinsic called "internal/task.start".
+// This intrinsic accepts the function pointer and a pointer to a struct containing the function's arguments.
+//
+// Before this pass, an intrinsic called "internal/task.Pause" is used to express suspensions of the current goroutine.
+//
+// This pass first accumulates a list of blocking functions.
+// A function is considered "blocking" if it calls "internal/task.Pause" or any other blocking function.
+//
+// Blocking calls are implemented by turning blocking functions into a coroutine.
+// The body of each blocking function is modified to start a new coroutine, and to return after the first suspend.
+// After calling a blocking function, the caller coroutine suspends.
+// The caller also provides a buffer to store the return value into.
+// When a blocking function returns, the return value is written into this buffer and then the caller is queued to run.
+//
+// Goroutine starts which invoke non-blocking functions are implemented as direct calls.
+// Goroutine starts are replaced with the creation of a new task data structure followed by a call to the start of the blocking function.
+// The task structure is populated with a "noop" coroutine before invoking the blocking function.
+// When the blocking function returns, it resumes this "noop" coroutine which does nothing.
+// The goroutine starter is able to continue after the first suspend of the started goroutine.
+//
+// The transformation of a function to a coroutine is accomplished using LLVM's coroutines system (https://llvm.org/docs/Coroutines.html).
+// The simplest implementation of a coroutine inserts a suspend point after every blocking call.
+//
+// Transforming blocking functions into coroutines and calls into suspend points is extremely expensive.
+// In many cases, a blocking call is followed immediately by a function terminator (a return or an "unreachable" instruction).
+// This is a blocking "tail call".
+// In a non-returning tail call (call to a non-returning function, such as an infinite loop), the coroutine can exit without any extra work.
+// In a returning tail call, the returned value must either be the return of the call or a value known before the call.
+// If the return value of the caller is the return of the callee, the coroutine can exit without any extra work and the tailed call will instead return to the caller of the caller.
+// If the return value is known in advance, this result can be stored into the parent's return buffer before the call so that a suspend is unnecessary.
+// If the callee returns an unnecessary value, a return buffer can be allocated on the heap so that it will outlive the coroutine.
+//
+// In the implementation of time.Sleep, the current task is pushed onto a timer queue and then suspended.
+// Since the only suspend point is a call to "internal/task.Pause" followed by a return, there is no need to transform this into a coroutine.
+// This generalizes to all blocking functions in which all suspend points can be elided.
+// This optimization saves a substantial amount of binary size.
+func LowerCoroutines(mod llvm.Module, needStackSlots bool) error {
+	ctx := mod.Context()
+
+	builder := ctx.NewBuilder()
+	defer builder.Dispose()
+
+	target := llvm.NewTargetData(mod.DataLayout())
+	defer target.Dispose()
+
+	pass := &coroutineLoweringPass{
+		mod:     mod,
+		ctx:     ctx,
+		builder: builder,
+		target:  target,
+	}
+
+	err := pass.load()
+	if err != nil {
+		return err
+	}
+
+	// Supply task operands to async calls.
+	pass.supplyTaskOperands()
+
+	// Analyze async returns.
+	pass.returnAnalysisPass()
+
+	// Categorize async calls.
+	pass.categorizeCalls()
+
+	// Lower async functions.
+	pass.lowerFuncsPass()
+
+	// Lower calls to internal/task.Current.
+	pass.lowerCurrent()
+
+	// Lower goroutine starts.
+	pass.lowerStartsPass()
+
+	// Fix annotations on async call params.
+	pass.fixAnnotations()
+
+	if needStackSlots {
+		// Set up garbage collector tracking of tasks at start.
+		err = pass.trackGoroutines()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// asyncFunc is a metadata container for an asynchronous function.
+type asyncFunc struct {
+	// fn is the underlying function pointer.
+	fn llvm.Value
+
+	// rawTask is the parameter where the task pointer is passed in.
+	rawTask llvm.Value
+
+	// callers is a set of all functions which call this async function.
+	callers map[llvm.Value]struct{}
+
+	// returns is a map of terminal basic blocks to their return kinds.
+	returns map[llvm.BasicBlock]returnKind
+
+	// calls is the set of all calls in the asyncFunc.
+	// normalCalls is the set of all intermideate suspending calls in the asyncFunc.
+	// tailCalls is the set of all tail calls in the asyncFunc.
+	calls, normalCalls, tailCalls map[llvm.Value]struct{}
+}
+
+// coroutineLoweringPass is a goroutine lowering pass which is used with the "coroutines" scheduler.
+type coroutineLoweringPass struct {
+	mod     llvm.Module
+	ctx     llvm.Context
+	builder llvm.Builder
+	target  llvm.TargetData
+
+	// asyncFuncs is a map of all asyncFuncs.
+	// The map keys are function pointers.
+	asyncFuncs map[llvm.Value]*asyncFunc
+
+	// calls is a slice of all of the async calls in the module.
+	calls []llvm.Value
+
+	i8ptr llvm.Type
+
+	// memory management functions from the runtime
+	alloc, free llvm.Value
+
+	// coroutine intrinsics
+	start, pause, current                                   llvm.Value
+	setState, setRetPtr, getRetPtr, returnTo, returnCurrent llvm.Value
+	createTask                                              llvm.Value
+
+	// llvm.coro intrinsics
+	coroId, coroSize, coroBegin, coroSuspend, coroEnd, coroFree, coroSave llvm.Value
+}
+
+// findAsyncFuncs finds all asynchronous functions.
+// A function is considered asynchronous if it calls an asynchronous function or intrinsic.
+func (c *coroutineLoweringPass) findAsyncFuncs() {
+	asyncs := map[llvm.Value]*asyncFunc{}
+	calls := []llvm.Value{}
+
+	// Use a breadth-first search to find all async functions.
+	worklist := []llvm.Value{c.pause}
+	for len(worklist) > 0 {
+		// Pop a function off the worklist.
+		fn := worklist[len(worklist)-1]
+		worklist = worklist[:len(worklist)-1]
+
+		// Get task pointer argument.
+		task := fn.LastParam()
+		if fn != c.pause && (task.IsNil() || task.Name() != "parentHandle") {
+			panic("trying to make exported function async: " + fn.Name())
+		}
+
+		// Search all uses of the function while collecting callers.
+		callers := map[llvm.Value]struct{}{}
+		for use := fn.FirstUse(); !use.IsNil(); use = use.NextUse() {
+			user := use.User()
+			if user.IsACallInst().IsNil() {
+				// User is not a call instruction, so this is irrelevant.
+				continue
+			}
+			if user.CalledValue() != fn {
+				// Not the called value.
+				continue
+			}
+
+			// Add to calls list.
+			calls = append(calls, user)
+
+			// Get the caller.
+			caller := user.InstructionParent().Parent()
+
+			// Add as caller.
+			callers[caller] = struct{}{}
+
+			if _, ok := asyncs[caller]; ok {
+				// Already marked caller as async.
+				continue
+			}
+
+			// Mark the caller as async.
+			// Use nil as a temporary value. It will be replaced later.
+			asyncs[caller] = nil
+
+			// Put the caller on the worklist.
+			worklist = append(worklist, caller)
+		}
+
+		asyncs[fn] = &asyncFunc{
+			fn:      fn,
+			rawTask: task,
+			callers: callers,
+		}
+	}
+
+	c.asyncFuncs = asyncs
+	c.calls = calls
+}
+
+func (c *coroutineLoweringPass) load() error {
+	// Find memory management functions from the runtime.
+	c.alloc = c.mod.NamedFunction("runtime.alloc")
+	if c.alloc.IsNil() {
+		return ErrMissingIntrinsic{"runtime.alloc"}
+	}
+	c.free = c.mod.NamedFunction("runtime.free")
+	if c.free.IsNil() {
+		return ErrMissingIntrinsic{"runtime.free"}
+	}
+
+	// Find intrinsics.
+	c.pause = c.mod.NamedFunction("internal/task.Pause")
+	if c.pause.IsNil() {
+		return ErrMissingIntrinsic{"internal/task.Pause"}
+	}
+	c.start = c.mod.NamedFunction("internal/task.start")
+	if c.start.IsNil() {
+		return ErrMissingIntrinsic{"internal/task.start"}
+	}
+	c.current = c.mod.NamedFunction("internal/task.Current")
+	if c.current.IsNil() {
+		return ErrMissingIntrinsic{"internal/task.Current"}
+	}
+	c.setState = c.mod.NamedFunction("(*internal/task.Task).setState")
+	if c.setState.IsNil() {
+		return ErrMissingIntrinsic{"(*internal/task.Task).setState"}
+	}
+	c.setRetPtr = c.mod.NamedFunction("(*internal/task.Task).setReturnPtr")
+	if c.setRetPtr.IsNil() {
+		return ErrMissingIntrinsic{"(*internal/task.Task).setReturnPtr"}
+	}
+	c.getRetPtr = c.mod.NamedFunction("(*internal/task.Task).getReturnPtr")
+	if c.getRetPtr.IsNil() {
+		return ErrMissingIntrinsic{"(*internal/task.Task).getReturnPtr"}
+	}
+	c.returnTo = c.mod.NamedFunction("(*internal/task.Task).returnTo")
+	if c.returnTo.IsNil() {
+		return ErrMissingIntrinsic{"(*internal/task.Task).returnTo"}
+	}
+	c.returnCurrent = c.mod.NamedFunction("(*internal/task.Task).returnCurrent")
+	if c.returnCurrent.IsNil() {
+		return ErrMissingIntrinsic{"(*internal/task.Task).returnCurrent"}
+	}
+	c.createTask = c.mod.NamedFunction("internal/task.createTask")
+	if c.createTask.IsNil() {
+		return ErrMissingIntrinsic{"internal/task.createTask"}
+	}
+
+	// Find async functions.
+	c.findAsyncFuncs()
+
+	// Get i8* type.
+	c.i8ptr = llvm.PointerType(c.ctx.Int8Type(), 0)
+
+	// Build LLVM coroutine intrinsic.
+	coroIdType := llvm.FunctionType(c.ctx.TokenType(), []llvm.Type{c.ctx.Int32Type(), c.i8ptr, c.i8ptr, c.i8ptr}, false)
+	c.coroId = llvm.AddFunction(c.mod, "llvm.coro.id", coroIdType)
+
+	sizeT := c.alloc.Param(0).Type()
+	coroSizeType := llvm.FunctionType(sizeT, nil, false)
+	c.coroSize = llvm.AddFunction(c.mod, "llvm.coro.size.i"+strconv.Itoa(sizeT.IntTypeWidth()), coroSizeType)
+
+	coroBeginType := llvm.FunctionType(c.i8ptr, []llvm.Type{c.ctx.TokenType(), c.i8ptr}, false)
+	c.coroBegin = llvm.AddFunction(c.mod, "llvm.coro.begin", coroBeginType)
+
+	coroSuspendType := llvm.FunctionType(c.ctx.Int8Type(), []llvm.Type{c.ctx.TokenType(), c.ctx.Int1Type()}, false)
+	c.coroSuspend = llvm.AddFunction(c.mod, "llvm.coro.suspend", coroSuspendType)
+
+	coroEndType := llvm.FunctionType(c.ctx.Int1Type(), []llvm.Type{c.i8ptr, c.ctx.Int1Type()}, false)
+	c.coroEnd = llvm.AddFunction(c.mod, "llvm.coro.end", coroEndType)
+
+	coroFreeType := llvm.FunctionType(c.i8ptr, []llvm.Type{c.ctx.TokenType(), c.i8ptr}, false)
+	c.coroFree = llvm.AddFunction(c.mod, "llvm.coro.free", coroFreeType)
+
+	coroSaveType := llvm.FunctionType(c.ctx.TokenType(), []llvm.Type{c.i8ptr}, false)
+	c.coroSave = llvm.AddFunction(c.mod, "llvm.coro.save", coroSaveType)
+
+	return nil
+}
+
+// lowerStartSync lowers a goroutine start of a synchronous function to a synchronous call.
+func (c *coroutineLoweringPass) lowerStartSync(start llvm.Value) {
+	c.builder.SetInsertPointBefore(start)
+
+	// Get function to call.
+	fn := start.Operand(0).Operand(0)
+
+	// Create the list of params for the call.
+	paramTypes := fn.Type().ElementType().ParamTypes()
+	params := llvmutil.EmitPointerUnpack(c.builder, c.mod, start.Operand(1), paramTypes[:len(paramTypes)-1])
+	params = append(params, llvm.Undef(c.i8ptr))
+
+	// Generate call to function.
+	c.builder.CreateCall(fn, params, "")
+
+	// Remove start call.
+	start.EraseFromParentAsInstruction()
+}
+
+// supplyTaskOperands fills in the task operands of async calls.
+func (c *coroutineLoweringPass) supplyTaskOperands() {
+	var curCalls []llvm.Value
+	for use := c.current.FirstUse(); !use.IsNil(); use = use.NextUse() {
+		curCalls = append(curCalls, use.User())
+	}
+	for _, call := range append(curCalls, c.calls...) {
+		c.builder.SetInsertPointBefore(call)
+		task := c.asyncFuncs[call.InstructionParent().Parent()].rawTask
+		call.SetOperand(call.OperandsCount()-2, task)
+	}
+}
+
+// returnKind is a classification of a type of function terminator.
+type returnKind uint8
+
+const (
+	// returnNormal is a terminator that returns a value normally from a function.
+	returnNormal returnKind = iota
+
+	// returnVoid is a terminator that exits normally without returning a value.
+	returnVoid
+
+	// returnVoidTail is a terminator which is a tail call to a void-returning function in a void-returning function.
+	returnVoidTail
+
+	// returnTail is a terinator which is a tail call to a value-returning function where the value is returned by the callee.
+	returnTail
+
+	// returnDeadTail is a terminator which is a call to a non-returning asynchronous function.
+	returnDeadTail
+
+	// returnAlternateTail is a terminator which is a tail call to a value-returning function where a previously acquired value is returned by the callee.
+	returnAlternateTail
+
+	// returnDitchedTail is a terminator which is a tail call to a value-returning function, where the callee returns void.
+	returnDitchedTail
+
+	// returnDelayedValue is a terminator in which a void-returning tail call is followed by a return of a previous value.
+	returnDelayedValue
+)
+
+// isAsyncCall returns whether the specified call is async.
+func (c *coroutineLoweringPass) isAsyncCall(call llvm.Value) bool {
+	_, ok := c.asyncFuncs[call.CalledValue()]
+	return ok
+}
+
+// analyzeFuncReturns analyzes and classifies the returns of a function.
+func (c *coroutineLoweringPass) analyzeFuncReturns(fn *asyncFunc) {
+	returns := map[llvm.BasicBlock]returnKind{}
+	if fn.fn == c.pause {
+		// Skip pause.
+		fn.returns = returns
+		return
+	}
+
+	for _, bb := range fn.fn.BasicBlocks() {
+		last := bb.LastInstruction()
+		switch last.InstructionOpcode() {
+		case llvm.Ret:
+			// Check if it is a void return.
+			isVoid := fn.fn.Type().ElementType().ReturnType().TypeKind() == llvm.VoidTypeKind
+
+			// Analyze previous instruction.
+			prev := llvm.PrevInstruction(last)
+			switch {
+			case prev.IsNil():
+				fallthrough
+			case prev.IsACallInst().IsNil():
+				fallthrough
+			case !c.isAsyncCall(prev):
+				// This is not any form of asynchronous tail call.
+				if isVoid {
+					returns[bb] = returnVoid
+				} else {
+					returns[bb] = returnNormal
+				}
+			case isVoid:
+				if prev.CalledValue().Type().ElementType().ReturnType().TypeKind() == llvm.VoidTypeKind {
+					// This is a tail call to a void-returning function from a function with a void return.
+					returns[bb] = returnVoidTail
+				} else {
+					// This is a tail call to a value-returning function from a function with a void return.
+					// The returned value will be ditched.
+					returns[bb] = returnDitchedTail
+				}
+			case last.Operand(0) == prev:
+				// This is a regular tail call. The return of the callee is returned to the parent.
+				returns[bb] = returnTail
+			case prev.CalledValue().Type().ElementType().ReturnType().TypeKind() == llvm.VoidTypeKind:
+				// This is a tail call that returns a previous value after waiting on a void function.
+				returns[bb] = returnDelayedValue
+			default:
+				// This is a tail call that returns a value that is available before the function call.
+				returns[bb] = returnAlternateTail
+			}
+		case llvm.Unreachable:
+			prev := llvm.PrevInstruction(last)
+
+			if prev.IsNil() || prev.IsACallInst().IsNil() || !c.isAsyncCall(prev) {
+				// This unreachable instruction does not behave as an asynchronous return.
+				continue
+			}
+
+			// This is an asyncnhronous tail call to function that does not return.
+			returns[bb] = returnDeadTail
+		}
+	}
+
+	fn.returns = returns
+}
+
+// returnAnalysisPass runs an analysis pass which classifies the returns of all async functions.
+func (c *coroutineLoweringPass) returnAnalysisPass() {
+	for _, async := range c.asyncFuncs {
+		c.analyzeFuncReturns(async)
+	}
+}
+
+// categorizeCalls categorizes all asynchronous calls into regular vs. async and matches them to their callers.
+func (c *coroutineLoweringPass) categorizeCalls() {
+	// Sort calls into their respective callers.
+	for _, async := range c.asyncFuncs {
+		async.calls = map[llvm.Value]struct{}{}
+	}
+	for _, call := range c.calls {
+		c.asyncFuncs[call.InstructionParent().Parent()].calls[call] = struct{}{}
+	}
+
+	// Seperate regular and tail calls.
+	for _, async := range c.asyncFuncs {
+		// Find all tail calls (of any kind).
+		tails := map[llvm.Value]struct{}{}
+		for ret, kind := range async.returns {
+			switch kind {
+			case returnVoidTail, returnTail, returnDeadTail, returnAlternateTail, returnDitchedTail, returnDelayedValue:
+				// This is a tail return. The previous instruction is a tail call.
+				tails[llvm.PrevInstruction(ret.LastInstruction())] = struct{}{}
+			}
+		}
+
+		// Find all regular calls.
+		regulars := map[llvm.Value]struct{}{}
+		for call := range async.calls {
+			if _, ok := tails[call]; ok {
+				// This is a tail call.
+				continue
+			}
+
+			regulars[call] = struct{}{}
+		}
+
+		async.tailCalls = tails
+		async.normalCalls = regulars
+	}
+}
+
+// lowerFuncsPass lowers all functions, turning them into coroutines if necessary.
+func (c *coroutineLoweringPass) lowerFuncsPass() {
+	for _, fn := range c.asyncFuncs {
+		if fn.fn == c.pause {
+			// Skip. It is an intrinsic.
+			continue
+		}
+
+		if len(fn.normalCalls) == 0 {
+			// No suspend points. Lower without turning it into a coroutine.
+			c.lowerFuncFast(fn)
+		} else {
+			// There are suspend points, so it is necessary to turn this into a coroutine.
+			c.lowerFuncCoro(fn)
+		}
+	}
+}
+
+func (async *asyncFunc) hasValueStoreReturn() bool {
+	for _, kind := range async.returns {
+		switch kind {
+		case returnNormal, returnAlternateTail, returnDelayedValue:
+			return true
+		}
+	}
+
+	return false
+}
+
+// heapAlloc creates a heap allocation large enough to hold the supplied type.
+// The allocation is returned as a raw i8* pointer.
+// This allocation is not automatically tracked by the garbage collector, and should thus be stored into a tracked memory object immediately.
+func (c *coroutineLoweringPass) heapAlloc(t llvm.Type, name string) llvm.Value {
+	sizeT := c.alloc.FirstParam().Type()
+	size := llvm.ConstInt(sizeT, c.target.TypeAllocSize(t), false)
+	return c.builder.CreateCall(c.alloc, []llvm.Value{size, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, name)
+}
+
+// lowerFuncFast lowers an async function that has no suspend points.
+func (c *coroutineLoweringPass) lowerFuncFast(fn *asyncFunc) {
+	// Get return type.
+	retType := fn.fn.Type().ElementType().ReturnType()
+
+	// Get task value.
+	c.insertPointAfterAllocas(fn.fn)
+	task := c.builder.CreateCall(c.current, []llvm.Value{llvm.Undef(c.i8ptr), fn.rawTask}, "task")
+
+	// Get return pointer if applicable.
+	var rawRetPtr, retPtr llvm.Value
+	if fn.hasValueStoreReturn() {
+		rawRetPtr = c.builder.CreateCall(c.getRetPtr, []llvm.Value{task, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "ret.ptr")
+		retType = fn.fn.Type().ElementType().ReturnType()
+		retPtr = c.builder.CreateBitCast(rawRetPtr, llvm.PointerType(retType, 0), "ret.ptr.bitcast")
+	}
+
+	// Lower returns.
+	for ret, kind := range fn.returns {
+		// Get terminator.
+		terminator := ret.LastInstruction()
+
+		// Get tail call if applicable.
+		var call llvm.Value
+		switch kind {
+		case returnVoidTail, returnTail, returnDeadTail, returnAlternateTail, returnDitchedTail, returnDelayedValue:
+			call = llvm.PrevInstruction(terminator)
+		}
+
+		switch kind {
+		case returnNormal:
+			c.builder.SetInsertPointBefore(terminator)
+
+			// Store value into return pointer.
+			c.builder.CreateStore(terminator.Operand(0), retPtr)
+
+			// Resume caller.
+			c.builder.CreateCall(c.returnCurrent, []llvm.Value{task, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "")
+
+			// Erase return argument.
+			terminator.SetOperand(0, llvm.Undef(retType))
+		case returnVoid:
+			c.builder.SetInsertPointBefore(terminator)
+
+			// Resume caller.
+			c.builder.CreateCall(c.returnCurrent, []llvm.Value{task, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "")
+		case returnVoidTail:
+			// Nothing to do. There is already a tail call followed by a void return.
+		case returnTail:
+			// Erase return argument.
+			terminator.SetOperand(0, llvm.Undef(retType))
+		case returnDeadTail:
+			// Replace unreachable with immediate return, without resuming the caller.
+			c.builder.SetInsertPointBefore(terminator)
+			if retType.TypeKind() == llvm.VoidTypeKind {
+				c.builder.CreateRetVoid()
+			} else {
+				c.builder.CreateRet(llvm.Undef(retType))
+			}
+			terminator.EraseFromParentAsInstruction()
+		case returnAlternateTail:
+			c.builder.SetInsertPointBefore(call)
+
+			// Store return value.
+			c.builder.CreateStore(terminator.Operand(0), retPtr)
+
+			// Heap-allocate a return buffer for the discarded return.
+			alternateBuf := c.heapAlloc(call.Type(), "ret.alternate")
+			c.builder.CreateCall(c.setRetPtr, []llvm.Value{task, alternateBuf, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "")
+
+			// Erase return argument.
+			terminator.SetOperand(0, llvm.Undef(retType))
+		case returnDitchedTail:
+			c.builder.SetInsertPointBefore(call)
+
+			// Heap-allocate a return buffer for the discarded return.
+			ditchBuf := c.heapAlloc(call.Type(), "ret.ditch")
+			c.builder.CreateCall(c.setRetPtr, []llvm.Value{task, ditchBuf, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "")
+		case returnDelayedValue:
+			c.builder.SetInsertPointBefore(call)
+
+			// Store value into return pointer.
+			c.builder.CreateStore(terminator.Operand(0), retPtr)
+
+			// Erase return argument.
+			terminator.SetOperand(0, llvm.Undef(retType))
+		}
+
+		// Delete call if it is a pause, because it has already been lowered.
+		if !call.IsNil() && call.CalledValue() == c.pause {
+			call.EraseFromParentAsInstruction()
+		}
+	}
+}
+
+// insertPointAfterAllocas sets the insert point of the builder to be immediately after the last alloca in the entry block.
+func (c *coroutineLoweringPass) insertPointAfterAllocas(fn llvm.Value) {
+	inst := fn.EntryBasicBlock().FirstInstruction()
+	for !inst.IsAAllocaInst().IsNil() {
+		inst = llvm.NextInstruction(inst)
+	}
+	c.builder.SetInsertPointBefore(inst)
+}
+
+// lowerCallReturn lowers the return value of an async call by creating a return buffer and loading the returned value from it.
+func (c *coroutineLoweringPass) lowerCallReturn(caller *asyncFunc, call llvm.Value) {
+	// Get return type.
+	retType := call.Type()
+	if retType.TypeKind() == llvm.VoidTypeKind {
+		// Void return. Nothing to do.
+		return
+	}
+
+	// Create alloca for return buffer.
+	alloca := llvmutil.CreateInstructionAlloca(c.builder, c.mod, retType, call, "call.return")
+
+	// Store new return buffer into task before call.
+	c.builder.SetInsertPointBefore(call)
+	task := c.builder.CreateCall(c.current, []llvm.Value{llvm.Undef(c.i8ptr), caller.rawTask}, "call.task")
+	retPtr := c.builder.CreateBitCast(alloca, c.i8ptr, "call.return.bitcast")
+	c.builder.CreateCall(c.setRetPtr, []llvm.Value{task, retPtr, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "")
+
+	// Load return value after call.
+	c.builder.SetInsertPointBefore(llvm.NextInstruction(call))
+	ret := c.builder.CreateLoad(alloca, "call.return.load")
+
+	// Replace call value with loaded return.
+	call.ReplaceAllUsesWith(ret)
+}
+
+// lowerFuncCoro transforms an async function into a coroutine by lowering async operations to `llvm.coro` intrinsics.
+// See https://llvm.org/docs/Coroutines.html for more information on these intrinsics.
+func (c *coroutineLoweringPass) lowerFuncCoro(fn *asyncFunc) {
+	returnType := fn.fn.Type().ElementType().ReturnType()
+
+	// Prepare coroutine state.
+	c.insertPointAfterAllocas(fn.fn)
+	// %coro.id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+	coroId := c.builder.CreateCall(c.coroId, []llvm.Value{
+		llvm.ConstInt(c.ctx.Int32Type(), 0, false),
+		llvm.ConstNull(c.i8ptr),
+		llvm.ConstNull(c.i8ptr),
+		llvm.ConstNull(c.i8ptr),
+	}, "coro.id")
+	// %coro.size = call i32 @llvm.coro.size.i32()
+	coroSize := c.builder.CreateCall(c.coroSize, []llvm.Value{}, "coro.size")
+	// %coro.alloc = call i8* runtime.alloc(i32 %coro.size)
+	coroAlloc := c.builder.CreateCall(c.alloc, []llvm.Value{coroSize, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "coro.alloc")
+	// %coro.state = call noalias i8* @llvm.coro.begin(token %coro.id, i8* %coro.alloc)
+	coroState := c.builder.CreateCall(c.coroBegin, []llvm.Value{coroId, coroAlloc}, "coro.state")
+	// Store state into task.
+	task := c.builder.CreateCall(c.current, []llvm.Value{llvm.Undef(c.i8ptr), fn.rawTask}, "task")
+	parentState := c.builder.CreateCall(c.setState, []llvm.Value{task, coroState, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "task.state.parent")
+	// Get return pointer if needed.
+	var retPtr llvm.Value
+	if fn.hasValueStoreReturn() {
+		retPtr = c.builder.CreateCall(c.getRetPtr, []llvm.Value{task, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "task.retPtr")
+		retPtr = c.builder.CreateBitCast(retPtr, llvm.PointerType(fn.fn.Type().ElementType().ReturnType(), 0), "task.retPtr.bitcast")
+	}
+
+	// Build suspend block.
+	// This is executed when the coroutine is about to suspend.
+	suspend := c.ctx.AddBasicBlock(fn.fn, "suspend")
+	c.builder.SetInsertPointAtEnd(suspend)
+	// %unused = call i1 @llvm.coro.end(i8* %coro.state, i1 false)
+	c.builder.CreateCall(c.coroEnd, []llvm.Value{coroState, llvm.ConstInt(c.ctx.Int1Type(), 0, false)}, "unused")
+	// Insert return.
+	if returnType.TypeKind() == llvm.VoidTypeKind {
+		c.builder.CreateRetVoid()
+	} else {
+		c.builder.CreateRet(llvm.Undef(returnType))
+	}
+
+	// Build cleanup block.
+	// This is executed before the function returns in order to clean up resources.
+	cleanup := c.ctx.AddBasicBlock(fn.fn, "cleanup")
+	c.builder.SetInsertPointAtEnd(cleanup)
+	// %coro.memFree = call i8* @llvm.coro.free(token %coro.id, i8* %coro.state)
+	coroMemFree := c.builder.CreateCall(c.coroFree, []llvm.Value{coroId, coroState}, "coro.memFree")
+	// call i8* runtime.free(i8* %coro.memFree)
+	c.builder.CreateCall(c.free, []llvm.Value{coroMemFree, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "")
+	// Branch to suspend block.
+	c.builder.CreateBr(suspend)
+
+	// Restore old state before tail calls.
+	for call := range fn.tailCalls {
+		if fn.returns[call.InstructionParent()] == returnDeadTail {
+			// Callee never returns, so the state restore is ineffectual.
+			continue
+		}
+
+		c.builder.SetInsertPointBefore(call)
+		c.builder.CreateCall(c.setState, []llvm.Value{task, parentState, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "coro.state.restore")
+	}
+
+	// Lower returns.
+	for ret, kind := range fn.returns {
+		// Get terminator instruction.
+		terminator := ret.LastInstruction()
+
+		// Get tail call if applicable.
+		var call llvm.Value
+		switch kind {
+		case returnVoidTail, returnTail, returnDeadTail, returnAlternateTail, returnDitchedTail, returnDelayedValue:
+			call = llvm.PrevInstruction(terminator)
+		}
+
+		switch kind {
+		case returnNormal:
+			c.builder.SetInsertPointBefore(terminator)
+
+			// Store value into return pointer.
+			c.builder.CreateStore(terminator.Operand(0), retPtr)
+
+			// Resume caller.
+			c.builder.CreateCall(c.returnTo, []llvm.Value{task, parentState, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "")
+		case returnVoid:
+			c.builder.SetInsertPointBefore(terminator)
+
+			// Resume caller.
+			c.builder.CreateCall(c.returnTo, []llvm.Value{task, parentState, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "")
+		case returnVoidTail, returnTail, returnDeadTail:
+			// Nothing to do.
+		case returnAlternateTail:
+			c.builder.SetInsertPointBefore(call)
+
+			// Store return value.
+			c.builder.CreateStore(ret.LastInstruction().Operand(0), retPtr)
+
+			// Heap-allocate a return buffer for the discarded return.
+			alternateBuf := c.heapAlloc(call.Type(), "ret.alternate")
+			c.builder.CreateCall(c.setRetPtr, []llvm.Value{task, alternateBuf, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "")
+		case returnDitchedTail:
+			c.builder.SetInsertPointBefore(call)
+
+			// Heap-allocate a return buffer for the discarded return.
+			ditchBuf := c.heapAlloc(call.Type(), "ret.ditch")
+			c.builder.CreateCall(c.setRetPtr, []llvm.Value{task, ditchBuf, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "")
+		case returnDelayedValue:
+			c.builder.SetInsertPointBefore(call)
+
+			// Store return value.
+			c.builder.CreateStore(ret.LastInstruction().Operand(0), retPtr)
+		}
+
+		// Delete call if it is a pause, because it has already been lowered.
+		if !call.IsNil() && call.CalledValue() == c.pause {
+			call.EraseFromParentAsInstruction()
+		}
+
+		// Replace terminator with branch to cleanup.
+		terminator.EraseFromParentAsInstruction()
+		c.builder.SetInsertPointAtEnd(ret)
+		c.builder.CreateBr(cleanup)
+	}
+
+	// Lower regular calls.
+	for call := range fn.normalCalls {
+		// Lower return value of call.
+		c.lowerCallReturn(fn, call)
+
+		// Get originating basic block.
+		bb := call.InstructionParent()
+
+		// Split block.
+		wakeup := llvmutil.SplitBasicBlock(c.builder, call, llvm.NextBasicBlock(bb), "wakeup")
+
+		// Insert suspension and switch.
+		c.builder.SetInsertPointAtEnd(bb)
+		// %coro.save = call token @llvm.coro.save(i8* %coro.state)
+		save := c.builder.CreateCall(c.coroSave, []llvm.Value{coroState}, "coro.save")
+		// %call.suspend = llvm.coro.suspend(token %coro.save, i1 false)
+		// switch i8 %call.suspend, label %suspend [i8 0, label %wakeup
+		//                                          i8 1, label %cleanup]
+		suspendValue := c.builder.CreateCall(c.coroSuspend, []llvm.Value{save, llvm.ConstInt(c.ctx.Int1Type(), 0, false)}, "call.suspend")
+		sw := c.builder.CreateSwitch(suspendValue, suspend, 2)
+		sw.AddCase(llvm.ConstInt(c.ctx.Int8Type(), 0, false), wakeup)
+		sw.AddCase(llvm.ConstInt(c.ctx.Int8Type(), 1, false), cleanup)
+
+		// Delete call if it is a pause, because it has already been lowered.
+		if call.CalledValue() == c.pause {
+			call.EraseFromParentAsInstruction()
+		}
+	}
+}
+
+// lowerCurrent lowers calls to internal/task.Current to bitcasts.
+func (c *coroutineLoweringPass) lowerCurrent() error {
+	taskType := c.current.Type().ElementType().ReturnType()
+	deleteQueue := []llvm.Value{}
+	for use := c.current.FirstUse(); !use.IsNil(); use = use.NextUse() {
+		// Get user.
+		user := use.User()
+
+		if user.IsACallInst().IsNil() || user.CalledValue() != c.current {
+			return errorAt(user, "unexpected non-call use of task.Current")
+		}
+
+		// Replace with bitcast.
+		c.builder.SetInsertPointBefore(user)
+		raw := user.Operand(1)
+		if !raw.IsAUndefValue().IsNil() || raw.IsNull() {
+			return errors.New("undefined task")
+		}
+		task := c.builder.CreateBitCast(raw, taskType, "task.current")
+		user.ReplaceAllUsesWith(task)
+		deleteQueue = append(deleteQueue, user)
+	}
+
+	// Delete calls.
+	for _, inst := range deleteQueue {
+		inst.EraseFromParentAsInstruction()
+	}
+
+	return nil
+}
+
+// lowerStart lowers a goroutine start into a task creation and call or a synchronous call.
+func (c *coroutineLoweringPass) lowerStart(start llvm.Value) {
+	c.builder.SetInsertPointBefore(start)
+
+	// Get function to call.
+	fn := start.Operand(0).Operand(0)
+
+	if _, ok := c.asyncFuncs[fn]; !ok {
+		// Turn into synchronous call.
+		c.lowerStartSync(start)
+		return
+	}
+
+	// Create the list of params for the call.
+	paramTypes := fn.Type().ElementType().ParamTypes()
+	params := llvmutil.EmitPointerUnpack(c.builder, c.mod, start.Operand(1), paramTypes[:len(paramTypes)-1])
+
+	// Create task.
+	task := c.builder.CreateCall(c.createTask, []llvm.Value{llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "start.task")
+	rawTask := c.builder.CreateBitCast(task, c.i8ptr, "start.task.bitcast")
+	params = append(params, rawTask)
+
+	// Generate a return buffer if necessary.
+	returnType := fn.Type().ElementType().ReturnType()
+	if returnType.TypeKind() == llvm.VoidTypeKind {
+		// No return buffer necessary for a void return.
+	} else {
+		// Check for any undead returns.
+		var undead bool
+		for _, kind := range c.asyncFuncs[fn].returns {
+			if kind != returnDeadTail {
+				// This return results in a value being eventually stored.
+				undead = true
+				break
+			}
+		}
+		if undead {
+			// The function stores a value into a return buffer, so we need to create one.
+			retBuf := c.heapAlloc(returnType, "ret.ditch")
+			c.builder.CreateCall(c.setRetPtr, []llvm.Value{task, retBuf, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "")
+		}
+	}
+
+	// Generate call to function.
+	c.builder.CreateCall(fn, params, "")
+
+	// Erase start call.
+	start.EraseFromParentAsInstruction()
+}
+
+// lowerStartsPass lowers all goroutine starts.
+func (c *coroutineLoweringPass) lowerStartsPass() {
+	starts := []llvm.Value{}
+	for use := c.start.FirstUse(); !use.IsNil(); use = use.NextUse() {
+		starts = append(starts, use.User())
+	}
+	for _, start := range starts {
+		c.lowerStart(start)
+	}
+}
+
+func (c *coroutineLoweringPass) fixAnnotations() {
+	for f := range c.asyncFuncs {
+		// These properties were added by the functionattrs pass. Remove
+		// them, because now we start using the parameter.
+		// https://llvm.org/docs/Passes.html#functionattrs-deduce-function-attributes
+		for _, kind := range []string{"nocapture", "readnone"} {
+			kindID := llvm.AttributeKindID(kind)
+			n := f.ParamsCount()
+			for i := 0; i <= n; i++ {
+				f.RemoveEnumAttributeAtIndex(i, kindID)
+			}
+		}
+	}
+}
+
+// trackGoroutines adds runtime.trackPointer calls to track goroutine starts and data.
+func (c *coroutineLoweringPass) trackGoroutines() error {
+	trackPointer := c.mod.NamedFunction("runtime.trackPointer")
+	if trackPointer.IsNil() {
+		return ErrMissingIntrinsic{"runtime.trackPointer"}
+	}
+
+	trackFunctions := []llvm.Value{c.createTask, c.setState, c.getRetPtr}
+	for _, fn := range trackFunctions {
+		for use := fn.FirstUse(); !use.IsNil(); use = use.NextUse() {
+			call := use.User()
+
+			c.builder.SetInsertPointBefore(llvm.NextInstruction(call))
+			ptr := call
+			if ptr.Type() != c.i8ptr {
+				ptr = c.builder.CreateBitCast(call, c.i8ptr, "")
+			}
+			c.builder.CreateCall(trackPointer, []llvm.Value{ptr, llvm.Undef(c.i8ptr), llvm.Undef(c.i8ptr)}, "")
+		}
+	}
+
+	return nil
+}

transform/errors.go

@@ -46,3 +46,12 @@ func getPosition(val llvm.Value) token.Position {
 		return token.Position{}
 	}
 }
+
+// ErrMissingIntrinsic is an error indicating that a required intrinsic was not found in the module.
+type ErrMissingIntrinsic struct {
+	Name string
+}
+
+func (err ErrMissingIntrinsic) Error() string {
+	return "missing intrinsic: " + err.Name
+}

transform/func-lowering.go

@@ -181,29 +181,15 @@ func LowerFuncValues(mod llvm.Module) {
 				// Remove some casts, checks, and the old call which we're going
 				// to replace.
 				for _, callIntPtr := range getUses(getFuncPtrCall) {
-					if !callIntPtr.IsACallInst().IsNil() && callIntPtr.CalledValue().Name() == "runtime.makeGoroutine" {
-						// Special case for runtime.makeGoroutine.
-						for _, inttoptr := range getUses(callIntPtr) {
-							if inttoptr.IsAIntToPtrInst().IsNil() {
-								panic("expected a inttoptr")
-							}
-							for _, use := range getUses(inttoptr) {
-								addFuncLoweringSwitch(mod, builder, funcID, use, func(funcPtr llvm.Value, params []llvm.Value) llvm.Value {
-									// The function lowering switch code passes in a parent handle value.
-									// Set the parent handle to null here because it is irrelevant to goroutine starts.
-									i8ptrType := llvm.PointerType(ctx.Int8Type(), 0)
-									params[len(params)-1] = llvm.ConstPointerNull(i8ptrType)
-									calleeValue := builder.CreatePtrToInt(funcPtr, uintptrType, "")
-									makeGoroutine := mod.NamedFunction("runtime.makeGoroutine")
-									calleeValue = builder.CreateCall(makeGoroutine, []llvm.Value{calleeValue, llvm.Undef(i8ptrType), llvm.ConstNull(i8ptrType)}, "")
-									calleeValue = builder.CreateIntToPtr(calleeValue, funcPtr.Type(), "")
-									builder.CreateCall(calleeValue, params, "")
-									return llvm.Value{} // void so no return value
-								}, functions)
-								use.EraseFromParentAsInstruction()
-							}
-							inttoptr.EraseFromParentAsInstruction()
-						}
+					if !callIntPtr.IsACallInst().IsNil() && callIntPtr.CalledValue().Name() == "internal/task.start" {
+						// Special case for goroutine starts.
+						addFuncLoweringSwitch(mod, builder, funcID, callIntPtr, func(funcPtr llvm.Value, params []llvm.Value) llvm.Value {
+							i8ptrType := llvm.PointerType(ctx.Int8Type(), 0)
+							calleeValue := builder.CreatePtrToInt(funcPtr, uintptrType, "")
+							start := mod.NamedFunction("internal/task.start")
+							builder.CreateCall(start, []llvm.Value{calleeValue, callIntPtr.Operand(1), llvm.Undef(i8ptrType), llvm.ConstNull(i8ptrType)}, "")
+							return llvm.Value{} // void so no return value
+						}, functions)
 						callIntPtr.EraseFromParentAsInstruction()
 						continue
 					}

transform/goroutine_test.go

@@ -0,0 +1,16 @@
+package transform
+
+import (
+	"testing"
+	"tinygo.org/x/go-llvm"
+)
+
+func TestGoroutineLowering(t *testing.T) {
+	t.Parallel()
+	testTransform(t, "testdata/coroutines", func(mod llvm.Module) {
+		err := LowerCoroutines(mod, false)
+		if err != nil {
+			panic(err)
+		}
+	})
+}

transform/testdata/coroutines.ll

@@ -0,0 +1,120 @@
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7m-none-eabi"
+
+%"internal/task.state" = type { i8* }
+%"internal/task.Task" = type { %"internal/task.Task", i8*, i32, %"internal/task.state" }
+
+declare void @"internal/task.start"(i32, i8*, i8*, i8*)
+declare void @"internal/task.Pause"(i8*, i8*)
+
+declare void @runtime.scheduler(i8*, i8*)
+
+declare i8* @runtime.alloc(i32, i8*, i8*)
+declare void @runtime.free(i8*, i8*, i8*)
+
+declare %"internal/task.Task"* @"internal/task.Current"(i8*, i8*)
+
+declare i8* @"(*internal/task.Task).setState"(%"internal/task.Task"*, i8*, i8*, i8*)
+declare void @"(*internal/task.Task).setReturnPtr"(%"internal/task.Task"*, i8*, i8*, i8*)
+declare i8* @"(*internal/task.Task).getReturnPtr"(%"internal/task.Task"*, i8*, i8*)
+declare void @"(*internal/task.Task).returnTo"(%"internal/task.Task"*, i8*, i8*, i8*)
+declare void @"(*internal/task.Task).returnCurrent"(%"internal/task.Task"*, i8*, i8*)
+declare %"internal/task.Task"* @"internal/task.createTask"(i8*, i8*)
+
+declare void @callMain(i8*, i8*)
+
+; Test a simple sleep-like scenario.
+declare void @enqueueTimer(%"internal/task.Task"*, i64, i8*, i8*)
+
+define void @sleep(i64, i8*, i8* %parentHandle) {
+entry:
+  %2 = call %"internal/task.Task"* @"internal/task.Current"(i8* undef, i8* null)
+  call void @enqueueTimer(%"internal/task.Task"* %2, i64 %0, i8* undef, i8* null)
+  call void @"internal/task.Pause"(i8* undef, i8* null)
+  ret void
+}
+
+; Test a delayed value return.
+define i32 @delayedValue(i32, i64, i8*, i8* %parentHandle) {
+entry:
+  call void @sleep(i64 %1, i8* undef, i8* null)
+  ret i32 %0
+}
+
+; Test a deadlocking async func.
+define void @deadlock(i8*, i8* %parentHandle) {
+entry:
+  call void @"internal/task.Pause"(i8* undef, i8* null)
+  unreachable
+}
+
+; Test a regular tail call.
+define i32 @tail(i32, i64, i8*, i8* %parentHandle) {
+entry:
+  %3 = call i32 @delayedValue(i32 %0, i64 %1, i8* undef, i8* null)
+  ret i32 %3
+}
+
+; Test a ditching tail call.
+define void @ditchTail(i32, i64, i8*, i8* %parentHandle) {
+entry:
+  %3 = call i32 @delayedValue(i32 %0, i64 %1, i8* undef, i8* null)
+  ret void
+}
+
+; Test a void tail call.
+define void @voidTail(i32, i64, i8*, i8* %parentHandle) {
+entry:
+  call void @ditchTail(i32 %0, i64 %1, i8* undef, i8* null)
+  ret void
+}
+
+; Test a tail call returning an alternate value.
+define i32 @alternateTail(i32, i32, i64, i8*, i8* %parentHandle) {
+entry:
+  %4 = call i32 @delayedValue(i32 %1, i64 %2, i8* undef, i8* null)
+  ret i32 %0
+}
+
+; Test a normal return from a coroutine.
+; This must be turned into a coroutine.
+define i1 @coroutine(i32, i64, i8*, i8* %parentHandle) {
+entry:
+  %3 = call i32 @delayedValue(i32 %0, i64 %1, i8* undef, i8* null)
+  %4 = icmp eq i32 %3, 0
+  ret i1 %4
+}
+
+; Normal function which should not be transformed.
+define void @doNothing(i8*, i8*) {
+entry:
+  ret void
+}
+
+; Goroutine that sleeps and does nothing.
+; Should be a void tail call.
+define void @sleepGoroutine(i8*, i8* %parentHandle) {
+  call void @sleep(i64 1000000, i8* undef, i8* null)
+  ret void
+}
+
+; Program main function.
+define void @progMain(i8*, i8* %parentHandle) {
+entry:
+  ; Call a sync func in a goroutine.
+  call void @"internal/task.start"(i32 ptrtoint (void (i8*, i8*)* @doNothing to i32), i8* undef, i8* undef, i8* null)
+  ; Call an async func in a goroutine.
+  call void @"internal/task.start"(i32 ptrtoint (void (i8*, i8*)* @sleepGoroutine to i32), i8* undef, i8* undef, i8* null)
+  ; Sleep a bit.
+  call void @sleep(i64 2000000, i8* undef, i8* null)
+  ; Done.
+  ret void
+}
+
+; Entrypoint of runtime.
+define void @main() {
+entry:
+  call void @"internal/task.start"(i32 ptrtoint (void (i8*, i8*)* @progMain to i32), i8* undef, i8* undef, i8* null)
+  call void @runtime.scheduler(i8* undef, i8* null)
+  ret void
+}

transform/testdata/coroutines.out.ll

@@ -0,0 +1,176 @@
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "armv7m-none-eabi"
+
+%"internal/task.Task" = type { %"internal/task.Task", i8*, i32, %"internal/task.state" }
+%"internal/task.state" = type { i8* }
+
+declare void @"internal/task.start"(i32, i8*, i8*, i8*)
+declare void @"internal/task.Pause"(i8*, i8*)
+
+declare void @runtime.scheduler(i8*, i8*)
+
+declare i8* @runtime.alloc(i32, i8*, i8*)
+declare void @runtime.free(i8*, i8*, i8*)
+
+declare %"internal/task.Task"* @"internal/task.Current"(i8*, i8*)
+
+declare i8* @"(*internal/task.Task).setState"(%"internal/task.Task"*, i8*, i8*, i8*)
+
+declare void @"(*internal/task.Task).setReturnPtr"(%"internal/task.Task"*, i8*, i8*, i8*)
+declare i8* @"(*internal/task.Task).getReturnPtr"(%"internal/task.Task"*, i8*, i8*)
+
+declare void @"(*internal/task.Task).returnTo"(%"internal/task.Task"*, i8*, i8*, i8*)
+declare void @"(*internal/task.Task).returnCurrent"(%"internal/task.Task"*, i8*, i8*)
+
+declare %"internal/task.Task"* @"internal/task.createTask"(i8*, i8*)
+
+declare void @callMain(i8*, i8*)
+
+declare void @enqueueTimer(%"internal/task.Task"*, i64, i8*, i8*)
+define void @sleep(i64, i8*, i8* %parentHandle) {
+entry:
+  %task.current = bitcast i8* %parentHandle to %"internal/task.Task"*
+  %task.current1 = bitcast i8* %parentHandle to %"internal/task.Task"*
+  call void @enqueueTimer(%"internal/task.Task"* %task.current1, i64 %0, i8* undef, i8* null)
+  ret void
+}
+
+define i32 @delayedValue(i32, i64, i8*, i8* %parentHandle) {
+entry:
+  %task.current = bitcast i8* %parentHandle to %"internal/task.Task"*
+  %ret.ptr = call i8* @"(*internal/task.Task).getReturnPtr"(%"internal/task.Task"* %task.current, i8* undef, i8* undef)
+  %ret.ptr.bitcast = bitcast i8* %ret.ptr to i32*
+  store i32 %0, i32* %ret.ptr.bitcast
+  call void @sleep(i64 %1, i8* undef, i8* %parentHandle)
+  ret i32 undef
+}
+
+define void @deadlock(i8*, i8* %parentHandle) {
+entry:
+  %task.current = bitcast i8* %parentHandle to %"internal/task.Task"*
+  ret void
+}
+
+define i32 @tail(i32, i64, i8*, i8* %parentHandle) {
+entry:
+  %task.current = bitcast i8* %parentHandle to %"internal/task.Task"*
+  %3 = call i32 @delayedValue(i32 %0, i64 %1, i8* undef, i8* %parentHandle)
+  ret i32 undef
+}
+
+define void @ditchTail(i32, i64, i8*, i8* %parentHandle) {
+entry:
+  %task.current = bitcast i8* %parentHandle to %"internal/task.Task"*
+  %ret.ditch = call i8* @runtime.alloc(i32 4, i8* undef, i8* undef)
+  call void @"(*internal/task.Task).setReturnPtr"(%"internal/task.Task"* %task.current, i8* %ret.ditch, i8* undef, i8* undef)
+  %3 = call i32 @delayedValue(i32 %0, i64 %1, i8* undef, i8* %parentHandle)
+  ret void
+}
+
+define void @voidTail(i32, i64, i8*, i8* %parentHandle) {
+entry:
+  %task.current = bitcast i8* %parentHandle to %"internal/task.Task"*
+  call void @ditchTail(i32 %0, i64 %1, i8* undef, i8* %parentHandle)
+  ret void
+}
+
+define i32 @alternateTail(i32, i32, i64, i8*, i8* %parentHandle) {
+entry:
+  %task.current = bitcast i8* %parentHandle to %"internal/task.Task"*
+  %ret.ptr = call i8* @"(*internal/task.Task).getReturnPtr"(%"internal/task.Task"* %task.current, i8* undef, i8* undef)
+  %ret.ptr.bitcast = bitcast i8* %ret.ptr to i32*
+  store i32 %0, i32* %ret.ptr.bitcast
+  %ret.alternate = call i8* @runtime.alloc(i32 4, i8* undef, i8* undef)
+  call void @"(*internal/task.Task).setReturnPtr"(%"internal/task.Task"* %task.current, i8* %ret.alternate, i8* undef, i8* undef)
+  %4 = call i32 @delayedValue(i32 %1, i64 %2, i8* undef, i8* %parentHandle)
+  ret i32 undef
+}
+
+define i1 @coroutine(i32, i64, i8*, i8* %parentHandle) {
+entry:
+  %call.return = alloca i32
+  %coro.id = call token @llvm.coro.id(i32 0, i8* null, i8* null, i8* null)
+  %coro.size = call i32 @llvm.coro.size.i32()
+  %coro.alloc = call i8* @runtime.alloc(i32 %coro.size, i8* undef, i8* undef)
+  %coro.state = call i8* @llvm.coro.begin(token %coro.id, i8* %coro.alloc)
+  %task.current2 = bitcast i8* %parentHandle to %"internal/task.Task"*
+  %task.state.parent = call i8* @"(*internal/task.Task).setState"(%"internal/task.Task"* %task.current2, i8* %coro.state, i8* undef, i8* undef)
+  %task.retPtr = call i8* @"(*internal/task.Task).getReturnPtr"(%"internal/task.Task"* %task.current2, i8* undef, i8* undef)
+  %task.retPtr.bitcast = bitcast i8* %task.retPtr to i1*
+  %call.return.bitcast = bitcast i32* %call.return to i8*
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %call.return.bitcast)
+  %task.current = bitcast i8* %parentHandle to %"internal/task.Task"*
+  %call.return.bitcast1 = bitcast i32* %call.return to i8*
+  call void @"(*internal/task.Task).setReturnPtr"(%"internal/task.Task"* %task.current, i8* %call.return.bitcast1, i8* undef, i8* undef)
+  %3 = call i32 @delayedValue(i32 %0, i64 %1, i8* undef, i8* %parentHandle)
+  %coro.save = call token @llvm.coro.save(i8* %coro.state)
+  %call.suspend = call i8 @llvm.coro.suspend(token %coro.save, i1 false)
+  switch i8 %call.suspend, label %suspend [
+    i8 0, label %wakeup
+    i8 1, label %cleanup
+  ]
+
+wakeup:                                           ; preds = %entry
+  %4 = load i32, i32* %call.return
+  call void @llvm.lifetime.end.p0i8(i64 4, i8* %call.return.bitcast)
+  %5 = icmp eq i32 %4, 0
+  store i1 %5, i1* %task.retPtr.bitcast
+  call void @"(*internal/task.Task).returnTo"(%"internal/task.Task"* %task.current2, i8* %task.state.parent, i8* undef, i8* undef)
+  br label %cleanup
+
+suspend:                                          ; preds = %entry, %cleanup
+  %unused = call i1 @llvm.coro.end(i8* %coro.state, i1 false)
+  ret i1 undef
+
+cleanup:                                          ; preds = %entry, %wakeup
+  %coro.memFree = call i8* @llvm.coro.free(token %coro.id, i8* %coro.state)
+  call void @runtime.free(i8* %coro.memFree, i8* undef, i8* undef)
+  br label %suspend
+}
+
+define void @doNothing(i8*, i8*) {
+entry:
+  ret void
+}
+
+define void @sleepGoroutine(i8*, i8* %parentHandle) {
+  %task.current = bitcast i8* %parentHandle to %"internal/task.Task"*
+  call void @sleep(i64 1000000, i8* undef, i8* %parentHandle)
+  ret void
+}
+
+define void @progMain(i8*, i8* %parentHandle) {
+entry:
+  %task.current = bitcast i8* %parentHandle to %"internal/task.Task"*
+  call void @doNothing(i8* undef, i8* undef)
+  %start.task = call %"internal/task.Task"* @"internal/task.createTask"(i8* undef, i8* undef)
+  %start.task.bitcast = bitcast %"internal/task.Task"* %start.task to i8*
+  call void @sleepGoroutine(i8* undef, i8* %start.task.bitcast)
+  call void @sleep(i64 2000000, i8* undef, i8* %parentHandle)
+  ret void
+}
+
+define void @main() {
+entry:
+  %start.task = call %"internal/task.Task"* @"internal/task.createTask"(i8* undef, i8* undef)
+  %start.task.bitcast = bitcast %"internal/task.Task"* %start.task to i8*
+  call void @progMain(i8* undef, i8* %start.task.bitcast)
+  call void @runtime.scheduler(i8* undef, i8* null)
+  ret void
+}
+
+declare token @llvm.coro.id(i32, i8* readnone, i8* nocapture readonly, i8*) #0
+declare i32 @llvm.coro.size.i32() #1
+declare i8* @llvm.coro.begin(token, i8* writeonly) #2
+declare i8 @llvm.coro.suspend(token, i1) #2
+declare i1 @llvm.coro.end(i8*, i1) #2
+declare i8* @llvm.coro.free(token, i8* nocapture readonly) #0
+declare token @llvm.coro.save(i8*) #2
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #3
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #3
+
+attributes #0 = { argmemonly nounwind readonly }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind }
+attributes #3 = { argmemonly nounwind }

transform/testdata/func-lowering.ll

@@ -15,7 +15,7 @@ target triple = "wasm32-unknown-unknown-wasm"
 
 declare i32 @runtime.getFuncPtr(i8*, i32, %runtime.typecodeID*, i8*, i8*)
 
-declare i32 @runtime.makeGoroutine(i32, i8*, i8*)
+declare void @"internal/task.start"(i32, i8*, i8*, i8*)
 
 declare void @runtime.nilPanic(i8*, i8*)
 
@@ -71,13 +71,10 @@ fpcall.next:
   ret void
 }
 
-; Special case for runtime.makeGoroutine.
+; Special case for internal/task.start.
 define void @sleepFuncValue(i8*, i32, i8* nocapture readnone %context, i8* nocapture readnone %parentHandle) {
 entry:
   %2 = call i32 @runtime.getFuncPtr(i8* %0, i32 %1, %runtime.typecodeID* @"reflect/types.type:func:{basic:int}{}", i8* undef, i8* null)
-  %3 = call i32 @runtime.makeGoroutine(i32 %2, i8* undef, i8* null)
-  %4 = inttoptr i32 %3 to void (i32, i8*, i8*)*
-  call void %4(i32 8, i8* %0, i8* null)
+  call void @"internal/task.start"(i32 %2, i8* null, i8* undef, i8* null)
   ret void
 }
-

transform/testdata/func-lowering.out.ll

@@ -15,7 +15,7 @@ target triple = "wasm32-unknown-unknown-wasm"
 
 declare i32 @runtime.getFuncPtr(i8*, i32, %runtime.typecodeID*, i8*, i8*)
 
-declare i32 @runtime.makeGoroutine(i32, i8*, i8*)
+declare void @"internal/task.start"(i32, i8*, i8*, i8*)
 
 declare void @runtime.nilPanic(i8*, i8*)
 
@@ -102,15 +102,11 @@ func.nil:
   unreachable
 
 func.call1:
-  %2 = call i32 @runtime.makeGoroutine(i32 ptrtoint (void (i32, i8*, i8*)* @"main$1" to i32), i8* undef, i8* null)
-  %3 = inttoptr i32 %2 to void (i32, i8*, i8*)*
-  call void %3(i32 8, i8* %0, i8* null)
+  call void @"internal/task.start"(i32 ptrtoint (void (i32, i8*, i8*)* @"main$1" to i32), i8* null, i8* undef, i8* null)
   br label %func.next
 
 func.call2:
-  %4 = call i32 @runtime.makeGoroutine(i32 ptrtoint (void (i32, i8*, i8*)* @"main$2" to i32), i8* undef, i8* null)
-  %5 = inttoptr i32 %4 to void (i32, i8*, i8*)*
-  call void %5(i32 8, i8* %0, i8* null)
+  call void @"internal/task.start"(i32 ptrtoint (void (i32, i8*, i8*)* @"main$2" to i32), i8* null, i8* undef, i8* null)
   br label %func.next
 
 func.next: