diff --git a/Gopkg.lock b/Gopkg.lock index 2e8e47ce7..d9459dad4 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -3,30 +3,20 @@ [[projects]] branch = "master" - digest = "1:f250e2a6d7e4f9ebc5ba37e5e2ec91b46eb1399ee43f2fdaeb20cd4fd1aeee59" + digest = "1:747e7010f73dc26e5cb298444f6f1b4442ae155ab87e44080686b94a5f9d6c90" name = "github.com/aykevl/go-llvm" packages = ["."] pruneopts = "UT" - revision = "d8539684f173a591ea9474d6262ac47ef2277d64" + revision = "8a0a627130a54562f55b1c2a9d7b8019ff14d9f1" [[projects]] branch = "master" - digest = "1:d1102ae84d8c9318db4ce2ad2673eb2bf54569ab2a4a5d57e70d8aef726b681d" + digest = "1:84316faef4ea12d34dde3b3e6dab682715a23b1c2bb8ab82cec9ab619766e214" name = "golang.org/x/tools" packages = [ "go/ast/astutil", - "go/buildutil", - "go/gcexportdata", - "go/internal/cgo", - "go/internal/gcimporter", - "go/loader", - "go/packages", "go/ssa", - "go/ssa/ssautil", "go/types/typeutil", - "internal/fastwalk", - "internal/gopathwalk", - "internal/semver", ] pruneopts = "UT" revision = "3e7aa9e59977626dc60433e9aeadf1bb63d28295" @@ -36,9 +26,7 @@ analyzer-version = 1 input-imports = [ "github.com/aykevl/go-llvm", - "golang.org/x/tools/go/loader", "golang.org/x/tools/go/ssa", - "golang.org/x/tools/go/ssa/ssautil", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/compiler/calls.go b/compiler/calls.go index e172c008e..7893aefb4 100644 --- a/compiler/calls.go +++ b/compiler/calls.go @@ -21,7 +21,8 @@ func (c *Compiler) createRuntimeCall(fnName string, args []llvm.Value, name stri } fn := c.ir.GetFunction(member.(*ssa.Function)) if !fn.IsExported() { - args = append(args, llvm.Undef(c.i8ptrType)) // unused context parameter + args = append(args, llvm.Undef(c.i8ptrType)) // unused context parameter + args = append(args, llvm.ConstPointerNull(c.i8ptrType)) // coroutine handle } return c.createCall(fn.LLVMFn, args, name) } diff --git a/compiler/compiler.go b/compiler/compiler.go index c37ed34d5..b2477bcfd 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -59,12 +59,6 @@ type Compiler struct { intType llvm.Type i8ptrType llvm.Type // for convenience uintptrType llvm.Type - coroIdFunc llvm.Value - coroSizeFunc llvm.Value - coroBeginFunc llvm.Value - coroSuspendFunc llvm.Value - coroEndFunc llvm.Value - coroFreeFunc llvm.Value initFuncs []llvm.Value interfaceInvokeWrappers []interfaceInvokeWrapper ir *ir.Program @@ -77,10 +71,7 @@ type Frame struct { blockExits map[*ssa.BasicBlock]llvm.BasicBlock // these are the exit blocks currentBlock *ssa.BasicBlock phis []Phi - blocking bool taskHandle llvm.Value - cleanupBlock llvm.BasicBlock - suspendBlock llvm.BasicBlock deferPtr llvm.Value difunc llvm.Metadata allDeferFuncs []interface{} @@ -133,24 +124,6 @@ func NewCompiler(pkgName string, config Config) (*Compiler, error) { } c.i8ptrType = llvm.PointerType(c.ctx.Int8Type(), 0) - coroIdType := llvm.FunctionType(c.ctx.TokenType(), []llvm.Type{c.ctx.Int32Type(), c.i8ptrType, c.i8ptrType, c.i8ptrType}, false) - c.coroIdFunc = llvm.AddFunction(c.mod, "llvm.coro.id", coroIdType) - - coroSizeType := llvm.FunctionType(c.ctx.Int32Type(), nil, false) - c.coroSizeFunc = llvm.AddFunction(c.mod, "llvm.coro.size.i32", coroSizeType) - - coroBeginType := llvm.FunctionType(c.i8ptrType, []llvm.Type{c.ctx.TokenType(), c.i8ptrType}, false) - c.coroBeginFunc = llvm.AddFunction(c.mod, "llvm.coro.begin", coroBeginType) - - coroSuspendType := llvm.FunctionType(c.ctx.Int8Type(), []llvm.Type{c.ctx.TokenType(), c.ctx.Int1Type()}, false) - c.coroSuspendFunc = llvm.AddFunction(c.mod, "llvm.coro.suspend", coroSuspendType) - - coroEndType := llvm.FunctionType(c.ctx.Int1Type(), []llvm.Type{c.i8ptrType, c.ctx.Int1Type()}, false) - c.coroEndFunc = llvm.AddFunction(c.mod, "llvm.coro.end", coroEndType) - - coroFreeType := llvm.FunctionType(c.i8ptrType, []llvm.Type{c.ctx.TokenType(), c.i8ptrType}, false) - c.coroFreeFunc = llvm.AddFunction(c.mod, "llvm.coro.free", coroFreeType) - return c, nil } @@ -237,12 +210,8 @@ func (c *Compiler) Compile(mainPath string) error { c.ir = ir.NewProgram(lprogram, mainPath) - // Run some DCE and analysis passes. The results are later used by the - // compiler. - c.ir.SimpleDCE() // remove most dead code - c.ir.AnalyseCallgraph() // set up callgraph - c.ir.AnalyseBlockingRecursive() // make all parents of blocking calls blocking (transitively) - c.ir.AnalyseGoCalls() // check whether we need a scheduler + // Run a simple dead code elimination pass. + c.ir.SimpleDCE() // Initialize debug information. c.cu = c.dibuilder.CreateCompileUnit(llvm.DICompileUnit{ @@ -387,33 +356,19 @@ func (c *Compiler) Compile(mainPath string) error { block := c.ctx.AddBasicBlock(initFn.LLVMFn, "entry") c.builder.SetInsertPointAtEnd(block) for _, fn := range c.initFuncs { - c.builder.CreateCall(fn, []llvm.Value{llvm.Undef(c.i8ptrType)}, "") + c.builder.CreateCall(fn, []llvm.Value{llvm.Undef(c.i8ptrType), llvm.Undef(c.i8ptrType)}, "") } c.builder.CreateRetVoid() - // Add a wrapper for the main.main function, either calling it directly or - // setting up the scheduler with it. - mainWrapper := c.ir.GetFunction(c.ir.Program.ImportedPackage("runtime").Members["mainWrapper"].(*ssa.Function)) - mainWrapper.LLVMFn.SetLinkage(llvm.InternalLinkage) - mainWrapper.LLVMFn.SetUnnamedAddr(true) - if c.Debug { - difunc, err := c.attachDebugInfo(mainWrapper) - if err != nil { - return err - } - pos := c.ir.Program.Fset.Position(mainWrapper.Pos()) - c.builder.SetCurrentDebugLocation(uint(pos.Line), uint(pos.Column), difunc, llvm.Metadata{}) - } - block = c.ctx.AddBasicBlock(mainWrapper.LLVMFn, "entry") - c.builder.SetInsertPointAtEnd(block) + // Conserve for goroutine lowering. Without marking these as external, they + // would be optimized away. realMain := c.mod.NamedFunction(c.ir.MainPkg().Pkg.Path() + ".main") - if c.ir.NeedsScheduler() { - coroutine := c.builder.CreateCall(realMain, []llvm.Value{llvm.ConstPointerNull(c.i8ptrType), llvm.Undef(c.i8ptrType)}, "") - c.createRuntimeCall("scheduler", []llvm.Value{coroutine}, "") - } else { - c.builder.CreateCall(realMain, []llvm.Value{llvm.Undef(c.i8ptrType)}, "") - } - c.builder.CreateRetVoid() + realMain.SetLinkage(llvm.ExternalLinkage) // keep alive until goroutine lowering + c.mod.NamedFunction("runtime.alloc").SetLinkage(llvm.ExternalLinkage) + c.mod.NamedFunction("runtime.free").SetLinkage(llvm.ExternalLinkage) + c.mod.NamedFunction("runtime.sleepTask").SetLinkage(llvm.ExternalLinkage) + c.mod.NamedFunction("runtime.activateTask").SetLinkage(llvm.ExternalLinkage) + c.mod.NamedFunction("runtime.scheduler").SetLinkage(llvm.ExternalLinkage) // see: https://reviews.llvm.org/D18355 c.mod.AddNamedMetadataOperand("llvm.module.flags", @@ -535,7 +490,8 @@ func (c *Compiler) getLLVMType(goType types.Type) (llvm.Type, error) { } // make a closure type (with a function pointer type inside): // {context, funcptr} - paramTypes = append(paramTypes, c.i8ptrType) + paramTypes = append(paramTypes, c.i8ptrType) // context + paramTypes = append(paramTypes, c.i8ptrType) // parent coroutine ptr := llvm.PointerType(llvm.FunctionType(returnType, paramTypes, false), 0) ptr = c.ctx.StructType([]llvm.Type{c.i8ptrType, ptr}, false) return ptr, nil @@ -676,16 +632,10 @@ func (c *Compiler) parseFuncDecl(f *ir.Function) (*Frame, error) { locals: make(map[ssa.Value]llvm.Value), blockEntries: make(map[*ssa.BasicBlock]llvm.BasicBlock), blockExits: make(map[*ssa.BasicBlock]llvm.BasicBlock), - blocking: c.ir.IsBlocking(f), } var retType llvm.Type - if frame.blocking { - if f.Signature.Results() != nil { - return nil, c.makeError(f.Function.Pos(), "todo: return values in blocking function") - } - retType = c.i8ptrType - } else if f.Signature.Results() == nil { + if f.Signature.Results() == nil { retType = c.ctx.VoidType() } else if f.Signature.Results().Len() == 1 { var err error @@ -706,9 +656,6 @@ func (c *Compiler) parseFuncDecl(f *ir.Function) (*Frame, error) { } var paramTypes []llvm.Type - if frame.blocking { - paramTypes = append(paramTypes, c.i8ptrType) // parent coroutine - } for _, param := range f.Params { paramType, err := c.getLLVMType(param.Type()) if err != nil { @@ -721,7 +668,8 @@ func (c *Compiler) parseFuncDecl(f *ir.Function) (*Frame, error) { // Add an extra parameter as the function context. This context is used in // closures and bound methods, but should be optimized away when not used. if !f.IsExported() { - paramTypes = append(paramTypes, c.i8ptrType) + paramTypes = append(paramTypes, c.i8ptrType) // context + paramTypes = append(paramTypes, c.i8ptrType) // parent coroutine } fnType := llvm.FunctionType(retType, paramTypes, false) @@ -1095,10 +1043,6 @@ func (c *Compiler) parseFunc(frame *Frame) error { frame.blockEntries[block] = llvmBlock frame.blockExits[block] = llvmBlock } - if frame.blocking { - frame.cleanupBlock = c.ctx.AddBasicBlock(frame.fn.LLVMFn, "task.cleanup") - frame.suspendBlock = c.ctx.AddBasicBlock(frame.fn.LLVMFn, "task.suspend") - } entryBlock := frame.blockEntries[frame.fn.Blocks[0]] c.builder.SetInsertPointAtEnd(entryBlock) @@ -1137,10 +1081,14 @@ func (c *Compiler) parseFunc(frame *Frame) error { // Load free variables from the context. This is a closure (or bound // method). - if len(frame.fn.FreeVars) != 0 { - context := frame.fn.LLVMFn.LastParam() + var context llvm.Value + if !frame.fn.IsExported() { + parentHandle := frame.fn.LLVMFn.LastParam() + parentHandle.SetName("parentHandle") + context = llvm.PrevParam(parentHandle) context.SetName("context") - + } + if len(frame.fn.FreeVars) != 0 { // Determine the context type. It's a struct containing all variables. freeVarTypes := make([]llvm.Type, 0, len(frame.fn.FreeVars)) for _, freeVar := range frame.fn.FreeVars { @@ -1186,39 +1134,6 @@ func (c *Compiler) parseFunc(frame *Frame) error { c.deferInitFunc(frame) } - if frame.blocking { - // Coroutine initialization. - taskState := c.builder.CreateAlloca(c.mod.GetTypeByName("runtime.taskState"), "task.state") - stateI8 := c.builder.CreateBitCast(taskState, c.i8ptrType, "task.state.i8") - id := c.builder.CreateCall(c.coroIdFunc, []llvm.Value{ - llvm.ConstInt(c.ctx.Int32Type(), 0, false), - stateI8, - llvm.ConstNull(c.i8ptrType), - llvm.ConstNull(c.i8ptrType), - }, "task.token") - size := c.builder.CreateCall(c.coroSizeFunc, nil, "task.size") - if c.targetData.TypeAllocSize(size.Type()) > c.targetData.TypeAllocSize(c.uintptrType) { - size = c.builder.CreateTrunc(size, c.uintptrType, "task.size.uintptr") - } else if c.targetData.TypeAllocSize(size.Type()) < c.targetData.TypeAllocSize(c.uintptrType) { - size = c.builder.CreateZExt(size, c.uintptrType, "task.size.uintptr") - } - data := c.createRuntimeCall("alloc", []llvm.Value{size}, "task.data") - frame.taskHandle = c.builder.CreateCall(c.coroBeginFunc, []llvm.Value{id, data}, "task.handle") - - // Coroutine cleanup. Free resources associated with this coroutine. - c.builder.SetInsertPointAtEnd(frame.cleanupBlock) - mem := c.builder.CreateCall(c.coroFreeFunc, []llvm.Value{id, frame.taskHandle}, "task.data.free") - c.createRuntimeCall("free", []llvm.Value{mem}, "") - // re-insert parent coroutine - c.createRuntimeCall("yieldToScheduler", []llvm.Value{frame.fn.LLVMFn.FirstParam()}, "") - c.builder.CreateBr(frame.suspendBlock) - - // Coroutine suspend. A call to llvm.coro.suspend() will branch here. - c.builder.SetInsertPointAtEnd(frame.suspendBlock) - c.builder.CreateCall(c.coroEndFunc, []llvm.Value{frame.taskHandle, llvm.ConstInt(c.ctx.Int1Type(), 0, false)}, "unused") - c.builder.CreateRet(frame.taskHandle) - } - // Fill blocks with instructions. for _, block := range frame.fn.DomPreorder() { if c.DumpSSA { @@ -1283,25 +1198,38 @@ func (c *Compiler) parseInstr(frame *Frame, instr ssa.Instruction) error { case *ssa.Defer: return c.emitDefer(frame, instr) case *ssa.Go: - if instr.Common().Method != nil { + if instr.Call.IsInvoke() { return c.makeError(instr.Pos(), "todo: go on method receiver") } + callee := instr.Call.StaticCallee() + if callee == nil { + return c.makeError(instr.Pos(), "todo: go on non-direct function (function pointer, etc.)") + } + calleeFn := c.ir.GetFunction(callee) - // Execute non-blocking calls (including builtins) directly. - // parentHandle param is ignored. - if !c.ir.IsBlocking(c.ir.GetFunction(instr.Common().Value.(*ssa.Function))) { - _, err := c.parseCall(frame, instr.Common(), llvm.Value{}) - return err // probably nil + // Mark this function as a 'go' invocation and break invalid + // interprocedural optimizations. For example, heap-to-stack + // transformations are not sound as goroutines can outlive their parent. + calleeType := calleeFn.LLVMFn.Type() + calleeValue := c.builder.CreateBitCast(calleeFn.LLVMFn, c.i8ptrType, "") + calleeValue = c.createRuntimeCall("makeGoroutine", []llvm.Value{calleeValue}, "") + calleeValue = c.builder.CreateBitCast(calleeValue, calleeType, "") + + // Get all function parameters to pass to the goroutine. + var params []llvm.Value + for _, param := range instr.Call.Args { + val, err := c.parseExpr(frame, param) + if err != nil { + return err + } + params = append(params, val) + } + if !calleeFn.IsExported() { + params = append(params, llvm.Undef(c.i8ptrType)) // context parameter + params = append(params, llvm.Undef(c.i8ptrType)) // parent coroutine handle } - // Start this goroutine. - // parentHandle is nil, as the goroutine has no parent frame (it's a new - // stack). - handle, err := c.parseCall(frame, instr.Common(), llvm.Value{}) - if err != nil { - return err - } - c.createRuntimeCall("yieldToScheduler", []llvm.Value{handle}, "") + c.createCall(calleeValue, params, "") return nil case *ssa.If: cond, err := c.parseExpr(frame, instr.Cond) @@ -1341,45 +1269,31 @@ func (c *Compiler) parseInstr(frame *Frame, instr ssa.Instruction) error { c.builder.CreateUnreachable() return nil case *ssa.Return: - if frame.blocking { - if len(instr.Results) != 0 { - return c.makeError(instr.Pos(), "todo: return values from blocking function") + if len(instr.Results) == 0 { + c.builder.CreateRetVoid() + return nil + } else if len(instr.Results) == 1 { + val, err := c.parseExpr(frame, instr.Results[0]) + if err != nil { + return err } - // Final suspend. - continuePoint := c.builder.CreateCall(c.coroSuspendFunc, []llvm.Value{ - llvm.ConstNull(c.ctx.TokenType()), - llvm.ConstInt(c.ctx.Int1Type(), 1, false), // final=true - }, "") - sw := c.builder.CreateSwitch(continuePoint, frame.suspendBlock, 2) - sw.AddCase(llvm.ConstInt(c.ctx.Int8Type(), 1, false), frame.cleanupBlock) + c.builder.CreateRet(val) return nil } else { - if len(instr.Results) == 0 { - c.builder.CreateRetVoid() - return nil - } else if len(instr.Results) == 1 { - val, err := c.parseExpr(frame, instr.Results[0]) - if err != nil { - return err - } - c.builder.CreateRet(val) - return nil - } else { - // Multiple return values. Put them all in a struct. - retVal, err := c.getZeroValue(frame.fn.LLVMFn.Type().ElementType().ReturnType()) - if err != nil { - return err - } - for i, result := range instr.Results { - val, err := c.parseExpr(frame, result) - if err != nil { - return err - } - retVal = c.builder.CreateInsertValue(retVal, val, i, "") - } - c.builder.CreateRet(retVal) - return nil + // Multiple return values. Put them all in a struct. + retVal, err := c.getZeroValue(frame.fn.LLVMFn.Type().ElementType().ReturnType()) + if err != nil { + return err } + for i, result := range instr.Results { + val, err := c.parseExpr(frame, result) + if err != nil { + return err + } + retVal = c.builder.CreateInsertValue(retVal, val, i, "") + } + c.builder.CreateRet(retVal) + return nil } case *ssa.RunDefers: return c.emitRunDefers(frame) @@ -1606,17 +1520,8 @@ func (c *Compiler) parseBuiltin(frame *Frame, args []ssa.Value, callName string, } } -func (c *Compiler) parseFunctionCall(frame *Frame, args []ssa.Value, llvmFn, context llvm.Value, blocking bool, parentHandle llvm.Value) (llvm.Value, error) { +func (c *Compiler) parseFunctionCall(frame *Frame, args []ssa.Value, llvmFn, context llvm.Value, exported bool) (llvm.Value, error) { var params []llvm.Value - if blocking { - if parentHandle.IsNil() { - // Started from 'go' statement. - params = append(params, llvm.ConstNull(c.i8ptrType)) - } else { - // Blocking function calls another blocking function. - params = append(params, parentHandle) - } - } for _, param := range args { val, err := c.parseExpr(frame, param) if err != nil { @@ -1625,60 +1530,20 @@ func (c *Compiler) parseFunctionCall(frame *Frame, args []ssa.Value, llvmFn, con params = append(params, val) } - if !context.IsNil() { + if !exported { // This function takes a context parameter. // Add it to the end of the parameter list. params = append(params, context) + + // Parent coroutine handle. + params = append(params, llvm.Undef(c.i8ptrType)) } - if frame.blocking && llvmFn.Name() == "time.Sleep" { - // Set task state to TASK_STATE_SLEEP and set the duration. - c.createRuntimeCall("sleepTask", []llvm.Value{frame.taskHandle, params[0]}, "") - - // Yield to scheduler. - continuePoint := c.builder.CreateCall(c.coroSuspendFunc, []llvm.Value{ - llvm.ConstNull(c.ctx.TokenType()), - llvm.ConstInt(c.ctx.Int1Type(), 0, false), - }, "") - wakeup := c.ctx.InsertBasicBlock(llvm.NextBasicBlock(c.builder.GetInsertBlock()), "task.wakeup") - sw := c.builder.CreateSwitch(continuePoint, frame.suspendBlock, 2) - sw.AddCase(llvm.ConstInt(c.ctx.Int8Type(), 0, false), wakeup) - sw.AddCase(llvm.ConstInt(c.ctx.Int8Type(), 1, false), frame.cleanupBlock) - c.builder.SetInsertPointAtEnd(wakeup) - - return llvm.Value{}, nil - } - - result := c.createCall(llvmFn, params, "") - if blocking && !parentHandle.IsNil() { - // Calling a blocking function as a regular function call. - // This is done by passing the current coroutine as a parameter to the - // new coroutine and dropping the current coroutine from the scheduler - // (with the TASK_STATE_CALL state). When the subroutine is finished, it - // will reactivate the parent (this frame) in it's destroy function. - - c.createRuntimeCall("yieldToScheduler", []llvm.Value{result}, "") - - // Set task state to TASK_STATE_CALL. - c.createRuntimeCall("waitForAsyncCall", []llvm.Value{frame.taskHandle}, "") - - // Yield to the scheduler. - continuePoint := c.builder.CreateCall(c.coroSuspendFunc, []llvm.Value{ - llvm.ConstNull(c.ctx.TokenType()), - llvm.ConstInt(c.ctx.Int1Type(), 0, false), - }, "") - resume := c.ctx.InsertBasicBlock(llvm.NextBasicBlock(c.builder.GetInsertBlock()), "task.callComplete") - sw := c.builder.CreateSwitch(continuePoint, frame.suspendBlock, 2) - sw.AddCase(llvm.ConstInt(c.ctx.Int8Type(), 0, false), resume) - sw.AddCase(llvm.ConstInt(c.ctx.Int8Type(), 1, false), frame.cleanupBlock) - c.builder.SetInsertPointAtEnd(resume) - } - return result, nil + return c.createCall(llvmFn, params, ""), nil } -func (c *Compiler) parseCall(frame *Frame, instr *ssa.CallCommon, parentHandle llvm.Value) (llvm.Value, error) { +func (c *Compiler) parseCall(frame *Frame, instr *ssa.CallCommon) (llvm.Value, error) { if instr.IsInvoke() { - // TODO: blocking methods (needs analysis) fnCast, args, err := c.getInvokeCall(frame, instr) if err != nil { return llvm.Value{}, err @@ -1821,7 +1686,7 @@ func (c *Compiler) parseCall(frame *Frame, instr *ssa.CallCommon, parentHandle l } else { context = llvm.Undef(c.i8ptrType) } - return c.parseFunctionCall(frame, instr.Args, targetFunc.LLVMFn, context, c.ir.IsBlocking(targetFunc), parentHandle) + return c.parseFunctionCall(frame, instr.Args, targetFunc.LLVMFn, context, targetFunc.IsExported()) } // Builtin or function pointer. @@ -1833,13 +1698,12 @@ func (c *Compiler) parseCall(frame *Frame, instr *ssa.CallCommon, parentHandle l if err != nil { return llvm.Value{}, err } - // TODO: blocking function pointers (needs analysis) // 'value' is a closure, not a raw function pointer. // Extract the function pointer and the context pointer. // closure: {context, function pointer} context := c.builder.CreateExtractValue(value, 0, "") value = c.builder.CreateExtractValue(value, 1, "") - return c.parseFunctionCall(frame, instr.Args, value, context, false, parentHandle) + return c.parseFunctionCall(frame, instr.Args, value, context, false) } } @@ -1954,7 +1818,7 @@ func (c *Compiler) parseExpr(frame *Frame, expr ssa.Value) (llvm.Value, error) { case *ssa.Call: // Passing the current task here to the subroutine. It is only used when // the subroutine is blocking. - return c.parseCall(frame, expr.Common(), frame.taskHandle) + return c.parseCall(frame, expr.Common()) case *ssa.ChangeInterface: // Do not change between interface types: always use the underlying // (concrete) type in the type number of the interface. Every method @@ -3129,8 +2993,10 @@ func (c *Compiler) ExternalInt64AsPtr() error { // Only change externally visible functions (exports and imports). continue } - if strings.HasPrefix(fn.Name(), "llvm.") { - // Do not try to modify the signature of internal LLVM functions. + if strings.HasPrefix(fn.Name(), "llvm.") || strings.HasPrefix(fn.Name(), "runtime.") { + // Do not try to modify the signature of internal LLVM functions and + // assume that runtime functions are only temporarily exported for + // coroutine lowering. continue } diff --git a/compiler/defer.go b/compiler/defer.go index 02f545a17..cb9f2785e 100644 --- a/compiler/defer.go +++ b/compiler/defer.go @@ -243,6 +243,9 @@ func (c *Compiler) emitRunDefers(frame *Frame) error { // with a strict calling convention. forwardParams = append(forwardParams, llvm.Undef(c.i8ptrType)) + // Parent coroutine handle. + forwardParams = append(forwardParams, llvm.Undef(c.i8ptrType)) + fnPtr, _, err := c.getInvokeCall(frame, callback) if err != nil { return err @@ -277,6 +280,9 @@ func (c *Compiler) emitRunDefers(frame *Frame) error { // function, but we have to pass one anyway. forwardParams = append(forwardParams, llvm.Undef(c.i8ptrType)) + // Parent coroutine handle. + forwardParams = append(forwardParams, llvm.Undef(c.i8ptrType)) + // Call real function. c.createCall(callback.LLVMFn, forwardParams, "") @@ -305,6 +311,9 @@ func (c *Compiler) emitRunDefers(frame *Frame) error { forwardParams = append(forwardParams, forwardParam) } + // Parent coroutine handle. + forwardParams = append(forwardParams, llvm.Undef(c.i8ptrType)) + // Call deferred function. c.createCall(fn.LLVMFn, forwardParams, "") diff --git a/compiler/goroutine-lowering.go b/compiler/goroutine-lowering.go new file mode 100644 index 000000000..c5abaf1ea --- /dev/null +++ b/compiler/goroutine-lowering.go @@ -0,0 +1,473 @@ +package compiler + +// This file lowers goroutine pseudo-functions into coroutines scheduled by a +// scheduler at runtime. It uses coroutine support in LLVM for this +// transformation: https://llvm.org/docs/Coroutines.html +// +// For example, take the following code: +// +// func main() { +// go foo() +// time.Sleep(2 * time.Second) +// println("some other operation") +// bar() +// println("done") +// } +// +// func foo() { +// for { +// println("foo!") +// time.Sleep(time.Second) +// } +// } +// +// func bar() { +// time.Sleep(time.Second) +// println("blocking operation completed) +// } +// +// It is transformed by the IR generator in compiler.go into the following +// pseudo-Go code: +// +// func main() { +// fn := runtime.makeGoroutine(foo) +// fn() +// time.Sleep(2 * time.Second) +// println("some other operation") +// bar() // imagine an 'await' keyword in front of this call +// println("done") +// } +// +// func foo() { +// for { +// println("foo!") +// time.Sleep(time.Second) +// } +// } +// +// func bar() { +// time.Sleep(time.Second) +// println("blocking operation completed) +// } +// +// The pass in this file transforms this code even further, to the following +// async/await style pseudocode: +// +// func main(parent) { +// hdl := llvm.makeCoroutine() +// foo(nil) // do not pass the parent coroutine: this is an independent goroutine +// runtime.sleepTask(hdl, 2 * time.Second) // ask the scheduler to re-activate this coroutine at the right time +// llvm.suspend(hdl) // suspend point +// println("some other operation") +// bar(hdl) // await, pass a continuation (hdl) to bar +// llvm.suspend(hdl) // suspend point, wait for the callee to re-activate +// println("done") +// runtime.activateTask(parent) // re-activate the parent (nop, there is no parent) +// } +// +// func foo(parent) { +// hdl := llvm.makeCoroutine() +// for { +// println("foo!") +// runtime.sleepTask(hdl, time.Second) // ask the scheduler to re-activate this coroutine at the right time +// llvm.suspend(hdl) // suspend point +// } +// } +// +// func bar(parent) { +// hdl := llvm.makeCoroutine() +// runtime.sleepTask(hdl, time.Second) // ask the scheduler to re-activate this coroutine at the right time +// llvm.suspend(hdl) // suspend point +// println("blocking operation completed) +// runtime.activateTask(parent) // re-activate the parent coroutine before returning +// } +// +// The real LLVM code is more complicated, but this is the general idea. +// +// The LLVM coroutine passes will then process this file further transforming +// these three functions into coroutines. Most of the actual work is done by the +// scheduler, which runs in the background scheduling all coroutines. + +import ( + "errors" + "strings" + + "github.com/aykevl/go-llvm" +) + +type asyncFunc struct { + taskHandle llvm.Value + cleanupBlock llvm.BasicBlock + suspendBlock llvm.BasicBlock + unreachableBlock llvm.BasicBlock +} + +// LowerGoroutines is a pass called during optimization that transforms the IR +// into one where all blocking functions are turned into goroutines and blocking +// calls into await calls. +func (c *Compiler) LowerGoroutines() error { + needsScheduler, err := c.markAsyncFunctions() + if err != nil { + return err + } + + uses := getUses(c.mod.NamedFunction("runtime.callMain")) + if len(uses) != 1 || uses[0].IsACallInst().IsNil() { + panic("expected exactly 1 call of runtime.callMain, check the entry point") + } + mainCall := uses[0] + + // Replace call of runtime.callMain() with a real call to main.main(), + // optionally followed by a call to runtime.scheduler(). + c.builder.SetInsertPointBefore(mainCall) + realMain := c.mod.NamedFunction(c.ir.MainPkg().Pkg.Path() + ".main") + c.builder.CreateCall(realMain, []llvm.Value{llvm.Undef(c.i8ptrType), llvm.ConstPointerNull(c.i8ptrType)}, "") + if needsScheduler { + c.createRuntimeCall("scheduler", nil, "") + } + mainCall.EraseFromParentAsInstruction() + + if !needsScheduler { + go_scheduler := c.mod.NamedFunction("go_scheduler") + if !go_scheduler.IsNil() { + // This is the WebAssembly backend. + // There is no need to export the go_scheduler function, but it is + // still exported. Make sure it is optimized away. + go_scheduler.SetLinkage(llvm.InternalLinkage) + } + } + + // main.main was set to external linkage during IR construction. Set it to + // internal linkage to enable interprocedural optimizations. + realMain.SetLinkage(llvm.InternalLinkage) + c.mod.NamedFunction("runtime.alloc").SetLinkage(llvm.InternalLinkage) + c.mod.NamedFunction("runtime.free").SetLinkage(llvm.InternalLinkage) + c.mod.NamedFunction("runtime.sleepTask").SetLinkage(llvm.InternalLinkage) + c.mod.NamedFunction("runtime.activateTask").SetLinkage(llvm.InternalLinkage) + c.mod.NamedFunction("runtime.scheduler").SetLinkage(llvm.InternalLinkage) + + return nil +} + +// markAsyncFunctions does the bulk of the work of lowering goroutines. It +// determines whether a scheduler is needed, and if it is, it transforms +// blocking operations into goroutines and blocking calls into await calls. +// +// It does the following operations: +// * Find all blocking functions. +// * Determine whether a scheduler is necessary. If not, it skips the +// following operations. +// * Transform call instructions into await calls. +// * Transform return instructions into final suspends. +// * Set up the coroutine frames for async functions. +// * Transform blocking calls into their async equivalents. +func (c *Compiler) markAsyncFunctions() (needsScheduler bool, err error) { + sleep := c.mod.NamedFunction("time.Sleep") + if sleep.IsNil() { + // There are no blocking operations, so no need to transform anything. + return false, c.lowerMakeGoroutineCalls() + } + + // Find all async functions. + // Keep reducing this worklist by marking a function as recursively async + // from the worklist and pushing all its parents that are non-async. + // This is somewhat similar to a worklist in a mark-sweep garbage collector: + // the work items are then grey objects. + worklist := []llvm.Value{sleep} + asyncFuncs := make(map[llvm.Value]*asyncFunc) + asyncList := make([]llvm.Value, 0, 4) + for len(worklist) != 0 { + // Pick the topmost. + f := worklist[len(worklist)-1] + worklist = worklist[:len(worklist)-1] + if _, ok := asyncFuncs[f]; ok { + continue // already processed + } + // Add to set of async functions. + asyncFuncs[f] = &asyncFunc{} + asyncList = append(asyncList, f) + + // Add all callees to the worklist. + for _, use := range getUses(f) { + if use.IsConstant() && use.Opcode() == llvm.BitCast { + bitcastUses := getUses(use) + for _, call := range bitcastUses { + if call.IsACallInst().IsNil() || call.CalledValue().Name() != "runtime.makeGoroutine" { + return false, errors.New("async function " + f.Name() + " incorrectly used in bitcast, expected runtime.makeGoroutine") + } + } + // This is a go statement. Do not mark the parent as async, as + // starting a goroutine is not a blocking operation. + continue + } + if use.IsACallInst().IsNil() { + // Not a call instruction. Maybe a store to a global? In any + // case, this requires support for async calls across function + // pointers which is not yet supported. + return false, errors.New("async function " + f.Name() + " used as function pointer") + } + parent := use.InstructionParent().Parent() + for i := 0; i < use.OperandsCount()-1; i++ { + if use.Operand(i) == f { + return false, errors.New("async function " + f.Name() + " used as function pointer in " + parent.Name()) + } + } + worklist = append(worklist, parent) + } + } + + // Check whether a scheduler is needed. + makeGoroutine := c.mod.NamedFunction("runtime.makeGoroutine") + if c.GOOS == "js" && strings.HasPrefix(c.Triple, "wasm") { + // JavaScript always needs a scheduler, as in general no blocking + // operations are possible. Blocking operations block the browser UI, + // which is very bad. + needsScheduler = true + } else { + // Only use a scheduler when an async goroutine is started. When the + // goroutine is not async (does not do any blocking operation), no + // scheduler is necessary as it can be called directly. + for _, use := range getUses(makeGoroutine) { + // Input param must be const bitcast of function. + bitcast := use.Operand(0) + if !bitcast.IsConstant() || bitcast.Opcode() != llvm.BitCast { + panic("expected const bitcast operand of runtime.makeGoroutine") + } + goroutine := bitcast.Operand(0) + if _, ok := asyncFuncs[goroutine]; ok { + needsScheduler = true + break + } + } + } + + if !needsScheduler { + // No scheduler is needed. Do not transform all functions here. + // However, make sure that all go calls (which are all non-async) are + // transformed into regular calls. + return false, c.lowerMakeGoroutineCalls() + } + + // Create a few LLVM intrinsics for coroutine support. + + coroIdType := llvm.FunctionType(c.ctx.TokenType(), []llvm.Type{c.ctx.Int32Type(), c.i8ptrType, c.i8ptrType, c.i8ptrType}, false) + coroIdFunc := llvm.AddFunction(c.mod, "llvm.coro.id", coroIdType) + + coroSizeType := llvm.FunctionType(c.ctx.Int32Type(), nil, false) + coroSizeFunc := llvm.AddFunction(c.mod, "llvm.coro.size.i32", coroSizeType) + + coroBeginType := llvm.FunctionType(c.i8ptrType, []llvm.Type{c.ctx.TokenType(), c.i8ptrType}, false) + coroBeginFunc := llvm.AddFunction(c.mod, "llvm.coro.begin", coroBeginType) + + coroSuspendType := llvm.FunctionType(c.ctx.Int8Type(), []llvm.Type{c.ctx.TokenType(), c.ctx.Int1Type()}, false) + coroSuspendFunc := llvm.AddFunction(c.mod, "llvm.coro.suspend", coroSuspendType) + + coroEndType := llvm.FunctionType(c.ctx.Int1Type(), []llvm.Type{c.i8ptrType, c.ctx.Int1Type()}, false) + coroEndFunc := llvm.AddFunction(c.mod, "llvm.coro.end", coroEndType) + + coroFreeType := llvm.FunctionType(c.i8ptrType, []llvm.Type{c.ctx.TokenType(), c.i8ptrType}, false) + coroFreeFunc := llvm.AddFunction(c.mod, "llvm.coro.free", coroFreeType) + + // Transform all async functions into coroutines. + for _, f := range asyncList { + if f == sleep { + continue + } + + frame := asyncFuncs[f] + frame.cleanupBlock = c.ctx.AddBasicBlock(f, "task.cleanup") + frame.suspendBlock = c.ctx.AddBasicBlock(f, "task.suspend") + frame.unreachableBlock = c.ctx.AddBasicBlock(f, "task.unreachable") + + // Scan for async calls and return instructions that need to have + // suspend points inserted. + var asyncCalls []llvm.Value + var returns []llvm.Value + for bb := f.EntryBasicBlock(); !bb.IsNil(); bb = llvm.NextBasicBlock(bb) { + for inst := bb.FirstInstruction(); !inst.IsNil(); inst = llvm.NextInstruction(inst) { + if !inst.IsACallInst().IsNil() { + callee := inst.CalledValue() + if _, ok := asyncFuncs[callee]; !ok || callee == sleep { + continue + } + asyncCalls = append(asyncCalls, inst) + } else if !inst.IsAReturnInst().IsNil() { + returns = append(returns, inst) + } + } + } + + // Coroutine setup. + c.builder.SetInsertPointBefore(f.EntryBasicBlock().FirstInstruction()) + taskState := c.builder.CreateAlloca(c.mod.GetTypeByName("runtime.taskState"), "task.state") + stateI8 := c.builder.CreateBitCast(taskState, c.i8ptrType, "task.state.i8") + id := c.builder.CreateCall(coroIdFunc, []llvm.Value{ + llvm.ConstInt(c.ctx.Int32Type(), 0, false), + stateI8, + llvm.ConstNull(c.i8ptrType), + llvm.ConstNull(c.i8ptrType), + }, "task.token") + size := c.builder.CreateCall(coroSizeFunc, nil, "task.size") + if c.targetData.TypeAllocSize(size.Type()) > c.targetData.TypeAllocSize(c.uintptrType) { + size = c.builder.CreateTrunc(size, c.uintptrType, "task.size.uintptr") + } else if c.targetData.TypeAllocSize(size.Type()) < c.targetData.TypeAllocSize(c.uintptrType) { + size = c.builder.CreateZExt(size, c.uintptrType, "task.size.uintptr") + } + data := c.createRuntimeCall("alloc", []llvm.Value{size}, "task.data") + frame.taskHandle = c.builder.CreateCall(coroBeginFunc, []llvm.Value{id, data}, "task.handle") + + // Modify async calls so this function suspends right after the child + // returns, because the child is probably not finished yet. Wait until + // the child reactivates the parent. + for _, inst := range asyncCalls { + inst.SetOperand(inst.OperandsCount()-2, frame.taskHandle) + + // Split this basic block. + await := c.splitBasicBlock(inst, llvm.NextBasicBlock(c.builder.GetInsertBlock()), "task.await") + + // Set task state to TASK_STATE_CALL. + c.builder.SetInsertPointAtEnd(inst.InstructionParent()) + + // Suspend. + continuePoint := c.builder.CreateCall(coroSuspendFunc, []llvm.Value{ + llvm.ConstNull(c.ctx.TokenType()), + llvm.ConstInt(c.ctx.Int1Type(), 0, false), + }, "") + sw := c.builder.CreateSwitch(continuePoint, frame.suspendBlock, 2) + sw.AddCase(llvm.ConstInt(c.ctx.Int8Type(), 0, false), await) + sw.AddCase(llvm.ConstInt(c.ctx.Int8Type(), 1, false), frame.cleanupBlock) + } + + // Replace return instructions with suspend points that should + // reactivate the parent coroutine. + for _, inst := range returns { + if inst.OperandsCount() == 0 { + // These properties were added by the functionattrs pass. + // Remove them, because now we start using the parameter. + // https://llvm.org/docs/Passes.html#functionattrs-deduce-function-attributes + for _, kind := range []string{"nocapture", "readnone"} { + kindID := llvm.AttributeKindID(kind) + f.RemoveEnumAttributeAtIndex(f.ParamsCount(), kindID) + } + + // Reactivate the parent coroutine. This adds it back to + // the run queue, so it is started again by the + // scheduler when possible (possibly right after the + // following suspend). + c.builder.SetInsertPointBefore(inst) + + parentHandle := f.LastParam() + c.createRuntimeCall("activateTask", []llvm.Value{parentHandle}, "") + + // Suspend this coroutine. + // It would look like this is unnecessary, but if this + // suspend point is left out, it leads to undefined + // behavior somehow (with the unreachable instruction). + continuePoint := c.builder.CreateCall(coroSuspendFunc, []llvm.Value{ + llvm.ConstNull(c.ctx.TokenType()), + llvm.ConstInt(c.ctx.Int1Type(), 1, false), + }, "ret") + sw := c.builder.CreateSwitch(continuePoint, frame.suspendBlock, 2) + sw.AddCase(llvm.ConstInt(c.ctx.Int8Type(), 0, false), frame.unreachableBlock) + sw.AddCase(llvm.ConstInt(c.ctx.Int8Type(), 1, false), frame.cleanupBlock) + inst.EraseFromParentAsInstruction() + } else { + panic("todo: return value from coroutine") + } + } + + // Coroutine cleanup. Free resources associated with this coroutine. + c.builder.SetInsertPointAtEnd(frame.cleanupBlock) + mem := c.builder.CreateCall(coroFreeFunc, []llvm.Value{id, frame.taskHandle}, "task.data.free") + c.createRuntimeCall("free", []llvm.Value{mem}, "") + c.builder.CreateBr(frame.suspendBlock) + + // Coroutine suspend. A call to llvm.coro.suspend() will branch here. + c.builder.SetInsertPointAtEnd(frame.suspendBlock) + c.builder.CreateCall(coroEndFunc, []llvm.Value{frame.taskHandle, llvm.ConstInt(c.ctx.Int1Type(), 0, false)}, "unused") + returnType := f.Type().ElementType().ReturnType() + if returnType.TypeKind() == llvm.VoidTypeKind { + c.builder.CreateRetVoid() + } else { + c.builder.CreateRet(llvm.Undef(returnType)) + } + + // Coroutine exit. All final suspends (return instructions) will branch + // here. + c.builder.SetInsertPointAtEnd(frame.unreachableBlock) + c.builder.CreateUnreachable() + } + + // Transform calls to time.Sleep() into coroutine suspend points. + for _, sleepCall := range getUses(sleep) { + // sleepCall must be a call instruction. + frame := asyncFuncs[sleepCall.InstructionParent().Parent()] + duration := sleepCall.Operand(0) + + // Set task state to TASK_STATE_SLEEP and set the duration. + c.builder.SetInsertPointBefore(sleepCall) + c.createRuntimeCall("sleepTask", []llvm.Value{frame.taskHandle, duration}, "") + + // Yield to scheduler. + continuePoint := c.builder.CreateCall(coroSuspendFunc, []llvm.Value{ + llvm.ConstNull(c.ctx.TokenType()), + llvm.ConstInt(c.ctx.Int1Type(), 0, false), + }, "") + wakeup := c.splitBasicBlock(sleepCall, llvm.NextBasicBlock(c.builder.GetInsertBlock()), "task.wakeup") + c.builder.SetInsertPointBefore(sleepCall) + sw := c.builder.CreateSwitch(continuePoint, frame.suspendBlock, 2) + sw.AddCase(llvm.ConstInt(c.ctx.Int8Type(), 0, false), wakeup) + sw.AddCase(llvm.ConstInt(c.ctx.Int8Type(), 1, false), frame.cleanupBlock) + sleepCall.EraseFromParentAsInstruction() + } + + return true, c.lowerMakeGoroutineCalls() +} + +// Lower runtime.makeGoroutine calls to regular call instructions. This is done +// after the regular goroutine transformations. The started goroutines are +// either non-blocking (in which case they can be called directly) or blocking, +// in which case they will ask the scheduler themselves to be rescheduled. +func (c *Compiler) lowerMakeGoroutineCalls() error { + // The following Go code: + // go startedGoroutine() + // + // Is translated to the following during IR construction, to preserve the + // fact that this function should be called as a new goroutine. + // %0 = call i8* @runtime.makeGoroutine(i8* bitcast (void (i8*, i8*)* @main.startedGoroutine to i8*), i8* undef, i8* null) + // %1 = bitcast i8* %0 to void (i8*, i8*)* + // call void %1(i8* undef, i8* undef) + // + // This function rewrites it to a direct call: + // call void @main.startedGoroutine(i8* undef, i8* null) + + makeGoroutine := c.mod.NamedFunction("runtime.makeGoroutine") + for _, goroutine := range getUses(makeGoroutine) { + bitcastIn := goroutine.Operand(0) + origFunc := bitcastIn.Operand(0) + uses := getUses(goroutine) + if len(uses) != 1 || uses[0].IsABitCastInst().IsNil() { + return errors.New("expected exactly 1 bitcast use of runtime.makeGoroutine") + } + bitcastOut := uses[0] + uses = getUses(bitcastOut) + if len(uses) != 1 || uses[0].IsACallInst().IsNil() { + return errors.New("expected exactly 1 call use of runtime.makeGoroutine bitcast") + } + realCall := uses[0] + + // Create call instruction. + var params []llvm.Value + for i := 0; i < realCall.OperandsCount()-1; i++ { + params = append(params, realCall.Operand(i)) + } + params[len(params)-1] = llvm.ConstPointerNull(c.i8ptrType) // parent coroutine handle (must be nil) + c.builder.SetInsertPointBefore(realCall) + c.builder.CreateCall(origFunc, params, "") + realCall.EraseFromParentAsInstruction() + bitcastOut.EraseFromParentAsInstruction() + goroutine.EraseFromParentAsInstruction() + } + + return nil +} diff --git a/compiler/interface.go b/compiler/interface.go index 407ec8bb1..fa36fcf08 100644 --- a/compiler/interface.go +++ b/compiler/interface.go @@ -330,6 +330,8 @@ func (c *Compiler) getInvokeCall(frame *Frame, instr *ssa.CallCommon) (llvm.Valu // Add the context parameter. An interface call never takes a context but we // have to supply the parameter anyway. args = append(args, llvm.Undef(c.i8ptrType)) + // Add the parent goroutine handle. + args = append(args, llvm.Undef(c.i8ptrType)) return fnCast, args, nil } diff --git a/compiler/llvm.go b/compiler/llvm.go new file mode 100644 index 000000000..721012f80 --- /dev/null +++ b/compiler/llvm.go @@ -0,0 +1,95 @@ +package compiler + +import ( + "github.com/aykevl/go-llvm" +) + +// This file contains helper functions for LLVM that are not exposed in the Go +// bindings. + +// Return a list of values (actually, instructions) where this value is used as +// an operand. +func getUses(value llvm.Value) []llvm.Value { + if value.IsNil() { + return nil + } + var uses []llvm.Value + use := value.FirstUse() + for !use.IsNil() { + uses = append(uses, use.User()) + use = use.NextUse() + } + return uses +} + +// splitBasicBlock splits a LLVM basic block into two parts. All instructions +// after afterInst are moved into a new basic block (created right after the +// current one) with the given name. +func (c *Compiler) splitBasicBlock(afterInst llvm.Value, insertAfter llvm.BasicBlock, name string) llvm.BasicBlock { + oldBlock := afterInst.InstructionParent() + newBlock := c.ctx.InsertBasicBlock(insertAfter, name) + var nextInstructions []llvm.Value // values to move + + // Collect to-be-moved instructions. + inst := afterInst + for { + inst = llvm.NextInstruction(inst) + if inst.IsNil() { + break + } + nextInstructions = append(nextInstructions, inst) + } + + // Move instructions. + c.builder.SetInsertPointAtEnd(newBlock) + for _, inst := range nextInstructions { + inst.RemoveFromParentAsInstruction() + c.builder.Insert(inst) + } + + // Find PHI nodes to update. + var phiNodes []llvm.Value // PHI nodes to update + for bb := insertAfter.Parent().FirstBasicBlock(); !bb.IsNil(); bb = llvm.NextBasicBlock(bb) { + for inst := bb.FirstInstruction(); !inst.IsNil(); inst = llvm.NextInstruction(inst) { + if inst.IsAPHINode().IsNil() { + continue + } + needsUpdate := false + incomingCount := inst.IncomingCount() + for i := 0; i < incomingCount; i++ { + if inst.IncomingBlock(i) == oldBlock { + needsUpdate = true + break + } + } + if !needsUpdate { + // PHI node has no incoming edge from the old block. + continue + } + phiNodes = append(phiNodes, inst) + } + } + + // Update PHI nodes. + for _, phi := range phiNodes { + c.builder.SetInsertPointBefore(phi) + newPhi := c.builder.CreatePHI(phi.Type(), "") + incomingCount := phi.IncomingCount() + incomingVals := make([]llvm.Value, incomingCount) + incomingBlocks := make([]llvm.BasicBlock, incomingCount) + for i := 0; i < incomingCount; i++ { + value := phi.IncomingValue(i) + block := phi.IncomingBlock(i) + if block == oldBlock { + block = newBlock + } + incomingVals[i] = value + incomingBlocks[i] = block + } + newPhi.AddIncoming(incomingVals, incomingBlocks) + phi.ReplaceAllUsesWith(newPhi) + phi.EraseFromParentAsInstruction() + } + + return newBlock +} diff --git a/compiler/optimizer.go b/compiler/optimizer.go index b78a4fb86..15225f2ec 100644 --- a/compiler/optimizer.go +++ b/compiler/optimizer.go @@ -52,9 +52,18 @@ func (c *Compiler) Optimize(optLevel, sizeLevel int, inlinerThreshold uint) erro // Run TinyGo-specific interprocedural optimizations. c.OptimizeAllocs() c.OptimizeStringToBytes() + + err := c.LowerGoroutines() + if err != nil { + return err + } } else { // Must be run at any optimization level. c.LowerInterfaces() + err := c.LowerGoroutines() + if err != nil { + return err + } } if err := c.Verify(); err != nil { return errors.New("optimizations caused a verification failure") @@ -70,6 +79,13 @@ func (c *Compiler) Optimize(optLevel, sizeLevel int, inlinerThreshold uint) erro } } + // Run function passes again, because without it, llvm.coro.size.i32() + // doesn't get lowered. + for fn := c.mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) { + funcPasses.RunFunc(fn) + } + funcPasses.FinalizeFunc() + // Run module passes. modPasses := llvm.NewPassManager() defer modPasses.Dispose() @@ -324,18 +340,3 @@ func (c *Compiler) hasFlag(call, param llvm.Value, kind string) bool { } return true } - -// Return a list of values (actually, instructions) where this value is used as -// an operand. -func getUses(value llvm.Value) []llvm.Value { - if value.IsNil() { - return nil - } - var uses []llvm.Value - use := value.FirstUse() - for !use.IsNil() { - uses = append(uses, use.User()) - use = use.NextUse() - } - return uses -} diff --git a/interp/interp.go b/interp/interp.go index c0f018c5a..9d56df43c 100644 --- a/interp/interp.go +++ b/interp/interp.go @@ -63,7 +63,7 @@ func Run(mod llvm.Module, targetData llvm.TargetData, debug bool) error { return errors.New("expected all instructions in " + name + " to be *.init() calls") } pkgName := initName[:len(initName)-5] - _, err := e.Function(call.CalledValue(), []Value{&LocalValue{e, undefPtr}}, pkgName) + _, err := e.Function(call.CalledValue(), []Value{&LocalValue{e, undefPtr}, &LocalValue{e, undefPtr}}, pkgName) if err == ErrUnreachable { break } diff --git a/ir/ir.go b/ir/ir.go index 242755abe..75885f158 100644 --- a/ir/ir.go +++ b/ir/ir.go @@ -18,31 +18,26 @@ import ( // View on all functions, types, and globals in a program, with analysis // results. type Program struct { - Program *ssa.Program - LoaderProgram *loader.Program - mainPkg *ssa.Package - Functions []*Function - functionMap map[*ssa.Function]*Function - Globals []*Global - globalMap map[*ssa.Global]*Global - comments map[string]*ast.CommentGroup - NamedTypes []*NamedType - needsScheduler bool - goCalls []*ssa.Go + Program *ssa.Program + LoaderProgram *loader.Program + mainPkg *ssa.Package + Functions []*Function + functionMap map[*ssa.Function]*Function + Globals []*Global + globalMap map[*ssa.Global]*Global + comments map[string]*ast.CommentGroup + NamedTypes []*NamedType } // Function or method. type Function struct { *ssa.Function LLVMFn llvm.Value - linkName string // go:linkname, go:export, go:interrupt - exported bool // go:export - nobounds bool // go:nobounds - blocking bool // calculated by AnalyseBlockingRecursive - flag bool // used by dead code elimination - interrupt bool // go:interrupt - parents []*Function // calculated by AnalyseCallgraph - children []*Function // calculated by AnalyseCallgraph + linkName string // go:linkname, go:export, go:interrupt + exported bool // go:export + nobounds bool // go:nobounds + flag bool // used by dead code elimination + interrupt bool // go:interrupt } // Global variable, possibly constant. diff --git a/ir/passes.go b/ir/passes.go index 792700b75..e73bf34cc 100644 --- a/ir/passes.go +++ b/ir/passes.go @@ -56,110 +56,6 @@ func signature(sig *types.Signature) string { return s } -// Fill in parents of all functions. -// -// All packages need to be added before this pass can run, or it will produce -// incorrect results. -func (p *Program) AnalyseCallgraph() { - for _, f := range p.Functions { - // Clear, if AnalyseCallgraph has been called before. - f.children = nil - f.parents = nil - - for _, block := range f.Blocks { - for _, instr := range block.Instrs { - switch instr := instr.(type) { - case *ssa.Call: - if instr.Common().IsInvoke() { - continue - } - switch call := instr.Call.Value.(type) { - case *ssa.Builtin: - // ignore - case *ssa.Function: - if isCGoInternal(call.Name()) { - continue - } - child := p.GetFunction(call) - if child.CName() != "" { - continue // assume non-blocking - } - if child.RelString(nil) == "time.Sleep" { - f.blocking = true - } - f.children = append(f.children, child) - } - } - } - } - } - for _, f := range p.Functions { - for _, child := range f.children { - child.parents = append(child.parents, f) - } - } -} - -// Analyse which functions are recursively blocking. -// -// Depends on AnalyseCallgraph. -func (p *Program) AnalyseBlockingRecursive() { - worklist := make([]*Function, 0) - - // Fill worklist with directly blocking functions. - for _, f := range p.Functions { - if f.blocking { - worklist = append(worklist, f) - } - } - - // Keep reducing this worklist by marking a function as recursively blocking - // from the worklist and pushing all its parents that are non-blocking. - // This is somewhat similar to a worklist in a mark-sweep garbage collector. - // The work items are then grey objects. - for len(worklist) != 0 { - // Pick the topmost. - f := worklist[len(worklist)-1] - worklist = worklist[:len(worklist)-1] - for _, parent := range f.parents { - if !parent.blocking { - parent.blocking = true - worklist = append(worklist, parent) - } - } - } -} - -// Check whether we need a scheduler. A scheduler is only necessary when there -// are go calls that start blocking functions (if they're not blocking, the go -// function can be turned into a regular function call). -// -// Depends on AnalyseBlockingRecursive. -func (p *Program) AnalyseGoCalls() { - p.goCalls = nil - for _, f := range p.Functions { - for _, block := range f.Blocks { - for _, instr := range block.Instrs { - switch instr := instr.(type) { - case *ssa.Go: - p.goCalls = append(p.goCalls, instr) - } - } - } - } - for _, instr := range p.goCalls { - switch instr := instr.Call.Value.(type) { - case *ssa.Builtin: - case *ssa.Function: - if p.functionMap[instr].blocking { - p.needsScheduler = true - } - default: - panic("unknown go call function type") - } - } -} - // Simple pass that removes dead code. This pass makes later analysis passes // more useful. func (p *Program) SimpleDCE() { @@ -239,21 +135,3 @@ func (p *Program) SimpleDCE() { } p.Functions = livefunctions } - -// Whether this function needs a scheduler. -// -// Depends on AnalyseGoCalls. -func (p *Program) NeedsScheduler() bool { - return p.needsScheduler -} - -// Whether this function blocks. Builtins are also accepted for convenience. -// They will always be non-blocking. -// -// Depends on AnalyseBlockingRecursive. -func (p *Program) IsBlocking(f *Function) bool { - if !p.needsScheduler { - return false - } - return f.blocking -} diff --git a/src/runtime/runtime.go b/src/runtime/runtime.go index d7b8d244a..c1e47f0fa 100644 --- a/src/runtime/runtime.go +++ b/src/runtime/runtime.go @@ -10,8 +10,8 @@ const Compiler = "tgo" // package. func initAll() -// The compiler will insert the call to main.main() here, depending on whether -// the scheduler is necessary. +// A function call to this function is replaced withone of the following, +// depending on whether the scheduler is necessary: // // Without scheduler: // @@ -19,9 +19,9 @@ func initAll() // // With scheduler: // -// coroutine := main.main(nil) -// scheduler(coroutine) -func mainWrapper() +// main.main() +// scheduler() +func callMain() func GOMAXPROCS(n int) int { // Note: setting GOMAXPROCS is ignored. diff --git a/src/runtime/runtime_atsamd21g18.go b/src/runtime/runtime_atsamd21g18.go index 3682089d6..f1ee98000 100644 --- a/src/runtime/runtime_atsamd21g18.go +++ b/src/runtime/runtime_atsamd21g18.go @@ -15,7 +15,7 @@ type timeUnit int64 func main() { preinit() initAll() - mainWrapper() + callMain() abort() } @@ -238,6 +238,8 @@ type isrFlag bool var timerWakeup isrFlag +const asyncScheduler = false + // sleepTicks should sleep for d number of microseconds. func sleepTicks(d timeUnit) { for d != 0 { diff --git a/src/runtime/runtime_avr.go b/src/runtime/runtime_avr.go index deeaa4ed1..78c6c41ce 100644 --- a/src/runtime/runtime_avr.go +++ b/src/runtime/runtime_avr.go @@ -41,7 +41,7 @@ func main() { preinit() initAll() postinit() - mainWrapper() + callMain() abort() } @@ -71,6 +71,8 @@ func putchar(c byte) { machine.UART0.WriteByte(c) } +const asyncScheduler = false + // Sleep this number of ticks of 16ms. // // TODO: not very accurate. Improve accuracy by calibrating on startup and every diff --git a/src/runtime/runtime_nrf.go b/src/runtime/runtime_nrf.go index 41524ae0d..e6009abe2 100644 --- a/src/runtime/runtime_nrf.go +++ b/src/runtime/runtime_nrf.go @@ -20,7 +20,7 @@ func main() { systemInit() preinit() initAll() - mainWrapper() + callMain() abort() } @@ -50,6 +50,8 @@ func putchar(c byte) { machine.UART0.WriteByte(c) } +const asyncScheduler = false + func sleepTicks(d timeUnit) { for d != 0 { ticks() // update timestamp diff --git a/src/runtime/runtime_qemu.go b/src/runtime/runtime_qemu.go index 5f16b4be8..5e7d35656 100644 --- a/src/runtime/runtime_qemu.go +++ b/src/runtime/runtime_qemu.go @@ -20,11 +20,13 @@ var timestamp timeUnit func main() { preinit() initAll() - mainWrapper() + callMain() arm.SemihostingCall(arm.SemihostingReportException, arm.SemihostingApplicationExit) abort() } +const asyncScheduler = false + func sleepTicks(d timeUnit) { // TODO: actually sleep here for the given time. timestamp += d diff --git a/src/runtime/runtime_stm32.go b/src/runtime/runtime_stm32.go index b1eb98df6..0717e7cff 100644 --- a/src/runtime/runtime_stm32.go +++ b/src/runtime/runtime_stm32.go @@ -8,6 +8,6 @@ type timeUnit int64 func main() { preinit() initAll() - mainWrapper() + callMain() abort() } diff --git a/src/runtime/runtime_stm32f103xx.go b/src/runtime/runtime_stm32f103xx.go index b20470bef..23925c33e 100644 --- a/src/runtime/runtime_stm32f103xx.go +++ b/src/runtime/runtime_stm32f103xx.go @@ -107,6 +107,8 @@ func initTIM() { arm.EnableIRQ(stm32.IRQ_TIM3) } +const asyncScheduler = false + // sleepTicks should sleep for specific number of microseconds. func sleepTicks(d timeUnit) { for d != 0 { diff --git a/src/runtime/runtime_unix.go b/src/runtime/runtime_unix.go index 1d49b6484..1749e70ce 100644 --- a/src/runtime/runtime_unix.go +++ b/src/runtime/runtime_unix.go @@ -46,8 +46,8 @@ func main() int { // Run initializers of all packages. initAll() - // Compiler-generated wrapper to main.main(). - mainWrapper() + // Compiler-generated call to main.main(). + callMain() // For libc compatibility. return 0 @@ -57,6 +57,8 @@ func putchar(c byte) { _putchar(int(c)) } +const asyncScheduler = false + func sleepTicks(d timeUnit) { usleep(uint(d) / 1000) } diff --git a/src/runtime/runtime_wasm.go b/src/runtime/runtime_wasm.go index 4abc1cd71..541751a24 100644 --- a/src/runtime/runtime_wasm.go +++ b/src/runtime/runtime_wasm.go @@ -6,11 +6,9 @@ import ( "unsafe" ) -type timeUnit int64 +type timeUnit float64 // time in milliseconds, just like Date.now() in JavaScript -const tickMicros = 1 - -var timestamp timeUnit +const tickMicros = 1000000 //go:export io_get_stdout func io_get_stdout() int32 @@ -32,21 +30,27 @@ func _start() { //go:export cwa_main func cwa_main() { initAll() // _start is not called by olin/cwa so has to be called here - mainWrapper() + callMain() } func putchar(c byte) { resource_write(stdout, &c, 1) } -func sleepTicks(d timeUnit) { - // TODO: actually sleep here for the given time. - timestamp += d +//go:export go_scheduler +func go_scheduler() { + scheduler() } -func ticks() timeUnit { - return timestamp -} +const asyncScheduler = true + +// This function is called by the scheduler. +// Schedule a call to runtime.scheduler, do not actually sleep. +//go:export runtime.sleepTicks +func sleepTicks(d timeUnit) + +//go:export runtime.ticks +func ticks() timeUnit // Abort executes the wasm 'unreachable' instruction. func abort() { diff --git a/src/runtime/scheduler.go b/src/runtime/scheduler.go index d2c8b6065..52301c186 100644 --- a/src/runtime/scheduler.go +++ b/src/runtime/scheduler.go @@ -9,14 +9,13 @@ package runtime // * A blocking function that calls a non-blocking function is called as // usual. // * A blocking function that calls a blocking function passes its own -// coroutine handle as a parameter to the subroutine and will make sure it's -// own coroutine is removed from the scheduler. When the subroutine returns, -// it will re-insert the parent into the scheduler. +// coroutine handle as a parameter to the subroutine. When the subroutine +// returns, it will re-insert the parent into the scheduler. // Note that a goroutine is generally called a 'task' for brevity and because // that's the more common term among RTOSes. But a goroutine and a task are // basically the same thing. Although, the code often uses the word 'task' to -// refer to both a coroutine and a goroutine, as most of the scheduler isn't -// aware of the difference. +// refer to both a coroutine and a goroutine, as most of the scheduler doesn't +// care about the difference. // // For more background on coroutines in LLVM: // https://llvm.org/docs/Coroutines.html @@ -45,25 +44,19 @@ func (t *coroutine) _promise(alignment int32, from bool) unsafe.Pointer // Get the promise belonging to a task. func (t *coroutine) promise() *taskState { - return (*taskState)(t._promise(4, false)) + return (*taskState)(t._promise(int32(unsafe.Alignof(taskState{})), false)) } +func makeGoroutine(*uint8) *uint8 + // State/promise of a task. Internally represented as: // -// {i8 state, i32 data, i8* next} +// {i8* next, i32/i64 data} type taskState struct { - state uint8 - data uint32 - next *coroutine + next *coroutine + data uint } -// Various states a task can be in. -const ( - TASK_STATE_RUNNABLE = iota - TASK_STATE_SLEEP - TASK_STATE_CALL // waiting for a sub-coroutine -) - // Queues used by the scheduler. // // TODO: runqueueFront can be removed by making the run queue a circular linked @@ -89,48 +82,28 @@ func scheduleLogTask(msg string, t *coroutine) { } } -// Set the task state to sleep for a given time. +// Set the task to sleep for a given time. // // This is a compiler intrinsic. func sleepTask(caller *coroutine, duration int64) { if schedulerDebug { - println(" set state sleep:", caller, uint32(duration/tickMicros)) + println(" set sleep:", caller, uint(duration/tickMicros)) } promise := caller.promise() - promise.state = TASK_STATE_SLEEP - promise.data = uint32(duration / tickMicros) // TODO: longer durations + promise.data = uint(duration / tickMicros) // TODO: longer durations + addSleepTask(caller) } -// Wait for the result of an async call. This means that the parent goroutine -// will be removed from the runqueue and be rescheduled by the callee. +// Add a non-queued task to the run queue. // -// This is a compiler intrinsic. -func waitForAsyncCall(caller *coroutine) { - scheduleLogTask(" set state call:", caller) - promise := caller.promise() - promise.state = TASK_STATE_CALL -} - -// Add a task to the runnable or sleep queue, depending on the state. -// -// This is a compiler intrinsic. -func yieldToScheduler(t *coroutine) { - if t == nil { +// This is a compiler intrinsic, and is called from a callee to reactivate the +// caller. +func activateTask(task *coroutine) { + if task == nil { return } - // See what we should do with this task: try to execute it directly - // again or let it sleep for a bit. - promise := t.promise() - if promise.state == TASK_STATE_CALL { - scheduleLogTask(" set waiting for call:", t) - return // calling an async task, the subroutine will re-active the parent - } else if promise.state == TASK_STATE_SLEEP && promise.data != 0 { - scheduleLogTask(" set sleeping:", t) - addSleepTask(t) - } else { - scheduleLogTask(" set runnable:", t) - runqueuePushBack(t) - } + scheduleLogTask(" set runnable:", task) + runqueuePushBack(task) } // Add this task to the end of the run queue. May also destroy the task if it's @@ -145,9 +118,6 @@ func runqueuePushBack(t *coroutine) { if t.promise().next != nil { panic("runtime: runqueuePushBack: expected next task to be nil") } - if t.promise().state != TASK_STATE_RUNNABLE { - panic("runtime: runqueuePushBack: expected task state to be runnable") - } } if runqueueBack == nil { // empty runqueue scheduleLogTask(" add to runqueue front:", t) @@ -169,10 +139,6 @@ func runqueuePopFront() *coroutine { } if schedulerDebug { println(" runqueuePopFront:", t) - // Sanity checking. - if t.promise().state != TASK_STATE_RUNNABLE { - panic("runtime: runqueuePopFront: task not runnable") - } } promise := t.promise() runqueueFront = promise.next @@ -190,9 +156,6 @@ func addSleepTask(t *coroutine) { if t.promise().next != nil { panic("runtime: addSleepTask: expected next task to be nil") } - if t.promise().state != TASK_STATE_SLEEP { - panic("runtime: addSleepTask: task not sleeping") - } } now := ticks() if sleepQueue == nil { @@ -236,11 +199,7 @@ func addSleepTask(t *coroutine) { } // Run the scheduler until all tasks have finished. -// It takes an initial task (main.main) to bootstrap. -func scheduler(main *coroutine) { - // Initial task. - yieldToScheduler(main) - +func scheduler() { // Main scheduler loop. for { scheduleLog("\n schedule") @@ -254,7 +213,6 @@ func scheduler(main *coroutine) { promise := t.promise() sleepQueueBaseTime += timeUnit(promise.data) sleepQueue = promise.next - promise.state = TASK_STATE_RUNNABLE promise.next = nil runqueuePushBack(t) } @@ -271,9 +229,15 @@ func scheduler(main *coroutine) { } timeLeft := timeUnit(sleepQueue.promise().data) - (now - sleepQueueBaseTime) if schedulerDebug { - println(" sleeping...", sleepQueue, uint32(timeLeft)) + println(" sleeping...", sleepQueue, uint(timeLeft)) } sleepTicks(timeUnit(timeLeft)) + if asyncScheduler { + // The sleepTicks function above only sets a timeout at which + // point the scheduler will be called again. It does not really + // sleep. + break + } continue } @@ -281,8 +245,5 @@ func scheduler(main *coroutine) { scheduleLog(" <- runqueuePopFront") scheduleLogTask(" run:", t) t.resume() - - // Add the just resumed task to the run queue or the sleep queue. - yieldToScheduler(t) } } diff --git a/targets/wasm_exec.js b/targets/wasm_exec.js index c693a3b08..2f7b3d233 100644 --- a/targets/wasm_exec.js +++ b/targets/wasm_exec.js @@ -213,6 +213,17 @@ } }, + // func ticks() float64 + "runtime.ticks": () => { + return timeOrigin + performance.now(); + }, + + // func sleepTicks(timeout float64) + "runtime.sleepTicks": (timeout) => { + // Do not sleep, only reactivate scheduler after the given timeout. + setTimeout(this._inst.exports.go_scheduler, timeout); + }, + // func stringVal(value string) ref "syscall/js.stringVal": (ret_ptr, value_ptr, value_len) => { const s = loadString(value_ptr, value_len); diff --git a/testdata/coroutines.go b/testdata/coroutines.go index 18b1c0a87..68a1ae249 100644 --- a/testdata/coroutines.go +++ b/testdata/coroutines.go @@ -9,6 +9,18 @@ func main() { println("main 2") time.Sleep(2 * time.Millisecond) println("main 3") + + // Await a blocking call. This must create a new coroutine. + println("wait:") + wait() + println("end waiting") + + // Run a non-blocking call in a goroutine. This should be turned into a + // regular call, so should be equivalent to calling nowait() without 'go' + // prefix. + go nowait() + time.Sleep(time.Millisecond) + println("done with non-blocking goroutine") } func sub() { @@ -16,3 +28,13 @@ func sub() { time.Sleep(2 * time.Millisecond) println("sub 2") } + +func wait() { + println(" wait start") + time.Sleep(time.Millisecond) + println(" wait end") +} + +func nowait() { + println("non-blocking goroutine") +} diff --git a/testdata/coroutines.txt b/testdata/coroutines.txt index c8b209da7..d5e2d74f9 100644 --- a/testdata/coroutines.txt +++ b/testdata/coroutines.txt @@ -3,3 +3,9 @@ sub 1 main 2 sub 2 main 3 +wait: + wait start + wait end +end waiting +non-blocking goroutine +done with non-blocking goroutine