diff --git a/compiler/compiler.go b/compiler/compiler.go index 673fe8931..c025c0f7a 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -156,6 +156,7 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con Compiler: "gc", // must be one of the recognized compilers BuildTags: c.BuildTags(), }, + Tests: c.TestConfig.CompileTestBinary, TypeChecker: types.Config{ Sizes: &stdSizes{ IntSize: int64(c.targetData.TypeAllocSize(c.intType)), @@ -169,33 +170,17 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con ClangHeaders: c.ClangHeaders, } - if strings.HasSuffix(pkgName, ".go") { - _, err = lprogram.ImportFile(pkgName) - if err != nil { - return c.mod, nil, []error{err} - } - } else { - _, err = lprogram.Import(pkgName, wd, token.Position{ - Filename: "build command-line-arguments", - }) - if err != nil { - return c.mod, nil, []error{err} - } - } - - _, err = lprogram.Import("runtime", "", token.Position{ - Filename: "build default import", - }) + err = lprogram.Load(pkgName) if err != nil { return c.mod, nil, []error{err} } - err = lprogram.Parse(c.TestConfig.CompileTestBinary) + err = lprogram.Parse() if err != nil { return c.mod, nil, []error{err} } - c.ir = ir.NewProgram(lprogram, pkgName) + c.ir = ir.NewProgram(lprogram) // Run a simple dead code elimination pass. err = c.ir.SimpleDCE() @@ -339,8 +324,11 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con // Gather the list of (C) file paths that should be included in the build. var extraFiles []string for _, pkg := range c.ir.LoaderProgram.Sorted() { - for _, file := range pkg.CFiles { - extraFiles = append(extraFiles, filepath.Join(pkg.Package.Dir, file)) + for _, file := range pkg.OtherFiles { + switch strings.ToLower(filepath.Ext(file)) { + case ".c": + extraFiles = append(extraFiles, file) + } } } diff --git a/ir/ir.go b/ir/ir.go index 7506400df..89ba5581f 100644 --- a/ir/ir.go +++ b/ir/ir.go @@ -63,23 +63,13 @@ const ( ) // Create and initialize a new *Program from a *ssa.Program. -func NewProgram(lprogram *loader.Program, mainPath string) *Program { +func NewProgram(lprogram *loader.Program) *Program { program := lprogram.LoadSSA() program.Build() // Find the main package, which is a bit difficult when running a .go file // directly. - mainPkg := program.ImportedPackage(mainPath) - if mainPkg == nil { - for _, pkgInfo := range program.AllPackages() { - if pkgInfo.Pkg.Name() == "main" { - if mainPkg != nil { - panic("more than one main package found") - } - mainPkg = pkgInfo - } - } - } + mainPkg := program.ImportedPackage(lprogram.MainPkg.PkgPath) if mainPkg == nil { panic("could not find main package") } @@ -87,21 +77,10 @@ func NewProgram(lprogram *loader.Program, mainPath string) *Program { // Make a list of packages in import order. packageList := []*ssa.Package{} packageSet := map[string]struct{}{} - worklist := []string{"runtime", mainPath} + worklist := []string{"runtime", lprogram.MainPkg.PkgPath} for len(worklist) != 0 { pkgPath := worklist[0] - var pkg *ssa.Package - if pkgPath == mainPath { - pkg = mainPkg // necessary for compiling individual .go files - } else { - pkg = program.ImportedPackage(pkgPath) - } - if pkg == nil { - // Non-SSA package (e.g. cgo). - packageSet[pkgPath] = struct{}{} - worklist = worklist[1:] - continue - } + pkg := program.ImportedPackage(pkgPath) if _, ok := packageSet[pkgPath]; ok { // Package already in the final package list. worklist = worklist[1:] diff --git a/loader/errors.go b/loader/errors.go index c25652ee1..e0414204b 100644 --- a/loader/errors.go +++ b/loader/errors.go @@ -1,10 +1,5 @@ package loader -import ( - "go/token" - "strings" -) - // Errors contains a list of parser errors or a list of typechecker errors for // the given package. type Errors struct { @@ -15,25 +10,3 @@ type Errors struct { func (e Errors) Error() string { return "could not compile: " + e.Errs[0].Error() } - -// ImportCycleErrors is returned when encountering an import cycle. The list of -// packages is a list from the root package to the leaf package that imports one -// of the packages in the list. -type ImportCycleError struct { - Packages []string - ImportPositions []token.Position -} - -func (e *ImportCycleError) Error() string { - var msg strings.Builder - msg.WriteString("import cycle:\n\t") - msg.WriteString(strings.Join(e.Packages, "\n\t")) - msg.WriteString("\n at ") - for i, pos := range e.ImportPositions { - if i > 0 { - msg.WriteString(", ") - } - msg.WriteString(pos.String()) - } - return msg.String() -} diff --git a/loader/goroot.go b/loader/goroot.go index e76f598a1..fbebb5690 100644 --- a/loader/goroot.go +++ b/loader/goroot.go @@ -10,8 +10,10 @@ import ( "io/ioutil" "math/rand" "os" + "os/exec" "path" "path/filepath" + "runtime" "strconv" "github.com/tinygo-org/tinygo/compileopts" @@ -58,8 +60,12 @@ func GetCachedGoroot(config *compileopts.Config) (string, error) { return "", err } + // Remove the temporary directory if it wasn't moved to the right place + // (for example, when there was an error). + defer os.RemoveAll(tmpgoroot) + for _, name := range []string{"bin", "lib", "pkg"} { - err = os.Symlink(filepath.Join(goroot, name), filepath.Join(tmpgoroot, name)) + err = symlink(filepath.Join(goroot, name), filepath.Join(tmpgoroot, name)) if err != nil { return "", err } @@ -72,8 +78,8 @@ func GetCachedGoroot(config *compileopts.Config) (string, error) { if err != nil { if os.IsExist(err) { // Another invocation of TinyGo also seems to have created a GOROOT. - // Use that one instead and delete ours. - os.RemoveAll(tmpgoroot) + // Use that one instead. Our new GOROOT will be automatically + // deleted by the defer above. return cachedgoroot, nil } return "", err @@ -91,7 +97,7 @@ func mergeDirectory(goroot, tinygoroot, tmpgoroot, importPath string, overrides // root, so simply make a symlink. newname := filepath.Join(tmpgoroot, "src", importPath) oldname := filepath.Join(tinygoroot, "src", importPath) - return os.Symlink(oldname, newname) + return symlink(oldname, newname) } // Merge subdirectories. Start by making the directory to merge. @@ -117,7 +123,7 @@ func mergeDirectory(goroot, tinygoroot, tmpgoroot, importPath string, overrides // A file, so symlink this. newname := filepath.Join(tmpgoroot, "src", importPath, e.Name()) oldname := filepath.Join(tinygoroot, "src", importPath, e.Name()) - err := os.Symlink(oldname, newname) + err := symlink(oldname, newname) if err != nil { return err } @@ -143,7 +149,7 @@ func mergeDirectory(goroot, tinygoroot, tmpgoroot, importPath string, overrides } newname := filepath.Join(tmpgoroot, "src", importPath, e.Name()) oldname := filepath.Join(goroot, "src", importPath, e.Name()) - err := os.Symlink(oldname, newname) + err := symlink(oldname, newname) if err != nil { return err } @@ -174,3 +180,46 @@ func pathsToOverride(needsSyscallPackage bool) map[string]bool { } return paths } + +// symlink creates a symlink or something similar. On Unix-like systems, it +// always creates a symlink. On Windows, it tries to create a symlink and if +// that fails, creates a hardlink or directory junction instead. +// +// Note that while Windows 10 does support symlinks and allows them to be +// created using os.Symlink, it requires developer mode to be enabled. +// Therefore provide a fallback for when symlinking is not possible. +// Unfortunately this fallback only works when TinyGo is installed on the same +// filesystem as the TinyGo cache and the Go installation (which is usually the +// C drive). +func symlink(oldname, newname string) error { + symlinkErr := os.Symlink(oldname, newname) + if runtime.GOOS == "windows" && symlinkErr != nil { + // Fallback for when developer mode is disabled. + // Note that we return the symlink error even if something else fails + // later on. This is because symlinks are the easiest to support + // (they're also used on Linux and MacOS) and enabling them is easy: + // just enable developer mode. + st, err := os.Stat(oldname) + if err != nil { + return symlinkErr + } + if st.IsDir() { + // Make a directory junction. There may be a way to do this + // programmatically, but it involves a lot of magic. Use the mklink + // command built into cmd instead (mklink is a builtin, not an + // external command). + err := exec.Command("cmd", "/k", "mklink", "/J", newname, oldname).Run() + if err != nil { + return symlinkErr + } + } else { + // Make a hard link. + err := os.Link(oldname, newname) + if err != nil { + return symlinkErr + } + } + return nil // success + } + return symlinkErr +} diff --git a/loader/loader.go b/loader/loader.go index 545c1a05c..a190b7563 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -3,10 +3,10 @@ package loader import ( "bytes" "errors" + "fmt" "go/ast" "go/build" "go/parser" - "go/scanner" "go/token" "go/types" "os" @@ -16,13 +16,16 @@ import ( "text/template" "github.com/tinygo-org/tinygo/cgo" + "github.com/tinygo-org/tinygo/goenv" + "golang.org/x/tools/go/packages" ) // Program holds all packages and some metadata about the program as a whole. type Program struct { - mainPkg string Build *build.Context + Tests bool Packages map[string]*Package + MainPkg *Package sorted []*Package fset *token.FileSet TypeChecker types.Config @@ -35,85 +38,114 @@ type Program struct { // Package holds a loaded package, its imports, and its parsed files. type Package struct { *Program - *build.Package - Imports map[string]*Package - Importing bool - Files []*ast.File - Pkg *types.Package + *packages.Package + Files []*ast.File + Pkg *types.Package types.Info } -// Import loads the given package relative to srcDir (for the vendor directory). -// It only loads the current package without recursion. -func (p *Program) Import(path, srcDir string, pos token.Position) (*Package, error) { +// Load loads the given package with all dependencies (including the runtime +// package). Call .Parse() afterwards to parse all Go files (including CGo +// processing, if necessary). +func (p *Program) Load(importPath string) error { if p.Packages == nil { p.Packages = make(map[string]*Package) } - // Load this package. - ctx := p.Build - buildPkg, err := ctx.Import(path, srcDir, build.ImportComment) + err := p.loadPackage(importPath) if err != nil { - return nil, scanner.Error{ - Pos: pos, - Msg: err.Error(), // TODO: define a new error type that will wrap the inner error - } + return err } - if existingPkg, ok := p.Packages[buildPkg.ImportPath]; ok { - // Already imported, or at least started the import. - return existingPkg, nil + p.MainPkg = p.sorted[len(p.sorted)-1] + if _, ok := p.Packages["runtime"]; !ok { + // The runtime package wasn't loaded. Although `go list -deps` seems to + // return the full dependency list, there is no way to get those + // packages from the go/packages package. Therefore load the runtime + // manually and add it to the list of to-be-compiled packages + // (duplicates are already filtered). + return p.loadPackage("runtime") } - p.sorted = nil // invalidate the sorted order of packages - pkg := p.newPackage(buildPkg) - p.Packages[buildPkg.ImportPath] = pkg - - if p.mainPkg == "" { - p.mainPkg = buildPkg.ImportPath - } - - return pkg, nil + return nil } -// ImportFile loads and parses the import statements in the given path and -// creates a pseudo-package out of it. -func (p *Program) ImportFile(path string) (*Package, error) { - if p.Packages == nil { - p.Packages = make(map[string]*Package) +func (p *Program) loadPackage(importPath string) error { + cgoEnabled := "0" + if p.Build.CgoEnabled { + cgoEnabled = "1" } - if _, ok := p.Packages[path]; ok { - // unlikely - return nil, errors.New("loader: cannot import file that is already imported as package: " + path) - } - - file, err := p.parseFile(path, parser.ImportsOnly) + pkgs, err := packages.Load(&packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | packages.NeedImports | packages.NeedDeps, + Env: append(os.Environ(), "GOROOT="+p.Build.GOROOT, "GOOS="+p.Build.GOOS, "GOARCH="+p.Build.GOARCH, "CGO_ENABLED="+cgoEnabled), + BuildFlags: []string{"-tags", strings.Join(p.Build.BuildTags, " ")}, + Tests: p.Tests, + }, importPath) if err != nil { - return nil, err + return err } - buildPkg := &build.Package{ - Dir: filepath.Dir(path), - ImportPath: path, - GoFiles: []string{filepath.Base(path)}, + var pkg *packages.Package + if p.Tests { + // We need the second package. Quoting from the docs: + // > For example, when using the go command, loading "fmt" with Tests=true + // > returns four packages, with IDs "fmt" (the standard package), + // > "fmt [fmt.test]" (the package as compiled for the test), + // > "fmt_test" (the test functions from source files in package fmt_test), + // > and "fmt.test" (the test binary). + pkg = pkgs[1] + } else { + if len(pkgs) != 1 { + return fmt.Errorf("expected exactly one package while importing %s, got %d", importPath, len(pkgs)) + } + pkg = pkgs[0] } - for _, importSpec := range file.Imports { - buildPkg.Imports = append(buildPkg.Imports, importSpec.Path.Value[1:len(importSpec.Path.Value)-1]) - } - p.sorted = nil // invalidate the sorted order of packages - pkg := p.newPackage(buildPkg) - p.Packages[buildPkg.ImportPath] = pkg + var importError *Errors + var addPackages func(pkg *packages.Package) + addPackages = func(pkg *packages.Package) { + if _, ok := p.Packages[pkg.PkgPath]; ok { + return + } + pkg2 := p.newPackage(pkg) + p.Packages[pkg.PkgPath] = pkg2 + if len(pkg.Errors) != 0 { + if importError != nil { + // There was another error reported already. Do not report + // errors from multiple packages at once. + return + } + importError = &Errors{ + Pkg: pkg2, + } + for _, err := range pkg.Errors { + importError.Errs = append(importError.Errs, err) + } + return + } - if p.mainPkg == "" { - p.mainPkg = buildPkg.ImportPath - } + // Get the list of imports (sorted alphabetically). + names := make([]string, 0, len(pkg.Imports)) + for name := range pkg.Imports { + names = append(names, name) + } + sort.Strings(names) - return pkg, nil + // Add all the imports. + for _, name := range names { + addPackages(pkg.Imports[name]) + } + + p.sorted = append(p.sorted, pkg2) + } + addPackages(pkg) + if importError != nil { + return importError + } + return nil } // newPackage instantiates a new *Package object with initialized members. -func (p *Program) newPackage(pkg *build.Package) *Package { +func (p *Program) newPackage(pkg *packages.Package) *Package { return &Package{ Program: p, Package: pkg, - Imports: make(map[string]*Package, len(pkg.Imports)), Info: types.Info{ Types: make(map[ast.Expr]types.TypeAndValue), Defs: make(map[*ast.Ident]types.Object), @@ -128,87 +160,25 @@ func (p *Program) newPackage(pkg *build.Package) *Package { // Sorted returns a list of all packages, sorted in a way that no packages come // before the packages they depend upon. func (p *Program) Sorted() []*Package { - if p.sorted == nil { - p.sort() - } return p.sorted } -func (p *Program) sort() { - p.sorted = nil - packageList := make([]*Package, 0, len(p.Packages)) - packageSet := make(map[string]struct{}, len(p.Packages)) - worklist := make([]string, 0, len(p.Packages)) - for path := range p.Packages { - worklist = append(worklist, path) - } - sort.Strings(worklist) - for len(worklist) != 0 { - pkgPath := worklist[0] - pkg := p.Packages[pkgPath] - - if _, ok := packageSet[pkgPath]; ok { - // Package already in the final package list. - worklist = worklist[1:] - continue - } - - unsatisfiedImports := make([]string, 0) - for _, pkg := range pkg.Imports { - if _, ok := packageSet[pkg.ImportPath]; ok { - continue - } - unsatisfiedImports = append(unsatisfiedImports, pkg.ImportPath) - } - sort.Strings(unsatisfiedImports) - if len(unsatisfiedImports) == 0 { - // All dependencies of this package are satisfied, so add this - // package to the list. - packageList = append(packageList, pkg) - packageSet[pkgPath] = struct{}{} - worklist = worklist[1:] - } else { - // Prepend all dependencies to the worklist and reconsider this - // package (by not removing it from the worklist). At that point, it - // must be possible to add it to packageList. - worklist = append(unsatisfiedImports, worklist...) - } - } - - p.sorted = packageList -} - -// Parse recursively imports all packages, parses them, and typechecks them. +// Parse parses all packages and typechecks them. // // The returned error may be an Errors error, which contains a list of errors. // // Idempotent. -func (p *Program) Parse(compileTestBinary bool) error { - includeTests := compileTestBinary - - // Load all imports - for _, pkg := range p.Sorted() { - err := pkg.importRecursively(includeTests) - if err != nil { - if err, ok := err.(*ImportCycleError); ok { - if pkg.ImportPath != err.Packages[0] { - err.Packages = append([]string{pkg.ImportPath}, err.Packages...) - } - } - return err - } - } - +func (p *Program) Parse() error { // Parse all packages. for _, pkg := range p.Sorted() { - err := pkg.Parse(includeTests) + err := pkg.Parse() if err != nil { return err } } - if compileTestBinary { - err := p.SwapTestMain() + if p.Tests { + err := p.swapTestMain() if err != nil { return err } @@ -225,7 +195,7 @@ func (p *Program) Parse(compileTestBinary bool) error { return nil } -func (p *Program) SwapTestMain() error { +func (p *Program) swapTestMain() error { var tests []string isTestFunc := func(f *ast.FuncDecl) bool { @@ -235,8 +205,7 @@ func (p *Program) SwapTestMain() error { } return false } - mainPkg := p.Packages[p.mainPkg] - for _, f := range mainPkg.Files { + for _, f := range p.MainPkg.Files { for i, d := range f.Decls { switch v := d.(type) { case *ast.FuncDecl: @@ -287,7 +256,7 @@ func main () { if err != nil { return err } - path := filepath.Join(p.mainPkg, "$testmain.go") + path := filepath.Join(p.MainPkg.Dir, "$testmain.go") if p.fset == nil { p.fset = token.NewFileSet() @@ -297,7 +266,7 @@ func main () { if err != nil { return err } - mainPkg.Files = append(mainPkg.Files, newMain) + p.MainPkg.Files = append(p.MainPkg.Files, newMain) return nil } @@ -313,34 +282,41 @@ func (p *Program) parseFile(path string, mode parser.Mode) (*ast.File, error) { return nil, err } defer rd.Close() - relpath := path - if filepath.IsAbs(path) { - rp, err := filepath.Rel(p.Dir, path) - if err == nil { - relpath = rp + diagnosticPath := path + if strings.HasPrefix(path, p.Build.GOROOT+string(filepath.Separator)) { + // If this file is part of the synthetic GOROOT, try to infer the + // original path. + relpath := path[len(p.Build.GOROOT)+1:] + tinygoPath := filepath.Join(p.TINYGOROOT, relpath) + if _, err := os.Stat(tinygoPath); err == nil { + diagnosticPath = tinygoPath + } + realgorootPath := filepath.Join(goenv.Get("GOROOT"), relpath) + if _, err := os.Stat(realgorootPath); err == nil { + diagnosticPath = realgorootPath } } - return parser.ParseFile(p.fset, relpath, rd, mode) + return parser.ParseFile(p.fset, diagnosticPath, rd, mode) } // Parse parses and typechecks this package. // // Idempotent. -func (p *Package) Parse(includeTests bool) error { +func (p *Package) Parse() error { if len(p.Files) != 0 { return nil } // Load the AST. // TODO: do this in parallel. - if p.ImportPath == "unsafe" { + if p.PkgPath == "unsafe" { // Special case for the unsafe package. Don't even bother loading // the files. p.Pkg = types.Unsafe return nil } - files, err := p.parseFiles(includeTests) + files, err := p.parseFiles() if err != nil { return err } @@ -367,7 +343,7 @@ func (p *Package) Check() error { // Do typechecking of the package. checker.Importer = p - typesPkg, err := checker.Check(p.ImportPath, p.fset, p.Files, &p.Info) + typesPkg, err := checker.Check(p.PkgPath, p.fset, p.Files, &p.Info) if err != nil { if err, ok := err.(Errors); ok { return err @@ -379,22 +355,14 @@ func (p *Package) Check() error { } // parseFiles parses the loaded list of files and returns this list. -func (p *Package) parseFiles(includeTests bool) ([]*ast.File, error) { +func (p *Package) parseFiles() ([]*ast.File, error) { // TODO: do this concurrently. var files []*ast.File var fileErrs []error - var gofiles []string - if includeTests { - gofiles = make([]string, 0, len(p.GoFiles)+len(p.TestGoFiles)) - gofiles = append(gofiles, p.GoFiles...) - gofiles = append(gofiles, p.TestGoFiles...) - } else { - gofiles = p.GoFiles - } - - for _, file := range gofiles { - f, err := p.parseFile(filepath.Join(p.Package.Dir, file), parser.ParseComments) + var cgoFiles []*ast.File + for _, file := range p.GoFiles { + f, err := p.parseFile(file, parser.ParseComments) if err != nil { fileErrs = append(fileErrs, err) continue @@ -403,19 +371,15 @@ func (p *Package) parseFiles(includeTests bool) ([]*ast.File, error) { fileErrs = append(fileErrs, err) continue } + for _, importSpec := range f.Imports { + if importSpec.Path.Value == `"C"` { + cgoFiles = append(cgoFiles, f) + } + } files = append(files, f) } - for _, file := range p.CgoFiles { - path := filepath.Join(p.Package.Dir, file) - f, err := p.parseFile(path, parser.ParseComments) - if err != nil { - fileErrs = append(fileErrs, err) - continue - } - files = append(files, f) - } - if len(p.CgoFiles) != 0 { - cflags := append(p.CFlags, "-I"+p.Package.Dir) + if len(cgoFiles) != 0 { + cflags := append(p.CFlags, "-I"+filepath.Dir(p.GoFiles[0])) if p.ClangHeaders != "" { cflags = append(cflags, "-Xclang", "-internal-isystem", "-Xclang", p.ClangHeaders) } @@ -439,58 +403,8 @@ func (p *Package) Import(to string) (*types.Package, error) { return types.Unsafe, nil } if _, ok := p.Imports[to]; ok { - return p.Imports[to].Pkg, nil + return p.Packages[p.Imports[to].PkgPath].Pkg, nil } else { return nil, errors.New("package not imported: " + to) } } - -// importRecursively calls Program.Import() on all imported packages, and calls -// importRecursively() on the imported packages as well. -// -// Idempotent. -func (p *Package) importRecursively(includeTests bool) error { - p.Importing = true - - imports := p.Package.Imports - if includeTests { - imports = append(imports, p.Package.TestImports...) - } - - for _, to := range imports { - if to == "C" { - // Do CGo processing in a later stage. - continue - } - if _, ok := p.Imports[to]; ok { - continue - } - // Find error location. - var pos token.Position - if len(p.Package.ImportPos[to]) > 0 { - pos = p.Package.ImportPos[to][0] - } else { - pos = token.Position{Filename: p.Package.ImportPath} - } - importedPkg, err := p.Program.Import(to, p.Package.Dir, pos) - if err != nil { - if err, ok := err.(*ImportCycleError); ok { - err.Packages = append([]string{p.ImportPath}, err.Packages...) - } - return err - } - if importedPkg.Importing { - return &ImportCycleError{[]string{p.ImportPath, importedPkg.ImportPath}, p.ImportPos[to]} - } - err = importedPkg.importRecursively(false) - if err != nil { - if err, ok := err.(*ImportCycleError); ok { - err.Packages = append([]string{p.ImportPath}, err.Packages...) - } - return err - } - p.Imports[to] = importedPkg - } - p.Importing = false - return nil -} diff --git a/main.go b/main.go index 540c6beae..6a3b2830c 100644 --- a/main.go +++ b/main.go @@ -714,7 +714,7 @@ func printCompilerError(logln func(...interface{}), err error) { } } case loader.Errors: - logln("#", err.Pkg.ImportPath) + logln("#", err.Pkg.PkgPath) for _, err := range err.Errs { logln(err) }