diff --git a/compiler/compiler.go b/compiler/compiler.go index 5ed85c053..d4dedf7f7 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -137,64 +137,25 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con c.funcPtrAddrSpace = dummyFunc.Type().PointerAddressSpace() dummyFunc.EraseFromParentAsFunction() - // Prefix the GOPATH with the system GOROOT, as GOROOT is already set to - // the TinyGo root. - overlayGopath := goenv.Get("GOPATH") - if overlayGopath == "" { - overlayGopath = goenv.Get("GOROOT") - } else { - overlayGopath = goenv.Get("GOROOT") + string(filepath.ListSeparator) + overlayGopath - } - wd, err := os.Getwd() if err != nil { return c.mod, nil, nil, []error{err} } + goroot, err := loader.GetCachedGoroot(c.Config) + if err != nil { + return c.mod, nil, []error{err} + } lprogram := &loader.Program{ Build: &build.Context{ GOARCH: c.GOARCH(), GOOS: c.GOOS(), - GOROOT: goenv.Get("GOROOT"), + GOROOT: goroot, GOPATH: goenv.Get("GOPATH"), CgoEnabled: c.CgoEnabled(), UseAllFiles: false, Compiler: "gc", // must be one of the recognized compilers BuildTags: c.BuildTags(), }, - OverlayBuild: &build.Context{ - GOARCH: c.GOARCH(), - GOOS: c.GOOS(), - GOROOT: goenv.Get("TINYGOROOT"), - GOPATH: overlayGopath, - CgoEnabled: c.CgoEnabled(), - UseAllFiles: false, - Compiler: "gc", // must be one of the recognized compilers - BuildTags: c.BuildTags(), - }, - OverlayPath: func(path string) string { - // Return the (overlay) import path when it should be overlaid, and - // "" if it should not. - if strings.HasPrefix(path, tinygoPath+"/src/") { - // Avoid issues with packages that are imported twice, one from - // GOPATH and one from TINYGOPATH. - path = path[len(tinygoPath+"/src/"):] - } - switch path { - case "machine", "os", "reflect", "runtime", "runtime/interrupt", "runtime/volatile", "sync", "testing", "internal/reflectlite", "internal/bytealg", "internal/task": - return path - default: - if strings.HasPrefix(path, "device/") || strings.HasPrefix(path, "examples/") { - return path - } else if path == "syscall" { - for _, tag := range c.BuildTags() { - if tag == "baremetal" || tag == "darwin" { - return path - } - } - } - } - return "" - }, TypeChecker: types.Config{ Sizes: &stdSizes{ IntSize: int64(c.targetData.TypeAllocSize(c.intType)), diff --git a/loader/goroot.go b/loader/goroot.go new file mode 100644 index 000000000..7947d15e2 --- /dev/null +++ b/loader/goroot.go @@ -0,0 +1,240 @@ +package loader + +// This file constructs a new temporary GOROOT directory by merging both the +// standard Go GOROOT and the GOROOT from TinyGo using symlinks. + +import ( + "crypto/sha512" + "encoding/hex" + "errors" + "fmt" + "io/ioutil" + "math/rand" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strconv" + + "github.com/tinygo-org/tinygo/compileopts" + "github.com/tinygo-org/tinygo/goenv" +) + +// GetCachedGoroot creates a new GOROOT by merging both the standard GOROOT and +// the GOROOT from TinyGo using lots of symbolic links. +func GetCachedGoroot(config *compileopts.Config) (string, error) { + goroot := goenv.Get("GOROOT") + if goroot == "" { + return "", errors.New("could not determine GOROOT") + } + tinygoroot := goenv.Get("TINYGOROOT") + if tinygoroot == "" { + return "", errors.New("could not determine TINYGOROOT") + } + + // Determine the location of the cached GOROOT. + version, err := goenv.GorootVersionString(goroot) + if err != nil { + return "", err + } + // This hash is really a cache key, that contains (hopefully) enough + // information to make collisions unlikely during development. + // By including the Go version and TinyGo version, cache collisions should + // not happen outside of development. + hash := sha512.New512_256() + fmt.Fprintln(hash, goroot) + fmt.Fprintln(hash, version) + fmt.Fprintln(hash, goenv.Version) + fmt.Fprintln(hash, tinygoroot) + gorootsHash := hash.Sum(nil) + gorootsHashHex := hex.EncodeToString(gorootsHash[:]) + cachedgoroot := filepath.Join(goenv.Get("GOCACHE"), "goroot-"+version+"-"+gorootsHashHex) + if needsSyscallPackage(config.BuildTags()) { + cachedgoroot += "-syscall" + } + + if _, err := os.Stat(cachedgoroot); err == nil { + return cachedgoroot, nil + } + tmpgoroot := cachedgoroot + ".tmp" + strconv.Itoa(rand.Int()) + err = os.MkdirAll(tmpgoroot, 0777) + if err != nil { + return "", err + } + + // Remove the temporary directory if it wasn't moved to the right place + // (for example, when there was an error). + defer os.RemoveAll(tmpgoroot) + + for _, name := range []string{"bin", "lib", "pkg"} { + err = symlink(filepath.Join(goroot, name), filepath.Join(tmpgoroot, name)) + if err != nil { + return "", err + } + } + err = mergeDirectory(goroot, tinygoroot, tmpgoroot, "", pathsToOverride(needsSyscallPackage(config.BuildTags()))) + if err != nil { + return "", err + } + err = os.Rename(tmpgoroot, cachedgoroot) + if err != nil { + if os.IsExist(err) { + // Another invocation of TinyGo also seems to have created a GOROOT. + // Use that one instead. Our new GOROOT will be automatically + // deleted by the defer above. + return cachedgoroot, nil + } + return "", err + } + return cachedgoroot, nil +} + +// mergeDirectory merges two roots recursively. The tmpgoroot is the directory +// that will be created by this call by either symlinking the directory from +// goroot or tinygoroot, or by creating the directory and merging the contents. +func mergeDirectory(goroot, tinygoroot, tmpgoroot, importPath string, overrides map[string]bool) error { + if mergeSubdirs, ok := overrides[importPath+"/"]; ok { + if !mergeSubdirs { + // This directory and all subdirectories should come from the TinyGo + // root, so simply make a symlink. + newname := filepath.Join(tmpgoroot, "src", importPath) + oldname := filepath.Join(tinygoroot, "src", importPath) + return symlink(oldname, newname) + } + + // Merge subdirectories. Start by making the directory to merge. + err := os.Mkdir(filepath.Join(tmpgoroot, "src", importPath), 0777) + if err != nil { + return err + } + + // Symlink all files from TinyGo, and symlink directories from TinyGo + // that need to be overridden. + tinygoEntries, err := ioutil.ReadDir(filepath.Join(tinygoroot, "src", importPath)) + if err != nil { + return err + } + for _, e := range tinygoEntries { + if e.IsDir() { + // A directory, so merge this thing. + err := mergeDirectory(goroot, tinygoroot, tmpgoroot, path.Join(importPath, e.Name()), overrides) + if err != nil { + return err + } + } else { + // A file, so symlink this. + newname := filepath.Join(tmpgoroot, "src", importPath, e.Name()) + oldname := filepath.Join(tinygoroot, "src", importPath, e.Name()) + err := symlink(oldname, newname) + if err != nil { + return err + } + } + } + + // Symlink all directories from $GOROOT that are not part of the TinyGo + // overrides. + gorootEntries, err := ioutil.ReadDir(filepath.Join(goroot, "src", importPath)) + if err != nil { + return err + } + for _, e := range gorootEntries { + if !e.IsDir() { + // Don't merge in files from Go. Otherwise we'd end up with a + // weird syscall package with files from both roots. + continue + } + if _, ok := overrides[path.Join(importPath, e.Name())+"/"]; ok { + // Already included above, so don't bother trying to create this + // symlink. + continue + } + newname := filepath.Join(tmpgoroot, "src", importPath, e.Name()) + oldname := filepath.Join(goroot, "src", importPath, e.Name()) + err := symlink(oldname, newname) + if err != nil { + return err + } + } + } + return nil +} + +// needsSyscallPackage returns whether the syscall package should be overriden +// with the TinyGo version. This is the case on some targets. +func needsSyscallPackage(buildTags []string) bool { + for _, tag := range buildTags { + if tag == "baremetal" || tag == "darwin" { + return true + } + } + return false +} + +// The boolean indicates whether to merge the subdirs. True means merge, false +// means use the TinyGo version. +func pathsToOverride(needsSyscallPackage bool) map[string]bool { + paths := map[string]bool{ + "/": true, + "device/": false, + "examples/": false, + "internal/": true, + "internal/bytealg/": false, + "internal/reflectlite/": false, + "internal/task/": false, + "machine/": false, + "os/": true, + "reflect/": false, + "runtime/": false, + "sync/": true, + "testing/": false, + } + if needsSyscallPackage { + paths["syscall/"] = true // include syscall/js + } + return paths +} + +// symlink creates a symlink or something similar. On Unix-like systems, it +// always creates a symlink. On Windows, it tries to create a symlink and if +// that fails, creates a hardlink or directory junction instead. +// +// Note that while Windows 10 does support symlinks and allows them to be +// created using os.Symlink, it requires developer mode to be enabled. +// Therefore provide a fallback for when symlinking is not possible. +// Unfortunately this fallback only works when TinyGo is installed on the same +// filesystem as the TinyGo cache and the Go installation (which is usually the +// C drive). +func symlink(oldname, newname string) error { + symlinkErr := os.Symlink(oldname, newname) + if runtime.GOOS == "windows" && symlinkErr != nil { + // Fallback for when developer mode is disabled. + // Note that we return the symlink error even if something else fails + // later on. This is because symlinks are the easiest to support + // (they're also used on Linux and MacOS) and enabling them is easy: + // just enable developer mode. + st, err := os.Stat(oldname) + if err != nil { + return symlinkErr + } + if st.IsDir() { + // Make a directory junction. There may be a way to do this + // programmatically, but it involves a lot of magic. Use the mklink + // command built into cmd instead (mklink is a builtin, not an + // external command). + err := exec.Command("cmd", "/k", "mklink", "/J", newname, oldname).Run() + if err != nil { + return symlinkErr + } + } else { + // Make a hard link. + err := os.Link(oldname, newname) + if err != nil { + return symlinkErr + } + } + return nil // success + } + return symlinkErr +} diff --git a/loader/loader.go b/loader/loader.go index 28ade8d2e..e5de53c1f 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -16,14 +16,13 @@ import ( "text/template" "github.com/tinygo-org/tinygo/cgo" + "github.com/tinygo-org/tinygo/goenv" ) // Program holds all packages and some metadata about the program as a whole. type Program struct { mainPkg string Build *build.Context - OverlayBuild *build.Context - OverlayPath func(path string) string Packages map[string]*Package sorted []*Package fset *token.FileSet @@ -55,10 +54,6 @@ func (p *Program) Import(path, srcDir string, pos token.Position) (*Package, err // Load this package. ctx := p.Build - if newPath := p.OverlayPath(path); newPath != "" { - ctx = p.OverlayBuild - path = newPath - } buildPkg, err := ctx.Import(path, srcDir, build.ImportComment) if err != nil { return nil, scanner.Error{ @@ -320,14 +315,30 @@ func (p *Program) parseFile(path string, mode parser.Mode) (*ast.File, error) { return nil, err } defer rd.Close() - relpath := path - if filepath.IsAbs(path) { - rp, err := filepath.Rel(p.Dir, path) - if err == nil { - relpath = rp + diagnosticPath := path + if strings.HasPrefix(path, p.Build.GOROOT+string(filepath.Separator)) { + // If this file is part of the synthetic GOROOT, try to infer the + // original path. + relpath := path[len(filepath.Join(p.Build.GOROOT, "src"))+1:] + realgorootPath := filepath.Join(goenv.Get("GOROOT"), "src", relpath) + if _, err := os.Stat(realgorootPath); err == nil { + diagnosticPath = realgorootPath + } + maybeInTinyGoRoot := false + for prefix := range pathsToOverride(needsSyscallPackage(p.Build.BuildTags)) { + if !strings.HasPrefix(relpath, prefix) { + continue + } + maybeInTinyGoRoot = true + } + if maybeInTinyGoRoot { + tinygoPath := filepath.Join(p.TINYGOROOT, "src", relpath) + if _, err := os.Stat(tinygoPath); err == nil { + diagnosticPath = tinygoPath + } } } - return parser.ParseFile(p.fset, relpath, rd, mode) + return parser.ParseFile(p.fset, diagnosticPath, rd, mode) } // Parse parses and typechecks this package. diff --git a/main.go b/main.go index 64f7036c3..0d711ab35 100644 --- a/main.go +++ b/main.go @@ -630,6 +630,36 @@ func getDefaultPort() (port string, err error) { return d[0], nil } +// runGoList runs the `go list` command but using the configuration used for +// TinyGo. +func runGoList(config *compileopts.Config, flagJSON, flagDeps bool, pkgs []string) error { + goroot, err := loader.GetCachedGoroot(config) + if err != nil { + return err + } + args := []string{"list"} + if flagJSON { + args = append(args, "-json") + } + if flagDeps { + args = append(args, "-deps") + } + if len(config.BuildTags()) != 0 { + args = append(args, "-tags", strings.Join(config.BuildTags(), " ")) + } + args = append(args, pkgs...) + cgoEnabled := "0" + if config.CgoEnabled() { + cgoEnabled = "1" + } + cmd := exec.Command("go", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), "GOROOT="+goroot, "GOOS="+config.GOOS(), "GOARCH="+config.GOARCH(), "CGO_ENABLED="+cgoEnabled) + cmd.Run() + return nil +} + func usage() { fmt.Fprintln(os.Stderr, "TinyGo is a Go compiler for small places.") fmt.Fprintln(os.Stderr, "version:", goenv.Version) @@ -641,12 +671,27 @@ func usage() { fmt.Fprintln(os.Stderr, " flash: compile and flash to the device") fmt.Fprintln(os.Stderr, " gdb: run/flash and immediately enter GDB") fmt.Fprintln(os.Stderr, " env: list environment variables used during build") + fmt.Fprintln(os.Stderr, " list: run go list using the TinyGo root") fmt.Fprintln(os.Stderr, " clean: empty cache directory ("+goenv.Get("GOCACHE")+")") fmt.Fprintln(os.Stderr, " help: print this help text") fmt.Fprintln(os.Stderr, "\nflags:") flag.PrintDefaults() } +// try to make the path relative to the current working directory. If any error +// occurs, this error is ignored and the absolute path is returned instead. +func tryToMakePathRelative(dir string) string { + wd, err := os.Getwd() + if err != nil { + return dir + } + relpath, err := filepath.Rel(wd, dir) + if err != nil { + return dir + } + return relpath +} + // printCompilerError prints compiler errors using the provided logger function // (similar to fmt.Println). // @@ -654,8 +699,24 @@ func usage() { // to limitations in the LLVM bindings. func printCompilerError(logln func(...interface{}), err error) { switch err := err.(type) { - case types.Error, scanner.Error: + case types.Error: + printCompilerError(logln, scanner.Error{ + Pos: err.Fset.Position(err.Pos), + Msg: err.Msg, + }) + case scanner.Error: + if !strings.HasPrefix(err.Pos.Filename, filepath.Join(goenv.Get("GOROOT"), "src")) && !strings.HasPrefix(err.Pos.Filename, filepath.Join(goenv.Get("TINYGOROOT"), "src")) { + // This file is not from the standard library (either the GOROOT or + // the TINYGOROOT). Make the path relative, for easier reading. + // Ignore any errors in the process (falling back to the absolute + // path). + err.Pos.Filename = tryToMakePathRelative(err.Pos.Filename) + } logln(err) + case scanner.ErrorList: + for _, scannerErr := range err { + printCompilerError(logln, *scannerErr) + } case *interp.Error: logln("#", err.ImportPath) logln(err.Error()) @@ -674,11 +735,11 @@ func printCompilerError(logln func(...interface{}), err error) { case loader.Errors: logln("#", err.Pkg.ImportPath) for _, err := range err.Errs { - logln(err) + printCompilerError(logln, err) } case *builder.MultiError: for _, err := range err.Errs { - logln(err) + printCompilerError(logln, err) } default: logln("error:", err) @@ -695,6 +756,13 @@ func handleCompilerError(err error) { } func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "No command-line arguments supplied.") + usage() + os.Exit(1) + } + command := os.Args[1] + outpath := flag.String("o", "", "output filename") opt := flag.String("opt", "z", "optimization level: 0, 1, 2, s, z") gc := flag.String("gc", "", "garbage collector to use (none, leaking, extalloc, conservative)") @@ -715,12 +783,11 @@ func main() { wasmAbi := flag.String("wasm-abi", "js", "WebAssembly ABI conventions: js (no i64 params) or generic") heapSize := flag.String("heap-size", "1M", "default heap size in bytes (only supported by WebAssembly)") - if len(os.Args) < 2 { - fmt.Fprintln(os.Stderr, "No command-line arguments supplied.") - usage() - os.Exit(1) + var flagJSON, flagDeps *bool + if command == "list" { + flagJSON = flag.Bool("json", false, "print data in JSON format") + flagDeps = flag.Bool("deps", false, "") } - command := os.Args[1] // Early command processing, before commands are interpreted by the Go flag // library. @@ -886,6 +953,18 @@ func main() { fmt.Printf("build tags: %s\n", strings.Join(config.BuildTags(), " ")) fmt.Printf("garbage collector: %s\n", config.GC()) fmt.Printf("scheduler: %s\n", config.Scheduler()) + case "list": + config, err := builder.NewConfig(options) + if err != nil { + fmt.Fprintln(os.Stderr, err) + usage() + os.Exit(1) + } + err = runGoList(config, *flagJSON, *flagDeps, flag.Args()) + if err != nil { + fmt.Fprintln(os.Stderr, "failed to run `go list`:", err) + os.Exit(1) + } case "clean": // remove cache directory err := os.RemoveAll(goenv.Get("GOCACHE"))