From f70dc70247e0632a477fffda2a473f6aec51e649 Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Thu, 5 Mar 2020 16:23:23 +0100 Subject: [PATCH] loader: merge roots from both Go and TinyGo in a cached directory This commit changes the way that packages are looked up. Instead of working around the loader package by modifying the GOROOT variable for specific packages, create a new GOROOT using symlinks. This GOROOT is cached for the specified configuration (Go version, underlying GOROOT path, TinyGo path, whether to override the syscall package). This will also enable go module support in the future. --- compiler/compiler.go | 49 ++---------- loader/goroot.go | 176 +++++++++++++++++++++++++++++++++++++++++++ loader/loader.go | 6 -- main.go | 59 +++++++++++++-- 4 files changed, 235 insertions(+), 55 deletions(-) create mode 100644 loader/goroot.go diff --git a/compiler/compiler.go b/compiler/compiler.go index e4cd8d4bd..673fe8931 100644 --- a/compiler/compiler.go +++ b/compiler/compiler.go @@ -137,64 +137,25 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con c.funcPtrAddrSpace = dummyFunc.Type().PointerAddressSpace() dummyFunc.EraseFromParentAsFunction() - // Prefix the GOPATH with the system GOROOT, as GOROOT is already set to - // the TinyGo root. - overlayGopath := goenv.Get("GOPATH") - if overlayGopath == "" { - overlayGopath = goenv.Get("GOROOT") - } else { - overlayGopath = goenv.Get("GOROOT") + string(filepath.ListSeparator) + overlayGopath - } - wd, err := os.Getwd() if err != nil { return c.mod, nil, []error{err} } + goroot, err := loader.GetCachedGoroot(c.Config) + if err != nil { + return c.mod, nil, []error{err} + } lprogram := &loader.Program{ Build: &build.Context{ GOARCH: c.GOARCH(), GOOS: c.GOOS(), - GOROOT: goenv.Get("GOROOT"), + GOROOT: goroot, GOPATH: goenv.Get("GOPATH"), CgoEnabled: c.CgoEnabled(), UseAllFiles: false, Compiler: "gc", // must be one of the recognized compilers BuildTags: c.BuildTags(), }, - OverlayBuild: &build.Context{ - GOARCH: c.GOARCH(), - GOOS: c.GOOS(), - GOROOT: goenv.Get("TINYGOROOT"), - GOPATH: overlayGopath, - CgoEnabled: c.CgoEnabled(), - UseAllFiles: false, - Compiler: "gc", // must be one of the recognized compilers - BuildTags: c.BuildTags(), - }, - OverlayPath: func(path string) string { - // Return the (overlay) import path when it should be overlaid, and - // "" if it should not. - if strings.HasPrefix(path, tinygoPath+"/src/") { - // Avoid issues with packages that are imported twice, one from - // GOPATH and one from TINYGOPATH. - path = path[len(tinygoPath+"/src/"):] - } - switch path { - case "machine", "os", "reflect", "runtime", "runtime/interrupt", "runtime/volatile", "sync", "testing", "internal/reflectlite", "internal/task": - return path - default: - if strings.HasPrefix(path, "device/") || strings.HasPrefix(path, "examples/") { - return path - } else if path == "syscall" { - for _, tag := range c.BuildTags() { - if tag == "baremetal" || tag == "darwin" { - return path - } - } - } - } - return "" - }, TypeChecker: types.Config{ Sizes: &stdSizes{ IntSize: int64(c.targetData.TypeAllocSize(c.intType)), diff --git a/loader/goroot.go b/loader/goroot.go new file mode 100644 index 000000000..e76f598a1 --- /dev/null +++ b/loader/goroot.go @@ -0,0 +1,176 @@ +package loader + +// This file constructs a new temporary GOROOT directory by merging both the +// standard Go GOROOT and the GOROOT from TinyGo using symlinks. + +import ( + "crypto/sha512" + "encoding/hex" + "errors" + "io/ioutil" + "math/rand" + "os" + "path" + "path/filepath" + "strconv" + + "github.com/tinygo-org/tinygo/compileopts" + "github.com/tinygo-org/tinygo/goenv" +) + +// GetCachedGoroot creates a new GOROOT by merging both the standard GOROOT and +// the GOROOT from TinyGo using lots of symbolic links. +func GetCachedGoroot(config *compileopts.Config) (string, error) { + goroot := goenv.Get("GOROOT") + if goroot == "" { + return "", errors.New("could not determine GOROOT") + } + tinygoroot := goenv.Get("TINYGOROOT") + if tinygoroot == "" { + return "", errors.New("could not determine TINYGOROOT") + } + + needsSyscallPackage := false + for _, tag := range config.BuildTags() { + if tag == "baremetal" || tag == "darwin" { + needsSyscallPackage = true + } + } + + // Determine the location of the cached GOROOT. + version, err := goenv.GorootVersionString(goroot) + if err != nil { + return "", err + } + gorootsHash := sha512.Sum512_256([]byte(goroot + "\x00" + tinygoroot)) + gorootsHashHex := hex.EncodeToString(gorootsHash[:]) + cachedgoroot := filepath.Join(goenv.Get("GOCACHE"), "goroot-"+version+"-"+gorootsHashHex) + if needsSyscallPackage { + cachedgoroot += "-syscall" + } + + if _, err := os.Stat(cachedgoroot); err == nil { + return cachedgoroot, nil + } + tmpgoroot := cachedgoroot + ".tmp" + strconv.Itoa(rand.Int()) + err = os.MkdirAll(tmpgoroot, 0777) + if err != nil { + return "", err + } + + for _, name := range []string{"bin", "lib", "pkg"} { + err = os.Symlink(filepath.Join(goroot, name), filepath.Join(tmpgoroot, name)) + if err != nil { + return "", err + } + } + err = mergeDirectory(goroot, tinygoroot, tmpgoroot, "", pathsToOverride(needsSyscallPackage)) + if err != nil { + return "", err + } + err = os.Rename(tmpgoroot, cachedgoroot) + if err != nil { + if os.IsExist(err) { + // Another invocation of TinyGo also seems to have created a GOROOT. + // Use that one instead and delete ours. + os.RemoveAll(tmpgoroot) + return cachedgoroot, nil + } + return "", err + } + return cachedgoroot, nil +} + +// mergeDirectory merges two roots recursively. The tmpgoroot is the directory +// that will be created by this call by either symlinking the directory from +// goroot or tinygoroot, or by creating the directory and merging the contents. +func mergeDirectory(goroot, tinygoroot, tmpgoroot, importPath string, overrides map[string]bool) error { + if mergeSubdirs, ok := overrides[importPath+"/"]; ok { + if !mergeSubdirs { + // This directory and all subdirectories should come from the TinyGo + // root, so simply make a symlink. + newname := filepath.Join(tmpgoroot, "src", importPath) + oldname := filepath.Join(tinygoroot, "src", importPath) + return os.Symlink(oldname, newname) + } + + // Merge subdirectories. Start by making the directory to merge. + err := os.Mkdir(filepath.Join(tmpgoroot, "src", importPath), 0777) + if err != nil { + return err + } + + // Symlink all files from TinyGo, and symlink directories from TinyGo + // that need to be overridden. + tinygoEntries, err := ioutil.ReadDir(filepath.Join(tinygoroot, "src", importPath)) + if err != nil { + return err + } + for _, e := range tinygoEntries { + if e.IsDir() { + // A directory, so merge this thing. + err := mergeDirectory(goroot, tinygoroot, tmpgoroot, path.Join(importPath, e.Name()), overrides) + if err != nil { + return err + } + } else { + // A file, so symlink this. + newname := filepath.Join(tmpgoroot, "src", importPath, e.Name()) + oldname := filepath.Join(tinygoroot, "src", importPath, e.Name()) + err := os.Symlink(oldname, newname) + if err != nil { + return err + } + } + } + + // Symlink all directories from $GOROOT that are not part of the TinyGo + // overrides. + gorootEntries, err := ioutil.ReadDir(filepath.Join(goroot, "src", importPath)) + if err != nil { + return err + } + for _, e := range gorootEntries { + if !e.IsDir() { + // Don't merge in files from Go. Otherwise we'd end up with a + // weird syscall package with files from both roots. + continue + } + if _, ok := overrides[path.Join(importPath, e.Name())+"/"]; ok { + // Already included above, so don't bother trying to create this + // symlink. + continue + } + newname := filepath.Join(tmpgoroot, "src", importPath, e.Name()) + oldname := filepath.Join(goroot, "src", importPath, e.Name()) + err := os.Symlink(oldname, newname) + if err != nil { + return err + } + } + } + return nil +} + +// The boolean indicates whether to merge the subdirs. True means merge, false +// means use the TinyGo version. +func pathsToOverride(needsSyscallPackage bool) map[string]bool { + paths := map[string]bool{ + "/": true, + "device/": false, + "examples/": false, + "internal/": true, + "internal/reflectlite/": false, + "internal/task/": false, + "machine/": false, + "os/": true, + "reflect/": false, + "runtime/": false, + "sync/": true, + "testing/": false, + } + if needsSyscallPackage { + paths["syscall/"] = true // include syscall/js + } + return paths +} diff --git a/loader/loader.go b/loader/loader.go index 5682bfa4a..545c1a05c 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -22,8 +22,6 @@ import ( type Program struct { mainPkg string Build *build.Context - OverlayBuild *build.Context - OverlayPath func(path string) string Packages map[string]*Package sorted []*Package fset *token.FileSet @@ -54,10 +52,6 @@ func (p *Program) Import(path, srcDir string, pos token.Position) (*Package, err // Load this package. ctx := p.Build - if newPath := p.OverlayPath(path); newPath != "" { - ctx = p.OverlayBuild - path = newPath - } buildPkg, err := ctx.Import(path, srcDir, build.ImportComment) if err != nil { return nil, scanner.Error{ diff --git a/main.go b/main.go index 64aa62b59..2dbb2a583 100644 --- a/main.go +++ b/main.go @@ -641,6 +641,36 @@ func getDefaultPort() (port string, err error) { return d[0], nil } +// runGoList runs the `go list` command but using the configuration used for +// TinyGo. +func runGoList(config *compileopts.Config, flagJSON, flagDeps bool, pkgs []string) error { + goroot, err := loader.GetCachedGoroot(config) + if err != nil { + return err + } + args := []string{"list"} + if flagJSON { + args = append(args, "-json") + } + if flagDeps { + args = append(args, "-deps") + } + if len(config.BuildTags()) != 0 { + args = append(args, "-tags", strings.Join(config.BuildTags(), " ")) + } + args = append(args, pkgs...) + cgoEnabled := "0" + if config.CgoEnabled() { + cgoEnabled = "1" + } + cmd := exec.Command("go", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Env = append(os.Environ(), "GOROOT="+goroot, "GOOS="+config.GOOS(), "GOARCH="+config.GOARCH(), "CGO_ENABLED="+cgoEnabled) + cmd.Run() + return nil +} + func usage() { fmt.Fprintln(os.Stderr, "TinyGo is a Go compiler for small places.") fmt.Fprintln(os.Stderr, "version:", version) @@ -652,6 +682,7 @@ func usage() { fmt.Fprintln(os.Stderr, " flash: compile and flash to the device") fmt.Fprintln(os.Stderr, " gdb: run/flash and immediately enter GDB") fmt.Fprintln(os.Stderr, " env: list environment variables used during build") + fmt.Fprintln(os.Stderr, " list: run go list using the TinyGo root") fmt.Fprintln(os.Stderr, " clean: empty cache directory ("+goenv.Get("GOCACHE")+")") fmt.Fprintln(os.Stderr, " help: print this help text") fmt.Fprintln(os.Stderr, "\nflags:") @@ -706,6 +737,13 @@ func handleCompilerError(err error) { } func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "No command-line arguments supplied.") + usage() + os.Exit(1) + } + command := os.Args[1] + outpath := flag.String("o", "", "output filename") opt := flag.String("opt", "z", "optimization level: 0, 1, 2, s, z") gc := flag.String("gc", "", "garbage collector to use (none, leaking, extalloc, conservative)") @@ -726,12 +764,11 @@ func main() { wasmAbi := flag.String("wasm-abi", "js", "WebAssembly ABI conventions: js (no i64 params) or generic") heapSize := flag.String("heap-size", "1M", "default heap size in bytes (only supported by WebAssembly)") - if len(os.Args) < 2 { - fmt.Fprintln(os.Stderr, "No command-line arguments supplied.") - usage() - os.Exit(1) + var flagJSON, flagDeps *bool + if command == "list" { + flagJSON = flag.Bool("json", false, "print data in JSON format") + flagDeps = flag.Bool("deps", false, "") } - command := os.Args[1] // Early command processing, before commands are interpreted by the Go flag // library. @@ -895,6 +932,18 @@ func main() { fmt.Printf("build tags: %s\n", strings.Join(config.BuildTags(), " ")) fmt.Printf("garbage collector: %s\n", config.GC()) fmt.Printf("scheduler: %s\n", config.Scheduler()) + case "list": + config, err := builder.NewConfig(options) + if err != nil { + fmt.Fprintln(os.Stderr, err) + usage() + os.Exit(1) + } + err = runGoList(config, *flagJSON, *flagDeps, flag.Args()) + if err != nil { + fmt.Fprintln(os.Stderr, "failed to run `go list`:", err) + os.Exit(1) + } case "clean": // remove cache directory err := os.RemoveAll(goenv.Get("GOCACHE"))