loader: merge roots from both Go and TinyGo in a cached directory

This commit changes the way that packages are looked up. Instead of
working around the loader package by modifying the GOROOT variable for
specific packages, create a new GOROOT using symlinks. This GOROOT is
cached for the specified configuration (Go version, underlying GOROOT
path, TinyGo path, whether to override the syscall package).

This will also enable go module support in the future.

compiler/compiler.go

@@ -137,64 +137,25 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con
 	c.funcPtrAddrSpace = dummyFunc.Type().PointerAddressSpace()
 	dummyFunc.EraseFromParentAsFunction()
 
-	// Prefix the GOPATH with the system GOROOT, as GOROOT is already set to
-	// the TinyGo root.
-	overlayGopath := goenv.Get("GOPATH")
-	if overlayGopath == "" {
-		overlayGopath = goenv.Get("GOROOT")
-	} else {
-		overlayGopath = goenv.Get("GOROOT") + string(filepath.ListSeparator) + overlayGopath
-	}
-
 	wd, err := os.Getwd()
 	if err != nil {
 		return c.mod, nil, []error{err}
 	}
+	goroot, err := loader.GetCachedGoroot(c.Config)
+	if err != nil {
+		return c.mod, nil, []error{err}
+	}
 	lprogram := &loader.Program{
 		Build: &build.Context{
 			GOARCH:      c.GOARCH(),
 			GOOS:        c.GOOS(),
-			GOROOT:      goenv.Get("GOROOT"),
+			GOROOT:      goroot,
 			GOPATH:      goenv.Get("GOPATH"),
 			CgoEnabled:  c.CgoEnabled(),
 			UseAllFiles: false,
 			Compiler:    "gc", // must be one of the recognized compilers
 			BuildTags:   c.BuildTags(),
 		},
-		OverlayBuild: &build.Context{
-			GOARCH:      c.GOARCH(),
-			GOOS:        c.GOOS(),
-			GOROOT:      goenv.Get("TINYGOROOT"),
-			GOPATH:      overlayGopath,
-			CgoEnabled:  c.CgoEnabled(),
-			UseAllFiles: false,
-			Compiler:    "gc", // must be one of the recognized compilers
-			BuildTags:   c.BuildTags(),
-		},
-		OverlayPath: func(path string) string {
-			// Return the (overlay) import path when it should be overlaid, and
-			// "" if it should not.
-			if strings.HasPrefix(path, tinygoPath+"/src/") {
-				// Avoid issues with packages that are imported twice, one from
-				// GOPATH and one from TINYGOPATH.
-				path = path[len(tinygoPath+"/src/"):]
-			}
-			switch path {
-			case "machine", "os", "reflect", "runtime", "runtime/interrupt", "runtime/volatile", "sync", "testing", "internal/reflectlite", "internal/task":
-				return path
-			default:
-				if strings.HasPrefix(path, "device/") || strings.HasPrefix(path, "examples/") {
-					return path
-				} else if path == "syscall" {
-					for _, tag := range c.BuildTags() {
-						if tag == "baremetal" || tag == "darwin" {
-							return path
-						}
-					}
-				}
-			}
-			return ""
-		},
 		TypeChecker: types.Config{
 			Sizes: &stdSizes{
 				IntSize:  int64(c.targetData.TypeAllocSize(c.intType)),

loader/goroot.go

@@ -0,0 +1,176 @@
+package loader
+
+// This file constructs a new temporary GOROOT directory by merging both the
+// standard Go GOROOT and the GOROOT from TinyGo using symlinks.
+
+import (
+	"crypto/sha512"
+	"encoding/hex"
+	"errors"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+
+	"github.com/tinygo-org/tinygo/compileopts"
+	"github.com/tinygo-org/tinygo/goenv"
+)
+
+// GetCachedGoroot creates a new GOROOT by merging both the standard GOROOT and
+// the GOROOT from TinyGo using lots of symbolic links.
+func GetCachedGoroot(config *compileopts.Config) (string, error) {
+	goroot := goenv.Get("GOROOT")
+	if goroot == "" {
+		return "", errors.New("could not determine GOROOT")
+	}
+	tinygoroot := goenv.Get("TINYGOROOT")
+	if tinygoroot == "" {
+		return "", errors.New("could not determine TINYGOROOT")
+	}
+
+	needsSyscallPackage := false
+	for _, tag := range config.BuildTags() {
+		if tag == "baremetal" || tag == "darwin" {
+			needsSyscallPackage = true
+		}
+	}
+
+	// Determine the location of the cached GOROOT.
+	version, err := goenv.GorootVersionString(goroot)
+	if err != nil {
+		return "", err
+	}
+	gorootsHash := sha512.Sum512_256([]byte(goroot + "\x00" + tinygoroot))
+	gorootsHashHex := hex.EncodeToString(gorootsHash[:])
+	cachedgoroot := filepath.Join(goenv.Get("GOCACHE"), "goroot-"+version+"-"+gorootsHashHex)
+	if needsSyscallPackage {
+		cachedgoroot += "-syscall"
+	}
+
+	if _, err := os.Stat(cachedgoroot); err == nil {
+		return cachedgoroot, nil
+	}
+	tmpgoroot := cachedgoroot + ".tmp" + strconv.Itoa(rand.Int())
+	err = os.MkdirAll(tmpgoroot, 0777)
+	if err != nil {
+		return "", err
+	}
+
+	for _, name := range []string{"bin", "lib", "pkg"} {
+		err = os.Symlink(filepath.Join(goroot, name), filepath.Join(tmpgoroot, name))
+		if err != nil {
+			return "", err
+		}
+	}
+	err = mergeDirectory(goroot, tinygoroot, tmpgoroot, "", pathsToOverride(needsSyscallPackage))
+	if err != nil {
+		return "", err
+	}
+	err = os.Rename(tmpgoroot, cachedgoroot)
+	if err != nil {
+		if os.IsExist(err) {
+			// Another invocation of TinyGo also seems to have created a GOROOT.
+			// Use that one instead and delete ours.
+			os.RemoveAll(tmpgoroot)
+			return cachedgoroot, nil
+		}
+		return "", err
+	}
+	return cachedgoroot, nil
+}
+
+// mergeDirectory merges two roots recursively. The tmpgoroot is the directory
+// that will be created by this call by either symlinking the directory from
+// goroot or tinygoroot, or by creating the directory and merging the contents.
+func mergeDirectory(goroot, tinygoroot, tmpgoroot, importPath string, overrides map[string]bool) error {
+	if mergeSubdirs, ok := overrides[importPath+"/"]; ok {
+		if !mergeSubdirs {
+			// This directory and all subdirectories should come from the TinyGo
+			// root, so simply make a symlink.
+			newname := filepath.Join(tmpgoroot, "src", importPath)
+			oldname := filepath.Join(tinygoroot, "src", importPath)
+			return os.Symlink(oldname, newname)
+		}
+
+		// Merge subdirectories. Start by making the directory to merge.
+		err := os.Mkdir(filepath.Join(tmpgoroot, "src", importPath), 0777)
+		if err != nil {
+			return err
+		}
+
+		// Symlink all files from TinyGo, and symlink directories from TinyGo
+		// that need to be overridden.
+		tinygoEntries, err := ioutil.ReadDir(filepath.Join(tinygoroot, "src", importPath))
+		if err != nil {
+			return err
+		}
+		for _, e := range tinygoEntries {
+			if e.IsDir() {
+				// A directory, so merge this thing.
+				err := mergeDirectory(goroot, tinygoroot, tmpgoroot, path.Join(importPath, e.Name()), overrides)
+				if err != nil {
+					return err
+				}
+			} else {
+				// A file, so symlink this.
+				newname := filepath.Join(tmpgoroot, "src", importPath, e.Name())
+				oldname := filepath.Join(tinygoroot, "src", importPath, e.Name())
+				err := os.Symlink(oldname, newname)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// Symlink all directories from $GOROOT that are not part of the TinyGo
+		// overrides.
+		gorootEntries, err := ioutil.ReadDir(filepath.Join(goroot, "src", importPath))
+		if err != nil {
+			return err
+		}
+		for _, e := range gorootEntries {
+			if !e.IsDir() {
+				// Don't merge in files from Go. Otherwise we'd end up with a
+				// weird syscall package with files from both roots.
+				continue
+			}
+			if _, ok := overrides[path.Join(importPath, e.Name())+"/"]; ok {
+				// Already included above, so don't bother trying to create this
+				// symlink.
+				continue
+			}
+			newname := filepath.Join(tmpgoroot, "src", importPath, e.Name())
+			oldname := filepath.Join(goroot, "src", importPath, e.Name())
+			err := os.Symlink(oldname, newname)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// The boolean indicates whether to merge the subdirs. True means merge, false
+// means use the TinyGo version.
+func pathsToOverride(needsSyscallPackage bool) map[string]bool {
+	paths := map[string]bool{
+		"/":                     true,
+		"device/":               false,
+		"examples/":             false,
+		"internal/":             true,
+		"internal/reflectlite/": false,
+		"internal/task/":        false,
+		"machine/":              false,
+		"os/":                   true,
+		"reflect/":              false,
+		"runtime/":              false,
+		"sync/":                 true,
+		"testing/":              false,
+	}
+	if needsSyscallPackage {
+		paths["syscall/"] = true // include syscall/js
+	}
+	return paths
+}

loader/loader.go

@@ -22,8 +22,6 @@ import (
 type Program struct {
 	mainPkg      string
 	Build        *build.Context
-	OverlayBuild *build.Context
-	OverlayPath  func(path string) string
 	Packages     map[string]*Package
 	sorted       []*Package
 	fset         *token.FileSet
@@ -54,10 +52,6 @@ func (p *Program) Import(path, srcDir string, pos token.Position) (*Package, err
 
 	// Load this package.
 	ctx := p.Build
-	if newPath := p.OverlayPath(path); newPath != "" {
-		ctx = p.OverlayBuild
-		path = newPath
-	}
 	buildPkg, err := ctx.Import(path, srcDir, build.ImportComment)
 	if err != nil {
 		return nil, scanner.Error{

main.go

@@ -641,6 +641,36 @@ func getDefaultPort() (port string, err error) {
 	return d[0], nil
 }
 
+// runGoList runs the `go list` command but using the configuration used for
+// TinyGo.
+func runGoList(config *compileopts.Config, flagJSON, flagDeps bool, pkgs []string) error {
+	goroot, err := loader.GetCachedGoroot(config)
+	if err != nil {
+		return err
+	}
+	args := []string{"list"}
+	if flagJSON {
+		args = append(args, "-json")
+	}
+	if flagDeps {
+		args = append(args, "-deps")
+	}
+	if len(config.BuildTags()) != 0 {
+		args = append(args, "-tags", strings.Join(config.BuildTags(), " "))
+	}
+	args = append(args, pkgs...)
+	cgoEnabled := "0"
+	if config.CgoEnabled() {
+		cgoEnabled = "1"
+	}
+	cmd := exec.Command("go", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Env = append(os.Environ(), "GOROOT="+goroot, "GOOS="+config.GOOS(), "GOARCH="+config.GOARCH(), "CGO_ENABLED="+cgoEnabled)
+	cmd.Run()
+	return nil
+}
+
 func usage() {
 	fmt.Fprintln(os.Stderr, "TinyGo is a Go compiler for small places.")
 	fmt.Fprintln(os.Stderr, "version:", version)
@@ -652,6 +682,7 @@ func usage() {
 	fmt.Fprintln(os.Stderr, "  flash: compile and flash to the device")
 	fmt.Fprintln(os.Stderr, "  gdb:   run/flash and immediately enter GDB")
 	fmt.Fprintln(os.Stderr, "  env:   list environment variables used during build")
+	fmt.Fprintln(os.Stderr, "  list:  run go list using the TinyGo root")
 	fmt.Fprintln(os.Stderr, "  clean: empty cache directory ("+goenv.Get("GOCACHE")+")")
 	fmt.Fprintln(os.Stderr, "  help:  print this help text")
 	fmt.Fprintln(os.Stderr, "\nflags:")
@@ -706,6 +737,13 @@ func handleCompilerError(err error) {
 }
 
 func main() {
+	if len(os.Args) < 2 {
+		fmt.Fprintln(os.Stderr, "No command-line arguments supplied.")
+		usage()
+		os.Exit(1)
+	}
+	command := os.Args[1]
+
 	outpath := flag.String("o", "", "output filename")
 	opt := flag.String("opt", "z", "optimization level: 0, 1, 2, s, z")
 	gc := flag.String("gc", "", "garbage collector to use (none, leaking, extalloc, conservative)")
@@ -726,12 +764,11 @@ func main() {
 	wasmAbi := flag.String("wasm-abi", "js", "WebAssembly ABI conventions: js (no i64 params) or generic")
 	heapSize := flag.String("heap-size", "1M", "default heap size in bytes (only supported by WebAssembly)")
 
-	if len(os.Args) < 2 {
+	var flagJSON, flagDeps *bool
-		fmt.Fprintln(os.Stderr, "No command-line arguments supplied.")
+	if command == "list" {
-		usage()
+		flagJSON = flag.Bool("json", false, "print data in JSON format")
-		os.Exit(1)
+		flagDeps = flag.Bool("deps", false, "")
 	}
-	command := os.Args[1]
 
 	// Early command processing, before commands are interpreted by the Go flag
 	// library.
@@ -895,6 +932,18 @@ func main() {
 		fmt.Printf("build tags:        %s\n", strings.Join(config.BuildTags(), " "))
 		fmt.Printf("garbage collector: %s\n", config.GC())
 		fmt.Printf("scheduler:         %s\n", config.Scheduler())
+	case "list":
+		config, err := builder.NewConfig(options)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			usage()
+			os.Exit(1)
+		}
+		err = runGoList(config, *flagJSON, *flagDeps, flag.Args())
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "failed to run `go list`:", err)
+			os.Exit(1)
+		}
 	case "clean":
 		// remove cache directory
 		err := os.RemoveAll(goenv.Get("GOCACHE"))