main: add -json flag support to go env

This is needed for VS Code support.

builder/config.go

@@ -21,7 +21,7 @@ func NewConfig(options *compileopts.Options) (*compileopts.Config, error) {
 	if goroot == "" {
 		return nil, errors.New("cannot locate $GOROOT, please set it manually")
 	}
-	major, minor, err := getGorootVersion(goroot)
+	major, minor, err := goenv.GetGorootVersion(goroot)
 	if err != nil {
 		return nil, fmt.Errorf("could not read version from GOROOT (%v): %v", goroot, err)
 	}

builder/env.go

@@ -1,71 +1,13 @@
 package builder
 
 import (
-	"errors"
-	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"regexp"
 	"sort"
-	"strings"
 )
 
-// getGorootVersion returns the major and minor version for a given GOROOT path.
-// If the goroot cannot be determined, (0, 0) is returned.
-func getGorootVersion(goroot string) (major, minor int, err error) {
-	s, err := GorootVersionString(goroot)
-	if err != nil {
-		return 0, 0, err
-	}
-
-	if s == "" || s[:2] != "go" {
-		return 0, 0, errors.New("could not parse Go version: version does not start with 'go' prefix")
-	}
-
-	parts := strings.Split(s[2:], ".")
-	if len(parts) < 2 {
-		return 0, 0, errors.New("could not parse Go version: version has less than two parts")
-	}
-
-	// Ignore the errors, we don't really handle errors here anyway.
-	var trailing string
-	n, err := fmt.Sscanf(s, "go%d.%d%s", &major, &minor, &trailing)
-	if n == 2 && err == io.EOF {
-		// Means there were no trailing characters (i.e., not an alpha/beta)
-		err = nil
-	}
-	if err != nil {
-		return 0, 0, fmt.Errorf("failed to parse version: %s", err)
-	}
-	return
-}
-
-// GorootVersionString returns the version string as reported by the Go
-// toolchain for the given GOROOT path. It is usually of the form `go1.x.y` but
-// can have some variations (for beta releases, for example).
-func GorootVersionString(goroot string) (string, error) {
-	if data, err := ioutil.ReadFile(filepath.Join(
-		goroot, "src", "runtime", "internal", "sys", "zversion.go")); err == nil {
-
-		r := regexp.MustCompile("const TheVersion = `(.*)`")
-		matches := r.FindSubmatch(data)
-		if len(matches) != 2 {
-			return "", errors.New("Invalid go version output:\n" + string(data))
-		}
-
-		return string(matches[1]), nil
-
-	} else if data, err := ioutil.ReadFile(filepath.Join(goroot, "VERSION")); err == nil {
-		return string(data), nil
-
-	} else {
-		return "", err
-	}
-}
-
 // getClangHeaderPath returns the path to the built-in Clang headers. It tries
 // multiple locations, which should make it find the directory when installed in
 // various ways.

compiler/compiler.go

@@ -137,64 +137,26 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con
 	c.funcPtrAddrSpace = dummyFunc.Type().PointerAddressSpace()
 	dummyFunc.EraseFromParentAsFunction()
 
-	// Prefix the GOPATH with the system GOROOT, as GOROOT is already set to
-	// the TinyGo root.
-	overlayGopath := goenv.Get("GOPATH")
-	if overlayGopath == "" {
-		overlayGopath = goenv.Get("GOROOT")
-	} else {
-		overlayGopath = goenv.Get("GOROOT") + string(filepath.ListSeparator) + overlayGopath
-	}
-
 	wd, err := os.Getwd()
 	if err != nil {
 		return c.mod, nil, []error{err}
 	}
+	goroot, err := loader.GetCachedGoroot(c.Config)
+	if err != nil {
+		return c.mod, nil, []error{err}
+	}
 	lprogram := &loader.Program{
 		Build: &build.Context{
 			GOARCH:      c.GOARCH(),
 			GOOS:        c.GOOS(),
-			GOROOT:      goenv.Get("GOROOT"),
+			GOROOT:      goroot,
 			GOPATH:      goenv.Get("GOPATH"),
 			CgoEnabled:  c.CgoEnabled(),
 			UseAllFiles: false,
 			Compiler:    "gc", // must be one of the recognized compilers
 			BuildTags:   c.BuildTags(),
 		},
-		OverlayBuild: &build.Context{
-			GOARCH:      c.GOARCH(),
-			GOOS:        c.GOOS(),
-			GOROOT:      goenv.Get("TINYGOROOT"),
-			GOPATH:      overlayGopath,
-			CgoEnabled:  c.CgoEnabled(),
-			UseAllFiles: false,
-			Compiler:    "gc", // must be one of the recognized compilers
-			BuildTags:   c.BuildTags(),
-		},
-		OverlayPath: func(path string) string {
-			// Return the (overlay) import path when it should be overlaid, and
-			// "" if it should not.
-			if strings.HasPrefix(path, tinygoPath+"/src/") {
-				// Avoid issues with packages that are imported twice, one from
-				// GOPATH and one from TINYGOPATH.
-				path = path[len(tinygoPath+"/src/"):]
-			}
-			switch path {
-			case "machine", "os", "reflect", "runtime", "runtime/interrupt", "runtime/volatile", "sync", "testing", "internal/reflectlite", "internal/task":
-				return path
-			default:
-				if strings.HasPrefix(path, "device/") || strings.HasPrefix(path, "examples/") {
-					return path
-				} else if path == "syscall" {
-					for _, tag := range c.BuildTags() {
-						if tag == "baremetal" || tag == "darwin" {
-							return path
-						}
-					}
-				}
-			}
-			return ""
-		},
+		Tests: c.TestConfig.CompileTestBinary,
 		TypeChecker: types.Config{
 			Sizes: &stdSizes{
 				IntSize:  int64(c.targetData.TypeAllocSize(c.intType)),
@@ -208,33 +170,17 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con
 		ClangHeaders: c.ClangHeaders,
 	}
 
-	if strings.HasSuffix(pkgName, ".go") {
-		_, err = lprogram.ImportFile(pkgName)
-		if err != nil {
-			return c.mod, nil, []error{err}
-		}
-	} else {
-		_, err = lprogram.Import(pkgName, wd, token.Position{
-			Filename: "build command-line-arguments",
-		})
-		if err != nil {
-			return c.mod, nil, []error{err}
-		}
-	}
-
-	_, err = lprogram.Import("runtime", "", token.Position{
-		Filename: "build default import",
-	})
+	err = lprogram.Load(pkgName)
 	if err != nil {
 		return c.mod, nil, []error{err}
 	}
 
-	err = lprogram.Parse(c.TestConfig.CompileTestBinary)
+	err = lprogram.Parse()
 	if err != nil {
 		return c.mod, nil, []error{err}
 	}
 
-	c.ir = ir.NewProgram(lprogram, pkgName)
+	c.ir = ir.NewProgram(lprogram)
 
 	// Run a simple dead code elimination pass.
 	err = c.ir.SimpleDCE()
@@ -378,8 +324,11 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con
 	// Gather the list of (C) file paths that should be included in the build.
 	var extraFiles []string
 	for _, pkg := range c.ir.LoaderProgram.Sorted() {
-		for _, file := range pkg.CFiles {
-			extraFiles = append(extraFiles, filepath.Join(pkg.Package.Dir, file))
+		for _, file := range pkg.OtherFiles {
+			switch strings.ToLower(filepath.Ext(file)) {
+			case ".c":
+				extraFiles = append(extraFiles, file)
+			}
 		}
 	}
 

goenv/version.go

@@ -0,0 +1,64 @@
+package goenv
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+// GetGorootVersion returns the major and minor version for a given GOROOT path.
+// If the goroot cannot be determined, (0, 0) is returned.
+func GetGorootVersion(goroot string) (major, minor int, err error) {
+	s, err := GorootVersionString(goroot)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	if s == "" || s[:2] != "go" {
+		return 0, 0, errors.New("could not parse Go version: version does not start with 'go' prefix")
+	}
+
+	parts := strings.Split(s[2:], ".")
+	if len(parts) < 2 {
+		return 0, 0, errors.New("could not parse Go version: version has less than two parts")
+	}
+
+	// Ignore the errors, we don't really handle errors here anyway.
+	var trailing string
+	n, err := fmt.Sscanf(s, "go%d.%d%s", &major, &minor, &trailing)
+	if n == 2 && err == io.EOF {
+		// Means there were no trailing characters (i.e., not an alpha/beta)
+		err = nil
+	}
+	if err != nil {
+		return 0, 0, fmt.Errorf("failed to parse version: %s", err)
+	}
+	return
+}
+
+// GorootVersionString returns the version string as reported by the Go
+// toolchain for the given GOROOT path. It is usually of the form `go1.x.y` but
+// can have some variations (for beta releases, for example).
+func GorootVersionString(goroot string) (string, error) {
+	if data, err := ioutil.ReadFile(filepath.Join(
+		goroot, "src", "runtime", "internal", "sys", "zversion.go")); err == nil {
+
+		r := regexp.MustCompile("const TheVersion = `(.*)`")
+		matches := r.FindSubmatch(data)
+		if len(matches) != 2 {
+			return "", errors.New("Invalid go version output:\n" + string(data))
+		}
+
+		return string(matches[1]), nil
+
+	} else if data, err := ioutil.ReadFile(filepath.Join(goroot, "VERSION")); err == nil {
+		return string(data), nil
+
+	} else {
+		return "", err
+	}
+}

ir/ir.go

@@ -63,23 +63,13 @@ const (
 )
 
 // Create and initialize a new *Program from a *ssa.Program.
-func NewProgram(lprogram *loader.Program, mainPath string) *Program {
+func NewProgram(lprogram *loader.Program) *Program {
 	program := lprogram.LoadSSA()
 	program.Build()
 
 	// Find the main package, which is a bit difficult when running a .go file
 	// directly.
-	mainPkg := program.ImportedPackage(mainPath)
-	if mainPkg == nil {
-		for _, pkgInfo := range program.AllPackages() {
-			if pkgInfo.Pkg.Name() == "main" {
-				if mainPkg != nil {
-					panic("more than one main package found")
-				}
-				mainPkg = pkgInfo
-			}
-		}
-	}
+	mainPkg := program.ImportedPackage(lprogram.MainPkg.PkgPath)
 	if mainPkg == nil {
 		panic("could not find main package")
 	}
@@ -87,21 +77,10 @@ func NewProgram(lprogram *loader.Program, mainPath string) *Program {
 	// Make a list of packages in import order.
 	packageList := []*ssa.Package{}
 	packageSet := map[string]struct{}{}
-	worklist := []string{"runtime", mainPath}
+	worklist := []string{"runtime", lprogram.MainPkg.PkgPath}
 	for len(worklist) != 0 {
 		pkgPath := worklist[0]
-		var pkg *ssa.Package
-		if pkgPath == mainPath {
-			pkg = mainPkg // necessary for compiling individual .go files
-		} else {
-			pkg = program.ImportedPackage(pkgPath)
-		}
-		if pkg == nil {
-			// Non-SSA package (e.g. cgo).
-			packageSet[pkgPath] = struct{}{}
-			worklist = worklist[1:]
-			continue
-		}
+		pkg := program.ImportedPackage(pkgPath)
 		if _, ok := packageSet[pkgPath]; ok {
 			// Package already in the final package list.
 			worklist = worklist[1:]

loader/errors.go

@@ -1,10 +1,5 @@
 package loader
 
-import (
-	"go/token"
-	"strings"
-)
-
 // Errors contains a list of parser errors or a list of typechecker errors for
 // the given package.
 type Errors struct {
@@ -15,25 +10,3 @@ type Errors struct {
 func (e Errors) Error() string {
 	return "could not compile: " + e.Errs[0].Error()
 }
-
-// ImportCycleErrors is returned when encountering an import cycle. The list of
-// packages is a list from the root package to the leaf package that imports one
-// of the packages in the list.
-type ImportCycleError struct {
-	Packages        []string
-	ImportPositions []token.Position
-}
-
-func (e *ImportCycleError) Error() string {
-	var msg strings.Builder
-	msg.WriteString("import cycle:\n\t")
-	msg.WriteString(strings.Join(e.Packages, "\n\t"))
-	msg.WriteString("\n at ")
-	for i, pos := range e.ImportPositions {
-		if i > 0 {
-			msg.WriteString(", ")
-		}
-		msg.WriteString(pos.String())
-	}
-	return msg.String()
-}

loader/goroot.go

@@ -0,0 +1,225 @@
+package loader
+
+// This file constructs a new temporary GOROOT directory by merging both the
+// standard Go GOROOT and the GOROOT from TinyGo using symlinks.
+
+import (
+	"crypto/sha512"
+	"encoding/hex"
+	"errors"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strconv"
+
+	"github.com/tinygo-org/tinygo/compileopts"
+	"github.com/tinygo-org/tinygo/goenv"
+)
+
+// GetCachedGoroot creates a new GOROOT by merging both the standard GOROOT and
+// the GOROOT from TinyGo using lots of symbolic links.
+func GetCachedGoroot(config *compileopts.Config) (string, error) {
+	goroot := goenv.Get("GOROOT")
+	if goroot == "" {
+		return "", errors.New("could not determine GOROOT")
+	}
+	tinygoroot := goenv.Get("TINYGOROOT")
+	if tinygoroot == "" {
+		return "", errors.New("could not determine TINYGOROOT")
+	}
+
+	needsSyscallPackage := false
+	for _, tag := range config.BuildTags() {
+		if tag == "baremetal" || tag == "darwin" {
+			needsSyscallPackage = true
+		}
+	}
+
+	// Determine the location of the cached GOROOT.
+	version, err := goenv.GorootVersionString(goroot)
+	if err != nil {
+		return "", err
+	}
+	gorootsHash := sha512.Sum512_256([]byte(goroot + "\x00" + tinygoroot))
+	gorootsHashHex := hex.EncodeToString(gorootsHash[:])
+	cachedgoroot := filepath.Join(goenv.Get("GOCACHE"), "goroot-"+version+"-"+gorootsHashHex)
+	if needsSyscallPackage {
+		cachedgoroot += "-syscall"
+	}
+
+	if _, err := os.Stat(cachedgoroot); err == nil {
+		return cachedgoroot, nil
+	}
+	tmpgoroot := cachedgoroot + ".tmp" + strconv.Itoa(rand.Int())
+	err = os.MkdirAll(tmpgoroot, 0777)
+	if err != nil {
+		return "", err
+	}
+
+	// Remove the temporary directory if it wasn't moved to the right place
+	// (for example, when there was an error).
+	defer os.RemoveAll(tmpgoroot)
+
+	for _, name := range []string{"bin", "lib", "pkg"} {
+		err = symlink(filepath.Join(goroot, name), filepath.Join(tmpgoroot, name))
+		if err != nil {
+			return "", err
+		}
+	}
+	err = mergeDirectory(goroot, tinygoroot, tmpgoroot, "", pathsToOverride(needsSyscallPackage))
+	if err != nil {
+		return "", err
+	}
+	err = os.Rename(tmpgoroot, cachedgoroot)
+	if err != nil {
+		if os.IsExist(err) {
+			// Another invocation of TinyGo also seems to have created a GOROOT.
+			// Use that one instead. Our new GOROOT will be automatically
+			// deleted by the defer above.
+			return cachedgoroot, nil
+		}
+		return "", err
+	}
+	return cachedgoroot, nil
+}
+
+// mergeDirectory merges two roots recursively. The tmpgoroot is the directory
+// that will be created by this call by either symlinking the directory from
+// goroot or tinygoroot, or by creating the directory and merging the contents.
+func mergeDirectory(goroot, tinygoroot, tmpgoroot, importPath string, overrides map[string]bool) error {
+	if mergeSubdirs, ok := overrides[importPath+"/"]; ok {
+		if !mergeSubdirs {
+			// This directory and all subdirectories should come from the TinyGo
+			// root, so simply make a symlink.
+			newname := filepath.Join(tmpgoroot, "src", importPath)
+			oldname := filepath.Join(tinygoroot, "src", importPath)
+			return symlink(oldname, newname)
+		}
+
+		// Merge subdirectories. Start by making the directory to merge.
+		err := os.Mkdir(filepath.Join(tmpgoroot, "src", importPath), 0777)
+		if err != nil {
+			return err
+		}
+
+		// Symlink all files from TinyGo, and symlink directories from TinyGo
+		// that need to be overridden.
+		tinygoEntries, err := ioutil.ReadDir(filepath.Join(tinygoroot, "src", importPath))
+		if err != nil {
+			return err
+		}
+		for _, e := range tinygoEntries {
+			if e.IsDir() {
+				// A directory, so merge this thing.
+				err := mergeDirectory(goroot, tinygoroot, tmpgoroot, path.Join(importPath, e.Name()), overrides)
+				if err != nil {
+					return err
+				}
+			} else {
+				// A file, so symlink this.
+				newname := filepath.Join(tmpgoroot, "src", importPath, e.Name())
+				oldname := filepath.Join(tinygoroot, "src", importPath, e.Name())
+				err := symlink(oldname, newname)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// Symlink all directories from $GOROOT that are not part of the TinyGo
+		// overrides.
+		gorootEntries, err := ioutil.ReadDir(filepath.Join(goroot, "src", importPath))
+		if err != nil {
+			return err
+		}
+		for _, e := range gorootEntries {
+			if !e.IsDir() {
+				// Don't merge in files from Go. Otherwise we'd end up with a
+				// weird syscall package with files from both roots.
+				continue
+			}
+			if _, ok := overrides[path.Join(importPath, e.Name())+"/"]; ok {
+				// Already included above, so don't bother trying to create this
+				// symlink.
+				continue
+			}
+			newname := filepath.Join(tmpgoroot, "src", importPath, e.Name())
+			oldname := filepath.Join(goroot, "src", importPath, e.Name())
+			err := symlink(oldname, newname)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// The boolean indicates whether to merge the subdirs. True means merge, false
+// means use the TinyGo version.
+func pathsToOverride(needsSyscallPackage bool) map[string]bool {
+	paths := map[string]bool{
+		"/":                     true,
+		"device/":               false,
+		"examples/":             false,
+		"internal/":             true,
+		"internal/reflectlite/": false,
+		"internal/task/":        false,
+		"machine/":              false,
+		"os/":                   true,
+		"reflect/":              false,
+		"runtime/":              false,
+		"sync/":                 true,
+		"testing/":              false,
+	}
+	if needsSyscallPackage {
+		paths["syscall/"] = true // include syscall/js
+	}
+	return paths
+}
+
+// symlink creates a symlink or something similar. On Unix-like systems, it
+// always creates a symlink. On Windows, it tries to create a symlink and if
+// that fails, creates a hardlink or directory junction instead.
+//
+// Note that while Windows 10 does support symlinks and allows them to be
+// created using os.Symlink, it requires developer mode to be enabled.
+// Therefore provide a fallback for when symlinking is not possible.
+// Unfortunately this fallback only works when TinyGo is installed on the same
+// filesystem as the TinyGo cache and the Go installation (which is usually the
+// C drive).
+func symlink(oldname, newname string) error {
+	symlinkErr := os.Symlink(oldname, newname)
+	if runtime.GOOS == "windows" && symlinkErr != nil {
+		// Fallback for when developer mode is disabled.
+		// Note that we return the symlink error even if something else fails
+		// later on. This is because symlinks are the easiest to support
+		// (they're also used on Linux and MacOS) and enabling them is easy:
+		// just enable developer mode.
+		st, err := os.Stat(oldname)
+		if err != nil {
+			return symlinkErr
+		}
+		if st.IsDir() {
+			// Make a directory junction. There may be a way to do this
+			// programmatically, but it involves a lot of magic. Use the mklink
+			// command built into cmd instead (mklink is a builtin, not an
+			// external command).
+			err := exec.Command("cmd", "/k", "mklink", "/J", newname, oldname).Run()
+			if err != nil {
+				return symlinkErr
+			}
+		} else {
+			// Make a hard link.
+			err := os.Link(oldname, newname)
+			if err != nil {
+				return symlinkErr
+			}
+		}
+		return nil // success
+	}
+	return symlinkErr
+}

loader/loader.go

@@ -3,10 +3,10 @@ package loader
 import (
 	"bytes"
 	"errors"
+	"fmt"
 	"go/ast"
 	"go/build"
 	"go/parser"
-	"go/scanner"
 	"go/token"
 	"go/types"
 	"os"
@@ -16,15 +16,16 @@ import (
 	"text/template"
 
 	"github.com/tinygo-org/tinygo/cgo"
+	"github.com/tinygo-org/tinygo/goenv"
+	"golang.org/x/tools/go/packages"
 )
 
 // Program holds all packages and some metadata about the program as a whole.
 type Program struct {
-	mainPkg      string
 	Build        *build.Context
-	OverlayBuild *build.Context
-	OverlayPath  func(path string) string
+	Tests        bool
 	Packages     map[string]*Package
+	MainPkg      *Package
 	sorted       []*Package
 	fset         *token.FileSet
 	TypeChecker  types.Config
@@ -37,89 +38,114 @@ type Program struct {
 // Package holds a loaded package, its imports, and its parsed files.
 type Package struct {
 	*Program
-	*build.Package
-	Imports   map[string]*Package
-	Importing bool
-	Files     []*ast.File
-	Pkg       *types.Package
+	*packages.Package
+	Files []*ast.File
+	Pkg   *types.Package
 	types.Info
 }
 
-// Import loads the given package relative to srcDir (for the vendor directory).
-// It only loads the current package without recursion.
-func (p *Program) Import(path, srcDir string, pos token.Position) (*Package, error) {
+// Load loads the given package with all dependencies (including the runtime
+// package). Call .Parse() afterwards to parse all Go files (including CGo
+// processing, if necessary).
+func (p *Program) Load(importPath string) error {
 	if p.Packages == nil {
 		p.Packages = make(map[string]*Package)
 	}
 
-	// Load this package.
-	ctx := p.Build
-	if newPath := p.OverlayPath(path); newPath != "" {
-		ctx = p.OverlayBuild
-		path = newPath
-	}
-	buildPkg, err := ctx.Import(path, srcDir, build.ImportComment)
+	err := p.loadPackage(importPath)
 	if err != nil {
-		return nil, scanner.Error{
-			Pos: pos,
-			Msg: err.Error(), // TODO: define a new error type that will wrap the inner error
-		}
+		return err
 	}
-	if existingPkg, ok := p.Packages[buildPkg.ImportPath]; ok {
-		// Already imported, or at least started the import.
-		return existingPkg, nil
+	p.MainPkg = p.sorted[len(p.sorted)-1]
+	if _, ok := p.Packages["runtime"]; !ok {
+		// The runtime package wasn't loaded. Although `go list -deps` seems to
+		// return the full dependency list, there is no way to get those
+		// packages from the go/packages package. Therefore load the runtime
+		// manually and add it to the list of to-be-compiled packages
+		// (duplicates are already filtered).
+		return p.loadPackage("runtime")
 	}
-	p.sorted = nil // invalidate the sorted order of packages
-	pkg := p.newPackage(buildPkg)
-	p.Packages[buildPkg.ImportPath] = pkg
-
-	if p.mainPkg == "" {
-		p.mainPkg = buildPkg.ImportPath
-	}
-
-	return pkg, nil
+	return nil
 }
 
-// ImportFile loads and parses the import statements in the given path and
-// creates a pseudo-package out of it.
-func (p *Program) ImportFile(path string) (*Package, error) {
-	if p.Packages == nil {
-		p.Packages = make(map[string]*Package)
+func (p *Program) loadPackage(importPath string) error {
+	cgoEnabled := "0"
+	if p.Build.CgoEnabled {
+		cgoEnabled = "1"
 	}
-	if _, ok := p.Packages[path]; ok {
-		// unlikely
-		return nil, errors.New("loader: cannot import file that is already imported as package: " + path)
-	}
-
-	file, err := p.parseFile(path, parser.ImportsOnly)
+	pkgs, err := packages.Load(&packages.Config{
+		Mode:       packages.NeedName | packages.NeedFiles | packages.NeedImports | packages.NeedDeps,
+		Env:        append(os.Environ(), "GOROOT="+p.Build.GOROOT, "GOOS="+p.Build.GOOS, "GOARCH="+p.Build.GOARCH, "CGO_ENABLED="+cgoEnabled),
+		BuildFlags: []string{"-tags", strings.Join(p.Build.BuildTags, " ")},
+		Tests:      p.Tests,
+	}, importPath)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	buildPkg := &build.Package{
-		Dir:        filepath.Dir(path),
-		ImportPath: path,
-		GoFiles:    []string{filepath.Base(path)},
+	var pkg *packages.Package
+	if p.Tests {
+		// We need the second package. Quoting from the docs:
+		// > For example, when using the go command, loading "fmt" with Tests=true
+		// > returns four packages, with IDs "fmt" (the standard package),
+		// > "fmt [fmt.test]" (the package as compiled for the test),
+		// > "fmt_test" (the test functions from source files in package fmt_test),
+		// > and "fmt.test" (the test binary).
+		pkg = pkgs[1]
+	} else {
+		if len(pkgs) != 1 {
+			return fmt.Errorf("expected exactly one package while importing %s, got %d", importPath, len(pkgs))
+		}
+		pkg = pkgs[0]
 	}
-	for _, importSpec := range file.Imports {
-		buildPkg.Imports = append(buildPkg.Imports, importSpec.Path.Value[1:len(importSpec.Path.Value)-1])
-	}
-	p.sorted = nil // invalidate the sorted order of packages
-	pkg := p.newPackage(buildPkg)
-	p.Packages[buildPkg.ImportPath] = pkg
+	var importError *Errors
+	var addPackages func(pkg *packages.Package)
+	addPackages = func(pkg *packages.Package) {
+		if _, ok := p.Packages[pkg.PkgPath]; ok {
+			return
+		}
+		pkg2 := p.newPackage(pkg)
+		p.Packages[pkg.PkgPath] = pkg2
+		if len(pkg.Errors) != 0 {
+			if importError != nil {
+				// There was another error reported already. Do not report
+				// errors from multiple packages at once.
+				return
+			}
+			importError = &Errors{
+				Pkg: pkg2,
+			}
+			for _, err := range pkg.Errors {
+				importError.Errs = append(importError.Errs, err)
+			}
+			return
+		}
 
-	if p.mainPkg == "" {
-		p.mainPkg = buildPkg.ImportPath
-	}
+		// Get the list of imports (sorted alphabetically).
+		names := make([]string, 0, len(pkg.Imports))
+		for name := range pkg.Imports {
+			names = append(names, name)
+		}
+		sort.Strings(names)
 
-	return pkg, nil
+		// Add all the imports.
+		for _, name := range names {
+			addPackages(pkg.Imports[name])
+		}
+
+		p.sorted = append(p.sorted, pkg2)
+	}
+	addPackages(pkg)
+	if importError != nil {
+		return importError
+	}
+	return nil
 }
 
 // newPackage instantiates a new *Package object with initialized members.
-func (p *Program) newPackage(pkg *build.Package) *Package {
+func (p *Program) newPackage(pkg *packages.Package) *Package {
 	return &Package{
 		Program: p,
 		Package: pkg,
-		Imports: make(map[string]*Package, len(pkg.Imports)),
 		Info: types.Info{
 			Types:      make(map[ast.Expr]types.TypeAndValue),
 			Defs:       make(map[*ast.Ident]types.Object),
@@ -134,87 +160,25 @@ func (p *Program) newPackage(pkg *build.Package) *Package {
 // Sorted returns a list of all packages, sorted in a way that no packages come
 // before the packages they depend upon.
 func (p *Program) Sorted() []*Package {
-	if p.sorted == nil {
-		p.sort()
-	}
 	return p.sorted
 }
 
-func (p *Program) sort() {
-	p.sorted = nil
-	packageList := make([]*Package, 0, len(p.Packages))
-	packageSet := make(map[string]struct{}, len(p.Packages))
-	worklist := make([]string, 0, len(p.Packages))
-	for path := range p.Packages {
-		worklist = append(worklist, path)
-	}
-	sort.Strings(worklist)
-	for len(worklist) != 0 {
-		pkgPath := worklist[0]
-		pkg := p.Packages[pkgPath]
-
-		if _, ok := packageSet[pkgPath]; ok {
-			// Package already in the final package list.
-			worklist = worklist[1:]
-			continue
-		}
-
-		unsatisfiedImports := make([]string, 0)
-		for _, pkg := range pkg.Imports {
-			if _, ok := packageSet[pkg.ImportPath]; ok {
-				continue
-			}
-			unsatisfiedImports = append(unsatisfiedImports, pkg.ImportPath)
-		}
-		sort.Strings(unsatisfiedImports)
-		if len(unsatisfiedImports) == 0 {
-			// All dependencies of this package are satisfied, so add this
-			// package to the list.
-			packageList = append(packageList, pkg)
-			packageSet[pkgPath] = struct{}{}
-			worklist = worklist[1:]
-		} else {
-			// Prepend all dependencies to the worklist and reconsider this
-			// package (by not removing it from the worklist). At that point, it
-			// must be possible to add it to packageList.
-			worklist = append(unsatisfiedImports, worklist...)
-		}
-	}
-
-	p.sorted = packageList
-}
-
-// Parse recursively imports all packages, parses them, and typechecks them.
+// Parse parses all packages and typechecks them.
 //
 // The returned error may be an Errors error, which contains a list of errors.
 //
 // Idempotent.
-func (p *Program) Parse(compileTestBinary bool) error {
-	includeTests := compileTestBinary
-
-	// Load all imports
-	for _, pkg := range p.Sorted() {
-		err := pkg.importRecursively(includeTests)
-		if err != nil {
-			if err, ok := err.(*ImportCycleError); ok {
-				if pkg.ImportPath != err.Packages[0] {
-					err.Packages = append([]string{pkg.ImportPath}, err.Packages...)
-				}
-			}
-			return err
-		}
-	}
-
+func (p *Program) Parse() error {
 	// Parse all packages.
 	for _, pkg := range p.Sorted() {
-		err := pkg.Parse(includeTests)
+		err := pkg.Parse()
 		if err != nil {
 			return err
 		}
 	}
 
-	if compileTestBinary {
-		err := p.SwapTestMain()
+	if p.Tests {
+		err := p.swapTestMain()
 		if err != nil {
 			return err
 		}
@@ -231,7 +195,7 @@ func (p *Program) Parse(compileTestBinary bool) error {
 	return nil
 }
 
-func (p *Program) SwapTestMain() error {
+func (p *Program) swapTestMain() error {
 	var tests []string
 
 	isTestFunc := func(f *ast.FuncDecl) bool {
@@ -241,8 +205,7 @@ func (p *Program) SwapTestMain() error {
 		}
 		return false
 	}
-	mainPkg := p.Packages[p.mainPkg]
-	for _, f := range mainPkg.Files {
+	for _, f := range p.MainPkg.Files {
 		for i, d := range f.Decls {
 			switch v := d.(type) {
 			case *ast.FuncDecl:
@@ -293,7 +256,7 @@ func main () {
 	if err != nil {
 		return err
 	}
-	path := filepath.Join(p.mainPkg, "$testmain.go")
+	path := filepath.Join(p.MainPkg.Dir, "$testmain.go")
 
 	if p.fset == nil {
 		p.fset = token.NewFileSet()
@@ -303,7 +266,7 @@ func main () {
 	if err != nil {
 		return err
 	}
-	mainPkg.Files = append(mainPkg.Files, newMain)
+	p.MainPkg.Files = append(p.MainPkg.Files, newMain)
 
 	return nil
 }
@@ -319,34 +282,41 @@ func (p *Program) parseFile(path string, mode parser.Mode) (*ast.File, error) {
 		return nil, err
 	}
 	defer rd.Close()
-	relpath := path
-	if filepath.IsAbs(path) {
-		rp, err := filepath.Rel(p.Dir, path)
-		if err == nil {
-			relpath = rp
+	diagnosticPath := path
+	if strings.HasPrefix(path, p.Build.GOROOT+string(filepath.Separator)) {
+		// If this file is part of the synthetic GOROOT, try to infer the
+		// original path.
+		relpath := path[len(p.Build.GOROOT)+1:]
+		tinygoPath := filepath.Join(p.TINYGOROOT, relpath)
+		if _, err := os.Stat(tinygoPath); err == nil {
+			diagnosticPath = tinygoPath
+		}
+		realgorootPath := filepath.Join(goenv.Get("GOROOT"), relpath)
+		if _, err := os.Stat(realgorootPath); err == nil {
+			diagnosticPath = realgorootPath
 		}
 	}
-	return parser.ParseFile(p.fset, relpath, rd, mode)
+	return parser.ParseFile(p.fset, diagnosticPath, rd, mode)
 }
 
 // Parse parses and typechecks this package.
 //
 // Idempotent.
-func (p *Package) Parse(includeTests bool) error {
+func (p *Package) Parse() error {
 	if len(p.Files) != 0 {
 		return nil
 	}
 
 	// Load the AST.
 	// TODO: do this in parallel.
-	if p.ImportPath == "unsafe" {
+	if p.PkgPath == "unsafe" {
 		// Special case for the unsafe package. Don't even bother loading
 		// the files.
 		p.Pkg = types.Unsafe
 		return nil
 	}
 
-	files, err := p.parseFiles(includeTests)
+	files, err := p.parseFiles()
 	if err != nil {
 		return err
 	}
@@ -373,7 +343,7 @@ func (p *Package) Check() error {
 	// Do typechecking of the package.
 	checker.Importer = p
 
-	typesPkg, err := checker.Check(p.ImportPath, p.fset, p.Files, &p.Info)
+	typesPkg, err := checker.Check(p.PkgPath, p.fset, p.Files, &p.Info)
 	if err != nil {
 		if err, ok := err.(Errors); ok {
 			return err
@@ -385,22 +355,14 @@ func (p *Package) Check() error {
 }
 
 // parseFiles parses the loaded list of files and returns this list.
-func (p *Package) parseFiles(includeTests bool) ([]*ast.File, error) {
+func (p *Package) parseFiles() ([]*ast.File, error) {
 	// TODO: do this concurrently.
 	var files []*ast.File
 	var fileErrs []error
 
-	var gofiles []string
-	if includeTests {
-		gofiles = make([]string, 0, len(p.GoFiles)+len(p.TestGoFiles))
-		gofiles = append(gofiles, p.GoFiles...)
-		gofiles = append(gofiles, p.TestGoFiles...)
-	} else {
-		gofiles = p.GoFiles
-	}
-
-	for _, file := range gofiles {
-		f, err := p.parseFile(filepath.Join(p.Package.Dir, file), parser.ParseComments)
+	var cgoFiles []*ast.File
+	for _, file := range p.GoFiles {
+		f, err := p.parseFile(file, parser.ParseComments)
 		if err != nil {
 			fileErrs = append(fileErrs, err)
 			continue
@@ -409,19 +371,15 @@ func (p *Package) parseFiles(includeTests bool) ([]*ast.File, error) {
 			fileErrs = append(fileErrs, err)
 			continue
 		}
+		for _, importSpec := range f.Imports {
+			if importSpec.Path.Value == `"C"` {
+				cgoFiles = append(cgoFiles, f)
+			}
+		}
 		files = append(files, f)
 	}
-	for _, file := range p.CgoFiles {
-		path := filepath.Join(p.Package.Dir, file)
-		f, err := p.parseFile(path, parser.ParseComments)
-		if err != nil {
-			fileErrs = append(fileErrs, err)
-			continue
-		}
-		files = append(files, f)
-	}
-	if len(p.CgoFiles) != 0 {
-		cflags := append(p.CFlags, "-I"+p.Package.Dir)
+	if len(cgoFiles) != 0 {
+		cflags := append(p.CFlags, "-I"+filepath.Dir(p.GoFiles[0]))
 		if p.ClangHeaders != "" {
 			cflags = append(cflags, "-Xclang", "-internal-isystem", "-Xclang", p.ClangHeaders)
 		}
@@ -445,58 +403,8 @@ func (p *Package) Import(to string) (*types.Package, error) {
 		return types.Unsafe, nil
 	}
 	if _, ok := p.Imports[to]; ok {
-		return p.Imports[to].Pkg, nil
+		return p.Packages[p.Imports[to].PkgPath].Pkg, nil
 	} else {
 		return nil, errors.New("package not imported: " + to)
 	}
 }
-
-// importRecursively calls Program.Import() on all imported packages, and calls
-// importRecursively() on the imported packages as well.
-//
-// Idempotent.
-func (p *Package) importRecursively(includeTests bool) error {
-	p.Importing = true
-
-	imports := p.Package.Imports
-	if includeTests {
-		imports = append(imports, p.Package.TestImports...)
-	}
-
-	for _, to := range imports {
-		if to == "C" {
-			// Do CGo processing in a later stage.
-			continue
-		}
-		if _, ok := p.Imports[to]; ok {
-			continue
-		}
-		// Find error location.
-		var pos token.Position
-		if len(p.Package.ImportPos[to]) > 0 {
-			pos = p.Package.ImportPos[to][0]
-		} else {
-			pos = token.Position{Filename: p.Package.ImportPath}
-		}
-		importedPkg, err := p.Program.Import(to, p.Package.Dir, pos)
-		if err != nil {
-			if err, ok := err.(*ImportCycleError); ok {
-				err.Packages = append([]string{p.ImportPath}, err.Packages...)
-			}
-			return err
-		}
-		if importedPkg.Importing {
-			return &ImportCycleError{[]string{p.ImportPath, importedPkg.ImportPath}, p.ImportPos[to]}
-		}
-		err = importedPkg.importRecursively(false)
-		if err != nil {
-			if err, ok := err.(*ImportCycleError); ok {
-				err.Packages = append([]string{p.ImportPath}, err.Packages...)
-			}
-			return err
-		}
-		p.Imports[to] = importedPkg
-	}
-	p.Importing = false
-	return nil
-}

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
@@ -124,6 +125,7 @@ func Build(pkgName, outpath string, options *compileopts.Options) error {
 
 // Test runs the tests in the given package.
 func Test(pkgName string, options *compileopts.Options) error {
+	options.TestConfig.CompileTestBinary = true
 	config, err := builder.NewConfig(options)
 	if err != nil {
 		return err
@@ -135,7 +137,6 @@ func Test(pkgName string, options *compileopts.Options) error {
 	// For details: https://github.com/golang/go/issues/21360
 	config.Target.BuildTags = append(config.Target.BuildTags, "test")
 
-	options.TestConfig.CompileTestBinary = true
 	return builder.Build(pkgName, ".elf", config, func(tmppath string) error {
 		cmd := exec.Command(tmppath)
 		cmd.Stdout = os.Stdout
@@ -641,6 +642,36 @@ func getDefaultPort() (port string, err error) {
 	return d[0], nil
 }
 
+// runGoList runs the `go list` command but using the configuration used for
+// TinyGo.
+func runGoList(config *compileopts.Config, flagJSON, flagDeps bool, pkgs []string) error {
+	goroot, err := loader.GetCachedGoroot(config)
+	if err != nil {
+		return err
+	}
+	args := []string{"list"}
+	if flagJSON {
+		args = append(args, "-json")
+	}
+	if flagDeps {
+		args = append(args, "-deps")
+	}
+	if len(config.BuildTags()) != 0 {
+		args = append(args, "-tags", strings.Join(config.BuildTags(), " "))
+	}
+	args = append(args, pkgs...)
+	cgoEnabled := "0"
+	if config.CgoEnabled() {
+		cgoEnabled = "1"
+	}
+	cmd := exec.Command("go", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	cmd.Env = append(os.Environ(), "GOROOT="+goroot, "GOOS="+config.GOOS(), "GOARCH="+config.GOARCH(), "CGO_ENABLED="+cgoEnabled)
+	cmd.Run()
+	return nil
+}
+
 func usage() {
 	fmt.Fprintln(os.Stderr, "TinyGo is a Go compiler for small places.")
 	fmt.Fprintln(os.Stderr, "version:", version)
@@ -652,6 +683,7 @@ func usage() {
 	fmt.Fprintln(os.Stderr, "  flash: compile and flash to the device")
 	fmt.Fprintln(os.Stderr, "  gdb:   run/flash and immediately enter GDB")
 	fmt.Fprintln(os.Stderr, "  env:   list environment variables used during build")
+	fmt.Fprintln(os.Stderr, "  list:  run go list using the TinyGo root")
 	fmt.Fprintln(os.Stderr, "  clean: empty cache directory ("+goenv.Get("GOCACHE")+")")
 	fmt.Fprintln(os.Stderr, "  help:  print this help text")
 	fmt.Fprintln(os.Stderr, "\nflags:")
@@ -683,7 +715,7 @@ func printCompilerError(logln func(...interface{}), err error) {
 			}
 		}
 	case loader.Errors:
-		logln("#", err.Pkg.ImportPath)
+		logln("#", err.Pkg.PkgPath)
 		for _, err := range err.Errs {
 			logln(err)
 		}
@@ -706,6 +738,13 @@ func handleCompilerError(err error) {
 }
 
 func main() {
+	if len(os.Args) < 2 {
+		fmt.Fprintln(os.Stderr, "No command-line arguments supplied.")
+		usage()
+		os.Exit(1)
+	}
+	command := os.Args[1]
+
 	outpath := flag.String("o", "", "output filename")
 	opt := flag.String("opt", "z", "optimization level: 0, 1, 2, s, z")
 	gc := flag.String("gc", "", "garbage collector to use (none, leaking, extalloc, conservative)")
@@ -726,12 +765,13 @@ func main() {
 	wasmAbi := flag.String("wasm-abi", "js", "WebAssembly ABI conventions: js (no i64 params) or generic")
 	heapSize := flag.String("heap-size", "1M", "default heap size in bytes (only supported by WebAssembly)")
 
-	if len(os.Args) < 2 {
-		fmt.Fprintln(os.Stderr, "No command-line arguments supplied.")
-		usage()
-		os.Exit(1)
+	var flagJSON, flagDeps *bool
+	if command == "list" || command == "env" {
+		flagJSON = flag.Bool("json", false, "print data in JSON format")
+	}
+	if command == "list" {
+		flagDeps = flag.Bool("deps", false, "")
 	}
-	command := os.Args[1]
 
 	// Early command processing, before commands are interpreted by the Go flag
 	// library.
@@ -895,6 +935,18 @@ func main() {
 		fmt.Printf("build tags:        %s\n", strings.Join(config.BuildTags(), " "))
 		fmt.Printf("garbage collector: %s\n", config.GC())
 		fmt.Printf("scheduler:         %s\n", config.Scheduler())
+	case "list":
+		config, err := builder.NewConfig(options)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			usage()
+			os.Exit(1)
+		}
+		err = runGoList(config, *flagJSON, *flagDeps, flag.Args())
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "failed to run `go list`:", err)
+			os.Exit(1)
+		}
 	case "clean":
 		// remove cache directory
 		err := os.RemoveAll(goenv.Get("GOCACHE"))
@@ -906,20 +958,36 @@ func main() {
 		usage()
 	case "version":
 		goversion := "<unknown>"
-		if s, err := builder.GorootVersionString(goenv.Get("GOROOT")); err == nil {
+		if s, err := goenv.GorootVersionString(goenv.Get("GOROOT")); err == nil {
 			goversion = s
 		}
 		fmt.Printf("tinygo version %s %s/%s (using go version %s and LLVM version %s)\n", version, runtime.GOOS, runtime.GOARCH, goversion, llvm.Version)
 	case "env":
-		if flag.NArg() == 0 {
-			// Show all environment variables.
-			for _, key := range goenv.Keys {
-				fmt.Printf("%s=%#v\n", key, goenv.Get(key))
+		if *flagJSON {
+			keys := goenv.Keys
+			if flag.NArg() != 0 {
+				// Show only one (or a few) environment variables.
+				keys = flag.Args()
 			}
+			// Show environment variables in JSON format.
+			env := make(map[string]string)
+			for _, key := range keys {
+				env[key] = goenv.Get(key)
+			}
+			encoder := json.NewEncoder(os.Stdout)
+			encoder.SetIndent("", "\t")
+			encoder.Encode(env)
 		} else {
-			// Show only one (or a few) environment variables.
-			for i := 0; i < flag.NArg(); i++ {
-				fmt.Println(goenv.Get(flag.Arg(i)))
+			if flag.NArg() == 0 {
+				// Show all environment variables.
+				for _, key := range goenv.Keys {
+					fmt.Printf("%s=%#v\n", key, goenv.Get(key))
+				}
+			} else {
+				// Show only one (or a few) environment variables.
+				for i := 0; i < flag.NArg(); i++ {
+					fmt.Println(goenv.Get(flag.Arg(i)))
+				}
 			}
 		}
 	default:

main_test.go

@@ -73,7 +73,7 @@ func TestCompiler(t *testing.T) {
 		t.Run("ARM64Linux", func(t *testing.T) {
 			runPlatTests("aarch64--linux-gnu", matches, t)
 		})
-		goVersion, err := builder.GorootVersionString(goenv.Get("GOROOT"))
+		goVersion, err := goenv.GorootVersionString(goenv.Get("GOROOT"))
 		if err != nil {
 			t.Error("could not get Go version:", err)
 			return