builder: parallelize most of the build

This commit parallelizes almost everything that can currently be
parallelized. With that, it also introduces a framework for easily
parallelizing other parts of the compiler.

Code for baremetal targets already compiles slightly faster because it
can parallelize the compilation of supporting assembly files. However,
the speedup is especially noticeable when libraries (compiler-rt,
picolibc) also need to be compiled: they will be compiled in parallel
next to the Go files using all available cores. On my dual core laptop
(4 cores if you count hyperthreading) this cuts compilation time roughly
in half when compiling something for a Cortex-M board after running
`tinygo clean`.

builder/library.go

@@ -1,7 +1,6 @@
 package builder
 
 import (
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -40,13 +39,33 @@ func (l *Library) sourcePaths(target string) []string {
 }
 
 // Load the library archive, possibly generating and caching it if needed.
-func (l *Library) Load(target string) (path string, err error) {
+// The resulting file is stored in the provided tmpdir, which is expected to be
+// removed after the Load call.
+func (l *Library) Load(target, tmpdir string) (path string, err error) {
+	path, job, err := l.load(target, tmpdir)
+	if err != nil {
+		return "", err
+	}
+	if job != nil {
+		jobs := append([]*compileJob{job}, job.dependencies...)
+		err = runJobs(jobs)
+	}
+	return path, err
+}
+
+// load returns a path to the library file for the given target, loading it from
+// cache if possible. It will return a non-zero compiler job if the library
+// wasn't cached, this job (and its dependencies) must be run before the library
+// path is valid.
+// The provided tmpdir will be used to store intermediary files and possibly the
+// output archive file, it is expected to be removed after use.
+func (l *Library) load(target, tmpdir string) (path string, job *compileJob, err error) {
 	// Try to load a precompiled library.
 	precompiledPath := filepath.Join(goenv.Get("TINYGOROOT"), "pkg", target, l.name+".a")
 	if _, err := os.Stat(precompiledPath); err == nil {
 		// Found a precompiled library for this OS/architecture. Return the path
 		// directly.
-		return precompiledPath, nil
+		return precompiledPath, nil, nil
 	}
 
 	outfile := l.name + "-" + target + ".a"
@@ -54,19 +73,21 @@ func (l *Library) Load(target string) (path string, err error) {
 	// Try to fetch this library from the cache.
 	if path, err := cacheLoad(outfile, commands["clang"][0], l.sourcePaths(target)); path != "" || err != nil {
 		// Cache hit.
-		return path, err
+		return path, nil, err
 	}
 	// Cache miss, build it now.
 
-	dirPrefix := "tinygo-" + l.name
-	remapDir := filepath.Join(os.TempDir(), dirPrefix)
-	dir, err := ioutil.TempDir(os.TempDir(), dirPrefix)
+	remapDir := filepath.Join(os.TempDir(), "tinygo-"+l.name)
+	dir := filepath.Join(tmpdir, "build-lib-"+l.name)
+	err = os.Mkdir(dir, 0777)
 	if err != nil {
-		return "", err
+		return "", nil, err
 	}
-	defer os.RemoveAll(dir)
 
 	// Precalculate the flags to the compiler invocation.
+	// Note: -fdebug-prefix-map is necessary to make the output archive
+	// reproducible. Otherwise the temporary directory is stored in the archive
+	// itself, which varies each run.
 	args := append(l.cflags(), "-c", "-Oz", "-g", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+target, "-fdebug-prefix-map="+dir+"="+remapDir)
 	if strings.HasPrefix(target, "arm") || strings.HasPrefix(target, "thumb") {
 		args = append(args, "-fshort-enums", "-fomit-frame-pointer", "-mfloat-abi=soft")
@@ -78,28 +99,44 @@ func (l *Library) Load(target string) (path string, err error) {
 		args = append(args, "-march=rv64gc", "-mabi=lp64")
 	}
 
-	// Compile all sources.
+	// Create job to put all the object files in a single archive. This archive
+	// file is the (static) library file.
 	var objs []string
+	arpath := filepath.Join(dir, l.name+".a")
+	job = &compileJob{
+		description: "ar " + l.name + ".a",
+		run: func() error {
+			// Create an archive of all object files.
+			err := makeArchive(arpath, objs)
+			if err != nil {
+				return err
+			}
+			// Store this archive in the cache.
+			_, err = cacheStore(arpath, outfile, commands["clang"][0], l.sourcePaths(target))
+			return err
+		},
+	}
+
+	// Create jobs to compile all sources. These jobs are depended upon by the
+	// archive job above, so must be run first.
 	for _, srcpath := range l.sourcePaths(target) {
+		srcpath := srcpath // avoid concurrency issues by redefining inside the loop
 		objpath := filepath.Join(dir, filepath.Base(srcpath)+".o")
 		objs = append(objs, objpath)
-		// Note: -fdebug-prefix-map is necessary to make the output archive
-		// reproducible. Otherwise the temporary directory is stored in the
-		// archive itself, which varies each run.
-		err := runCCompiler("clang", append(args, "-o", objpath, srcpath)...)
-		if err != nil {
-			return "", &commandError{"failed to build", srcpath, err}
-		}
+		job.dependencies = append(job.dependencies, &compileJob{
+			description: "compile " + srcpath,
+			run: func() error {
+				var compileArgs []string
+				compileArgs = append(compileArgs, args...)
+				compileArgs = append(compileArgs, "-o", objpath, srcpath)
+				err := runCCompiler("clang", compileArgs...)
+				if err != nil {
+					return &commandError{"failed to build", srcpath, err}
+				}
+				return nil
+			},
+		})
 	}
 
-	// Put all the object files in a single archive. This archive file will be
-	// used to statically link this library.
-	arpath := filepath.Join(dir, l.name+".a")
-	err = makeArchive(arpath, objs)
-	if err != nil {
-		return "", err
-	}
-
-	// Store this archive in the cache.
-	return cacheStore(arpath, outfile, commands["clang"][0], l.sourcePaths(target))
+	return arpath, job, nil
 }