builder: build static binaries using musl on Linux

This commit adds support for musl-libc and uses it by default on Linux.
The main benefit of it is that binaries are always statically linked
instead of depending on the host libc, even when using CGo.

Advantages:
  - The resulting binaries are always statically linked.
  - No need for any tools on the host OS, like a compiler, linker, or
    libc in a release build of TinyGo.
  - This also simplifies cross compilation as no cross compiler is
    needed (it's all built into the TinyGo release build).

Disadvantages:
  - Binary size increases by 5-6 kilobytes if -no-debug is used. Binary
    size increases by a much larger margin when debugging symbols are
    included (the default behavior) because musl is built with debugging
    symbols enabled.
  - Musl does things a bit differently than glibc, and some CGo code
    might rely on the glibc behavior.
  - The first build takes a bit longer because musl needs to be built.

As an additional bonus, time is now obtained from the system in a way
that fixes the Y2038 problem because musl has been a bit more agressive
in switching to 64-bit time_t.

builder/build.go

@@ -92,6 +92,13 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 	root := goenv.Get("TINYGOROOT")
 	var libcDependencies []*compileJob
 	switch config.Target.Libc {
+	case "musl":
+		job, err := Musl.load(config, dir)
+		if err != nil {
+			return err
+		}
+		libcDependencies = append(libcDependencies, dummyCompileJob(filepath.Join(filepath.Dir(job.result), "crt1.o")))
+		libcDependencies = append(libcDependencies, job)
 	case "picolibc":
 		libcJob, err := Picolibc.load(config, dir)
 		if err != nil {
@@ -578,12 +585,15 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 			// while we're at it. Relocations can only be compressed when debug
 			// information is stripped.
 			ldflags = append(ldflags, "--strip-debug", "--compress-relocations")
+		} else if config.Target.Linker == "ld.lld" {
+			// ld.lld is also used on Linux.
+			ldflags = append(ldflags, "--strip-debug")
 		} else {
 			switch config.GOOS() {
 			case "linux":
 				// Either real linux or an embedded system (like AVR) that
 				// pretends to be Linux. It's a ELF linker wrapped by GCC in any
-				// case.
+				// case (not ld.lld - that case is handled above).
 				ldflags = append(ldflags, "-Wl,--strip-debug")
 			case "darwin":
 				// MacOS (darwin) doesn't have a linker flag to strip debug

builder/builtins.go

@@ -158,11 +158,11 @@ var aeabiBuiltins = []string{
 // For more information, see: https://compiler-rt.llvm.org/
 var CompilerRT = Library{
 	name: "compiler-rt",
-	cflags: func(headerPath string) []string {
+	cflags: func(target, headerPath string) []string {
 		return []string{"-Werror", "-Wall", "-std=c11", "-nostdlibinc"}
 	},
 	sourceDir: "lib/compiler-rt/lib/builtins",
-	sources: func(target string) []string {
+	librarySources: func(target string) []string {
 		builtins := append([]string{}, genericBuiltins...) // copy genericBuiltins
 		if strings.HasPrefix(target, "arm") || strings.HasPrefix(target, "thumb") {
 			builtins = append(builtins, aeabiBuiltins...)

builder/library.go

@@ -17,31 +17,19 @@ type Library struct {
 	name string
 
 	// makeHeaders creates a header include dir for the library
-	makeHeaders func(includeDir string) error
+	makeHeaders func(target, includeDir string) error
 
 	// cflags returns the C flags specific to this library
-	cflags func(headerPath string) []string
+	cflags func(target, headerPath string) []string
 
 	// The source directory, relative to TINYGOROOT.
 	sourceDir string
 
 	// The source files, relative to sourceDir.
-	sources func(target string) []string
-}
+	librarySources func(target string) []string
 
-// fullPath returns the full path to the source directory.
-func (l *Library) fullPath() string {
-	return filepath.Join(goenv.Get("TINYGOROOT"), l.sourceDir)
-}
-
-// sourcePaths returns a slice with the full paths to the source files.
-func (l *Library) sourcePaths(target string) []string {
-	sources := l.sources(target)
-	paths := make([]string, len(sources))
-	for i, name := range sources {
-		paths[i] = filepath.Join(l.fullPath(), name)
-	}
-	return paths
+	// The source code for the crt1.o file, relative to sourceDir.
+	crt1Source string
 }
 
 // Load the library archive, possibly generating and caching it if needed.
@@ -90,6 +78,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ
 
 	// Make headers if needed.
 	headerPath := filepath.Join(outdir, "include")
+	target := config.Triple()
 	if l.makeHeaders != nil {
 		if _, err = os.Stat(headerPath); err != nil {
 			temporaryHeaderPath, err := ioutil.TempDir(outdir, "include.tmp*")
@@ -97,7 +86,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ
 				return nil, err
 			}
 			defer os.RemoveAll(temporaryHeaderPath)
-			err = l.makeHeaders(temporaryHeaderPath)
+			err = l.makeHeaders(target, temporaryHeaderPath)
 			if err != nil {
 				return nil, err
 			}
@@ -119,11 +108,16 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ
 	// Note: -fdebug-prefix-map is necessary to make the output archive
 	// reproducible. Otherwise the temporary directory is stored in the archive
 	// itself, which varies each run.
-	target := config.Triple()
-	args := append(l.cflags(headerPath), "-c", "-Oz", "-g", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+target, "-fdebug-prefix-map="+dir+"="+remapDir)
+	args := append(l.cflags(target, headerPath), "-c", "-Oz", "-g", "-ffunction-sections", "-fdata-sections", "-Wno-macro-redefined", "--target="+target, "-fdebug-prefix-map="+dir+"="+remapDir)
 	cpu := config.CPU()
 	if cpu != "" {
-		args = append(args, "-mcpu="+cpu)
+		// X86 has deprecated the -mcpu flag, so we need to use -march instead.
+		// However, ARM has not done this.
+		if strings.HasPrefix(target, "i386") || strings.HasPrefix(target, "x86_64") {
+			args = append(args, "-march="+cpu)
+		} else {
+			args = append(args, "-mcpu="+cpu)
+		}
 	}
 	if strings.HasPrefix(target, "arm") || strings.HasPrefix(target, "thumb") {
 		args = append(args, "-fshort-enums", "-fomit-frame-pointer", "-mfloat-abi=soft")
@@ -162,9 +156,15 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ
 
 	// Create jobs to compile all sources. These jobs are depended upon by the
 	// archive job above, so must be run first.
-	for _, srcpath := range l.sourcePaths(target) {
-		srcpath := srcpath // avoid concurrency issues by redefining inside the loop
-		objpath := filepath.Join(dir, filepath.Base(srcpath)+".o")
+	for _, path := range l.librarySources(target) {
+		// Strip leading "../" parts off the path.
+		cleanpath := path
+		for strings.HasPrefix(cleanpath, "../") {
+			cleanpath = cleanpath[3:]
+		}
+		srcpath := filepath.Join(goenv.Get("TINYGOROOT"), l.sourceDir, path)
+		objpath := filepath.Join(dir, cleanpath+".o")
+		os.MkdirAll(filepath.Dir(objpath), 0o777)
 		objs = append(objs, objpath)
 		job.dependencies = append(job.dependencies, &compileJob{
 			description: "compile " + srcpath,
@@ -181,5 +181,31 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ
 		})
 	}
 
+	// Create crt1.o job, if needed.
+	// Add this as a (fake) dependency to the ar file so it gets compiled.
+	// (It could be done in parallel with creating the ar file, but it probably
+	// won't make much of a difference in speed).
+	if l.crt1Source != "" {
+		srcpath := filepath.Join(goenv.Get("TINYGOROOT"), l.sourceDir, l.crt1Source)
+		job.dependencies = append(job.dependencies, &compileJob{
+			description: "compile " + srcpath,
+			run: func(*compileJob) error {
+				var compileArgs []string
+				compileArgs = append(compileArgs, args...)
+				tmpfile, err := ioutil.TempFile(outdir, "crt1.o.tmp*")
+				if err != nil {
+					return err
+				}
+				tmpfile.Close()
+				compileArgs = append(compileArgs, "-o", tmpfile.Name(), srcpath)
+				err = runCCompiler(compileArgs...)
+				if err != nil {
+					return &commandError{"failed to build", srcpath, err}
+				}
+				return os.Rename(tmpfile.Name(), filepath.Join(outdir, "crt1.o"))
+			},
+		})
+	}
+
 	return job, nil
 }

builder/musl.go

@@ -0,0 +1,162 @@
+package builder
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/tinygo-org/tinygo/compileopts"
+	"github.com/tinygo-org/tinygo/goenv"
+)
+
+var Musl = Library{
+	name: "musl",
+	makeHeaders: func(target, includeDir string) error {
+		bits := filepath.Join(includeDir, "bits")
+		err := os.Mkdir(bits, 0777)
+		if err != nil {
+			return err
+		}
+
+		arch := compileopts.MuslArchitecture(target)
+		muslDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib", "musl")
+
+		// Create the file alltypes.h.
+		f, err := os.Create(filepath.Join(bits, "alltypes.h"))
+		if err != nil {
+			return err
+		}
+		infiles := []string{
+			filepath.Join(muslDir, "arch", arch, "bits", "alltypes.h.in"),
+			filepath.Join(muslDir, "include", "alltypes.h.in"),
+		}
+		for _, infile := range infiles {
+			data, err := ioutil.ReadFile(infile)
+			if err != nil {
+				return err
+			}
+			lines := strings.Split(string(data), "\n")
+			for _, line := range lines {
+				if strings.HasPrefix(line, "TYPEDEF ") {
+					matches := regexp.MustCompile(`TYPEDEF (.*) ([^ ]*);`).FindStringSubmatch(line)
+					value := matches[1]
+					name := matches[2]
+					line = fmt.Sprintf("#if defined(__NEED_%s) && !defined(__DEFINED_%s)\ntypedef %s %s;\n#define __DEFINED_%s\n#endif\n", name, name, value, name, name)
+				}
+				if strings.HasPrefix(line, "STRUCT ") {
+					matches := regexp.MustCompile(`STRUCT * ([^ ]*) (.*);`).FindStringSubmatch(line)
+					name := matches[1]
+					value := matches[2]
+					line = fmt.Sprintf("#if defined(__NEED_struct_%s) && !defined(__DEFINED_struct_%s)\nstruct %s %s;\n#define __DEFINED_struct_%s\n#endif\n", name, name, name, value, name)
+				}
+				f.WriteString(line + "\n")
+			}
+		}
+		f.Close()
+
+		// Create the file syscall.h.
+		f, err = os.Create(filepath.Join(bits, "syscall.h"))
+		if err != nil {
+			return err
+		}
+		data, err := ioutil.ReadFile(filepath.Join(muslDir, "arch", arch, "bits", "syscall.h.in"))
+		if err != nil {
+			return err
+		}
+		_, err = f.Write(bytes.ReplaceAll(data, []byte("__NR_"), []byte("SYS_")))
+		if err != nil {
+			return err
+		}
+		f.Close()
+
+		return nil
+	},
+	cflags: func(target, headerPath string) []string {
+		arch := compileopts.MuslArchitecture(target)
+		muslDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib/musl")
+		return []string{
+			"-std=c99",            // same as in musl
+			"-D_XOPEN_SOURCE=700", // same as in musl
+			// Musl triggers some warnings and we don't want to show any
+			// warnings while compiling (only errors or silence), so disable
+			// specific warnings that are triggered in musl.
+			"-Werror",
+			"-Wno-logical-op-parentheses",
+			"-Wno-bitwise-op-parentheses",
+			"-Wno-shift-op-parentheses",
+			"-Wno-ignored-attributes",
+			"-Wno-string-plus-int",
+			"-Qunused-arguments",
+			// Select include dirs. Don't include standard library includes
+			// (that would introduce host dependencies and other complications),
+			// but do include all the include directories expected by musl.
+			"-nostdlibinc",
+			"-I" + muslDir + "/arch/" + arch,
+			"-I" + muslDir + "/arch/generic",
+			"-I" + muslDir + "/src/include",
+			"-I" + muslDir + "/src/internal",
+			"-I" + headerPath,
+			"-I" + muslDir + "/include",
+		}
+	},
+	sourceDir: "lib/musl/src",
+	librarySources: func(target string) []string {
+		arch := compileopts.MuslArchitecture(target)
+		globs := []string{
+			"env/*.c",
+			"errno/*.c",
+			"exit/*.c",
+			"internal/defsysinfo.c",
+			"internal/libc.c",
+			"internal/syscall_ret.c",
+			"internal/vdso.c",
+			"malloc/*.c",
+			"mman/*.c",
+			"signal/*.c",
+			"stdio/*.c",
+			"string/*.c",
+			"thread/" + arch + "/*.s",
+			"thread/" + arch + "/*.c",
+			"thread/*.c",
+			"time/*.c",
+			"unistd/*.c",
+		}
+
+		var sources []string
+		seenSources := map[string]struct{}{}
+		basepath := goenv.Get("TINYGOROOT") + "/lib/musl/src/"
+		for _, pattern := range globs {
+			matches, err := filepath.Glob(basepath + pattern)
+			if err != nil {
+				// From the documentation:
+				// > Glob ignores file system errors such as I/O errors reading
+				// > directories. The only possible returned error is
+				// > ErrBadPattern, when pattern is malformed.
+				// So the only possible error is when the (statically defined)
+				// pattern is wrong. In other words, a programming bug.
+				panic("could not glob source dirs: " + err.Error())
+			}
+			for _, match := range matches {
+				relpath, err := filepath.Rel(basepath, match)
+				if err != nil {
+					// Not sure if this is even possible.
+					panic(err)
+				}
+				// Make sure architecture specific files override generic files.
+				id := strings.ReplaceAll(relpath, "/"+arch+"/", "/")
+				if _, ok := seenSources[id]; ok {
+					// Already seen this file, skipping this (generic) file.
+					continue
+				}
+				seenSources[id] = struct{}{}
+				sources = append(sources, relpath)
+			}
+		}
+		return sources
+	},
+	crt1Source: "../crt/crt1.c", // lib/musl/crt/crt1.c
+}

builder/picolibc.go

@@ -11,14 +11,14 @@ import (
 // based on newlib.
 var Picolibc = Library{
 	name: "picolibc",
-	makeHeaders: func(includeDir string) error {
+	makeHeaders: func(target, includeDir string) error {
 		f, err := os.Create(filepath.Join(includeDir, "picolibc.h"))
 		if err != nil {
 			return err
 		}
 		return f.Close()
 	},
-	cflags: func(headerPath string) []string {
+	cflags: func(target, headerPath string) []string {
 		picolibcDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib/picolibc/newlib/libc")
 		return []string{
 			"-Werror",
@@ -34,7 +34,7 @@ var Picolibc = Library{
 		}
 	},
 	sourceDir: "lib/picolibc/newlib/libc",
-	sources: func(target string) []string {
+	librarySources: func(target string) []string {
 		return picolibcSources
 	},
 }