From 632357ffb92123d9d24c596394f874157c355f6d Mon Sep 17 00:00:00 2001 From: Ayke van Laethem Date: Thu, 20 Mar 2025 14:41:52 +0100 Subject: [PATCH] builder: build wasi-libc inside TinyGo Instead of relying on a build when TinyGo is being built, do it like all other libraries when it is needed. This brings a few benefits: * No more running `make wasi-libc` on the command line as a special case for WebAssembly. The generic `git submodule update --init` step is now enough for wasi-libc. * It becomes much easier to customize the build per system. For example: include/exclude malloc as needed, disable/enable bulk memory operations per target, etc. --- .circleci/config.yml | 8 - .github/workflows/build-macos.yml | 9 - .github/workflows/linux.yml | 18 -- .github/workflows/windows.yml | 9 - GNUmakefile | 44 +++-- builder/build.go | 10 +- builder/library.go | 7 + builder/musl.go | 71 ++++---- builder/wasilibc.go | 278 ++++++++++++++++++++++++++++++ compileopts/config.go | 4 +- flake.nix | 9 - 11 files changed, 362 insertions(+), 105 deletions(-) create mode 100644 builder/wasilibc.go diff --git a/.circleci/config.yml b/.circleci/config.yml index a3a052c28..257e5971a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,14 +73,6 @@ commands: - go-cache-v4-{{ checksum "go.mod" }} - llvm-source-linux - run: go install -tags=llvm<> . - - restore_cache: - keys: - - wasi-libc-sysroot-systemclang-v7 - - run: make wasi-libc - - save_cache: - key: wasi-libc-sysroot-systemclang-v7 - paths: - - lib/wasi-libc/sysroot - when: condition: <> steps: diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 25b597178..46ec2cc7c 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -89,15 +89,6 @@ jobs: with: key: ${{ steps.cache-llvm-build.outputs.cache-primary-key }} path: llvm-build - - name: Cache wasi-libc sysroot - uses: actions/cache@v4 - id: cache-wasi-libc - with: - key: wasi-libc-sysroot-${{ matrix.os }}-v1 - path: lib/wasi-libc/sysroot - - name: Build wasi-libc - if: steps.cache-wasi-libc.outputs.cache-hit != 'true' - run: make wasi-libc - name: make gen-device run: make -j3 gen-device - name: Test TinyGo diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 0a8fabf1f..803759892 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -104,15 +104,6 @@ jobs: run: | apk add cmake samurai python3 make binaryen STATIC=1 - - name: Cache wasi-libc - uses: actions/cache@v4 - id: cache-wasi-libc - with: - key: wasi-libc-sysroot-linux-alpine-v2 - path: lib/wasi-libc/sysroot - - name: Build wasi-libc - if: steps.cache-wasi-libc.outputs.cache-hit != 'true' - run: make wasi-libc - name: Install fpm run: | gem install --version 4.0.7 public_suffix @@ -258,15 +249,6 @@ jobs: - name: Build Binaryen if: steps.cache-binaryen.outputs.cache-hit != 'true' run: make binaryen - - name: Cache wasi-libc - uses: actions/cache@v4 - id: cache-wasi-libc - with: - key: wasi-libc-sysroot-linux-asserts-v6 - path: lib/wasi-libc/sysroot - - name: Build wasi-libc - if: steps.cache-wasi-libc.outputs.cache-hit != 'true' - run: make wasi-libc - run: make gen-device -j4 - name: Test TinyGo run: make ASSERT=1 test diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 42365f59b..692790baf 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -91,15 +91,6 @@ jobs: with: key: ${{ steps.cache-llvm-build.outputs.cache-primary-key }} path: llvm-build - - name: Cache wasi-libc sysroot - uses: actions/cache@v4 - id: cache-wasi-libc - with: - key: wasi-libc-sysroot-v5 - path: lib/wasi-libc/sysroot - - name: Build wasi-libc - if: steps.cache-wasi-libc.outputs.cache-hit != 'true' - run: make wasi-libc - name: Cache Go cache uses: actions/cache@v4 with: diff --git a/GNUmakefile b/GNUmakefile index ebcff1e51..d8770c325 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -261,13 +261,6 @@ build/wasm-opt$(EXE): cp lib/binaryen/bin/wasm-opt$(EXE) build/wasm-opt$(EXE) endif -# Build wasi-libc sysroot -.PHONY: wasi-libc -wasi-libc: lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a -lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a: - @if [ ! -e lib/wasi-libc/Makefile ]; then echo "Submodules have not been downloaded. Please download them using:\n git submodule update --init"; exit 1; fi - cd lib/wasi-libc && $(MAKE) -j4 EXTRA_CFLAGS="-O2 -g -DNDEBUG -mnontrapping-fptoint -msign-ext" MALLOC_IMPL=none CC="$(CLANG)" AR=$(LLVM_AR) NM=$(LLVM_NM) - # Generate WASI syscall bindings WASM_TOOLS_MODULE=go.bytecodealliance.org .PHONY: wasi-syscall @@ -295,7 +288,7 @@ check-nodejs-version: tinygo: ## Build the TinyGo compiler @if [ ! -f "$(LLVM_BUILDDIR)/bin/llvm-config" ]; then echo "Fetch and build LLVM first by running:"; echo " $(MAKE) llvm-source"; echo " $(MAKE) $(LLVM_BUILDDIR)"; exit 1; fi CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOENVFLAGS) $(GO) build -buildmode exe -o build/tinygo$(EXE) -tags "byollvm osusergo" . -test: wasi-libc check-nodejs-version +test: check-nodejs-version CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags "byollvm osusergo" $(GOTESTPKGS) # Standard library packages that pass tests on darwin, linux, wasi, and windows, but take over a minute in wasi @@ -528,9 +521,9 @@ test-corpus: CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml test-corpus-fast: CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus -short . -corpus=testdata/corpus.yaml -test-corpus-wasi: wasi-libc +test-corpus-wasi: CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml -target=wasip1 -test-corpus-wasip2: wasi-libc +test-corpus-wasip2: CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test $(GOTESTFLAGS) -timeout=1h -buildmode exe -tags byollvm -run TestCorpus . -corpus=testdata/corpus.yaml -target=wasip2 .PHONY: testchdir @@ -941,7 +934,7 @@ endif wasmtest: $(GO) test ./tests/wasm -build/release: tinygo gen-device wasi-libc $(if $(filter 1,$(USE_SYSTEM_BINARYEN)),,binaryen) +build/release: tinygo gen-device $(if $(filter 1,$(USE_SYSTEM_BINARYEN)),,binaryen) @mkdir -p build/release/tinygo/bin @mkdir -p build/release/tinygo/lib/bdwgc @mkdir -p build/release/tinygo/lib/clang/include @@ -956,7 +949,7 @@ build/release: tinygo gen-device wasi-libc $(if $(filter 1,$(USE_SYSTEM_BINARYEN @mkdir -p build/release/tinygo/lib/nrfx @mkdir -p build/release/tinygo/lib/picolibc/newlib/libc @mkdir -p build/release/tinygo/lib/picolibc/newlib/libm - @mkdir -p build/release/tinygo/lib/wasi-libc/libc-bottom-half/headers + @mkdir -p build/release/tinygo/lib/wasi-libc/libc-bottom-half @mkdir -p build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch @mkdir -p build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src @mkdir -p build/release/tinygo/lib/wasi-cli/ @@ -1019,15 +1012,36 @@ endif @cp -rp lib/picolibc/newlib/libm/common build/release/tinygo/lib/picolibc/newlib/libm @cp -rp lib/picolibc/newlib/libm/math build/release/tinygo/lib/picolibc/newlib/libm @cp -rp lib/picolibc-stdio.c build/release/tinygo/lib - @cp -rp lib/wasi-libc/libc-bottom-half/headers/public build/release/tinygo/lib/wasi-libc/libc-bottom-half/headers + @cp -rp lib/wasi-libc/libc-bottom-half/cloudlibc build/release/tinygo/lib/wasi-libc/libc-bottom-half + @cp -rp lib/wasi-libc/libc-bottom-half/headers build/release/tinygo/lib/wasi-libc/libc-bottom-half + @cp -rp lib/wasi-libc/libc-bottom-half/sources build/release/tinygo/lib/wasi-libc/libc-bottom-half + @cp -rp lib/wasi-libc/libc-top-half/headers build/release/tinygo/lib/wasi-libc/libc-top-half @cp -rp lib/wasi-libc/libc-top-half/musl/arch/generic build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch @cp -rp lib/wasi-libc/libc-top-half/musl/arch/wasm32 build/release/tinygo/lib/wasi-libc/libc-top-half/musl/arch + @cp -rp lib/wasi-libc/libc-top-half/musl/include build/release/tinygo/lib/wasi-libc/libc-top-half/musl + @cp -rp lib/wasi-libc/libc-top-half/musl/src/conf build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/dirent build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/env build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/errno build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/exit build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/fcntl build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/fenv build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src @cp -rp lib/wasi-libc/libc-top-half/musl/src/include build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src @cp -rp lib/wasi-libc/libc-top-half/musl/src/internal build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/legacy build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/locale build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src @cp -rp lib/wasi-libc/libc-top-half/musl/src/math build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/misc build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/multibyte build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/network build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/stat build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/stdio build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/stdlib build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src @cp -rp lib/wasi-libc/libc-top-half/musl/src/string build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src - @cp -rp lib/wasi-libc/libc-top-half/musl/include build/release/tinygo/lib/wasi-libc/libc-top-half/musl - @cp -rp lib/wasi-libc/sysroot build/release/tinygo/lib/wasi-libc/sysroot + @cp -rp lib/wasi-libc/libc-top-half/musl/src/thread build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/time build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/musl/src/unistd build/release/tinygo/lib/wasi-libc/libc-top-half/musl/src + @cp -rp lib/wasi-libc/libc-top-half/sources build/release/tinygo/lib/wasi-libc/libc-top-half @cp -rp lib/wasi-cli/wit build/release/tinygo/lib/wasi-cli/wit @cp -rp llvm-project/compiler-rt/lib/builtins build/release/tinygo/lib/compiler-rt-builtins @cp -rp llvm-project/compiler-rt/LICENSE.TXT build/release/tinygo/lib/compiler-rt-builtins diff --git a/builder/build.go b/builder/build.go index d921f95a0..ee5f97ec5 100644 --- a/builder/build.go +++ b/builder/build.go @@ -14,7 +14,6 @@ import ( "fmt" "go/types" "hash/crc32" - "io/fs" "math/bits" "os" "os/exec" @@ -168,11 +167,12 @@ func Build(pkgName, outpath, tmpdir string, config *compileopts.Config) (BuildRe defer unlock() libcDependencies = append(libcDependencies, libcJob) case "wasi-libc": - path := filepath.Join(root, "lib/wasi-libc/sysroot/lib/wasm32-wasi/libc.a") - if _, err := os.Stat(path); errors.Is(err, fs.ErrNotExist) { - return BuildResult{}, errors.New("could not find wasi-libc, perhaps you need to run `make wasi-libc`?") + libcJob, unlock, err := libWasiLibc.load(config, tmpdir) + if err != nil { + return BuildResult{}, err } - libcDependencies = append(libcDependencies, dummyCompileJob(path)) + defer unlock() + libcDependencies = append(libcDependencies, libcJob) case "wasmbuiltins": libcJob, unlock, err := libWasmBuiltins.load(config, tmpdir) if err != nil { diff --git a/builder/library.go b/builder/library.go index 0c73a515a..53ae5d9d6 100644 --- a/builder/library.go +++ b/builder/library.go @@ -25,6 +25,9 @@ type Library struct { // cflags returns the C flags specific to this library cflags func(target, headerPath string) []string + // cflagsForFile returns additional C flags for a particular source file. + cflagsForFile func(path string) []string + // needsLibc is set to true if this library needs libc headers. needsLibc bool @@ -226,6 +229,7 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ } for _, path := range paths { // Strip leading "../" parts off the path. + path := path cleanpath := path for strings.HasPrefix(cleanpath, "../") { cleanpath = cleanpath[3:] @@ -239,6 +243,9 @@ func (l *Library) load(config *compileopts.Config, tmpdir string) (job *compileJ run: func(*compileJob) error { var compileArgs []string compileArgs = append(compileArgs, args...) + if l.cflagsForFile != nil { + compileArgs = append(compileArgs, l.cflagsForFile(path)...) + } compileArgs = append(compileArgs, "-o", objpath, srcpath) if config.Options.PrintCommands != nil { config.Options.PrintCommands("clang", compileArgs...) diff --git a/builder/musl.go b/builder/musl.go index dc03be46f..419533255 100644 --- a/builder/musl.go +++ b/builder/musl.go @@ -12,6 +12,45 @@ import ( "github.com/tinygo-org/tinygo/goenv" ) +// Create the alltypes.h file from the appropriate alltypes.h.in files. +func buildMuslAllTypes(arch, muslDir, outputBitsDir string) error { + // Create the file alltypes.h. + f, err := os.Create(filepath.Join(outputBitsDir, "alltypes.h")) + if err != nil { + return err + } + infiles := []string{ + filepath.Join(muslDir, "arch", arch, "bits", "alltypes.h.in"), + filepath.Join(muslDir, "include", "alltypes.h.in"), + } + for _, infile := range infiles { + data, err := os.ReadFile(infile) + if err != nil { + return err + } + lines := strings.Split(string(data), "\n") + for _, line := range lines { + if strings.HasPrefix(line, "TYPEDEF ") { + matches := regexp.MustCompile(`TYPEDEF (.*) ([^ ]*);`).FindStringSubmatch(line) + value := matches[1] + name := matches[2] + line = fmt.Sprintf("#if defined(__NEED_%s) && !defined(__DEFINED_%s)\ntypedef %s %s;\n#define __DEFINED_%s\n#endif\n", name, name, value, name, name) + } + if strings.HasPrefix(line, "STRUCT ") { + matches := regexp.MustCompile(`STRUCT * ([^ ]*) (.*);`).FindStringSubmatch(line) + name := matches[1] + value := matches[2] + line = fmt.Sprintf("#if defined(__NEED_struct_%s) && !defined(__DEFINED_struct_%s)\nstruct %s %s;\n#define __DEFINED_struct_%s\n#endif\n", name, name, name, value, name) + } + _, err := f.WriteString(line + "\n") + if err != nil { + return err + } + } + } + return f.Close() +} + var libMusl = Library{ name: "musl", makeHeaders: func(target, includeDir string) error { @@ -24,41 +63,13 @@ var libMusl = Library{ arch := compileopts.MuslArchitecture(target) muslDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib", "musl") - // Create the file alltypes.h. - f, err := os.Create(filepath.Join(bits, "alltypes.h")) + err = buildMuslAllTypes(arch, muslDir, bits) if err != nil { return err } - infiles := []string{ - filepath.Join(muslDir, "arch", arch, "bits", "alltypes.h.in"), - filepath.Join(muslDir, "include", "alltypes.h.in"), - } - for _, infile := range infiles { - data, err := os.ReadFile(infile) - if err != nil { - return err - } - lines := strings.Split(string(data), "\n") - for _, line := range lines { - if strings.HasPrefix(line, "TYPEDEF ") { - matches := regexp.MustCompile(`TYPEDEF (.*) ([^ ]*);`).FindStringSubmatch(line) - value := matches[1] - name := matches[2] - line = fmt.Sprintf("#if defined(__NEED_%s) && !defined(__DEFINED_%s)\ntypedef %s %s;\n#define __DEFINED_%s\n#endif\n", name, name, value, name, name) - } - if strings.HasPrefix(line, "STRUCT ") { - matches := regexp.MustCompile(`STRUCT * ([^ ]*) (.*);`).FindStringSubmatch(line) - name := matches[1] - value := matches[2] - line = fmt.Sprintf("#if defined(__NEED_struct_%s) && !defined(__DEFINED_struct_%s)\nstruct %s %s;\n#define __DEFINED_struct_%s\n#endif\n", name, name, name, value, name) - } - f.WriteString(line + "\n") - } - } - f.Close() // Create the file syscall.h. - f, err = os.Create(filepath.Join(bits, "syscall.h")) + f, err := os.Create(filepath.Join(bits, "syscall.h")) if err != nil { return err } diff --git a/builder/wasilibc.go b/builder/wasilibc.go new file mode 100644 index 000000000..9b8ad7a5e --- /dev/null +++ b/builder/wasilibc.go @@ -0,0 +1,278 @@ +package builder + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/tinygo-org/tinygo/goenv" +) + +var libWasiLibc = Library{ + name: "wasi-libc", + makeHeaders: func(target, includeDir string) error { + bits := filepath.Join(includeDir, "bits") + err := os.Mkdir(bits, 0777) + if err != nil { + return err + } + + muslDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib", "wasi-libc/libc-top-half/musl") + err = buildMuslAllTypes("wasm32", muslDir, bits) + if err != nil { + return err + } + + // See MUSL_OMIT_HEADERS in the Makefile. + omitHeaders := map[string]struct{}{ + "syslog.h": {}, + "wait.h": {}, + "ucontext.h": {}, + "paths.h": {}, + "utmp.h": {}, + "utmpx.h": {}, + "lastlog.h": {}, + "elf.h": {}, + "link.h": {}, + "pwd.h": {}, + "shadow.h": {}, + "grp.h": {}, + "mntent.h": {}, + "netdb.h": {}, + "resolv.h": {}, + "pty.h": {}, + "dlfcn.h": {}, + "setjmp.h": {}, + "ulimit.h": {}, + "wordexp.h": {}, + "spawn.h": {}, + "termios.h": {}, + "libintl.h": {}, + "aio.h": {}, + + "stdarg.h": {}, + "stddef.h": {}, + + "pthread.h": {}, + } + + for _, glob := range [][2]string{ + {"libc-bottom-half/headers/public/*.h", ""}, + {"libc-bottom-half/headers/public/wasi/*.h", "wasi"}, + {"libc-top-half/musl/arch/wasm32/bits/*.h", "bits"}, + {"libc-top-half/musl/include/*.h", ""}, + {"libc-top-half/musl/include/netinet/*.h", "netinet"}, + {"libc-top-half/musl/include/sys/*.h", "sys"}, + } { + matches, _ := filepath.Glob(filepath.Join(goenv.Get("TINYGOROOT"), "lib/wasi-libc", glob[0])) + outDir := filepath.Join(includeDir, glob[1]) + os.MkdirAll(outDir, 0o777) + for _, match := range matches { + name := filepath.Base(match) + if _, ok := omitHeaders[name]; ok { + continue + } + data, err := os.ReadFile(match) + if err != nil { + return err + } + err = os.WriteFile(filepath.Join(outDir, name), data, 0o666) + if err != nil { + return err + } + } + } + + return nil + }, + cflags: func(target, headerPath string) []string { + libcDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib/wasi-libc") + return []string{ + "-Werror", + "-Wall", + "-std=gnu11", + "-nostdlibinc", + "-mnontrapping-fptoint", "-msign-ext", + "-Wno-null-pointer-arithmetic", "-Wno-unused-parameter", "-Wno-sign-compare", "-Wno-unused-variable", "-Wno-unused-function", "-Wno-ignored-attributes", "-Wno-missing-braces", "-Wno-ignored-pragmas", "-Wno-unused-but-set-variable", "-Wno-unknown-warning-option", + "-Wno-parentheses", "-Wno-shift-op-parentheses", "-Wno-bitwise-op-parentheses", "-Wno-logical-op-parentheses", "-Wno-string-plus-int", "-Wno-dangling-else", "-Wno-unknown-pragmas", + "-DNDEBUG", + "-D__wasilibc_printscan_no_long_double", + "-D__wasilibc_printscan_full_support_option=\"long double support is disabled\"", + "-isystem", headerPath, + "-I" + libcDir + "/libc-top-half/musl/src/include", + "-I" + libcDir + "/libc-top-half/musl/src/internal", + "-I" + libcDir + "/libc-top-half/musl/arch/wasm32", + "-I" + libcDir + "/libc-top-half/musl/arch/generic", + "-I" + libcDir + "/libc-top-half/headers/private", + } + }, + cflagsForFile: func(path string) []string { + if strings.HasPrefix(path, "libc-bottom-half"+string(os.PathSeparator)) { + libcDir := filepath.Join(goenv.Get("TINYGOROOT"), "lib/wasi-libc") + return []string{ + "-I" + libcDir + "/libc-bottom-half/headers/private", + "-I" + libcDir + "/libc-bottom-half/cloudlibc/src/include", + "-I" + libcDir + "/libc-bottom-half/cloudlibc/src", + } + } + return nil + }, + sourceDir: func() string { return filepath.Join(goenv.Get("TINYGOROOT"), "lib/wasi-libc") }, + librarySources: func(target string) ([]string, error) { + type filePattern struct { + glob string + exclude []string + } + + // See: LIBC_TOP_HALF_MUSL_SOURCES in the Makefile + globs := []filePattern{ + // Top half: mostly musl sources. + {glob: "libc-top-half/sources/*.c"}, + {glob: "libc-top-half/musl/src/conf/*.c"}, + {glob: "libc-top-half/musl/src/internal/*.c", exclude: []string{ + "procfdname.c", "syscall.c", "syscall_ret.c", "vdso.c", "version.c", + }}, + {glob: "libc-top-half/musl/src/locale/*.c", exclude: []string{ + "dcngettext.c", "textdomain.c", "bind_textdomain_codeset.c"}}, + {glob: "libc-top-half/musl/src/math/*.c", exclude: []string{ + "__signbit.c", "__signbitf.c", "__signbitl.c", + "__fpclassify.c", "__fpclassifyf.c", "__fpclassifyl.c", + "ceilf.c", "ceil.c", + "floorf.c", "floor.c", + "truncf.c", "trunc.c", + "rintf.c", "rint.c", + "nearbyintf.c", "nearbyint.c", + "sqrtf.c", "sqrt.c", + "fabsf.c", "fabs.c", + "copysignf.c", "copysign.c", + "fminf.c", "fmaxf.c", + "fmin.c", "fmax.c,", + }}, + {glob: "libc-top-half/musl/src/multibyte/*.c"}, + {glob: "libc-top-half/musl/src/stdio/*.c", exclude: []string{ + "vfwscanf.c", "vfwprintf.c", // long double is unsupported + "__lockfile.c", "flockfile.c", "funlockfile.c", "ftrylockfile.c", + "rename.c", + "tmpnam.c", "tmpfile.c", "tempnam.c", + "popen.c", "pclose.c", + "remove.c", + "gets.c"}}, + {glob: "libc-top-half/musl/src/stdlib/*.c"}, + {glob: "libc-top-half/musl/src/string/*.c", exclude: []string{ + "strsignal.c"}}, + + // Bottom half: connect top half to WASI equivalents. + {glob: "libc-bottom-half/cloudlibc/src/libc/*/*.c"}, + {glob: "libc-bottom-half/cloudlibc/src/libc/sys/*/*.c"}, + {glob: "libc-bottom-half/sources/*.c"}, + } + + // See: LIBC_TOP_HALF_MUSL_SOURCES in the Makefile + sources := []string{ + "libc-top-half/musl/src/misc/a64l.c", + "libc-top-half/musl/src/misc/basename.c", + "libc-top-half/musl/src/misc/dirname.c", + "libc-top-half/musl/src/misc/ffs.c", + "libc-top-half/musl/src/misc/ffsl.c", + "libc-top-half/musl/src/misc/ffsll.c", + "libc-top-half/musl/src/misc/fmtmsg.c", + "libc-top-half/musl/src/misc/getdomainname.c", + "libc-top-half/musl/src/misc/gethostid.c", + "libc-top-half/musl/src/misc/getopt.c", + "libc-top-half/musl/src/misc/getopt_long.c", + "libc-top-half/musl/src/misc/getsubopt.c", + "libc-top-half/musl/src/misc/uname.c", + "libc-top-half/musl/src/misc/nftw.c", + "libc-top-half/musl/src/errno/strerror.c", + "libc-top-half/musl/src/network/htonl.c", + "libc-top-half/musl/src/network/htons.c", + "libc-top-half/musl/src/network/ntohl.c", + "libc-top-half/musl/src/network/ntohs.c", + "libc-top-half/musl/src/network/inet_ntop.c", + "libc-top-half/musl/src/network/inet_pton.c", + "libc-top-half/musl/src/network/inet_aton.c", + "libc-top-half/musl/src/network/in6addr_any.c", + "libc-top-half/musl/src/network/in6addr_loopback.c", + "libc-top-half/musl/src/fenv/fenv.c", + "libc-top-half/musl/src/fenv/fesetround.c", + "libc-top-half/musl/src/fenv/feupdateenv.c", + "libc-top-half/musl/src/fenv/fesetexceptflag.c", + "libc-top-half/musl/src/fenv/fegetexceptflag.c", + "libc-top-half/musl/src/fenv/feholdexcept.c", + "libc-top-half/musl/src/exit/exit.c", + "libc-top-half/musl/src/exit/atexit.c", + "libc-top-half/musl/src/exit/assert.c", + "libc-top-half/musl/src/exit/quick_exit.c", + "libc-top-half/musl/src/exit/at_quick_exit.c", + "libc-top-half/musl/src/time/strftime.c", + "libc-top-half/musl/src/time/asctime.c", + "libc-top-half/musl/src/time/asctime_r.c", + "libc-top-half/musl/src/time/ctime.c", + "libc-top-half/musl/src/time/ctime_r.c", + "libc-top-half/musl/src/time/wcsftime.c", + "libc-top-half/musl/src/time/strptime.c", + "libc-top-half/musl/src/time/difftime.c", + "libc-top-half/musl/src/time/timegm.c", + "libc-top-half/musl/src/time/ftime.c", + "libc-top-half/musl/src/time/gmtime.c", + "libc-top-half/musl/src/time/gmtime_r.c", + "libc-top-half/musl/src/time/timespec_get.c", + "libc-top-half/musl/src/time/getdate.c", + "libc-top-half/musl/src/time/localtime.c", + "libc-top-half/musl/src/time/localtime_r.c", + "libc-top-half/musl/src/time/mktime.c", + "libc-top-half/musl/src/time/__tm_to_secs.c", + "libc-top-half/musl/src/time/__month_to_secs.c", + "libc-top-half/musl/src/time/__secs_to_tm.c", + "libc-top-half/musl/src/time/__year_to_secs.c", + "libc-top-half/musl/src/time/__tz.c", + "libc-top-half/musl/src/fcntl/creat.c", + "libc-top-half/musl/src/dirent/alphasort.c", + "libc-top-half/musl/src/dirent/versionsort.c", + "libc-top-half/musl/src/env/__stack_chk_fail.c", + "libc-top-half/musl/src/env/clearenv.c", + "libc-top-half/musl/src/env/getenv.c", + "libc-top-half/musl/src/env/putenv.c", + "libc-top-half/musl/src/env/setenv.c", + "libc-top-half/musl/src/env/unsetenv.c", + "libc-top-half/musl/src/unistd/posix_close.c", + "libc-top-half/musl/src/stat/futimesat.c", + "libc-top-half/musl/src/legacy/getpagesize.c", + "libc-top-half/musl/src/thread/thrd_sleep.c", + } + + basepath := goenv.Get("TINYGOROOT") + "/lib/wasi-libc/" + for _, pattern := range globs { + matches, err := filepath.Glob(basepath + pattern.glob) + if err != nil { + // From the documentation: + // > Glob ignores file system errors such as I/O errors reading + // > directories. The only possible returned error is + // > ErrBadPattern, when pattern is malformed. + // So the only possible error is when the (statically defined) + // pattern is wrong. In other words, a programming bug. + return nil, fmt.Errorf("wasi-libc: could not glob source dirs: %w", err) + } + if len(matches) == 0 { + return nil, fmt.Errorf("wasi-libc: did not find any files for pattern %#v", pattern) + } + excludeSet := map[string]struct{}{} + for _, exclude := range pattern.exclude { + excludeSet[exclude] = struct{}{} + } + for _, match := range matches { + if _, ok := excludeSet[filepath.Base(match)]; ok { + continue + } + relpath, err := filepath.Rel(basepath, match) + if err != nil { + // Not sure if this is even possible. + return nil, err + } + sources = append(sources, relpath) + } + } + return sources, nil + }, +} diff --git a/compileopts/config.go b/compileopts/config.go index 411cd7c84..9fa482952 100644 --- a/compileopts/config.go +++ b/compileopts/config.go @@ -371,10 +371,10 @@ func (c *Config) LibcCFlags() []string { "-isystem", filepath.Join(root, "lib", "musl", "include"), } case "wasi-libc": - root := goenv.Get("TINYGOROOT") + path := c.LibcPath("wasi-libc") return []string{ "-nostdlibinc", - "-isystem", root + "/lib/wasi-libc/sysroot/include", + "-isystem", filepath.Join(path, "include"), } case "wasmbuiltins": // nothing to add (library is purely for builtins) diff --git a/flake.nix b/flake.nix index 25ffc7020..136e1014f 100644 --- a/flake.nix +++ b/flake.nix @@ -21,7 +21,6 @@ # make llvm-source # fetch compiler-rt # git submodule update --init # fetch lots of other libraries and SVD files # make gen-device -j4 # build src/device/*/*.go files -# make wasi-libc # build support for wasi/wasm # # With this, you should have an environment that can compile anything - except # for the Xtensa architecture (ESP8266/ESP32) because support for that lives in @@ -64,14 +63,6 @@ #openocd ]; shellHook= '' - # Configure CLANG, LLVM_AR, and LLVM_NM for `make wasi-libc`. - # Without setting these explicitly, Homebrew versions might be used - # or the default `ar` and `nm` tools might be used (which don't - # support wasi). - export CLANG="clang-18 -resource-dir ${llvmPackages_18.clang.cc.lib}/lib/clang/18" - export LLVM_AR=llvm-ar - export LLVM_NM=llvm-nm - # Make `make smoketest` work (the default is `md5`, while Nix only # has `md5sum`). export MD5SUM=md5sum