builder: fixed a problem with multiple process build cases (cc)

.circleci/config.yml

@@ -119,7 +119,7 @@ commands:
             - lib/wasi-libc/sysroot
       - run: go test -v -tags=llvm<<parameters.llvm>> ./cgo ./compileopts ./compiler ./interp ./transform .
       - run: make gen-device -j4
-      - run: make smoketest XTENSA=0
+      - run: make smoketest XTENSA=0 RUN_SMOKETEST_JOBS=4
       - run: make tinygo-test
       - run: make wasmtest
       - save_cache:
@@ -195,7 +195,7 @@ commands:
             - ~/.cache/go-build
             - /go/pkg/mod
       - run: make gen-device -j4
-      - run: make smoketest TINYGO=build/tinygo
+      - run: make smoketest TINYGO=build/tinygo RUN_SMOKETEST_JOBS=4
   build-linux:
     steps:
       - checkout
@@ -278,7 +278,7 @@ commands:
             tar -C ~/lib -xf /tmp/tinygo.linux-amd64.tar.gz
             ln -s ~/lib/tinygo/bin/tinygo /go/bin/tinygo
             tinygo version
-      - run: make smoketest
+      - run: make smoketest RUN_SMOKETEST_JOBS=4
   build-macos:
     steps:
       - checkout
@@ -357,7 +357,7 @@ commands:
             tar -C /usr/local/opt -xf /tmp/tinygo.darwin-amd64.tar.gz
             ln -s /usr/local/opt/tinygo/bin/tinygo /usr/local/bin/tinygo
             tinygo version
-      - run: make smoketest AVR=0
+      - run: make smoketest AVR=0 RUN_SMOKETEST_JOBS=4
       - save_cache:
           key: go-cache-macos-v3-{{ checksum "go.mod" }}-{{ .Environment.CIRCLE_BUILD_NUM }}
           paths:

Makefile

@@ -202,8 +202,11 @@ tinygo-test:
 	$(TINYGO) test text/scanner
 	$(TINYGO) test unicode/utf8
 
-.PHONY: smoketest
+.PHONY: smoketest smoketest-commands
 smoketest:
+	@go run ./src/cmd/run-smoketest make smoketest-commands
+
+smoketest-commands:
 	$(TINYGO) version
 	# test all examples (except pwm)
 	$(TINYGO) build -size short -o test.hex -target=pca10040            examples/blinky1

builder/build.go

@@ -300,7 +300,13 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 				if err != nil {
 					return err
 				}
-				return os.Rename(f.Name(), bitcodePath)
+
+				// Rename may fail if another process is trying to write to
+				// the same file.  However, in this case, the failure is
+				// acceptable because the result of the other process can be
+				// used.
+				os.Rename(f.Name(), bitcodePath)
+				return nil
 			},
 		}
 		jobs = append(jobs, job)

builder/buildcache.go

@@ -84,7 +84,7 @@ func copyFile(src, dst string) error {
 		return err
 	}
 	defer inf.Close()
-	outpath := dst + ".tmp"
+	outpath := src + ".tmp"
 	outf, err := os.Create(outpath)
 	if err != nil {
 		return err
@@ -101,5 +101,10 @@ func copyFile(src, dst string) error {
 		return err
 	}
 
-	return os.Rename(dst+".tmp", dst)
+	// Rename may fail if another process is trying to write to
+	// the same file.  However, in this case, the failure is
+	// acceptable because the result of the other process can be
+	// used.
+	os.Rename(outpath, dst)
+	return nil
 }

builder/cc.go

@@ -167,10 +167,7 @@ func compileAndCacheCFile(abspath, tmpdir string, cflags []string, printCommands
 	if err != nil {
 		return "", err
 	}
-	err = os.Rename(f.Name(), depfileCachePath)
-	if err != nil {
-		return "", err
-	}
+	os.Rename(f.Name(), depfileCachePath)
 
 	// Move temporary object file to final location.
 	outpath, err := makeCFileCachePath(dependencySlice, depfileNameHash)

go.mod

@@ -9,7 +9,9 @@ require (
 	github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf
 	github.com/marcinbor85/gohex v0.0.0-20200531091804-343a4b548892
 	github.com/mattn/go-colorable v0.1.8
+	github.com/sago35/ochan v0.0.0-20200313013834-e0f46da4579f // indirect
 	go.bug.st/serial v1.1.2
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78
 	golang.org/x/tools v0.0.0-20200216192241-b320d3a0f5a2
 	tinygo.org/x/go-llvm v0.0.0-20210325115028-e7b85195e81c

go.sum

@@ -28,6 +28,8 @@ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sago35/ochan v0.0.0-20200313013834-e0f46da4579f h1:bHOcIl3AODQMzleLeSGs3xbIvAzhcgLTUwd8IUKqLMI=
+github.com/sago35/ochan v0.0.0-20200313013834-e0f46da4579f/go.mod h1:55Pg0jnassdnFxlhS8HIU5pdYhEBYftGaknIP4bBUq8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 go.bug.st/serial v1.0.0 h1:ogEPzrllCsnG00EqKRjeYvPRsO7NJW6DqykzkdD6E/k=
@@ -41,6 +43,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 h1:ZBzSG/7F4eNKz2L3GE9o300RX0Az1Bw5HF7PDraD+qU=

src/cmd/run-smoketest/main.go

@@ -0,0 +1,219 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/md5"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/sago35/ochan"
+	"golang.org/x/sync/errgroup"
+)
+
+func main() {
+	threads := runtime.NumCPU()
+	if j, ok := os.LookupEnv(`RUN_SMOKETEST_JOBS`); ok {
+		n, err := strconv.ParseInt(j, 0, 0)
+		if err == nil {
+			threads = int(n)
+		}
+	}
+
+	flag.IntVar(&threads, "threads", threads, "threads of make smoketest")
+	flag.Parse()
+
+	if len(os.Args) < 2 {
+		fmt.Printf("usage: %s make smoketest\n", filepath.Base(os.Args[0]))
+		os.Exit(1)
+	}
+
+	commands, err := getMakeCommands(flag.Args())
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
+
+	err = run(commands, threads)
+	if err != nil {
+		log.Fatalf(err.Error())
+	}
+}
+
+func getMakeCommands(args []string) ([]string, error) {
+	var buf bytes.Buffer
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Args = append(cmd.Args, "-n")
+	cmd.Stdout = &buf
+	cmd.Stderr = &buf
+	err := cmd.Run()
+	if err != nil {
+		return nil, err
+	}
+
+	commands := []string{}
+	scanner := bufio.NewScanner(&buf)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		fields := strings.Fields(line)
+		if filepath.Base(fields[0]) == "tinygo" && (fields[1] == "build" || fields[1] == "version") {
+			commands = append(commands, line)
+		} else if strings.HasPrefix(line, "#") {
+			commands = append(commands, line)
+		}
+	}
+
+	return commands, nil
+}
+
+func run(commands []string, threads int) error {
+	ch := make(chan string, 1000)
+	o := ochan.NewOchan(ch, 100)
+
+	go func() {
+		limit := make(chan struct{}, threads)
+		var eg errgroup.Group
+
+		for _, command := range commands {
+			command := command
+			select {
+			case limit <- struct{}{}:
+				och := o.GetCh()
+				eg.Go(func() error {
+					var err error
+
+					if runtime.GOOS == `windows` {
+						command = convertToWindowsPath(command)
+					}
+
+					fields := strings.Fields(command)
+					if filepath.Base(fields[0]) == "tinygo" && fields[1] == "build" {
+						err = buildAndmd5sum(command, och)
+					} else if strings.HasPrefix(command, "#") {
+						och <- fmt.Sprintf("%s\n", command)
+					} else {
+						err = runCommand(command, och)
+					}
+					close(och)
+					<-limit
+					return err
+				})
+			}
+		}
+
+		if err := eg.Wait(); err != nil {
+			log.Fatal(err)
+		}
+
+		o.Wait()
+		close(ch)
+	}()
+
+	for s := range ch {
+		fmt.Print(s)
+	}
+
+	return nil
+}
+
+func runCommand(command string, ch chan string) error {
+	var buf bytes.Buffer
+
+	fmt.Fprintf(&buf, "%s\n", command)
+
+	args := strings.Fields(command)
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Stdout = &buf
+	cmd.Stderr = &buf
+
+	err := cmd.Run()
+	if err != nil {
+		ch <- buf.String()
+		return err
+	}
+
+	ch <- buf.String()
+	return nil
+}
+
+func buildAndmd5sum(command string, ch chan string) error {
+	var buf bytes.Buffer
+
+	fmt.Fprintf(&buf, "%s\n", command)
+
+	args := strings.Fields(command)
+
+	output := ""
+	tmpOutput := ""
+	tmpdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		return err
+	}
+	for i := range args {
+		if args[i] == "-o" {
+			output = args[i+1]
+			ext := filepath.Ext(output)
+			tmpOutput = filepath.Join(tmpdir, fmt.Sprintf("%s.%s", filepath.Base(output), ext))
+			args[i+1] = tmpOutput
+		} else if strings.HasPrefix(args[i], "-o=") {
+			output = args[i][3:]
+			ext := filepath.Ext(output)
+			tmpOutput = filepath.Join(tmpdir, fmt.Sprintf("%s.%s", filepath.Base(output), ext))
+			args[i] = fmt.Sprintf("-o=%s", tmpOutput)
+		}
+	}
+
+	cmd := exec.Command(args[0], args[1:]...)
+	cmd.Stdout = &buf
+	cmd.Stderr = &buf
+
+	err = cmd.Run()
+	if err != nil {
+		ch <- buf.String()
+		return err
+	}
+
+	md5str, err := calcMD5(tmpOutput)
+	if err != nil {
+		ch <- buf.String()
+		return err
+	}
+	fmt.Fprintf(&buf, "%s  %s\n", md5str, output)
+
+	ch <- buf.String()
+	return nil
+}
+
+func calcMD5(f string) (string, error) {
+	r, err := os.Open(f)
+	if err != nil {
+		return "", err
+	}
+	defer r.Close()
+
+	h := md5.New()
+	if _, err := io.Copy(h, r); err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%x", h.Sum(nil)), nil
+}
+
+func convertToWindowsPath(command string) string {
+	if !strings.HasPrefix(command, `/`) {
+		return command
+	}
+
+	command = fmt.Sprintf("%c:%s", command[1], command[2:])
+	return command
+}