tools: process SVD files in parallel

This speeds up the generation of SVD files, for example as part of a
package build (where parallelism may not be desired, or tools should
auto-detect the number of threads to use).

.circleci/config.yml

@@ -89,7 +89,7 @@ commands:
               # formatting of generated files.
               name: Check Go code formatting
               command: make fmt-check
-      - run: make gen-device -j4
+      - run: make gen-device
       - run: make smoketest XTENSA=0
       - save_cache:
           key: go-cache-v3-{{ checksum "go.mod" }}-{{ .Environment.CIRCLE_BUILD_NUM }}

.github/workflows/build-macos.yml

@@ -88,7 +88,7 @@ jobs:
         if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
         run: make wasi-libc
       - name: make gen-device
-        run: make -j3 gen-device
+        run: make gen-device
       - name: Test TinyGo
         shell: bash
         run: make test GOTESTFLAGS="-short"

.github/workflows/linux.yml

@@ -255,7 +255,7 @@ jobs:
       - name: Build wasi-libc
         if: steps.cache-wasi-libc.outputs.cache-hit != 'true'
         run: make wasi-libc
-      - run: make gen-device -j4
+      - run: make gen-device
       - name: Test TinyGo
         run: make ASSERT=1 test
       - name: Build TinyGo

.github/workflows/sizediff.yml

@@ -49,7 +49,7 @@ jobs:
           path: |
             ~/.cache/go-build
             ~/go/pkg/mod
-      - run: make gen-device -j4
+      - run: make gen-device
       - name: Download drivers repo
         run: git clone https://github.com/tinygo-org/drivers.git
       - name: Save HEAD

.github/workflows/windows.yml

@@ -98,7 +98,7 @@ jobs:
         run: |
           scoop install wasmtime
       - name: make gen-device
-        run: make -j3 gen-device
+        run: make gen-device
       - name: Test TinyGo
         shell: bash
         run: make test GOTESTFLAGS="-short"

tools/gen-device-svd/gen-device-svd.go

@@ -10,9 +10,11 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"text/template"
 	"unicode"
 )
@@ -1441,6 +1443,31 @@ __isr_vector:
 	return w.Flush()
 }
 
+// Process a single SVD file, synchronously.
+func processFile(infile, outdir, sourceURL, interruptSystem string) error {
+	device, err := readSVD(infile, sourceURL)
+	if err != nil {
+		return fmt.Errorf("failed to read: %w", err)
+	}
+	err = writeGo(outdir, device, interruptSystem)
+	if err != nil {
+		return fmt.Errorf("failed to write Go file: %w", err)
+	}
+	switch interruptSystem {
+	case "software":
+		// Nothing to do.
+	case "hardware":
+		err = writeAsm(outdir, device)
+		if err != nil {
+			return fmt.Errorf("failed to write assembly file: %w", err)
+		}
+	default:
+		return fmt.Errorf("unknown interrupt system: %s", interruptSystem)
+	}
+	return nil
+}
+
+// Process an entire directory, in parallel.
 func generate(indir, outdir, sourceURL, interruptSystem string) error {
 	if _, err := os.Stat(indir); errors.Is(err, fs.ErrNotExist) {
 		fmt.Fprintln(os.Stderr, "cannot find input directory:", indir)
@@ -1448,35 +1475,52 @@ func generate(indir, outdir, sourceURL, interruptSystem string) error {
 	}
 	os.MkdirAll(outdir, 0777)
 
+	// Read list of SVD files to process.
 	infiles, err := filepath.Glob(filepath.Join(indir, "*.svd"))
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "could not read .svd files:", err)
 		os.Exit(1)
 	}
 	sort.Strings(infiles)
-	for _, infile := range infiles {
+
-		fmt.Println(infile)
+	// Start worker goroutines.
-		device, err := readSVD(infile, sourceURL)
+	var wg sync.WaitGroup
-		if err != nil {
+	workChan := make(chan string)
-			return fmt.Errorf("failed to read: %w", err)
+	errChan := make(chan error, 1)
-		}
+	for i := 0; i < runtime.NumCPU(); i++ {
-		err = writeGo(outdir, device, interruptSystem)
+		go func() {
-		if err != nil {
+			for infile := range workChan {
-			return fmt.Errorf("failed to write Go file: %w", err)
+				err := processFile(infile, outdir, sourceURL, interruptSystem)
-		}
+				if err != nil {
-		switch interruptSystem {
+					// Store error to errChan if no error was stored before.
-		case "software":
+					select {
-			// Nothing to do.
+					case errChan <- err:
-		case "hardware":
+					default:
-			err = writeAsm(outdir, device)
+					}
-			if err != nil {
+				}
-				return fmt.Errorf("failed to write assembly file: %w", err)
+				wg.Done()
 			}
-		default:
+		}()
-			return fmt.Errorf("unknown interrupt system: %s", interruptSystem)
+	}
-		}
+
+	// Submit all jobs to the goroutines.
+	wg.Add(len(infiles))
+	for _, filepath := range infiles {
+		fmt.Println(filepath)
+		workChan <- filepath
+	}
+	close(workChan)
+
+	// Wait until all workers have finished.
+	wg.Wait()
+
+	// Check for an error.
+	select {
+	case err := <-errChan:
+		return err
+	default:
+		return nil
 	}
-	return nil
 }
 
 func main() {