diff --git a/Makefile b/Makefile index 4e3e67c2b..4bf5a2150 100644 --- a/Makefile +++ b/Makefile @@ -254,8 +254,12 @@ TEST_PACKAGES_WASI = \ .PHONY: tinygo-test tinygo-test: $(TINYGO) test $(TEST_PACKAGES) +tinygo-bench: + $(TINYGO) test -bench . $(TEST_PACKAGES) tinygo-test-wasi: $(TINYGO) test -target wasi $(TEST_PACKAGES_WASI) +tinygo-bench-wasi: + $(TINYGO) test -target wasi -bench . $(TEST_PACKAGES_WASI) .PHONY: smoketest smoketest: diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go index 88aec366f..c293e37da 100644 --- a/src/testing/benchmark.go +++ b/src/testing/benchmark.go @@ -8,6 +8,9 @@ package testing import ( "fmt" + "io" + "math" + "strings" "time" ) @@ -41,14 +44,17 @@ type InternalBenchmark struct { // affecting benchmark results. type B struct { common - hasSub bool // TODO: should be in common, and atomic - start time.Time // TODO: should be in common - duration time.Duration // TODO: should be in common - N int - benchFunc func(b *B) - benchTime benchTimeFlag - timerOn bool - result BenchmarkResult + hasSub bool // TODO: should be in common, and atomic + start time.Time // TODO: should be in common + duration time.Duration // TODO: should be in common + context *benchContext + N int + benchFunc func(b *B) + bytes int64 + missingBytes bool // one of the subbenchmarks does not have bytes set. + benchTime benchTimeFlag + timerOn bool + result BenchmarkResult } // StartTimer starts timing a test. This function is called automatically @@ -82,15 +88,13 @@ func (b *B) ResetTimer() { // SetBytes records the number of bytes processed in a single operation. // If this is called, the benchmark will report ns/op and MB/s. -func (b *B) SetBytes(n int64) { - panic("testing: unimplemented: B.SetBytes") -} +func (b *B) SetBytes(n int64) { b.bytes = n } // ReportAllocs enables malloc statistics for this benchmark. // It is equivalent to setting -test.benchmem, but it only affects the // benchmark function that calls ReportAllocs. func (b *B) ReportAllocs() { - panic("testing: unimplemented: B.ReportAllocs") + return // TODO: implement } // runN runs a single benchmark for the specified number of iterations. @@ -119,13 +123,31 @@ func max(x, y int64) int64 { // run1 runs the first iteration of benchFunc. It reports whether more // iterations of this benchmarks should be run. func (b *B) run1() bool { + if ctx := b.context; ctx != nil { + // Extend maxLen, if needed. + if n := len(b.name); n > ctx.maxLen { + ctx.maxLen = n + 8 // Add additional slack to avoid too many jumps in size. + } + } b.runN(1) return !b.hasSub } // run executes the benchmark. func (b *B) run() { + if b.context != nil { + // Running go test --test.bench + b.processBench(b.context) // calls doBench and prints results + } else { + // Running func Benchmark. + b.doBench() + } +} + +func (b *B) doBench() BenchmarkResult { + // in upstream, this uses a goroutine b.launch() + return b.result } // launch launches the benchmark function. It gradually increases the number @@ -159,13 +181,14 @@ func (b *B) launch() { n = min(n, 1e9) b.runN(int(n)) } - b.result = BenchmarkResult{b.N, b.duration} + b.result = BenchmarkResult{b.N, b.duration, b.bytes} } // BenchmarkResult contains the results of a benchmark run. type BenchmarkResult struct { - N int // The number of iterations. - T time.Duration // The total time taken. + N int // The number of iterations. + T time.Duration // The total time taken. + Bytes int64 // Bytes processed in one iteration. } // NsPerOp returns the "ns/op" metric. @@ -176,6 +199,14 @@ func (r BenchmarkResult) NsPerOp() int64 { return r.T.Nanoseconds() / int64(r.N) } +// mbPerSec returns the "MB/s" metric. +func (r BenchmarkResult) mbPerSec() float64 { + if r.Bytes <= 0 || r.T <= 0 || r.N <= 0 { + return 0 + } + return (float64(r.Bytes) * float64(r.N) / 1e6) / r.T.Seconds() +} + // AllocsPerOp returns the "allocs/op" metric, // which is calculated as r.MemAllocs / r.N. func (r BenchmarkResult) AllocsPerOp() int64 { @@ -188,10 +219,71 @@ func (r BenchmarkResult) AllocedBytesPerOp() int64 { return 0 // Dummy version to allow running e.g. golang.org/test/fibo.go } +// String returns a summary of the benchmark results. +// It follows the benchmark result line format from +// https://golang.org/design/14313-benchmark-format, not including the +// benchmark name. +// Extra metrics override built-in metrics of the same name. +// String does not include allocs/op or B/op, since those are reported +// by MemString. +func (r BenchmarkResult) String() string { + buf := new(strings.Builder) + fmt.Fprintf(buf, "%8d", r.N) + + // Get ns/op as a float. + ns := float64(r.T.Nanoseconds()) / float64(r.N) + if ns != 0 { + buf.WriteByte('\t') + prettyPrint(buf, ns, "ns/op") + } + + if mbs := r.mbPerSec(); mbs != 0 { + fmt.Fprintf(buf, "\t%7.2f MB/s", mbs) + } + return buf.String() +} + +func prettyPrint(w io.Writer, x float64, unit string) { + // Print all numbers with 10 places before the decimal point + // and small numbers with four sig figs. Field widths are + // chosen to fit the whole part in 10 places while aligning + // the decimal point of all fractional formats. + var format string + switch y := math.Abs(x); { + case y == 0 || y >= 999.95: + format = "%10.0f %s" + case y >= 99.995: + format = "%12.1f %s" + case y >= 9.9995: + format = "%13.2f %s" + case y >= 0.99995: + format = "%14.3f %s" + case y >= 0.099995: + format = "%15.4f %s" + case y >= 0.0099995: + format = "%16.5f %s" + case y >= 0.00099995: + format = "%17.6f %s" + default: + format = "%18.7f %s" + } + fmt.Fprintf(w, format, x, unit) +} + +type benchContext struct { + maxLen int // The largest recorded benchmark name. +} + func runBenchmarks(benchmarks []InternalBenchmark) bool { if len(benchmarks) == 0 { return true } + ctx := &benchContext{} + for _, Benchmark := range benchmarks { + if l := len(Benchmark.Name); l > ctx.maxLen { + ctx.maxLen = l + } + } main := &B{ common: common{ name: "Main", @@ -199,30 +291,55 @@ func runBenchmarks(benchmarks []InternalBenchmark) bool { benchTime: benchTime, benchFunc: func(b *B) { for _, Benchmark := range benchmarks { - if flagVerbose { - fmt.Printf("=== RUN %s\n", Benchmark.Name) - } b.Run(Benchmark.Name, Benchmark.F) - fmt.Printf("--- Result: %d ns/op\n", b.result.NsPerOp()) } }, + context: ctx, } main.runN(1) return true } +// processBench runs bench b and prints the results. +func (b *B) processBench(ctx *benchContext) { + benchName := b.name + + if ctx != nil { + fmt.Printf("%-*s\t", ctx.maxLen, benchName) + } + r := b.doBench() + if b.failed { + // The output could be very long here, but probably isn't. + // We print it all, regardless, because we don't want to trim the reason + // the benchmark failed. + fmt.Printf("--- FAIL: %s\n%s", benchName, "") // b.output) + return + } + if ctx != nil { + results := r.String() + fmt.Println(results) + } +} + // Run benchmarks f as a subbenchmark with the given name. It reports // true if the subbenchmark succeeded. // // A subbenchmark is like any other benchmark. A benchmark that calls Run at // least once will not be measured itself and will be called once with N=1. func (b *B) Run(name string, f func(b *B)) bool { + if b.level > 0 { + name = b.name + "/" + name + } b.hasSub = true sub := &B{ - common: common{name: name}, + common: common{ + name: name, + level: b.level + 1, + }, benchFunc: f, benchTime: b.benchTime, + context: b.context, } if sub.run1() { sub.run() @@ -240,6 +357,15 @@ func (b *B) add(other BenchmarkResult) { // in sequence in a single benchmark. r.N = 1 r.T += time.Duration(other.NsPerOp()) + if other.Bytes == 0 { + // Summing Bytes is meaningless in aggregate if not all subbenchmarks + // set it. + b.missingBytes = true + r.Bytes = 0 + } + if !b.missingBytes { + r.Bytes += other.Bytes + } } // A PB is used by RunParallel for running parallel benchmarks. diff --git a/src/testing/benchmark_test.go b/src/testing/benchmark_test.go index 49612d3d4..7fbd2cf12 100644 --- a/src/testing/benchmark_test.go +++ b/src/testing/benchmark_test.go @@ -48,3 +48,8 @@ func TestBenchmark(t *testing.T) { t.Errorf("Expected speedup >= 0.3, got %f", speedup) } } + +func BenchmarkSub(b *testing.B) { + b.Run("Fast", func(b *testing.B) { BenchmarkFastNonASCII(b) }) + b.Run("Slow", func(b *testing.B) { BenchmarkSlowNonASCII(b) }) +} diff --git a/src/testing/testing.go b/src/testing/testing.go index ee4056ebd..3eac03ddf 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -48,6 +48,7 @@ type common struct { failed bool // Test or benchmark has failed. skipped bool // Test of benchmark has been skipped. finished bool // Test function has completed. + level int // Nesting depth of test or benchmark. name string // Name of test or benchmark. }