Add benchmark function for mbt (#3255)

mbt

@@ -185,6 +185,7 @@ Usage:
                                  Download/setup ARM toolchain only
   ./mbt doctor                   Validate host setup and report gaps
   ./mbt shell [options]          Open a shell with toolchain exported for this session
+  ./mbt benchmark [options]      Benchmark clean-build time across generators and ccache modes
 
 Build options:
   -d, --device <name>            Device: hackrf-one | hackrf-pro | portarf
@@ -205,12 +206,24 @@ Build options:
   -y, --non-interactive          Fail instead of prompting when required input is missing
   -h, --help                     Show this help
 
+Benchmark options (in addition to -d/--device, -j/--jobs, --target, -B, --cmake-arg):
+  --bench-generators <list>      Comma list of generators to test: ninja,make (default: all installed)
+  --bench-ccache <list>          Comma list of ccache modes: off,cold,warm (default: off,cold,warm)
+                                   off  = ccache disabled (baseline)
+                                   cold = ccache on, empty cache (first-build/populate cost)
+                                   warm = ccache on, pre-populated cache (the ccache speedup)
+                                 Each cell is a full clean build in a dedicated dir (default: ./build-bench),
+                                 using an isolated CCACHE_DIR so your real ccache is never touched.
+
 Examples:
   ./mbt
   ./mbt build --device hackrf-one
   ./mbt build --device hackrf-pro --no-submodule-sync
   ./mbt build --device portarf --no-toolchain --no-deps
   ./mbt build --device hackrf-one --cmake-arg=-DVERSION_STRING=my-build
+  ./mbt benchmark --device hackrf-one --no-deps --no-submodule-sync
+  ./mbt benchmark --device hackrf-one --bench-generators ninja --bench-ccache off,cold,warm
+  ./mbt benchmark --device hackrf-one --bench-generators ninja,make --bench-ccache off
 EOF
 }
 
@@ -308,9 +321,28 @@ mbt_detect_cached_device() {
     return 0
 }
 
+# libopencm3 builds in-source (inside the submodule tree), so deleting the CMake
+# build dir alone does NOT rebuild it. Clean it explicitly for a true clean build.
+# Set quiet=1 to suppress make output (used by the benchmark).
+mbt_clean_libopencm3() {
+    local quiet="${1:-0}"
+    local libopencm3_dir="$SCRIPT_PATH/hackrf/firmware/libopencm3"
+
+    if [[ ! -d "$libopencm3_dir" ]]; then
+        mbt_error "libopencm3 directory not found: $libopencm3_dir"
+        return 1
+    fi
+
+    if [[ "$quiet" -eq 1 ]]; then
+        make -C "$libopencm3_dir" clean >/dev/null 2>&1
+    else
+        mbt_log "cleaning libopencm3: $libopencm3_dir"
+        make -C "$libopencm3_dir" clean
+    fi
+}
+
 mbt_clean() {
     local build_dir="$1"
-    local libopencm3_dir="$SCRIPT_PATH/hackrf/firmware/libopencm3"
 
     mbt_log "cleaning build directory: $build_dir"
     if ! rm -rf "$build_dir"; then
@@ -318,19 +350,63 @@ mbt_clean() {
         return 1
     fi
 
-    if [[ -d "$libopencm3_dir" ]]; then
+    mbt_clean_libopencm3 0
-        mbt_log "cleaning libopencm3: $libopencm3_dir"
-        make -C "$libopencm3_dir" clean
-    else
-        mbt_error "libopencm3 directory not found: $libopencm3_dir"
-        return 1
-    fi
 }
 
 mbt_report_build_error() {
     printf '%s\n' 'Build error happened. Try to fix your code, or run ./mbt --clean and try again' >&2
 }
 
+# Print the device-specific CMake args, one per line (empty for hackrf-one).
+mbt_device_args() {
+    case "$1" in
+    hackrf-one)
+        : # no extra args
+        ;;
+    hackrf-pro)
+        printf '%s\n' "-DBOARD=PRALINE"
+        ;;
+    portarf)
+        printf '%s\n' "-DFLASH_MB_SIZE=2" "-DFLASH_MB_LIMIT_SIZE=2"
+        ;;
+    *)
+        return 1
+        ;;
+    esac
+}
+
+# Normalize a user-supplied generator name to a canonical CMake generator name.
+mbt_resolve_generator() {
+    local raw
+    raw="$(printf "%s" "$1" | tr '[:upper:]' '[:lower:]')"
+    case "$raw" in
+    ninja)
+        printf "Ninja"
+        ;;
+    make|makefiles|unixmakefiles|"unix makefiles")
+        printf "Unix Makefiles"
+        ;;
+    *)
+        printf "%s" "$1"
+        ;;
+    esac
+}
+
+# Print the command required by a canonical CMake generator (empty if unknown).
+mbt_generator_command() {
+    case "$1" in
+    Ninja)
+        printf "ninja"
+        ;;
+    "Unix Makefiles")
+        printf "make"
+        ;;
+    *)
+        printf ""
+        ;;
+    esac
+}
+
 mbt_build() {
     local build_dir="$1"
     local generator="$2"
@@ -372,20 +448,8 @@ mbt_build() {
     mkdir -p "$build_dir"
 
     device_args=()
-    case "$device" in
+    mbt_device_args "$device" >/dev/null || mbt_die "unsupported device preset: $device"
-    hackrf-one)
+    mapfile -t device_args < <(mbt_device_args "$device")
-        device_args+=()
-        ;;
-    hackrf-pro)
-        device_args+=("-DBOARD=PRALINE")
-        ;;
-    portarf)
-        device_args+=("-DFLASH_MB_SIZE=2" "-DFLASH_MB_LIMIT_SIZE=2")
-        ;;
-    *)
-        mbt_die "unsupported device preset: $device"
-        ;;
-    esac
 
     if [[ "$generator" == "Ninja" ]] && ! mbt_command_exists ninja; then
         if mbt_command_exists make; then
@@ -430,6 +494,258 @@ mbt_build() {
     # have to hard code the name here cuz it's too expensive to read cmake for just a file name.
 }
 
+# Compute elapsed wall-clock seconds (with 2 decimals) between two `date +%s.%N` stamps.
+mbt_elapsed() {
+    awk -v s="$1" -v e="$2" 'BEGIN { printf "%.2f", e - s }'
+}
+
+# Format seconds as a human-friendly "MmSS.ss" (or "SS.ss" under a minute).
+mbt_format_seconds() {
+    awk -v t="$1" 'BEGIN {
+        if (t < 0) t = 0
+        m = int(t / 60)
+        s = t - m * 60
+        if (m > 0) printf "%dm%05.2fs", m, s
+        else printf "%.2fs", s
+    }'
+}
+
+# Run one full clean build and print elapsed seconds on stdout.
+# All progress/logging goes to stderr; build output is captured to a per-cell log.
+# Args: cell_dir generator use_ccache device jobs target ccache_dir log_file  cmake_extra_array_name
+mbt_bench_cell() {
+    local cell_dir="$1" generator="$2" use_ccache="$3" device="$4" jobs="$5"
+    local target="$6" ccache_dir="$7" log_file="$8"
+    shift 8
+    local -n cmake_extra_ref="$1"
+    local -a cmake_extra_args=("${cmake_extra_ref[@]}")
+    local -a device_args cmake_args build_args
+    local start end status=0
+
+    rm -rf "$cell_dir"
+    mkdir -p "$cell_dir"
+
+    # libopencm3 builds in-source, so it survives `rm -rf "$cell_dir"`. Without
+    # this, only the first cell would pay its compile cost and every later cell
+    # would look artificially fast, biasing the whole comparison.
+    mbt_clean_libopencm3 1 || true
+
+    mapfile -t device_args < <(mbt_device_args "$device")
+
+    cmake_args=(
+        -S "$SCRIPT_PATH"
+        -B "$cell_dir"
+        -G "$generator"
+        "-DUSE_CCACHE=$use_ccache"
+    )
+    cmake_args+=("${device_args[@]}")
+    cmake_args+=("${cmake_extra_args[@]}")
+
+    build_args=(--build "$cell_dir" --parallel "$jobs")
+    if [[ -n "$target" ]]; then
+        build_args+=(--target "$target")
+    fi
+
+    start="$(date +%s.%N)"
+    (
+        if [[ -n "$ccache_dir" ]]; then
+            export CCACHE_DIR="$ccache_dir"
+        fi
+        cmake "${cmake_args[@]}" && cmake "${build_args[@]}"
+    ) >"$log_file" 2>&1 || status=$?
+    end="$(date +%s.%N)"
+
+    if [[ "$status" -ne 0 ]]; then
+        printf "FAILED"
+        return "$status"
+    fi
+
+    mbt_elapsed "$start" "$end"
+}
+
+# Populate a ccache directory without timing (used when a warm run is requested
+# but no cold run precedes it to fill the cache).
+mbt_bench_populate() {
+    local cell_dir="$1" generator="$2" device="$3" jobs="$4" target="$5"
+    local ccache_dir="$6" log_file="$7"
+    shift 7
+    mbt_bench_cell "$cell_dir" "$generator" "ON" "$device" "$jobs" "$target" \
+        "$ccache_dir" "$log_file" "$1" >/dev/null
+}
+
+mbt_benchmark() {
+    local base_dir="$1" device="$2" jobs="$3" target="$4"
+    local generators_csv="$5" ccache_csv="$6"
+    shift 6
+    local -n bench_cmake_extra_ref="$1"
+    local -a bench_extra_args=("${bench_cmake_extra_ref[@]}")
+
+    local -a want_generators=() generators=() modes=()
+    local raw g cmd mode
+
+    # Resolve requested generators (auto = every generator whose tool is present).
+    if [[ -z "$generators_csv" || "$generators_csv" == "auto" ]]; then
+        want_generators=("Ninja" "Unix Makefiles")
+    else
+        IFS=',' read -ra raw_gens <<< "$generators_csv"
+        for raw in "${raw_gens[@]}"; do
+            [[ -z "$raw" ]] && continue
+            want_generators+=("$(mbt_resolve_generator "$raw")")
+        done
+    fi
+
+    for g in "${want_generators[@]}"; do
+        cmd="$(mbt_generator_command "$g")"
+        if [[ -z "$cmd" ]]; then
+            mbt_error "skipping unknown generator: $g"
+            continue
+        fi
+        if ! mbt_command_exists "$cmd"; then
+            mbt_error "skipping generator '$g' ($cmd not installed)."
+            continue
+        fi
+        generators+=("$g")
+    done
+
+    if [[ "${#generators[@]}" -eq 0 ]]; then
+        mbt_die "no usable generators for benchmark (need ninja and/or make)."
+    fi
+
+    # Resolve ccache modes: off | cold | warm.
+    if [[ -z "$ccache_csv" ]]; then
+        ccache_csv="off,cold,warm"
+    fi
+    local -A mode_seen=()
+    IFS=',' read -ra raw_modes <<< "$ccache_csv"
+    for mode in "${raw_modes[@]}"; do
+        mode="$(printf "%s" "$mode" | tr '[:upper:]' '[:lower:]')"
+        [[ -z "$mode" ]] && continue
+        case "$mode" in
+        off | cold | warm) ;;
+        on) mode="cold" ;; # treat plain "on" as a cold measurement
+        *) mbt_die "unknown ccache mode: $mode (use off, cold, warm)" ;;
+        esac
+        if [[ -z "${mode_seen[$mode]:-}" ]]; then
+            modes+=("$mode")
+            mode_seen[$mode]=1
+        fi
+    done
+    [[ "${#modes[@]}" -eq 0 ]] && mbt_die "no ccache modes selected."
+
+    local needs_ccache=0
+    for mode in "${modes[@]}"; do
+        [[ "$mode" != "off" ]] && needs_ccache=1
+    done
+    if [[ "$needs_ccache" -eq 1 ]] && ! mbt_command_exists ccache; then
+        mbt_die "ccache modes requested but ccache is not installed. Install it or use --bench-ccache off."
+    fi
+
+    mkdir -p "$base_dir"
+    local log_dir="$base_dir/logs"
+    mkdir -p "$log_dir"
+
+    mbt_log "benchmark: device=$device jobs=$jobs generators=[${generators[*]}] ccache=[${modes[*]}]"
+    [[ -n "$target" ]] && mbt_log "benchmark: target=$target"
+    mbt_log "benchmark: results and per-cell logs under $base_dir"
+    mbt_log "benchmark: each cell is a full clean build; this can take a while."
+
+    # Parallel result arrays.
+    local -a r_gen=() r_mode=() r_secs=() r_ok=()
+    local cell_dir ccache_dir log_file secs cell_status label
+
+    for g in "${generators[@]}"; do
+        # Per-generator, isolated ccache dir so "cold" is truly cold and the
+        # user's real cache is never touched. Cache keys are generator-agnostic,
+        # so each generator needs its own fresh dir.
+        ccache_dir=""
+        if [[ "$needs_ccache" -eq 1 ]]; then
+            ccache_dir="$base_dir/ccache-$(printf "%s" "$g" | tr ' ' '-')"
+            rm -rf "$ccache_dir"
+            mkdir -p "$ccache_dir"
+        fi
+
+        local cold_done=0
+        for mode in "${modes[@]}"; do
+            label="$g / ccache=$mode"
+            log_file="$log_dir/$(printf "%s" "$g" | tr ' ' '-')-$mode.log"
+
+            case "$mode" in
+            off)
+                mbt_log "running: $label"
+                secs="$(mbt_bench_cell "$base_dir/cell" "$g" "OFF" "$device" \
+                    "$jobs" "$target" "" "$log_file" bench_extra_args)" && cell_status=0 || cell_status=$?
+                ;;
+            cold)
+                rm -rf "$ccache_dir" && mkdir -p "$ccache_dir"
+                mbt_log "running: $label (empty cache)"
+                secs="$(mbt_bench_cell "$base_dir/cell" "$g" "ON" "$device" \
+                    "$jobs" "$target" "$ccache_dir" "$log_file" bench_extra_args)" && cell_status=0 || cell_status=$?
+                [[ "$cell_status" -eq 0 ]] && cold_done=1
+                ;;
+            warm)
+                if [[ "$cold_done" -eq 0 ]]; then
+                    mbt_log "priming cache for: $label (untimed)"
+                    rm -rf "$ccache_dir" && mkdir -p "$ccache_dir"
+                    if ! mbt_bench_populate "$base_dir/cell" "$g" "$device" \
+                        "$jobs" "$target" "$ccache_dir" "$log_dir/$(printf "%s" "$g" | tr ' ' '-')-prime.log" bench_extra_args; then
+                        mbt_error "cache priming failed for $label; skipping."
+                        r_gen+=("$g"); r_mode+=("$mode"); r_secs+=("0"); r_ok+=("0")
+                        continue
+                    fi
+                fi
+                mbt_log "running: $label (populated cache)"
+                secs="$(mbt_bench_cell "$base_dir/cell" "$g" "ON" "$device" \
+                    "$jobs" "$target" "$ccache_dir" "$log_file" bench_extra_args)" && cell_status=0 || cell_status=$?
+                ;;
+            esac
+
+            if [[ "$cell_status" -ne 0 ]]; then
+                mbt_error "cell failed: $label (see $log_file)"
+                r_gen+=("$g"); r_mode+=("$mode"); r_secs+=("0"); r_ok+=("0")
+            else
+                mbt_log "  -> $(mbt_format_seconds "$secs") ($secs s)"
+                r_gen+=("$g"); r_mode+=("$mode"); r_secs+=("$secs"); r_ok+=("1")
+            fi
+        done
+    done
+
+    rm -rf "$base_dir/cell"
+
+    # Summary table.
+    printf "\n"
+    mbt_log "benchmark summary (device=$device, jobs=$jobs)"
+    printf "  %-16s %-8s %12s %10s\n" "generator" "ccache" "time" "vs off"
+    printf "  %-16s %-8s %12s %10s\n" "---------" "------" "----" "------"
+
+    local i baseline speedup
+    for ((i = 0; i < ${#r_gen[@]}; i++)); do
+        # Find the "off" baseline for the same generator, if measured.
+        baseline=""
+        local j
+        for ((j = 0; j < ${#r_gen[@]}; j++)); do
+            if [[ "${r_gen[$j]}" == "${r_gen[$i]}" && "${r_mode[$j]}" == "off" && "${r_ok[$j]}" == "1" ]]; then
+                baseline="${r_secs[$j]}"
+                break
+            fi
+        done
+
+        if [[ "${r_ok[$i]}" != "1" ]]; then
+            printf "  %-16s %-8s %12s %10s\n" "${r_gen[$i]}" "${r_mode[$i]}" "FAILED" "-"
+            continue
+        fi
+
+        if [[ -n "$baseline" && "${r_mode[$i]}" != "off" ]]; then
+            speedup="$(awk -v b="$baseline" -v t="${r_secs[$i]}" 'BEGIN { if (t > 0) printf "%.2fx", b / t; else printf "-" }')"
+        else
+            speedup="-"
+        fi
+        printf "  %-16s %-8s %12s %10s\n" "${r_gen[$i]}" "${r_mode[$i]}" "$(mbt_format_seconds "${r_secs[$i]}")" "$speedup"
+    done
+    printf "\n"
+    mbt_log "'vs off' > 1x means faster than the same generator's ccache=off build."
+    mbt_log "cold = empty cache (populate cost); warm = pre-populated cache (the ccache win)."
+}
+
 main() {
     local command="build"
     local quickstart=0
@@ -439,11 +755,14 @@ main() {
     local with_submodule_sync=1
     local with_toolchain=1
     local build_dir="$SCRIPT_PATH/build"
+    local build_dir_explicit=0
     local generator="Ninja"
     local jobs
     local target=""
     local clean_build=0
     local non_interactive=0
+    local benchmark_generators=""
+    local benchmark_ccache=""
     local -a cmake_extra_args=()
     local -a build_extra_args=()
 
@@ -457,7 +776,7 @@ main() {
 
     if [[ "$#" -gt 0 ]]; then
         case "$1" in
-        build|deps|toolchain|doctor|shell)
+        build|deps|toolchain|doctor|shell|benchmark)
             command="$1"
             shift
             ;;
@@ -515,8 +834,27 @@ main() {
         -B|--build-dir)
             [[ "$#" -ge 2 ]] || mbt_die "$1 requires a value"
             build_dir="$2"
+            build_dir_explicit=1
             shift 2
             ;;
+        --bench-generators)
+            [[ "$#" -ge 2 ]] || mbt_die "$1 requires a value"
+            benchmark_generators="$2"
+            shift 2
+            ;;
+        --bench-generators=*)
+            benchmark_generators="${1#*=}"
+            shift
+            ;;
+        --bench-ccache)
+            [[ "$#" -ge 2 ]] || mbt_die "$1 requires a value"
+            benchmark_ccache="$2"
+            shift 2
+            ;;
+        --bench-ccache=*)
+            benchmark_ccache="${1#*=}"
+            shift
+            ;;
         -G|--generator)
             [[ "$#" -ge 2 ]] || mbt_die "$1 requires a value"
             generator="$2"
@@ -598,6 +936,13 @@ main() {
         ;;
     build)
         ;;
+    benchmark)
+        # Never benchmark into the user's working ./build dir; a benchmark
+        # wipes its build dir between cells. Use a dedicated dir by default.
+        if [[ "$build_dir_explicit" -eq 0 ]]; then
+            build_dir="$SCRIPT_PATH/build-bench"
+        fi
+        ;;
     *)
         mbt_die "unsupported command: $command"
         ;;
@@ -640,6 +985,12 @@ main() {
         mbt_log "quickstart mode enabled."
     fi
 
+    if [[ "$command" == "benchmark" ]]; then
+        mbt_benchmark "$build_dir" "$device" "$jobs" "$target" \
+            "$benchmark_generators" "$benchmark_ccache" cmake_extra_args
+        exit $?
+    fi
+
     mbt_build "$build_dir" "$generator" "$use_ccache" "$device" "$jobs" "$target" "$clean_build" \
         cmake_extra_args[@] build_extra_args[@]
 }