diff --git a/mbt b/mbt index e7dea65eb..3326602e2 100755 --- a/mbt +++ b/mbt @@ -185,6 +185,7 @@ Usage: Download/setup ARM toolchain only ./mbt doctor Validate host setup and report gaps ./mbt shell [options] Open a shell with toolchain exported for this session + ./mbt benchmark [options] Benchmark clean-build time across generators and ccache modes Build options: -d, --device Device: hackrf-one | hackrf-pro | portarf @@ -205,12 +206,24 @@ Build options: -y, --non-interactive Fail instead of prompting when required input is missing -h, --help Show this help +Benchmark options (in addition to -d/--device, -j/--jobs, --target, -B, --cmake-arg): + --bench-generators Comma list of generators to test: ninja,make (default: all installed) + --bench-ccache Comma list of ccache modes: off,cold,warm (default: off,cold,warm) + off = ccache disabled (baseline) + cold = ccache on, empty cache (first-build/populate cost) + warm = ccache on, pre-populated cache (the ccache speedup) + Each cell is a full clean build in a dedicated dir (default: ./build-bench), + using an isolated CCACHE_DIR so your real ccache is never touched. + Examples: ./mbt ./mbt build --device hackrf-one ./mbt build --device hackrf-pro --no-submodule-sync ./mbt build --device portarf --no-toolchain --no-deps ./mbt build --device hackrf-one --cmake-arg=-DVERSION_STRING=my-build + ./mbt benchmark --device hackrf-one --no-deps --no-submodule-sync + ./mbt benchmark --device hackrf-one --bench-generators ninja --bench-ccache off,cold,warm + ./mbt benchmark --device hackrf-one --bench-generators ninja,make --bench-ccache off EOF } @@ -308,9 +321,28 @@ mbt_detect_cached_device() { return 0 } +# libopencm3 builds in-source (inside the submodule tree), so deleting the CMake +# build dir alone does NOT rebuild it. Clean it explicitly for a true clean build. +# Set quiet=1 to suppress make output (used by the benchmark). +mbt_clean_libopencm3() { + local quiet="${1:-0}" + local libopencm3_dir="$SCRIPT_PATH/hackrf/firmware/libopencm3" + + if [[ ! -d "$libopencm3_dir" ]]; then + mbt_error "libopencm3 directory not found: $libopencm3_dir" + return 1 + fi + + if [[ "$quiet" -eq 1 ]]; then + make -C "$libopencm3_dir" clean >/dev/null 2>&1 + else + mbt_log "cleaning libopencm3: $libopencm3_dir" + make -C "$libopencm3_dir" clean + fi +} + mbt_clean() { local build_dir="$1" - local libopencm3_dir="$SCRIPT_PATH/hackrf/firmware/libopencm3" mbt_log "cleaning build directory: $build_dir" if ! rm -rf "$build_dir"; then @@ -318,19 +350,63 @@ mbt_clean() { return 1 fi - if [[ -d "$libopencm3_dir" ]]; then - mbt_log "cleaning libopencm3: $libopencm3_dir" - make -C "$libopencm3_dir" clean - else - mbt_error "libopencm3 directory not found: $libopencm3_dir" - return 1 - fi + mbt_clean_libopencm3 0 } mbt_report_build_error() { printf '%s\n' 'Build error happened. Try to fix your code, or run ./mbt --clean and try again' >&2 } +# Print the device-specific CMake args, one per line (empty for hackrf-one). +mbt_device_args() { + case "$1" in + hackrf-one) + : # no extra args + ;; + hackrf-pro) + printf '%s\n' "-DBOARD=PRALINE" + ;; + portarf) + printf '%s\n' "-DFLASH_MB_SIZE=2" "-DFLASH_MB_LIMIT_SIZE=2" + ;; + *) + return 1 + ;; + esac +} + +# Normalize a user-supplied generator name to a canonical CMake generator name. +mbt_resolve_generator() { + local raw + raw="$(printf "%s" "$1" | tr '[:upper:]' '[:lower:]')" + case "$raw" in + ninja) + printf "Ninja" + ;; + make|makefiles|unixmakefiles|"unix makefiles") + printf "Unix Makefiles" + ;; + *) + printf "%s" "$1" + ;; + esac +} + +# Print the command required by a canonical CMake generator (empty if unknown). +mbt_generator_command() { + case "$1" in + Ninja) + printf "ninja" + ;; + "Unix Makefiles") + printf "make" + ;; + *) + printf "" + ;; + esac +} + mbt_build() { local build_dir="$1" local generator="$2" @@ -372,20 +448,8 @@ mbt_build() { mkdir -p "$build_dir" device_args=() - case "$device" in - hackrf-one) - device_args+=() - ;; - hackrf-pro) - device_args+=("-DBOARD=PRALINE") - ;; - portarf) - device_args+=("-DFLASH_MB_SIZE=2" "-DFLASH_MB_LIMIT_SIZE=2") - ;; - *) - mbt_die "unsupported device preset: $device" - ;; - esac + mbt_device_args "$device" >/dev/null || mbt_die "unsupported device preset: $device" + mapfile -t device_args < <(mbt_device_args "$device") if [[ "$generator" == "Ninja" ]] && ! mbt_command_exists ninja; then if mbt_command_exists make; then @@ -430,6 +494,258 @@ mbt_build() { # have to hard code the name here cuz it's too expensive to read cmake for just a file name. } +# Compute elapsed wall-clock seconds (with 2 decimals) between two `date +%s.%N` stamps. +mbt_elapsed() { + awk -v s="$1" -v e="$2" 'BEGIN { printf "%.2f", e - s }' +} + +# Format seconds as a human-friendly "MmSS.ss" (or "SS.ss" under a minute). +mbt_format_seconds() { + awk -v t="$1" 'BEGIN { + if (t < 0) t = 0 + m = int(t / 60) + s = t - m * 60 + if (m > 0) printf "%dm%05.2fs", m, s + else printf "%.2fs", s + }' +} + +# Run one full clean build and print elapsed seconds on stdout. +# All progress/logging goes to stderr; build output is captured to a per-cell log. +# Args: cell_dir generator use_ccache device jobs target ccache_dir log_file cmake_extra_array_name +mbt_bench_cell() { + local cell_dir="$1" generator="$2" use_ccache="$3" device="$4" jobs="$5" + local target="$6" ccache_dir="$7" log_file="$8" + shift 8 + local -n cmake_extra_ref="$1" + local -a cmake_extra_args=("${cmake_extra_ref[@]}") + local -a device_args cmake_args build_args + local start end status=0 + + rm -rf "$cell_dir" + mkdir -p "$cell_dir" + + # libopencm3 builds in-source, so it survives `rm -rf "$cell_dir"`. Without + # this, only the first cell would pay its compile cost and every later cell + # would look artificially fast, biasing the whole comparison. + mbt_clean_libopencm3 1 || true + + mapfile -t device_args < <(mbt_device_args "$device") + + cmake_args=( + -S "$SCRIPT_PATH" + -B "$cell_dir" + -G "$generator" + "-DUSE_CCACHE=$use_ccache" + ) + cmake_args+=("${device_args[@]}") + cmake_args+=("${cmake_extra_args[@]}") + + build_args=(--build "$cell_dir" --parallel "$jobs") + if [[ -n "$target" ]]; then + build_args+=(--target "$target") + fi + + start="$(date +%s.%N)" + ( + if [[ -n "$ccache_dir" ]]; then + export CCACHE_DIR="$ccache_dir" + fi + cmake "${cmake_args[@]}" && cmake "${build_args[@]}" + ) >"$log_file" 2>&1 || status=$? + end="$(date +%s.%N)" + + if [[ "$status" -ne 0 ]]; then + printf "FAILED" + return "$status" + fi + + mbt_elapsed "$start" "$end" +} + +# Populate a ccache directory without timing (used when a warm run is requested +# but no cold run precedes it to fill the cache). +mbt_bench_populate() { + local cell_dir="$1" generator="$2" device="$3" jobs="$4" target="$5" + local ccache_dir="$6" log_file="$7" + shift 7 + mbt_bench_cell "$cell_dir" "$generator" "ON" "$device" "$jobs" "$target" \ + "$ccache_dir" "$log_file" "$1" >/dev/null +} + +mbt_benchmark() { + local base_dir="$1" device="$2" jobs="$3" target="$4" + local generators_csv="$5" ccache_csv="$6" + shift 6 + local -n bench_cmake_extra_ref="$1" + local -a bench_extra_args=("${bench_cmake_extra_ref[@]}") + + local -a want_generators=() generators=() modes=() + local raw g cmd mode + + # Resolve requested generators (auto = every generator whose tool is present). + if [[ -z "$generators_csv" || "$generators_csv" == "auto" ]]; then + want_generators=("Ninja" "Unix Makefiles") + else + IFS=',' read -ra raw_gens <<< "$generators_csv" + for raw in "${raw_gens[@]}"; do + [[ -z "$raw" ]] && continue + want_generators+=("$(mbt_resolve_generator "$raw")") + done + fi + + for g in "${want_generators[@]}"; do + cmd="$(mbt_generator_command "$g")" + if [[ -z "$cmd" ]]; then + mbt_error "skipping unknown generator: $g" + continue + fi + if ! mbt_command_exists "$cmd"; then + mbt_error "skipping generator '$g' ($cmd not installed)." + continue + fi + generators+=("$g") + done + + if [[ "${#generators[@]}" -eq 0 ]]; then + mbt_die "no usable generators for benchmark (need ninja and/or make)." + fi + + # Resolve ccache modes: off | cold | warm. + if [[ -z "$ccache_csv" ]]; then + ccache_csv="off,cold,warm" + fi + local -A mode_seen=() + IFS=',' read -ra raw_modes <<< "$ccache_csv" + for mode in "${raw_modes[@]}"; do + mode="$(printf "%s" "$mode" | tr '[:upper:]' '[:lower:]')" + [[ -z "$mode" ]] && continue + case "$mode" in + off | cold | warm) ;; + on) mode="cold" ;; # treat plain "on" as a cold measurement + *) mbt_die "unknown ccache mode: $mode (use off, cold, warm)" ;; + esac + if [[ -z "${mode_seen[$mode]:-}" ]]; then + modes+=("$mode") + mode_seen[$mode]=1 + fi + done + [[ "${#modes[@]}" -eq 0 ]] && mbt_die "no ccache modes selected." + + local needs_ccache=0 + for mode in "${modes[@]}"; do + [[ "$mode" != "off" ]] && needs_ccache=1 + done + if [[ "$needs_ccache" -eq 1 ]] && ! mbt_command_exists ccache; then + mbt_die "ccache modes requested but ccache is not installed. Install it or use --bench-ccache off." + fi + + mkdir -p "$base_dir" + local log_dir="$base_dir/logs" + mkdir -p "$log_dir" + + mbt_log "benchmark: device=$device jobs=$jobs generators=[${generators[*]}] ccache=[${modes[*]}]" + [[ -n "$target" ]] && mbt_log "benchmark: target=$target" + mbt_log "benchmark: results and per-cell logs under $base_dir" + mbt_log "benchmark: each cell is a full clean build; this can take a while." + + # Parallel result arrays. + local -a r_gen=() r_mode=() r_secs=() r_ok=() + local cell_dir ccache_dir log_file secs cell_status label + + for g in "${generators[@]}"; do + # Per-generator, isolated ccache dir so "cold" is truly cold and the + # user's real cache is never touched. Cache keys are generator-agnostic, + # so each generator needs its own fresh dir. + ccache_dir="" + if [[ "$needs_ccache" -eq 1 ]]; then + ccache_dir="$base_dir/ccache-$(printf "%s" "$g" | tr ' ' '-')" + rm -rf "$ccache_dir" + mkdir -p "$ccache_dir" + fi + + local cold_done=0 + for mode in "${modes[@]}"; do + label="$g / ccache=$mode" + log_file="$log_dir/$(printf "%s" "$g" | tr ' ' '-')-$mode.log" + + case "$mode" in + off) + mbt_log "running: $label" + secs="$(mbt_bench_cell "$base_dir/cell" "$g" "OFF" "$device" \ + "$jobs" "$target" "" "$log_file" bench_extra_args)" && cell_status=0 || cell_status=$? + ;; + cold) + rm -rf "$ccache_dir" && mkdir -p "$ccache_dir" + mbt_log "running: $label (empty cache)" + secs="$(mbt_bench_cell "$base_dir/cell" "$g" "ON" "$device" \ + "$jobs" "$target" "$ccache_dir" "$log_file" bench_extra_args)" && cell_status=0 || cell_status=$? + [[ "$cell_status" -eq 0 ]] && cold_done=1 + ;; + warm) + if [[ "$cold_done" -eq 0 ]]; then + mbt_log "priming cache for: $label (untimed)" + rm -rf "$ccache_dir" && mkdir -p "$ccache_dir" + if ! mbt_bench_populate "$base_dir/cell" "$g" "$device" \ + "$jobs" "$target" "$ccache_dir" "$log_dir/$(printf "%s" "$g" | tr ' ' '-')-prime.log" bench_extra_args; then + mbt_error "cache priming failed for $label; skipping." + r_gen+=("$g"); r_mode+=("$mode"); r_secs+=("0"); r_ok+=("0") + continue + fi + fi + mbt_log "running: $label (populated cache)" + secs="$(mbt_bench_cell "$base_dir/cell" "$g" "ON" "$device" \ + "$jobs" "$target" "$ccache_dir" "$log_file" bench_extra_args)" && cell_status=0 || cell_status=$? + ;; + esac + + if [[ "$cell_status" -ne 0 ]]; then + mbt_error "cell failed: $label (see $log_file)" + r_gen+=("$g"); r_mode+=("$mode"); r_secs+=("0"); r_ok+=("0") + else + mbt_log " -> $(mbt_format_seconds "$secs") ($secs s)" + r_gen+=("$g"); r_mode+=("$mode"); r_secs+=("$secs"); r_ok+=("1") + fi + done + done + + rm -rf "$base_dir/cell" + + # Summary table. + printf "\n" + mbt_log "benchmark summary (device=$device, jobs=$jobs)" + printf " %-16s %-8s %12s %10s\n" "generator" "ccache" "time" "vs off" + printf " %-16s %-8s %12s %10s\n" "---------" "------" "----" "------" + + local i baseline speedup + for ((i = 0; i < ${#r_gen[@]}; i++)); do + # Find the "off" baseline for the same generator, if measured. + baseline="" + local j + for ((j = 0; j < ${#r_gen[@]}; j++)); do + if [[ "${r_gen[$j]}" == "${r_gen[$i]}" && "${r_mode[$j]}" == "off" && "${r_ok[$j]}" == "1" ]]; then + baseline="${r_secs[$j]}" + break + fi + done + + if [[ "${r_ok[$i]}" != "1" ]]; then + printf " %-16s %-8s %12s %10s\n" "${r_gen[$i]}" "${r_mode[$i]}" "FAILED" "-" + continue + fi + + if [[ -n "$baseline" && "${r_mode[$i]}" != "off" ]]; then + speedup="$(awk -v b="$baseline" -v t="${r_secs[$i]}" 'BEGIN { if (t > 0) printf "%.2fx", b / t; else printf "-" }')" + else + speedup="-" + fi + printf " %-16s %-8s %12s %10s\n" "${r_gen[$i]}" "${r_mode[$i]}" "$(mbt_format_seconds "${r_secs[$i]}")" "$speedup" + done + printf "\n" + mbt_log "'vs off' > 1x means faster than the same generator's ccache=off build." + mbt_log "cold = empty cache (populate cost); warm = pre-populated cache (the ccache win)." +} + main() { local command="build" local quickstart=0 @@ -439,11 +755,14 @@ main() { local with_submodule_sync=1 local with_toolchain=1 local build_dir="$SCRIPT_PATH/build" + local build_dir_explicit=0 local generator="Ninja" local jobs local target="" local clean_build=0 local non_interactive=0 + local benchmark_generators="" + local benchmark_ccache="" local -a cmake_extra_args=() local -a build_extra_args=() @@ -457,7 +776,7 @@ main() { if [[ "$#" -gt 0 ]]; then case "$1" in - build|deps|toolchain|doctor|shell) + build|deps|toolchain|doctor|shell|benchmark) command="$1" shift ;; @@ -515,8 +834,27 @@ main() { -B|--build-dir) [[ "$#" -ge 2 ]] || mbt_die "$1 requires a value" build_dir="$2" + build_dir_explicit=1 shift 2 ;; + --bench-generators) + [[ "$#" -ge 2 ]] || mbt_die "$1 requires a value" + benchmark_generators="$2" + shift 2 + ;; + --bench-generators=*) + benchmark_generators="${1#*=}" + shift + ;; + --bench-ccache) + [[ "$#" -ge 2 ]] || mbt_die "$1 requires a value" + benchmark_ccache="$2" + shift 2 + ;; + --bench-ccache=*) + benchmark_ccache="${1#*=}" + shift + ;; -G|--generator) [[ "$#" -ge 2 ]] || mbt_die "$1 requires a value" generator="$2" @@ -598,6 +936,13 @@ main() { ;; build) ;; + benchmark) + # Never benchmark into the user's working ./build dir; a benchmark + # wipes its build dir between cells. Use a dedicated dir by default. + if [[ "$build_dir_explicit" -eq 0 ]]; then + build_dir="$SCRIPT_PATH/build-bench" + fi + ;; *) mbt_die "unsupported command: $command" ;; @@ -640,6 +985,12 @@ main() { mbt_log "quickstart mode enabled." fi + if [[ "$command" == "benchmark" ]]; then + mbt_benchmark "$build_dir" "$device" "$jobs" "$target" \ + "$benchmark_generators" "$benchmark_ccache" cmake_extra_args + exit $? + fi + mbt_build "$build_dir" "$generator" "$use_ccache" "$device" "$jobs" "$target" "$clean_build" \ cmake_extra_args[@] build_extra_args[@] }