From c8ef4c1ec9699a9c2532e6fb8f88868999d47ff4 Mon Sep 17 00:00:00 2001 From: Jochen Hoenle <173445474+hoe-jo@users.noreply.github.com> Date: Thu, 21 May 2026 08:54:36 +0200 Subject: [PATCH] [rules score] add coverage report --- .bazelrc | 8 ++ .gitignore | 2 + MODULE.bazel | 13 +++ README.md | 14 ++- coverage/BUILD | 6 ++ coverage/README.md | 71 ++++++++++++++- coverage/run_combined_coverage.sh | 143 ++++++++++++++++++++++++++++++ 7 files changed, 255 insertions(+), 2 deletions(-) create mode 100755 coverage/run_combined_coverage.sh diff --git a/.bazelrc b/.bazelrc index 4a34f05f..9b663e7c 100644 --- a/.bazelrc +++ b/.bazelrc @@ -23,5 +23,13 @@ build:info --//bazel/rules/rules_score:verbosity=info # debug build: complete output including debug/trace from all tools build:debug --//bazel/rules/rules_score:verbosity=debug +# Standard combined coverage (Rust + Python, no Ferrocene required) +# Usage: bazel coverage --config=coverage +# Then run: bazel run //coverage:combined_report +coverage:coverage --combined_report=lcov +coverage:coverage --instrumentation_filter=//plantuml,//validation,//manual_analysis,-//plantuml/parser/integration_test,-//validation/core/integration_test +coverage:coverage --@rules_rust//rust/settings:extra_rustc_flag=-Clink-dead-code +coverage:coverage --@rules_rust//rust/settings:extra_rustc_flag=-Ccodegen-units=1 + # Import AI checker custom configuration try-import %workspace%/.bazelrc.ai_checker diff --git a/.gitignore b/.gitignore index 2d4ee55f..51a160c7 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ external .clwb/ __pycache__ +.ruff_cache/ +coverage-html/ diff --git a/MODULE.bazel b/MODULE.bazel index 2519da66..1a6c2547 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -33,6 +33,7 @@ bazel_dep(name = "score_rust_policies", version = "0.0.2") bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2") bazel_dep(name = "flatbuffers", version = "25.9.23") +bazel_dep(name = "download_utils", version = "1.2.2") # flatbuffers depends on this transitively, but older grpc-java version # The main problem is that there the command `bazel mod deps` is broken, which @@ -137,6 +138,7 @@ PYTHON_VERSION = "3.12" python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain( + configure_coverage_tool = True, is_default = True, python_version = PYTHON_VERSION, ) @@ -211,6 +213,17 @@ multitool.hub( ) use_repo(multitool, "yamlfmt_hub") +############################################################################### +# lcov deb package (provides genhtml + lcov for combined coverage reports) +############################################################################### +deb = use_repo_rule("@download_utils//download/deb:defs.bzl", "download_deb") + +deb( + name = "lcov_deb", + integrity = "sha256-Ip14IkKavqBtkQ7mh6AXzr/6YyHpvSAZ0veMmw1+N80=", + urls = ["https://archive.ubuntu.com/ubuntu/pool/universe/l/lcov/lcov_2.0-4ubuntu2_all.deb"], +) + register_toolchains( "//bazel/rules/rules_score:sphinx_default_toolchain", ) diff --git a/README.md b/README.md index b4d336e9..b7762941 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,19 @@ See the individual README files for detailed usage instructions and configuratio | **python_basics** | Python development utilities and testing | [README](python_basics/README.md) | | **starpls** | Starlark language server support | [README](starpls/README.md) | | **tools** | Formatters & Linters | [README](tools/README.md) | -| **coverage** | Ferrocene Rust coverage workflow | [README](coverage/README.md) | +| **coverage** | Rust + Python coverage reports | [README](coverage/README.md) | + +## Coverage + +Generate a combined Rust + Python HTML coverage report for `plantuml`, `validation`, +and `manual_analysis`: + +```bash +bazel run //coverage:combined_report +``` + +See [coverage/README.md](coverage/README.md) for full details, options, and the +Ferrocene Rust coverage workflow. ## Usage Examples diff --git a/coverage/BUILD b/coverage/BUILD index 758507e9..3553984b 100644 --- a/coverage/BUILD +++ b/coverage/BUILD @@ -36,3 +36,9 @@ sh_binary( srcs = ["llvm_profile_wrapper.sh"], visibility = ["//visibility:public"], ) + +sh_binary( + name = "combined_report", + srcs = ["run_combined_coverage.sh"], + data = ["@lcov_deb//:srcs"], +) diff --git a/coverage/README.md b/coverage/README.md index 713b9a85..f428dd1a 100644 --- a/coverage/README.md +++ b/coverage/README.md @@ -4,7 +4,76 @@ Copyright (c) 2026 Contributors to the Eclipse Foundation SPDX-License-Identifier: Apache-2.0 --> -# Ferrocene Rust Coverage +# Coverage + +## Combined Rust + Python Coverage + +The `//coverage:combined_report` target generates a single HTML coverage report +for all Rust and Python tools in the repository using Bazel's built-in +coverage support (`bazel coverage`) and `genhtml`. + +### Usage + +```bash +bazel run //coverage:combined_report +``` + +This runs `bazel coverage --config=coverage` for `//plantuml/...`, +`//validation/...` and `//manual_analysis/...`, merges all LCOV data, and +renders the report to `/coverage-html/index.html`. + +Custom output directory: + +```bash +bazel run //coverage:combined_report -- --out-dir /tmp/my-coverage +``` + +Custom target set: + +```bash +bazel run //coverage:combined_report -- --targets "//plantuml/... //validation/core/..." +``` + +### How it works + +1. `bazel coverage --config=coverage` compiles Rust with `-Cinstrument-coverage` + and wraps Python tests with `coverage.py` (via `rules_python`'s built-in + `configure_coverage_tool`). +2. Bazel merges all per-test LCOV files into one `_coverage_report.dat` + (controlled by `--combined_report=lcov`). +3. `--instrumentation_filter` limits instrumentation to the three tool + packages, excluding external dependencies and generated code. +4. Test infrastructure files (`integration_test/`, `tests/`) are excluded from + instrumentation via `--instrumentation_filter`; external Python files are + removed via `lcov --remove`. +5. The HTML report uses a high-coverage threshold of **95 %** (green) and the + default medium threshold of 75 % (yellow). +6. `genhtml` and `lcov` are downloaded hermetically via the `download_utils` + Bazel module (`@lcov_deb`) — no system installation of `lcov` is required. + +### .bazelrc config + +The `coverage:coverage` config in `.bazelrc` provides the required flags: + +``` +coverage:coverage --combined_report=lcov +coverage:coverage --instrumentation_filter=//plantuml,//validation,//manual_analysis,-//plantuml/parser/integration_test,-//validation/core/integration_test +coverage:coverage --@rules_rust//rust/settings:extra_rustc_flag=-Clink-dead-code +coverage:coverage --@rules_rust//rust/settings:extra_rustc_flag=-Ccodegen-units=1 +``` + +You can also run `bazel coverage` directly without the script (requires `genhtml` +from the system `lcov` package): + +```bash +bazel coverage --config=coverage //plantuml/... //validation/... //manual_analysis/... +genhtml "$(bazel info output_path)/_coverage/_coverage_report.dat" \ + --output-directory coverage-html/ +``` + +--- + +## Ferrocene Rust Coverage This directory provides the Ferrocene Rust coverage workflow for Bazel-based projects. It uses Ferrocene's `symbol-report` and `blanket` tools to generate diff --git a/coverage/run_combined_coverage.sh b/coverage/run_combined_coverage.sh new file mode 100755 index 00000000..a439cceb --- /dev/null +++ b/coverage/run_combined_coverage.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +set -euo pipefail + +usage() { + cat <<'USAGE' +Generate a combined Rust + Python coverage HTML report. + +Runs 'bazel coverage' for plantuml, validation and manual_analysis, then +renders a single HTML report via genhtml. + +Usage: + bazel run //coverage:combined_report -- [options] + +Options: + --out-dir Output directory for the HTML report. + Default: /coverage-html + --targets Space-separated list of Bazel target patterns. + Default: //plantuml/... //validation/... //manual_analysis/... + --help Show this help. + +Requirements: + genhtml and lcov must be available either via the Bazel-managed @lcov_deb + runfiles (automatic when run via 'bazel run //coverage:combined_report') or + installed system-wide (apt install lcov). +USAGE +} + +OUT_DIR="" +TARGETS="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --out-dir) + OUT_DIR="$2"; shift 2 ;; + --targets) + TARGETS="$2"; shift 2 ;; + -h|--help) + usage; exit 0 ;; + *) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 ;; + esac +done + +# When invoked via 'bazel run', BUILD_WORKSPACE_DIRECTORY is set to the workspace +# root. We must cd into it before calling nested bazel commands, because Bazel +# refuses to be invoked from inside its own output tree. +WORKSPACE_DIR="${BUILD_WORKSPACE_DIRECTORY:-$(bazel info workspace 2>/dev/null)}" +cd "$WORKSPACE_DIR" + +if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="${WORKSPACE_DIR}/coverage-html" +fi + +if [[ -z "$TARGETS" ]]; then + TARGETS="//plantuml/... //validation/... //manual_analysis/..." +fi + +# --------------------------------------------------------------------------- +# Resolve lcov tools: prefer Bazel-managed binaries from @lcov_deb runfiles +# so that no system lcov installation is required. Fall back to PATH. +# --------------------------------------------------------------------------- +_tool_path() { + local name="$1" + local found="" + # Search runfiles for the Bazel-managed binary (works regardless of the + # canonical repo name Bazel assigns under bzlmod, e.g. +_repo_rules+lcov_deb) + if [[ -n "${RUNFILES_DIR:-}" ]]; then + found=$(find "${RUNFILES_DIR}" -path "*/lcov_deb/usr/bin/${name}" -type f 2>/dev/null | head -1) + fi + # Fall back to system PATH + if [[ -z "${found}" ]]; then + found=$(command -v "${name}" 2>/dev/null || true) + fi + echo "${found}" +} + +GENHTML="$(_tool_path genhtml)" +LCOV="$(_tool_path lcov)" + +if [[ -z "$GENHTML" ]]; then + echo "ERROR: 'genhtml' not found. Run via 'bazel run //coverage:combined_report' or install 'lcov'." >&2 + exit 1 +fi +if [[ -z "$LCOV" ]]; then + echo "ERROR: 'lcov' not found. Run via 'bazel run //coverage:combined_report' or install 'lcov'." >&2 + exit 1 +fi + +# When using the Bazel-managed tools, set PERL5LIB so Perl finds lcovutil.pm. +if [[ -n "${RUNFILES_DIR:-}" ]]; then + lcov_lib=$(find "${RUNFILES_DIR}" -path "*/lcov_deb/usr/lib/lcov" -type d 2>/dev/null | head -1) + if [[ -n "${lcov_lib}" ]]; then + export PERL5LIB="${lcov_lib}${PERL5LIB:+:${PERL5LIB}}" + fi +fi + +echo "==> Running bazel coverage --config=coverage ${TARGETS}" +# shellcheck disable=SC2086 # word-splitting of TARGETS is intentional +bazel coverage --config=coverage $TARGETS + +DAT_FILE="$(bazel info output_path)/_coverage/_coverage_report.dat" + +if [[ ! -f "$DAT_FILE" ]]; then + echo "ERROR: Coverage data not found at ${DAT_FILE}" >&2 + echo " Make sure at least one test ran successfully." >&2 + exit 1 +fi + +echo "==> Generating HTML report in ${OUT_DIR}" +mkdir -p "$OUT_DIR" + +# Remove files that should not count towards coverage: +# - external/ : Python files from rules_python internals captured by coverage.py +# lcov --ignore-errors unused prevents a failure when none of the patterns match. +FILTERED_DAT="${OUT_DIR}/filtered_coverage.dat" +"$LCOV" --remove "$DAT_FILE" \ + "external/*" \ + --output-file "$FILTERED_DAT" \ + --ignore-errors unused + +"$GENHTML" "$FILTERED_DAT" \ + --output-directory "$OUT_DIR" \ + --legend \ + --title "Combined Rust + Python Coverage" \ + --rc genhtml_hi_limit=95 \ + --ignore-errors source + +echo "" +echo "Coverage report: ${OUT_DIR}/index.html"