diff --git a/.gitignore b/.gitignore index 04bd016fc..f2092a19e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ /.idea/ +/.metals/ +/.bloop/ /scripts/docker/m2/ /python/geobrix/.venv/ /python/geobrix/dist/ @@ -48,4 +50,6 @@ __pycache__/ !/.claude/qc-judge/ /input/ .venv-pyrx/ +.venv-host-ci/ +.venv-host-pyrx/ test-logs/bench/ diff --git a/CLAUDE.md b/CLAUDE.md index 3a1d36fd1..451db8a26 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,6 +57,19 @@ Use `gbx:docker:start` / `gbx:docker:exec` rather than `docker run` directly. Th Default Maven profile is **`skipScoverage`** for fast compile/test (`mvn clean package -DskipTests`). Coverage commands explicitly trigger the `standard` profile. +### Building/testing on the host (arca, outside the container) + +The container is still the canonical path, but the project also builds and tests directly on an arca host (Ubuntu 24.04 noble, x86_64) via a `$HOME`-local GDAL install — useful when Docker isn't wanted. This host setup lives in the **`geobrix-arca` plugin** (marketplace `experimental/general`), not in this repo — install it with `isaac plugin add geobrix-arca@experimental`. Its two skills: + +- **`geobrix-gdal-env`** — provisions native GDAL into `$HOME`. Its `setup-host-gdal.sh` adds the fingerprint-pinned UbuntuGIS PPA, downloads the pinned `.debs` (GDAL `3.11.4+dfsg-1~noble0`, matching `scripts/build-gdal-artifacts.sh`), extracts them under `$HOME/.local/gdal` (no root, no `/usr` writes), reuses the committed JNI (`scripts/gdal311/libgdalalljni.so` — a plain committed file, **not** LFS), and bridges the two `/usr` paths the Scala hardcodes. **The LFS platform tarball is unreachable from arca** (`lfs.github.com` is firewall-blocked), so the PPA-download path is the one that works here — not extracting the committed tarball. + - **Each fresh session:** `setup-host-gdal.sh --bridges-only` recreates the two root-owned `/usr` symlinks (they don't persist; `$HOME/.local/gdal` does). + - **In every build/test shell:** `source ~/.local/geobrix-gdal-env.sh`. It pins **Java 17** (arca defaults to 21; pom targets `release=17`), and sets `LD_LIBRARY_PATH` (so the JNI resolves `libgdal.so.37` + deps), `PROJ_DATA`/`GDAL_DATA`, and `PYTHONPATH` (extracted `osgeo`/`numpy` — required or GDAL's embedded-Python VRT pixel functions in RST_MapAlgebra/CombineAvg/DerivedBand/vegetation-indices silently no-op and the output raster is null). +- **`geobrix-metals-nav`** — fixes Cursor/Metals code navigation on arca (the managed Metals is wired to the monorepo Bazel BSP, so geobrix imports underline as unresolved). Generates an analysis-only Bloop config (Scala 2.13.18 to match the mtags arca's Metals ships; the pom/build/JAR stay 2.13.16) and steers Metals to it. + +Then build with `mvn clean package -DskipTests -PskipScoverage`, or run suites with `mvn test -PskipScoverage -DskipTests=false -Dsuites='...'`. For `/Volumes` suites, symlink `/Volumes` → `sample-data/Volumes` (mirrors the container mount); the full sample bundles still require `gbx:data:download`. + +**Caveat:** the `gbx:test:*` commands are hardwired to `docker exec geobrix-dev` and do **not** run on the host — invoke `mvn` directly, or extend those commands with a `--host` mode. + ## Commands (the `gbx:*` palette) The repo has **50 `gbx:*` commands** in `scripts/commands/` (each is a `.md` registration + a `.sh` implementation). They handle Docker setup, env vars, log paths (`--log filename` → `test-logs/filename`), and profile selection. Originally registered for Cursor's command palette (hence the `.md` files), they're now invoked directly from any shell or via the Task tool. diff --git a/scripts/commands/common.sh b/scripts/commands/common.sh index f29bbb1a7..6495597cd 100755 --- a/scripts/commands/common.sh +++ b/scripts/commands/common.sh @@ -186,6 +186,121 @@ validate_set() { esac } +# Assert the arca host GDAL environment is active (LD_LIBRARY_PATH points at the $HOME-local +# GDAL install and gdalinfo is on PATH). The --host test paths need native GDAL in the forked +# Spark JVM; that is provided by sourcing ~/.local/geobrix-gdal-env.sh (geobrix-arca plugin), +# NOT by this repo. We only assert it — we never source it (it's user/plugin-owned). +# Returns non-zero with a remediation message if the env is not active. +require_host_gdal_env() { + if ! command -v gdalinfo >/dev/null 2>&1 || [[ "${LD_LIBRARY_PATH:-}" != *".local/gdal"* ]]; then + echo -e "${RED}❌ Host GDAL environment not active.${NC}" >&2 + echo -e "${YELLOW} --host mode needs native GDAL on the arca host. Source the env first:${NC}" >&2 + echo -e "${YELLOW} source ~/.local/geobrix-gdal-env.sh${NC}" >&2 + echo -e "${YELLOW} (provisioned by the geobrix-arca plugin's geobrix-gdal-env skill).${NC}" >&2 + return 1 + fi + return 0 +} + +# Ensure a host test venv exists, built from one of CI's exact hash-pinned locks via uv, with the +# geobrix package installed editable+no-deps. Mirrors CI's TWO-environment split (CI never runs all +# Python tests in one env) — pick the venv by the "kind" arg: +# +# ci -> $PROJECT_ROOT/.venv-host-ci from requirements-ci.txt (~27 pkgs: pyspark, py4j, +# numpy, pytest; NO rasterio/pandas/pdal). Matches CI's python_build "heavy" job, which +# runs `pytest test -m "not integration" --ignore=test/pyrx --ignore=test/pyvx`. +# pyrx -> $PROJECT_ROOT/.venv-host-pyrx from requirements-pyrx-ci.txt (~104 pkgs: rasterio, +# shapely, pandas, pyarrow, h3, mapbox-vector-tile, vizx stack). Matches CI's pyrx_build +# "light" job (test/pyrx test/ds test/pyvx test/pygx test/pmtiles_light test/stac +# test/vizx test/sample) and is also the right env for the doc-tests (rasterio/pandas). +# +# Both locks are pure wheels on arca (neither contains pdal, which is source-only and needs native +# PDAL the container builds but arca lacks — so no package filtering is required, unlike the +# container image lock). uv is required (stdlib `python3 -m venv` yields a pip-less venv on arca — +# ensurepip is absent). The index is taken from ambient PIP_INDEX_URL/UV_INDEX_URL — never hardcoded. +# +# gdal/osgeo are NOT in either lock (CI installs gdal[numpy] from the apt-matched sdist); on the host +# they are provided on PYTHONPATH by the sourced arca env, additive to the venv at runtime. +# +# A stamp holds the sha256 of the source lock so re-runs skip the install unless the lock changed. +# Set GBX_REBUILD_VENV=1 to force a rebuild. Echoes the venv bin dir; callers use +# "$(ensure_host_test_venv )/python -m pytest". +ensure_host_test_venv() { + local kind="${1:-pyrx}" + local venv_dir reqs + case "$kind" in + ci) venv_dir="${PROJECT_ROOT}/.venv-host-ci"; reqs="${PROJECT_ROOT}/python/geobrix/requirements-ci.txt" ;; + pyrx) venv_dir="${PROJECT_ROOT}/.venv-host-pyrx"; reqs="${PROJECT_ROOT}/python/geobrix/requirements-pyrx-ci.txt" ;; + *) echo -e "${RED}❌ ensure_host_test_venv: unknown kind '$kind' (expected ci|pyrx)${NC}" >&2; return 1 ;; + esac + local stamp="${venv_dir}/.gbx-reqs-stamp" + local py_version="${GBX_HOST_PY_VERSION:-3.12}" + + if ! command -v uv >/dev/null 2>&1; then + echo -e "${RED}❌ uv not found on PATH — required to build the host test venv.${NC}" >&2 + echo -e "${YELLOW} Install uv: https://docs.astral.sh/uv/ (or use the geobrix-arca plugin).${NC}" >&2 + return 1 + fi + if [ ! -f "$reqs" ]; then + echo -e "${RED}❌ Pinned requirements not found: $reqs${NC}" >&2 + return 1 + fi + + # Stamp on the source lock's hash: if the committed lock changes, the venv rebuilds. + local want_hash cur_hash + want_hash="$(sha256sum "$reqs" | awk '{print $1}')" + cur_hash="$(cat "$stamp" 2>/dev/null || true)" + + if [ "${GBX_REBUILD_VENV:-0}" = "1" ] || [ ! -x "${venv_dir}/bin/python" ] || [ "$want_hash" != "$cur_hash" ]; then + echo -e "${CYAN}🐍 Building host test venv (${kind}) at ${YELLOW}${venv_dir}${CYAN} from $(basename "$reqs")...${NC}" >&2 + [ "${GBX_REBUILD_VENV:-0}" = "1" ] && rm -rf "$venv_dir" + uv venv "$venv_dir" --python "$py_version" >&2 \ + || { echo -e "${RED}❌ uv venv failed${NC}" >&2; return 1; } + uv pip install --python "${venv_dir}/bin/python" --require-hashes -r "$reqs" >&2 \ + || { echo -e "${RED}❌ uv pip install ($kind lock) failed — check PIP_INDEX_URL/proxy coverage${NC}" >&2; return 1; } + uv pip install --python "${venv_dir}/bin/python" --no-deps -e "${PROJECT_ROOT}/python/geobrix" >&2 \ + || { echo -e "${RED}❌ editable geobrix install failed${NC}" >&2; return 1; } + printf '%s\n' "$want_hash" > "$stamp" + echo -e "${GREEN}✅ Host test venv (${kind}) ready.${NC}" >&2 + fi + + echo "${venv_dir}/bin" +} + +# Prepare the process environment so a host --host pytest run drives Spark + rasterio correctly. +# Call with the venv bin dir (from ensure_host_test_venv) BEFORE launching pytest. Two exports the +# forked Spark JVM's Python workers and rasterio need: +# - PYSPARK_PYTHON / PYSPARK_DRIVER_PYTHON = the venv interpreter, so Spark workers use the venv +# (pandas/pyarrow for Arrow UDFs live there, not in system python3). These MUST be real exports +# in the current shell — a `VAR=x eval "..."` command-prefix does NOT propagate to the python +# grandchild, so workers would silently fall back to system python3 (ModuleNotFound: pandas). +# - unset PROJ_DATA / PROJ_LIB so the venv rasterio uses its own bundled proj.db (layout >=6); +# the arca env points these at the older $HOME GDAL proj.db (layout 3), which rasterio rejects +# (CRSError "... another PROJ installation"). unset (not empty-string) is required — an empty +# PROJ_DATA is a search path of "", not a fallback to bundled data. The JVM GDAL sets PROJ_LIB +# internally via SetConfigOption (the /usr/share/proj bridge), so the heavy tier is unaffected. +# Usage: activate_host_python_env "$VENV_BIN" +activate_host_python_env() { + local venv_bin="$1" + export PYSPARK_PYTHON="${venv_bin}/python" + export PYSPARK_DRIVER_PYTHON="${venv_bin}/python" + unset PROJ_DATA PROJ_LIB +} + +# The light-tier test dirs (single source of truth: python/geobrix/test/conftest.py _LIGHT_TEST_DIRS). +# Their modules import light-only deps (rasterio/shapely/pandas/h3/…) at collection time, so they run +# only in the pyrx venv; the ci venv's conftest collect_ignore skips them (rasterio absent). Echoed +# space-separated. Falls back to a hardcoded list only if the conftest can't be parsed. +host_light_test_dirs() { + local conftest="${PROJECT_ROOT}/python/geobrix/test/conftest.py" + local dirs + dirs="$(awk '/_LIGHT_TEST_DIRS *= *\[/{f=1;next} f&&/\]/{f=0} f{gsub(/[",[:space:]]/,""); if($0!="") print}' "$conftest" 2>/dev/null | tr '\n' ' ')" + if [ -z "${dirs// /}" ]; then + dirs="bench ds pyrx pyvx pygx pmtiles_light stac vizx sample" + fi + echo "$dirs" +} + # Run a command inside the isolated pyrx venv (host, no Docker). # Usage: run_in_pyrx_venv "" # Requires gbx:venv:sync to have been run (venv at $PROJECT_ROOT/.venv-pyrx). @@ -209,4 +324,6 @@ run_in_pyrx_venv() { export RED GREEN YELLOW BLUE CYAN NC DOCKER_MAVEN_ENV export -f check_docker resolve_log_path setup_log_file show_banner show_separator \ print_report_link open_report generate_timestamp warn_if_jar_stale \ - print_banner print_separator setup_log run_in_pyrx_venv validate_set 2>/dev/null || true + print_banner print_separator setup_log run_in_pyrx_venv validate_set \ + require_host_gdal_env ensure_host_test_venv activate_host_python_env \ + host_light_test_dirs 2>/dev/null || true diff --git a/scripts/commands/gbx-test-docs.md b/scripts/commands/gbx-test-docs.md index c31f8a49e..164e781ff 100644 --- a/scripts/commands/gbx-test-docs.md +++ b/scripts/commands/gbx-test-docs.md @@ -20,6 +20,8 @@ bash scripts/commands/gbx-test-docs.sh [OPTIONS] **Common** +- `--host` – Run on the host (arca), not the Docker container. Passed through to each child suite (python-docs, sql-docs, scala-docs). Requires `source ~/.local/geobrix-gdal-env.sh` first (provisioned by the `geobrix-arca` plugin). See "Host mode" below. +- `--rebuild-venv` – (with `--host`) force-rebuild the host test venv; forwarded to the venv-based child suites (python-docs, sql-docs). - `--log ` – Log file (filename → `test-logs/`). - `--markers ` – Pytest markers for Python (e.g. `"not slow"`). - `--include-integration` – Include Python integration tests (excluded by default). @@ -30,6 +32,10 @@ bash scripts/commands/gbx-test-docs.sh [OPTIONS] - `--no-sample-data-root` – Do **not** set `GBX_SAMPLE_DATA_ROOT` (use your env or path_config default; e.g. full bundle). - `--help` – Help and examples. +## Host mode (arca, no Docker) + +With `--host` the orchestrator runs on the host instead of `docker exec geobrix-dev`, forwarding `--host` to each child suite (python-docs, sql-docs, scala-docs). Prerequisites: `source ~/.local/geobrix-gdal-env.sh` (native GDAL + Java 17 + PYTHONPATH) and `uv` on PATH, with `PIP_INDEX_URL` pointing at the internal pip proxy. The venv-based child suites build a host test venv from the exact CI-pinned locks via `uv` on first run — `.venv-host-pyrx` from `python/geobrix/requirements-pyrx-ci.txt` (light-tier deps); neither CI lock contains `pdal` (source-only, unbuildable on arca); `--rebuild-venv` is forwarded to them. The scala-docs child runs `mvn` directly and needs only the sourced GDAL env. See the `geobrix-arca` plugin for the full setup. + **Sample data (default):** The command sets `GBX_SAMPLE_DATA_ROOT=/Volumes/main/default/test-data` inside the container so doc tests use the minimal bundle (host path `sample-data/Volumes/main/default/test-data`). This is required for running docs unit tests on remote/CI. Use `--no-sample-data-root` to leave it unset (e.g. to use a full bundle or your own env). ## Examples @@ -38,6 +44,10 @@ bash scripts/commands/gbx-test-docs.sh [OPTIONS] # Full run with build bash scripts/commands/gbx-test-docs.sh +# On the arca host (no Docker) — source the GDAL env first +source ~/.local/geobrix-gdal-env.sh +bash scripts/commands/gbx-test-docs.sh --host + # Fast run (skip build), with log. Uses in-repo minimal bundle; no download. bash scripts/commands/gbx-test-docs.sh --skip-build --log docs.log diff --git a/scripts/commands/gbx-test-docs.sh b/scripts/commands/gbx-test-docs.sh index 1c1786f9a..2e5ad99cc 100644 --- a/scripts/commands/gbx-test-docs.sh +++ b/scripts/commands/gbx-test-docs.sh @@ -21,6 +21,7 @@ show_help() { echo -e " ${GREEN}--test ${NC} Python: single test node id" echo "" echo -e "${CYAN}Common options:${NC}" + echo -e " ${GREEN}--host${NC} Run on the host (arca), not Docker. Requires ${YELLOW}source ~/.local/geobrix-gdal-env.sh${NC} first." echo -e " ${GREEN}--log ${NC} Write output to log (filename → test-logs/)" echo -e " ${GREEN}--markers ${NC} Pytest markers for Python (e.g. \"not slow\")" echo -e " ${GREEN}--include-integration${NC} Include Python integration tests (excluded by default)" @@ -47,6 +48,7 @@ SCALA_SUITE="tests.docs.scala.*" PYTHON_ONLY=false SCALA_ONLY=false SET_SAMPLE_DATA_ROOT=true +USE_HOST=false # Pass-through for Python phase (only one of these set) SUITE_VAL="" PATH_VAL="" @@ -54,6 +56,14 @@ TEST_VAL="" while [[ $# -gt 0 ]]; do case $1 in + --host) + USE_HOST=true + shift + ;; + --rebuild-venv) + export GBX_REBUILD_VENV=1 + shift + ;; --suite) case "$2" in quickstart|api|readers|rasterx|advanced|setup) @@ -128,13 +138,18 @@ done cd "$PROJECT_ROOT" show_banner "📚 GeoBrix: All Documentation Tests" -check_docker setup_log_file "$LOG_PATH" -mkdir -p "$PROJECT_ROOT/sample-data/Volumes/main/default/geobrix_samples" -if ! docker exec geobrix-dev test -d /Volumes 2>/dev/null; then - echo -e "${RED}❌ /Volumes not found. Start with: ./scripts/docker/start_docker_with_volumes.sh${NC}" - exit 1 +if [ "$USE_HOST" = true ]; then + require_host_gdal_env || exit 1 + echo -e "${CYAN}🖥️ Host mode (arca) — passing --host to child suites${NC}" +else + check_docker + mkdir -p "$PROJECT_ROOT/sample-data/Volumes/main/default/geobrix_samples" + if ! docker exec geobrix-dev test -d /Volumes 2>/dev/null; then + echo -e "${RED}❌ /Volumes not found. Start with: ./scripts/docker/start_docker_with_volumes.sh${NC}" + exit 1 + fi fi [ "$SKIP_BUILD" = true ] && echo -e "${CYAN}⏭️ Skipping build (--skip-build)${NC}" @@ -146,6 +161,7 @@ TOTAL_EXIT=0 PYTHON_ARR=() SQL_ARR=() SCALA_ARR=() +[ "$USE_HOST" = true ] && { PYTHON_ARR+=(--host); SQL_ARR+=(--host); SCALA_ARR+=(--host); } [ "$SKIP_BUILD" = true ] && { PYTHON_ARR+=(--skip-build); SQL_ARR+=(--skip-build); SCALA_ARR+=(--skip-build); } [ "$SET_SAMPLE_DATA_ROOT" = false ] && { PYTHON_ARR+=(--no-sample-data-root); SQL_ARR+=(--no-sample-data-root); SCALA_ARR+=(--no-sample-data-root); } [ -n "$MARKERS_VAL" ] && [ "$INCLUDE_INTEGRATION" = false ] && { PYTHON_ARR+=(--markers "$MARKERS_VAL"); SQL_ARR+=(--markers "$MARKERS_VAL"); } diff --git a/scripts/commands/gbx-test-function-info.md b/scripts/commands/gbx-test-function-info.md index 2ce753bc7..175d66615 100644 --- a/scripts/commands/gbx-test-function-info.md +++ b/scripts/commands/gbx-test-function-info.md @@ -11,9 +11,15 @@ bash scripts/commands/gbx-test-function-info.sh [OPTIONS] ## Options - `--skip-generate` - Skip the generator; run only pytest in `docs/tests-function-info/` +- `--host` - Run on the host (arca), not the Docker container. Requires `source ~/.local/geobrix-gdal-env.sh` first (provisioned by the `geobrix-arca` plugin); builds/reuses `.venv-host-pyrx` from the pinned CI lock and runs against a host-built JAR. See "Host mode" below. +- `--rebuild-venv` - (with `--host`) force-rebuild the host test venv. - `--log ` - Write output to log file - `--help` - Display help +## Host mode (arca, no Docker) + +With `--host` the command runs directly on the host instead of `docker exec geobrix-dev`. Prerequisites: `source ~/.local/geobrix-gdal-env.sh` (native GDAL + Java 17 + PYTHONPATH) and `uv` on PATH, with `PIP_INDEX_URL` pointing at the internal pip proxy. The pytest registers functions via the built JAR; no sample data is needed. The first run builds a host test venv from the exact CI-pinned lock via `uv` — `.venv-host-pyrx` from `python/geobrix/requirements-pyrx-ci.txt` (the light-tier deps: rasterio/pandas/h3/vizx). Neither CI lock contains `pdal` (source-only, unbuildable on arca and not needed here). See the `geobrix-arca` plugin for the full setup. + ## Default behavior (inside Docker) 1. **Generate**: `python3 docs/scripts/generate-function-info.py` in container @@ -29,6 +35,10 @@ bash scripts/commands/gbx-test-function-info.sh [OPTIONS] # Full run: generate then test gbx:test:function-info +# On the arca host (no Docker) — source the GDAL env first +source ~/.local/geobrix-gdal-env.sh +gbx:test:function-info --host + # Only run tests (do not regenerate JSON) gbx:test:function-info --skip-generate diff --git a/scripts/commands/gbx-test-function-info.sh b/scripts/commands/gbx-test-function-info.sh index ac63add89..7104cde1f 100644 --- a/scripts/commands/gbx-test-function-info.sh +++ b/scripts/commands/gbx-test-function-info.sh @@ -20,6 +20,9 @@ show_help() { echo "" echo -e "${CYAN}Options:${NC}" echo -e " ${GREEN}--skip-generate${NC} Skip the generator step; run only tests" + echo -e " ${GREEN}--host${NC} Run on the host (arca), not Docker. Requires ${YELLOW}source ~/.local/geobrix-gdal-env.sh${NC}" + echo -e " first; builds/uses ${YELLOW}.venv-host${NC} and a built JAR." + echo -e " ${GREEN}--rebuild-venv${NC} (with --host) force-rebuild the host test venv" echo -e " ${GREEN}--log ${NC} Write output to log file" echo -e " ${GREEN}--help${NC} Show this help" echo "" @@ -27,12 +30,21 @@ show_help() { SKIP_GENERATE=false LOG_PATH="" +USE_HOST=false while [[ $# -gt 0 ]]; do case $1 in --skip-generate) SKIP_GENERATE=true shift ;; + --host) + USE_HOST=true + shift + ;; + --rebuild-venv) + export GBX_REBUILD_VENV=1 + shift + ;; --log) LOG_PATH=$(resolve_log_path "$2") shift 2 @@ -52,31 +64,53 @@ done cd "$PROJECT_ROOT" || exit 1 show_banner "🧪 GeoBrix: Function-Info Tests" -check_docker setup_log_file "$LOG_PATH" -# Run generator and pytest inside container (paths under /root/geobrix) -RUN_CMD="set -e +EXIT=0 +if [ "$USE_HOST" = true ]; then + # --- Host (arca) path: no Docker. The pytest registers functions via the built JAR. --- + require_host_gdal_env || exit 1 + VENV_BIN=$(ensure_host_test_venv pyrx) || exit 1 + # Wire Spark workers to the venv python and unset PROJ_DATA/PROJ_LIB (see common.sh). + activate_host_python_env "$VENV_BIN" + warn_if_jar_stale "$PROJECT_ROOT" + unset JAVA_TOOL_OPTIONS + export JUPYTER_PLATFORM_DIRS=1 + if [ "$SKIP_GENERATE" = false ]; then + echo -e "${CYAN}Step 1: Generate function-info.json from doc SQL examples...${NC}" + "$VENV_BIN/python" docs/scripts/generate-function-info.py || EXIT=$? + echo "" + fi + if [ $EXIT -eq 0 ]; then + echo -e "${CYAN}Step 2: Run function-info tests (DESCRIBE output + coverage, host)...${NC}" + "$VENV_BIN/python" -m pytest docs/tests-function-info/ -v -s --tb=short || EXIT=$? + fi +else + # --- Docker path (unchanged) --- + check_docker + + # Run generator and pytest inside container (paths under /root/geobrix) + RUN_CMD="set -e unset JAVA_TOOL_OPTIONS export JUPYTER_PLATFORM_DIRS=1 cd /root/geobrix " -if [ "$SKIP_GENERATE" = false ]; then - RUN_CMD="$RUN_CMD + if [ "$SKIP_GENERATE" = false ]; then + RUN_CMD="$RUN_CMD echo 'Step 1: Generate function-info.json from doc SQL examples (fails if any registered function has no doc example)...' python3 docs/scripts/generate-function-info.py echo '' " -fi + fi -RUN_CMD="$RUN_CMD + RUN_CMD="$RUN_CMD echo 'Step 2: Run function-info tests (DESCRIBE output + coverage)...' python3 -m pytest docs/tests-function-info/ -v -s --tb=short " -EXIT=0 -docker exec geobrix-dev /bin/bash -c "$RUN_CMD" || EXIT=$? + docker exec geobrix-dev /bin/bash -c "$RUN_CMD" || EXIT=$? +fi show_separator if [ $EXIT -eq 0 ]; then diff --git a/scripts/commands/gbx-test-notebooks.md b/scripts/commands/gbx-test-notebooks.md index c27f32239..638545914 100644 --- a/scripts/commands/gbx-test-notebooks.md +++ b/scripts/commands/gbx-test-notebooks.md @@ -14,11 +14,19 @@ bash scripts/commands/gbx-test-notebooks.sh [OPTIONS] **Common** +- `--host` – Run on the host (arca), not the Docker container. Requires `source ~/.local/geobrix-gdal-env.sh` first (provisioned by the `geobrix-arca` plugin); builds/reuses `.venv-host-pyrx` from the pinned CI lock and runs against a host-built JAR. See "Host mode" below. +- `--rebuild-venv` – (with `--host`) force-rebuild the host test venv. - `--log ` – Write output to log (filename → `test-logs/`). - `--path ` – Limit scope: subdir (e.g. `sample-data`, `fixtures`), a single `.ipynb`, or a test file (e.g. `test_notebook_via_script.py`). With a `.py` path, runs **pytest** for that file instead of the cell-by-cell runner. - `--include-integration` – Include full-notebook execution tests when running **pytest** (default: **false**). - `--help` – Help and examples. +## Host mode (arca, no Docker) + +With `--host` the command runs directly on the host instead of `docker exec geobrix-dev`. Prerequisites: `source ~/.local/geobrix-gdal-env.sh` (native GDAL + Java 17 + PYTHONPATH) and `uv` on PATH, with `PIP_INDEX_URL` pointing at the internal pip proxy. The notebook runner executes from the host venv against the built JAR. The first run builds a host test venv from the exact CI-pinned lock via `uv` — `.venv-host-pyrx` from `python/geobrix/requirements-pyrx-ci.txt` (the light-tier deps: rasterio/pandas/h3/vizx), plus `nbformat`/`nbconvert` (needed by the runner, installed on demand). Neither CI lock contains `pdal` (source-only, unbuildable on arca and not needed here). Unlike the container, the host path runs the notebooks directly in `.venv-host-pyrx` (`GBX_NOTEBOOK_ISOLATED_ENV=0`) rather than a nested `python -m venv` (which fails on arca — no `ensurepip`/`python3-venv`). + +**Note (bare host):** notebooks that write sample bundles to the literal `/Volumes` path (the data-download notebook) fail on a bare host with `Permission denied: '/Volumes'` — those need a real UC Volumes mount (`sudo ln -sfn "$PWD/sample-data/Volumes" /Volumes`) and, for Sentinel-2 fixtures, `pystac-client`/`planetary-computer`. The runner itself works on host; these are data/mount limitations. See the `geobrix-arca` plugin for the full setup. + **Read/write path behavior (absolute vs relative)** - **Default**: Absolute, non-temp paths in notebook cells are **remapped** to the cell-by-cell workdir so reads and writes go under a temp directory and runs are fully isolated. That includes **`/Volumes/`** so you can test setup bundles without touching real volume data; existence checks (e.g. `Path("/Volumes/...").exists()`) and file reads see the workdir tree. Paths under `/tmp` (or `tempfile.gettempdir()`) are left unchanged; relative paths are unchanged. @@ -41,6 +49,10 @@ bash scripts/commands/gbx-test-notebooks.sh [OPTIONS] # Cell-by-cell run of fixtures + sample-data notebooks (default) bash scripts/commands/gbx-test-notebooks.sh +# On the arca host (no Docker) — source the GDAL env first +source ~/.local/geobrix-gdal-env.sh +bash scripts/commands/gbx-test-notebooks.sh --host + # Only sample-data notebooks bash scripts/commands/gbx-test-notebooks.sh --path sample-data diff --git a/scripts/commands/gbx-test-notebooks.sh b/scripts/commands/gbx-test-notebooks.sh index 109dced46..3b35d5f9d 100644 --- a/scripts/commands/gbx-test-notebooks.sh +++ b/scripts/commands/gbx-test-notebooks.sh @@ -12,6 +12,9 @@ show_help() { echo -e " ${GREEN}gbx:test:notebooks${NC} ${YELLOW}[options]${NC}" echo "" echo -e "${CYAN}Options:${NC}" + echo -e " ${GREEN}--host${NC} Run on the host (arca), not Docker. Requires ${YELLOW}source ~/.local/geobrix-gdal-env.sh${NC}" + echo -e " first; builds/uses ${YELLOW}.venv-host${NC}." + echo -e " ${GREEN}--rebuild-venv${NC} (with --host) force-rebuild the host test venv" echo -e " ${GREEN}--allow-absolute-reads${NC} Do not remap absolute read paths under workdir (default: remap so reads go under temp workdir)" echo -e " ${GREEN}--allow-absolute-writes${NC} Do not remap absolute write paths under workdir (default: remap so writes go under temp workdir)" echo -e " ${GREEN}--include-integration${NC} Include full-notebook execution tests (pytest only; default: exclude)" @@ -39,11 +42,19 @@ LOG_PATH="" INCLUDE_INTEGRATION=false ALLOW_ABSOLUTE_READS=false ALLOW_ABSOLUTE_WRITES=false -TEST_PATH="/root/geobrix/notebooks/tests" +USE_HOST=false PATH_ARG="" while [[ $# -gt 0 ]]; do case $1 in + --host) + USE_HOST=true + shift + ;; + --rebuild-venv) + export GBX_REBUILD_VENV=1 + shift + ;; --allow-absolute-reads) ALLOW_ABSOLUTE_READS=true shift @@ -65,7 +76,6 @@ while [[ $# -gt 0 ]]; do # If user passes notebooks/tests/..., strip that prefix. PATH_ARG="$2" [[ "$PATH_ARG" == notebooks/tests/* ]] && PATH_ARG="${PATH_ARG#notebooks/tests/}" - TEST_PATH="/root/geobrix/notebooks/tests/$PATH_ARG" shift 2 ;; --help|-h) @@ -87,12 +97,9 @@ cd "$PROJECT_ROOT" if [[ -n "${PATH_ARG:-}" && "$PATH_ARG" == *.py && "$PATH_ARG" != */* ]]; then NBTEST="$PROJECT_ROOT/notebooks/tests" if [[ ! -f "$NBTEST/$PATH_ARG" ]]; then - FOUND= for sub in sample-data fixtures; do if [[ -f "$NBTEST/$sub/$PATH_ARG" ]]; then PATH_ARG="$sub/$PATH_ARG" - TEST_PATH="/root/geobrix/notebooks/tests/$PATH_ARG" - FOUND=1 break fi done @@ -100,29 +107,13 @@ if [[ -n "${PATH_ARG:-}" && "$PATH_ARG" == *.py && "$PATH_ARG" != */* ]]; then fi show_banner "📓 GeoBrix: Notebook Tests" -check_docker setup_log_file "$LOG_PATH" -# Ensure sample-data Volumes structure exists on host (mount target for start_docker_with_volumes.sh) -mkdir -p "$PROJECT_ROOT/sample-data/Volumes/main/default/geobrix_samples" - -# Notebook tests need /Volumes so the notebook can use get_volumes_path() and run_*_bundle() -if ! docker exec geobrix-dev test -d /Volumes 2>/dev/null; then - echo -e "${RED}❌ /Volumes not found in container. Start the container with the Volumes mount:${NC}" - echo -e " ${YELLOW}./scripts/docker/start_docker_with_volumes.sh${NC}" - echo "" - echo "Then run this command again so notebook tests can use sample-data paths." - exit 1 -fi - # Single entry point: always use isolation (venv + GBX_NOTEBOOK_ISOLATED=1). Path can be a test file (.py) or notebook scope. PATH_ARG="${PATH_ARG:-}" -# Single-quote the path for safe embedding in the docker exec bash -c string (handles parens, spaces, etc.) -# Produces: 'some/path with (parens).ipynb' — single-quote-safe by escaping any embedded single quotes +# Single-quote the path for safe embedding in the exec bash -c string (handles parens, spaces, etc.) PATH_ARG_ESCAPED="${PATH_ARG//\'/\'\\\'\'}" NOTEBOOK_VERBOSITY="${GBX_NOTEBOOK_VERBOSITY:-}" -PYTEST_EXTRA="-s" -[[ "$NOTEBOOK_VERBOSITY" = "quiet" ]] && PYTEST_EXTRA="" if [[ -n "$PATH_ARG" ]]; then echo -e "${CYAN}🎯 Path: ${YELLOW}$PATH_ARG${NC}" @@ -138,7 +129,49 @@ if [[ "$PATH_ARG" == *.py ]]; then fi echo "" -RUN_CMD="set -e +if [ "$USE_HOST" = true ]; then + # --- Host (arca) path: no Docker; run the notebook runner from .venv-host --- + require_host_gdal_env || exit 1 + VENV_BIN=$(ensure_host_test_venv pyrx) || exit 1 + # The notebook runner needs nbformat/nbconvert (not in the pyrx lock — the runner normally installs + # them into its own nested venv). We run directly in .venv-host-pyrx (see below), so ensure they're + # present here. Idempotent; uv no-ops when already installed. + if ! "$VENV_BIN/python" -c "import nbformat, nbconvert" >/dev/null 2>&1; then + echo -e "${CYAN}Installing nbformat/nbconvert into the host venv...${NC}" + uv pip install --python "$VENV_BIN/python" -q nbformat nbconvert \ + || { echo -e "${RED}❌ failed to install nbformat/nbconvert${NC}"; exit 1; } + fi + # Wire Spark workers to the venv python and unset PROJ_DATA/PROJ_LIB (see common.sh). + activate_host_python_env "$VENV_BIN" + unset JAVA_TOOL_OPTIONS + export JUPYTER_PLATFORM_DIRS=1 + # Do NOT force the runner's nested-venv isolation on host: .venv-host-pyrx (plus nbformat/nbconvert + # installed just above) is already the isolated env, and the runner's `python -m venv` nested-venv + # path fails on arca (no ensurepip / python3-venv). Running directly in .venv-host-pyrx is the isolation. + export GBX_NOTEBOOK_ISOLATED_ENV=0 + [ -n "$NOTEBOOK_VERBOSITY" ] && export GBX_NOTEBOOK_VERBOSITY="$NOTEBOOK_VERBOSITY" + [ "$INCLUDE_INTEGRATION" = true ] && export GBX_NOTEBOOK_INCLUDE_INTEGRATION=1 + [ "$ALLOW_ABSOLUTE_READS" = true ] && export GBX_NOTEBOOK_ALLOW_ABSOLUTE_READS=1 + [ "$ALLOW_ABSOLUTE_WRITES" = true ] && export GBX_NOTEBOOK_ALLOW_ABSOLUTE_WRITES=1 + eval "\"$VENV_BIN/python\" \"$PROJECT_ROOT/notebooks/tests/run_notebooks_cell_by_cell.py\" ${PATH_ARG:+'$PATH_ARG_ESCAPED'}" + EXIT_CODE=$? +else + # --- Docker path (unchanged) --- + check_docker + + # Ensure sample-data Volumes structure exists on host (mount target for start_docker_with_volumes.sh) + mkdir -p "$PROJECT_ROOT/sample-data/Volumes/main/default/geobrix_samples" + + # Notebook tests need /Volumes so the notebook can use get_volumes_path() and run_*_bundle() + if ! docker exec geobrix-dev test -d /Volumes 2>/dev/null; then + echo -e "${RED}❌ /Volumes not found in container. Start the container with the Volumes mount:${NC}" + echo -e " ${YELLOW}./scripts/docker/start_docker_with_volumes.sh${NC}" + echo "" + echo "Then run this command again so notebook tests can use sample-data paths." + exit 1 + fi + + RUN_CMD="set -e unset JAVA_TOOL_OPTIONS export JUPYTER_PLATFORM_DIRS=1 export GBX_NOTEBOOK_TESTS_DOCKER=1 @@ -156,9 +189,10 @@ pip install --no-deps -e /root/geobrix/python/geobrix --break-system-packages -q python3 /root/geobrix/notebooks/tests/run_notebooks_cell_by_cell.py ${PATH_ARG:+'$PATH_ARG_ESCAPED'} " -# -t allocates a pseudo-TTY so testbook's Jupyter kernel doesn't hang (common in Docker) -docker exec -t geobrix-dev /bin/bash -c "$RUN_CMD" -EXIT_CODE=$? + # -t allocates a pseudo-TTY so testbook's Jupyter kernel doesn't hang (common in Docker) + docker exec -t geobrix-dev /bin/bash -c "$RUN_CMD" + EXIT_CODE=$? +fi echo "" show_separator diff --git a/scripts/commands/gbx-test-python-docs.md b/scripts/commands/gbx-test-python-docs.md index 4bf553736..92eac03f8 100644 --- a/scripts/commands/gbx-test-python-docs.md +++ b/scripts/commands/gbx-test-python-docs.md @@ -58,6 +58,8 @@ bash scripts/commands/gbx-test-python-docs.sh [OPTIONS] **Other** +- `--host` – Run on the host (arca), not the Docker container. Requires `source ~/.local/geobrix-gdal-env.sh` first (provisioned by the `geobrix-arca` plugin); builds/reuses `.venv-host-pyrx` from the pinned CI lock and a host-built JAR. See "Host mode" below. +- `--rebuild-venv` – (with `--host`) force-rebuild the host test venv. - `--log ` – Log file (filename → `test-logs/`). Prefer timestamped names for tracking. - `--markers ` – Pytest markers (e.g. `"not slow"`). - `--include-integration` – Include integration tests (excluded by default). @@ -65,12 +67,20 @@ bash scripts/commands/gbx-test-python-docs.sh [OPTIONS] - `--no-sample-data-root` – Do **not** set `GBX_SAMPLE_DATA_ROOT` (use your env or path_config default). - `--help` – Help and suite timing. +## Host mode (arca, no Docker) + +With `--host` the command runs directly on the host instead of `docker exec geobrix-dev`. Prerequisites: `source ~/.local/geobrix-gdal-env.sh` (native GDAL + Java 17 + PYTHONPATH) and `uv` on PATH, with `PIP_INDEX_URL` pointing at the internal pip proxy. The first run builds `.venv-host-pyrx` from `python/geobrix/requirements-pyrx-ci.txt` (the exact light-tier CI pins: rasterio/pandas/h3/vizx) via `uv`; neither CI lock contains `pdal` (source-only, unbuildable on arca and not needed here). Sample data reads from the on-disk `sample-data/…/test-data` mirror via `GBX_SAMPLE_DATA_ROOT`. See the `geobrix-arca` plugin for the full setup. + ## Examples ```bash # Quickstart only, no build, with log (typical during edits) bash scripts/commands/gbx-test-python-docs.sh --suite quickstart --skip-build --log quickstart.log +# On the arca host (no Docker) — source the GDAL env first +source ~/.local/geobrix-gdal-env.sh +bash scripts/commands/gbx-test-python-docs.sh --host + # Single failing test bash scripts/commands/gbx-test-python-docs.sh --test quickstart/test_examples.py::test_convert_to_databricks_geometry_with_nyc_data --skip-build diff --git a/scripts/commands/gbx-test-python-docs.sh b/scripts/commands/gbx-test-python-docs.sh index e2810d65b..6689ff0c4 100755 --- a/scripts/commands/gbx-test-python-docs.sh +++ b/scripts/commands/gbx-test-python-docs.sh @@ -26,6 +26,9 @@ show_help() { echo -e " ${YELLOW}integration${NC} varies integration/ (DBR or integration env; use with --include-integration or run alone)" echo "" echo -e "${CYAN}Other options:${NC}" + echo -e " ${GREEN}--host${NC} Run on the host (arca), not Docker. Requires ${YELLOW}source ~/.local/geobrix-gdal-env.sh${NC}" + echo -e " first; builds/uses ${YELLOW}.venv-host${NC} from the pinned lock." + echo -e " ${GREEN}--rebuild-venv${NC} (with --host) force-rebuild the host test venv" echo -e " ${GREEN}--log ${NC} Write output to log (use timestamp for tracking: python-docs-\$(date +%Y%m%d-%H%M%S).log)" echo -e " ${GREEN}--markers ${NC} Pytest markers (e.g. \"not slow\")" echo -e " ${GREEN}--include-integration${NC} Include integration tests (excluded by default)" @@ -41,13 +44,13 @@ show_help() { echo "" } -# Parse arguments -BASE="/root/geobrix/docs/tests/python" -TEST_PATH="${BASE}/" +# Parse arguments. REL_PATH is relative to docs/tests/python; each mode prefixes it. +REL_PATH="" LOG_PATH="" MARKERS="-m 'not integration'" INCLUDE_INTEGRATION=false SKIP_BUILD=false +USE_HOST=false # Default: set sample data root so doc tests use minimal bundle (required for remote/CI) SET_SAMPLE_DATA_ROOT=true @@ -55,16 +58,16 @@ while [[ $# -gt 0 ]]; do case $1 in --test) # Single test: path relative to docs/tests/python, e.g. quickstart/test_examples.py::test_foo - TEST_PATH="${BASE}/$2" + REL_PATH="$2" shift 2 ;; --suite) case "$2" in quickstart|api|readers|rasterx|advanced|setup) - TEST_PATH="${BASE}/$2/" + REL_PATH="$2/" ;; integration) - TEST_PATH="${BASE}/$2/" + REL_PATH="$2/" INCLUDE_INTEGRATION=true MARKERS="" ;; @@ -77,9 +80,17 @@ while [[ $# -gt 0 ]]; do shift 2 ;; --path) - TEST_PATH="${BASE}/$2" + REL_PATH="$2" shift 2 ;; + --host) + USE_HOST=true + shift + ;; + --rebuild-venv) + export GBX_REBUILD_VENV=1 + shift + ;; --log) LOG_PATH=$(resolve_log_path "$2") shift 2 @@ -114,55 +125,95 @@ while [[ $# -gt 0 ]]; do esac done -# When running the default full suite (entire docs/tests/python/), exclude api/ +# When running the default full suite (entire docs/tests/python/, i.e. no REL_PATH), exclude api/ # so gbx:test:python-docs and gbx:test:sql-docs stay isolated. Use --suite api to run SQL/API tests. SKIP_SQL_TESTS=false -if [ "$TEST_PATH" = "${BASE}/" ]; then +if [ -z "$REL_PATH" ]; then SKIP_SQL_TESTS=true fi -IGNORE_API_ARG="" -if [ "$SKIP_SQL_TESTS" = true ]; then - IGNORE_API_ARG="--ignore=${BASE}/api" -fi cd "$PROJECT_ROOT" show_banner "📚 GeoBrix: Python Documentation Tests" -check_docker setup_log_file "$LOG_PATH" -# Ensure sample-data Volumes structure exists on host (mount target for start_docker_with_volumes.sh) -mkdir -p "$PROJECT_ROOT/sample-data/Volumes/main/default/geobrix_samples" +if [ "$USE_HOST" = true ]; then + # --- Host (arca) path: no Docker; build JAR on host, run pytest from .venv-host --- + require_host_gdal_env || exit 1 + local_base="$PROJECT_ROOT/docs/tests/python" + TEST_PATH="${local_base}/${REL_PATH}" + # Real args array (space-safe), not an eval-interpolated string, so a repo path with spaces + # doesn't word-split the --ignore value. + ignore_args=() + [ "$SKIP_SQL_TESTS" = true ] && ignore_args=(--ignore="${local_base}/api") -# Volumes must be mounted so tests can use sample data (minimal bundle in-repo or full at geobrix_samples) -if ! docker exec geobrix-dev test -d /Volumes 2>/dev/null; then - echo -e "${RED}❌ /Volumes not found in container. Start the container with the Volumes mount:${NC}" - echo -e " ${YELLOW}./scripts/docker/start_docker_with_volumes.sh${NC}" + echo -e "${CYAN}🎯 Test path: ${YELLOW}$TEST_PATH${NC} ${CYAN}(host)${NC}" + [ "$SKIP_SQL_TESTS" = true ] && echo -e "${CYAN}🚫 Excluding api/ (use gbx:test:sql-docs or --suite api)${NC}" + [ "$INCLUDE_INTEGRATION" = true ] && echo -e "${CYAN}🐢 Including integration tests${NC}" || echo -e "${CYAN}⚡ Excluding integration tests${NC}" + [ "$SKIP_BUILD" = true ] && echo -e "${CYAN}⏭️ Skipping build${NC}" echo "" - echo "Then run this command again so tests can run." - exit 1 -fi -echo -e "${CYAN}🎯 Test path: ${YELLOW}$TEST_PATH${NC}" -if [ "$SKIP_SQL_TESTS" = true ]; then - echo -e "${CYAN}🚫 Excluding api/ (SQL/API tests; use gbx:test:sql-docs or --suite api to run them)${NC}" -fi -if [ "$INCLUDE_INTEGRATION" = true ]; then - echo -e "${CYAN}🐢 Including integration tests (may be slow)${NC}" + VENV_BIN=$(ensure_host_test_venv pyrx) || exit 1 + # Wire Spark workers to the venv python and unset PROJ_DATA/PROJ_LIB (see common.sh). + activate_host_python_env "$VENV_BIN" + + if [ "$SKIP_BUILD" != true ]; then + show_separator + echo -e "${CYAN}Building JAR (mvn package -DskipTests -PskipScoverage)...${NC}" + show_separator + (cd "$PROJECT_ROOT" && mvn package -DskipTests -q -PskipScoverage) || exit $? + echo "" + fi + + unset JAVA_TOOL_OPTIONS + export JUPYTER_PLATFORM_DIRS=1 + [ "$SET_SAMPLE_DATA_ROOT" = true ] && export GBX_SAMPLE_DATA_ROOT="$PROJECT_ROOT/sample-data/Volumes/main/default/test-data" + show_separator + echo -e "${CYAN}Running Python documentation tests (host)...${NC}" + show_separator + marker_args=(); [ -n "$MARKERS" ] && eval "marker_args=($MARKERS)" + "$VENV_BIN/python" -m pytest "$TEST_PATH" -v --tb=short --color=yes "${marker_args[@]}" "${ignore_args[@]}" + EXIT_CODE=$? else - echo -e "${CYAN}⚡ Excluding integration tests (fast mode)${NC}" -fi -if [ "$SKIP_BUILD" = true ]; then - echo -e "${CYAN}⏭️ Skipping Maven and Python build (--skip-build)${NC}" -fi -echo "" + # --- Docker path (unchanged) --- + check_docker + BASE="/root/geobrix/docs/tests/python" + TEST_PATH="${BASE}/${REL_PATH}" + IGNORE_API_ARG="" + [ "$SKIP_SQL_TESTS" = true ] && IGNORE_API_ARG="--ignore=${BASE}/api" + + # Ensure sample-data Volumes structure exists on host (mount target for start_docker_with_volumes.sh) + mkdir -p "$PROJECT_ROOT/sample-data/Volumes/main/default/geobrix_samples" -# Use minimal bundle path in container so doc tests pass on remote/CI (unless --no-sample-data-root) -SAMPLE_DATA_ROOT_EXPORT="" -[ "$SET_SAMPLE_DATA_ROOT" = true ] && SAMPLE_DATA_ROOT_EXPORT="export GBX_SAMPLE_DATA_ROOT=/Volumes/main/default/test-data" + # Volumes must be mounted so tests can use sample data (minimal bundle in-repo or full at geobrix_samples) + if ! docker exec geobrix-dev test -d /Volumes 2>/dev/null; then + echo -e "${RED}❌ /Volumes not found in container. Start the container with the Volumes mount:${NC}" + echo -e " ${YELLOW}./scripts/docker/start_docker_with_volumes.sh${NC}" + echo "" + echo "Then run this command again so tests can run." + exit 1 + fi -# Run pre-steps and pytest inside Docker (single bash -c so env and cwd carry through) -RUN_CMD="set -e + echo -e "${CYAN}🎯 Test path: ${YELLOW}$TEST_PATH${NC}" + if [ "$SKIP_SQL_TESTS" = true ]; then + echo -e "${CYAN}🚫 Excluding api/ (SQL/API tests; use gbx:test:sql-docs or --suite api to run them)${NC}" + fi + if [ "$INCLUDE_INTEGRATION" = true ]; then + echo -e "${CYAN}🐢 Including integration tests (may be slow)${NC}" + else + echo -e "${CYAN}⚡ Excluding integration tests (fast mode)${NC}" + fi + if [ "$SKIP_BUILD" = true ]; then + echo -e "${CYAN}⏭️ Skipping Maven and Python build (--skip-build)${NC}" + fi + echo "" + + # Use minimal bundle path in container so doc tests pass on remote/CI (unless --no-sample-data-root) + SAMPLE_DATA_ROOT_EXPORT="" + [ "$SET_SAMPLE_DATA_ROOT" = true ] && SAMPLE_DATA_ROOT_EXPORT="export GBX_SAMPLE_DATA_ROOT=/Volumes/main/default/test-data" + + # Run pre-steps and pytest inside Docker (single bash -c so env and cwd carry through) + RUN_CMD="set -e unset JAVA_TOOL_OPTIONS export JUPYTER_PLATFORM_DIRS=1 $SAMPLE_DATA_ROOT_EXPORT @@ -203,8 +254,9 @@ echo '━━━━━━━━━━━━━━━━━━━━━━━━ python3 -m pytest $TEST_PATH -v $MARKERS --tb=short --color=yes $IGNORE_API_ARG " -docker exec geobrix-dev /bin/bash -c "$RUN_CMD" -EXIT_CODE=$? + docker exec geobrix-dev /bin/bash -c "$RUN_CMD" + EXIT_CODE=$? +fi echo "" show_separator diff --git a/scripts/commands/gbx-test-python.md b/scripts/commands/gbx-test-python.md index 2ef2b503b..cfd8c46b5 100644 --- a/scripts/commands/gbx-test-python.md +++ b/scripts/commands/gbx-test-python.md @@ -11,11 +11,24 @@ bash scripts/commands/gbx-test-python.sh [OPTIONS] ## Options - `--path ` - Run specific test file or directory +- `--host` - Run on the host (arca), not the Docker container. Requires `source ~/.local/geobrix-gdal-env.sh` first (provisioned by the `geobrix-arca` plugin) and a host-built JAR. Mirrors CI's two-environment split — see "Host mode" below. +- `--rebuild-venv` - (with `--host`) force-rebuild the host test venvs. - `--log ` - Write output to log file (supports filename, relative, or absolute path) - `--with-integration` - Include `@pytest.mark.integration` tests (network downloads, slow). Excluded by default. - `--markers ` - Override marker filter with a custom pytest expression (e.g. `"not slow"`). Disables the default `not integration` filter. - `--help` - Display help message +## Host mode (arca, no Docker) + +With `--host` the command runs directly on the host instead of `docker exec geobrix-dev`. There is no build step — the tests run against the already-built assembly JAR. Prerequisites: `source ~/.local/geobrix-gdal-env.sh` (native GDAL + Java 17 + PYTHONPATH) and `uv` on PATH, with `PIP_INDEX_URL` pointing at the internal pip proxy. + +**Two-environment split (mirrors CI).** CI never runs all Python tests in one environment, and neither does `--host` — running everything in one bloated venv causes cross-suite SparkSession/fixture errors. Instead a default `--host` run does two legs, each in its own uv venv built from the corresponding CI lock: + +- **Heavy leg** — `.venv-host-ci` from `requirements-ci.txt` (~27 pkgs: pyspark, py4j, numpy, pytest; no rasterio/pandas). Runs `test/` minus the light dirs. Matches CI's `python_build` job. +- **Light leg** — `.venv-host-pyrx` from `requirements-pyrx-ci.txt` (~104 pkgs: rasterio, shapely, pandas, pyarrow, h3, mapbox-vector-tile, vizx). Runs `test/{pyrx,pyvx,pygx,pmtiles_light,stac,vizx,ds,sample}`. Matches CI's `pyrx_build` job. + +A single `--path` is routed to whichever venv fits (light dirs → pyrx venv, else ci venv). `osgeo` comes from the sourced arca env's PYTHONPATH (equivalent to CI's apt `python3-gdal`); neither lock contains `pdal` (source-only, unbuildable on arca — and not needed by these suites). See the `geobrix-arca` plugin for the full setup. + ## Default marker filter By default the script runs with `-m "not integration"`, matching CI's `python_build` action. This excludes `python/geobrix/test/sample/test_sample_bundle.py::test_run_*_bundle_returns_dict_shape`, which download hundreds of MB of sample data. @@ -28,6 +41,10 @@ Opt in with `--with-integration` (drops the filter entirely) or `--markers ${NC} Specific test directory or file" + echo -e " ${GREEN}--path ${NC} Specific test directory or file (repo-relative)" + echo -e " ${GREEN}--host${NC} Run on the host (arca) instead of the Docker container. Requires" + echo -e " ${YELLOW}source ~/.local/geobrix-gdal-env.sh${NC} first; builds/uses ${YELLOW}.venv-host${NC}." + echo -e " ${GREEN}--rebuild-venv${NC} (with --host) force-rebuild the host test venv from the pinned lock" echo -e " ${GREEN}--log ${NC} Write output to log file" echo -e " ${GREEN}--with-integration${NC} Include ${YELLOW}@pytest.mark.integration${NC} tests (network downloads, slow); excluded by default" echo -e " ${GREEN}--markers ${NC} Override marker filter with a pytest expression (e.g. 'not slow'); disables the default 'not integration' filter" @@ -26,25 +29,35 @@ show_help() { echo -e " ${YELLOW}/abs/path/file.log${NC} → /abs/path/file.log" echo "" echo -e "${CYAN}Examples:${NC}" - echo -e " ${YELLOW}gbx:test:python${NC} ${CYAN}# unit tests only (default)${NC}" + echo -e " ${YELLOW}gbx:test:python${NC} ${CYAN}# unit tests only (default, Docker)${NC}" + echo -e " ${YELLOW}gbx:test:python --host${NC} ${CYAN}# run on the arca host (no Docker)${NC}" echo -e " ${YELLOW}gbx:test:python --with-integration${NC} ${CYAN}# unit + integration (network)${NC}" echo -e " ${YELLOW}gbx:test:python --path python/geobrix/test/rasterx/${NC}" echo -e " ${YELLOW}gbx:test:python --markers 'not slow' --log python-tests.log${NC}" echo "" } -# Parse arguments -TEST_PATH="/root/geobrix/python/geobrix/test/" +# Parse arguments. REL_PATH is repo-relative; each mode prefixes it (container /root/geobrix vs host $PROJECT_ROOT). +REL_PATH="python/geobrix/test/" LOG_PATH="" +USE_HOST=false # Default: exclude integration tests (network downloads); matches CI's python_build action. MARKERS="-m 'not integration'" while [[ $# -gt 0 ]]; do case $1 in --path) - TEST_PATH="/root/geobrix/$2" + REL_PATH="$2" shift 2 ;; + --host) + USE_HOST=true + shift + ;; + --rebuild-venv) + export GBX_REBUILD_VENV=1 + shift + ;; --log) LOG_PATH=$(resolve_log_path "$2") shift 2 @@ -73,13 +86,16 @@ done cd "$PROJECT_ROOT" show_banner "🐍 GeoBrix: Python Tests (Non-Docs)" -check_docker setup_log_file "$LOG_PATH" # Python tests run against the assembly JAR (spark.jars); warn if it predates Scala sources. warn_if_jar_stale "$PROJECT_ROOT" -echo -e "${CYAN}🎯 Test path: ${YELLOW}$TEST_PATH${NC}" +if [ "$USE_HOST" = true ]; then + echo -e "${CYAN}🎯 Test path: ${YELLOW}$PROJECT_ROOT/$REL_PATH${NC} ${CYAN}(host)${NC}" +else + echo -e "${CYAN}🎯 Test path: ${YELLOW}/root/geobrix/$REL_PATH${NC}" +fi if [ -n "$MARKERS" ]; then echo -e "${CYAN}🏷️ Markers: ${YELLOW}$MARKERS${NC}" else @@ -92,13 +108,62 @@ echo -e "${CYAN}Running tests...${NC}" show_separator echo "" -# Build pytest command -PYTEST_CMD="unset JAVA_TOOL_OPTIONS && \ - cd /root/geobrix && \ - python3 -m pytest $TEST_PATH -v --tb=short --color=yes $MARKERS" +if [ "$USE_HOST" = true ]; then + require_host_gdal_env || exit 1 + unset JAVA_TOOL_OPTIONS + export JUPYTER_PLATFORM_DIRS=1 + + # Runs pytest for one leg in the given venv kind. activate_host_python_env exports + # PYSPARK_PYTHON/PYSPARK_DRIVER_PYTHON (real exports — a VAR=x eval prefix would NOT reach the + # Spark Python workers) and unsets PROJ_DATA/PROJ_LIB (rasterio bundled proj.db). Runs in a + # subshell so those env changes don't leak between the two legs. Paths pass as real positional + # args (space-safe, no eval); only $MARKERS ("-m 'not integration'") is eval-split into an array. + _run_leg() { # $1=kind $2..=pytest paths + local kind="$1"; shift + local vbin + vbin=$(ensure_host_test_venv "$kind") || return 1 + local marker_args=() + [ -n "$MARKERS" ] && eval "marker_args=($MARKERS)" + ( + activate_host_python_env "$vbin" + "$vbin/python" -m pytest "$@" -v --tb=short --color=yes "${marker_args[@]}" + ) + } + + LIGHT_DIRS="$(host_light_test_dirs)" + EXIT_CODE=0 + if [ "$REL_PATH" = "python/geobrix/test/" ]; then + # Default full run — mirror CI's two-environment split (CI never runs all unit tests in one env). + # Heavy leg (requirements-ci.txt): run the whole test/ tree; python/geobrix/test/conftest.py's + # dependency-aware collect_ignore skips the light dirs automatically (rasterio absent), so we do + # NOT hand-maintain an --ignore list here. Light leg (requirements-pyrx-ci.txt): the light dirs, + # sourced from that same conftest's _LIGHT_TEST_DIRS via host_light_test_dirs (keeps them in sync). + local_root="$PROJECT_ROOT/python/geobrix/test" + light_paths=() + for d in $LIGHT_DIRS; do light_paths+=("$local_root/$d"); done -docker exec geobrix-dev /bin/bash -c "$PYTEST_CMD" -EXIT_CODE=$? + echo -e "${CYAN}▶ Heavy leg (requirements-ci.txt): test/ (light dirs auto-skipped by conftest)${NC}" + _run_leg ci "$local_root" || EXIT_CODE=$? + echo "" + echo -e "${CYAN}▶ Light leg (requirements-pyrx-ci.txt): $LIGHT_DIRS${NC}" + _run_leg pyrx "${light_paths[@]}" || EXIT_CODE=$? + else + # Explicit --path: route to the light venv if the path is under a light dir, else the heavy venv. + kind=ci + for d in $LIGHT_DIRS; do + case "$REL_PATH" in *"test/$d"|*"test/$d/"*) kind=pyrx; break ;; esac + done + echo -e "${CYAN}▶ Leg ($kind venv): $REL_PATH${NC}" + _run_leg "$kind" "$PROJECT_ROOT/$REL_PATH" || EXIT_CODE=$? + fi +else + check_docker + PYTEST_CMD="unset JAVA_TOOL_OPTIONS && \ + cd /root/geobrix && \ + python3 -m pytest /root/geobrix/$REL_PATH -v --tb=short --color=yes $MARKERS" + docker exec geobrix-dev /bin/bash -c "$PYTEST_CMD" + EXIT_CODE=$? +fi echo "" show_separator diff --git a/scripts/commands/gbx-test-scala-docs.md b/scripts/commands/gbx-test-scala-docs.md index dca91cbe9..9e4e3b4d3 100644 --- a/scripts/commands/gbx-test-scala-docs.md +++ b/scripts/commands/gbx-test-scala-docs.md @@ -10,12 +10,17 @@ bash scripts/commands/gbx-test-scala-docs.sh [OPTIONS] ## Options +- `--host` – Run `mvn` on the host (arca), not the Docker container. Requires `source ~/.local/geobrix-gdal-env.sh` first (provisioned by the `geobrix-arca` plugin). See "Host mode" below. - `--suite ` – Maven suite pattern (default: `tests.docs.scala.*`). Example: `docs.tests.scala.api.*` - `--log ` – Log file (filename → `test-logs/`, absolute path as-is) - `--skip-build` – Skip Maven compile before test (optional; `mvn test` still compiles) - `--no-sample-data-root` – Do **not** set `GBX_SAMPLE_DATA_ROOT` (use env or full-bundle default in Scala) - `--help` – Display help message +## Host mode (arca, no Docker) + +With `--host` the command runs `mvn test` directly on the host instead of `docker exec geobrix-dev`. Scala doc-tests are a pure JVM + native GDAL path, so this only needs `source ~/.local/geobrix-gdal-env.sh` (native GDAL + Java 17 + PYTHONPATH) first — no Python test venv, so `--rebuild-venv` does not apply. Sample data reads from the on-disk `sample-data/…/test-data` mirror via `GBX_SAMPLE_DATA_ROOT`. See the `geobrix-arca` plugin for the full setup. + **Sample data (default):** The command sets `GBX_SAMPLE_DATA_ROOT=/Volumes/main/default/test-data` in the container so doc tests use the minimal bundle (required for remote/CI). Use `--no-sample-data-root` to leave it unset so Scala uses the full-bundle default path. ## Examples @@ -24,6 +29,10 @@ bash scripts/commands/gbx-test-scala-docs.sh [OPTIONS] # Run all Scala documentation tests bash scripts/commands/gbx-test-scala-docs.sh +# On the arca host (no Docker) — source the GDAL env first +source ~/.local/geobrix-gdal-env.sh +bash scripts/commands/gbx-test-scala-docs.sh --host + # Run specific docs test suite bash scripts/commands/gbx-test-scala-docs.sh --suite tests.docs.scala.packages.* diff --git a/scripts/commands/gbx-test-scala-docs.sh b/scripts/commands/gbx-test-scala-docs.sh index cd502e389..ba91e61e2 100755 --- a/scripts/commands/gbx-test-scala-docs.sh +++ b/scripts/commands/gbx-test-scala-docs.sh @@ -12,6 +12,7 @@ show_help() { echo -e " ${GREEN}gbx:test:scala-docs${NC} ${YELLOW}[options]${NC}" echo "" echo -e "${CYAN}Options:${NC}" + echo -e " ${GREEN}--host${NC} Run mvn on the host (arca), not Docker. Requires ${YELLOW}source ~/.local/geobrix-gdal-env.sh${NC} first." echo -e " ${GREEN}--log ${NC} Write output to log (filename → test-logs/)" echo -e " ${GREEN}--suite ${NC} Maven suite pattern (default: tests.docs.scala.*)" echo -e " ${GREEN}--skip-build${NC} Skip Maven compile before test (mvn test still compiles)" @@ -32,11 +33,16 @@ show_help() { LOG_PATH="" SUITE_PATTERN="tests.docs.scala.*" SKIP_BUILD=false +USE_HOST=false # Default: set sample data root so doc tests use minimal bundle (required for remote/CI) SET_SAMPLE_DATA_ROOT=true while [[ $# -gt 0 ]]; do case $1 in + --host) + USE_HOST=true + shift + ;; --log) LOG_PATH=$(resolve_log_path "$2") shift 2 @@ -69,7 +75,6 @@ done cd "$PROJECT_ROOT" show_banner "📚 GeoBrix: Scala Documentation Tests" -check_docker setup_log_file "$LOG_PATH" echo -e "${CYAN}🎯 Suite: ${YELLOW}$SUITE_PATTERN${NC}" @@ -80,14 +85,27 @@ echo -e "${CYAN}Running Scala doc tests...${NC}" show_separator echo "" -# Use minimal bundle path in container unless --no-sample-data-root (then Scala uses full-bundle default) -SAMPLE_DATA_ROOT_MAVEN="" -[ "$SET_SAMPLE_DATA_ROOT" = true ] && SAMPLE_DATA_ROOT_MAVEN="export GBX_SAMPLE_DATA_ROOT=/Volumes/main/default/test-data && " +if [ "$USE_HOST" = true ]; then + # --- Host (arca) path: run mvn directly (no Docker). Scala doc-tests are pure JVM + native GDAL. --- + require_host_gdal_env || exit 1 + unset JAVA_TOOL_OPTIONS + export JUPYTER_PLATFORM_DIRS=1 + [ "$SET_SAMPLE_DATA_ROOT" = true ] && export GBX_SAMPLE_DATA_ROOT="$PROJECT_ROOT/sample-data/Volumes/main/default/test-data" + (cd "$PROJECT_ROOT" && mvn test -PskipScoverage -Dsuites="$SUITE_PATTERN") + EXIT_CODE=$? +else + # --- Docker path (unchanged) --- + check_docker -MVN_CMD="unset JAVA_TOOL_OPTIONS && export JUPYTER_PLATFORM_DIRS=1 && ${SAMPLE_DATA_ROOT_MAVEN}cd /root/geobrix && mvn test -Dsuites='$SUITE_PATTERN'" + # Use minimal bundle path in container unless --no-sample-data-root (then Scala uses full-bundle default) + SAMPLE_DATA_ROOT_MAVEN="" + [ "$SET_SAMPLE_DATA_ROOT" = true ] && SAMPLE_DATA_ROOT_MAVEN="export GBX_SAMPLE_DATA_ROOT=/Volumes/main/default/test-data && " -docker exec geobrix-dev /bin/bash -c "$MVN_CMD" -EXIT_CODE=$? + MVN_CMD="unset JAVA_TOOL_OPTIONS && export JUPYTER_PLATFORM_DIRS=1 && ${SAMPLE_DATA_ROOT_MAVEN}cd /root/geobrix && mvn test -Dsuites='$SUITE_PATTERN'" + + docker exec geobrix-dev /bin/bash -c "$MVN_CMD" + EXIT_CODE=$? +fi echo "" # pytest-style short summary when logging (dedupe by test name, explain minimal bundle) diff --git a/scripts/commands/gbx-test-sql-docs.md b/scripts/commands/gbx-test-sql-docs.md index b5e213024..70d718b5c 100644 --- a/scripts/commands/gbx-test-sql-docs.md +++ b/scripts/commands/gbx-test-sql-docs.md @@ -19,6 +19,8 @@ bash scripts/commands/gbx-test-sql-docs.sh [OPTIONS] **Common** +- `--host` – Run on the host (arca), not the Docker container. Requires `source ~/.local/geobrix-gdal-env.sh` first (provisioned by the `geobrix-arca` plugin); builds/reuses `.venv-host-pyrx` from the pinned CI lock and a host-built JAR. See "Host mode" below. +- `--rebuild-venv` – (with `--host`) force-rebuild the host test venv. - `--log ` – Log file (filename → `test-logs/`). - `--markers ` – Pytest markers (e.g. `"not slow"`). - `--include-integration` – Include integration tests (excluded by default). @@ -26,7 +28,11 @@ bash scripts/commands/gbx-test-sql-docs.sh [OPTIONS] - `--no-sample-data-root` – Do **not** set `GBX_SAMPLE_DATA_ROOT` (use your env or path_config default). - `--help` – Help and examples. -**Sample data (default):** Like `gbx:test:python-docs`, this command sets `GBX_SAMPLE_DATA_ROOT=/Volumes/main/default/test-data` in the container for the minimal bundle (required for remote/CI). Use `--no-sample-data-root` to leave it unset. +**Sample data (default):** Like `gbx:test:python-docs`, this command sets `GBX_SAMPLE_DATA_ROOT=/Volumes/main/default/test-data` (in the container, or the on-disk `sample-data/…/test-data` mirror on the host) for the minimal bundle (required for remote/CI). Use `--no-sample-data-root` to leave it unset. + +## Host mode (arca, no Docker) + +With `--host` the command runs directly on the host instead of `docker exec geobrix-dev`. Prerequisites: `source ~/.local/geobrix-gdal-env.sh` (native GDAL + Java 17 + PYTHONPATH) and `uv` on PATH, with `PIP_INDEX_URL` pointing at the internal pip proxy. The first run builds a host test venv from the exact CI-pinned lock via `uv` — `.venv-host-pyrx` from `python/geobrix/requirements-pyrx-ci.txt` (the light-tier deps: rasterio/pandas/h3/vizx). Neither CI lock contains `pdal` (source-only, unbuildable on arca and not needed here). See the `geobrix-arca` plugin for the full setup. ## Examples @@ -34,6 +40,10 @@ bash scripts/commands/gbx-test-sql-docs.sh [OPTIONS] # API/SQL doc tests only, skip build (uses in-repo minimal bundle) bash scripts/commands/gbx-test-sql-docs.sh --skip-build +# On the arca host (no Docker) — source the GDAL env first +source ~/.local/geobrix-gdal-env.sh +bash scripts/commands/gbx-test-sql-docs.sh --host + # Single test file with log bash scripts/commands/gbx-test-sql-docs.sh --path api/test_sql_api.py --skip-build --log sql-docs.log diff --git a/scripts/commands/gbx-test-sql-docs.sh b/scripts/commands/gbx-test-sql-docs.sh index c21dd3c98..858c7bf87 100644 --- a/scripts/commands/gbx-test-sql-docs.sh +++ b/scripts/commands/gbx-test-sql-docs.sh @@ -20,6 +20,9 @@ show_help() { echo -e " ${GREEN}--path ${NC} File or dir relative to docs/tests/python/ (default: api/)" echo "" echo -e "${CYAN}Common options:${NC}" + echo -e " ${GREEN}--host${NC} Run on the host (arca), not Docker. Requires ${YELLOW}source ~/.local/geobrix-gdal-env.sh${NC}" + echo -e " first; builds/uses ${YELLOW}.venv-host${NC} from the pinned lock." + echo -e " ${GREEN}--rebuild-venv${NC} (with --host) force-rebuild the host test venv" echo -e " ${GREEN}--log ${NC} Write output to log (filename → test-logs/)" echo -e " ${GREEN}--markers ${NC} Pytest markers (e.g. \"not slow\")" echo -e " ${GREEN}--include-integration${NC} Include integration tests (excluded by default)" @@ -29,30 +32,40 @@ show_help() { echo "" echo -e "${CYAN}Examples:${NC}" echo -e " ${YELLOW}gbx:test:sql-docs --skip-build${NC}" + echo -e " ${YELLOW}gbx:test:sql-docs --host${NC} ${CYAN}# on the arca host (no Docker)${NC}" echo -e " ${YELLOW}gbx:test:sql-docs --test api/test_sql_api.py --skip-build${NC}" echo -e " ${YELLOW}gbx:test:sql-docs --log sql-docs.log${NC}" echo "" } -BASE="/root/geobrix/docs/tests/python" -TEST_PATH="${BASE}/api/" +# REL_PATH is relative to docs/tests/python; each mode prefixes it (container BASE vs host). +REL_PATH="api/" LOG_PATH="" MARKERS="-m 'not integration'" INCLUDE_INTEGRATION=false SKIP_BUILD=false +USE_HOST=false # Default: set sample data root so doc tests use minimal bundle (required for remote/CI) SET_SAMPLE_DATA_ROOT=true while [[ $# -gt 0 ]]; do case $1 in --test) - TEST_PATH="${BASE}/$2" + REL_PATH="$2" shift 2 ;; --path) - TEST_PATH="${BASE}/$2" + REL_PATH="$2" shift 2 ;; + --host) + USE_HOST=true + shift + ;; + --rebuild-venv) + export GBX_REBUILD_VENV=1 + shift + ;; --log) LOG_PATH=$(resolve_log_path "$2") shift 2 @@ -89,24 +102,59 @@ done cd "$PROJECT_ROOT" show_banner "📚 GeoBrix: SQL Documentation Tests" -check_docker setup_log_file "$LOG_PATH" -mkdir -p "$PROJECT_ROOT/sample-data/Volumes/main/default/geobrix_samples" -if ! docker exec geobrix-dev test -d /Volumes 2>/dev/null; then - echo -e "${RED}❌ /Volumes not found in container. Start with: ./scripts/docker/start_docker_with_volumes.sh${NC}" - exit 1 -fi +if [ "$USE_HOST" = true ]; then + # --- Host (arca) path: no Docker; build JAR on host, run pytest from .venv-host --- + require_host_gdal_env || exit 1 + echo -e "${CYAN}🎯 Test path: ${YELLOW}$PROJECT_ROOT/docs/tests/python/$REL_PATH${NC} ${CYAN}(host)${NC}" + [ "$SKIP_BUILD" = true ] && echo -e "${CYAN}⏭️ Skipping build (--skip-build)${NC}" + echo "" -echo -e "${CYAN}🎯 Test path: ${YELLOW}$TEST_PATH${NC}" -[ "$SKIP_BUILD" = true ] && echo -e "${CYAN}⏭️ Skipping build (--skip-build)${NC}" -echo "" + # On host, sample data reads from the on-disk mirror via GBX_SAMPLE_DATA_ROOT (path_config honors it); + # no /Volumes symlink needed. + SAMPLE_DATA_ROOT="$PROJECT_ROOT/sample-data/Volumes/main/default/test-data" + + VENV_BIN=$(ensure_host_test_venv pyrx) || exit 1 + # Wire Spark workers to the venv python and unset PROJ_DATA/PROJ_LIB (see common.sh). + activate_host_python_env "$VENV_BIN" -# Use minimal bundle path in container so doc tests pass on remote/CI (unless --no-sample-data-root) -SAMPLE_DATA_ROOT_EXPORT="" -[ "$SET_SAMPLE_DATA_ROOT" = true ] && SAMPLE_DATA_ROOT_EXPORT="export GBX_SAMPLE_DATA_ROOT=/Volumes/main/default/test-data" + if [ "$SKIP_BUILD" != true ]; then + show_separator + echo -e "${CYAN}Building JAR (mvn package -DskipTests -PskipScoverage)...${NC}" + show_separator + (cd "$PROJECT_ROOT" && mvn package -DskipTests -q -PskipScoverage) || exit $? + echo "" + fi -RUN_CMD="set -e + unset JAVA_TOOL_OPTIONS + export JUPYTER_PLATFORM_DIRS=1 + [ "$SET_SAMPLE_DATA_ROOT" = true ] && export GBX_SAMPLE_DATA_ROOT="$SAMPLE_DATA_ROOT" + show_separator + echo -e "${CYAN}Running SQL/API documentation tests (host)...${NC}" + show_separator + marker_args=(); [ -n "$MARKERS" ] && eval "marker_args=($MARKERS)" + "$VENV_BIN/python" -m pytest "$PROJECT_ROOT/docs/tests/python/$REL_PATH" -v --tb=short --color=yes "${marker_args[@]}" + EXIT_CODE=$? +else + # --- Docker path (unchanged) --- + check_docker + + mkdir -p "$PROJECT_ROOT/sample-data/Volumes/main/default/geobrix_samples" + if ! docker exec geobrix-dev test -d /Volumes 2>/dev/null; then + echo -e "${RED}❌ /Volumes not found in container. Start with: ./scripts/docker/start_docker_with_volumes.sh${NC}" + exit 1 + fi + + echo -e "${CYAN}🎯 Test path: ${YELLOW}/root/geobrix/docs/tests/python/$REL_PATH${NC}" + [ "$SKIP_BUILD" = true ] && echo -e "${CYAN}⏭️ Skipping build (--skip-build)${NC}" + echo "" + + # Use minimal bundle path in container so doc tests pass on remote/CI (unless --no-sample-data-root) + SAMPLE_DATA_ROOT_EXPORT="" + [ "$SET_SAMPLE_DATA_ROOT" = true ] && SAMPLE_DATA_ROOT_EXPORT="export GBX_SAMPLE_DATA_ROOT=/Volumes/main/default/test-data" + + RUN_CMD="set -e unset JAVA_TOOL_OPTIONS export JUPYTER_PLATFORM_DIRS=1 $SAMPLE_DATA_ROOT_EXPORT @@ -124,11 +172,12 @@ fi echo '━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━' echo 'Running SQL/API documentation tests...' echo '━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━' -python3 -m pytest $TEST_PATH -v $MARKERS --tb=short --color=yes +python3 -m pytest /root/geobrix/docs/tests/python/$REL_PATH -v $MARKERS --tb=short --color=yes " -docker exec geobrix-dev /bin/bash -c "$RUN_CMD" -EXIT_CODE=$? + docker exec geobrix-dev /bin/bash -c "$RUN_CMD" + EXIT_CODE=$? +fi echo "" # Short test summary when logging (pytest-style: FAILED/SKIPPED + totals)