Skip to content

test/scripts: harden rc.12 devnet test suite from comprehensive RC validation #719

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
branarakic merged 3 commits into from
May 27, 2026
Merged

test/scripts: harden rc.12 devnet test suite from comprehensive RC validation #719

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d1347d7
test/scripts: harden rc.12 devnet test suite from comprehensive RC va…
branarakic May 27, 2026
8f848f3
test/scripts: read auth.token from disk; pre-flight all N nodes
branarakic May 27, 2026
a0370be
test/scripts: address PR #719 review feedback (8 bug fixes)
branarakic May 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
352 changes: 352 additions & 0 deletions scripts/devnet-comprehensive.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,352 @@
#!/usr/bin/env bash
#
# Comprehensive devnet test orchestrator.
#
# Runs (in order, on top of an already-started 6-node devnet):
# 1. v10-rc-validation.sh (15-section API smoke)
# 2. _devnet-full-sweep.sh (baseline harnesses)
# 3. rc11 recovery tests (promote-crash + shutdown-mid-publish)
# 4. rfc38-all aggregator (lu5/lu5-pub/lu7/lu8/lu9/lu10/e2e/xcg/mm/scale/lj)
# 5. rc.12 feature probes (hub-rotation, multi-RPC failover, libp2p tunables,
# CG-phonebook agent discovery, structured ACK rejection reasons)
# 6. node-ui smoke
# 7. soak suite (libp2p / SWM / RS)
#
# Bash 3.2 compatible (uses parallel indexed arrays instead of
# `declare -A`).
#
# IMPORTANT: do NOT edit this file while an instance is running — bash 3.2
# re-reads the script as it executes, and a mid-run byte shift will derail
# the parser (we've observed double-emitted PASS/FAIL log lines under that
# race). Wait for the run to finish, or duplicate the script before
# editing.
#
# Env knobs:
# RESULTS_DIR override the output directory
# (default: $REPO_ROOT/.devnet/comprehensive-results/<ts>)
# SKIP_SOAK=1 skip the long soak suite
# SOAK_ONLY=1 run only the soak suite
# SKIP_PROBES=1 skip the rc.12-specific probes
# SKIP_RFC38_EXTRAS=1 skip the rfc38-all suite
# SKIP_UI=1 skip the node-ui smoke
# FAIL_FAST=1 stop on first FAIL
# SOAK_RS_SECONDS length of the devnet-soak-rs run (default 1800)
# SOAK_LIBP2P_CYCLES libp2p-soak cycle count (default 5; each cycle ~60s)
# SOAK_SWM_CYCLES swm-soak cycle count (default 10)

set -u

REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
TS=$(date -u +'%Y%m%dT%H%M%SZ')
RESULTS="${RESULTS_DIR:-$REPO_ROOT/.devnet/comprehensive-results/$TS}"
mkdir -p "$RESULTS"
# `latest` symlink for convenience. Use the absolute path as the target so
# it resolves correctly even when RESULTS_DIR is overridden to a directory
# outside the default `<parent>/<ts>` layout. -sfn makes re-runs atomically
# replace any prior link without leaving a "latest/latest" trail.
ln -sfn "$RESULTS" "$(dirname "$RESULTS")/latest" 2>/dev/null || true

log() { echo "[orch $(date -u +'%H:%M:%S')] $*" | tee -a "$RESULTS/orchestrator.log"; }

# ── Pre-flight ───────────────────────────────────────────────────
log "Pre-flight: devnet status"
HARDHAT_PORT="${HARDHAT_PORT:-8545}"
API_PORT_BASE="${API_PORT_BASE:-9201}"
DEVNET_DIR="${DEVNET_DIR:-$REPO_ROOT/.devnet}"

if ! curl -sf "http://127.0.0.1:$HARDHAT_PORT" -X POST -H "Content-Type: application/json" \
-d '{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}' > /dev/null 2>&1; then
log "FATAL: Hardhat not responding on :$HARDHAT_PORT — start the devnet first ($REPO_ROOT/scripts/devnet.sh start)"
exit 2
fi
AUTH=$(grep -v '^#' "$DEVNET_DIR/node1/auth.token" 2>/dev/null | head -1 || echo "")
if [ -z "$AUTH" ]; then
log "FATAL: no auth token at $DEVNET_DIR/node1/auth.token"
exit 2
fi
if ! curl -sf -H "Authorization: Bearer $AUTH" "http://127.0.0.1:$API_PORT_BASE/api/status" > /dev/null 2>&1; then
log "FATAL: node 1 not responding on :$API_PORT_BASE"
exit 2
fi
# Don't just trust node 1 — earlier orchestrator runs let v10-rc-validation
# and 4 sibling suites fail downstream because one node had silently died
# overnight. Check every node we'll actually exercise so triage time isn't
# wasted on "test broke" when the truth is "devnet broke".
NUM_NODES="${NUM_NODES:-6}"
DOWN_NODES=""
for n in $(seq 1 "$NUM_NODES"); do
port=$((API_PORT_BASE + n - 1))
code=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $AUTH" \
"http://127.0.0.1:$port/api/status" 2>/dev/null || echo "000")
if [ "$code" != "200" ]; then
DOWN_NODES="${DOWN_NODES} node${n}(port=$port,http=$code)"
fi
done
if [ -n "$DOWN_NODES" ]; then
log "FATAL: not all $NUM_NODES nodes are reachable. Down:${DOWN_NODES}"
log "Hint: ./scripts/devnet.sh stop && ./scripts/devnet.sh start"
exit 2
fi
export DKG_AUTH="$AUTH"
log "Devnet is up — all $NUM_NODES nodes healthy. Results dir: $RESULTS"

# ── Suite registry (parallel arrays; bash 3.2 compatible) ───────
SUITE_IDS=()
SUITE_CMDS=()
SUITE_GROUPS=()
SUITE_RESULTS=()
SUITE_LOGS=()
SUITE_ELAPSEDS=()

register() {
SUITE_IDS+=("$1")
SUITE_GROUPS+=("$2")
SUITE_CMDS+=("$3")
SUITE_RESULTS+=("PENDING")
SUITE_LOGS+=("")
SUITE_ELAPSEDS+=("0")
}

# Group: smoke
register "v10-rc-validation" "smoke" "$REPO_ROOT/scripts/v10-rc-validation.sh"

# Group: sweep
register "devnet-full-sweep" "sweep" "$REPO_ROOT/scripts/_devnet-full-sweep.sh"

# Group: rc11 recovery
register "rc11-promote-crash" "rc11-recovery" "$REPO_ROOT/scripts/devnet-test-rc11-promote-crash-recovery.sh"
register "rc11-shutdown-mid" "rc11-recovery" "$REPO_ROOT/scripts/devnet-test-rc11-shutdown-mid-publish.sh"

# Group: rfc38 extras (the all-aggregator runs lu5/lu5-pub/lu7/lu8/lu9/lu10/e2e/xcg/mm/scale/lj)
if [ "${SKIP_RFC38_EXTRAS:-0}" != "1" ]; then
register "rfc38-all" "rfc38-extras" "$REPO_ROOT/scripts/devnet-test-rfc38-all.sh"
fi

# Group: rc.12 feature probes
if [ "${SKIP_PROBES:-0}" != "1" ]; then
for p in hub-rotation multi-rpc-failover libp2p-tunables cg-phonebook ack-rejection-reasons; do
register "probe-${p}" "rc12-probes" "$REPO_ROOT/scripts/devnet-probe-${p}.sh"
done
fi

# Group: node-ui smoke
if [ "${SKIP_UI:-0}" != "1" ]; then
register "node-ui-smoke" "node-ui" "$REPO_ROOT/scripts/devnet-test-node-ui-smoke.sh"
fi

# Group: soak (LONG)
if [ "${SKIP_SOAK:-0}" != "1" ]; then
SOAK_RECIPIENT_PEER=$(curl -sf -H "Authorization: Bearer $AUTH" \
"http://127.0.0.1:$((API_PORT_BASE + 1))/api/status" 2>/dev/null \
| python3 -c "import sys,json;print(json.load(sys.stdin).get('peerId',''))" 2>/dev/null || echo "")
SOAK_LIBP2P_CYCLES="${SOAK_LIBP2P_CYCLES:-5}"
SOAK_SWM_CYCLES="${SOAK_SWM_CYCLES:-10}"
SOAK_RS_SECONDS="${SOAK_RS_SECONDS:-1800}"

register "libp2p-soak-short" "soak" \
"env DKG_HOME=$DEVNET_DIR/node1 DKG_AUTH=$AUTH API=http://127.0.0.1:$API_PORT_BASE RECIPIENT_PEER_ID=$SOAK_RECIPIENT_PEER RECIPIENT=devnet-node-2 SENDER_TAG=rc12 TOTAL_CYCLES=$SOAK_LIBP2P_CYCLES INTERVAL_S=60 $REPO_ROOT/scripts/libp2p-soak-test.sh"

# SWM soak — solo mode (PEERS_EXPECTED unset). Confirms write-tag
# rate on local SWM survives N × 30s cycles; cross-peer delivery
# is already exercised by rfc38-multi-member + rfc38-cross-cg.
# PEERS_EXPECTED is a comma-separated TAG list (not a count); set
# only if running concurrent operators sharing a SOAK_COHORT_ID.
register "swm-soak-short" "soak" \
"env DKG_HOME=$DEVNET_DIR/node1 DKG_AUTH=$AUTH API=http://127.0.0.1:$API_PORT_BASE SWM_CG_PUBLIC=devnet-test SWM_CG_CURATED=devnet-isolation SWM_INTERVAL_S=30 SWM_TOTAL_CYCLES=$SOAK_SWM_CYCLES SENDER_TAG=rc12 $REPO_ROOT/scripts/swm-soak-test.sh"

register "devnet-soak-rs" "soak" \
"$REPO_ROOT/scripts/devnet-soak-rs.sh 1 $SOAK_RS_SECONDS"
fi

# Apply SOAK_ONLY filter
if [ "${SOAK_ONLY:-0}" = "1" ]; then
NEW_IDS=()
NEW_CMDS=()
NEW_GROUPS=()
NEW_RESULTS=()
NEW_LOGS=()
NEW_ELAPSEDS=()
i=0
while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
if [ "${SUITE_GROUPS[$i]}" = "soak" ]; then
NEW_IDS+=("${SUITE_IDS[$i]}")
NEW_CMDS+=("${SUITE_CMDS[$i]}")
NEW_GROUPS+=("${SUITE_GROUPS[$i]}")
NEW_RESULTS+=("PENDING")
NEW_LOGS+=("")
NEW_ELAPSEDS+=("0")
fi
i=$((i + 1))
done
SUITE_IDS=("${NEW_IDS[@]}")
SUITE_CMDS=("${NEW_CMDS[@]}")
SUITE_GROUPS=("${NEW_GROUPS[@]}")
SUITE_RESULTS=("${NEW_RESULTS[@]}")
SUITE_LOGS=("${NEW_LOGS[@]}")
SUITE_ELAPSEDS=("${NEW_ELAPSEDS[@]}")
fi

log "Registered ${#SUITE_IDS[@]} suite(s):"
i=0
while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
log " - ${SUITE_IDS[$i]} [${SUITE_GROUPS[$i]}]"
i=$((i + 1))
done

# ── Run loop ────────────────────────────────────────────────────
START=$(date +%s)
TOTAL_PASS=0
TOTAL_FAIL=0
TOTAL_MISSING=0

i=0
while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
id="${SUITE_IDS[$i]}"
cmd="${SUITE_CMDS[$i]}"
group="${SUITE_GROUPS[$i]}"
logfile="$RESULTS/${id}.log"
SUITE_LOGS[$i]="$logfile"

# Extract the script path (last token in the command, possibly after env vars)
bare_path=""
for tok in $cmd; do
case "$tok" in
*.sh) bare_path="$tok" ;;
esac
done
[ -z "$bare_path" ] && bare_path=$(echo "$cmd" | awk '{print $NF}')

if [ ! -e "$bare_path" ]; then
log "MISSING: $id ($bare_path)"
SUITE_RESULTS[$i]="MISSING"
TOTAL_MISSING=$((TOTAL_MISSING + 1))
i=$((i + 1))
if [ "${FAIL_FAST:-0}" = "1" ]; then
log "FAIL_FAST=1 — aborting"
break
fi
continue
fi

log "============================================================"
log "RUN $id [$group]"
log "============================================================"
suite_start=$(date +%s)
( cd "$REPO_ROOT" && bash -c "$cmd" ) > "$logfile" 2>&1
ec=$?
suite_end=$(date +%s)
elapsed=$((suite_end - suite_start))
SUITE_ELAPSEDS[$i]="$elapsed"

if [ "$ec" -eq 0 ]; then
SUITE_RESULTS[$i]="PASS"
TOTAL_PASS=$((TOTAL_PASS + 1))
log "PASS $id (${elapsed}s)"
else
SUITE_RESULTS[$i]="FAIL:$ec"
TOTAL_FAIL=$((TOTAL_FAIL + 1))
log "FAIL $id (exit=$ec, ${elapsed}s)"
log " last 12 lines of $logfile:"
tail -n 12 "$logfile" 2>/dev/null | sed 's/^/ /' | tee -a "$RESULTS/orchestrator.log"
if [ "${FAIL_FAST:-0}" = "1" ]; then
log "FAIL_FAST=1 — aborting"
break
fi
fi
i=$((i + 1))
done

END=$(date +%s)
WALL=$((END - START))

# ── Reports ─────────────────────────────────────────────────────
log ""
log "============================================================"
log "DONE — ${WALL}s wall (~$((WALL/60))m)"
log "PASS=$TOTAL_PASS FAIL=$TOTAL_FAIL MISSING=$TOTAL_MISSING TOTAL=${#SUITE_IDS[@]}"
log "============================================================"

# Markdown report
MD="$RESULTS/REPORT.md"
{
echo "# Comprehensive devnet test report"
echo
echo "- **Started**: $(date -u -r $START +'%Y-%m-%dT%H:%M:%SZ')"
echo "- **Ended**: $(date -u -r $END +'%Y-%m-%dT%H:%M:%SZ')"
echo "- **Wall**: ${WALL}s (~$((WALL/60))m)"
echo "- **Branch**: $(cd "$REPO_ROOT" && git rev-parse --abbrev-ref HEAD) @ $(cd "$REPO_ROOT" && git rev-parse --short HEAD)"
echo "- **Results dir**: \`$RESULTS\`"
echo
echo "## Summary"
echo
echo "| | count |"
echo "|---|---|"
echo "| PASS | $TOTAL_PASS |"
echo "| FAIL | $TOTAL_FAIL |"
echo "| MISSING | $TOTAL_MISSING |"
echo "| Total registered | ${#SUITE_IDS[@]} |"
echo
echo "## Suites"
echo
echo "| id | group | result | elapsed | log |"
echo "|---|---|---|---:|---|"
i=0
while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
logf=$(basename "${SUITE_LOGS[$i]}")
echo "| \`${SUITE_IDS[$i]}\` | ${SUITE_GROUPS[$i]} | ${SUITE_RESULTS[$i]} | ${SUITE_ELAPSEDS[$i]}s | \`$logf\` |"
i=$((i + 1))
done
echo
if [ "$TOTAL_FAIL" -gt 0 ]; then
echo "## Failures — last 25 lines of each failing log"
echo
i=0
while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
case "${SUITE_RESULTS[$i]}" in
FAIL:*)
echo "### ${SUITE_IDS[$i]}"
echo
echo '```'
tail -n 25 "${SUITE_LOGS[$i]}" 2>/dev/null || echo "(no log)"
echo '```'
echo
;;
esac
i=$((i + 1))
done
fi
} > "$MD"

# JSON report
JSON="$RESULTS/REPORT.json"
{
echo "{"
echo " \"startedAt\": \"$(date -u -r $START +'%Y-%m-%dT%H:%M:%SZ')\","
echo " \"endedAt\": \"$(date -u -r $END +'%Y-%m-%dT%H:%M:%SZ')\","
echo " \"wallSeconds\": $WALL,"
echo " \"branch\": \"$(cd "$REPO_ROOT" && git rev-parse --abbrev-ref HEAD)\","
echo " \"commit\": \"$(cd "$REPO_ROOT" && git rev-parse HEAD)\","
echo " \"totals\": { \"pass\": $TOTAL_PASS, \"fail\": $TOTAL_FAIL, \"missing\": $TOTAL_MISSING, \"registered\": ${#SUITE_IDS[@]} },"
echo " \"suites\": ["
first=1
i=0
while [ "$i" -lt "${#SUITE_IDS[@]}" ]; do
[ "$first" -eq 0 ] && echo ","
first=0
printf ' { "id": "%s", "group": "%s", "result": "%s", "elapsedSeconds": %s, "log": "%s" }' \
"${SUITE_IDS[$i]}" "${SUITE_GROUPS[$i]}" "${SUITE_RESULTS[$i]}" "${SUITE_ELAPSEDS[$i]}" \
"$(basename "${SUITE_LOGS[$i]}")"
i=$((i + 1))
done
echo
echo " ]"
echo "}"
} > "$JSON"

log "Report: $MD"
log "JSON: $JSON"

if [ "$TOTAL_FAIL" -gt 0 ]; then
exit 1
fi
exit 0
Loading
Loading