test: run nightly bench of block capacity

.github/workflows/nightly-spartan-bench.yml

@@ -223,17 +223,122 @@ jobs:
               --data "$data"
           fi
 
+  block-capacity-benchmark:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          ref: merge-train/spartan
+
+      - name: Determine nightly tag
+        id: nightly-tag
+        run: |
+          if [[ -n "${{ inputs.nightly_tag }}" ]]; then
+            nightly_tag="${{ inputs.nightly_tag }}"
+          else
+            current_version=$(jq -r '."."' .release-please-manifest.json)
+            nightly_tag="${current_version}-spartan.$(date -u +%Y%m%d)"
+          fi
+          echo "nightly_tag=$nightly_tag" >> $GITHUB_OUTPUT
+          echo "Using nightly tag: $nightly_tag"
+
+      - name: Check if Docker image exists
+        run: |
+          DOCKER_IMAGE="aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}"
+          echo "Checking if Docker image exists: $DOCKER_IMAGE"
+          if docker manifest inspect "$DOCKER_IMAGE" > /dev/null 2>&1; then
+            echo "Docker image exists: $DOCKER_IMAGE"
+          else
+            echo "Docker image does not exist: $DOCKER_IMAGE"
+            exit 1
+          fi
+
+      - name: Run block capacity benchmarks
+        timeout-minutes: 240
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          RUN_ID: ${{ github.run_id }}
+          AWS_SHUTDOWN_TIME: 240
+          NO_SPOT: 1
+        run: |
+          ./.github/ci3.sh network-block-capacity-bench block-capacity nightly-block-capacity "aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}"
+
+      - name: Cleanup network resources
+        if: always()
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          BUILD_INSTANCE_SSH_KEY: ${{ secrets.BUILD_INSTANCE_SSH_KEY }}
+          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+          GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
+          NO_SPOT: 1
+        run: ./.github/ci3.sh network-teardown block-capacity nightly-block-capacity
+
+      - name: Download benchmarks
+        if: always()
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          if ./ci.sh gh-spartan-block-capacity-bench; then
+            echo "ENABLE_DEPLOY_BENCH=1" >> $GITHUB_ENV
+          fi
+
+      - name: Upload benchmarks
+        if: always() && env.ENABLE_DEPLOY_BENCH == '1'
+        uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29
+        with:
+          name: Spartan
+          benchmark-data-dir-path: "bench/next"
+          tool: "customSmallerIsBetter"
+          output-file-path: ./bench-out/bench.json
+          gh-repository: github.com/AztecProtocol/benchmark-page-data
+          github-token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }}
+          auto-push: true
+          ref: ${{ github.event.workflow_run.head_sha || github.sha }}
+          alert-threshold: "120%"
+          comment-on-alert: false
+          fail-on-alert: false
+          max-items-in-chart: 100
+
+      - name: Notify Slack on failure
+        if: failure() && github.event_name != 'workflow_dispatch'
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+        run: |
+          if [ -n "${SLACK_BOT_TOKEN}" ]; then
+            read -r -d '' data <<EOF || true
+            {
+              "channel": "#alerts-next-scenario",
+              "text": "Nightly Block Capacity Benchmarks FAILED (nightly tag ${{ steps.nightly-tag.outputs.nightly_tag }}): <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>"
+            }
+          EOF
+            curl -X POST https://slack.com/api/chat.postMessage \
+              -H "Authorization: Bearer $SLACK_BOT_TOKEN" \
+              -H "Content-type: application/json" \
+              --data "$data"
+          fi
+
   status:
     runs-on: ubuntu-latest
-    needs: [benchmark, proving-benchmark]
+    needs: [benchmark, proving-benchmark, block-capacity-benchmark]
     if: always()
     steps:
       - name: Check benchmark results
         run: |
-          if [[ "${{ needs.benchmark.result }}" != "success" || "${{ needs.proving-benchmark.result }}" != "success" ]]; then
+          if [[ "${{ needs.benchmark.result }}" != "success" || "${{ needs.proving-benchmark.result }}" != "success" || "${{ needs.block-capacity-benchmark.result }}" != "success" ]]; then
             echo "One or more benchmark jobs failed"
             echo "benchmark: ${{ needs.benchmark.result }}"
             echo "proving-benchmark: ${{ needs.proving-benchmark.result }}"
+            echo "block-capacity-benchmark: ${{ needs.block-capacity-benchmark.result }}"
             exit 1
           fi
           echo "All benchmark jobs succeeded"

bootstrap.sh

@@ -704,6 +704,30 @@ case "$cmd" in
     bench_merge
     cache_upload spartan-proving-bench-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json
     ;;
+  "ci-network-block-capacity-bench")
+    # Args: <env_file> <namespace> [docker_image]
+    # Deploys network and runs block capacity benchmarks. Cleanup should be done separately.
+    export CI=1
+    env_file="${1:?env_file is required}"
+    namespace="${2:?namespace is required}"
+    docker_image="${3:-}"
+    build
+    # If no docker image provided, build and push to aztecdev
+    if [ -z "$docker_image" ]; then
+      release-image/bootstrap.sh push_pr
+      docker_image="aztecprotocol/aztecdev:$(git rev-parse HEAD)"
+    fi
+    # Set up environment and deploy using spartan
+    export NAMESPACE="$namespace"
+    export AZTEC_DOCKER_IMAGE="$docker_image"
+    spartan/bootstrap.sh network_deploy "${env_file}"
+    # Run block capacity benchmarks
+    spartan/bootstrap.sh block_capacity_bench "${env_file}"
+    rm -rf bench-out
+    mkdir -p bench-out
+    bench_merge
+    cache_upload spartan-block-capacity-bench-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json
+    ;;
   "ci-network-teardown")
     # Args: <env_file> <namespace>
     # Tears down a deployed network.

ci.sh

@@ -244,6 +244,14 @@ case "$cmd" in
     export INSTANCE_POSTFIX="n-proving-bench"
     bootstrap_ec2 "./bootstrap.sh ci-network-proving-bench $*"
     ;;
+  network-block-capacity-bench)
+    # Args: <scenario> <namespace> [docker_image]
+    # Deploys network and runs block capacity benchmarks.
+    export CI_DASHBOARD="network"
+    export JOB_ID="x-${2:?namespace is required}-network-block-capacity-bench" CPUS=16
+    export INSTANCE_POSTFIX="n-block-cap-bench"
+    bootstrap_ec2 "./bootstrap.sh ci-network-block-capacity-bench $*"
+    ;;
   network-teardown)
     # Args: <scenario> <namespace>
     export CI_DASHBOARD="network"
@@ -390,7 +398,7 @@ case "$cmd" in
   ########################
   # BENCHMARK PROCESSING #
   ########################
-  gh-bench|gh-deploy-bench|gh-spartan-bench|gh-spartan-proving-bench)
+  gh-bench|gh-deploy-bench|gh-spartan-bench|gh-spartan-proving-bench|gh-spartan-block-capacity-bench)
     cache_download ${cmd#gh-}-$(git rev-parse HEAD^{tree}).tar.gz
     ;;
 

noir-projects/noir-contracts/contracts/test/benchmarking_contract/src/main.nr

@@ -11,7 +11,15 @@ pub contract Benchmarking {
         macros::{functions::external, storage::storage},
         messages::message_delivery::MessageDelivery,
         note::note_getter_options::NoteGetterOptions,
-        protocol::address::AztecAddress,
+        oracle::random::random,
+        protocol::{
+            address::{AztecAddress, EthAddress},
+            constants::{
+                CONTRACT_CLASS_LOG_SIZE_IN_FIELDS, MAX_L2_TO_L1_MSGS_PER_CALL,
+                MAX_NOTE_HASHES_PER_CALL, MAX_NULLIFIERS_PER_CALL, MAX_PRIVATE_LOGS_PER_CALL,
+                PRIVATE_LOG_SIZE_IN_FIELDS,
+            },
+        },
         state_vars::{Map, Owned, PrivateSet, PublicMutable},
     };
     use field_note::FieldNote;
@@ -61,4 +69,68 @@ pub contract Benchmarking {
     fn sha256_hash_1024(data: [u8; 1024]) -> [u8; 32] {
         sha256::sha256_var(data, data.len())
     }
+
+    // Lightest possible private transaction: empty app circuit, no state changes, no public calls.
+    #[external("private")]
+    fn noop() {}
+
+    #[external("private")]
+    fn emit_nullifiers() {
+        // Safety: Benchmarking code
+        let random_seed = unsafe { random() };
+        for i in 0..MAX_NULLIFIERS_PER_CALL {
+            self.context.push_nullifier(random_seed + (i as Field));
+        }
+    }
+
+    #[external("private")]
+    fn emit_note_hashes() {
+        // Safety: Benchmarking code
+        let random_seed = unsafe { random() };
+
+        for i in 0..MAX_NOTE_HASHES_PER_CALL {
+            self.context.push_note_hash(random_seed + (i as Field));
+        }
+    }
+
+    #[external("private")]
+    fn emit_l2_to_l1_msgs() {
+        // Safety: Benchmarking code
+        let random_seed = unsafe { random() };
+
+        for i in 0..MAX_L2_TO_L1_MSGS_PER_CALL {
+            let recipient = EthAddress::from_field((random_seed as u128) as Field + (i as Field));
+            self.context.message_portal(recipient, random_seed + (i + 1) as Field);
+        }
+    }
+
+    #[external("private")]
+    fn emit_private_logs() {
+        // Safety: Benchmarking code
+        let random_seed = unsafe { random() };
+
+        for i in 0..MAX_PRIVATE_LOGS_PER_CALL {
+            let mut log = [0; PRIVATE_LOG_SIZE_IN_FIELDS];
+            for j in 0..PRIVATE_LOG_SIZE_IN_FIELDS {
+                log[i] = random_seed + (i * MAX_PRIVATE_LOGS_PER_CALL + j) as Field;
+            }
+            self.context.emit_private_log(log, PRIVATE_LOG_SIZE_IN_FIELDS);
+        }
+    }
+
+    #[external("private")]
+    fn emit_contract_class_log() {
+        // Safety: Benchmarking code
+        let random_seed = unsafe { random() };
+
+        let mut log = [0; CONTRACT_CLASS_LOG_SIZE_IN_FIELDS];
+        for i in 0..log.len() {
+            log[i] = random_seed + (i as Field);
+        }
+        self.context.emit_contract_class_log(log);
+    }
+
+    // Lightest possible private transaction: empty app circuit, no state changes, no public calls.
+    #[external("public")]
+    fn noop_pub() {}
 }

spartan/.gitignore

@@ -16,6 +16,7 @@ environments/*
 !environments/devnet-avm-prover.env
 !environments/devnet-next.env
 !environments/devnet.env
+!environments/block-capacity.env
 !environments/ignition-fisherman.env
 !environments/next-net.env
 !environments/next-scenario.env

spartan/bootstrap.sh

@@ -144,6 +144,11 @@ function proving_bench_cmds {
   echo "$(hash):TIMEOUT=${timeout} TPS=${tps} BENCH_OUTPUT=bench-out/n_tps_prove.${tps}tps.bench.json $root/yarn-project/end-to-end/scripts/run_test.sh simple n_tps_prove.test.ts"
 }
 
+function block_capacity_bench_cmds {
+  local timeout=7200  # 2h
+  echo "$(hash):TIMEOUT=${timeout} BENCH_OUTPUT=bench-out/block_capacity.bench.json $root/yarn-project/end-to-end/scripts/run_test.sh simple block_capacity.test.ts"
+}
+
 function network_bench {
   rm -rf bench-out
   mkdir -p bench-out
@@ -170,6 +175,19 @@ function proving_bench {
   proving_bench_cmds | parallelize 1
 }
 
+function block_capacity_bench {
+  rm -rf bench-out
+  mkdir -p bench-out
+
+  local env_file="$1"
+  source_network_env $env_file
+
+  echo_header "spartan block capacity bench"
+  gcp_auth
+  export K8S_ENRICHER=${K8S_ENRICHER:-1}
+  block_capacity_bench_cmds | parallelize 1
+}
+
 function ensure_eth_balances {
   amount="$1"
   # if ETHEREUM_HOST is not set, use the first RPC URL
@@ -234,7 +252,7 @@ case "$cmd" in
     run_network_tests "$1" "$2"
     ;;
 
-  network_tests|network_tests_1|network_tests_2|network_bench|proving_bench)
+  network_tests|network_tests_1|network_tests_2|network_bench|proving_bench|block_capacity_bench)
     env_file="$1"
     $cmd "$env_file"
     ;;

spartan/environments/block-capacity.env

@@ -0,0 +1,50 @@
+NAMESPACE=${NAMESPACE:-block-capacity}
+CLUSTER=aztec-gke-private
+GCP_REGION=us-west1-a
+
+AZTEC_EPOCH_DURATION=8
+AZTEC_SLOT_DURATION=72
+AZTEC_PROOF_SUBMISSION_EPOCHS=4
+AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=1
+AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=1
+
+CREATE_ETH_DEVNET=true
+DESTROY_NAMESPACE=true
+DESTROY_AZTEC_INFRA=true
+CREATE_ROLLUP_CONTRACTS=true
+REDEPLOY_ROLLUP_CONTRACTS=true
+
+ETHEREUM_CHAIN_ID=1337
+LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk"
+FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
+
+OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET
+
+VALIDATOR_REPLICAS=1
+VALIDATORS_PER_NODE=48
+PUBLISHERS_PER_VALIDATOR_KEY=1
+VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000
+VALIDATOR_RESOURCE_PROFILE="prod-hi-tps"
+
+REAL_VERIFIER=false
+
+RPC_REPLICAS=1
+RPC_INGRESS_ENABLED=false
+
+PROVER_REPLICAS=10
+PROVER_RESOURCE_PROFILE="dev"
+PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000
+PROVER_AGENT_POLL_INTERVAL_MS=10000
+PUBLISHERS_PER_PROVER=1
+
+PROVER_TEST_DELAY_TYPE=realistic
+DEBUG_FORCE_TX_PROOF_VERIFICATION=true
+
+SEQ_MAX_TX_PER_BLOCK=72000 # 1000 tps
+SEQ_MIN_TX_PER_BLOCK=0
+SEQ_ENFORCE_TIME_TABLE=true
+P2P_MAX_TX_POOL_SIZE=1000000000
+DEBUG_P2P_INSTRUMENT_MESSAGES=true
+
+LOG_LEVEL="debug; info: json-rpc, simulator"
+

spartan/terraform/deploy-aztec-infra/values/validator-resources-prod-hi-tps.yaml

@@ -0,0 +1,23 @@
+validator:
+  nodeSelector:
+    local-ssd: "false"
+    node-type: "network"
+    cores: "8"
+    hi-mem: "true"
+  node:
+    resources:
+      requests:
+        cpu: "7.5"
+        memory: "55Gi"
+
+    nodeJsOptions:
+      - "--max-old-space-size=61440"
+  statefulSet:
+    volumeClaimTemplates:
+      - metadata:
+          name: data
+        spec:
+          accessModes: [ReadWriteOnce]
+          resources:
+            requests:
+              storage: 16Gi