diff --git a/.bedrock_agentcore.yaml b/.bedrock_agentcore.yaml index 0808de8..9ec1809 100644 --- a/.bedrock_agentcore.yaml +++ b/.bedrock_agentcore.yaml @@ -1,23 +1,127 @@ -agentRuntimeName: jira-readiness-agent -entrypoint: src/agent/main.py -runtime: python3.12 -buildContext: . -networkConfig: - type: PUBLIC -observabilityConfig: - enabled: true -authorizerConfiguration: - customJWTAuthorizer: - discoveryUrl: ${AGENTCORE_JWT_DISCOVERY_URL} - allowedAudience: ["${AGENTCORE_JWT_AUDIENCE}"] -environmentVariables: - AGENT_PROFILE: aws - AWS_REGION: ${AWS_REGION} - BEDROCK_MODEL_ID: us.anthropic.claude-sonnet-4-5-20250929-v1:0 - MCP_BASE_URL: http://mcp-internal.internal:8081/mcp -secretsManagerArns: - - ${MCP_BEARER_TOKEN_ARN} - - ${JIRA_API_TOKEN_ARN} - - ${ANTHROPIC_API_KEY_ARN} - - ${BEDROCK_GUARDRAIL_IDENTIFIER_ARN} - - ${BEDROCK_GUARDRAIL_VERSION_ARN} +default_agent: jira_readiness_agent +agents: + create_agent: + name: create_agent + language: python + node_version: '20' + entrypoint: src/agent/server.py + deployment_type: container + runtime_type: null + platform: linux/arm64 + container_runtime: docker + source_path: /Users/davidparry/code/github/aws-agent-core + aws: + execution_role: arn:aws:iam::577638396504:role/jira-readiness-agent-execution + execution_role_auto_create: false + account: '577638396504' + region: us-east-1 + ecr_repository: null + ecr_auto_create: false + s3_path: null + s3_auto_create: false + network_configuration: + network_mode: PUBLIC + network_mode_config: null + protocol_configuration: + server_protocol: HTTP + observability: + enabled: true + lifecycle_configuration: + idle_runtime_session_timeout: null + max_lifetime: null + bedrock_agentcore: + agent_id: null + agent_arn: null + agent_session_id: null + codebuild: + project_name: null + execution_role: null + source_bucket: null + memory: + mode: STM_ONLY + memory_id: create_agent_mem-d9eb8hDNS3 + memory_arn: arn:aws:bedrock-agentcore:us-east-1:577638396504:memory/create_agent_mem-d9eb8hDNS3 + memory_name: create_agent_mem + event_expiry_days: 30 + first_invoke_memory_check_done: true + was_created_by_toolkit: true + identity: + credential_providers: [] + workload: null + aws_jwt: + enabled: false + audiences: [] + signing_algorithm: ES384 + issuer_url: null + duration_seconds: 300 + authorizer_configuration: null + request_header_configuration: null + oauth_configuration: null + api_key_env_var_name: null + api_key_credential_provider_name: null + is_generated_by_agentcore_create: false + jira_readiness_agent: + name: jira_readiness_agent + language: python + node_version: '20' + entrypoint: src/agent/server.py + deployment_type: container + runtime_type: null + platform: linux/arm64 + container_runtime: docker + source_path: /Users/davidparry/code/github/aws-agent-core + aws: + execution_role: arn:aws:iam::577638396504:role/jira-readiness-agent-execution + execution_role_auto_create: false + account: '577638396504' + region: us-east-1 + ecr_repository: 577638396504.dkr.ecr.us-east-1.amazonaws.com/bedrock-agentcore-jira_readiness_agent + ecr_auto_create: false + s3_path: null + s3_auto_create: false + network_configuration: + network_mode: VPC + network_mode_config: + security_groups: + - sg-0f3c0db822d3d7cda + subnets: + - subnet-0fef3db39a146229e + - subnet-007eaee970b15a41d + protocol_configuration: + server_protocol: HTTP + observability: + enabled: true + lifecycle_configuration: + idle_runtime_session_timeout: null + max_lifetime: null + bedrock_agentcore: + agent_id: jira_readiness_agent-cnVZj34P86 + agent_arn: arn:aws:bedrock-agentcore:us-east-1:577638396504:runtime/jira_readiness_agent-cnVZj34P86 + agent_session_id: null + codebuild: + project_name: bedrock-agentcore-jira_readiness_agent-builder + execution_role: arn:aws:iam::577638396504:role/AmazonBedrockAgentCoreSDKCodeBuild-us-east-1-846bb58ffc + source_bucket: bedrock-agentcore-codebuild-sources-577638396504-us-east-1 + memory: + mode: STM_ONLY + memory_id: jira_readiness_agent_mem-25jINP8uDf + memory_arn: arn:aws:bedrock-agentcore:us-east-1:577638396504:memory/jira_readiness_agent_mem-25jINP8uDf + memory_name: jira_readiness_agent_mem + event_expiry_days: 30 + first_invoke_memory_check_done: true + was_created_by_toolkit: false + identity: + credential_providers: [] + workload: null + aws_jwt: + enabled: false + audiences: [] + signing_algorithm: ES384 + issuer_url: null + duration_seconds: 300 + authorizer_configuration: null + request_header_configuration: null + oauth_configuration: null + api_key_env_var_name: null + api_key_credential_provider_name: null + is_generated_by_agentcore_create: false diff --git a/.cursorignore b/.cursorignore new file mode 100644 index 0000000..74a99d0 --- /dev/null +++ b/.cursorignore @@ -0,0 +1,12 @@ +.coverage +htmlcov/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.hypothesis/ +.import_linter_cache/ +.env +.env.local +.env.*.local +*.pem +*.key \ No newline at end of file diff --git a/.dockerignore b/.dockerignore index 4504c57..a69daf4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,25 @@ # Keep the build context tight so docker layer caching is effective and # developer-only state never leaks into the image. The runtime image # only needs `pyproject.toml`, `README.md`, and `src/` (see Dockerfile). +# +# IMPORTANT — `agentcore deploy` does NOT honor this file. +# The bedrock-agentcore-starter-toolkit hardcodes its own bundled +# `dockerignore.template` (verified in +# bedrock_agentcore_starter_toolkit.services.codebuild +# ::CodeBuildService._parse_dockerignore +# and `utils/runtime/package.py::_get_ignore_patterns`). The toolkit +# template excludes `terraform/`, `cdk/`, `tests/`, `docs/`, and +# `mcp/lambda/`, but NOT `mcp/` itself. +# +# `mcp/` no longer carries the ~62 MB Spring Boot fat-JAR (it is +# resolved from `${MCP_INTERNAL_REPO}/build/libs` at build time — see +# `scripts/build_mcp_image.sh` and `mcp/README.md`), so the size driver +# behind the original AgentCore source.zip bloat is gone. The +# stash-and-restore in `scripts/bootstrap_agentcore_runtime.sh` is +# retained as topology hygiene (the runtime container does not consume +# anything from `mcp/`) and as defence-in-depth against future growth. +# Tracked upstream under "AgentCore source-zip filtering" in +# `docs/DEFERRED.md`. # ============================================================================= # VCS / IDE / cache state. diff --git a/.env.local.example b/.env.local.example index 2de0477..c4632df 100644 --- a/.env.local.example +++ b/.env.local.example @@ -29,11 +29,20 @@ AGENT_PROFILE=local # `DynamoDbTokenBudgetEnforcer`. AGENT_BUDGET_SCOPE=local-pod-dev -# --- LLM provider (Anthropic direct, local profile) -------------------------- -# Consumed by `anthropic_model_factory(api_key=...)`. +# --- LLM provider (local profile) ------------------------------------------- +# `anthropic` uses the hosted Anthropic API. `ollama` runs fully local via +# `langchain_ollama.ChatOllama` and does not require ANTHROPIC_API_KEY. +LLM_PROVIDER=anthropic +# Consumed only when LLM_PROVIDER=anthropic or DESIGN_LLM_PROVIDER=anthropic. ANTHROPIC_API_KEY=sk-ant-FILL-IN # Pinned in lockstep with BEDROCK_MODEL_ID for production parity. ANTHROPIC_MODEL=claude-sonnet-4-5-20250929 +# Consumed only when LLM_PROVIDER=ollama or DESIGN_LLM_PROVIDER=ollama. +OLLAMA_BASE_URL=http://ollama:11434 +OLLAMA_MODEL=llama3.1 +# Optional; defaults to LLM_PROVIDER. Set to `ollama` to run the designer +# locally even when the assessor stays on Anthropic. +DESIGN_LLM_PROVIDER=anthropic # --- MCP transport (HTTP-streamable) ----------------------------------------- # Cluster-internal URL; `localhost` variant only applies when port-forwarded. @@ -71,9 +80,20 @@ AGENT_JIRA_ACCOUNT_ID=local-bot-account-id AGENT_JIRA_EMAIL=local-bot@example.test # --- Observability ----------------------------------------------------------- -# Selects the tracer factory: `jaeger` locally, `xray` in production. +# Selects the tracer factory: `jaeger` (default, span-to-log fallback when +# the OTLP SDK is missing), `otlp` (strict OTLP/HTTP — fails loudly on +# missing SDK or endpoint), `noop` (no tracer), or `xray` (production AWS +# profile only). OBSERVABILITY_BACKEND=jaeger +# OTLP/HTTP exporter endpoint. Defaults to the Jaeger sidecar's OTLP +# receiver under `deploy/local/jaeger.yaml`; redirect to a real OTel +# Collector / Datadog / Grafana Tempo / etc. by overriding here. The +# `OBSERVABILITY_BACKEND=otlp` peer choice **requires** this env var to +# be reachable; the `OBSERVABILITY_BACKEND=jaeger` choice degrades to a +# span-to-log shim if the endpoint is unreachable. +OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318/v1/traces + # --- Webhook signing --------------------------------------------------------- # HMAC-SHA256 secret shared between Atlassian (webhook registration) and # `agent.infrastructure.signature.HmacSha256SignatureVerifier`. The local @@ -85,7 +105,7 @@ WEBHOOK_HMAC_SECRET=local-dev-secret # value left blank or 0 disables that particular cap. The default window is # 1 hour (TOKEN_BUDGET_WINDOW_SECONDS=3600). # -# Wired into `agent.application.token_governance.GovernedAssessmentLanguageModel` +# Wired into `agent.application.token_governance.GovernedAgenticChatModel` # via `agent.composition.TokenGovernanceConfig`. The application enforces # them in addition to the AWS profile's hard cap (AgentCore Harness # `maxTokens` enforced at the runtime layer, terraform/agentcore-runtime). diff --git a/.gitattributes b/.gitattributes index bf0792c..ea73d8c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,21 +1,21 @@ # ============================================================================= -# Git LFS pins for the local mcp-internal Spring Boot JAR. +# Git LFS pins. # -# The `mcp-internal` HTTP-streamable MCP server ships as a Spring Boot -# fat-JAR (`ai.qodo.mcp.InternalMcpApplication`, ~62 MB). -# It is the build context for `mcp/Dockerfile` and is referenced by -# `scripts/build_mcp_image.sh`. Storing it in Git LFS keeps `git clone` -# fast on shallow checkouts and keeps the regular pack files small. +# Historically `mcp/*.jar filter=lfs diff=lfs merge=lfs -text` lived here +# so the ~62 MB `mcp-internal-*.jar` Spring Boot fat-JAR could ride along +# in this repository without bloating the regular pack files. # -# Developer prerequisite (one-time, host-level): -# brew install git-lfs && git lfs install +# That pin has been removed: the JAR is now resolved from outside this +# repository at build time by `scripts/build_mcp_image.sh` +# (--jar / MCP_INTERNAL_JAR / ${MCP_INTERNAL_REPO}/build/libs/mcp-internal-*.jar +# in that resolution order). The script stages the resolved JAR into +# `mcp/.build/mcp-internal.jar` for the duration of `docker build` and +# removes it on exit, so nothing ever needs to live in the working tree. # -# After cloning the repository: -# git lfs pull +# See `mcp/README.md` "External JAR contract" for the developer-facing +# rationale and the matching CI wire-up in `.github/workflows/smoke.yml` +# (downloads a Release artifact from the upstream `mcp-internal` repo). # -# CI / production digest pinning is owned by `scripts/promote_mcp_image.sh` -# (the production-only `mcp-internal` image-pin / bearer-token rotation -# seam) and is independent of the local LFS pin. +# Re-introduce an `lfs` pin here only if a future asset (a real binary +# fixture, a test corpus, etc.) genuinely needs to ride in-tree. # ============================================================================= - -mcp/*.jar filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 929e45f..2a28674 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,11 @@ name: CI on: push: branches: [main] + # Release tags also fan out into the ``sbom`` job below so the + # CycloneDX SBOM gets committed to ``docs/sbom/sbom.cdx.json``. + # PR builds keep getting the artifact-only path (no commit), so + # untrusted forks cannot push to the protected branch. + tags: ["v*", "release/*"] pull_request: permissions: @@ -15,7 +20,7 @@ jobs: timeout-minutes: 15 strategy: matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.12", "3.13", "3.14"] steps: - name: Checkout repository uses: actions/checkout@v4 @@ -38,7 +43,7 @@ jobs: run: mypy - name: Architecture lint (import-linter sealed-test-seams contract) - # Wave 4 sealed-test-seams contract enforced by ``import-linter`` + # Sealed-test-seams contract enforced by ``import-linter`` # (see ``[tool.importlinter]`` in pyproject.toml). Production # code in the ``agent`` package must never import from # ``agent.composition._test_seams``; a regression here would @@ -48,7 +53,7 @@ jobs: - name: Validate structured-log schema (docs/structured_log_schema.json) # Cheap structural gate: ensures the schema parses as JSON and # carries the JSON-Schema $schema marker the JsonStructuredLogger - # validation hook depends on (M5.9 closure). + # validation hook depends on. # Full Draft-2020-12 validation against captured log records # is gated behind STRUCTURED_LOG_VALIDATE=1 at runtime. run: | @@ -87,6 +92,9 @@ jobs: print(f"structured_log_schema.json OK ({len(properties)} properties)") PY + - name: Validate vendored model pricing snapshot + run: python scripts/check_model_prices.py + - name: Unit tests with 100% line + branch coverage run: pytest @@ -111,12 +119,90 @@ jobs: path: htmlcov retention-days: 30 + # Assert the LocalStack seed schemas defined by + # ``make seed-localstack`` match the DynamoDB adapter contracts under + # ``src/agent/infrastructure/``. Two stages: + # + # 1. Static stage: parse the Makefile and adapter source code so the + # contract drift is caught even when LocalStack is unavailable. + # Cheap, runs on every PR. + # + # 2. Live stage (``--mode=describe``): bring up LocalStack as a + # GitHub Actions service container, run ``make seed-localstack`` + # against it, then ``describe_table`` every contract to assert + # the partition key, sort key, and TTL attribute match the + # adapter expectation. This catches the cases where the Makefile + # looks right but the AWS CLI invocation has a typo or a + # truncated attribute name, before it bites at smoke time. + seed-schema-contract: + name: seed-schema-contract + runs-on: ubuntu-latest + timeout-minutes: 10 + services: + localstack: + image: localstack/localstack:3.9 + ports: + - 4566:4566 + env: + SERVICES: dynamodb,s3,sqs + AWS_DEFAULT_REGION: us-east-1 + options: >- + --health-cmd "curl -fsS http://localhost:4566/_localstack/health || exit 1" + --health-interval 5s + --health-timeout 5s + --health-retries 30 + env: + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_DEFAULT_REGION: us-east-1 + LOCALSTACK_ENDPOINT: http://localhost:4566 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + + - name: Install AWS CLI v2 + run: | + curl -fsS "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o awscliv2.zip + unzip -q awscliv2.zip + sudo ./aws/install --update + aws --version + + - name: Install boto3 (for the live describe-table check) + run: | + python -m pip install --upgrade pip + pip install boto3 + + - name: Static schema-contract check (no LocalStack required) + run: python scripts/check_seed_schemas.py --mode=static + + - name: Wait for LocalStack readiness + run: | + for i in $(seq 1 30); do + if curl -fsS "${LOCALSTACK_ENDPOINT}/_localstack/health" >/dev/null 2>&1; then + echo "LocalStack ready after ${i}s" + exit 0 + fi + sleep 1 + done + echo "LocalStack did not become ready within 30s"; exit 1 + + - name: make seed-localstack + run: make seed-localstack + + - name: Live schema-contract check (describe_table on LocalStack) + run: python scripts/check_seed_schemas.py --mode=describe + # `manifest-dryrun` runs `kubectl apply -k --dry-run=client` against the # local-profile kustomize bases so a malformed Deployment / Service / - # ConfigMap fails CI before it reaches `make local-up`. This is the - # M5.8 deliverable (c). The job is independent of the - # Python `quality-gates` job above and is allowed to fail-fast on its - # own without blocking the unit-test matrix. + # ConfigMap fails CI before it reaches `make local-up`. The job is + # independent of the Python `quality-gates` job above and is allowed + # to fail-fast on its own without blocking the unit-test matrix. # # We spin up a throwaway `kind` cluster so kubectl's API-discovery # call (which `kubectl apply --dry-run=client` always performs to map @@ -137,13 +223,13 @@ jobs: - name: Install kubectl uses: azure/setup-kubectl@v4 with: - version: v1.30.0 + version: v1.31.0 - name: Bring up an ephemeral kind cluster (for API discovery) uses: helm/kind-action@v1 with: - version: v0.24.0 - node_image: kindest/node:v1.30.0 + version: v0.25.0 + node_image: kindest/node:v1.31.0 cluster_name: aws-agent-core-ci - name: kubectl apply -k deploy/local/ --dry-run=client @@ -162,9 +248,8 @@ jobs: deploy/local/overlays/host-mount/ \ | kubectl apply -f - --dry-run=client - # Wave 10 deliverable: build the agent-runtime container image and - # generate a CycloneDX SBOM via `syft`. This closes grade.md §11 - # ("Tooling discipline / SBOM") and gives the AppSec team a + # Build the agent-runtime container image and generate a CycloneDX + # SBOM via `syft`. Gives the AppSec team a # machine-readable inventory of every transitive dependency the # production image carries. The job runs on every PR (so a # surprising new transitive dep surfaces immediately at review @@ -180,9 +265,20 @@ jobs: name: agent-runtime image + CycloneDX SBOM runs-on: ubuntu-latest timeout-minutes: 15 + # Release-tag builds need write access to push the SBOM back to + # ``docs/sbom/sbom.cdx.json``. Forked-PR builds keep the default + # ``contents: read`` from the workflow-level ``permissions`` block + # because GitHub strips ``write`` for fork PRs. + permissions: + contents: write steps: - name: Checkout repository uses: actions/checkout@v4 + with: + # The persist-credentials default keeps the auto-provisioned + # ``GITHUB_TOKEN`` on disk so the release-tag commit step + # below can push to the tag's parent branch. + persist-credentials: true - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -229,3 +325,351 @@ jobs: else echo "::warning::sbom.cdx.json is missing or empty" fi + + # WS4 closure: persist the CycloneDX SBOM under + # ``docs/sbom/sbom.cdx.json`` whenever the workflow runs on a + # release tag. The committed file becomes the canonical + # release-time inventory of every transitive dependency the + # agent-runtime image carries; AppSec consumers do not have to + # download a workflow artifact to audit a release. We use the + # auto-provisioned ``GITHUB_TOKEN`` which is scoped to + # ``contents: write`` on this job (see the job-level permissions + # block above). Forked-PR builds skip this step because their + # token is read-only. + - name: Persist SBOM under docs/sbom/ on release tags + if: startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/tags/release/') + run: | + set -euo pipefail + if [[ ! -s sbom.cdx.json ]]; then + echo "::error::sbom.cdx.json is missing or empty; refusing to persist" + exit 1 + fi + mkdir -p docs/sbom + cp sbom.cdx.json docs/sbom/sbom.cdx.json + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add docs/sbom/sbom.cdx.json + if git diff --cached --quiet; then + echo "SBOM already up-to-date for ${GITHUB_REF#refs/tags/}; nothing to commit." + exit 0 + fi + git commit -m "ci(sbom): refresh CycloneDX SBOM for ${GITHUB_REF#refs/tags/}" + # Push to the default branch so the SBOM lands on ``main`` for + # downstream consumers; the tag itself is immutable. + git push origin "HEAD:${GITHUB_REF_NAME%-rc*}" || \ + git push origin "HEAD:main" + + # Architecture conformance gate. Asserts that every contract + # Protocol has a consumer, the thread-id format matches the docs, + # and the import-linter contract list mirrors the documented + # layering. + conformance: + name: architecture conformance + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + + - name: Install package and dev tooling + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run architecture conformance suite + run: pytest tests/architecture --no-cov -v + + # Docs/code parity gate. Fails when an in-doc link is broken, + # a cited symbol cannot be imported, or the deferred-work register + # has a high-risk item past its review date. + docs-code-sync: + name: docs/code parity + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + + - name: Install package + run: | + python -m pip install --upgrade pip + pip install -e . + + - name: Run docs/code-sync check + run: python scripts/check_docs_code_sync.py + + # Unused-seam detector. Fails when an ``agent.contracts`` + # Protocol has no consumer in application/graph/composition/infrastructure. + unused-seams: + name: unused contract seams + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + + - name: Install package + run: | + python -m pip install --upgrade pip + pip install -e . + + - name: Run unused-seam check + run: python scripts/check_unused_seams.py + + # Canonical-event-identifier gate. Asserts every callsite of + # ``StructuredLogger.info / .warning / .error`` uses an identifier + # from ``agent.infrastructure.logging.CANONICAL_EVENT_IDENTIFIERS`` + # and that both copies of ``structured_log_schema.json`` carry the + # same enum. Catches log-event drift on the fast PR path so + # operators don't discover stale identifiers in dashboards + # post-deploy. + event-identifiers: + name: canonical event identifiers + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + + - name: Install package + run: | + python -m pip install --upgrade pip + pip install -e . + + - name: Run event-identifier check + run: python scripts/check_event_identifiers.py + + # WS4 closure: langsmith dormancy gate. ``langchain-core`` declares a + # hard dep on ``langsmith`` but this codebase deliberately keeps the + # package dormant -- no direct imports, no live ``Client`` instances, + # no SaaS traffic when ``LANGSMITH_TRACING=true`` is set. This job + # locks that posture in CI so a future regression cannot silently + # start exporting traces to a third party. + langsmith-dormancy: + name: langsmith dormancy + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + + - name: Install package + run: | + python -m pip install --upgrade pip + pip install -e . + + - name: Run langsmith dormancy check + run: python scripts/check_langsmith_unreachable.py + + # Native-provider availability probe. When the AWS provider + # ships ``aws_bedrockagentcore_runtime`` / ``_memory`` resources but + # ``var.use_native_provider`` is still ``false``, the + # ``native_provider_required`` resource fails the plan -- this job + # surfaces that drift on every PR so the team is forced to flip + # the toggle (vs. silently keeping the cli-fallback path). + tf-providers-availability-probe: + name: tf providers availability probe + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.13" + terraform_wrapper: false + + - name: terraform init -backend=false + working-directory: terraform/agentcore-runtime + run: terraform init -backend=false -input=false + + - name: terraform plan (drift-aware fallback path) + working-directory: terraform/agentcore-runtime + env: + # ``terraform plan`` against the drift-aware fallback path + # does NOT need real AWS credentials -- the plan is a no-op + # when the local-exec provisioner does not run -- but we + # still need the AWS provider to initialize. Use the + # mock credentials the LocalStack pipeline already uses. + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_DEFAULT_REGION: us-east-1 + run: | + terraform plan -input=false \ + -var "use_native_provider=false" \ + -var "agent_account_id=000000000000" \ + -var "agent_runtime_name=aws-agent-core-runtime" \ + -var "agent_runtime_image=000000000000.dkr.ecr.us-east-1.amazonaws.com/aws-agent-core/agent-runtime:probe" \ + -var "agent_runtime_role_arn=arn:aws:iam::000000000000:role/aws-agent-core-runtime" \ + -var "kms_key_arn=arn:aws:kms:us-east-1:000000000000:key/00000000-0000-0000-0000-000000000000" \ + -refresh=false || true + + - name: terraform output -raw native_provider_available + working-directory: terraform/agentcore-runtime + # The probe writes ``true`` / ``false`` to a state file; we + # read the captured value via ``terraform output``. When the + # native provider becomes available, the + # ``null_resource.native_provider_required`` precondition + # fails the plan and this job goes red so the operator is + # forced to flip ``var.use_native_provider``. + run: | + if terraform output -raw native_provider_available 2>/dev/null | grep -q "true"; then + echo "::error::Native AWS provider supports aws_bedrockagentcore_runtime but use_native_provider is false. Flip the toggle." + exit 1 + fi + echo "Native provider not yet available; cli-fallback remains in effect." + + # SCP plan canary. Renders ``terraform/organizations/`` to + # JSON and asserts the SCP policy resource (and, with the default + # ``var.attach_scp = true``, at least one attachment) is present. + # No AWS credentials required -- the canary parses the plan JSON + # purely client-side via stdlib ``json``. + iam-policy-canary-scp-plan: + name: iam_policy_canary --scp-plan + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + + - name: Set up Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.13" + terraform_wrapper: false + + - name: Install package + run: | + python -m pip install --upgrade pip + pip install -e . + + - name: terraform init -backend=false + working-directory: terraform/organizations + run: terraform init -backend=false -input=false + + - name: terraform plan -> JSON + working-directory: terraform/organizations + env: + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_DEFAULT_REGION: us-east-1 + run: | + terraform plan -input=false \ + -var "audit_account_id=000000000000" \ + -var 'raw_payloads_bucket_arn=arn:aws:s3:::aws-agent-core-raw-payloads' \ + -var 'target_ou_ids=["ou-aaaa-aaaaaaaa"]' \ + -refresh=false \ + -out plan.out + terraform show -json plan.out > plan.json + + - name: Run SCP plan canary (--expect-attachment per the default) + run: | + python scripts/iam_policy_canary.py \ + --scp-plan terraform/organizations/plan.json \ + --expect-attachment + + # Multi-tenant integration suite. Runs the integration tests + # that exercise two tenants concurrently against LocalStack. A + # tenant-leak regression (e.g. co-locating two tenants on the same + # ``thread_id``) shows up here before it can ship. + multi-tenant-integration: + name: multi-tenant integration + runs-on: ubuntu-latest + timeout-minutes: 10 + services: + localstack: + image: localstack/localstack:3.9 + ports: + - 4566:4566 + env: + SERVICES: dynamodb,s3,sqs + AWS_DEFAULT_REGION: us-east-1 + options: >- + --health-cmd "curl -fsS http://localhost:4566/_localstack/health || exit 1" + --health-interval 5s + --health-timeout 5s + --health-retries 30 + env: + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_DEFAULT_REGION: us-east-1 + LOCALSTACK_ENDPOINT_URL: http://localhost:4566 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + + - name: Install package and dev tooling + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Wait for LocalStack readiness + run: | + for i in $(seq 1 30); do + if curl -fsS "${LOCALSTACK_ENDPOINT_URL}/_localstack/health" >/dev/null 2>&1; then + echo "LocalStack ready after ${i}s" + exit 0 + fi + sleep 1 + done + echo "LocalStack did not become ready within 30s"; exit 1 + + - name: Seed LocalStack + run: make seed-localstack + + - name: Run multi-tenant integration suite + # We select tests under tests/integration/multitenant/ when + # the directory exists. Until the suite is fully fleshed + # out the marker selector keeps the gate low-friction; the + # job is PR-required so the absence of the directory fails + # the gate. + run: | + if [[ ! -d tests/integration/multitenant ]]; then + echo "::error::tests/integration/multitenant/ missing; create the multi-tenant integration suite." + exit 1 + fi + pytest tests/integration/multitenant -m integration --no-cov -v diff --git a/.github/workflows/iam-canary.yml b/.github/workflows/iam-canary.yml index 41c13a4..223c2ca 100644 --- a/.github/workflows/iam-canary.yml +++ b/.github/workflows/iam-canary.yml @@ -101,11 +101,11 @@ jobs: echo "run=${run}" >>"${GITHUB_OUTPUT}" echo "reason=${reason}" >>"${GITHUB_OUTPUT}" - - name: Set up Python 3.12 + - name: Set up Python 3.13 if: steps.detect.outputs.run == 'true' uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" cache: pip - name: Install boto3 + PyYAML (canary lazy-imports both) diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml index e78f92d..37e36d7 100644 --- a/.github/workflows/mutation.yml +++ b/.github/workflows/mutation.yml @@ -1,8 +1,8 @@ name: Weekly mutation testing (advisory) -# Wave 9 deliverable. Runs `mutmut` on a focused subset of the -# application layer (`event_invariants` + `recursion_guard`) — the -# two modules whose invariants short-circuit the load-bearing +# Runs `mutmut` on a focused subset of the application layer +# (`recursion_guard`, `webhook_handler`, etc.) -- the modules whose +# invariants short-circuit the load-bearing # safety paths. A surviving mutant indicates a missing test # assertion; the workflow uploads the HTML report as an artifact # so the on-call engineer can drill in without reproducing the @@ -30,7 +30,7 @@ concurrency: jobs: mutation: - name: mutmut event_invariants + recursion_guard + name: mutmut application-layer invariants runs-on: ubuntu-latest timeout-minutes: 45 env: @@ -44,10 +44,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Python 3.12 + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" cache: pip - name: Install dev tooling (includes mutmut) @@ -55,7 +55,7 @@ jobs: python -m pip install --upgrade pip pip install -e ".[dev]" - - name: Run mutmut (event_invariants + recursion_guard) + - name: Run mutmut (application-layer invariants) # `mutmut run` mutates the modules listed under # `[tool.mutmut].paths_to_mutate` in pyproject.toml and runs # the suite listed under `tests_dir`. Exit code 1 is "some diff --git a/.github/workflows/sandbox-apply.yml b/.github/workflows/sandbox-apply.yml new file mode 100644 index 0000000..7bc1e18 --- /dev/null +++ b/.github/workflows/sandbox-apply.yml @@ -0,0 +1,124 @@ +############################################################################### +# Manual-approval sandbox apply workflow. +# +# Provisions and tears down the three load-bearing IaC modules against a +# disposable sandbox AWS account so the drift-aware ``null_resource`` +# fallbacks can be validated end-to-end: +# +# * ``terraform/agentcore-runtime`` — runtime + memory ARNs. +# * ``terraform/observability-cloudwatch-genai`` — Application Signals. +# * ``terraform/kms-secrets`` — KMS rotation cycle. +# +# The job asserts that every ``runtime_arn`` / ``memory_store_arn`` / +# ``transaction_search_status`` output is non-``cli-fallback:*`` before +# running the teardown step, so a sandbox apply that silently fell back +# to the placeholder string is failed loudly rather than passing +# vacuously. +# +# Triggered manually via ``workflow_dispatch`` so accidental commits +# never spin up sandbox resources. GitHub Environments protection on +# the ``sandbox-aws`` environment requires a human approver before +# the AWS credentials are exposed to the runner. +############################################################################### + +name: sandbox-apply + +on: + workflow_dispatch: + inputs: + module: + description: "IaC module to apply (use 'all' for the full sweep)" + required: true + default: "all" + type: choice + options: + - all + - agentcore-runtime + - observability-cloudwatch-genai + - kms-secrets + schedule: + # Nightly cadence so the drift-aware fallback path is + # exercised every 24h against the sandbox account. The + # ``environment: sandbox-aws`` protection rules below still + # require a human approver before credentials are issued, so the + # cron just queues the run -- it does not auto-apply. + - cron: "0 6 * * *" + +permissions: + contents: read + id-token: write + +jobs: + sandbox-apply: + runs-on: ubuntu-latest + environment: sandbox-aws + concurrency: + # Default the concurrency key to ``all`` for the nightly cron + # so two scheduled runs serialise rather than racing. + group: sandbox-apply-${{ github.event.inputs.module || 'all' }} + cancel-in-progress: false + env: + SANDBOX_MODULE: ${{ github.event.inputs.module || 'all' }} + + steps: + - name: Check out repo + uses: actions/checkout@v4 + + - name: Configure AWS credentials (sandbox) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.SANDBOX_TF_ROLE_ARN }} + aws-region: ${{ vars.SANDBOX_AWS_REGION }} + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_version: "1.13" + + - name: Apply agentcore-runtime + if: env.SANDBOX_MODULE == 'all' || env.SANDBOX_MODULE == 'agentcore-runtime' + working-directory: terraform/agentcore-runtime + run: | + terraform init -input=false + terraform apply -auto-approve + runtime_arn=$(terraform output -raw runtime_arn) + memory_arn=$(terraform output -raw memory_store_arn) + if [[ "$runtime_arn" == cli-fallback:* ]]; then + echo "::error::runtime_arn is still a cli-fallback placeholder: $runtime_arn" + exit 1 + fi + if [[ "$memory_arn" == cli-fallback:* ]]; then + echo "::error::memory_store_arn is still a cli-fallback placeholder: $memory_arn" + exit 1 + fi + echo "runtime_arn=$runtime_arn" + echo "memory_arn=$memory_arn" + + - name: Apply observability-cloudwatch-genai + if: env.SANDBOX_MODULE == 'all' || env.SANDBOX_MODULE == 'observability-cloudwatch-genai' + working-directory: terraform/observability-cloudwatch-genai + run: | + terraform init -input=false + terraform apply -auto-approve + + - name: Apply kms-secrets and exercise rotation cycle + if: env.SANDBOX_MODULE == 'all' || env.SANDBOX_MODULE == 'kms-secrets' + working-directory: terraform/kms-secrets + run: | + terraform init -input=false + terraform apply -auto-approve + # Trigger a rotation against the in-scope secret kinds. + if [ -n "${{ secrets.SANDBOX_HMAC_SECRET_ID }}" ]; then + aws secretsmanager rotate-secret \ + --secret-id "${{ secrets.SANDBOX_HMAC_SECRET_ID }}" \ + --rotate-immediately + fi + + - name: Teardown + if: always() && (env.SANDBOX_MODULE == 'all' || env.SANDBOX_MODULE != 'kms-secrets') + run: | + for dir in terraform/agentcore-runtime terraform/observability-cloudwatch-genai; do + if [ -d "$dir/.terraform" ]; then + (cd "$dir" && terraform destroy -auto-approve || true) + fi + done diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index ae5986c..75e5864 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -1,7 +1,7 @@ name: Local-stack smoke -# Nightly CI workflow that runs `make smoke` end-to-end so the smoke -# target no longer exits 0 when the runtime is wedged. The workflow +# Nightly CI workflow that runs `make smoke` end-to-end so a wedged +# runtime fails the job (non-zero exit). The workflow # brings up the full local profile (kind + LocalStack + mcp-internal # stub + agent-runtime) on a GitHub-hosted runner once per night, runs # both the happy and recursion smoke fixtures through @@ -57,15 +57,30 @@ jobs: # (tests/integration/test_smoke_local.py) reuse the same # port-forwarded endpoint the Makefile smoke targets hit. AGENT_LOCAL_TARGET: http://localhost:8080/invocations + # Upstream `mcp-internal` repository the smoke job pulls the + # Spring Boot fat-JAR from. The JAR is no longer tracked in this + # repo (see .gitattributes / mcp/README.md "External JAR + # contract"); the workflow downloads it from a GitHub Release of + # the upstream project below. + # + # `MCP_INTERNAL_REPO_SLUG` is the `/` to fetch from + # (override via repository variables if the project is mirrored + # under a different org). `MCP_INTERNAL_VERSION` pins the release + # tag this aws-agent-core revision is qualified against; bump it + # in lock-step with `mcp/README.md` whenever the upstream JAR is + # rotated. The matching release on the upstream repo MUST attach + # an asset named `mcp-internal-${MCP_INTERNAL_VERSION}.jar`. + MCP_INTERNAL_REPO_SLUG: ${{ vars.MCP_INTERNAL_REPO_SLUG || 'qodo-ai/mcp-internal' }} + MCP_INTERNAL_VERSION: ${{ vars.MCP_INTERNAL_VERSION || '1.0.4' }} steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Python 3.12 + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" cache: pip - name: Install dev tooling (pytest + coverage gate parity) @@ -76,7 +91,7 @@ jobs: - name: Install kubectl uses: azure/setup-kubectl@v4 with: - version: v1.30.0 + version: v1.31.0 - name: Install AWS CLI v2 (for `make seed-localstack`) run: | @@ -96,24 +111,64 @@ jobs: run: | scripts/build_local_image.sh - - name: Build the mcp-internal image (best-effort) - # `mcp/` is intentionally lightweight and the image is built - # from a JAR fetched via Git LFS. When the JAR is unavailable - # (LFS quota exhausted, etc.) we skip the build and let the - # smoke happy-path test depend on the Pod's ImagePullBackOff - # diagnostics surfacing in the artifact bundle. + - name: Download mcp-internal JAR from upstream Release + # The mcp-internal JAR is load-bearing for the smoke pipeline + # because the SoftNoopWriter calls + # ``jira_set_issue_property`` against it on every + # ``issue_updated`` invocation. When the JAR is unavailable + # the workflow fails fast with an actionable pointer to + # ``mcp/README.md`` instead of pretending to be green -- + # the agent-runtime would otherwise silently violate the + # ``write_count >= 1`` invariant on every soft-noop pass. + # + # The JAR is no longer carried in this repository (see + # .gitattributes / mcp/README.md). It is downloaded from a + # GitHub Release of the upstream ${{ env.MCP_INTERNAL_REPO_SLUG }} + # project, pinned to ${{ env.MCP_INTERNAL_VERSION }}. The + # release on that repo MUST attach an asset named + # `mcp-internal-${MCP_INTERNAL_VERSION}.jar`. + # + # `MCP_INTERNAL_TOKEN` is a repo-level Actions secret holding a + # PAT (or a fine-grained token / GitHub App token) with `Contents: + # Read` on the upstream repo. The default `GITHUB_TOKEN` cannot + # cross-repo, so this PAT is required even for public assets if + # the upstream repo is private. If the upstream repo is public + # and you want to skip the secret, you can replace the `gh` + # invocation with a plain `curl -fL` against the public release + # asset URL. + env: + GH_TOKEN: ${{ secrets.MCP_INTERNAL_TOKEN || secrets.GITHUB_TOKEN }} run: | - if [[ -f mcp/Dockerfile ]] && ls mcp/mcp-internal-*.jar >/dev/null 2>&1; then - scripts/build_mcp_image.sh - else - echo "::warning::mcp-internal JAR not present; skipping build (the deployment will pull from registry or CrashLoopBackOff)" + if [[ ! -f mcp/Dockerfile ]]; then + echo "::error::mcp/Dockerfile missing -- see mcp/README.md for the upstream deployment contract." + exit 1 + fi + mkdir -p .ci-cache + ASSET="mcp-internal-${MCP_INTERNAL_VERSION}.jar" + DEST=".ci-cache/${ASSET}" + echo "Fetching ${ASSET} from ${MCP_INTERNAL_REPO_SLUG} release v${MCP_INTERNAL_VERSION}..." + if ! gh release download "v${MCP_INTERNAL_VERSION}" \ + --repo "${MCP_INTERNAL_REPO_SLUG}" \ + --pattern "${ASSET}" \ + --output "${DEST}" \ + --clobber; then + echo "::error::Failed to download ${ASSET} from ${MCP_INTERNAL_REPO_SLUG} release v${MCP_INTERNAL_VERSION}. Verify (a) the release exists and attaches that asset, (b) MCP_INTERNAL_TOKEN has Contents:Read on the upstream repo, and (c) the MCP_INTERNAL_REPO_SLUG / MCP_INTERNAL_VERSION repo variables are correct." + exit 1 fi + echo "MCP_INTERNAL_JAR=${PWD}/${DEST}" >> "${GITHUB_ENV}" + + - name: Build the mcp-internal image (smoke gate) + # `scripts/build_mcp_image.sh` reads MCP_INTERNAL_JAR (set by + # the previous step) and stages it into mcp/.build/ for the + # docker build. + run: | + scripts/build_mcp_image.sh - name: Bring up an ephemeral kind cluster uses: helm/kind-action@v1 with: - version: v0.24.0 - node_image: kindest/node:v1.30.0 + version: v0.25.0 + node_image: kindest/node:v1.31.0 cluster_name: aws-agent-core-smoke - name: Load locally-built images into the kind cluster @@ -140,16 +195,13 @@ jobs: run: | kubectl rollout status -n "${NAMESPACE}" deploy/agent-runtime --timeout=300s - - name: Wait for mcp-internal to become Ready (best-effort) - # Some PRs intentionally land without an mcp-internal image - # (LFS-backed JAR not yet present). We continue past the - # rollout failure so the agent-runtime smoke happy-path can - # surface its own diagnostic. If the deployment IS present, - # we honour its rollout result. + - name: Wait for mcp-internal to become Ready (mandatory) + # The mcp-internal Pod is load-bearing for the SoftNoopWriter + # ``jira_set_issue_property`` gate, so the workflow fails + # fast if the rollout doesn't settle. See + # ``mcp/README.md`` for the upstream deployment contract. run: | - if kubectl get -n "${NAMESPACE}" deploy/mcp-internal >/dev/null 2>&1; then - kubectl rollout status -n "${NAMESPACE}" deploy/mcp-internal --timeout=300s || true - fi + kubectl rollout status -n "${NAMESPACE}" deploy/mcp-internal --timeout=300s - name: Port-forward LocalStack to localhost:4566 (background) # `make seed-localstack` defaults to ${LOCALSTACK_ENDPOINT} which @@ -198,10 +250,39 @@ jobs: run: | make smoke-recursion + - name: Smoke test — SoftNoopWriter against mcp-internal + # Assert the deployed mcp-internal JAR exposes + # ``jira_set_issue_property`` so the SoftNoopWriter can stamp + # the invisible audit property the EventInvariantPolicy + # depends on. A missing endpoint here means a regression in + # the JAR contract; fail with a pointer to mcp/README.md. + run: | + PORT=$(kubectl get svc -n "${NAMESPACE}" mcp-internal -o jsonpath='{.spec.ports[0].port}' 2>/dev/null || echo "8081") + kubectl port-forward -n "${NAMESPACE}" svc/mcp-internal "${PORT}:${PORT}" >mcp-pf.log 2>&1 & + MCP_PID=$! + echo "${MCP_PID}" > mcp-pf.pid + for i in {1..30}; do + if curl -fsS "http://localhost:${PORT}/actuator/health" >/dev/null 2>&1; then + break + fi + if [[ $i -eq 30 ]]; then + echo "::error::mcp-internal port-forward did not come up; see mcp/README.md." + cat mcp-pf.log || true + exit 1 + fi + sleep 1 + done + BODY='{"jsonrpc":"2.0","id":1,"method":"tools/list","params":{}}' + if ! curl -fsS -H "Content-Type: application/json" -H "Authorization: Bearer ${MCP_INTERNAL_TOKEN:-DAVIDSUPERSECRETTOKEN}" \ + -d "${BODY}" "http://localhost:${PORT}/mcp" | tee mcp-tools.json | grep -q '"jira_set_issue_property"'; then + echo "::error::mcp-internal JAR did not advertise jira_set_issue_property; the SoftNoopWriter contract is broken. See mcp/README.md." + exit 1 + fi + kill "${MCP_PID}" 2>/dev/null || true + - name: Integration tests via pytest -m integration # Re-run the same smoke fixtures from the pytest entrypoint to - # exercise the test-runner wiring (replaces the prior placeholder - # integration test). + # exercise the test-runner wiring for integration tests. run: | pytest tests/integration -m integration --no-cov -v @@ -236,15 +317,15 @@ jobs: retention-days: 14 aws-profile-smoke: - # Wave 6 (lifts §4 / §7): nightly round-trip of the canonical + # Nightly round-trip of the canonical # comment_created_human.json fixture through the deployed AgentCore # Runtime via the `agentcore invoke` CLI. Runs against a sandbox # AWS account using OIDC-issued short-lived credentials (no static # keys) and is intentionally decoupled from `local-stack-smoke` so # a kind-cluster regression cannot mask an AWS-profile regression - # (and vice versa). When the secrets / role are not yet - # provisioned, the test inside `tests/integration/test_smoke_aws.py` - # auto-skips on missing env vars so this job stays green. + # (and vice versa). Scheduled CI sets REQUIRE_AWS_PROFILE_SMOKE=1 so + # missing sandbox secrets fail visibly instead of producing a vacuous + # green skip. name: AWS sandbox + agentcore invoke runs-on: ubuntu-latest timeout-minutes: 15 @@ -254,15 +335,23 @@ jobs: env: AWS_REGION: us-east-1 + REQUIRE_AWS_PROFILE_SMOKE: "1" + # GitHub Actions does not expose the `secrets` context inside + # step-level ``if:`` expressions ("Unrecognized named-value: + # 'secrets'"), so we project the sandbox role ARN through the + # job-level ``env`` block and gate the credentials step on the + # ``env`` value instead. See actions/runner#520 for the + # upstream constraint. + AWS_AGENTCORE_SANDBOX_ROLE_ARN: ${{ secrets.AWS_AGENTCORE_SANDBOX_ROLE_ARN }} steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Set up Python 3.12 + - name: Set up Python 3.13 uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" cache: pip - name: Install pytest + agentcore CLI @@ -276,11 +365,14 @@ jobs: # the repo settings to assume a least-privilege role in the # sandbox account that is allowed only to invoke the deployed # AgentCore Runtime. When the secret is absent, the step is - # a no-op and the integration test below auto-skips. - if: ${{ secrets.AWS_AGENTCORE_SANDBOX_ROLE_ARN != '' }} + # a no-op and the integration test below auto-skips. We + # gate on ``env`` rather than ``secrets`` because the latter + # is not a valid named value in step-level ``if:`` + # conditions (GitHub Actions parser rejects it). + if: ${{ env.AWS_AGENTCORE_SANDBOX_ROLE_ARN != '' }} uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ secrets.AWS_AGENTCORE_SANDBOX_ROLE_ARN }} + role-to-assume: ${{ env.AWS_AGENTCORE_SANDBOX_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} - name: Run AWS-profile integration smoke diff --git a/.github/workflows/soak.yml b/.github/workflows/soak.yml new file mode 100644 index 0000000..b805b95 --- /dev/null +++ b/.github/workflows/soak.yml @@ -0,0 +1,140 @@ +name: Soak (weekly) + +# WS5 closure: a weekly burst-rate soak test against the LocalStack-backed +# agent runtime. The job is intentionally NOT part of the per-PR pipeline +# (it would push the wall-clock past the GitHub-hosted-runner 6-hour +# ceiling for any contributor running ``make ci`` locally), so we +# schedule it via cron and expose ``workflow_dispatch`` for ad-hoc runs. +# +# Failures here surface in the weekly digest the on-call rotates through; +# the soak suite asserts: +# * No request returns HTTP non-2xx. +# * Recursion-guard skip path always reports ``status="skipped"``. +# * Happy-path DLQ rate stays under SOAK_DLQ_RATE_BUDGET. +# +# See `tests/integration/perf/locustfile.py` for the assertion surface. + +on: + schedule: + # Every Monday at 08:00 UTC. The hour is deliberately offset from + # the per-PR pipeline's peak (00:00-06:00 UTC) so the runner pool is + # uncontended. + - cron: "0 8 * * 1" + workflow_dispatch: + inputs: + run_time: + description: "locust --run-time (e.g. 5m, 30m, 1h)" + required: false + default: "10m" + users: + description: "Concurrent locust users" + required: false + default: "20" + +permissions: + contents: read + +jobs: + soak: + name: LocalStack burst-rate soak + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_DEFAULT_REGION: us-east-1 + LOCALSTACK_ENDPOINT: http://localhost:4566 + AGENT_LOCAL_TARGET: http://localhost:8080/invocations + SOAK_USERS: ${{ github.event.inputs.users || '20' }} + SOAK_SPAWN: "10" + SOAK_RUN_TIME: ${{ github.event.inputs.run_time || '10m' }} + SOAK_DLQ_RATE_BUDGET: "0.10" + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + + - name: Install package + [perf] extra + run: | + python -m pip install --upgrade pip + pip install -e ".[perf]" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build agent-runtime image + run: | + scripts/build_local_image.sh + + - name: Start LocalStack + run: | + docker run -d --rm \ + --name localstack \ + -p 4566:4566 \ + -e SERVICES=dynamodb,s3,sqs \ + -e DEFAULT_REGION=${AWS_DEFAULT_REGION} \ + localstack/localstack:latest + # Wait for LocalStack to start. 60s upper bound is generous — + # GitHub-hosted runners typically clear LocalStack readiness in + # under 15s. + for i in $(seq 1 60); do + if curl -fs http://localhost:4566/_localstack/health >/dev/null 2>&1; then + echo "LocalStack ready (attempt $i)" + break + fi + sleep 1 + done + + - name: Seed LocalStack tables / bucket / queue + run: | + make seed-localstack + + - name: Boot agent runtime container + run: | + docker run -d --rm \ + --name agent-runtime \ + --network host \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ + -e AWS_DEFAULT_REGION \ + -e LOCALSTACK_ENDPOINT \ + -e AGENT_PROFILE=local \ + -e OBSERVABILITY_BACKEND=noop \ + aws-agent-core/agent-runtime:local + for i in $(seq 1 60); do + if curl -fs http://localhost:8080/health >/dev/null 2>&1; then + echo "agent-runtime ready (attempt $i)" + break + fi + sleep 1 + done + + - name: Run locust soak + run: | + make soak + + - name: Capture agent-runtime logs (always) + if: always() + run: | + docker logs agent-runtime > agent-runtime.log 2>&1 || true + docker logs localstack > localstack.log 2>&1 || true + + - name: Upload runtime / LocalStack logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: soak-logs + path: | + agent-runtime.log + localstack.log + retention-days: 14 + + - name: Tear down docker stack + if: always() + run: | + docker stop agent-runtime || true + docker stop localstack || true diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index fbc7701..f73d82c 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -52,7 +52,7 @@ jobs: - name: Set up Terraform uses: hashicorp/setup-terraform@v3 with: - terraform_version: 1.9.5 + terraform_version: "1.13" terraform_wrapper: false - name: Set up tflint diff --git a/.gitignore b/.gitignore index 603d22c..ea1ea19 100644 --- a/.gitignore +++ b/.gitignore @@ -113,6 +113,14 @@ terraform.rc # ============================================================================= .agentcore/ +# Per-deploy evidence directory written by the aws-deploy-plan.md / runbook helpers. +# Holds saved Terraform plans (.state/.tfplan), human-readable +# plan summaries (.state/.plan.txt), JSON plan dumps +# (.state/.plan.json), and post-apply outputs +# (.state/.outputs.json). All are operator-local; everything in +# here is treated as sensitive and must never be committed. +.state/ + # ============================================================================= # AI assistant local config (per-developer; never team-wide) # ============================================================================= @@ -170,6 +178,18 @@ logs/ mcp/.env +# ============================================================================= +# mcp-internal Spring Boot fat-JAR (external dependency) +# ============================================================================= +# The ~62 MB `mcp-internal-*.jar` is no longer tracked in this repository; +# `scripts/build_mcp_image.sh` resolves it from outside the repo +# (--jar / MCP_INTERNAL_JAR / ${MCP_INTERNAL_REPO}/build/libs) and stages +# it into `mcp/.build/mcp-internal.jar` for the duration of `docker build`. +# Both patterns below are belt-and-braces guards so a stray copy never +# slips into a commit. +mcp/*.jar +mcp/.build/ + # ============================================================================= # Terraform — production-only IaC hardening # ============================================================================= @@ -183,6 +203,12 @@ mcp/.env terraform/**/.terraform/ terraform/**/.terraform.tfstate.lock.info +# Saved plan files. Binary plan files (`-out=...`) embed the resolved +# values of every variable, including secret ARNs, so they're treated +# the same as state on disk. +terraform/**/tfplan +terraform/**/*.tfplan + # State files (must live in the remote backend, never on disk in git). terraform/**/terraform.tfstate terraform/**/terraform.tfstate.backup @@ -197,3 +223,34 @@ terraform/**/*.tfvars.json # .terraform.lock.hcl is intentionally committed (provider lock file) — do NOT # add an ignore pattern for it. +# Backend configuration uses Terraform's "partial backend config" pattern: +# - terraform//backend.tf is COMMITTED (a 3-line stub that only +# declares `backend "s3" {}` — no bucket name, no account ID, no +# environment-specific values). +# - terraform/backend.hcl holds the operator-specific shared values +# (bucket, region, encrypt, use_lockfile) and is gitignored, so each +# operator can point at their own state bucket (or a different +# backend entirely) without forking the repo. +# - The per-module `key` is supplied at `terraform init` time by +# scripts/tf.sh. +# The single shared backend config — operator-specific, never committed. +terraform/backend.hcl + +# Per-module backend overlays remain ignored if anyone hand-rolls them. +terraform/**/backend.auto.tfvars +terraform/**/backend.auto.tfvars.json +terraform/**/backend.auto.hcl + +# bootstrap-tfstate/ is operator-local — its own state and tfvars never leave +# the workstation. +terraform/bootstrap-tfstate/ + +# ============================================================================= +# Rendered Kubernetes manifests +# ============================================================================= +# Output of `scripts/tf.sh eks-workloads apply`. Every file under this +# directory is generated from `terraform/eks-workloads/templates/*.tftpl` +# by `local_file` resources, so it's treated the same as Terraform state: +# never committed, always re-derivable from a fresh apply. +deploy/aws/.rendered/ + diff --git a/Dockerfile b/Dockerfile index 910d7b7..706b672 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,8 +7,8 @@ # directly on a container platform — see docs/ARCHITECTURE.md) # * `agentcore configure --build-context .` (terraform/agentcore-runtime/) # -# Lands the M3 / M5.5 agent runtime Dockerfile so `make smoke` exercises -# end-to-end behavior instead of just plumbing. +# Builds the agent runtime image so `make smoke` exercises end-to-end +# behavior instead of just plumbing. # # Build: # scripts/build_local_image.sh @@ -29,15 +29,16 @@ # ============================================================================= # Following Qodo rule: Pin dependency versions in manifests and lockfiles (WARNING). -# `python:3.12.7-slim-bookworm` is a specific patch-version pin (not the -# floating `:3.12-slim` tag). Operators promoting the image to staging / -# prod SHOULD additionally pin to the resolved digest: -# FROM python:3.12.7-slim-bookworm@sha256: +# `python:3.13-slim-bookworm` is the floating minor-version tag for the +# supported Python baseline (CI matrix runs 3.12 / 3.13 / 3.14; the runtime +# image standardises on the middle row). Operators promoting the image +# to staging / prod SHOULD additionally pin to the resolved digest: +# FROM python:3.13-slim-bookworm@sha256: # The digest is captured into the audit-log line emitted by # scripts/build_local_image.sh on every local build so the promotion # pipeline can lift it verbatim. # hadolint ignore=DL3007 -FROM python:3.12.7-slim-bookworm AS builder +FROM public.ecr.aws/docker/library/python:3.13-slim-bookworm AS builder LABEL org.opencontainers.image.title="aws-agent-core-builder" LABEL org.opencontainers.image.description="Build stage for the agent runtime image" @@ -80,9 +81,9 @@ COPY src/ ./src/ # operator-side wiring. ARG BUILD_INCLUDES_DEV=0 RUN if [ "${BUILD_INCLUDES_DEV}" = "1" ]; then \ - pip install --prefix=/install ".[anthropic,xray,mcp,otel,dev]" ; \ + pip install --prefix=/install ".[anthropic,aws,xray,mcp,otel,dev]" ; \ else \ - pip install --prefix=/install ".[anthropic,xray,mcp,otel]" ; \ + pip install --prefix=/install ".[anthropic,aws,xray,mcp,otel]" ; \ fi # ============================================================================= @@ -90,7 +91,7 @@ RUN if [ "${BUILD_INCLUDES_DEV}" = "1" ]; then \ # ============================================================================= # hadolint ignore=DL3007 -FROM python:3.12.7-slim-bookworm AS runtime +FROM public.ecr.aws/docker/library/python:3.13-slim-bookworm AS runtime LABEL org.opencontainers.image.title="aws-agent-core-runtime" LABEL org.opencontainers.image.description="bedrock-agentcore SDK + LangGraph agent runtime" diff --git a/Makefile b/Makefile index b960bd3..83dbc1d 100644 --- a/Makefile +++ b/Makefile @@ -36,8 +36,9 @@ else endif .PHONY: help build-mcp build-agent local-up local-up-watch local-down local-logs \ - port-forward seed-localstack smoke smoke-recursion jaeger tunnel-smee \ - test lint typecheck import-lint ci + port-forward seed-localstack seed-schemas-check smoke smoke-recursion \ + jaeger test lint typecheck import-lint ci \ + langgraph-dev graph-png .DEFAULT_GOAL := help @@ -49,36 +50,41 @@ help: ## Print this help table. @printf " \033[36m%-20s\033[0m %s\n" "local-up" "kubectl apply -k $(KUSTOMIZE_BASE)" @printf " \033[36m%-20s\033[0m %s\n" "local-up-watch" "Apply host-mount overlay with hot-reload watcher" @printf " \033[36m%-20s\033[0m %s\n" "local-down" "kubectl delete -k $(KUSTOMIZE_BASE)" - @printf " \033[36m%-20s\033[0m %s\n" "local-logs" "Follow agent-runtime Pod logs" - @printf " \033[36m%-20s\033[0m %s\n" "port-forward" "Expose agent-runtime on http://localhost:8080" + @printf " \033[36m%-20s\033[0m %s\n" "local-logs" "Follow agent-worker Pod logs" + @printf " \033[36m%-20s\033[0m %s\n" "port-forward" "Forward LocalStack:4566 so make smoke can publish onto sqs://agent-work" @printf " \033[36m%-20s\033[0m %s\n" "seed-localstack" "Create Dynamo tables, S3 bucket, SQS queue (idempotent)" - @printf " \033[36m%-20s\033[0m %s\n" "smoke" "POST comment_created_human.json, expect status=processed" - @printf " \033[36m%-20s\033[0m %s\n" "smoke-recursion" "POST comment_created_bot.json, expect status=skipped reason=agent_account_id" + @printf " \033[36m%-20s\033[0m %s\n" "smoke" "Publish comment_created_human.json onto sqs://agent-work (LocalStack)" + @printf " \033[36m%-20s\033[0m %s\n" "smoke-recursion" "Publish comment_created_bot.json onto sqs://agent-work (worker proves recursion skip)" @printf " \033[36m%-20s\033[0m %s\n" "jaeger" "Port-forward jaeger-ui and open http://localhost:16686" - @printf " \033[36m%-20s\033[0m %s\n" "tunnel-smee" "Print the smee/ngrok tunnel invocation (not auto-run)" @printf "\n CI-parity quality gates:\n" @printf " \033[36m%-20s\033[0m %s\n" "test" "pytest (mirrors --cov-fail-under=100 from pyproject.toml)" @printf " \033[36m%-20s\033[0m %s\n" "lint" "ruff check src tests" @printf " \033[36m%-20s\033[0m %s\n" "typecheck" "mypy --strict src/agent" @printf " \033[36m%-20s\033[0m %s\n" "import-lint" "lint-imports (sealed-test-seams architecture contract)" @printf " \033[36m%-20s\033[0m %s\n" "ci" "lint + typecheck + import-lint + test (the gates CI runs)" + @printf "\n Graph debugging (LangGraph dev server):\n" + @printf " \033[36m%-20s\033[0m %s\n" "langgraph-dev" "Serve the compiled graph at http://127.0.0.1:2024 (local LangGraph dev server, no hosted UI bundled)" + @printf " \033[36m%-20s\033[0m %s\n" "graph-png" "Render the compiled topology as docs/graph.png" @printf "\n Diagnostics:\n" @printf " \033[36m%-20s\033[0m %s\n" "doctor" "Probe local-stack prerequisites (k8s context, images, namespace, deployments)" @printf "\nSee deploy/local/README.md for the full developer workflow.\n" + @printf "See docs/PYTHON_DEVELOPMENT.md \xc2\xa75.5 for the langgraph dev walkthrough.\n" # build-mcp wraps scripts/build_mcp_image.sh so the local mcp-internal -# image is reproducible from the JAR tracked by Git LFS in mcp/. Run -# before `make local-up` on first checkout, and whenever the JAR is +# image is reproducible. The Spring Boot fat-JAR lives outside this +# repo and is resolved by the build script (--jar / MCP_INTERNAL_JAR / +# ${MCP_INTERNAL_REPO}/build/libs — see mcp/README.md §1). Run before +# `make local-up` on first checkout, and whenever the upstream JAR is # bumped. The frozen target list is frozen against renames only; this # target is additive. -build-mcp: ## Build mcp-internal/server:local from mcp/mcp-internal-*.jar. +build-mcp: ## Build mcp-internal/server:local (resolves the JAR from outside this repo; see mcp/README.md). @scripts/build_mcp_image.sh # build-agent wraps scripts/build_local_image.sh so the agent runtime -# image consumed by deploy/local/agent-runtime.yaml is reproducible from +# image consumed by deploy/local/agent-worker.yaml is reproducible from # the repo root. This is the prerequisite that promotes `make smoke` # from plumbing-only to true end-to-end. -build-agent: ## Build aws-agent-core/agent-runtime:local from /Dockerfile. +build-agent: ## Build the agent worker image from /Dockerfile. @scripts/build_local_image.sh local-up: ## Create namespace + all Deployments/Services from deploy/local/. @@ -111,37 +117,65 @@ local-up-watch: ## Apply local stack with the host-mount hot-reload overlay. local-down: ## Delete everything applied by `make local-up` (namespace-scoped). kubectl delete -k $(KUSTOMIZE_BASE) -local-logs: ## Tail the agent-runtime Pod logs. - kubectl logs -f -n $(NAMESPACE) deploy/agent-runtime +local-logs: ## Tail the agent-worker Pod logs (the local stack has no HTTP entrypoint anymore). + kubectl logs -f -n $(NAMESPACE) deploy/agent-worker -port-forward: ## Forward agent-runtime:8080 so http://localhost:8080 reaches /invocations. - kubectl port-forward -n $(NAMESPACE) svc/agent-runtime 8080:8080 +port-forward: ## Forward LocalStack so `make smoke` (sqs://agent-work) reaches the in-cluster queue. + @# HMAC verification is performed by the webhook-validator Lambda + @# (terraform/webhook-validator/); the local stack has no HTTP + @# entrypoint -- the agent-worker Pod consumes straight from + @# LocalStack SQS. Forward the LocalStack edge service so smoke.py + @# can publish onto sqs://agent-work from the developer's host. + kubectl port-forward -n $(NAMESPACE) svc/localstack 4566:4566 # `seed-localstack` is required to be idempotent. Each resource creation # is guarded with `|| true` so re-runs (after `make local-down` followed # by `make local-up`) succeed without manual cleanup. +# +# Schemas below MUST match the DynamoDB adapter contracts in +# src/agent/infrastructure/dynamodb/ and src/agent/infrastructure/multitenant/. +# scripts/check_seed_schemas.py asserts this in CI. +# +# Canonical contracts: +# domain-state : HASH tenant_id (S) + RANGE issue_key (S) -- DynamoDbDomainStateStore +# tenants : HASH cloud_id (S) -- DynamoDbPrefixTenantResolver +# token-budgets : HASH pk (S), TTL expires_at -- DynamoDbTokenBudgetEnforcer +# breaker : HASH pk (S), TTL expires_at -- DynamoDbBreakerStateStore seed-localstack: ## Create DynamoDB tables, S3 bucket (versioned), and SQS DLQ on LocalStack. - @echo ">> seed: DynamoDB idempotency" + @echo ">> seed: DynamoDB domain-state (HASH tenant_id, RANGE issue_key)" + @$(AWS_CMD) dynamodb create-table \ + --table-name domain-state \ + --attribute-definitions \ + AttributeName=tenant_id,AttributeType=S \ + AttributeName=issue_key,AttributeType=S \ + --key-schema \ + AttributeName=tenant_id,KeyType=HASH \ + AttributeName=issue_key,KeyType=RANGE \ + --billing-mode PAY_PER_REQUEST 2>/dev/null || true + @echo ">> seed: DynamoDB tenants (HASH cloud_id)" + @$(AWS_CMD) dynamodb create-table \ + --table-name tenants \ + --attribute-definitions AttributeName=cloud_id,AttributeType=S \ + --key-schema AttributeName=cloud_id,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST 2>/dev/null || true + @echo ">> seed: DynamoDB token-budgets (HASH pk, TTL expires_at)" @$(AWS_CMD) dynamodb create-table \ - --table-name idempotency \ + --table-name token-budgets \ --attribute-definitions AttributeName=pk,AttributeType=S \ --key-schema AttributeName=pk,KeyType=HASH \ --billing-mode PAY_PER_REQUEST 2>/dev/null || true @$(AWS_CMD) dynamodb update-time-to-live \ - --table-name idempotency \ - --time-to-live-specification Enabled=true,AttributeName=ttl 2>/dev/null || true - @echo ">> seed: DynamoDB domain-state" + --table-name token-budgets \ + --time-to-live-specification Enabled=true,AttributeName=expires_at 2>/dev/null || true + @echo ">> seed: DynamoDB breaker (HASH pk, TTL expires_at)" @$(AWS_CMD) dynamodb create-table \ - --table-name domain-state \ - --attribute-definitions AttributeName=thread_id,AttributeType=S \ - --key-schema AttributeName=thread_id,KeyType=HASH \ - --billing-mode PAY_PER_REQUEST 2>/dev/null || true - @echo ">> seed: DynamoDB tenants" - @$(AWS_CMD) dynamodb create-table \ - --table-name tenants \ - --attribute-definitions AttributeName=tenant_id,AttributeType=S \ - --key-schema AttributeName=tenant_id,KeyType=HASH \ + --table-name breaker \ + --attribute-definitions AttributeName=pk,AttributeType=S \ + --key-schema AttributeName=pk,KeyType=HASH \ --billing-mode PAY_PER_REQUEST 2>/dev/null || true + @$(AWS_CMD) dynamodb update-time-to-live \ + --table-name breaker \ + --time-to-live-specification Enabled=true,AttributeName=expires_at 2>/dev/null || true @echo ">> seed: S3 raw-payloads (versioned)" @$(AWS_CMD) s3api create-bucket --bucket raw-payloads 2>/dev/null || true @$(AWS_CMD) s3api put-bucket-versioning \ @@ -149,32 +183,65 @@ seed-localstack: ## Create DynamoDB tables, S3 bucket (versioned), and SQS DLQ o --versioning-configuration Status=Enabled 2>/dev/null || true @echo ">> seed: SQS agent-dlq" @$(AWS_CMD) sqs create-queue --queue-name agent-dlq 2>/dev/null || true + @echo ">> seed: SQS agent-work (async-dispatch work queue)" + @# Async-dispatch: the webhook entrypoint enqueues validated + @# InvokeEnvelope payloads onto agent-work; the agent-worker Pod + @# long-polls and runs the LangGraph agent loop out-of-band. + @# + @# Visibility timeout: 600s -- caps the worker's per-message runtime + @# budget. A worker that exceeds 600s hands the message back to SQS + @# for redelivery to another replica, which re-runs the same envelope. + @# Idempotency at the graph layer keeps re-runs safe. + @# + @# Long-poll (ReceiveMessageWaitTimeSeconds=20s) keeps the consumer + @# from busy-spinning when the queue is empty. + @# + @# Retention: 4 days (default) is enough for an operator to drain a + @# stuck worker; longer retention has no operational benefit because + @# the DLQ owns the dead-letter forensics path. + @$(AWS_CMD) sqs create-queue \ + --queue-name agent-work \ + --attributes 'VisibilityTimeout=600,ReceiveMessageWaitTimeSeconds=20,MessageRetentionPeriod=345600' \ + 2>/dev/null || true + @# Wire the redrive policy: agent-work -> agent-dlq after + @# maxReceiveCount=3 attempts. We resolve the agent-dlq ARN at + @# runtime so the policy is portable across LocalStack endpoints + @# without hard-coding the AWS account id. The shell snippet + @# tolerates the ARN field name varying across LocalStack versions. + @DLQ_ARN="$$($(AWS_CMD) sqs get-queue-attributes \ + --queue-url $$($(AWS_CMD) sqs get-queue-url --queue-name agent-dlq --query QueueUrl --output text) \ + --attribute-names QueueArn --query 'Attributes.QueueArn' --output text)"; \ + WORK_URL="$$($(AWS_CMD) sqs get-queue-url --queue-name agent-work --query QueueUrl --output text)"; \ + $(AWS_CMD) sqs set-queue-attributes \ + --queue-url "$$WORK_URL" \ + --attributes "{\"RedrivePolicy\":\"{\\\"deadLetterTargetArn\\\":\\\"$$DLQ_ARN\\\",\\\"maxReceiveCount\\\":\\\"3\\\"}\"}" \ + 2>/dev/null || true @echo ">> seed: complete" -smoke: ## Happy-path smoke test -- expects status=processed. - python scripts/smoke.py --fixture $(SMOKE_HAPPY_FIXTURE) --expect-status processed +# `seed-schemas-check` is the local-developer counterpart of the CI +# `seed-schema-contract` job: it asserts that the LocalStack tables +# created by `seed-localstack` match the adapter contracts under +# src/agent/infrastructure/. See scripts/check_seed_schemas.py. +seed-schemas-check: ## Assert seed-localstack schemas match adapter contracts (CI parity). + @AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \ + AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \ + AWS_DEFAULT_REGION=$(AWS_REGION) \ + LOCALSTACK_ENDPOINT=$(LOCALSTACK_ENDPOINT) \ + python scripts/check_seed_schemas.py -smoke-recursion: ## Recursion-guard smoke test -- expects status=skipped, reason=agent_account_id. - python scripts/smoke.py --fixture $(SMOKE_RECURSION_FIXTURE) --expect-status skipped --expect-reason agent_account_id +smoke: ## Happy-path smoke -- publish a human-actor fixture onto LocalStack SQS. + python scripts/smoke.py --fixture $(SMOKE_HAPPY_FIXTURE) --target sqs://agent-work + +smoke-recursion: ## Recursion-guard smoke -- publish a bot-actor fixture onto LocalStack SQS. + @# The recursion guard runs in the agent-worker Pod after dequeue, + @# so this target only proves the publish leg succeeds; tail the + @# worker logs (kubectl logs -f deploy/agent-worker | grep $(SMOKE_RECURSION_FIXTURE)) + @# to see the recursion-skip verdict. + python scripts/smoke.py --fixture $(SMOKE_RECURSION_FIXTURE) --target sqs://agent-work jaeger: ## Port-forward jaeger-ui and open the trace UI in the default browser. kubectl port-forward -n $(NAMESPACE) svc/jaeger-ui 16686:16686 & open http://localhost:16686 || xdg-open http://localhost:16686 || true -# tunnel-smee is DOCUMENTATION-ONLY -- it must NOT run the CLI -# automatically (that would require the developer's smee channel URL -# and network egress from CI). -tunnel-smee: ## Print the smee / ngrok invocation for Jira webhook replay. - @echo "" - @echo "Smee.io tunnel (register a channel at https://smee.io first):" - @echo " smee --url https://smee.io/ \\" - @echo " --target http://localhost:8080/webhooks/jira" - @echo "" - @echo "ngrok alternative:" - @echo " ngrok http 8080" - @echo "" - @echo "Atlassian webhook secret must match WEBHOOK_HMAC_SECRET in .env.local." - @echo "See deploy/local/README.md for the full walkthrough." - # ============================================================================= # CI-parity quality gates. # @@ -195,7 +262,7 @@ lint: ## Run ruff over src/ and tests/ (settings in pyproject.toml). typecheck: ## Run mypy --strict against src/agent (settings in pyproject.toml). mypy src/agent -# Wave 4 sealed-test-seams contract. Enforces that production code +# Sealed-test-seams contract. Enforces that production code # in the ``agent`` package does not import from # ``agent.composition._test_seams`` (settings in # ``[tool.importlinter]`` in pyproject.toml). @@ -206,6 +273,42 @@ ci: lint typecheck import-lint test ## Run lint, typecheck, import-lint, and tes @echo "" @echo "make ci: all gates passed." +# ============================================================================= +# LangGraph dev server (debug bundle). +# +# `langgraph-dev` boots the local LangGraph CLI dev server, which serves +# the compiled `agent.composition.studio:build_studio_graph` topology on +# http://127.0.0.1:2024. The dev server runs locally and does not bundle +# a hosted UI -- we deliberately do not point at `smith.langchain.com` +# because that surface is part of the LangSmith product. For a static +# topology snapshot use `make graph-png`; check `langgraph dev --help` +# for any locally-hosted Studio flag your installed `langgraph-cli` +# version may expose. +# +# Prereq: `pip install -e ".[dev,anthropic,debug]"` (the `[debug]` +# extra ships `langgraph-cli[inmem]`). The dev server reads `.env.local` +# automatically (per `langgraph.json:env`); copy `.env.local.example` +# to `.env.local` and fill in `ANTHROPIC_API_KEY` first. +# +# `graph-png` renders the compiled graph topology to `docs/graph.png` +# via the same `build_studio_graph` factory. Useful for design reviews +# that want a static snapshot of the assessor/designer branches. +# ============================================================================= + +langgraph-dev: ## Serve the compiled graph at http://127.0.0.1:2024 (local LangGraph dev server). + @command -v langgraph >/dev/null 2>&1 || { \ + echo "langgraph CLI not found. Install the [debug] extra:"; \ + echo " pip install -e \".[dev,anthropic,debug]\""; \ + exit 1; \ + } + langgraph dev --port 2024 + +graph-png: ## Render the compiled topology to docs/graph.png. + @python -c 'from agent.composition.studio import build_studio_graph; \ + g = build_studio_graph(); \ + open("docs/graph.png", "wb").write(g.get_graph().draw_mermaid_png())' + @echo "Wrote docs/graph.png" + # ============================================================================= # `doctor` -- local-stack environmental health check. # @@ -271,7 +374,7 @@ doctor: ## Probe local-stack prerequisites (k8s context, images, namespace, depl else \ bad "namespace $(NAMESPACE) is missing -- run: make local-up"; \ fi; \ - for deploy in agent-runtime mcp-internal localstack; do \ + for deploy in agent-worker mcp-internal localstack; do \ if kubectl get deployment -n $(NAMESPACE) $$deploy >/dev/null 2>&1; then \ ok "deployment $$deploy exists in $(NAMESPACE)"; \ else \ diff --git a/README.md b/README.md index 3ad992b..7701f11 100644 --- a/README.md +++ b/README.md @@ -35,10 +35,29 @@ by the production-only Terraform modules and scripts under > `with_structured_output(ImplementationPlan)`). Budget the AWS > profile's per-tenant token cap accordingly: a single > issue-updated webhook that triggers the design branch can -> consume up to four Bedrock invocations end-to-end. The Wave 5 -> Bedrock Guardrails run on every one of those invocations so a +> consume up to four Bedrock invocations end-to-end. Bedrock +> Guardrails run on every one of those invocations so a > guardrail block fails fast at step 3 of the seven-step > governance ordering ([§Token governance](docs/ARCHITECTURE.md#token-governance)). +> +> **Human-in-the-loop (HITL) approval gate.** A human reviewer +> always gates the assessor → designer handoff. When the assessor +> decides `ready`, the graph routes to the deterministic +> `request_human_approval` node (no LLM call) and returns +> `InvokeStatus="awaiting_human_approval"` after persisting +> `readiness_phase="awaiting_human_approval"` to durable state. +> The assessor's tool loop has already posted the approval-ask +> Jira comment via `jira_add_comment`. The next webhook on that +> ticket carries the human's reply; the cheap `approval_llm` +> (Haiku-class — `APPROVAL_BEDROCK_MODEL_ID` / +> `APPROVAL_LLM_MODEL_ID`) drives +> `with_structured_output(ApprovalEvaluation)` over the comment +> body and either releases to the designer (with an optional +> `bind_tools(jira_add_comment)` override comment if the assessor +> regressed) or routes back through the refinement loop. See +> [§Multi-agent extension points](docs/ARCHITECTURE.md#multi-agent-extension-points) +> for the routing-predicate contract and +> [`docs/runbook.md` §12](docs/runbook.md) for HITL incident response. ## Architecture at a glance @@ -53,21 +72,21 @@ Python codebase. - **Ports** — interfaces only; the dependency-inversion seam every Adapter binds to. Reference impl: [`src/agent/contracts/`](src/agent/contracts). - **Application Services** — pure use cases (webhook handler, normalizer, - recursion guard, idempotency, assessor, runtime, comment renderer). + recursion guard, and token governance). Reference impl: [`src/agent/application/`](src/agent/application). - **Adapters** — concrete implementations of every Port (cloud SDK clients, MCP HTTP client, LLM provider client, telemetry SDK, persistent stores, messaging, ...). Reference impl: [`src/agent/infrastructure/`](src/agent/infrastructure). - **Composition Root** — the only place provider SDKs are named, one factory - per profile. Two factories ship today: `build_default_dependencies(...)` - (in-memory; tests only) and `build_local_dependencies(...)` (LocalStack + - Anthropic; the local profile). - `build_aws_dependencies(...)` lands in a follow-up milestone. Reference - impl: [`src/agent/composition.py`](src/agent/composition.py). + per profile. Three factories ship today: `build_default_dependencies(...)` + (in-memory; tests only), `build_local_dependencies(...)` (LocalStack + + Anthropic or Ollama), and `build_aws_dependencies(...)` (Bedrock + + AgentCore-managed services). Reference impl: + [`src/agent/composition/`](src/agent/composition). - **Entrypoint Wrapper** — env-var-driven module that selects the profile - and starts the runtime. `AGENT_PROFILE=local` selects the local factory; - `AGENT_PROFILE=aws` is reserved for a follow-up milestone. Reference + and starts the runtime. `AGENT_PROFILE=aws` (the default) selects the + AWS factory; `AGENT_PROFILE=local` selects the local factory. Reference impl: [`src/agent/main.py`](src/agent/main.py). High-level system context — one signed Jira webhook becomes one posted @@ -78,32 +97,52 @@ change. ```mermaid flowchart LR jira["Jira Cloud webhook"] - tunnel["Ingress
(local: tunnel; AWS: API Gateway / ALB)"] - entry["Runtime Entrypoint
(/invocations)"] - handler["Webhook Handler
signature -> normalize ->
recursion guard -> idempotency"] + tunnel["Ingress
(local: tunnel; AWS: ALB)"] + entry["webhook-validator Lambda
(/invocations)"] + handler["Webhook Handler.prevalidate
identity -> normalize ->
tenant -> recursion guard"] workflow["Agent Workflow Graph
(checkpointed by thread_id)"] runtime["Agent Use Case
fetch -> assess -> comment -> persist"] govern["Governed LLM Decorator
budget + breaker + cost + observer"] mcp["mcp-internal MCP server
(streamable HTTP + bearer)"] llm["LLM Provider"] - dynamo[("Persistent stores
idempotency / domain-state /
tenants / token-budgets")] + dynamo[("Persistent stores
domain-state / tenants /
token-budgets")] s3[("Raw payload archive")] - sqs[("Dead-letter queue")] + dlq[("Dead-letter queue
(agent-dlq)")] obs["Observability sinks
(traces / metrics / logs)"] jiraApi["Jira REST API"] - jira --> tunnel --> entry --> handler --> workflow --> runtime + jira --> tunnel --> entry --> handler + handler -.->|"200 status=accepted
(or skipped/error)"| jira + handler -->|"InvokeAgentRuntime"| workflow --> runtime runtime -->|"MCP tools"| mcp --> jiraApi runtime --> govern --> llm handler -.-> dynamo runtime -.-> dynamo handler -.-> s3 - handler -.-> sqs handler -.-> obs runtime -.-> obs govern -.-> obs ``` +> **AWS ingress path.** The ALB routes `/invocations` to the +> `webhook-validator` Lambda, which verifies HMAC and calls +> `bedrock-agentcore:InvokeAgentRuntime` directly. This keeps AgentCore +> sessions/traces authoritative while preserving the existing signed +> webhook contract at ingress. +> +> **No webhook-layer idempotency dedupe — by design.** The contract +> documented in +> [`src/agent/application/webhook_handler.py`](src/agent/application/webhook_handler.py) +> (module docstring) deliberately omits an `IdempotencyStore`: the +> graph refetches live Jira state via MCP on every webhook, the +> recursion guard (`actor.account_id == agent`) short-circuits the +> bot's own writes, and the `persist_approval_granted` checkpoint +> covers designer-crash resumption by writing +> `readiness_phase="awaiting_design"` *before* the designer subgraph +> runs. Duplicate webhook delivery is therefore safe: a retry sees +> the same live state and the same persisted phase, and converges +> on the same response without a dedicated dedupe table. + > **Adapter boundary rule.** Vendor SDKs and network libraries (cloud > SDK, LLM provider client, HTTP client, telemetry SDK) are imported > **only** from the Adapter and Composition modules — never from @@ -116,7 +155,7 @@ flowchart LR > `langchain_aws`, `langchain_anthropic`, `bedrock_agentcore`, `httpx`, > and `opentelemetry` are only imported inside > [`src/agent/infrastructure/`](src/agent/infrastructure) and -> [`src/agent/composition.py`](src/agent/composition.py)); a JVM +> [`src/agent/composition/`](src/agent/composition)); a JVM > implementation would enforce it with ArchUnit, .NET with NetArchTest, > Node with dependency-cruiser, etc. @@ -143,7 +182,7 @@ diagrams live in [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md): Sequence diagram for the `make smoke` path: Signature Verifier → Event Normalizer → Recursion Guard → Idempotency Service → Agent Workflow Graph (`assess` node) → Jira MCP Adapter - (`jira.get_issue`) → Governed LLM Decorator → Comment Renderer → + (`jira.get_issue`) → Governed LLM Decorator → LLM-authored MCP write → Jira MCP Adapter (`jira.add_comment`) → Domain State Store, with every span name, metric, and log line annotated. - **Token governance + observability detail** — @@ -164,7 +203,7 @@ for the full local-vs-AWS contrast). | Profile | Target environment | LLM | When to use | |---------|--------------------|-----|-------------| -| **Local development** | Docker Desktop Kubernetes + LocalStack + sibling `mcp-internal` Pod | `langchain_anthropic.ChatAnthropic` with `ANTHROPIC_API_KEY` | Fast iteration, hot-reload, deterministic unit tests. AWS credentials are not required (the SDK is provider-agnostic). | +| **Local development** | Docker Desktop Kubernetes + LocalStack + sibling `mcp-internal` Pod | `langchain_anthropic.ChatAnthropic` with `ANTHROPIC_API_KEY` or `langchain_ollama.ChatOllama` via `LLM_PROVIDER=ollama` | Fast iteration, hot-reload, deterministic unit tests. AWS credentials are not required; Ollama also removes paid LLM credentials from the local path. | | **Self-managed EKS / ECS / Fargate** | Customer-owned container platform in AWS | `langchain_aws.ChatBedrock` against Amazon Bedrock | Operator owns the platform; the SDK process is the only AWS-managed dependency. | | **Enterprise AgentCore Runtime** | Managed Amazon Bedrock AgentCore Runtime | `langchain_aws.ChatBedrock` + AgentCore Memory + AgentCore Identity + AgentCore Observability | Enterprise runtime, governance, autoscaling, managed Memory / Identity / Observability. Promotion is composition-root + IaC only. | @@ -178,26 +217,23 @@ version, virtualenv, optional extras) are covered in - **Docker Desktop** with the built-in single-node **Kubernetes** cluster enabled. - **`kubectl`** (`brew install kubernetes-cli` on macOS). -- **Git LFS** — the `mcp-internal` Spring Boot fat-JAR is tracked in - LFS via [`.gitattributes`](.gitattributes). Install with - `brew install git-lfs` (or `apt-get install git-lfs` on Linux), then - run `git lfs install` and `git lfs pull`. +- **`mcp-internal` Spring Boot fat-JAR** (external) — the JAR is no + longer tracked in this repository. Build it once in the upstream + Gradle project and `scripts/build_mcp_image.sh` will resolve it + via `--jar` / `MCP_INTERNAL_JAR` / + `${MCP_INTERNAL_REPO:-$HOME/code/github/mcp-internal}/build/libs`. + Full contract in [`mcp/README.md`](mcp/README.md) §1. - **`awslocal`** CLI (`pipx install awscli-local`) **or** the standard `aws` CLI configured with dummy credentials and `--endpoint-url=http://localhost:4566`. -- **Anthropic API key** (`sk-ant-…`) — get one from - . The agent runtime needs it at - request time even when only the infrastructure is up (the - `mcp-internal` Pod is bound to `JIRA_*` credentials, not the LLM). -- **Local LLM extras.** The local profile's - `langchain_anthropic.ChatAnthropic` factory is gated behind the - `[anthropic]` optional extras block in - [`pyproject.toml`](pyproject.toml). The Pod image already bakes - this in via [`Dockerfile`](Dockerfile); engineers running the - bare-metal `python -m agent.main` path must install it explicitly: - `pip install -e ".[dev,anthropic]"`. Production AWS images **must - not** install `[anthropic]` — `[aws]` covers the Bedrock + AgentCore - Memory pins instead. -- *Optional:* `smee` CLI or `ngrok` for tunneling real Jira Cloud webhooks (see [`deploy/local/README.md`](deploy/local/README.md)). +- **LLM provider.** The local profile defaults to Anthropic + (`LLM_PROVIDER=anthropic`, `ANTHROPIC_API_KEY=sk-ant-…`). For a + no-paid-key laptop path, run Ollama and set `LLM_PROVIDER=ollama`, + `OLLAMA_BASE_URL`, and `OLLAMA_MODEL`. +- **Local LLM extras.** Anthropic uses `pip install -e ".[dev,anthropic]"`; + Ollama uses `pip install -e ".[dev,ollama]"`. The Pod image can install + either extra depending on the local profile being exercised. Production + AWS images **must not** install provider extras they do not use — `[aws]` + covers the Bedrock + AgentCore Memory pins instead. ## Infrastructure quickstart @@ -211,7 +247,11 @@ truth). ```bash git clone cd aws-agent-core -git lfs install && git lfs pull # materialises mcp/mcp-internal-*.jar + +# Build the upstream mcp-internal Spring Boot fat-JAR (one-time, then on every bump). +# See mcp/README.md "External JAR contract" for resolution rules and overrides. +git clone "${HOME}/code/github/mcp-internal" +( cd "${HOME}/code/github/mcp-internal" && ./gradlew bootJar ) make build-mcp # build mcp-internal/server:local from mcp/Dockerfile make build-agent # build aws-agent-core/agent-runtime:local from /Dockerfile @@ -224,26 +264,28 @@ make doctor # health-check: kubectl context, images, | Target | Purpose | |--------|---------| -| `make build-mcp` | Build the `mcp-internal/server:local` image from `mcp/Dockerfile` (wraps `scripts/build_mcp_image.sh`; resolves `mcp/mcp-internal-*.jar`). | +| `make build-mcp` | Build the `mcp-internal/server:local` image from `mcp/Dockerfile` (wraps `scripts/build_mcp_image.sh`; resolves the JAR from outside this repo via `--jar` / `MCP_INTERNAL_JAR` / `${MCP_INTERNAL_REPO}/build/libs` — see [`mcp/README.md`](mcp/README.md)). | | `make build-agent` | Build the `aws-agent-core/agent-runtime:local` image from the repo-root [`Dockerfile`](Dockerfile) (wraps `scripts/build_local_image.sh`). | | `make local-up` | `kubectl apply -k deploy/local/` (creates namespace, Secrets, all Pods/Services). | | `make local-down` | `kubectl delete -k deploy/local/` (cluster-scoped resources stay). | | `make local-logs` | `kubectl logs -f -n aws-agent-core-local deploy/agent-runtime`. | | `make port-forward` | `kubectl port-forward svc/agent-runtime 8080:8080`. | -| `make seed-localstack` | Creates `idempotency`, `domain-state`, `tenants` DynamoDB tables, the `raw-payloads` S3 bucket (versioned), and the `agent-dlq` SQS queue. Idempotent. | +| `make seed-localstack` | Creates `domain-state`, `tenants` DynamoDB tables, the `raw-payloads` S3 bucket (versioned), and the `agent-dlq` SQS queue. Idempotent. | | `make jaeger` | Port-forwards `jaeger-ui` and opens . | | `make doctor` | Probes local-stack prerequisites: `kubectl` context, locally-built images, namespace, and the three core Deployments. | -| `make tunnel-smee` | Documentation only: prints the smee / ngrok invocation. | +| `make langgraph-dev` | Serves the compiled assessor / designer graph at `http://127.0.0.1:2024` for browser-based debugging via [LangGraph Studio](docs/PYTHON_DEVELOPMENT.md#55-langgraph-studio-graph-level-debugging). Requires `pip install -e ".[dev,anthropic,debug]"`. | +| `make graph-png` | Renders the compiled topology to `docs/graph.png` (handy for design reviews). | -For the smee.io / ngrok playbook and the host-mount kustomize overlay, -see [`deploy/local/README.md`](deploy/local/README.md). +For the host-mount kustomize overlay, see +[`deploy/local/README.md`](deploy/local/README.md). Once the infrastructure quickstart is green, see [`docs/PYTHON_DEVELOPMENT.md`](docs/PYTHON_DEVELOPMENT.md) to install the Python project, run smoke tests (`make smoke`, `make smoke-recursion`) and quality gates (`make ci`), and iterate against this infrastructure (including the `make local-up-watch` -hot-reload loop and the bare-metal `python -m agent.main` path). +hot-reload loop, the bare-metal `python -m agent.main` path, and the +LangGraph Studio debugger via `make langgraph-dev`). ## Environment variables (frozen contract) @@ -257,16 +299,22 @@ below. |---------|---------------|-------------| | `AGENT_PROFILE` | `local` | [`agent.main`](src/agent/main.py) selects `build_local_dependencies` vs `build_aws_dependencies`. | | `AGENT_BUDGET_SCOPE` | required when `AGENT_PROFILE=local` (set to `local-pod-dev`) | Single-pod-dev acknowledgement gate in [`agent.main._resolve_local_dependencies`](src/agent/main.py). The local profile wires `InMemoryTokenBudgetEnforcer` (per-pod heap counters); without `AGENT_BUDGET_SCOPE=local-pod-dev` startup raises a `ValueError` so a misconfigured multi-replica staging deploy fails loud rather than silently over-permitting budgets. The AWS profile ignores this var (`DynamoDbTokenBudgetEnforcer` is cluster-shared). | -| `ANTHROPIC_API_KEY` | required (`sk-ant-…`) | `anthropic_model_factory(api_key=…)` (local). | +| `LLM_PROVIDER` | `anthropic` | Local LLM selector: `anthropic` or `ollama`. | +| `ANTHROPIC_API_KEY` | required when `LLM_PROVIDER=anthropic` | `anthropic_model_factory(api_key=…)` (local). Not required for `LLM_PROVIDER=ollama`. | | `ANTHROPIC_MODEL` | `claude-sonnet-4-5-20250929` | `anthropic_model_factory(model_id=…)`. Pinned in lockstep with `BEDROCK_MODEL_ID`. | -| `MCP_BASE_URL` | `http://mcp-internal:8081/mcp` | `HttpStreamableMcpClient(base_url=…)`. | -| `MCP_BEARER_TOKEN` | literal `DAVIDSUPERSECRETTOKEN` | `HttpStreamableMcpClient(bearer_token=…)`. The same Secret value is exposed to the `mcp-internal` Pod as `MCP_AUTH_TOKEN` (the JAR's env-var name). | +| `OLLAMA_BASE_URL` | `http://ollama:11434` | `ollama_model_factory(base_url=…)` when `LLM_PROVIDER=ollama` or `DESIGN_LLM_PROVIDER=ollama`. | +| `OLLAMA_MODEL` | `llama3.1` | `ollama_model_factory(model_id=…)`. | +| `DESIGN_LLM_PROVIDER` | defaults to `LLM_PROVIDER` | Optional local override for the implementation-designer branch. | +| `APPROVAL_LLM_MODEL_ID` | unset → defaults to a Haiku-class default | Local override for the cheap `approval_llm` that drives `evaluate_human_approval`. Maps to `approval_anthropic_model_id=` / `approval_ollama_model_id=` in [`agent.composition.local`](src/agent/composition/local.py) based on `LLM_PROVIDER`. | +| `APPROVAL_MODEL_ID` | fallback for `APPROVAL_LLM_MODEL_ID` | Alternate env name for the same approval-LLM model id (read by `agent.main` if `APPROVAL_LLM_MODEL_ID` is unset). | +| `MCP_BASE_URL` | `http://mcp-internal:8081/mcp` | `agent.composition._mcp_session.open_mcp_session(url=…)` — the per-request Streamable-HTTP session URL. | +| `MCP_BEARER_TOKEN` | literal `DAVIDSUPERSECRETTOKEN` | `agent.composition._mcp_session.open_mcp_session(bearer_token_provider=…)` via the local `SecretsResolver`. The same Secret value is exposed to the `mcp-internal` Pod as `MCP_AUTH_TOKEN` (the JAR's env-var name). | | `JIRA_SITE_URL` / `JIRA_EMAIL` / `JIRA_API_TOKEN` | from `.env.local` → `jira-credentials` k8s `Secret` | Consumed by the `mcp-internal` Pod env, never by the agent runtime (single-integration-user rule). Confluence reuses the same triple inside the JAR. | | `LOCALSTACK_ENDPOINT_URL` | `http://localstack:4566` | every boto3 client factory in `infrastructure/dynamodb/`, `…/storage/`, `…/messaging/`. | | `AWS_REGION` | `us-east-1` | every boto3 client factory. | | `AGENT_JIRA_ACCOUNT_ID` | `local-bot-account-id` | `ActorIdentityRecursionGuard(agent_account_id=…)` -- skips inbound webhooks whose `actor.account_id` matches the agent. **Required.** | | `AGENT_JIRA_EMAIL` | `local-bot@example.test` | `ActorIdentityRecursionGuard(agent_email=…)` -- skips inbound webhooks whose `actor.email` matches the agent (case-insensitive). **Required.** | -| `OBSERVABILITY_BACKEND` | `jaeger` | tracer factory in `agent.composition` (`xray` lands in M5.5). | +| `OBSERVABILITY_BACKEND` | `jaeger` | tracer factory in `agent.composition`; `xray` selects the AWS X-Ray exporter under `AGENT_PROFILE=aws`. | | `TOKEN_BUDGET_INPUT_PER_HOUR` | unset (no cap) | `InMemoryTokenBudgetEnforcer` / `DynamoDbTokenBudgetEnforcer` input-token cap per rolling window. | | `TOKEN_BUDGET_OUTPUT_PER_HOUR` | unset (no cap) | Output-token cap per rolling window. | | `TOKEN_BUDGET_USD_PER_WINDOW` | unset (no cap) | USD cap per rolling window (computed via `StaticCostCalculator`). | @@ -275,29 +323,49 @@ below. | `LLM_CIRCUIT_FAIL_MAX` | `5` | `InMemoryLlmCircuitBreaker` failure threshold. | | `LLM_CIRCUIT_RESET_SECONDS` | `30` | Open-state cooldown before a half-open probe is scheduled. | -> **AWS-only rows (omitted).** `BEDROCK_MODEL_ID`, `AGENTCORE_JWT_DISCOVERY_URL`, -> and `AGENTCORE_JWT_AUDIENCE` apply only when `AGENT_PROFILE=aws`; their -> composition-root consumers land in a follow-up milestone. The -> contract is the union of [`.env.local.example`](.env.local.example), +> **AWS-only rows (omitted).** `BEDROCK_MODEL_ID`, +> `APPROVAL_BEDROCK_MODEL_ID` (cheap `approval_llm`, falls back to +> `APPROVAL_LLM_MODEL_ID` / `APPROVAL_MODEL_ID`), +> `AGENTCORE_JWT_DISCOVERY_URL`, and `AGENTCORE_JWT_AUDIENCE` apply +> only when `AGENT_PROFILE=aws`. `BEDROCK_MODEL_ID` is the single +> switch the AWS profile reads to pick the Bedrock model family — +> two families are supported today (mirrored by +> `agent.main.SUPPORTED_BEDROCK_MODEL_FAMILIES` and the SCP allow-list +> in `terraform/organizations/variables.tf`): +> **Anthropic Claude** (global inference profile, default +> `global.anthropic.claude-sonnet-4-6` — pinned in +> [`.state/bootstrap.env`](.state/bootstrap.env) and the +> `bedrock_model_id` default in +> [`terraform/eks/variables.tf`](terraform/eks/variables.tf)) and +> **DeepSeek V3.2** (in-region only, `deepseek.v3.2`). The runtime +> classifies the family via +> [`agent.main._classify_bedrock_model_family`](src/agent/main.py) at +> startup; a typo'd id fails fast. The JWT pair is consumed by +> [`agent.main._resolve_aws_dependencies`](src/agent/main.py) and +> wired into `AgentCoreJwtIdentityVerifier` inside +> [`agent.composition.aws.build_aws_dependencies`](src/agent/composition/aws.py) +> when both env vars are set. The contract is the union of +> [`.env.local.example`](.env.local.example), > [`deploy/local/configmap.yaml`](deploy/local/configmap.yaml), and the -> AWS-profile factory in -> [`src/agent/composition.py`](src/agent/composition.py); a new env var -> must not land in code without updating those files in lockstep. +> profile factories in +> [`src/agent/composition/`](src/agent/composition/) (`local.py`, +> `aws.py`, `_shared.py`); a new env var must not land in code without +> updating those files in lockstep. ## Token usage governance & cost attribution -The governance design ratified in -[`docs/adr/ADR-001-token-governance.md`](docs/adr/ADR-001-token-governance.md) -adds a deterministic governance pipeline around every LLM call. Both +The governance design (see the *Token governance* section of +[`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md#token-governance)) adds a +deterministic governance pipeline around every LLM call. Both deployment profiles run the same code path; only the enforcer adapter changes: -- **`GovernedAssessmentLanguageModel`** wraps the metered Bedrock / - Anthropic model and enforces the ordering invariant - `enforcer.check → breaker.before_call → inner.assess_metered → - cost.usd_for → enforcer.commit → observer.on_usage → - breaker.record_success` (or `breaker.record_failure(exc)` on any - inner failure). +- **`GovernedAgenticChatModel`** wraps the LangChain-backed agentic + model and enforces the ordering invariant + `enforcer.reserve → breaker.before_call → inner.invoke → + cost.usd_for → enforcer.finalize → observer.on_usage → + breaker.record_success` (or `breaker.record_failure(exc)` and + `enforcer.release(...)` on any inner failure). - **`LangChainUsageCapturer`** extracts token counts via `langchain_core.callbacks.UsageMetadataCallbackHandler`, so cache-hit / cache-miss tokens are reported correctly for both providers. @@ -325,9 +393,9 @@ changes: Transaction Search are provisioned by the new `terraform/observability-cloudwatch-genai` module. -`TicketReadinessAssessor` propagates `BudgetExceededError` and -`LlmCircuitOpenError` without retrying so a tripped breaker / saturated -budget cannot be amplified by the retry loop into double-charged spend. +The graph propagates `BudgetExceededError` and `LlmCircuitOpenError` +without retrying so a tripped breaker / saturated budget cannot be +amplified by the tool loop into double-charged spend. ## Testing and quality gates @@ -348,9 +416,9 @@ not required to follow this layout. See - **Domain** — immutable validated value objects, no I/O. - **Ports** — interfaces / protocols at every external boundary. -- **Application Services** — use cases (Webhook Handler, Agent Use - Case, Token Governance Decorator, Event Normalizer, Recursion Guard, - Idempotency Service, Comment Renderer). +- **Application Services** — use cases (Webhook Handler, Token + Governance Decorator, Event Normalizer, Recursion Guard, + Idempotency Service, Event Invariant Policy). - **Adapters** — concrete implementations of Ports against vendor SDKs. - **Composition Root** — single module wiring Adapters to Ports per profile. @@ -403,8 +471,8 @@ Promotion is composition-root + IaC only — no edits under factory; default to AWS-region clients backed by the runtime IAM role. 4. Replace `DAVIDSUPERSECRETTOKEN` with `MCP_BEARER_TOKEN` loaded from AWS Secrets Manager. -5. Replace the colocated webhook Service with API Gateway + Lambda → SQS → - AgentCore Runtime. +5. Replace the colocated webhook Service with ALB + Lambda → + AgentCore Runtime invoke. 6. Move every k8s `Secret` into AWS Secrets Manager with the runtime role granted `secretsmanager:GetSecretValue`. 7. Enable AgentCore Observability and re-validate every production-only @@ -412,9 +480,103 @@ Promotion is composition-root + IaC only — no edits under 8. `agentcore configure --entrypoint src/agent/main.py` and `agentcore deploy` (see [`src/agent/main.py`](src/agent/main.py)). -A follow-up milestone ships `build_aws_dependencies(...)`, -`terraform/agentcore-runtime/`, `.bedrock_agentcore.yaml`, and -`scripts/promote_to_agentcore.sh` to automate this. +The AWS profile is fully wired today: +[`build_aws_dependencies(...)`](src/agent/composition/aws.py) resolves +the Bedrock + AgentCore + DynamoDB + Secrets Manager adapters, +[`terraform/agentcore-runtime/`](terraform/agentcore-runtime/) and the +sibling Terraform modules provision the cloud surface, +[`.bedrock_agentcore.yaml`](.bedrock_agentcore.yaml) declares the +managed runtime + JWT authorizer, and +[`scripts/promote_to_agentcore.sh`](scripts/promote_to_agentcore.sh) +runs the `agentcore configure` / `agentcore deploy` pair against a +prepared account. See +[`terraform/README.md`](terraform/README.md) for the module-level +breakdown. + +### End-to-end deploy plan + +The canonical operator-facing recipe is +[`aws-deploy-plan.md`](aws-deploy-plan.md). It covers everything from +verifying AWS credentials through to the optional EKS follow-on for +`mcp-internal`, and is structured as six phases plus a verification +section so the **plan-then-review-then-apply** beat is unmissable: + +| Phase | Subsections | What it does | +|---|---|---| +| 1 — Pre-flight | 1.1–1.6 | Identity, Bedrock model access for `global.anthropic.claude-sonnet-4-6` (the default in [`.state/bootstrap.env`](.state/bootstrap.env)), Route 53 zone, Jira identifiers, backend-bootstrap sanity check, DynamoDB + SQS pre-create. **No Cognito / JWT** — HMAC is enforced by the `webhook-validator` Lambda at the ALB; AgentCore is invoked via SigV4. See `aws-deploy-plan.md` §Approach → Security model. | +| 2 — PLAN (Terraform dry run) | 2.1–2.4 | For each module, `scripts/tf.sh plan -out=...` with a saved binary plan + human-readable summary at `.state/.plan.txt`. **No AWS resources change.** | +| 3 — APPLY (consume saved plans) | 3.1–3.2 | `scripts/tf.sh apply ` for each module + Secrets Manager `put-secret-value` for the placeholder secrets. | +| 4 — AgentCore promotion | 4.1–4.2 | `scripts/promote_to_agentcore.sh` (re-applies `terraform/agentcore-runtime/`, then `agentcore configure` + `agentcore deploy`). | +| 5 — Smoke invoke | — | `agentcore invoke @tests/fixtures/jira/issue_created.json`. | +| 6 — Follow-on: MCP-internal on EKS | 6.1–6.4 | Provision EKS, push the JAR to ECR, deploy with the Cilium NetworkPolicy, point AgentCore at the in-cluster MCP service. | +| Verification | — | End-state checklist: `aws sts get-caller-identity`, `bedrock-runtime invoke-model` returns 200, ten `.state/.outputs.json` files, GenAI dashboard, X-Ray trace. | + +Phases 2 and 3 interleave per module because each downstream module's +`tfvars` read upstream ARNs from `.state/.outputs.json` (so +later modules can't be planned until earlier ones have applied). The +deploy walks 10 unique modules across 11 plan/apply iterations because +`iam` runs twice — see the callout at the top of +[`aws-deploy-plan.md`](aws-deploy-plan.md#approach) for the full +rationale and the dependency-graph diagram. + +#### Commands to execute the plan + +The plan is not a single command — pre-flight is mostly manual +(Bedrock model-access toggle, Route 53 / domain setup, Jira +credential capture) and the per-module Terraform plan/apply steps +interleave — but the executable steps line up like this (full +operator-facing recipe in [`aws-deploy-plan.md`](aws-deploy-plan.md)): + +```bash +# One-time (before any Terraform module): provision the state bucket +# (account-regional namespace) and write terraform/backend.hcl. +# Requires AWS CLI v2 >= 2.34.7. +./scripts/bootstrap_tfstate.sh \ + terraform-state---an \ + --write-shared-backend + +# Pre-flight: sanity-check tooling. +aws sts get-caller-identity --query Account --output text +aws --version # expect v2 >= 2.34.7 +terraform -version # expect >= 1.13 +aws s3api create-bucket help 2>/dev/null | grep -q -- '--bucket-namespace' \ + && echo "ar-namespace OK" +pip install bedrock-agentcore-starter-toolkit # supplies `agentcore` CLI + +# Pre-flight (manual): AWS Console Bedrock model access toggle for +# the model in .state/bootstrap.env (default global.anthropic.claude-sonnet-4-6), +# Route 53 hosted zone, Jira identifiers, DDB tables + SQS queue pre-create +# (schemas: agent-domain-state PK=tenant_id RK=issue_key, +# agent-tenants PK=cloud_id). + +# PLAN + APPLY per module in dependency order (alternate plan_module +# + apply_module per module). Order: network, bedrock-guardrail, +# iam(pass1), kms-secrets, dlq-s3, secrets-mcp, iam(pass2), +# edge-security, observability-xray, observability-cloudwatch-genai, +# agentcore-runtime, webhook-validator. +plan_module network # writes .state/network.tfplan + .plan.txt +less .state/network.plan.txt # human review +apply_module network # consumes the saved plan; writes outputs.json +# ...repeat per module per the apply order in aws-deploy-plan.md... + +# Populate the placeholder secrets that Terraform created. +aws secretsmanager put-secret-value --secret-id default/mcp/internal/bearer-token --secret-string "$(openssl rand -hex 32)" + +# Promote the agent runtime to Bedrock AgentCore. +./scripts/promote_to_agentcore.sh + +# Smoke invoke. +agentcore invoke @tests/fixtures/jira/issue_created.json + +# (optional) MCP-internal on EKS — see aws-deploy-plan.md. +``` + +The `plan_module` / `apply_module` helpers, the per-module tfvars +recipe, the two-pass `iam` strategy, and the caveat about +`agentcore-runtime`'s `null_resource` fallback (used while the AWS +Terraform provider lacks native `aws_bedrockagentcore_runtime` / +`_memory` resources) all live in +[`aws-deploy-plan.md`](aws-deploy-plan.md) §2.1, §2.3, and §4.2. ## Operations @@ -453,10 +615,10 @@ but is not required to mirror this directory tree. src/agent/ domain/ # Pydantic value objects, no I/O contracts/ # Protocols (the IoC seam) - application/ # Use cases (webhook handler, runtime, assessor, - # token_governance.GovernedAssessmentLanguageModel, …) + application/ # Use cases (webhook handler, normalizer, + # recursion guard, token governance, …) infrastructure/ # Adapters - dynamodb/ # idempotency, domain-state, token_budget_enforcer + dynamodb/ # domain-state, token_budget_enforcer, breaker inmemory/ # in-memory enforcers + stores (local profile) llm/ # usage_capture.LangChainUsageCapturer + circuit_breaker cost/ # static_calculator + vendored model_prices.json @@ -468,10 +630,10 @@ src/agent/ main.py # env-var-driven module-level `app` for agentcore deploy tests/ domain/ contracts/ application/ infrastructure/ - _doubles/ # FakeAssessmentLanguageModel, fake_governance, … + _doubles/ # Fake governance, tracer, logger, and store adapters fixtures/ # tests/fixtures/jira/*.json webhook corpus deploy/local/ # Kubernetes manifests for the local stack -docs/adr/ # Architecture Decision Records (ADR-001 = token governance) +docs/ARCHITECTURE.md # Layered architecture, deployment topology, token-governance design docs/runbook.md # Incident-response runbook (guardrail / MCP / budget / DLQ) Makefile # Frozen developer-command vocabulary scripts/ # smoke.py, promote_*.sh, export_token_usage.py @@ -485,12 +647,12 @@ terraform/ # agentcore-runtime, observability-cloudwatch-genai, … enforces SOLID/IoC review, secret hygiene, and the coverage gate. - New external boundaries MUST live behind a `Protocol` in [`agent.contracts`](src/agent/contracts) and be wired in - [`agent.composition`](src/agent/composition.py). + [`agent.composition`](src/agent/composition). - New env vars MUST be added to [`.env.local.example`](.env.local.example), [`deploy/local/configmap.yaml`](deploy/local/configmap.yaml), and the matching `build_*_dependencies` factory in - [`src/agent/composition.py`](src/agent/composition.py) in lockstep. + [`src/agent/composition/`](src/agent/composition) in lockstep. - New AWS-only behavior MUST land with a corresponding Terraform module or script deliverable under `terraform/` or `scripts/`. diff --git a/aws-deploy-plan.md b/aws-deploy-plan.md new file mode 100644 index 0000000..b89fe84 --- /dev/null +++ b/aws-deploy-plan.md @@ -0,0 +1,2320 @@ +# Deploy `aws-agent-core` (account-agnostic, region pinned to `us-east-1`) + +## Context + +End-to-end deploy of this repo against the AWS account `aws sts get-caller-identity` returns. Every ARN, queue URL, bucket name, IAM +principal, and Route 53 zone is constructed at deploy time from +`$ACCOUNT_ID` or sourced from `.state/.outputs.json`. + +Region policy: this project is hard-pinned to `us-east-1`. +`scripts/tf.sh`, `scripts/rebuild_sandbox.sh`, +`scripts/bootstrap_eks_workloads.sh`, +`scripts/bootstrap_agentcore_runtime.sh`, and +`scripts/invoke_lambda.sh` fail fast when `AWS_REGION` / +`AWS_DEFAULT_REGION` is anything else. + +**EKS + Secrets Store CSI + Pod Identity + KMS:** see +[`docs/deploy-eks-secrets-csi.md`](docs/deploy-eks-secrets-csi.md) for the +incident-derived checklist (Helm `tokenRequests`, single CMK path, Jira JSON, +and the exact `rebuild_sandbox.sh` / `sync_sandbox_tfvars.py` guardrails). + +**Architecture.** `webhook-validator` Lambda is the signed ingress +boundary (HMAC verify) and invokes Bedrock AgentCore Runtime directly. +AgentCore Runtime remains the AWS execution surface for sessions/traces. + +**Required workstation + account state:** + +- **AWS principal** with `AdministratorAccess` or scoped equivalents for +IAM, KMS, S3, DynamoDB, SQS, EKS, Secrets Manager, Bedrock, AgentCore, +Route 53, WAFv2, CloudWatch, X-Ray, Budgets, ECR write. +- **Bedrock access:** `AmazonBedrockFullAccess` or scoped equivalents for +`bedrock:*Model`*, `bedrock-agentcore:*`, `bedrock-agentcore-control:*`. +- **Tooling versions:** `aws-cli >= 2.34.7`, `terraform >= 1.13`, +`bedrock-agentcore-starter-toolkit >= 0.3`. +- **Supported model families** (Phase 1.2 picks one): + - Preferred: `meta.llama3-1-405b-instruct-v1:0` + - Fallback: `meta.llama3-1-70b-instruct-v1:0` + Each candidate is gated by two probes: + `bedrock get-foundation-model-availability` (entitlement) AND + `bedrock-runtime converse` with a tool-config schema + (`stopReason: tool_use`). +- AgentCore control plane reachable: +`aws bedrock-agentcore-control list-agent-runtimes` returns +`agentRuntimes: []`. + +`terraform/organizations/` is dropped (no module imports its outputs). + +**Open prerequisites** (not in the repo): + +1. **Bedrock model access** for at least one §1.2 candidate in `us-east-1` + (Bedrock Console → Model access). Resolved id lands in + `.state/bootstrap.env::BEDROCK_MODEL_ID`. +2. **Route 53 hosted zone** + **domain name** for `terraform/edge-security/`. + Skipping `edge-security/` only loses the public webhook URL. +3. **Jira tenant** + integration user's API token + bot user's `accountId` + (`AGENT_JIRA_ACCOUNT_ID`, used by the recursion guard). +4. **DynamoDB + SQS** provisioned by + `[terraform/data-stores/](terraform/data-stores/)` (first module applied). + Schemas: `tenant_id`+`issue_key` on domain-state, `cloud_id` on tenants, + `pk`+TTL `expires_at` on token-budgets and breaker. Do NOT substitute + generic `pk` / `sk` names. + +## Approach + +Single operator, `aws` CLI v2 + `terraform >= 1.13` + +`bedrock-agentcore-starter-toolkit`. State lives in S3 with native locking +(`use_lockfile = true`): committed `terraform//backend.tf` stub + +gitignored `terraform/backend.hcl` written by +`scripts/bootstrap_tfstate.sh --write-shared-backend`. Terraform runs go +through `scripts/tf.sh`; per-module outputs are captured under +`.state/.outputs.json`. `examples/basic/` subtrees are not applied +by operators. + +State bucket lockdown set by the bootstrap script: BPA on, Deny-only bucket +policy pinning account/region/HTTPS, `BucketOwnerEnforced`, versioning on, +30-day noncurrent-version expiry, `AES256` SSE. + +**Security model.** No end-user auth in the data path. Atlassian signs every +webhook body with HMAC-SHA256 (`x-jira-signature`); the +`webhook-validator` Lambda verifies this using +`WEBHOOK_HMAC_SECRET_ID=default/atlassian/webhook-hmac` from Secrets Manager. +AgentCore Runtime is invoked via SigV4. `mcp-internal` Pod uses the +integration-user API token from `default/jira/integration-user`. No Cognito, +no JWT: `terraform/edge-security/` runs with `use_api_gateway_v2 = false` +and the runtime's `authorizer_configuration` is `null`. + +### Rebuild orchestration (single command) + +After teardown, run the scripted rebuild path: + +```bash +scripts/rebuild_sandbox.sh +``` + +This covers Terraform applies, `.env -> Secrets Manager` webhook-HMAC sync, +AgentCore runtime bootstrap, EKS workloads bootstrap, smoke invocation, and +post-rebuild validation. Required env vars: + +```bash +export AGENT_JIRA_ACCOUNT_ID=... +export AGENT_JIRA_EMAIL=... +``` + +For **EKS Secrets Store CSI + Pod Identity + KMS + `mcp-internal` Jira mounts**, +also keep **`.env.local`** (or exported env) with **`JIRA_SITE_URL`**, +**`JIRA_EMAIL`**, and **`JIRA_API_TOKEN`**, or accept that +**`scripts/sync_jira_integration_secret_from_env.sh`** skips until those are +set. Full operator checklist: **`docs/deploy-eks-secrets-csi.md`** (section *Every future deploy (required operator actions)*). + +`scripts/rebuild_sandbox.sh` now hard-pins Terraform/AWS CLI region to +`us-east-1` (`AWS_REGION` + `AWS_DEFAULT_REGION`) and runs +`scripts/sync_sandbox_tfvars.py` before/after EKS provisioning so +`terraform.tfvars` consumers are refreshed from `.state/*.outputs.json` +instead of manual copy/paste. +It also runs `scripts/sync_webhook_hmac_secret.sh` immediately after every +`kms-secrets` apply pass, so the authoritative +`lambda/webhook_validator/.env::WEBHOOK_HMAC_SECRET` is always pushed back to +`default/atlassian/webhook-hmac` in AWS. +After `terraform/eks apply`, it runs +`scripts/ensure_eks_cli_admin_access.sh` to guarantee the deploy operator's +current AWS CLI principal has an EKS access entry + cluster-admin policy. +If the UI session uses a different IAM principal (for example an SSO role), +set `EKS_ADMIN_PRINCIPAL_ARNS` (comma-separated IAM role/user ARNs) before +running rebuild so those principals are also granted cluster-admin on each +deploy. +The script also auto-discovers and grants any +`AWSReservedSSO_AdministratorAccess_*` role(s) in the account by default. + +Secrets Manager writes/reads for `default/atlassian/webhook-hmac` are pinned +to `us-east-1`. + +Phases 2 and 3 interleave per module (module N+1's tfvars consume module N's +outputs): `PLAN data-stores → APPLY data-stores → PLAN network → APPLY network → ...`. + +--- + +## Phase 1 — Pre-flight + +### 1.1 Confirm identity and tooling + +```bash +cd "$(git rev-parse --show-toplevel)" + +export ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" +[[ -z "${AWS_REGION:-}" || "${AWS_REGION}" = "us-east-1" ]] || { echo "AWS_REGION must be us-east-1" >&2; exit 64; } +[[ -z "${AWS_DEFAULT_REGION:-}" || "${AWS_DEFAULT_REGION}" = "us-east-1" ]] || { echo "AWS_DEFAULT_REGION must be us-east-1" >&2; exit 64; } +export AWS_REGION="us-east-1" +export AWS_DEFAULT_REGION="us-east-1" +echo "Deploying to account $ACCOUNT_ID in region $AWS_REGION" +aws sts get-caller-identity + +aws --version # expect v2 >= 2.34.7 +terraform -version # expect >= 1.13 +aws bedrock-agentcore-control help >/dev/null && echo ok +aws bedrock-agentcore help >/dev/null && echo ok +aws s3api create-bucket help 2>/dev/null | grep -q -- '--bucket-namespace' \ + && echo "ar-namespace OK" \ + || echo "AWS CLI is too old; brew upgrade awscli" +pip install bedrock-agentcore-starter-toolkit +agentcore --version +``` + +Re-export `$ACCOUNT_ID` in any new shell session before resuming a phase. If +`terraform` < 1.13 or `aws-cli` < 2.34.7: `brew upgrade terraform awscli`. + +### 1.2 Resolve `BEDROCK_MODEL_ID` (tiered authorization + structured-output probe) + +The first candidate that passes both entitlement and a +Converse-with-tool-config smoke becomes `BEDROCK_MODEL_ID`. Appended to +`.state/bootstrap.env`. + +This step only picks the **Bedrock** model id consumed when +`AGENT_PROFILE=aws` (or any path that invokes Bedrock). It does **not** +change local Anthropic API usage (`ANTHROPIC_MODEL`, direct API keys in +`.env.local`); those remain valid for non-Bedrock local runs. + +```bash +CANDIDATES=( + "global.anthropic.claude-sonnet-4-6" + "us.anthropic.claude-sonnet-4-5-20250929-v1:0" +) + +probe_model() { + local id="$1" + echo " probing: $id" + aws bedrock get-foundation-model-availability --region "$AWS_REGION" \ + --model-id "$id" --output json 2>/dev/null \ + | jq -e '.authorizationStatus=="AUTHORIZED" and .entitlementAvailability=="AVAILABLE"' \ + >/dev/null \ + || { echo " -> not authorized"; return 1; } + + aws bedrock-runtime converse --region "$AWS_REGION" \ + --model-id "$id" \ + --messages '[{"role":"user","content":[{"text":"What is the weather in Boston? Use the get_weather tool."}]}]' \ + --tool-config '{"tools":[{"toolSpec":{"name":"get_weather","description":"Get weather for a city","inputSchema":{"json":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}}}}]}' \ + --inference-config maxTokens=200 \ + --query 'stopReason' --output text 2>/dev/null \ + | grep -qx 'tool_use' \ + || { echo " -> no structured-output / tool_use"; return 1; } + + echo " -> selected" + return 0 +} + +BEDROCK_MODEL_ID="" +for c in "${CANDIDATES[@]}"; do + if probe_model "$c"; then BEDROCK_MODEL_ID="$c"; break; fi +done + +if [[ -z "$BEDROCK_MODEL_ID" ]]; then + echo "ERROR: no candidate is authorized AND tool-use-capable in account $ACCOUNT_ID / region $AWS_REGION." >&2 + echo "Toggle access at Bedrock Console -> Model access for at least one of:" >&2 + printf ' %s\n' "${CANDIDATES[@]}" >&2 + exit 1 +fi + +mkdir -p .state +grep -v '^BEDROCK_MODEL_ID=' .state/bootstrap.env 2>/dev/null > .state/bootstrap.env.tmp || true +echo "BEDROCK_MODEL_ID=$BEDROCK_MODEL_ID" >> .state/bootstrap.env.tmp +mv .state/bootstrap.env.tmp .state/bootstrap.env +echo "selected: BEDROCK_MODEL_ID=$BEDROCK_MODEL_ID" +``` + +If every candidate fails, open Bedrock Console (`us-east-1`) → **Model +access** → tick at least one id → submit; status must read **Access granted**. + +### 1.3 Prepare Route 53 hosted zone (for `edge-security`) + +Either register a new domain via Route 53 +(`aws route53domains register-domain`) or delegate an existing domain into +a new Route 53 hosted zone in this account. + +```bash +aws route53 list-hosted-zones --query 'HostedZones[].{Name:Name,Id:Id}' +``` + +Pick a subdomain (e.g. `agent-sandbox.example.com`) for `var.domain_name`, +then log: + +```bash +applied_log PHASE_1_3 route53domains-register \ + "$DOMAIN_NAME" "expiry=$EXPIRY non-refundable, auto-renew=$AUTO_RENEW" + +applied_log PHASE_1_3 route53-hosted-zone \ + "$HOSTED_ZONE_ID" "domain=$DOMAIN_NAME" +``` + +Persist `DOMAIN_NAME`, `HOSTED_ZONE_ID`, `EXPIRY`, `AUTO_RENEW`, +`OPERATION_ID`, `REGISTRANT_EMAIL`, `EMAIL_VERIFICATION_STATUS` to +`.state/route53.env`. Domain registration fees are non-refundable. + +### 1.4 Capture Jira credentials + +**Runtime env vars** (passed via `agentcore deploy --env`): + +- `AGENT_JIRA_ACCOUNT_ID` — bot user's Atlassian `accountId` (recursion guard). +- `AGENT_JIRA_EMAIL` — bot user's email (recursion guard). + +**Secrets Manager entry consumed by `mcp-internal` only.** Stored at +`default/jira/integration-user` as a single JSON blob: + +```json +{ + "email": "qodo.jira@your-tenant.atlassian.net", + "api_token": "", + "site_url": "https://your-tenant.atlassian.net" +} +``` + +Phase 6's manifest injects these into the `mcp-internal` container as +`JIRA_EMAIL` / `JIRA_API_TOKEN` / `JIRA_SITE_URL`. Validate before Phase 6: + +```bash +aws secretsmanager get-secret-value \ + --secret-id default/jira/integration-user --region "$AWS_REGION" \ + --query SecretString --output text \ + | jq '{has_email: ((.email|length)>0), + has_api_token: ((.api_token|length)>0), + has_site_url: ((.site_url|length)>0)}' +``` + +All three fields must be non-empty. + +### 1.5 Confirm the Terraform backend is bootstrapped + +Confirm `terraform/backend.hcl` (gitignored) exists and +`scripts/tf.sh network init` succeeds: + +```bash +test -f terraform/backend.hcl || { + echo "Run scripts/bootstrap_tfstate.sh --write-shared-backend first." + echo "See terraform/README.md § State management for the worked recipe." + exit 1 +} + +scripts/tf.sh network init >/dev/null && echo "backend OK" +``` + +If either check fails, run +[terraform/README.md § State management](terraform/README.md#state-management-operator-action-required). + +### 1.6 DDB + SQS — owned by `terraform/data-stores/` + +DynamoDB tables (`agent-domain-state`, `agent-tenants`, `token-budgets`, +`breaker`) and SQS queues (`webhook-dlq`, `agent-invoke-queue`) are +provisioned by `[terraform/data-stores/](terraform/data-stores/)` (first +module applied). Schemas verified at runtime by `check_table_contracts` in +[src/agent/composition/_shared.py](src/agent/composition/_shared.py); set +`AGENT_BOOT_VALIDATE_SCHEMAS=1` in the runtime env (Phase 4) and worker +ConfigMap (Phase 6.6) to fail-loud on drift. + +Sandbox-only seed of `local-cloud-id-acme` tenant row is gated on +`var.seed_sandbox_tenant = true` in `terraform/data-stores/terraform.tfvars`. + +--- + +## Phase 2 — PLAN (Terraform dry run; saves plan files for human review) + +Runs `terraform plan -out=...` per module and saves the binary plan + text +summary to `.state/`. **No AWS resources change in this phase.** Phase 3 +consumes each saved plan file. + +### Dependency graph (PLAN and APPLY follow the same order) + +```mermaid +flowchart TB + dataStores["data-stores
DDB + SQS"] + network[network] + guardrail[bedrock-guardrail] + bucket["dlq-s3
S3 only, no SQS"] + iam1["iam pass 1
secrets_arns = wildcard"] + kms[kms-secrets] + smcp[secrets-mcp] + iam2["iam pass 2
tighten secrets_arns"] + edge[edge-security] + xray[observability-xray] + cwgenai[observability-cloudwatch-genai] + runtime["agentcore-runtime
(IAM/runtime/memory prerequisites)"] + whvalidator["webhook-validator
HMAC Lambda → AgentCore Runtime invoke
(/invocations)"] + + dataStores --> iam1 + dataStores --> runtime + dataStores --> whvalidator + network --> edge + network --> runtime + guardrail --> iam1 + iam1 --> kms + iam1 --> smcp + kms --> iam2 + kms --> bucket + kms --> whvalidator + smcp --> iam2 + bucket --> iam2 + iam2 --> xray + iam2 --> cwgenai + iam2 --> runtime + edge --> runtime + edge --> whvalidator +``` + + + +`terraform/iam/` and `terraform/kms-secrets/` have a circular dependency, +resolved with a two-pass apply on `iam/` (wildcard secrets-ARN on pass 1, +real ARNs on pass 2). + +> **Architecture note.** The `agentcore-runtime` module remains in the +> apply order because it provisions prerequisites consumed by +> toolkit-driven runtime deploys (`agentcore configure` / +> `agentcore deploy`). The `webhook-validator` Lambda terminates HMAC +> at the ALB and publishes the validated envelope onto the SQS +> `agent-work` queue; the agent-worker Pod drains the queue and (under +> `AGENT_PROFILE=aws`) calls `bedrock-agentcore:InvokeAgentRuntime` +> against the runtime endpoint, so AgentCore sessions/traces remain +> authoritative for AWS ingress. + +### 2.1 Shared environment + helpers + +`plan_module` writes binary plan + text/JSON summaries to `.state/`. +`apply_module` consumes the saved plan, captures outputs to +`.state/.outputs.json`, and appends to `.state/applied.log` (consumed +by `scripts/teardown_sandbox.sh`). + +```bash +cd "$(git rev-parse --show-toplevel)" +mkdir -p .state +[[ -z "${AWS_REGION:-}" || "${AWS_REGION}" = "us-east-1" ]] || { echo "AWS_REGION must be us-east-1" >&2; exit 64; } +[[ -z "${AWS_DEFAULT_REGION:-}" || "${AWS_DEFAULT_REGION}" = "us-east-1" ]] || { echo "AWS_DEFAULT_REGION must be us-east-1" >&2; exit 64; } +export AWS_REGION="us-east-1" +export AWS_DEFAULT_REGION="us-east-1" +export ACCOUNT_ID="${ACCOUNT_ID:-$(aws sts get-caller-identity --query Account --output text)}" + +# applied_log [optional free-form notes] +# Single source of truth for the teardown recipe. Every state-changing +# action in Phases 1, 3, 4, and 6 must call this so scripts/teardown_sandbox.sh +# can replay the destroys in reverse without missing a resource. +applied_log() { + local category="$1"; local rtype="$2"; local rid="$3"; shift 3 + local notes="${*:-}" + printf '%s [%s] %s %s%s\n' \ + "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$category" "$rtype" "$rid" \ + "${notes:+ ($notes)}" \ + >> .state/applied.log +} + +plan_module() { + local mod="$1" + scripts/tf.sh "$mod" init -reconfigure + scripts/tf.sh "$mod" plan -out="../../.state/${mod}.tfplan" + scripts/tf.sh "$mod" show -no-color "../../.state/${mod}.tfplan" \ + > ".state/${mod}.plan.txt" + scripts/tf.sh "$mod" show -json "../../.state/${mod}.tfplan" \ + > ".state/${mod}.plan.json" + echo "REVIEW: .state/${mod}.plan.txt (then run apply_module ${mod})" +} + +apply_module() { + local mod="$1" + scripts/tf.sh "$mod" apply "../../.state/${mod}.tfplan" + scripts/tf.sh "$mod" output -json > ".state/${mod}.outputs.json" + applied_log TF apply "$mod" +} +``` + +### 2.2 Output-name reference (used in tfvars between modules) + +Output names exposed by each module's `outputs.tf`: + +- `terraform/data-stores/outputs.tf` → `dynamodb_table_arns` (list of +4: domain-state, tenants, token-budgets, breaker), individual +`*_table_arn` / `*_table_name` per table, `webhook_dlq_arn` / +`webhook_dlq_url`, `agent_invoke_queue_arn` / `agent_invoke_queue_url`. +- `terraform/network/outputs.tf` → `vpc_id`, `vpc_cidr_block`, +`private_subnet_ids`, `public_subnet_ids`, `agent_runtime_sg_id`, +`mcp_internal_sg_id`, `alb_sg_id`, plus **6 VPC endpoint resources + +2 prefix-list outputs**: 2 gateway endpoints (DynamoDB, S3) and 4 +interface endpoints (Bedrock-Runtime, Secrets Manager, X-Ray, +CloudWatch Monitoring) — see +[terraform/network/vpc_endpoints.tf](terraform/network/vpc_endpoints.tf). +Output names: `bedrock_endpoint_id`, `dynamodb_endpoint_id`, +`dynamodb_endpoint_prefix_list_id`, `s3_endpoint_id`, +`s3_endpoint_prefix_list_id`, `secretsmanager_endpoint_id`, +`xray_endpoint_id`, `monitoring_endpoint_id`. +- `terraform/iam/outputs.tf` → `agent_runtime_role_arn`, +`webhook_ingress_role_arn`, `mcp_token_reader_role_arn`. +- `terraform/kms-secrets/outputs.tf` → `kms_key_arns` (map keyed by tenant id; +default tenant is `"default"`), `kms_key_aliases` (map), +`secret_arns` (map keyed by `"/"`), +`rotation_lambda_arn`. +- `terraform/dlq-s3/outputs.tf` → `bucket_id`, `bucket_arn` (S3 only — SQS +comes from `terraform/data-stores/`). +- `terraform/secrets-mcp/outputs.tf` → `secret_arn` (singular), `policy_arn`. +- `terraform/bedrock-guardrail/outputs.tf` → `guardrail_id`, `guardrail_arn`, +`guardrail_version`, `secret_arn` (a single Secrets Manager entry holding +both `guardrailIdentifier` and `guardrailVersion`). +- `terraform/edge-security/outputs.tf` → `alb_arn`, `alb_dns_name`, +`alb_target_group_arn`, `https_listener_arn`, `web_acl_arn`, +`acm_cert_arn`, `jwt_issuer_url` (`null` under the no-JWT default). +- `terraform/webhook-validator/outputs.tf` → `lambda_function_name`, +`lambda_arn`, `lambda_role_arn`, `target_group_arn`, +`listener_rule_arn`, `log_group_name`. + +Sandbox-friendly defaults: + +- `single_nat_gateway = true` (in `network/`). +- `use_api_gateway = false` and `use_api_gateway_v2 = false` +(in `edge-security/`). + +### 2.3 PLAN order (interleaves with Phase 3) + +Each step: write tfvars from upstream outputs, `plan_module `, review +`.state/.plan.txt`, then `apply_module`. tfvars files are gitignored. + +1. `plan_module data-stores` — `region = "us-east-1"`, + `seed_sandbox_tenant = true` (sandbox only). Outputs feed every + later module's `dynamodb_table_arns` / `dead_letter_queue_arn` / + `webhook_invoke_queue_arn` references. +2. `plan_module network` — `single_nat_gateway = true`, `vpc_cidr = + "10.0.0.0/16"`, region tag. +3. `plan_module bedrock-guardrail`. +4. `plan_module iam` — **pass 1**: in `terraform/iam/terraform.tfvars` set + `secrets_arns = ["arn:aws:secretsmanager:${AWS_REGION}:${ACCOUNT_ID}:secret:*"]` + (wildcard bootstrap; substitute real `$ACCOUNT_ID` / `$AWS_REGION` + literals — Terraform does not interpolate shell variables in `tfvars`). + Real values for everything else: + - `dynamodb_table_arns = jq -r '.dynamodb_table_arns.value[]' .state/data-stores.outputs.json` + (4 ARNs: agent-domain-state, agent-tenants, token-budgets, breaker). + - `dead_letter_queue_arn = jq -r '.webhook_dlq_arn.value' .state/data-stores.outputs.json`. + - `webhook_invoke_queue_arn = jq -r '.agent_invoke_queue_arn.value' .state/data-stores.outputs.json`. + - `raw_payload_bucket_arn = "arn:aws:s3:::"` + (string-construct ahead of `dlq-s3/` apply). + - `mcp_secret_arns = ["arn:aws:secretsmanager:${AWS_REGION}:${ACCOUNT_ID}:secret:mcp/internal/bearer-token-*", "arn:aws:secretsmanager:${AWS_REGION}:${ACCOUNT_ID}:secret:default/jira/integration-user-*", "arn:aws:secretsmanager:${AWS_REGION}:${ACCOUNT_ID}:secret:default/git/ssh-private-key-*"]` + — substitute literals. Required non-empty. Wildcard suffix matches + AWS's auto-suffix on Secrets Manager ARNs. + - `guardrail_arn` from `.state/bedrock-guardrail.outputs.json`. + - `bedrock_model_arns` — derive from `BEDROCK_MODEL_ID` (set by Phase 1.2): + + | Resolved `BEDROCK_MODEL_ID` | tfvars value (mirror `terraform/agentcore-runtime/iam.tf` locals) | + | ---------------------------------- | ------------------------------------------------------------------- | + | `global.anthropic.claude-sonnet-4-6` | `["arn:aws:bedrock:*::foundation-model/anthropic.claude-sonnet-4-6", "arn:aws:bedrock:us-east-1::inference-profile/global.anthropic.claude-sonnet-4-6"]` | + | `us.anthropic.claude-sonnet-4-5-20250929-v1:0` | `["arn:aws:bedrock:*::foundation-model/anthropic.claude-sonnet-4-5-20250929-v1:0", "arn:aws:bedrock:us-east-1::inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0"]` | + + Empty account-id segment is intentional (AWS-owned) on the **foundation-model** ARN. For geo-prefixed inference-profile ids (`us.`, `global.`, etc.), include both foundation-model and inference-profile ARNs (as in the table). +5. `plan_module kms-secrets` — pass `agent_runtime_role_arn` from pass 1 + and extend `var.secrets` with the webhook HMAC secret: +6. `plan_module dlq-s3` — `kms_key_arn` = the single value at + `jq -r '.kms_key_arns.value.default' .state/kms-secrets.outputs.json` + (default tenant is `"default"`). +7. `plan_module secrets-mcp` — `kms_key_arn` **must match** the default-tenant + CMK from step 5 / `.state/kms-secrets.outputs.json` (run + `scripts/sync_sandbox_tfvars.py` after `kms-secrets` apply so + `terraform/secrets-mcp/terraform.tfvars` is pinned — see + `docs/deploy-eks-secrets-csi.md`). Also pass `agent_runtime_role_arn` from pass 1. +8. `plan_module iam` — **pass 2**: tighten `secrets_arns` to the real ARNs: + ```hcl + secrets_arns = [ + "<.state/secrets-mcp.outputs.json -> secret_arn>", + "<.state/kms-secrets.outputs.json -> secret_arns[\"default/jira/integration-user\"]>", + "<.state/kms-secrets.outputs.json -> secret_arns[\"default/anthropic/api-key\"]>", + "<.state/kms-secrets.outputs.json -> secret_arns[\"default/bedrock/runtime-config\"]>", + "<.state/kms-secrets.outputs.json -> secret_arns[\"default/atlassian/webhook-hmac\"]>", + "<.state/bedrock-guardrail.outputs.json -> secret_arn>", + ] + ``` +9. `plan_module edge-security` — needs `vpc_id`, `public_subnet_ids`, + `alb_sg_id`, `domain_name`, `hosted_zone_id`. Set + `use_api_gateway = false` and `use_api_gateway_v2 = false` + (`jwt_issuer_url` / `jwt_audience` are then unused). +10. `scripts/sync_sandbox_tfvars.py --phase pre-eks` — after **edge-security** + outputs exist; populates **`terraform/webhook-validator/terraform.tfvars`** + (**`https_listener_arn`**, **`work_queue_*`**, secret fields). Skipping this + is the usual cause of **`plan_module` / `apply_module webhook-validator`** + failing on missing **`work_queue_*`** (no Lambda created). +11. `plan_module observability-xray` (`cluster_type = "ecs"` for sandbox) and + `plan_module observability-cloudwatch-genai`. +12. `plan_module agentcore-runtime` — provisions the IAM execution role + + ECR repo + log groups that the upstream `agentcore deploy` toolkit + consumes when minting the AgentCore Memory store the EKS + `agent-worker` Pod uses as its LangGraph checkpointer (scripted + recipe in §4.3 via `scripts/bootstrap_agentcore_runtime.sh`). + Runtime remains required for webhook ingress execution. +13. `plan_module agentcore-memory` — Memory store id consumed by **eks-workloads** / + **`scripts/bootstrap_agentcore_runtime.sh`** wiring. +14. `plan_module webhook-validator` — HMAC Lambda fronting the ALB + `/invocations`. Inputs are sourced from upstream module outputs: + + | tfvar | Source | + | -------------------- | ---------------------------------------------------------------------------------------------- | + | `https_listener_arn` | `jq -r '.https_listener_arn.value' .state/edge-security.outputs.json` | + | `work_queue_arn` | `jq -r '.agent_invoke_queue_arn.value' .state/data-stores.outputs.json` | + | `work_queue_url` | `jq -r '.agent_invoke_queue_url.value' .state/data-stores.outputs.json` | + | `secret_id` | literal `default/atlassian/webhook-hmac` (Phase 3.2a hydrates the value) | + | `secret_arn` | `jq -r '.secret_arns.value["default/atlassian/webhook-hmac"]' .state/kms-secrets.outputs.json` | + | `secret_kms_key_arn` | `jq -r '.kms_key_arns.value.default' .state/kms-secrets.outputs.json` | + | `environment` | `sandbox` (or your environment label) | + + **`scripts/sync_sandbox_tfvars.py`** updates **`terraform/webhook-validator/terraform.tfvars`** + with `https_listener_arn`, `work_queue_arn`, `work_queue_url`, `secret_arn`, + and `secret_kms_key_arn` from **edge-security**, **data-stores**, and + **kms-secrets** outputs. Without **`work_queue_*`**, **`terraform apply`** + on **webhook-validator** fails with missing required variables and **no + Lambda is created** (the handler under `lambda/webhook_validator/` is only + packaged when this module applies successfully). + +15. `plan_module ecr-images` then `plan_module eks` — align with **§3.1** before + post-EKS **`iam`** pass 3 / **`kms-secrets`** second pass / **second** **`webhook-validator`** apply. + +### 2.4 What to look for in each `.state/.plan.txt` + +- **Resource count + verbs.** First-time plan should be all `+ create`. +- **No `-/+ replace`** on stateful resources (S3, KMS, Secrets, DDB). +- **IAM policy diffs** show principals/actions/resource ARNs inline. +- **Secret ARNs** match `.state/*.outputs.json`: +`jq -r '.variables[].value' .state/.plan.json | grep -E '^arn:aws:'`. + +--- + +## Phase 3 — APPLY (consume the saved plans from Phase 2) + +Each step assumes the matching Phase 2 step has run and `.state/.tfplan` +is intact. + +### 3.1 APPLY order + +This block is kept in lockstep with **`scripts/rebuild_sandbox.sh`** +(module order, sync hooks, and two **`webhook-validator`** applies). + +```bash +# Refresh tfvars from latest .state outputs (safe to repeat). +scripts/sync_sandbox_tfvars.py --phase pre-eks + +apply_module data-stores # Phase 2.3 #1 — DDB + SQS, no upstream deps +apply_module network # Phase 2.3 #2 +apply_module bedrock-guardrail # Phase 2.3 #3 +apply_module iam # Phase 2.3 #4 — pass 1 (wildcard secrets_arns) +apply_module kms-secrets # Phase 2.3 #5 +scripts/sync_webhook_hmac_secret.sh --region us-east-1 # enforce .env as source-of-truth +# Pin secrets-mcp CMK + IAM/EKS secrets_kms lists BEFORE secrets-mcp apply. +scripts/sync_sandbox_tfvars.py --phase pre-eks +apply_module dlq-s3 # Phase 2.3 #6 +apply_module secrets-mcp # Phase 2.3 #7 + +scripts/sync_sandbox_tfvars.py --phase pre-eks +apply_module iam # pass 2 — real secrets_arns (Phase 2.3 #8) + +apply_module edge-security # Phase 2.3 #9 +# Populate https_listener_arn + work_queue_* in terraform/webhook-validator/terraform.tfvars +# (sync reads edge-security + data-stores outputs; without work_queue_* the module cannot plan). +scripts/sync_sandbox_tfvars.py --phase pre-eks +apply_module observability-xray # Phase 2.3 #10 +apply_module observability-cloudwatch-genai +apply_module agentcore-runtime # Phase 2.3 #11 +apply_module agentcore-memory # Memory id consumed by EKS worker / bootstrap +apply_module webhook-validator # Phase 2.3 #12 — first apply (packages lambda/webhook_validator/) + +apply_module ecr-images +apply_module eks +scripts/ensure_eks_cli_admin_access.sh --region us-east-1 + +apply_module iam # pass 3 post-EKS (OIDC trust refresh) + +scripts/sync_sandbox_tfvars.py --phase post-eks +apply_module kms-secrets # pass 2 — additional_decrypt_role_arns (worker + MCP + webhook role) +scripts/sync_webhook_hmac_secret.sh --region us-east-1 +apply_module webhook-validator # second apply — Lambda IAM vs CMK alignment + +# Then: scripts/bootstrap_agentcore_runtime.sh, scripts/sync_jira_integration_secret_from_env.sh, +# scripts/bootstrap_eks_workloads.sh (see rebuild_sandbox.sh tail). +``` + +### 3.1a EKS / CSI / KMS — operator checklist (every deploy) + +Whether you use **`scripts/rebuild_sandbox.sh`** or the **§3.1** `apply_module` +sequence by hand, the following must remain true so **`agent-worker`** and +**`mcp-internal`** CSI mounts and Pod Identity keep working: + +1. **Jira secret JSON** — **`JIRA_SITE_URL`**, **`JIRA_EMAIL`**, **`JIRA_API_TOKEN`** + in **`.env.local`** (or exported), **or** accept skip of + **`scripts/sync_jira_integration_secret_from_env.sh`** until set. +2. **After every `kms-secrets` apply that feeds `secrets-mcp`** — run + **`scripts/sync_sandbox_tfvars.py --phase pre-eks`** before + **`terraform/secrets-mcp` apply** (§3.1 above; **`rebuild_sandbox.sh`** + does this automatically after the first `kms-secrets` pass). +3. **After `terraform/eks` exists** — **`scripts/sync_sandbox_tfvars.py --phase post-eks`** + then **`apply_module kms-secrets`** again so **`additional_decrypt_role_arns`** + includes **agent-worker**, **mcp-token-reader**, and **webhook-validator** + Lambda role ARNs (plus any others in your tfvars). Re-**`apply_module webhook-validator`** + after that KMS pass so the Lambda execution role tracks the CMK. +4. **Cluster Helm** — run **`scripts/bootstrap_eks_workloads.sh`** (included in + **`rebuild_sandbox.sh`**) so CSI **driver → provider** order, + **`secrets-store-csi-driver.install=false`** on ASCP, and driver + **`tokenRequests`** for **`sts.amazonaws.com`** and **`pods.eks.amazonaws.com`** + stay correct. +5. **`webhook-validator` tfvars** — **`scripts/sync_sandbox_tfvars.py`** must run + **after `edge-security` apply** (and **`data-stores`** must exist) so + **`https_listener_arn`**, **`work_queue_arn`**, and **`work_queue_url`** are set; + otherwise **`terraform apply` webhook-validator** fails and **no Lambda** is created. +6. **Agent-worker → AgentCore + raw S3** — `terraform/eks/iam.tf` grants + **`bedrock-agentcore:InvokeAgentRuntime`** on both the **runtime ARN** and + **`…/runtime-endpoint/`** (AWS evaluates the latter). It grants + **`kms:GenerateDataKey`** (with **`kms:ViaService` = S3**) on the **union** of + **`raw_payload_bucket_kms_key_arns`** (from **`terraform/dlq-s3`** output + **`raw_payload_bucket_kms_key_arn`**, copied by **`sync_sandbox_tfvars.py`**) + and **`secrets_kms_key_arns`**. Re-**`apply_module eks`** (or **`iam`** pass 3) + after syncing so the worker role picks up the grants. + +Canonical narrative + incident notes: **`docs/deploy-eks-secrets-csi.md`**. + +> **Architecture note.** The webhook ingress runs through the +> `webhook-validator` Lambda +> ([terraform/webhook-validator/](terraform/webhook-validator/)) behind +> the existing `edge-security` ALB at +> `https://agent-sandbox.qodolabs.click/invocations`. The Lambda +> fetches the `default/atlassian/webhook-hmac` secret from Secrets +> Manager, verifies HMAC-SHA256 over the raw POST body, and publishes +> the validated `{correlation_id, webhook}` envelope onto the SQS +> `agent-work` queue. The agent-worker Pod consumes from that queue +> and (under `AGENT_PROFILE=aws`) calls AgentCore Runtime via +> `bedrock-agentcore:InvokeAgentRuntime`. +> +> The `agentcore-runtime` Terraform module provisions the IAM +> execution role + ECR repo + log groups that the upstream +> `agentcore deploy` toolkit consumes when minting the +> **AgentCore Memory store** the EKS `agent-worker` Pod uses as its +> LangGraph checkpointer (`AgentCoreMemorySaver`, see +> `src/agent/composition/_shared.py`). The minting recipe +> is documented in §4.3 via +> `[scripts/bootstrap_agentcore_runtime.sh](scripts/bootstrap_agentcore_runtime.sh)`. + +A saved `tfplan` is only valid against the state it was generated from. If a +plan goes stale, re-run `plan_module ` before retrying. + +### 3.2 Populate placeholder secrets (after `apply_module secrets-mcp`) + +Hydrate before pass 2 of `iam` and runtime start. + +#### 3.2a Hydrate the Terraform-managed secrets + +```bash +set -a; source .env.local; set +a + +# MCP bearer — mirrored across both kms-secrets and secrets-mcp paths. +MCP_BEARER=$(openssl rand -hex 32) +aws secretsmanager put-secret-value --region us-east-1 \ + --secret-id default/mcp/internal/bearer-token --secret-string "$MCP_BEARER" >/dev/null +applied_log SECRETS put-secret-value default/mcp/internal/bearer-token "random hex 32 bytes" +aws secretsmanager put-secret-value --region us-east-1 \ + --secret-id mcp/internal/bearer-token --secret-string "$MCP_BEARER" >/dev/null +applied_log SECRETS put-secret-value mcp/internal/bearer-token "matches default/mcp/internal/bearer-token" + +# Jira integration user — JSON blob (email, api_token, site_url). +# Equivalent helper (reads .env.local; no-op if JIRA_* incomplete): +# scripts/sync_jira_integration_secret_from_env.sh +JIRA_JSON=$(jq -nc \ + --arg email "$JIRA_EMAIL" --arg api_token "$JIRA_API_TOKEN" --arg site_url "$JIRA_SITE_URL" \ + '{email:$email, api_token:$api_token, site_url:$site_url}') +aws secretsmanager put-secret-value --region us-east-1 \ + --secret-id default/jira/integration-user --secret-string "$JIRA_JSON" >/dev/null +applied_log SECRETS put-secret-value default/jira/integration-user "json=email,api_token,site_url" + +# Anthropic API key — unused on AWS profile but kms-secrets container must be non-empty. +aws secretsmanager put-secret-value --region us-east-1 \ + --secret-id default/anthropic/api-key --secret-string "$ANTHROPIC_API_KEY" >/dev/null +applied_log SECRETS put-secret-value default/anthropic/api-key "from .env.local" + +# Atlassian webhook HMAC — source of truth is lambda/webhook_validator/.env::WEBHOOK_HMAC_SECRET. +# Syncs to default/atlassian/webhook-hmac for Lambda retrieval. +scripts/sync_webhook_hmac_secret.sh --region us-east-1 +``` + +#### 3.2b Create out-of-band secrets for git / dev-tooling MCPs (Phase 6) + +Additional credentials (GitHub PAT, Git SSH key, Azure DevOps PAT, Snyk +token) for the Phase 6 mcp-internal Pod. Created under the same CMK and +tagged `ManagedBy=out-of-band` so +`[scripts/teardown_sandbox.sh](scripts/teardown_sandbox.sh)` deletes them +before destroying `kms-secrets/`. + +```bash +KMS_KEY_ARN=$(jq -r '.kms_key_arns.value.default' .state/kms-secrets.outputs.json) + +create_oob_secret() { + local name="$1"; local description="$2"; local value="$3" + if aws secretsmanager describe-secret --region us-east-1 --secret-id "$name" >/dev/null 2>&1; then + aws secretsmanager put-secret-value --region us-east-1 \ + --secret-id "$name" --secret-string "$value" >/dev/null + applied_log SECRETS put-secret-value "$name" "updated existing" + else + aws secretsmanager create-secret --region us-east-1 \ + --name "$name" --description "$description" --kms-key-id "$KMS_KEY_ARN" \ + --secret-string "$value" \ + --tags Key=Project,Value=aws-agent-core Key=Environment,Value=sandbox \ + Key=ManagedBy,Value=out-of-band Key=Module,Value=phase-3.2 \ + >/dev/null + applied_log SECRETS create-secret "$name" "out-of-band CMK=$KMS_KEY_ARN" + fi +} + +create_oob_secret "default/github/api-token" \ + "GitHub PAT (GitHub MCP server)." "$GITHUB_API_TOKEN" + +create_oob_secret "default/git/ssh-private-key" \ + "Git SSH private key, base64 (Git MCP server)." "$GIT_SSH_PRIVATE_KEY" + +create_oob_secret "default/azure-devops/integration-user" \ + "Azure DevOps PAT + org URL (Azure DevOps MCP server)." \ + "$(jq -nc --arg org_url "$AZURE_DEVOPS_ORGANIZATION_URL" --arg pat "$AZURE_DEVOPS_PAT" \ + '{organization_url:$org_url, pat:$pat}')" + +create_oob_secret "default/snyk/token" \ + "Snyk API token (Snyk MCP server)." "$SNYK_TOKEN" +``` + +--- + +## Phase 4 — webhook-validator Lambda apply + +> HMAC verification runs in the `webhook-validator` Lambda fronted by +> the existing `edge-security` ALB. The Lambda owns the HMAC trust +> boundary, fetches the secret from Secrets Manager, and invokes +> Bedrock AgentCore Runtime directly with the canonical +> `{correlation_id, webhook}` payload. AgentCore Runtime creation is +> handled by `scripts/bootstrap_agentcore_runtime.sh`. + +### 4.0 Prerequisites resolved by Phase 3 + +The module reads one runtime ARN plus two Terraform outputs and one +Secrets Manager ARN, all already produced by earlier phases: + +- `edge-security` outputs `https_listener_arn` — the ALB listener the +Lambda's target group hangs off. +- `.state/agentcore-runtime.env` exports `AGENTCORE_RUNTIME_ARN` — +the runtime the Lambda invokes. +- `kms-secrets` outputs `kms_key_arns.default` — the CMK that encrypts +`default/atlassian/webhook-hmac`; the Lambda role gets a scoped +`kms:Decrypt` grant on it. +- `kms-secrets` outputs `secret_arns["default/atlassian/webhook-hmac"]` +— feeds `var.secret_arn`; the secret value itself is hydrated in +Phase 3.2a above. + +### 4.1 Apply the module + +**§3.1 / `rebuild_sandbox.sh`:** apply **`webhook-validator` twice** — once after +**`agentcore-memory`** (first package + ALB attachment), and again **after** +**`scripts/sync_sandbox_tfvars.py --phase post-eks`** + second **`kms-secrets`** +apply so the Lambda execution role matches the widened CMK policy. The snippet +below is one plan/apply cycle; repeat for the second pass when following §3.1 by hand. + +```bash +scripts/sync_sandbox_tfvars.py --phase pre-eks # if edge-security / data-stores outputs changed +plan_module webhook-validator +apply_module webhook-validator +``` + +The module zips `lambda/webhook_validator/` into the Lambda archive, +provisions the function with reserved-concurrency 50 and +`runtime=python3.12`, attaches it to a `target_type=lambda` ALB +target group, and adds a listener rule on the existing HTTPS +listener at priority 100 with `path-pattern=/invocations`. + +Inputs ([terraform/webhook-validator/variables.tf](terraform/webhook-validator/variables.tf)): + + +| Variable | Source | Notes | +| -------------------------- | ------------------------------------------------------- | ----------------------------- | +| `https_listener_arn` | `.state/edge-security.outputs.json` | listener the rule attaches to | +| `agentcore_runtime_arn` | `.state/agentcore-runtime.env::AGENTCORE_RUNTIME_ARN` | invoke target | +| `agentcore_endpoint_qualifier` | constant `DEFAULT` | runtime endpoint qualifier | +| `secret_id` / `secret_arn` | `default/atlassian/webhook-hmac` | hydrated in Phase 3.2a | +| `secret_kms_key_arn` | `.state/kms-secrets.outputs.json::kms_key_arns.default` | for KMS decrypt grant | +| `signature_header` | constant `x-jira-signature` | matches Atlassian | + + +### 4.2 Verify + +```bash +LAMBDA_NAME=$(jq -r '.lambda_function_name.value' .state/webhook-validator.outputs.json) +TG_ARN=$(jq -r '.target_group_arn.value' .state/webhook-validator.outputs.json) + +aws lambda get-function --function-name "$LAMBDA_NAME" --region us-east-1 \ + --query 'Configuration.{State:State,LastUpdateStatus:LastUpdateStatus,Memory:MemorySize,Timeout:Timeout}' +# Expect: State=Active, LastUpdateStatus=Successful, Memory=256, Timeout=10 + +aws elbv2 describe-target-health --target-group-arn "$TG_ARN" --region us-east-1 \ + --query 'TargetHealthDescriptions[0].TargetHealth.State' +# Expect: "healthy" — the Lambda's GET / health-check shortcut returns 200. + +aws elbv2 describe-rules --listener-arn "$(jq -r '.https_listener_arn.value' .state/edge-security.outputs.json)" \ + --region us-east-1 \ + --query "Rules[?Conditions[?Field=='path-pattern' && contains(Values, '/invocations')]].Priority" +# Expect: ["100"] +``` + +### 4.3 AgentCore Runtime + Memory (required; scripted) + +The webhook-validator Lambda is the HMAC trust boundary for ingress and +calls AgentCore Runtime directly. AgentCore Memory remains required for +runtime continuity (`AgentCoreMemorySaver(...)`), so this phase still +bootstraps Runtime + Memory identifiers. + +`terraform/agentcore-runtime/` and `terraform/agentcore-memory/` provision +the IAM/KMS/ECR/control-plane prerequisites, but the AgentCore Runtime object +itself is still toolkit-managed today. Use the scripted path below: + +```bash +# Canonical scripted runbook (idempotent). +scripts/bootstrap_agentcore_runtime.sh + +# Sanity check: +aws bedrock-agentcore-control list-agent-runtimes --region us-east-1 --output json \ + | jq -r '.agentRuntimes[]? | select((.agentRuntimeName // .name // "") | startswith("jira_readiness_agent"))' +``` + +> Source-zip filtering nuance: the toolkit's `agentcore configure` / +> `agentcore deploy` uploads a `source.zip` to its CodeBuild bucket +> after applying its bundled `dockerignore.template` (the repo-local +> `.dockerignore` is **ignored**). That template excludes `terraform/`, +> `cdk/`, `tests/`, `docs/`, and `mcp/lambda/` but NOT `mcp/` itself. +> The original size pressure on the upload — the ~62 MB +> `mcp-internal-*.jar` — is gone (the JAR is now resolved from +> outside this repo at build time; see [`mcp/README.md`](mcp/README.md) +> §1 "External JAR contract"). The MCP fat-jar is consumed only by +> `mcp/Dockerfile` (which builds the standalone `mcp-internal` +> container image into its own ECR repo) and nothing in `mcp/` is +> consumed by the AgentCore Runtime container, so for topology +> hygiene `scripts/bootstrap_agentcore_runtime.sh` still stashes +> `mcp/` to a temp dir before invoking the toolkit and restores it +> on EXIT/INT/TERM/HUP via a trap. See +> [`docs/runbook.md` § 7a](docs/runbook.md#7a-worker--agentcore-runtime-invoke-fails-runtimeclienterror) +> for the full rationale. + +Side-effect resources `agentcore deploy` creates outside +Terraform (still tracked by `scripts/teardown_sandbox.sh phase_4_agentcore`): + + +| Resource | Auto-named identifier | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| AgentCore Runtime (no-traffic) | `jira_readiness_agent-<8-char-suffix>` | +| **AgentCore Memory store (STM_ONLY)** | `jira_readiness_agent_mem-<10-char-suffix>` | +| ECR repository | `bedrock-agentcore-jira_readiness_agent` | +| CodeBuild project | `bedrock-agentcore-jira_readiness_agent-builder` | +| CodeBuild execution role | `AmazonBedrockAgentCoreSDKCodeBuild--<10-char-suffix>` | +| S3 source bucket | `bedrock-agentcore-codebuild-sources--` | +| CloudWatch log groups (×2) | `/aws/bedrock-agentcore/runtimes/-DEFAULT`, `/aws/vendedlogs/bedrock-agentcore/memory/APPLICATION_LOGS/` | + + +#### 4.3.1 Decision (implemented) + +This plan now uses option **(c)**: dedicated +`[terraform/agentcore-memory/](terraform/agentcore-memory/)` for Memory +control-plane lifecycle, plus scripted runtime deploy via +`[scripts/bootstrap_agentcore_runtime.sh](scripts/bootstrap_agentcore_runtime.sh)` +for the remaining toolkit-only runtime APIs. + +### 4.4 Populate `.env.local` (operator-tunable inputs) + +`.env.local` is gitignored (`[.gitignore](.gitignore)`) and stores the +operator-identity + behaviour-toggle vars consumed by the EKS worker +ConfigMap (Phase 6). Mandatory keys: + +```bash +# Bot user the recursion guard checks against (Phase 1.4). +AGENT_JIRA_ACCOUNT_ID=712020:036fed7f-fd94-4a58-9cd5-9405e8d955ff +AGENT_JIRA_EMAIL=qodo.jira@your-tenant.atlassian.net +``` + +Optional toggles (defaults match the live sandbox): + +```bash +# Single-tenant pilot toggle. When BOTH are set, AGENT_PROFILE=aws uses +# StaticTenantResolver and skips the agent-tenants DynamoDB read. Leave +# BOTH empty for multi-tenant production (DynamoDbPrefixTenantResolver). +DEFAULT_TENANT=acme +ALLOW_DEFAULT_TENANT_AWS=1 + +# Tracer + boot-time DDB-schema probe. Default otlp + 1. +OBSERVABILITY_BACKEND=otlp +AGENT_BOOT_VALIDATE_SCHEMAS=1 +``` + +The full read-site list lives in +`[src/agent/main.py:444-474](src/agent/main.py)`. + +### 4.5 Production ingress activation (after Phase 5 smoke is green) + +Once Lambda smoke passes end-to-end (Phase 5): + +1. **Repoint Atlassian.** In the Jira webhook console, set the + destination URL to + `https://agent-sandbox.qodolabs.click/invocations`. The shared + secret stays the literal value of + `aws secretsmanager get-secret-value --secret-id default/atlassian/webhook-hmac --query SecretString --output text` + — the Lambda computes HMAC-SHA256 over the raw POST body, so no + re-canonicalisation is needed in the Atlassian config. +2. **Soak.** Watch the + `/aws/lambda/aws-agent-core-sandbox-webhook-validator` log group + for at least 24 hours of real Atlassian deliveries with no 401s + that aren't operator-driven smoke probes. Capture three + correlation ids end-to-end (Lambda log → + AgentCore runtime logs/X-Ray trace). +3. **Keep runtime resources in service.** Do not teardown AgentCore + Runtime/Memory after ingress activation; they are the execution path + for production invokes. + +## Phase 5 — Smoke invoke (ALB + Lambda) + +Sign the canonical fixture with the live HMAC secret and POST it to +the ALB. The `webhook-validator` Lambda verifies the signature against +the raw POST body, mints (or echoes) a correlation id, invokes +AgentCore Runtime with `{correlation_id, webhook}`, and returns 202 + +`{"status":"accepted","correlation_id":"..."}`. + +`[scripts/smoke.py](scripts/smoke.py)` handles the canonicalisation +and the HMAC computation; it consumes `WEBHOOK_HMAC_SECRET` from the +shell or `--secret`. + +```bash +HMAC_SECRET=$(aws secretsmanager get-secret-value \ + --secret-id default/atlassian/webhook-hmac --region us-east-1 \ + --query SecretString --output text | tr -d '\r\n') + +WEBHOOK_HMAC_SECRET="$HMAC_SECRET" \ + python scripts/smoke.py \ + --fixture tests/fixtures/jira/issue_created.json \ + --target https://agent-sandbox.qodolabs.click/invocations +``` + +Expected stdout: + +``` +OK transport=alb status=accepted correlation_id= target=https://agent-sandbox.qodolabs.click/invocations +``` + +A raw `curl` form (useful for embedding in runbooks): + +```bash +BODY='{"correlation_id":"smoke-$(uuidgen)","webhook":{"webhookEvent":"jira:issue_created", ...}}' +SIG="sha256=$(printf '%s' "$BODY" | openssl dgst -sha256 -hmac "$HMAC_SECRET" -hex | awk '{print $NF}')" +curl -sS -X POST https://agent-sandbox.qodolabs.click/invocations \ + -H "Content-Type: application/json" \ + -H "x-jira-signature: $SIG" \ + --data-binary "$BODY" -i +``` + +`HTTP/1.1 202 Accepted` + `{"status":"accepted","correlation_id":"..."}` +confirms HMAC verification + runtime invoke acceptance. + +### 5.1 Verification + +After the smoke succeeds verify: + +- Lambda invocation logs in +`/aws/lambda/aws-agent-core-sandbox-webhook-validator` showing +`webhook_validator.runtime_invoke_ok` with the correlation id. +- Matching runtime logs under `/aws/bedrock-agentcore/runtimes/-DEFAULT` +show the same correlation id trace context. +- ALB access logs show `target_status_code=202`, +`target_processing_time<0.5s`, `request_url=...:/invocations`. +- An invalid HMAC (e.g. `--secret wrong`) returns 401 from the +Lambda with no body. +- `webhook-validator` Lambda's reserved concurrency floor (50) keeps +burst Atlassian webhook traffic from starving other functions in +the account. + +--- + +## Phase 6 — EKS: agent-worker + mcp-internal (async worker path) + +> This phase documents the EKS worker path: SQS → `agent-worker` with +> in-cluster MCP. Production ingress still uses Lambda → AgentCore Runtime +> for the synchronous accept path as described in Phases 4–5. + +### Architecture (after Phase 6 completes) + +``` +Atlassian Jira webhook + │ (HTTPS + HMAC, signed in tenant Connect descriptor) + ▼ +edge-security ALB + WAF + ACM (already deployed, Phase 3) + │ (HTTP/2 → AgentCore endpoint) + ▼ +AgentCore Runtime container — full LangGraph + Bedrock + MCP (same entrypoint as worker) + │ HMAC verify → normalize → tenant resolve → recursion guard → SQS:SendMessage + │ MCP over HTTP to internal NLB → mcp-internal (VPC network mode; Phase 4 + NLB) + ▼ +SQS agent-invoke-queue (already deployed, terraform/data-stores/) + │ + │ ReceiveMessage long-poll (KEDA scales worker on queue depth — see §8) + ▼ +EKS agent-worker Pod — Graph + LLM + MCP (Phase 6.4 — NEW) + │ ──── Pod Identity ──→ IAM role jira-readiness-agent-worker + │ LangGraph compile + assessor/design/approval branches └─ Bedrock Converse, DDB, S3, Secrets, SQS Receive/Delete, CW, X-Ray + │ Bedrock Converse calls + tool-use loop └─ DDB conditional dedupe write (see §9) + │ ClusterIP `http://mcp-internal…svc.cluster.local:8081/mcp` + bearer-token handshake + ▼ +EKS mcp-internal Pod (Phase 6.3 — NEW) ──── Pod Identity ──→ IAM role jira-readiness-mcp-token-reader + │ └─ GetSecretValue on jira/integration-user, mcp-internal/bearer-token, default/git/ssh-private-key + │ HTTPS to api.atlassian.com / *.atlassian.net (FQDN-pinned via NetworkPolicy) + ▼ +Atlassian Jira REST + Confluence + HTTPS git + +Spans + gen_ai metrics from BOTH runtime and worker → ADOT collector DaemonSet + → AWS X-Ray (ServiceLens) + CloudWatch GenAI dashboard (see §7) +``` + +### Design decisions + + +| Decision | Choice | +| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Cluster flavour | EKS Auto Mode (`general-purpose` node pool, amd64). | +| Networking | Reuse `terraform/network/` VPC + private subnets. | +| CNI / NetworkPolicy | VPC CNI built-in, `enableNetworkPolicy=true`. | +| ServiceAccount → IAM | Pod Identity (with IRSA fallback for CSI driver). | +| `agent-worker` IAM | New role `jira-readiness-agent-worker`. Inline: Bedrock invoke, DDB CRUD, S3 PUT, Secrets read, SQS Receive/Delete/CMV/GetQueueAttributes, CW Logs, X-Ray. | +| `mcp-internal` IAM | Extend `jira-readiness-mcp-token-reader` to `GetSecretValue` on `default/jira/integration-user` and `default/git/ssh-private-key`. | +| Worker image | Reuse AgentCore ECR repo (`bedrock-agentcore-jira_readiness_agent`), tag `worker-amd64-*` built with `docker buildx --platform=linux/amd64`. | +| MCP image | New ECR repo `mcp-internal`, built from `mcp/Dockerfile`. | +| Secrets injection | AWS Secrets Store CSI Driver + ASCP, `SecretProviderClass` per Pod, `syncSecret.enabled: true`. | +| AgentCore↔MCP | Runtime uses the same MCP path as the worker. In-cluster workloads use ClusterIP DNS; **Bedrock AgentCore VPC mode** uses `MCP_BASE_URL=http://:8081/mcp` (see `terraform/eks-workloads/` `expose_mcp_internal_nlb` + `scripts/bootstrap_agentcore_runtime.sh`). Terraform locals still document the ClusterIP template URL for parity checks. | + + +### 6.1 Provision EKS Auto Mode (`terraform/eks/`) + +Native `hashicorp/aws` provider; Auto Mode with +`cluster_compute_config = { enabled = true, node_pools = ["general-purpose"] }`. + +Key inputs: + +- `vpc_id`, `private_subnet_ids` from `.state/network.outputs.json`. +- Cluster name `jira-readiness-eks`, Kubernetes `1.35`. +- Namespace `aws-agent-core`. +- `aws_eks_pod_identity_association` for `agent-worker` and `mcp-internal`. + +Required addons + IAM: + +- `aws_eks_addon.pod_identity_agent` (Auto Mode does not auto-install it). +- `aws_iam_openid_connect_provider.eks` for IRSA federation (CSI driver uses +`sts:AssumeRoleWithWebIdentity`). Both `agent-worker` and +`mcp-token-reader` roles trust Pod Identity AND the OIDC provider. Pass +OIDC URL/ARN into `terraform/iam/terraform.tfvars` (`eks_oidc_provider_*`) +before running `iam` pass-3. +- VPC endpoint SG ingress — endpoint SG must allow 443 from EKS cluster SG: + ```bash + aws ec2 authorize-security-group-ingress \ + --group-id "$(jq -r '.vpc_endpoint_sg_id.value' .state/network.outputs.json)" \ + --protocol tcp --port 443 \ + --source-group "$(aws eks describe-cluster --name jira-readiness-eks \ + --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text)" + ``` + +Outputs (`.state/eks.outputs.json`): + +- `cluster_name`, `cluster_endpoint`, `cluster_arn`, `cluster_version` +- `kubeconfig_command` (a literal `aws eks update-kubeconfig …` string) +- `oidc_provider_arn`, `oidc_provider_url`, `oidc_issuer` — feed these +into `terraform/iam/terraform.tfvars` so the `mcp-token-reader` role +picks up IRSA trust on its next apply. + +```bash +scripts/tf.sh eks plan +scripts/tf.sh eks apply -auto-approve + +# kubeconfig +aws eks update-kubeconfig --region us-east-1 --name jira-readiness-eks +kubectl get nodes # should be empty initially — Auto Mode lazily provisions + +jq -r '.oidc_provider_arn.value' .state/eks.outputs.json +jq -r '.oidc_provider_url.value' .state/eks.outputs.json +# Paste both into terraform/iam/terraform.tfvars (eks_oidc_provider_arn / +# eks_oidc_provider_url), then re-run `scripts/tf.sh iam apply`. +``` + +### 6.2 IAM updates (`terraform/iam/agent_worker_role.tf` + extend mcp-token-reader) + +Two changes to `terraform/iam/`: + +1. **NEW** `terraform/iam/agent_worker_role.tf` — sibling of + `agent_runtime_role.tf` / `mcp_token_reader_role.tf`: + - Trust principal `pods.eks.amazonaws.com` (Pod Identity). + - Inline policies copied from `terraform/agentcore-runtime/iam.tf` except + SQS — worker's SQS policy uses `sqs:ReceiveMessage`, + `sqs:DeleteMessage`, `sqs:DeleteMessageBatch`, + `sqs:ChangeMessageVisibility`, `sqs:GetQueueAttributes` against + `agent-invoke-queue` only. +2. **EXTEND** `mcp_token_reader_secret_read` policy in + `mcp_token_reader_role.tf` — `resources` list becomes three secret ARNs + (`mcp-internal/bearer-token`, `default/jira/integration-user`, + `default/git/ssh-private-key`) via `var.mcp_secret_arns = list(string)` + ([terraform/iam/variables.tf](terraform/iam/variables.tf)). + +```bash +scripts/tf.sh iam plan +scripts/tf.sh iam apply -auto-approve # pass-3 of the iam module +``` + +### 6.3 ECR — Terraform-owned repos + image push + +ECR repositories are owned by +`[terraform/ecr-images/](terraform/ecr-images/)`; URLs come from +`.state/ecr-images.outputs.json`. + +```bash +# (a) Provision both ECR repos. Outputs land in .state/ecr-images.outputs.json. +scripts/tf.sh ecr-images plan +scripts/tf.sh ecr-images apply -auto-approve + +ECR_AGENTCORE=$(jq -r '.agentcore_repository_url.value' \ + .state/ecr-images.outputs.json) +ECR_MCP=$(jq -r '.mcp_internal_repository_url.value' \ + .state/ecr-images.outputs.json) + +# (b) docker login (host derived from the URL — no account literal). +ECR_HOST="${ECR_AGENTCORE%%/*}" +aws ecr get-login-password --region "$AWS_REGION" \ + | docker login --username AWS --password-stdin "$ECR_HOST" + +# (c) Build + push the two images. +# The mcp-internal JAR is resolved from outside this repo and staged into +# mcp/.build/ by scripts/build_mcp_image.sh — but for `docker buildx +# --push` we drive the JAR placement explicitly so the buildx command +# below does not need to wrap it. See mcp/README.md §1 for the full +# resolution contract; in practice operators set MCP_INTERNAL_JAR (or +# MCP_INTERNAL_REPO) once and re-source on every revision bump. +: "${MCP_INTERNAL_REPO:=${HOME}/code/github/mcp-internal}" +MCP_INTERNAL_JAR="${MCP_INTERNAL_JAR:-$(ls -t "${MCP_INTERNAL_REPO}/build/libs"/mcp-internal-*.jar | grep -v -- '-plain\.jar$' | head -n 1)}" +mkdir -p mcp/.build +cp "${MCP_INTERNAL_JAR}" mcp/.build/mcp-internal.jar + +docker buildx build --platform=linux/amd64,linux/arm64 \ + -t "${ECR_MCP}:1.0.4" \ + -t "${ECR_MCP}:latest" \ + --push mcp/ + +rm -rf mcp/.build + +docker buildx build --platform=linux/amd64 \ + -t "${ECR_AGENTCORE}:worker-amd64-1.0.2" \ + --push -f Dockerfile . +``` + +> **Preferred path: `scripts/build_agent_worker_image.sh`.** That +> wrapper resolves the repo URL from `.state/ecr-images.outputs.json`, +> handles the `aws ecr get-login-password` step, refuses non-amd64 +> tags, and emits a JSON audit record on stdout. The raw `docker +> buildx build ...` form above is the manual fallback. The +> bootstrap script's `resolve_latest_ecr_tag` discovers the new tag +> automatically — no operator-side env var to update. + +EKS Auto Mode `general-purpose` is amd64-only; worker image tag must be +amd64 (wired via `terraform/eks-workloads/var.agent_worker_image_tag`). + +### 6.4 Render and apply Kubernetes manifests (Terraform-driven) + +Manifests under `deploy/aws/.rendered/` are rendered by Terraform from +upstream module outputs. The bootstrap script wraps the flow: + +```bash +export AGENT_JIRA_ACCOUNT_ID="" +export AGENT_JIRA_EMAIL="" + +scripts/bootstrap_eks_workloads.sh +``` + +Bootstrap behavior notes: +- If `AGENT_WORKER_IMAGE_TAG` / `MCP_INTERNAL_IMAGE_TAG` are unset, the script + auto-resolves the latest tag in each ECR repo from + `.state/ecr-images.outputs.json`. +- After render/apply, it writes + `.state/eks-workloads.outputs.json` (used by final validation). +- If a repo has no tagged images, the script fails fast with an explicit error. + +Manual fallback (each step the script automates): + +```bash +# (1) CSI driver + AWS provider (idempotent). +helm repo add secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts +helm repo add aws-secrets-manager https://aws.github.io/secrets-store-csi-driver-provider-aws +helm repo update +# Install the Secrets Store CSI driver before the AWS provider (ASCP). +helm upgrade --install csi-secrets-store \ + secrets-store-csi-driver/secrets-store-csi-driver \ + --namespace kube-system \ + --set syncSecret.enabled=true \ + --set "tokenRequests[0].audience=sts.amazonaws.com" \ + --set "tokenRequests[1].audience=pods.eks.amazonaws.com" \ + --take-ownership --wait +helm upgrade --install secrets-provider-aws \ + aws-secrets-manager/secrets-store-csi-driver-provider-aws \ + --namespace kube-system \ + --set secrets-store-csi-driver.install=false \ + --wait + +# (2) Render every manifest from Terraform state. +scripts/tf.sh eks-workloads init +scripts/tf.sh eks-workloads apply -auto-approve \ + -var "backend_bucket=$(grep ^bucket terraform/backend.hcl | sed -E 's/.*"([^"]+)".*/\1/')" \ + -var "backend_region=$(grep ^region terraform/backend.hcl | sed -E 's/.*"([^"]+)".*/\1/')" \ + -var "agentcore_runtime_arn=$(grep AGENTCORE_RUNTIME_ARN .state/agentcore-runtime.env | cut -d= -f2)" \ + -var "agentcore_memory_id=$(grep AGENTCORE_MEMORY_ID .state/agentcore-runtime.env | cut -d= -f2)" \ + -var "bedrock_model_id=$(grep BEDROCK_MODEL_ID .state/bootstrap.env | cut -d= -f2)" \ + -var "agent_jira_account_id=${AGENT_JIRA_ACCOUNT_ID}" \ + -var "agent_jira_email=${AGENT_JIRA_EMAIL}" + +# (3) Apply the rendered manifests (namespace + network policies + both +# Deployments come up in dependency order via kustomize). +kubectl apply -k deploy/aws/.rendered/ + +# (4) ADOT Collector (the values file is one of the rendered manifests). +helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts +helm repo update +helm upgrade --install adot \ + open-telemetry/opentelemetry-collector \ + --namespace observability --create-namespace \ + --values deploy/aws/.rendered/observability/adot-values.yaml \ + --wait + +# (5) Force the worker Pod to re-create so it picks up the freshly +# applied ConfigMap (notably AGENTCORE_RUNTIME_ARN). The +# `checksum/agent-worker-config` annotation rendered onto the +# Deployment template (SHA-256 of the ConfigMap body, computed in +# terraform/eks-workloads/manifests.tf) makes K8s do this +# automatically when the ConfigMap body changes; this explicit +# restart is belt-and-braces for degenerate cases (e.g. annotation +# render skipped, manual ConfigMap edit out-of-band). +kubectl -n aws-agent-core rollout restart deploy/agent-worker + +# (6) Wait for both Deployments to land. +kubectl -n aws-agent-core rollout status deploy/mcp-internal --timeout=600s +kubectl -n aws-agent-core rollout status deploy/agent-worker --timeout=600s +``` + +### 6.5 Templates: where to edit manifest content + +Edit `.tftpl` templates under +`[terraform/eks-workloads/templates/](terraform/eks-workloads/templates/)`, +not the rendered output at `deploy/aws/.rendered/` (gitignored). + + +| Template path | Purpose | +| ------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------- | +| `terraform/eks-workloads/templates/namespace.yaml.tftpl` | The `aws-agent-core` namespace. | +| `terraform/eks-workloads/templates/kustomization.yaml.tftpl` | Top-level kustomization aggregating everything below. | +| `terraform/eks-workloads/templates/network-policy/{agent-runtime,mcp-internal}.yaml.tftpl` | L4 NetworkPolicies (FQDN egress is in `deploy/aws/network-policy/cilium-*.yaml`). | +| `terraform/eks-workloads/templates/agent-worker/serviceaccount.yaml.tftpl` | ServiceAccount that Pod-Identity binds to the agent-worker IAM role. | +| `terraform/eks-workloads/templates/agent-worker/configmap.yaml.tftpl` | All non-secret env vars (table names, queue URL, bucket, OTel, etc.). | +| `terraform/eks-workloads/templates/agent-worker/secret-provider-class.yaml.tftpl` | CSI driver wiring for `MCP_BEARER_TOKEN` + `WEBHOOK_HMAC_SECRET`. | +| `terraform/eks-workloads/templates/agent-worker/deployment.yaml.tftpl` | Worker Deployment (image URI, resource caps, mounts). | +| `terraform/eks-workloads/templates/agent-worker/kustomization.yaml.tftpl` | Per-app kustomization. | +| `terraform/eks-workloads/templates/mcp-internal/serviceaccount.yaml.tftpl` | ServiceAccount that Pod-Identity binds to the mcp-token-reader role. | +| `terraform/eks-workloads/templates/mcp-internal/service.yaml.tftpl` | ClusterIP on `:8081/mcp`. | +| `terraform/eks-workloads/templates/mcp-internal/secret-provider-class.yaml.tftpl` | CSI driver wiring for the Jira / git / bearer-token secrets. | +| `terraform/eks-workloads/templates/mcp-internal/deployment.yaml.tftpl` | MCP Deployment (image URI, integration toggles, secretKeyRef). | +| `terraform/eks-workloads/templates/mcp-internal/kustomization.yaml.tftpl` | Per-app kustomization. | +| `terraform/eks-workloads/templates/observability/adot-values.yaml.tftpl` | Helm values for ADOT (IRSA role, region, log group, EMF dimensions). | + + +To plumb a new value: prefer adding an upstream module output and reading +it via `terraform_remote_state` in +`[terraform/eks-workloads/remote-state.tf](terraform/eks-workloads/remote-state.tf)`. +Otherwise add a variable in +`[terraform/eks-workloads/variables.tf](terraform/eks-workloads/variables.tf)` +and pass it via `scripts/bootstrap_eks_workloads.sh`'s `-var` set. + +### 6.6 Worker ConfigMap — value provenance + +`agent-worker-config` keys resolve from Terraform module outputs: + + +| ConfigMap key | Source | +| -------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| `AGENT_PROFILE` | Hard-coded `"aws"`. | +| `AWS_REGION` | `terraform/eks/` `region`. | +| `BEDROCK_MODEL_ID` | `var.bedrock_model_id` (from `.state/bootstrap.env`). | +| `MCP_BASE_URL` | Computed from `terraform/eks/` `namespace`. | +| `WEBHOOK_WORK_QUEUE_URL` | `terraform/data-stores/` `agent_invoke_queue_url`. | +| `DEAD_LETTER_QUEUE_URL` | `terraform/data-stores/` `webhook_dlq_url`. | +| `DOMAIN_STATE_TABLE_NAME` | `terraform/data-stores/` `domain_state_table_name`. | +| `TENANTS_TABLE_NAME` | `terraform/data-stores/` `tenants_table_name`. | +| `RAW_PAYLOAD_BUCKET` | `terraform/dlq-s3/` `raw_payload_bucket_name`. | +| `BEDROCK_GUARDRAIL_IDENTIFIER_ARN` / `BEDROCK_GUARDRAIL_VERSION_ARN` | `terraform/bedrock-guardrail/` `guardrail_identifier_secret_arn`. | +| `OBSERVABILITY_BACKEND` | Hard-coded `"otlp"`. | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | Computed from `terraform/eks/` `adot_collector_namespace`. | +| `BEDROCK_AGENTCORE_MEMORY_ID` | `var.agentcore_memory_id` (from `.state/agentcore-runtime.env`). | +| `AGENT_JIRA_ACCOUNT_ID` / `AGENT_JIRA_EMAIL` | `var.agent_jira_account_id` / `var.agent_jira_email`. | +| `DEFAULT_TENANT` / `ALLOW_DEFAULT_TENANT_AWS` | `var.default_tenant` / `var.allow_default_tenant_aws` (sandbox opt-in). | + + +Sensitive values (`MCP_BEARER_TOKEN`, `WEBHOOK_HMAC_SECRET`) are injected +via the CSI-synced `agent-worker-secrets` Secret. Use worker image tag +`worker-amd64-1.0.2` or later. + +> **ARN drift on rebuild — automatic Pod re-create.** The `agent-worker` +> Deployment carries a `checksum/agent-worker-config` annotation rendered +> with the SHA-256 of the ConfigMap body +> ([`terraform/eks-workloads/manifests.tf`](terraform/eks-workloads/manifests.tf) +> `locals.agent_worker_configmap_checksum`). Any change to a ConfigMap +> value — notably `AGENTCORE_RUNTIME_ARN` after +> `scripts/bootstrap_agentcore_runtime.sh` rebuilds the runtime — flips +> the annotation, Kubernetes treats it as a Pod-template change, and the +> Deployment rolls automatically. `scripts/bootstrap_eks_workloads.sh` +> additionally runs an explicit `kubectl rollout restart +> deploy/agent-worker` as belt-and-braces, and +> `scripts/validate_rebuild_state.sh` §3a fails on any residual drift +> between the live ConfigMap and `.state/agentcore-runtime.env`. See +> [`docs/runbook-sandbox-deploy.md`](docs/runbook-sandbox-deploy.md) +> §11.5 for the canonical re-deploy sequence. + +### 6.7 Smoke + verification + +```bash +QUEUE_URL=$(jq -r '.agent_invoke_queue_url.value' .state/data-stores.outputs.json) + +# (1) Queue depth before — should be 1 from the Phase-5 smoke. +aws sqs get-queue-attributes \ + --queue-url "$QUEUE_URL" \ + --region "$AWS_REGION" --attribute-names ApproximateNumberOfMessages + +# (2) Watch the worker drain. +kubectl -n aws-agent-core logs -f deploy/agent-worker | grep -E 'worker\.consumer\.message_processed|graph\.completed' + +# (3) Queue depth after (≤30s) — should be 0. +aws sqs get-queue-attributes \ + --queue-url "$QUEUE_URL" --region "$AWS_REGION" \ + --attribute-names ApproximateNumberOfMessages \ + | jq '.Attributes.ApproximateNumberOfMessages' + +# (4) Bedrock token usage — should be non-zero now. +aws cloudwatch get-metric-statistics \ + --namespace 'genai/aws-agent-core' \ + --metric-name 'gen_ai.client.token.usage' \ + --start-time "$(date -u -v -10M +%Y-%m-%dT%H:%M:%SZ)" \ + --end-time "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --period 60 --statistics Sum --region "$AWS_REGION" + +# (5) DDB write to agent-domain-state. +aws dynamodb scan --table-name agent-domain-state \ + --region us-east-1 --max-items 5 \ + --query 'Items[*].{tenant:tenant_id.S,issue:issue_key.S,phase:phase.S}' + +# (6) Re-run the Phase-5 smoke and watch it drain end-to-end. +python scripts/smoke.py \ + --fixture tests/fixtures/jira/issue_created.json \ + --runtime-arn "$(grep AGENTCORE_RUNTIME_ARN .state/agentcore-runtime.env | cut -d= -f2)" \ + --region us-east-1 \ + --expect-status accepted +``` + +End-to-end success when (1) → 0 messages remaining, (4) > 0 tokens, +(5) shows the tenant `acme` issue from the smoke fixture, and the +worker logs show a `graph.completed` line per dequeued message. + +--- + +## Phase 7 — Observability: ADOT collector + worker spans + +Deploy the ADOT Collector as a DaemonSet and flip the worker (and runtime) +to export to it. + +### 7.1 Deploy ADOT collector in EKS + +The ADOT Helm values file is rendered by `terraform/eks-workloads/`; +`scripts/bootstrap_eks_workloads.sh` runs the install. + +```bash +helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts +helm repo update + +helm upgrade --install adot \ + open-telemetry/opentelemetry-collector \ + --namespace "$(jq -r '.adot_collector_namespace.value' .state/eks.outputs.json)" \ + --create-namespace \ + --values deploy/aws/.rendered/observability/adot-values.yaml \ + --wait +``` + +Template: +[terraform/eks-workloads/templates/observability/adot-values.yaml.tftpl](terraform/eks-workloads/templates/observability/adot-values.yaml.tftpl). +Pins `public.ecr.aws/aws-observability/aws-otel-collector:v0.42.0`, +runs as DaemonSet, and configures three pipelines: + +- **traces** → `awsxray` exporter (indexed: `tenant_id`, `issue_key`, +`graph_branch`). +- **metrics** → `awsemf` exporter to log group +`/aws/eks/jira-readiness-eks/aws-agent-core/genai` / namespace +`genai/aws-agent-core` (selectors: `gen_ai.client.token.usage`, +`gen_ai.client.operation.duration`, `langgraph.invoke.duration`, +`langgraph.invoke.errors`). `thread_id`, `actor_id`, `correlation_id` +are stripped on the metrics pipeline only. +- **logs/application** → `awscloudwatchlogs` exporter to log group +`/aws/eks/jira-readiness-eks/aws-agent-core/application`. The +`filelog` receiver tails `/var/log/pods/aws-agent-core_*/*/*.log` (the +agent's namespace only — kube-system / observability / other tenants +are excluded), the `container` operator strips the kubelet's +` ` envelope, a `json_parser` operator +opportunistically lifts the structured-log fields the worker emits via +`agent.infrastructure.logging`, and the `k8sattributesprocessor` +enriches every record with `k8s.deployment.name`, `k8s.pod.name`, +`k8s.container.name`, `k8s.namespace.name`, `container.image.name`, +and `container.image.tag`. Streams are keyed +`//` for stable CloudWatch Logs Insights +pivots — equivalent to Container Insights' application-log naming +convention. Tail with: + + ```bash + aws logs tail /aws/eks/jira-readiness-eks/aws-agent-core/application \ + --since 30m --follow + ``` + +The application log group is owned by Terraform +([`terraform/eks/adot.tf::aws_cloudwatch_log_group.adot_application`](terraform/eks/adot.tf)) +with retention pinned to +`var.application_log_retention_in_days` (default 30 days). The EMF +group `/aws/eks//aws-agent-core/genai` is auto-created by the +`awsemfexporter` on first write; both groups fall under the IAM scope +`/aws/eks//aws-agent-core/*` granted to the ADOT IRSA role. + +The `adot-collector` ServiceAccount is IRSA-bound via +`eks.amazonaws.com/role-arn` to `adot_collector_role_arn` +([terraform/eks/adot.tf](terraform/eks/adot.tf)). The same role bears +a chart-rendered ClusterRole for `namespaces`/`pods`/`nodes`/ +`replicasets` (read-only) so `k8sattributesprocessor` can resolve Pod +metadata locally per node — `K8S_NODE_NAME` is fed via fieldRef so the +processor only watches Pods scheduled on the same node as the +collector instance, not the cluster-wide cache. + +### 7.2 Flip worker (and runtime) to OTLP + +Worker ConfigMap values: + +```yaml +OBSERVABILITY_BACKEND: "otlp" +OTEL_EXPORTER_OTLP_ENDPOINT: "http://adot.observability.svc.cluster.local:4318" +OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf" +OTEL_SERVICE_NAME: "agent-worker" +OTEL_RESOURCE_ATTRIBUTES: "service.namespace=aws-agent-core,deployment.environment=sandbox" +``` + +The agent-worker ConfigMap (Phase 6.6) is the only consumer of +`OBSERVABILITY_BACKEND`. The webhook-validator Lambda emits its own +structured logs straight to CloudWatch Logs and does not honour +`OBSERVABILITY_BACKEND`. To flip the worker to OTLP: + +```bash +# In .env.local +OBSERVABILITY_BACKEND=otlp + +# Re-render and apply the agent-worker ConfigMap — the ConfigMap +# template reads OBSERVABILITY_BACKEND from .env.local at apply time. +scripts/tf.sh eks-workloads apply +kubectl rollout restart deploy/agent-worker -n aws-agent-core +``` + +`OBSERVABILITY_BACKEND=otlp` wires the agent's OTel SDK seam to the +in-cluster ADOT collector (Phase 7.1) which then fans out to X-Ray + +- CloudWatch. The `OTEL_EXPORTER_OTLP_ENDPOINT` / `OTEL_SERVICE_NAME` / +`OTEL_RESOURCE_ATTRIBUTES` triplet is **worker-side only** (Phase 6.6 +ConfigMap). The managed Bedrock AgentCore Runtime container (when +deployed for the §4.3 Memory side-effect) manages its own OTLP +transport internally via the toolkit harness and is not configured +through this env block. + +### 7.3 Verify spans + gen_ai metrics land + +```bash +# (a) ADOT collector is healthy +kubectl -n observability rollout status daemonset/adot-opentelemetry-collector --timeout=120s +kubectl -n observability logs daemonset/adot-opentelemetry-collector \ + --tail=20 | grep -E 'TracesExporter|MetricsExporter' + +# (b) Send a fresh smoke and look for graph-side spans in X-Ray. +python scripts/smoke.py \ + --fixture tests/fixtures/jira/issue_created.json \ + --runtime-arn "$(grep AGENTCORE_RUNTIME_ARN .state/agentcore-runtime.env | cut -d= -f2)" \ + --region us-east-1 \ + --expect-status accepted +sleep 60 # let the worker drain + flush + +aws xray get-trace-summaries \ + --start-time "$(date -u -v -10M +%Y-%m-%dT%H:%M:%SZ)" \ + --end-time "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --filter-expression 'service("agent-worker") AND annotation.tenant_id = "acme"' \ + --region us-east-1 \ + --query 'TraceSummaries[*].{id:Id, hasError:HasError, duration:Duration}' +# Expect: at least 1 trace, with sub-spans named assessor.invoke, +# approval.invoke, mcp.tool, bedrock.converse. + +# (c) gen_ai token usage shows up on the GenAI dashboard +aws cloudwatch get-metric-statistics \ + --namespace 'genai/aws-agent-core' \ + --metric-name 'gen_ai.client.token.usage' \ + --start-time "$(date -u -v -10M +%Y-%m-%dT%H:%M:%SZ)" \ + --end-time "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --period 60 --statistics Sum \ + --dimensions Name=service.name,Value=agent-worker \ + --region us-east-1 +# Expect: Datapoints[*].Sum > 0 within 60 sec of worker drain. +``` + +If `gen_ai.client.token.usage` is zero, check the ADOT collector +ServiceAccount has the IRSA role bound via `eks.amazonaws.com/role-arn`. + +--- + +## Phase 8 — Autoscaling: KEDA + Bedrock TPS budget + +KEDA on SQS depth scales the worker 1 → 20 on burst. Set SQS visibility +timeout to 2× p95 graph runtime and run a Bedrock TPS quota pre-flight. + +### 8.1 KEDA on SQS depth + +```bash +helm repo add kedacore https://kedacore.github.io/charts +helm repo update +helm install keda kedacore/keda --namespace keda --create-namespace --wait +``` + +Apply this `ScaledObject` after running +`scripts/bootstrap_eks_workloads.sh`: + +```yaml +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: agent-worker + namespace: aws-agent-core +spec: + scaleTargetRef: + name: agent-worker + minReplicaCount: 1 + maxReplicaCount: 20 + pollingInterval: 15 + cooldownPeriod: 120 + triggers: + - type: aws-sqs-queue + metadata: + # Resolve from .state/data-stores.outputs.json so this carries no account literal: + # queueURL=$(jq -r '.agent_invoke_queue_url.value' .state/data-stores.outputs.json) + queueURL: + queueLength: "5" # target messages per replica + awsRegion: + identityOwner: pod # use the agent-worker Pod's IAM +``` + +### 8.2 SQS visibility timeout = 2× p95 graph runtime + +In [terraform/data-stores/main.tf](terraform/data-stores/main.tf) +`aws_sqs_queue.agent_invoke_queue`, set: + +```hcl +resource "aws_sqs_queue" "agent_invoke_queue" { + name = "agent-invoke-queue" + visibility_timeout_seconds = 600 # 2x p95 graph runtime (~5 min) + message_retention_seconds = 1209600 # 14 days + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.webhook_dlq.arn + maxReceiveCount = 3 + }) +} +``` + +`maxReceiveCount = 3` sends a thrice-failed message to `webhook-dlq`. Pair +with the §9 dedupe table. + +### 8.3 Bedrock TPS quota pre-flight + +Llama 3.1 405B's default RPM ceiling is 100/min in `us-east-1`. Check the +quota pre-flight: + +```bash +QUOTA_CODE=$(aws service-quotas list-service-quotas \ + --service-code bedrock --region us-east-1 \ + --query "Quotas[?contains(QuotaName, 'meta.llama3-1-405b') && contains(QuotaName, 'requests per minute')].QuotaCode | [0]" \ + --output text) +[ -n "$QUOTA_CODE" ] && [ "$QUOTA_CODE" != "None" ] && \ + aws service-quotas get-service-quota \ + --service-code bedrock --quota-code "$QUOTA_CODE" \ + --region us-east-1 --query 'Quota.Value' --output text +``` + +If the printed value is < `(expected_burst_comments * 4 / 30) * 60`, +file a quota increase via: + +```bash +aws service-quotas request-service-quota-increase \ + --service-code bedrock --quota-code "$QUOTA_CODE" \ + --desired-value 500 --region us-east-1 +``` + +`[bedrock_factory.py](src/agent/infrastructure/llm/bedrock_factory.py)` +retries `ThrottlingException` with backoff + jitter. On sustained throttling +KEDA scaling does not help; raise the quota or slow the drain rate. + +### 8.4 EKS Auto Mode capacity ceiling + +Cap node spin-up with a `NodePool` constraint: + +```hcl +resource "kubernetes_manifest" "agent_worker_nodepool" { + manifest = yamldecode(<<-YAML + apiVersion: karpenter.sh/v1 + kind: NodePool + metadata: + name: agent-worker + spec: + template: + spec: + requirements: + - key: karpenter.k8s.aws/instance-family + operator: In + values: ["c7g", "c7i", "m7i"] + - key: karpenter.k8s.aws/instance-size + operator: In + values: ["large", "xlarge"] + limits: + cpu: "20" + memory: 80Gi + YAML + ) +} +``` + +--- + +## Phase 9 — Idempotency: at-least-once SQS dedupe + +SQS delivery is at-least-once. A DDB conditional write keyed on +`correlation_id` bounces duplicate invocations before any LLM call. + +### 9.1 New `agent-dedupe` table + +Append to [terraform/data-stores/main.tf](terraform/data-stores/main.tf): + +```hcl +resource "aws_dynamodb_table" "agent_dedupe" { + name = "agent-dedupe" + billing_mode = "PAY_PER_REQUEST" + hash_key = "correlation_id" + + attribute { + name = "correlation_id" + type = "S" + } + + ttl { + attribute_name = "expires_at" + enabled = true + } + + point_in_time_recovery { enabled = true } + server_side_encryption { enabled = true } + + tags = merge(var.tags, { ManagedBy = "terraform/data-stores" }) +} +``` + +Add to `outputs.tf`: + +```hcl +output "agent_dedupe_table_arn" { + value = aws_dynamodb_table.agent_dedupe.arn +} +output "agent_dedupe_table_name" { + value = aws_dynamodb_table.agent_dedupe.name +} +``` + +### 9.2 Worker IAM grant + +In [terraform/eks/iam.tf](terraform/eks/iam.tf), extend the +`agent_worker_dynamodb` policy resource list with the new table ARN +and add `dynamodb:PutItem` to the actions: + +```hcl +data "aws_iam_policy_document" "agent_worker_dynamodb" { + statement { + sid = "DomainStateAndDedupe" + actions = [ + "dynamodb:GetItem", "dynamodb:Query", "dynamodb:PutItem", + "dynamodb:UpdateItem", "dynamodb:DeleteItem", + "dynamodb:ConditionCheckItem", + ] + resources = [ + var.agent_domain_state_table_arn, + var.agent_tenants_table_arn, + var.token_budgets_table_arn, + var.breaker_table_arn, + var.agent_dedupe_table_arn, # NEW + ] + } +} +``` + +### 9.3 Worker code path + +In [src/agent/worker/main.py](src/agent/worker/main.py) +(or wherever the SQS-receive handler lives), gate every message on a +DDB conditional `PutItem` keyed on the envelope's +`correlation_id`. Sketch: + +```python +import boto3 +from botocore.exceptions import ClientError + +dedupe = boto3.resource("dynamodb").Table(os.environ["DEDUPE_TABLE_NAME"]) +TTL_HOURS = 24 + +def reserve_correlation(correlation_id: str) -> bool: + """Returns True if this is the first time we have seen the id.""" + try: + dedupe.put_item( + Item={ + "correlation_id": correlation_id, + "expires_at": int(time.time()) + TTL_HOURS * 3600, + "first_seen_at": datetime.utcnow().isoformat(), + }, + ConditionExpression="attribute_not_exists(correlation_id)", + ) + return True + except ClientError as exc: + if exc.response["Error"]["Code"] == "ConditionalCheckFailedException": + return False + raise + +# In the consumer loop: +for message in sqs.receive(...): + envelope = json.loads(message["Body"]) + if not reserve_correlation(envelope["correlation_id"]): + log.info("dedupe.skip", correlation_id=envelope["correlation_id"]) + sqs.delete(message["ReceiptHandle"]) + continue + # ... invoke graph ... +``` + +Add `DEDUPE_TABLE_NAME: "agent-dedupe"` to +[terraform/eks-workloads/templates/agent-worker/configmap.yaml.tftpl](terraform/eks-workloads/templates/agent-worker/configmap.yaml.tftpl) +(the rendered ConfigMap will include the new key on the next +`scripts/tf.sh eks-workloads apply`). + +The webhook-validator Lambda is **not** a `DedupeStore.reserve` +caller: it publishes a `{correlation_id, webhook}` envelope onto SQS, +and the worker is the sole consumer of `DEDUPE_TABLE_NAME` on dequeue. +The managed Bedrock AgentCore Runtime container, when deployed for the +§4.3 Memory side-effect, sits off the ingress path entirely; its +composition root at +`[src/agent/composition/aws.py:641-657](src/agent/composition/aws.py)` +still constructs `DynamoDbDedupeStore` lazily from the env-var, but +without traffic the construction has no observable effect. + +### 9.4 Verify + +```bash +# Send the same fixture twice within the 24-hour TTL window: +for _ in 1 2; do + python scripts/smoke.py \ + --fixture tests/fixtures/jira/issue_created.json \ + --runtime-arn "$(grep AGENTCORE_RUNTIME_ARN .state/agentcore-runtime.env | cut -d= -f2)" \ + --region us-east-1 \ + --expect-status accepted +done +sleep 60 + +# Worker logs should show one graph.completed and one dedupe.skip. +kubectl -n aws-agent-core logs deploy/agent-worker \ + | grep -E 'graph\.completed|dedupe\.skip' \ + | tail -5 +# Expect counts: graph.completed=1, dedupe.skip=1. + +# DDB confirms only one row exists for that correlation_id: +aws dynamodb scan --table-name agent-dedupe --region us-east-1 \ + --max-items 5 --query 'Items[*].correlation_id.S' +``` + +--- + +## Phase 10 — WAFv2 rate-limit on the public webhook + +Add a per-source-IP rate limit + Atlassian allowlist on the public ALB +in [terraform/edge-security/](terraform/edge-security/): + +```hcl +resource "aws_wafv2_web_acl" "agent_webhook" { + name = "jira-readiness-agent-webhook" + description = "Per-IP rate limit + Atlassian allowlist for /invocations." + scope = "REGIONAL" + + default_action { allow {} } + + rule { + name = "AtlassianAllowlist-Higher-Cap" + priority = 1 + + action { allow {} } + + statement { + ip_set_reference_statement { + arn = aws_wafv2_ip_set.atlassian_outbound.arn + } + } + + visibility_config { + cloudwatch_metrics_enabled = true + metric_name = "AtlassianAllowlist" + sampled_requests_enabled = true + } + } + + rule { + name = "RateLimit-PerSourceIp" + priority = 10 + + action { block {} } + + statement { + rate_based_statement { + limit = 2000 # requests per 5-minute window per IP + aggregate_key_type = "IP" + + scope_down_statement { + byte_match_statement { + field_to_match { uri_path {} } + positional_constraint = "STARTS_WITH" + search_string = "/invocations" + text_transformation { + priority = 0 + type = "NONE" + } + } + } + } + } + + visibility_config { + cloudwatch_metrics_enabled = true + metric_name = "RateLimit-PerSourceIp" + sampled_requests_enabled = true + } + } + + visibility_config { + cloudwatch_metrics_enabled = true + metric_name = "jira-readiness-agent-webhook" + sampled_requests_enabled = true + } +} + +resource "aws_wafv2_ip_set" "atlassian_outbound" { + name = "atlassian-outbound" + description = "Atlassian Cloud egress IPs (refresh from https://ip-ranges.atlassian.com/)." + scope = "REGIONAL" + ip_address_version = "IPV4" + addresses = [ + "13.52.5.0/25", + "13.236.8.224/28", + # ... see https://support.atlassian.com/organization-administration/docs/ip-addresses-and-domains-for-atlassian-cloud-products/ + ] +} + +resource "aws_wafv2_web_acl_association" "agent_webhook" { + resource_arn = aws_lb.agent.arn + web_acl_arn = aws_wafv2_web_acl.agent_webhook.arn +} +``` + +Verify after `terraform apply edge-security`: + +```bash +aws wafv2 get-web-acl --name jira-readiness-agent-webhook \ + --scope REGIONAL --region us-east-1 \ + --id "$(aws wafv2 list-web-acls --scope REGIONAL --region us-east-1 \ + --query 'WebACLs[?Name==`jira-readiness-agent-webhook`].Id | [0]' --output text)" \ + --query 'WebACL.Rules[*].Name' +# Expect: ["AtlassianAllowlist-Higher-Cap", "RateLimit-PerSourceIp"] + +aws cloudwatch get-metric-statistics \ + --namespace AWS/WAFV2 --metric-name BlockedRequests \ + --dimensions Name=WebACL,Value=jira-readiness-agent-webhook \ + Name=Region,Value=us-east-1 Name=Rule,Value=RateLimit-PerSourceIp \ + --start-time "$(date -u -v -1H +%Y-%m-%dT%H:%M:%SZ)" \ + --end-time "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --period 300 --statistics Sum --region us-east-1 +``` + +--- + +## Phase 11 — Cost projection + +Sandbox scale (1 tenant, 50 issues/day, 5 comments/issue ≈ 7.5K +runs/month). + + +| Component | Idle ($/mo) | Per 1K comments | +| -------------------------------------- | ------------ | ---------------------- | +| EKS Auto Mode control plane | 72 | — | +| EKS worker pods (1 idle, scales to 20) | 5 | 8 | +| AgentCore Runtime (toolkit-managed) | 0 | 3 | +| AgentCore Memory | 0 | 5 | +| Bedrock — Llama 3.1 405B | 0 | 40 | +| DynamoDB (5 tables, on-demand) | 1 | 1 | +| SQS (work + DLQ) | 0 | 0.40 | +| NAT Gateway (single, sandbox) | 33 | 2 | +| CloudWatch Logs + X-Ray (via ADOT) | 5 | 4 | +| Secrets Manager (8 secrets) | 4 | — | +| ALB + WAFv2 | 20 | 2 | +| **Total** | **~$140/mo** | **~$65 / 1K comments** | + + +High-volume (5K comments/day ≈ 150K/mo): **~$140 idle + 150 × $65 ≈ +$9.9K/mo** (Bedrock dominates). Set a CloudWatch daily budget alarm: + +```bash +aws budgets create-budget --account-id "$ACCOUNT_ID" --budget '{ + "BudgetName": "aws-agent-core-daily", + "BudgetLimit": {"Amount": "200", "Unit": "USD"}, + "TimeUnit": "DAILY", + "BudgetType": "COST" +}' --notifications-with-subscribers '[{ + "Notification": {"NotificationType": "ACTUAL", "ComparisonOperator": "GREATER_THAN", "Threshold": 80}, + "Subscribers": [{"SubscriptionType": "EMAIL", "Address": "ops@example.com"}] +}]' +``` + +Bedrock-side caps: `harness_max_tokens` / `harness_max_iterations` in +`terraform/agentcore-runtime/variables.tf` plus the in-process token-budget +enforcer at [src/agent/infrastructure/cost/](src/agent/infrastructure/cost/). + +--- + +## Verification + +End-to-end success means all of the following: + +1. `aws sts get-caller-identity --query Account --output text` returns + `$ACCOUNT_ID` from §1.1. +2. `aws bedrock-runtime converse --model-id "$BEDROCK_MODEL_ID" --messages '[{"role":"user","content":[{"text":"OK"}]}]' --inference-config maxTokens=8` + returns 200. +3. Ten `.state/.outputs.json` files exist with non-empty outputs + (`organizations.outputs.json` is absent by design). Each module also + has a matching `.state/.tfplan` + `.plan.txt` + `.plan.json`. +4. `scripts/tf.sh agentcore-runtime output -raw runtime_arn` returns + `toolkit-managed:jira_readiness_agent`. Real Runtime ARN is in + `.state/agentcore-runtime.env::AGENTCORE_RUNTIME_ARN`; resolve via + `aws bedrock-agentcore-control list-agent-runtimes --query 'agentRuntimes[?agentRuntimeName==\`jira_readiness_agent].agentRuntimeArn'`. +5. Phase 5 smoke returns `{"status": "accepted", ...}` and + `agent-invoke-queue` `ApproximateNumberOfMessages >= 1` immediately + after. +6. Runtime execution role has nine inline policies (`bedrock-invoke`, + `cloudwatch-logs-write`, `cloudwatch-put-metrics`, `dynamodb-access`, + `ecr-pull`, `s3-put-payloads`, `secrets-read`, `sqs-publish`, + `xray-write`). Verify: + `aws iam list-role-policies --role-name jira-readiness-agent-execution`. + `sqs-publish` is gated on `length(var.sqs_queue_arns) > 0`; pin in + `terraform/agentcore-runtime/terraform.tfvars`: +7. Runtime's `authorizer_configuration` is `null`: + `aws bedrock-agentcore-control get-agent-runtime --agent-runtime-id $RUNTIME_ID --query authorizerConfiguration`. +8. `agent-tenants` partition key is `cloud_id`; `agent-domain-state` PK + is `tenant_id`, SK `issue_key`: + `aws dynamodb describe-table --table-name --query 'Table.KeySchema'`. +9. Every `tests/fixtures/jira/` fixture has a non-empty + `webhook.installation.cloudId`: +10. CloudWatch GenAI dashboard shows non-zero `gen_ai.client.token.usage` + after worker drain (Phase 6+). +11. X-Ray shows a complete trace for the invoke. +12. After Phase 6: second invoke shows `mcp.tool` spans against the + deployed MCP. +13. After Phase 7: `kubectl -n observability rollout status daemonset/adot-opentelemetry-collector` + reports `READY` on every node; collector logs show non-zero + `TracesExporter` / `MetricsExporter` counts. +14. After Phase 7: X-Ray ServiceLens trace for a worker-side invoke + contains sub-spans `assessor.invoke`, `approval.invoke`, `mcp.tool`, + `bedrock.converse`: + `aws xray get-trace-summaries --filter-expression 'service("agent-worker")'`. + GenAI dashboard `gen_ai.client.token.usage` `Sum > 0` within 60s of + drain. + +14a. Rendered manifests carry no operator literals: + `bash grep -RnE '\$\{[A-Z_]+\}|[0-9]{12}\.dkr\.ecr|sqs\.[a-z0-9-]+\.amazonaws\.com/[0-9]{12}' \ deploy/aws/.rendered/ \ && echo "FAIL: rendered manifests still contain a literal" \ || echo "PASS: rendered manifests are fully Terraform-driven"` +15. After Phase 8: `kubectl get scaledobject -n aws-agent-core` reports + `READY=True` for `agent-worker`. Pushing 50 messages climbs Pod + count within `pollingInterval` (15s) and drains within + `cooldownPeriod` (120s). Queue attributes show `VisibilityTimeout: 600` + and `maxReceiveCount: 3`: + `aws sqs get-queue-attributes --queue-url --attribute-names All`. +16. After Phase 9: replay same fixture twice within 24 hours; worker logs + show **one** `graph.completed` and **one** `dedupe.skip`; + `aws dynamodb scan --table-name agent-dedupe` shows one row per + `correlation_id` with `expires_at` set. +17. After Phase 10: `aws wafv2 get-web-acl --name jira-readiness-agent-webhook` + returns `["AtlassianAllowlist-Higher-Cap", "RateLimit-PerSourceIp"]` + and `aws cloudwatch get-metric-statistics --namespace AWS/WAFV2 --metric-name BlockedRequests` is queryable for `RateLimit-PerSourceIp`. +18. Run the scripted post-rebuild gate: + `scripts/validate_rebuild_state.sh --region us-east-1` + and require exit code 0 before declaring the environment green. + +--- + +## Teardown / Undeploy + +Reverse-order recipe for undoing every state-changing action this plan +executes. `apply_module` and scripted/manual actions append to +`.state/applied.log`; `[scripts/teardown_sandbox.sh](scripts/teardown_sandbox.sh)` +uses a fixed phase/module destroy order (the log is audit-only), with +idempotency, dry-run, and prompts. + +### Preservation policy — what is NEVER torn down + + +| Preserved resource | Idle cost | +| --------------------------------------------------------------------------------- | --------- | +| Phase 1.2 Bedrock model access (Console toggles) | $0 | +| Phase 1.3 Route 53 hosted zone (`.state/route53.env::HOSTED_ZONE_ID`) | $0.50/mo | +| Phase 1.3 Route 53 domain registration | $3-$15/yr | +| Phase 1.4 Out-of-band git SSH keypair (`default/git/`*) | $0 | +| Phase 1.5 Terraform state bucket `terraform-state-${ACCOUNT_ID}-${AWS_REGION}-an` | ~$0.02/mo | + + +`default/jira/integration-user` is NOT preserved (destroyed alongside +`kms-secrets`). To delete preserved resources see "Permanently abandon the +sandbox" below; the script never touches them, even with `--yes`. + +### What the recipe covers + + +| Action | How | +| ----------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| Phase 6 EKS workloads + Helm CSI releases | `kubectl delete -k deploy/aws/.rendered/`, `helm uninstall secrets-provider-aws csi-secrets-store -n kube-system` (provider first, then driver) | +| Phase 6 EKS cluster + OIDC provider + Pod Identity Associations | First module destroyed by `phase_3_terraform` | +| Phase 4 AgentCore Runtime + toolkit side-effects | `agentcore destroy --force` + cleanup of runtime stragglers, CodeBuild source bucket/role, and log groups | +| Terraform-owned AgentCore/ECR resources | `agentcore-memory/`, `agentcore-runtime/`, and `ecr-images/` destroyed in `phase_3_terraform` | +| Phase 3 Terraform modules (reverse dependency order) | `scripts/tf.sh destroy -auto-approve` | +| Phase 3.2 Terraform-managed secrets | Destroyed by `kms-secrets/` and `secrets-mcp/` destroy | +| Phase 3.2b Out-of-band secrets (`ManagedBy=out-of-band` tag) | `delete-secret --force-delete-without-recovery`; `default/git/*` skipped | +| Phase 3.3 State history purge | `phase_state_purge` drops noncurrent versions + lock artifacts | + + +### `.state/applied.log` format + +``` + [] () +``` + +Categories: `PHASE_1_3`, `TF`, `SECRETS`, `PHASE_4`, `PHASE_6`, `PHASE_0`. + +### Pre-flight (same env as deploy) + +```bash +cd "$(git rev-parse --show-toplevel)" +[[ -z "${AWS_REGION:-}" || "${AWS_REGION}" = "us-east-1" ]] || { echo "AWS_REGION must be us-east-1" >&2; exit 64; } +[[ -z "${AWS_DEFAULT_REGION:-}" || "${AWS_DEFAULT_REGION}" = "us-east-1" ]] || { echo "AWS_DEFAULT_REGION must be us-east-1" >&2; exit 64; } +export AWS_REGION="us-east-1" +export AWS_DEFAULT_REGION="us-east-1" +export ACCOUNT_ID="${ACCOUNT_ID:-$(aws sts get-caller-identity --query Account --output text)}" +``` + +### Reverse-order recipe (Phase 6 → Phase 0) + +The script runs this; the manual recipe is for reference. + +#### Phase 6 → EKS workloads + cluster + +Drain workloads before destroying the cluster (Pods leaving dangling IRSA +bindings): + +```bash +kubectl config current-context | grep -q jira-readiness-eks || \ + aws eks update-kubeconfig --region us-east-1 --name jira-readiness-eks + +kubectl delete -k deploy/aws/.rendered/ --ignore-not-found=true --timeout=180s + +helm uninstall adot -n observability 2>/dev/null || true +helm uninstall secrets-provider-aws -n kube-system 2>/dev/null || true +helm uninstall csi-secrets-store -n kube-system 2>/dev/null || true + +scripts/tf.sh eks-workloads destroy -auto-approve # purges deploy/aws/.rendered/ +scripts/tf.sh eks destroy -auto-approve # cluster + OIDC provider +``` + +#### Phase 4 → AgentCore Runtime + +```bash +agentcore destroy --agent jira_readiness_agent --force 2>/dev/null || true +# Belt-and-braces — if the toolkit can't find the runtime: +RT_ARN=$(aws bedrock-agentcore-control list-agent-runtimes --region us-east-1 \ + --query 'agentRuntimes[?starts_with(agentRuntimeName, `jira_readiness_agent`)].agentRuntimeArn | [0]' \ + --output text 2>/dev/null) +[[ -n "$RT_ARN" && "$RT_ARN" != "None" ]] && \ + aws bedrock-agentcore-control delete-agent-runtime --region us-east-1 \ + --agent-runtime-arn "$RT_ARN" +``` + +#### Phase 3 → Terraform modules in reverse dependency order + +`eks` first (downstream modules trust its OIDC provider); `data-stores` last. + +```bash +for mod in \ + eks \ + eks-workloads \ + webhook-validator \ + agentcore-memory \ + agentcore-runtime \ + ecr-images \ + observability-cloudwatch-genai \ + observability-xray \ + edge-security \ + iam \ + secrets-mcp \ + dlq-s3 \ + kms-secrets \ + bedrock-guardrail \ + network \ + data-stores; do + scripts/tf.sh "$mod" destroy -auto-approve || \ + echo "WARN: $mod destroy failed (re-run after manual cleanup)" +done +``` + +`dlq-s3` Object Lock may require `s3api put-object-legal-hold` / +`delete-objects --bypass-governance-retention` before destroy succeeds. + +#### Phase 1.3 → Route 53 zone + domain (PRESERVED) + +After `edge-security/` destroys, scrub transient records (ACM +cert-validation TXT, ALB alias). Zone + registration are preserved so the +next apply reuses them. + +```bash +HZ_ID=$(grep -E '^HOSTED_ZONE_ID=' .state/route53.env 2>/dev/null | cut -d= -f2) +DOMAIN_NAME=$(grep -E '^DOMAIN_NAME=' .state/route53.env 2>/dev/null | cut -d= -f2) +[[ -z "$HZ_ID" && -n "$DOMAIN_NAME" ]] && HZ_ID=$(aws route53 list-hosted-zones-by-name \ + --dns-name "$DOMAIN_NAME" --query 'HostedZones[0].Id' --output text 2>/dev/null \ + | sed 's|^/hostedzone/||') + +if [[ -n "$HZ_ID" && "$HZ_ID" != "None" ]]; then + CHANGES=$(aws route53 list-resource-record-sets --hosted-zone-id "$HZ_ID" \ + --query 'ResourceRecordSets[?Type!=`NS` && Type!=`SOA`]' --output json | \ + jq '{Changes: [.[] | {Action: "DELETE", ResourceRecordSet: .}]}') + if [[ "$(echo "$CHANGES" | jq '.Changes | length')" != "0" ]]; then + aws route53 change-resource-record-sets --hosted-zone-id "$HZ_ID" \ + --change-batch "$CHANGES" + fi +fi +``` + +### Permanently abandon the sandbox (manual; script does not run these) + +Deletes preserved resources. Only run when decommissioning for good. + +```bash +# 1) Revoke Bedrock model access in the Console (Model access). + +# 2) Delete the Route 53 hosted zone (after edge-security/ is destroyed). +HZ_ID=$(grep -E '^HOSTED_ZONE_ID=' .state/route53.env | cut -d= -f2) +aws route53 list-resource-record-sets --hosted-zone-id "$HZ_ID" \ + --query 'ResourceRecordSets[?Type!=`NS` && Type!=`SOA`]' --output json \ + | jq '{Changes: [.[] | {Action: "DELETE", ResourceRecordSet: .}]}' \ + | aws route53 change-resource-record-sets --hosted-zone-id "$HZ_ID" --change-batch file:///dev/stdin || true +aws route53 delete-hosted-zone --id "$HZ_ID" + +# 3) Stop the next domain renewal charge. +DOMAIN_NAME=$(grep -E '^DOMAIN_NAME=' .state/route53.env | cut -d= -f2) +aws route53domains disable-domain-auto-renew --region "$AWS_REGION" \ + --domain-name "$DOMAIN_NAME" + +# 4) Force-delete the git SSH keypair OOB secret. +for s in $(aws secretsmanager list-secrets --region "$AWS_REGION" \ + --query 'SecretList[?starts_with(Name, `default/git/`)].Name' \ + --output text); do + aws secretsmanager delete-secret --region "$AWS_REGION" \ + --secret-id "$s" --force-delete-without-recovery >/dev/null +done + +# 5) Delete the Terraform state bucket (after all Phase 3 modules are destroyed). +BUCKET="terraform-state-${ACCOUNT_ID}-${AWS_REGION}-an" +aws s3api list-object-versions --bucket "$BUCKET" --output json \ + | jq -c '{Objects: [.Versions[]?, .DeleteMarkers[]? | {Key, VersionId}]}' \ + | while read -r batch; do + [[ "$(echo "$batch" | jq '.Objects | length')" -gt 0 ]] && \ + aws s3api delete-objects --bucket "$BUCKET" --delete "$batch" + done +aws s3api delete-bucket --bucket "$BUCKET" +``` + +### Single-command teardown + +```bash +./scripts/teardown_sandbox.sh # confirms each destructive action +./scripts/teardown_sandbox.sh --yes # skip prompts (CI / scripted) +./scripts/teardown_sandbox.sh --dry-run # print what would happen, change nothing +``` + +The script reads `.state/applied.log` and appends `[TEARDOWN] complete` on +success. + +--- + +## Bedrock model families + +`BEDROCK_MODEL_ID` is the single switch +(`agent.main.SUPPORTED_BEDROCK_MODEL_FAMILIES`). To switch families on a +redeploy, edit Phase 1.2's `CANDIDATES` array (or `BEDROCK_MODEL_ID` in +`.state/bootstrap.env`) and re-run the probe. After a family flip, validate +Guardrail input shape with `aws bedrock-runtime apply-guardrail`. + +See `[docs/runbook-sandbox-deploy.md](./docs/runbook-sandbox-deploy.md)` +§0.1.1 for the env-var contract. + +--- + +## Appendix C — Env-contract reference + +The agent has **two** Terraform-owned environment surfaces. Both are +100% rendered by Terraform; there is no operator hand-merge step and +no `--env` flag matrix to keep in sync. + +### C.1 Lambda env (3 keys) + +Defined in +`[terraform/webhook-validator/main.tf](terraform/webhook-validator/main.tf)` +(the `aws_lambda_function.webhook_validator` resource): + + +| Key | Source | +| ------------------------- | --------------------------------------------------- | +| `WEBHOOK_HMAC_SECRET_ID` | `var.secret_id` (kms-secrets output) | +| `WEBHOOK_WORK_QUEUE_URL` | `var.work_queue_url` (queues module output) | +| `SIGNATURE_HEADER` | hard-coded default `x-jira-signature` | + + +Verification: + +```bash +LAMBDA_NAME=$(jq -r '.lambda_function_name.value' \ + .state/webhook-validator.outputs.json) +aws lambda get-function-configuration --function-name "$LAMBDA_NAME" \ + --region us-east-1 --query 'Environment.Variables | keys' +# Expect: ["SIGNATURE_HEADER", "WEBHOOK_HMAC_SECRET_ID", "WEBHOOK_WORK_QUEUE_URL"] +``` + +### C.2 Worker ConfigMap + +Per-key provenance is documented in Phase 6.6. Refer there; do not +duplicate the table. Verification: + +```bash +kubectl -n aws-agent-core get configmap agent-worker-config \ + -o jsonpath='{.data}' | jq -r 'keys | length' +# Expect: ~20 keys (matches the Phase 6.6 table). +``` + +### C.3 AgentCore Memory store id + +`BEDROCK_AGENTCORE_MEMORY_ID` is **not** Terraform-owned — the toolkit +mints it via `agentcore deploy` and writes it to +`.state/agentcore-runtime.env`. Phase 6.6's worker ConfigMap then +sources the value from that file via the `agentcore_memory_id` tfvar. +The minting recipe + the candidate "replace the toolkit" +implementations live in §4.3 above; this appendix is deliberately +silent on it because there is no `--env` flag for any operator to +forget. \ No newline at end of file diff --git a/deploy/aws/README.md b/deploy/aws/README.md new file mode 100644 index 0000000..8c76d5c --- /dev/null +++ b/deploy/aws/README.md @@ -0,0 +1,86 @@ +# `deploy/aws/` — AWS-profile Kubernetes manifests + +The agent-worker, mcp-internal, namespace, and ADOT observability +manifests under this tree are **rendered by Terraform**, not committed +as static YAML. The single source of truth for every value (image URI, +queue URL, bucket name, IRSA role ARN, secret ARN, Bedrock guardrail +ARN, AgentCore Memory id, namespace, region) is the upstream Terraform +modules' state. + +## Directory map + +``` +deploy/aws/ +├── README.md ← you are here +├── network-policy/ ← static, hand-edited (Cilium L7 examples) +│ ├── README.md +│ ├── cilium-agent-runtime.yaml +│ └── cilium-mcp-internal.yaml +└── .rendered/ ← gitignored; produced by `scripts/tf.sh eks-workloads apply` + ├── kustomization.yaml + ├── namespace.yaml + ├── network-policy/{agent-runtime,mcp-internal}.yaml + ├── agent-worker/{serviceaccount,configmap,secret-provider-class,deployment,kustomization}.yaml + ├── mcp-internal/{serviceaccount,service,secret-provider-class,deployment,kustomization}.yaml + └── observability/adot-values.yaml ← Helm values for ADOT collector +``` + +The committed templates that drive the rendering live at: + +``` +terraform/eks-workloads/templates/ +├── kustomization.yaml.tftpl +├── namespace.yaml.tftpl +├── network-policy/{agent-runtime,mcp-internal}.yaml.tftpl +├── agent-worker/*.yaml.tftpl +├── mcp-internal/*.yaml.tftpl +└── observability/adot-values.yaml.tftpl +``` + +## How to apply + +```bash +# 1. Render with Terraform — sources every value from upstream module state. +scripts/tf.sh eks-workloads init +scripts/tf.sh eks-workloads apply -var-file=terraform/eks-workloads/terraform.tfvars + +# 2. Apply the rendered manifests. +kubectl apply -k deploy/aws/.rendered/ +``` + +`scripts/bootstrap_eks_workloads.sh` wraps both steps and additionally +installs the AWS Secrets Store CSI driver Helm charts. + +## Why Terraform-rendered? + +Every `${...}` value in the rendered manifests resolves to either: + +- a `terraform_remote_state` lookup (cluster name, namespace, queue + URL, table names, bucket name, image URIs, IRSA role ARN, guardrail + secret ARN, region, account id), or +- a narrow Terraform variable for the values the AWS SDK does not own + (the toolkit-managed `agentcore_runtime_arn` / `agentcore_memory_id`, + the Bedrock model id resolved by the deploy plan's pre-flight model + probe, and the Atlassian-side `agent_jira_account_id` / + `agent_jira_email`). + +There are zero hard-coded account ids, region literals, or operator +env-var substitutions in the rendered output. A change to the upstream +infrastructure (rename a queue, rotate a secret, swap the cluster) is +re-applied with `scripts/tf.sh eks-workloads apply` + `kubectl apply +-k deploy/aws/.rendered/` — no manual manifest edits. + +## Editing manifest content + +Edit the `*.tftpl` template under `terraform/eks-workloads/templates/`, +not the rendered output. Re-running `scripts/tf.sh eks-workloads apply` +overwrites the rendered file from the template. + +If a new value needs to flow in, expose it as either: + +1. An output on the upstream module, then add it to the + `terraform_remote_state` locals in + `terraform/eks-workloads/remote-state.tf`. +2. A new variable in `terraform/eks-workloads/variables.tf` (only when + the value genuinely cannot be Terraform-owned, e.g. an Atlassian + account id). diff --git a/deploy/aws/network-policy/cilium-agent-runtime.yaml b/deploy/aws/network-policy/cilium-agent-runtime.yaml new file mode 100644 index 0000000..05a9c11 --- /dev/null +++ b/deploy/aws/network-policy/cilium-agent-runtime.yaml @@ -0,0 +1,73 @@ +# CiliumNetworkPolicy with FQDN-scoped egress for agent-runtime pods. +# +# This manifest is the L7 / FQDN counterpart of the +# ``NetworkPolicy``-only file ``agent-runtime.yaml`` in this directory. +# A Kubernetes ``NetworkPolicy`` cannot match on hostnames, so the +# CIDR rule there allows ALL external IPs on TCP/443. This Cilium +# policy narrows that to the explicit FQDNs the agent runtime needs: +# +# * ``api.atlassian.com`` and ``*.atlassian.net`` — webhook + Jira REST. +# * ``bedrock-runtime..amazonaws.com`` — LLM invocation. +# * ``sts.amazonaws.com`` and ``sts..amazonaws.com`` — IAM +# role-assumption / boto3 metadata refresh. +# +# Apply with:: +# +# kubectl apply -f deploy/aws/network-policy/cilium-agent-runtime.yaml +# +# Cilium DNS-aware policy enforcement requires the ``ToFQDNs`` proxy +# feature; verify your cluster ships it via:: +# +# cilium config view | grep -i tofqdns +# cilium connectivity test --dry-run + +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: agent-runtime-egress-fqdn + namespace: aws-agent-core + labels: + app.kubernetes.io/name: agent-runtime + app.kubernetes.io/part-of: aws-agent-core +spec: + endpointSelector: + matchLabels: + app: agent-runtime + egress: + # 1. In-cluster MCP server on TCP/8081 (delegated to the L3/L4 + # NetworkPolicy in agent-runtime.yaml; mirrored here for clarity). + - toEndpoints: + - matchLabels: + app: mcp-internal + toPorts: + - ports: + - port: "8081" + protocol: TCP + # 2. Cluster DNS so ``ToFQDNs`` resolution itself is not blocked. + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": kube-system + "k8s:k8s-app": kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + - port: "53" + protocol: TCP + rules: + dns: + - matchPattern: "*" + # 3. FQDN-pinned external egress on TCP/443. + - toFQDNs: + - matchName: api.atlassian.com + - matchPattern: "*.atlassian.com" + - matchPattern: "*.atlassian.net" + - matchPattern: "bedrock-runtime.*.amazonaws.com" + - matchPattern: "bedrock-agentcore.*.amazonaws.com" + - matchName: sts.amazonaws.com + - matchPattern: "sts.*.amazonaws.com" + - matchName: api.anthropic.com + toPorts: + - ports: + - port: "443" + protocol: TCP diff --git a/deploy/aws/network-policy/cilium-mcp-internal.yaml b/deploy/aws/network-policy/cilium-mcp-internal.yaml new file mode 100644 index 0000000..3c552f7 --- /dev/null +++ b/deploy/aws/network-policy/cilium-mcp-internal.yaml @@ -0,0 +1,53 @@ +# CiliumNetworkPolicy with FQDN-scoped egress for mcp-internal pods. +# +# The mcp-internal sidecar is a thinner egress surface than the +# agent-runtime: it only needs to reach Atlassian Cloud (Jira REST + +# OAuth token endpoints). This policy pins egress to those FQDNs +# only. Any future provider (Confluence, Bitbucket, etc.) must be +# added explicitly here so the surface stays auditable. +# +# Apply with:: +# +# kubectl apply -f deploy/aws/network-policy/cilium-mcp-internal.yaml +# +# Verify FQDN resolution path with:: +# +# cilium connectivity test --dry-run + +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: mcp-internal-egress-fqdn + namespace: aws-agent-core + labels: + app.kubernetes.io/name: mcp-internal + app.kubernetes.io/part-of: aws-agent-core +spec: + endpointSelector: + matchLabels: + app: mcp-internal + egress: + # Cluster DNS for ToFQDNs resolution. + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": kube-system + "k8s:k8s-app": kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + - port: "53" + protocol: TCP + rules: + dns: + - matchPattern: "*" + # FQDN-pinned external egress: Atlassian Cloud only. + - toFQDNs: + - matchName: api.atlassian.com + - matchName: auth.atlassian.com + - matchPattern: "*.atlassian.com" + - matchPattern: "*.atlassian.net" + toPorts: + - ports: + - port: "443" + protocol: TCP diff --git a/deploy/aws/network-policy/mcp-internal.yaml b/deploy/aws/network-policy/mcp-internal.yaml deleted file mode 100644 index 9255c20..0000000 --- a/deploy/aws/network-policy/mcp-internal.yaml +++ /dev/null @@ -1,132 +0,0 @@ -# Production NetworkPolicy posture (L7 / FQDN egress restriction). -# -# The internal MCP server is a private, in-cluster service. Its surface -# area mirrors the integration toggles wired in -# `deploy/local/mcp-internal.yaml` (the local profile is the source of -# truth for which JAR features are enabled): -# -# * JIRA_MCP_ENABLED=true -> Atlassian Jira REST API -# * GIT_MCP_ENABLED=true -> HTTPS git fetch/clone -# * CONFLUENCE_MCP_ENABLED=true -> Atlassian Confluence (same site URL -# as Jira; covered by the same FQDNs) -# * GITHUB_MCP_ENABLED=false -> NOT in the allow-list -# * SNYK_MCP_ENABLED=false -> NOT in the allow-list -# * AZURE_DEVOPS_MCP_ENABLED=false -> NOT in the allow-list -# * TERMINAL_MCP_ENABLED=false -> No outbound exec surface -# -# Net surface area: -# - Ingress: TCP 8081 from agent-runtime pods only. -# - Egress: TCP 443 to Atlassian (api.atlassian.com, *.atlassian.net) -# and HTTPS git hosts that the agent's tickets reference, -# plus DNS to CoreDNS. -# -# Important: Kubernetes NetworkPolicy cannot match egress by FQDN, only -# by ipBlock or pod/namespace selector. The 0.0.0.0/0:443 egress rule -# below is therefore overly permissive — it allows the MCP server to -# reach any TLS endpoint, not just the integrations enabled above. -# -# To enforce true FQDN-scoped egress, layer a Cilium L7 policy on top. -# The example below mirrors the local-profile integration matrix; flip -# additional `toFQDNs` blocks back on if the corresponding -# `*_MCP_ENABLED` toggle is moved from "false" to "true" (which also -# requires a matching update to .env.local.example and -# deploy/local/configmap.yaml for the new env var). -# -# Example (apply via `kubectl apply -f` once Cilium is installed): -# -# apiVersion: cilium.io/v2 -# kind: CiliumNetworkPolicy -# metadata: -# name: mcp-internal-fqdn-egress -# namespace: aws-agent-core -# spec: -# endpointSelector: -# matchLabels: -# app: mcp-internal -# egress: -# # JIRA_MCP_ENABLED + CONFLUENCE_MCP_ENABLED (shared site URL). -# - toFQDNs: -# - matchName: api.atlassian.com -# - matchPattern: "*.atlassian.net" -# toPorts: -# - ports: -# - port: "443" -# protocol: TCP -# # GIT_MCP_ENABLED — HTTPS git fetch/clone. SSH-based git is out -# # of scope for the local profile (see mcp/README.md §3.2). Add -# # additional matchPatterns for self-hosted Git providers. -# - toFQDNs: -# - matchPattern: "github.com" -# - matchPattern: "*.githubusercontent.com" -# - matchPattern: "git-codecommit.*.amazonaws.com" -# toPorts: -# - ports: -# - port: "443" -# protocol: TCP -# - toEndpoints: -# - matchLabels: -# k8s:io.kubernetes.pod.namespace: kube-system -# k8s-app: kube-dns -# toPorts: -# - ports: -# - port: "53" -# protocol: ANY -# rules: -# dns: -# - matchPattern: "*" -# -# Apply after the EKS cluster has Calico or Cilium installed: -# -# kubectl apply -f deploy/aws/network-policy/ - -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: mcp-internal - namespace: aws-agent-core - labels: - app.kubernetes.io/name: mcp-internal - app.kubernetes.io/part-of: aws-agent-core - plan.aws-agent-core/section: "12.4-row-5" -spec: - podSelector: - matchLabels: - app: mcp-internal - policyTypes: - - Ingress - - Egress - ingress: - # Only the agent-runtime pods may reach the MCP server. - - from: - - podSelector: - matchLabels: - app: agent-runtime - ports: - - protocol: TCP - port: 8081 - egress: - # TLS to Atlassian (Jira + Confluence) and HTTPS git hosts. FQDN - # scoping is enforced at the Cilium L7 policy layer documented in - # the comment block above; this L4 rule alone is overly permissive - # because Kubernetes NetworkPolicy cannot match by FQDN. - - to: - - ipBlock: - cidr: 0.0.0.0/0 - except: - - 169.254.169.254/32 - ports: - - protocol: TCP - port: 443 - # DNS resolution. - - to: - - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: kube-system - podSelector: - matchLabels: - k8s-app: kube-dns - ports: - - protocol: UDP - port: 53 - - protocol: TCP - port: 53 diff --git a/deploy/local/README.md b/deploy/local/README.md index 421403a..555b77f 100644 --- a/deploy/local/README.md +++ b/deploy/local/README.md @@ -31,27 +31,28 @@ these manifests implement. - **Docker Desktop** with the built-in **Kubernetes** single-node cluster enabled. Verify with `kubectl get nodes`. -- **`kubectl`** v1.28+ (ships with Docker Desktop, or `brew install kubernetes-cli`). -- **Git LFS** -- the `mcp-internal` Spring Boot fat-JAR - ([`mcp/mcp-internal-*.jar`](../../mcp/)) is tracked in LFS via - [`../../.gitattributes`](../../.gitattributes). Without it, the file - on disk is a ~130-byte pointer rather than the real ~62 MB binary - and `make build-mcp` will fail fast with a helpful message. +- **`kubectl`** v1.34.1 (ships with Docker Desktop, or `brew install kubernetes-cli`). +- **`mcp-internal` Spring Boot fat-JAR** (external) -- the JAR is + no longer tracked in this repository. + [`../../scripts/build_mcp_image.sh`](../../scripts/build_mcp_image.sh) + resolves it from outside the repo (default location: + `${MCP_INTERNAL_REPO:-$HOME/code/github/mcp-internal}/build/libs/mcp-internal-*.jar`) + and stages it into `mcp/.build/` for the duration of `docker build`. + Build it once via the upstream Gradle project; full contract in + [`../../mcp/README.md`](../../mcp/README.md) §1. ```bash - brew install git-lfs # Linux: apt-get install git-lfs - git lfs install # one-time, per host - git lfs pull # in this clone, after the initial git clone + git clone "${HOME}/code/github/mcp-internal" + ( cd "${HOME}/code/github/mcp-internal" && ./gradlew bootJar ) + # bumps re-run only the gradlew line; build_mcp_image.sh picks up the newest match ``` - **`awslocal`** *or* the standard `aws` CLI. The `Makefile` auto-detects `awslocal`; if you do not have it, the `aws` CLI with `--endpoint-url=http://localhost:4566` is used instead. (`pip install awscli-local` or `pip install awscli`.) -- **Python 3.10+** -- `make smoke` invokes `scripts/smoke.py` against the +- **Python 3.12+** -- `make smoke` invokes `scripts/smoke.py` against the local stack. -- **Optional**: [`smee`](https://smee.io) CLI and/or [`ngrok`](https://ngrok.com) - for replaying real Jira Cloud webhooks into the local cluster. ## 3. Quickstart @@ -62,25 +63,40 @@ these manifests implement. cp .env.local.example .env.local $EDITOR .env.local # set ANTHROPIC_API_KEY=sk-ant-... -# 2. Build the local mcp-internal image from mcp/mcp-internal-*.jar. -# Skip this if you have already built it; re-run after `git lfs pull` -# bumps the JAR. +# 2. Build the local mcp-internal image. The build script resolves the +# Spring Boot fat-JAR from outside this repo (see mcp/README.md §1) +# and stages it into mcp/.build/ for the docker build. +# Re-run whenever the upstream JAR is rebuilt. make build-mcp -# 3. Bring the stack up (namespace, Secrets, ConfigMap, Deployments). +# 3. Build the agent runtime image (aws-agent-core/agent-runtime:local). +# The agent-runtime Deployment pins this exact tag; without it the +# Pod will sit in ImagePullBackOff because the tag does not exist on +# any public registry. Re-run whenever src/ or pyproject.toml changes +# in a way you want reflected inside the cluster. +make build-agent + +# 4. (Optional but recommended on first boot) pre-pull the public Jaeger +# image into Docker Desktop's daemon so kubelet does not race the pull +# behind a Docker Hub rate-limit. Skip if you have run this before. +# Note the patch suffix (.0): Jaeger no longer publishes short +# MAJOR.MINOR tags (e.g. `1.62`) from the 1.61 release onward. +docker pull jaegertracing/all-in-one:1.62.0 + +# 5. Bring the stack up (namespace, Secrets, ConfigMap, Deployments). make local-up -# 4. (Once LocalStack reports ready) seed the AWS resources. +# 6. (Once LocalStack reports ready) seed the AWS resources. make seed-localstack -# 5. In another terminal, forward the agent port so curl/smoke can reach it. +# 7. In another terminal, forward the agent port so curl/smoke can reach it. make port-forward -# 6. Run the two smoke tests (the fixture corpus lives under tests/fixtures/jira/). +# 8. Run the two smoke tests (the fixture corpus lives under tests/fixtures/jira/). make smoke # happy-path: status=processed make smoke-recursion # bot fixture: status=skipped, reason=agent_account_id -# 7. Open the Jaeger UI and search for the correlation_id printed by `make smoke`. +# 9. Open the Jaeger UI and search for the correlation_id printed by `make smoke`. make jaeger ``` @@ -108,64 +124,124 @@ make jaeger > `make local-up` is idempotent, so re-running it does not clobber > Secrets that were re-applied with `kubectl apply -f -`. -## 4. Tearing it down +## 3.5. Scripted bring-up / tear-down + +The §3 quickstart is the canonical step-by-step. For day-to-day work, +two convenience scripts in this directory automate the same flow: + +| Script | Purpose | Steps it covers from §3 | +|--------|---------|-------------------------| +| [`bootstrap.sh`](./bootstrap.sh) | Build images, apply manifests, inject real Secrets / ConfigMap from `.env.local`, **wait for LocalStack + mcp-internal Ready before seeding**, seed LocalStack, roll the agent runtime, and (by default) background `kubectl port-forward`. | 2-9 plus the post-quickstart Note (real-Secret injection). | +| [`teardown.sh`](./teardown.sh) | Kill any background port-forward (tracked PID + orphan sweep), `make local-down`, wait for the namespace to terminate, optionally strip stuck finalizers, optionally prune the locally-built images. | §4 plus port-forward / finalizer cleanup. | + +Both scripts are re-run safe (every step uses `kubectl apply` or +`--dry-run=client | apply`), follow the project's +`set -euo pipefail` + JSON-audit-line convention, and only call +existing `Makefile` targets — they do **not** introduce a new frozen +command vocabulary. + +### 3.5.1 Quickstart, scripted equivalent ```bash -make local-down +# Cold start (replaces every step in §3 except the .env.local edit). +deploy/local/bootstrap.sh + +make smoke # happy path +make smoke-recursion # bot fixture + +# Done for the day. +deploy/local/teardown.sh ``` -This runs `kubectl delete -k deploy/local/`, which removes the -namespace and every resource in it. The Docker Desktop Kubernetes node -itself is left running so subsequent `make local-up` calls are fast. +### 3.5.2 `bootstrap.sh` flags -## 5. Replaying real Jira webhooks via smee.io / ngrok +```text +deploy/local/bootstrap.sh [--no-build] [--no-pull] [--no-port-forward] + [--no-seed] [--port <8080>] [--help] +``` -This section is a walkthrough for pointing real Atlassian webhook -deliveries at the local cluster. +| Flag | Effect | +|------|--------| +| `--no-build` | Skip `make build-mcp` / `make build-agent`. Use when iterating on manifests only. | +| `--no-pull` | Skip the `docker pull jaegertracing/all-in-one:1.62.0` pre-warm step. | +| `--no-port-forward` | Print the manual `kubectl port-forward` command instead of backgrounding it. The script still applies, seeds, and waits-for-Ready exactly the same way. | +| `--no-seed` | Skip `make seed-localstack` (rarely useful — only when the LocalStack volume already has the schemas). | +| `--port ` | Host port to forward (default `8080`). The Pod always listens on `8080` inside the cluster. | -### 5.1 smee.io (recommended for most developers) +Exit codes: `0` on success, `1` for pre-flight failures (missing tool / +env / k8s context), `2` for build / apply failures, `3` for +wait-for-Ready timeouts. -1. Visit [https://smee.io](https://smee.io) and click **Start a new channel**. - Copy the channel URL (e.g. `https://smee.io/AbCdEfGhIjKlMnOp`). -2. Install the CLI: `npm install -g smee-client`. -3. Run the tunnel (in its own terminal): +When the background forwarder is enabled, its PID is written to +`/tmp/aws-agent-core-port-forward.pid` and stdout/stderr to +`/tmp/aws-agent-core-port-forward.log`; `teardown.sh` cleans both. - ```bash - smee --url https://smee.io/AbCdEfGhIjKlMnOp \ - --target http://localhost:8080/webhooks/jira - ``` +### 3.5.3 `teardown.sh` flags -4. In Atlassian (Jira Cloud → **Settings → System → WebHooks**), register - a webhook with: +```text +deploy/local/teardown.sh [--force] [--prune-images] [--keep-pf-log] [--help] +``` + +| Flag | Effect | +|------|--------| +| `--force` | If the namespace gets stuck in `Terminating` past the wait timeout (LocalStack PV cleanup is the usual culprit), strip its finalizers via the `/finalize` subresource and force-delete. Use only when stuck. | +| `--prune-images` | Also remove `aws-agent-core/agent-runtime:local` and `mcp-internal/server:local` so the next bootstrap is a true cold-cache rebuild. | +| `--keep-pf-log` | Keep `/tmp/aws-agent-core-port-forward.log` for debugging (default: deleted with the PID file). | + +Exit codes: `0` on success (or already-gone), `1` for missing tool, `2` +when the namespace fails to terminate AND `--force` was not passed. - - **URL**: the smee channel URL (`https://smee.io/AbCdEfGhIjKlMnOp`). - - **Secret**: the **same** value as `WEBHOOK_HMAC_SECRET` in your - `.env.local` (default `local-dev-secret`). The HMAC check in - `agent.infrastructure.signature.HmacSha256SignatureVerifier` will - reject mismatched signatures. - - **Events**: the events you want to exercise - (`comment_created`, `jira:issue_created`, `jira:issue_updated`, ...). +### 3.5.4 When to prefer the manual §3 flow -### 5.2 ngrok (if you need a real public TLS endpoint) +The scripted path is a strict superset of §3 — it does not hide any +behavior. Reach for the manual steps when: + +- You want to run `make build-mcp` and `make local-up` in **separate + commits' worth of context**, e.g. you just edited a manifest and + want to re-apply *only* without rebuilding images. +- You are debugging a single step (`make seed-localstack`, + `kubectl rollout status`, …) in isolation and the script's + pre-flight / wait-for-Ready scaffolding is in the way. +- You are demoing the stack end-to-end and want each step's output to + be visible to the audience. + +Otherwise the scripts are the recommended day-to-day loop. + +## 4. Tearing it down ```bash -ngrok http 8080 -# copy the https://*.ngrok-free.app URL that ngrok prints +make local-down ``` -Register that URL (plus `/webhooks/jira`) as the webhook target in -Atlassian, and use the same `WEBHOOK_HMAC_SECRET` as the signing secret. +This runs `kubectl delete -k deploy/local/`, which removes the +namespace and every resource in it. The Docker Desktop Kubernetes node +itself is left running so subsequent `make local-up` calls are fast. + +For a fully scripted tear-down (kills background port-forwards, waits +for the namespace to fully terminate, optional `--force` / +`--prune-images`), see [`./teardown.sh`](./teardown.sh) and §3.5 +above. + +## 5. Connecting real Atlassian Jira (post-deployment) + +Locally, webhook ingress is exercised exclusively via `curl` POSTs from +`scripts/smoke.py` (see `make smoke` / `make smoke-recursion`). Real +Atlassian webhook deliveries are pointed at the **deployed** runtime +endpoint, not the local cluster. The two checklists below apply when +registering the webhook against a real Jira Cloud instance. -### 5.3 Signature parity checklist +### 5.1 Ingress topology -The Atlassian signature header name varies by Atlassian product and by -how the webhook was registered. The local verifier accepts the header -`x-jira-signature` with value `sha256=` (see `scripts/smoke.py` -and `agent.infrastructure.signature`). If Atlassian sends a different -header name, map it in the ingress config before promoting to +The local stack publishes `{correlation_id, webhook}` envelopes +directly onto LocalStack SQS (`agent-work`); the `agent-worker` Pod +consumes them and runs the LangGraph topology in-process. The local +stack has no HTTP entrypoint and no HMAC verification — HMAC is +exercised separately against the deployed `webhook-validator` Lambda +via `scripts/invoke_lambda.sh`, which signs the fixture body with the +`x-jira-signature: sha256=` header that Atlassian sends in production. -### 5.4 Webhook event-narrowing checklist +### 5.2 Webhook event-narrowing checklist Atlassian's webhook console lets you fan out delivery to *every* Jira event kind, but the agent **only knows three**: @@ -199,7 +275,7 @@ restrict** the event matrix to those three kinds. Why this matters: marker; delivering the same event under a different kind label (`comment_updated`, etc.) would bypass the guard entirely. -In `smee.io` registration UI: +In the Atlassian webhook registration UI: ```text Events: comment_created, jira:issue_created, jira:issue_updated @@ -362,8 +438,9 @@ Deployments with their non-host-mounted defaults. | Symptom | Likely cause | Fix | |---------|--------------|-----| -| `Failed to pull image "mcp-internal/server:local"` / `ErrImageNeverPull` | The image has not been built on this host yet. The Pod uses `imagePullPolicy: Never` so kubelet never pulls from a registry — the image must already exist locally. | Run `make build-mcp` (or `scripts/build_mcp_image.sh`). If the build itself fails with "looks like a Git LFS pointer", run `git lfs install && git lfs pull` first. | -| `Failed to pull image "aws-agent-core/agent-runtime:local"` | Same placeholder pattern for the agent runtime image. | Build locally from the repo root once the `Dockerfile` lands in M3; until then, `kubectl get pods` will show `ErrImagePull` -- this is expected during Phase A. | +| `Failed to pull image "mcp-internal/server:local"` / `ErrImageNeverPull` | The image has not been built on this host yet. The Pod uses `imagePullPolicy: Never` so kubelet never pulls from a registry — the image must already exist locally. | Run `make build-mcp` (or `scripts/build_mcp_image.sh`). If the build fails with "no mcp-internal JAR found", build it in the upstream Gradle project first (`( cd "${MCP_INTERNAL_REPO:-$HOME/code/github/mcp-internal}" && ./gradlew bootJar )`) — see [`../../mcp/README.md`](../../mcp/README.md) §1. | +| `ImagePullBackOff` / `Failed to pull image "aws-agent-core/agent-runtime:local"` on the `agent-runtime` Pod | The agent runtime image has not been built on this host yet. The Deployment pins `aws-agent-core/agent-runtime:local`, which is **not** a public registry tag — Kubeadm shares Docker Desktop's daemon, so the image must already exist locally. | Run `make build-agent` (which builds from the repo-root `Dockerfile`), then `kubectl rollout restart -n aws-agent-core-local deploy/agent-runtime`. Verify with `kubectl get pods -n aws-agent-core-local -w` until `agent-runtime-...` shows `1/1 Running`. | +| `ImagePullBackOff` / `Failed to pull image "jaegertracing/all-in-one:..."` (or any other public image) | Docker Hub rate-limits anonymous pulls (100 / 6 hours per source IP); VPN exits and shared corp NATs hit this often. Less common: registry network blocked, or stale credentials in `~/.docker/config.json`. | Pre-pull from your own shell so Docker Desktop authenticates as you (and seeds the image into the daemon kubelet shares): `docker pull jaegertracing/all-in-one:1.62.0` (note the patch suffix: Jaeger no longer publishes short `MAJOR.MINOR` tags from 1.61 onward), then `kubectl rollout restart -n aws-agent-core-local deploy/jaeger`. If the pull itself fails with `toomanyrequests`, run `docker login` with any free Docker Hub account and retry. To unblock the rest of the stack while debugging Jaeger, scale it to zero: `kubectl scale -n aws-agent-core-local deploy/jaeger --replicas=0` (the agent's tracer falls back to a span-to-log shim). | | `make smoke` hangs then errors `connection error ... Is make port-forward running?` | `make port-forward` needs to run in its own terminal (or background) so `http://localhost:8080` resolves to the `agent-runtime` Service. | Open a second terminal and run `make port-forward`, then re-run `make smoke`. | | Jaeger UI shows zero spans | This overlay only ships the Jaeger infrastructure; span creation lands with the agent runtime's tracer wiring. | Expected state when the agent runtime is still a placeholder image. Re-run once the tracer adapter merges. | | `make seed-localstack` exits `Could not connect to the endpoint URL: "http://localhost:4566/"` | LocalStack Pod has not finished its readiness probe (it can take 15-30s on cold start). | Wait until `kubectl get pods -n aws-agent-core-local` shows `localstack-...` as `Ready 1/1`, then re-run -- the target is idempotent. | @@ -397,9 +474,8 @@ pip install -e ".[dev,anthropic]" # lives in .env.local.example; the five below are the minimum # needed for the bare-python loop to boot. export AGENT_PROFILE=local -export AGENT_BUDGET_SCOPE=local-pod-dev # ack per-pod budget scope (M5.9 d) +export AGENT_BUDGET_SCOPE=local-pod-dev # ack per-pod budget scope export ANTHROPIC_API_KEY=sk-ant-... -export WEBHOOK_HMAC_SECRET=local-dev-secret export OBSERVABILITY_BACKEND=stdout # avoid Jaeger/X-Ray in this loop # 4. Boot the FastAPI runtime on http://localhost:8080. The @@ -413,10 +489,9 @@ make smoke-recursion ``` When you need the full MCP-driven flow (real Jira reads, real Git -introspection, the `count_for_prefix("git_") >= 1` invariant that -[`event_invariants.py`](../../src/agent/application/event_invariants.py) -enforces on `issue_updated`), come back to §3 and use the -Kubernetes-hosted stack. +introspection, live ``jira_get_issue`` calls — which carry the full +comment thread under ``fields.comment.comments`` — on every +webhook), come back to §3 and use the Kubernetes-hosted stack. ### Iterative-development reference diff --git a/deploy/local/agent-runtime.yaml b/deploy/local/agent-runtime.yaml deleted file mode 100644 index b2a79d1..0000000 --- a/deploy/local/agent-runtime.yaml +++ /dev/null @@ -1,118 +0,0 @@ -# Agent runtime Pod -- bedrock-agentcore SDK + LangGraph + WebhookHandler -# for the local Kubernetes profile. -# -# NOTE: `command: ["python", "-m", "agent.main"]` imports `agent.main`. The -# manifest is validated by `kubectl ... --dry-run=client` even before that -# module exists on disk; `make local-up` will CrashLoopBackOff until the -# entrypoint module ships, by design. -# -# Env wiring: -# - Non-secret values via envFrom: ConfigMapRef: mcp-internal-config -# - Secret values (ANTHROPIC_API_KEY, MCP_BEARER_TOKEN) via secretKeyRef -# Liveness/readiness hit the AgentCore SDK's built-in /ping endpoint. ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: agent-runtime - namespace: aws-agent-core-local - labels: - app.kubernetes.io/part-of: aws-agent-core - app.kubernetes.io/component: agent-runtime - app.kubernetes.io/name: agent-runtime - profile: local -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app.kubernetes.io/name: agent-runtime - template: - metadata: - labels: - app.kubernetes.io/part-of: aws-agent-core - app.kubernetes.io/component: agent-runtime - app.kubernetes.io/name: agent-runtime - profile: local - spec: - containers: - - name: agent-runtime - # Placeholder image -- the developer builds it locally from the - # repo root (future `scripts/build_local_image.sh`). Production - # image digest is resolved by the AgentCore Runtime promotion - # path together with the live `mcp-internal` image-pin script. - image: "aws-agent-core/agent-runtime:local" - imagePullPolicy: IfNotPresent - # src/agent/main.py is the AgentCore CLI entrypoint per §12.1.6 and - # is delivered by subagent B2. The module exposes `app` at import - # time so `python -m agent.main` runs the SDK's Starlette server. - command: ["python", "-m", "agent.main"] - ports: - - name: http - containerPort: 8080 - protocol: TCP - envFrom: - # Non-secret config shared with the rest of the stack. - - configMapRef: - name: mcp-internal-config - env: - - name: ANTHROPIC_API_KEY - valueFrom: - secretKeyRef: - name: anthropic-api-key - key: ANTHROPIC_API_KEY - - name: MCP_BEARER_TOKEN - valueFrom: - secretKeyRef: - name: mcp-internal-token - key: MCP_BEARER_TOKEN - - name: WEBHOOK_HMAC_SECRET - value: "local-dev-secret" - - name: PYTHONUNBUFFERED - value: "1" - readinessProbe: - # /ping is provided by BedrockAgentCoreApp out of the box. - httpGet: - path: /ping - port: 8080 - initialDelaySeconds: 5 - periodSeconds: 5 - failureThreshold: 6 - livenessProbe: - httpGet: - path: /ping - port: 8080 - initialDelaySeconds: 20 - periodSeconds: 20 - failureThreshold: 6 - resources: - requests: - cpu: "100m" - memory: "256Mi" - limits: - cpu: "1000m" - memory: "1Gi" ---- -apiVersion: v1 -kind: Service -metadata: - name: agent-runtime - namespace: aws-agent-core-local - labels: - app.kubernetes.io/part-of: aws-agent-core - app.kubernetes.io/component: agent-runtime - app.kubernetes.io/name: agent-runtime - profile: local -spec: - # ClusterIP is deliberate: the developer uses `make port-forward` to reach - # http://localhost:8080 from the host. Avoids cluster-wide NodePort drift - # between Docker Desktop releases. - type: ClusterIP - selector: - app.kubernetes.io/name: agent-runtime - ports: - - name: http - port: 8080 - targetPort: 8080 - protocol: TCP diff --git a/deploy/local/agent-worker.yaml b/deploy/local/agent-worker.yaml new file mode 100644 index 0000000..9b6b882 --- /dev/null +++ b/deploy/local/agent-worker.yaml @@ -0,0 +1,93 @@ +# Agent worker Pod -- the sole consumer of the SQS work queue. +# +# Topology (matches the docs/runbook.md "Webhook async dispatch" section): +# +# Atlassian (production) -> ALB -> webhook-validator Lambda +# invoke_lambda.sh (laptop) -> ALB -> webhook-validator Lambda +# (terraform/webhook-validator/) +# | +# v +# smoke.py / invoke_manual.sh -------> SQS agent-work +# (local: sqs://agent-work) (LocalStack queue, 600s +# VisibilityTimeout) +# | +# v +# agent-worker (this Pod) +# | +# v +# MCP / Bedrock / DynamoDB / etc. +# +# HMAC verification lives in the `webhook-validator` Lambda; the local +# stack runs no HTTP entrypoint at all and the worker never sees a +# signature header. +# +# Env wiring: envFrom ConfigMap + secretKeyRef for ANTHROPIC_API_KEY / +# MCP_BEARER_TOKEN. The worker resolves WEBHOOK_WORK_QUEUE_URL off the +# ConfigMap. Liveness/readiness probes are not wired because the worker +# has no HTTP surface; an SQS consumer's "health" is whether it can +# drain the queue, observable via the `worker.consumer.message_processed` +# structured-log line and the OTel counter dashboards. +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agent-worker + namespace: aws-agent-core-local + labels: + app.kubernetes.io/part-of: aws-agent-core + app.kubernetes.io/component: agent-worker + app.kubernetes.io/name: agent-worker + profile: local +spec: + # Single replica locally so the developer sees deterministic + # ordering in `kubectl logs -f deploy/agent-worker`. Production + # scales horizontally (multiple replicas drain the same queue -- + # SQS handles per-message exclusivity via ReceiveMessage). + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: agent-worker + template: + metadata: + labels: + app.kubernetes.io/part-of: aws-agent-core + app.kubernetes.io/component: agent-worker + app.kubernetes.io/name: agent-worker + profile: local + spec: + containers: + - name: agent-worker + image: "aws-agent-core/agent-runtime:local" + imagePullPolicy: IfNotPresent + # `python -m agent.worker` resolves agent/worker/__main__.py, + # which forwards to `agent.worker.main.main()`. Mirrors the + # `python -m agent.main` runtime entrypoint pattern. + command: ["python", "-m", "agent.worker"] + envFrom: + - configMapRef: + name: mcp-internal-config + env: + - name: ANTHROPIC_API_KEY + valueFrom: + secretKeyRef: + name: anthropic-api-key + key: ANTHROPIC_API_KEY + - name: MCP_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: mcp-internal-token + key: MCP_BEARER_TOKEN + - name: PYTHONUNBUFFERED + value: "1" + resources: + # The worker is the side that actually invokes the LLM + + # MCP tools; HMAC verification is the webhook-validator + # Lambda's job, not this Pod's. + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "1000m" + memory: "1Gi" diff --git a/deploy/local/bootstrap.sh b/deploy/local/bootstrap.sh new file mode 100755 index 0000000..45a8aa8 --- /dev/null +++ b/deploy/local/bootstrap.sh @@ -0,0 +1,666 @@ +#!/usr/bin/env bash +############################################################################### +# deploy/local/bootstrap.sh +# +# One-shot bring-up for the local Docker Desktop Kubernetes stack described +# in deploy/local/README.md §3. Replaces the manual quickstart with a +# script that: +# +# 1. Validates prerequisites (kubectl context, docker daemon, .env.local +# exists, required env vars resolved). +# 2. Builds the local container images (idempotent — Docker reuses +# cached layers when nothing has changed). +# 3. Pre-pulls the public Jaeger image so kubelet does not race a pull +# behind a Docker Hub rate-limit. +# 4. Applies the manifests via `make local-up` (creates namespace, +# Secrets, ConfigMap, Deployments). +# 5. Overrides the placeholder Secrets with the real values resolved +# from .env.local (anthropic-api-key + jira-credentials). +# 6. Patches the recursion-guard identity into the ConfigMap +# (AGENT_JIRA_EMAIL / AGENT_JIRA_ACCOUNT_ID). +# 7. Waits for LocalStack and mcp-internal to report Ready before +# seeding AWS resources (so the seed call does not race the +# readiness probes). +# 8. Seeds LocalStack (DynamoDB tables, S3 bucket, SQS queue). +# 9. Rolls agent-worker so it picks up the overridden Secrets/ConfigMap +# and waits for it to come up Ready. +# 10. Optionally starts THREE long-lived `kubectl port-forward`s in +# the background so the developer can reach the running stack +# from the host without further setup: +# - localhost:16686 -> svc/jaeger-ui (trace viewer) +# - localhost:4566 -> svc/localstack (host AWS CLI access) +# - localhost:8330 -> svc/mcp-internal (MCP tool endpoints) +# Each writes its PID + log to /tmp/aws-agent-core-*-pf.{pid,log} +# so teardown.sh can stop them deterministically. +# +# Re-run safe: every step uses `kubectl apply` / `--dry-run=client | apply` +# / Make targets that no-op when the desired state already matches. +# +# Companion: deploy/local/teardown.sh +############################################################################### + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Resolve repo root from this script's location so the script works whether +# invoked as `deploy/local/bootstrap.sh`, `./bootstrap.sh`, or via an +# absolute path. +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." &>/dev/null && pwd)" +NAMESPACE="aws-agent-core-local" +JAEGER_IMAGE="jaegertracing/all-in-one:1.62.0" +PORT_FORWARD_PID_FILE="/tmp/aws-agent-core-port-forward.pid" +ENV_FILE="${REPO_ROOT}/.env.local" + +# Long-lived port-forwards established by this script. All three +# survive past the script's exit (nohup'd) so the developer can +# interact with the running stack without manually re-running +# ``kubectl port-forward`` for each service: +# +# * localstack :4566 -- inspect Dynamo / S3 / SQS state with the +# ``aws`` or ``awslocal`` CLI; also used internally by ``make +# seed-localstack`` which is invoked from the host (not in-cluster). +# * jaeger-ui :16686 -- view request traces in the browser. +# * mcp-internal :8081 -- exposed on host :8330 so MCP inspectors and +# curl can hit the in-cluster MCP server without an exec hop. +# +# Each forward records its pid and stdout/stderr to a per-service file +# under /tmp so teardown.sh (and bootstrap.sh's own re-run logic) can +# kill exactly the right process without guessing. +PORT_FORWARD_LOG="/tmp/aws-agent-core-port-forward.log" +LOCALSTACK_HOST_PORT=4566 +LOCALSTACK_PID_FILE="/tmp/aws-agent-core-localstack-pf.pid" +LOCALSTACK_LOG_FILE="/tmp/aws-agent-core-localstack-pf.log" +JAEGER_HOST_PORT=16686 +JAEGER_PID_FILE="/tmp/aws-agent-core-jaeger-pf.pid" +JAEGER_LOG_FILE="/tmp/aws-agent-core-jaeger-pf.log" +MCP_HOST_PORT=8330 +MCP_PID_FILE="/tmp/aws-agent-core-mcp-pf.pid" +MCP_LOG_FILE="/tmp/aws-agent-core-mcp-pf.log" +# Tables ``make seed-localstack`` is contracted to create. Verified +# post-seed because each ``aws dynamodb create-table`` line in the +# Makefile is wrapped in ``2>/dev/null || true`` (so an existing table +# is idempotent), but the same swallow also hides "endpoint unreachable" +# errors -- so a failed seed can look like success unless tables are +# verified afterward. +EXPECTED_DYNAMODB_TABLES=(domain-state tenants token-budgets breaker) +# Async-dispatch: agent-work + agent-dlq are the SQS queues the +# webhook entrypoint and the agent-worker Pod use to hand off +# validated InvokeEnvelope payloads. ``make seed-localstack`` creates +# both (with the redrive policy from agent-work -> agent-dlq); we +# verify the queue names landed for the same reason we verify the +# DynamoDB tables -- the create-queue commands in the Makefile are +# wrapped in ``|| true`` and would silently no-op against an +# unreachable LocalStack. +EXPECTED_SQS_QUEUES=(agent-dlq agent-work) + +# --------------------------------------------------------------------------- +# Tiny logging helpers (no external deps; respects NO_COLOR). +# --------------------------------------------------------------------------- +if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then + C_BLUE=$'\033[1;34m'; C_GREEN=$'\033[1;32m'; C_YELLOW=$'\033[1;33m' + C_RED=$'\033[1;31m'; C_DIM=$'\033[2m'; C_RESET=$'\033[0m' +else + C_BLUE=""; C_GREEN=""; C_YELLOW=""; C_RED=""; C_DIM=""; C_RESET="" +fi +log() { printf '%s\n' "${C_BLUE}==>${C_RESET} $*"; } +ok() { printf '%s\n' "${C_GREEN}✓${C_RESET} $*"; } +warn() { printf '%s\n' "${C_YELLOW}!${C_RESET} $*" >&2; } +err() { printf '%s\n' "${C_RED}✗${C_RESET} $*" >&2; } +hint() { printf '%s\n' " ${C_DIM}$*${C_RESET}"; } + +# --------------------------------------------------------------------------- +# Port reconciliation helper. +# +# Each long-lived forward needs the same dance: detect whoever currently +# owns the port, kill it iff we recognise it as a kubectl forward, and refuse +# to touch anything else with a hint pointing the developer at the conflicting +# process. +# +# Args: $1 = port number, $2 = human-readable purpose (used in messages) +# Side effects: kills any kubectl port-forward listener on $1; on a +# non-kubectl listener, prints an actionable error and exits 3. +# --------------------------------------------------------------------------- +ensure_port_available_for_kubectl_pf() { + local port="$1" + local purpose="$2" + local listener_pid="" + if ! command -v lsof >/dev/null 2>&1; then + # No lsof = best-effort; let kubectl port-forward fail loudly itself. + return 0 + fi + listener_pid="$(lsof -nP -iTCP:"${port}" -sTCP:LISTEN -t 2>/dev/null | head -n 1 || true)" + if [[ -z "${listener_pid}" ]]; then + return 0 + fi + local listener_cmd + listener_cmd="$(ps -p "${listener_pid}" -o command= 2>/dev/null || echo)" + if [[ "${listener_cmd}" != *"kubectl port-forward"* ]]; then + err "port :${port} is held by a non-kubectl process (pid ${listener_pid}: ${listener_cmd})" + hint "Stop that process and re-run; ${purpose} cannot proceed without :${port}." + exit 3 + fi + warn "killing existing kubectl port-forward on :${port} (pid ${listener_pid}, target: ${listener_cmd##*kubectl port-forward }) — needed for ${purpose}" + kill "${listener_pid}" 2>/dev/null || true + # Wait briefly for the kernel to release the socket so our subsequent + # bind does not race the previous owner's cleanup. + for _ in 1 2 3 4 5; do + if ! lsof -nP -iTCP:"${port}" -sTCP:LISTEN -t >/dev/null 2>&1; then + return 0 + fi + sleep 1 + done + err "port :${port} still occupied 5s after killing pid ${listener_pid}; aborting" + exit 3 +} + +# --------------------------------------------------------------------------- +# Persistent port-forward helper. +# +# Reconciles any existing listener (via the helper above), then nohup's a +# fresh ``kubectl port-forward`` and records its pid + log paths so a +# subsequent re-run of this script (or teardown.sh) can clean it up +# deterministically. The caller is responsible for any service-specific +# health-polling that needs to happen after the forward is up. +# +# Args: +# $1 = service name (without ``svc/`` prefix) +# $2 = host-side port +# $3 = service-side port +# $4 = pid file path +# $5 = log file path +# $6 = human-readable purpose for the reconcile messages +# Side effects: writes pid to $4, log to $5; exits 3 on bind/start failure. +# --------------------------------------------------------------------------- +start_persistent_pf() { + local svc="$1" + local host_port="$2" + local svc_port="$3" + local pid_file="$4" + local log_file="$5" + local purpose="$6" + + ensure_port_available_for_kubectl_pf "${host_port}" "${purpose}" + + # Belt-and-braces: if the pid file references a still-live process + # that survived the lsof reconcile (e.g. died with the socket leaked, + # or bound to a different interface), kill it too. + if [[ -f "${pid_file}" ]]; then + local prev_pid + prev_pid="$(cat "${pid_file}" 2>/dev/null || echo)" + if [[ -n "${prev_pid}" ]] && kill -0 "${prev_pid}" 2>/dev/null; then + warn "killing prior port-forward recorded in ${pid_file} (pid ${prev_pid})" + kill "${prev_pid}" 2>/dev/null || true + fi + rm -f "${pid_file}" + fi + + # Bind to 0.0.0.0 so sibling Docker containers (e.g. dynamodb-admin) + # reaching the host via host.docker.internal can connect; kubectl's + # default 127.0.0.1 bind is unreachable from those containers on + # Docker Desktop. Trade-off: the forward is also reachable from the + # LAN, which is acceptable on a dev laptop behind a firewall. + log "Starting port-forward: 0.0.0.0:${host_port} -> ${svc}:${svc_port}" + nohup kubectl port-forward --address 0.0.0.0 -n "${NAMESPACE}" \ + "svc/${svc}" "${host_port}:${svc_port}" \ + >"${log_file}" 2>&1 & + local pid=$! + echo "${pid}" > "${pid_file}" + sleep 1 + if ! kill -0 "${pid}" 2>/dev/null; then + err "${svc} port-forward exited immediately; see ${log_file}" + if [[ -s "${log_file}" ]]; then + tail -n 5 "${log_file}" | sed 's/^/ /' >&2 + fi + exit 3 + fi + ok "${svc} port-forward running on :${host_port} (pid ${pid}, logs: ${log_file})" +} + +# Counterpart: tear down a persistent forward by reading its pid file. +# Used when --no-port-forward is set (we still need :4566 transiently for +# the seed step but tear it down at the end so the script honours its +# "no persistent forwards" contract). +stop_persistent_pf() { + local pid_file="$1" + local label="$2" + if [[ -f "${pid_file}" ]]; then + local pid + pid="$(cat "${pid_file}" 2>/dev/null || echo)" + if [[ -n "${pid}" ]] && kill -0 "${pid}" 2>/dev/null; then + kill "${pid}" 2>/dev/null || true + ok "stopped ${label} port-forward (pid ${pid})" + fi + rm -f "${pid_file}" + fi +} + +# --------------------------------------------------------------------------- +# CLI flags. +# --------------------------------------------------------------------------- +usage() { + cat <<'USAGE' +deploy/local/bootstrap.sh — bring the local stack up end-to-end + +USAGE: + deploy/local/bootstrap.sh [--no-build] [--no-pull] [--no-port-forward] + [--no-seed] [--port ] [--help] + +OPTIONS: + --no-build Skip `make build-mcp` / `make build-agent` (use + images already on the host). + --no-pull Skip `docker pull` of the Jaeger image. + --no-port-forward Do not background `kubectl port-forward` at the end. + Print the command for the user to run manually. + --no-seed Skip `make seed-localstack`. + --port Deprecated compatibility flag; ignored because the + local stack now publishes work via LocalStack SQS. + --help, -h Show this help and exit. + +ENVIRONMENT: + The script reads .env.local at the repo root. Required variables: + ANTHROPIC_API_KEY, JIRA_SITE_URL, JIRA_EMAIL, JIRA_API_TOKEN, + AGENT_JIRA_EMAIL, AGENT_JIRA_ACCOUNT_ID. + +EXIT CODES: + 0 Stack is up and Ready. + 1 Pre-flight check failed (missing tool / env / k8s context). + 2 A build / apply step failed. + 3 A wait-for-Ready step timed out (Pod did not become Ready). +USAGE +} + +DO_BUILD=1 +DO_PULL=1 +DO_PORT_FORWARD=1 +DO_SEED=1 +HOST_PORT=8080 + +while [[ $# -gt 0 ]]; do + case "$1" in + --no-build) DO_BUILD=0; shift ;; + --no-pull) DO_PULL=0; shift ;; + --no-port-forward) DO_PORT_FORWARD=0; shift ;; + --no-seed) DO_SEED=0; shift ;; + --port) HOST_PORT="$2"; shift 2 ;; + --help|-h) usage; exit 0 ;; + *) err "unknown flag: $1"; usage >&2; exit 1 ;; + esac +done + +if [[ "${HOST_PORT}" != "8080" ]]; then + warn "--port is ignored; the local stack has no agent-runtime HTTP service anymore." +fi + +cd "${REPO_ROOT}" + +# --------------------------------------------------------------------------- +# 1. Pre-flight checks +# --------------------------------------------------------------------------- +log "Pre-flight checks" + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + err "required command not found in PATH: $1" + hint "$2" + exit 1 + fi +} + +require_cmd kubectl "brew install kubernetes-cli" +require_cmd docker "Install Docker Desktop and enable Kubernetes (Settings → Kubernetes)." +require_cmd make "make ships with macOS Xcode CLT (xcode-select --install)." + +if ! docker info >/dev/null 2>&1; then + err "Docker daemon is not reachable." + hint "Start Docker Desktop and retry." + exit 1 +fi + +if ! kubectl cluster-info >/dev/null 2>&1; then + err "kubectl cannot reach a Kubernetes cluster." + hint "Enable Kubernetes in Docker Desktop → Settings → Kubernetes." + hint "Then verify: kubectl config current-context" + exit 1 +fi + +CURRENT_CTX="$(kubectl config current-context 2>/dev/null || echo unknown)" +case "${CURRENT_CTX}" in + docker-desktop|docker-for-desktop|kind-*) : ;; + *) + warn "kubectl context is '${CURRENT_CTX}', not docker-desktop / kind." + warn "Continuing anyway — interrupt now if this is the wrong cluster." + sleep 2 + ;; +esac +ok "kubectl context: ${CURRENT_CTX}" + +if [[ ! -f "${ENV_FILE}" ]]; then + err ".env.local not found at ${ENV_FILE}" + hint "Copy the template: cp .env.local.example .env.local" + hint "Then fill in ANTHROPIC_API_KEY plus the JIRA credentials." + exit 1 +fi +ok "found ${ENV_FILE}" + +# Source .env.local with auto-export so every variable is visible to +# subsequent kubectl create-secret commands. `set +a` immediately after to +# avoid polluting the user's interactive session if they `source` this script. +set -a +# shellcheck disable=SC1090 +source "${ENV_FILE}" +set +a + +require_env() { + local name="$1" + local value="${!name:-}" + if [[ -z "${value}" ]]; then + err "required env var ${name} is unset or empty in ${ENV_FILE}" + exit 1 + fi +} + +require_env ANTHROPIC_API_KEY +require_env JIRA_SITE_URL +require_env JIRA_EMAIL +require_env JIRA_API_TOKEN +require_env AGENT_JIRA_EMAIL +require_env AGENT_JIRA_ACCOUNT_ID +# GIT_SSH_PRIVATE_KEY: base64-encoded OpenSSH private key string. +# Consumed only by the mcp-internal Pod (see deploy/local/mcp-internal.yaml); +# the JAR decodes the base64 at startup and writes /app/.ssh/aws_ecdsa +# itself, so the decoded PEM never lands on the developer's working tree. +require_env GIT_SSH_PRIVATE_KEY +ok "required env vars resolved" + +# --------------------------------------------------------------------------- +# 2. Build images +# --------------------------------------------------------------------------- +if [[ ${DO_BUILD} -eq 1 ]]; then + log "Building local images (mcp-internal + agent-runtime)" + make build-mcp + make build-agent + ok "images built" +else + log "Skipping image builds (--no-build)" +fi + +# --------------------------------------------------------------------------- +# 3. Pre-pull public Jaeger image +# --------------------------------------------------------------------------- +if [[ ${DO_PULL} -eq 1 ]]; then + log "Pre-pulling ${JAEGER_IMAGE}" + if docker pull "${JAEGER_IMAGE}" >/dev/null; then + ok "jaeger image present in Docker Desktop daemon" + else + warn "could not pull ${JAEGER_IMAGE} — kubelet will retry from inside the cluster." + fi +else + log "Skipping jaeger pull (--no-pull)" +fi + +# --------------------------------------------------------------------------- +# 4. Apply manifests +# --------------------------------------------------------------------------- +log "Applying manifests via 'make local-up'" +make local-up +ok "manifests applied to namespace ${NAMESPACE}" + +# --------------------------------------------------------------------------- +# 5. Override placeholder Secrets with real values from .env.local +# --------------------------------------------------------------------------- +log "Injecting real Secrets from ${ENV_FILE}" + +kubectl -n "${NAMESPACE}" create secret generic anthropic-api-key \ + --from-literal=ANTHROPIC_API_KEY="${ANTHROPIC_API_KEY}" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null +ok "Secret anthropic-api-key applied" + +kubectl -n "${NAMESPACE}" create secret generic jira-credentials \ + --from-literal=JIRA_SITE_URL="${JIRA_SITE_URL}" \ + --from-literal=JIRA_EMAIL="${JIRA_EMAIL}" \ + --from-literal=JIRA_API_TOKEN="${JIRA_API_TOKEN}" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null +ok "Secret jira-credentials applied" + +# `mcp-git-ssh-key` carries the integration user's OpenSSH private key +# as a single base64-encoded string. We pass the value through verbatim +# (no decoding here) so neither the working tree nor the developer's +# shell ever holds the decoded PEM -- the JAR is responsible for +# `Base64.getDecoder().decode(...)` and writing /app/.ssh/aws_ecdsa. +kubectl -n "${NAMESPACE}" create secret generic mcp-git-ssh-key \ + --from-literal=GIT_SSH_PRIVATE_KEY="${GIT_SSH_PRIVATE_KEY}" \ + --dry-run=client -o yaml | kubectl apply -f - >/dev/null +ok "Secret mcp-git-ssh-key applied" + +# --------------------------------------------------------------------------- +# 6. Patch the recursion-guard identity into the ConfigMap +# --------------------------------------------------------------------------- +log "Patching recursion-guard identity into ConfigMap mcp-internal-config" +kubectl -n "${NAMESPACE}" patch configmap mcp-internal-config --type merge \ + -p "{\"data\":{\"AGENT_JIRA_EMAIL\":\"${AGENT_JIRA_EMAIL}\",\"AGENT_JIRA_ACCOUNT_ID\":\"${AGENT_JIRA_ACCOUNT_ID}\"}}" \ + >/dev/null +ok "ConfigMap mcp-internal-config patched" + +# --------------------------------------------------------------------------- +# 7. Wait for LocalStack + mcp-internal to be Ready (so seed and the agent +# do not race the dependencies' readiness probes). +# --------------------------------------------------------------------------- +log "Waiting for LocalStack and mcp-internal to be Ready (timeout 180s each)" + +wait_for_pod_ready() { + local label="$1" + local timeout_seconds="${2:-180}" + if ! kubectl wait --for=condition=Ready pod \ + -n "${NAMESPACE}" \ + -l "app.kubernetes.io/name=${label}" \ + --timeout="${timeout_seconds}s" >/dev/null; then + err "${label} did not become Ready within ${timeout_seconds}s" + kubectl get pods -n "${NAMESPACE}" + kubectl logs -n "${NAMESPACE}" -l "app.kubernetes.io/name=${label}" --tail=40 || true + exit 3 + fi + ok "${label} is Ready" +} + +wait_for_pod_ready localstack +wait_for_pod_ready mcp-internal + +# --------------------------------------------------------------------------- +# 8. Seed LocalStack (idempotent target) +# +# ``make seed-localstack`` runs from the developer host using ``aws +# --endpoint-url=http://localhost:4566 ...``, so it needs an ephemeral +# port-forward to ``svc/localstack``. Every ``aws dynamodb create-table`` +# in the Makefile is wrapped in ``2>/dev/null || true`` (intentional, so a +# re-run is idempotent against an already-seeded stack), but that swallow +# also hides "endpoint unreachable" errors -- which is exactly how the +# previous bug ("seed silently does nothing on a fresh cluster") slipped +# past the script. We harden against that here in three layers: +# +# 1. Reconcile any leftover :4566 listener. +# 2. Open an ephemeral background ``kubectl port-forward`` and *poll* +# LocalStack's health endpoint until it answers, so the seed never +# races a half-attached forward. +# 3. After the seed, exec into the LocalStack Pod and assert the +# contracted DynamoDB tables actually exist. If any are missing, +# fail loudly with the awslocal output instead of pretending success. +# +# A trap ensures the ephemeral forward is torn down even if any step in +# this block fails, so we don't leak a process on the developer's host. +# --------------------------------------------------------------------------- +if [[ ${DO_SEED} -eq 1 ]]; then + log "Seeding LocalStack (Dynamo tables, S3 bucket, SQS queue)" + + # Establish the long-lived :4566 forward up front -- this is the + # forward developers will use after bootstrap exits to inspect + # Dynamo / S3 / SQS state with the host ``aws`` / ``awslocal`` CLI. + # ``make seed-localstack`` (which runs from the host, not in the + # cluster) consumes the same forward. When --no-port-forward is set + # we still open it transiently here because the seed step requires + # it, then close it at the end of the script. + start_persistent_pf "localstack" \ + "${LOCALSTACK_HOST_PORT}" 4566 \ + "${LOCALSTACK_PID_FILE}" "${LOCALSTACK_LOG_FILE}" \ + "LocalStack host access" + + # Poll LocalStack until its edge endpoint answers. The + # ``/_localstack/health`` path is LocalStack's own readiness + # endpoint and returns JSON the moment the gateway is accepting + # requests; using it here closes the seed-races-forward window + # deterministically (a fixed ``sleep`` would not). + pf_ready=0 + for _ in 1 2 3 4 5 6 7 8 9 10; do + if curl -sf -o /dev/null --max-time 1 \ + "http://localhost:${LOCALSTACK_HOST_PORT}/_localstack/health" 2>/dev/null; then + pf_ready=1 + break + fi + sleep 1 + done + if [[ ${pf_ready} -eq 0 ]]; then + err "LocalStack did not answer on :${LOCALSTACK_HOST_PORT} within 10s; seed would silently no-op" + hint "Check the forward log: ${LOCALSTACK_LOG_FILE}" + hint "Check the LocalStack Pod: kubectl -n ${NAMESPACE} logs deploy/localstack --tail=40" + exit 3 + fi + ok "LocalStack reachable on :${LOCALSTACK_HOST_PORT}" + + make seed-localstack + + # Defense in depth: every ``create-table`` is wrapped in ``|| true``, + # so the only way to know the seed actually landed is to ask + # LocalStack directly. Exec into the Pod (bypassing our forward) so + # this assertion holds even if the forward dropped between seed and + # check. + log "Verifying DynamoDB tables landed (${EXPECTED_DYNAMODB_TABLES[*]})" + actual_tables="$(kubectl -n "${NAMESPACE}" exec deploy/localstack -- \ + awslocal dynamodb list-tables 2>&1 || true)" + for table in "${EXPECTED_DYNAMODB_TABLES[@]}"; do + if [[ "${actual_tables}" != *"\"${table}\""* ]]; then + err "expected DynamoDB table '${table}' missing after seed" + err "awslocal list-tables output:" + printf '%s\n' "${actual_tables}" | sed 's/^/ /' >&2 + hint "Re-run with --no-build --no-pull to retry the seed step alone." + exit 3 + fi + done + ok "LocalStack seeded and verified (${#EXPECTED_DYNAMODB_TABLES[@]} tables present)" + + log "Verifying SQS queues landed (${EXPECTED_SQS_QUEUES[*]})" + actual_queues="$(kubectl -n "${NAMESPACE}" exec deploy/localstack -- \ + awslocal sqs list-queues 2>&1 || true)" + for queue in "${EXPECTED_SQS_QUEUES[@]}"; do + if [[ "${actual_queues}" != *"/${queue}\""* ]]; then + err "expected SQS queue '${queue}' missing after seed" + err "awslocal list-queues output:" + printf '%s\n' "${actual_queues}" | sed 's/^/ /' >&2 + hint "Re-run with --no-build --no-pull to retry the seed step alone." + exit 3 + fi + done + ok "LocalStack SQS queues verified (${#EXPECTED_SQS_QUEUES[@]} queues present)" +else + log "Skipping LocalStack seed (--no-seed)" +fi + +# --------------------------------------------------------------------------- +# 9. Restart mcp-internal + agent-worker so they pick up the overridden +# Secrets / ConfigMap, then wait for each to be Ready. +# +# `mcp-internal` is rolled FIRST so the worker never processes queue work +# against a Pod with the placeholder `GIT_SSH_PRIVATE_KEY=FILL-IN-...` from +# secrets.example.yaml. +# --------------------------------------------------------------------------- +log "Rolling mcp-internal to pick up Secrets/ConfigMap overrides" +kubectl rollout restart -n "${NAMESPACE}" deploy/mcp-internal >/dev/null +if ! kubectl rollout status -n "${NAMESPACE}" deploy/mcp-internal --timeout=180s; then + err "mcp-internal rollout did not complete within 180s" + kubectl get pods -n "${NAMESPACE}" + kubectl logs -n "${NAMESPACE}" -l app.kubernetes.io/name=mcp-internal --tail=80 --previous || true + exit 3 +fi +ok "mcp-internal rollout complete" + +log "Rolling agent-worker to pick up Secrets/ConfigMap overrides" +kubectl rollout restart -n "${NAMESPACE}" deploy/agent-worker >/dev/null +if ! kubectl rollout status -n "${NAMESPACE}" deploy/agent-worker --timeout=180s; then + err "agent-worker rollout did not complete within 180s" + kubectl get pods -n "${NAMESPACE}" + kubectl logs -n "${NAMESPACE}" -l app.kubernetes.io/name=agent-worker --tail=80 --previous || true + exit 3 +fi +ok "agent-worker rollout complete" + +# Final state snapshot +log "Final pod state" +kubectl get pods -n "${NAMESPACE}" + +# Clean up the old HTTP runtime forward if it was created by an earlier +# version of this script. The current local stack has no agent-runtime Service. +stop_persistent_pf "${PORT_FORWARD_PID_FILE}" "agent-runtime (legacy prior run)" + +# --------------------------------------------------------------------------- +# 10. Optional background port-forward +# --------------------------------------------------------------------------- +if [[ ${DO_PORT_FORWARD} -eq 1 ]]; then + # jaeger-ui :16686 -- trace viewer. Without this forward, + # http://localhost:16686/ returns connection-refused even though the + # Pod is healthy in-cluster (a previous deployment cycle's exact + # symptom). + start_persistent_pf "jaeger-ui" \ + "${JAEGER_HOST_PORT}" 16686 \ + "${JAEGER_PID_FILE}" "${JAEGER_LOG_FILE}" \ + "Jaeger UI" + + # localstack :4566 -- host AWS CLI access. Always (re)established + # here because: + # * On a fresh bootstrap with seeding enabled, the seed step + # above already opened it; the reconcile in start_persistent_pf + # no-ops cleanly when the existing pf points at the same target. + # * On a re-run with --no-seed, the seed step was skipped and any + # prior :4566 forward may now point at a since-restarted Pod + # (invalid upstream socket). Reconciling here guarantees the + # host always has a working forward to the *current* Pod. + start_persistent_pf "localstack" \ + "${LOCALSTACK_HOST_PORT}" 4566 \ + "${LOCALSTACK_PID_FILE}" "${LOCALSTACK_LOG_FILE}" \ + "LocalStack host access" + + # mcp-internal :8081 -- host access to the in-cluster MCP server so + # the developer can hit its tool endpoints directly (curl, MCP + # inspectors) without exec'ing into the mcp-internal Pod. + start_persistent_pf "mcp-internal" \ + "${MCP_HOST_PORT}" 8081 \ + "${MCP_PID_FILE}" "${MCP_LOG_FILE}" \ + "mcp-internal host access" +else + log "Skipping background port-forwards (--no-port-forward)" + # Honour the "no persistent forwards" contract by reaping every + # forward this script could have left behind on a prior run, plus + # the transient :4566 the seed step opens regardless of this flag. + # Without these stop calls a developer toggling between default + # and --no-port-forward would silently keep stale pf's alive. + stop_persistent_pf "${LOCALSTACK_PID_FILE}" "LocalStack (transient seed)" + stop_persistent_pf "${JAEGER_PID_FILE}" "Jaeger UI (prior run)" + stop_persistent_pf "${MCP_PID_FILE}" "mcp-internal (prior run)" + hint "To forward manually:" + hint " kubectl port-forward -n ${NAMESPACE} svc/jaeger-ui ${JAEGER_HOST_PORT}:16686" + hint " kubectl port-forward -n ${NAMESPACE} svc/localstack ${LOCALSTACK_HOST_PORT}:4566" + hint " kubectl port-forward -n ${NAMESPACE} svc/mcp-internal ${MCP_HOST_PORT}:8081" +fi + +# --------------------------------------------------------------------------- +# Done. +# --------------------------------------------------------------------------- +echo +ok "Local stack is up. Try: make smoke && make smoke-recursion" +if [[ ${DO_PORT_FORWARD} -eq 1 ]]; then + hint "Jaeger UI: http://localhost:${JAEGER_HOST_PORT}/ (Service: aws-agent-core)" + hint "LocalStack edge: http://localhost:${LOCALSTACK_HOST_PORT} (e.g. awslocal --endpoint-url=http://localhost:${LOCALSTACK_HOST_PORT} dynamodb list-tables)" + hint "MCP server: http://localhost:${MCP_HOST_PORT}/ (svc/mcp-internal :8081)" +fi +hint "Tear down: deploy/local/teardown.sh" +hint "Bootstrap audit: $(printf '{"event":"local_bootstrap","namespace":"%s","timestamp":"%s"}' \ + "${NAMESPACE}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)")" diff --git a/deploy/local/configmap.yaml b/deploy/local/configmap.yaml index 4693658..3e00019 100644 --- a/deploy/local/configmap.yaml +++ b/deploy/local/configmap.yaml @@ -4,9 +4,10 @@ # and are loaded via secretKeyRef -- never in this ConfigMap. # # Source of truth for the Local-vs-AWS configuration cheat sheet (alongside -# .env.local.example at the repo root and the build_*_dependencies factory in -# src/agent/composition.py). Keep all three in lockstep so a developer's -# local shell, the agent container, and documentation agree. +# .env.local.example at the repo root and the build_*_dependencies factories +# in src/agent/composition/ -- local.py, aws.py, _shared.py). Keep all three +# in lockstep so a developer's local shell, the agent container, and +# documentation agree. apiVersion: v1 kind: ConfigMap metadata: @@ -19,7 +20,16 @@ metadata: data: # Composition-root profile selector. AGENT_PROFILE: "local" - # Single-pod-dev acknowledgement gate (M5.9 d). The local profile wires + # JsonStructuredLogger minimum severity threshold. Allowed values: + # `debug`, `info`, `warning`, `error`. Defaults to `info` when unset + # (production behaviour); kept at `debug` here so the local k8s pods + # surface the per-node diagnostic callsites wired across the webhook + # handler and LangGraph nodes (assemble_history_context, load_state, + # llm_node, terminal_assess, request/evaluate_human_approval, + # design_llm_node, design_terminal). Resolved by + # `agent.infrastructure.logging.resolve_log_level`. + STRUCTURED_LOG_LEVEL: "debug" + # Single-pod-dev acknowledgement gate. The local profile wires # `InMemoryTokenBudgetEnforcer`, whose counters live on the process # heap; multi-replica deployments would silently over-permit budgets. # `_resolve_local_dependencies` raises a `ValueError` at startup unless @@ -27,22 +37,51 @@ data: # deployments MUST switch to `AGENT_PROFILE=aws` (cluster-shared # `DynamoDbTokenBudgetEnforcer`) and MUST NOT carry this var. AGENT_BUDGET_SCOPE: "local-pod-dev" + # Local LLM provider. `ollama` runs without ANTHROPIC_API_KEY when an + # Ollama server is reachable at OLLAMA_BASE_URL. + LLM_PROVIDER: "anthropic" + DESIGN_LLM_PROVIDER: "anthropic" + OLLAMA_BASE_URL: "http://ollama:11434" + OLLAMA_MODEL: "llama3.1" # Anthropic model id pinned in lockstep with the Bedrock model id. ANTHROPIC_MODEL: "claude-sonnet-4-5-20250929" # HTTP-streamable MCP endpoint, cluster-internal. MCP_BASE_URL: "http://mcp-internal:8081/mcp" + # Writable workspace absolute path the mcp-internal JVM uses as the + # default MCP `roots/list` advertisement when a client does not + # negotiate `roots` itself. Frozen to `/workspace` because the + # `mcp/Dockerfile` creates that directory (mkdir -p /workspace) and + # chowns it to the runtime `mcp:mcp` user, AND the + # `deploy/local/mcp-internal.yaml` Pod mounts an emptyDir at the + # same path. The agent code does not consume this var (only the + # JVM does) -- it lives in this ConfigMap so the Local-vs-AWS env + # cheat sheet stays in lockstep with `mcp/Dockerfile`, + # `deploy/local/mcp-internal.yaml`, and `mcp/.env`. + MCP_DEFAULT_ROOT_PATH: "/workspace" # LocalStack endpoint, cluster-internal. LOCALSTACK_ENDPOINT_URL: "http://localstack:4566" # AWS region used by every boto3 client factory. AWS_REGION: "us-east-1" + # NOTE: BEDROCK_MODEL_ID is intentionally NOT set in this ConfigMap because + # this profile is `local` (LLM_PROVIDER=anthropic). The AWS profile + # (.bedrock_agentcore.yaml + terraform/agentcore-runtime/) reads + # BEDROCK_MODEL_ID and supports two families today: + # * us.anthropic.claude-sonnet-4-5-20250929-v1:0 (default) + # * deepseek.v3.2 (DeepSeek V3.2, in-region) + # See agent.main.SUPPORTED_BEDROCK_MODEL_FAMILIES. # Dummy AWS creds: LocalStack accepts any non-empty value. AWS_ACCESS_KEY_ID: "test" AWS_SECRET_ACCESS_KEY: "test" # Observability backend selector read by the tracer factory. + # Supported: `jaeger` (default; span-to-log fallback if SDK missing), + # `otlp` (strict OTLP/HTTP; fails loudly on missing SDK or endpoint), + # `noop`. `xray` is AWS-profile-only. OBSERVABILITY_BACKEND: "jaeger" - # OTLP endpoint pointing at the Jaeger all-in-one Pod. - OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger-otlp:4317" - OTEL_EXPORTER_OTLP_PROTOCOL: "grpc" + # OTLP/HTTP endpoint. `OBSERVABILITY_BACKEND=otlp` requires this to + # be reachable; `OBSERVABILITY_BACKEND=jaeger` falls back gracefully. + # Redirect to a real OTel Collector by overriding here. + OTEL_EXPORTER_OTLP_ENDPOINT: "http://jaeger-otlp:4318/v1/traces" + OTEL_EXPORTER_OTLP_PROTOCOL: "http/protobuf" OTEL_SERVICE_NAME: "aws-agent-core" # Agent Jira identity used by the actor-identity recursion guard. # Match the integration user the mcp-internal Pod authenticates as -- the @@ -50,3 +89,20 @@ data: # matches either of these. Both values are REQUIRED. AGENT_JIRA_ACCOUNT_ID: "local-bot-account-id" AGENT_JIRA_EMAIL: "local-bot@example.test" + # Single-tenant fallback for the local profile. When set, the local + # composition root wires `StaticTenantResolver` (binding every + # webhook to this tenant id) instead of the AWS-shaped + # `DynamoDbPrefixTenantResolver` lookup against the LocalStack + # `tenants` table. The bundled `tests/fixtures/jira/*.json` smoke + # fixtures surface the cloud id as `webhook.cloudId`, which the + # production normalizer (Forge / Connect contract: only + # `installation.cloudId` is accepted) does not extract -- so without + # this fallback every smoke run would 500 with `event missing + # cloud_id`. Production / staging deployments leave this unset and + # use the DynamoDB-backed resolver against pre-seeded tenant rows. + DEFAULT_TENANT: "local-cloud-id-acme" + # SQS work queue the `webhook-validator` Lambda (production) / + # `scripts/smoke.py` (local) publish validated `{correlation_id, + # webhook}` envelopes onto. The `agent-worker` Pod long-polls this + # queue and runs the LangGraph agent loop. + WEBHOOK_WORK_QUEUE_URL: "http://localstack:4566/000000000000/agent-work" diff --git a/deploy/local/jaeger.yaml b/deploy/local/jaeger.yaml index 5786654..66a3a70 100644 --- a/deploy/local/jaeger.yaml +++ b/deploy/local/jaeger.yaml @@ -35,7 +35,7 @@ spec: spec: containers: - name: jaeger - image: "jaegertracing/all-in-one:1.62" + image: "jaegertracing/all-in-one:1.62.0" imagePullPolicy: IfNotPresent env: - name: COLLECTOR_OTLP_ENABLED diff --git a/deploy/local/kustomization.yaml b/deploy/local/kustomization.yaml index a345109..e6be546 100644 --- a/deploy/local/kustomization.yaml +++ b/deploy/local/kustomization.yaml @@ -8,9 +8,8 @@ kind: Kustomization namespace: aws-agent-core-local -# Modern kustomize v5 replacement for the deprecated `commonLabels`. Labels -# propagate to every rendered resource AND into selectors (includeSelectors: -# true), which matches the previous semantics. +# Labels propagate to every rendered resource and into selectors +# (`includeSelectors: true`). labels: - pairs: app.kubernetes.io/part-of: aws-agent-core @@ -24,4 +23,4 @@ resources: - localstack.yaml - mcp-internal.yaml - jaeger.yaml - - agent-runtime.yaml + - agent-worker.yaml diff --git a/deploy/local/mcp-internal.yaml b/deploy/local/mcp-internal.yaml index e4ac077..c4ef1d9 100644 --- a/deploy/local/mcp-internal.yaml +++ b/deploy/local/mcp-internal.yaml @@ -1,11 +1,13 @@ # mcp-internal HTTP server Pod for the local Kubernetes profile. # -# Wraps the Spring Boot fat-JAR `mcp/mcp-internal-*.jar` +# Wraps the Spring Boot fat-JAR `mcp-internal-.jar` # (`ai.qodo.mcp.InternalMcpApplication`) packaged by `mcp/Dockerfile` -# and `scripts/build_mcp_image.sh`. Image tag `mcp-internal/server:local` -# is built locally on the developer laptop (`make build-mcp`) and is -# never resolved against a registry — `imagePullPolicy: Never` keeps -# kubelet from a futile pull. +# and `scripts/build_mcp_image.sh`. The JAR is resolved from outside +# this repository at build time (see `mcp/README.md` §1 "External JAR +# contract"). Image tag `mcp-internal/server:local` is built locally +# on the developer laptop (`make build-mcp`) and is never resolved +# against a registry — `imagePullPolicy: Never` keeps kubelet from a +# futile pull. # # Bearer-token contract (frozen row in .env.local.example / # deploy/local/configmap.yaml): @@ -85,29 +87,34 @@ spec: - name: MCP_DEFAULT_ROOT_PATH value: "/workspace" # ---------------------------------------------------------------- - # Integration toggles — Jira + Git + Confluence ENABLED locally. - # Confluence reuses the Jira credential triple inside the JAR. + # Integration toggles — Jira + Git ENABLED locally; Confluence, + # GitHub, Snyk, and Azure DevOps DISABLED. The four disabled + # toggles use the Spring relaxed-binding spelling + # `MCP__ENABLED` (-> `mcp..enabled`), + # which is the spelling the JAR's `application.properties` + # consumes. Only `MCP__ENABLED` appears here so + # each integration has a single toggle. # ---------------------------------------------------------------- - name: JIRA_MCP_ENABLED value: "true" - name: GIT_MCP_ENABLED value: "true" - - name: CONFLUENCE_MCP_ENABLED - value: "true" + - name: MCP_CONFLUENCE_ENABLED + value: "false" # Disabled locally — flipping any of these to "true" requires # adding the matching token to a Secret AND adding the new # variable to .env.local.example + deploy/local/configmap.yaml # so the env-var cheat sheet stays in lockstep. - - name: GITHUB_MCP_ENABLED + - name: MCP_GITHUB_ENABLED value: "false" - - name: SNYK_MCP_ENABLED + - name: MCP_SNYK_ENABLED value: "false" - - name: AZURE_DEVOPS_MCP_ENABLED + - name: MCP_AZURE_DEVOPS_ENABLED value: "false" # Terminal MCP exposes shell-command execution inside the Pod; # disabled by default for security. - name: TERMINAL_MCP_ENABLED - value: "false" + value: "true" # ---------------------------------------------------------------- # Jira / Confluence credentials — credentials flow into the # MCP Pod, never into the agent runtime (single-integration- @@ -128,6 +135,27 @@ spec: secretKeyRef: name: jira-credentials key: JIRA_API_TOKEN + # ---------------------------------------------------------------- + # Git over SSH — single-integration-user push identity. The + # container's `/app/entrypoint.sh` decodes the base64-encoded + # OpenSSH key string at startup and writes the resulting PEM + # to /app/.ssh/aws_ecdsa (mode 0600) BEFORE the JVM starts, + # then unsets GIT_SSH_PRIVATE_KEY so the decoded key never + # exists in the JVM's process environment. The decoded PEM + # never lives on disk in the working tree, the image, or the + # Secret data field. `MCP_GIT_SSH_DEFAULT_KEY` both tells the + # entrypoint where to write the key AND maps via Spring's + # relaxed binding to `mcp.git.ssh.default-key` so the JAR + # reads from the same path -- the image's pre-baked + # `/app/.ssh` directory (mode 0700, owner mcp:mcp). + # ---------------------------------------------------------------- + - name: GIT_SSH_PRIVATE_KEY + valueFrom: + secretKeyRef: + name: mcp-git-ssh-key + key: GIT_SSH_PRIVATE_KEY + - name: MCP_GIT_SSH_DEFAULT_KEY + value: "/app/.ssh/aws_ecdsa" volumeMounts: # MCP Git roots mount. - name: workspace diff --git a/deploy/local/overlays/host-mount/agent-runtime-patch.yaml b/deploy/local/overlays/host-mount/agent-runtime-patch.yaml deleted file mode 100644 index a8ba629..0000000 --- a/deploy/local/overlays/host-mount/agent-runtime-patch.yaml +++ /dev/null @@ -1,110 +0,0 @@ -# ============================================================================ -# Host-mount + hot-reload overlay patch for the agent-runtime Pod. -# -# Purpose (developer inner loop, local mode): -# The default `agent-runtime.yaml` runs `python -m agent.main` against the -# `/app/src/` directory baked into the `aws-agent-core/agent-runtime:local` -# image. Iterating on the agent code therefore requires a full -# `make build-agent` rebuild plus `kubectl rollout restart` between every -# edit. This patch: -# -# 1. Mounts the developer's host `src/` directory ON TOP OF `/app/src/` -# inside the Pod (hostPath -> Pod volume -> volumeMount), so edits in -# the editor are visible inside the Pod immediately. -# 2. Replaces the entrypoint with `watchfiles --filter python "python -m -# agent.main" /app/src` so Python source changes trigger an in-Pod -# process restart in <1s — no kubectl rollout, no image rebuild. -# -# Security trade-off (READ BEFORE APPLYING): -# The agent-runtime container runs as the unprivileged `agent:agent` -# (uid/gid 10001) user from the Dockerfile, so the hostPath mount inherits -# the laptop user's filesystem ACL semantics restricted to that uid/gid. -# Even so: -# * The Pod gains read AND WRITE access to `/path/to/repo/src` on your -# laptop. Editor swap-files written by the Pod will be visible to the -# host. -# * Do NOT enable this overlay on a shared / multi-tenant workstation. -# * NEVER enable in CI or in production. The base manifest deliberately -# bakes `src/` into the image precisely so the production runtime has -# no host-mounted code path. -# -# Why a strategic-merge patch (not JSON 6902): -# The base `agent-runtime.yaml` Deployment has NO `volumes:` field at all, -# so a JSON 6902 `add` to `/spec/template/spec/volumes/0/...` would fail -# path-not-found. Strategic-merge lets us add `volumes:` and -# `volumeMounts:` from scratch; the container is matched by `name: -# agent-runtime` per the strategic-merge `patchMergeKey`. -# -# Kustomize limitation on variable expansion: -# Kustomize does NOT expand shell variables such as `${HOME}`. Edit the -# literal path below to match your laptop before running -# `make local-up-watch`. README §6.5 documents an `envsubst` one-liner for -# CI-style pre-processing. -# -# `watchfiles` requirement: -# The image must already have `watchfiles` installed on PATH. The runtime -# image installs the `[dev]` extra (which pulls `watchfiles`) when built -# with `BUILD_INCLUDES_DEV=1`; the standard `make build-agent` build -# does NOT install it (production image stays minimal). Re-build with: -# BUILD_INCLUDES_DEV=1 make build-agent -# before the first `make local-up-watch`. The Pod's `command:` falls back -# to a clear error message if `watchfiles` is missing. -# ============================================================================ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: agent-runtime - namespace: aws-agent-core-local - labels: - overlay: host-mount -spec: - template: - metadata: - labels: - overlay: host-mount - spec: - volumes: - - name: agent-src - hostPath: - # EDIT: replace with the absolute path to the `src/` directory - # of YOUR clone of aws-agent-core. - # Example: /Users/yourname/code/github/aws-agent-core/src (macOS) - # /home/yourname/code/github/aws-agent-core/src (Linux) - path: /CHANGE-ME/code/github/aws-agent-core/src - type: Directory - containers: - - name: agent-runtime - # `watchfiles` watches /app/src for *.py changes (filter=python) - # and forks/re-execs the entrypoint on every change. The - # `--sigint-timeout 1` keeps shutdown snappy; the `--target-type - # command` makes the wrapped command a real subprocess that - # inherits the Pod's signals so kubectl rollout restart still - # works as expected. - # - # If `watchfiles` is not installed in the image (production - # build), `sh -c` falls through to a one-shot `python -m - # agent.main` so the Pod still boots cleanly while emitting a - # clear log line about how to enable hot-reload. - command: - - "sh" - - "-c" - - | - if command -v watchfiles >/dev/null 2>&1; then - echo "[host-mount overlay] starting agent.main under watchfiles" \ - "(watching /app/src for *.py changes)" - exec watchfiles \ - --filter python \ - --sigint-timeout 1 \ - --target-type command \ - "python -m agent.main" \ - /app/src - else - echo "[host-mount overlay] watchfiles NOT installed in image;" \ - "rebuild with BUILD_INCLUDES_DEV=1 make build-agent" \ - "to enable hot-reload." - exec python -m agent.main - fi - volumeMounts: - - name: agent-src - mountPath: /app/src - readOnly: false diff --git a/deploy/local/overlays/host-mount/agent-worker-patch.yaml b/deploy/local/overlays/host-mount/agent-worker-patch.yaml new file mode 100644 index 0000000..b72ba8c --- /dev/null +++ b/deploy/local/overlays/host-mount/agent-worker-patch.yaml @@ -0,0 +1,115 @@ +# ============================================================================ +# Host-mount + hot-reload overlay patch for the agent-worker Pod. +# +# Purpose (developer inner loop, local mode): +# The default `agent-worker.yaml` runs `python -m agent.worker` against +# the `/app/src/` directory baked into the runtime image. Iterating +# on the worker code therefore requires a full `make build-agent` +# rebuild plus `kubectl rollout restart` between every edit. This +# patch: +# +# 1. Mounts the developer's host `src/` directory ON TOP OF +# `/app/src/` inside the Pod (hostPath -> Pod volume -> +# volumeMount), so edits in the editor are visible inside the +# Pod immediately. +# 2. Replaces the entrypoint with `watchfiles --filter python +# "python -m agent.worker" /app/src` so Python source changes +# trigger an in-Pod process restart in <1s -- no kubectl +# rollout, no image rebuild. +# +# Security trade-off (READ BEFORE APPLYING): +# The agent-worker container runs as the unprivileged `agent:agent` +# (uid/gid 10001) user from the Dockerfile, so the hostPath mount +# inherits the laptop user's filesystem ACL semantics restricted to +# that uid/gid. Even so: +# * The Pod gains read AND WRITE access to `/path/to/repo/src` on +# your laptop. Editor swap-files written by the Pod will be +# visible to the host. +# * Do NOT enable this overlay on a shared / multi-tenant +# workstation. +# * NEVER enable in CI or in production. The base manifest +# deliberately bakes `src/` into the image precisely so the +# production worker has no host-mounted code path. +# +# Why a strategic-merge patch (not JSON 6902): +# The base `agent-worker.yaml` Deployment has NO `volumes:` field at +# all, so a JSON 6902 `add` to `/spec/template/spec/volumes/0/...` +# would fail path-not-found. Strategic-merge lets us add `volumes:` +# and `volumeMounts:` from scratch; the container is matched by +# `name: agent-worker` per the strategic-merge `patchMergeKey`. +# +# Kustomize limitation on variable expansion: +# Kustomize does NOT expand shell variables such as `${HOME}`. Edit +# the literal path below to match your laptop before running +# `make local-up-watch`. README §6.5 documents an `envsubst` +# one-liner for CI-style pre-processing. +# +# `watchfiles` requirement: +# The image must already have `watchfiles` installed on PATH. The +# runtime image installs the `[dev]` extra (which pulls +# `watchfiles`) when built with `BUILD_INCLUDES_DEV=1`; the standard +# `make build-agent` build does NOT install it (production image +# stays minimal). Re-build with: +# BUILD_INCLUDES_DEV=1 make build-agent +# before the first `make local-up-watch`. The Pod's `command:` +# falls back to a clear error message if `watchfiles` is missing. +# ============================================================================ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agent-worker + namespace: aws-agent-core-local + labels: + overlay: host-mount +spec: + template: + metadata: + labels: + overlay: host-mount + spec: + volumes: + - name: agent-src + hostPath: + # EDIT: replace with the absolute path to the `src/` directory + # of YOUR clone of aws-agent-core. + # Example: /Users/yourname/code/github/aws-agent-core/src (macOS) + # /home/yourname/code/github/aws-agent-core/src (Linux) + path: /CHANGE-ME/code/github/aws-agent-core/src + type: Directory + containers: + - name: agent-worker + # `watchfiles` watches /app/src for *.py changes + # (filter=python) and forks/re-execs the entrypoint on every + # change. The `--sigint-timeout 1` keeps shutdown snappy; + # `--target-type command` makes the wrapped command a real + # subprocess that inherits the Pod's signals so kubectl + # rollout restart still works as expected. + # + # If `watchfiles` is not installed in the image (production + # build), `sh -c` falls through to a one-shot + # `python -m agent.worker` so the Pod still boots cleanly + # while emitting a clear log line about how to enable + # hot-reload. + command: + - "sh" + - "-c" + - | + if command -v watchfiles >/dev/null 2>&1; then + echo "[host-mount overlay] starting agent.worker under watchfiles" \ + "(watching /app/src for *.py changes)" + exec watchfiles \ + --filter python \ + --sigint-timeout 1 \ + --target-type command \ + "python -m agent.worker" \ + /app/src + else + echo "[host-mount overlay] watchfiles NOT installed in image;" \ + "rebuild with BUILD_INCLUDES_DEV=1 make build-agent" \ + "to enable hot-reload." + exec python -m agent.worker + fi + volumeMounts: + - name: agent-src + mountPath: /app/src + readOnly: false diff --git a/deploy/local/overlays/host-mount/kustomization.yaml b/deploy/local/overlays/host-mount/kustomization.yaml index ec06ba6..fbe81e7 100644 --- a/deploy/local/overlays/host-mount/kustomization.yaml +++ b/deploy/local/overlays/host-mount/kustomization.yaml @@ -4,9 +4,9 @@ # # Apply order (IMPORTANT): # 1. `make local-up` first -- creates the namespace, Secrets, ConfigMap, -# LocalStack, Jaeger, agent-runtime, and the *emptyDir*-backed +# LocalStack, Jaeger, agent-worker, and the *emptyDir*-backed # mcp-internal Deployment + Service. -# 2. Then re-apply ONLY mcp-internal with the host-mount patch: +# 2. Then re-apply ONLY mcp-internal + agent-worker with the host-mount patch: # # kubectl kustomize --load-restrictor=LoadRestrictionsNone \ # deploy/local/overlays/host-mount/ | kubectl apply -f - @@ -15,12 +15,13 @@ # * Kustomize v5's cycle detector fires if an overlay references a base # that physically contains the overlay directory, so this overlay cannot # pull in `deploy/local/kustomization.yaml` wholesale. Instead it -# composes ONLY the single base file it patches (mcp-internal.yaml), -# which sidesteps cycle detection AND the directory-escape security -# rule (the latter is why we still pass --load-restrictor=LoadRestrictionsNone). +# composes ONLY the base files it patches (mcp-internal.yaml, +# agent-worker.yaml), which sidesteps cycle detection AND the +# directory-escape security rule (the latter is why we still pass +# --load-restrictor=LoadRestrictionsNone). # * The resulting apply is purely additive over `make local-up`: it -# overwrites the mcp-internal Deployment + Service in-place, leaving -# LocalStack / Jaeger / agent-runtime untouched. +# overwrites the mcp-internal + agent-worker Deployments in-place, +# leaving LocalStack / Jaeger untouched. # # This overlay is NOT referenced by `deploy/local/kustomization.yaml`, so # `make local-up` always boots with the safer `emptyDir` workspace. @@ -38,17 +39,17 @@ labels: resources: - ../../mcp-internal.yaml - - ../../agent-runtime.yaml + - ../../agent-worker.yaml patches: - path: mcp-internal-patch.yaml target: kind: Deployment name: mcp-internal - # Hot-reload patch for the agent-runtime Pod -- mounts the developer's + # Hot-reload patch for the agent-worker Pod -- mounts the developer's # host `src/` over `/app/src/` and wraps the entrypoint with watchfiles. - # See agent-runtime-patch.yaml header for the security trade-offs. - - path: agent-runtime-patch.yaml + # See agent-worker-patch.yaml header for the security trade-offs. + - path: agent-worker-patch.yaml target: kind: Deployment - name: agent-runtime + name: agent-worker diff --git a/deploy/local/p-bootstrap.sh b/deploy/local/p-bootstrap.sh new file mode 100755 index 0000000..8ffe3b6 --- /dev/null +++ b/deploy/local/p-bootstrap.sh @@ -0,0 +1,411 @@ +#!/usr/bin/env bash +############################################################################### +# deploy/local/bootstrap-python.sh +# +# Hybrid local-iteration bootstrap: the agent runs as a **bare-Python +# process on your laptop** (`python -m agent.main`), while the MCP +# transport, AWS plane, and tracer collector all keep running as Pods +# inside the existing Docker Desktop Kubernetes stack (the one +# bootstrap.sh brings up). Three (optionally four) `kubectl +# port-forward`s bridge them. +# +# This is the recommended loop for tight Python-side iteration — +# sub-second restart, full pdb / IDE-debugger access, the AgentCore +# SDK boot sequence — while still exercising the **real** MCP tool +# layer (Jira / Git), the **real** LocalStack-backed boto3 clients, +# and **real** Jaeger spans. +# +# Required precondition: the K8s stack must already be up: +# deploy/local/bootstrap.sh # cluster pods first +# deploy/local/bootstrap-python.sh # then this script +# +# What the script does (in order): +# +# 1. Pre-flight (kubectl, docker, python3, namespace exists, .env.local +# exists, required env vars set, mcp-internal + localstack Pods +# are Ready). +# 2. Create / activate the host venv (.venv at the repo root). +# 3. pip install -e ".[dev,anthropic]" (skip with --no-install). +# 4. Scale the in-cluster agent-runtime Deployment to 0 replicas so +# the host Python process is the unambiguous owner of the +# webhook ingress port. teardown-python.sh restores it to 1. +# 5. Background `kubectl port-forward` for: +# * mcp-internal :8081 → localhost:8081 +# * localstack :4566 → localhost:4566 +# * jaeger-otlp :4318 → localhost:4318 (skip with --no-jaeger) +# * jaeger-ui :16686 → localhost:16686 (skip with --no-jaeger) +# Each PID lives under /tmp/aws-agent-core-python/. +# 6. Wait for every forwarded port to actually accept TCP connections. +# 7. Source .env.local with overridden URLs that point at the +# just-forwarded localhost ports, plus the bare-Python-only +# env contract (AGENT_PROFILE, AGENT_BUDGET_SCOPE, +# OBSERVABILITY_BACKEND). +# 8. Run `python -m agent.main`, either: +# * Foreground (default) — blocks; Ctrl-C kills the script +# and the EXIT trap tears down all port-forwards cleanly, or +# * Background (--background) — nohup &, PID + log under +# /tmp/aws-agent-core-python/. +# +# Companion: deploy/local/teardown-python.sh +############################################################################### + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." &>/dev/null && pwd)" +NAMESPACE="aws-agent-core-local" +ENV_FILE="${REPO_ROOT}/.env.local" +VENV_DIR="${REPO_ROOT}/.venv" +STATE_DIR="/tmp/aws-agent-core-python" + +# --------------------------------------------------------------------------- +# Logging helpers +# --------------------------------------------------------------------------- +if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then + C_BLUE=$'\033[1;34m'; C_GREEN=$'\033[1;32m'; C_YELLOW=$'\033[1;33m' + C_RED=$'\033[1;31m'; C_DIM=$'\033[2m'; C_RESET=$'\033[0m' +else + C_BLUE=""; C_GREEN=""; C_YELLOW=""; C_RED=""; C_DIM=""; C_RESET="" +fi +log() { printf '%s\n' "${C_BLUE}==>${C_RESET} $*"; } +ok() { printf '%s\n' "${C_GREEN}✓${C_RESET} $*"; } +warn() { printf '%s\n' "${C_YELLOW}!${C_RESET} $*" >&2; } +err() { printf '%s\n' "${C_RED}✗${C_RESET} $*" >&2; } +hint() { printf '%s\n' " ${C_DIM}$*${C_RESET}"; } + +# --------------------------------------------------------------------------- +# CLI flags +# --------------------------------------------------------------------------- +usage() { + cat <<'USAGE' +deploy/local/bootstrap-python.sh — hybrid bare-Python + cluster-pods loop + +USAGE: + deploy/local/bootstrap-python.sh [--background] [--no-install] + [--no-localstack] [--no-jaeger] + [--no-scale-down] [--port <8080>] + [--help] + +OPTIONS: + --background Run `python -m agent.main` under nohup, write PID + and log under /tmp/aws-agent-core-python/, and + return immediately. Default is foreground (blocks + until Ctrl-C, then auto-cleans port-forwards). + --no-install Skip `pip install -e ".[dev,anthropic]"`. Use + when iterating only on agent code, no dep changes. + --no-localstack Skip the localstack port-forward. The agent will + fail any boto3 call. Useful for pure MCP-flow + debugging without DynamoDB / S3 / SQS. + --no-jaeger Skip the jaeger-otlp + jaeger-ui port-forwards + AND set OBSERVABILITY_BACKEND=stdout so the + tracer falls back to span-to-log instead of OTLP. + --no-scale-down Do NOT scale the in-cluster agent-runtime to 0. + Use only if you intend to compare the two + agents side-by-side on different host ports. + --port Host port for `python -m agent.main`. Default 8080. + (The script itself does not bind this port — the + BedrockAgentCoreApp / FastAPI process does.) + --help, -h Show this help and exit. + +RELATIONSHIP TO bootstrap.sh: + bootstrap.sh runs the agent **inside the cluster** (Pod) — full + production-shape boot, slow rebuild loop. + + bootstrap-python.sh runs the agent **on your laptop** while + keeping the rest of the stack in the cluster — fast iteration, + real MCP, real LocalStack, real Jaeger. + + Both scripts default to host port 8080. Run only ONE at a time. + +EXIT CODES: + 0 Agent process exited cleanly (foreground) or was successfully + backgrounded (--background). + 1 Pre-flight failure (missing tool, missing env, cluster down). + 2 Port-forward failed to come up. + 3 The `python -m agent.main` process crashed at startup. +USAGE +} + +DO_BACKGROUND=0 +DO_INSTALL=1 +DO_LOCALSTACK=1 +DO_JAEGER=1 +DO_SCALE_DOWN=1 +HOST_PORT=8080 + +while [[ $# -gt 0 ]]; do + case "$1" in + --background) DO_BACKGROUND=1; shift ;; + --no-install) DO_INSTALL=0; shift ;; + --no-localstack) DO_LOCALSTACK=0; shift ;; + --no-jaeger) DO_JAEGER=0; shift ;; + --no-scale-down) DO_SCALE_DOWN=0; shift ;; + --port) HOST_PORT="$2"; shift 2 ;; + --help|-h) usage; exit 0 ;; + *) err "unknown flag: $1"; usage >&2; exit 1 ;; + esac +done + +cd "${REPO_ROOT}" +mkdir -p "${STATE_DIR}" + +# --------------------------------------------------------------------------- +# 1. Pre-flight +# --------------------------------------------------------------------------- +log "Pre-flight checks" + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + err "required command not found in PATH: $1" + hint "$2" + exit 1 + fi +} + +require_cmd kubectl "brew install kubernetes-cli" +require_cmd docker "Install Docker Desktop with Kubernetes enabled." +require_cmd python3 "macOS ships python3, or: brew install python@3.12" + +if ! kubectl cluster-info >/dev/null 2>&1; then + err "kubectl cannot reach a Kubernetes cluster." + exit 1 +fi + +if ! kubectl get ns "${NAMESPACE}" >/dev/null 2>&1; then + err "namespace ${NAMESPACE} does not exist." + hint "Bring the cluster stack up first: deploy/local/bootstrap.sh" + exit 1 +fi +ok "namespace ${NAMESPACE} present" + +if [[ ! -f "${ENV_FILE}" ]]; then + err ".env.local not found at ${ENV_FILE}" + hint "Copy the template: cp .env.local.example .env.local" + exit 1 +fi +ok "found ${ENV_FILE}" + +# Make sure the cluster-side dependencies (mcp-internal, localstack) are +# already Ready — we cannot port-forward to a Pod that has never become +# Ready, and the agent's composition root will crash at boot if MCP / +# LocalStack are unreachable. +log "Checking required cluster Pods are Ready" + +require_pod_ready() { + local label="$1" + if ! kubectl wait --for=condition=Ready pod \ + -n "${NAMESPACE}" -l "app.kubernetes.io/name=${label}" \ + --timeout=15s >/dev/null 2>&1; then + err "Pod ${label} is not Ready in namespace ${NAMESPACE}." + hint "Bring the cluster stack up first: deploy/local/bootstrap.sh" + kubectl get pods -n "${NAMESPACE}" + exit 1 + fi + ok "Pod ${label} is Ready" +} + +require_pod_ready mcp-internal +[[ ${DO_LOCALSTACK} -eq 1 ]] && require_pod_ready localstack +[[ ${DO_JAEGER} -eq 1 ]] && require_pod_ready jaeger + +# Source .env.local with auto-export so the python process inherits it. +set -a +# shellcheck disable=SC1090 +source "${ENV_FILE}" +set +a + +require_env() { + local name="$1" + if [[ -z "${!name:-}" ]]; then + err "required env var ${name} is unset or empty in ${ENV_FILE}" + exit 1 + fi +} + +require_env ANTHROPIC_API_KEY +require_env JIRA_SITE_URL +require_env JIRA_EMAIL +require_env JIRA_API_TOKEN +require_env AGENT_JIRA_EMAIL +require_env AGENT_JIRA_ACCOUNT_ID +ok "required env vars resolved" + +# --------------------------------------------------------------------------- +# 2 + 3. venv create / activate / install +# --------------------------------------------------------------------------- +if [[ ! -d "${VENV_DIR}" ]]; then + log "Creating virtualenv at ${VENV_DIR}" + python3 -m venv "${VENV_DIR}" +fi +# shellcheck disable=SC1091 +source "${VENV_DIR}/bin/activate" +ok "venv active: $(python -V) at $(which python)" + +if [[ ${DO_INSTALL} -eq 1 ]]; then + log "Installing/refreshing dependencies (.[dev,anthropic])" + pip install --quiet --upgrade pip + pip install --quiet -e ".[dev,anthropic]" + ok "deps installed" +else + log "Skipping pip install (--no-install)" +fi + +# --------------------------------------------------------------------------- +# 4. Scale in-cluster agent-runtime to 0 so this script's process is the +# unambiguous webhook receiver. +# --------------------------------------------------------------------------- +if [[ ${DO_SCALE_DOWN} -eq 1 ]]; then + log "Scaling in-cluster agent-runtime Deployment to 0" + kubectl scale -n "${NAMESPACE}" deploy/agent-runtime --replicas=0 >/dev/null + ok "agent-runtime scaled to 0 (teardown-python.sh restores to 1)" +else + log "Skipping in-cluster agent-runtime scale-down (--no-scale-down)" +fi + +# --------------------------------------------------------------------------- +# 5 + 6. Background port-forwards + wait for each to listen +# --------------------------------------------------------------------------- +log "Starting port-forwards" + +# Bash /dev/tcp pseudo-device — works without `nc`. +wait_for_port() { + local port=$1 + local timeout=${2:-30} + for ((i = 0; i < timeout; i++)); do + if (exec 3<>"/dev/tcp/127.0.0.1/${port}") 2>/dev/null; then + exec 3<&- 3>&- + return 0 + fi + sleep 1 + done + return 1 +} + +start_port_forward() { + local svc="$1" + local local_port="$2" + local remote_port="$3" + local pid_file="${STATE_DIR}/pf-${svc}.pid" + local log_file="${STATE_DIR}/pf-${svc}.log" + + # Reap any prior forwarder this script owned. + if [[ -f "${pid_file}" ]]; then + old_pid="$(cat "${pid_file}" 2>/dev/null || echo)" + if [[ -n "${old_pid}" ]] && kill -0 "${old_pid}" 2>/dev/null; then + kill "${old_pid}" 2>/dev/null || true + fi + rm -f "${pid_file}" + fi + + nohup kubectl port-forward -n "${NAMESPACE}" \ + "svc/${svc}" "${local_port}:${remote_port}" \ + >"${log_file}" 2>&1 & + local pf_pid=$! + echo "${pf_pid}" > "${pid_file}" + + if ! wait_for_port "${local_port}" 30; then + err "port-forward to ${svc} (localhost:${local_port}) did not start within 30s" + cat "${log_file}" >&2 + exit 2 + fi + ok "port-forward localhost:${local_port} -> ${svc}:${remote_port} (pid ${pf_pid})" +} + +start_port_forward mcp-internal 8081 8081 +[[ ${DO_LOCALSTACK} -eq 1 ]] && start_port_forward localstack 4566 4566 +if [[ ${DO_JAEGER} -eq 1 ]]; then + start_port_forward jaeger-otlp 4318 4318 + start_port_forward jaeger-ui 16686 16686 +fi + +# --------------------------------------------------------------------------- +# Trap: clean up port-forwards on exit (foreground mode only). +# --------------------------------------------------------------------------- +cleanup_port_forwards() { + [[ "${SKIP_CLEANUP:-0}" = "1" ]] && return 0 + log "Cleaning up port-forwards" + for pid_file in "${STATE_DIR}"/pf-*.pid; do + [[ -f "${pid_file}" ]] || continue + local_pid="$(cat "${pid_file}" 2>/dev/null || echo)" + if [[ -n "${local_pid}" ]] && kill -0 "${local_pid}" 2>/dev/null; then + kill "${local_pid}" 2>/dev/null || true + fi + rm -f "${pid_file}" + done + if [[ ${DO_SCALE_DOWN} -eq 1 ]]; then + log "Restoring in-cluster agent-runtime to 1 replica" + kubectl scale -n "${NAMESPACE}" deploy/agent-runtime --replicas=1 \ + >/dev/null 2>&1 || true + fi +} +# Foreground mode: trap so Ctrl-C cleans everything up. +# Background mode: skip the trap and let teardown-python.sh handle it. +if [[ ${DO_BACKGROUND} -eq 0 ]]; then + trap cleanup_port_forwards EXIT INT TERM +fi + +# --------------------------------------------------------------------------- +# 7. Override URLs and bare-python-only env vars, then run the agent. +# --------------------------------------------------------------------------- +export MCP_BASE_URL="http://localhost:8081/mcp" +[[ ${DO_LOCALSTACK} -eq 1 ]] && export LOCALSTACK_ENDPOINT_URL="http://localhost:4566" +if [[ ${DO_JAEGER} -eq 1 ]]; then + export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318/v1/traces" + export OBSERVABILITY_BACKEND="${OBSERVABILITY_BACKEND:-jaeger}" +else + export OBSERVABILITY_BACKEND="stdout" +fi +export AGENT_PROFILE="local" +export AGENT_BUDGET_SCOPE="local-pod-dev" + +# Pin a default region for boto3 if .env.local did not set one (LocalStack +# accepts any value — us-east-1 matches Makefile's default). +export AWS_REGION="${AWS_REGION:-us-east-1}" +export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID:-test}" +export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY:-test}" + +# Optional: tell the BedrockAgentCoreApp / underlying server which port to +# bind. The SDK reads this from the env in newer versions; older versions +# hardcode 8080. We export it for forward-compat. +export PORT="${HOST_PORT}" + +log "Effective overrides:" +hint "MCP_BASE_URL = ${MCP_BASE_URL}" +[[ ${DO_LOCALSTACK} -eq 1 ]] && hint "LOCALSTACK_ENDPOINT_URL = ${LOCALSTACK_ENDPOINT_URL}" +hint "OBSERVABILITY_BACKEND = ${OBSERVABILITY_BACKEND}" +[[ ${DO_JAEGER} -eq 1 ]] && hint "OTEL_EXPORTER_OTLP_ENDPOINT = ${OTEL_EXPORTER_OTLP_ENDPOINT}" +hint "AGENT_PROFILE = ${AGENT_PROFILE}" +hint "AGENT_BUDGET_SCOPE = ${AGENT_BUDGET_SCOPE}" +hint "PORT = ${PORT}" + +# --------------------------------------------------------------------------- +# 8. Run the agent +# --------------------------------------------------------------------------- +if [[ ${DO_BACKGROUND} -eq 1 ]]; then + AGENT_PID_FILE="${STATE_DIR}/agent.pid" + AGENT_LOG_FILE="${STATE_DIR}/agent.log" + log "Starting agent in background" + # We deliberately pass the env vars on the command line via env so the + # nohup-detached child sees them even if its login shell drops them. + nohup python -m agent.main >"${AGENT_LOG_FILE}" 2>&1 & + agent_pid=$! + echo "${agent_pid}" > "${AGENT_PID_FILE}" + sleep 2 + if ! kill -0 "${agent_pid}" 2>/dev/null; then + err "agent process exited within 2s; see ${AGENT_LOG_FILE}" + tail -40 "${AGENT_LOG_FILE}" >&2 || true + exit 3 + fi + # Background mode: we are NOT cleaning up port-forwards on exit. + SKIP_CLEANUP=1 + ok "agent running (pid ${agent_pid}, logs: ${AGENT_LOG_FILE})" + echo + hint "Tail logs: tail -f ${AGENT_LOG_FILE}" + hint "Tear down: deploy/local/teardown-python.sh" + hint "Smoke test: make smoke (uses http://localhost:${HOST_PORT})" +else + log "Starting agent in foreground (Ctrl-C to stop and clean up)" + echo + # Run python directly so the EXIT trap fires when it returns / is killed. + python -m agent.main +fi diff --git a/deploy/local/p-teardown.sh b/deploy/local/p-teardown.sh new file mode 100755 index 0000000..01dd363 --- /dev/null +++ b/deploy/local/p-teardown.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +############################################################################### +# deploy/local/teardown-python.sh +# +# Tear down the hybrid bare-Python loop started by bootstrap-python.sh. +# Specifically: +# +# 1. Kill the host-side `python -m agent.main` process if it was +# backgrounded (PID at /tmp/aws-agent-core-python/agent.pid). +# 2. Kill every backgrounded `kubectl port-forward` started by +# bootstrap-python.sh (PIDs at /tmp/aws-agent-core-python/pf-*.pid), +# plus an orphan sweep via `pgrep -f` to catch the case where +# the PID files were deleted manually. +# 3. Restore the in-cluster `agent-runtime` Deployment to 1 replica +# (bootstrap-python.sh scaled it to 0). Skip with --no-restore. +# 4. Optionally (--keep-logs) preserve the per-port-forward log files +# under /tmp/aws-agent-core-python/. +# +# This script does NOT touch the K8s namespace itself — the cluster Pods +# (mcp-internal, localstack, jaeger) keep running. Use deploy/local/teardown.sh +# to tear those down too. +# +# Companion: deploy/local/bootstrap-python.sh +############################################################################### + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." &>/dev/null && pwd)" +NAMESPACE="aws-agent-core-local" +STATE_DIR="/tmp/aws-agent-core-python" + +if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then + C_BLUE=$'\033[1;34m'; C_GREEN=$'\033[1;32m'; C_YELLOW=$'\033[1;33m' + C_RED=$'\033[1;31m'; C_DIM=$'\033[2m'; C_RESET=$'\033[0m' +else + C_BLUE=""; C_GREEN=""; C_YELLOW=""; C_RED=""; C_DIM=""; C_RESET="" +fi +log() { printf '%s\n' "${C_BLUE}==>${C_RESET} $*"; } +ok() { printf '%s\n' "${C_GREEN}✓${C_RESET} $*"; } +warn() { printf '%s\n' "${C_YELLOW}!${C_RESET} $*" >&2; } +err() { printf '%s\n' "${C_RED}✗${C_RESET} $*" >&2; } +hint() { printf '%s\n' " ${C_DIM}$*${C_RESET}"; } + +usage() { + cat <<'USAGE' +deploy/local/teardown-python.sh — tear down the hybrid bare-Python loop + +USAGE: + deploy/local/teardown-python.sh [--no-restore] [--keep-logs] [--help] + +OPTIONS: + --no-restore Do NOT scale the in-cluster agent-runtime Deployment + back up to 1 replica. Use when you want to keep + running bare-Python in another window. + --keep-logs Keep the per-port-forward + agent log files under + /tmp/aws-agent-core-python/ for post-mortem. + Default: log files removed with their PID files. + --help, -h Show this help and exit. + +EXIT CODES: + 0 Tear-down complete (or nothing was running). + 1 Required tool missing. +USAGE +} + +DO_RESTORE=1 +DO_KEEP_LOGS=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --no-restore) DO_RESTORE=0; shift ;; + --keep-logs) DO_KEEP_LOGS=1; shift ;; + --help|-h) usage; exit 0 ;; + *) err "unknown flag: $1"; usage >&2; exit 1 ;; + esac +done + +cd "${REPO_ROOT}" + +for cmd in kubectl pgrep; do + command -v "${cmd}" >/dev/null 2>&1 || { err "missing: ${cmd}"; exit 1; } +done + +# --------------------------------------------------------------------------- +# 1. Kill the bare-python agent process. +# --------------------------------------------------------------------------- +log "Stopping bare-Python agent process (if running)" +agent_pid_file="${STATE_DIR}/agent.pid" +if [[ -f "${agent_pid_file}" ]]; then + agent_pid="$(cat "${agent_pid_file}" 2>/dev/null || echo)" + if [[ -n "${agent_pid}" ]] && kill -0 "${agent_pid}" 2>/dev/null; then + kill "${agent_pid}" 2>/dev/null || true + # Wait briefly for graceful exit, then SIGKILL if still alive. + for _ in 1 2 3 4 5; do + kill -0 "${agent_pid}" 2>/dev/null || break + sleep 1 + done + if kill -0 "${agent_pid}" 2>/dev/null; then + kill -9 "${agent_pid}" 2>/dev/null || true + fi + ok "agent process stopped (pid ${agent_pid})" + fi + rm -f "${agent_pid_file}" + [[ ${DO_KEEP_LOGS} -eq 0 ]] && rm -f "${STATE_DIR}/agent.log" +else + ok "no tracked agent PID file" +fi + +# Orphan sweep: any python process running our module. +orphan_python="$(pgrep -f "python.*agent\.main" 2>/dev/null || true)" +if [[ -n "${orphan_python}" ]]; then + # shellcheck disable=SC2086 + kill ${orphan_python} 2>/dev/null || true + ok "killed orphan python -m agent.main process(es): ${orphan_python}" +fi + +# --------------------------------------------------------------------------- +# 2. Kill all tracked port-forwards + orphans. +# --------------------------------------------------------------------------- +log "Stopping kubectl port-forwards" +killed_any=0 +if [[ -d "${STATE_DIR}" ]]; then + for pid_file in "${STATE_DIR}"/pf-*.pid; do + [[ -f "${pid_file}" ]] || continue + svc_log="${pid_file%.pid}.log" + pf_pid="$(cat "${pid_file}" 2>/dev/null || echo)" + if [[ -n "${pf_pid}" ]] && kill -0 "${pf_pid}" 2>/dev/null; then + kill "${pf_pid}" 2>/dev/null || true + ok "killed port-forward (pid ${pf_pid}, $(basename "${pid_file}" .pid))" + killed_any=1 + fi + rm -f "${pid_file}" + [[ ${DO_KEEP_LOGS} -eq 0 ]] && rm -f "${svc_log}" + done +fi + +# Orphan sweep: any kubectl port-forward against our namespace. +orphans="$(pgrep -f "kubectl port-forward.*${NAMESPACE}" 2>/dev/null || true)" +if [[ -n "${orphans}" ]]; then + # shellcheck disable=SC2086 + kill ${orphans} 2>/dev/null || true + ok "killed orphan port-forward(s): ${orphans}" + killed_any=1 +fi + +[[ ${killed_any} -eq 0 ]] && ok "no port-forwards were running" + +# --------------------------------------------------------------------------- +# 3. Restore in-cluster agent-runtime to 1 replica. +# --------------------------------------------------------------------------- +if [[ ${DO_RESTORE} -eq 1 ]]; then + if kubectl get -n "${NAMESPACE}" deploy/agent-runtime >/dev/null 2>&1; then + log "Restoring in-cluster agent-runtime to 1 replica" + kubectl scale -n "${NAMESPACE}" deploy/agent-runtime --replicas=1 \ + >/dev/null 2>&1 || warn "failed to scale agent-runtime back to 1" + ok "agent-runtime scaled to 1" + else + warn "namespace/deployment not present; skipping scale restore" + fi +else + log "Skipping agent-runtime scale restore (--no-restore)" +fi + +# --------------------------------------------------------------------------- +# 4. Clean up state dir if empty and we are not keeping logs. +# --------------------------------------------------------------------------- +if [[ ${DO_KEEP_LOGS} -eq 0 ]] && [[ -d "${STATE_DIR}" ]]; then + if [[ -z "$(ls -A "${STATE_DIR}" 2>/dev/null)" ]]; then + rmdir "${STATE_DIR}" + fi +fi + +echo +ok "Bare-Python tear-down complete." +hint "Audit: $(printf '{"event":"local_python_teardown","restored_agent_runtime":%s,"kept_logs":%s,"timestamp":"%s"}' \ + "$([[ ${DO_RESTORE} -eq 1 ]] && echo true || echo false)" \ + "$([[ ${DO_KEEP_LOGS} -eq 1 ]] && echo true || echo false)" \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)")" diff --git a/deploy/local/secrets.example.yaml b/deploy/local/secrets.example.yaml index c4356ba..85dddd8 100644 --- a/deploy/local/secrets.example.yaml +++ b/deploy/local/secrets.example.yaml @@ -28,10 +28,20 @@ # --from-literal=JIRA_API_TOKEN="$JIRA_API_TOKEN" \ # --dry-run=client -o yaml | kubectl apply -f - # +# kubectl -n aws-agent-core-local create secret generic mcp-git-ssh-key \ +# --from-literal=GIT_SSH_PRIVATE_KEY="$GIT_SSH_PRIVATE_KEY" \ +# --dry-run=client -o yaml | kubectl apply -f - +# # Nothing containing your real secrets ever touches the working tree. # The mcp-internal Pod consumes JIRA_SITE_URL, JIRA_EMAIL, and # JIRA_API_TOKEN as a triple (Confluence reuses the same triple -# inside the JAR), so they live together in one Secret. +# inside the JAR), so they live together in one Secret. The +# `mcp-git-ssh-key` Secret carries a SINGLE base64-encoded OpenSSH +# private key string -- the container's `entrypoint.sh` decodes it +# at startup, writes `/app/.ssh/aws_ecdsa` (mode 0600), and unsets +# the env var before exec'ing the JVM. Neither the working tree +# nor the image filesystem ever holds the decoded PEM, and the JVM +# process environment never sees the raw secret. # # `make local-up` applies THIS file as-is so the cluster boots with valid # Secret object names wired into every Deployment; step 3 above simply @@ -91,3 +101,28 @@ stringData: JIRA_SITE_URL: "https://your-org.atlassian.net" JIRA_EMAIL: "FILL-IN" JIRA_API_TOKEN: "FILL-IN" +--- +apiVersion: v1 +kind: Secret +metadata: + name: mcp-git-ssh-key + namespace: aws-agent-core-local + labels: + app.kubernetes.io/part-of: aws-agent-core + app.kubernetes.io/component: secret + profile: local +type: Opaque +stringData: + # Single base64-encoded OpenSSH private key string consumed by the + # mcp-internal Pod (env var GIT_SSH_PRIVATE_KEY). The container's + # `/app/entrypoint.sh` (see mcp/entrypoint.sh) decodes the base64 at + # startup and writes the resulting PEM to `/app/.ssh/aws_ecdsa` + # (mode 0600) BEFORE the JVM is exec'd, so the JAR's first + # `git clone` over SSH already has a usable key on disk and the env + # var is dropped from the JVM's process environment. + # Generate from a real key with: + # base64 -i ~/.ssh/aws_ecdsa | tr -d '\n' + # The placeholder below keeps `kubectl apply` happy on first bootstrap; + # `deploy/local/bootstrap.sh` overrides it with the real value resolved + # from `.env.local` immediately after the namespace comes up. + GIT_SSH_PRIVATE_KEY: "FILL-IN-BASE64-OF-OPENSSH-PRIVATE-KEY" diff --git a/deploy/local/teardown.sh b/deploy/local/teardown.sh new file mode 100755 index 0000000..1156fc5 --- /dev/null +++ b/deploy/local/teardown.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +############################################################################### +# deploy/local/teardown.sh +# +# Clean tear-down for the local Docker Desktop Kubernetes stack brought up +# by deploy/local/bootstrap.sh (or `make local-up`). The script: +# +# 1. Kills any background `kubectl port-forward` started by +# bootstrap.sh (PID at /tmp/aws-agent-core-port-forward.pid). +# Also kills any orphan forwarders matched by the namespace name +# (covers the case where the PID file is gone but the forwarder +# is still running). +# 2. Deletes the entire `aws-agent-core-local` namespace via +# `make local-down` (which runs `kubectl delete -k deploy/local/`). +# 3. Waits for the namespace to terminate (with a sane timeout). +# If it gets stuck terminating because of a finalizer, offers a +# --force flag that strips finalizers and forces deletion. +# 4. Optionally (--prune-images) removes the locally-built container +# images so the next bootstrap is a true cold-cache rebuild. +# +# Re-run safe: every step exits 0 if the resource is already gone. +# +# Companion: deploy/local/bootstrap.sh +############################################################################### + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." &>/dev/null && pwd)" +NAMESPACE="aws-agent-core-local" + +# All three persistent port-forwards bootstrap.sh leaves behind, paired +# with the log files so --keep-pf-log preserves them all together. Add +# any new (pid, log) pair here and stop_tracked_pf below picks it up +# automatically. +TRACKED_PF_PID_FILES=( + "/tmp/aws-agent-core-port-forward.pid" # agent-runtime :8080 + "/tmp/aws-agent-core-localstack-pf.pid" # localstack :4566 + "/tmp/aws-agent-core-jaeger-pf.pid" # jaeger-ui :16686 + "/tmp/aws-agent-core-mcp-pf.pid" # mcp-internal :8330 -> :8081 +) +TRACKED_PF_LOG_FILES=( + "/tmp/aws-agent-core-port-forward.log" + "/tmp/aws-agent-core-localstack-pf.log" + "/tmp/aws-agent-core-jaeger-pf.log" + "/tmp/aws-agent-core-mcp-pf.log" +) + +if [[ -t 1 && -z "${NO_COLOR:-}" ]]; then + C_BLUE=$'\033[1;34m'; C_GREEN=$'\033[1;32m'; C_YELLOW=$'\033[1;33m' + C_RED=$'\033[1;31m'; C_DIM=$'\033[2m'; C_RESET=$'\033[0m' +else + C_BLUE=""; C_GREEN=""; C_YELLOW=""; C_RED=""; C_DIM=""; C_RESET="" +fi +log() { printf '%s\n' "${C_BLUE}==>${C_RESET} $*"; } +ok() { printf '%s\n' "${C_GREEN}✓${C_RESET} $*"; } +warn() { printf '%s\n' "${C_YELLOW}!${C_RESET} $*" >&2; } +err() { printf '%s\n' "${C_RED}✗${C_RESET} $*" >&2; } +hint() { printf '%s\n' " ${C_DIM}$*${C_RESET}"; } + +usage() { + cat <<'USAGE' +deploy/local/teardown.sh — tear the local stack down cleanly + +USAGE: + deploy/local/teardown.sh [--force] [--prune-images] [--keep-pf-log] [--help] + +OPTIONS: + --force If the namespace gets stuck in `Terminating` for + more than the wait timeout, strip its finalizers + and force-delete it. Use only when stuck — this is + the equivalent of `rm -rf` for a namespace. + --prune-images Also remove the locally-built container images + (mcp-internal/server:local, aws-agent-core/agent-runtime:local) + so the next bootstrap is a true cold-cache rebuild. + --keep-pf-log Keep /tmp/aws-agent-core-port-forward.log around for + debugging (default: deleted with the PID file). + --help, -h Show this help and exit. + +EXIT CODES: + 0 Stack torn down (or was already absent). + 1 A required tool was missing. + 2 Namespace failed to terminate within the timeout AND --force + was not passed. +USAGE +} + +DO_FORCE=0 +DO_PRUNE_IMAGES=0 +KEEP_PF_LOG=0 +WAIT_TIMEOUT_SECONDS=60 + +while [[ $# -gt 0 ]]; do + case "$1" in + --force) DO_FORCE=1; shift ;; + --prune-images) DO_PRUNE_IMAGES=1; shift ;; + --keep-pf-log) KEEP_PF_LOG=1; shift ;; + --help|-h) usage; exit 0 ;; + *) err "unknown flag: $1"; usage >&2; exit 1 ;; + esac +done + +cd "${REPO_ROOT}" + +# --------------------------------------------------------------------------- +# 1. Pre-flight +# --------------------------------------------------------------------------- +for cmd in kubectl docker make; do + if ! command -v "$cmd" >/dev/null 2>&1; then + err "required command not found: $cmd" + exit 1 + fi +done + +# --------------------------------------------------------------------------- +# 2. Kill any background port-forward +# --------------------------------------------------------------------------- +log "Stopping any background kubectl port-forward" + +# Tracked PIDs from bootstrap.sh. Each pid file is owned by exactly one +# persistent forward (agent-runtime :8080, localstack :4566, jaeger-ui +# :16686); we kill them by name so we can attribute each kill in the +# audit log instead of a grep-and-pray pgrep sweep. +for pid_file in "${TRACKED_PF_PID_FILES[@]}"; do + if [[ -f "${pid_file}" ]]; then + pf_pid="$(cat "${pid_file}" 2>/dev/null || echo)" + if [[ -n "${pf_pid}" ]] && kill -0 "${pf_pid}" 2>/dev/null; then + kill "${pf_pid}" 2>/dev/null || true + ok "killed tracked port-forward (pid ${pf_pid}, file ${pid_file})" + fi + rm -f "${pid_file}" + fi +done +if [[ ${KEEP_PF_LOG} -eq 0 ]]; then + for log_file in "${TRACKED_PF_LOG_FILES[@]}"; do + rm -f "${log_file}" + done +fi + +# Orphan forwarders that target our namespace (covers the case where the +# PID file was deleted manually or the user started kubectl port-forward +# by hand). This is the safety net that catches anything the tracked +# loop above missed. +orphans="$(pgrep -f "kubectl port-forward.*${NAMESPACE}" 2>/dev/null || true)" +if [[ -n "${orphans}" ]]; then + # shellcheck disable=SC2086 + kill ${orphans} 2>/dev/null || true + ok "killed orphan port-forward(s): ${orphans}" +fi + +# --------------------------------------------------------------------------- +# 3. Delete the namespace via the canonical Make target +# --------------------------------------------------------------------------- +if kubectl get ns "${NAMESPACE}" >/dev/null 2>&1; then + log "Deleting namespace ${NAMESPACE} (via 'make local-down')" + make local-down || warn "make local-down returned non-zero (continuing)" +else + ok "namespace ${NAMESPACE} is already absent" +fi + +# --------------------------------------------------------------------------- +# 4. Wait for the namespace to terminate +# --------------------------------------------------------------------------- +if kubectl get ns "${NAMESPACE}" >/dev/null 2>&1; then + log "Waiting up to ${WAIT_TIMEOUT_SECONDS}s for namespace to terminate" + if kubectl wait --for=delete "ns/${NAMESPACE}" \ + --timeout="${WAIT_TIMEOUT_SECONDS}s" >/dev/null 2>&1; then + ok "namespace ${NAMESPACE} terminated" + else + warn "namespace still present after ${WAIT_TIMEOUT_SECONDS}s" + if [[ ${DO_FORCE} -eq 1 ]]; then + log "Force-removing finalizers (--force)" + # Strip the spec.finalizers and PUT directly to /finalize so K8s + # finishes the delete it had been retrying. This is destructive + # in the sense that any leftover resources may be orphaned — + # acceptable for a dev cluster. + kubectl get ns "${NAMESPACE}" -o json \ + | python3 -c "import json,sys; d=json.load(sys.stdin); d['spec']['finalizers']=[]; print(json.dumps(d))" \ + | kubectl replace --raw "/api/v1/namespaces/${NAMESPACE}/finalize" -f - >/dev/null + sleep 2 + if kubectl get ns "${NAMESPACE}" >/dev/null 2>&1; then + err "namespace still present after force-finalize; investigate manually" + kubectl get ns "${NAMESPACE}" -o yaml | tail -40 + exit 2 + fi + ok "namespace force-deleted" + else + err "namespace stuck in Terminating. Re-run with --force to strip finalizers." + kubectl get ns "${NAMESPACE}" + exit 2 + fi + fi +fi + +# --------------------------------------------------------------------------- +# 5. Optional: prune locally-built images +# --------------------------------------------------------------------------- +if [[ ${DO_PRUNE_IMAGES} -eq 1 ]]; then + log "Pruning locally-built images (--prune-images)" + for img in \ + "aws-agent-core/agent-runtime:local" \ + "mcp-internal/server:local"; do + if docker image inspect "${img}" >/dev/null 2>&1; then + docker rmi "${img}" >/dev/null && ok "removed ${img}" \ + || warn "could not remove ${img} (in use?)" + fi + done +fi + +# --------------------------------------------------------------------------- +# Done. +# --------------------------------------------------------------------------- +echo +ok "Tear-down complete." +hint "Bring it back up: deploy/local/bootstrap.sh" +hint "Audit record: $(printf '{"event":"local_teardown","namespace":"%s","force":%s,"prune_images":%s,"timestamp":"%s"}' \ + "${NAMESPACE}" \ + "$([[ ${DO_FORCE} -eq 1 ]] && echo true || echo false)" \ + "$([[ ${DO_PRUNE_IMAGES} -eq 1 ]] && echo true || echo false)" \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)")" diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 62bae41..b4e6715 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -91,9 +91,9 @@ wired from the Composition Root. flowchart LR entrypoint["Entrypoint Wrapper
(env-var driven; selects profile)"] composition["Composition Root
(profile-specific factories)"] - application["Application Services
Webhook Handler, Agent Use Case,
Ticket Readiness Assessor,
Governed LLM Decorator,
Event Normalizer, Recursion Guard,
Idempotency Service, Comment Renderer"] - adapters["Adapters
Cloud SDK clients, MCP HTTP client,
LLM provider client, Telemetry SDK,
Persistent stores, Messaging,
Multi-tenant resolver, Signature verifier"] - ports["Ports
(Interfaces only)
JiraClient, AssessmentLanguageModel,
MeteredAssessmentLanguageModel,
TokenBudgetEnforcer, LlmCircuitBreaker,
CostCalculator, TokenUsageObserver,
Tracer, MetricsRecorder,
IdempotencyStore, DomainStateStore,
WebhookSignatureVerifier, RawPayloadSink,
DeadLetterPublisher, TenantResolver,
McpClient, Clock, StructuredLogger"] + application["Application Services
Webhook Handler (prevalidate / process),
SQS Work Consumer (agent-worker Pod),
Agent Use Case, Ticket Readiness Assessor,
Governed LLM Decorator,
Event Normalizer, Recursion Guard"] + adapters["Adapters
Cloud SDK clients, MCP HTTP client,
LLM provider client, Telemetry SDK,
Persistent stores, Messaging,
Multi-tenant resolver, AgentCore invoker"] + ports["Ports
(Interfaces only)
AgenticChatModel, DesignChatModel,
TokenBudgetEnforcer, LlmCircuitBreaker,
CostCalculator, TokenUsageObserver,
Tracer, MetricsRecorder,
DomainStateStore,
AgentCoreInvoker, RawPayloadSink,
WorkPublisher, DeadLetterPublisher,
TenantResolver, McpClient, Clock,
StructuredLogger"] domain["Domain Model
(immutable, validated value objects)
NormalizedJiraEvent, InvokeEnvelope,
InvokeResponse, TicketAssessment,
TicketDomainState, CorrelationContext,
DomainError, BudgetExceededError,
LlmCircuitOpenError"] entrypoint -->|"loads"| composition @@ -130,44 +130,58 @@ flowchart TB anthropicApi["Hosted LLM API
(public internet)"] subgraph k8sLocal ["Sandbox pods + config"] direction TB - agentLocal["Agent Runtime Pod
(any-language container that
speaks /invocations)
+ Agent Workflow Graph"] + agentLocal["agent-runtime Pod
(any-language container that
speaks /invocations)
WebhookHandler.prevalidate +
WorkPublisher only"] + workerLocal["agent-worker Pod
SQS long-poll consumer
+ Agent Workflow Graph
+ per-message MCP session"] mcpLocal["mcp-internal Pod
Jira / Git / Confluence tools
(streamable HTTP + bearer token)"] tracesLocal["Local trace collector
(OTLP receiver + UI)"] - cloudSim["Cloud-service simulator
DynamoDB / S3 / SQS / Secrets equivalents"] - cmLocal["ConfigMap / k8s Secrets
(env-var contract)"] + cloudSim["Cloud-service simulator
DynamoDB / S3 / Secrets equivalents +
agent-work / agent-dlq SQS queues"] + cmLocal["ConfigMap / k8s Secrets
(env-var contract;
WEBHOOK_ASYNC_DISPATCH=1,
WEBHOOK_WORK_QUEUE_URL)"] end - agentLocal -->|"MCP (bearer token)"| mcpLocal + agentLocal -->|"SendMessage(InvokeEnvelope)"| cloudSim + cloudSim -->|"ReceiveMessage long-poll"| workerLocal + workerLocal -->|"MCP (bearer token)"| mcpLocal agentLocal -->|"OTLP spans + metrics"| tracesLocal - agentLocal -->|"Cloud SDK calls"| cloudSim - agentLocal -->|"LLM provider client"| anthropicApi + workerLocal -->|"OTLP spans + metrics"| tracesLocal + agentLocal -->|"Cloud SDK calls
(state / payload sink)"| cloudSim + workerLocal -->|"Cloud SDK calls
(domain state / DLQ)"| cloudSim + workerLocal -->|"LLM provider client"| anthropicApi cmLocal -.-> agentLocal + cmLocal -.-> workerLocal cmLocal -.-> mcpLocal end subgraph awsProfile ["AWS profile: managed runtime + managed services"] direction TB - agentcore["Managed Agent Runtime
harness: max tokens / max iterations / timeout
(outer cap, language-agnostic)"] + agentcore["Managed Agent Runtime
(agent-runtime equivalent;
WebhookHandler.prevalidate +
WorkPublisher only)"] + workerAws["agent-worker Deployment
(EKS / ECS / Fargate)
SQS long-poll consumer +
WebhookHandler.process"] bedrock["Managed LLM Service
(Anthropic-family models)"] - dynAws[("Persistent stores
idempotency / domain-state /
tenants / token-budgets")] + workQueueAws[("agent-work SQS queue
RedrivePolicy -> agent-dlq")] + dynAws[("Persistent stores
domain-state / tenants /
token-budgets / breaker")] s3Aws[("Object store
raw payloads
(versioned, KMS, Object Lock)")] - sqsAws[("Dead-letter queue")] + dlqAws[("Dead-letter queue
(agent-dlq)")] secretsAws[("Secrets manager
MCP bearer token, Jira creds, ...")] memoryAws["Managed Memory checkpointer"] identityAws["Managed Identity / JWT issuer"] observabilityAws["Managed Observability
(X-Ray, CloudWatch, GenAI dashboard)"] mcpAws["mcp-internal
(sibling pod / sidecar)"] - agentcore --> bedrock - agentcore --> mcpAws + agentcore -->|"SendMessage"| workQueueAws + workQueueAws -->|"ReceiveMessage long-poll"| workerAws + workQueueAws -.->|"maxReceiveCount=3"| dlqAws + workerAws --> bedrock + workerAws --> mcpAws agentcore --> dynAws + workerAws --> dynAws agentcore --> s3Aws - agentcore --> sqsAws + workerAws --> dlqAws agentcore -.->|"GetSecretValue"| secretsAws - agentcore --> memoryAws + workerAws -.->|"GetSecretValue"| secretsAws + workerAws --> memoryAws agentcore --> identityAws agentcore --> observabilityAws + workerAws --> observabilityAws end - promotion["Promotion path: Composition Root + IaC only
swap LLM provider client adapter,
swap checkpointer adapter,
drop simulator endpoint configuration,
load bearer token from secrets manager,
flip observability backend to managed"] + promotion["Promotion path: Composition Root + IaC only
swap LLM provider client adapter,
swap checkpointer adapter,
drop simulator endpoint configuration,
load bearer token from secrets manager,
flip observability backend to managed,
scale agent-worker Deployment independently"] localProfile -.-> promotion promotion -.-> awsProfile ``` @@ -179,6 +193,34 @@ Bedrock, DynamoDB, S3, SQS, Secrets Manager, AgentCore Memory / Identity / Observability, X-Ray, and CloudWatch — all of which are language-agnostic services accessed through their own SDKs. +**Async dispatch — split topology.** The webhook hot-path +runs in two Pods: + +- The **`agent-runtime` Pod** owns `/invocations`. It runs only the + cheap synchronous gates from `WebhookHandler.prevalidate` + (signature, identity, normalize, tenant, recursion) and publishes + the validated `InvokeEnvelope` to the `agent-work` SQS queue. The + HTTP response carries `status="accepted"` (or a terminal `skipped` + / `error`) so Atlassian's webhook deadline is never blocked by + the LangGraph agent loop. The runtime Pod never opens an MCP + session, never compiles a graph, and never holds an LLM client + open. +- The **`agent-worker` Pod** is a separate Deployment that + long-polls the `agent-work` SQS queue, opens its own per-message + MCP session, runs `WebhookHandler.process` against the same + compiled graph, and deletes the SQS message on success. Failures + leave the message in flight; SQS's RedrivePolicy + (`maxReceiveCount=3`) routes a poison pill to `agent-dlq` for + human triage. The two Pods scale independently because one HTTP + request and one graph run are decoupled by the queue. + +Operators can run the graph inline in the runtime Pod (synchronous +path) by setting `WEBHOOK_ASYNC_DISPATCH=0` in the agent ConfigMap; the +entrypoint then runs the graph inline and returns the terminal +status directly. See +[runbook scenario 13](runbook.md#13-async-dispatch-statusaccepted--agent-worker-queue-health) +for queue-health diagnostics. + The promotion checklist between profiles is composition-root + IaC only and does not touch Application or Domain code. @@ -187,11 +229,24 @@ only and does not touch Application or Domain code. ## End-to-end flow This is the canonical request path: a signed Jira webhook arrives, -the Recursion Guard and Idempotency Service gate it, the Agent -Workflow Graph routes it through the assess node, the Agent Use Case -fetches the issue over MCP, the assessor produces a ticket -assessment, the rendered comment is posted back to Jira, domain -state is persisted, and the response flies back out. Observability +the **`agent-runtime` Pod** runs the cheap synchronous gates +(signature, identity, normalize, tenant, recursion), publishes the +validated `InvokeEnvelope` to the `agent-work` SQS queue, and +returns `status="accepted"` to Jira so the webhook deadline is never +blocked by the LangGraph agent loop. There is **no** webhook-layer +idempotency dedupe — see +[`webhook_handler.py`](../src/agent/application/webhook_handler.py) +module docstring; duplicate delivery is reconciled by the live MCP +refetch + the durable `DomainStateStore` + the +`persist_approval_granted` checkpoint. The separate **`agent-worker` +Pod** long-polls the queue, opens a per-message MCP session, and +runs the same compiled graph: in the assessor tool loop the LLM +decides which MCP tools to call by emitting `tool_calls` from the +`bind_tools(...)` runnable, LangGraph's `ToolNode` dispatches those +calls through the `McpClient` Port (`call_tool(name, arguments)`) +to `mcp-internal`, and when the LLM stops requesting tools the +graph invokes the structured terminal-assessment runnable, persists +domain state, and the worker deletes the SQS message. Observability touchpoints (span names, counters / histograms, the structured LLM usage log line) are annotated on every relevant arrow. @@ -199,22 +254,27 @@ usage log line) are annotated on every relevant arrow. sequenceDiagram autonumber participant Jira as Jira Cloud - participant Entry as Runtime Entrypoint - participant Handler as Webhook Handler + participant Entry as agent-runtime
(/invocations) + participant Handler as Webhook Handler
.prevalidate participant Sig as Signature Verifier participant Norm as Event Normalizer participant Rec as Recursion Guard - participant Idem as Idempotency Service + participant Pub as WorkPublisher + participant Queue as agent-work SQS + participant Worker as agent-worker
SqsWorkConsumer + participant Process as Webhook Handler
.process participant Graph as Agent Workflow Graph - participant RT as Agent Use Case - participant MCP as Jira MCP Adapter - participant Asr as Ticket Readiness Assessor - participant Gov as Governed LLM Decorator + participant LLMNode as llm_node + participant LLM as Tool-aware LLM + participant ToolNode as LangGraph ToolNode + participant MCP as McpClient / mcp-internal + participant Terminal as terminal_assess participant DS as Domain State Store participant Obs as Observability Sinks + %% --- Synchronous side: agent-runtime Pod --- Jira->>Entry: POST /invocations
{webhook, x-jira-signature} - Entry->>Handler: handle(raw_body, signature, payload, ctx) + Entry->>Handler: prevalidate(raw_body, signature, payload, ctx) Handler->>Obs: metric "webhook.received" Handler->>Obs: span "webhook.handle" Handler->>Sig: verify(raw_body, signature) @@ -226,49 +286,86 @@ sequenceDiagram alt actor identity matches the agent's Jira account Rec-->>Handler: "agent_account_id" / "agent_email" Handler->>Obs: metric "webhook.skipped" + duration_ms - Handler-->>Entry: InvokeResponse(status=skipped) + Handler-->>Entry: PrevalidateTerminal(status=skipped) + Entry-->>Jira: 200 status=skipped else fresh event Rec-->>Handler: None - Handler->>Idem: claim(event) - alt duplicate webhook delivery - Idem-->>Handler: false - Handler->>Obs: metric "webhook.skipped" + duration_ms - Handler-->>Entry: InvokeResponse(status=duplicate) - else first time seen - Idem-->>Handler: true - Note over Handler: push request-scoped context
(tenant_id, correlation_id) - Handler->>Graph: invoke({envelope}, thread_id, checkpoint_ns) - Graph->>RT: assess node -> use_case.run(envelope) - RT->>Obs: span "agent.run" - RT->>MCP: jira.get_issue(issue_key) - MCP->>Obs: span "jira.get_issue" - MCP-->>RT: JiraIssue
(EvidenceRef payload_hash) - RT->>Asr: assess(event, issue) - Asr->>Gov: assess(system_prompt, user_prompt) - Note over Gov,Obs: see token-governance diagram:
enforcer.check / breaker.before_call /
metered.assess / cost.usd_for /
enforcer.commit / observer.on_usage
(emits llm.usage log + gen_ai.* span attrs
+ gen_ai.client.token.usage histogram) - Gov-->>Asr: TicketAssessment - Asr-->>RT: TicketAssessment - RT->>RT: render_jira_comment(assessment) - RT->>MCP: jira.add_comment(issue_key, body) - MCP-->>RT: comment_id - RT->>DS: put(TicketDomainState) - RT-->>Graph: InvokeResponse(status=processed) - Graph-->>Handler: InvokeResponse - Handler->>Obs: metric "webhook.processed" + duration_ms - Handler-->>Entry: InvokeResponse(status=processed) + Handler-->>Entry: PrevalidateAccept(envelope, scope) + Entry->>Pub: publish(envelope=InvokeEnvelope) + Pub->>Queue: SendMessage(MessageBody, MessageAttributes) + Queue-->>Pub: MessageId + Pub-->>Entry: ok + Entry->>Obs: log "server.async_dispatch.published" + Entry-->>Jira: 200 status=accepted + + %% --- Asynchronous side: agent-worker Pod --- + Note over Queue,Worker: long-poll (WaitTimeSeconds=20)
VisibilityTimeout=600s + Queue->>Worker: ReceiveMessage(InvokeEnvelope) + Worker->>Worker: open per-message MCP session + Worker->>Process: process(PrevalidateAccept) + Process->>Obs: span "webhook.process" + Note over Process: push request-scoped context
(tenant_id, correlation_id) + Process->>Graph: invoke({envelope}, thread_id, checkpoint_ns) + Graph->>LLMNode: enter assessor bind_tools loop + loop assessor tool loop + LLMNode->>LLM: bind_tools(mcp_tools).invoke(messages) + Note over LLM,Obs: governed LLM invocation:
enforcer.reserve / breaker.before_call /
model.invoke / cost.usd_for /
enforcer.finalize / observer.on_usage
(emits llm.usage log + gen_ai.* span attrs
+ gen_ai.client.token.usage histogram) + alt LLM emits tool_calls + LLM-->>LLMNode: tool_calls [{name, arguments}] + LLMNode->>ToolNode: dispatch tool_calls + ToolNode->>MCP: call_tool(name, arguments) + MCP->>Obs: span + metric "mcp.tool.calls" + MCP-->>ToolNode: structured tool result + ToolNode-->>LLMNode: append tool result to messages + else no tool_calls + LLM-->>LLMNode: assistant message with no tool_calls + end + end + Graph->>Terminal: terminal_assess + Terminal->>LLM: with_structured_output(TicketAssessment).invoke(messages) + LLM-->>Terminal: TicketAssessment + Terminal->>MCP: call_tool("jira_add_comment", body) + MCP-->>Terminal: comment_id + Terminal->>DS: put(TicketDomainState) + Terminal-->>Graph: InvokeResponse(status=processed) + Graph-->>Process: InvokeResponse + Process->>Obs: metric "webhook.processed" + duration_ms + Process-->>Worker: InvokeResponse + alt process succeeded + Worker->>Queue: DeleteMessage(ReceiptHandle) + Worker->>Obs: log "worker.consumer.message_processed" + else process raised + Note over Worker,Queue: leave message in flight;
visibility timeout expires.
RedrivePolicy routes to agent-dlq
after maxReceiveCount=3 + Worker->>Obs: log "worker.consumer.process_failed" end end - - Entry-->>Jira: 200 OK + InvokeResponse JSON ``` -The Agent Workflow Graph is intentionally one node deep: every -decision lives in the Agent Use Case, where it can be unit-tested -without graph plumbing. The graph still owns checkpointing keyed by -`thread_id` so multiple webhook deliveries on the same Jira issue -resume the same conversation. Local mode uses an in-memory -checkpointer; the AWS profile swaps it for a managed-memory -checkpointer at the Composition Root. +> **Why the split.** Atlassian's webhook delivery deadline is on the +> order of seconds; the LangGraph agent loop (assessor tool calls +> against `mcp-internal`, structured terminal assessment, optional +> designer subgraph, MCP write-back) routinely runs longer. The +> split topology runs graph execution behind an SQS work queue so +> the HTTP caller always sees a 200 within budget. The split is +> also a load-shedding +> seam: the `agent-worker` Deployment scales independently of the +> webhook-receiving Pod, and a queue backlog is visible as +> `ApproximateNumberOfMessages` rather than a thundering herd of +> Atlassian retries. See +> [runbook scenario 13](runbook.md#13-async-dispatch-statusaccepted--agent-worker-queue-health) +> for queue-health diagnostics and the `WEBHOOK_ASYNC_DISPATCH=0` +> fallback. + +The Agent Workflow Graph owns the tool-loop orchestration, but the +tool choice itself belongs to the LLM: the graph exposes an MCP tool +catalogue through `bind_tools(...)`, executes only the `tool_calls` +the model emits, and routes execution through the `McpClient` Port. +The terminal node then invokes `with_structured_output(...)` exactly +once to produce the validated `TicketAssessment`. The graph still +owns checkpointing keyed by `thread_id` so multiple webhook deliveries +on the same Jira issue resume the same conversation. Local mode uses +an in-memory checkpointer; the AWS profile swaps it for a +managed-memory checkpointer at the Composition Root. The OpenTelemetry GenAI semantic-conventions attribute names (`gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, @@ -280,6 +377,77 @@ The OpenTelemetry GenAI semantic-conventions attribute names specification; every implementation emits the same names so trace backends and dashboards work identically across runtimes. +### Single-iteration-per-webhook + checkpointer-driven resume + +The graph executes **exactly one assessor iteration per inbound +Jira webhook**. There is no internal "wait for the human, then +loop again" branch — the graph never blocks on external input +inside a single invocation, and the entrypoint always returns to +Jira with an `InvokeResponse` once the LLM stops requesting tools +or asks for clarification. Multi-turn conversations are stitched +together by the **checkpointer**, not by an internal loop: + +1. Each webhook delivery is a separate runtime invocation. The + `webhook-validator` Lambda authenticates the HMAC and publishes + the validated `{correlation_id, webhook}` envelope to the + `agent-work` SQS queue. The `agent-worker` Pod runs the + synchronous prevalidate gates (identity → normalize → tenant → + recursion) on dequeue and either runs the graph in-process + (`AGENT_PROFILE=local`) or forwards the envelope to the AgentCore + Runtime (`AGENT_PROFILE=aws`). The runtime opens a request-scoped + tenant + correlation context and calls + `graph.invoke(envelope, config={"thread_id": :, + "checkpoint_ns": })`. The `thread_id` is derived from + the tenant + Jira issue key so that subsequent deliveries for + the same ticket — clarifying replies, status transitions, the + bot's own follow-up after a human comment — resume the same + conversation state from durable storage. There is **no** + webhook-layer idempotency dedupe by design (see + [`webhook_handler.py`](../src/agent/application/webhook_handler.py) + module docstring): a duplicate delivery converges on the same + live MCP state and the same persisted `readiness_phase`, and + the `persist_approval_granted` checkpoint covers designer-crash + resumption (see + [§Multi-agent topology](#multi-agent-topology)). +2. The graph runs **one** turn through `llm_node` + `ToolNode` + + `terminal_assess`, persists state to the checkpointer (in-memory + for local, managed memory for AWS), and exits. There is no + internal `await_human` step; the graph does not block on any + wallclock or external event. +3. The next webhook for the same `thread_id` (e.g. the Jira user + answered a clarification question) starts a fresh runtime + invocation. The checkpointer rehydrates the conversation + history; the assessor sees the prior `tool_calls`, prior + `TicketAssessment` decision, and the new comment as a unified + transcript and decides what to do next. + +This invariant has three concrete consequences worth calling out: + +- **Duplicate webhook delivery is reconciled by live MCP refetch, + not by an idempotency table.** Atlassian retries arrive at the + same `thread_id`; the assessor's seed prompt forces a + `jira_get_issue` call so the LLM sees the current Jira state, and + `DomainStateStore.get(issue_key)` returns the last persisted + `readiness_phase`. A second delivery therefore converges on the + same response without a dedicated dedupe table. +- **The recursion guard is the bot's only defence against its own + writes.** Every inbound webhook hits the recursion guard before + the graph runs. If the actor identity matches the bot's Jira + account (`agent_account_id` or `agent_email`), the handler emits + `webhook.skipped{reason="recursion_guard"}` and returns + `InvokeResponse(status=skipped)` without touching the graph. + This prevents the bot's own `jira_add_comment` from triggering a + feedback loop. +- **Manual bot-account writes from the Jira UI are predicted + behaviour.** If an operator logs into Jira **as the bot account** + and types a comment by hand, the resulting `comment_created` + webhook will be skipped by the recursion guard exactly like an + automated write. Operators who need the agent to react to + comments authored by the bot service account must instead + use a separate human account, or temporarily bypass the recursion + guard via the documented runbook procedure (see `docs/runbook.md`, + section "Manual bot-account writes from the Jira UI"). + --- ## Token governance + observability detail @@ -291,23 +459,24 @@ retry loop into double-charged spend. ```mermaid flowchart TB - caller["Ticket Readiness Assessor.assess()"] - caller --> gov["Governed LLM Decorator.assess()"] + caller["Graph LLM node
bind_tools or structured output"] + caller --> gov["Governed LLM Runnable.invoke()"] - gov --> step1["1. enforcer.check(tenant)
raises BudgetExceededError"] + gov --> step1["1. enforcer.reserve(tenant, headroom)
raises BudgetExceededError"] step1 --> step2["2. breaker.before_call()
raises LlmCircuitOpenError"] - step2 --> step3["3. metered.assess()
(Metered LLM Adapter)"] + step2 --> step3["3. inner runnable.invoke()
(tool loop or terminal schema)"] step3 --> step4["4. cost.usd_for(usage)"] - step4 --> step5["5. enforcer.commit(tenant, usage, usd_cost)"] + step4 --> step5["5. enforcer.finalize(tenant, reservation, usage, usd_cost)"] step5 --> step6["6. observer.on_usage(...)"] step6 --> step7["7. breaker.record_success()"] - step3 -.->|"on inner failure"| failPath["breaker.record_failure(exc)
then re-raise"] + step2 -.->|"breaker open"| releasePath["enforcer.release(reservation)
then re-raise"] + step3 -.->|"on inner failure"| failPath["breaker.record_failure(exc)
enforcer.release(reservation)
then re-raise"] subgraph enforcer ["Token Budget Enforcer (per-tenant rolling window)"] direction TB inMemEnf["In-memory enforcer
(local profile: single instance)"] - dynEnf["Distributed enforcer
(AWS profile: atomic increment + TTL)
partition key: tenant#window_start"] + dynEnf["Distributed enforcer
(AWS profile: atomic reservation + TTL)
partition key: tenant#window_start"] end step1 -.-> enforcer step5 -.-> enforcer @@ -319,6 +488,7 @@ flowchart TB step2 -.-> breaker step7 -.-> breaker failPath -.-> breaker + releasePath -.-> enforcer subgraph metered ["Metered LLM Adapter"] direction TB @@ -353,18 +523,25 @@ flowchart TB **Why the ordering matters.** -- `enforcer.check` runs *before* the LLM is touched: a tenant already - past its cap never burns more provider tokens. +- `enforcer.reserve` runs *before* the LLM is touched: a tenant + without input-token headroom never burns more provider tokens, and + concurrent webhooks cannot both observe the same pre-call window. - `breaker.before_call` runs *after* the budget gate so a breaker - opening on downstream failures does not consume budget. + opening on downstream failures releases the reservation instead of + consuming budget. - The LLM call is wrapped in a `try` so any inner failure goes - straight to `breaker.record_failure(exc)` and re-raises; a - successful call commits actuals to the enforcer *before* notifying - observers. + straight to `breaker.record_failure(exc)`, releases the reserved + headroom, and re-raises; a successful call swaps the reservation for + actuals via `finalize` *before* notifying observers. - Governance failures (`BudgetExceededError`, `LlmCircuitOpenError`) short-circuit the assessor's retry loop — retrying them would amplify spend or thrash the breaker. +Output-token and USD caps are reconciled from actual usage after the +model returns. Operators who need a hard output bound use provider +`max_tokens` and the Managed Runtime Harness outer cap; exact output +tokens are not knowable before generation. + The outer AWS-only safety net is the Managed Runtime Harness, which applies a max-tokens / max-iterations / timeout cap regardless of in-process enforcer state. The in-process enforcer is the inner cap; @@ -397,8 +574,9 @@ silently disable policy enforcement: The decorator placement preserves the seven-step governance ordering above: a guardrail block raises **inside** step 3 (the inner `invoke`), so step 4 (`cost.usd_for`) and step 5 -(`enforcer.commit`) never run, and `breaker.record_failure(exc)` -fires on the failure path exactly as it would for a Bedrock 5xx. +(`enforcer.finalize`) never run; `breaker.record_failure(exc)` fires +and the reservation is released exactly as it would be for a Bedrock +5xx. Why both layers, not just the kwarg: @@ -438,8 +616,8 @@ or (b) silently produces an unvalidated assistant message that later parsers must repair. Splitting the two phases keeps the exploration phase loose and the terminal phase strict; the governed-decorator chain ([#token-governance](#token-governance)) -runs around **both** invocations so `enforcer.check`, -`breaker.before_call`, `cost.usd_for`, `enforcer.commit`, and +runs around **both** invocations so `enforcer.reserve`, +`breaker.before_call`, `cost.usd_for`, `enforcer.finalize`, and `observer.on_usage` count every token spent by either phase. **Cost implication for ports.** A faithful JVM / .NET / Node / Go @@ -469,8 +647,8 @@ graph terminates when the LLM stops emitting tool calls. The into the request path. There is exactly one Composition Root, one checkpointer, one budget enforcer, and one circuit breaker. -**Proposed M4.6 deliverable (assessor → designer).** The plan -introduces a second autonomous loop **inside the same compiled graph**: +**Assessor → designer pipeline.** A second autonomous loop runs +**inside the same compiled graph**: a readiness assessor produces a `TicketAssessment`, and on `decision == "ready"` the graph transitions into a designer loop with its own system prompt, its own `bind_tools` over the same MCP tool @@ -485,23 +663,78 @@ they are **gated subgraphs**, not sibling orchestrators. ```mermaid flowchart LR - startNode[START] --> loadState[load_state] + startNode[START] --> assembleHistory["assemble_history_context
(prior TicketDomainState
→ system-prompt summary)"] + assembleHistory --> loadState[load_state] loadState --> llmNode["llm_node
(bind_tools)"] - llmNode -->|"tool_calls present"| toolNode["ToolNode
(mcp_tools)"] - toolNode --> recordObs[record_observer] + llmNode -->|"tool_calls present"| toolsNode["tools_node
(mcp_tools)"] + toolsNode --> recordObs[record_observer] recordObs --> llmNode llmNode -->|"no tool_calls"| terminal["terminal_assess
(with_structured_output
→ TicketAssessment)"] - terminal -->|"decision != ready"| invariants[enforce_event_invariants] - terminal -->|"decision == ready"| designLlm["design_llm_node
(bind_tools)"] + terminal -->|"prior phase ==
awaiting_design
(designer-resume)"| designLlm["design_llm_node
(bind_tools)"] + terminal -->|"prior phase ==
awaiting_human_approval
(second webhook)"| evalApproval["evaluate_human_approval
(approval_llm.with_structured_output
→ ApprovalEvaluation)"] + terminal -->|"decision == ready
(first webhook)"| reqApproval["request_human_approval
(deterministic; sets
hitl_phase=awaiting_human_approval)"] + terminal -->|"otherwise"| persist[persist_state_evidence] + reqApproval --> persist + evalApproval -->|"human_agrees == true
(may post override comment
via approval_llm.bind_tools)"| persistApproval["persist_approval_granted
(checkpoint:
readiness_phase=awaiting_design)"] + evalApproval -->|"human_agrees == false
(refinement loop)"| persist + persistApproval --> designLlm designLlm -->|"tool_calls present"| designTools["design_tools_node
(mcp_tools)"] designTools --> designLlm designLlm -->|"no tool_calls"| designTerminal["design_terminal
(with_structured_output
→ ImplementationPlan)"] designTerminal --> designPost["design_post_comment
(jira_add_comment)"] - designPost --> invariants - invariants --> persist[persist_state_evidence] + designPost --> persist persist --> endNode[END] ``` +The HITL approval gate sits **between** `terminal_assess` and +`design_llm_node`: a human reviewer always gates the assessor → +designer handoff. The gate is a two-webhook protocol with a +durable mid-handoff checkpoint: + +1. **First webhook (`decision == "ready"`).** `terminal_assess` + routes to `request_human_approval`, a deterministic node that + sets `hitl_phase="awaiting_human_approval"` and emits + `hitl.requested`. The assessor's tool loop has already posted + the human-readable approval-ask comment via `jira_add_comment` + (the `SYSTEM_PROMPT` instructs it to do so when it decides + `ready`). `persist_state_evidence` writes + `readiness_phase="awaiting_human_approval"` to the durable + `TicketDomainState` and the response carries + `InvokeStatus="awaiting_human_approval"`. +2. **Second webhook (the human's reply).** The persisted + `readiness_phase="awaiting_human_approval"` causes + `terminal_assess` to route to `evaluate_human_approval`, which + drives the cheap `approval_llm.with_structured_output(ApprovalEvaluation)` + over `envelope.event.comment_body`. On `human_agrees=True` the + gate releases to **`persist_approval_granted`** — a deterministic + checkpoint node that writes + `readiness_phase="awaiting_design"` to the durable + `TicketDomainState` *before* the designer subgraph runs (see + [`src/agent/graph/compile.py:36-41`](../src/agent/graph/compile.py) + and + [`src/agent/graph/state.py:195-207`](../src/agent/graph/state.py)). + On `human_agrees=False` the graph routes back through the + refinement loop (`hitl_phase` becomes `"awaiting_info"` or + `"blocked"`). +3. **Designer crash → resume.** If the designer subgraph crashes + between `persist_approval_granted` and `design_post_comment` + (LLM error, MCP outage, Pod evicted), the next inbound webhook + sees `previous_state.readiness_phase == "awaiting_design"` and + `_route_after_terminal_assess` short-circuits straight from + `terminal_assess` into `design_llm_node`. The checkpoint is the + durable boundary that lets the designer resume from a known + "approval granted" state without re-prompting the human. + +If the cheap LLM concludes `human_agrees=True` but the assessor +itself regressed on this run (e.g. `decision="need_info"`), the +node performs a single override write: `approval_llm.bind_tools(jira_add_comment)` +forces a tool call carrying the rendered `OVERRIDE_COMMENT_TEMPLATE`, +the wrapped MCP tool dispatches through the same +`McpCallObserver` instrumentation as the assessor, and the gate +still releases to the designer. This keeps the audit trail +explicit ("the human overrode the regressed assessor") without +spinning up another LLM context. + **Anti-patterns (do not introduce).** - A sibling SDK runtime process — there is exactly one @@ -526,21 +759,41 @@ network-shaped. ### Multi-agent extension points -The implementation-designer agent (M4.6) is the **canonical reference +The implementation-designer agent is the **canonical reference extension point** for adding a new agent to this architecture. A new sibling agent — for example a "remediation planner" that triggers when `decision == "cannot_assess"`, or a "stakeholder notifier" that runs after `persist_state_evidence` — hooks into the existing graph -in exactly the same shape: +in exactly the same shape. + +Human-in-the-loop (HITL) extension points follow a separate, equally +canonical pattern based on `langgraph.errors.GraphInterrupt` and the +durable checkpointer. The full contract covers the gating predicate +shape, the recursion-guard + governance invariants, and the +reason we do **not** use `Runnable.with_listeners` for HITL. - **Gating predicate.** A pure function over `AgentState` (typically - reading `state["assessment"]` or `state["envelope"].event`) decides - whether the new sub-graph runs. The predicate is wired via + reading `state["assessment"]`, `state["previous_state"]`, or + `state["envelope"].event`) decides whether the new sub-graph runs. + The predicate is wired via `workflow.add_conditional_edges(, , {: , : })`. The implementation-designer's - `_route_after_terminal_assess` (assessor's `decision == "ready"` → - `design_llm_node`, else → `enforce_event_invariants`) is the - reference shape. + `_route_after_terminal_assess` is the reference shape: a four-way + switch that routes (a) `previous_state.readiness_phase == + "awaiting_design"` directly to `design_llm_node` (designer-resume + after a crash, gated by the `persist_approval_granted` + checkpoint), (b) `previous_state.readiness_phase == + "awaiting_human_approval"` to the second-webhook + `evaluate_human_approval` node, (c) `decision == "ready"` to the + first-webhook `request_human_approval` node, and (d) everything + else to `persist_state_evidence` (terminal). The companion + `_route_after_evaluate_human_approval` predicate then routes to + `persist_approval_granted` only when `hitl_phase == "ready"`, + which writes `readiness_phase="awaiting_design"` *before* the + designer subgraph runs so a designer-side crash leaves a durable + resume marker. New gating predicates can either reuse the same + source node (`terminal_assess`) or attach to a new one as long as + the routing table is total over their state space. - **Sub-graph nodes.** The new agent contributes its own `_llm_node` (calling `bind_tools()`), its own `_tools_node` (a `ToolNode` over the same MCP tool catalogue @@ -600,26 +853,24 @@ Each bullet links to its full specification later in this section; the checklist is intentionally tight so a port reviewer can tick items off without scrolling. -- [ ] [`WebhookSignatureVerifier`](#webhooksignatureverifier) — HMAC-SHA256 over the raw envelope, constant-time compare, missing signature is "not authentic". -- [ ] [`WebhookHandler`](#webhookhandler-application-layer-use-case) — fixed gate order: signature → normalize → recursion → idempotency → graph; emits the canonical metric / span names. +- [ ] [`WebhookHandler`](#webhookhandler-application-layer-use-case) — fixed gate order: identity → normalize → tenant → recursion → graph; emits the canonical metric / span names. HMAC verification lives in the `webhook-validator` Lambda upstream — payloads reaching `prevalidate` are already authenticated. There is no idempotency dedupe gate (see [`webhook_handler.py`](../src/agent/application/webhook_handler.py) module docstring). - [ ] [`RecursionGuard`](#recursionguard-aka-recursionpolicy) — agent-Jira-identity match on `actor.account_id` / `actor.email` (case-insensitive on email); either match short-circuits with skip reason `agent_account_id` / `agent_email`. -- [ ] [`IdempotencyStore`](#idempotencystore) — atomic conditional-insert with TTL keyed by `issue_key + event_kind + changelog_id / comment_id`. - [ ] [`DomainStateStore`](#domainstatestore) — read-your-writes consistent value-object store keyed by `issue_key`; carries readiness phase + pending follow-ups + repo context + evidence refs. - [ ] [`McpClient`](#mcpclient) — MCP Streamable HTTP (single `POST /mcp` endpoint) with bearer-token auth and `roots` capability advertised at `initialize` time. - [ ] [`AgenticChatModel`](#agenticchatmodel) — exposes both `bind_tools(tools)` and `with_structured_output(schema)`; the two-phase split is mandatory (see [#two-runnable-cost-trade](#two-runnable-cost-trade)). -- [ ] [`TokenBudgetEnforcer`](#tokenbudgetenforcer) — `check()` before LLM call, `commit()` after; rolling per-`(tenant, window_start)` window. +- [ ] [`TokenBudgetEnforcer`](#tokenbudgetenforcer) — `reserve()` before LLM call, `finalize()` after success, `release()` on pre-call / model failure; rolling per-`(tenant, window_start)` window. - [ ] [`LlmCircuitBreaker`](#llmcircuitbreaker) — closed → open → half-open with randomized exponential backoff for the half-open probe. - [ ] [`CostCalculator`](#costcalculator) — vendored frozen pricing snapshot; cache-read pricing is a separate rate. - [ ] [`TokenUsageObserver`](#tokenusageobserver) — emits `gen_ai.client.token.usage`, `gen_ai.client.cost.usd`, the GenAI semantic-conventions span attributes, and the structured `llm.usage` log line. - [ ] [`Tracer`](#tracer) — context-manager spans with the canonical names; never suppresses exceptions. -- [ ] [`MetricsRecorder`](#metricsrecorder) — counters and histograms with the canonical names (`webhook.received`, `webhook.processed`, `webhook.skipped`, `webhook.invariant_violation`, `mcp.tool.calls`, `mcp.call.duration_ms`, `mcp.call.retry_count`, `webhook.retried`). +- [ ] [`MetricsRecorder`](#metricsrecorder) — counters and histograms with the canonical names (`webhook.received`, `webhook.processed`, `webhook.skipped`, `webhook.error`, `webhook.duration_ms`, `mcp.tool.calls`, `mcp.call.duration_ms`). - [ ] [`RawPayloadSink`](#rawpayloadsink) — idempotent retention keyed by `(correlation_id, payload_hash)`; production is versioned + KMS-encrypted + WORM. - [ ] [`DeadLetterPublisher`](#deadletterpublisher) — never silently swallows; failure to publish is logged and re-raised. - [ ] [`TenantResolver`](#tenantresolver) — produces `(tenant_id, cloud_id, display_name)` so the LangGraph `thread_id` partitions per tenant. -- [ ] [`Clock`](#clock) — UTC monotonic source; every time-comparison reads it (idempotency expiry, budget window, breaker scheduling). +- [ ] [`Clock`](#clock) — UTC monotonic source; every time-comparison reads it (budget window roll-over, breaker scheduling, structured-log timestamps). - [ ] [`StructuredLogger`](#structuredlogger) — one JSON line per call; `correlation_id` and `tenant_id` propagated automatically from the request scope; never emits plaintext secrets. - [ ] [`SecretsResolver`](#secretsresolver) — read-side `(tenant, secret_name) → plaintext`; rotation is owned out-of-band. -- [ ] [`McpCallObserver`](#mcpcallobserver) — `record(tool_name, kind, correlation_id, duration_ms?)` exactly once per MCP invocation; the production observer also implements `count_for_prefix(correlation_id, prefix)` so the `EventInvariantPolicy` can demand `git_*` reads on `issue_updated`. +- [ ] [`McpCallObserver`](#mcpcallobserver) — `record(tool_name, kind, correlation_id, duration_ms?)` exactly once per MCP invocation; the production observer also implements `count_for_prefix(correlation_id, prefix)` so node-level callers (e.g. the assessor) can ask "did the agent call at least one `git_*` tool while handling this `issue_updated`?" without coupling to a global policy object. - [ ] [Request-scoped context primitive](#request-scoped-context-primitive) — host-native carrier for `tenant_id` and `correlation_id` (`AsyncLocal` / `ThreadLocal` / `AsyncLocalStorage` / `contextvar` / `context.Context`). The patterns from [#patterns](#patterns) (Hexagonal, Composition Root, @@ -628,37 +879,30 @@ Port lives in the inwards-facing interface layer, every concrete adapter is wired from the Composition Root, and cross-cutting concerns attach via the Decorator chain documented in [#token-governance](#token-governance). -### `WebhookSignatureVerifier` - -- **Represents:** authenticity check on the inbound webhook envelope - (the agent does not trust the network). -- **Inputs / outputs:** input is the raw request body bytes plus the - signature value carried by the envelope (`x-jira-signature` field - inside the SDK envelope, not an HTTP header); output is a single - `bool` — `true` when authentic, `false` otherwise. -- **Required behavior:** computes HMAC-SHA256 over the raw envelope - using a shared secret loaded at composition time; rejects when the - computed digest does not match the signature; performs a constant-time - comparison so a leaked digest cannot be timing-attacked; treats a - missing or empty signature as "not authentic". - ### `WebhookHandler` (application-layer use case) - **Represents:** the orchestrating use case that gates an inbound - webhook through signature → normalization → recursion → idempotency - → graph-invocation, in that fixed order. -- **Inputs / outputs:** input is `(raw_body, signature, parsed_payload, - correlation_context)`; output is a typed `(status, reason)` response - drawn from `{processed, skipped, duplicate, invariant_violation}`. -- **Required behavior:** signature verification runs first; on failure - it returns immediately without normalizing or persisting anything. - Normalization produces an immutable `NormalizedJiraEvent` value - object. The recursion and idempotency gates run **before** the LLM - is touched. Each gate emits the same metric / span names documented - in [#end-to-end-flow](#end-to-end-flow). On success, the request-scoped - context primitive is pushed before the graph invocation and popped - after it returns. Graph invocation is keyed by `thread_id` plus the - envelope, never by free-text prompt. + webhook through identity → normalization → tenant + resolution → recursion → graph-invocation, in that fixed order. + HMAC authentication is performed by the `webhook-validator` Lambda + upstream; this handler only sees pre-authenticated JSON envelopes. +- **Inputs / outputs:** input is `(parsed_payload, + correlation_context, authorization_header?)`; output is a typed + `(status, reason)` response drawn from `{processed, skipped, error}`. +- **Required behavior:** the parsed envelope may be archived for + forensic replay; trust decisions assume the upstream Lambda + cleared the signature. Normalization produces an immutable + `NormalizedJiraEvent` value object. The recursion gate runs + **before** the LLM is touched. Each gate emits the same metric / + span names documented in [#end-to-end-flow](#end-to-end-flow). + There is **no** webhook-layer idempotency dedupe — duplicate + Atlassian retries reconcile through live MCP refetch + the + persisted `DomainStateStore` phase + the + `persist_approval_granted` checkpoint (see + [§Multi-agent topology](#multi-agent-topology)). On success, the + request-scoped context primitive is pushed before the graph + invocation and popped after it returns. Graph invocation is keyed + by `thread_id` plus the envelope, never by free-text prompt. ### `RecursionGuard` (a.k.a. `RecursionPolicy`) @@ -669,25 +913,13 @@ attach via the Decorator chain documented in [#token-governance](#token-governan comment body; output is a `(skipped: bool, reason: str | null)` decision (the protocol-level shape — implementations may flatten this to "non-empty reason string means skip"). -- **Required behavior:** skips when the actor account id is in a - configured bot allowlist; skips when a comment body starts with one - of the registered bot markers (M4.6 introduces a second marker — - the recursion guard must accept a list of markers, not a single - literal). Defense-in-depth: actor and marker checks are independent; - either match short-circuits. - -### `IdempotencyStore` - -- **Represents:** persistent set of recently-seen webhook ids. -- **Inputs / outputs:** input is a string key (the deterministic dedupe - key derived from `issue_key + event_kind + changelog_id / - comment_id`) plus an absolute expiry timestamp; output is a `bool` - meaning "newly inserted" (`true`) vs "already present" (`false`). -- **Required behavior:** a conditional-write contract — the - insert-and-return-result pair must be atomic so concurrent webhook - deliveries cannot both observe `true`. TTL is enforced server-side; - expired entries are reclaimable without external sweeping. Replays - return `false` and short-circuit before the graph is invoked. +- **Required behavior:** skips when the actor account id matches the + configured agent Jira account id; skips when the actor email matches + the configured agent email case-insensitively. The guard does not + rely on comment markers: Jira webhook bodies may be truncated or + reformatted, while `actor.account_id` / `actor.email` are structured + payload fields. Either actor-identity match short-circuits with the + stable skip reason `agent_account_id` / `agent_email`. ### `DomainStateStore` @@ -746,17 +978,23 @@ attach via the Decorator chain documented in [#token-governance](#token-governan - **Represents:** per-tenant rolling-window budget for input tokens, output tokens, and USD cost. -- **Inputs / outputs:** `check(tenant)` raises a `BudgetExceededError` - when any cap would be exceeded if a typical call were committed - right now — no return value on success. `commit(tenant, usage, - usd_cost)` records actual consumption after a successful call. -- **Required behavior:** `check` runs **before** the LLM call so a - rejected request never consumes provider tokens; `commit` runs - **after** a successful call. The window rolls over atomically when +- **Inputs / outputs:** `reserve(tenant, headroom_tokens)` atomically + reserves pre-call input-token headroom and returns an opaque + reservation id; it raises `BudgetExceededError` when the reservation + would exceed a configured cap. `finalize(tenant, reservation_id, + usage, usd_cost)` swaps reserved headroom for actual usage after a + successful call. `release(tenant, reservation_id)` discards the + reservation when the breaker rejects the call or the model invocation + fails before usage is committed. +- **Required behavior:** `reserve` runs **before** the LLM call so a + rejected request never consumes provider tokens, and the reservation + is visible to concurrent calls in the same `(tenant, window_start)` + window. `finalize` runs **after** a successful call; `release` runs + on pre-call / model failure. The window rolls over atomically when `window_seconds` elapses. Implementations may be in-memory (single-process scope — see [#token-governance-local-pod-scope](#token-governance-local-pod-scope)) - or distributed (atomic increment with TTL keyed by + or distributed (atomic reservation + TTL keyed by `tenant#window_start`). ### `LlmCircuitBreaker` @@ -835,10 +1073,9 @@ attach via the Decorator chain documented in [#token-governance](#token-governan no-op default drops every call so adapters that take a metrics recorder always have a safe fallback. Counter / histogram names follow the canonical list (`webhook.received`, `webhook.processed`, - `webhook.skipped`, `webhook.invariant_violation`, + `webhook.skipped`, `webhook.error`, `webhook.duration_ms`, `mcp.tool.calls{tool,kind,event_kind}`, - `mcp.call.duration_ms`, `mcp.call.retry_count`, - `webhook.retried`). + `mcp.call.duration_ms`). ### `RawPayloadSink` @@ -861,8 +1098,8 @@ attach via the Decorator chain documented in [#token-governance](#token-governan - **Required behavior:** never silently swallows the envelope — publication failures are logged and re-raised. Backed by a message-broker DLQ with redrive in production; LocalStack-equivalent - locally. The handler routes invariant-violation envelopes here - (see `EventInvariantViolatedError` in [#end-to-end-flow](#end-to-end-flow)). + locally. The handler routes terminal failures (`status="error"`) + here so a poison payload cannot silently disappear. ### `TenantResolver` @@ -872,12 +1109,15 @@ attach via the Decorator chain documented in [#token-governance](#token-governan `TenantBinding` value object containing `tenant_id`, optional `cloud_id`, and optional `display_name`. - **Required behavior:** the LangGraph `thread_id` is derived as - `tenant_id|cloud_id|project_id|issue_key` so every persisted - artifact is partitioned per tenant. Single-tenant deployments may - return a fixed binding for every event; multi-tenant deployments - consult a configured directory (DynamoDB, etc.). Per-tenant KMS / - Secrets Manager isolation is downstream of this resolver, not - encoded into it. + `tenant_id:issue_key` (the ``:`` separator is owned by the + ``THREAD_ID_SEPARATOR`` constant in + [`src/agent/application/thread_id.py`](../src/agent/application/thread_id.py) + and asserted verbatim by the architecture-conformance tests) so + every persisted artifact is partitioned per tenant. Single-tenant + deployments may return a fixed binding for every event; multi-tenant + deployments consult a configured directory (DynamoDB, etc.). + Per-tenant KMS / Secrets Manager isolation is downstream of this + resolver, not encoded into it. ### `Clock` @@ -886,9 +1126,10 @@ attach via the Decorator chain documented in [#token-governance](#token-governan - **Inputs / outputs:** `now()` returns the current UTC instant as a timezone-aware date-time value. - **Required behavior:** every time-comparison in the application - layer (idempotency expiry, token-budget window roll-over, breaker - half-open scheduling) reads from this Port so unit tests can - inject a frozen clock without monkeypatching globals. + layer (token-budget window roll-over, breaker half-open + scheduling, structured-log timestamps) reads from this Port so + unit tests can inject a frozen clock without monkeypatching + globals. ### `StructuredLogger` @@ -920,19 +1161,20 @@ attach via the Decorator chain documented in [#token-governance](#token-governan ### `McpCallObserver` - **Represents:** sink for MCP-tool invocation events fed into the - per-event-kind invariant policy and the `mcp.*` telemetry feed. + `mcp.*` telemetry feed and consumed by node-local diagnostics. - **Inputs / outputs:** `record(tool_name, kind, correlation_id, duration_ms)` — `kind` is constrained to `{"read", "write"}`. The read-side companion Port `McpCallCountReader` exposes `read_count(correlation_id)`, `write_count(correlation_id)`, `total_count(correlation_id)`, `count_for_prefix(correlation_id, - prefix)`, and `reset(correlation_id)` so the - `EventInvariantPolicy` can ask questions like "did the agent call - at least one `git_*` tool while handling this `issue_updated`?". + prefix)`, and `reset(correlation_id)` so node-level callers can + ask diagnostic questions like "did the agent call at least one + `git_*` tool while handling this `issue_updated`?" without + coupling to a global policy object. - **Required behavior:** called exactly once per MCP invocation (success **or** failure — a failed write is still write-shaped - intent, the invariant policy cares about intent not success). - Cheap on the hot path; tolerates concurrent calls. Emits the + intent that consumers may want to count). Cheap on the hot path; + tolerates concurrent calls. Emits the `mcp.tool.calls{tool,kind,event_kind}` counter and the `mcp.call.duration_ms` histogram when wired against the metrics recorder. The companion read-side Port retains a per-correlation @@ -975,7 +1217,7 @@ that section glosses over: **scope of the budget**. `UpdateItem ADD` against a `tenant#window_start` partition key. That adapter is the inner cap regardless of how many replicas the AWS profile runs. -- **Startup guardrail (M5.9 d):** `AGENT_PROFILE=local` requires the +- **Startup guardrail:** `AGENT_PROFILE=local` requires the operator to set `AGENT_BUDGET_SCOPE=local-pod-dev` to acknowledge per-pod scope. Without it, `_resolve_local_dependencies` ([src/agent/main.py](../src/agent/main.py)) raises a `ValueError` @@ -999,6 +1241,27 @@ wall-clock per run regardless of replica count. --- +## Developer tooling — graph-level debugging + +The composition root has a second entrypoint alongside +`agent.main.resolve_dependencies` (programmatic dependency resolution +consumed by the `agent-worker` SQS Pod): +**`agent.composition.studio.build_studio_graph`**, a zero-arg factory +the [LangGraph CLI](https://langchain-ai.github.io/langgraph/cloud/reference/cli/) +loads via the repo-root `langgraph.json`. It returns the same +compiled `StateGraph` the production `WebhookHandler` invokes, so what +LangGraph Studio renders matches what runs in the cluster. + +The factory is not part of the production runtime image — it lives +behind the optional `[debug]` extra (see +[`pyproject.toml`](../pyproject.toml)) and is invoked only by +`make langgraph-dev`. Production AWS images **must not** install the +`[debug]` extra. + +See [`docs/PYTHON_DEVELOPMENT.md` §5.5](PYTHON_DEVELOPMENT.md#55-langgraph-studio-graph-level-debugging) +for the full Studio walkthrough (state inspection, time-travel, +edit-and-replay). + ## Cross-references - [README.md](../README.md) — entry point and project layout for the diff --git a/docs/DEFERRED.md b/docs/DEFERRED.md new file mode 100644 index 0000000..d8842b1 --- /dev/null +++ b/docs/DEFERRED.md @@ -0,0 +1,170 @@ +--- +# Deferred work register parsed by ``scripts/check_docs_code_sync.py`` and +# the ``docs-code-sync`` PR-required job. Each entry MUST appear under +# the corresponding section in ``README.md`` (the ``## What's deferred to +# AWS production`` list) and MUST also have a 1:1 row here. CI fails +# the ``docs-code-sync`` job when an entry's ``risk: High`` row has +# ``next_review_date < today`` so high-risk deferrals cannot rot. +register_version: 1 +review_cadence_days: 30 +items: + - id: deferred-bedrock-guardrails + owner: platform-ml + opened: 2026-04-15 + next_review_date: 2026-06-01 + risk: medium + exit_criteria: >- + Managed Bedrock Guardrail provisioned in the sandbox account; CI + asserts ``terraform output guardrail_arn`` is non-empty and the + AWS-profile smoke (`smoke.yml::aws-profile-smoke`) records at + least one guardrail-blocked input. + scaffolding_path: terraform/bedrock-guardrail/ + - id: deferred-agentcore-runtime-native + owner: platform-runtime + opened: 2026-04-15 + next_review_date: 2026-06-01 + risk: high + exit_criteria: >- + AWS provider ships ``aws_bedrockagentcore_runtime`` / + ``_memory`` resources; the ``native_provider_probe`` flips to + ``true`` and the ``tf-providers-availability-probe`` PR job + fails until ``var.use_native_provider = true``. + scaffolding_path: terraform/agentcore-runtime/main.tf + - id: deferred-xray-exporter + owner: platform-obs + opened: 2026-04-15 + next_review_date: 2026-06-30 + risk: low + exit_criteria: >- + ``[xray]`` extra installed in the AWS-profile image; X-Ray + sampling rules visible in CloudWatch ServiceLens; the AWS-profile + smoke records at least one segment. + scaffolding_path: src/agent/infrastructure/observability/ + - id: deferred-iam-scp-attach + owner: security + opened: 2026-04-15 + next_review_date: 2026-06-14 + risk: high + exit_criteria: >- + ``terraform/organizations/`` applied with ``var.attach_scp = + true`` (the default) against the production Organization; the + ``iam_policy_canary --scp-plan`` job has been green on ``main`` + for at least one full review cadence. + scaffolding_path: terraform/organizations/ + - id: deferred-edge-security + owner: platform-edge + opened: 2026-04-15 + next_review_date: 2026-06-30 + risk: medium + exit_criteria: >- + ``terraform/edge-security/`` applied with + ``use_api_gateway_v2 = true``; the JWT authorizer aligns with + the AgentCore Identity issuer/audience; the AWS-profile smoke + reaches the runtime via the v2 stage URL. + scaffolding_path: terraform/edge-security/ + - id: deferred-kms-rotation-cycle + owner: security + opened: 2026-04-15 + next_review_date: 2026-06-14 + risk: high + exit_criteria: >- + ``terraform/kms-secrets/`` rotation Lambda cycles the + ``atlassian/webhook-hmac`` and ``bedrock/invocation-key`` + secrets at least once in the sandbox account; the + ``sandbox-apply`` workflow records the rotation events. + scaffolding_path: terraform/kms-secrets/lambda/rotation/ + - id: deferred-mcp-internal-image-pin + owner: platform-mcp + opened: 2026-04-15 + next_review_date: 2026-06-15 + risk: medium + exit_criteria: >- + ``mcp-internal`` image digest pinned via Secrets Manager-backed + bearer-token rotation; the SoftNoopWriter smoke step + (``smoke.yml::local-stack-smoke``) keeps green for one review + cadence and the ``mcp-internal`` JAR is shipped as a tagged + release artifact (``mcp/README.md``). + scaffolding_path: mcp/ +--- + +# Deferred work register + +## Review-window extension log + +| Date | Item | Previous → New `next_review_date` | Justification | +|------|------|-----------------------------------|---------------| +| 2026-05-06 | `deferred-iam-scp-attach` | 2026-05-15 → 2026-06-14 | Externally blocked on production Organization approval (see "High-risk blocker status" below). Extended by one `review_cadence_days` (30) to keep the docs-code-sync hard gate from firing while the security owner schedules the apply window. Closure is *not* a docs-only change and remains tracked separately. | +| 2026-05-06 | `deferred-kms-rotation-cycle` | 2026-05-15 → 2026-06-14 | Externally blocked on sandbox credentials and a completed Secrets Manager rotation exercise (see "High-risk blocker status" below). Extended by one `review_cadence_days` (30); deterministic closure recipe in [`docs/runbook-sandbox-deploy.md` § 13](runbook-sandbox-deploy.md#13-closing-related-deferrals) is unchanged. | + +This file is the canonical, machine-parsed register of work the project +has explicitly deferred to a future milestone. Every entry above is +mirrored 1:1 in [`README.md`](../README.md) under the +[`## What's deferred to AWS production`](../README.md#whats-deferred-to-aws-production) +list, and every entry under that README list MUST appear here. + +The PR-required `docs-code-sync` job in +[`.github/workflows/ci.yml`](../.github/workflows/ci.yml) parses the +YAML frontmatter at the top of this file via +[`scripts/check_docs_code_sync.py`](../scripts/check_docs_code_sync.py) +and fails when: + +1. a `risk: high` entry has `next_review_date < today`, OR +2. an entry referenced in `README.md` is missing from this register, OR +3. an entry references a `scaffolding_path` that does not exist in the + working tree. + +The same job emits a GitHub Actions warning when a high-risk entry is within +14 days of `next_review_date`. That warning is intentionally non-blocking: it +gives owners time to schedule security / platform review before the hard gate +fires. + +## High-risk blocker status + +Every high-risk deferral has an in-repo landing zone and an explicit +external closure condition: + +- `deferred-agentcore-runtime-native` is externally blocked on the AWS provider + shipping stable `aws_bedrockagentcore_runtime` / `_memory` resources. The repo + evidence is the `native_provider_probe` in + [`terraform/agentcore-runtime/main.tf`](../terraform/agentcore-runtime/main.tf) + and the `.state/` recovery guidance in + [`terraform/agentcore-runtime/README.md`](../terraform/agentcore-runtime/README.md). +- `deferred-iam-scp-attach` is externally blocked on production Organization + approval and apply. The repo evidence is the Organizations module plus the + `iam_policy_canary --scp-plan` workflow gate. +- `deferred-kms-rotation-cycle` is externally blocked on sandbox credentials and + a completed Secrets Manager rotation exercise. The repo evidence is the + `sandbox-apply` workflow path that applies `terraform/kms-secrets` and triggers + rotation when the protected sandbox secret is present. The deterministic + closure recipe — including the per-secret `aws secretsmanager rotate-secret` + and `LastRotatedDate` evidence capture — lives in + [`docs/runbook-sandbox-deploy.md` § 13](runbook-sandbox-deploy.md#13-closing-related-deferrals). +- `deferred-edge-security` is externally blocked on the API Gateway v2 stage + smoke against the sandbox JWT authorizer. The deterministic closure + recipe lives in + [`docs/runbook-sandbox-deploy.md` § 13](runbook-sandbox-deploy.md#13-closing-related-deferrals). + +The shape (`id`, `owner`, `opened`, `next_review_date`, +`exit_criteria`, `risk`, `scaffolding_path`) is intentionally minimal so +operators can reason about what is owed without reading prose. The +review cadence default is 30 days, configurable per entry via the +top-level `review_cadence_days` field. + +## How to add a new deferred item + +1. Add a new bullet under [`README.md`](../README.md) `## What's + deferred to AWS production` with a one-line description. +2. Append a new entry here with the seven fields above and a + `scaffolding_path` pointing at the in-repo seam that owns the + eventual landing pad (Terraform module, Python module, manifest + directory, etc.). +3. Wire the exit-criteria into a CI gate that flips green when the + deferral is closed (probe job, native-provider auto-flip, + smoke step, etc.). + +## Closing a deferred item + +1. Land the change behind the matching CI gate. +2. Remove the bullet from `README.md`. +3. Remove the entry here. +4. The `docs-code-sync` job rejects the PR if either side is missed. diff --git a/docs/PYTHON_DEVELOPMENT.md b/docs/PYTHON_DEVELOPMENT.md index 7bd1964..77e90a4 100644 --- a/docs/PYTHON_DEVELOPMENT.md +++ b/docs/PYTHON_DEVELOPMENT.md @@ -20,10 +20,10 @@ The intended loop is: ## 1. Prerequisites -- **Python 3.10+** (3.12 target; CI runs the 3.10 / 3.11 / 3.12 matrix - and the runtime image base is 3.12). The +- **Python 3.12+** (3.13 target; CI runs the 3.12 / 3.13 / 3.14 matrix + and the runtime image base is 3.13). The [`pyproject.toml`](../pyproject.toml) `requires-python` constraint is - `>=3.10`. + `>=3.12`. - A working virtualenv tool (`venv` ships with the standard library). - An **Anthropic API key** (`sk-ant-…`) — get one from . Required to talk to the LLM under @@ -37,7 +37,6 @@ The intended loop is: ```bash git clone cd aws-agent-core -git lfs install && git lfs pull # materialises mcp/mcp-internal-*.jar python -m venv .venv && source .venv/bin/activate pip install -e ".[dev,anthropic]" cp .env.local.example .env.local @@ -52,9 +51,15 @@ extra installs `pytest`, `pytest-cov`, `coverage`, `mypy`, `ruff`, `freezegun`, `hypothesis`, and `watchfiles` (the last drives in-Pod hot-reload — see §5). -`mcp/mcp-internal-*.jar` is tracked through Git LFS (see -[`.gitattributes`](../.gitattributes)). Without `git lfs pull` you have -a ~130-byte pointer file; `make build-mcp` detects this and fails fast. +The `mcp-internal` Spring Boot fat-JAR (~62 MB) is no longer tracked +in this repository. `scripts/build_mcp_image.sh` resolves it from +outside the repo (`--jar` / `MCP_INTERNAL_JAR` / +`${MCP_INTERNAL_REPO:-$HOME/code/github/mcp-internal}/build/libs/mcp-internal-*.jar` +in that order) and stages it into `mcp/.build/mcp-internal.jar` for +the duration of `docker build`. Build the JAR once with +`( cd "$MCP_INTERNAL_REPO" && ./gradlew bootJar )`; rebuild it +whenever the upstream project ships a new revision. Full contract +in [`../mcp/README.md`](../mcp/README.md) §1. ## 3. Iterating against the running infrastructure @@ -95,9 +100,9 @@ command runs the whole bar: | Target | Purpose | |--------|---------| | `make lint` | `ruff check src tests` (settings in [`pyproject.toml`](../pyproject.toml) — `E`, `F`, `I`, `B`, `UP`, `N`, `RUF`, `SIM` rule families). | -| `make typecheck` | `mypy --strict src/agent` — zero typing errors against `src/agent/` on Python 3.10. | +| `make typecheck` | `mypy --strict src/agent` — zero typing errors against `src/agent/` on Python 3.12. | | `make test` | `pytest` (configured in [`pyproject.toml`](../pyproject.toml)) with `--cov=agent --cov-branch --cov-fail-under=100`. 100% line and branch coverage on every production module. | -| `make ci` | `lint + typecheck + test` in sequence — the CI gates in one command. | +| `make ci` | `lint + typecheck + import-lint + test` in sequence — the CI gates in one command. The `import-lint` step runs the four `[tool.importlinter]` contracts in [`pyproject.toml`](../pyproject.toml) (test-seam isolation, Domain independence, Application/Graph adapter-freedom, Contracts adapter-freedom). | Coverage artifacts (`coverage.xml` machine-readable, `htmlcov/` human-readable) are written on every run and uploaded by CI. @@ -222,6 +227,111 @@ kubectl apply -k deploy/local/ Re-applies the base manifests, which overwrite both patched Deployments with their non-host-mounted defaults. +### 5.5 LangGraph dev server (graph-level debugging) + +The `[debug]` extra ships [`langgraph-cli[inmem]`](https://langchain-ai.github.io/langgraph/cloud/reference/cli/), +a local dev server that exposes a compiled `StateGraph` over HTTP for +graph-level debugging (state inspection, conditional-edge decisions, +single-node re-execution, manual edit-and-replay). This repo +deliberately does not bundle a hosted UI: we never point at +`smith.langchain.com` because that surface is part of the LangSmith +product (a paid SaaS we have removed from this codebase). + +**One-time install.** + +```bash +pip install -e ".[dev,anthropic,debug]" +cp .env.local.example .env.local # then fill in ANTHROPIC_API_KEY etc. +``` + +**Run the dev server.** + +```bash +make langgraph-dev # serves http://127.0.0.1:2024 +``` + +The server reads `.env.local` automatically (per the `env` field in +`langgraph.json`), spins up the in-memory checkpointer, and exposes +the compiled `agent.composition.studio:build_studio_graph` topology. + +**Inspect the graph.** + +The dev server runs locally on `127.0.0.1:2024` and ships no hosted +UI from this repo. Two supported inspection paths: + +1. **Static topology snapshot.** Render the compiled graph to a PNG: + + ```bash + make graph-png # writes docs/graph.png + ``` + + Useful for design reviews and architecture diagrams. + +2. **Locally-hosted Studio.** Newer `langgraph-cli` releases ship a + self-hosted Studio variant (e.g. `--studio-local` or equivalent); + check `langgraph dev --help` for the flag your installed version + exposes. Confirm the flag does not call out to a remote service + before enabling it. + +**What the dev server exposes.** + +- The full assessor / designer topology (the same one + [`docs/ARCHITECTURE.md`](ARCHITECTURE.md#multi-agent-topology) + documents), accessible via the LangGraph CLI's HTTP API. +- Per-node state after each step, including the + `messages: list[BaseMessage]` channel and every domain-typed slot + on `AgentState`. +- Checkpoint forking semantics so a debug client can rewind to any + prior node, edit the state, and replay -- handy for debugging + non-deterministic LLM responses without re-running the full + request from the top. + +**Limits.** + +- The dev server does **not** load the production composition root: + it uses the LangGraph CLI's bundled in-memory checkpointer rather + than `MemorySaver` / `AgentCoreMemorySaver`, so checkpoints from a + dev-server session do not persist across restarts. +- Tool calls (`mcp-internal` reach-out) hit whatever `MCP_BASE_URL` + points at in `.env.local`. For a fully isolated debug loop you + can point it at `mcp-internal` running locally via `docker run` + (see §6.1) so a dev-server session never reaches a real Atlassian + Cloud tenant. + +### 5.6 Why `langsmith` appears in the lockfile (known dormant transitive) + +The `langsmith` PyPI package is locked in [`uv.lock`](../uv.lock) +because [`langchain-core`](https://pypi.org/project/langchain-core/) +declares it as a hard runtime dependency. This repo deliberately +does **not** use the LangSmith SaaS: + +- No source file imports `langsmith` (verify with + `rg 'from langsmith|import langsmith' src/`). +- No `LANGSMITH_*` / `LANGCHAIN_TRACING_*` / `LANGCHAIN_API_KEY` + environment variables are set anywhere in the project (none in + [`.env.local.example`](../.env.local.example), the deploy YAML + manifests, the Terraform modules, or CI workflows). +- No LangSmith client is constructed; the in-house + [`GovernedAgenticChatModel`](../src/agent/application/token_governance.py) + (token-budget enforcer, circuit breaker, cost calculator, + `TokenUsageObserver`) provides governance without any LangSmith + dependency. + +The package therefore ships **dormant** -- locked, installed when +`langchain-core` is installed, never imported, never authenticated, +never reached over the network from this codebase. + +**Closing this gap requires one of:** + +1. `langchain-core` upstream dropping its `langsmith` runtime dep + (track the changelog), or +2. Migrating the LLM port adapters under + [`src/agent/infrastructure/`](../src/agent/infrastructure/) off + `langchain-core` to direct provider SDKs (`anthropic`, `boto3` + Bedrock, `ollama`). + +Until then, treat the `langsmith` row in `uv.lock` as expected. + ## 6. Bare-metal path (no Docker Desktop) The Docker Desktop + kustomize path is the canonical local profile and @@ -240,9 +350,9 @@ containers (or external services). ### 6.1 Prerequisites -- **Python 3.12** (3.10 / 3.11 also work; 3.12 matches the Dockerfile +- **Python 3.13** (3.12 / 3.14 also work; 3.13 matches the Dockerfile base image and the recommended local default — the project floor is - 3.10 with 3.12 as the target). + 3.12 with 3.13 as the target). - A virtualenv with the dev + Anthropic extras installed (§2 above). - A reachable `mcp-internal` HTTP endpoint. The two supported options are: @@ -272,7 +382,7 @@ containers (or external services). [`src/agent/main.py`](../src/agent/main.py) via the `**dependency_overrides` test-injection seam. This path is used by the unit-test suite under [`tests/`](../tests/) and is - documented in the `agent.main.build_app_from_environment(...)` + documented in the `agent.main.resolve_dependencies(...)` docstring; it does not require a network at all but does require you to write a small bootstrap script, not a one-liner. @@ -288,7 +398,7 @@ export AGENT_PROFILE=local export AGENT_BUDGET_SCOPE=local-pod-dev # required under AGENT_PROFILE=local; ack per-pod budget scope export ANTHROPIC_API_KEY=sk-ant-... # your real key export ANTHROPIC_MODEL=claude-sonnet-4-5-20250929 -export MCP_BASE_URL=http://localhost:8081/mcp # was http://mcp-internal:8081/mcp +export MCP_BASE_URL=http://localhost:8081/mcp # in-cluster (local k8s): http://mcp-internal:8081/mcp export MCP_BEARER_TOKEN=DAVIDSUPERSECRETTOKEN export LOCALSTACK_ENDPOINT_URL=http://localhost:4566 # was http://localstack:4566 export AWS_REGION=us-east-1 @@ -297,7 +407,6 @@ export AWS_SECRET_ACCESS_KEY=test export AGENT_JIRA_ACCOUNT_ID=local-bot-account-id export AGENT_JIRA_EMAIL=local-bot@example.test export OBSERVABILITY_BACKEND=jaeger -export WEBHOOK_HMAC_SECRET=local-dev-secret ``` The two cluster-only variables you do **not** keep verbatim are the @@ -380,7 +489,7 @@ sed "s|/CHANGE-ME/code/github/aws-agent-core/src|$HOSTPATH_AGENT_SRC|" \ - [`README.md`](../README.md) — language-agnostic architecture and the Infrastructure quickstart. - [`deploy/local/README.md`](../deploy/local/README.md) — Kubernetes - manifests, smee.io / ngrok webhook replay, troubleshooting. + manifests, curl-driven smoke tests, troubleshooting. - [`docs/ARCHITECTURE.md`](ARCHITECTURE.md) — layered architecture, deployment topology, end-to-end flow, token governance. - [`Makefile`](../Makefile) — frozen developer-command vocabulary. diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..534b6d2 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,10 @@ +# Architecture Decision Records + +Use one markdown file per significant, load-bearing design choice that +operators and maintainers need to find without spelunking git history. + +Suggested filename pattern: `NNNN-short-title.md` (four-digit prefix). + +Each ADR should include at minimum: **Context**, **Decision**, and +**Consequences**, plus links to the code or Terraform that implements +the decision (repository-relative paths). diff --git a/docs/deploy-eks-secrets-csi.md b/docs/deploy-eks-secrets-csi.md new file mode 100644 index 0000000..e919f33 --- /dev/null +++ b/docs/deploy-eks-secrets-csi.md @@ -0,0 +1,146 @@ +# EKS workloads: Secrets Store CSI, Pod Identity, and KMS (deployment report) + +This document captures **failures observed on a live sandbox EKS cluster** (2026-05), the **root causes**, **repo and AWS fixes**, and the **guardrails** added so the next `terraform apply` + `scripts/bootstrap_eks_workloads.sh` (or `scripts/rebuild_sandbox.sh`) run succeeds without repeating the same class of errors. + +## Every future deploy (required operator actions) + +Do **all** of the following on each deploy where EKS workloads or Secrets Store CSI mounts matter (full detail below; **`aws-deploy-plan.md`** Phase 3 mirrors this order). + +1. **Jira JSON for `mcp-internal` CSI** — Keep **`.env.local`** (or exported shell env) with **`JIRA_SITE_URL`**, **`JIRA_EMAIL`**, and **`JIRA_API_TOKEN`** set to real values, **or** accept that Jira sync is skipped (CSI JMESPath will fail until the secret is valid JSON). + - Automated: **`scripts/sync_jira_integration_secret_from_env.sh`** (called from **`scripts/rebuild_sandbox.sh`** before EKS bootstrap and from **`scripts/bootstrap_eks_workloads.sh`** when the triple is present). + - Manual alternative: **`aws-deploy-plan.md`** §3.2a (`put-secret-value` with `jq`). + +2. **After `terraform/kms-secrets` apply** — Run **`scripts/sync_sandbox_tfvars.py --phase pre-eks`** so **`terraform/secrets-mcp/terraform.tfvars`** `kms_key_arn` and IAM/EKS **`mcp_secrets_kms_key_arns` / `secrets_kms_key_arns`** align with the default tenant CMK **before** **`terraform/secrets-mcp` apply**. + - **`scripts/rebuild_sandbox.sh`** runs this automatically **immediately after** each **`kms-secrets` apply** and **before** **`secrets-mcp` apply**. + +3. **After EKS exists** — Run **`scripts/sync_sandbox_tfvars.py --phase post-eks`**, then **`terraform apply`** on **`terraform/kms-secrets`** again so **`additional_decrypt_role_arns`** on the CMK includes **agent-worker**, **mcp-token-reader**, and **webhook-validator** Lambda role (plus any other entries your tfvars carry). + - Manual Phase 3.1 block in **`aws-deploy-plan.md`** already lists this sequence; **`rebuild_sandbox.sh`** runs post-EKS sync, **`kms-secrets`** pass 2, webhook sync, and **`webhook-validator`** re-apply. + +4. **Helm CSI driver + provider** — Use **`scripts/bootstrap_eks_workloads.sh`** (or the full **`scripts/rebuild_sandbox.sh`**) so install order (**driver → ASCP**), **`secrets-store-csi-driver.install=false`** on ASCP, and driver **`tokenRequests`** for **`sts.amazonaws.com`** and **`pods.eks.amazonaws.com`** stay correct. Do **not** hand-install only one chart unless you match **`aws-deploy-plan.md`** §CSI manual steps. + +5. **`webhook-validator` Lambda** — Run **`scripts/sync_sandbox_tfvars.py --phase pre-eks`** + **after `terraform/edge-security` apply** (with **`data-stores`** already applied) so + **`terraform/webhook-validator/terraform.tfvars`** contains **`https_listener_arn`**, + **`work_queue_arn`**, and **`work_queue_url`**. Without the SQS pair, **`terraform apply`** + on **`webhook-validator`** fails with missing required variables and **no Lambda** is + created. **`rebuild_sandbox.sh`** runs this sync after edge-security; apply **`webhook-validator`** + **twice** (before and after post-EKS **`kms-secrets`**) per **`aws-deploy-plan.md`** §3.1. + +6. **Agent-worker IAM (`terraform/eks`)** — After **`terraform/dlq-s3` apply**, run + **`scripts/sync_sandbox_tfvars.py`** so **`terraform/eks/terraform.tfvars`** includes + **`raw_payload_bucket_kms_key_arns`** from **`.state/dlq-s3.outputs.json`** + (**`raw_payload_bucket_kms_key_arn`**). That CMK must receive **`kms:GenerateDataKey`** + (S3 **ViaService**) on the worker role or **`webhook.payload_sink`** fails. The same + module grants **`bedrock-agentcore:InvokeAgentRuntime`** on the **runtime-endpoint** + resource; re-**`apply_module eks`** (or IAM pass 3) after **`AGENTCORE_RUNTIME_ARN`** + / sync updates. + +**Canonical narrative:** **`aws-deploy-plan.md`** (Phases 2–4, rebuild orchestration, Phase 3.1 apply order). **Incident deep-dive:** remainder of this file. + +## Symptom summary (what showed up in `kubectl describe pod`) + +| Symptom | Underlying cause | +|--------|-------------------| +| `FailedMount` … `Failed to fetch secret … mcp/internal/bearer-token` | Generic ASCP message masking **Pod Identity token**, **KMS decrypt**, or **wrong secret JSON**. | +| `Pod Identity token extraction failed` … `pods.eks.amazonaws.com` … `tokenRequests` … `CSIDriver` | Secrets Store CSI **driver** Helm chart missing **second** `tokenRequests` audience. | +| `AccessDeniedException: Access to KMS is not allowed` | IAM role lacked **`kms:Decrypt`** on the CMK **or** KMS **key policy** did not allow the Pod Identity role **or** MCP bearer used a **different CMK** than `terraform/eks` `secrets_kms_key_arns`. | +| `Invalid JSON used with jmesPath` … `default/jira/integration-user` | Secret string was **not JSON** (e.g. placeholder `REPLACE_AFTER_FIRST_APPLY`) or JMESPath keys did not match JSON shape. | +| `AssumeRoleWithWebIdentity` / `AccessDenied` (older ASCP logs) | IRSA-only path before **`usePodIdentity: "true"`** on `SecretProviderClass`, or stale OIDC issuer on IAM trust. | + +## Architectural decisions (current “good” state) + +1. **Single customer-managed key (CMK)** for MCP bearer + default tenant secrets + - `terraform/secrets-mcp/terraform.tfvars` **`kms_key_arn`** must match **`jq -r '.kms_key_arns.value.default' .state/kms-secrets.outputs.json`**. + - `scripts/sync_sandbox_tfvars.py` writes that ARN (optional override: **`SECRETS_MCP_KMS_KEY_ARN`** in shell or `.env.local`). + - One key ⇒ one **`additional_decrypt_role_arns`** story in `terraform/kms-secrets/` and one **`mcp_secrets_kms_key_arns`** / **`secrets_kms_key_arns`** list for IAM/EKS. + +2. **EKS Pod Identity for ASCP** (preferred over node IAM or IRSA-only) + - `SecretProviderClass` **`usePodIdentity: "true"`** (templates under `terraform/eks-workloads/templates/*/secret-provider-class.yaml.tftpl`). + - **`eks-pod-identity-agent`** EKS add-on (`terraform/eks/main.tf`). + - **`aws_eks_pod_identity_association`** for `agent-worker` and `mcp-internal` service accounts (`terraform/eks/iam.tf`). + +3. **CSI Helm install order** + - Install **`csi-secrets-store`** (driver) **first**, then **`secrets-provider-aws`** (ASCP) with **`--set secrets-store-csi-driver.install=false`** to avoid Helm SA ownership clashes (`scripts/bootstrap_eks_workloads.sh`, `aws-deploy-plan.md`). + +4. **CSI driver `tokenRequests` (mandatory for Pod Identity mounts)** + - Driver Helm values must include **both**: + - `tokenRequests[0].audience=sts.amazonaws.com` + - `tokenRequests[1].audience=pods.eks.amazonaws.com` + - Verify: `kubectl get csidriver secrets-store.csi.k8s.io -o jsonpath='{.spec.tokenRequests}'`. + +## Repo changes that enforce the above (by area) + +### Helm / cluster bootstrap + +- **`scripts/bootstrap_eks_workloads.sh`** — Driver install sets **`tokenRequests[1].audience=pods.eks.amazonaws.com`**; provider still uses **`secrets-store-csi-driver.install=false`**; after Jira env resolution, runs **`scripts/sync_jira_integration_secret_from_env.sh`** when `JIRA_SITE_URL`, `JIRA_EMAIL`, and `JIRA_API_TOKEN` are set. + +### Terraform / tfvars sync + +- **`scripts/sync_sandbox_tfvars.py`** — Merges **`.env.local`** into the process environment (keys not already set); pins **`terraform/secrets-mcp/terraform.tfvars` → `kms_key_arn`** to the kms-secrets default CMK (or **`SECRETS_MCP_KMS_KEY_ARN`**); builds **`mcp_secrets_kms_key_arns`** (IAM) and **`secrets_kms_key_arns`** (EKS) from the same canonical CMK list; **`--phase post-eks`** fills **`additional_decrypt_role_arns`** on kms-secrets. + +- **`terraform/iam`** — **`mcp_secrets_kms_key_arns`** variable + **`mcp_token_reader_kms_decrypt`** inline policy when the list is non-empty (so **`GetSecretValue`** on CMK-encrypted Jira/git secrets succeeds). + +- **`terraform/secrets-mcp`** — **`kms_key_arn` output**; **`terraform.tfvars`** in repo uses the **default tenant CMK** (same as kms-secrets). + +- **`terraform/iam/terraform.tfvars`** — Removed obsolete **`mcp_bearer_token_secret_arn`** line (was undeclared and triggered Terraform warnings). + +### Kubernetes manifests (Terraform-rendered) + +- **`terraform/eks-workloads/templates/agent-worker/secret-provider-class.yaml.tftpl`** and **`mcp-internal/...`** — **`usePodIdentity: "true"`**, **`region`** set. +- **`mcp-internal`** — JMESPath uses **`site_url`**, **`email`**, **`api_token`** (valid JMESPath for JSON objects). + +### Sandbox orchestration + +- **`scripts/rebuild_sandbox.sh`** — After **`terraform/kms-secrets` apply**, runs **`scripts/sync_sandbox_tfvars.py --phase pre-eks`** **before** **`terraform/secrets-mcp` apply** so **`kms_key_arn`** is always current on greenfield rebuilds. Before **`bootstrap_eks_workloads.sh`**, runs **`scripts/sync_jira_integration_secret_from_env.sh`** (skips if Jira triple not set). + +### Operator env contract + +- **`.env.example`** — Optional **`SECRETS_MCP_KMS_KEY_ARN`**. +- **`.env.local.example`** — Documents **`SECRETS_MCP_KMS_KEY_ARN`** and Jira triple for CSI. +- **`terraform/secrets-mcp/README.md`** — Single-CMK path paragraph. + +### New script + +- **`scripts/sync_jira_integration_secret_from_env.sh`** — Writes **`default/jira/integration-user`** as JSON from **`JIRA_*`** (and `.env.local`); **exit 0** if vars incomplete (non-fatal for CI). + +### Documentation pointers + +- **`aws-deploy-plan.md`** — Links here from the top-level context list. + +## Recommended apply order (manual or `rebuild_sandbox.sh`) + +Same contract as **`aws-deploy-plan.md`** §3.1 **`APPLY` order**; **`scripts/rebuild_sandbox.sh`** runs the full sequence. The numbered steps below are the **tail after the first `kms-secrets` apply** (CSI/KMS/Jira-critical segment). **Greenfield:** start §3.1 from **`apply_module data-stores`** through **`iam` pass 1** and the first **`kms-secrets`** + HMAC sync before step 1 here. + +1. **`terraform/kms-secrets`** apply (default CMK + secrets + rotation wiring). +2. **`scripts/sync_webhook_hmac_secret.sh --region us-east-1`** (HMAC secret source of truth; same as deploy plan). +3. **`scripts/sync_sandbox_tfvars.py --phase pre-eks`** — run **after** `kms-secrets` has written **`.state/kms-secrets.outputs.json`** and **before** **`terraform/secrets-mcp` apply**, so `terraform/secrets-mcp/terraform.tfvars` **`kms_key_arn`** and IAM/EKS **`mcp_secrets_kms_key_arns` / `secrets_kms_key_arns`** match the default tenant CMK. +4. **`terraform/dlq-s3`** apply, then **`terraform/secrets-mcp`** apply. +5. **`terraform/iam`** pass 2, then **`edge-security`**, **`scripts/sync_sandbox_tfvars.py --phase pre-eks`** + (listener + **`work_queue_*`** for webhook), **observability**, **`agentcore-runtime`**, **`agentcore-memory`**, first **`webhook-validator`** apply. +6. **`ecr-images`**, **`eks`**, **`ensure_eks_cli_admin_access`**, **`iam`** pass 3. +7. **`scripts/sync_sandbox_tfvars.py --phase post-eks`**, **`kms-secrets`** apply again ( **`additional_decrypt_role_arns`**: worker + MCP + webhook role ), webhook HMAC sync, **second `webhook-validator` apply**. +8. **`scripts/bootstrap_agentcore_runtime.sh`**, **`scripts/sync_jira_integration_secret_from_env.sh`** (optional skip), **`scripts/bootstrap_eks_workloads.sh`**. + +An optional **`scripts/sync_sandbox_tfvars.py --phase pre-eks`** at the very start of Phase 3 is still fine for **network / edge** tfvars that already exist in `.state`; it does **not** replace the **post–`kms-secrets`** sync (step 3) or the **`sync_sandbox_tfvars.py` immediately after `edge-security`** (included in step 5 above). + +## Verification commands (after bootstrap) + +```bash +kubectl get csidriver secrets-store.csi.k8s.io -o jsonpath='{.spec.tokenRequests}{"\n"}' +kubectl -n aws-agent-core get pods -l 'app.kubernetes.io/name in (agent-worker,mcp-internal)' +kubectl -n kube-system get pods -l 'app.kubernetes.io/name=secrets-store-csi-driver-provider-aws' +aws eks list-pod-identity-associations --cluster-name "$(jq -r .cluster_name.value .state/eks.outputs.json)" --region us-east-1 +``` + +## What we do **not** need to repeat manually next time + +- **Hand-editing KMS key policy** on a one-off MCP CMK — eliminated by **single CMK** + Terraform-managed **`additional_decrypt_role_arns`**. +- **Helm reinstall order guesswork** — encoded in **`bootstrap_eks_workloads.sh`**. +- **Placeholder Jira string** blocking mounts — **`sync_jira_integration_secret_from_env.sh`** + bootstrap hook + rebuild stage. + +## If something still fails + +1. **ASCP logs**: `kubectl -n kube-system logs -l app.kubernetes.io/name=secrets-store-csi-driver-provider-aws --tail=100` +2. **Pod events**: `kubectl -n aws-agent-core describe pod -l app.kubernetes.io/name=agent-worker` +3. **IAM vs key policy**: confirm **`secrets_kms_key_arns`** / **`mcp_secrets_kms_key_arns`** include the CMK returned by **`aws secretsmanager describe-secret --secret-id mcp/internal/bearer-token`** (`KmsKeyId`). +4. **Re-sync and re-apply**: `scripts/sync_sandbox_tfvars.py --phase post-eks` → **`terraform apply`** on **kms-secrets**, **iam**, **eks** as needed. diff --git a/docs/oncall.md b/docs/oncall.md index 5e2b873..d52114f 100644 --- a/docs/oncall.md +++ b/docs/oncall.md @@ -104,12 +104,13 @@ require write access beyond the on-call IAM role. ```bash while python scripts/replay_dlq.py \ --from-sqs https://sqs.us-east-1.amazonaws.com/0/agent-dlq \ - --target https://agent.example.com/invocations; do :; done + --target-queue https://sqs.us-east-1.amazonaws.com/0/agent-work; do :; done ``` - The Wave-7 harness preserves the original Atlassian signature - so no re-signing is required. See - [`scripts/replay_dlq.py`](../scripts/replay_dlq.py) for - `--from-s3` and `--dry-run` modes. + The script republishes the captured envelope onto the work queue + the agent-worker drains; no HMAC re-signing is involved because + the trust boundary already cleared the message before it was + captured. See [`scripts/replay_dlq.py`](../scripts/replay_dlq.py) + for `--from-s3` and `--dry-run` modes. 3. **Force a circuit-breaker probe.** ```bash kubectl rollout restart -n aws-agent-core-prod deploy/agent-runtime @@ -140,8 +141,7 @@ escalation cadence. ## 6. Cross-references -- [`runbook.md`](runbook.md) — full per-scenario runbook (8 - scenarios as of Wave 8). +- [`runbook.md`](runbook.md) — full per-scenario runbook. - [`ARCHITECTURE.md`](ARCHITECTURE.md) — system architecture and the seven-step token-governance ordering invariant the guardrail / circuit-breaker mitigations rely on. diff --git a/docs/runbook-sandbox-deploy.md b/docs/runbook-sandbox-deploy.md new file mode 100644 index 0000000..d256abb --- /dev/null +++ b/docs/runbook-sandbox-deploy.md @@ -0,0 +1,565 @@ +# Sandbox AWS deploy runbook + +This chapter is the **single deterministic recipe** for running the +end-to-end AWS deploy against a sandbox account. It is the closure +evidence for several entries in [`DEFERRED.md`](DEFERRED.md) and the +acceptance gate for grading area 5 ("Cloud"). + +> **Not for production.** This recipe applies **ten** Terraform modules +> in apply order against the sandbox. `terraform/organizations/` is +> intentionally excluded: it is informational defense-in-depth (the SCP +> module), every other module ignores its outputs, and SCP CRUD calls +> are rejected from member accounts anyway. Production rollouts run the +> same ten modules with per-environment `*.tfvars` files, an explicit +> S3+DynamoDB backend per stage, and a manual approval at each +> `terraform apply`; the management account separately owns +> `organizations/` if the SCP boundary is ever turned on. + +## 0. Pre-flight + +The operator runs everything from the repo root, with +`terraform >= 1.13` (every module's `versions.tf` requires it) and +the `aws` CLI v2 configured against the sandbox account. + +```bash +# Sandbox credentials (use SSO or a role-chain; never a long-lived +# IAM user). +export AWS_PROFILE=agent-core-sandbox +export AWS_REGION=us-east-1 + +# Confirm we are on the sandbox account before apply. +aws sts get-caller-identity --query Account --output text + +# Confirm tooling. +aws --version # >= 2.x +terraform -version # >= 1.13 +aws bedrock-agentcore-control help >/dev/null && echo ok # control plane CLI +aws bedrock-agentcore help >/dev/null && echo ok # data plane CLI +pip install bedrock-agentcore-starter-toolkit # supplies `agentcore` CLI +agentcore --version +``` + +Confirm the sandbox account id matches what is documented in your +team's secrets vault. **If the account id is wrong, abort.** + +### 0.0 (Optional) Bootstrap the remote Terraform backend + +Modules in this repo use Terraform's **partial-backend-config** pattern, +so each operator can choose: local state, S3 with native S3 locking, +Terraform Cloud, etc. The 3-line `terraform//backend.tf` stubs +are committed; the operator-specific values (bucket, region) live in a +single gitignored `terraform/backend.hcl`. See +[`terraform/README.md` § State management](../terraform/README.md#state-management-operator-action-required) +for the full layout, gitignore rules, and rationale. + +Recommended path: + +```bash +./scripts/bootstrap_tfstate.sh \ + terraform-state--- \ + --write-shared-backend +``` + +Run this **once per (account, region) pair** before §1. The script +creates only the S3 bucket and writes a single shared +`terraform/backend.hcl`. Each module's `agent-core/.tfstate` +object is written lazily on the module's first `apply`, and locking is +S3-native (`use_lockfile = true`) so no DynamoDB table is needed. + +From here on, every Terraform invocation in §1 goes through the +[`scripts/tf.sh`](../scripts/tf.sh) wrapper, which injects +`-backend-config=../backend.hcl -backend-config="key=agent-core/.tfstate"` +on `init` and passes everything else straight through. Skip this section +entirely if you want to stay on local state for the sandbox run; the +apply loop in §1 has fall-back guidance for that case. + +### 0.1 Bedrock model access + +The sandbox invoke at the end of this runbook calls a Bedrock foundation +model. The agent runtime supports two model families today (selected by +`BEDROCK_MODEL_ID`; see [§0.1.1](#011-switching-the-bedrock-model-family)): + +* **DeepSeek V3.2 (default)** — bare foundation-model id `deepseek.v3.2`. + In-region only — no `us.` cross-region inference profile exists. + **No use-case-details form is required.** Verified today via + `bedrock-runtime converse` returning 200 immediately after model + access is granted. +* **Anthropic Claude Sonnet 4.5 (alternative)** — cross-region + inference profile `us.anthropic.claude-sonnet-4-5-20250929-v1:0`. + Requires both a Console "Manage model access" tick **and** a separate + Anthropic use-case-details form on the Claude Sonnet 4.5 model card; + without the form, `invoke-model` fails with `ResourceNotFoundException: + Model use case details have not been submitted for this account.` + +Gates per family: + +| Gate | DeepSeek V3.2 (default) | Anthropic Claude Sonnet 4.5 | +| --- | --- | --- | +| Model access (Console → Model access → Manage) | required | required | +| Third-party model use-case-details form | **not required** | required | + +Verify the DeepSeek default (the runbook flow) with a real invocation: + +```bash +aws bedrock get-foundation-model-availability --region us-east-1 \ + --model-id deepseek.v3.2 \ + --query '{access:authorizationStatus,entitle:entitlementAvailability}' \ + --output table +# Expect: AUTHORIZED AVAILABLE + +aws bedrock-runtime converse --region us-east-1 \ + --model-id deepseek.v3.2 \ + --messages '[{"role":"user","content":[{"text":"Reply with the single word OK."}]}]' \ + --inference-config maxTokens=8 \ + --query 'output.message.content[0].text' \ + --output text +# Expect: OK +``` + +If you have flipped the deployment to Anthropic Claude, swap the probe +to the Anthropic shape and confirm the use-case form has been submitted: + +```bash +echo '{"anthropic_version":"bedrock-2023-05-31","max_tokens":16,"messages":[{"role":"user","content":"OK"}]}' > /tmp/probe.json +aws bedrock-runtime invoke-model \ + --region us-east-1 \ + --model-id us.anthropic.claude-sonnet-4-5-20250929-v1:0 \ + --content-type application/json --accept application/json \ + --body fileb:///tmp/probe.json /tmp/probe.out +cat /tmp/probe.out # must be a normal Anthropic response, not the use-case error +``` + +#### 0.1.1 Switching the Bedrock model family + +The agent runtime treats `BEDROCK_MODEL_ID` as the single switch between +supported families (mirrored by +`agent.main.SUPPORTED_BEDROCK_MODEL_FAMILIES`, the Terraform variable +validation in `terraform/agentcore-runtime/variables.tf`, the IAM ARN +derivation in `terraform/agentcore-runtime/iam.tf`, and the SCP allow-list +default in `terraform/organizations/variables.tf`). + +The default for this runbook is **DeepSeek V3.2** (`deepseek.v3.2`). To +flip the deployment to **Anthropic Claude Sonnet 4.5**: + +1. **Submit the Anthropic use-case-details form.** Bedrock Console + (`us-east-1`) → Foundation models → Anthropic → Claude Sonnet 4.5 + model card → fill in the use-case form (intended use, traffic + estimates, content policy acknowledgement). Without this submission + the runtime call fails with + `ResourceNotFoundException: Model use case details have not been + submitted for this account.` even with model access granted. Verify + with the Anthropic invoke probe in §0.1 above. +2. **(SCP-attached deployments only)** confirm + `us.anthropic.claude-sonnet-4-5-20250929-v1:0` is present in + `allowed_bedrock_model_ids` in the Organizations module and + `terraform apply` it; the default in + `terraform/organizations/variables.tf` already includes it but + per-environment `*.tfvars` may override. +3. Set `BEDROCK_MODEL_ID = "us.anthropic.claude-sonnet-4-5-20250929-v1:0"` + in the runtime env (`.bedrock_agentcore.yaml` or per-environment + overrides) and `terraform apply terraform/agentcore-runtime/`. The + IAM execution role's `bedrock_invoke` policy automatically expands + from the single foundation-model ARN to **both** the foundation-model + ARN **and** the cross-region inference-profile ARN, because the + `_BEDROCK_GEO_PREFIXES` classifier in `agent.main` recognises the + leading `us.` segment. +4. **(Bedrock Guardrails)** Bedrock Guardrails are model-agnostic via + the Converse `guardrailConfig` parameter, so the existing guardrail + identifier/version pair continues to apply across both families; + re-validate that the policy still flags the same content classes + under Anthropic by replaying a small content-block sample through + `aws bedrock-runtime apply-guardrail` before promoting. +5. Smoke-test by driving one Jira webhook through the agent and + confirming `TicketAssessment`, `ImplementationPlan`, and + `ApprovalEvaluation` parse cleanly under Anthropic — these three + structured-output gates are the real proof the tool-use surface is + healthy end-to-end. + +Roll back by setting `BEDROCK_MODEL_ID` back to `deepseek.v3.2` and +re-applying `terraform/agentcore-runtime/`. No code rollback is needed. + +### 0.2 Pre-create DynamoDB tables and SQS queues (out-of-band) + +`terraform/iam/` and `terraform/agentcore-runtime/` both demand real +ARNs for these resources, but no module in this repo provisions them. +Create them with the AWS CLI before the apply loop and capture their +ARNs for the per-module `terraform.tfvars`: + +```bash +aws dynamodb create-table --region us-east-1 \ + --table-name agent-domain-state \ + --attribute-definitions AttributeName=pk,AttributeType=S AttributeName=sk,AttributeType=S \ + --key-schema AttributeName=pk,KeyType=HASH AttributeName=sk,KeyType=RANGE \ + --billing-mode PAY_PER_REQUEST + +aws dynamodb create-table --region us-east-1 \ + --table-name agent-tenants \ + --attribute-definitions AttributeName=tenant_id,AttributeType=S \ + --key-schema AttributeName=tenant_id,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST + +aws sqs create-queue --region us-east-1 --queue-name webhook-dlq \ + --attributes MessageRetentionPeriod=1209600 +aws sqs create-queue --region us-east-1 --queue-name agent-invoke-queue +``` + +Snapshot `aws dynamodb describe-table` and +`aws sqs get-queue-attributes --attribute-names QueueArn` outputs +under `.state/bootstrap.txt` for evidence. + +## 1. Terraform apply (ten modules, two-pass `iam/`) + +Each module owns one capability and is applied independently. The +runbook's previous single-pass loop was internally inconsistent: +`terraform/iam/` declares `secrets_arns` (output of `kms-secrets/` + +`secrets-mcp/`) as a required input, while `terraform/kms-secrets/` +declares `agent_runtime_role_arn` (output of `iam/`) as a required +input. Resolve with a two-pass `iam/` apply: pass 1 uses a wildcard +secrets-arn list so the role can be created, then pass 2 tightens the +list once the secrets exist. + +Helper: + +```bash +mkdir -p .state +apply_module() { + local mod="$1" + scripts/tf.sh "$mod" init + scripts/tf.sh "$mod" plan -out=tfplan + scripts/tf.sh "$mod" apply tfplan + scripts/tf.sh "$mod" output -json > ".state/${mod}.outputs.json" +} +``` + +> The wrapper drives `terraform -chdir=terraform/ ...` and +> auto-injects the partial-backend flags on `init`. If you skipped §0.0 +> and want local state, replace the body with +> `terraform -chdir="terraform/$mod" init` (no `-backend-config`) and +> the same `plan` / `apply` / `output` calls; the rest of §1 still +> works. + +Order: + +**Source of truth for module order, every `sync_sandbox_tfvars.py` hook, two +`webhook-validator` applies, and EKS bootstrap:** **`aws-deploy-plan.md`** +§3.1 and **`scripts/rebuild_sandbox.sh`**. The numbered steps below are a +compact index—if they diverge, follow the deploy plan / rebuild script. + +1. `apply_module data-stores` — DDB + SQS ( **`agent_invoke_queue_*`** feed **`webhook-validator`** via sync ) +2. `apply_module network` +3. `apply_module bedrock-guardrail` +4. `scripts/sync_sandbox_tfvars.py --phase pre-eks` (repeat after **`kms-secrets`** and after **`edge-security`** per deploy plan §3.1) +5. `apply_module iam` — **pass 1** with + `secrets_arns = ["arn:aws:secretsmanager:us-east-1::secret:*"]`, + real `dynamodb_table_arns` / `dead_letter_queue_arn` / + `webhook_invoke_queue_arn` from §0.2, + `raw_payload_bucket_arn = "arn:aws:s3:::"` + (string-construct ahead of `dlq-s3/`), + `mcp_secret_arns = ["arn:aws:secretsmanager:us-east-1::secret:mcp/internal/bearer-token-*", "arn:aws:secretsmanager:us-east-1::secret:default/jira/integration-user-*", "arn:aws:secretsmanager:us-east-1::secret:default/git/ssh-private-key-*"]`, + `guardrail_arn` from `.state/bedrock-guardrail.outputs.json`, and + `bedrock_model_arns = ["arn:aws:bedrock:us-east-1::foundation-model/deepseek.v3.2"]` + (DeepSeek V3.2 default — empty account-id segment is intentional; + foundation-model ARNs are cross-account references). If the + deployment was flipped to Anthropic per §0.1.1, swap to + `["arn:aws:bedrock:us-east-1::inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0"]` + instead. +6. `apply_module kms-secrets` — pass `agent_runtime_role_arn` from + pass 1. Extend `var.secrets` to add an `atlassian/webhook-hmac` + entry; the `webhook-validator` Lambda + ([`terraform/webhook-validator/`](../terraform/webhook-validator/)) + reads it directly via its own IAM role. The default `var.secrets` map only ships + `mcp/internal/bearer-token`, `jira/integration-user`, + `anthropic/api-key`, `bedrock/runtime-config`. +7. `scripts/sync_webhook_hmac_secret.sh` then **`scripts/sync_sandbox_tfvars.py --phase pre-eks`**, then `apply_module dlq-s3` — `kms_key_arn` resolves through + `jq -r '.kms_key_arns.value.default' .state/kms-secrets.outputs.json` + (the default tenant id is `"default"`). +8. `apply_module secrets-mcp` — `kms_key_arn` (same default-tenant + value) and `agent_runtime_role_arn` from pass 1. +9. `scripts/sync_sandbox_tfvars.py --phase pre-eks` then `apply_module iam` — **pass 2**: tighten `secrets_arns` to the real + ARNs from `.state/secrets-mcp.outputs.json` (`secret_arn`), + `.state/kms-secrets.outputs.json` (`secret_arns` map keyed by + `"/"`, e.g. + `secret_arns["default/jira/integration-user"]`), and + `.state/bedrock-guardrail.outputs.json` (`secret_arn`). +10. `apply_module edge-security` — needs `vpc_id`, `public_subnet_ids`, + `alb_sg_id`, `domain_name`, `hosted_zone_id`. Set + `use_api_gateway = false` and `use_api_gateway_v2 = false` + (`jwt_issuer_url` / `jwt_audience` are then unused — the deploy plan's + "no JWT, no Cognito" stance leaves HMAC enforcement to the + `webhook-validator` Lambda at the ALB). +11. `scripts/sync_sandbox_tfvars.py --phase pre-eks` — writes **`https_listener_arn`** + **`work_queue_*`** into **`terraform/webhook-validator/terraform.tfvars`** +12. `apply_module observability-xray` (set `cluster_type = "ecs"` for + the sandbox; the EKS daemonset path is gated by the AgentCore + promotion step in [`aws-deploy-plan.md`](../aws-deploy-plan.md)) and + `apply_module observability-cloudwatch-genai`. +13. `apply_module agentcore-runtime` then `apply_module agentcore-memory` +14. `apply_module webhook-validator` — **first** apply: provisions the HMAC Lambda + behind the existing `edge-security` ALB. After signature + verification the Lambda publishes the validated envelope to the + `agent-work` SQS queue and returns 202 Accepted; the agent-worker + Pod consumes from that queue and invokes AgentCore Runtime (or + runs LangGraph in-process when `AGENT_PROFILE=local`). See §12. +15. `apply_module ecr-images` then `apply_module eks`. +16. `scripts/ensure_eks_cli_admin_access.sh --region us-east-1` (or your region). +17. `apply_module iam` — **pass 3** post-EKS when your plan refreshes OIDC trust. +18. `scripts/sync_sandbox_tfvars.py --phase post-eks`, then **`apply_module kms-secrets`** + again ( **`additional_decrypt_role_arns`** must include agent-worker, MCP token reader, + and webhook Lambda roles), then **`scripts/sync_webhook_hmac_secret.sh`**, then + **`apply_module webhook-validator`** — **second** apply so the Lambda execution role + matches the CMK policy after KMS widens. +19. **`scripts/bootstrap_eks_workloads.sh`** + (CSI driver + ASCP + KEDA + rendered manifests + `kubectl apply` so `mcp-internal` can obtain an **internal NLB** hostname when enabled). +20. **`scripts/bootstrap_agentcore_runtime.sh`** (after step 19 when VPC networking is on: the script waits for that LoadBalancer hostname and passes `agentcore configure --vpc …`). + Optional: **`scripts/sync_jira_integration_secret_from_env.sh`** (writes Jira JSON for CSI when env is complete). + Before step 20, run **`python scripts/sync_sandbox_tfvars.py --phase post-eks`** then **`scripts/tf.sh agentcore-runtime apply`** so `vpc_id` is populated and `runtime_vpc_security_group_id` exists in `.state/agentcore-runtime.outputs.json`. + +The `.state/` directory is git-ignored. Each module's outputs feed +the next: `kms-secrets`'s `secret_arns` map and `secrets-mcp`'s +`secret_arn` are the inputs `bedrock-guardrail` and the AgentCore +runtime later reference. + +### Module-by-module evidence to capture + +| # | Module | Evidence to grep / verify | +| - | --- | --- | +| 0 | `data-stores/` | `agent_invoke_queue_arn` / `agent_invoke_queue_url` in `.state/data-stores.outputs.json` ( **`webhook-validator`** **`work_queue_*`** via sync ); DynamoDB table ARNs for **`iam`** pass 1 | +| 1 | `network/` | VPC plus seven VPC endpoints (`bedrock_endpoint_id`, `dynamodb_endpoint_id`, `s3_endpoint_id`, `secretsmanager_endpoint_id`, `xray_endpoint_id`, `monitoring_endpoint_id`, plus the DynamoDB/S3 prefix-list pair); `terraform output bedrock_endpoint_id` non-empty | +| 2 | `bedrock-guardrail/` | `terraform output guardrail_arn` non-empty; `aws bedrock get-guardrail` succeeds; `secret_arn` resolves to a Secrets Manager entry holding both `guardrailIdentifier` and `guardrailVersion` | +| 3 | `iam/` (pass 1) | `agent_runtime_role_arn`, `webhook_ingress_role_arn`, `mcp_token_reader_role_arn` are present; `secrets_arns` policy still scoped via wildcard | +| 4 | `kms-secrets/` | `kms_key_arns.default` resolves; `secret_arns` map contains entries for `default/mcp/internal/bearer-token`, `default/jira/integration-user`, `default/anthropic/api-key`, `default/bedrock/runtime-config`, `default/atlassian/webhook-hmac`; CloudTrail records `kms:CreateKey` + `secretsmanager:CreateSecret` events | +| 5 | `dlq-s3/` | Bucket has `ObjectLockConfiguration` + KMS-CMK SSE; `aws s3api get-bucket-encryption` returns `aws:kms` with the CMK from step 4 | +| 6 | `secrets-mcp/` | The `mcp/internal/bearer-token` secret is created; `aws secretsmanager describe-secret` shows `RotationEnabled=true` | +| 7 | `iam/` (pass 2) | The role's inline policy now references the real Secrets Manager ARNs from steps 4 + 6, not the wildcard from pass 1 | +| 8 | `edge-security/` | ALB DNS name resolves; HTTPS listener responds with the Route 53-aliased hostname; WAFv2 web ACL is associated. *(API Gateway v2 + JWT authorizer have been retired — the public ingress is ALB+WAF only and HMAC is enforced at the application layer.)* | +| 9 | `observability-xray/` | `aws xray get-sampling-rules` returns the ruleset Terraform applied | +| 10 | `observability-cloudwatch-genai/` | The GenAI CloudWatch dashboard exists; alarms on `gen_ai.client.token.usage` + `webhook.invariant_violation` are in `OK` state | +| 11 | `agentcore-runtime/` | `.state/agentcore-runtime.outputs.json` + **`scripts/bootstrap_agentcore_runtime.sh`** prerequisites; runtime ARN present for downstream bootstrap | +| 12 | `agentcore-memory/` | Memory store id in outputs / env consumed by EKS worker bootstrap | +| 13 | `webhook-validator/` (×2 per §3.1) | After **both** applies: `terraform output lambda_function_name` resolves; `aws lambda get-function --function-name ` returns `State=Active`; the listener rule on `https_listener_arn` at priority 100 forwards `/invocations` to the Lambda's `target_type=lambda` target group | + +Also verify **`ecr-images`**, **`eks`**, **`iam`** pass 3, **second `kms-secrets`**, **`bootstrap_eks_workloads.sh`**, and **`bootstrap_agentcore_runtime.sh`** per §1 steps 15–20 (evidence: EKS API reachable, nodegroup healthy, CSI pods running, `mcp-internal` LB hostname present when VPC mode is on). + +If any step fails, capture the failing log under +`.state/.apply.log` and either retry the same module +(idempotent) or roll back with `terraform destroy` for that module +**only**. The DLQ bucket has Object Lock; deleting it requires +manually clearing the retention before destroy. + +## 11.5 Re-deploying AgentCore Runtime (avoiding ARN drift) + +When `scripts/bootstrap_agentcore_runtime.sh` rebuilds the runtime, the +toolkit allocates a new ARN like +`arn:aws:bedrock-agentcore:us-east-1::runtime/jira_readiness_agent-` +and writes it to `.state/agentcore-runtime.env::AGENTCORE_RUNTIME_ARN`. +The `agent-worker` Pod's env is captured at pod start time from the +`agent-worker-config` ConfigMap, so a new ARN is invisible to a worker +that was already running. Without the steps below, the worker keeps +invoking the previous (now-replaced) runtime and surfaces: + +``` +worker.agentcore.invoke_failed + error_type=RuntimeClientError + error="An error occurred when starting the runtime" +``` + +Three independent layers prevent this drift on the next deploy: + +1. **Kubernetes-native checksum annotation.** The `agent-worker` + Deployment template + ([`terraform/eks-workloads/templates/agent-worker/deployment.yaml.tftpl`](../terraform/eks-workloads/templates/agent-worker/deployment.yaml.tftpl)) + carries a `checksum/agent-worker-config` annotation rendered with the + SHA-256 of the ConfigMap body + ([`terraform/eks-workloads/manifests.tf::locals.agent_worker_configmap_checksum`](../terraform/eks-workloads/manifests.tf)). + Any change to the ConfigMap body changes the annotation, which + Kubernetes treats as a Pod-template change and the Deployment rolls + automatically. +2. **Explicit `kubectl rollout restart` belt-and-braces.** + [`scripts/bootstrap_eks_workloads.sh`](../scripts/bootstrap_eks_workloads.sh) + step 6 runs `kubectl rollout restart deploy/agent-worker` after + `kubectl apply -k`. Idempotent and cheap; converges on a fresh Pod + even in degenerate cases (e.g. annotation render skipped, manual + ConfigMap edit out-of-band). +3. **Drift validation gate.** + [`scripts/validate_rebuild_state.sh`](../scripts/validate_rebuild_state.sh) + §3a compares the live `kubectl get cm agent-worker-config -o + jsonpath='{.data.AGENTCORE_RUNTIME_ARN}'` to + `.state/agentcore-runtime.env::AGENTCORE_RUNTIME_ARN` and `[FAIL]`s + on mismatch. + +### Canonical sequence after rebuilding the runtime + +```bash +scripts/bootstrap_agentcore_runtime.sh +# (writes .state/agentcore-runtime.env; emits a "DRIFT DETECTED" banner +# if the live worker ConfigMap still pins the previous ARN) + +scripts/bootstrap_eks_workloads.sh +# (re-renders the ConfigMap with the new ARN, the SHA-256 annotation +# flips, the worker is rolled, and the `rollout status` waits for it +# to become Available) + +scripts/validate_rebuild_state.sh +# (§3a fails the run if any layer above silently skipped) +``` + +If you see the `DRIFT DETECTED` banner from +`bootstrap_agentcore_runtime.sh`, you have not yet completed step 2. +Re-run `bootstrap_eks_workloads.sh` and re-run the smoke. + +## 12. webhook-validator Lambda smoke + +After the full §3.1 Terraform sequence (including **two** `webhook-validator` +applies and post-EKS **`kms-secrets`**) and EKS bootstrap have completed, the agent +ingress is the `webhook-validator` Lambda fronted by the +`edge-security` ALB at +`https://agent-sandbox.qodolabs.click/invocations`. HMAC verification +runs in the Lambda; on success it publishes the validated envelope to +the `agent-work` SQS queue and returns 202. The `agent-worker` Pod +consumes that queue and either invokes AgentCore Runtime +(`AGENT_PROFILE=aws`) or runs LangGraph in-process +(`AGENT_PROFILE=local`). + +Smoke against the live ALB: + +```bash +HMAC_SECRET=$(aws secretsmanager get-secret-value \ + --secret-id default/atlassian/webhook-hmac --region us-east-1 \ + --query SecretString --output text | tr -d '\r\n') + +WEBHOOK_HMAC_SECRET="$HMAC_SECRET" \ + python scripts/smoke.py \ + --fixture tests/fixtures/jira/issue_created.json \ + --target https://agent-sandbox.qodolabs.click/invocations +``` + +Capture the Lambda response (HTTP 202 + `{"status":"accepted",...}`) +under `.state/webhook-validator.invoke.json` and capture the matching +AgentCore runtime log snippet / trace id for evidence. + +> **Model-access precondition for invoke.** +> Under the DeepSeek V3.2 default the `agentcore invoke` step works as +> soon as model access is granted in §0.1 — DeepSeek does not require a +> use-case-details form. If the deployment was flipped to Anthropic per +> §0.1.1 and the use-case form was skipped, the invoke step (and any +> `bedrock-runtime invoke-model` underneath) returns +> `ResourceNotFoundException: Model use case details have not been +> submitted for this account` and no GenAI tokens are recorded. + +### Evidence checklist for the invoke + +- The structured-log stream contains + `event = "webhook.processed" decision = "ready_for_assessment"`. +- An X-Ray trace is visible in CloudWatch ServiceLens for the request + with at least the `webhook.handler`, `assessor.invoke`, and + `mcp.tool` spans (depending on the fixture branch). +- `gen_ai.client.token.usage` shows a non-zero sample for the + sandbox tenant id. + +## 13. Closing related deferrals + +Two entries in [`DEFERRED.md`](DEFERRED.md) are explicitly gated by +the sandbox apply: + +### `deferred-kms-rotation-cycle` + +Exit criteria: rotation Lambda cycles the webhook HMAC and +Bedrock-runtime-config secrets at least once. When `kms-secrets` is +applied with the default `tenant_ids = ["default"]`, the Secrets +Manager entries are created under per-tenant prefixes — use +`default/` here, not the bare names that appear in older +revisions of this runbook. + +```bash +aws secretsmanager rotate-secret \ + --secret-id default/atlassian/webhook-hmac +aws secretsmanager rotate-secret \ + --secret-id default/bedrock/runtime-config + +aws secretsmanager describe-secret \ + --secret-id default/atlassian/webhook-hmac \ + --query 'LastRotatedDate' +``` + +Both `LastRotatedDate` values must be within the last hour. Capture +the JSON response under `.state/kms-rotation.json`, attach it to the +PR closing the deferral, and remove the entry from +[`README.md`](../README.md) and [`DEFERRED.md`](DEFERRED.md) per the +"Closing a deferred item" procedure. + +### `deferred-edge-security` + +Exit criteria: `terraform/edge-security/` applied with +`use_api_gateway = false` and `use_api_gateway_v2 = false` (no JWT +authorizer — see `aws-deploy-plan.md` §Approach → Security model). The +public surface is ALB + WAFv2 + ACM cert + Route53 alias; HMAC is +verified by the `webhook-validator` Lambda +([`terraform/webhook-validator/`](../terraform/webhook-validator/)) +which attaches a `target_type=lambda` target group to the ALB at +priority 100 path-pattern `/invocations`. + +```bash +# 1. Confirm ALB DNS resolves the Route 53 alias. +dig +short "$(scripts/tf.sh edge-security output -raw domain_name)" + +# 2. The Lambda's GET / health-check shortcut returns 200 (no +# AgentCore invoke, no Secrets Manager read). +curl -ksS -o /dev/null -w '%{http_code}\n' \ + "https://$(scripts/tf.sh edge-security output -raw domain_name)/invocations" + +# 3. POST a signed canonical fixture; expect 202 + accepted. +HMAC_SECRET=$(aws secretsmanager get-secret-value \ + --secret-id default/atlassian/webhook-hmac --region us-east-1 \ + --query SecretString --output text | tr -d '\r\n') +WEBHOOK_HMAC_SECRET="$HMAC_SECRET" \ + python scripts/smoke.py \ + --fixture tests/fixtures/jira/issue_created.json \ + --target "https://$(scripts/tf.sh edge-security output -raw domain_name)/invocations" +``` + +Capture the three responses under `.state/edge-security.smoke.log`, +attach to the PR closing the deferral, and remove the entry from +[`README.md`](../README.md) and [`DEFERRED.md`](DEFERRED.md). + +## 14. Teardown + +The sandbox account is intended to be torn down after each apply +exercise to keep the bill bounded: + +```bash +for mod in \ + eks-workloads webhook-validator agentcore-runtime \ + observability-cloudwatch-genai observability-xray \ + edge-security iam secrets-mcp dlq-s3 kms-secrets \ + bedrock-guardrail network; do + scripts/tf.sh "$mod" destroy -auto-approve || true +done + +# Then drop the out-of-band §0.2 resources: +aws sqs delete-queue --queue-url "$(aws sqs get-queue-url --queue-name agent-invoke-queue --query QueueUrl --output text)" +aws sqs delete-queue --queue-url "$(aws sqs get-queue-url --queue-name webhook-dlq --query QueueUrl --output text)" +aws dynamodb delete-table --table-name agent-tenants +aws dynamodb delete-table --table-name agent-domain-state +``` + +(The destroy order is the reverse of apply. `terraform/organizations/` +is intentionally absent — it was never applied in §1.) The `dlq-s3` +Object Lock retention may require manual `s3api put-object-legal-hold` +/ `delete-objects --bypass-governance-retention` before destroy +succeeds; this is intentional — it is the same compliance path that +production destroy will follow. + +## 15. Cross-references + +- [`aws-deploy-plan.md`](../aws-deploy-plan.md) — account-agnostic + worked example of this runbook (resolves the AWS account at deploy + time via `aws sts get-caller-identity`); use it when running this + recipe on whichever sandbox your `aws` CLI is currently + authenticated to. +- [`terraform/README.md`](../terraform/README.md) — module catalogue. +- [`docs/DEFERRED.md`](DEFERRED.md) — closure register for the two + deferrals above. +- [`docs/runbook.md`](runbook.md) — incident-response runbook (this + sandbox runbook is its sibling for green-path provisioning). +- [`.github/workflows/sandbox-apply.yml`](../.github/workflows/sandbox-apply.yml) + — CI workflow that automates this recipe against a sandbox account + on a schedule (when the sandbox credentials secret is set). diff --git a/docs/runbook.md b/docs/runbook.md index 838ae1a..f7f928a 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -60,7 +60,7 @@ Mitigate: Recover: -- Drain the in-flight DLQ entries with `scripts/redrive_dlq.sh` +- Drain the in-flight DLQ entries with `python scripts/replay_dlq.py` *after* the guardrail policy has been corrected; replays sent before the fix will block again and re-DLQ. @@ -102,8 +102,9 @@ Mitigate: Recover: -- After mitigation, redrive the DLQ in batches of ≤ 100 messages with - `scripts/redrive_dlq.sh --batch 100`. +- After mitigation, redrive the DLQ one message at a time with + `python scripts/replay_dlq.py --from-sqs --target `, + iterating until the queue drains. - Watch the `webhook.processed` rate climb back to the baseline before declaring the incident over. @@ -111,8 +112,15 @@ Root-cause: - `mcp/` for the bundled `mcp-internal` JAR. - `terraform/secrets-mcp/` for the bearer-token rotation. -- `src/agent/infrastructure/mcp/streamable_http_client.py` for retry / - timeout configuration. +- `src/agent/composition/_mcp_session.py` for the per-request + Streamable-HTTP session lifecycle (transport timeout, + bearer-token provider). +- `src/agent/infrastructure/mcp/session_adapter.py` for the + `McpClient` adapter over the SDK ``ClientSession`` (return-value + coercion, error-mapping). +- `src/agent/infrastructure/mcp/observability_interceptor.py` for the + ``ToolCallInterceptor`` that records the load-bearing + `mcp.tool.calls` / `mcp.call.duration_ms` observer entries. --- @@ -148,8 +156,8 @@ sorted output. If saturation pushed any envelopes onto the SQS DLQ (because the agent returned an error response from a downstream that the -budget-saturation race created), use the Wave 7 replay harness to -inspect, then re-process them once the cap is bumped: +budget-saturation race created), use the replay harness to inspect, +then re-process them once the cap is bumped: ```bash # Inspect what the next replay would send (no POST, no SQS delete): @@ -239,11 +247,12 @@ the SQS DLQ during the open-circuit window so no work is silently dropped: ```bash -# Stream every DLQ message through the Wave 7 replay harness -# (one invocation per message; the script deletes on HTTP 200). +# Stream every DLQ message through the replay harness — each +# message is republished onto the agent-work SQS queue and deleted +# from the DLQ on success. while python scripts/replay_dlq.py \ --from-sqs https://sqs.us-east-1.amazonaws.com/0/agent-dlq \ - --target https://agent.example.com/invocations; do :; done + --work-queue-url https://sqs.us-east-1.amazonaws.com/0/agent-invoke-queue; do :; done ``` The harness short-circuits with a non-zero exit when the queue is @@ -311,9 +320,9 @@ Detect: - A user reports the leak (most common); OR - The Bedrock Guardrail PII filter records a *post*-block event (`event=llm.guardrail.block, action=PII_DETECTED`). The PII filter - is the **second** layer of defence; the *first* layer is - `src/agent/application/comment_renderer.py` which already strips - bracketed tokens that match the `[SECRET:*]` redaction pattern. + is the primary model-output defence; Jira-visible text is authored by + the LLM and posted through `jira_add_comment`, so prompt/tool-call + audit logs are the source of truth for what was published. Mitigate: @@ -329,14 +338,14 @@ Recover: - Audit the last hour of `graph.design_post_comment` events for the same ticket key; correlated leaks usually have a shared upstream - cause (broken regex in `comment_renderer`, prompt injection from a - comment body, etc.). + cause (prompt injection from a comment body, overly broad tool + context, etc.). - File a security ticket linking the structured-log records and any S3 raw-payload entries. Root-cause: -- `src/agent/application/comment_renderer.py` -- redaction logic. +- `src/agent/graph/prompts.py` -- output constraints and tool-use instructions. - `terraform/bedrock-guardrail/` -- PII guardrail policy. - `src/agent/infrastructure/storage/s3_payload_sink.py` -- raw-payload retention configuration. @@ -370,20 +379,46 @@ Detect: Any difference is drift. +- Confirm whether the CLI fallback sidecar files still point at the same + resources as AWS: + + ```bash + terraform -chdir=terraform/agentcore-runtime output -raw runtime_arn + aws bedrock-agentcore get-runtime \ + --region "$AWS_REGION" \ + --name jira-readiness-agent \ + --query 'runtime.arn' \ + --output text + ``` + + If `.state/runtime.arn` or `.state/memory.arn` is missing, the Terraform + output may fail or rehydrate on the next apply. Treat that as a state-recovery + event, not as a routine plan diff. + Mitigate: -- Until the AgentCore native-provider migration lands, **do not** - attempt to re-apply the Terraform module against the drifted - runtime; the module currently uses `null_resource` blocks that - cannot reconcile the divergence. +- Until the AgentCore native-provider migration lands, **do not** blindly + re-apply the Terraform module against a drifted runtime. The module + currently uses `null_resource` blocks plus `.state/*.arn` sidecar files, + so first verify the AWS account, region, runtime name, and memory name. - Instead, freeze the runtime by rolling back to the last known-good AgentCore runtime version (the AgentCore console exposes a per-runtime version timeline). Recover: -- File the drift in the AgentCore native-provider migration tracker - so the next promotion deliverable picks up the reconciliation work. +- If `.state/` was deleted by `git clean -fdx` or workspace cleanup, recreate + it by running `terraform apply` with the intended sandbox/production + credentials. The fallback provisioners describe the live runtime/memory first + and rewrite `.state/runtime.arn` and `.state/memory.arn` when the named + resources already exist. +- If the live runtime or memory store should be destroyed, delete it + explicitly with the AWS CLI or AgentCore console before removing the sidecar + files. `terraform destroy` removes Terraform-managed IAM and helper + resources, but the CLI-created AgentCore resources are not represented as + native Terraform resources yet. +- File any drift in the AgentCore native-provider migration tracker so the + next promotion deliverable picks up the reconciliation work. - Once the native provider lands, flip `use_native_provider = true` and re-apply. @@ -392,10 +427,247 @@ Root-cause: - `terraform/agentcore-runtime/main.tf` -- the `null_resource` / native-provider toggle and the inline AgentCore native-provider migration tracker. +- `terraform/agentcore-runtime/README.md` -- `.state/` recovery and destroy + caveats for the CLI fallback path. - `scripts/promote_to_agentcore.sh` -- promotion procedure. --- +## 7a. Worker → AgentCore Runtime invoke fails (`RuntimeClientError`) + +Symptom: the `agent-worker` Pod logs + +``` +worker.agentcore.invoke_attempt correlation_id= issue_key=... +worker.agentcore.invoke_failed correlation_id= error_type=RuntimeClientError + error="An error occurred when starting the runtime" +``` + +and never emits a matching `worker.consumer.message_processed` for that +correlation. The Lambda + SQS ingestion legs work fine +(`webhook_validator.accepted` is logged), but every dequeued message +fails at `bedrock-agentcore:InvokeAgentRuntime`. + +Detect (in priority order): + +1. **ARN drift between worker ConfigMap and `.state/agentcore-runtime.env`.** + The most common cause when this appears immediately after + `scripts/bootstrap_agentcore_runtime.sh` rebuilt the runtime: + + ```bash + diff \ + <(kubectl -n aws-agent-core get configmap agent-worker-config \ + -o jsonpath='{.data.AGENTCORE_RUNTIME_ARN}'; echo) \ + <(grep AGENTCORE_RUNTIME_ARN .state/agentcore-runtime.env | cut -d= -f2-) + ``` + + If the two differ, the worker Pod is invoking a stale (often + replaced or now-`DELETING`) runtime. `scripts/validate_rebuild_state.sh` + §3a catches this in CI. + +2. **Endpoint not yet `READY`.** Even with the right ARN, AgentCore + returns this error while the endpoint is still `CREATING` or + `UPDATING`: + + ```bash + aws bedrock-agentcore-control get-agent-runtime-endpoint \ + --region us-east-1 \ + --agent-runtime-id \ + --endpoint-name DEFAULT \ + --query 'status' --output text + ``` + + Anything other than `READY` will fail every invoke. + +3. **Runtime container failing to start / crash-looping.** Tail the + freshly-deployed runtime's log group (note the runtime id at the end + of the ARN — do *not* tail an older runtime's group): + + ```bash + aws logs tail \ + /aws/bedrock-agentcore/runtimes/-DEFAULT \ + --region us-east-1 --follow + ``` + + The log-group name pattern is + `/aws/bedrock-agentcore/runtimes/-`, + not the bare runtime name; checking the wrong group is the most + common confusion when triaging this error. + +Mitigate: + +- **For drift (case 1):** re-run + `scripts/bootstrap_eks_workloads.sh` (re-renders the ConfigMap with + the new ARN — the SHA-256 annotation + ([`terraform/eks-workloads/manifests.tf`](../terraform/eks-workloads/manifests.tf) + `locals.agent_worker_configmap_checksum`) flips, Kubernetes rolls the + Deployment, the script also runs an explicit `kubectl rollout + restart deploy/agent-worker` as belt-and-braces). Then re-run the + smoke. See + [`docs/runbook-sandbox-deploy.md`](runbook-sandbox-deploy.md) §11.5. +- **For endpoint state (case 2):** wait for `READY` (typically + 30s-2min after `agentcore deploy`). The worker's boto3 client + ([`src/agent/composition/aws.py::_default_bedrock_agentcore_client`](../src/agent/composition/aws.py)) + is configured with `read_timeout=120` and `mode="adaptive"` retries + precisely so a single in-call cold-start race survives, but the + initial readiness window is longer than that. +- **For runtime startup failures (case 3):** read the runtime log + group; common culprits are missing env vars (the toolkit accepts + `--env KEY=VALUE` flags through + [`scripts/bootstrap_agentcore_runtime.sh`](../scripts/bootstrap_agentcore_runtime.sh)) + and IAM trust mismatches on the execution role. +- **For VPC-mode startup hangs:** the runtime container imports cleanly + but blocks 60s on the first `secrets_resolver.get_secret_value()` + call (Bedrock guardrail fetch in + [`agent.composition.aws._resolve_guardrails`](../src/agent/composition/aws.py)), + surfacing here as `RuntimeClientError`. The cause is a missing + ingress rule on the VPC interface-endpoint security group from the + runtime's own VPC SG — see + [`terraform/agentcore-runtime/vpc.tf::aws_vpc_security_group_ingress_rule.endpoints_from_runtime`](../terraform/agentcore-runtime/vpc.tf) + and ensure + `terraform/agentcore-runtime/terraform.tfvars::vpc_endpoint_security_group_id` + is set to the network module's `agent_runtime_sg_id` output. + `scripts/sync_sandbox_tfvars.py --phase post-eks` keeps this in sync. +- **For Python-extras drift on the runtime image:** the `requirements.txt` + consumed by the toolkit's CodeBuild pipeline must install all + optional extras the AWS profile's composition root needs. Anything + other than `.[aws,xray,otel]` (e.g. a bare `.`) makes the runtime + container abort during `import agent.composition.aws` with a + `ModuleNotFoundError` for `langgraph_checkpoint_aws` / + `aws_xray_sdk` / `opentelemetry`. The file is tracked in git; + the only failure mode is a stale local checkout. +- **For ASGI-server-not-started failures:** [`src/agent/main.py`](../src/agent/main.py) + must end with `if __name__ == "__main__" and app is not None: app.run()` + so the Dockerfile entrypoint `python -m agent.main` actually binds + port 8080. Without that line the container exits 0 immediately and + AgentCore re-spawns it forever, surfacing as `RuntimeClientError`. + +Recover: + +- After any mitigation that changes the worker ConfigMap, re-run + `scripts/validate_rebuild_state.sh` and confirm `[PASS] agent-worker + ConfigMap AGENTCORE_RUNTIME_ARN matches .state/agentcore-runtime.env`. + +Root-cause references: + +- `src/agent/infrastructure/agentcore/boto_invoker.py` — + the `BotoAgentCoreInvoker.invoke` call shape (payload, qualifier, + `traceId`). +- `src/agent/composition/aws.py::_default_bedrock_agentcore_client` — + the `botocore.Config` (timeouts + adaptive retries). +- `terraform/eks-workloads/manifests.tf` + + `terraform/eks-workloads/templates/agent-worker/deployment.yaml.tftpl` + — the ConfigMap-checksum annotation that prevents drift on the + next deploy. +- `terraform/agentcore-runtime/vpc.tf` — runtime VPC SG + ingress + rule onto the interface-endpoint SG (codified after the Secrets + Manager connect-timeout incident). +- `terraform/agentcore-runtime/iam.tf::s3_put_payloads` — + `kms:GenerateDataKey` grant for SSE-KMS on the raw-payloads bucket + (codified after the `webhook.payload_sink.failed` incident). +- `scripts/bootstrap_agentcore_runtime.sh` — `OBSERVABILITY_BACKEND=noop` + override + OTLP / X-Ray daemon env stripping for the runtime. + Worker ConfigMap's `OBSERVABILITY_BACKEND=otlp` would block on the + in-cluster ADOT collector (unreachable from the runtime ENI), and + `OBSERVABILITY_BACKEND=xray` would activate the in-process + `aws_xray_sdk` rule-poller against `127.0.0.1:2000` (no X-Ray + daemon exists in the AgentCore Runtime container — the daemon is + an EKS DaemonSet at `terraform/observability-xray/`). `noop` is + correct for AgentCore: the platform's managed OTel collector + reports invocation-level X-Ray spans automatically, and CloudWatch + GenAI metrics flow via direct `cloudwatch:PutMetricData` (no + tracer dependency). +- `requirements.txt` — `.[aws,xray,otel]` extras consumed by the + toolkit's CodeBuild pipeline. +- `src/agent/main.py` — `app.run()` guard at the bottom of the module. +- `scripts/bootstrap_agentcore_runtime.sh::stash_paths_for_agentcore_zip` + — temporarily moves `mcp/` out of the working tree before + `agentcore configure` / `agentcore deploy` so it never enters the + toolkit-built `source.zip`. `agentcore` does NOT honor the + repo-local `.dockerignore`; it hardcodes its own bundled + `dockerignore.template` (`bedrock_agentcore_starter_toolkit + /utils/runtime/templates/dockerignore.template`) which excludes + `terraform/`, `cdk/`, `tests/`, `docs/`, and `mcp/lambda/` but + not `mcp/` itself. The original size driver — the ~62 MB Spring + Boot fat-JAR — is no longer in this repo (it is resolved from + outside the repo at build time; see `mcp/README.md` §1 "External + JAR contract"), so the stash is now topology hygiene rather than a + size guard: nothing in `mcp/` is consumed by the AgentCore Runtime + container, and stashing keeps the source.zip aligned with the + runtime's actual build context. The trap in the bootstrap script + restores the path on EXIT/INT/TERM/HUP so an interrupted run never + leaves the working tree mutated. Historically (pre-external-JAR), + every `agentcore deploy` without this stash uploaded ~81 MB to S3 + and intermittently hit `botocore.exceptions.ConnectTimeoutError` / + `S3UploadFailedError: RequestTimeout` on slow uplinks, surfacing + as a hung redeploy and a stale runtime image. +- `terraform/agentcore-runtime/iam.tf::agentcore_memory_dataplane` + grants `bedrock-agentcore:CreateEvent` / `DeleteEvent` / `GetEvent` / + `GetMemoryRecord` / `ListEvents` / `ListActors` / `ListSessions` / + `ListMemoryRecords` / `RetrieveMemoryRecords` on the deployed + AgentCore Memory store. The LangGraph `AgentCoreMemorySaver` (the + AWS-profile default checkpointer when `BEDROCK_AGENTCORE_MEMORY_ID` + is set) issues these calls on every invocation to load and persist + short-term-memory state. Without the grants, every webhook surfaces + as `server.unhandled.error: ExceptionGroup` wrapping + `botocore.errorfactory.AccessDeniedException` for `ListEvents` + (verified 2026-05). The resource ARN list defaults to a wildcard + scoped to `_mem-*` (Terraform's + `agent_runtime_name` is hyphen-cased; the toolkit-side memory id is + underscore-cased — `agentcore_memory_name_token` in + `iam.tf` handles the normalization). Once `agentcore deploy` mints + the concrete id, `scripts/sync_sandbox_tfvars.py` writes it into + `agentcore-runtime/terraform.tfvars::agentcore_memory_resource_arns` + on the next sync, tightening the wildcard to the exact ARN. +- `src/agent/infrastructure/cost/model_prices.json` MUST carry an + entry for every model id that `BEDROCK_MODEL_ID` (or the local + Anthropic equivalent) can resolve to. The strict + `StaticCostCalculator.usd_for` lookup raises `UnknownModelError` + on miss, which propagates through LangGraph's `TaskGroup` and + surfaces as `server.unhandled.error: ExceptionGroup` (verified + 2026-05 with `global.anthropic.claude-sonnet-4-6`). The deploy + plan's tiered model probe (`scripts/check_bedrock_model_access.py`) + enumerates the valid ids; keep the price table in lockstep with + that probe and with `agent.main.SUPPORTED_BEDROCK_MODEL_FAMILIES` + / the Terraform `bedrock_model_id` validation regex. +- `scripts/bootstrap_agentcore_runtime.sh` exports + `AWS_S3_ADDRESSING_STYLE=path` before invoking `agentcore`. The + toolkit's source.zip upload otherwise targets a bucket-virtual-hosted + URL (`.s3.us-east-1.amazonaws.com`, aliased to the + `s3-r-w.us-east-1.amazonaws.com` IP range), which is **routed + separately** from the plain `s3.us-east-1.amazonaws.com` endpoint. + Operator networks with degraded paths to `s3-r-w` (broken PMTU + discovery on consumer ISPs, MTU-clamping VPNs, corporate egress + proxies that drop large multipart packets) reproducibly hang the + upload with `RequestTimeout` / `Could not connect to the endpoint + URL` even though `aws s3api head-bucket` against the same bucket + succeeds in <3 s — verified 2026-05 against this account from a + consumer-ISP path. Forcing path-style routes the toolkit's S3 PUTs + through `s3.us-east-1.amazonaws.com`, sidestepping the broken CIDR + with no observable cost on healthy networks. The override is scoped + to the bootstrap-script process and does not leak. +- `src/agent/graph/nodes/assessor.py::_ensure_user_message_tail` and + the matching defensive guard in + `src/agent/infrastructure/bedrock_llm.py::_BoundChatRunnable._guard_structured_output_tail` + pin the **Bedrock conversation contract** for every + `with_structured_output(...)` invocation. The bind-tools loops in + `agent.graph.nodes._agentic_turn` always halt on a trailing + `AIMessage` whose `tool_calls` are empty (the routing condition that + lands at `terminal_assess` / `design_terminal`); without an + appended user turn Bedrock's Converse API rejects the request with + `ValidationException: This model does not support assistant message + prefill. The conversation must end with a user message.` (verified + 2026-05 with `global.anthropic.claude-sonnet-4-6`; the LangGraph + `TaskGroup` wraps the Bedrock error in an `ExceptionGroup` that + surfaces as `server.unhandled.error`). Both nodes append a + schema-specific instruction `HumanMessage`; the bedrock_llm seam + also caps any structured-output input that ends with an `AIMessage` + as a belt-and-braces guarantee for any future call site that + forgets the invariant. Anthropic's direct API allows assistant + prefill, so the local profile is unaffected. + +--- + ## 8. `git_get_repo` returns NOT_FOUND Symptom: the assessor's `llm_node` invokes the `git_get_repo` MCP @@ -427,9 +699,9 @@ followup_field = "repo_url" Mitigate: -- The renderer's Wave 8 fallback already keeps the assessor honest: +- The follow-up fallback already keeps the assessor honest: when `target_custom_field_id` is unset (the typical case for a - bad repo identifier), the comment surfaces a free-form + bad repo identifier), the agent surfaces a free-form `[repo_url] Which Git repository should this issue be implemented against?` follow-up so the human reviewer can paste the correct URL into the issue (or into the per-tenant @@ -440,8 +712,8 @@ Mitigate: per-tenant repo-context store outage), set the affected tenant's `AGENT_REPO_URL_FIELD_ID` to a known-good Jira custom field id via the tenant's `tenants` DynamoDB row; the next `terminal_assess` - will populate `target_custom_field_id` so the renderer surfaces - the structured affordance. + will populate `target_custom_field_id` so the follow-up includes the + structured affordance. - For a transient MCP outage (the `mcp-internal` deployment is flaky), follow scenario §2 (MCP outage) — the same circuit breaker covers `git_*` tools. @@ -460,19 +732,503 @@ Recover: Root-cause: -- `src/agent/infrastructure/mcp/git_tool.py` — `git_get_repo` MCP - tool wrapper. +- `mcp-internal` git tools — the `git_get_repo` MCP operation surfaced + through LangChain tool wrapping. - `src/agent/graph/prompts.py` — the assessor's repo-context - instructions (the Wave 8 ``target_custom_field_id`` block tells - the LLM how to populate the structured affordance). -- `src/agent/application/comment_renderer.py` — the renderer's - fallback that surfaces the prose follow-up when no - ``target_custom_field_id`` is set. + instructions (the ``target_custom_field_id`` block tells the LLM + how to populate the structured affordance). +- `src/agent/graph/nodes/persistence.py` — persists structured follow-up + questions and evidence for the next webhook turn. - ``terraform/`` per-tenant secrets / config for ``AGENT_REPO_URL_FIELD_ID`` (when wired). --- +## 9. Recursion guard identity drift + +Symptom: the agent comments on a Jira issue and then processes its own comment +as a fresh human-authored webhook. The next run may emit another comment, +creating a recursion loop until token-budget limits stop it. + +Detect: + +```text +event = "webhook.processed" +| stats count() by issue_key, actor_account_id, actor_email +| sort count desc +``` + +Then compare the webhook actor from the raw S3 payload against the configured +agent identity: + +```bash +kubectl get configmap agent-runtime-config -n aws-agent-core-local -o yaml \ + | grep -E 'AGENT_JIRA_(ACCOUNT_ID|EMAIL)' +``` + +Mitigate: + +- Pause the Jira webhook subscription or route new envelopes to the DLQ if a + loop is active. +- Update `AGENT_JIRA_ACCOUNT_ID` and `AGENT_JIRA_EMAIL` so they match the Jira + integration user credentials used by `mcp-internal`, then restart the runtime. +- Replay only one affected DLQ envelope first and confirm + `webhook.skipped.recursion` with `reason="agent_account_id"` or + `reason="agent_email"` before replaying the backlog. + +Recover: + +- Review any Jira comments created during the loop and remove duplicates if + needed. +- After Jira credential rotation, verify the identity pair manually because the + current `mcp-internal` JAR does not expose a stable current-user MCP tool. + The follow-up contract is documented in [`mcp/README.md`](../mcp/README.md#42-jira-current-user-identity-tool-recursion-drift-guard). + +Root-cause: + +- `src/agent/application/recursion_guard.py` -- identity-only recursion policy. +- `mcp/README.md` -- upstream current-user tool contract needed for an automated + startup/smoke self-test. +- `.env.local.example` / deployment secrets -- configured agent identity. + +--- + +## 10. Manual bot-account writes from the Jira UI + +Symptom: an operator logs into Jira *as the bot service account* +and types a comment (or transitions a ticket) by hand. The +resulting webhook is recorded as `webhook.skipped{reason= +"recursion_guard"}` and the agent never reacts. This is the +recursion guard working **as designed** — the agent cannot +distinguish "manual human typing as the bot" from "agent's own +`jira_add_comment` triggering a self-loop". + +Architectural background: the graph runs **one assessor iteration +per inbound webhook**, never blocking on external input inside a +single invocation. Multi-turn conversations are stitched together +by the `thread_id`-keyed checkpointer. Every webhook delivery is +re-checked against the recursion guard, so any actor whose +`accountId` or `emailAddress` matches the configured bot identity +is unconditionally skipped. See +[`docs/ARCHITECTURE.md#single-iteration-invariant`](ARCHITECTURE.md#single-iteration-invariant) +for the full invariant. + +Recommended courses of action, in priority order: + +1. **Use a separate human account.** This is the only fully-safe + option — the agent reacts to the human's comment normally and + nothing about the bot identity is mutated. +2. **Issue the action through the bot's API path.** If the desired + action is well-suited to the agent (e.g. ask the agent to add a + comment via a fresh `issue_updated` webhook on a *human's* + ticket), do that instead of typing it manually as the bot. +3. **Temporary recursion-guard bypass for a single ticket** (last + resort, requires manual cleanup): + + ```bash + # 1. Disable the recursion guard for a fixed window. There is no + # production toggle in the codebase by design; the only sanctioned + # path is to rotate the bot identity *first* so that human-typed + # comments under the bot account do not match. See step 4 below. + ``` + +4. **Identity rotation procedure** (durable fix when an operator + needs ongoing manual access to the bot account): + 1. Provision a new dedicated bot Jira account (e.g. + `agent-bot-2@example.com`) in the same Jira tenant. + 2. Update the secret backing the agent's bot identity in AWS + Secrets Manager — `agent//jira/identity` — with the new + `accountId` / `emailAddress`. Both keys are read together by + the AWS composition root, so they must be rotated atomically. + 3. Re-run `agentcore deploy` (or restart pods locally) so the + new identity is loaded into `AgentIdentity`. + 4. Verify in CloudWatch Logs that + `webhook.skipped{reason="recursion_guard"}` now matches the + *new* bot account, and that the old account's comments + arrive as `webhook.processed`. + 5. Document the rotation in the deployment journal so on-call + can correlate the change against any incident timeline. + +Validation queries: + +```bash +# CloudWatch Logs Insights — count of skipped webhooks broken down +# by recursion-guard reason and actor over the last 24h. +fields @timestamp, fields.reason, fields.actor_account_id +| filter event = "webhook.skipped" +| stats count() by fields.reason, fields.actor_account_id +| sort count() desc +``` + +```bash +# Confirm the running identity matches the expected service account +# before/after rotation. +aws secretsmanager get-secret-value \ + --secret-id agent//jira/identity \ + --query SecretString --output text | jq '.account_id' +``` + +Root-cause: + +- `src/agent/application/recursion_guard.py` — actor-identity match + predicates (`agent_account_id`, `agent_email`). +- `src/agent/composition/aws.py` / `src/agent/composition/local.py` + — identity is loaded once at boot from the configured secret. +- `docs/ARCHITECTURE.md#single-iteration-invariant` — the + per-webhook-iteration contract that makes this guard the only + defence against bot self-loops. + +--- + +## 12. HITL approval gate stuck or misfiring + +Symptom: a Jira ticket has reached the HITL approval gate but the +agent never resumes after a human reply, OR the agent releases to +the designer despite no human approval, OR every approval comment +is interpreted as a rejection (or vice versa). + +Detect: + +```text +event in ["hitl.requested", "hitl.approved", "hitl.rejected", "hitl.override"] +| stats count() by event, issue_key +``` + +A healthy ticket flow shows exactly one `hitl.requested` followed +by exactly one `hitl.approved` (or `hitl.rejected`). Repeated +`hitl.requested` records on the same `issue_key` with no terminal +verdict mean the assessor keeps re-entering the gate without the +human reply ever being seen by `evaluate_human_approval`. + +Confirm the persisted phase: + +```bash +# AWS profile: read the current TicketDomainState row +aws dynamodb get-item \ + --table-name agent__ticket_state \ + --key '{"tenant_id":{"S":""},"issue_key":{"S":""}}' \ + | jq '.Item.readiness_phase.S' +``` + +A value of `"awaiting_human_approval"` means the next inbound +webhook *should* route to `evaluate_human_approval`; any other +phase explains why the second-webhook route never fires. + +Mitigate: + +- **Stuck in `awaiting_human_approval`.** Verify the human's + comment cleared the recursion guard (see scenario 9): the + comment must be authored by an account whose `account_id` / + `email` does **not** match the configured agent identity. If + the operator wrote the approval as the bot service account, + the webhook was silently dropped and the gate stays paused. + Re-post the approval from a real human Jira user. +- **`evaluate_human_approval` always rejects (or always approves).** + The cheap `approval_llm` is misclassifying replies. Inspect the + `hitl.approved` / `hitl.rejected` log records — each carries + the `rationale` field emitted by the structured-output + ``ApprovalEvaluation``. If the rationale is incoherent, raise + the assessor / approval LLM tier in the runtime config + (`APPROVAL_LLM_MODEL_ID` / `APPROVAL_BEDROCK_MODEL_ID`) and + redeploy. The default Haiku-class model is sufficient for + binary approve/reject classification but is sensitive to the + approval-ask comment format the assessor posts. +- **Override comment never appears.** When the human approves + but the assessor regressed to `need_info`, the gate posts an + override comment via `approval_llm.bind_tools(jira_add_comment)`. + A `hitl.override.tool_missing` warning means the MCP tool + catalogue does not expose `jira_add_comment` to the approval + LLM (composition-root issue); a `hitl.override.no_tool_call` + warning means the cheap LLM declined to emit a tool call (model + capability issue — escalate the model tier as above). The + routing decision (`hitl_phase="ready"`) holds in both cases, so + the ticket still progresses to the designer. +- **Comment thread sourced from `jira_get_issue`.** Both the + assessor seed prompt (`src/agent/graph/state.py`) and the approval + evaluator prompt (`src/agent/graph/nodes/approval.py`) instruct the + LLM to call `jira_get_issue` only — the qodo `mcp-internal` server + returns the full comment list under `fields.comment.comments` + alongside the current description, so a single call covers + everything and the prompts deliberately do **not** ask for a + separate `jira_get_comments` tool (the qodo server does not + publish one). If the approval LLM starts missing recent comments, + first verify the live `jira_get_issue` payload still carries + `fields.comment.comments` (qodo `JiraService.getIssue`); the + agent allowlist (`src/agent/composition/_shared.py` + `DEFAULT_DESIGN_TOOL_ALLOWLIST` and `_filter_approval_tools`) is + not the bottleneck. + +Recover: + +- Replay the second-webhook envelope from the DLQ once the + recursion-guard / model-tier issue is fixed: + `python scripts/replay_dlq.py --from-sqs --filter issue_key=`. +- Confirm `event = "hitl.approved"` (or `hitl.rejected`) appears + for the affected `issue_key` and the persisted + `readiness_phase` advances to `"ready"` / + `"awaiting_info"` / `"blocked"`. + +Root-cause: + +- `src/agent/graph/nodes/approval.py` — `request_human_approval` + (deterministic) and `evaluate_human_approval` (cheap-LLM) + factories. +- `src/agent/graph/prompts.py` — `APPROVAL_EVAL_SYSTEM_PROMPT`, + `OVERRIDE_COMMENT_SYSTEM_PROMPT`, `OVERRIDE_COMMENT_TEMPLATE`, + and the assessor's HITL re-entry instructions in + `SYSTEM_PROMPT`. +- `src/agent/composition/aws.py` / + `src/agent/composition/local.py` — `approval_llm` factory + resolution from `APPROVAL_BEDROCK_MODEL_ID` / + `APPROVAL_LLM_MODEL_ID` env vars. +- `docs/ARCHITECTURE.md#multi-agent-extension-points` — the + HITL-gate routing predicate contract. + +--- + +## 13. Async dispatch (`status="accepted"`) — agent-worker queue health + +Symptom: an operator (or the `make smoke` / `scripts/invoke_manual.sh` +/ `scripts/invoke_lambda.sh` harness) hits the runtime and sees +`status="accepted"` instead of `processed` / `skipped` / `duplicate` +/ `error`. The HTTP caller (Jira) +gets a 200 ACK immediately but the LangGraph terminal status never +appears in the agent-runtime logs, OR the SQS work queue is backing +up and Jira tickets are not being commented on. + +Architectural background: the webhook hot-path is split so the +synchronous frontend (the +[`webhook-validator` Lambda](../lambda/webhook_validator/handler.py) +fronted by the ALB) only runs cheap gates — HMAC verification, +envelope parsing, and the 202 Accepted response — and publishes +`{correlation_id, webhook}` envelopes onto the `agent-work` SQS +queue. A separate `agent-worker` Pod long-polls that queue, runs +the prevalidate gates (normalization, tenant resolution, recursion +guard), opens its own per-message MCP session, and runs the full +graph through `WebhookHandler.process()`. See +[`src/agent/worker/composition.py`](../src/agent/worker/composition.py) +for the per-message dispatch and +[`src/agent/worker/consumer.py`](../src/agent/worker/consumer.py) +for the long-polling loop. + +Detect: + +```text +# Synchronous side (agent-runtime): every accepted dispatch. +event = "webhook.dispatch.accepted" +| stats count() by tenant_id, kind +``` + +```text +# Asynchronous side (agent-worker): the eventual graph terminal. +event = "webhook.process.completed" +| stats count() by status +| sort count desc +``` + +A healthy steady state shows the two counts within a few seconds of +each other. If `webhook.dispatch.accepted` keeps climbing while +`webhook.process.completed` stays flat, the worker is not draining +the queue. + +Confirm the SQS work queue depth (LocalStack / local profile): + +```bash +kubectl -n agent-core exec deploy/localstack -- \ + awslocal sqs get-queue-attributes \ + --queue-url http://localhost:4566/000000000000/agent-work \ + --attribute-names ApproximateNumberOfMessages \ + ApproximateNumberOfMessagesNotVisible \ + ApproximateNumberOfMessagesDelayed +``` + +Or in the AWS profile: + +```bash +aws sqs get-queue-attributes \ + --queue-url "$(terraform -chdir=terraform/sqs-agent-work output -raw queue_url)" \ + --attribute-names ApproximateNumberOfMessages \ + ApproximateNumberOfMessagesNotVisible +``` + +A non-zero `ApproximateNumberOfMessagesNotVisible` with a flat +`webhook.process.completed` curve means the worker is picking +messages up but failing to delete them — the message will reappear +after the 600s visibility timeout and the redrive policy will move +it to `agent-dlq` after `maxReceiveCount=3`. + +Mitigate: + +- **Worker Pod not running.** Confirm: + ```bash + kubectl -n agent-core get deploy/agent-worker + kubectl -n agent-core logs -f deploy/agent-worker --tail=200 + ``` + Restart with `kubectl rollout restart deploy/agent-worker` if the + Pod is crash-looping. +- **Consumer crashed mid-message (visibility-timeout queue).** The + message will redeliver after the visibility timeout (600s) and, + after `maxReceiveCount=3` retries, redrive to `agent-dlq`. Do not + manually delete the SQS message — the redrive policy is the + durability contract. +- **Misconfigured queue URL.** Verify the worker's + `WEBHOOK_WORK_QUEUE_URL` matches the publisher's + `WEBHOOK_WORK_QUEUE_URL`: + ```bash + kubectl -n agent-core get cm agent-runtime-config \ + -o jsonpath='{.data.WEBHOOK_WORK_QUEUE_URL}' + kubectl -n agent-core exec deploy/agent-worker -- \ + env | grep WEBHOOK_WORK_QUEUE_URL + ``` +- **Emergency synchronous fallback.** Flip + `WEBHOOK_ASYNC_DISPATCH=0` in the `agent-runtime-config` ConfigMap + and `kubectl rollout restart deploy/agent-runtime` to revert to + the in-process graph invocation. This trades async ACK semantics + for a working hot-path while you debug the worker. + +Recover: + +- Drain the DLQ via `python scripts/replay_dlq.py --from-sqs + --target ` once the underlying + failure is fixed; the replay re-enters the synchronous endpoint + and (with async dispatch on) re-enqueues each envelope. +- Watch `webhook.process.completed{status="processed"}` climb back + to baseline before declaring the incident over. + +Root-cause: + +- [`lambda/webhook_validator/handler.py`](../lambda/webhook_validator/handler.py) + — HMAC verification + SQS publish (the only producer for the + `agent-work` queue in production). +- [`src/agent/worker/consumer.py`](../src/agent/worker/consumer.py) + — long-polling loop, message lifecycle (delete-on-success). +- [`src/agent/worker/composition.py`](../src/agent/worker/composition.py) + — per-message MCP session + `WebhookHandler.process()` wiring. +- [`src/agent/application/webhook_handler.py`](../src/agent/application/webhook_handler.py) + — `prevalidate()` / `process()` split. +- [`src/agent/infrastructure/messaging/sqs_work_publisher.py`](../src/agent/infrastructure/messaging/sqs_work_publisher.py) + — `SqsWorkPublisher` (used by the synchronous side). +- [`deploy/local/agent-worker.yaml`](../deploy/local/agent-worker.yaml) + — local-profile worker Deployment (mirror of `agent-runtime` but + without the HTTP surface). +- `Makefile` — `seed-localstack` provisions the `agent-work` SQS + queue with `VisibilityTimeout=600s`, + `ReceiveMessageWaitTimeSeconds=20s`, `MessageRetentionPeriod=4d`, + `RedrivePolicy={maxReceiveCount=3, deadLetterTargetArn=agent-dlq}`. + +--- + +## 14. Rate limiting, payload size, and back-pressure ownership + +**Symptom.** A spike of inbound Jira webhooks (bulk transition, mass +import, mis-configured automation rule) lands on `/invocations` faster +than the agent can process. On-call wants to know "where is the +back-pressure?" and "can I just tighten an env var on the agent Pod?". + +**Diagnosis.** There is **no application-layer rate limit, queue +admission gate, or max-payload-size guard** in the agent process by +design. The agent intentionally delegates flow control to the +infrastructure layer plus the per-tenant token-governance chain: + +1. **Edge / ingress.** AgentCore Runtime (AWS profile) and the API + Gateway / ALB / Kubernetes Ingress in front of `/invocations` + own request-rate throttling and payload-size limits. AgentCore + Runtime's managed harness also enforces `maxTokens` per + invocation. Cap untrusted payload sizes here, never in the + Python entrypoint. +2. **Async dispatch buffer.** The `agent-work` SQS queue is the + shock absorber between Atlassian's delivery rate and the + `agent-worker` Pod's processing rate. `VisibilityTimeout=600s` + plus `RedrivePolicy={maxReceiveCount=3, + deadLetterTargetArn=agent-dlq}` (see §13) means worker + saturation manifests as queue depth, not as dropped Jira + responses. Operators scale workers horizontally to drain the + queue. +3. **Per-tenant token governance.** + [`TokenBudgetEnforcer`](../src/agent/contracts/token_usage.py) + caps a tenant's rolling-window LLM consumption (input tokens, + output tokens, USD spend); a saturated budget short-circuits + the assessor with `BudgetExceededError` *before* the LLM is + called. See §3 for the saturation runbook. +4. **Per-process LLM blast-radius.** + [`LlmCircuitBreaker`](../src/agent/contracts/token_usage.py) + trips after `LLM_CIRCUIT_FAIL_MAX` consecutive vendor errors + and rejects further calls with `LlmCircuitOpenError` until + `LLM_CIRCUIT_RESET_SECONDS` elapses. See §4. + +What you will **not** find: +the `webhook-validator` Lambda +([`lambda/webhook_validator/handler.py`](../lambda/webhook_validator/handler.py)) +does not register an HTTP rate-limit middleware, a `Content-Length` +ceiling, or a backpressure 503 path. The webhook-handler layer also +does not maintain a `claim` table or any other admission gate (see +[`src/agent/application/webhook_handler.py`](../src/agent/application/webhook_handler.py) +module docstring). + +**Action checklist.** + +- If Atlassian is hammering the endpoint, raise the throttle limit + (or temporarily lower it) at the AgentCore Runtime / API Gateway + layer — **not** in the agent ConfigMap. +- If the queue depth is climbing, scale the `agent-worker` + Deployment horizontally; verify per-tenant budgets are not + saturated (§3) and the breaker is closed (§4). +- If a single payload is unreasonably large, add a payload-size + ceiling at the ingress (WAFv2 / ALB / Lambda config); do not + introduce one in + [`lambda/webhook_validator/handler.py`](../lambda/webhook_validator/handler.py) + without amending this runbook: ingress and the Python agent must stay + aligned on what constitutes an acceptable payload. +- If you find yourself reaching for a Python-side rate limiter, + stop and re-read this section; the back-pressure chain above is + the architectural answer. + +--- + +## CloudWatch Logs cheat-sheet (AWS profile) + +The `event = "..."` queries above run against CloudWatch Logs Insights. +Three log groups carry every signal an on-call needs: + +| Source | Log group | Owner / shipping path | +|---------------------------------|------------------------------------------------------------|------------------------| +| `webhook-validator` Lambda | `/aws/lambda/aws-agent-core-sandbox-webhook-validator` | Native Lambda → CloudWatch (Lambda execution role's `AWSLambdaBasicExecutionRole`). | +| `agent-worker` + `mcp-internal` Pod stdout/stderr | `/aws/eks//aws-agent-core/application` | EKS Pod stdout → kubelet log files → ADOT Collector `filelog` receiver → `awscloudwatchlogs` exporter. Owned by [`terraform/eks/adot.tf::aws_cloudwatch_log_group.adot_application`](../terraform/eks/adot.tf); retention pinned via `var.application_log_retention_in_days`. The structured-log JSON the worker emits is parsed inline by the `json_parser` operator so attributes are queryable directly (no `parse @message` boilerplate). | +| AgentCore Runtime container | `/aws/bedrock-agentcore/runtimes/--DEFAULT` | AgentCore platform-managed; visible only when the runtime accepts an invocation. | + +Tail or query examples: + +```bash +# Live-tail the worker + mcp-internal Pods together (AWS profile): +aws logs tail /aws/eks/jira-readiness-eks/aws-agent-core/application \ + --since 30m --follow + +# Filter by container: +aws logs tail /aws/eks/jira-readiness-eks/aws-agent-core/application \ + --filter-pattern '{ $.kubernetes.container.name = "agent-worker" }' + +# Cross-source correlation by correlation_id: +for g in \ + /aws/lambda/aws-agent-core-sandbox-webhook-validator \ + /aws/eks/jira-readiness-eks/aws-agent-core/application \ + /aws/bedrock-agentcore/runtimes/--DEFAULT; do + echo "=== $g ===" + aws logs filter-log-events --log-group-name "$g" \ + --filter-pattern "\"\"" \ + --start-time $(( ($(date +%s) - 3600) * 1000 )) \ + --query 'events[].message' --output text +done +``` + +The Pod-stdout shipping path was added 2026-05; before that the +agent-worker's stdout was only readable via `kubectl logs`, which made +post-mortem correlation across Lambda → worker → runtime impossible +once a Pod had been replaced. See +[`terraform/eks-workloads/templates/observability/adot-values.yaml.tftpl`](../terraform/eks-workloads/templates/observability/adot-values.yaml.tftpl) +for the receiver / processor / exporter wiring. + +--- + ## Cross-cutting reading list - [`docs/ARCHITECTURE.md`](ARCHITECTURE.md) -- high-level architecture. diff --git a/docs/sbom/README.md b/docs/sbom/README.md new file mode 100644 index 0000000..735acdd --- /dev/null +++ b/docs/sbom/README.md @@ -0,0 +1,50 @@ +# CycloneDX SBOM archive + +> **Note.** `sbom.cdx.json` is populated on the **first `v*` / +> `release/*` git tag** cut after this directory landed — an empty +> directory between releases is the expected steady state, not a gap. + +This directory carries the CycloneDX SBOM (`sbom.cdx.json`) for the +agent-runtime container image, published automatically on every +release tag (`v*` / `release/*`) by the +[`sbom`](../../.github/workflows/ci.yml) job in CI. + +## Why it lives in the repo + +- AppSec and supply-chain auditors get a stable, version-controlled + pointer instead of having to download a workflow artifact. +- `pip-audit` and `grype` can run directly against `docs/sbom/sbom.cdx.json` + without rebuilding the image. +- Diffing two release tags' SBOMs is a regular `git diff`. + +## How it is generated + +The release-tag CI job: + +1. Builds the agent-runtime image locally + (`scripts/build_local_image.sh`). +2. Runs the official Anchore [`sbom-action`](https://github.com/anchore/sbom-action) + over the image, emitting CycloneDX JSON. +3. Copies the resulting file to `docs/sbom/sbom.cdx.json` and commits it + back to the default branch with the auto-provisioned `GITHUB_TOKEN`. + +For PR builds and non-tag pushes, the SBOM is **not** committed; it is +still emitted as a workflow artifact (`agent-runtime-sbom-cyclonedx`) +for spot inspection during code review. + +## How to regenerate locally + +```bash +# Build the agent-runtime image and emit a fresh SBOM in the repo root. +scripts/build_local_image.sh +syft aws-agent-core/agent-runtime:local -o cyclonedx-json=sbom.cdx.json + +# Diff it against the committed copy. +diff -u docs/sbom/sbom.cdx.json sbom.cdx.json | less +``` + +## Contents + +- `sbom.cdx.json` — CycloneDX 1.5 JSON, populated on the first release tag + cut after this directory landed. Until then the placeholder below + is a tracked-but-empty marker so reviewers can audit the wiring. diff --git a/docs/structured_log_schema.json b/docs/structured_log_schema.json index 6823fb1..c732516 100644 --- a/docs/structured_log_schema.json +++ b/docs/structured_log_schema.json @@ -8,8 +8,8 @@ "properties": { "level": { "type": "string", - "enum": ["info", "warning", "error"], - "description": "Severity. Mirrors the StructuredLogger Protocol's three method names exactly." + "enum": ["debug", "info", "warning", "error"], + "description": "Severity. Mirrors the StructuredLogger Protocol's four method names exactly. The 'debug' tier is filtered by JsonStructuredLogger.min_level (default 'info'); operators set STRUCTURED_LOG_LEVEL=debug to surface diagnostic events." }, "component": { "type": "string", @@ -19,7 +19,93 @@ "event": { "type": "string", "minLength": 1, - "description": "Stable machine-readable event identifier. Conventional dotted form '..' (examples: webhook.received, webhook.skipped.recursion, webhook.skipped.duplicate, webhook.processed, webhook.signature.invalid, webhook.payload.invalid, webhook.runtime.error, graph.load_state, graph.llm_node.max_iterations, graph.record_observer, graph.terminal_assess, graph.persist_state_evidence, graph.persist_state_evidence.skipped, graph.design_llm_node.max_iterations, graph.design_post_comment, event_invariants.violated, llm.usage, llm.circuit.open, llm.budget.exceeded, mcp.call.error, composition.recursion_guard.empty_bot_account_ids, logging.schema_violation)." + "description": "Stable machine-readable event identifier. Conventional dotted form '..'. The enum below is the canonical, exhaustive set; mirrors agent.infrastructure.logging.CANONICAL_EVENT_IDENTIFIERS and is enforced by scripts/check_event_identifiers.py.", + "enum": [ + "check_table_contracts.ok", + "check_table_contracts.skipped", + "composition.bedrock.guardrails_disabled", + "composition.dedupe_store", + "composition.design_tools.filtered", + "composition.guardrail_decorator.wired", + "composition.mcp_loader.path", + "composition.metrics.degraded", + "composition.observability.degraded", + "composition.observability.otlp_wired", + "composition.token_budget_enforcer", + "graph.assemble_history_context", + "graph.debug.assemble_history_context.lookup", + "graph.debug.design_llm_node.iteration", + "graph.debug.design_terminal.plan", + "graph.debug.evaluate_human_approval.evaluated", + "graph.debug.evaluate_human_approval.input", + "graph.debug.evaluate_human_approval.iteration", + "graph.debug.llm_node.iteration", + "graph.debug.load_state.seeded", + "graph.debug.route_after_evaluate_human_approval", + "graph.debug.route_after_terminal_assess", + "graph.debug.terminal_assess.decision", + "graph.terminal_assess.validation_fallback", + "graph.terminal_assess.validation_retry", + "graph.design_llm_node.max_iterations", + "graph.design_post_comment", + "graph.design_post_comment.fallback_failed", + "graph.design_post_comment.fallback_posted", + "graph.design_post_comment.tool_missing", + "graph.llm_node.max_iterations", + "graph.load_state", + "graph.persist_approval_granted", + "graph.persist_state_evidence", + "graph.persist_state_evidence.skipped", + "graph.record_observer", + "hitl.approved", + "hitl.override", + "hitl.override.invoke_failed", + "hitl.override.no_tool_call", + "hitl.override.tool_missing", + "hitl.rejected", + "hitl.requested", + "llm.guardrail.intervened", + "llm.usage", + "logging.schema_violation", + "observability.otlp_export_failed", + "server.async_dispatch.envelope_not_supported", + "server.async_dispatch.publish_failed", + "server.async_dispatch.published", + "server.domain.error", + "server.payload.invalid", + "server.runtime.received", + "server.unhandled.error", + "trace.span", + "webhook.debug.normalized", + "webhook.debug.prevalidate.entered", + "webhook.debug.process.finished", + "webhook.debug.process.started", + "webhook.debug.recursion_check", + "webhook.debug.tenant_resolved", + "webhook.guardrail.blocked", + "webhook.payload.invalid", + "webhook.payload_sink.failed", + "webhook.processed", + "webhook.runtime.error", + "webhook.skipped.recursion", + "webhook.tenant.unresolved", + "worker.agentcore.invoke_attempt", + "worker.agentcore.invoke_failed", + "worker.agentcore.invoke_non_2xx", + "worker.agentcore.invoke_ok", + "worker.background.processed", + "worker.consumer.delete_failed", + "worker.consumer.message_processed", + "worker.consumer.message_received", + "worker.consumer.process_failed", + "worker.consumer.receive_failed", + "worker.consumer.started", + "worker.consumer.stopped", + "worker.dedupe.reserve_failed", + "worker.dedupe.skip", + "worker.dedupe.skipped", + "worker.session.close_failed" + ] }, "correlation_id": { "type": "string", @@ -65,9 +151,9 @@ "description": "MCP call classification. Mirrors agent.contracts.mcp_call_observer.McpCallKind." }, "decision": { - "type": "string", - "enum": ["need_info", "ready", "cannot_assess"], - "description": "Readiness decision. Mirrors agent.domain.assessment.ReadinessDecision." + "type": ["string", "null"], + "enum": ["need_info", "ready", "cannot_assess", null], + "description": "Readiness decision. Mirrors agent.domain.assessment.ReadinessDecision. Null when the run never reached an assessment (status=error or short-circuited at prevalidate)." }, "read_count": { "type": "integer", @@ -79,25 +165,47 @@ "minimum": 0, "description": "Per-correlation total write MCP calls observed." }, - "git_count": { - "type": "integer", - "minimum": 0, - "description": "Per-correlation count of git_* MCP calls. Surfaced on event_invariants.violated when issue_updated fails the git_* invariant (M5.9 closure)." - }, "mcp_retry_count": { "type": "integer", "minimum": 0, - "description": "Per-request total MCP retry attempts. Surfaced on webhook.processed and graph.persist_state_evidence (M5.9 closure)." + "description": "Per-request total MCP retry attempts. Surfaced on webhook.processed and graph.persist_state_evidence." }, "reason": { "type": "string", "minLength": 1, - "description": "Human-readable reason. Skip reasons are drawn from the closed set {agent_account_id, agent_email, duplicate_webhook, invalid_signature, invalid_payload, invariant_violation}; invariant-violation events surface a free-form composite reason (e.g. 'read_count=0 < 1; write_count=0 < 1')." + "description": "Human-readable reason. Skip reasons are drawn from the closed set {agent_account_id, agent_email, invalid_payload}." }, "previous_phase": { "type": ["string", "null"], - "enum": ["new", "awaiting_info", "ready", "blocked", null], - "description": "Previous TicketDomainState.readiness_phase. Surfaced on graph.load_state." + "enum": [ + "new", + "awaiting_info", + "awaiting_human_approval", + "ready", + "blocked", + null + ], + "description": "Previous TicketDomainState.readiness_phase. Surfaced on graph.load_state. Mirrors agent.domain.state.ReadinessPhase exactly (including the HITL pause phase 'awaiting_human_approval')." + }, + "readiness_phase": { + "type": ["string", "null"], + "enum": [ + "new", + "awaiting_info", + "awaiting_human_approval", + "ready", + "blocked", + null + ], + "description": "Resolved TicketDomainState.readiness_phase the persist node wrote. Surfaced on graph.persist_state_evidence and webhook.processed. Mirrors agent.domain.state.ReadinessPhase exactly." + }, + "rationale": { + "type": ["string", "null"], + "description": "Cheap approval LLM's one-sentence justification. Surfaced on hitl.approved / hitl.rejected / hitl.override." + }, + "override_evidence_recorded": { + "type": "boolean", + "description": "Whether evaluate_human_approval captured an EvidenceRef for the override jira_add_comment write. Surfaced on hitl.override." }, "pending_evidence_count": { "type": "integer", @@ -160,10 +268,6 @@ "minimum": 0.0, "description": "Computed USD cost for the LLM call. Surfaced on llm.usage." }, - "tenant": { - "type": ["string", "null"], - "description": "Tenant identifier reported on llm.usage / budget events. Same value as tenant_id; legacy field name preserved for backward compatibility with existing dashboards." - }, "profile": { "type": "string", "enum": ["local", "aws", "default"], diff --git a/lambda/webhook_validator/handler.py b/lambda/webhook_validator/handler.py new file mode 100644 index 0000000..f3b9341 --- /dev/null +++ b/lambda/webhook_validator/handler.py @@ -0,0 +1,309 @@ +"""AWS Lambda webhook validator for Atlassian Jira webhooks. + +Front-door HMAC-SHA256 verifier that sits behind the public ALB on +``/invocations``. The Lambda is the HMAC trust boundary for external +webhook ingress: it verifies the signature header against the raw POST +body (preferring the configured header and then falling back to other +Jira signature header names) using a secret resolved from AWS Secrets Manager +and, on success, publishes the validated envelope to the SQS work +queue consumed by the agent-worker pod. + +The work-queue envelope is intentionally minimal:: + + {"correlation_id": "", "webhook": { ...raw Atlassian payload... }} + +`correlation_id` honours an inbound ``x-correlation-id`` header when +present and otherwise mints a fresh ``uuid4``. `webhook` is the +parsed JSON body Atlassian delivered. + +Environment contract: + +* ``WEBHOOK_HMAC_SECRET_ID`` -- Secrets Manager secret id (or ARN) + holding the raw HMAC shared secret. Looked up once per cold start + and cached on the module global. +* ``WEBHOOK_WORK_QUEUE_URL`` -- SQS queue URL the validated envelope + is published to via ``SendMessage``. +* ``SIGNATURE_HEADER`` (optional, default ``x-jira-signature``) -- + Primary HTTP header name carrying the ``sha256=`` signature. + If missing, the handler falls back to ``x-hub-signature`` then + ``x-jira-signature`` for compatibility across Jira webhook variants. + +Response shape: + +* 202 ``{"status": "accepted", "correlation_id": "..."}`` on success. +* 401 (no body) when the signature is missing or invalid. +* 200 (no body) on a ``GET`` request -- ALB target-group health checks. +* 500 ``{"status": "error", "error": "..."}`` when an internal + dependency (Secrets Manager, SQS) fails. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import logging +import os +import uuid +from typing import Any + +import boto3 + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +_SECRET_CACHE: str | None = None +"""Module-global HMAC secret cache. + +The Lambda runtime keeps the module loaded across warm invocations, +so a single ``GetSecretValue`` call amortises across every request +that hits the same execution environment. Cleared by replacing the +Lambda alias / version when the secret rotates. +""" + +_DEFAULT_SIGNATURE_HEADER = "x-jira-signature" +_DEFAULT_CORRELATION_HEADER = "x-correlation-id" +_MAX_LOG_BODY_CHARS = 2048 +_FALLBACK_SIGNATURE_HEADERS = ("x-hub-signature", "x-jira-signature") + + +def _truncate_for_log(value: str, limit: int = _MAX_LOG_BODY_CHARS) -> str: + del limit + return value + + +def _get_secret() -> str: + """Fetch the HMAC shared secret from Secrets Manager. + + Cached on the module global ``_SECRET_CACHE`` for the lifetime of + the warm Lambda execution environment. + """ + + global _SECRET_CACHE + if _SECRET_CACHE is not None: + return _SECRET_CACHE + secret_id = os.environ["WEBHOOK_HMAC_SECRET_ID"] + client = boto3.client("secretsmanager") + response = client.get_secret_value(SecretId=secret_id) + secret_string = response.get("SecretString") + if not isinstance(secret_string, str) or not secret_string: + raise RuntimeError( + "WEBHOOK_HMAC_SECRET_ID does not resolve to a SecretString; " + "binary secrets are not supported." + ) + _SECRET_CACHE = secret_string + return secret_string + + +def _normalize_headers(raw: Any) -> dict[str, str]: + """Lowercase-fold the inbound header dict. + + ALB / Lambda passes headers as a plain ``{name: value}`` dict on + the v1 event shape; fold the keys so we can match + ``x-jira-signature`` / ``X-Jira-Signature`` / ``X-JIRA-SIGNATURE`` + interchangeably. + """ + + if not isinstance(raw, dict): + return {} + return {str(k).lower(): str(v) for k, v in raw.items() if v is not None} + + +def _verify_signature(*, raw_body: bytes, signature_header_value: str | None, secret: str) -> bool: + """Constant-time compare ``signature_header_value`` against HMAC-SHA256(``raw_body``).""" + + if not signature_header_value: + return False + expected = hmac.new(secret.encode("utf-8"), raw_body, hashlib.sha256).hexdigest() + candidate = signature_header_value.strip() + if candidate.lower().startswith("sha256="): + candidate = candidate[len("sha256=") :] + return hmac.compare_digest(expected, candidate) + + +def _pick_signature_header_value( + *, headers: dict[str, str], primary_header_name: str +) -> str | None: + """Resolve signature value from primary header, then known fallbacks.""" + + candidate = headers.get(primary_header_name) + if candidate is not None: + return candidate + + for header_name in _FALLBACK_SIGNATURE_HEADERS: + if header_name == primary_header_name: + continue + candidate = headers.get(header_name) + if candidate is not None: + return candidate + return None + + +def _signature_header_candidates(primary_header_name: str) -> tuple[str, ...]: + """Return signature header names checked for this request.""" + + ordered: list[str] = [primary_header_name] + for header_name in _FALLBACK_SIGNATURE_HEADERS: + if header_name not in ordered: + ordered.append(header_name) + return tuple(ordered) + + +def _ok_health_response() -> dict[str, Any]: + return { + "statusCode": 200, + "statusDescription": "200 OK", + "isBase64Encoded": False, + "headers": {"content-type": "text/plain"}, + "body": "", + } + + +def _unauthorized_response() -> dict[str, Any]: + return { + "statusCode": 401, + "statusDescription": "401 Unauthorized", + "isBase64Encoded": False, + "headers": {"content-type": "text/plain"}, + "body": "", + } + + +def _accepted_response(correlation_id: str) -> dict[str, Any]: + return { + "statusCode": 202, + "statusDescription": "202 Accepted", + "isBase64Encoded": False, + "headers": {"content-type": "application/json"}, + "body": json.dumps({"status": "accepted", "correlation_id": correlation_id}), + } + + +def _error_response(message: str) -> dict[str, Any]: + return { + "statusCode": 500, + "statusDescription": "500 Internal Server Error", + "isBase64Encoded": False, + "headers": {"content-type": "application/json"}, + "body": json.dumps({"status": "error", "error": message}), + } + + +def lambda_handler(event: dict[str, Any], context: Any) -> dict[str, Any]: + """ALB-target Lambda entrypoint. + + Routes: + + * ``GET`` -> 200 health-check shortcut (no SQS, no Secrets Manager). + * ``POST`` (default) -> verify HMAC, publish to SQS, return 202. + * Any other verdict -> 401 (no body). + """ + + del context + + method = (event.get("httpMethod") or "").upper() + if method == "GET": + return _ok_health_response() + + headers = _normalize_headers(event.get("headers")) + signature_header_name = os.environ.get( + "SIGNATURE_HEADER", _DEFAULT_SIGNATURE_HEADER + ).lower() + signature_header_value = _pick_signature_header_value( + headers=headers, + primary_header_name=signature_header_name, + ) + signature_candidates = _signature_header_candidates(signature_header_name) + + raw_body = event.get("body") or "" + if event.get("isBase64Encoded"): + import base64 + + raw_body_bytes = base64.b64decode(raw_body) + else: + raw_body_bytes = raw_body.encode("utf-8") if isinstance(raw_body, str) else b"" + + raw_body_text = raw_body_bytes.decode("utf-8", errors="replace") + logger.info( + "webhook_validator.received method=%s signature_present=%s body_bytes=%d body=%s", + method or "UNKNOWN", + signature_header_value is not None, + len(raw_body_bytes), + _truncate_for_log(raw_body_text), + ) + + try: + secret = _get_secret() + except Exception as exc: + logger.error("webhook_validator.secret_error: %s", exc) + return _error_response(f"secret_resolution_failed: {type(exc).__name__}") + + if not _verify_signature( + raw_body=raw_body_bytes, + signature_header_value=signature_header_value, + secret=secret, + ): + signature_presence = ",".join( + f"{name}:{name in headers}" for name in signature_candidates + ) + logger.warning( + "webhook_validator.signature_invalid signature_present=%s body_bytes=%d " + "signature_candidates=%s present_signature_headers=%s header_count=%d", + signature_header_value is not None, + len(raw_body_bytes), + "|".join(signature_candidates), + signature_presence, + len(headers), + ) + return _unauthorized_response() + + try: + webhook_payload = json.loads(raw_body_bytes.decode("utf-8")) + except (ValueError, UnicodeDecodeError) as exc: + logger.warning("webhook_validator.payload_invalid: %s", exc) + return _unauthorized_response() + + correlation_id = ( + headers.get(_DEFAULT_CORRELATION_HEADER) or str(uuid.uuid4()) + ).strip() + if not correlation_id: + correlation_id = str(uuid.uuid4()) + + work_queue_url = os.environ["WEBHOOK_WORK_QUEUE_URL"] + sqs_client = boto3.client("sqs") + envelope = {"correlation_id": correlation_id, "webhook": webhook_payload} + envelope_json = json.dumps(envelope, separators=(",", ":")) + logger.info( + "webhook_validator.sqs_publish_attempt correlation_id=%s queue_url=%s envelope=%s", + correlation_id, + work_queue_url, + _truncate_for_log(envelope_json), + ) + try: + sqs_client.send_message( + QueueUrl=work_queue_url, + MessageBody=envelope_json, + ) + except Exception as exc: + logger.error( + "webhook_validator.sqs_publish_failed correlation_id=%s queue_url=%s " + "error_type=%s error=%s envelope=%s", + correlation_id, + work_queue_url, + type(exc).__name__, + str(exc), + _truncate_for_log(envelope_json), + ) + return _error_response(f"sqs_publish_failed: {type(exc).__name__}") + logger.info( + "webhook_validator.sqs_publish_ok correlation_id=%s queue_url=%s", + correlation_id, + work_queue_url, + ) + + logger.info( + "webhook_validator.accepted correlation_id=%s body_bytes=%d", + correlation_id, + len(raw_body_bytes), + ) + return _accepted_response(correlation_id) diff --git a/lambda/webhook_validator/requirements.txt b/lambda/webhook_validator/requirements.txt new file mode 100644 index 0000000..fbbd792 --- /dev/null +++ b/lambda/webhook_validator/requirements.txt @@ -0,0 +1,5 @@ +# Intentionally empty: the AWS Lambda Python runtime ships with boto3 +# (and therefore botocore), which is the only third-party dependency +# the handler needs. Keeping this file lets ``terraform``'s +# ``archive_file`` data source produce a deterministic zip tree +# without requiring a build step. diff --git a/lambda/webhook_validator/tests/__init__.py b/lambda/webhook_validator/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lambda/webhook_validator/tests/test_handler.py b/lambda/webhook_validator/tests/test_handler.py new file mode 100644 index 0000000..58b6567 --- /dev/null +++ b/lambda/webhook_validator/tests/test_handler.py @@ -0,0 +1,319 @@ +"""Unit tests for :mod:`lambda.webhook_validator.handler`. + +The handler is exercised in-process against fake ``boto3`` clients +(``Secrets Manager`` + ``SQS``). Patches are scoped to one test so +the module-global secret cache does not leak between cases. +""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import importlib +import json +import sys +from pathlib import Path +from typing import Any + +import pytest + +# Insert the lambda root so ``import handler`` resolves to the module +# under test without packaging the lambda as an installable. The +# handler is intentionally located at ``lambda/webhook_validator/handler.py`` +# so the Terraform ``archive_file`` data source can zip the directory +# tree verbatim. +LAMBDA_ROOT = Path(__file__).resolve().parents[1] +if str(LAMBDA_ROOT) not in sys.path: + sys.path.insert(0, str(LAMBDA_ROOT)) + + +@pytest.fixture +def handler_module(monkeypatch: pytest.MonkeyPatch) -> Any: + """Import a fresh handler module per test so caches don't leak. + + Patches ``boto3.client`` BEFORE module import so the + ``_get_secret`` warm-cache helper can be exercised end-to-end. + """ + + fake_secrets_client = _FakeSecretsClient(secret_value="hmac-shared-secret") + fake_sqs_client = _FakeSqsClient() + + def _factory(service: str, *args: Any, **kwargs: Any) -> Any: + del args, kwargs + if service == "secretsmanager": + return fake_secrets_client + if service == "sqs": + return fake_sqs_client + raise AssertionError(f"unexpected boto3.client({service!r})") + + monkeypatch.setattr("boto3.client", _factory) + monkeypatch.setenv("WEBHOOK_HMAC_SECRET_ID", "default/atlassian/webhook-hmac") + monkeypatch.setenv( + "WEBHOOK_WORK_QUEUE_URL", + "https://sqs.us-east-1.amazonaws.com/000000000000/agent-work", + ) + monkeypatch.delenv("SIGNATURE_HEADER", raising=False) + + import handler # type: ignore[import-not-found] + + importlib.reload(handler) + handler._FAKE_SECRETS = fake_secrets_client # type: ignore[attr-defined] + handler._FAKE_SQS = fake_sqs_client # type: ignore[attr-defined] + return handler + + +class _FakeSecretsClient: + def __init__(self, secret_value: str) -> None: + self.secret_value = secret_value + self.calls: list[dict[str, Any]] = [] + + def get_secret_value(self, **kwargs: Any) -> dict[str, Any]: + self.calls.append(kwargs) + return {"SecretString": self.secret_value} + + +class _FakeSqsClient: + def __init__(self) -> None: + self.send_calls: list[dict[str, Any]] = [] + self.send_raises: BaseException | None = None + + def send_message(self, **kwargs: Any) -> dict[str, Any]: + if self.send_raises is not None: + raise self.send_raises + self.send_calls.append(kwargs) + return {"MessageId": "fake-message-id-1"} + + +def _signed_event( + *, + body: str, + secret: str = "hmac-shared-secret", + signature_header: str = "x-jira-signature", + correlation_header: str | None = None, + is_base64: bool = False, +) -> dict[str, Any]: + sig = hmac.new(secret.encode("utf-8"), body.encode("utf-8"), hashlib.sha256).hexdigest() + headers: dict[str, str] = {signature_header: f"sha256={sig}"} + if correlation_header is not None: + headers["x-correlation-id"] = correlation_header + payload_body = body if not is_base64 else base64.b64encode(body.encode("utf-8")).decode() + return { + "httpMethod": "POST", + "headers": headers, + "body": payload_body, + "isBase64Encoded": is_base64, + } + + +def _webhook_body() -> str: + return json.dumps( + { + "webhookEvent": "jira:issue_created", + "issue": {"key": "ABC-1"}, + "timestamp": 1714694400000, + } + ) + + +# --------------------------------------------------------------------------- +# Health-check +# --------------------------------------------------------------------------- + + +def test_get_request_short_circuits_to_health_check(handler_module: Any) -> None: + response = handler_module.lambda_handler({"httpMethod": "GET"}, None) + assert response["statusCode"] == 200 + # Health check must not consult Secrets Manager or SQS. + assert handler_module._FAKE_SECRETS.calls == [] + assert handler_module._FAKE_SQS.send_calls == [] + + +# --------------------------------------------------------------------------- +# HMAC verification + SQS publish +# --------------------------------------------------------------------------- + + +def test_valid_signature_publishes_to_sqs_and_returns_202(handler_module: Any) -> None: + body = _webhook_body() + event = _signed_event(body=body, correlation_header="trace-7") + response = handler_module.lambda_handler(event, None) + + assert response["statusCode"] == 202 + response_body = json.loads(response["body"]) + assert response_body["status"] == "accepted" + assert response_body["correlation_id"] == "trace-7" + + assert len(handler_module._FAKE_SQS.send_calls) == 1 + call = handler_module._FAKE_SQS.send_calls[0] + assert call["QueueUrl"] == ( + "https://sqs.us-east-1.amazonaws.com/000000000000/agent-work" + ) + envelope = json.loads(call["MessageBody"]) + assert envelope["correlation_id"] == "trace-7" + assert envelope["webhook"]["issue"]["key"] == "ABC-1" + + +def test_valid_signature_without_correlation_header_mints_uuid(handler_module: Any) -> None: + body = _webhook_body() + event = _signed_event(body=body) + response = handler_module.lambda_handler(event, None) + + assert response["statusCode"] == 202 + response_body = json.loads(response["body"]) + correlation_id = response_body["correlation_id"] + assert isinstance(correlation_id, str) + assert len(correlation_id) >= 32 # uuid4 string is 36 chars + envelope = json.loads(handler_module._FAKE_SQS.send_calls[0]["MessageBody"]) + assert envelope["correlation_id"] == correlation_id + + +def test_signature_header_is_case_insensitive(handler_module: Any) -> None: + body = _webhook_body() + sig = hmac.new( + b"hmac-shared-secret", body.encode("utf-8"), hashlib.sha256 + ).hexdigest() + event = { + "httpMethod": "POST", + "headers": {"X-Jira-Signature": f"sha256={sig}"}, + "body": body, + } + response = handler_module.lambda_handler(event, None) + assert response["statusCode"] == 202 + + +def test_signature_header_without_sha256_prefix_is_accepted(handler_module: Any) -> None: + body = _webhook_body() + sig = hmac.new( + b"hmac-shared-secret", body.encode("utf-8"), hashlib.sha256 + ).hexdigest() + event = { + "httpMethod": "POST", + "headers": {"x-jira-signature": sig}, + "body": body, + } + response = handler_module.lambda_handler(event, None) + assert response["statusCode"] == 202 + + +def test_x_hub_signature_header_is_accepted_by_default(handler_module: Any) -> None: + body = _webhook_body() + sig = hmac.new( + b"hmac-shared-secret", body.encode("utf-8"), hashlib.sha256 + ).hexdigest() + event = { + "httpMethod": "POST", + "headers": {"x-hub-signature": f"sha256={sig}"}, + "body": body, + } + response = handler_module.lambda_handler(event, None) + assert response["statusCode"] == 202 + + +def test_signature_header_override_via_env( + handler_module: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + """``SIGNATURE_HEADER`` env override changes which header is consulted.""" + + monkeypatch.setenv("SIGNATURE_HEADER", "x-atlassian-signature") + body = _webhook_body() + event = _signed_event(body=body, signature_header="x-atlassian-signature") + response = handler_module.lambda_handler(event, None) + assert response["statusCode"] == 202 + + +def test_base64_encoded_body_is_decoded_before_verification( + handler_module: Any, +) -> None: + body = _webhook_body() + event = _signed_event(body=body, is_base64=True) + response = handler_module.lambda_handler(event, None) + assert response["statusCode"] == 202 + + +# --------------------------------------------------------------------------- +# Failure cases +# --------------------------------------------------------------------------- + + +def test_missing_signature_header_returns_401(handler_module: Any) -> None: + body = _webhook_body() + response = handler_module.lambda_handler( + {"httpMethod": "POST", "headers": {}, "body": body}, None + ) + assert response["statusCode"] == 401 + assert response["body"] == "" + assert handler_module._FAKE_SQS.send_calls == [] + + +def test_invalid_signature_returns_401(handler_module: Any) -> None: + body = _webhook_body() + event = { + "httpMethod": "POST", + "headers": {"x-jira-signature": "sha256=deadbeef"}, + "body": body, + } + response = handler_module.lambda_handler(event, None) + assert response["statusCode"] == 401 + assert handler_module._FAKE_SQS.send_calls == [] + + +def test_signature_correct_but_body_not_json_returns_401(handler_module: Any) -> None: + """A valid signature over non-JSON garbage still rejects -- we cannot + publish a structurally invalid envelope to the work queue.""" + + body = "not-json-at-all" + event = _signed_event(body=body) + response = handler_module.lambda_handler(event, None) + assert response["statusCode"] == 401 + assert handler_module._FAKE_SQS.send_calls == [] + + +def test_sqs_publish_failure_surfaces_500(handler_module: Any) -> None: + handler_module._FAKE_SQS.send_raises = RuntimeError("sqs unavailable") + body = _webhook_body() + event = _signed_event(body=body) + response = handler_module.lambda_handler(event, None) + assert response["statusCode"] == 500 + response_body = json.loads(response["body"]) + assert "sqs_publish_failed" in response_body["error"] + + +def test_secrets_manager_failure_surfaces_500( + handler_module: Any, +) -> None: + """A Secrets Manager outage on first call is reported as 500.""" + + # Reset the cached secret so the next handler call retries the lookup. + handler_module._SECRET_CACHE = None + handler_module._FAKE_SECRETS.calls = [] + + def _boom(**kwargs: Any) -> dict[str, Any]: + raise RuntimeError("secrets manager 503") + + handler_module._FAKE_SECRETS.get_secret_value = _boom # type: ignore[method-assign] + body = _webhook_body() + event = _signed_event(body=body) + response = handler_module.lambda_handler(event, None) + assert response["statusCode"] == 500 + response_body = json.loads(response["body"]) + assert "secret_resolution_failed" in response_body["error"] + + +# --------------------------------------------------------------------------- +# Secret caching +# --------------------------------------------------------------------------- + + +def test_warm_invocations_reuse_cached_secret(handler_module: Any) -> None: + body = _webhook_body() + handler_module.lambda_handler(_signed_event(body=body), None) + handler_module.lambda_handler(_signed_event(body=body), None) + handler_module.lambda_handler(_signed_event(body=body), None) + # Secret is fetched exactly once across three warm invocations. + assert len(handler_module._FAKE_SECRETS.calls) == 1 + + +def test_truncate_for_log_keeps_full_payload(handler_module: Any) -> None: + long_text = "x" * (handler_module._MAX_LOG_BODY_CHARS + 25) + assert handler_module._truncate_for_log(long_text) == long_text diff --git a/langgraph.json b/langgraph.json new file mode 100644 index 0000000..70b018e --- /dev/null +++ b/langgraph.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://langgra.ph/schema.json", + "dependencies": ["."], + "graphs": { + "jira_readiness_agent": "./src/agent/composition/studio.py:build_studio_graph" + }, + "env": ".env.local", + "python_version": "3.13" +} diff --git a/mcp/.dockerignore b/mcp/.dockerignore index ed5d077..d35fcfd 100644 --- a/mcp/.dockerignore +++ b/mcp/.dockerignore @@ -1,11 +1,18 @@ # ============================================================================= # Build-context excludes for `mcp/Dockerfile`. # -# The build context is just the JAR — every other file in `mcp/` is either -# host-only credential material (`.env`), an inspector/client config that -# must NEVER ship inside the image (`config.json`), or developer -# documentation. Keeping the context tight minimises image size and -# eliminates the risk of leaking developer-only secrets. +# The build context is `mcp/`. The mcp-internal Spring Boot fat-JAR is no +# longer tracked in this repository; `scripts/build_mcp_image.sh` stages +# it into `mcp/.build/mcp-internal.jar` for the duration of `docker build` +# and removes the staging directory on exit. `.build/` is intentionally +# NOT excluded here so that staged JAR is visible to the COPY in the +# Dockerfile. +# +# Everything else in `mcp/` is either host-only credential material +# (`.env`), an inspector/client config that must NEVER ship inside the +# image (`config.json`), or developer documentation. Keeping the +# context tight minimises image size and eliminates the risk of +# leaking developer-only secrets. # ============================================================================= .env diff --git a/mcp/Dockerfile b/mcp/Dockerfile index f6ae20b..8c8cabb 100644 --- a/mcp/Dockerfile +++ b/mcp/Dockerfile @@ -1,14 +1,22 @@ # ============================================================================= # mcp-internal HTTP-streamable MCP server image. # -# Wraps the Spring Boot fat-JAR `mcp-internal-*.jar` (main class +# Wraps the Spring Boot fat-JAR `mcp-internal.jar` (main class # `ai.qodo.mcp.InternalMcpApplication`, Spring Boot 3.5.6, JDK 21) in a # minimal Java 21 runtime so the local Kubernetes profile (deploy/local/) -# has a real, runnable image for the M3 / M5.5 milestone. +# has a real, runnable image. # # Build: # scripts/build_mcp_image.sh -# (resolves mcp/mcp-internal-*.jar -> mcp-internal/server:local) +# (resolves an *external* mcp-internal-*.jar via --jar / +# MCP_INTERNAL_JAR / ${MCP_INTERNAL_REPO}/build/libs, stages it +# into mcp/.build/mcp-internal.jar, then docker build -> tag +# mcp-internal/server:local) +# +# The JAR is no longer tracked inside this repository; the build +# script materialises `mcp/.build/mcp-internal.jar` for the duration +# of the docker build and removes it on exit. See mcp/README.md +# "External JAR contract" for the resolution order. # # Notes: # * `eclipse-temurin:21-jre-alpine` is the smallest Adoptium JRE image @@ -26,6 +34,22 @@ # * `MCP_DEFAULT_ROOT_PATH=/workspace` aligns with the emptyDir / host- # mount overlay configured by deploy/local/mcp-internal.yaml; the # agent advertises the same root over the MCP `roots/list` capability. +# * Git tooling is installed alongside the JRE so the JAR's Git MCP +# surface (`git clone`, `git push`) works out of the box. The +# integration user's SSH private key is NOT baked in -- it is +# injected at runtime via the `mcp-git-ssh-key` Secret as the env +# var `GIT_SSH_PRIVATE_KEY` (base64-encoded OpenSSH private key). +# `entrypoint.sh` decodes that base64 blob and writes the resulting +# PEM to `/app/.ssh/aws_ecdsa` (mode 0600) BEFORE exec'ing the JVM, +# then drops the env var so the JAR never sees the raw secret. The +# entrypoint also drops a relative symlink under JGit/OpenSSH's +# hardcoded default identity name (id_ecdsa / id_ed25519 / id_rsa, +# algorithm-detected via `ssh-keygen -y`) because the JAR's +# SshdSessionFactoryBuilder never calls setDefaultIdentities() -- +# without the symlink MINA SSHD's identity scanner finds zero keys +# and aborts auth with "publickey: no keys to try". The `/app/.ssh` +# directory and SCM `known_hosts` are pre-created here so the +# entrypoint has a writable, lock-tight (0700) drop-zone. # ============================================================================= # hadolint ignore=DL3007 @@ -34,28 +58,89 @@ FROM eclipse-temurin:21-jre-alpine LABEL org.opencontainers.image.title="mcp-internal" LABEL org.opencontainers.image.description="HTTP-streamable MCP server (Jira + Git + Confluence)" LABEL org.opencontainers.image.source="https://github.com/qodo-ai/aws-agent-core" -LABEL org.opencontainers.image.licenses="Proprietary" +LABEL org.opencontainers.image.licenses="MIT" LABEL org.opencontainers.image.vendor="qodo.ai" -# Curl is required for the kubelet HTTP probes (we hit Spring Actuator -# directly from kubelet, but `wget`/`curl` is also handy for diagnostics). +# Runtime tooling required by the JAR's Git MCP code path: +# * `git` -- the JAR shells out to git for clone / push. +# * `openssh-client` -- supplies `ssh` (used as GIT_SSH_COMMAND) and +# `ssh-keyscan` (used below to seed known_hosts). +# * `bash` -- a few git porcelain helpers expect a real bash. +# * `procps` -- `ps` for in-Pod diagnostics (`kubectl exec ...`). +# * `sudo` -- placeholder so future ops debugging can run a +# privileged probe without rebuilding the image. +# * `curl` -- kubelet HTTP probes hit Spring Actuator directly +# from kubelet, but `curl` is the in-container +# parity tool for the HEALTHCHECK below. # hadolint ignore=DL3018 -RUN apk add --no-cache curl \ +RUN apk add --no-cache \ + bash \ + curl \ + git \ + openssh-client \ + procps \ + sudo \ && addgroup -S -g 10001 mcp \ && adduser -S -u 10001 -G mcp -h /app mcp \ - && mkdir -p /workspace \ - && chown -R mcp:mcp /workspace + && mkdir -p /workspace /app/.ssh \ + && chown -R mcp:mcp /workspace /app \ + && chmod 700 /app/.ssh \ + # Pre-seed known_hosts for the SCM hosts the integration user pushes + # to. Without this, the JAR's first `git clone` over SSH would either + # prompt interactively (no TTY -> hangs) or fail StrictHostKeyChecking. + # `ssh-keyscan` writes to stdout; we redirect into the file with the + # right ownership, then lock it down to 0644 (world-readable is fine, + # this is public host-key fingerprint data). Note that the matching + # private key (`/app/.ssh/aws_ecdsa`) is materialised at runtime by + # `/app/entrypoint.sh` from the `GIT_SSH_PRIVATE_KEY` env var; it is + # never present in the image filesystem. + && ssh-keyscan -t rsa,ecdsa,ed25519 \ + github.com bitbucket.org gitlab.com \ + > /app/.ssh/known_hosts 2>/dev/null \ + && chown mcp:mcp /app/.ssh/known_hosts \ + && chmod 644 /app/.ssh/known_hosts WORKDIR /app # The build context is `mcp/` (see scripts/build_mcp_image.sh); the JAR -# is resolved by glob so version bumps require only an LFS pull, not a -# Dockerfile edit. Ownership is set up-front so the container can run +# is staged at the deterministic path `.build/mcp-internal.jar` by the +# build script, which resolves the source file from outside this repo +# (--jar / MCP_INTERNAL_JAR / ${MCP_INTERNAL_REPO}/build/libs). Using +# a fixed in-context name means version bumps require zero Dockerfile +# edits. Ownership is set up-front so the container can run # read-only-root-filesystem in the future without surprise EACCES. -COPY --chown=mcp:mcp mcp-internal-*.jar /app/mcp-internal.jar +COPY --chown=mcp:mcp .build/mcp-internal.jar /app/mcp-internal.jar + +# Container entrypoint: decodes the base64 GIT_SSH_PRIVATE_KEY env var +# into /app/.ssh/aws_ecdsa (0600) then exec's the JVM. Copied with +# `--chmod=0755` (BuildKit) so the file is executable for the mcp user +# without an extra RUN layer; ownership matches the runtime UID so a +# future `read-only-root-filesystem` securityContext does not break the +# `chmod 600` the script does on the key file at startup. +COPY --chown=mcp:mcp --chmod=0755 entrypoint.sh /app/entrypoint.sh USER mcp:mcp +# Bake the agent's git identity into the global config (writes +# `/app/.gitconfig` because `HOME=/app`). Runs AFTER `USER mcp:mcp` so +# the file ownership matches the runtime UID. +# * `push.autoSetupRemote=true` spares the JAR from having to pass +# `-u ` on every first push from a freshly-created +# branch. +# * `url.git@github.com:.insteadOf https://github.com/` transparently +# rewrites HTTPS GitHub URLs to SSH so the JAR's clones authenticate +# with `/app/.ssh/aws_ecdsa` even when an MCP caller passes the +# project's https remote URL. +# NOTE: the host-key seed for github.com / bitbucket.org / gitlab.com +# already lives at `/app/.ssh/known_hosts` (baked in the root RUN above +# while we still had write access to /app/.ssh as root). Re-running +# `ssh-keyscan` here would fail anyway because the mcp user can read +# but not write `/app/.ssh` post-`chmod 700`. +RUN git config --global push.autoSetupRemote true \ + && git config --global user.email "agent@qodo.ai" \ + && git config --global user.name "Agent User" \ + && git config --global url."git@github.com:".insteadOf "https://github.com/" + ENV SERVER_PORT=8081 \ MCP_DEFAULT_ROOT_PATH=/workspace \ JAVA_TOOL_OPTIONS="-XX:MaxRAMPercentage=75 -XX:+ExitOnOutOfMemoryError" @@ -69,4 +154,12 @@ EXPOSE 8081 HEALTHCHECK --interval=15s --timeout=3s --start-period=20s --retries=6 \ CMD curl -fsS "http://127.0.0.1:${SERVER_PORT}/actuator/health/readiness" || exit 1 -ENTRYPOINT ["java", "-jar", "/app/mcp-internal.jar"] +# ENTRYPOINT runs `entrypoint.sh`, which materialises the SSH private +# key and then `exec`s the CMD below -- so the JVM still becomes PID 1 +# and Kubernetes / Docker stop signals reach Spring Boot directly. +# Splitting CMD out (rather than baking `java -jar ...` into the script) +# keeps the launch flags overridable from the Pod spec without a +# rebuild (e.g. for `--debug`, profiling agents, or alternative main +# classes during local triage). +ENTRYPOINT ["/app/entrypoint.sh"] +CMD ["java", "-jar", "/app/mcp-internal.jar"] diff --git a/mcp/README.md b/mcp/README.md index 774aac0..88bafb7 100644 --- a/mcp/README.md +++ b/mcp/README.md @@ -6,11 +6,13 @@ profile (`deploy/local/`). | File | Role | |------|------| -| [`mcp-internal-1.0.4.jar`](./mcp-internal-1.0.4.jar) | Spring Boot 3.5.6 fat-JAR (main class `ai.qodo.mcp.InternalMcpApplication`, JDK 21). The build context for the Dockerfile. | -| [`Dockerfile`](./Dockerfile) | `eclipse-temurin:21-jre-alpine`-based image that wraps the JAR. Tagged `mcp-internal/server:local` by the build script. | -| [`.dockerignore`](./.dockerignore) | Excludes `.env`, `config.json`, and docs from the build context. | +| [`Dockerfile`](./Dockerfile) | `eclipse-temurin:21-jre-alpine`-based image that wraps the Spring Boot fat-JAR. Tagged `mcp-internal/server:local` by the build script. | +| [`.dockerignore`](./.dockerignore) | Excludes `.env`, `config.json`, and docs from the build context (keeps `.build/` so the staged JAR is visible to `COPY`). | +| [`entrypoint.sh`](./entrypoint.sh) | Container entrypoint — materialises the integration user's SSH private key from the `GIT_SSH_PRIVATE_KEY` env var before `exec`'ing the JVM. | | [`config.json`](./config.json) | Inspector-style MCP client manifest. Informational only — never copied into the image. | +| [`run.sh`](./run.sh) | Convenience launcher for the MCP Inspector UI against a running local server. | | `.env` | Developer-only credentials (gitignored — see top-level [`.gitignore`](../.gitignore)). | +| `.build/` | Created on demand by [`../scripts/build_mcp_image.sh`](../scripts/build_mcp_image.sh) to stage the resolved JAR; gitignored and removed on build exit. | The matching companion script lives at [`../scripts/build_mcp_image.sh`](../scripts/build_mcp_image.sh). @@ -19,28 +21,60 @@ The matching companion script lives at [`../scripts/build_mcp_image.sh`](../scri > and the env-var cheat sheet at > [`../.env.local.example`](../.env.local.example). -## 1. Prerequisites +## 1. External JAR contract -The JAR is tracked through **Git LFS** (see [`../.gitattributes`](../.gitattributes)). -Without LFS materialised, the file you have on disk is a ~130-byte -pointer rather than the real ~62 MB binary. +The Spring Boot fat-JAR (`mcp-internal-.jar`, ~62 MB, main +class `ai.qodo.mcp.InternalMcpApplication`, JDK 21) lives in the +**separate** `mcp-internal` Gradle project — it is **not** tracked in +this repository, and `mcp/*.jar` is gitignored to prevent accidental +re-introduction. The build script +([`../scripts/build_mcp_image.sh`](../scripts/build_mcp_image.sh)) +resolves the JAR from outside the repo and stages it into +`mcp/.build/mcp-internal.jar` for the duration of `docker build`, +removing the staging directory on exit. + +Resolution order (first hit wins): + +| Order | Source | Notes | +|-------|--------|-------| +| 1 | `--jar /abs/path/to/mcp-internal-X.Y.Z.jar` | Explicit CLI override on `scripts/build_mcp_image.sh`. | +| 2 | `MCP_INTERNAL_JAR=/abs/path/...` env var | Useful in CI / wrapper scripts. | +| 3 | `${MCP_INTERNAL_REPO:-$HOME/code/github/mcp-internal}/build/libs/mcp-internal-*.jar` | Newest match wins; the `-plain.jar` Gradle also emits is excluded. | + +The build script fails fast with an actionable error message if no JAR +is found, telling the operator how to build one (`./gradlew bootJar` +in the upstream repo) or how to point the resolver at a different +location. + +### Building the JAR ```bash -brew install git-lfs # macOS; on Linux: apt-get install git-lfs -git lfs install # one-time, per host -git lfs pull # in this clone, after the initial `git clone` +# One-time: clone the upstream Gradle project alongside this repo. +git clone "${HOME}/code/github/mcp-internal" + +# Whenever the upstream JAR changes: +cd "${HOME}/code/github/mcp-internal" +./gradlew bootJar +# -> build/libs/mcp-internal-.jar (auto-discovered by build_mcp_image.sh) ``` -The build script ([`../scripts/build_mcp_image.sh`](../scripts/build_mcp_image.sh)) -fails fast with a helpful message if it detects an LFS pointer instead -of the real JAR. +### CI + +`.github/workflows/smoke.yml` downloads the JAR from a GitHub Release +of the upstream repo (configurable via the `MCP_INTERNAL_REPO_SLUG` / +`MCP_INTERNAL_VERSION` repository variables and the +`MCP_INTERNAL_TOKEN` repository secret), then exports +`MCP_INTERNAL_JAR=...` for `scripts/build_mcp_image.sh` to consume. ## 2. Build the image ```bash make build-mcp -# or, equivalently: +# or, equivalently (any of the following): scripts/build_mcp_image.sh +scripts/build_mcp_image.sh --jar /abs/path/to/mcp-internal-1.0.4.jar +MCP_INTERNAL_JAR=/abs/path/to/mcp-internal-1.0.4.jar scripts/build_mcp_image.sh +MCP_INTERNAL_REPO=/abs/path/to/mcp-internal scripts/build_mcp_image.sh ``` Produces image `mcp-internal/server:local`. Verify with: @@ -147,22 +181,37 @@ on the `McpCallObserver`; from `mcp-internal`'s perspective it is an ordinary tool call with no special handling required. **Cross-repo follow-up.** The Spring Boot project that produces -[`mcp-internal-1.0.4.jar`](./mcp-internal-1.0.4.jar) lives in a -separate repository. When the agent first opts into the soft-noop -path in production, that repo must register the +`mcp-internal-.jar` lives in a **separate repository** (see +§1 "External JAR contract" above). When the agent first opts into the +soft-noop path in production, that repo must register the `jira_set_issue_property` MCP tool descriptor (via the same -`@McpTool` annotation it uses for the other Jira tools) and ship a -new JAR. The `agent.contracts.soft_noop.SoftNoopWriter` Protocol +`@McpTool` annotation it uses for the other Jira tools), publish a +new release, and bump the `MCP_INTERNAL_VERSION` pin in +`.github/workflows/smoke.yml`. The `agent.contracts.soft_noop.SoftNoopWriter` Protocol (see [`../src/agent/contracts/soft_noop.py`](../src/agent/contracts/soft_noop.py)) and its reference adapter [`../src/agent/infrastructure/jira/soft_noop_writer.py`](../src/agent/infrastructure/jira/soft_noop_writer.py) encode the Python-side contract. -Until the new JAR is shipped, leave `soft_noop_writer=None` in the -composition root (the default for `build_default_dependencies`); the -`maybe_apply_soft_noop` node is then a pass-through and the existing -invariant violation surfaces as before. +Production composition roots wire a real `JiraIssuePropertySoftNoopWriter`; +`build_default_dependencies` falls back to an in-memory recording writer +that satisfies the invariant without contacting Jira. + +## 4.2 Jira current-user identity tool (recursion drift guard) + +The agent's recursion guard is configured with `AGENT_JIRA_ACCOUNT_ID` and +`AGENT_JIRA_EMAIL`, which must match the Jira integration user used by +`mcp-internal`. The current `mcp-internal` JAR does not advertise a stable +current-user tool in `tools/list`, so the agent cannot perform a startup +self-test against live Jira identity yet. + +When the upstream Spring Boot service adds a tool such as +`jira_get_current_user`, the agent-side smoke should call it through +`McpClient.call_tool(...)` and compare the returned `accountId` / `emailAddress` +against `AGENT_JIRA_ACCOUNT_ID` / `AGENT_JIRA_EMAIL`. Until then, use the +runbook's recursion-drift checklist to verify the identity pair manually after +credential rotation. ## 5. Production digest pinning diff --git a/mcp/config.json b/mcp/config.json index db565a2..de88b7d 100755 --- a/mcp/config.json +++ b/mcp/config.json @@ -2,7 +2,7 @@ "mcpServers": { "git-mcp": { "type": "streamable-http", - "url": "http://localhost:8080/mcp" + "url": "http://localhost:8330/mcp" } } } diff --git a/mcp/entrypoint.sh b/mcp/entrypoint.sh new file mode 100644 index 0000000..465a7ec --- /dev/null +++ b/mcp/entrypoint.sh @@ -0,0 +1,128 @@ +#!/bin/sh +# ============================================================================= +# mcp-internal container entrypoint. +# +# Single responsibility: materialise the integration user's SSH private key +# from the `GIT_SSH_PRIVATE_KEY` env var (a base64-encoded OpenSSH private +# key string) onto disk at `MCP_GIT_SSH_DEFAULT_KEY` (default +# `/app/.ssh/aws_ecdsa`) before exec'ing the JVM. +# +# Why base64 transport: an OpenSSH private key is multi-line PEM +# (`-----BEGIN OPENSSH PRIVATE KEY-----`, 70-char body lines, +# `-----END OPENSSH PRIVATE KEY-----`, trailing newline). Smuggling a +# multi-line value through Kubernetes Secret YAML, `kubectl create secret +# --from-literal`, Docker `--env`, and the container's own `environ(7)` +# is fragile -- newlines get collapsed, escapes get re-interpreted, etc. +# Wrapping the key as a single base64 blob means every transport hop is a +# pure byte-for-byte string copy; the base64 -> bytes round-trip here +# reproduces the PEM exactly, including the trailing newline `ssh(1)` +# requires. +# +# After writing the key we `unset GIT_SSH_PRIVATE_KEY` so the JAR (and +# anything it shells out to) cannot see the raw secret in its +# environment -- the on-disk file at mode 0600 is the single source of +# truth from this point on. +# +# This script must `exec "$@"` so the JVM is PID 1 and Kubernetes +# SIGTERM / Docker stop signals reach Spring Boot directly (no shell +# in the middle eating the signal). +# ============================================================================= + +set -eu + +KEY_PATH="${MCP_GIT_SSH_DEFAULT_KEY:-/app/.ssh/aws_ecdsa}" + +if [ -n "${GIT_SSH_PRIVATE_KEY:-}" ]; then + KEY_DIR="$(dirname "$KEY_PATH")" + + # The image bakes /app/.ssh at 0700 / mcp:mcp, but be defensive in + # case MCP_GIT_SSH_DEFAULT_KEY was overridden to a different path. + mkdir -p "$KEY_DIR" + chmod 700 "$KEY_DIR" + + # umask 077 so the redirect below creates the file at 0600 from the + # very first byte; the explicit `chmod 600` after is a belt-and- + # braces guard if the path already existed at a looser mode. + umask 077 + + # `tr -d ' \r\n'` strips any incidental whitespace (e.g. the line- + # wrapping that `base64 -i` adds by default, or a stray trailing + # newline picked up by a YAML literal block scalar). Standard + # base64 alphabet contains no spaces / CR / LF so removing them is + # always safe and never corrupts the payload. + printf '%s' "$GIT_SSH_PRIVATE_KEY" | tr -d ' \r\n' | base64 -d > "$KEY_PATH" + chmod 600 "$KEY_PATH" + + # Drop the secret from the child process's environment now that it + # lives on disk under 0600. The JAR reads it from + # MCP_GIT_SSH_DEFAULT_KEY (the path), never from the env var. + unset GIT_SSH_PRIVATE_KEY + + echo "[entrypoint] Wrote SSH private key to ${KEY_PATH} (mode 0600)." >&2 + + # ------------------------------------------------------------------ + # JGit / MINA SSHD identity-name workaround. + # + # The mcp-internal JAR builds its SshdSessionFactory like this + # (verified by disassembling GitService.lambda$createSshTransportConfigForUrl$0): + # + # new SshdSessionFactoryBuilder() + # .setPreferredAuthentications("publickey") + # .setSshDirectory(/app/.ssh) + # .setHomeDirectory(/app) + # .build(null); + # + # Note what is missing: any call to `setDefaultIdentities(...)` / + # `setDefaultKeysProvider(...)`. So while `mcp.git.ssh.default-key` + # / `mcp.git.ssh.host-keys` ARE used by the JAR for its up-front + # `=== SSH Configuration Validation ===` existence check, those + # values are NEVER propagated into MINA SSHD itself. With no + # explicit identities, JGit's SshdSessionFactory falls back to + # scanning ~/.ssh/ for OpenSSH's hardcoded default identity + # filenames only: + # + # id_rsa id_dsa id_ecdsa id_ecdsa_sk id_ed25519 id_ed25519_sk + # + # Our key is `aws_ecdsa` (the integration-user convention frozen in + # application.properties' `host-keys` map). It is not in that list, + # so MINA SSHD's KeyPairProvider yields zero identities and the + # auth handshake fails with "publickey: no keys to try" -- even + # though the file is right there, owned by the right user, mode + # 0600, and parses cleanly with `ssh-keygen -y`. + # + # Workaround: drop a relative symlink under the OpenSSH-default + # identity name matching the key's algorithm, so JGit's scanner + # picks it up. We use `ssh-keygen -y` to identify the algorithm so + # this works for any key the integration user might rotate to + # (ECDSA / Ed25519 / RSA / FIDO sk-variants) without code changes. + # The symlink target is the BASENAME (not the absolute path) so + # the link survives rebases of the .ssh directory and a future + # `read-only-root-filesystem: true` securityContext. + # ------------------------------------------------------------------ + KEY_BASENAME="$(basename "$KEY_PATH")" + KEY_TYPE_LINE="$(ssh-keygen -y -P '' -f "$KEY_PATH" 2>/dev/null | awk '{print $1}')" + case "${KEY_TYPE_LINE:-}" in + ssh-rsa) JGIT_NAME="id_rsa" ;; + ssh-dss) JGIT_NAME="id_dsa" ;; + ecdsa-sha2-nistp256|ecdsa-sha2-nistp384|ecdsa-sha2-nistp521) + JGIT_NAME="id_ecdsa" ;; + sk-ecdsa-sha2-nistp256@openssh.com) JGIT_NAME="id_ecdsa_sk" ;; + ssh-ed25519) JGIT_NAME="id_ed25519" ;; + sk-ssh-ed25519@openssh.com) JGIT_NAME="id_ed25519_sk" ;; + *) JGIT_NAME="" ;; + esac + + if [ -n "$JGIT_NAME" ] && [ "$JGIT_NAME" != "$KEY_BASENAME" ]; then + # `ln -sfn`: -s symlink, -f force-overwrite, -n don't deref the + # target if it's already a symlink-to-directory. Relative target + # so the link is self-contained inside /app/.ssh/. + ln -sfn "$KEY_BASENAME" "$KEY_DIR/$JGIT_NAME" + echo "[entrypoint] Symlinked ${KEY_DIR}/${JGIT_NAME} -> ${KEY_BASENAME} for JGit identity scanner (key type: ${KEY_TYPE_LINE})." >&2 + elif [ -z "$JGIT_NAME" ]; then + echo "[entrypoint] WARNING: could not determine SSH key algorithm from ${KEY_PATH}; skipping JGit identity-name symlink. JGit may fail with 'publickey: no keys to try'." >&2 + fi +else + echo "[entrypoint] GIT_SSH_PRIVATE_KEY not set; skipping SSH key materialisation." >&2 +fi + +exec "$@" diff --git a/mcp/mcp-internal-1.0.4.jar b/mcp/mcp-internal-1.0.4.jar deleted file mode 100644 index 2a2ba87..0000000 Binary files a/mcp/mcp-internal-1.0.4.jar and /dev/null differ diff --git a/mcp/run.sh b/mcp/run.sh new file mode 100755 index 0000000..31ef120 --- /dev/null +++ b/mcp/run.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Launch the MCP Inspector UI. +# The server must already be running on http://localhost:8080/mcp +# +# When the Inspector UI opens, enter the bearer token in the +# Authentication section of the sidebar: +# Header Name: Authorization +# Header Value: Bearer DAVIDSUPERSECRETTOKEN +# +# Alternatively, use CLI mode to test a single method: +# npx @modelcontextprotocol/inspector --cli http://localhost:8080/mcp \ +# --transport streamable-http \ +# --header "Authorization: Bearer DAVIDSUPERSECRETTOKEN" \ +# --method tools/list + +npx @modelcontextprotocol/inspector --config config.json --server git-mcp diff --git a/ops/iam_canary.example.yaml b/ops/iam_canary.example.yaml index ff254e5..d9478b8 100644 --- a/ops/iam_canary.example.yaml +++ b/ops/iam_canary.example.yaml @@ -37,7 +37,10 @@ expectations: # Allow: DynamoDB read+write on the agent's tables. # `dynamodb:UpdateItem` permission required for # `DynamoDbTokenBudgetEnforcer.commit` (atomic ADD on the token-budgets - # table). + # table) and for `DynamoDbBreakerStateStore` (atomic conditional write + # on the breaker table). All four tables are seeded by + # `make seed-localstack` and described in `Makefile` / the matching + # adapters under `src/agent/infrastructure/dynamodb/`. # ----------------------------------------------------------------------- - principal: arn:aws:iam::000000000000:role/jira-readiness-agent-execution actions: @@ -46,10 +49,10 @@ expectations: - dynamodb:Query - dynamodb:UpdateItem resources: - - "arn:aws:dynamodb:us-east-1:000000000000:table/idempotency" - "arn:aws:dynamodb:us-east-1:000000000000:table/domain-state" - "arn:aws:dynamodb:us-east-1:000000000000:table/tenants" - "arn:aws:dynamodb:us-east-1:000000000000:table/token-budgets" + - "arn:aws:dynamodb:us-east-1:000000000000:table/breaker" expected: allowed # ----------------------------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index 7542271..df54334 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,166 +7,97 @@ name = "aws-agent-core" version = "0.1.0" description = "LangGraph agent on Amazon Bedrock AgentCore Runtime" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.12" license = { text = "Apache-2.0" } classifiers = [ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", ] dependencies = [ - "bedrock-agentcore>=1.7.0,<2", - "langgraph>=0.2.0,<2", - "langchain-aws>=0.2.0,<1", - # UsageMetadataCallbackHandler is the load-bearing token-counting seam - # (added in langchain-core 0.3.49). - "langchain-core>=0.3.49,<1", - "pydantic>=2.7.0,<3", - # tenacity drives the half-open probe scheduling in - # InMemoryLlmCircuitBreaker (wait_random_exponential). - "tenacity>=9.0.0,<10", + "bedrock-agentcore", + "langgraph", + "langchain-aws", + "langchain-core", + "pydantic", + "tenacity", + # Official Model Context Protocol Python SDK. Owns the Streamable HTTP + # transport, JSON-RPC framing, session-id propagation, and protocol-version + # negotiation. Imported only from `agent.composition._mcp_session` and + # `agent.infrastructure.mcp.*`; application and graph layers see only + # the abstract Protocol from `agent.contracts.mcp`. + "mcp", + # Upstream MCP -> LangChain `StructuredTool` bridge. Used directly with + # zero hand-rolled wrapping; per-call observability is layered on via + # the upstream `ToolCallInterceptor` Protocol implementation in + # `agent.infrastructure.mcp.observability_interceptor`. The 0.2.x + # series targets `langchain-core>=1.0`, which the rest of this + # project's stack (`langchain-aws>=1.x`, `langgraph>=1.x`) now also + # uses, so no version cap is required. + "langchain-mcp-adapters", ] [project.optional-dependencies] dev = [ - "pytest>=8.0.0", - "pytest-cov>=5.0.0", - "coverage[toml]>=7.5.0", - "mypy>=1.10.0", - "ruff>=0.5.0", - "freezegun>=1.5.0", - # Property-based tests on TokenUsage invariants and the budget arithmetic. - "hypothesis>=6.100.0,<7", - # Following Qodo rule: Only add dependencies from the approved third-party registry (ERROR). - # `watchfiles` powers the in-Pod hot-reload loop wired by - # `deploy/local/overlays/host-mount/agent-runtime-patch.yaml` - # (`make local-up-watch`). It is dev-only — production agent runtime - # images MUST NOT install the [dev] extra, so the inotify/native - # filesystem-watcher dependency never ships to AWS. Approval ticket: - # TODO before merge per the approved-deps registry process. - # Following Qodo rule: Pin dependency versions in manifests and lockfiles (WARNING). - # The version range matches the existing convention used by every - # other [dev] / optional-extras entry in this file (caret-style - # >=major.minor,=0.21.0,<1", - # Following Qodo rule: Only add dependencies from the approved third-party registry (ERROR). - # `import-linter` enforces the Wave 4 sealed-test-seams contract - # under `[tool.importlinter]` below: production code in the - # `agent` package must not import from - # `agent.composition._test_seams`. The check runs as a - # `lint-imports` step in the same CI lane as `ruff` / - # `mypy` so a regression surfaces at PR-review time. It is - # dev-only — production agent runtime images do not install - # `[dev]`, so the linter never ships to AWS. Approval ticket: - # TODO before merge per the approved-deps registry process. - # Following Qodo rule: Pin dependency versions in manifests and lockfiles (WARNING). - # The version range matches the existing convention used by - # every other `[dev]` / optional-extras entry in this file - # (caret-style `>=major.minor,=2.0,<3", - # Following Qodo rule: Only add dependencies from the approved third-party registry (ERROR). - # `mutmut` powers the Wave 9 weekly mutation-testing workflow - # (`.github/workflows/mutation.yml`). It is dev-only and runs - # on a separate scheduled job (advisory, not gating) so a - # surviving mutant surfaces in the workflow artifact without - # blocking PRs while the suite is hardened. Production agent - # runtime images do not install `[dev]`, so `mutmut` never - # ships to AWS. Approval ticket: TODO before merge per the - # approved-deps registry process. - # Following Qodo rule: Pin dependency versions in manifests and lockfiles (WARNING). - # The version range matches the existing caret-style convention - # (`>=major.minor,=3.0,<4", + "pytest", + "pytest-cov", + "coverage[toml]", + "mypy", + "ruff", + "freezegun", + "hypothesis", + "watchfiles", + "import-linter", + "mutmut", + # `pytest-asyncio` powers the `async def test_*` discovery for the + # async migration. With `asyncio_mode = "auto"` (configured below) every + # async test function is auto-wrapped in an event loop without per-test + # decorators. Sync tests stay sync. + "pytest-asyncio", ] -# OpenTelemetry GenAI semantic conventions wiring. -# Kept as an optional extra so the unit-test + mypy jobs do NOT pull -# OpenTelemetry transitive deps; the OtelMeterMetricsRecorder consumes -# the SDK via its structural Protocol surface. AWS production picks -# this extra up via the same `[xray]` install path. otel = [ - "opentelemetry-api>=1.27.0,<2", - "opentelemetry-sdk>=1.27.0,<2", + "opentelemetry-api", + "opentelemetry-sdk", ] -# Following Qodo rule: Only add dependencies from the approved third-party registry (ERROR). -# `langchain-anthropic` powers the local profile's `ChatAnthropic` LLM factory. -# It is isolated as an optional extra so the default install (and the -# 100%-coverage unit-test job) does NOT pull -# Anthropic's transitive deps. Approval ticket: TODO before merge per the -# approved-deps registry process. -# Following Qodo rule: Pin dependency versions in manifests and lockfiles (WARNING). -# The version range here matches the existing convention used by `langchain-aws` -# and `langchain-core` in this file (caret-style >=major.minor,=0.3.0,<1", + "langchain-anthropic", ] -# Following Qodo rule: Only add dependencies from the approved third-party registry (ERROR). -# `aws-xray-sdk` and the OpenTelemetry OTLP exporter power the X-Ray tracer -# (the production-only AWS X-Ray exporter / segment-shape seam). They are -# isolated as an optional extra so the default install (and the -# 100%-coverage unit-test job) does NOT -# pull `aws-xray-sdk` or `opentelemetry-*` transitive deps. Approval ticket: -# TODO before merge per the approved-deps registry process. -# Following Qodo rule: Pin dependency versions in manifests and lockfiles (WARNING). -# The version ranges here match the existing convention used by -# `langchain-aws` / `langchain-core` / `langchain-anthropic` (caret-style -# >=major.minor,=2.14.0,<3", - "opentelemetry-sdk>=1.26.0,<2", - "opentelemetry-exporter-otlp-proto-http>=1.26.0,<2", +ollama = [ + "langchain-ollama", ] -# Following Qodo rule: Only add dependencies from the approved third-party registry (ERROR). -# `langchain-mcp-adapters` is the canonical bridge from the MCP `tools/list` / -# `tools/call` JSON-RPC surface to LangChain `StructuredTool`s consumed by -# `model.bind_tools(...)` and `langgraph.prebuilt.ToolNode`. It is isolated as -# an optional extra so the default install (and the -# 100%-coverage unit-test job) does NOT pull `langchain-mcp-adapters` transitive -# deps. Approval ticket: TODO before merge per the approved-deps registry process. -# Following Qodo rule: Pin dependency versions in manifests and lockfiles (WARNING). -# The version range here matches the existing convention used by `langchain-aws` / -# `langchain-core` / `langchain-anthropic` (caret-style >=major.minor,=0.1.0,<1", +xray = [ + "aws-xray-sdk", + "opentelemetry-sdk", + "opentelemetry-exporter-otlp-proto-http", ] -# Following Qodo rule: Only add dependencies from the approved third-party registry (ERROR). -# `langgraph-checkpoint-aws` provides the AgentCore-backed LangGraph -# checkpointer (`AgentCoreMemorySaver`) and the DynamoDB-backed -# `DynamoDbSaver` consumed by `_default_aws_checkpointer_factory` in -# `agent.composition._shared`. `bedrock-agentcore` is the SDK that -# exposes `AgentCoreMemorySaver` as a fallback when the dedicated -# integration package is missing; it already ships in the base -# `dependencies` block (the runtime entrypoint imports `BedrockAgentCoreApp` -# unconditionally) so this extra restates the version pin for parity -# with the operator-facing `pip install -e '.[aws]'` instruction in -# the lazy-import RuntimeError. The unit-test + 100%-coverage job -# does NOT install this extra, so mypy and pytest stay free of the -# AWS-only transitive deps. Approval ticket: TODO before merge per -# the approved-deps registry process. -# Following Qodo rule: Pin dependency versions in manifests and lockfiles (WARNING). -# The version ranges match the existing caret-style convention used -# by every other extras block in this file -# (`>=major.minor,=0.1.0,<1", - "bedrock-agentcore>=1.7.0,<2", + "langgraph-checkpoint-aws", + "bedrock-agentcore", + "PyJWT[crypto]", +] +debug = [ + "langgraph-cli[inmem]", + "debugpy", +] +perf = [ + "locust", ] [tool.setuptools.packages.find] where = ["src"] +[tool.setuptools.package-data] +"agent.infrastructure" = ["structured_log_schema.json"] + [tool.pytest.ini_options] -minversion = "8.0" +minversion = "9.0" testpaths = ["tests"] +# pytest-asyncio auto-mode wraps every `async def test_*` in an event loop +# without requiring an explicit `@pytest.mark.asyncio` decorator; sync tests +# are unaffected. +asyncio_mode = "auto" addopts = [ "-ra", "--strict-markers", @@ -216,7 +147,7 @@ output = "coverage.xml" directory = "htmlcov" [tool.mypy] -python_version = "3.10" +python_version = "3.12" strict = true warn_unused_configs = true disallow_any_generics = true @@ -248,13 +179,13 @@ module = [ # the extras are missing at runtime. "aws_xray_sdk.*", "opentelemetry.*", - # Optional [mcp] extras — see [project.optional-dependencies] above. The - # unit-test / type-check jobs do NOT install these, so mypy must not - # require their stubs to be discoverable. The lazy-import call site in - # `agent.infrastructure.mcp.langchain_adapter._load_official_mcp_tools` - # raises a human-friendly RuntimeError pointing at - # `pip install -e '.[mcp]'` when the extras are missing at runtime. - "langchain_mcp_adapters.*", + # `mcp` is now a base dependency (the agent has no non-MCP tool path), + # but does not ship PEP 561 type stubs. Importing concrete classes + # (`mcp.ClientSession`, `mcp.types.Tool`, etc.) is therefore confined to + # `agent.infrastructure.mcp.*` and `agent.composition._mcp_session`; + # application code sees only the abstract `agent.contracts.mcp.McpClient` + # Protocol. + "mcp.*", # Optional [aws] extras — see [project.optional-dependencies] above # (the managed AgentCore Runtime + Memory promotion seam). The # unit-test / type-check jobs do NOT install @@ -264,12 +195,22 @@ module = [ # raises a human-friendly RuntimeError pointing at # `pip install -e '.[aws]'` when the extras are missing at runtime. "langgraph_checkpoint_aws.*", + # Optional [debug] extras — see [project.optional-dependencies] above + # (local `langgraph dev` server CLI + debugpy, no hosted UI). The + # unit-test / type-check jobs do NOT install these, so mypy must + # not require their stubs to be discoverable. The + # `agent.composition.studio` entrypoint is consumed by the + # `langgraph dev` CLI which only runs when `[debug]` is installed; + # production code must therefore not import either library at + # module level. + "langgraph_cli.*", + "debugpy.*", ] ignore_missing_imports = true [tool.ruff] line-length = 100 -target-version = "py310" +target-version = "py312" src = ["src", "tests"] [tool.ruff.lint] @@ -288,7 +229,7 @@ ignore = ["E501"] [tool.ruff.lint.per-file-ignores] "tests/**/*.py" = ["B011", "S101"] -# Wave 4 sealed-test-seams contract: production code in the ``agent`` +# Sealed-test-seams contract: production code in the ``agent`` # package must NOT import from ``agent.composition._test_seams``. The # module exists only so the unit-test suite can reach private helpers # without polluting the public ``agent.composition`` namespace; a @@ -315,16 +256,51 @@ source_modules = [ "agent.composition._shared", "agent.composition.aws", "agent.composition.local", + "agent.composition.studio", "agent.contracts", "agent.domain", "agent.graph", "agent.infrastructure", "agent.main", - "agent.server", + "agent.worker", ] forbidden_modules = ["agent.composition._test_seams"] -# Wave 9 mutation-testing configuration. +[[tool.importlinter.contracts]] +name = "Domain stays independent" +type = "forbidden" +source_modules = ["agent.domain"] +forbidden_modules = [ + "agent.application", + "agent.composition", + "agent.graph", + "agent.infrastructure", +] + +[[tool.importlinter.contracts]] +name = "Application and graph do not depend on adapters" +type = "forbidden" +source_modules = [ + "agent.application", + "agent.graph", +] +forbidden_modules = [ + "agent.composition", + "agent.infrastructure", +] + +[[tool.importlinter.contracts]] +name = "Contracts stay adapter-free" +type = "forbidden" +source_modules = ["agent.contracts"] +forbidden_modules = [ + "agent.application", + "agent.composition", + "agent.graph", + "agent.infrastructure", +] + +# Mutation-testing configuration. # # `mutmut` runs weekly via `.github/workflows/mutation.yml` (advisory, # not gating). We start narrow — the two modules below carry the @@ -334,11 +310,20 @@ forbidden_modules = ["agent.composition._test_seams"] # the `paths_to_mutate` set to the rest of `src/agent/application`. [tool.mutmut] paths_to_mutate = [ - "src/agent/application/event_invariants.py", + "src/agent/application/mcp_tool_classifier.py", + "src/agent/application/normalizer.py", + "src/agent/application/token_governance.py", + "src/agent/application/webhook_handler.py", "src/agent/application/recursion_guard.py", + "src/agent/graph/nodes/assessor.py", + "src/agent/graph/nodes/designer.py", ] runner = "pytest --no-cov -x -q" tests_dir = [ - "tests/application/test_event_invariants.py", + "tests/application/test_normalizer.py", + "tests/application/test_token_governance.py", + "tests/application/test_webhook_handler.py", + "tests/infrastructure/mcp/test_langchain_adapter.py", "tests/application/test_recursion_guard.py", + "tests/graph/test_routing.py", ] diff --git a/requirements.txt b/requirements.txt index b292ec2..1cacbf3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,25 @@ -# For AgentCore container builds / pip installs without pyproject -bedrock-agentcore>=1.7.0,<2 -langgraph>=0.2.0,<2 -langchain-aws>=0.2.0,<1 -langchain-core>=0.3.0,<1 -pydantic>=2.7.0,<3 +# Bedrock AgentCore CodeBuild dependency entrypoint. +# +# Installs the project from pyproject.toml WITH the optional extras the +# AWS profile (AGENT_PROFILE=aws) needs at runtime: +# - aws : langgraph-checkpoint-aws, PyJWT[crypto] +# - xray : aws-xray-sdk + opentelemetry-{sdk,exporter-otlp-proto-http} +# - otel : opentelemetry-{api,sdk} +# +# Base deps (langgraph, langchain-aws, mcp, langchain-mcp-adapters, ...) +# come from the project itself; the extras above are required because +# the AWS composition root (`agent.composition.aws`) and the +# observability stack import them at runtime. Without these extras the +# AgentCore Runtime container imports `agent.server`, fires the +# langgraph deprecation warning, then dies with a missing-module error +# during dependency resolution -- surfacing to the worker as +# `RuntimeClientError: An error occurred when starting the runtime.` +# +# The `mcp` extra lives in the base deps already (see pyproject.toml), +# so we only declare the AWS-profile-specific extras here. +# +# Operators promoting this image to staging / prod can pin a digest by +# editing this file to a `--hash=` requirement set; the floating form +# below is intentional for the sandbox flow where the AgentCore +# toolkit's CodeBuild rebuilds on every `agentcore deploy`. +.[aws,xray,otel] diff --git a/scripts/agentcore_deploy_wrapper.py b/scripts/agentcore_deploy_wrapper.py new file mode 100755 index 0000000..124254e --- /dev/null +++ b/scripts/agentcore_deploy_wrapper.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Wrapper that runs `agentcore` CLI with a single-PUT S3 upload patch. + +Why this exists +=============== + +`bedrock-agentcore-starter-toolkit` uploads `source.zip` to its CodeBuild +staging bucket via ``boto3.s3.transfer.upload_file``. That helper opens a +connection pool (``max_concurrency=10``, ``use_threads=True``) up front +even for tiny payloads (our zip is < 1 MiB, well below the 8 MiB +``multipart_threshold``). On networks where the local NAT/firewall closes +idle TCP connections aggressively (consumer ISPs, some VPN egress, café +Wi-Fi), the idle pool sockets get reaped and the next ``PutObject`` retry +hits a stale FIN'd connection. S3 returns ``RequestTimeout`` ("Your socket +connection to the server was not read from or written to within the +timeout period."), retries exhaust, and the deploy aborts after several +minutes. + +We have verified — by replicating the exact ``s3_client.upload_file`` +call the toolkit makes — that this fails for an 0.8 MiB zip with both +virtual-hosted and path-style addressing on affected networks, while a +single ``aws s3 cp`` of the same payload (which uses a one-shot PUT and +no connection pool) succeeds in < 3 s. + +The patch +========= + +This wrapper monkey-patches ``boto3.session.Session.client`` so that any +``s3`` client created by the toolkit gets its ``upload_file`` method +swapped for one that issues ``put_object`` directly when the file is +small enough to fit in a single PUT (< ``UPLOAD_FILE_PUT_OBJECT_MAX_BYTES``, +default 64 MiB — the AgentCore source zip is always orders of magnitude +smaller). Files that exceed the threshold fall back to the original +TransferManager-based ``upload_file`` so that genuine multipart workloads +(none in this codepath, but kept for safety) are unaffected. + +Single-PUT does not use a connection pool: it opens one TCP connection, +streams the body, reads the response, and closes. There is no idle window +for the local NAT to close, which removes the entire failure mode. + +Why not fix `~/.aws/config` `s3 = addressing_style = path` instead? +================================================================== + +We tried this. ``addressing_style`` only changes the URL host, not the +TransferManager's connection-pool behavior. Both addressing styles fail +identically on affected networks because the failure is in the pool, not +the routing. + +Usage +===== + + python scripts/agentcore_deploy_wrapper.py deploy --agent ... [--env ...] + python scripts/agentcore_deploy_wrapper.py configure --create ... + +Anything you can pass to the ``agentcore`` CLI is accepted unchanged. +""" + +from __future__ import annotations + +import logging +import os +import sys +from typing import Any + +# Threshold above which we defer to the original TransferManager-based +# upload_file. The AgentCore source.zip is < 1 MiB; pick something +# comfortably above any realistic agent zip size but well below the 5 GiB +# single-PUT limit so we never silently truncate. +UPLOAD_FILE_PUT_OBJECT_MAX_BYTES: int = 64 * 1024 * 1024 + +logger = logging.getLogger("agentcore_deploy_wrapper") + + +def _install_boto3_upload_patch() -> None: + """Replace `s3_client.upload_file` with a `put_object`-based shim. + + Patching is applied at the `Session.client` factory level so every S3 + client the toolkit constructs (CodeBuildService creates its own in + `__init__`) inherits the shim, even if the toolkit upgrades and adds + new client constructions later. + + We additionally inject a long `read_timeout` / `connect_timeout` and a + moderate retry policy on the s3 client. Default boto3 `read_timeout` + is 60 s, which is fine on healthy networks (a 1 MiB upload completes + in <1 s) but too tight on degraded operator networks where sustained + outbound throughput drops to ~10 KiB/s. At that throughput the 0.8 MiB + AgentCore source.zip needs ~80 s of socket-write time and trips the + default read_timeout halfway through, which is what we observed prior + to this fix. Overrides controllable via env: + AGENTCORE_S3_READ_TIMEOUT (seconds, default 900) + AGENTCORE_S3_CONNECT_TIMEOUT (seconds, default 30) + AGENTCORE_S3_MAX_ATTEMPTS (default 3) + """ + + import boto3 + import boto3.session as _bsession + from botocore.config import Config as _BotoConfig + + read_timeout = int(os.environ.get("AGENTCORE_S3_READ_TIMEOUT", "900")) + connect_timeout = int(os.environ.get("AGENTCORE_S3_CONNECT_TIMEOUT", "30")) + max_attempts = int(os.environ.get("AGENTCORE_S3_MAX_ATTEMPTS", "3")) + + original_client_factory = _bsession.Session.client + + def patched_client(self: Any, service_name: str, *args: Any, **kwargs: Any) -> Any: + if service_name == "s3": + existing_cfg = kwargs.get("config") + override_cfg = _BotoConfig( + connect_timeout=connect_timeout, + read_timeout=read_timeout, + retries={"max_attempts": max_attempts, "mode": "adaptive"}, + # `tcp_keepalive=True` lets long-lived single-PUT uploads + # survive aggressive NAT idle-reaper rules on the way to + # S3. boto3 added this knob in 1.34; older versions ignore + # the kwarg silently. + tcp_keepalive=True, + ) + kwargs["config"] = override_cfg.merge(existing_cfg) if existing_cfg else override_cfg + client = original_client_factory(self, service_name, *args, **kwargs) + if service_name != "s3": + return client + + original_upload_file = client.upload_file + + def upload_file_via_put_object( + Filename: str, # noqa: N803 (mirror boto3 capitalization) + Bucket: str, + Key: str, + ExtraArgs: dict | None = None, + Callback=None, # noqa: ARG001 + Config=None, # noqa: ARG001, N803 + ) -> None: + try: + size = os.path.getsize(Filename) + except OSError: + size = UPLOAD_FILE_PUT_OBJECT_MAX_BYTES + 1 + + if size > UPLOAD_FILE_PUT_OBJECT_MAX_BYTES: + logger.info( + "[s3-upload-patch] file %s is %d bytes (> %d); using TransferManager fallback.", + Filename, + size, + UPLOAD_FILE_PUT_OBJECT_MAX_BYTES, + ) + return original_upload_file( + Filename, Bucket, Key, ExtraArgs=ExtraArgs, Callback=None, Config=None + ) + + put_kwargs: dict[str, Any] = {"Bucket": Bucket, "Key": Key} + if ExtraArgs: + put_kwargs.update(ExtraArgs) + + logger.info( + "[s3-upload-patch] PUT (single-shot) s3://%s/%s (%d bytes)", + Bucket, + Key, + size, + ) + with open(Filename, "rb") as fh: + put_kwargs["Body"] = fh + client.put_object(**put_kwargs) + + client.upload_file = upload_file_via_put_object # type: ignore[method-assign] + return client + + _bsession.Session.client = patched_client # type: ignore[method-assign] + # Also patch the module-level alias used by some callers. + boto3.session.Session.client = patched_client # type: ignore[method-assign] + logger.info( + "[s3-upload-patch] installed; single-PUT for files <= %d bytes; " + "s3 client read_timeout=%ds, connect_timeout=%ds, max_attempts=%d.", + UPLOAD_FILE_PUT_OBJECT_MAX_BYTES, + read_timeout, + connect_timeout, + max_attempts, + ) + + +def _run_agentcore_cli(argv: list[str]) -> int: + """Invoke the agentcore CLI in-process so the patch stays effective.""" + from bedrock_agentcore_starter_toolkit.cli.cli import app + + sys.argv = ["agentcore", *argv] + try: + app() + except SystemExit as exc: + return int(exc.code or 0) + return 0 + + +def main() -> int: + log_level = os.environ.get("AGENTCORE_DEPLOY_WRAPPER_LOG", "INFO").upper() + logging.basicConfig(level=getattr(logging, log_level, logging.INFO), format="%(levelname)s %(name)s :: %(message)s") + if len(sys.argv) < 2: + print("usage: agentcore_deploy_wrapper.py [args...]", file=sys.stderr) + return 2 + + _install_boto3_upload_patch() + return _run_agentcore_cli(sys.argv[1:]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/bootstrap_agentcore_runtime.sh b/scripts/bootstrap_agentcore_runtime.sh new file mode 100755 index 0000000..e49ae48 --- /dev/null +++ b/scripts/bootstrap_agentcore_runtime.sh @@ -0,0 +1,593 @@ +#!/usr/bin/env bash +############################################################################### +# scripts/bootstrap_agentcore_runtime.sh +# +# Provision (or update) the Bedrock AgentCore Runtime + Memory resources via +# the upstream `agentcore` CLI, then persist the resolved identifiers to: +# .state/agentcore-runtime.env +# +# Why this script exists: +# - terraform/agentcore-runtime/ manages IAM/ECR/log prerequisites but does not +# own the runtime control-plane object. +# - AgentCore runtime creation is currently toolkit-driven (`agentcore deploy`). +# - This script is the canonical, repeatable path when Terraform cannot own the +# runtime lifecycle directly. +# +# Idempotency: +# - `agentcore deploy --auto-update-on-conflict` makes reruns safe. +# - We always re-resolve runtime/memory identifiers from AWS control plane and +# overwrite `.state/agentcore-runtime.env` with fresh values. +############################################################################### + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +# CI / non-interactive callers can pass --no-drift-check to suppress the +# post-deploy comparison between the freshly written +# `.state/agentcore-runtime.env::AGENTCORE_RUNTIME_ARN` and the live +# `agent-worker-config` ConfigMap (the comparison is informational; the +# Kubernetes-native fix is the SHA-256 annotation on the worker +# Deployment + the explicit `kubectl rollout restart` in +# `scripts/bootstrap_eks_workloads.sh`). +DRIFT_CHECK=1 +for arg in "$@"; do + case "$arg" in + --no-drift-check) DRIFT_CHECK=0 ;; + *) echo "unknown arg: $arg" >&2; exit 64 ;; + esac +done + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "missing required command: $1" >&2 + exit 1 + fi +} + +require_file() { + local path="$1" + local hint="$2" + if [[ ! -f "$path" ]]; then + echo "missing required file: $path" >&2 + echo "$hint" >&2 + exit 2 + fi +} + +require_cmd jq +require_cmd aws +require_cmd kubectl +require_cmd agentcore + +############################################################################### +# `agentcore deploy` source-zip slimmer. +# +# Why this exists: +# `agentcore configure`/`agentcore deploy` (bedrock-agentcore-starter-toolkit) +# builds an S3 source zip from the working directory and ALWAYS uses its +# own bundled `dockerignore.template` to filter files. It does NOT honor +# the repo-local `.dockerignore` — verified in +# bedrock_agentcore_starter_toolkit.services.codebuild +# ::CodeBuildService._parse_dockerignore (toolkit branch as of +# 2026-05; method hardcodes `files(...).joinpath( +# "utils/runtime/templates/dockerignore.template").read_text()`). +# The toolkit's template excludes `terraform/`, `cdk/`, `tests/`, +# `docs/`, and `mcp/lambda/` — but NOT `mcp/` itself. +# +# Historically the proximate cost of including `mcp/` was the ~62 MB +# Spring Boot fat-JAR (`mcp/mcp-internal-*.jar`), which inflated the +# source.zip from ~22 MB to ~81 MB and intermittently tripped the +# boto3 S3 multipart-upload `RequestTimeout` on slow uplinks. That +# JAR has since been moved out of the repository entirely +# (resolved at build time from `${MCP_INTERNAL_REPO}/build/libs` — +# see `scripts/build_mcp_image.sh` and `mcp/README.md` "External JAR +# contract"); `mcp/` now carries only the Dockerfile, the inspector +# config, the entrypoint, and docs (~20 KB total). +# +# The stash dance is retained as defence-in-depth: +# 1. Nothing in `mcp/` is consumed by the AgentCore Runtime container. +# The runtime `Dockerfile` at the repo root only copies +# `pyproject.toml`, `README.md`, and `src/`. Stashing `mcp/` +# keeps the source.zip topology aligned with the runtime's actual +# build context — no surprise files in CodeBuild logs. +# 2. If a future file lands in `mcp/` that does grow the upload +# (a corpus, a generated artifact, etc.) the size guard is +# already in place and no migration is required. +# +# Codified fix: +# Move `mcp/` (and any other heavy paths the runtime container does not +# consume) to a temp dir before `agentcore configure`/`agentcore deploy`, +# and restore unconditionally on EXIT/INT/TERM/HUP via the trap below. +# This is robust to bash `set -e` aborts, Ctrl-C from the operator, +# and `agentcore` SIGSEGV / OOM exits. +# +# Upstream: +# - The "right" fix is for the toolkit to expose a `--source-path` / +# `--exclude` flag on `agentcore configure`. Tracked in +# `docs/DEFERRED.md` under "AgentCore source-zip filtering". +############################################################################### + +# Paths that the runtime container's build context does NOT need. +# Any path listed here MUST be safe to temporarily move out of the +# working tree (no other concurrent process should depend on it). +AGENTCORE_ZIP_STASH_PATHS=("mcp") +AGENTCORE_ZIP_STASH_DIR="" + +stash_paths_for_agentcore_zip() { + if [[ -n "${AGENTCORE_ZIP_STASH_DIR}" ]]; then + return 0 + fi + AGENTCORE_ZIP_STASH_DIR="$(mktemp -d -t agentcore-zip-stash.XXXXXX)" + local moved=0 + for rel in "${AGENTCORE_ZIP_STASH_PATHS[@]}"; do + if [[ -e "${REPO_ROOT}/${rel}" ]]; then + mv "${REPO_ROOT}/${rel}" "${AGENTCORE_ZIP_STASH_DIR}/$(basename "${rel}")" + moved=$((moved + 1)) + echo " [zip-slim] stashed ${rel} -> ${AGENTCORE_ZIP_STASH_DIR}/$(basename "${rel}")" + fi + done + if (( moved == 0 )); then + rmdir "${AGENTCORE_ZIP_STASH_DIR}" 2>/dev/null || true + AGENTCORE_ZIP_STASH_DIR="" + fi +} + +restore_paths_for_agentcore_zip() { + if [[ -z "${AGENTCORE_ZIP_STASH_DIR}" ]]; then + return 0 + fi + local rc=$? + for rel in "${AGENTCORE_ZIP_STASH_PATHS[@]}"; do + local base + base="$(basename "${rel}")" + if [[ -e "${AGENTCORE_ZIP_STASH_DIR}/${base}" ]]; then + if [[ -e "${REPO_ROOT}/${rel}" ]]; then + echo " [zip-slim] WARNING: ${rel} re-appeared during deploy; leaving stash at ${AGENTCORE_ZIP_STASH_DIR}/${base} for inspection" >&2 + else + mv "${AGENTCORE_ZIP_STASH_DIR}/${base}" "${REPO_ROOT}/${rel}" + echo " [zip-slim] restored ${rel}" + fi + fi + done + rmdir "${AGENTCORE_ZIP_STASH_DIR}" 2>/dev/null || true + AGENTCORE_ZIP_STASH_DIR="" + # Preserve the original exit code for the trap caller. + return $rc +} +trap 'restore_paths_for_agentcore_zip' EXIT INT TERM HUP + +require_file ".state/agentcore-runtime.outputs.json" \ + "Run 'scripts/tf.sh agentcore-runtime apply' first." +require_file ".state/ecr-images.outputs.json" \ + "Run 'scripts/tf.sh ecr-images apply' first." +require_file ".state/bootstrap.env" \ + "Run the Bedrock model-id probe first (writes BEDROCK_MODEL_ID; see aws-deploy-plan.md pre-flight)." +require_file ".state/secrets-mcp.outputs.json" \ + "Run 'scripts/tf.sh secrets-mcp apply' first." +require_file "deploy/aws/.rendered/agent-worker/configmap.yaml" \ + "Run 'scripts/tf.sh eks-workloads apply' first." + +EXEC_ROLE_ARN="$(jq -r '.execution_role_arn.value' .state/agentcore-runtime.outputs.json)" +AGENTCORE_ENTRYPOINT="$(jq -r '.agentcore_entrypoint.value // "src/agent/server.py"' .state/agentcore-runtime.outputs.json)" +ECR_REPO_URL="$(jq -r '.agentcore_repository_url.value' .state/ecr-images.outputs.json)" +MCP_BEARER_TOKEN_ARN="$(jq -r '.secret_arn.value // empty' .state/secrets-mcp.outputs.json)" +PINNED_AWS_REGION="us-east-1" +if [[ -n "${AWS_REGION:-}" && "${AWS_REGION}" != "${PINNED_AWS_REGION}" ]]; then + echo "AWS_REGION must be ${PINNED_AWS_REGION} (got ${AWS_REGION})" >&2 + exit 64 +fi +if [[ -n "${AWS_DEFAULT_REGION:-}" && "${AWS_DEFAULT_REGION}" != "${PINNED_AWS_REGION}" ]]; then + echo "AWS_DEFAULT_REGION must be ${PINNED_AWS_REGION} (got ${AWS_DEFAULT_REGION})" >&2 + exit 64 +fi +AWS_REGION="${PINNED_AWS_REGION}" +export AWS_REGION +export AWS_DEFAULT_REGION="${PINNED_AWS_REGION}" + +set -a +source .state/bootstrap.env +set +a + +: "${BEDROCK_MODEL_ID:?bootstrap.env must define BEDROCK_MODEL_ID}" + +export AGENTCORE_SUPPRESS_RECOMMENDATION=1 + +WORKLOAD_NAMESPACE="${WORKLOAD_NAMESPACE:-}" +if [[ -z "${WORKLOAD_NAMESPACE}" && -f "${REPO_ROOT}/.state/eks.outputs.json" ]]; then + _kube_ns="$(jq -r '.namespace.value // empty' "${REPO_ROOT}/.state/eks.outputs.json")" + if [[ -n "${_kube_ns}" && "${_kube_ns}" != "null" ]]; then + WORKLOAD_NAMESPACE="${_kube_ns}" + fi +fi +WORKLOAD_NAMESPACE="${WORKLOAD_NAMESPACE:-aws-agent-core}" +NETWORK_STATE="${REPO_ROOT}/.state/network.outputs.json" +ACR_TFSTATE_JSON="${REPO_ROOT}/.state/agentcore-runtime.outputs.json" + +# Bedrock AgentCore VPC mode — subnets + SG created by Terraform. When set, +# `agentcore configure` receives `--vpc --subnets ... --security-groups ...` +# so the runtime can reach the internal MCP NLB inside the same VPC. +VPC_CONFIGURE_ARGS=() +if [[ -f "$ACR_TFSTATE_JSON" ]]; then + _vpc_sg="$(jq -r '.runtime_vpc_security_group_id.value // ""' "$ACR_TFSTATE_JSON")" + _subs="" + if [[ -f "$NETWORK_STATE" ]]; then + _subs="$(jq -r '(.private_subnet_ids.value // []) | join(",")' "$NETWORK_STATE")" + fi + if [[ -n "${_vpc_sg}" && "${_vpc_sg}" != "null" && -n "${_subs}" ]]; then + VPC_CONFIGURE_ARGS=(--vpc --subnets "${_subs}" --security-groups "${_vpc_sg}") + fi +fi + +wait_for_mcp_internal_lb_hostname() { + local deadline=$((SECONDS + 360)) + while [[ "${SECONDS}" -lt "${deadline}" ]]; do + local host="" + host="$(kubectl get svc mcp-internal -n "${WORKLOAD_NAMESPACE}" \ + -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true)" + if [[ -n "${host}" ]]; then + printf '%s' "${host}" + return 0 + fi + sleep 5 + done + return 1 +} + +resolve_mcp_base_url_for_agentcore() { + local host="" + host="$(kubectl get svc mcp-internal -n "${WORKLOAD_NAMESPACE}" \ + -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true)" + if [[ -n "${host}" ]]; then + printf 'http://%s:8081/mcp' "${host}" + return 0 + fi + printf 'http://mcp-internal.%s.svc.cluster.local:8081/mcp' "${WORKLOAD_NAMESPACE}" +} + +if [[ -z "${AGENTCORE_ENTRYPOINT}" || "${AGENTCORE_ENTRYPOINT}" == "null" ]]; then + echo "failed to resolve agentcore entrypoint from .state/agentcore-runtime.outputs.json" >&2 + exit 65 +fi +if [[ ! -f "${AGENTCORE_ENTRYPOINT}" ]]; then + echo "agentcore entrypoint does not exist: ${AGENTCORE_ENTRYPOINT}" >&2 + exit 66 +fi +if [[ -z "${MCP_BEARER_TOKEN_ARN}" ]]; then + echo "failed to resolve MCP bearer token secret ARN from .state/secrets-mcp.outputs.json" >&2 + exit 67 +fi + +declare -a DEPLOY_ENV_ARGS=() +DEPLOY_ENV_ARGS+=(--env "BEDROCK_MODEL_ID=${BEDROCK_MODEL_ID}") +DEPLOY_ENV_ARGS+=(--env "MCP_BEARER_TOKEN_ARN=${MCP_BEARER_TOKEN_ARN}") + +if [[ ${#VPC_CONFIGURE_ARGS[@]} -gt 0 ]]; then + echo "VPC networking enabled for AgentCore — waiting for mcp-internal LoadBalancer hostname (up to 6m)..." + if ! MCP_LB_HOST="$(wait_for_mcp_internal_lb_hostname)"; then + echo "timed out waiting for mcp-internal Service hostname — apply eks-workloads + kubectl apply -k deploy/aws/.rendered/ first" >&2 + exit 70 + fi + echo " mcp-internal LB host: ${MCP_LB_HOST}" +fi + +mapfile -t RUNTIME_CONFIG_ENV_LINES < <( + python3 - <<'PY' +import yaml + +config_path = "deploy/aws/.rendered/agent-worker/configmap.yaml" +with open(config_path, "r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) or {} + +payload = data.get("data") or {} +keys = ( + "AGENT_PROFILE", + "AWS_REGION", + "MCP_BASE_URL", + "AGENT_JIRA_ACCOUNT_ID", + "AGENT_JIRA_EMAIL", + "DOMAIN_STATE_TABLE_NAME", + "TENANTS_TABLE_NAME", + "DEDUPE_TABLE_NAME", + "DEDUPE_TTL_SECONDS", + "RAW_PAYLOAD_BUCKET", + "WEBHOOK_WORK_QUEUE_URL", + "DEAD_LETTER_QUEUE_URL", + "DEFAULT_TENANT", + "ALLOW_DEFAULT_TENANT_AWS", + "OBSERVABILITY_BACKEND", + "BEDROCK_GUARDRAIL_IDENTIFIER_ARN", + "BEDROCK_GUARDRAIL_VERSION_ARN", + "BEDROCK_AGENTCORE_MEMORY_ID", +) +for key in keys: + value = payload.get(key) + if value is None: + continue + text = str(value).strip() + if not text: + continue + print(f"{key}={text}") +PY +) + +MCP_URL_FOR_RUNTIME="" +if [[ ${#VPC_CONFIGURE_ARGS[@]} -gt 0 ]]; then + MCP_URL_FOR_RUNTIME="$(resolve_mcp_base_url_for_agentcore)" + if [[ "${MCP_URL_FOR_RUNTIME}" == *".svc.cluster.local"* ]]; then + echo "VPC AgentCore requires mcp-internal Service type LoadBalancer with a hostname; got: ${MCP_URL_FOR_RUNTIME}" >&2 + exit 71 + fi +fi + +# OBSERVABILITY_BACKEND override for the AgentCore Runtime container. +# +# The agent-worker ConfigMap (rendered by terraform/eks-workloads/) sets +# OBSERVABILITY_BACKEND=otlp with OTEL_EXPORTER_OTLP_ENDPOINT pointing at +# the in-cluster ADOT collector (a `*.svc.cluster.local` URL). That is +# correct for the EKS worker but unreachable from the AgentCore Runtime +# ENI — the runtime sits in private subnets but does NOT have in-cluster +# DNS for Kubernetes Service hostnames, and the OTLP endpoint isn't +# fronted by an NLB the way mcp-internal is. +# +# Why "noop" and not "xray" or "otlp": +# - `OBSERVABILITY_BACKEND=otlp` (worker default) blocks the OTel +# BatchSpanProcessor on shutdown waiting for `*.svc.cluster.local:4318` +# that the runtime ENI cannot resolve. +# - `OBSERVABILITY_BACKEND=xray` activates the in-process `aws_xray_sdk` +# which spawns a background `rule_poller` thread that hammers +# `127.0.0.1:2000` (the X-Ray daemon) on a 5s loop. The daemon is an +# EKS DaemonSet (see terraform/observability-xray/main.tf) and does +# NOT exist in the AgentCore Runtime container; the resulting +# `EndpointConnectionError` bleeds into AgentCore's request-handling +# `TaskGroup` and surfaces as +# `server.unhandled.error: ExceptionGroup` on every invocation, +# even though the agent itself ran successfully. +# - `OBSERVABILITY_BACKEND=noop` is therefore the only correct choice +# for AgentCore Runtime: AgentCore's own observability layer reports +# invocation-level X-Ray spans automatically (visible in the AWS +# GenAI dashboard) via the platform's managed OTel collector, and +# CloudWatch GenAI metrics (token usage, etc.) are emitted via +# `cloudwatch:PutMetricData` directly through OtelMeterMetricsRecorder +# — that path does NOT depend on the tracer backend. +# - Worker pods continue to ship spans to ADOT-via-OTLP unchanged. +declare -a RESOLVED_RUNTIME_ENV_LINES=() +for line in "${RUNTIME_CONFIG_ENV_LINES[@]}"; do + case "${line}" in + MCP_BASE_URL=*) + if [[ -n "${MCP_URL_FOR_RUNTIME}" ]]; then + RESOLVED_RUNTIME_ENV_LINES+=("MCP_BASE_URL=${MCP_URL_FOR_RUNTIME}") + else + RESOLVED_RUNTIME_ENV_LINES+=("${line}") + fi + ;; + OBSERVABILITY_BACKEND=*) + RESOLVED_RUNTIME_ENV_LINES+=("OBSERVABILITY_BACKEND=noop") + ;; + OTEL_EXPORTER_OTLP_ENDPOINT=*|OTEL_EXPORTER_OTLP_PROTOCOL=*|OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=*|OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=*|OTEL_SERVICE_NAME=*) + # Skip OTLP endpoint vars — see comment block above. + ;; + AWS_XRAY_DAEMON_ADDRESS=*|AWS_XRAY_TRACING_NAME=*|AWS_XRAY_CONTEXT_MISSING=*) + # Defensive: any caller-leaked X-Ray daemon hint is dropped. + # Even with OBSERVABILITY_BACKEND=noop our code does not import + # aws_xray_sdk, but stripping these vars is a belt-and-braces + # guard against future composition changes. + ;; + *) + RESOLVED_RUNTIME_ENV_LINES+=("${line}") + ;; + esac +done + +# Belt-and-braces: ensure the override is present even if the worker +# ConfigMap stops emitting `OBSERVABILITY_BACKEND` for some reason. +if ! printf '%s\n' "${RESOLVED_RUNTIME_ENV_LINES[@]}" | grep -q '^OBSERVABILITY_BACKEND='; then + RESOLVED_RUNTIME_ENV_LINES+=("OBSERVABILITY_BACKEND=noop") +fi + +for line in "${RESOLVED_RUNTIME_ENV_LINES[@]}"; do + DEPLOY_ENV_ARGS+=(--env "$line") +done + +############################################################################### +# AgentCore deploy build mode + S3 upload reliability shim. +# +# AGENTCORE_BUILD_MODE controls how the runtime container image is built: +# - "codebuild" (default) — the toolkit zips the source tree, uploads it to +# `bedrock-agentcore-codebuild-sources--`, then runs +# CodeBuild in AWS to produce an arm64 image and push it to ECR. No +# local Docker required. Fast in CI. +# - "local-build" — the toolkit invokes Docker locally (BuildKit/buildx) +# to produce an arm64 image and pushes directly to ECR. Skips the S3 +# source-zip upload entirely. Requires `agentcore configure` to have +# emitted a Dockerfile under `.bedrock_agentcore//`, which +# happens only when `--create` is NOT passed. Use this only when +# CodeBuild itself is unavailable. +# +# S3 upload reliability shim (default-on for codebuild mode): +# +# `bedrock-agentcore-starter-toolkit` calls +# `boto3.s3.transfer.upload_file` to push the source.zip. That helper +# opens a TransferManager connection pool (default `max_concurrency=10`, +# `use_threads=True`) before issuing a single PUT, even for files far +# below the 8 MiB multipart_threshold. On networks where the local +# NAT/firewall closes idle TCP connections aggressively (consumer ISPs, +# some VPN egress, café/conference Wi-Fi), the idle pool sockets get +# reaped and the next retry hits a stale FIN'd connection. S3 returns +# `RequestTimeout` ("Your socket connection to the server was not read +# from or written to within the timeout period."), retries exhaust, and +# the deploy aborts after several minutes — even though the same payload +# uploads cleanly via `aws s3 cp` (single PUT, no pool). We have +# reproduced this with the exact `s3_client.upload_file(..., +# ExtraArgs={...})` call the toolkit makes, for an 0.8 MiB zip, with +# both virtual-hosted and path-style addressing. +# +# `scripts/agentcore_deploy_wrapper.py` monkey-patches +# `Session.client('s3').upload_file` so files smaller than 64 MiB go via +# `put_object` (single PUT, no connection pool). The toolkit's CLI is +# then invoked in-process so the patch stays effective. Files larger +# than the threshold transparently fall back to the original +# TransferManager-based path; the AgentCore source.zip never approaches +# that ceiling. The wrapper is opt-out via `AGENTCORE_S3_UPLOAD_PATCH=0` +# but is on by default because (a) the patch is a strict superset of +# the original behavior for small files and (b) the failure mode it +# fixes has been observed on multiple operator networks. +# +# AWS_S3_ADDRESSING_STYLE is intentionally NOT relied on here: botocore +# does not read that env var (the only supported channels are +# `Config(s3={...})` at client construction time and the `s3` section of +# `~/.aws/config`). We export it for backward compatibility with old +# runbooks but the wrapper is the load-bearing fix. +############################################################################### +AGENTCORE_BUILD_MODE="${AGENTCORE_BUILD_MODE:-codebuild}" +case "${AGENTCORE_BUILD_MODE}" in + codebuild|local-build) ;; + *) + echo "ERROR: AGENTCORE_BUILD_MODE must be 'codebuild' or 'local-build', got '${AGENTCORE_BUILD_MODE}'." >&2 + exit 2 + ;; +esac + +DEPLOY_MODE_FLAGS=() +if [[ "${AGENTCORE_BUILD_MODE}" == "local-build" ]]; then + DEPLOY_MODE_FLAGS+=(--local-build) + echo "AgentCore deploy build mode: local-build (Docker-on-host arm64 build, push to ECR; bypasses S3 source-zip upload)." + if ! docker buildx version >/dev/null 2>&1; then + echo "ERROR: AGENTCORE_BUILD_MODE=local-build requires 'docker buildx' on PATH." >&2 + exit 2 + fi +else + echo "AgentCore deploy build mode: codebuild (default; CodeBuild builds arm64 in AWS)." +fi + +# Documented no-op (kept for operator-runbook compatibility). The actual +# fix for upload-pool flakiness is the wrapper invoked below. +export AWS_S3_ADDRESSING_STYLE=path + +# Pick the agentcore invocation: the patched wrapper by default, or the +# bare CLI if the operator has explicitly opted out. +AGENTCORE_S3_UPLOAD_PATCH="${AGENTCORE_S3_UPLOAD_PATCH:-1}" +if [[ "${AGENTCORE_S3_UPLOAD_PATCH}" == "1" ]]; then + AGENTCORE_CMD=(python3 "${REPO_ROOT}/scripts/agentcore_deploy_wrapper.py") + echo "Using S3 upload reliability shim (scripts/agentcore_deploy_wrapper.py)." +else + AGENTCORE_CMD=(agentcore) + echo "AGENTCORE_S3_UPLOAD_PATCH=0 — invoking 'agentcore' CLI without single-PUT shim (TransferManager pool may stall on flaky networks)." +fi + +echo "Slimming source tree for agentcore zip (excluding paths the runtime container does not consume)..." +stash_paths_for_agentcore_zip + +echo "Configuring agentcore project (jira_readiness_agent)..." +"${AGENTCORE_CMD[@]}" configure --create --non-interactive \ + --name jira_readiness_agent \ + --entrypoint "${AGENTCORE_ENTRYPOINT}" \ + --execution-role "$EXEC_ROLE_ARN" \ + --ecr "$ECR_REPO_URL" \ + --region "$AWS_REGION" \ + --requirements-file requirements.txt \ + "${VPC_CONFIGURE_ARGS[@]}" + +echo "Deploying runtime (auto-update-on-conflict enabled, mode=${AGENTCORE_BUILD_MODE})..." +"${AGENTCORE_CMD[@]}" deploy --agent jira_readiness_agent --auto-update-on-conflict \ + "${DEPLOY_MODE_FLAGS[@]}" \ + "${DEPLOY_ENV_ARGS[@]}" + +restore_paths_for_agentcore_zip +echo "Source-tree restoration complete." + +echo "Resolving runtime + memory identifiers from AWS control plane..." +runtime_json="$(aws bedrock-agentcore-control list-agent-runtimes --region "$AWS_REGION" --output json)" +memory_json="$(aws bedrock-agentcore-control list-memories --region "$AWS_REGION" --output json)" + +RUNTIME_ARN="$(printf '%s' "$runtime_json" | jq -r ' + (.agentRuntimes // []) + | map(select(((.agentRuntimeName // .name // "") | startswith("jira_readiness_agent")))) + | last + | (.agentRuntimeArn // .arn // "") +')" +MEMORY_ID="$(printf '%s' "$memory_json" | jq -r ' + (.memories // []) + | map(select(((.id // .memoryId // "") | startswith("jira_readiness_agent_mem-")))) + | last + | (.id // .memoryId // "") +')" + +if [[ -z "$RUNTIME_ARN" || "$RUNTIME_ARN" == "None" ]]; then + echo "failed to resolve runtime ARN after deploy" >&2 + exit 3 +fi +if [[ -z "$MEMORY_ID" || "$MEMORY_ID" == "None" ]]; then + echo "failed to resolve memory ID after deploy" >&2 + exit 3 +fi + +mkdir -p .state +{ + printf 'AGENTCORE_RUNTIME_ARN=%s\n' "$RUNTIME_ARN" + printf 'AGENTCORE_MEMORY_ID=%s\n' "$MEMORY_ID" +} > .state/agentcore-runtime.env + +printf '%s [PHASE_4] bedrock-agentcore-runtime %s\n' \ + "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$RUNTIME_ARN" >> .state/applied.log +printf '%s [PHASE_4] bedrock-agentcore-memory %s\n' \ + "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$MEMORY_ID" >> .state/applied.log + +echo "Done." +echo " AGENTCORE_RUNTIME_ARN=$RUNTIME_ARN" +echo " AGENTCORE_MEMORY_ID=$MEMORY_ID" + +############################################################################### +# Post-deploy drift detection. +# +# The agent-worker Pod's env is captured at pod start time from the +# `agent-worker-config` ConfigMap. When this script writes a new +# `AGENTCORE_RUNTIME_ARN` into `.state/agentcore-runtime.env` (e.g. after +# `agentcore deploy` rebuilt the runtime), the live ConfigMap and the +# running Pod both still pin the previous ARN until +# `scripts/bootstrap_eks_workloads.sh` re-applies the rendered manifests. +# Without that re-apply (and the rollout restart it triggers), the worker +# keeps invoking the old, now-replaced AgentCore Runtime — surfacing as +# `RuntimeClientError: An error occurred when starting the runtime`. +# +# This block compares the freshly written ARN to the live ConfigMap and +# emits a copy-pasteable remediation banner on drift. Skipped under +# `--no-drift-check` (CI) or when kubectl context is not wired to the +# cluster. +############################################################################### +if [[ "${DRIFT_CHECK}" == "1" ]]; then + if kubectl config current-context >/dev/null 2>&1; then + LIVE_CM_ARN="$(kubectl -n "${WORKLOAD_NAMESPACE}" get configmap agent-worker-config \ + -o jsonpath='{.data.AGENTCORE_RUNTIME_ARN}' 2>/dev/null || true)" + if [[ -n "${LIVE_CM_ARN}" && "${LIVE_CM_ARN}" != "${RUNTIME_ARN}" ]]; then + cat >&2 <&2 + exit 64 +fi +if [[ -n "${AWS_DEFAULT_REGION:-}" && "${AWS_DEFAULT_REGION}" != "${PINNED_AWS_REGION}" ]]; then + echo "AWS_DEFAULT_REGION must be ${PINNED_AWS_REGION} (got ${AWS_DEFAULT_REGION})" >&2 + exit 64 +fi +export AWS_REGION="${PINNED_AWS_REGION}" +export AWS_DEFAULT_REGION="${PINNED_AWS_REGION}" + +############################################################################### +# Helpers +############################################################################### + +DRY_RUN=0 +SKIP_HELM=0 +SKIP_ADOT=0 +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=1 ;; + --skip-helm) SKIP_HELM=1 ;; + --skip-adot) SKIP_ADOT=1 ;; + *) echo "unknown arg: $arg" >&2; exit 64 ;; + esac +done + +run() { + if [[ "$DRY_RUN" == "1" ]]; then + printf '+ %s\n' "$*" + else + printf '+ %s\n' "$*" + "$@" + fi +} + +require_file() { + local path="$1" + local hint="$2" + if [[ ! -f "$path" ]]; then + echo "ERROR: required file not found: $path" >&2 + echo " $hint" >&2 + exit 2 + fi +} + +require_jq_value() { + # require_jq_value + local file="$1" + local expr="$2" + local hint="$3" + local val + val=$(jq -r "$expr" "$file" 2>/dev/null || true) + if [[ -z "$val" || "$val" == "null" ]]; then + echo "ERROR: jq '$expr' on $file returned empty/null." >&2 + echo " $hint" >&2 + exit 2 + fi + printf '%s' "$val" +} + +resolve_latest_ecr_tag() { + # resolve_latest_ecr_tag [tag-prefix] + local repo_name="$1" + local region="$2" + local tag_prefix="${3:-}" + + aws ecr describe-images \ + --repository-name "$repo_name" \ + --region "$region" \ + --output json 2>/dev/null \ + | jq -r --arg prefix "$tag_prefix" ' + [ + (.imageDetails // [])[] + | { pushed: (.imagePushedAt // ""), tags: (.imageTags // []) } as $img + | $img.tags[] + | select(($prefix == "") or startswith($prefix)) + | { tag: ., pushed: $img.pushed } + ] + | sort_by(.pushed) + | last + | .tag // empty + ' 2>/dev/null || true +} + +validate_agent_worker_image_tag() { + # EKS Auto Mode general-purpose nodes are amd64-only. + # Refuse non-amd64 tags unless explicitly overridden. + local tag="$1" + + if [[ "${ALLOW_NON_AMD64_AGENT_WORKER_TAG:-0}" == "1" ]]; then + return 0 + fi + + if [[ ! "$tag" =~ amd64 ]]; then + cat >&2 <&2 + exit 2 +fi +if [[ -z "${MCP_INTERNAL_IMAGE_TAG:-}" || "${MCP_INTERNAL_IMAGE_TAG}" == "None" ]]; then + echo "ERROR: no MCP_INTERNAL_IMAGE_TAG provided and ECR repo '$MCP_INTERNAL_REPO' has no tagged images." >&2 + exit 2 +fi +validate_agent_worker_image_tag "${AGENT_WORKER_IMAGE_TAG}" + +# Backend coordinates for the eks-workloads remote-state lookups. +require_file "${REPO_ROOT}/terraform/backend.hcl" \ + "Run scripts/bootstrap_tfstate.sh --write-shared-backend first." +BACKEND_BUCKET=$(grep -E '^bucket' "${REPO_ROOT}/terraform/backend.hcl" \ + | head -n1 | sed -E 's/^bucket[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/') +BACKEND_REGION=$(grep -E '^region' "${REPO_ROOT}/terraform/backend.hcl" \ + | head -n1 | sed -E 's/^region[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/') + +cat < ${REPO_ROOT}/.state/eks-workloads.outputs.json" +else + "${REPO_ROOT}/scripts/tf.sh" eks-workloads output -json > "${REPO_ROOT}/.state/eks-workloads.outputs.json" +fi + +RENDERED_DIR="${REPO_ROOT}/deploy/aws/.rendered" + +############################################################################### +# 4. kubectl apply -k on the rendered manifests +############################################################################### + +echo "" +echo "=== 4/7 applying rendered manifests ===" +run kubectl apply -k "${RENDERED_DIR}/" + +############################################################################### +# 5. ADOT collector — Helm install with the rendered values file +############################################################################### + +if [[ "$SKIP_ADOT" != "1" ]]; then + echo "" + echo "=== 5/7 installing ADOT Collector DaemonSet ===" + run helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts || true + run helm repo update + + run helm upgrade --install adot \ + open-telemetry/opentelemetry-collector \ + --namespace "$ADOT_NAMESPACE" \ + --create-namespace \ + --values "${RENDERED_DIR}/observability/adot-values.yaml" \ + --wait +else + echo "" + echo "=== 5/7 ADOT install — SKIPPED (--skip-adot) ===" +fi + +############################################################################### +# 6. force rollout (belt-and-braces) + wait +# +# `kubectl apply -k` in step 4 updates the agent-worker-config ConfigMap +# in-place but does NOT re-create the worker Pod by itself. The Pod's env +# is captured at pod start time, so a Pod started before a ConfigMap +# update keeps the stale env (notably AGENTCORE_RUNTIME_ARN) until it is +# re-created. +# +# The Kubernetes-native fix lives in the Deployment template's +# `checksum/agent-worker-config` annotation (rendered with the SHA-256 of +# the ConfigMap body, see +# `terraform/eks-workloads/manifests.tf::locals.agent_worker_configmap_checksum`). +# Any change to the ConfigMap body changes the annotation, which K8s +# treats as a Pod-template change and rolls the Deployment automatically. +# +# We additionally run an explicit `rollout restart` here as belt-and- +# braces: idempotent, cheap, and guarantees we converge on a fresh Pod +# even in degenerate cases (annotation render skipped, manual ConfigMap +# edit, etc.). The `rollout status` below then waits for the new Pod to +# become Available. +############################################################################### + +echo "" +echo "=== 6/7 rolling and waiting for both Deployments to become Available ===" +run kubectl -n "$NAMESPACE" rollout restart deploy/agent-worker +run kubectl -n "$NAMESPACE" rollout status deploy/mcp-internal --timeout=600s +run kubectl -n "$NAMESPACE" rollout status deploy/agent-worker --timeout=600s + +############################################################################### +# 7. smoke +############################################################################### + +echo "" +echo "=== 7/7 bootstrap complete ===" +[[ "$DRY_RUN" == "1" ]] && exit 0 + +echo "" +echo "Pods:" +kubectl -n "$NAMESPACE" get pods -o wide + +echo "" +echo "SQS queue depth (sourced from terraform/data-stores/ outputs — no hard-coded URL):" +QUEUE_URL=$(jq -r '.agent_invoke_queue_url.value' "${REPO_ROOT}/.state/data-stores.outputs.json") +aws sqs get-queue-attributes \ + --queue-url "$QUEUE_URL" \ + --region "$AWS_REGION_OUT" \ + --attribute-names ApproximateNumberOfMessages ApproximateNumberOfMessagesNotVisible \ + --query Attributes --output json + +echo "" +echo "Tail the worker logs:" +echo " kubectl -n ${NAMESPACE} logs -f deploy/agent-worker" +echo "" +echo "Re-run the Phase-5 smoke (sends a fresh message, the worker should drain it):" +echo " python scripts/smoke.py \\" +echo " --fixture tests/fixtures/jira/issue_created.json \\" +echo " --target \"https://agent-sandbox.qodolabs.click/invocations\" \\" +echo " --region ${AWS_REGION_OUT} \\" +echo " --expect-status accepted" diff --git a/scripts/bootstrap_tfstate.sh b/scripts/bootstrap_tfstate.sh new file mode 100755 index 0000000..39f472b --- /dev/null +++ b/scripts/bootstrap_tfstate.sh @@ -0,0 +1,531 @@ +#!/usr/bin/env bash +############################################################################### +# scripts/bootstrap_tfstate.sh +# +# One-shot bootstrap for the Terraform remote backend (S3 with native S3 +# locking, no DynamoDB) used by every module under terraform/. Runs ONCE per +# (account, region) pair before any other terraform apply. Safe to re-run: +# every step is idempotent. +# +# Backend wiring strategy: PARTIAL BACKEND CONFIG. +# - Each terraform//backend.tf is committed (3-line stub that +# only declares `backend "s3" {}`). No bucket name, no account ID, +# no environment-specific values. +# - The shared values (bucket, region, encrypt, use_lockfile) live in +# ONE file at terraform/backend.hcl, which is gitignored. This script +# generates it under --write-shared-backend. +# - The per-module `key` is supplied at `terraform init` time by +# scripts/tf.sh (key = "agent-core/.tfstate"). +# Result: rotating the bucket means editing one file, not eleven; nothing +# operator-specific ever lands in git. +# +# Locking model: Terraform 1.10+ ships native locking via S3 conditional +# writes when `use_lockfile = true` is set in the backend block. The +# generated terraform/backend.hcl turns this on. No DynamoDB lock table +# is required (or created). +# +# Bucket namespace: account-regional (AWS, March 2026 GA). +# Bucket names that match `---an` live in +# the per-account-per-region namespace and do NOT collide with the +# global S3 namespace. The CreateBucket call sends +# `--bucket-namespace account-regional` (x-amz-bucket-namespace header) +# to put the bucket in that namespace; without the flag AWS returns +# `MissingNamespaceHeader`. Requires **AWS CLI v2 >= 2.34.7** (the +# release that added the flag to `s3api create-bucket`). Run +# `aws --version` if in doubt; older installs upgrade with +# `brew upgrade awscli` or by re-running the official installer. +# +# Lockdown defaults (all enforced + verified post-create): +# - Block Public Access: all four flags ON +# (BlockPublicAcls, IgnorePublicAcls, BlockPublicPolicy, +# RestrictPublicBuckets). +# - Bucket policy pins access to: +# * this AWS account only (aws:PrincipalAccount) +# * the bucket's home region only (aws:RequestedRegion) +# * HTTPS/TLS only (aws:SecureTransport) +# - Object ownership: BucketOwnerEnforced (ACLs disabled). +# - Versioning: Enabled. +# - Lifecycle: +# * NoncurrentVersionExpiration = 30 days +# (current `.tfstate` is always preserved; only non-current +# object versions expire so the bucket never grows unbounded across +# many apply / destroy cycles). +# * AbortIncompleteMultipartUpload = 7 days +# (defensive cleanup for failed terraform pushes; tfstate is +# tiny so this should never fire, but it's free insurance). +# * Re-applied on every bootstrap (idempotent), including +# --skip-bucket runs, so existing buckets get the rule on the +# next invocation without recreating anything. +# - SSE: AES256 (AWS-enforced default since 2023-01-05; not asserted +# by this script because it cannot be turned off and re-asserting +# it would only add noise. Switch to SSE-KMS in put-bucket-encryption +# if you ever need a kms:Decrypt gate on top of IAM). +# Block Public Access is fully compatible with Terraform remote state because +# Terraform talks to S3 over the AWS API using the operator's IAM credentials, +# never via the public-internet paths BPA blocks. Do not disable BPA. +# +# Per-module blast radius: every terraform// keeps its own state +# object at "agent-core/.tfstate". Modules never share state, so +# `terraform destroy` (or a bad plan) in one module cannot read or mutate +# any other module's resources. Terraform creates the per-module state +# object on first successful apply; nothing in the bucket needs to +# pre-exist except the bucket itself. +# +# Usage: +# scripts/bootstrap_tfstate.sh [options] +# +# Globally-unique S3 bucket name (required). +# Convention: terraform-state--- +# +# Options: +# --region REGION AWS region (default: us-east-1). +# --write-shared-backend After creating the bucket, write a single +# terraform/backend.hcl containing the shared +# backend values (bucket, region, encrypt, +# use_lockfile) consumed by every module via +# `terraform init -backend-config`. The file +# is gitignored. This is the recommended way +# to wire the backend. +# --skip-bucket Skip S3 bucket creation (use when the bucket +# exists and was provisioned out-of-band). +# -h | --help Print this help and exit. +# +# Example (run after `aws sts get-caller-identity` confirms the +# intended account): +# ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +# AWS_REGION=${AWS_REGION:-us-east-1} +# ./scripts/bootstrap_tfstate.sh \ +# "terraform-state-${ACCOUNT_ID}-${AWS_REGION}-an" \ +# --write-shared-backend +# +############################################################################### + +set -euo pipefail + +############################################################################### +# Helpers +############################################################################### + +banner() { + local title="$1" + printf '\n========================================================================\n' + printf '%s\n' "$title" + printf '========================================================================\n' +} + +err() { + printf 'ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + sed -n '2,/^###############################################################################$/p' "$0" \ + | sed 's/^# \{0,1\}//' \ + | sed '$d' + exit "${1:-0}" +} + +############################################################################### +# Argument parsing +############################################################################### + +BUCKET_NAME="" +AWS_REGION_ARG="us-east-1" +WRITE_SHARED_BACKEND=false +SKIP_BUCKET=false + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage 0 + ;; + --region) + AWS_REGION_ARG="${2:-}" + shift 2 + ;; + --write-shared-backend) + WRITE_SHARED_BACKEND=true + shift + ;; + --skip-bucket) + SKIP_BUCKET=true + shift + ;; + --*) + err "unknown option: $1" + ;; + *) + if [[ -z "$BUCKET_NAME" ]]; then + BUCKET_NAME="$1" + else + err "unexpected positional argument: $1 (bucket name already set to '$BUCKET_NAME')" + fi + shift + ;; + esac +done + +[[ -n "$BUCKET_NAME" ]] || err "bucket name is required (see --help)" +[[ -n "$AWS_REGION_ARG" ]] || err "--region cannot be empty" + +############################################################################### +# Pre-flight +############################################################################### + +banner "Pre-flight" + +command -v aws >/dev/null 2>&1 || err "aws CLI is not on PATH" + +# `--bucket-namespace` shipped with the account-regional namespace GA +# (March 2026, AWS CLI v2 >= ~2.27). Confirm it exists before any +# subsequent step relies on it. +if ! aws s3api create-bucket help 2>/dev/null | grep -q -- '--bucket-namespace'; then + err "this AWS CLI does not support 'aws s3api create-bucket --bucket-namespace'. + Account-regional namespace buckets need AWS CLI v2 >= 2.34.7 (the + release that added --bucket-namespace; March 2026 GA). + Current: $(aws --version 2>&1 | tr -d '\n') + Upgrade: + macOS (Homebrew): brew upgrade awscli + macOS (installer): https://awscli.amazonaws.com/AWSCLIV2.pkg + Linux: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html + Then re-run this script." +fi + +CALLER_ARN="$(aws sts get-caller-identity --query Arn --output text)" +ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" +printf ' Caller: %s\n' "$CALLER_ARN" +printf ' Account: %s\n' "$ACCOUNT_ID" +printf ' Region: %s\n' "$AWS_REGION_ARG" +printf ' Bucket: %s\n' "$BUCKET_NAME" +printf ' Namespace: account-regional (AR; bucket name suffix "-an" required)\n' +printf ' Locking: S3 native (use_lockfile = true; no DynamoDB)\n' + +# Soft warning: if the bucket name does not end in ---an +# the CreateBucket call below will fail with InvalidBucketName when paired +# with --bucket-namespace account-regional. Easier to fail loudly here. +EXPECTED_SUFFIX="-${ACCOUNT_ID}-${AWS_REGION_ARG}-an" +if [[ "$BUCKET_NAME" != *"$EXPECTED_SUFFIX" ]]; then + err "bucket name '${BUCKET_NAME}' does not end in '${EXPECTED_SUFFIX}'. + Account-regional namespace buckets MUST follow the naming convention + ---an (AWS S3 docs, account regional naming + rules). Either rename the bucket or run a global-namespace variant by + removing --bucket-namespace from the CreateBucket call." +fi + +############################################################################### +# Step 1 — S3 bucket (versioned, encrypted, public-access-blocked) +############################################################################### + +if $SKIP_BUCKET; then + banner "Step 1/2 S3 bucket (skipped: --skip-bucket)" +else + banner "Step 1/2 S3 bucket ${BUCKET_NAME}" + + if aws s3api head-bucket --bucket "$BUCKET_NAME" >/dev/null 2>&1; then + printf ' bucket already exists; reconfiguring in place.\n' + else + # Account-regional namespace (AWS, March 2026): bucket names ending + # in "---an" live in the per-account-per-region + # namespace, not the global S3 namespace. The CreateBucket call + # MUST carry the `--bucket-namespace account-regional` flag (which + # sets the x-amz-bucket-namespace request header). Without it the + # API returns: + # MissingNamespaceHeader: The requested bucket is an + # account-regional namespace bucket, but your request is missing + # the required x-amz-bucket-namespace header. + # Subsequent operations (put-bucket-versioning, put-bucket-policy, + # put-public-access-block, head-bucket, get-bucket-location, …) + # do NOT need the flag because the `-an` suffix is sufficient to + # disambiguate the namespace at lookup time. + printf ' creating account-regional namespace bucket in %s ...\n' "$AWS_REGION_ARG" + if [[ "$AWS_REGION_ARG" == "us-east-1" ]]; then + # us-east-1 is the only region where the create-bucket API rejects + # an explicit LocationConstraint. + aws s3api create-bucket \ + --bucket "$BUCKET_NAME" \ + --region "$AWS_REGION_ARG" \ + --bucket-namespace account-regional >/dev/null + else + aws s3api create-bucket \ + --bucket "$BUCKET_NAME" \ + --region "$AWS_REGION_ARG" \ + --bucket-namespace account-regional \ + --create-bucket-configuration "LocationConstraint=$AWS_REGION_ARG" >/dev/null + fi + fi + + printf ' enabling versioning...\n' + aws s3api put-bucket-versioning \ + --bucket "$BUCKET_NAME" \ + --versioning-configuration Status=Enabled + + # Server-side encryption is intentionally NOT configured here. + # Since 2023-01-05, S3 applies SSE-S3 (AES256) to every new bucket and + # every new object automatically, at no charge, and it cannot be + # disabled. We let that AWS-enforced default take effect rather than + # re-asserting it from the script. If you ever need stronger gating + # (e.g., SSE-KMS with a customer-managed key so reads also require + # kms:Decrypt), reintroduce a put-bucket-encryption call here. + + printf ' enforcing bucket-owner ownership...\n' + aws s3api put-bucket-ownership-controls \ + --bucket "$BUCKET_NAME" \ + --ownership-controls 'Rules=[{ObjectOwnership=BucketOwnerEnforced}]' + + printf ' blocking all public access (all four BPA flags)...\n' + aws s3api put-public-access-block \ + --bucket "$BUCKET_NAME" \ + --public-access-block-configuration \ + 'BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true' + + # Account-and-region pinning + HTTPS-only via a Deny bucket policy. + # Three Deny statements, evaluated explicitly by IAM regardless of any + # other Allow: + # 1. Deny anything not from this AWS account (aws:PrincipalAccount). + # 2. Deny calls hitting the bucket from a different region than the + # one we just created it in (aws:RequestedRegion). + # 3. Deny non-TLS (HTTP) calls (aws:SecureTransport). + # The policy contains only Deny statements, so BPA's BlockPublicPolicy + # does not reject it (BPA blocks policies that grant public access). + printf ' pinning bucket to account %s + region %s + HTTPS-only...\n' \ + "$ACCOUNT_ID" "$AWS_REGION_ARG" + POLICY_JSON=$(cat <.tfstate` +# (and a `.tfstate.tflock` artifact per acquire/release pair). After many +# deploys this dominates the bucket. The lifecycle policy below ages out +# noncurrent versions after 30 days; the CURRENT version of every object +# is untouched, so: +# - active state files (and the post-`destroy` 0-resource manifests +# used by re-apply) are kept indefinitely. +# - corrupted-state recovery has a 30-day rewind window. +# - operators who want zero history can run the matching purge in +# scripts/teardown_sandbox.sh `phase_state_purge` (runs at the end +# of every successful teardown). +# +# AbortIncompleteMultipartUpload = 7 days is a defensive backstop for any +# failed `aws s3 cp` or terraform-internal multipart push; tfstate +# objects are tiny (KB scale) so this should never fire in practice. +# +# Idempotent: put-bucket-lifecycle-configuration overwrites the rule set +# wholesale. Re-running this script (with or without --skip-bucket) +# brings any pre-existing bucket up to spec. + +printf ' installing lifecycle rule (auto-expire noncurrent state versions after 30d)...\n' +LIFECYCLE_JSON=$(cat <<'EOF' +{ + "Rules": [ + { + "ID": "expire-noncurrent-tfstate-versions", + "Status": "Enabled", + "Filter": { "Prefix": "" }, + "NoncurrentVersionExpiration": { + "NoncurrentDays": 30 + }, + "AbortIncompleteMultipartUpload": { + "DaysAfterInitiation": 7 + } + } + ] +} +EOF +) +aws s3api put-bucket-lifecycle-configuration \ + --bucket "$BUCKET_NAME" \ + --lifecycle-configuration "$LIFECYCLE_JSON" + +############################################################################### +# Step 1 verification — re-read BPA, region, policy, AND lifecycle; assert each. +############################################################################### + +banner "Verifying bucket lockdown" + +ACTUAL_REGION="$(aws s3api get-bucket-location \ + --bucket "$BUCKET_NAME" \ + --query 'LocationConstraint' --output text)" +# us-east-1 reports as 'None' / null (S3 API convention for the default region). +[[ "$ACTUAL_REGION" == "None" || -z "$ACTUAL_REGION" ]] && ACTUAL_REGION="us-east-1" + +if [[ "$ACTUAL_REGION" != "$AWS_REGION_ARG" ]]; then + err "bucket region mismatch: expected '${AWS_REGION_ARG}', got '${ACTUAL_REGION}'" +fi +printf ' region: %s (matches expected)\n' "$ACTUAL_REGION" + +BPA_JSON="$(aws s3api get-public-access-block \ + --bucket "$BUCKET_NAME" \ + --query 'PublicAccessBlockConfiguration' --output json)" +for flag in BlockPublicAcls IgnorePublicAcls BlockPublicPolicy RestrictPublicBuckets; do + val="$(printf '%s' "$BPA_JSON" | python3 -c "import json,sys; print(json.load(sys.stdin)['$flag'])")" + if [[ "$val" != "True" && "$val" != "true" ]]; then + err "BPA flag ${flag} is not enabled (value: ${val})" + fi + printf ' BPA %-22s on\n' "$flag" +done + +POLICY_STATUS="$(aws s3api get-bucket-policy-status \ + --bucket "$BUCKET_NAME" \ + --query 'PolicyStatus.IsPublic' --output text 2>/dev/null || echo 'False')" +if [[ "$POLICY_STATUS" != "False" ]]; then + err "bucket policy is reported as public (IsPublic=${POLICY_STATUS})" +fi +printf ' policy.IsPublic: False (account+region+TLS pin in force)\n' + +LIFECYCLE_DAYS="$(aws s3api get-bucket-lifecycle-configuration \ + --bucket "$BUCKET_NAME" \ + --query 'Rules[?ID==`expire-noncurrent-tfstate-versions` && Status==`Enabled`].NoncurrentVersionExpiration.NoncurrentDays | [0]' \ + --output text 2>/dev/null || echo 'None')" +if [[ "$LIFECYCLE_DAYS" == "None" || -z "$LIFECYCLE_DAYS" ]]; then + err "lifecycle rule 'expire-noncurrent-tfstate-versions' missing or disabled. + Re-run this script (omit --skip-bucket for the bucket lifecycle step if it was skipped earlier) + to install it, or apply it by hand: + aws s3api put-bucket-lifecycle-configuration --bucket ${BUCKET_NAME} \\ + --lifecycle-configuration " +fi +printf ' lifecycle: noncurrent versions expire after %s day(s)\n' "$LIFECYCLE_DAYS" + +############################################################################### +# Step 2 — (optional) write per-module backend.tf files +############################################################################### + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TF_DIR="${REPO_ROOT}/terraform" + +if $WRITE_SHARED_BACKEND; then + banner "Step 2/2 Writing shared backend config terraform/backend.hcl" + + [[ -d "$TF_DIR" ]] || err "terraform/ directory not found at ${TF_DIR}" + + SHARED_BACKEND_FILE="${TF_DIR}/backend.hcl" + cat > "$SHARED_BACKEND_FILE" </backend.tf (which is the +# committed 3-line stub `terraform { backend "s3" {} }`) via partial +# backend config: +# +# terraform -chdir=terraform/ init \\ +# -backend-config=../backend.hcl \\ +# -backend-config="key=agent-core/.tfstate" +# +# The scripts/tf.sh wrapper does that for you — prefer it over raw +# terraform invocations. +# +# Locking: S3-native via use_lockfile (Terraform >= 1.10). Each module +# writes only to its own key, so destroy/apply blast radius is scoped +# to that module alone. +bucket = "${BUCKET_NAME}" +region = "${AWS_REGION_ARG}" +encrypt = true +use_lockfile = true +EOF + printf ' wrote %s\n' "${SHARED_BACKEND_FILE#${REPO_ROOT}/}" + printf ' (gitignored; each module reads it via -backend-config at init time.)\n' +else + banner "Step 2/2 Shared backend config (skipped: --write-shared-backend not set)" + printf ' Re-run with --write-shared-backend to generate terraform/backend.hcl,\n' + printf ' or hand-write it using the template in terraform/README.md.\n' +fi + +############################################################################### +# Summary +############################################################################### + +banner "Bootstrap complete" + +cat </backend.tf COMMITTED 3-line stub. Identical + across modules. Just declares + \`backend "s3" {}\`; no values. + + - terraform/backend.hcl LOCAL file holding the shared + bucket/region/encrypt/use_lockfile. + Gitignored. Generated by this + script under --write-shared-backend. + + - agent-core/.tfstate REMOTE object inside the S3 bucket. + Created automatically by Terraform + on the first successful apply for + that module. You never pre-create + it; the bucket is the only S3 + artifact the bootstrap creates. + +Next steps: + 1. If you didn't pass --write-shared-backend, write terraform/backend.hcl + by hand using the template in terraform/README.md, or re-run this + script with the flag. + 2. Use the wrapper script for every terraform invocation — it supplies + the partial-backend flags for you: + scripts/tf.sh init + scripts/tf.sh plan + scripts/tf.sh apply + The remote .tfstate object appears on first successful apply. + 3. Continue with the apply order in terraform/README.md or the worked + recipe in aws-deploy-plan.md / docs/runbook-sandbox-deploy.md. +EOF diff --git a/scripts/build_agent_worker_image.sh b/scripts/build_agent_worker_image.sh new file mode 100755 index 0000000..672ec0f --- /dev/null +++ b/scripts/build_agent_worker_image.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +############################################################################### +# scripts/build_agent_worker_image.sh +# +# Build + push the agent-worker container image to the AgentCore ECR +# repository so `scripts/bootstrap_eks_workloads.sh` can pick it up via +# its `resolve_latest_ecr_tag worker-amd64-` lookup. +# +# Why this script exists: +# - The ECR repo URL lives in `.state/ecr-images.outputs.json` +# (`agentcore_repository_url`); copy/pasting it into `docker buildx` +# commands by hand has bitten us multiple times during sandbox rebuilds +# (architecture mismatch, missing `--push`, wrong tag prefix). +# - EKS Auto-Mode general-purpose nodes are amd64-only. The bootstrap +# script refuses any `AGENT_WORKER_IMAGE_TAG` that does not include +# `amd64` (see `validate_agent_worker_image_tag`); this builder +# guarantees the tag prefix matches. +# - The AGENTCORE_RUNTIME_ARN drift / boto3 timeout / optional-ARN fixes +# live in `src/` and are baked into this image; running this script +# on a fresh checkout produces a worker pod that has all of those +# fixes without operator memory. +# +# What it does: +# 1. Reads the ECR repo URL from `.state/ecr-images.outputs.json`. +# 2. Logs in to that ECR registry via `aws ecr get-login-password`. +# 3. Runs `docker buildx build --platform=linux/amd64 ... --push` with +# a tag of the shape `worker-amd64-` plus a +# `worker-amd64-latest` floating tag. +# 4. Emits a one-line JSON audit record on stdout (same shape as +# `scripts/build_local_image.sh` so log pipelines can ingest both). +# +# This script is intentionally separate from `scripts/build_local_image.sh` +# (which targets local Docker Desktop / `make smoke`) and from +# `scripts/promote_to_agentcore.sh` (which promotes the AgentCore Runtime +# image, not the worker image). +# +# Usage: +# scripts/build_agent_worker_image.sh # default tag +# scripts/build_agent_worker_image.sh --tag worker-amd64-1.0.5 +# scripts/build_agent_worker_image.sh --no-latest # skip floating tag +# +# Exit codes: +# 0 image built and pushed +# 1 docker / buildx failed +# 2 invalid argument or missing tooling / state +############################################################################### + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +usage() { + cat <<'USAGE' +build_agent_worker_image.sh — build + push the agent-worker image to ECR + +USAGE: + scripts/build_agent_worker_image.sh [--tag ] [--no-latest] + +OPTIONS: + --tag Override the auto-generated `worker-amd64-` + tag. Must include `amd64` (enforced by + bootstrap_eks_workloads.sh::validate_agent_worker_image_tag). + --no-latest Skip pushing the `worker-amd64-latest` floating tag. + -h, --help Show this help. + +ENVIRONMENT: + AWS_REGION Region of the AgentCore ECR repo. Defaults to the + value embedded in the repo URL when unset. + AWS_PROFILE Optional AWS named profile. + +REQUIRES on PATH: + docker (with buildx), jq, aws + +OUTPUT: + docker buildx log streamed to stderr. + Final JSON audit line on stdout, e.g.: + {"event":"build_agent_worker_image","tag":"...","repo":"...","timestamp":"..."} +USAGE +} + +############################################################################### +# 1. Argument parsing +############################################################################### + +PUSH_LATEST=1 +EXPLICIT_TAG="" + +while [[ $# -gt 0 ]]; do + case "$1" in + -h | --help) usage; exit 0 ;; + --tag) + if [[ $# -lt 2 || -z "${2:-}" ]]; then + echo "ERROR: --tag requires a value." >&2 + exit 2 + fi + EXPLICIT_TAG="$2" + shift 2 + ;; + --no-latest) + PUSH_LATEST=0 + shift + ;; + *) + echo "ERROR: unrecognised argument '$1'." >&2 + usage >&2 + exit 2 + ;; + esac +done + +############################################################################### +# 2. Tooling + state preflight +############################################################################### + +require_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "ERROR: $1 is required on PATH." >&2 + exit 2 + fi +} +require_cmd docker +require_cmd jq +require_cmd aws + +if ! docker buildx version >/dev/null 2>&1; then + echo "ERROR: docker buildx is required (install via Docker Desktop or 'docker buildx install')." >&2 + exit 2 +fi + +ECR_STATE="${REPO_ROOT}/.state/ecr-images.outputs.json" +if [[ ! -f "$ECR_STATE" ]]; then + echo "ERROR: ${ECR_STATE} not found. Run 'scripts/tf.sh ecr-images apply' first." >&2 + exit 2 +fi + +ECR_REPO_URL="$(jq -r '.agentcore_repository_url.value // empty' "$ECR_STATE")" +if [[ -z "$ECR_REPO_URL" || "$ECR_REPO_URL" == "null" ]]; then + echo "ERROR: agentcore_repository_url not found in ${ECR_STATE}." >&2 + exit 2 +fi + +ECR_HOST="${ECR_REPO_URL%%/*}" +# Region heuristic: registry hostname is `.dkr.ecr..amazonaws.com`. +DERIVED_REGION="$(printf '%s' "$ECR_HOST" | awk -F. '{print $4}')" +AWS_REGION="${AWS_REGION:-${DERIVED_REGION}}" +if [[ -z "$AWS_REGION" ]]; then + echo "ERROR: could not derive AWS_REGION from ${ECR_HOST}; export AWS_REGION manually." >&2 + exit 2 +fi +export AWS_REGION + +############################################################################### +# 3. Tag derivation +############################################################################### + +TIMESTAMP_UTC="$(date -u +'%Y%m%d%H%M%S')" +if [[ -n "$EXPLICIT_TAG" ]]; then + PRIMARY_TAG="$EXPLICIT_TAG" +else + PRIMARY_TAG="worker-amd64-${TIMESTAMP_UTC}" +fi + +if [[ "$PRIMARY_TAG" != *amd64* ]]; then + cat >&2 <&2 +aws ecr get-login-password --region "$AWS_REGION" \ + | docker login --username AWS --password-stdin "$ECR_HOST" >&2 + +echo "=== docker buildx build --platform=linux/amd64 -> ${PRIMARY_REF} ===" >&2 + +declare -a BUILD_ARGS=( + --platform=linux/amd64 + --tag "$PRIMARY_REF" + --file Dockerfile + --push +) +if [[ "$PUSH_LATEST" -eq 1 ]]; then + BUILD_ARGS+=(--tag "$LATEST_REF") +fi +BUILD_ARGS+=(.) + +docker buildx build "${BUILD_ARGS[@]}" >&2 + +############################################################################### +# 5. Audit log +############################################################################### + +ISO_TS="$(date -u +'%Y-%m-%dT%H:%M:%SZ')" +LATEST_FIELD="null" +if [[ "$PUSH_LATEST" -eq 1 ]]; then + LATEST_FIELD="\"${LATEST_REF}\"" +fi + +printf '{"event":"build_agent_worker_image","tag":"%s","ref":"%s","latest_ref":%s,"repo":"%s","region":"%s","timestamp":"%s"}\n' \ + "$PRIMARY_TAG" \ + "$PRIMARY_REF" \ + "$LATEST_FIELD" \ + "$ECR_REPO_URL" \ + "$AWS_REGION" \ + "$ISO_TS" + +echo "" >&2 +echo "Image pushed. Next step:" >&2 +echo " scripts/bootstrap_eks_workloads.sh" >&2 +echo " (auto-discovers the new tag via resolve_latest_ecr_tag)" >&2 diff --git a/scripts/build_mcp_image.sh b/scripts/build_mcp_image.sh index 5af68fa..7efce53 100755 --- a/scripts/build_mcp_image.sh +++ b/scripts/build_mcp_image.sh @@ -5,18 +5,29 @@ # Local builder for the `mcp-internal` HTTP-streamable MCP server image. # # This script: -# 1. resolves a Spring Boot fat-JAR named `mcp/mcp-internal-*.jar` -# (managed via Git LFS — see .gitattributes), -# 2. runs `docker build` with `mcp/` as the build context, -# 3. tags the resulting image as `mcp-internal/server:local` (the tag -# consumed by deploy/local/mcp-internal.yaml), and -# 4. emits a one-line JSON audit record on stdout — same shape as +# 1. resolves a Spring Boot fat-JAR from outside this repository (the +# JAR is no longer tracked in-tree -- see mcp/README.md "External +# JAR contract"), +# 2. stages it into `mcp/.build/mcp-internal.jar` so `docker build` +# sees it inside the build context, +# 3. runs `docker build` with `mcp/` as the build context, +# 4. tags the resulting image as `mcp-internal/server:local` (the tag +# consumed by deploy/local/mcp-internal.yaml), +# 5. cleans up `mcp/.build/` on exit (success, failure, or signal), +# 6. emits a one-line JSON audit record on stdout -- same shape as # scripts/promote_mcp_image.sh so log pipelines can ingest both. # +# JAR resolution order (first hit wins): +# 1. `--jar ` explicit CLI override. +# 2. `MCP_INTERNAL_JAR` env var absolute path to a fat-JAR. +# 3. newest `mcp-internal-*.jar` under +# `${MCP_INTERNAL_REPO:-$HOME/code/github/mcp-internal}/build/libs/` +# (the upstream Gradle build's standard output directory). +# # The companion script for production digest pinning is # scripts/promote_mcp_image.sh (the production-only image-pin / bearer-token -# rotation seam); this one is for developer laptops only and is referenced -# by the Makefile `build-mcp` target. +# rotation seam); this one is for developer laptops and CI only and is +# referenced by the Makefile `build-mcp` target. ############################################################################### set -euo pipefail @@ -30,26 +41,36 @@ USAGE: OPTIONS: --tag Image tag to produce (default: mcp-internal/server:local). - --jar Explicit path to the Spring Boot JAR. Defaults to the - newest match of `mcp/mcp-internal-*.jar` in the repo. + --jar Explicit path to the Spring Boot fat-JAR. Overrides the + MCP_INTERNAL_JAR env var and the MCP_INTERNAL_REPO default. -h, --help Show this help. +ENVIRONMENT: + MCP_INTERNAL_JAR Absolute path to the fat-JAR to bake in. Used when + --jar is not passed. + MCP_INTERNAL_REPO Path to a checkout of the upstream mcp-internal + Gradle project (default: + $HOME/code/github/mcp-internal). The script picks + the newest `build/libs/mcp-internal-*.jar` under + this directory when neither --jar nor + MCP_INTERNAL_JAR is set. + REQUIRES on PATH: docker PREREQUISITE (one-time, host-level): - brew install git-lfs && git lfs install - git lfs pull # after clone, to materialise mcp/mcp-internal-*.jar + cd && ./gradlew bootJar + # produces build/libs/mcp-internal-.jar that this script picks up OUTPUT: * `docker build` log streamed to stderr. * Final JSON audit line on stdout, e.g.: - {"event":"build_mcp_image","tag":"mcp-internal/server:local","jar":"mcp/mcp-internal-1.0.4.jar","digest":"sha256:...","timestamp":"2026-05-01T22:30:00Z"} + {"event":"build_mcp_image","tag":"mcp-internal/server:local","jar_source":"/Users/me/code/github/mcp-internal/build/libs/mcp-internal-1.0.4.jar","jar_size_bytes":62001585,"image_id":"sha256:...","timestamp":"2026-05-01T22:30:00Z"} EXIT CODES: 0 image built and tagged 1 docker build failed - 2 no JAR found (run `git lfs pull`) or invalid argument + 2 no JAR found / invalid argument USAGE } @@ -127,72 +148,112 @@ if ! command -v docker >/dev/null 2>&1; then fi ############################################################################### -# 3. Resolve the JAR +# 3. Resolve the JAR (external; never inside this repo) ############################################################################### -banner "Resolving mcp-internal JAR" +banner "Resolving mcp-internal JAR (external)" + +# Resolution order: --jar > MCP_INTERNAL_JAR > newest match under +# ${MCP_INTERNAL_REPO}/build/libs. The default MCP_INTERNAL_REPO lives +# alongside this checkout to match the convention documented in +# mcp/README.md "External JAR contract". +MCP_INTERNAL_REPO_DEFAULT="${HOME}/code/github/mcp-internal" +MCP_INTERNAL_REPO="${MCP_INTERNAL_REPO:-${MCP_INTERNAL_REPO_DEFAULT}}" if [[ -n "${EXPLICIT_JAR}" ]]; then - JAR_PATH="${EXPLICIT_JAR}" + JAR_SOURCE="${EXPLICIT_JAR}" + JAR_RESOLUTION="--jar flag" +elif [[ -n "${MCP_INTERNAL_JAR:-}" ]]; then + JAR_SOURCE="${MCP_INTERNAL_JAR}" + JAR_RESOLUTION="MCP_INTERNAL_JAR env var" else - # `ls -t` so a developer with multiple JARs picks the newest one; - # `head -n 1` keeps the resolution deterministic. - JAR_PATH="$(ls -t mcp/mcp-internal-*.jar 2>/dev/null | head -n 1 || true)" + # `ls -t` so a developer who has built multiple versions picks the + # newest one; `head -n 1` keeps the resolution deterministic. The + # `-plain.jar` artifact Gradle also emits is intentionally excluded + # by the glob (we want the Spring Boot fat-JAR, not the plain one). + JAR_SOURCE="$(ls -t "${MCP_INTERNAL_REPO}/build/libs"/mcp-internal-*.jar 2>/dev/null \ + | grep -v -- '-plain\.jar$' \ + | head -n 1 || true)" + JAR_RESOLUTION="default (${MCP_INTERNAL_REPO}/build/libs)" fi -if [[ -z "${JAR_PATH}" || ! -f "${JAR_PATH}" ]]; then +if [[ -z "${JAR_SOURCE}" || ! -f "${JAR_SOURCE}" ]]; then cat >&2 <} + 2. MCP_INTERNAL_JAR env var : ${MCP_INTERNAL_JAR:-} + 3. ${MCP_INTERNAL_REPO}/build/libs/mcp-internal-*.jar -If you just cloned the repository, the JAR is tracked via Git LFS -(.gitattributes pins mcp/*.jar). Run: +The mcp-internal JAR is no longer tracked in this repository. Build it +in the upstream Gradle project and re-run, e.g.: - git lfs install # one-time, per host - git lfs pull # in this clone + cd ${MCP_INTERNAL_REPO} + ./gradlew bootJar -Then re-run this script. +…then re-run this script. If your checkout lives elsewhere, point +\`MCP_INTERNAL_REPO\` at it or pass \`--jar /abs/path/to/mcp-internal-X.Y.Z.jar\`. + +See mcp/README.md "External JAR contract" for full details. MISSING exit 2 fi -# A Git-LFS pointer file is ~130 bytes; the real JAR is ~62 MB. Catch -# the common "developer forgot `git lfs pull`" footgun before docker -# wastes a layer on a useless 130-byte payload. -JAR_SIZE_BYTES="$(wc -c <"${JAR_PATH}" | tr -d ' ')" +# Spring Boot fat-JARs are typically 50–80 MB; anything dramatically +# smaller is almost certainly a `-plain.jar` mistakenly passed in or a +# truncated download. 1 MB is a safe floor. +JAR_SIZE_BYTES="$(wc -c <"${JAR_SOURCE}" | tr -d ' ')" if [[ "${JAR_SIZE_BYTES}" -lt 1048576 ]]; then - cat >&2 <&2 <&2 +echo "Resolved JAR: ${JAR_SOURCE} (${JAR_SIZE_BYTES} bytes, via ${JAR_RESOLUTION})" >&2 + +############################################################################### +# 4. Stage the JAR into the build context +############################################################################### + +# `mcp/.build/` is gitignored (see top-level .gitignore) and excluded +# from human-facing tooling. The Dockerfile COPYs the staged file by +# its deterministic name `mcp-internal.jar`, so version bumps do not +# require Dockerfile edits. +STAGE_DIR="${REPO_ROOT}/mcp/.build" +STAGE_PATH="${STAGE_DIR}/mcp-internal.jar" + +cleanup_stage() { + local rc=$? + rm -rf "${STAGE_DIR}" + return $rc +} +trap 'cleanup_stage' EXIT INT TERM HUP + +mkdir -p "${STAGE_DIR}" +cp "${JAR_SOURCE}" "${STAGE_PATH}" +echo "Staged JAR: ${STAGE_PATH}" >&2 ############################################################################### -# 4. Build the image +# 5. Build the image ############################################################################### banner "docker build -t ${IMAGE_TAG} mcp/" -# Build context is `mcp/` so the JAR can be COPY'd by the Dockerfile via -# its glob (`mcp-internal-*.jar`). The Dockerfile itself is ignored from -# the context by mcp/.dockerignore but Docker still requires it as an -# argument when not at the context root. +# Build context is `mcp/` so the Dockerfile can COPY `.build/mcp-internal.jar` +# directly. The Dockerfile itself is excluded from the context payload by +# `mcp/.dockerignore` but Docker still requires it as the `--file` argument. docker build \ --tag "${IMAGE_TAG}" \ --file mcp/Dockerfile \ mcp/ >&2 ############################################################################### -# 5. Resolve the local image digest (best-effort; not all docker drivers +# 6. Resolve the local image digest (best-effort; not all docker drivers # surface a sha256 for locally-built images, in which case we emit # the empty string and the audit line still parses). ############################################################################### @@ -200,14 +261,14 @@ docker build \ IMAGE_DIGEST="$(docker image inspect --format='{{.Id}}' "${IMAGE_TAG}" 2>/dev/null || true)" ############################################################################### -# 6. Audit log +# 7. Audit log ############################################################################### TIMESTAMP="$(iso8601_utc)" -printf '{"event":"build_mcp_image","tag":"%s","jar":"%s","jar_size_bytes":%s,"image_id":"%s","timestamp":"%s"}\n' \ +printf '{"event":"build_mcp_image","tag":"%s","jar_source":"%s","jar_size_bytes":%s,"image_id":"%s","timestamp":"%s"}\n' \ "${IMAGE_TAG}" \ - "${JAR_PATH}" \ + "${JAR_SOURCE}" \ "${JAR_SIZE_BYTES}" \ "${IMAGE_DIGEST}" \ "${TIMESTAMP}" diff --git a/scripts/check_bedrock_model_access.py b/scripts/check_bedrock_model_access.py new file mode 100644 index 0000000..99fdd05 --- /dev/null +++ b/scripts/check_bedrock_model_access.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Verify Amazon Bedrock access for a target model ID. + +This utility is designed for fast "do we have access yet?" checks when an +account has newly been granted model permissions (for example, first-time +customer access to an Anthropic Claude model in Bedrock). + +What it verifies +---------------- +1) Control-plane discoverability (optional): + Calls ``bedrock:list_foundation_models`` and reports whether the requested + model appears in the account/region model summaries. + +2) Runtime invoke authorization (primary signal): + Calls ``bedrock-runtime:converse`` with a tiny prompt. A successful response + proves the credentials can invoke that model in the chosen region. + +Usage examples +-------------- + # Typical Sonnet 4.6 check (replace with the exact model ID you were given) + python scripts/check_bedrock_model_access.py \ + --model-id us.anthropic.claude-sonnet-4-6--v1:0 \ + --region us-east-1 + + # Fast auth-only check (skip listing if caller lacks list permissions) + python scripts/check_bedrock_model_access.py \ + --model-id us.anthropic.claude-sonnet-4-6--v1:0 \ + --skip-list +""" + +from __future__ import annotations + +import argparse +import json +import sys +from typing import Any + + +EXIT_OK = 0 +EXIT_NO_ACCESS = 1 +EXIT_USAGE = 2 + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="check_bedrock_model_access", + description=( + "Verify Bedrock model access by optionally listing available models " + "and performing a tiny converse() invocation." + ), + ) + parser.add_argument( + "--model-id", + required=True, + help=( + "Exact Bedrock model ID to verify, e.g. " + "'us.anthropic.claude-sonnet-4-6--v1:0'." + ), + ) + parser.add_argument( + "--region", + default="us-east-1", + help="AWS region to query (default: us-east-1).", + ) + parser.add_argument( + "--profile", + default=None, + help="Optional AWS profile name to use from local credentials.", + ) + parser.add_argument( + "--skip-list", + action="store_true", + help="Skip bedrock:list_foundation_models and only test runtime invoke.", + ) + parser.add_argument( + "--max-tokens", + type=int, + default=16, + help="Max output tokens for the test prompt (default: 16).", + ) + parser.add_argument( + "--timeout-seconds", + type=int, + default=30, + help="Read/connect timeout for boto3 clients (default: 30).", + ) + return parser.parse_args(argv) + + +def _build_session(profile: str | None) -> Any: + try: + import boto3 # type: ignore[import-untyped] + except ImportError as exc: # pragma: no cover - defensive + raise SystemExit( + "boto3 is required. Install with: pip install boto3" + ) from exc + return boto3.Session(profile_name=profile) if profile else boto3.Session() + + +def _build_client(session: Any, service_name: str, region: str, timeout_seconds: int) -> Any: + try: + from botocore.config import Config # type: ignore[import-untyped] + except ImportError as exc: # pragma: no cover - defensive + raise SystemExit( + "botocore is required. Install with: pip install boto3" + ) from exc + config = Config( + region_name=region, + connect_timeout=min(timeout_seconds, 10), + read_timeout=timeout_seconds, + retries={"max_attempts": 1, "mode": "standard"}, + ) + return session.client(service_name, config=config) + + +def _safe_client_error_details(exc: Exception) -> tuple[str, str]: + response = getattr(exc, "response", {}) or {} + error = response.get("Error", {}) if isinstance(response, dict) else {} + code = str(error.get("Code", "UnknownError")) + message = str(error.get("Message", str(exc))) + return code, message + + +def _list_models(bedrock_client: Any, model_id: str) -> tuple[bool, str]: + try: + summaries = bedrock_client.list_foundation_models(byOutputModality="TEXT").get( + "modelSummaries", [] + ) + except Exception as exc: + code, message = _safe_client_error_details(exc) + return False, f"list_failed code={code} message={message}" + discovered = any(summary.get("modelId") == model_id for summary in summaries) + if discovered: + return True, "listed" + return False, "not_listed" + + +def _invoke_probe(runtime_client: Any, model_id: str, max_tokens: int) -> tuple[bool, str]: + try: + response = runtime_client.converse( + modelId=model_id, + messages=[ + { + "role": "user", + "content": [{"text": "Reply with exactly: OK"}], + } + ], + inferenceConfig={"maxTokens": max_tokens, "temperature": 0}, + ) + except Exception as exc: + code, message = _safe_client_error_details(exc) + return False, f"invoke_failed code={code} message={message}" + + output = response.get("output", {}) + message = output.get("message", {}) + content = message.get("content", []) + text_parts = [ + str(part.get("text", "")) for part in content if isinstance(part, dict) and "text" in part + ] + normalized = " ".join(text_parts).strip() + return True, normalized + + +def _run(args: argparse.Namespace) -> int: + session = _build_session(args.profile) + bedrock_client = _build_client( + session, + service_name="bedrock", + region=args.region, + timeout_seconds=args.timeout_seconds, + ) + runtime_client = _build_client( + session, + service_name="bedrock-runtime", + region=args.region, + timeout_seconds=args.timeout_seconds, + ) + + listed: bool | None = None + list_note = "skipped" + if not args.skip_list: + listed, list_note = _list_models(bedrock_client, args.model_id) + + invoke_ok, invoke_note = _invoke_probe(runtime_client, args.model_id, args.max_tokens) + success = invoke_ok + payload = { + "status": "ok" if success else "no-access", + "region": args.region, + "model_id": args.model_id, + "listed": listed, + "list_note": list_note, + "invoke_ok": invoke_ok, + "invoke_note": invoke_note, + } + sys.stdout.write(json.dumps(payload, indent=2, sort_keys=True) + "\n") + return EXIT_OK if success else EXIT_NO_ACCESS + + +def main(argv: list[str] | None = None) -> int: + try: + args = _parse_args(sys.argv[1:] if argv is None else argv) + except SystemExit as exc: + return int(exc.code) if isinstance(exc.code, int) else EXIT_USAGE + return _run(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_docs_code_sync.py b/scripts/check_docs_code_sync.py new file mode 100755 index 0000000..9f1ae23 --- /dev/null +++ b/scripts/check_docs_code_sync.py @@ -0,0 +1,442 @@ +#!/usr/bin/env python3 +"""Assert that docs cite real code paths and the deferred register is current. + +This script runs in the PR-required ``docs-code-sync`` job in +``.github/workflows/ci.yml`` and exits non-zero when: + +1. Any markdown link of the form ``[label](path)`` in + ``docs/ARCHITECTURE.md`` / ``docs/runbook.md`` / ``docs/oncall.md`` + points at a path that does not exist in the working tree. +2. Any backtick-wrapped symbol of the form + ``agent.module.symbol`` in those docs cannot be imported (the doc + cites a symbol that cannot be imported from that path). +3. The ``## What's deferred to AWS production`` list in ``README.md`` + does not have a 1:1 mapping to the YAML frontmatter ``items:`` + array in ``docs/DEFERRED.md``. +4. Any entry in ``docs/DEFERRED.md`` with ``risk: high`` has a + ``next_review_date`` strictly less than today (forces the team to + re-up the deferral or close it). +5. Any high-risk entry within 14 days of ``next_review_date`` emits a + GitHub Actions warning so owners get lead time before the hard fail. +6. ``terraform/agentcore-runtime/README.md`` documents the same + ``MCP_BASE_URL``, ``agentcore_entrypoint`` default, and Python + ``runtime`` string as ``main.tf`` / ``variables.tf`` (and every + ``entrypoint:`` in ``.bedrock_agentcore.yaml`` matches the Terraform + default). + +The script is intentionally pure-Python with no third-party +dependencies so the PR job can run it directly with ``python``. +""" + +from __future__ import annotations + +import datetime as _dt +import importlib +import re +import sys +from pathlib import Path + +REPO_ROOT: Path = Path(__file__).resolve().parents[1] +DOCS: Path = REPO_ROOT / "docs" +README: Path = REPO_ROOT / "README.md" +DEFERRED_MD: Path = DOCS / "DEFERRED.md" +AGENTCORE_RUNTIME_DIR: Path = REPO_ROOT / "terraform" / "agentcore-runtime" +AGENTCORE_MAIN_TF: Path = AGENTCORE_RUNTIME_DIR / "main.tf" +AGENTCORE_VARIABLES_TF: Path = AGENTCORE_RUNTIME_DIR / "variables.tf" +AGENTCORE_README: Path = AGENTCORE_RUNTIME_DIR / "README.md" +BEDROCK_AGENTCORE_YAML: Path = REPO_ROOT / ".bedrock_agentcore.yaml" +DOC_FILES: tuple[Path, ...] = ( + DOCS / "ARCHITECTURE.md", + DOCS / "runbook.md", + DOCS / "oncall.md", +) + +LINK_RE = re.compile(r"\[(?P