docker-compose.stack.yml

# Production-shaped stack overlay (profile: stack), used by scripts/stack/*.sh
# and `make verify-stack`. It adds what the two-cell dev stack lacks and
# production has:
#   - a second Foghorn replica in cell B, so both cells have a leader and a
#     non-leader;
#   - an offchain Livepeer orchestrator+transcoder and gateway per cell, fronted
#     by nginx with the production media_ingest shape and discovered through
#     Quartermaster as a physical gateway (scripts/stack/seed.sql);
#   - a webhook receiver, Mailpit as SMTP, and the in-network scenario runner.
# Run it through scripts/stack/common.sh, which layers a generated overlay that
# resets every container_name and host port so several slots can coexist.

# scripts/stack/up.sh sets GO_LIVEPEER_IMAGE (in the slot env file) to the go-livepeer release
# scripts/resolve-golivepeer-release.sh resolves. The fallback only lets commands that create no
# container (down, logs) interpolate this file on a slot that was never brought up.
x-go-livepeer-image: &go-livepeer-image ${GO_LIVEPEER_IMAGE:-go-livepeer-unresolved-run-scripts-stack-up}

x-stack-logging: &stack-logging
  driver: "json-file"
  options:
    max-size: "50m"
    max-file: "3"

# Durable storage: one in-network S3 service (hostname s3) with one bucket per
# cell, so each cell keeps its own immutable backend. Every storage-aware
# service carries the literal tuple rather than interpolated env, so a service
# the runner recreates during fault injection keeps the same backend. Path-style
# presigned URLs (http://s3:8080/<bucket>/...) resolve for edges and the runner.
x-stack-s3-cell-a: &stack-s3-cell-a
  STORAGE_S3_ENDPOINT: http://s3:8080
  STORAGE_S3_BUCKET: fw-stack-central-primary
  STORAGE_S3_REGION: us-east-1
  STORAGE_S3_ACCESS_KEY: frameworks
  STORAGE_S3_SECRET_KEY: frameworks-secret
  STORAGE_S3_PREFIX: ""

x-stack-s3-cell-b: &stack-s3-cell-b
  STORAGE_S3_ENDPOINT: http://s3:8080
  STORAGE_S3_BUCKET: fw-stack-us-primary
  STORAGE_S3_REGION: us-east-1
  STORAGE_S3_ACCESS_KEY: frameworks
  STORAGE_S3_SECRET_KEY: frameworks-secret
  STORAGE_S3_PREFIX: ""

services:
  # Second control-plane replicas, on their own node (central-node-2, from
  # scripts/stack/seed.sql) as production runs them on separate hosts. The
  # network alias puts each behind the same name as the first replica, so every
  # client that dials commodore:19001 or quartermaster:19002 resolves both, the
  # way <service>.internal resolves every instance on the mesh.
  quartermaster-2:
    extends:
      file: docker-compose.yml
      service: quartermaster
    profiles: ["stack"]
    environment:
      QUARTERMASTER_HOST: quartermaster-2
      NODE_ID: central-node-2
    depends_on:
      postgres:
        condition: service_healthy
    networks:
      frameworks:
        aliases: [quartermaster]

  commodore-2:
    extends:
      file: docker-compose.yml
      service: commodore
    profiles: ["stack"]
    environment:
      COMMODORE_HOST: commodore-2
      NODE_ID: central-node-2
      SMTP_HOST: mailpit
      SMTP_PORT: "1025"
      SMTP_USER: ""
      SMTP_PASSWORD: ""
      SMTP_ALLOW_INSECURE: "true"
      FROM_EMAIL: stack@frameworks.network
    depends_on:
      postgres:
        condition: service_healthy
      quartermaster:
        condition: service_healthy
    networks:
      frameworks:
        aliases: [commodore]

  # Second cell-B replica. Its Quartermaster instance row and cluster
  # assignments come from scripts/stack/seed.sql before it starts.
  foghorn-b-2:
    extends:
      file: docker-compose.yml
      service: foghorn-b
    profiles: ["stack"]
    environment:
      <<: *stack-s3-cell-b
      FOGHORN_RELAY_ADVERTISE_ADDR: foghorn-b-2:18019
      FOGHORN_INSTANCE_ID: foghorn-b-2
      FOGHORN_HOST: foghorn-b-2
      PLAYBACK_WEBHOOK_ALLOW_PRIVATE_DESTINATIONS: "true"
    depends_on:
      postgres:
        condition: service_healthy
      quartermaster:
        condition: service_healthy
      foghorn-redis-b:
        condition: service_healthy

  # Offchain Livepeer, cell A: orchestrator+transcoder on CPU, gateway with the
  # production auth webhook into the cell's Foghorn, nginx front on the
  # physical infra FQDN Quartermaster advertises.
  livepeer-orch-a:
    image: *go-livepeer-image
    profiles: ["stack"]
    command:
      - -orchestrator
      - -transcoder
      - -network=offchain
      - -serviceAddr=livepeer-orch-a:8935
      - -httpAddr=0.0.0.0:8935
      - -cliAddr=127.0.0.1:7935
      - -orchSecret=stack-orch-secret
      - -v=4
    networks: [frameworks]
    cpus: 2.0
    mem_limit: 2g
    logging: *stack-logging

  livepeer-gateway-a:
    image: *go-livepeer-image
    profiles: ["stack"]
    command:
      - -gateway
      - -network=offchain
      - -httpAddr=0.0.0.0:8935
      - -httpIngest
      - -cliAddr=127.0.0.1:7935
      - -rtmpAddr=127.0.0.1:1936
      - -orchAddr=livepeer-orch-a:8935
      - -authWebhookUrl=http://foghorn:18008/webhooks/livepeer/auth
      - -trustedProxyCIDRs=${STACK_SUBNET:-10.231.1.0/24}
      - -v=4
    depends_on: [livepeer-orch-a]
    networks: [frameworks]
    mem_limit: 1g
    logging: *stack-logging

  livepeer-proxy-a:
    image: nginx:1.30.1-alpine@sha256:c819f83c54b0361f5557601bf5eb4943d09360e7a7fdf426afc466570f45874d
    profiles: ["stack"]
    environment:
      GATEWAY_FQDN: livepeer-gateway.stack-lp-a.infra.frameworks.network
      GATEWAY_UPSTREAM: livepeer-gateway-a:8935
    volumes:
      - ./scripts/stack/livepeer-proxy.conf.template:/etc/nginx/templates/default.conf.template:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/a.crt:/etc/frameworks/certs/gateway.crt:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/a.key:/etc/frameworks/certs/gateway.key:ro
    depends_on: [livepeer-gateway-a]
    networks:
      frameworks:
        aliases: [livepeer-gateway.stack-lp-a.infra.frameworks.network]
    logging: *stack-logging

  # Offchain Livepeer, cell B.
  livepeer-orch-b:
    image: *go-livepeer-image
    profiles: ["stack"]
    command:
      - -orchestrator
      - -transcoder
      - -network=offchain
      - -serviceAddr=livepeer-orch-b:8935
      - -httpAddr=0.0.0.0:8935
      - -cliAddr=127.0.0.1:7935
      - -orchSecret=stack-orch-secret
      - -v=4
    networks: [frameworks]
    cpus: 2.0
    mem_limit: 2g
    logging: *stack-logging

  livepeer-gateway-b:
    image: *go-livepeer-image
    profiles: ["stack"]
    command:
      - -gateway
      - -network=offchain
      - -httpAddr=0.0.0.0:8935
      - -httpIngest
      - -cliAddr=127.0.0.1:7935
      - -rtmpAddr=127.0.0.1:1936
      - -orchAddr=livepeer-orch-b:8935
      - -authWebhookUrl=http://foghorn-b:18008/webhooks/livepeer/auth
      - -trustedProxyCIDRs=${STACK_SUBNET:-10.231.1.0/24}
      - -v=4
    depends_on: [livepeer-orch-b]
    networks: [frameworks]
    mem_limit: 1g
    logging: *stack-logging

  livepeer-proxy-b:
    image: nginx:1.30.1-alpine@sha256:c819f83c54b0361f5557601bf5eb4943d09360e7a7fdf426afc466570f45874d
    profiles: ["stack"]
    environment:
      GATEWAY_FQDN: livepeer-gateway.stack-lp-b.infra.frameworks.network
      GATEWAY_UPSTREAM: livepeer-gateway-b:8935
    volumes:
      - ./scripts/stack/livepeer-proxy.conf.template:/etc/nginx/templates/default.conf.template:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/b.crt:/etc/frameworks/certs/gateway.crt:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/b.key:/etc/frameworks/certs/gateway.key:ro
    depends_on: [livepeer-gateway-b]
    networks:
      frameworks:
        aliases: [livepeer-gateway.stack-lp-b.infra.frameworks.network]
    logging: *stack-logging

  # Cell A's public control name in front of each of its Foghorn instances,
  # for the edge scenario 30 switches to TLS: the name resolves to both fronts,
  # as a production cell's foghorn.<cluster>.<root> resolves to its instances.
  # See scripts/stack/foghorn-control-tls.conf.template.
  foghorn-tls-a-1:
    image: nginx:1.30.1-alpine@sha256:c819f83c54b0361f5557601bf5eb4943d09360e7a7fdf426afc466570f45874d
    profiles: ["stack"]
    environment:
      FOGHORN_CONTROL_FQDN: foghorn.demo-media.stack.frameworks.network
      FOGHORN_UPSTREAM: foghorn:18029
    volumes:
      - ./scripts/stack/foghorn-control-tls.conf.template:/etc/nginx/templates/default.conf.template:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/foghorn-a.crt:/etc/frameworks/certs/foghorn.crt:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/foghorn-a.key:/etc/frameworks/certs/foghorn.key:ro
    networks:
      frameworks:
        aliases: [foghorn.demo-media.stack.frameworks.network]
    logging: *stack-logging

  foghorn-tls-a-2:
    image: nginx:1.30.1-alpine@sha256:c819f83c54b0361f5557601bf5eb4943d09360e7a7fdf426afc466570f45874d
    profiles: ["stack"]
    environment:
      FOGHORN_CONTROL_FQDN: foghorn.demo-media.stack.frameworks.network
      FOGHORN_UPSTREAM: foghorn-2:18029
    volumes:
      - ./scripts/stack/foghorn-control-tls.conf.template:/etc/nginx/templates/default.conf.template:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/foghorn-a.crt:/etc/frameworks/certs/foghorn.crt:ro
      - ${STACK_STATE_DIR:-./.stack/slot-1}/certs/foghorn-a.key:/etc/frameworks/certs/foghorn.key:ro
    networks:
      frameworks:
        aliases: [foghorn.demo-media.stack.frameworks.network]
    logging: *stack-logging

  webhook-receiver:
    image: python:3.13-alpine
    profiles: ["stack"]
    command: ["python3", "/app/webhook-receiver.py"]
    volumes:
      - ./scripts/stack/webhook-receiver.py:/app/webhook-receiver.py:ro
    networks: [frameworks]
    logging: *stack-logging

  # Chaos lets the registration scenario make SMTP fail once to exercise the
  # account-email outbox retry.
  mailpit:
    image: axllent/mailpit:v1.27
    profiles: ["stack"]
    command: ["--enable-chaos"]
    networks: [frameworks]
    logging: *stack-logging

  # Scenarios run here (docker compose exec). The Docker socket is for fault
  # injection into this slot's own project only: scripts/stack/lib.sh always
  # passes -p "$COMPOSE_PROJECT_NAME".
  stack-runner:
    build:
      context: scripts/stack
      dockerfile: runner.Dockerfile
    profiles: ["stack"]
    # Publishers are started from $(...) subshells and orphaned to PID 1; an init
    # reaps them so wait_publisher sees them exit instead of a lingering zombie.
    init: true
    environment:
      STACK_SLOT: ${STACK_SLOT:-1}
      STACK_STATE_DIR: /tmp/fw-stack
      COMPOSE_PROJECT_NAME: ${COMPOSE_PROJECT_NAME:-fw-stack-1}
      COMPOSE_PROFILES: ${COMPOSE_PROFILES:-edge,two-cell,stack,stack-s3-ceph}
      COMPOSE_FILE: /repo/docker-compose.yml:/repo/docker-compose.stack.yml:/repo/.stack/slot-${STACK_SLOT:-1}/reset.yml
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      CLICKHOUSE_USER: ${CLICKHOUSE_USER}
      CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
      SERVICE_TOKEN: ${SERVICE_TOKEN}
      JWT_SECRET: ${JWT_SECRET}
      GRPC_ALLOW_INSECURE: "true"
      SUPPLIER_NAME: ${SUPPLIER_NAME}
      SUPPLIER_ADDRESS: ${SUPPLIER_ADDRESS}
      SUPPLIER_VAT_NUMBER: ${SUPPLIER_VAT_NUMBER}
      SUPPLIER_REGISTRATION_NUMBER: ${SUPPLIER_REGISTRATION_NUMBER}
      SUPPLIER_COUNTRY: ${SUPPLIER_COUNTRY}
    volumes:
      - ./:/repo:ro
      - /var/run/docker.sock:/var/run/docker.sock
    networks: [frameworks]
    logging: *stack-logging

  # Stack-wide wiring into the existing services. The edge image uses the
  # production edge OS: the Mist tree scripts/stack/mist.Dockerfile builds links
  # against Ubuntu 24.04's C runtime.
  edge:
    image: ${STACK_EDGE_IMAGE:-frameworks-edge:dev}
    build:
      args:
        EDGE_BASE_IMAGE: ubuntu:24.04
    # Every Foghorn instance of the cell, as production renders the node's
    # control address list: a node whose instance goes away fails over to the
    # other one instead of staying without control.
    environment:
      FOGHORN_CONTROL_ADDR: foghorn:18029,foghorn-2:18029
    # The dev edge serves the repo's demo recordings from a bind mount; a stack
    # edge must never write into the source tree, so it gets a slot volume.
    volumes:
      - edge_a_storage:/data/storage
  edge-b:
    image: ${STACK_EDGE_IMAGE:-frameworks-edge:dev}
    environment:
      FOGHORN_CONTROL_ADDR: foghorn-b:18029,foghorn-b-2:18029
  foghorn:
    environment:
      <<: *stack-s3-cell-a
  foghorn-2:
    environment:
      <<: *stack-s3-cell-a
      PLAYBACK_WEBHOOK_ALLOW_PRIVATE_DESTINATIONS: "true"
      # Mist's balancer source for the edges this replica controls. The dev
      # fallback is <FOGHORN_HOST>:18008, but this replica serves public HTTP
      # on 18018.
      FOGHORN_URL: http://foghorn-2:18018
  foghorn-b:
    environment:
      <<: *stack-s3-cell-b
      PLAYBACK_WEBHOOK_ALLOW_PRIVATE_DESTINATIONS: "true"
  chandler:
    environment:
      <<: *stack-s3-cell-a
  storage-init:
    environment:
      <<: *stack-s3-cell-a
  # Cell B's descriptor: the same clean, one-time establishment storage-init
  # does for cell A, for the us-primary cell the two-cell seed creates.
  storage-init-b:
    extends:
      file: docker-compose.yml
      service: storage-init
    profiles: ["stack"]
    environment:
      <<: *stack-s3-cell-b
      CLUSTER_ID: us-primary

  # Ceph RGW is the S3 implementation production's provider runs, pinned as in
  # the storage conformance tests; its demo image is amd64-only, so arm64
  # engines get the pinned SeaweedFS instead. scripts/stack/common.sh enables
  # exactly one backend profile. The transparent test proxy answers s3:8080.
  s3-proxy:
    image: python:3.13-alpine
    profiles: ["stack"]
    command: ["python3", "/app/s3-fault-proxy.py"]
    environment:
      STACK_S3_UPSTREAM: s3-${STACK_S3:-ceph}:8080
      SERVICE_TOKEN: ${SERVICE_TOKEN}
    volumes:
      - ./scripts/stack/s3-fault-proxy.py:/app/s3-fault-proxy.py:ro
    networks:
      frameworks:
        aliases: [s3]
    logging: *stack-logging
  s3-ceph:
    image: quay.io/ceph/demo@sha256:522483cf07cfce6386b8e18a3edfa88a1b32c688dee231d0a6962b1513557723
    profiles: ["stack-s3-ceph"]
    environment:
      CEPH_DAEMON: demo
      DEMO_DAEMONS: osd rgw
      MON_IP: 127.0.0.1
      CEPH_PUBLIC_NETWORK: 0.0.0.0/0
      RGW_NAME: localhost
      CEPH_DEMO_UID: frameworks
      CEPH_DEMO_ACCESS_KEY: frameworks
      CEPH_DEMO_SECRET_KEY: frameworks-secret
    networks: [frameworks]
    logging: *stack-logging
  s3-seaweed:
    image: chrislusf/seaweedfs@sha256:ce9e796f1fe6f06968f4c04bdaf8f678dad9c8acdfef3d244133d71bfa6bf882
    profiles: ["stack-s3-seaweed"]
    # weed server's volume server defaults to 8080; move it so S3 answers there.
    command: ["server", "-s3", "-dir=/data", "-s3.port=8080", "-volume.port=18081"]
    networks: [frameworks]
    logging: *stack-logging
  commodore:
    environment:
      SMTP_HOST: mailpit
      SMTP_PORT: "1025"
      SMTP_USER: ""
      SMTP_PASSWORD: ""
      SMTP_ALLOW_INSECURE: "true"
      FROM_EMAIL: stack@frameworks.network
  purser:
    environment:
      SMTP_HOST: mailpit
      SMTP_PORT: "1025"
      SMTP_USER: ""
      SMTP_PASSWORD: ""
      SMTP_ALLOW_INSECURE: "true"
  lookout:
    environment:
      SMTP_HOST: mailpit
      SMTP_PORT: "1025"
      SMTP_USER: ""
      SMTP_PASSWORD: ""
      SMTP_ALLOW_INSECURE: "true"
  bosun:
    environment:
      SMTP_HOST: mailpit
      SMTP_PORT: "1025"
      SMTP_USER: ""
      SMTP_PASSWORD: ""
      SMTP_ALLOW_INSECURE: "true"
      BOSUN_ALLOW_PRIVATE_DESTINATIONS: "true"

networks:
  frameworks:
    ipam:
      config:
        - subnet: ${STACK_SUBNET:-10.231.1.0/24}

volumes:
  edge_a_storage: