docker-compose.yaml

# ============================================================================
# InsightEra Big Data Cluster - Docker Compose Configuration
# ============================================================================
# OPTIMIZED FOR: 12GB RAM / 2 Core VPS
# ============================================================================
# A production-ready big data analytics platform with:
# - Apache Hadoop (HDFS) for distributed storage
# - Apache Spark with Iceberg for data processing
# - Apache Hive for SQL queries
# - Apache Airflow for workflow orchestration
# - Apache Superset for data visualization
# - Prometheus + Grafana for monitoring
# ============================================================================
# MEMORY ALLOCATION (12GB Total):
#   - System Reserved:     1.5 GB
#   - PostgreSQL:          0.5 GB
#   - Redis:               0.25 GB
#   - Hadoop NameNode:     0.5 GB
#   - Hadoop DataNode:     0.5 GB (1 node only)
#   - Spark Master:        0.5 GB
#   - Spark Worker:        2.0 GB (1 worker only)
#   - Hive Metastore:      0.5 GB
#   - Airflow (all):       2.0 GB
#   - Iceberg REST:        0.25 GB
#   - Buffer/Swap:         1.5 GB
#   - Available for viz/monitoring: ~2 GB
# ============================================================================
# PROFILES:
#   - default:    Core services only (minimal footprint)
#   - full:       All services including monitoring
#   - monitoring: Add Prometheus + Grafana
#   - viz:        Add Superset visualization
# ============================================================================

x-airflow-common: &airflow-common
  image: apache/airflow:${AIRFLOW_VERSION:-2.8.0}
  deploy:
    resources:
      limits:
        memory: 768M
      reservations:
        memory: 384M
  environment: &airflow-common-env
    AIRFLOW__CORE__EXECUTOR: ${AIRFLOW_EXECUTOR:-CeleryExecutor}
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_AIRFLOW_DB}
    AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_AIRFLOW_DB}
    AIRFLOW__CELERY__BROKER_URL: redis://${REDIS_HOST}:${REDIS_PORT}/0
    AIRFLOW__CORE__FERNET_KEY: ${AIRFLOW__CORE__FERNET_KEY}
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
    AIRFLOW__CORE__LOAD_EXAMPLES: ${AIRFLOW_LOAD_EXAMPLES:-false}
    AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session'
    AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'
    AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW__WEBSERVER__SECRET_KEY}
    AIRFLOW__WEBSERVER__EXPOSE_CONFIG: 'true'
    # Reduce webserver workers for low memory (default is 4)
    AIRFLOW__WEBSERVER__WORKERS: '2'
    AIRFLOW__WEBSERVER__WORKER_REFRESH_INTERVAL: '1800'
    # Spark connection
    SPARK_HOME: /opt/spark
    HADOOP_CONF_DIR: /opt/hadoop/etc/hadoop
    # S3/MinIO connection for Iceberg
    AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER}
    AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD}
    AWS_REGION: us-east-1
  volumes:
    - ./airflow/dags:/opt/airflow/dags
    - ./airflow/logs:/opt/airflow/logs
    - ./airflow/plugins:/opt/airflow/plugins
    - ./airflow/config:/opt/airflow/config
    - ./hadoop/config:/opt/hadoop/etc/hadoop:ro
    - ./spark/config:/opt/spark/conf:ro
  user: "${AIRFLOW_UID:-50000}:0"
  networks:
    - insightera-network
  depends_on: &airflow-common-depends-on
    postgres:
      condition: service_healthy
    redis:
      condition: service_healthy

services:
  # ==========================================================================
  # STORAGE LAYER
  # ==========================================================================
  
  # PostgreSQL - Metadata store for Airflow, Hive, Superset
  postgres:
    image: postgres:${POSTGRES_VERSION:-15}
    container_name: insightera-postgres
    hostname: postgres
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 256M
    environment:
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
      POSTGRES_MULTIPLE_DATABASES: "${POSTGRES_AIRFLOW_DB},${POSTGRES_HIVE_DB},${POSTGRES_SUPERSET_DB}"
    volumes:
      - postgres-data:/var/lib/postgresql/data
      - ./docker/postgres/init-multiple-databases.sh:/docker-entrypoint-initdb.d/init-multiple-databases.sh:ro
    ports:
      - "${POSTGRES_PORT:-5432}:5432"
    networks:
      - insightera-network
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER}"]
      interval: 10s
      timeout: 5s
      retries: 5
      start_period: 10s

  # Redis - Message broker for Airflow Celery
  redis:
    image: redis:${REDIS_VERSION:-7}-alpine
    container_name: insightera-redis
    hostname: redis
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 256M
          cpus: '0.25'
        reservations:
          memory: 128M
    command: redis-server --maxmemory 200mb --maxmemory-policy allkeys-lru
    ports:
      - "${REDIS_PORT:-6379}:6379"
    volumes:
      - redis-data:/data
    networks:
      - insightera-network
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 10s
      timeout: 5s
      retries: 5

  # MongoDB - Document database for unstructured data and ETL sources
  mongodb:
    image: mongo:${MONGODB_VERSION:-7.0}
    container_name: insightera-mongodb
    hostname: mongodb
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 256M
    environment:
      MONGO_INITDB_ROOT_USERNAME: ${MONGODB_ROOT_USER:-insightera}
      MONGO_INITDB_ROOT_PASSWORD: ${MONGODB_ROOT_PASSWORD:-CHANGE_ME_IN_PRODUCTION}
      MONGO_INITDB_DATABASE: ${MONGODB_DATABASE:-insightera_data}
    ports:
      - "${MONGODB_PORT:-27017}:27017"
    volumes:
      - mongodb-data:/data/db
      - ./docker/mongodb/init-mongo.js:/docker-entrypoint-initdb.d/init-mongo.js:ro
    networks:
      - insightera-network
    healthcheck:
      test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s

  # MinIO - S3-compatible object storage for Iceberg
  # NOTE: Disabled when using Azure Data Lake Storage
  minio:
    image: minio/minio:${MINIO_VERSION:-latest}
    container_name: insightera-minio
    hostname: minio
    restart: unless-stopped
    profiles:
      - minio
      - full
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 256M
    command: server /data --console-address ":9001"
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
      MINIO_DOMAIN: ${MINIO_DOMAIN}
    ports:
      - "${MINIO_API_PORT:-9000}:9000"
      - "${MINIO_CONSOLE_PORT:-9001}:9001"
    volumes:
      - minio-data:/data
    networks:
      insightera-network:
        aliases:
          - warehouse.minio
          - iceberg.minio
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 20s
      retries: 3

  # MinIO Client - Initialize buckets (Only when using MinIO profile)
  minio-init:
    image: minio/mc:latest
    container_name: insightera-minio-init
    profiles:
      - minio
      - full
    depends_on:
      minio:
        condition: service_healthy
    networks:
      - insightera-network
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
    entrypoint: >
      /bin/sh -c "
      until mc alias set myminio http://minio:9000 $${MINIO_ROOT_USER} $${MINIO_ROOT_PASSWORD}; do
        echo 'Waiting for MinIO...'
        sleep 3
      done;
      mc mb --ignore-existing myminio/warehouse;
      mc mb --ignore-existing myminio/spark-logs;
      mc mb --ignore-existing myminio/airflow-logs;
      mc anonymous set public myminio/warehouse;
      echo 'MinIO buckets initialized successfully';
      exit 0;
      "

  # ==========================================================================
  # HADOOP HDFS LAYER
  # ==========================================================================

  # Hadoop NameNode
  namenode:
    build:
      context: ./docker/hadoop
      dockerfile: Dockerfile
    container_name: insightera-namenode
    hostname: namenode
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 256M
    command: ["hdfs", "namenode"]
    environment:
      HADOOP_HEAPSIZE_MIN: 256m
      HADOOP_HEAPSIZE_MAX: 384m
      HADOOP_NODE_TYPE: namenode
      CLUSTER_NAME: ${CLUSTER_NAME}
    ports:
      - "9870:9870"   # NameNode Web UI
      - "8020:8020"   # HDFS RPC
    volumes:
      - hadoop-namenode-data:/hadoop/dfs/name
      - ./hadoop/config:/opt/hadoop/etc/hadoop:ro
    networks:
      - insightera-network
    healthcheck:
      test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/namenode/9870' || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s

  # Hadoop DataNode 1
  datanode-1:
    build:
      context: ./docker/hadoop
      dockerfile: Dockerfile
    container_name: insightera-datanode-1
    hostname: datanode-1
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 256M
    command: ["hdfs", "datanode"]
    environment:
      HADOOP_HEAPSIZE_MIN: 256m
      HADOOP_HEAPSIZE_MAX: 384m
      HADOOP_NODE_TYPE: datanode
      SERVICE_PRECONDITION: "namenode:9870"
    volumes:
      - hadoop-datanode-1-data:/hadoop/dfs/data
      - ./hadoop/config:/opt/hadoop/etc/hadoop:ro
    networks:
      - insightera-network
    depends_on:
      namenode:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/datanode-1/9864' || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s

  # Hadoop DataNode 2 (Disabled by default for 8GB RAM - use profile 'full')
  datanode-2:
    build:
      context: ./docker/hadoop
      dockerfile: Dockerfile
    container_name: insightera-datanode-2
    hostname: datanode-2
    restart: unless-stopped
    profiles:
      - full
      - scale
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 256M
    command: ["hdfs", "datanode"]
    environment:
      HADOOP_HEAPSIZE_MIN: 256m
      HADOOP_HEAPSIZE_MAX: 384m
      HADOOP_NODE_TYPE: datanode
      SERVICE_PRECONDITION: "namenode:9870"
    volumes:
      - hadoop-datanode-2-data:/hadoop/dfs/data
      - ./hadoop/config:/opt/hadoop/etc/hadoop:ro
    networks:
      - insightera-network
    depends_on:
      namenode:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/datanode-2/9864' || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s

  # ==========================================================================
  # SPARK + ICEBERG LAYER
  # ==========================================================================

  # Spark Master
  spark-master:
    build:
      context: ./docker/spark
      dockerfile: Dockerfile
    container_name: insightera-spark-master
    hostname: spark-master
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 256M
    command: ["/opt/spark/sbin/start-master.sh"]
    environment:
      SPARK_MODE: master
      SPARK_MASTER_HOST: spark-master
      SPARK_MASTER_PORT: ${SPARK_MASTER_PORT:-7077}
      SPARK_MASTER_WEBUI_PORT: ${SPARK_MASTER_WEBUI_PORT:-8080}
      SPARK_DAEMON_MEMORY: 384m
      # Azure Data Lake Storage configuration
      AZURE_STORAGE_ACCOUNT: ${AZURE_STORAGE_ACCOUNT}
      AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY}
      AZURE_STORAGE_CONTAINER: ${AZURE_STORAGE_CONTAINER:-warehouse}
      # Fallback S3/MinIO configuration (if using MinIO profile)
      AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-}
      AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:-}
      AWS_REGION: us-east-1
      SPARK_EXTRA_CLASSPATH: /opt/spark/jars/*
    ports:
      - "${SPARK_MASTER_PORT:-7077}:7077"
      - "${SPARK_MASTER_WEBUI_PORT:-8080}:8080"
    volumes:
      - ./spark/config:/opt/spark/conf:ro
      - ./spark/apps:/opt/spark/apps
      - spark-logs:/opt/spark/logs
      - ./hadoop/config:/opt/hadoop/etc/hadoop:ro
    networks:
      - insightera-network
    depends_on:
      namenode:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:8080/ || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s

  # Spark Worker 1
  spark-worker-1:
    build:
      context: ./docker/spark
      dockerfile: Dockerfile
    container_name: insightera-spark-worker-1
    hostname: spark-worker-1
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 2048M
          cpus: '1.5'
        reservations:
          memory: 1024M
    command: ["/opt/spark/sbin/start-worker.sh", "spark://spark-master:7077"]
    environment:
      SPARK_MODE: worker
      SPARK_MASTER_URL: spark://spark-master:7077
      SPARK_WORKER_CORES: ${SPARK_WORKER_CORES:-2}
      SPARK_WORKER_MEMORY: ${SPARK_WORKER_MEMORY:-1536m}
      SPARK_EXECUTOR_MEMORY: 1024m
      SPARK_DRIVER_MEMORY: 512m
      # Azure Data Lake Storage configuration
      AZURE_STORAGE_ACCOUNT: ${AZURE_STORAGE_ACCOUNT}
      AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY}
      # Fallback S3/MinIO configuration
      AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-}
      AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:-}
      AWS_REGION: us-east-1
    volumes:
      - ./spark/config:/opt/spark/conf:ro
      - ./spark/apps:/opt/spark/apps
      - spark-logs:/opt/spark/logs
      - ./hadoop/config:/opt/hadoop/etc/hadoop:ro
    networks:
      - insightera-network
    depends_on:
      spark-master:
        condition: service_healthy

  # Spark Worker 2 (Disabled by default for 8GB RAM - use profile 'full' or 'scale')
  spark-worker-2:
    build:
      context: ./docker/spark
      dockerfile: Dockerfile
    container_name: insightera-spark-worker-2
    hostname: spark-worker-2
    restart: unless-stopped
    profiles:
      - full
      - scale
    deploy:
      resources:
        limits:
          memory: 1536M
          cpus: '1.0'
        reservations:
          memory: 1024M
    command: ["/opt/spark/sbin/start-worker.sh", "spark://spark-master:7077"]
    environment:
      SPARK_MODE: worker
      SPARK_MASTER_URL: spark://spark-master:7077
      SPARK_WORKER_CORES: ${SPARK_WORKER_CORES:-1}
      SPARK_WORKER_MEMORY: ${SPARK_WORKER_MEMORY:-1g}
      SPARK_EXECUTOR_MEMORY: 768m
      SPARK_DRIVER_MEMORY: 512m
      # Azure Data Lake Storage configuration
      AZURE_STORAGE_ACCOUNT: ${AZURE_STORAGE_ACCOUNT}
      AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY}
      # Fallback S3/MinIO configuration
      AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-}
      AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:-}
      AWS_REGION: us-east-1
    volumes:
      - ./spark/config:/opt/spark/conf:ro
      - ./spark/apps:/opt/spark/apps
      - spark-logs:/opt/spark/logs
      - ./hadoop/config:/opt/hadoop/etc/hadoop:ro
    networks:
      - insightera-network
    depends_on:
      spark-master:
        condition: service_healthy

  # Iceberg REST Catalog
  iceberg-rest:
    image: apache/iceberg-rest-fixture:latest
    container_name: insightera-iceberg-rest
    hostname: iceberg-rest
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 256M
          cpus: '0.25'
        reservations:
          memory: 128M
    ports:
      - "${ICEBERG_REST_PORT:-8181}:8181"
    environment:
      JAVA_OPTS: "-Xms128m -Xmx192m"
      # Azure Data Lake Storage configuration (primary)
      CATALOG_WAREHOUSE: ${AZURE_CATALOG_WAREHOUSE:-abfss://${AZURE_STORAGE_CONTAINER}@${AZURE_STORAGE_ACCOUNT}.dfs.core.windows.net/warehouse/}
      CATALOG_IO__IMPL: org.apache.iceberg.azure.adlsv2.ADLSFileIO
      AZURE_STORAGE_ACCOUNT: ${AZURE_STORAGE_ACCOUNT}
      AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY}
      # Fallback S3/MinIO configuration (if using MinIO profile)
      AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-}
      AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:-}
      AWS_REGION: us-east-1
      CATALOG_S3_ENDPOINT: http://minio:9000
      CATALOG_S3_PATH_STYLE_ACCESS: "true"
    networks:
      - insightera-network
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:8181/v1/config || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5

  # ==========================================================================
  # HIVE METASTORE LAYER
  # ==========================================================================

  # Hive Metastore
  hive-metastore:
    build:
      context: ./docker/hive
      dockerfile: Dockerfile
    container_name: insightera-hive-metastore
    hostname: hive-metastore
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 256M
    environment:
      SERVICE_NAME: metastore
      DB_DRIVER: postgres
      SERVICE_OPTS: >-
        -Djavax.jdo.option.ConnectionDriverName=org.postgresql.Driver
        -Djavax.jdo.option.ConnectionURL=jdbc:postgresql://${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_HIVE_DB}
        -Djavax.jdo.option.ConnectionUserName=${POSTGRES_USER}
        -Djavax.jdo.option.ConnectionPassword=${POSTGRES_PASSWORD}
      # Azure Data Lake Storage configuration
      AZURE_STORAGE_ACCOUNT: ${AZURE_STORAGE_ACCOUNT}
      AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY}
      # Fallback S3/MinIO configuration
      AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-}
      AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:-}
    ports:
      - "${HIVE_METASTORE_PORT:-9083}:9083"
    volumes:
      - ./hive/config:/hive_custom_conf:ro
      - hive-warehouse:/opt/hive/data/warehouse
    networks:
      - insightera-network
    depends_on:
      postgres:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "netstat -tln | grep 9083 || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 120s

  # HiveServer2 (Disabled by default for 8GB RAM - use profile 'full' or 'hive')
  hive-server:
    build:
      context: ./docker/hive
      dockerfile: Dockerfile
    container_name: insightera-hive-server
    hostname: hive-server
    restart: unless-stopped
    profiles:
      - full
      - hive
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 256M
    environment:
      SERVICE_NAME: hiveserver2
      IS_RESUME: "true"
      SERVICE_OPTS: >-
        -Dhive.metastore.uris=thrift://hive-metastore:9083
      # Azure Data Lake Storage configuration
      AZURE_STORAGE_ACCOUNT: ${AZURE_STORAGE_ACCOUNT}
      AZURE_STORAGE_KEY: ${AZURE_STORAGE_KEY}
      # Fallback S3/MinIO configuration
      AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER:-}
      AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD:-}
    ports:
      - "${HIVE_SERVER2_PORT:-10000}:10000"
      - "${HIVE_SERVER2_WEBUI_PORT:-10002}:10002"
    volumes:
      - ./hive/config:/hive_custom_conf:ro
      - hive-warehouse:/opt/hive/data/warehouse
    networks:
      - insightera-network
    depends_on:
      hive-metastore:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "netstat -tln | grep 10000 || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 120s

  # ==========================================================================
  # AIRFLOW ORCHESTRATION LAYER
  # ==========================================================================

  # Airflow Database Init
  airflow-init:
    <<: *airflow-common
    container_name: insightera-airflow-init
    entrypoint: /bin/bash
    command:
      - -c
      - |
        if [[ -z "${AIRFLOW_UID}" ]]; then
          echo
          echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m"
          echo "If you are on Linux, you SHOULD follow the instructions below to set "
          echo "AIRFLOW_UID in your environment"
          echo
        fi
        exec /entrypoint airflow version
    environment:
      <<: *airflow-common-env
      _AIRFLOW_DB_MIGRATE: 'true'
      _AIRFLOW_WWW_USER_CREATE: 'true'
      _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-admin}
      _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-admin}
    user: "0:0"

  # Airflow Webserver
  airflow-webserver:
    <<: *airflow-common
    container_name: insightera-airflow-webserver
    hostname: airflow-webserver
    command: webserver
    ports:
      - "${AIRFLOW_WEBSERVER_PORT:-8082}:8080"
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 30s
      timeout: 10s
      retries: 10
      start_period: 120s
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  # Airflow Scheduler
  airflow-scheduler:
    <<: *airflow-common
    container_name: insightera-airflow-scheduler
    hostname: airflow-scheduler
    command: scheduler
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8974/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  # Airflow Celery Worker
  airflow-worker:
    <<: *airflow-common
    container_name: insightera-airflow-worker
    hostname: airflow-worker
    # Remove stale pidfile before starting to prevent restart loops
    # Limit concurrency to 2 workers for low memory environments (default is 16 = CPU cores)
    command: bash -c "rm -f /opt/airflow/airflow-worker.pid && exec airflow celery worker --concurrency 2"
    restart: unless-stopped
    # Disable healthcheck - Airflow container has restricted permissions
    healthcheck:
      disable: true
    environment:
      <<: *airflow-common-env
      DUMB_INIT_SETSID: "0"
      # Limit Celery worker concurrency to prevent OOM
      AIRFLOW__CELERY__WORKER_CONCURRENCY: "2"
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  # Airflow Triggerer (for deferrable operators - disabled by default for 8GB RAM)
  airflow-triggerer:
    <<: *airflow-common
    container_name: insightera-airflow-triggerer
    hostname: airflow-triggerer
    command: triggerer
    restart: unless-stopped
    profiles:
      - full
      - airflow
    healthcheck:
      test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"']
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  # Flower - Celery monitoring (optional)
  flower:
    <<: *airflow-common
    container_name: insightera-flower
    hostname: flower
    command: celery flower
    profiles:
      - flower
    ports:
      - "5555:5555"
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 30s
    restart: unless-stopped
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  # ==========================================================================
  # SUPERSET VISUALIZATION LAYER
  # ==========================================================================

  # Superset (Disabled by default for 8GB RAM - use profile 'full' or 'viz')
  superset:
    build:
      context: ./docker/superset
      dockerfile: Dockerfile
    container_name: insightera-superset
    hostname: superset
    restart: unless-stopped
    profiles:
      - full
      - viz
    deploy:
      resources:
        limits:
          memory: 1000M
          cpus: '0.5'
        reservations:
          memory: 526M
    environment:
      SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY}
      SUPERSET_ADMIN_USER: ${SUPERSET_ADMIN_USER}
      SUPERSET_ADMIN_PASSWORD: ${SUPERSET_ADMIN_PASSWORD}
      SUPERSET_ADMIN_EMAIL: ${SUPERSET_ADMIN_EMAIL}
      DATABASE_HOST: ${POSTGRES_HOST}
      DATABASE_PORT: ${POSTGRES_PORT}
      DATABASE_DB: ${POSTGRES_SUPERSET_DB}
      DATABASE_USER: ${POSTGRES_USER}
      DATABASE_PASSWORD: ${POSTGRES_PASSWORD}
      REDIS_HOST: ${REDIS_HOST}
      REDIS_PORT: ${REDIS_PORT}
    ports:
      - "${SUPERSET_PORT:-8088}:8088"
    volumes:
      - ./superset/config:/app/pythonpath:ro
      - superset-home:/app/superset_home
    networks:
      - insightera-network
    depends_on:
      postgres:
        condition: service_healthy
      redis:
        condition: service_healthy
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8088/health"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s

  # ==========================================================================
  # MONITORING LAYER
  # ==========================================================================

  # Prometheus (Disabled by default for 8GB RAM - use profile 'full' or 'monitoring')
  prometheus:
    image: prom/prometheus:${PROMETHEUS_VERSION:-v2.48.0}
    container_name: insightera-prometheus
    hostname: prometheus
    restart: unless-stopped
    profiles:
      - full
      - monitoring
    deploy:
      resources:
        limits:
          memory: 256M
          cpus: '0.25'
        reservations:
          memory: 128M
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/etc/prometheus/console_libraries'
      - '--web.console.templates=/etc/prometheus/consoles'
      - '--storage.tsdb.retention.time=15d'
      - '--web.enable-lifecycle'
    ports:
      - "${PROMETHEUS_PORT:-9090}:9090"
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - ./prometheus/alerts:/etc/prometheus/alerts:ro
      - prometheus-data:/prometheus
    networks:
      - insightera-network
    healthcheck:
      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:9090/-/healthy"]
      interval: 30s
      timeout: 10s
      retries: 5

  # Grafana (Disabled by default for 8GB RAM - use profile 'full' or 'monitoring')
  grafana:
    image: grafana/grafana:${GRAFANA_VERSION:-10.2.2}
    container_name: insightera-grafana
    hostname: grafana
    restart: unless-stopped
    profiles:
      - full
      - monitoring
    deploy:
      resources:
        limits:
          memory: 256M
          cpus: '0.25'
        reservations:
          memory: 128M
    environment:
      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER}
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
      GF_USERS_ALLOW_SIGN_UP: 'false'
      GF_SERVER_ROOT_URL: "http://localhost:${GRAFANA_PORT:-3000}"
      GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-simple-json-datasource
    ports:
      - "${GRAFANA_PORT:-3000}:3000"
    volumes:
      - ./grafana/provisioning:/etc/grafana/provisioning:ro
      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
      - grafana-data:/var/lib/grafana
    networks:
      - insightera-network
    depends_on:
      prometheus:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3000/api/health || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5

  # Node Exporter - Host metrics (Disabled by default for 8GB RAM - use profile 'full' or 'monitoring')
  node-exporter:
    image: prom/node-exporter:latest
    container_name: insightera-node-exporter
    hostname: node-exporter
    restart: unless-stopped
    profiles:
      - full
      - monitoring
    deploy:
      resources:
        limits:
          memory: 64M
          cpus: '0.1'
        reservations:
          memory: 32M
    command:
      - '--path.rootfs=/host'
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/host:ro
    networks:
      - insightera-network
    ports:
      - "9100:9100"

  # cAdvisor - Container metrics (Disabled by default for 8GB RAM - use profile 'full' or 'monitoring')
  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.49.1
    container_name: insightera-cadvisor
    hostname: cadvisor
    restart: unless-stopped
    profiles:
      - full
      - monitoring
    command:
      - --store_container_labels=true
      - --disable_metrics=percpu,sched,tcp,udp,hugetlb,referenced_memory,cpu_topology,resctrl
    deploy:
      resources:
        limits:
          memory: 128M
          cpus: '0.25'
        reservations:
          memory: 64M
    privileged: true
    ports:
      - "8081:8080"
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /dev/disk/:/dev/disk:ro
    networks:
      - insightera-network

# ==========================================================================
# NETWORKS
# ==========================================================================
networks:
  insightera-network:
    driver: bridge
    ipam:
      config:
        - subnet: ${NETWORK_SUBNET:-172.28.0.0/16}

# ==========================================================================
# VOLUMES
# ==========================================================================
volumes:
  postgres-data:
    driver: local
  redis-data:
    driver: local
  mongodb-data:
    driver: local
  minio-data:
    driver: local
  hadoop-namenode-data:
    driver: local
  hadoop-datanode-1-data:
    driver: local
  hadoop-datanode-2-data:
    driver: local
  spark-logs:
    driver: local
  hive-warehouse:
    driver: local
  superset-home:
    driver: local
  prometheus-data:
    driver: local
  grafana-data:
    driver: local