From 0778bf6a2dd2441ef08f9a98a9fdecd479fde9ac Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 13 Oct 2026 19:10:45 +0530 Subject: [PATCH 1/2] Add CI that checks Snowflake and Trino share the Iceberg table test.sh runs script.sql on LocalStack for Snowflake, reads the table back from Trino through Polaris and checks the Iceberg files in S3. The catalog setup script now fails on HTTP errors, and make run uses the compose service, since polaris:8181 only resolves inside the compose network. Claude-Session: https://claude.ai/code/session_01DZfDmzUBPqNAu3yFKyfCNM --- .github/workflows/main.yml | 50 +++++++++++++++++ Makefile | 11 ++-- README.md | 36 ++++++++---- create-polaris-catalog.sh | 32 ++++++----- test.sh | 112 +++++++++++++++++++++++++++++++++++++ 5 files changed, 211 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/main.yml create mode 100755 test.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..f799670 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,50 @@ +name: Test the Polaris demo + +on: + push: + paths-ignore: + - '**.md' + branches: + - main + pull_request: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + name: Share an Iceberg table between Snowflake and Trino + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install the Snowflake connector + run: pip install snowflake-connector-python + + - name: Start LocalStack for Snowflake, Polaris and Trino + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + run: make start + + - name: Run tests + run: make test + + - name: Show container logs + if: failure() + run: | + docker compose ps -a + docker compose logs --tail 300 + + - name: Stop the services + if: always() + run: docker compose down diff --git a/Makefile b/Makefile index d8a5e81..807638f 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ usage: ## Show this help check: ## Check if all required prerequisites are installed @command -v docker > /dev/null 2>&1 || { echo "Docker is not installed. Please install Docker and try again."; exit 1; } - @command -v lstk > /dev/null 2>&1 || { echo "lstk is not installed. Please install lstk and try again."; exit 1; } + @command -v aws > /dev/null 2>&1 || { echo "The AWS CLI is not installed. Please install it and try again."; exit 1; } @echo "All required prerequisites are available." start: ## Start LocalStack and all services via Docker Compose @@ -22,7 +22,10 @@ stop: ## Stop all services logs: ## Save the LocalStack logs in a separate file @docker compose logs localstack > logs.txt -run: ## Run the Polaris catalog setup script - ./create-polaris-catalog.sh +run: ## Run the Polaris catalog setup again (make start already runs it) + docker compose run --rm create-polaris-catalog -.PHONY: usage check start stop logs run +test: ## Run script.sql and check Snowflake, Trino and S3 share the Iceberg table + ./test.sh + +.PHONY: usage check start stop logs run test diff --git a/README.md b/README.md index b2ae591..a0b5a2a 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ This project demonstrates the integration of Apache Polaris with LocalStack for - Docker and Docker Compose - AWS CLI (for testing with LocalStack) +- Python 3 with `snowflake-connector-python`, to run `make test` ## LocalStack Authentication @@ -32,13 +33,13 @@ make start ## Getting Started -1. Run the Polaris catalog setup: +1. Wait for all services to be healthy (especially Polaris). `make start` also runs the `create-polaris-catalog` container, which creates the `polaris` catalog once Polaris is healthy. + +2. To run the catalog setup again, use: ```bash make run ``` -2. Wait for all services to be healthy (especially Polaris) - ### AWS CLI Commands @@ -100,12 +101,24 @@ This demonstrates that the tables are interoperable between LocalStack Snowflake ## Testing -1. The Polaris catalog will be automatically created with the test bucket configuration. +`make test` runs the whole walkthrough against a freshly started stack (`make start`): + +```bash +pip install snowflake-connector-python +make test +``` + +It checks that: +- the `polaris` catalog exists +- `script.sql` runs on LocalStack for Snowflake, and the `SELECT` returns `test` and `foobar` +- Trino reads the same rows from `iceberg.test_namespace.iceberg_table` +- the Iceberg data and metadata files are in `s3://test-bucket/test/` + +The test creates `test-bucket` if it does not exist. `script.sql` is not idempotent, so run `make stop` and `make start` before running the test again. -2. You can verify the setup by: - - Checking Polaris API health: http://localhost:8181/healthcheck - - Accessing Trino UI: http://localhost:8080 - - Listing S3 buckets in LocalStack +You can also check the setup by hand: +- Accessing the Trino UI: http://localhost:8080 +- Listing S3 buckets in LocalStack ## Project Structure @@ -113,6 +126,7 @@ This demonstrates that the tables are interoperable between LocalStack Snowflake . ├── docker-compose.yml ├── create-polaris-catalog.sh +├── test.sh ├── trino-config/ │ └── catalog/ ├── script.sql @@ -122,11 +136,11 @@ This demonstrates that the tables are interoperable between LocalStack Snowflake ## Troubleshooting 1. If services fail to start: - - Check Docker logs: `docker-compose logs` + - Check Docker logs: `docker compose logs` - Verify environment variables are set correctly - Ensure ports are not in use 2. If Polaris catalog creation fails: - - Check LocalStack is running: `curl http://localhost:4566/health` + - Check LocalStack is running: `curl http://localhost:4566/_localstack/health` - Verify S3 bucket exists: `aws --endpoint-url=http://localhost:4566 s3 ls` - - Check Polaris logs: `docker-compose logs polaris` \ No newline at end of file + - Check Polaris logs: `docker compose logs polaris` \ No newline at end of file diff --git a/create-polaris-catalog.sh b/create-polaris-catalog.sh index 2e4875e..dc91a39 100644 --- a/create-polaris-catalog.sh +++ b/create-polaris-catalog.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # Wait for Polaris to be ready echo "Waiting for Polaris to be ready..." @@ -9,7 +9,7 @@ echo "Polaris is ready!" # Get bearer token echo "Getting bearer token..." -if ! output=$(curl -X POST -H "Polaris-Realm: default-realm" "http://polaris:8181/api/catalog/v1/oauth/tokens" \ +if ! output=$(curl -s -X POST -H "Polaris-Realm: default-realm" "http://polaris:8181/api/catalog/v1/oauth/tokens" \ -d "grant_type=client_credentials" \ -d "client_id=root" \ -d "client_secret=s3cr3t" \ @@ -20,7 +20,7 @@ fi token=$(echo "$output" | awk -F\" '{print $4}') -if [ "$token" == "unauthorized_client" ]; then +if [ -z "$token" ] || [ "$token" = "unauthorized_client" ]; then echo "Failed to get bearer token" exit 1 fi @@ -28,9 +28,9 @@ fi PRINCIPAL_TOKEN=$token echo "Bearer token obtained successfully" -# Create catalog +# Create catalog. A 409 means it already exists, for example when the script runs again. echo "Creating Polaris catalog..." -if ! curl -i -X POST -H "Authorization: Bearer $PRINCIPAL_TOKEN" -H 'Accept: application/json' -H 'Content-Type: application/json' \ +status=$(curl -s -o /tmp/create-catalog.json -w '%{http_code}' -X POST -H "Authorization: Bearer $PRINCIPAL_TOKEN" -H 'Accept: application/json' -H 'Content-Type: application/json' \ http://polaris:8181/api/management/v1/catalogs \ -d '{ "catalog": { @@ -52,19 +52,21 @@ if ! curl -i -X POST -H "Authorization: Bearer $PRINCIPAL_TOKEN" -H 'Accept: app "s3.endpoint": "http://localstack:4566" } } - }'; then - echo "Failed to create catalog" - exit 1 -fi -echo "Catalog created successfully" + }') +case "$status" in + 2*) echo "Catalog created successfully" ;; + 409) echo "Catalog already exists" ;; + *) echo "Failed to create catalog (HTTP $status)"; cat /tmp/create-catalog.json; exit 1 ;; +esac # Add TABLE_WRITE_DATA to catalog_admin role echo "Adding TABLE_WRITE_DATA privilege to catalog_admin role..." -if ! curl -i -X PUT -H "Authorization: Bearer $PRINCIPAL_TOKEN" -H 'Accept: application/json' -H 'Content-Type: application/json' \ +status=$(curl -s -o /tmp/grant.json -w '%{http_code}' -X PUT -H "Authorization: Bearer $PRINCIPAL_TOKEN" -H 'Accept: application/json' -H 'Content-Type: application/json' \ http://polaris:8181/api/management/v1/catalogs/polaris/catalog-roles/catalog_admin/grants \ - -d '{"type": "catalog", "privilege": "TABLE_WRITE_DATA"}'; then - echo "Failed to add TABLE_WRITE_DATA privilege" - exit 1 -fi + -d '{"type": "catalog", "privilege": "TABLE_WRITE_DATA"}') +case "$status" in + 2*) ;; + *) echo "Failed to add TABLE_WRITE_DATA privilege (HTTP $status)"; cat /tmp/grant.json; exit 1 ;; +esac echo "TABLE_WRITE_DATA privilege added successfully" echo "Polaris catalog setup completed successfully!" \ No newline at end of file diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..bc34f4e --- /dev/null +++ b/test.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Walk through the demo end to end on a freshly started stack (`make start`): +# 1. the create-polaris-catalog container set up the `polaris` catalog +# 2. script.sql runs on LocalStack for Snowflake and reads back its two rows +# 3. Trino reads the same Iceberg table through Polaris +# 4. the Iceberg data and metadata files are in the LocalStack S3 bucket +# Requires snowflake-connector-python (`pip install snowflake-connector-python`). +set -euo pipefail +cd "$(dirname "$0")" + +export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-test} +export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-test} +export AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-us-east-1} +AWS=${AWS:-aws --endpoint-url=http://localhost:4566} +POLARIS=${POLARIS:-http://localhost:8181} +BUCKET=test-bucket +MAX_ATTEMPTS=${MAX_ATTEMPTS:-90} + +# Wait until the one-shot catalog setup container has exited and check it succeeded +container="" +for attempt in $(seq 1 "$MAX_ATTEMPTS"); do + container=$(docker compose ps -aq create-polaris-catalog) + if [ -n "$container" ] && [ "$(docker inspect -f '{{.State.Status}}' "$container")" = "exited" ]; then + break + fi + echo "Attempt $attempt/$MAX_ATTEMPTS: waiting for the Polaris catalog setup" + sleep 5 +done +if [ -z "$container" ] || [ "$(docker inspect -f '{{.State.ExitCode}}' "$container")" != "0" ]; then + echo "Test failed: the create-polaris-catalog container did not finish successfully" + docker compose logs create-polaris-catalog || true + exit 1 +fi + +token=$(curl -sf -X POST -H "Polaris-Realm: default-realm" "$POLARIS/api/catalog/v1/oauth/tokens" \ + -d grant_type=client_credentials -d client_id=root -d client_secret=s3cr3t -d scope=PRINCIPAL_ROLE:ALL \ + | jq -r .access_token) +location=$(curl -sf -H "Authorization: Bearer $token" "$POLARIS/api/management/v1/catalogs/polaris" \ + | jq -r '.properties["default-base-location"]') +if [ "$location" != "s3://$BUCKET/test" ]; then + echo "Test failed: Polaris catalog 'polaris' has base location '$location'" + exit 1 +fi +echo "PASS: Polaris catalog 'polaris' exists with base location $location" + +$AWS s3api head-bucket --bucket "$BUCKET" 2>/dev/null || $AWS s3 mb "s3://$BUCKET" + +# Run script.sql statement by statement and print the rows of the final SELECT +rows=$(python3 - <<'EOF' +import re +import sys +import time + +import snowflake.connector + +for attempt in range(60): + try: + conn = snowflake.connector.connect( + user="test", password="test", account="test", database="test", + host="snowflake.localhost.localstack.cloud", + ) + break + except Exception as e: + print(f"Waiting for the Snowflake emulator: {e}", file=sys.stderr) + time.sleep(5) +else: + sys.exit("Test failed: could not connect to the Snowflake emulator") + +cur = conn.cursor() +cur.execute("CREATE DATABASE IF NOT EXISTS polaris_demo") +cur.execute("USE DATABASE polaris_demo") +cur.execute("USE SCHEMA public") + +sql = re.sub(r"^\s*--.*$", "", open("script.sql").read(), flags=re.M) +statements = [s.strip() for s in sql.split(";") if s.strip()] +for statement in statements: + print(f"> {statement.splitlines()[0]}", file=sys.stderr) + cur.execute(statement) +for row in sorted(r[0] for r in cur.fetchall()): + print(row) +EOF +) +if [ "$(echo "$rows" | tr '\n' ' ')" != "foobar test " ]; then + echo "Test failed: Snowflake returned '$rows' from iceberg_table" + exit 1 +fi +echo "PASS: script.sql created the Iceberg table and Snowflake reads back foobar and test" + +trino() { + docker compose exec -T trino trino --output-format CSV_UNQUOTED --execute "$1" +} +for attempt in $(seq 1 "$MAX_ATTEMPTS"); do + trino "SELECT 1" > /dev/null 2>&1 && break + echo "Attempt $attempt/$MAX_ATTEMPTS: waiting for Trino" + sleep 5 +done +rows=$(trino "SELECT c1 FROM iceberg.test_namespace.iceberg_table ORDER BY c1") +if [ "$(echo "$rows" | tr '\n' ' ')" != "foobar test " ]; then + echo "Test failed: Trino returned '$rows' from iceberg.test_namespace.iceberg_table" + exit 1 +fi +echo "PASS: Trino reads the same rows through Polaris" + +files=$($AWS s3 ls "s3://$BUCKET/test/" --recursive) +echo "$files" +if ! echo "$files" | grep -q '\.parquet$' || ! echo "$files" | grep -q '\.metadata\.json$'; then + echo "Test failed: no Parquet data files or Iceberg metadata in s3://$BUCKET/test/" + exit 1 +fi +echo "PASS: Iceberg data and metadata files are in s3://$BUCKET/test/" + +echo "Test passed: Snowflake and Trino share the Iceberg table through Polaris and LocalStack S3" From 903068db63fdeea11878da8c473ac6d7cf4d2509 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 13 Oct 2026 19:13:00 +0530 Subject: [PATCH 2/2] Let docker compose run without the auth token set docker-compose.yml required LOCALSTACK_AUTH_TOKEN, so make stop, make logs and docker compose exec failed unless the token was exported. make start still checks it. Claude-Session: https://claude.ai/code/session_01DZfDmzUBPqNAu3yFKyfCNM --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9f919f7..4627b11 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ services: - "127.0.0.1:4510-4559:4510-4559" - "127.0.0.1:443:443" environment: - - LOCALSTACK_AUTH_TOKEN=${LOCALSTACK_AUTH_TOKEN:?} + - LOCALSTACK_AUTH_TOKEN=${LOCALSTACK_AUTH_TOKEN:-} - DEBUG=1 - DOCKER_FLAGS='-e SF_LOG=trace' volumes: