Merge pull request #685 from sharphu/feature_db #1328
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 🚀 CI Pipeline | |
| on: | |
| push: | |
| branches: [main, master, 'feature/**'] | |
| pull_request: | |
| branches: [main, master, 'feature/**'] | |
| workflow_dispatch: | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| packages: write | |
| attestations: write | |
| id-token: write | |
| env: | |
| PYTHON: python3 | |
| jobs: | |
| # ============================================================================ | |
| # Stage 1: Project Detection & Setup | |
| # ============================================================================ | |
| detect-projects: | |
| name: 🔍 Detect Projects | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| outputs: | |
| matrix: ${{ steps.detect.outputs.matrix }} | |
| has-projects: ${{ steps.detect.outputs.has-projects }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Detect active projects | |
| id: detect | |
| run: | | |
| echo "🔍 Detecting projects..." | |
| # Generate project matrix | |
| matrix="{\"include\":[" | |
| first=true | |
| # Java projects | |
| if [[ -f "console/backend/pom.xml" ]]; then | |
| [[ "$first" == "false" ]] && matrix+="," | |
| matrix+="{\"name\":\"console-backend\",\"path\":\"console/backend\",\"type\":\"java\",\"setup\":\"java\",\"cache\":\"maven\"}" | |
| first=false | |
| echo "✅ Java: console/backend" | |
| fi | |
| # TypeScript projects | |
| if [[ -f "console/frontend/package.json" && -f "console/frontend/tsconfig.json" ]]; then | |
| [[ "$first" == "false" ]] && matrix+="," | |
| matrix+="{\"name\":\"console-frontend\",\"path\":\"console/frontend\",\"type\":\"typescript\",\"setup\":\"node\",\"cache\":\"npm\"}" | |
| first=false | |
| echo "✅ TypeScript: console/frontend" | |
| fi | |
| # Go projects | |
| if [[ -f "core/tenant/go.mod" ]]; then | |
| [[ "$first" == "false" ]] && matrix+="," | |
| matrix+="{\"name\":\"core-tenant\",\"path\":\"core/tenant\",\"type\":\"go\",\"setup\":\"go\",\"cache\":\"go\"}" | |
| first=false | |
| echo "✅ Go: core/tenant" | |
| fi | |
| # Python projects - use pyproject.toml detection | |
| for project in core/memory/database core/plugin/rpa core/plugin/link core/plugin/aitools core/agent core/knowledge core/workflow; do | |
| if [[ -f "$project/pyproject.toml" ]] || [[ -f "$project/requirements.txt" ]]; then | |
| [[ "$first" == "false" ]] && matrix+="," | |
| name=$(basename "$project") | |
| # Handle nested paths | |
| if [[ "$project" == core/memory/database ]]; then | |
| name="core-database" | |
| elif [[ "$project" == core/plugin/* ]]; then | |
| name="core-$(basename "$project")" | |
| fi | |
| matrix+="{\"name\":\"$name\",\"path\":\"$project\",\"type\":\"python\",\"setup\":\"python\",\"cache\":\"pip\"}" | |
| first=false | |
| echo "✅ Python: $project" | |
| fi | |
| done | |
| matrix+="]}" | |
| echo "matrix=$matrix" >> $GITHUB_OUTPUT | |
| # Fix: Ensure clean boolean output without extra whitespace | |
| if [[ "$first" == "false" ]]; then | |
| echo "has-projects=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "has-projects=false" >> $GITHUB_OUTPUT | |
| fi | |
| echo "🎯 Generated matrix: $matrix" | |
| # ============================================================================ | |
| # Stage 2: Quality Checks & Tests (Parallel by Project) | |
| # ============================================================================ | |
| check: | |
| name: 🔍 Check ${{ matrix.name }} | |
| runs-on: ubuntu-latest | |
| needs: [detect-projects] | |
| if: needs.detect-projects.outputs.has-projects == 'true' | |
| timeout-minutes: 5 | |
| strategy: | |
| matrix: ${{ fromJson(needs.detect-projects.outputs.matrix) }} | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Java | |
| if: matrix.setup == 'java' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: '21' | |
| cache: maven | |
| - name: Setup Node.js | |
| if: matrix.setup == 'node' | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: npm | |
| cache-dependency-path: console/frontend/package-lock.json | |
| - name: Setup Go | |
| if: matrix.setup == 'go' | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.23' | |
| cache-dependency-path: core/tenant/go.sum | |
| - name: Setup Python | |
| if: matrix.setup == 'python' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install uv | |
| if: matrix.setup == 'python' | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| echo "$HOME/.cargo/bin" >> $GITHUB_PATH | |
| - name: Quality check | |
| working-directory: ${{ matrix.path }} | |
| run: | | |
| echo "🔍 Running quality check for ${{ matrix.name }} (${{ matrix.type }})" | |
| case "${{ matrix.type }}" in | |
| java) | |
| echo "📦 Compiling Java project..." | |
| mvn clean compile | |
| echo "✨ Running Spotless format check..." | |
| mvn spotless:check | |
| echo "🔍 Running Checkstyle..." | |
| mvn checkstyle:check | |
| echo "🐛 Running SpotBugs..." | |
| mvn clean compile spotbugs:check | |
| echo "📊 Running PMD..." | |
| mvn clean compile pmd:check | |
| ;; | |
| typescript) | |
| echo "📦 Installing dependencies..." | |
| npm ci --legacy-peer-deps | |
| echo "✨ Checking format compliance..." | |
| UNFORMATTED=$(npx prettier --list-different "**/*.{ts,tsx,js,jsx,json,md}" 2>/dev/null || true) && \ | |
| if [ -n "$UNFORMATTED" ]; then \ | |
| echo "Files that need formatting:" && \ | |
| echo "$UNFORMATTED" && \ | |
| echo "Run 'npm run format' to fix formatting issues." && \ | |
| exit 1; \ | |
| fi && \ | |
| echo "🔍 Running TypeScript type checking..." && \ | |
| (npx tsc --noEmit --pretty || echo "⚠️ TypeScript type checking found errors, but continuing...") && \ | |
| echo "📋 Running ESLint..." && \ | |
| (npx eslint "**/*.{ts,tsx}" --format=stylish --max-warnings=0 || echo "⚠️ ESLint found warnings/errors, but continuing...") | |
| ;; | |
| go) | |
| echo "🔍 Running Go quality checks..." | |
| echo "🛠️ Installing Go tools..." | |
| go install golang.org/x/tools/cmd/goimports@latest | |
| go install github.com/fzipp/gocyclo/cmd/gocyclo@latest | |
| go install honnef.co/go/tools/cmd/[email protected] | |
| curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.5.0 | |
| export PATH=$PATH:$(go env GOPATH)/bin | |
| echo "✨ Checking goimports compliance..." | |
| UNFORMATTED=$(goimports -l .) && \ | |
| if [ -n "$UNFORMATTED" ]; then \ | |
| echo "Files that need formatting:" && \ | |
| echo "$UNFORMATTED" && \ | |
| echo "Run 'goimports -w .' to fix formatting issues." && \ | |
| exit 1; \ | |
| fi | |
| echo "🔍 Running go vet..." | |
| go vet ./... | |
| echo "🔍 Running gocyclo..." | |
| gocyclo -over 10 . || (echo "High cyclomatic complexity detected" && exit 1) | |
| echo "🔍 Running staticcheck..." | |
| PKGS=$(go list ./... 2>/dev/null) | |
| if [ -n "$PKGS" ]; then \ | |
| staticcheck $PKGS || exit 1; \ | |
| fi | |
| echo "📋 Running golangci-lint..." | |
| golangci-lint run ./... --timeout=5m | |
| ;; | |
| python) | |
| echo "🛠️ Installing Python quality tools..." | |
| python3 -m pip install black==24.4.2 isort==5.13.2 flake8==7.0.0 mypy==1.18.2 pylint==3.1.0 types-requests>=2.32.4.20250913 | |
| echo "🔍 Running Python quality checks..." | |
| echo "1. Running flake8 code style check..." | |
| python3 -m flake8 --max-line-length 88 --ignore=E203,W503,E501 --max-complexity 10 . | |
| echo "2. Checking isort import order..." | |
| python3 -m isort --check-only --profile black . | |
| echo "3. Checking black code format..." | |
| python3 -m black --check . | |
| echo "4. Running mypy type checking..." | |
| python3 -m mypy --disallow-untyped-defs --disallow-incomplete-defs --check-untyped-defs --no-implicit-optional --ignore-missing-imports --explicit-package-bases . | |
| echo "5. Running pylint code analysis..." | |
| python3 -m pylint --disable=import-error --max-line-length=88 --max-args=7 --max-locals=15 --max-returns=6 --max-branches=12 --max-statements=50 --fail-under=8.0 *.py | |
| ;; | |
| esac | |
| test: | |
| name: 🧪 Test ${{ matrix.name }} | |
| runs-on: ubuntu-latest | |
| needs: [detect-projects] | |
| if: needs.detect-projects.outputs.has-projects == 'true' | |
| timeout-minutes: 5 | |
| strategy: | |
| matrix: ${{ fromJson(needs.detect-projects.outputs.matrix) }} | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Java | |
| if: matrix.setup == 'java' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: '21' | |
| cache: maven | |
| - name: Setup Node.js | |
| if: matrix.setup == 'node' | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: npm | |
| cache-dependency-path: console/frontend/package-lock.json | |
| - name: Setup Go | |
| if: matrix.setup == 'go' | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.23' | |
| cache-dependency-path: core/tenant/go.sum | |
| - name: Setup Python | |
| if: matrix.setup == 'python' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install uv | |
| if: matrix.setup == 'python' | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| echo "$HOME/.cargo/bin" >> $GITHUB_PATH | |
| - name: Run tests with coverage | |
| working-directory: ${{ matrix.path }} | |
| run: | | |
| echo "🧪 Running tests with coverage for ${{ matrix.name }} (${{ matrix.type }})" | |
| case "${{ matrix.type }}" in | |
| java) | |
| echo "🧪 Running Java tests with JaCoCo coverage..." | |
| mvn org.jacoco:jacoco-maven-plugin:0.8.12:prepare-agent test org.jacoco:jacoco-maven-plugin:0.8.12:report | |
| echo "📊 Coverage report generated at target/site/jacoco/index.html" | |
| ;; | |
| typescript) | |
| echo "📦 Installing dependencies..." | |
| npm ci --legacy-peer-deps | |
| echo "🧪 Running TypeScript tests with coverage..." | |
| if [ -f package.json ] && grep -q '"test"' package.json; then | |
| # Check if test script is actually for unit testing (not dev server) | |
| if grep -q '"test":.*vite.*--host' package.json || grep -q '"test":.*dev' package.json; then | |
| echo "Test script appears to be dev server, skipping" | |
| else | |
| npm test -- --coverage --coverageReporters=text --coverageReporters=lcov || \ | |
| npm test -- --coverage || \ | |
| npm test | |
| fi | |
| else | |
| echo "No test script found in package.json" | |
| fi | |
| ;; | |
| go) | |
| echo "🧪 Running Go tests with coverage..." | |
| go test -coverprofile=coverage.out -covermode=atomic ./... | |
| echo "📊 Generating coverage report..." | |
| go tool cover -func=coverage.out | |
| ;; | |
| python) | |
| echo "📦 Installing dependencies..." | |
| if [ -f "uv.lock" ]; then | |
| echo "Installing dependencies with uv..." | |
| uv sync | |
| echo "🛠️ Installing coverage tools..." | |
| uv pip install pytest-cov | |
| echo "🧪 Running Python tests with coverage..." | |
| if [ -d "tests" ]; then | |
| uv run python -m pytest tests/ -v --cov=. --cov-report=term --cov-report=xml --cov-report=html | |
| else | |
| echo "No tests directory found" | |
| fi | |
| elif [ -f "requirements.txt" ]; then | |
| echo "Installing from requirements.txt..." | |
| python3 -m pip install -r requirements.txt | |
| echo "🛠️ Installing pytest and coverage tools..." | |
| python3 -m pip install pytest==8.0.0 pytest-cov | |
| echo "🧪 Running Python tests with coverage..." | |
| if [ -d "tests" ]; then | |
| python3 -m pytest tests/ -v --cov=. --cov-report=term --cov-report=xml --cov-report=html | |
| else | |
| echo "No tests directory found" | |
| fi | |
| elif [ -f "pyproject.toml" ]; then | |
| echo "Extracting dependencies from pyproject.toml..." | |
| sed -n '/^dependencies = \[/,/^\]/p' pyproject.toml | grep -E '^\s*"' | sed 's/^\s*"//' | sed 's/",\?$//' > /tmp/deps.txt | |
| echo "Installing dependencies..." | |
| python3 -m pip install -r /tmp/deps.txt | |
| echo "🛠️ Installing pytest and coverage tools..." | |
| python3 -m pip install pytest==8.0.0 pytest-cov | |
| echo "🧪 Running Python tests with coverage..." | |
| if [ -d "tests" ]; then | |
| python3 -m pytest tests/ -v --cov=. --cov-report=term --cov-report=xml --cov-report=html | |
| else | |
| echo "No tests directory found" | |
| fi | |
| fi | |
| ;; | |
| esac | |
| - name: Upload coverage reports | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: coverage-${{ matrix.name }} | |
| path: | | |
| ${{ matrix.path }}/**/target/site/jacoco/ | |
| ${{ matrix.path }}/coverage/ | |
| ${{ matrix.path }}/coverage.out | |
| ${{ matrix.path }}/coverage.xml | |
| ${{ matrix.path }}/htmlcov/ | |
| retention-days: 30 | |
| include-hidden-files: false | |
| - name: Display coverage summary | |
| if: always() | |
| working-directory: ${{ matrix.path }} | |
| run: | | |
| echo "📊 Coverage Summary for ${{ matrix.name }}" | |
| case "${{ matrix.type }}" in | |
| java) | |
| echo "✅ Java Coverage Report:" | |
| echo "================================================" | |
| # Find all jacoco.csv files in submodules | |
| CSV_FILES=$(find . -name "jacoco.csv" -path "*/site/jacoco/jacoco.csv" 2>/dev/null) | |
| if [ -n "$CSV_FILES" ]; then | |
| # Aggregate coverage from all modules | |
| awk -F',' ' | |
| FNR==1 {next} # Skip header of each file | |
| $2 != "" { | |
| branch_covered += $6; | |
| branch_missed += $5; | |
| line_covered += $8; | |
| line_missed += $7; | |
| method_covered += $12; | |
| method_missed += $11; | |
| } | |
| END { | |
| if ((line_covered + line_missed) > 0) { | |
| printf "Lines: %.1f%% (%d/%d)\n", (line_covered/(line_covered+line_missed))*100, line_covered, line_covered+line_missed; | |
| } | |
| if ((branch_covered + branch_missed) > 0) { | |
| printf "Branches: %.1f%% (%d/%d)\n", (branch_covered/(branch_covered+branch_missed))*100, branch_covered, branch_covered+branch_missed; | |
| } | |
| if ((method_covered + method_missed) > 0) { | |
| printf "Methods: %.1f%% (%d/%d)\n", (method_covered/(method_covered+method_missed))*100, method_covered, method_covered+method_missed; | |
| } | |
| } | |
| ' $CSV_FILES | |
| echo "================================================" | |
| echo "📄 Module reports:" | |
| find . -name "index.html" -path "*/site/jacoco/index.html" | while read report; do | |
| echo " - ${report#./}" | |
| done | |
| else | |
| echo "⚠️ No coverage reports found" | |
| fi | |
| ;; | |
| typescript) | |
| echo "✅ TypeScript Coverage Report:" | |
| echo "================================================" | |
| if [ -f coverage/lcov-report/index.html ]; then | |
| echo "Coverage report generated successfully" | |
| echo "📄 HTML report: coverage/lcov-report/index.html" | |
| if [ -f coverage/lcov.info ]; then | |
| echo "📄 LCOV report: coverage/lcov.info" | |
| fi | |
| echo "" | |
| echo "💡 Coverage summary is displayed above in test output" | |
| else | |
| echo "⚠️ No coverage report found" | |
| fi | |
| echo "================================================" | |
| ;; | |
| go) | |
| if [ -f coverage.out ]; then | |
| echo "✅ Go Coverage Report:" | |
| echo "================================================" | |
| go tool cover -func=coverage.out | tail -n 1 | |
| echo "================================================" | |
| echo "📄 Coverage profile: coverage.out" | |
| echo "💡 View HTML report: go tool cover -html=coverage.out" | |
| else | |
| echo "⚠️ No Go coverage report found" | |
| fi | |
| ;; | |
| python) | |
| echo "✅ Python Coverage Report:" | |
| echo "================================================" | |
| if [ -f coverage.xml ]; then | |
| echo "Coverage report generated successfully" | |
| if [ -f htmlcov/index.html ]; then | |
| echo "📄 HTML report: htmlcov/index.html" | |
| fi | |
| echo "📄 XML report: coverage.xml" | |
| echo "" | |
| echo "💡 Coverage summary is displayed above in test output" | |
| else | |
| echo "⚠️ No coverage report found" | |
| fi | |
| echo "================================================" | |
| ;; | |
| esac | |
| # ============================================================================ | |
| # Stage 4: Additional Checks | |
| # ============================================================================ | |
| comment-check: | |
| name: 💬 Comment Check | |
| runs-on: ubuntu-latest | |
| needs: [detect-projects] | |
| if: needs.detect-projects.outputs.has-projects == 'true' | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Check comment language | |
| run: | | |
| echo "💬 Checking comment language compliance..." | |
| chmod +x makefiles/check-comments.sh | |
| ./makefiles/check-comments.sh | |
| # ============================================================================ | |
| # Stage 5: Summary | |
| # ============================================================================ | |
| summary: | |
| name: 📊 Summary | |
| runs-on: ubuntu-latest | |
| needs: [detect-projects, check, test, comment-check] | |
| if: always() && needs.detect-projects.outputs.has-projects == 'true' | |
| timeout-minutes: 2 | |
| steps: | |
| - name: Generate summary | |
| run: | | |
| echo "=== 🚀 CI Pipeline Summary ===" | |
| echo "" | |
| echo "🔍 Project Detection: ${{ needs.detect-projects.result }}" | |
| echo "🔍 Quality Checks: ${{ needs.check.result }}" | |
| echo "🧪 Tests: ${{ needs.test.result }}" | |
| echo "💬 Comment Check: ${{ needs.comment-check.result }}" | |
| echo "" | |
| # Check overall success | |
| if [[ "${{ needs.detect-projects.result }}" == "success" && \ | |
| "${{ needs.check.result }}" == "success" && \ | |
| "${{ needs.test.result }}" == "success" && \ | |
| "${{ needs.comment-check.result }}" == "success" ]]; then | |
| echo "✅ 🎉 All checks passed! Ready for merge." | |
| else | |
| echo "❌ 🚨 Some checks failed. Please review the logs." | |
| exit 1 | |
| fi |