diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 49ecd09e..9e478d10 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -22,6 +22,17 @@ updates: commit-message: prefix: "chore(deps)" + - package-ecosystem: "pip" + directory: "/integration-tests" + schedule: + interval: "weekly" + day: "monday" + open-pull-requests-limit: 10 + labels: + - "dependencies" + commit-message: + prefix: "chore(deps)" + - package-ecosystem: "npm" directory: "/docs-site" schedule: diff --git a/.github/scripts/check_llama_index_updates.py b/.github/scripts/check_llama_index_updates.py new file mode 100644 index 00000000..63219d34 --- /dev/null +++ b/.github/scripts/check_llama_index_updates.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Check PyPI for llama-index-* releases newer than our pinned floors. + +Reads the requirements.txt files listed in the REQUIREMENTS_FILES env var +(whitespace-separated), extracts every ``llama-index*`` requirement and its +``>=`` floor, queries PyPI for the latest release of each, and writes a markdown +report of the packages that have a newer version available. + +Outputs (written to $GITHUB_OUTPUT when running under Actions): + has_updates : "true" if at least one package has a newer release, else "false" + +The markdown report is written to $GITHUB_WORKSPACE/llama_index_report.md. + +This is a best-effort nudge: a network or parse error for a single package is +logged and skipped rather than failing the job. +""" + +import json +import os +import re +import sys +import urllib.request +import urllib.error + +try: + from packaging.version import Version, InvalidVersion +except ImportError: # pragma: no cover - packaging ships with pip on the runner + print("packaging not available; install it in the workflow step", file=sys.stderr) + raise + +# Matches e.g. "llama-index-llms-bedrock-converse>=0.15.0". Only >= floors are +# tracked because that is the pinning style used across the toolkit. +REQ_LINE = re.compile(r'^(llama-index[a-z0-9-]*)\s*>=\s*([0-9][0-9A-Za-z.\-]*)\s*$') + +PYPI_URL = 'https://pypi.org/pypi/{pkg}/json' + + +def latest_version(pkg: str) -> str: + """Return the latest stable, non-yanked release of a package on PyPI. + + `info.version` can point at a pre-release and does not account for yanked + releases, so walk `releases` instead: skip pre-releases, versions with no + uploaded files, and versions whose files are all yanked, then return the + highest remaining version. Falls back to `info.version` only if that filter + leaves nothing. + """ + with urllib.request.urlopen(PYPI_URL.format(pkg=pkg), timeout=30) as resp: + data = json.load(resp) + + candidates = [] + for version, files in data.get('releases', {}).items(): + if not files or all(f.get('yanked') for f in files): + continue + try: + parsed = Version(version) + except InvalidVersion: + continue + if parsed.is_prerelease: + continue + candidates.append(parsed) + + return str(max(candidates)) if candidates else data['info']['version'] + + +def collect_requirements(files): + """Map each llama-index package to its lowest pinned floor and the files it appears in.""" + found = {} + for path in files: + if not path or not os.path.exists(path): + print(f'skipping missing requirements file: {path}', file=sys.stderr) + continue + with open(path, encoding='utf-8') as fh: + for raw in fh: + match = REQ_LINE.match(raw.strip()) + if not match: + continue + pkg, floor = match.group(1), match.group(2) + entry = found.setdefault(pkg, {'floor': floor, 'files': set()}) + entry['files'].add(path) + # Keep the lowest floor seen across files (most conservative). + try: + if Version(floor) < Version(entry['floor']): + entry['floor'] = floor + except InvalidVersion: + pass + return found + + +TEST_COMMANDS = """### Suggested tests to run before merging a bump + +```bash +# lexical-graph unit tests +cd lexical-graph && PYTHONPATH=src python -m pytest tests/ + +# byokg-rag unit tests +cd byokg-rag && PYTHONPATH=src python -m pytest tests/ +``` + +Pay particular attention to the Bedrock / LLM paths, which are most sensitive to +llama-index-llms-bedrock-converse and llama-index-llms-anthropic changes: + +```bash +cd lexical-graph && PYTHONPATH=src python -m pytest \\ + tests/unit/indexing/utils/test_batch_inference_utils.py \\ + tests/unit/indexing/utils/test_batch_inference_aws.py \\ + tests/unit/utils/test_llm_cache.py \\ + tests/unit/utils/test_llm_cache_cross_region.py +``` +""" + + +def build_report(updates): + """Render the markdown issue body for the packages that have newer releases.""" + lines = [ + 'The following LlamaIndex dependencies have newer releases on PyPI than ' + 'the floors pinned in this repo. Dependabot opens the actual bump PRs; ' + 'this issue tracks awareness and the tests to run.', + '', + '| Package | Pinned floor | Latest on PyPI | Requirements file(s) |', + '| --- | --- | --- | --- |', + ] + for pkg in sorted(updates): + info = updates[pkg] + files = '
'.join(sorted(info['files'])) + lines.append(f"| `{pkg}` | {info['floor']} | **{info['latest']}** | {files} |") + lines.extend(['', TEST_COMMANDS]) + return '\n'.join(lines) + + +def set_output(name: str, value: str) -> None: + out = os.environ.get('GITHUB_OUTPUT') + if not out: + return + with open(out, 'a', encoding='utf-8') as fh: + fh.write(f'{name}={value}\n') + + +def main() -> int: + files = os.environ.get('REQUIREMENTS_FILES', '').split() + requirements = collect_requirements(files) + if not requirements: + print('no llama-index requirements found', file=sys.stderr) + set_output('has_updates', 'false') + return 0 + + updates = {} + for pkg, info in sorted(requirements.items()): + try: + latest = latest_version(pkg) + except (urllib.error.URLError, urllib.error.HTTPError, KeyError, ValueError) as exc: + print(f'could not fetch {pkg} from PyPI: {exc}', file=sys.stderr) + continue + try: + is_newer = Version(latest) > Version(info['floor']) + except InvalidVersion: + print(f'could not compare versions for {pkg}: {info["floor"]} vs {latest}', file=sys.stderr) + continue + status = 'newer available' if is_newer else 'up to date' + print(f'{pkg}: floor {info["floor"]} -> latest {latest} ({status})') + if is_newer: + updates[pkg] = {**info, 'latest': latest} + + if not updates: + set_output('has_updates', 'false') + print('all llama-index dependencies are up to date') + return 0 + + report = build_report(updates) + report_path = os.path.join(os.environ.get('GITHUB_WORKSPACE', '.'), 'llama_index_report.md') + with open(report_path, 'w', encoding='utf-8') as fh: + fh.write(report) + set_output('has_updates', 'true') + print(f'{len(updates)} package(s) have newer releases; report written to {report_path}') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/workflows/llama-index-deps-check.yml b/.github/workflows/llama-index-deps-check.yml new file mode 100644 index 00000000..30cf3ea0 --- /dev/null +++ b/.github/workflows/llama-index-deps-check.yml @@ -0,0 +1,82 @@ +# Monthly check for newer llama-index-* releases than the >= floors pinned in the +# toolkit's requirements.txt files. When newer releases exist it opens (or +# refreshes) a single tracking issue that lists the available upgrades and the +# tests to run before merging a bump. Complements Dependabot, which opens the +# actual version-bump PRs; this workflow adds a curated, test-oriented nudge. +name: LlamaIndex Dependency Freshness + +on: + schedule: + # 08:00 UTC on the 1st of each month. + - cron: "0 8 1 * *" + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Install packaging + run: python -m pip install --disable-pip-version-check packaging + + - name: Check PyPI for newer llama-index releases + id: check + env: + REQUIREMENTS_FILES: >- + lexical-graph/src/graphrag_toolkit/lexical_graph/requirements.txt + byokg-rag/src/graphrag_toolkit/byokg_rag/requirements.txt + integration-tests/requirements-integ-test.txt + run: python .github/scripts/check_llama_index_updates.py + + - name: Open or refresh tracking issue + if: steps.check.outputs.has_updates == 'true' + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs'); + const title = 'chore(deps): LlamaIndex dependency updates available'; + const label = 'dependencies'; + const body = fs.readFileSync('llama_index_report.md', 'utf8') + + `\n\n_Last checked: ${new Date().toISOString().slice(0, 10)} ` + + `([run](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}))._`; + + // Reuse a single living issue rather than opening a new one each week. + const existing = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: label, + per_page: 100, + }); + const match = existing.data.find(i => i.title === title && !i.pull_request); + + if (match) { + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: match.number, + body, + }); + console.log(`Updated existing issue #${match.number}`); + } else { + const created = await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + labels: [label], + }); + console.log(`Created issue #${created.data.number}`); + } diff --git a/byokg-rag/src/graphrag_toolkit/byokg_rag/requirements.txt b/byokg-rag/src/graphrag_toolkit/byokg_rag/requirements.txt index 6b084e13..516155e6 100644 --- a/byokg-rag/src/graphrag_toolkit/byokg_rag/requirements.txt +++ b/byokg-rag/src/graphrag_toolkit/byokg_rag/requirements.txt @@ -6,7 +6,7 @@ colorama==0.4.6 faiss-cpu==1.9.0 langchain_huggingface==0.3.1 langchain_aws==0.2.33 -llama-index-embeddings-bedrock>=0.8.2 +llama-index-embeddings-bedrock>=0.9.0 numpy>=1.24.0,<2.0.0 pydantic>=2.8.2 pyyaml==6.0.3 diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/requirements.txt b/lexical-graph/src/graphrag_toolkit/lexical_graph/requirements.txt index 3f2b430d..ef66de4b 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/requirements.txt +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/requirements.txt @@ -4,10 +4,10 @@ anthropic-bedrock==0.8.0 boto3>=1.40.61 botocore>=1.40.61 json2xml==5.2.0 -llama-index-core>=0.14.23 -llama-index-embeddings-bedrock>=0.8.2 -llama-index-llms-anthropic>=0.11.8 -llama-index-llms-bedrock-converse>=0.14.16 +llama-index-core>=0.14.24 +llama-index-embeddings-bedrock>=0.9.0 +llama-index-llms-anthropic>=0.12.0 +llama-index-llms-bedrock-converse>=0.15.0 lru-dict==1.3.0 # Pinned below 3.10.3: that release's pathsec hardlink check (CWE-59) rejects # uv-hardlinked NLTK data files, breaking sentence splitting. See issue 468.