From 2a0828fbdfb4b864ec6dcd1393750bd91f7dc55d Mon Sep 17 00:00:00 2001 From: Disturbed Ocean Date: Thu, 19 Feb 2026 18:19:39 -0800 Subject: [PATCH 1/9] feat: Add Python 3.13t and 3.14t builds / wheels --- .github/workflows/test-python.yml | 14 ++++-- .github/workflows/wheels.yml | 14 +++--- obstore/src/lib.rs | 2 +- pyproject.toml | 21 ++++++--- tests/conftest.py | 12 ++++++ tests/store/test_local.py | 71 +++++++++++++++++-------------- tests/test_fsspec.py | 33 +++++++------- uv.lock | 34 ++++++++++++--- 8 files changed, 129 insertions(+), 72 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index c2537312..bb24096b 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -44,7 +44,7 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"] steps: - uses: actions/checkout@v4 @@ -57,18 +57,24 @@ jobs: uses: astral-sh/setup-uv@v5 with: enable-cache: true - version: "0.5.x" + version: "0.8.x" - name: Set up Python run: uv python install ${{ matrix.python-version }} - name: Build rust submodules run: | - uv run maturin develop -m obstore/Cargo.toml + uv run --python ${{ matrix.python-version }} maturin develop -m obstore/Cargo.toml - name: Run python tests + if: "!endsWith(matrix.python-version, 't')" run: | - uv run pytest + uv run --python ${{ matrix.python-version }} pytest tests + + - name: Run python tests with --require-gil-disabled + if: "endsWith(matrix.python-version, 't')" + run: | + uv run --python ${{ matrix.python-version }} pytest tests --require-gil-disabled # Ensure docs build without warnings - name: Check docs diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 9f63f5d0..8355f4ab 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -52,7 +52,7 @@ jobs: version: "0.5.x" - name: Install Python versions - run: uv python install 3.10 3.11 pypy3.11 + run: uv python install 3.10 3.11 3.13t 3.14t pypy3.11 - name: Build abi3-py311 wheels uses: PyO3/maturin-action@v1 @@ -66,7 +66,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i 3.10 -i pypy3.11 --manifest-path obstore/Cargo.toml + args: --release --out dist -i 3.10 -i 3.13t -i 3.14t -i pypy3.11 --manifest-path obstore/Cargo.toml sccache: "true" manylinux: ${{ matrix.platform.manylinux }} @@ -99,7 +99,7 @@ jobs: version: "0.5.x" - name: Install Python versions - run: uv python install 3.10 3.11 pypy3.11 + run: uv python install 3.10 3.11 3.13t 3.14t pypy3.11 - name: Build abi3-py311 wheels uses: PyO3/maturin-action@v1 @@ -113,7 +113,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i 3.10 -i pypy3.11 --manifest-path obstore/Cargo.toml + args: --release --out dist -i 3.10 -i 3.13t -i 3.14t -i pypy3.11 --manifest-path obstore/Cargo.toml sccache: "true" manylinux: musllinux_1_2 @@ -152,7 +152,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i 3.10 --manifest-path obstore/Cargo.toml + args: --release --out dist -i 3.10 -i 3.13t -i 3.14t --manifest-path obstore/Cargo.toml sccache: "true" - name: Upload wheels uses: actions/upload-artifact@v4 @@ -179,7 +179,7 @@ jobs: version: "0.5.x" - name: Install Python versions - run: uv python install 3.10 3.11 pypy3.11 + run: uv python install 3.10 3.11 3.13t 3.14t pypy3.11 - name: Build abi3-py311 wheels uses: PyO3/maturin-action@v1 @@ -192,7 +192,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i 3.10 -i pypy3.11 --manifest-path obstore/Cargo.toml + args: --release --out dist -i 3.10 -i 3.13t -i 3.14t -i pypy3.11 --manifest-path obstore/Cargo.toml sccache: "true" - name: Upload wheels uses: actions/upload-artifact@v4 diff --git a/obstore/src/lib.rs b/obstore/src/lib.rs index 1e10db0b..edc29fa3 100644 --- a/obstore/src/lib.rs +++ b/obstore/src/lib.rs @@ -43,7 +43,7 @@ fn check_debug_build(_py: Python) -> PyResult<()> { } /// A Python module implemented in Rust. -#[pymodule] +#[pymodule(gil_used = false)] fn _obstore(py: Python, m: &Bound) -> PyResult<()> { check_debug_build(py)?; diff --git a/pyproject.toml b/pyproject.toml index b25433ea..3eaf06a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,11 +10,11 @@ dependencies = [] dev-dependencies = [ "aiohttp-retry>=2.9.1", "aiohttp>=3.11.13", - "arro3-core>=0.4.2", + "arro3-core>=0.6.5", "azure-identity>=1.21.0", "boto3>=1.38.21", "docker>=7.1.0", - "fastapi>=0.115.12", # used in example but added here for pyright CI + "fastapi>=0.129.0", # used in example but added here for pyright CI "fsspec>=2024.10.0", "google-auth>=2.38.0", "griffe-inherited-docstrings>=1.0.1", @@ -24,22 +24,26 @@ dev-dependencies = [ "maturin>=1.12.0", "mike>=2.1.3", "minio>=7.2.16", - "mkdocs-material[imaging]>=9.6.3", + "mkdocs-material[imaging]>=9.7.2", "mkdocs-redirects>=1.2.2", "mkdocs>=1.6.1", "mkdocstrings-python>=1.13.0", "mkdocstrings>=0.27.0", "mypy>=1.15.0", "obspec>=0.1.0", + "pillow>=12.1.1", "pip>=24.2", "polars>=1.30.0", - "pyarrow>=17.0.0", - "pystac-client>=0.8.3", + "pyarrow>=23.0.0", + "pydantic>=2.12.5", "pystac>=1.10.1", + "pystac-client>=0.8.3", + "pytest>=8.3.3", "pytest-asyncio>=0.24.0", "pytest-mypy-plugins>=3.2.0", - "pytest>=8.3.3", + "pytest-freethreaded>=0.1.0;python_version>='3.13'", "python-dotenv>=1.0.1", + "pyzmq>=27.1.0", "ruff>=0.15.0", "tqdm>=4.67.1", "types-boto3[s3,sts]>=1.36.23", @@ -103,7 +107,10 @@ executionEnvironments = [ addopts = "-v --mypy-only-local-stub" asyncio_default_fixture_loop_scope = "function" testpaths = ["tests"] -markers = ["network: mark the test as requiring a network connection"] +markers = [ + "network: mark the test as requiring a network connection", + "freethreaded: mark test for free-threaded concurrent execution", +] [tool.mypy] files = ["obstore/python"] diff --git a/tests/conftest.py b/tests/conftest.py index ad15f08c..0fa456dc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ from __future__ import annotations import socket +import sys import time import warnings from typing import TYPE_CHECKING, Any @@ -13,6 +14,17 @@ from obstore.store import S3Store + +def pytest_configure(config: pytest.Config) -> None: + """Disable pytest-freethreaded's multi-thread defaults. + + Tests must opt-in via @pytest.mark.freethreaded(threads=N, iterations=M). + """ + if sys.version_info >= (3, 13) and hasattr(config.option, "threads"): + config.option.threads = 2 + config.option.iterations = 1 + + if TYPE_CHECKING: from collections.abc import Generator diff --git a/tests/store/test_local.py b/tests/store/test_local.py index e9ee2ebb..77acbe39 100644 --- a/tests/store/test_local.py +++ b/tests/store/test_local.py @@ -1,5 +1,6 @@ import pickle from pathlib import Path +from tempfile import TemporaryDirectory import pytest @@ -44,30 +45,36 @@ def test_local_from_url(): store = LocalStore.from_url(url) -def test_create_prefix(tmp_path: Path): - tmpdir = tmp_path / "abc" - assert not tmpdir.exists() - LocalStore(tmpdir, mkdir=True) - assert tmpdir.exists() +def test_create_prefix(): + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + tmpdir = tmp_path / "abc" + assert not tmpdir.exists() + LocalStore(tmpdir, mkdir=True) + assert tmpdir.exists() - # Assert that mkdir=True works even when the dir already exists - LocalStore(tmpdir, mkdir=True) - assert tmpdir.exists() + # Assert that mkdir=True works even when the dir already exists + LocalStore(tmpdir, mkdir=True) + assert tmpdir.exists() -def test_prefix_property(tmp_path: Path): - store = LocalStore(tmp_path) - assert store.prefix == tmp_path - assert isinstance(store.prefix, Path) - # Can pass it back to the store init - LocalStore(store.prefix) +def test_prefix_property(): + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + store = LocalStore(tmp_path) + assert store.prefix == tmp_path + assert isinstance(store.prefix, Path) + # Can pass it back to the store init + LocalStore(store.prefix) -def test_pickle(tmp_path: Path): - store = LocalStore(tmp_path) - store.put("path.txt", b"foo") - new_store: LocalStore = pickle.loads(pickle.dumps(store)) - assert new_store.get("path.txt").bytes() == b"foo" +def test_pickle(): + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + store = LocalStore(tmp_path) + store.put("path.txt", b"foo") + new_store: LocalStore = pickle.loads(pickle.dumps(store)) + assert new_store.get("path.txt").bytes() == b"foo" def test_eq(): @@ -79,18 +86,20 @@ def test_eq(): assert store != store3 -def test_local_store_percent_encoded(tmp_path: Path): - fname1 = "hello%20world.txt" - content1 = b"Hello, World!" - with (tmp_path / fname1).open("wb") as f: - f.write(content1) +def test_local_store_percent_encoded(): + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + fname1 = "hello%20world.txt" + content1 = b"Hello, World!" + with (tmp_path / fname1).open("wb") as f: + f.write(content1) - store = LocalStore(tmp_path) - assert store.get(fname1).bytes() == content1 + store = LocalStore(tmp_path) + assert store.get(fname1).bytes() == content1 - fname2 = "hello world.txt" - content2 = b"Hello, World! (with spaces)" - with (tmp_path / fname2).open("wb") as f: - f.write(content2) + fname2 = "hello world.txt" + content2 = b"Hello, World! (with spaces)" + with (tmp_path / fname2).open("wb") as f: + f.write(content2) - assert store.get(fname2).bytes() == content2 + assert store.get(fname2).bytes() == content2 diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index 0c89fcc3..c9a45097 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -2,6 +2,8 @@ import gc import os +from pathlib import Path +from tempfile import TemporaryDirectory from typing import TYPE_CHECKING from unittest.mock import patch @@ -14,8 +16,6 @@ from tests.conftest import TEST_BUCKET_NAME if TYPE_CHECKING: - from pathlib import Path - from obstore.store import ClientConfig, S3Config @@ -318,7 +318,7 @@ async def test_info_async(minio_bucket: tuple[S3Config, ClientConfig]): assert not await fs._isdir(f"{bucket}/dir_1/") -def test_put_files(minio_bucket: tuple[S3Config, ClientConfig], tmp_path: Path): +def test_put_files(minio_bucket: tuple[S3Config, ClientConfig]): """Test put new file to S3 synchronously.""" register("s3") fs = fsspec.filesystem( @@ -330,14 +330,15 @@ def test_put_files(minio_bucket: tuple[S3Config, ClientConfig], tmp_path: Path): bucket = minio_bucket[0].get("bucket") assert bucket is not None - test_data = "Hello, World!" - local_file_path = tmp_path / "test_file.txt" - local_file_path.write_text(test_data) + with TemporaryDirectory() as tmp: + test_data = "Hello, World!" + local_file_path = Path(tmp) / "test_file.txt" + local_file_path.write_text(test_data) - assert local_file_path.read_text() == test_data - remote_file_path = f"{bucket}/uploaded_test_file.txt" + assert local_file_path.read_text() == test_data + remote_file_path = f"{bucket}/uploaded_test_file.txt" - fs.put(str(local_file_path), remote_file_path) + fs.put(str(local_file_path), remote_file_path) # Verify file upload assert remote_file_path in fs.ls(f"{bucket}", detail=False, refresh=True) @@ -350,7 +351,6 @@ def test_put_files(minio_bucket: tuple[S3Config, ClientConfig], tmp_path: Path): @pytest.mark.asyncio async def test_put_files_async( minio_bucket: tuple[S3Config, ClientConfig], - tmp_path: Path, ): """Test put new file to S3 asynchronously.""" register("s3") @@ -361,14 +361,15 @@ async def test_put_files_async( asynchronous=True, ) - test_data = "Hello, World!" - local_file_path = tmp_path / "test_file.txt" - local_file_path.write_text(test_data) + with TemporaryDirectory() as tmp: + test_data = "Hello, World!" + local_file_path = Path(tmp) / "test_file.txt" + local_file_path.write_text(test_data) - assert local_file_path.read_text() == test_data - remote_file_path = f"{TEST_BUCKET_NAME}/uploaded_test_file.txt" + assert local_file_path.read_text() == test_data + remote_file_path = f"{TEST_BUCKET_NAME}/uploaded_test_file.txt" - await fs._put(str(local_file_path), remote_file_path) + await fs._put(str(local_file_path), remote_file_path) # Verify file upload assert remote_file_path in await fs._ls( diff --git a/uv.lock b/uv.lock index 200a7e4b..64680db2 100644 --- a/uv.lock +++ b/uv.lock @@ -3,7 +3,8 @@ revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14'", - "python_full_version >= '3.11' and python_full_version < '3.14'", + "python_full_version == '3.13.*'", + "python_full_version >= '3.11' and python_full_version < '3.13'", "python_full_version < '3.11'", ] @@ -1166,7 +1167,8 @@ version = "9.10.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14'", - "python_full_version >= '3.11' and python_full_version < '3.14'", + "python_full_version == '3.13.*'", + "python_full_version >= '3.11' and python_full_version < '3.13'", ] dependencies = [ { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, @@ -2668,6 +2670,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] +[[package]] +name = "pytest-freethreaded" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest", marker = "python_full_version >= '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/09/3bf5fb5321463551787c41bd1af128c1eff018a55e1bb91af0501df54773/pytest_freethreaded-0.1.0.tar.gz", hash = "sha256:9cb4f412db3abb5a8ffdcb29a2d42e058c41f16437c0a1815101651b485dcf45", size = 6439, upload-time = "2024-10-03T09:09:56.817Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/e0/f379af92804357ac186d2d90ec73a1534183477de1e32e1a4f0bb86b85da/pytest_freethreaded-0.1.0-py3-none-any.whl", hash = "sha256:5f2beb6026857f274162d0aa131fe605b8c6d2a3b0c2d35a2af14721959398f1", size = 5286, upload-time = "2024-10-03T09:09:54.716Z" }, +] + [[package]] name = "pytest-mypy-plugins" version = "3.3.0" @@ -3267,15 +3281,19 @@ dev = [ { name = "mkdocstrings-python" }, { name = "mypy" }, { name = "obspec" }, + { name = "pillow" }, { name = "pip" }, { name = "polars" }, { name = "pyarrow" }, + { name = "pydantic" }, { name = "pystac" }, { name = "pystac-client" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-freethreaded", marker = "python_full_version >= '3.13'" }, { name = "pytest-mypy-plugins" }, { name = "python-dotenv" }, + { name = "pyzmq" }, { name = "ruff" }, { name = "tqdm" }, { name = "types-boto3", extra = ["s3", "sts"] }, @@ -3289,11 +3307,11 @@ dev = [ dev = [ { name = "aiohttp", specifier = ">=3.11.13" }, { name = "aiohttp-retry", specifier = ">=2.9.1" }, - { name = "arro3-core", specifier = ">=0.4.2" }, + { name = "arro3-core", specifier = ">=0.6.5" }, { name = "azure-identity", specifier = ">=1.21.0" }, { name = "boto3", specifier = ">=1.38.21" }, { name = "docker", specifier = ">=7.1.0" }, - { name = "fastapi", specifier = ">=0.115.12" }, + { name = "fastapi", specifier = ">=0.129.0" }, { name = "fsspec", specifier = ">=2024.10.0" }, { name = "google-auth", specifier = ">=2.38.0" }, { name = "griffe", specifier = ">=1.6.0" }, @@ -3304,21 +3322,25 @@ dev = [ { name = "mike", specifier = ">=2.1.3" }, { name = "minio", specifier = ">=7.2.16" }, { name = "mkdocs", specifier = ">=1.6.1" }, - { name = "mkdocs-material", extras = ["imaging"], specifier = ">=9.6.3" }, + { name = "mkdocs-material", extras = ["imaging"], specifier = ">=9.7.2" }, { name = "mkdocs-redirects", specifier = ">=1.2.2" }, { name = "mkdocstrings", specifier = ">=0.27.0" }, { name = "mkdocstrings-python", specifier = ">=1.13.0" }, { name = "mypy", specifier = ">=1.15.0" }, { name = "obspec", specifier = ">=0.1.0" }, + { name = "pillow", specifier = ">=12.1.1" }, { name = "pip", specifier = ">=24.2" }, { name = "polars", specifier = ">=1.30.0" }, - { name = "pyarrow", specifier = ">=17.0.0" }, + { name = "pyarrow", specifier = ">=23.0.0" }, + { name = "pydantic", specifier = ">=2.12.5" }, { name = "pystac", specifier = ">=1.10.1" }, { name = "pystac-client", specifier = ">=0.8.3" }, { name = "pytest", specifier = ">=8.3.3" }, { name = "pytest-asyncio", specifier = ">=0.24.0" }, + { name = "pytest-freethreaded", marker = "python_full_version >= '3.13'", specifier = ">=0.1.0" }, { name = "pytest-mypy-plugins", specifier = ">=3.2.0" }, { name = "python-dotenv", specifier = ">=1.0.1" }, + { name = "pyzmq", specifier = ">=27.1.0" }, { name = "ruff", specifier = ">=0.15.0" }, { name = "tqdm", specifier = ">=4.67.1" }, { name = "types-boto3", extras = ["s3", "sts"], specifier = ">=1.36.23" }, From 9a4734b926519c1a555e766e86502d3cc8142355 Mon Sep 17 00:00:00 2001 From: Disturbed Ocean Date: Mon, 23 Feb 2026 06:37:30 -0800 Subject: [PATCH 2/9] Try to fix minio --- tests/conftest.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0fa456dc..085d7d06 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -128,13 +128,19 @@ def minio_config() -> Generator[tuple[S3Config, ClientConfig], Any, None]: def minio_bucket( minio_config: tuple[S3Config, ClientConfig], ) -> Generator[tuple[S3Config, ClientConfig], Any, None]: + # Clean bucket before each test so tests always start with empty state, + # regardless of whether a previous test's teardown failed or was incomplete. + store = S3Store(config=minio_config[0], client_options=minio_config[1]) + objects = store.list().collect() + if paths := [obj["path"] for obj in objects]: + store.delete(paths) + yield minio_config - # Remove all files from bucket - store = S3Store(config=minio_config[0], client_options=minio_config[1]) + # Best-effort cleanup after the test as well. objects = store.list().collect() - paths = [obj["path"] for obj in objects] - store.delete(paths) + if paths := [obj["path"] for obj in objects]: + store.delete(paths) @pytest.fixture From 895337858c2a270e2193e9328d96e462181988ff Mon Sep 17 00:00:00 2001 From: Disturbed Ocean Date: Mon, 23 Feb 2026 07:59:20 -0800 Subject: [PATCH 3/9] Try a different approach with pytest-freethreaded --- tests/conftest.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 085d7d06..faabffc0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ import socket import sys +import sysconfig import time import warnings from typing import TYPE_CHECKING, Any @@ -18,11 +19,27 @@ def pytest_configure(config: pytest.Config) -> None: """Disable pytest-freethreaded's multi-thread defaults. - Tests must opt-in via @pytest.mark.freethreaded(threads=N, iterations=M). + On GIL-enabled Python, unregister the plugin entirely so its + pytest_runtest_call hook doesn't wrap tests in a ThreadPoolExecutor + (which breaks async tests run from non-main threads). + + On free-threaded Python, set conservative defaults so tests must opt-in + via @pytest.mark.freethreaded(threads=N, iterations=M). """ if sys.version_info >= (3, 13) and hasattr(config.option, "threads"): - config.option.threads = 2 - config.option.iterations = 1 + if sysconfig.get_config_var("Py_GIL_DISABLED"): + config.option.threads = 2 + config.option.iterations = 1 + else: + # Unregister the plugin on GIL-enabled builds so its + # pytest_runtest_call hook doesn't wrap tests in a + # ThreadPoolExecutor (breaks async tests from non-main threads). + try: + import pytest_freethreaded.plugin as ft_plugin + + config.pluginmanager.unregister(ft_plugin) + except (ImportError, ValueError): + pass if TYPE_CHECKING: From bc0ee4e37151eb4e99d98fe8f29455ad0bc2a6dc Mon Sep 17 00:00:00 2001 From: Disturbed Ocean Date: Mon, 23 Feb 2026 08:07:59 -0800 Subject: [PATCH 4/9] Bump the setup-uv version --- .github/workflows/docs.yml | 4 ++-- .github/workflows/test-python.yml | 4 ++-- .github/workflows/wheels.yml | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7cb1174c..f27e161d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -32,10 +32,10 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Install a specific version of uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v7 with: enable-cache: true - version: "0.5.x" + version: "0.10.x" - name: Set up Python 3.11 run: uv python install 3.11 diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index bb24096b..e985832d 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -54,10 +54,10 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Install uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v7 with: enable-cache: true - version: "0.8.x" + version: "0.10.x" - name: Set up Python run: uv python install ${{ matrix.python-version }} diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 8355f4ab..0743126b 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -46,10 +46,10 @@ jobs: - uses: actions/checkout@v4 - name: Install uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v7 with: enable-cache: true - version: "0.5.x" + version: "0.10.x" - name: Install Python versions run: uv python install 3.10 3.11 3.13t 3.14t pypy3.11 @@ -93,7 +93,7 @@ jobs: - uses: actions/checkout@v4 - name: Install uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v7 with: enable-cache: true version: "0.5.x" @@ -173,7 +173,7 @@ jobs: - uses: actions/checkout@v4 - name: Install uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@v7 with: enable-cache: true version: "0.5.x" From 27285c18df411e16fcf4906ad7f88ee92c523aba Mon Sep 17 00:00:00 2001 From: Disturbed Ocean Date: Mon, 23 Feb 2026 08:50:12 -0800 Subject: [PATCH 5/9] Back to 1 --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index faabffc0..6a2c1649 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,7 +28,7 @@ def pytest_configure(config: pytest.Config) -> None: """ if sys.version_info >= (3, 13) and hasattr(config.option, "threads"): if sysconfig.get_config_var("Py_GIL_DISABLED"): - config.option.threads = 2 + config.option.threads = 1 config.option.iterations = 1 else: # Unregister the plugin on GIL-enabled builds so its From 8892268a6be9031c806af432804715dca863b070 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 4 Mar 2026 14:43:53 -0500 Subject: [PATCH 6/9] modify testss --- tests/test_fsspec.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index c9a45097..e4273d48 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -212,13 +212,15 @@ def test_list( fs.pipe_file(f"{bucket}/afile", b"hello world") out = fs.ls(f"{bucket}", detail=False, refresh=True) - assert out == [f"{bucket}/afile"] + assert f"{bucket}/afile" in out fs.pipe_file(f"{bucket}/dir/bfile", b"data") out = fs.ls(f"{bucket}", detail=False, refresh=True) - assert out == [f"{bucket}/afile", f"{bucket}/dir"] + assert f"{bucket}/afile" in out + assert f"{bucket}/dir" in out out = fs.ls(f"{bucket}", detail=True, refresh=True) - assert out[0]["type"] == "file" - assert out[1]["type"] == "directory" + types = {o["name"]: o["type"] for o in out} + assert types.get(f"{bucket}/afile") == "file" + assert types.get(f"{bucket}/dir") == "directory" @pytest.mark.asyncio @@ -239,13 +241,15 @@ async def test_list_async( await fs._pipe_file(f"{bucket}/afile", b"hello world") out = await fs._ls(f"{bucket}", detail=False, refresh=True) - assert out == [f"{bucket}/afile"] + assert f"{bucket}/afile" in out await fs._pipe_file(f"{bucket}/dir/bfile", b"data") out = await fs._ls(f"{bucket}", detail=False, refresh=True) - assert out == [f"{bucket}/afile", f"{bucket}/dir"] + assert f"{bucket}/afile" in out + assert f"{bucket}/dir" in out out = await fs._ls(f"{bucket}", detail=True, refresh=True) - assert out[0]["type"] == "file" - assert out[1]["type"] == "directory" + types = {o["name"]: o["type"] for o in out} + assert types.get(f"{bucket}/afile") == "file" + assert types.get(f"{bucket}/dir") == "directory" def test_info(minio_bucket: tuple[S3Config, ClientConfig]): From 36948e99dca8070f1fe2102a56cfc0b1ef440931 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 4 Mar 2026 15:29:27 -0500 Subject: [PATCH 7/9] cleaner delete --- tests/conftest.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6a2c1649..fafb3c70 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -149,15 +149,13 @@ def minio_bucket( # regardless of whether a previous test's teardown failed or was incomplete. store = S3Store(config=minio_config[0], client_options=minio_config[1]) objects = store.list().collect() - if paths := [obj["path"] for obj in objects]: - store.delete(paths) + store.delete([obj["path"] for obj in objects]) yield minio_config # Best-effort cleanup after the test as well. objects = store.list().collect() - if paths := [obj["path"] for obj in objects]: - store.delete(paths) + store.delete([obj["path"] for obj in objects]) @pytest.fixture From ff691049b9e482ceb4f8318e3653b63623af90be Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Wed, 4 Mar 2026 15:49:38 -0500 Subject: [PATCH 8/9] Enable `generate-import-lib` pyo3 feature on windows --- .github/workflows/wheels.yml | 4 ++-- Cargo.lock | 10 ++++++++++ obstore/Cargo.toml | 2 ++ pyproject.toml | 5 ++--- uv.lock | 30 +++++++++++++++--------------- 5 files changed, 31 insertions(+), 20 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 7497ab8b..bcc31f58 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -145,14 +145,14 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist --features abi3-py311 -i 3.11 --manifest-path obstore/Cargo.toml + args: --release --out dist --features abi3-py311 --features generate-import-lib -i 3.11 --manifest-path obstore/Cargo.toml sccache: "true" - name: Build version-specific wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i 3.10 -i 3.13t -i 3.14t --manifest-path obstore/Cargo.toml + args: --release --out dist --features generate-import-lib -i 3.10 -i 3.13t -i 3.14t --manifest-path obstore/Cargo.toml sccache: "true" - name: Upload wheels uses: actions/upload-artifact@v4 diff --git a/Cargo.lock b/Cargo.lock index 93022e8d..f485548a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1576,6 +1576,7 @@ version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bf94ee265674bf76c09fa430b0e99c26e319c945d96ca0d5a8215f31bf81cf7" dependencies = [ + "python3-dll-a", "target-lexicon", ] @@ -1654,6 +1655,15 @@ dependencies = [ "url", ] +[[package]] +name = "python3-dll-a" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d381ef313ae70b4da5f95f8a4de773c6aa5cd28f73adec4b4a31df70b66780d8" +dependencies = [ + "cc", +] + [[package]] name = "quick-xml" version = "0.39.2" diff --git a/obstore/Cargo.toml b/obstore/Cargo.toml index d591cb0b..81ac0d07 100644 --- a/obstore/Cargo.toml +++ b/obstore/Cargo.toml @@ -19,6 +19,8 @@ crate-type = ["cdylib"] [features] abi3-py311 = ["pyo3/abi3-py311"] +# https://www.maturin.rs/distribution.html#cross-compile-to-windows +generate-import-lib = ["pyo3/generate-import-lib"] [dependencies] arrow = "57" diff --git a/pyproject.toml b/pyproject.toml index 544177bd..dc99d903 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,11 +42,10 @@ dev = [ "pyarrow>=17.0.0", "pystac-client>=0.8.3", "pystac>=1.10.1", - "pystac-client>=0.8.3", "pytest>=8.3.3", "pytest-asyncio>=0.24.0", "pytest-mypy-plugins>=3.2.0", - "pytest-freethreaded>=0.1.0;python_version>='3.13'", + "pytest-run-parallel>=0.3.0;python_version>='3.13'", "python-dotenv>=1.0.1", "pyzmq>=27.1.0", "ruff>=0.15.0", @@ -113,7 +112,7 @@ asyncio_default_fixture_loop_scope = "function" testpaths = ["tests"] markers = [ "network: mark the test as requiring a network connection", - "freethreaded: mark test for free-threaded concurrent execution", + "parallel_threads: mark test for concurrent thread execution", ] [tool.mypy] diff --git a/uv.lock b/uv.lock index 9e78e66b..516362e6 100644 --- a/uv.lock +++ b/uv.lock @@ -905,7 +905,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -2873,18 +2873,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075, upload-time = "2025-11-10T16:07:45.537Z" }, ] -[[package]] -name = "pytest-freethreaded" -version = "0.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytest", marker = "python_full_version >= '3.13'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/20/09/3bf5fb5321463551787c41bd1af128c1eff018a55e1bb91af0501df54773/pytest_freethreaded-0.1.0.tar.gz", hash = "sha256:9cb4f412db3abb5a8ffdcb29a2d42e058c41f16437c0a1815101651b485dcf45", size = 6439, upload-time = "2024-10-03T09:09:56.817Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/e0/f379af92804357ac186d2d90ec73a1534183477de1e32e1a4f0bb86b85da/pytest_freethreaded-0.1.0-py3-none-any.whl", hash = "sha256:5f2beb6026857f274162d0aa131fe605b8c6d2a3b0c2d35a2af14721959398f1", size = 5286, upload-time = "2024-10-03T09:09:54.716Z" }, -] - [[package]] name = "pytest-mypy-plugins" version = "3.3.0" @@ -2905,6 +2893,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4e/49/a5f212e829e2b05b31aad9e96b89e70cd73c7a604f74df34825eac5fd227/pytest_mypy_plugins-3.3.0-py3-none-any.whl", hash = "sha256:7b93f338609eace9405986be69fb52344dfb3ccbb96f3fc5ac3cbd257a34c5fc", size = 20798, upload-time = "2026-02-16T17:24:54.693Z" }, ] +[[package]] +name = "pytest-run-parallel" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest", marker = "python_full_version >= '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/91/14b1403b47f595d360af67624ee50eae2129f04f56e354ce6c04dbda9d93/pytest_run_parallel-0.8.2.tar.gz", hash = "sha256:ccc625bb79b3759a8c280a635a2332371af183020289c9215d04631b55ecfe3e", size = 66369, upload-time = "2026-01-14T09:55:33.797Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/ae/48c9c65807ecdfdd47abe8de308d805e2f3fd782838a0a54ed5505dae627/pytest_run_parallel-0.8.2-py3-none-any.whl", hash = "sha256:2a61aba89f56013859ca5c1efb660755bb6b0349ccfd1dbbef27a94b8c04fe4b", size = 19242, upload-time = "2026-01-14T09:55:32.742Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -3528,8 +3528,8 @@ dev = [ { name = "pystac-client" }, { name = "pytest" }, { name = "pytest-asyncio" }, - { name = "pytest-freethreaded", marker = "python_full_version >= '3.13'" }, { name = "pytest-mypy-plugins" }, + { name = "pytest-run-parallel", marker = "python_full_version >= '3.13'" }, { name = "python-dotenv" }, { name = "pyzmq" }, { name = "ruff" }, @@ -3577,8 +3577,8 @@ dev = [ { name = "pystac-client", specifier = ">=0.8.3" }, { name = "pytest", specifier = ">=8.3.3" }, { name = "pytest-asyncio", specifier = ">=0.24.0" }, - { name = "pytest-freethreaded", marker = "python_full_version >= '3.13'", specifier = ">=0.1.0" }, { name = "pytest-mypy-plugins", specifier = ">=3.2.0" }, + { name = "pytest-run-parallel", marker = "python_full_version >= '3.13'", specifier = ">=0.3.0" }, { name = "python-dotenv", specifier = ">=1.0.1" }, { name = "pyzmq", specifier = ">=27.1.0" }, { name = "ruff", specifier = ">=0.15.0" }, From 0054d7d841062f1f485edd92df6b354de588fc3b Mon Sep 17 00:00:00 2001 From: DisturbedOcean Date: Fri, 6 Mar 2026 09:28:11 -0800 Subject: [PATCH 9/9] fix: Use pytest-run-parallel --- .github/workflows/test-python.yml | 4 +-- pyproject.toml | 4 +++ tests/conftest.py | 50 +++++++++++++++++-------------- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 07ec8895..bfaad14c 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -71,10 +71,10 @@ jobs: run: | uv run --python ${{ matrix.python-version }} pytest tests - - name: Run python tests with --require-gil-disabled + - name: Run python tests with --parallel-threads=2 if: "endsWith(matrix.python-version, 't')" run: | - uv run --python ${{ matrix.python-version }} pytest tests --require-gil-disabled + uv run --python ${{ matrix.python-version }} pytest tests --parallel-threads=2 # Ensure docs build without warnings - name: Check docs diff --git a/pyproject.toml b/pyproject.toml index dc99d903..600156aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,10 @@ markers = [ "network: mark the test as requiring a network connection", "parallel_threads: mark test for concurrent thread execution", ] +thread_unsafe_fixtures = [ + "minio_bucket", + "minio_store", +] [tool.mypy] files = ["obstore/python"] diff --git a/tests/conftest.py b/tests/conftest.py index fafb3c70..10b25de7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,7 @@ from __future__ import annotations +import inspect import socket -import sys import sysconfig import time import warnings @@ -17,29 +17,33 @@ def pytest_configure(config: pytest.Config) -> None: - """Disable pytest-freethreaded's multi-thread defaults. - - On GIL-enabled Python, unregister the plugin entirely so its - pytest_runtest_call hook doesn't wrap tests in a ThreadPoolExecutor - (which breaks async tests run from non-main threads). - - On free-threaded Python, set conservative defaults so tests must opt-in - via @pytest.mark.freethreaded(threads=N, iterations=M). + """On free-threaded Python, enable parallel test execution by default.""" + if ( + sysconfig.get_config_var("Py_GIL_DISABLED") + and hasattr( + config.option, + "parallel_threads", + ) + and config.option.parallel_threads == 1 + ): + config.option.parallel_threads = "auto" + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + """Mark async tests as thread-unsafe. + + pytest-asyncio and pytest-run-parallel are incompatible: run-parallel wraps + the test function before asyncio can schedule it, producing an unawaited + coroutine. Marking every async test as thread_unsafe keeps them on the main + thread where pytest-asyncio expects to run them. + + See: https://github.com/Quansight-Labs/pytest-run-parallel/issues/14 """ - if sys.version_info >= (3, 13) and hasattr(config.option, "threads"): - if sysconfig.get_config_var("Py_GIL_DISABLED"): - config.option.threads = 1 - config.option.iterations = 1 - else: - # Unregister the plugin on GIL-enabled builds so its - # pytest_runtest_call hook doesn't wrap tests in a - # ThreadPoolExecutor (breaks async tests from non-main threads). - try: - import pytest_freethreaded.plugin as ft_plugin - - config.pluginmanager.unregister(ft_plugin) - except (ImportError, ValueError): - pass + for item in items: + if isinstance(item, pytest.Function) and inspect.iscoroutinefunction( + item.function, + ): + item.add_marker(pytest.mark.thread_unsafe) if TYPE_CHECKING: