diff --git a/.github/workflows/test-python.yml b/.github/workflows/test-python.yml index 08ec9df9..bfaad14c 100644 --- a/.github/workflows/test-python.yml +++ b/.github/workflows/test-python.yml @@ -64,11 +64,17 @@ jobs: - name: Build rust submodules run: | - uv run --python ${{ matrix.python-version }} maturin develop --uv -m obstore/Cargo.toml + uv run --python ${{ matrix.python-version }} maturin develop -m obstore/Cargo.toml - name: Run python tests + if: "!endsWith(matrix.python-version, 't')" run: | - uv run --python ${{ matrix.python-version }} pytest + uv run --python ${{ matrix.python-version }} pytest tests + + - name: Run python tests with --parallel-threads=2 + if: "endsWith(matrix.python-version, 't')" + run: | + uv run --python ${{ matrix.python-version }} pytest tests --parallel-threads=2 # Ensure docs build without warnings - name: Check docs diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 17cc2632..bcc31f58 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -52,7 +52,7 @@ jobs: version: "0.10.x" - name: Install Python versions - run: uv python install 3.10 3.11 pypy3.11 + run: uv python install 3.10 3.11 3.13t 3.14t pypy3.11 - name: Build abi3-py311 wheels uses: PyO3/maturin-action@v1 @@ -66,7 +66,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i 3.10 -i pypy3.11 --manifest-path obstore/Cargo.toml + args: --release --out dist -i 3.10 -i 3.13t -i 3.14t -i pypy3.11 --manifest-path obstore/Cargo.toml sccache: "true" manylinux: ${{ matrix.platform.manylinux }} @@ -99,7 +99,7 @@ jobs: version: "0.10.x" - name: Install Python versions - run: uv python install 3.10 3.11 pypy3.11 + run: uv python install 3.10 3.11 3.13t 3.14t pypy3.11 - name: Build abi3-py311 wheels uses: PyO3/maturin-action@v1 @@ -113,7 +113,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i 3.10 -i pypy3.11 --manifest-path obstore/Cargo.toml + args: --release --out dist -i 3.10 -i 3.13t -i 3.14t -i pypy3.11 --manifest-path obstore/Cargo.toml sccache: "true" manylinux: musllinux_1_2 @@ -145,14 +145,14 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist --features abi3-py311 -i 3.11 --manifest-path obstore/Cargo.toml + args: --release --out dist --features abi3-py311 --features generate-import-lib -i 3.11 --manifest-path obstore/Cargo.toml sccache: "true" - name: Build version-specific wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i 3.10 --manifest-path obstore/Cargo.toml + args: --release --out dist --features generate-import-lib -i 3.10 -i 3.13t -i 3.14t --manifest-path obstore/Cargo.toml sccache: "true" - name: Upload wheels uses: actions/upload-artifact@v4 @@ -179,7 +179,7 @@ jobs: version: "0.10.x" - name: Install Python versions - run: uv python install 3.10 3.11 pypy3.11 + run: uv python install 3.10 3.11 3.13t 3.14t pypy3.11 - name: Build abi3-py311 wheels uses: PyO3/maturin-action@v1 @@ -192,7 +192,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist -i 3.10 -i pypy3.11 --manifest-path obstore/Cargo.toml + args: --release --out dist -i 3.10 -i 3.13t -i 3.14t -i pypy3.11 --manifest-path obstore/Cargo.toml sccache: "true" - name: Upload wheels uses: actions/upload-artifact@v4 diff --git a/Cargo.lock b/Cargo.lock index 93022e8d..f485548a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1576,6 +1576,7 @@ version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bf94ee265674bf76c09fa430b0e99c26e319c945d96ca0d5a8215f31bf81cf7" dependencies = [ + "python3-dll-a", "target-lexicon", ] @@ -1654,6 +1655,15 @@ dependencies = [ "url", ] +[[package]] +name = "python3-dll-a" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d381ef313ae70b4da5f95f8a4de773c6aa5cd28f73adec4b4a31df70b66780d8" +dependencies = [ + "cc", +] + [[package]] name = "quick-xml" version = "0.39.2" diff --git a/obstore/Cargo.toml b/obstore/Cargo.toml index d591cb0b..81ac0d07 100644 --- a/obstore/Cargo.toml +++ b/obstore/Cargo.toml @@ -19,6 +19,8 @@ crate-type = ["cdylib"] [features] abi3-py311 = ["pyo3/abi3-py311"] +# https://www.maturin.rs/distribution.html#cross-compile-to-windows +generate-import-lib = ["pyo3/generate-import-lib"] [dependencies] arrow = "57" diff --git a/obstore/src/lib.rs b/obstore/src/lib.rs index 1e10db0b..edc29fa3 100644 --- a/obstore/src/lib.rs +++ b/obstore/src/lib.rs @@ -43,7 +43,7 @@ fn check_debug_build(_py: Python) -> PyResult<()> { } /// A Python module implemented in Rust. -#[pymodule] +#[pymodule(gil_used = false)] fn _obstore(py: Python, m: &Bound) -> PyResult<()> { check_debug_build(py)?; diff --git a/pyproject.toml b/pyproject.toml index c809d1e8..600156aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ examples = ["fastapi>=0.115.12", "tqdm>=4.67.1"] dev = [ "aiohttp-retry>=2.9.1", "aiohttp>=3.11.13", - "arro3-core>=0.4.2", + "arro3-core>=0.6.5", "azure-identity>=1.21.0", "boto3>=1.38.21", "docker>=7.1.0", @@ -42,10 +42,12 @@ dev = [ "pyarrow>=17.0.0", "pystac-client>=0.8.3", "pystac>=1.10.1", + "pytest>=8.3.3", "pytest-asyncio>=0.24.0", "pytest-mypy-plugins>=3.2.0", - "pytest>=8.3.3", + "pytest-run-parallel>=0.3.0;python_version>='3.13'", "python-dotenv>=1.0.1", + "pyzmq>=27.1.0", "ruff>=0.15.0", "types-boto3[s3,sts]>=1.36.23", "types-requests>=2.31.0.6", @@ -108,7 +110,14 @@ executionEnvironments = [ addopts = "-v --mypy-only-local-stub" asyncio_default_fixture_loop_scope = "function" testpaths = ["tests"] -markers = ["network: mark the test as requiring a network connection"] +markers = [ + "network: mark the test as requiring a network connection", + "parallel_threads: mark test for concurrent thread execution", +] +thread_unsafe_fixtures = [ + "minio_bucket", + "minio_store", +] [tool.mypy] files = ["obstore/python"] diff --git a/tests/conftest.py b/tests/conftest.py index ad15f08c..10b25de7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ from __future__ import annotations +import inspect import socket +import sysconfig import time import warnings from typing import TYPE_CHECKING, Any @@ -13,6 +15,37 @@ from obstore.store import S3Store + +def pytest_configure(config: pytest.Config) -> None: + """On free-threaded Python, enable parallel test execution by default.""" + if ( + sysconfig.get_config_var("Py_GIL_DISABLED") + and hasattr( + config.option, + "parallel_threads", + ) + and config.option.parallel_threads == 1 + ): + config.option.parallel_threads = "auto" + + +def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: + """Mark async tests as thread-unsafe. + + pytest-asyncio and pytest-run-parallel are incompatible: run-parallel wraps + the test function before asyncio can schedule it, producing an unawaited + coroutine. Marking every async test as thread_unsafe keeps them on the main + thread where pytest-asyncio expects to run them. + + See: https://github.com/Quansight-Labs/pytest-run-parallel/issues/14 + """ + for item in items: + if isinstance(item, pytest.Function) and inspect.iscoroutinefunction( + item.function, + ): + item.add_marker(pytest.mark.thread_unsafe) + + if TYPE_CHECKING: from collections.abc import Generator @@ -116,13 +149,17 @@ def minio_config() -> Generator[tuple[S3Config, ClientConfig], Any, None]: def minio_bucket( minio_config: tuple[S3Config, ClientConfig], ) -> Generator[tuple[S3Config, ClientConfig], Any, None]: + # Clean bucket before each test so tests always start with empty state, + # regardless of whether a previous test's teardown failed or was incomplete. + store = S3Store(config=minio_config[0], client_options=minio_config[1]) + objects = store.list().collect() + store.delete([obj["path"] for obj in objects]) + yield minio_config - # Remove all files from bucket - store = S3Store(config=minio_config[0], client_options=minio_config[1]) + # Best-effort cleanup after the test as well. objects = store.list().collect() - paths = [obj["path"] for obj in objects] - store.delete(paths) + store.delete([obj["path"] for obj in objects]) @pytest.fixture diff --git a/tests/store/test_local.py b/tests/store/test_local.py index e9ee2ebb..77acbe39 100644 --- a/tests/store/test_local.py +++ b/tests/store/test_local.py @@ -1,5 +1,6 @@ import pickle from pathlib import Path +from tempfile import TemporaryDirectory import pytest @@ -44,30 +45,36 @@ def test_local_from_url(): store = LocalStore.from_url(url) -def test_create_prefix(tmp_path: Path): - tmpdir = tmp_path / "abc" - assert not tmpdir.exists() - LocalStore(tmpdir, mkdir=True) - assert tmpdir.exists() +def test_create_prefix(): + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + tmpdir = tmp_path / "abc" + assert not tmpdir.exists() + LocalStore(tmpdir, mkdir=True) + assert tmpdir.exists() - # Assert that mkdir=True works even when the dir already exists - LocalStore(tmpdir, mkdir=True) - assert tmpdir.exists() + # Assert that mkdir=True works even when the dir already exists + LocalStore(tmpdir, mkdir=True) + assert tmpdir.exists() -def test_prefix_property(tmp_path: Path): - store = LocalStore(tmp_path) - assert store.prefix == tmp_path - assert isinstance(store.prefix, Path) - # Can pass it back to the store init - LocalStore(store.prefix) +def test_prefix_property(): + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + store = LocalStore(tmp_path) + assert store.prefix == tmp_path + assert isinstance(store.prefix, Path) + # Can pass it back to the store init + LocalStore(store.prefix) -def test_pickle(tmp_path: Path): - store = LocalStore(tmp_path) - store.put("path.txt", b"foo") - new_store: LocalStore = pickle.loads(pickle.dumps(store)) - assert new_store.get("path.txt").bytes() == b"foo" +def test_pickle(): + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + store = LocalStore(tmp_path) + store.put("path.txt", b"foo") + new_store: LocalStore = pickle.loads(pickle.dumps(store)) + assert new_store.get("path.txt").bytes() == b"foo" def test_eq(): @@ -79,18 +86,20 @@ def test_eq(): assert store != store3 -def test_local_store_percent_encoded(tmp_path: Path): - fname1 = "hello%20world.txt" - content1 = b"Hello, World!" - with (tmp_path / fname1).open("wb") as f: - f.write(content1) +def test_local_store_percent_encoded(): + with TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + fname1 = "hello%20world.txt" + content1 = b"Hello, World!" + with (tmp_path / fname1).open("wb") as f: + f.write(content1) - store = LocalStore(tmp_path) - assert store.get(fname1).bytes() == content1 + store = LocalStore(tmp_path) + assert store.get(fname1).bytes() == content1 - fname2 = "hello world.txt" - content2 = b"Hello, World! (with spaces)" - with (tmp_path / fname2).open("wb") as f: - f.write(content2) + fname2 = "hello world.txt" + content2 = b"Hello, World! (with spaces)" + with (tmp_path / fname2).open("wb") as f: + f.write(content2) - assert store.get(fname2).bytes() == content2 + assert store.get(fname2).bytes() == content2 diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index 0c89fcc3..e4273d48 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -2,6 +2,8 @@ import gc import os +from pathlib import Path +from tempfile import TemporaryDirectory from typing import TYPE_CHECKING from unittest.mock import patch @@ -14,8 +16,6 @@ from tests.conftest import TEST_BUCKET_NAME if TYPE_CHECKING: - from pathlib import Path - from obstore.store import ClientConfig, S3Config @@ -212,13 +212,15 @@ def test_list( fs.pipe_file(f"{bucket}/afile", b"hello world") out = fs.ls(f"{bucket}", detail=False, refresh=True) - assert out == [f"{bucket}/afile"] + assert f"{bucket}/afile" in out fs.pipe_file(f"{bucket}/dir/bfile", b"data") out = fs.ls(f"{bucket}", detail=False, refresh=True) - assert out == [f"{bucket}/afile", f"{bucket}/dir"] + assert f"{bucket}/afile" in out + assert f"{bucket}/dir" in out out = fs.ls(f"{bucket}", detail=True, refresh=True) - assert out[0]["type"] == "file" - assert out[1]["type"] == "directory" + types = {o["name"]: o["type"] for o in out} + assert types.get(f"{bucket}/afile") == "file" + assert types.get(f"{bucket}/dir") == "directory" @pytest.mark.asyncio @@ -239,13 +241,15 @@ async def test_list_async( await fs._pipe_file(f"{bucket}/afile", b"hello world") out = await fs._ls(f"{bucket}", detail=False, refresh=True) - assert out == [f"{bucket}/afile"] + assert f"{bucket}/afile" in out await fs._pipe_file(f"{bucket}/dir/bfile", b"data") out = await fs._ls(f"{bucket}", detail=False, refresh=True) - assert out == [f"{bucket}/afile", f"{bucket}/dir"] + assert f"{bucket}/afile" in out + assert f"{bucket}/dir" in out out = await fs._ls(f"{bucket}", detail=True, refresh=True) - assert out[0]["type"] == "file" - assert out[1]["type"] == "directory" + types = {o["name"]: o["type"] for o in out} + assert types.get(f"{bucket}/afile") == "file" + assert types.get(f"{bucket}/dir") == "directory" def test_info(minio_bucket: tuple[S3Config, ClientConfig]): @@ -318,7 +322,7 @@ async def test_info_async(minio_bucket: tuple[S3Config, ClientConfig]): assert not await fs._isdir(f"{bucket}/dir_1/") -def test_put_files(minio_bucket: tuple[S3Config, ClientConfig], tmp_path: Path): +def test_put_files(minio_bucket: tuple[S3Config, ClientConfig]): """Test put new file to S3 synchronously.""" register("s3") fs = fsspec.filesystem( @@ -330,14 +334,15 @@ def test_put_files(minio_bucket: tuple[S3Config, ClientConfig], tmp_path: Path): bucket = minio_bucket[0].get("bucket") assert bucket is not None - test_data = "Hello, World!" - local_file_path = tmp_path / "test_file.txt" - local_file_path.write_text(test_data) + with TemporaryDirectory() as tmp: + test_data = "Hello, World!" + local_file_path = Path(tmp) / "test_file.txt" + local_file_path.write_text(test_data) - assert local_file_path.read_text() == test_data - remote_file_path = f"{bucket}/uploaded_test_file.txt" + assert local_file_path.read_text() == test_data + remote_file_path = f"{bucket}/uploaded_test_file.txt" - fs.put(str(local_file_path), remote_file_path) + fs.put(str(local_file_path), remote_file_path) # Verify file upload assert remote_file_path in fs.ls(f"{bucket}", detail=False, refresh=True) @@ -350,7 +355,6 @@ def test_put_files(minio_bucket: tuple[S3Config, ClientConfig], tmp_path: Path): @pytest.mark.asyncio async def test_put_files_async( minio_bucket: tuple[S3Config, ClientConfig], - tmp_path: Path, ): """Test put new file to S3 asynchronously.""" register("s3") @@ -361,14 +365,15 @@ async def test_put_files_async( asynchronous=True, ) - test_data = "Hello, World!" - local_file_path = tmp_path / "test_file.txt" - local_file_path.write_text(test_data) + with TemporaryDirectory() as tmp: + test_data = "Hello, World!" + local_file_path = Path(tmp) / "test_file.txt" + local_file_path.write_text(test_data) - assert local_file_path.read_text() == test_data - remote_file_path = f"{TEST_BUCKET_NAME}/uploaded_test_file.txt" + assert local_file_path.read_text() == test_data + remote_file_path = f"{TEST_BUCKET_NAME}/uploaded_test_file.txt" - await fs._put(str(local_file_path), remote_file_path) + await fs._put(str(local_file_path), remote_file_path) # Verify file upload assert remote_file_path in await fs._ls( diff --git a/uv.lock b/uv.lock index 184fbfa1..516362e6 100644 --- a/uv.lock +++ b/uv.lock @@ -3,7 +3,8 @@ revision = 3 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.14'", - "python_full_version >= '3.11' and python_full_version < '3.14'", + "python_full_version == '3.13.*'", + "python_full_version >= '3.11' and python_full_version < '3.13'", "python_full_version < '3.11'", ] @@ -1249,7 +1250,8 @@ version = "9.10.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.14'", - "python_full_version >= '3.11' and python_full_version < '3.14'", + "python_full_version == '3.13.*'", + "python_full_version >= '3.11' and python_full_version < '3.13'", ] dependencies = [ { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, @@ -2891,6 +2893,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4e/49/a5f212e829e2b05b31aad9e96b89e70cd73c7a604f74df34825eac5fd227/pytest_mypy_plugins-3.3.0-py3-none-any.whl", hash = "sha256:7b93f338609eace9405986be69fb52344dfb3ccbb96f3fc5ac3cbd257a34c5fc", size = 20798, upload-time = "2026-02-16T17:24:54.693Z" }, ] +[[package]] +name = "pytest-run-parallel" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest", marker = "python_full_version >= '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/91/14b1403b47f595d360af67624ee50eae2129f04f56e354ce6c04dbda9d93/pytest_run_parallel-0.8.2.tar.gz", hash = "sha256:ccc625bb79b3759a8c280a635a2332371af183020289c9215d04631b55ecfe3e", size = 66369, upload-time = "2026-01-14T09:55:33.797Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/ae/48c9c65807ecdfdd47abe8de308d805e2f3fd782838a0a54ed5505dae627/pytest_run_parallel-0.8.2-py3-none-any.whl", hash = "sha256:2a61aba89f56013859ca5c1efb660755bb6b0349ccfd1dbbef27a94b8c04fe4b", size = 19242, upload-time = "2026-01-14T09:55:32.742Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -3515,7 +3529,9 @@ dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-mypy-plugins" }, + { name = "pytest-run-parallel", marker = "python_full_version >= '3.13'" }, { name = "python-dotenv" }, + { name = "pyzmq" }, { name = "ruff" }, { name = "types-boto3", extra = ["s3", "sts"] }, { name = "types-requests" }, @@ -3544,7 +3560,7 @@ examples = [ dev = [ { name = "aiohttp", specifier = ">=3.11.13" }, { name = "aiohttp-retry", specifier = ">=2.9.1" }, - { name = "arro3-core", specifier = ">=0.4.2" }, + { name = "arro3-core", specifier = ">=0.6.5" }, { name = "azure-identity", specifier = ">=1.21.0" }, { name = "boto3", specifier = ">=1.38.21" }, { name = "docker", specifier = ">=7.1.0" }, @@ -3562,7 +3578,9 @@ dev = [ { name = "pytest", specifier = ">=8.3.3" }, { name = "pytest-asyncio", specifier = ">=0.24.0" }, { name = "pytest-mypy-plugins", specifier = ">=3.2.0" }, + { name = "pytest-run-parallel", marker = "python_full_version >= '3.13'", specifier = ">=0.3.0" }, { name = "python-dotenv", specifier = ">=1.0.1" }, + { name = "pyzmq", specifier = ">=27.1.0" }, { name = "ruff", specifier = ">=0.15.0" }, { name = "types-boto3", extras = ["s3", "sts"], specifier = ">=1.36.23" }, { name = "types-requests", specifier = ">=2.31.0.6" },