From 60fe70ee5ad85eccfcdb6d33511e4a95f50e3e33 Mon Sep 17 00:00:00 2001 From: Fitz Elliott Date: Mon, 17 Aug 2026 10:53:04 -0400 Subject: [PATCH] slurp small downloads from oraclecloud * WB has a weird bug (also encountered with googledrive) where trying to download small files indirectly will hang after reading one chunk. "Indirectly" in this case means from MFR, which uses WB as a library to download cached file conversions. This commit applies the same fix to oraclecloud as we used for googlecloud. If the download response has a content-length header AND the header is less than a configurable number of bytes (default: 100k), then we slurp the file contents into memory and wrap it in a StringStream object. No idea why this is happening, but the divine Cargomides seems to appreciate it. --- tests/providers/oraclecloud/test_provider.py | 9 +++++++-- waterbutler/providers/oraclecloud/provider.py | 9 ++++++++- waterbutler/providers/oraclecloud/settings.py | 3 +++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/providers/oraclecloud/test_provider.py b/tests/providers/oraclecloud/test_provider.py index 3114bf662..e8c32d47c 100644 --- a/tests/providers/oraclecloud/test_provider.py +++ b/tests/providers/oraclecloud/test_provider.py @@ -228,7 +228,9 @@ async def test_download_file(self, mock_provider, file_wb_path, file_content): ): stream = await mock_provider.download(file_wb_path) - assert isinstance(stream, ResponseStreamReader) + # workaround: for small files, we slurp into memory and return a StringStream + # assert isinstance(stream, ResponseStreamReader) + assert isinstance(stream, StringStream) @pytest.mark.asyncio async def test_download_folder_raises(self, mock_provider, folder_wb_path): @@ -268,7 +270,10 @@ async def test_download_with_range(self, mock_provider, file_wb_path): ) as mocked: stream = await mock_provider.download(file_wb_path, range=(0, 6)) - assert isinstance(stream, ResponseStreamReader) + # workaround: for small files, we slurp into memory and return a StringStream + # assert isinstance(stream, ResponseStreamReader) + assert isinstance(stream, StringStream) + # Verify Range header was included in the signed headers call_kwargs = mocked.call_args assert call_kwargs.kwargs["headers"]["Range"] == "bytes=0-6" diff --git a/waterbutler/providers/oraclecloud/provider.py b/waterbutler/providers/oraclecloud/provider.py index 663d5b5bf..f10c84950 100644 --- a/waterbutler/providers/oraclecloud/provider.py +++ b/waterbutler/providers/oraclecloud/provider.py @@ -20,8 +20,9 @@ ) from waterbutler.core.path import WaterButlerPath from waterbutler.core.provider import BaseProvider -from waterbutler.core.streams import BaseStream, HashStreamWriter, ResponseStreamReader +from waterbutler.core.streams import BaseStream, HashStreamWriter, ResponseStreamReader, StringStream from waterbutler.core.utils import make_disposition +from waterbutler.providers.googlecloud import settings as pd_settings from waterbutler.providers.oraclecloud.metadata import ( BaseOracleCloudMetadata, OracleCloudFileMetadata, @@ -307,6 +308,12 @@ async def download( f"Object not found: {path}", code=HTTPStatus.NOT_FOUND ) + resp_size = resp.headers.get('Content-Length', None) + if resp_size is not None and int(resp_size) < pd_settings.MAX_SLURP_SIZE: + stream = StringStream(await resp.read()) + await resp.release() + return stream + return ResponseStreamReader(resp) async def delete(self, path: WaterButlerPath, *args, **kwargs) -> None: # type: ignore[override] diff --git a/waterbutler/providers/oraclecloud/settings.py b/waterbutler/providers/oraclecloud/settings.py index 6577c2bf5..8750714dd 100644 --- a/waterbutler/providers/oraclecloud/settings.py +++ b/waterbutler/providers/oraclecloud/settings.py @@ -1,3 +1,6 @@ from waterbutler import settings config = settings.child("ORACLECLOUD_PROVIDER_SETTINGS") + +# slurp downloads below this threshhold (in bytes) +MAX_SLURP_SIZE = 100 * 1000