Unify remote on-demand sequence retrieval across gtars bindings

.github/workflows/build-wasm-bindings.yml

@@ -13,4 +13,5 @@ jobs:
             - name: Build bindings and prepare for publish (rename package)
               run: |
                 wasm-pack build --release --target web gtars-wasm
-                jq '.name = "@databio/gtars" | .repository = {"type": "git", "url": "https://github.com/databio/gtars"}' gtars-wasm/pkg/package.json > tmp.json && mv tmp.json gtars-wasm/pkg/package.json
+                cp gtars-wasm/js/remote-refget-store.js gtars-wasm/js/remote-refget-store.d.ts gtars-wasm/pkg/
+                jq '.name = "@databio/gtars" | .repository = {"type": "git", "url": "https://github.com/databio/gtars"} | .files += ["remote-refget-store.js", "remote-refget-store.d.ts"] | .exports = {".": {"types": ("./" + .types), "default": ("./" + .main)}, "./remote": {"types": "./remote-refget-store.d.ts", "default": "./remote-refget-store.js"}}' gtars-wasm/pkg/package.json > tmp.json && mv tmp.json gtars-wasm/pkg/package.json

.github/workflows/rust-publish.yml

@@ -339,7 +339,8 @@ jobs:
         run: |
           npm install -g npm@latest   # OIDC trusted publishing requires npm >= 11.5.1
           wasm-pack build --release --target web gtars-wasm
-          jq '.name = "@databio/gtars" | .repository = {"type": "git", "url": "https://github.com/databio/gtars"}' gtars-wasm/pkg/package.json > tmp.json && mv tmp.json gtars-wasm/pkg/package.json
+          cp gtars-wasm/js/remote-refget-store.js gtars-wasm/js/remote-refget-store.d.ts gtars-wasm/pkg/
+          jq '.name = "@databio/gtars" | .repository = {"type": "git", "url": "https://github.com/databio/gtars"} | .files += ["remote-refget-store.js", "remote-refget-store.d.ts"] | .exports = {".": {"types": ("./" + .types), "default": ("./" + .main)}, "./remote": {"types": "./remote-refget-store.d.ts", "default": "./remote-refget-store.js"}}' gtars-wasm/pkg/package.json > tmp.json && mv tmp.json gtars-wasm/pkg/package.json
           if npm publish --provenance --access public gtars-wasm/pkg/ 2>npm_err.log; then
             echo "published @databio/gtars"
           elif grep -qiE "cannot publish over|previously published|already exists" npm_err.log; then

Makefile

@@ -1,6 +1,9 @@
 wasm:
 	wasm-pack build --target web --release gtars-wasm
-	jq '.name = "@databio/gtars" | .repository = {"type": "git", "url": "https://github.com/databio/gtars"}' gtars-wasm/pkg/package.json > tmp.json && mv tmp.json gtars-wasm/pkg/package.json
+	# Ship the hand-written JS layer (RemoteRefgetStore: byte-range + OPFS refget
+	# client) alongside the generated bindings, exposed as @databio/gtars/remote.
+	cp gtars-wasm/js/remote-refget-store.js gtars-wasm/js/remote-refget-store.d.ts gtars-wasm/pkg/
+	jq '.name = "@databio/gtars" | .repository = {"type": "git", "url": "https://github.com/databio/gtars"} | .files += ["remote-refget-store.js", "remote-refget-store.d.ts"] | .exports = {".": {"types": ("./" + .types), "default": ("./" + .main)}, "./remote": {"types": "./remote-refget-store.d.ts", "default": "./remote-refget-store.js"}}' gtars-wasm/pkg/package.json > tmp.json && mv tmp.json gtars-wasm/pkg/package.json
 
 test:
 	cargo test --all --workspace -- --nocapture

gtars-node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "gtars-node"
-version = "0.7.0"
+version = "0.7.1"
 edition = "2024"
 
 [lib]

gtars-node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@databio/gtars-node",
-  "version": "0.7.0",
+  "version": "0.7.1",
   "description": "Native Node.js bindings for gtars (genomic tools and region sets)",
   "main": "wrapper.js",
   "types": "wrapper.d.ts",
@@ -28,7 +28,7 @@
   "scripts": {
     "build": "napi build --platform --release",
     "build:debug": "napi build --platform",
-    "test": "node --test __test__/",
+    "test": "node --test __test__/*.test.mjs",
     "prepublishOnly": "napi prepublish -t npm"
   },
   "devDependencies": {

gtars-node/src/lib.rs

@@ -104,16 +104,28 @@ impl RefgetStore {
         }
     }
 
-    // -- Sequence retrieval (auto-loads stubs from disk/remote) --
+    // -- Sequence retrieval (flow 1: lean partial read) --
+    //
+    // These read only the bytes covering the requested region, resolving
+    // resident -> local `.seq` -> remote HTTP byte-range, and persist nothing.
+    // They do NOT implicitly download whole sequences; for repeat-heavy
+    // workloads call `loadSequence` (flow 3) first to cache the sequence.
 
     #[napi]
     pub fn get_sequence(&self, digest: String) -> Result<String> {
         let mut store = self.inner.lock().map_err(lock_err)?;
-        store.load_sequence(&digest).map_err(to_napi_err)?;
-        let record = store.get_sequence(&digest).map_err(to_napi_err)?;
-        record
-            .decode()
-            .ok_or_else(|| napi::Error::from_reason("Failed to decode sequence"))
+        // Ensure the sequence index (metadata stubs) is loaded so a remote
+        // sequence is locatable; this is cheap and a no-op once loaded. It does
+        // NOT download sequence bytes.
+        store.load_sequence_index().map_err(to_napi_err)?;
+        // Whole sequence via the lean partial-read path (no implicit caching).
+        let length = store
+            .get_sequence_metadata(&digest)
+            .ok_or_else(|| napi::Error::from_reason(format!("Sequence not found: {}", digest)))?
+            .length;
+        store
+            .get_substring(&digest, 0, length)
+            .map_err(to_napi_err)
     }
 
     #[napi]
@@ -126,7 +138,9 @@ impl RefgetStore {
         let start = bigint_to_u64(start, "start")?;
         let end = bigint_to_u64(end, "end")?;
         let mut store = self.inner.lock().map_err(lock_err)?;
-        store.load_sequence(&digest).map_err(to_napi_err)?;
+        // Cheap, no-op-once-loaded index load so remote sequences are locatable;
+        // does not download sequence bytes (those come via byte-range below).
+        store.load_sequence_index().map_err(to_napi_err)?;
         store
             .get_substring(&digest, start as usize, end as usize)
             .map_err(to_napi_err)
@@ -144,15 +158,31 @@ impl RefgetStore {
         let record = store
             .get_sequence_by_name(&collection_digest, &name)
             .map_err(to_napi_err)?;
-        // Get the digest so we can load the sequence data
+        // Resolve the digest + length, then serve the whole sequence via the
+        // lean partial-read path (no implicit whole-sequence download/cache).
         let seq_digest = record.metadata().sha512t24u.clone();
-        store.load_sequence(&seq_digest).map_err(to_napi_err)?;
-        let record = store
-            .get_sequence_by_name(&collection_digest, &name)
-            .map_err(to_napi_err)?;
-        record
-            .decode()
-            .ok_or_else(|| napi::Error::from_reason("Failed to decode sequence"))
+        let length = record.metadata().length;
+        store
+            .get_substring(&seq_digest, 0, length)
+            .map_err(to_napi_err)
+    }
+
+    // -- Flow 3: explicit whole-sequence download + cache --
+
+    /// Download a whole sequence and cache it (resident, and persisted to the
+    /// local cache dir when the store is configured to persist). Opt-in for
+    /// repeat-heavy workloads; subsequent reads are served from memory.
+    #[napi]
+    pub fn load_sequence(&self, digest: String) -> Result<()> {
+        let mut store = self.inner.lock().map_err(lock_err)?;
+        store.load_sequence(&digest).map_err(to_napi_err)
+    }
+
+    /// Download and cache every sequence in the store (flow 3, in bulk).
+    #[napi]
+    pub fn load_all_sequences(&self) -> Result<()> {
+        let mut store = self.inner.lock().map_err(lock_err)?;
+        store.load_all_sequences().map_err(to_napi_err)
     }
 
     // -- Listing / metadata --
@@ -312,11 +342,12 @@ impl RefgetStore {
         // Note: we deliberately do NOT call `load_sequence` here. Pulling the
         // entire sequence into memory before streaming defeats the O(1) memory
         // guarantee of `stream_sequence` — peak memory would grow to roughly
-        // 2x the encoded sequence for chromosome-sized reads. `stream_sequence`
-        // already falls back to the on-disk / remote byte source when the
-        // record is a `Stub`, so no pre-loading is required.
+        // 2x the encoded sequence for chromosome-sized reads. We only ensure the
+        // sequence *index* (metadata stubs) is loaded so a remote sequence is
+        // locatable; `stream_sequence` then byte-ranges the data itself.
         let reader: Box<dyn Read + Send> = {
-            let store = self.inner.lock().map_err(lock_err)?;
+            let mut store = self.inner.lock().map_err(lock_err)?;
+            store.load_sequence_index().map_err(to_napi_err)?;
             store
                 .stream_sequence(&digest, start, end)
                 .map_err(to_napi_err)?

gtars-python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "gtars-py"
-version = "0.9.1"
+version = "0.9.2"
 edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -28,7 +28,7 @@ vrs = ["dep:gtars-vrs", "gtars-vrs?/transcripts", "reftx"]
 [dependencies]
 anyhow = { workspace = true }
 serde_json = { workspace = true }
-pyo3 = { version = "0.27.1", features=["anyhow", "extension-module"] }
+pyo3 = { version = "0.29.0", features=["anyhow", "extension-module"] }
 openssl = { version = "0.10", features = ["vendored"] }
 
 # our code (optional, behind feature flags)

gtars-python/py_src/gtars/refget/__init__.pyi

@@ -253,6 +253,33 @@ class RetrievedSequence:
     def __repr__(self) -> str: ...
     def __str__(self) -> str: ...
 
+class SequenceStream:
+    """Lazy, O(1)-memory stream over a (sub)sequence's decoded ASCII bases.
+
+    Produced by ``RefgetStore.stream_sequence()`` /
+    ``ReadonlyRefgetStore.stream_sequence()`` (flow 2). Iterating yields ``str``
+    chunks of decoded bases as they are read from the backing source (resident
+    bytes, a local ``.seq``, or a remote HTTP byte-range), so peak memory stays
+    bounded regardless of region length.
+
+    Example::
+
+        stream = store.stream_sequence("chr1_digest", 0, 1_000_000)
+        total = 0
+        for chunk in stream:
+            total += len(chunk)
+    """
+
+    def __iter__(self) -> "SequenceStream": ...
+    def __next__(self) -> str: ...
+    def read_all(self) -> str:
+        """Read and return all remaining bases as a single string.
+
+        Exhausts the stream; defeats the O(1)-memory guarantee for the size of
+        the result.
+        """
+        ...
+
 class StorageMode(Enum):
     """Defines how sequence data is stored in the Refget store.
 
@@ -952,12 +979,15 @@ class RefgetStore:
         ...
 
     def get_substring(self, seq_digest: str, start: int, end: int) -> str:
-        """Extract a substring from a sequence.
+        """Extract a substring from a sequence (flow 1: lean partial read).
 
-        Retrieves a specific region from a sequence using 0-based, half-open
-        coordinates [start, end). Automatically loads sequence data if not
-        already cached (for lazy-loaded stores). Automatically strips "SQ."
-        prefix from digest if present.
+        Retrieves a specific region using 0-based, half-open coordinates
+        [start, end), reading only the bytes that cover the region. Source
+        resolution is resident -> local ``.seq`` -> remote HTTP byte-range, so a
+        remote-backed store is served directly with no manual preload and
+        without downloading or caching the whole sequence. For repeat-heavy
+        access call ``load_sequence()`` (flow 3) first. Automatically strips
+        "SQ." prefix from digest if present.
 
         Args:
             seq_digest: Sequence digest (SHA-512/24u), optionally with "SQ." prefix.
@@ -978,6 +1008,31 @@ class RefgetStore:
         """
         ...
 
+    def stream_sequence(
+        self,
+        seq_digest: str,
+        start: Optional[int] = None,
+        end: Optional[int] = None,
+        chunk_size: int = 65536,
+    ) -> SequenceStream:
+        """Stream a (sub)sequence as decoded bases without loading it (flow 2).
+
+        Returns a :class:`SequenceStream` that yields ``str`` chunks lazily,
+        reading from resident bytes, a local ``.seq``, or a remote HTTP
+        byte-range as needed. Peak memory is O(1) in the region length. Omit
+        ``start``/``end`` to stream the whole sequence.
+
+        Args:
+            seq_digest: Sequence digest (SHA-512/24u), optionally with "SQ." prefix.
+            start: Start position (0-based, inclusive). Defaults to 0.
+            end: End position (0-based, exclusive). Defaults to the sequence length.
+            chunk_size: Bytes read per chunk (default 64 KiB).
+
+        Returns:
+            A SequenceStream iterator of decoded base chunks.
+        """
+        ...
+
     # =========================================================================
     # Store Management
     # =========================================================================
@@ -1491,7 +1546,11 @@ class ReadonlyRefgetStore:
         ...
 
     def get_substring(self, seq_digest: str, start: int, end: int) -> str:
-        """Extract a substring from a sequence.
+        """Extract a substring from a sequence (flow 1: lean partial read).
+
+        Source resolution is resident -> local ``.seq`` -> remote HTTP
+        byte-range; remote stores are served with no preload and without
+        downloading whole sequences.
 
         Args:
             seq_digest: Sequence digest (SHA-512/24u).
@@ -1506,6 +1565,22 @@ class ReadonlyRefgetStore:
         """
         ...
 
+    def stream_sequence(
+        self,
+        seq_digest: str,
+        start: Optional[int] = None,
+        end: Optional[int] = None,
+        chunk_size: int = 65536,
+    ) -> SequenceStream:
+        """Stream a (sub)sequence as decoded bases without loading it (flow 2).
+
+        Returns a :class:`SequenceStream` yielding ``str`` chunks lazily from
+        resident bytes, a local ``.seq``, or a remote HTTP byte-range. Peak
+        memory is O(1) in the region length. Omit ``start``/``end`` to stream
+        the whole sequence.
+        """
+        ...
+
     # =========================================================================
     # Store Info
     # =========================================================================

gtars-python/src/lib.rs

@@ -90,7 +90,7 @@ fn gtars(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
         sys_modules.set_item("gtars.vrs", &vrs_bound)?;
         // Register the nested gtars.vrs.hgvs submodule so
         // `from gtars.vrs.hgvs import parse_hgvs` works.
-        let vrs_module_ref = vrs_bound.downcast::<PyModule>()?;
+        let vrs_module_ref = vrs_bound.cast::<PyModule>()?;
         let hgvs_mod = PyModule::new(py, "hgvs")?;
         vrs::hgvs::register(py, &hgvs_mod)?;
         vrs_module_ref.add_submodule(&hgvs_mod)?;