Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
9ff2c28
feat(fdb): switch to pyfdb 2.0
peshence Apr 8, 2026
956cb15
handle mixed step types
Apr 10, 2026
56a0cad
Update datacube.py
mathleur Apr 10, 2026
5695918
Update fdb.py
mathleur Apr 10, 2026
fc45637
Update datacube_type_change.py
mathleur Apr 10, 2026
13ec530
Update datacube_axis.py
mathleur Apr 10, 2026
e6621e7
fix tests and rethink how to find indexes between, one slow way for m…
mathleur Apr 10, 2026
1fb4cb0
remove cffi test dep
mathleur Apr 13, 2026
c650c50
qa
mathleur Apr 13, 2026
4bf5ff7
Add service-backed switching grid lookup
Apr 29, 2026
16f1a50
Add switching grid service tests
Apr 29, 2026
7c6e758
Add logging to switching grid service
Apr 30, 2026
a79546d
fix: fail when dynamic grid has been requested but fails
peshence May 1, 2026
eb2372d
unique list of timesteps
mathleur May 4, 2026
d9f9d63
chore(config): meaningful error when axis engine is not specified
peshence May 5, 2026
d1737cc
chore: remove grid service (moved to polytope-server
peshence May 7, 2026
c3765f5
fix(switching grids): don't fdb read if cache hits, handle missing ge…
peshence May 8, 2026
f322a99
chore(switching grids): add in memory caching and thread lock
peshence May 8, 2026
7f12a4c
chore: isort fixes
peshence May 8, 2026
0635bc7
chore: remove pre-commit default version to allow other dev envs
peshence May 8, 2026
0866b37
chore: black, flake8
peshence May 8, 2026
4f48c7c
fix: manual grib message size read for dynamic grid (remotefdb does n…
peshence May 8, 2026
7b16de8
fix: use first_element.length()
peshence May 8, 2026
645bb3f
chore: remove dynamic grid resolution
peshence May 11, 2026
d881dec
chore: revert pyfdb update to fix debian ci
peshence May 12, 2026
caf7dae
Merge pull request #523 from ecmwf/feat/fix_timestep_types
mathleur Jun 1, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
default_language_version:
python: python3.10

repos:
- repo: https://github.com/pycqa/isort
rev: 7.0.0
Expand All @@ -16,4 +15,4 @@ repos:
- repo: https://github.com/pycqa/flake8
rev: 7.3.0 # Use the latest version of flake8
hooks:
- id: flake8
- id: flake8
129 changes: 98 additions & 31 deletions polytope_feature/datacube/datacube_axis.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,39 +101,88 @@ def _remap_val_to_axis_range(self, value):
value = transformation._remap_val_to_axis_range(value, self)
return value

def find_standard_indices_between(self, indexes, low, up, datacube, method=None):
indexes_between_ranges = []

if self.name in datacube.complete_axes and self.name not in datacube.transformed_axes:
# Find the range of indexes between lower and upper
# https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html
# Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing
if method == "surrounding" or method == "nearest":
start = indexes.searchsorted(low, "left")
end = indexes.searchsorted(up, "right")
start = max(start - 1, 0)
end = min(end + 1, len(indexes))
indexes_between = indexes[start:end].to_list()
indexes_between_ranges.extend(indexes_between)
else:
start = indexes.searchsorted(low, "left")
end = indexes.searchsorted(up, "right")
indexes_between = indexes[start:end].to_list()
indexes_between_ranges.extend(indexes_between)
def _mixed_key(self, x):

if isinstance(x, pd.Timedelta):
return (0, x.total_seconds())
elif isinstance(x, str):
return (1, x)
else:
if method == "surrounding" or method == "nearest":
start = bisect.bisect_left(indexes, low)
end = bisect.bisect_right(indexes, up)
return (2, x)

def _is_mixed(self, indexes):
types = {type(x) for x in indexes}
return len(types) > 1

def find_standard_indices_between(self, indexes, low, up, datacube, method=None):

indexes_list = list(indexes)

# If homogeneous → use fast path
if not self._is_mixed(indexes_list):

if method in ("surrounding", "nearest"):
start = bisect.bisect_left(indexes_list, low)
end = bisect.bisect_right(indexes_list, up)
start = max(start - 1, 0)
end = min(end + 1, len(indexes))
indexes_between = indexes[start:end]
indexes_between_ranges.extend(indexes_between)
end = min(end + 1, len(indexes_list))
else:
lower_idx = bisect.bisect_left(indexes, low)
upper_idx = bisect.bisect_right(indexes, up)
indexes_between = indexes[lower_idx:upper_idx]
indexes_between_ranges.extend(indexes_between)
return indexes_between_ranges
start = bisect.bisect_left(indexes_list, low)
end = bisect.bisect_right(indexes_list, up)

return indexes_list[start:end]

# Mixed types → fallback (robust)
low_k = self._mixed_key(low)
up_k = self._mixed_key(up)

filtered = [x for x in indexes_list if low_k <= self._mixed_key(x) <= up_k]

if method in ("surrounding", "nearest") and filtered:
# add neighbors manually
first_idx = indexes_list.index(filtered[0])
last_idx = indexes_list.index(filtered[-1])

start = max(first_idx - 1, 0)
end = min(last_idx + 2, len(indexes_list))

return indexes_list[start:end]

return filtered

# def find_standard_indices_between(self, indexes, low, up, datacube, method=None):
# indexes_between_ranges = []

# if self.name in datacube.complete_axes and self.name not in datacube.transformed_axes:
# # Find the range of indexes between lower and upper
# # https://pandas.pydata.org/docs/reference/api/pandas.Index.searchsorted.html
# # Assumes the indexes are already sorted (could sort to be sure) and monotonically increasing
# if method == "surrounding" or method == "nearest":
# start = indexes.searchsorted(low, "left")
# end = indexes.searchsorted(up, "right")
# start = max(start - 1, 0)
# end = min(end + 1, len(indexes))
# indexes_between = indexes[start:end].to_list()
# indexes_between_ranges.extend(indexes_between)
# else:
# start = indexes.searchsorted(low, "left")
# end = indexes.searchsorted(up, "right")
# indexes_between = indexes[start:end].to_list()
# indexes_between_ranges.extend(indexes_between)
# else:
# if method == "surrounding" or method == "nearest":
# start = bisect.bisect_left(indexes, low)
# end = bisect.bisect_right(indexes, up)
# start = max(start - 1, 0)
# end = min(end + 1, len(indexes))
# indexes_between = indexes[start:end]
# indexes_between_ranges.extend(indexes_between)
# else:
# lower_idx = bisect.bisect_left(indexes, low)
# upper_idx = bisect.bisect_right(indexes, up)
# indexes_between = indexes[lower_idx:upper_idx]
# indexes_between_ranges.extend(indexes_between)
# return indexes_between_ranges

def find_indices_between(self, indexes_ranges, low, up, datacube, method=None):
indexes_between_ranges = self.find_standard_indices_between(indexes_ranges, low, up, datacube, method)
Expand Down Expand Up @@ -272,16 +321,34 @@ def __init__(self):
self.type = np.timedelta64(0, "s")
self.can_round = False

# def parse(self, value: Any) -> Any:
# if isinstance(value, np.str_):
# value = str(value)
# return pd.Timedelta(value)

# def to_float(self, value: pd.Timedelta):
# if isinstance(value, np.timedelta64):
# return value.astype("timedelta64[s]").astype(int)
# else:
# return float(value.value / 10**9)

def parse(self, value: Any) -> Any:
if isinstance(value, np.str_):
value = str(value)
if isinstance(value, str) and "-" in value:
return value
return pd.Timedelta(value)

def to_float(self, value: pd.Timedelta):
if isinstance(value, np.timedelta64):
return value.astype("timedelta64[s]").astype(int)
else:
return float(value.value / 10**9)
if isinstance(value, str):
if "-" in value:
return value
return float(value)
else:
return float(value.value / 10**9)

def from_float(self, value):
return pd.Timedelta(int(value), unit="s")
Expand Down
210 changes: 0 additions & 210 deletions polytope_feature/datacube/switching_grid_helper.py

This file was deleted.

Loading
Loading