From a508151a7872e0f1de096212ba38e192cfaec67a Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 25 Feb 2025 11:27:56 +0000 Subject: [PATCH 001/105] Fix examples index --- docs/Service/Examples/examples.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/Service/Examples/examples.md b/docs/Service/Examples/examples.md index 69d427998..2179a4103 100644 --- a/docs/Service/Examples/examples.md +++ b/docs/Service/Examples/examples.md @@ -12,11 +12,11 @@ ## Open Data * Timeseries -* Timeseries -* Timeseries -* Timeseries -* Timeseries -* Timeseries +* Polygon +* Vertical Profile +* Bounding Box +* Trajectory +* Country Cut-Out For examples of Polytope Feature Extraction on Destination Earth Digital Twin Data please visit the following Github Repo: https://github.com/destination-earth-digital-twins/polytope-examples From 95c35a4a29fe016bee56d6a00d65f6b96cee9dc7 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 25 Feb 2025 11:50:06 +0000 Subject: [PATCH 002/105] Fix link --- .../{od_boundingbox.ipynb => od_boundingbox_example.ipynb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/Service/Examples/OpenData/{od_boundingbox.ipynb => od_boundingbox_example.ipynb} (100%) diff --git a/docs/Service/Examples/OpenData/od_boundingbox.ipynb b/docs/Service/Examples/OpenData/od_boundingbox_example.ipynb similarity index 100% rename from docs/Service/Examples/OpenData/od_boundingbox.ipynb rename to docs/Service/Examples/OpenData/od_boundingbox_example.ipynb From f47211e182af52cc2c963b8ef7b44b896f46c605 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 26 Mar 2025 15:02:16 +0100 Subject: [PATCH 003/105] do not import in init for mapper types --- .../datacube_mappers/mapper_types/__init__.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/__init__.py b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/__init__.py index ba9a7b339..e69de29bb 100644 --- a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/__init__.py +++ b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/__init__.py @@ -1,5 +0,0 @@ -from .healpix import * -from .local_regular import * -from .octahedral import * -from .reduced_ll import * -from .regular import * From 07d1c0615e30c057ea724df9c5de5adc581d661f Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 27 Mar 2025 11:57:52 +0100 Subject: [PATCH 004/105] remove caching of healpix nested grid --- .../mapper_types/healpix_nested.py | 26 +++++++------------ 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py index da3f0f1ae..c90406ec1 100644 --- a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py +++ b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py @@ -13,11 +13,9 @@ def __init__(self, base_axis, mapped_axes, resolution, md5_hash=None, local_area self._first_axis_vals = self.first_axis_vals() self.compressed_grid_axes = [self._mapped_axes[1]] self.Nside = self._resolution - self._cached_longitudes = {} self.k = int(math.log2(self.Nside)) self.Npix = 12 * self.Nside * self.Nside self.Ncap = (self.Nside * (self.Nside - 1)) << 1 - self._healpix_longitudes = {} if md5_hash is not None: self.md5_hash = md5_hash else: @@ -59,9 +57,7 @@ def second_axis_vals(self, first_val): return values def second_axis_vals_from_idx(self, first_val_idx): - if first_val_idx not in self._healpix_longitudes: - self._healpix_longitudes[first_val_idx] = self.HEALPix_longitudes(first_val_idx) - values = self._healpix_longitudes[first_val_idx] + values = self.HEALPix_longitudes(first_val_idx) return values def HEALPix_nj(self, i): @@ -77,19 +73,15 @@ def HEALPix_nj(self, i): return self.HEALPix_nj(ni - 1 - i) def HEALPix_longitudes(self, i): - if i in self._cached_longitudes: - return self._cached_longitudes[i] - else: - Nj = self.HEALPix_nj(i) - step = 360.0 / Nj - start = ( - step / 2.0 - if i < self._resolution or 3 * self._resolution - 1 < i or (i + self._resolution) % 2 - else 0.0 - ) + Nj = self.HEALPix_nj(i) + step = 360.0 / Nj + start = ( + step / 2.0 + if i < self._resolution or 3 * self._resolution - 1 < i or (i + self._resolution) % 2 + else 0.0 + ) - longitudes = [start + n * step for n in range(Nj)] - self._cached_longitudes[i] = longitudes + longitudes = [start + n * step for n in range(Nj)] return longitudes def map_second_axis(self, first_val, lower, upper): From c3b22b71e800ea6cdb4d5c81ae067b68bb251b01 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 27 Mar 2025 12:11:50 +0100 Subject: [PATCH 005/105] update version and black --- .../datacube_mappers/mapper_types/healpix_nested.py | 4 +--- polytope_feature/version.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py index c90406ec1..5b5405f60 100644 --- a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py +++ b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py @@ -76,9 +76,7 @@ def HEALPix_longitudes(self, i): Nj = self.HEALPix_nj(i) step = 360.0 / Nj start = ( - step / 2.0 - if i < self._resolution or 3 * self._resolution - 1 < i or (i + self._resolution) % 2 - else 0.0 + step / 2.0 if i < self._resolution or 3 * self._resolution - 1 < i or (i + self._resolution) % 2 else 0.0 ) longitudes = [start + n * step for n in range(Nj)] diff --git a/polytope_feature/version.py b/polytope_feature/version.py index d0ab53db9..0d0374a07 100644 --- a/polytope_feature/version.py +++ b/polytope_feature/version.py @@ -1 +1 @@ -__version__ = "1.0.39" +__version__ = "1.0.40" From fca670ba40119295ddc41cdf5838598429aa6e4c Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 27 Mar 2025 14:04:50 +0100 Subject: [PATCH 006/105] add maturity badge --- readme.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 05eed3c21..2aaaf1fdf 100644 --- a/readme.md +++ b/readme.md @@ -13,6 +13,9 @@ Documentation Status + + Project Maturity +

Concept • @@ -30,8 +33,11 @@ Polytope is designed to extend different datacube backends: Polytope supports datacubes which have branching, non-uniform indexing, and even cyclic axes. If the datacube backend supports byte-addressability and efficient random access (either in-memory or direct from storage), **Polytope** can be used to dramatically decrease overall I/O load. -> [!WARNING] -> This project is BETA and will be experimental for the foreseeable future. Interfaces and functionality are likely to change, and the project itself may be scrapped. DO NOT use this software in any project/software that is operational. +> \[!IMPORTANT\] +> This software is **Incubating** and subject to ECMWF's guidelines on [Software Maturity](https://github.com/ecmwf/codex/raw/refs/heads/main/Project%20Maturity). + + ## Concept From 6d720f7fa9b95069bdec3f0308efea3abbd9b7d8 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 27 Mar 2025 16:35:59 +0100 Subject: [PATCH 007/105] remove some expensive computations --- .../mapper_types/healpix_nested.py | 64 +++++++++++++------ 1 file changed, 45 insertions(+), 19 deletions(-) diff --git a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py index 5b5405f60..ac8215ffb 100644 --- a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py +++ b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py @@ -87,26 +87,52 @@ def map_second_axis(self, first_val, lower, upper): return_vals = [val for val in axis_lines if lower <= val <= upper] return return_vals + # def axes_idx_to_healpix_idx(self, first_idx, second_idx): + # idx = 0 + # for i in range(self._resolution - 1): + # if i != first_idx: + # idx += 4 * (i + 1) + # else: + # idx += second_idx + # return idx + # for i in range(self._resolution - 1, 3 * self._resolution): + # if i != first_idx: + # idx += 4 * self._resolution + # else: + # idx += second_idx + # return idx + # for i in range(3 * self._resolution, 4 * self._resolution - 1): + # if i != first_idx: + # idx += 4 * (4 * self._resolution - 1 - i) + # else: + # idx += second_idx + # return idx + def axes_idx_to_healpix_idx(self, first_idx, second_idx): - idx = 0 - for i in range(self._resolution - 1): - if i != first_idx: - idx += 4 * (i + 1) - else: - idx += second_idx - return idx - for i in range(self._resolution - 1, 3 * self._resolution): - if i != first_idx: - idx += 4 * self._resolution - else: - idx += second_idx - return idx - for i in range(3 * self._resolution, 4 * self._resolution - 1): - if i != first_idx: - idx += 4 * (4 * self._resolution - 1 - i) - else: - idx += second_idx - return idx + res = self._resolution + # Directly compute index without unnecessary loops + # if first_idx < res - 1: + # return sum(4 * (i + 1) for i in range(first_idx)) + second_idx + # elif first_idx < 3 * res: + # return sum(4 * (i + 1) for i in range(res - 1)) + (first_idx - (res - 1)) * (4 * res) + second_idx + # else: + # return ( + # sum(4 * (i + 1) for i in range(res - 1)) + # + (2 * res + 1) * (4 * res) + # + sum(4 * (4 * res - 1 - i) for i in range(3 * res, first_idx)) + # + second_idx + # ) + # sum1 = (4 * np.arange(res - 1) + 4).sum() + sum1 = (2 * (res-1) * res) + # sum2 = (4 * np.arange(3 * res, first_idx) - 4 * res + 1).sum() if first_idx >= 3 * res else 0 + sum2 = 2 * (((res-1) * res) - ((4 * res - 1 - first_idx) * (4 * res - first_idx))) + + if first_idx < res - 1: + return (2 * first_idx * (first_idx + 1)) + second_idx + elif first_idx < 3 * res: + return sum1 + (first_idx - (res - 1)) * (4 * res) + second_idx + else: + return sum1 + (2 * res + 1) * (4 * res) + sum2 + second_idx def unmap(self, first_val, second_vals): tol = 1e-8 From 9db9c0d9115cc686cd2538c10b4e09a84de36338 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 27 Mar 2025 16:57:56 +0100 Subject: [PATCH 008/105] make ring_to_nested a numpy operation --- .../mapper_types/healpix_nested.py | 294 +++++++++++++----- 1 file changed, 217 insertions(+), 77 deletions(-) diff --git a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py index ac8215ffb..5b294991b 100644 --- a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py +++ b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py @@ -1,9 +1,31 @@ import math +import numpy as np from ..datacube_mappers import DatacubeMapper class NestedHealpixGridMapper(DatacubeMapper): + # def __init__(self, base_axis, mapped_axes, resolution, md5_hash=None, local_area=[], axis_reversed=None): + # # TODO: if local area is not empty list, raise NotImplemented + # self._mapped_axes = mapped_axes + # self._base_axis = base_axis + # self._resolution = resolution + # self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} + # self._first_axis_vals = self.first_axis_vals() + # self.compressed_grid_axes = [self._mapped_axes[1]] + # self.Nside = self._resolution + # self.k = int(math.log2(self.Nside)) + # self.Npix = 12 * self.Nside * self.Nside + # self.Ncap = (self.Nside * (self.Nside - 1)) << 1 + # if md5_hash is not None: + # self.md5_hash = md5_hash + # else: + # self.md5_hash = _md5_hash.get(resolution, None) + # if self._axis_reversed[mapped_axes[1]]: + # raise NotImplementedError("Healpix grid with second axis in decreasing order is not supported") + # if not self._axis_reversed[mapped_axes[0]]: + # raise NotImplementedError("Healpix grid with first axis in increasing order is not supported") + def __init__(self, base_axis, mapped_axes, resolution, md5_hash=None, local_area=[], axis_reversed=None): # TODO: if local area is not empty list, raise NotImplemented self._mapped_axes = mapped_axes @@ -11,6 +33,7 @@ def __init__(self, base_axis, mapped_axes, resolution, md5_hash=None, local_area self._resolution = resolution self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} self._first_axis_vals = self.first_axis_vals() + self._first_axis_vals_np_rounded = -np.round(np.array(self._first_axis_vals), decimals=8) self.compressed_grid_axes = [self._mapped_axes[1]] self.Nside = self._resolution self.k = int(math.log2(self.Nside)) @@ -75,11 +98,17 @@ def HEALPix_nj(self, i): def HEALPix_longitudes(self, i): Nj = self.HEALPix_nj(i) step = 360.0 / Nj - start = ( - step / 2.0 if i < self._resolution or 3 * self._resolution - 1 < i or (i + self._resolution) % 2 else 0.0 + # start = ( + # step / 2.0 if i < self._resolution or 3 * self._resolution - 1 < i or (i + self._resolution) % 2 else 0.0 + # ) + + # longitudes = [start + n * step for n in range(Nj)] + start = np.where( + (i < self._resolution) | (3 * self._resolution - 1 < i) | ((i + self._resolution) % 2 == 1), + step / 2.0, + 0.0 ) - - longitudes = [start + n * step for n in range(Nj)] + longitudes = start + np.arange(Nj) * step return longitudes def map_second_axis(self, first_val, lower, upper): @@ -134,98 +163,209 @@ def axes_idx_to_healpix_idx(self, first_idx, second_idx): else: return sum1 + (2 * res + 1) * (4 * res) + sum2 + second_idx + # def unmap(self, first_val, second_vals): + # tol = 1e-8 + # first_idx = next( + # (i for i, val in enumerate(self._first_axis_vals) if first_val[0] - tol <= val <= first_val[0] + tol), None + # ) + # if first_idx is None: + # return None + # second_axis_vals = self.second_axis_vals_from_idx(first_idx) + + # return_idxs = [] + # for second_val in second_vals: + # second_idx = next( + # (i for i, val in enumerate(second_axis_vals) if second_val - tol <= val <= second_val + tol), None + # ) + # if second_idx is None: + # return None + # healpix_index = self.axes_idx_to_healpix_idx(first_idx, second_idx) + # nested_healpix_index = self.ring_to_nested(healpix_index) + # return_idxs.append(nested_healpix_index) + # return return_idxs + + # def div_03(self, a, b): + # t = 1 if a >= (b << 1) else 0 + # a -= t * (b << 1) + # return (t << 1) + (1 if a >= b else 0) + + # def pll(self, f): + # pll_values = [1, 3, 5, 7, 0, 2, 4, 6, 1, 3, 5, 7] + # return pll_values[f] + + # def to_nest(self, f, ring, Nring, phi, shift): + # r = int(((2 + (f >> 2)) << self.k) - ring - 1) + # p = int(2 * phi - self.pll(f) * Nring - shift - 1) + # if p >= 2 * self.Nside: + # p -= 8 * self.Nside + # i = int((r + p)) >> 1 + # j = int((r - p)) >> 1 + + # return self.fij_to_nest(f, i, j, self.k) + + # def fij_to_nest(self, f, i, j, k): + # return (f << (2 * k)) + self.nest_encode_bits(i) + (self.nest_encode_bits(j) << 1) + + # def nest_encode_bits(self, i): + # __masks = [ + # 0x00000000FFFFFFFF, + # 0x0000FFFF0000FFFF, + # 0x00FF00FF00FF00FF, + # 0x0F0F0F0F0F0F0F0F, + # 0x3333333333333333, + # 0x5555555555555555, + # ] + # i = int(i) + # b = i & __masks[0] + # b = (b ^ (b << 16)) & __masks[1] + # b = (b ^ (b << 8)) & __masks[2] + # b = (b ^ (b << 4)) & __masks[3] + # b = (b ^ (b << 2)) & __masks[4] + # b = (b ^ (b << 1)) & __masks[5] + # return b + + # def ring_to_nested(self, idx): + # if idx < self.Ncap: + # # North polar cap + # Nring = (1 + self.int_sqrt(2 * idx + 1)) >> 1 + # phi = 1 + idx - 2 * Nring * (Nring - 1) + # f = self.div_03(phi - 1, Nring) + # return self.to_nest(f, Nring, Nring, phi, 0) + + # if self.Npix - self.Ncap <= idx: + # # South polar cap + # Nring = (1 + self.int_sqrt(2 * self.Npix - 2 * idx - 1)) >> 1 + # phi = 1 + idx + 2 * Nring * (Nring - 1) + 4 * Nring - self.Npix + # ring = 4 * self.Nside - Nring # (from South pole) + # f = self.div_03(phi - 1, Nring) + 8 + # return self.to_nest(f, ring, Nring, phi, 0) + # else: + # # Equatorial belt + # ip = idx - self.Ncap + # tmp = ip >> (self.k + 2) + + # phi = ip - tmp * 4 * self.Nside + 1 + # ring = tmp + self.Nside + + # ifm = 1 + ((phi - 1 - ((1 + tmp) >> 1)) >> self.k) + # ifp = 1 + ((phi - 1 - ((1 - tmp + 2 * self.Nside) >> 1)) >> self.k) + # f = (ifp | 4) if ifp == ifm else (ifp if ifp < ifm else (ifm + 8)) + + # return self.to_nest(f, ring, self.Nside, phi, ring & 1) + + # def int_sqrt(self, i): + # return int(math.sqrt(i + 0.5)) + def unmap(self, first_val, second_vals): - tol = 1e-8 - first_idx = next( - (i for i, val in enumerate(self._first_axis_vals) if first_val[0] - tol <= val <= first_val[0] + tol), None - ) - if first_idx is None: + # Convert to NumPy array for fast computation + idx = np.searchsorted(self._first_axis_vals_np_rounded, -np.round(first_val[0], decimals=8)) + if idx >= len(self._first_axis_vals_np_rounded): + return None + second_axis_vals = np.round(np.array(self.second_axis_vals_from_idx(idx)), decimals=8) + second_vals = np.round(np.array(second_vals), decimals=8) + second_idxs = np.searchsorted(second_axis_vals, second_vals) + valid_mask = second_idxs < len(second_axis_vals) + if not np.all(valid_mask): return None - second_axis_vals = self.second_axis_vals_from_idx(first_idx) - - return_idxs = [] - for second_val in second_vals: - second_idx = next( - (i for i, val in enumerate(second_axis_vals) if second_val - tol <= val <= second_val + tol), None - ) - if second_idx is None: - return None - healpix_index = self.axes_idx_to_healpix_idx(first_idx, second_idx) - nested_healpix_index = self.ring_to_nested(healpix_index) - return_idxs.append(nested_healpix_index) - return return_idxs + healpix_idxs = [self.axes_idx_to_healpix_idx(idx, sec_idx) for sec_idx in second_idxs] + # return [self.ring_to_nested(self.axes_idx_to_healpix_idx(idx, sec_idx)) for sec_idx in second_idxs] + # return [self.ring_to_nested(healpix_idx) for healpix_idx in healpix_idxs] + return self.ring_to_nested(np.asarray(healpix_idxs)) def div_03(self, a, b): - t = 1 if a >= (b << 1) else 0 + """Vectorized version of div_03""" + t = np.where(a >= (b << 1), 1, 0) a -= t * (b << 1) - return (t << 1) + (1 if a >= b else 0) + return (t << 1) + np.where(a >= b, 1, 0) def pll(self, f): - pll_values = [1, 3, 5, 7, 0, 2, 4, 6, 1, 3, 5, 7] + """Vectorized lookup for PLL values""" + pll_values = np.array([1, 3, 5, 7, 0, 2, 4, 6, 1, 3, 5, 7]) return pll_values[f] def to_nest(self, f, ring, Nring, phi, shift): - r = int(((2 + (f >> 2)) << self.k) - ring - 1) - p = int(2 * phi - self.pll(f) * Nring - shift - 1) - if p >= 2 * self.Nside: - p -= 8 * self.Nside - i = int((r + p)) >> 1 - j = int((r - p)) >> 1 + """Vectorized to_nest conversion""" + r = ((2 + (f >> 2)) << self.k) - ring - 1 + p = 2 * phi - self.pll(f) * Nring - shift - 1 + p = np.where(p >= 2 * self.Nside, p - 8 * self.Nside, p) + i = (r + p) >> 1 + j = (r - p) >> 1 return self.fij_to_nest(f, i, j, self.k) def fij_to_nest(self, f, i, j, k): - return (f << (2 * k)) + self.nest_encode_bits(i) + (self.nest_encode_bits(j) << 1) + """Vectorized nest encoding""" + return ( + # (f.astype(np.uint64) << np.uint64(2 * k)) + (f.astype(object) << (2*k)) + + self.nest_encode_bits(i) + + (self.nest_encode_bits(j).astype(np.uint64) << np.uint64(1)) + ) def nest_encode_bits(self, i): - __masks = [ - 0x00000000FFFFFFFF, - 0x0000FFFF0000FFFF, - 0x00FF00FF00FF00FF, - 0x0F0F0F0F0F0F0F0F, - 0x3333333333333333, - 0x5555555555555555, - ] - i = int(i) - b = i & __masks[0] - b = (b ^ (b << 16)) & __masks[1] - b = (b ^ (b << 8)) & __masks[2] - b = (b ^ (b << 4)) & __masks[3] - b = (b ^ (b << 2)) & __masks[4] - b = (b ^ (b << 1)) & __masks[5] - return b - - def ring_to_nested(self, idx): - if idx < self.Ncap: - # North polar cap - Nring = (1 + self.int_sqrt(2 * idx + 1)) >> 1 - phi = 1 + idx - 2 * Nring * (Nring - 1) - f = self.div_03(phi - 1, Nring) - return self.to_nest(f, Nring, Nring, phi, 0) - - if self.Npix - self.Ncap <= idx: - # South polar cap - Nring = (1 + self.int_sqrt(2 * self.Npix - 2 * idx - 1)) >> 1 - phi = 1 + idx + 2 * Nring * (Nring - 1) + 4 * Nring - self.Npix - ring = 4 * self.Nside - Nring # (from South pole) - f = self.div_03(phi - 1, Nring) + 8 - return self.to_nest(f, ring, Nring, phi, 0) - else: - # Equatorial belt - ip = idx - self.Ncap - tmp = ip >> (self.k + 2) - - phi = ip - tmp * 4 * self.Nside + 1 - ring = tmp + self.Nside + """Vectorized bit manipulation for HEALPix indexing""" + __masks = np.array( + [ + 0x00000000FFFFFFFF, + 0x0000FFFF0000FFFF, + 0x00FF00FF00FF00FF, + 0x0F0F0F0F0F0F0F0F, + 0x3333333333333333, + 0x5555555555555555, + ], + dtype=np.uint64, + ) - ifm = 1 + ((phi - 1 - ((1 + tmp) >> 1)) >> self.k) - ifp = 1 + ((phi - 1 - ((1 - tmp + 2 * self.Nside) >> 1)) >> self.k) - f = (ifp | 4) if ifp == ifm else (ifp if ifp < ifm else (ifm + 8)) + b = i.astype(np.uint64) & __masks[0] + b = (b ^ (b << np.uint64(16))) & __masks[1] + b = (b ^ (b << np.uint64(8))) & __masks[2] + b = (b ^ (b << np.uint64(4))) & __masks[3] + b = (b ^ (b << np.uint64(2))) & __masks[4] + b = (b ^ (b << np.uint64(1))) & __masks[5] + return b - return self.to_nest(f, ring, self.Nside, phi, ring & 1) + def int_sqrt(self, x): + """Efficient integer square root for arrays""" + return np.sqrt(x + 0.5).astype(int) - def int_sqrt(self, i): - return int(math.sqrt(i + 0.5)) + def ring_to_nested(self, idx): + """Vectorized ring_to_nested conversion""" + # idx = np.asarray(idx) # Ensure input is an array + + north_mask = idx < self.Ncap + south_mask = self.Npix - self.Ncap <= idx + + # North polar cap + Nring_north = (1 + self.int_sqrt(2 * idx + 1)) >> 1 + phi_north = 1 + idx - 2 * Nring_north * (Nring_north - 1) + f_north = self.div_03(phi_north - 1, Nring_north) + nested_north = self.to_nest(f_north, Nring_north, Nring_north, phi_north, 0) + + # South polar cap + Nring_south = (1 + self.int_sqrt(2 * self.Npix - 2 * idx - 1)) >> 1 + phi_south = 1 + idx + 2 * Nring_south * (Nring_south - 1) + 4 * Nring_south - self.Npix + ring_south = 4 * self.Nside - Nring_south + f_south = self.div_03(phi_south - 1, Nring_south) + 8 + nested_south = self.to_nest(f_south, ring_south, Nring_south, phi_south, 0) + + # Equatorial belt + ip = idx - self.Ncap + tmp = ip >> (self.k + 2) + + phi_equatorial = ip - tmp * 4 * self.Nside + 1 + ring_equatorial = tmp + self.Nside + + ifm = 1 + ((phi_equatorial - 1 - ((1 + tmp) >> 1)) >> self.k) + ifp = 1 + ((phi_equatorial - 1 - ((1 - tmp + 2 * self.Nside) >> 1)) >> self.k) + f_equatorial = np.where(ifp == ifm, ifp | 4, np.where(ifp < ifm, ifp, ifm + 8)) + + nested_equatorial = self.to_nest(f_equatorial, ring_equatorial, self.Nside, phi_equatorial, ring_equatorial & 1) + # nested_result = np.where(north_mask, nested_north, np.where(south_mask, nested_south, nested_equatorial)) + nested_result = np.empty_like(idx) # Preallocate array for performance + nested_result[north_mask] = nested_north[north_mask] + nested_result[south_mask] = nested_south[south_mask] + nested_result[~(north_mask | south_mask)] = nested_equatorial[~(north_mask | south_mask)] + return nested_result # md5 grid hash in form {resolution : hash} From 764366b075445679101f1c69afee7f35c6ede1bb Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 27 Mar 2025 17:03:12 +0100 Subject: [PATCH 009/105] clean up --- .../mapper_types/healpix_nested.py | 168 +----------------- 1 file changed, 5 insertions(+), 163 deletions(-) diff --git a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py index 5b294991b..226c8c359 100644 --- a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py +++ b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py @@ -1,31 +1,11 @@ import math + import numpy as np from ..datacube_mappers import DatacubeMapper class NestedHealpixGridMapper(DatacubeMapper): - # def __init__(self, base_axis, mapped_axes, resolution, md5_hash=None, local_area=[], axis_reversed=None): - # # TODO: if local area is not empty list, raise NotImplemented - # self._mapped_axes = mapped_axes - # self._base_axis = base_axis - # self._resolution = resolution - # self._axis_reversed = {mapped_axes[0]: True, mapped_axes[1]: False} - # self._first_axis_vals = self.first_axis_vals() - # self.compressed_grid_axes = [self._mapped_axes[1]] - # self.Nside = self._resolution - # self.k = int(math.log2(self.Nside)) - # self.Npix = 12 * self.Nside * self.Nside - # self.Ncap = (self.Nside * (self.Nside - 1)) << 1 - # if md5_hash is not None: - # self.md5_hash = md5_hash - # else: - # self.md5_hash = _md5_hash.get(resolution, None) - # if self._axis_reversed[mapped_axes[1]]: - # raise NotImplementedError("Healpix grid with second axis in decreasing order is not supported") - # if not self._axis_reversed[mapped_axes[0]]: - # raise NotImplementedError("Healpix grid with first axis in increasing order is not supported") - def __init__(self, base_axis, mapped_axes, resolution, md5_hash=None, local_area=[], axis_reversed=None): # TODO: if local area is not empty list, raise NotImplemented self._mapped_axes = mapped_axes @@ -98,15 +78,8 @@ def HEALPix_nj(self, i): def HEALPix_longitudes(self, i): Nj = self.HEALPix_nj(i) step = 360.0 / Nj - # start = ( - # step / 2.0 if i < self._resolution or 3 * self._resolution - 1 < i or (i + self._resolution) % 2 else 0.0 - # ) - - # longitudes = [start + n * step for n in range(Nj)] start = np.where( - (i < self._resolution) | (3 * self._resolution - 1 < i) | ((i + self._resolution) % 2 == 1), - step / 2.0, - 0.0 + (i < self._resolution) | (3 * self._resolution - 1 < i) | ((i + self._resolution) % 2 == 1), step / 2.0, 0.0 ) longitudes = start + np.arange(Nj) * step return longitudes @@ -116,45 +89,10 @@ def map_second_axis(self, first_val, lower, upper): return_vals = [val for val in axis_lines if lower <= val <= upper] return return_vals - # def axes_idx_to_healpix_idx(self, first_idx, second_idx): - # idx = 0 - # for i in range(self._resolution - 1): - # if i != first_idx: - # idx += 4 * (i + 1) - # else: - # idx += second_idx - # return idx - # for i in range(self._resolution - 1, 3 * self._resolution): - # if i != first_idx: - # idx += 4 * self._resolution - # else: - # idx += second_idx - # return idx - # for i in range(3 * self._resolution, 4 * self._resolution - 1): - # if i != first_idx: - # idx += 4 * (4 * self._resolution - 1 - i) - # else: - # idx += second_idx - # return idx - def axes_idx_to_healpix_idx(self, first_idx, second_idx): res = self._resolution - # Directly compute index without unnecessary loops - # if first_idx < res - 1: - # return sum(4 * (i + 1) for i in range(first_idx)) + second_idx - # elif first_idx < 3 * res: - # return sum(4 * (i + 1) for i in range(res - 1)) + (first_idx - (res - 1)) * (4 * res) + second_idx - # else: - # return ( - # sum(4 * (i + 1) for i in range(res - 1)) - # + (2 * res + 1) * (4 * res) - # + sum(4 * (4 * res - 1 - i) for i in range(3 * res, first_idx)) - # + second_idx - # ) - # sum1 = (4 * np.arange(res - 1) + 4).sum() - sum1 = (2 * (res-1) * res) - # sum2 = (4 * np.arange(3 * res, first_idx) - 4 * res + 1).sum() if first_idx >= 3 * res else 0 - sum2 = 2 * (((res-1) * res) - ((4 * res - 1 - first_idx) * (4 * res - first_idx))) + sum1 = 2 * (res - 1) * res + sum2 = 2 * (((res - 1) * res) - ((4 * res - 1 - first_idx) * (4 * res - first_idx))) if first_idx < res - 1: return (2 * first_idx * (first_idx + 1)) + second_idx @@ -163,99 +101,6 @@ def axes_idx_to_healpix_idx(self, first_idx, second_idx): else: return sum1 + (2 * res + 1) * (4 * res) + sum2 + second_idx - # def unmap(self, first_val, second_vals): - # tol = 1e-8 - # first_idx = next( - # (i for i, val in enumerate(self._first_axis_vals) if first_val[0] - tol <= val <= first_val[0] + tol), None - # ) - # if first_idx is None: - # return None - # second_axis_vals = self.second_axis_vals_from_idx(first_idx) - - # return_idxs = [] - # for second_val in second_vals: - # second_idx = next( - # (i for i, val in enumerate(second_axis_vals) if second_val - tol <= val <= second_val + tol), None - # ) - # if second_idx is None: - # return None - # healpix_index = self.axes_idx_to_healpix_idx(first_idx, second_idx) - # nested_healpix_index = self.ring_to_nested(healpix_index) - # return_idxs.append(nested_healpix_index) - # return return_idxs - - # def div_03(self, a, b): - # t = 1 if a >= (b << 1) else 0 - # a -= t * (b << 1) - # return (t << 1) + (1 if a >= b else 0) - - # def pll(self, f): - # pll_values = [1, 3, 5, 7, 0, 2, 4, 6, 1, 3, 5, 7] - # return pll_values[f] - - # def to_nest(self, f, ring, Nring, phi, shift): - # r = int(((2 + (f >> 2)) << self.k) - ring - 1) - # p = int(2 * phi - self.pll(f) * Nring - shift - 1) - # if p >= 2 * self.Nside: - # p -= 8 * self.Nside - # i = int((r + p)) >> 1 - # j = int((r - p)) >> 1 - - # return self.fij_to_nest(f, i, j, self.k) - - # def fij_to_nest(self, f, i, j, k): - # return (f << (2 * k)) + self.nest_encode_bits(i) + (self.nest_encode_bits(j) << 1) - - # def nest_encode_bits(self, i): - # __masks = [ - # 0x00000000FFFFFFFF, - # 0x0000FFFF0000FFFF, - # 0x00FF00FF00FF00FF, - # 0x0F0F0F0F0F0F0F0F, - # 0x3333333333333333, - # 0x5555555555555555, - # ] - # i = int(i) - # b = i & __masks[0] - # b = (b ^ (b << 16)) & __masks[1] - # b = (b ^ (b << 8)) & __masks[2] - # b = (b ^ (b << 4)) & __masks[3] - # b = (b ^ (b << 2)) & __masks[4] - # b = (b ^ (b << 1)) & __masks[5] - # return b - - # def ring_to_nested(self, idx): - # if idx < self.Ncap: - # # North polar cap - # Nring = (1 + self.int_sqrt(2 * idx + 1)) >> 1 - # phi = 1 + idx - 2 * Nring * (Nring - 1) - # f = self.div_03(phi - 1, Nring) - # return self.to_nest(f, Nring, Nring, phi, 0) - - # if self.Npix - self.Ncap <= idx: - # # South polar cap - # Nring = (1 + self.int_sqrt(2 * self.Npix - 2 * idx - 1)) >> 1 - # phi = 1 + idx + 2 * Nring * (Nring - 1) + 4 * Nring - self.Npix - # ring = 4 * self.Nside - Nring # (from South pole) - # f = self.div_03(phi - 1, Nring) + 8 - # return self.to_nest(f, ring, Nring, phi, 0) - # else: - # # Equatorial belt - # ip = idx - self.Ncap - # tmp = ip >> (self.k + 2) - - # phi = ip - tmp * 4 * self.Nside + 1 - # ring = tmp + self.Nside - - # ifm = 1 + ((phi - 1 - ((1 + tmp) >> 1)) >> self.k) - # ifp = 1 + ((phi - 1 - ((1 - tmp + 2 * self.Nside) >> 1)) >> self.k) - # f = (ifp | 4) if ifp == ifm else (ifp if ifp < ifm else (ifm + 8)) - - # return self.to_nest(f, ring, self.Nside, phi, ring & 1) - - # def int_sqrt(self, i): - # return int(math.sqrt(i + 0.5)) - def unmap(self, first_val, second_vals): # Convert to NumPy array for fast computation idx = np.searchsorted(self._first_axis_vals_np_rounded, -np.round(first_val[0], decimals=8)) @@ -268,8 +113,6 @@ def unmap(self, first_val, second_vals): if not np.all(valid_mask): return None healpix_idxs = [self.axes_idx_to_healpix_idx(idx, sec_idx) for sec_idx in second_idxs] - # return [self.ring_to_nested(self.axes_idx_to_healpix_idx(idx, sec_idx)) for sec_idx in second_idxs] - # return [self.ring_to_nested(healpix_idx) for healpix_idx in healpix_idxs] return self.ring_to_nested(np.asarray(healpix_idxs)) def div_03(self, a, b): @@ -297,7 +140,7 @@ def fij_to_nest(self, f, i, j, k): """Vectorized nest encoding""" return ( # (f.astype(np.uint64) << np.uint64(2 * k)) - (f.astype(object) << (2*k)) + (f.astype(object) << (2 * k)) + self.nest_encode_bits(i) + (self.nest_encode_bits(j).astype(np.uint64) << np.uint64(1)) ) @@ -360,7 +203,6 @@ def ring_to_nested(self, idx): f_equatorial = np.where(ifp == ifm, ifp | 4, np.where(ifp < ifm, ifp, ifm + 8)) nested_equatorial = self.to_nest(f_equatorial, ring_equatorial, self.Nside, phi_equatorial, ring_equatorial & 1) - # nested_result = np.where(north_mask, nested_north, np.where(south_mask, nested_south, nested_equatorial)) nested_result = np.empty_like(idx) # Preallocate array for performance nested_result[north_mask] = nested_north[north_mask] nested_result[south_mask] = nested_south[south_mask] From d8c8c21607a0fa33e903ec8a671a2b9e57fb8d47 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 27 Mar 2025 17:04:41 +0100 Subject: [PATCH 010/105] make work on fdb --- polytope_feature/datacube/backends/fdb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polytope_feature/datacube/backends/fdb.py b/polytope_feature/datacube/backends/fdb.py index 64304e379..645bdcd46 100644 --- a/polytope_feature/datacube/backends/fdb.py +++ b/polytope_feature/datacube/backends/fdb.py @@ -205,11 +205,11 @@ def remove_duplicates_in_request_ranges(self, fdb_node_ranges, current_start_idx original_fdb_node_range_vals = [] new_current_start_idx = [] for j, idx in enumerate(sub_lat_idxs): - if idx not in seen_indices: + if idx.tolist() not in seen_indices: # NOTE: need to remove it from the values in the corresponding tree node # NOTE: need to read just the range we give to gj original_fdb_node_range_vals.append(actual_fdb_node[0].values[j]) - seen_indices.add(idx) + seen_indices.add(idx.tolist()) new_current_start_idx.append(idx) if original_fdb_node_range_vals != []: actual_fdb_node[0].values = tuple(original_fdb_node_range_vals) From e0510fbee2b65fa66c1feb626da7409e995e1351 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 27 Mar 2025 17:08:16 +0100 Subject: [PATCH 011/105] upgrade version --- polytope_feature/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope_feature/version.py b/polytope_feature/version.py index 0d0374a07..db0784c9a 100644 --- a/polytope_feature/version.py +++ b/polytope_feature/version.py @@ -1 +1 @@ -__version__ = "1.0.40" +__version__ = "1.0.41" From 30632b4ee1b5ac9aa69017cde8fcd19ab1b2541c Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 1 Apr 2025 09:44:48 +0200 Subject: [PATCH 012/105] fix bug --- polytope_feature/datacube/backends/fdb.py | 4 ++-- .../datacube_mappers/mapper_types/healpix_nested.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/polytope_feature/datacube/backends/fdb.py b/polytope_feature/datacube/backends/fdb.py index 645bdcd46..64304e379 100644 --- a/polytope_feature/datacube/backends/fdb.py +++ b/polytope_feature/datacube/backends/fdb.py @@ -205,11 +205,11 @@ def remove_duplicates_in_request_ranges(self, fdb_node_ranges, current_start_idx original_fdb_node_range_vals = [] new_current_start_idx = [] for j, idx in enumerate(sub_lat_idxs): - if idx.tolist() not in seen_indices: + if idx not in seen_indices: # NOTE: need to remove it from the values in the corresponding tree node # NOTE: need to read just the range we give to gj original_fdb_node_range_vals.append(actual_fdb_node[0].values[j]) - seen_indices.add(idx.tolist()) + seen_indices.add(idx) new_current_start_idx.append(idx) if original_fdb_node_range_vals != []: actual_fdb_node[0].values = tuple(original_fdb_node_range_vals) diff --git a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py index 226c8c359..4ea3e17bd 100644 --- a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py +++ b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/healpix_nested.py @@ -113,7 +113,7 @@ def unmap(self, first_val, second_vals): if not np.all(valid_mask): return None healpix_idxs = [self.axes_idx_to_healpix_idx(idx, sec_idx) for sec_idx in second_idxs] - return self.ring_to_nested(np.asarray(healpix_idxs)) + return self.ring_to_nested(np.asarray(healpix_idxs)).tolist() def div_03(self, a, b): """Vectorized version of div_03""" From 55830fb29949c91b324b508046c41d94a9f35d77 Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 1 Apr 2025 09:54:36 +0200 Subject: [PATCH 013/105] update version --- polytope_feature/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope_feature/version.py b/polytope_feature/version.py index db0784c9a..0d6c7d97e 100644 --- a/polytope_feature/version.py +++ b/polytope_feature/version.py @@ -1 +1 @@ -__version__ = "1.0.41" +__version__ = "1.0.42" From 772fe820bd8507a8b1641eb22161f2d713717bd0 Mon Sep 17 00:00:00 2001 From: Chris Bradley Date: Tue, 8 Apr 2025 14:12:47 +0100 Subject: [PATCH 014/105] Update gribjump interface to use iterator --- polytope_feature/datacube/backends/fdb.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/polytope_feature/datacube/backends/fdb.py b/polytope_feature/datacube/backends/fdb.py index 64304e379..6352560d3 100644 --- a/polytope_feature/datacube/backends/fdb.py +++ b/polytope_feature/datacube/backends/fdb.py @@ -140,7 +140,7 @@ def get(self, requests: TensorIndexTree, context=None): logging.debug("The requests we give GribJump are: %s", printed_list_to_gj) logging.info("Requests given to GribJump extract for %s", context) try: - output_values = self.gj.extract(complete_list_complete_uncompressed_requests, context) + iterator = self.gj.extract(complete_list_complete_uncompressed_requests, context) except Exception as e: if "BadValue: Grid hash mismatch" in str(e): logging.info("Error is: %s", e) @@ -152,10 +152,7 @@ def get(self, requests: TensorIndexTree, context=None): raise e logging.info("Requests extracted from GribJump for %s", context) - if logging.root.level <= logging.DEBUG: - printed_output_values = output_values[::1000] - logging.debug("GribJump outputs: %s", printed_output_values) - self.assign_fdb_output_to_nodes(output_values, complete_fdb_decoding_info) + self.assign_fdb_output_to_nodes(iterator, complete_fdb_decoding_info) def get_fdb_requests( self, @@ -321,9 +318,8 @@ def get_last_layer_before_leaf(self, requests, leaf_path, current_idx, fdb_range fdb_range_n[i].append(c) return (current_idx, fdb_range_n) - def assign_fdb_output_to_nodes(self, output_values, fdb_requests_decoding_info): - for k in range(len(output_values)): - request_output_values = output_values[k] + def assign_fdb_output_to_nodes(self, output_iterator, fdb_requests_decoding_info): + for k, result in enumerate(output_iterator): ( original_indices, fdb_node_ranges, @@ -331,13 +327,12 @@ def assign_fdb_output_to_nodes(self, output_values, fdb_requests_decoding_info): sorted_fdb_range_nodes = [fdb_node_ranges[i] for i in original_indices] for i in range(len(sorted_fdb_range_nodes)): n = sorted_fdb_range_nodes[i][0] - if len(request_output_values[0]) == 0: + if len(result.values) == 0: # If we are here, no data was found for this path in the fdb none_array = [None] * len(n.values) n.result.extend(none_array) else: - interm_request_output_values = request_output_values[0][i][0] - n.result.extend(interm_request_output_values) + n.result.extend(result.values) def sort_fdb_request_ranges(self, current_start_idx, lat_length, fdb_node_ranges): (new_fdb_node_ranges, new_current_start_idx) = self.remove_duplicates_in_request_ranges( From 29c0a1b5d5ed887f86c5fffe2f6c7a6f0f46b077 Mon Sep 17 00:00:00 2001 From: Chris Bradley Date: Tue, 8 Apr 2025 18:46:17 +0100 Subject: [PATCH 015/105] Extend one range at a time --- polytope_feature/datacube/backends/fdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope_feature/datacube/backends/fdb.py b/polytope_feature/datacube/backends/fdb.py index 6352560d3..b69138c8d 100644 --- a/polytope_feature/datacube/backends/fdb.py +++ b/polytope_feature/datacube/backends/fdb.py @@ -332,7 +332,7 @@ def assign_fdb_output_to_nodes(self, output_iterator, fdb_requests_decoding_info none_array = [None] * len(n.values) n.result.extend(none_array) else: - n.result.extend(result.values) + n.result.extend(result.values[i]) def sort_fdb_request_ranges(self, current_start_idx, lat_length, fdb_node_ranges): (new_fdb_node_ranges, new_current_start_idx) = self.remove_duplicates_in_request_ranges( From 19f8d1cca447377aa1be7b8ad2ae1be25bf19076 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 9 Apr 2025 11:37:39 +0200 Subject: [PATCH 016/105] WIP --- .gitignore | 3 ++- tests/conftest.py | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/conftest.py diff --git a/.gitignore b/.gitignore index 742930a33..a4face77a 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ new_polytope_venv *.json venv_python3_11 *.txt -tests/data \ No newline at end of file +tests/data +venv_gj_iterator \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 000000000..747aecb1a --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,25 @@ +import pytest + +import pathlib + + +@pytest.fixture +def load_fdb_data_from_nexus(): + # TODO: load all the relevant data files we want in the fdb from nexus + pass + + +@pytest.fixture +def fdb_data_path(request) -> pathlib.Path: + """ + Provides path to test data at '/tests/fdb_data' + """ + path = request.config.rootpath / "tests" / "fdb_data" + assert path.exists() + return path + + +@pytest.fixture(scope="session") +def fdb_store_operational(fdb_data_path) -> pathlib.Path: + # TODO: once we have the test data in the fdb_data folder, and we have a path to it, load it all in an fdb + pass From 6a61a42baf416d46c3da4686b72047f96d975e60 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 9 Apr 2025 13:56:30 +0200 Subject: [PATCH 017/105] update version --- .gitignore | 3 ++- polytope_feature/version.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 742930a33..a4face77a 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ new_polytope_venv *.json venv_python3_11 *.txt -tests/data \ No newline at end of file +tests/data +venv_gj_iterator \ No newline at end of file diff --git a/polytope_feature/version.py b/polytope_feature/version.py index 0d6c7d97e..6849410aa 100644 --- a/polytope_feature/version.py +++ b/polytope_feature/version.py @@ -1 +1 @@ -__version__ = "1.0.42" +__version__ = "1.1.0" From f4cf1964459eb3ec595a7097baf9e952c166e03d Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 14 Apr 2025 11:02:06 +0200 Subject: [PATCH 018/105] add pytest fixtures to add fdb data --- tests/conftest.py | 95 ++++++++++++++++++++++++++++++- tests/fdb_data/schema | 29 ++++++++++ tests/test_ecmwf_oper_data_fdb.py | 4 +- 3 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 tests/fdb_data/schema diff --git a/tests/conftest.py b/tests/conftest.py index 747aecb1a..f03c5f9ac 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,15 @@ +from requests.exceptions import HTTPError import pytest import pathlib +import yaml +import os +import shutil + +import tempfile +import requests +from .helper_functions import HTTPError +import pyfdb @pytest.fixture @@ -10,7 +19,7 @@ def load_fdb_data_from_nexus(): @pytest.fixture -def fdb_data_path(request) -> pathlib.Path: +def fdb_path(request) -> pathlib.Path: """ Provides path to test data at '/tests/fdb_data' """ @@ -20,6 +29,86 @@ def fdb_data_path(request) -> pathlib.Path: @pytest.fixture(scope="session") -def fdb_store_operational(fdb_data_path) -> pathlib.Path: +def fdb_store_operational_setup(fdb_path, tmp_path, downloaded_data_test_files) -> pathlib.Path: # TODO: once we have the test data in the fdb_data folder, and we have a path to it, load it all in an fdb - pass + + # Set up FDB config and schemas + db_store_path = tmp_path / "db_store" + db_store_path.mkdir(exist_ok=True) + schema_path = tmp_path / "schema" + config = dict( + type="local", + engine="toc", + schema=str(schema_path), + spaces=[ + dict( + handler="Default", + roots=[ + {"path": str(db_store_path)}, + ], + ) + ], + ) + config_path = tmp_path / "config.yaml" + config_path.write_text(yaml.dump(config)) + shutil.copy(fdb_path / "schema", schema_path) + os.environ["FDB5_CONFIG_FILE"] = str(config_path) + + # TODO: download data from the internet + # grib_files = data_path.glob("*.grib") + # grib_files = [] + # for file_name in downloaded_data_test_files: + # # grib_files = [data_path / "synth11.grib"] + # grib_files.append(file_name) + fdb = pyfdb.FDB() + # for f in grib_files: + for f in downloaded_data_test_files: + fdb.archive(f.read_bytes()) + return tmp_path + + +@pytest.fixture(scope="session") +def shared_temp_data_dir(tmp_path_factory): + # This creates a unique temp dir for the whole test session + temp_dir = tmp_path_factory.mktemp("shared_fdb_data") + return temp_dir + + +@pytest.fixture(scope="session") +def downloaded_data_test_files(shared_temp_data_dir): + files_to_download = [ + # ("https://example.com/file1.csv", "file1.csv"), + # ("https://example.com/file2.csv", "file2.csv"), + # ("https://example.com/file3.csv", "file3.csv"), + ("https://get.ecmwf.int/test-data/polytope/test-data/foo.grib", "foo.grib") + ] + + downloaded_paths = [] + + for (url, filename) in files_to_download: + path = shared_temp_data_dir / filename + if not path.exists(): + response = requests.get(url) + if response.status_code != 200: + raise HTTPError(f"HTTP {response.status_code} - Failed to download data.") + # path.write_bytes(response.content) + with open(path, "wb") as f: + f.write(response.content) + downloaded_paths.append(path) + + return downloaded_paths + + +# @pytest.fixture(scope="session") +# def downloaded_test_file(shared_temp_data_dir, nexus_url, filename): + +# local_file_path = shared_temp_data_dir / filename + +# if not os.path.exists(local_file_path): +# response = requests.get(nexus_url) +# if response.status_code != 200: +# raise HTTPError(f"HTTP {response.status_code} - Failed to download data.") +# with open(local_file_path, "wb") as f: +# f.write(response.content) + +# yield local_file_path diff --git a/tests/fdb_data/schema b/tests/fdb_data/schema new file mode 100644 index 000000000..b0e7497db --- /dev/null +++ b/tests/fdb_data/schema @@ -0,0 +1,29 @@ +# Default types + +param: Param; +step: Step; +date: Date; +levelist: Double; +grid: Grid; +expver: Expver; +time: Time; +number: Integer; + +######################################################## +# The are the rules matching most of the fields +# oper/dcda +[ class, expver, stream=oper/dcda/scda, date, time, domain? + [ type, levtype + [ step, levelist?, param ]] +] +# enfo +[ class, expver, stream=enfo/efov/eefo, date, time, domain + [ type, levtype + [ step, quantile?, number?, levelist?, param ]] +] + +# waef/weov +[ class, expver, stream=waef/weov/weef, date, time, domain + [ type, levtype + [ step, number?, param, frequency?, direction? ]] +] \ No newline at end of file diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index e848716c8..147396262 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -70,8 +70,8 @@ def test_fdb_datacube(self): assert len(result.leaves) == 3 assert len(result.leaves[0].result) == 3 - @pytest.mark.fdb - def test_fdb_datacube_point(self): + # @pytest.mark.fdb + def test_fdb_datacube_point(self, downloaded_data_test_files): import pygribjump as gj request = Request( From 1ca85649bc63a42b0c62a466854df73af2a7c7b6 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 14 Apr 2025 11:03:46 +0200 Subject: [PATCH 019/105] isort --- tests/conftest.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index f03c5f9ac..2992e0b07 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,15 +1,13 @@ -from requests.exceptions import HTTPError -import pytest - -import pathlib -import yaml import os +import pathlib import shutil -import tempfile +import pyfdb +import pytest import requests +import yaml + from .helper_functions import HTTPError -import pyfdb @pytest.fixture From 2e11478e64ef80b282b5c9a9ca861f6a4dbd4ef4 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 14 Apr 2025 11:04:38 +0200 Subject: [PATCH 020/105] black --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2992e0b07..1b191ffd7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -83,7 +83,7 @@ def downloaded_data_test_files(shared_temp_data_dir): downloaded_paths = [] - for (url, filename) in files_to_download: + for url, filename in files_to_download: path = shared_temp_data_dir / filename if not path.exists(): response = requests.get(url) From 5a0d809f03aaed20ac1d15b43415f59ace72c756 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 14 Apr 2025 11:12:13 +0200 Subject: [PATCH 021/105] add test requirements --- tests/requirements_test.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index e48f1da1a..f54f99219 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -4,4 +4,6 @@ cffi eccodes h5netcdf h5py -earthkit-data \ No newline at end of file +earthkit-data +pyfdb +pygribjump \ No newline at end of file From 05e46222708e835b9dde5cd8c0a1995b462bfb19 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 14 Apr 2025 11:16:24 +0200 Subject: [PATCH 022/105] try to install pygribjump --- tests/requirements_test.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index f54f99219..ce80c602a 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -6,4 +6,5 @@ h5netcdf h5py earthkit-data pyfdb -pygribjump \ No newline at end of file +# pygribjump +git+ssh://git@github.com/ecmwf/gribjump.git@develop \ No newline at end of file From 39b02de08ce604d5ec0c957c49699069857ebec6 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 14 Apr 2025 11:19:37 +0200 Subject: [PATCH 023/105] try to install pygribjump --- tests/requirements_test.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index ce80c602a..6d2e2299d 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -7,4 +7,5 @@ h5py earthkit-data pyfdb # pygribjump -git+ssh://git@github.com/ecmwf/gribjump.git@develop \ No newline at end of file +# git+ssh://git@github.com/ecmwf/gribjump.git@develop +git+https://github.com/ecmwf/gribjump.git@develop From 98f7457b796a0819a51e213457c44d9be637afc1 Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 15 Apr 2025 15:34:55 +0200 Subject: [PATCH 024/105] add downstream ci --- .github/ci-config.yml | 13 + .github/workflows/cd.yml | 11 + .github/workflows/ci.yaml | 337 +++++++++++++++----------- .github/workflows/label-public-pr.yml | 10 + .gitignore | 3 +- 5 files changed, 225 insertions(+), 149 deletions(-) create mode 100644 .github/ci-config.yml create mode 100644 .github/workflows/cd.yml create mode 100644 .github/workflows/label-public-pr.yml diff --git a/.github/ci-config.yml b/.github/ci-config.yml new file mode 100644 index 000000000..baaa89424 --- /dev/null +++ b/.github/ci-config.yml @@ -0,0 +1,13 @@ +dependencies: | + ecmwf/ecbuild + MathisRosenhauer/libaec@master + ecmwf/eccodes + ecmwf/eckit@develop + ecmwf/metkit + ecmwf/fdb + ecmwf/gribjump +dependency_cmake_options: | + ecmwf/gribjump: "-DENABLE_FDB_BUILD_TOOLS=ON" +dependency_branch: develop +parallelism_factor: 8 +self_build: false \ No newline at end of file diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 000000000..ae4568c03 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,11 @@ +name: cd + +on: + push: + tags: + - '**' + +jobs: + deploy: + uses: ecmwf-actions/reusable-workflows/.github/workflows/create-package.yml@v2 + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 24a5887c9..d0f92d80b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,162 +1,203 @@ +# name: ci +# on: +# # Trigger the workflow on push to master or develop, except tag creation +# push: +# branches: +# - 'main' +# - 'develop' +# # Trigger the workflow on pull request +# pull_request: ~ +# # Trigger the workflow manually +# workflow_dispatch: ~ +# # Trigger after public PR approved for CI +# pull_request_target: +# types: [labeled] +# release: +# types: [created] +# jobs: +# qa: +# name: qa +# runs-on: ubuntu-20.04 +# steps: +# - name: Checkout Repository +# uses: actions/checkout@v3 +# with: +# repository: ${{ inputs.repository }} +# ref: ${{ inputs.ref }} + +# - name: Setup Python +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ inputs.python_version }} + +# - name: Install Python Dependencies +# run: | +# python -m pip install --upgrade pip +# python -m pip install black flake8 isort +# - name: Check isort +# run: isort --check . + +# - name: Check black +# run: black --check . + +# - name: Check flake8 +# run: flake8 . +# setup: +# name: setup +# runs-on: ubuntu-20.04 +# outputs: +# matrix: ${{ steps.set-matrix.outputs.matrix }} +# inputs: ${{ steps.prepare-inputs.outputs.inputs }} +# inputs-for-ubuntu: ${{ steps.prepare-inputs.outputs.inputs-for-ubuntu }} +# steps: +# - name: Set Matrix +# id: set-matrix +# shell: bash -eux {0} +# run: | +# MATRIX=$(cat << 'EOS' +# name: +# - gnu-11@ubuntu-22.04 +# - clang-14@ubuntu-22.04 +# include: +# - name: gnu-11@ubuntu-22.04 +# os: ubuntu-22.04 +# compiler: gnu-11 +# compiler_cc: gcc-11 +# compiler_cxx: g++-11 +# compiler_fc: gfortran-11 +# - name: clang-14@ubuntu-22.04 +# os: ubuntu-22.04 +# compiler: clang-14 +# compiler_cc: clang-14 +# compiler_cxx: clang++-14 +# compiler_fc: gfortran-11 +# # Xcode compiler requires empty environment variables, so we pass null (~) here +# EOS +# ) +# SKIP_MATRIX_JOBS=$(cat << 'EOS' +# ${{ inputs.skip_matrix_jobs }} +# EOS +# ) +# SELECT_NAME_COND="1 != 1" +# SELECT_INCLUDE_COND="1 != 1" +# for skip_job in $SKIP_MATRIX_JOBS; do SELECT_NAME_COND="$SELECT_NAME_COND or . == \"$skip_job\""; SELECT_INCLUDE_COND="$SELECT_INCLUDE_COND or .name == \"$skip_job\""; done +# echo matrix=$(echo "$MATRIX" | yq eval "del(.name[] | select($SELECT_NAME_COND)) | del(.include[] | select($SELECT_INCLUDE_COND))" --output-format json --indent 0 -) >> $GITHUB_OUTPUT +# - name: Prepare build-package Inputs +# id: prepare-inputs +# shell: bash -eux {0} +# run: | +# echo inputs=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '.' --output-format json --indent 0 -) >> $GITHUB_OUTPUT +# echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-20.04","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT +# test: +# name: test +# needs: +# - qa +# - setup +# strategy: +# fail-fast: false +# matrix: ${{ fromJson(needs.setup.outputs.matrix) }} +# runs-on: ${{ matrix.os }} +# steps: +# - uses: actions/checkout@v4 +# - name: Install eccodes and Dependencies +# id: install-dependencies +# uses: ecmwf-actions/build-package@v2 +# with: +# self_build: false +# dependencies: | +# ecmwf/ecbuild@develop +# MathisRosenhauer/libaec@master +# ecmwf/eccodes@develop + +# - name: Setup Python +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ inputs.python_version }} + +# - name: Install Python Dependencies +# run: | +# python -m pip install --upgrade pip +# python -m pip install pytest pytest-cov +# python -m pip install -r requirements.txt +# python -m pip install -r ./tests/requirements_test.txt + +# - name: Verify Source Distribution +# shell: bash -eux {0} +# run: | +# python setup.py sdist +# python -m pip install dist/* +# - name: Run Tests with Repository Code +# env: +# LD_LIBRARY_PATH: ${{ steps.install-dependencies.outputs.lib_path }} +# shell: bash -eux {0} +# run: | +# DYLD_LIBRARY_PATH=${{ env.LD_LIBRARY_PATH }} python -m pytest -m "not fdb" tests --cov=./ --cov-report=xml +# python -m coverage report + +# - name: Upload coverage to Codecov +# uses: codecov/codecov-action@v4 +# with: +# files: coverage.xml +# deploy: +# needs: test +# if: ${{ github.event_name == 'release' }} +# name: Upload to Pypi +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - name: Set up Python +# uses: actions/setup-python@v2 +# with: +# python-version: '3.8' +# - name: Install dependencies +# run: | +# python -m pip install --upgrade pip +# pip install build twine +# - name: Build and publish +# env: +# TWINE_USERNAME: "__token__" +# TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} +# run: | +# python -m build +# twine upload dist/* + name: ci + on: # Trigger the workflow on push to master or develop, except tag creation push: branches: - - 'main' - - 'develop' + - "master" + - "develop" + tags-ignore: + - "**" + # Trigger the workflow on pull request pull_request: ~ + # Trigger the workflow manually workflow_dispatch: ~ + # Trigger after public PR approved for CI pull_request_target: types: [labeled] - release: - types: [created] -jobs: - qa: - name: qa - runs-on: ubuntu-20.04 - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - with: - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python_version }} - - - name: Install Python Dependencies - run: | - python -m pip install --upgrade pip - python -m pip install black flake8 isort - - name: Check isort - run: isort --check . - - name: Check black - run: black --check . - - - name: Check flake8 - run: flake8 . - setup: - name: setup - runs-on: ubuntu-20.04 - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - inputs: ${{ steps.prepare-inputs.outputs.inputs }} - inputs-for-ubuntu: ${{ steps.prepare-inputs.outputs.inputs-for-ubuntu }} - steps: - - name: Set Matrix - id: set-matrix - shell: bash -eux {0} - run: | - MATRIX=$(cat << 'EOS' - name: - - gnu-11@ubuntu-22.04 - - clang-14@ubuntu-22.04 - include: - - name: gnu-11@ubuntu-22.04 - os: ubuntu-22.04 - compiler: gnu-11 - compiler_cc: gcc-11 - compiler_cxx: g++-11 - compiler_fc: gfortran-11 - - name: clang-14@ubuntu-22.04 - os: ubuntu-22.04 - compiler: clang-14 - compiler_cc: clang-14 - compiler_cxx: clang++-14 - compiler_fc: gfortran-11 - # Xcode compiler requires empty environment variables, so we pass null (~) here - EOS - ) - SKIP_MATRIX_JOBS=$(cat << 'EOS' - ${{ inputs.skip_matrix_jobs }} - EOS - ) - SELECT_NAME_COND="1 != 1" - SELECT_INCLUDE_COND="1 != 1" - for skip_job in $SKIP_MATRIX_JOBS; do SELECT_NAME_COND="$SELECT_NAME_COND or . == \"$skip_job\""; SELECT_INCLUDE_COND="$SELECT_INCLUDE_COND or .name == \"$skip_job\""; done - echo matrix=$(echo "$MATRIX" | yq eval "del(.name[] | select($SELECT_NAME_COND)) | del(.include[] | select($SELECT_INCLUDE_COND))" --output-format json --indent 0 -) >> $GITHUB_OUTPUT - - name: Prepare build-package Inputs - id: prepare-inputs - shell: bash -eux {0} - run: | - echo inputs=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '.' --output-format json --indent 0 -) >> $GITHUB_OUTPUT - echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-20.04","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT - test: - name: test - needs: - - qa - - setup - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.setup.outputs.matrix) }} - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - name: Install eccodes and Dependencies - id: install-dependencies - uses: ecmwf-actions/build-package@v2 - with: - self_build: false - dependencies: | - ecmwf/ecbuild@develop - MathisRosenhauer/libaec@master - ecmwf/eccodes@develop - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python_version }} - - - name: Install Python Dependencies - run: | - python -m pip install --upgrade pip - python -m pip install pytest pytest-cov - python -m pip install -r requirements.txt - python -m pip install -r ./tests/requirements_test.txt - - - name: Verify Source Distribution - shell: bash -eux {0} - run: | - python setup.py sdist - python -m pip install dist/* - - name: Run Tests with Repository Code - env: - LD_LIBRARY_PATH: ${{ steps.install-dependencies.outputs.lib_path }} - shell: bash -eux {0} - run: | - DYLD_LIBRARY_PATH=${{ env.LD_LIBRARY_PATH }} python -m pytest -m "not fdb" tests --cov=./ --cov-report=xml - python -m coverage report +jobs: + # Run CI including downstream packages on self-hosted runners + downstream-ci: + name: downstream-ci + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main + with: + polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} + codecov_upload: true + secrets: inherit - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - files: coverage.xml - deploy: - needs: test - if: ${{ github.event_name == 'release' }} - name: Upload to Pypi - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.8' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build twine - - name: Build and publish - env: - TWINE_USERNAME: "__token__" - TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - run: | - python -m build - twine upload dist/* \ No newline at end of file + # Build downstream packages on HPC + downstream-ci-hpc: + name: downstream-ci-hpc + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main + with: + polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} + secrets: inherit diff --git a/.github/workflows/label-public-pr.yml b/.github/workflows/label-public-pr.yml new file mode 100644 index 000000000..52f3b87c5 --- /dev/null +++ b/.github/workflows/label-public-pr.yml @@ -0,0 +1,10 @@ +# Manage labels of pull requests that originate from forks +name: label-public-pr + +on: + pull_request_target: + types: [opened, synchronize] + +jobs: + label: + uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 742930a33..a4face77a 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ new_polytope_venv *.json venv_python3_11 *.txt -tests/data \ No newline at end of file +tests/data +venv_gj_iterator \ No newline at end of file From 18a8c8095d0468ce4bf3789105d3cbaabb75bb2c Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 15 Apr 2025 15:40:06 +0200 Subject: [PATCH 025/105] add hpc github ci config and fix branch of downstream ci --- .github/ci-hpc-config.yml | 19 +++++++++++++++++++ .github/workflows/ci.yaml | 6 +++--- 2 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 .github/ci-hpc-config.yml diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml new file mode 100644 index 000000000..456c4c0f2 --- /dev/null +++ b/.github/ci-hpc-config.yml @@ -0,0 +1,19 @@ +build: + python: "3.10" + modules: + - ninja + - aec + dependencies: + - ecmwf/ecbuild@develop + - ecmwf/eccodes@develop + - ecmwf/eckit@develop + - ecmwf/metkit@develop + - ecmwf/fdb@develop + - ecmwf/gribjump@develop + dependency_cmake_options: + - "ecmwf/gribjump: '-DENABLE_FDB_BUILD_TOOLS=ON'" + - "ecmwf/fdb: '-DENABLE_LUSTRE=OFF'" + parallel: 64 + env: + - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples + - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d0f92d80b..bd5c837c0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -167,7 +167,7 @@ on: # Trigger the workflow on push to master or develop, except tag creation push: branches: - - "master" + - "main" - "develop" tags-ignore: - "**" @@ -187,7 +187,7 @@ jobs: downstream-ci: name: downstream-ci if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@feature/add_polytope with: polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} codecov_upload: true @@ -197,7 +197,7 @@ jobs: downstream-ci-hpc: name: downstream-ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/add_polytope with: polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} secrets: inherit From b529204c941abd5da9519ab1eb884822495866da Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 15 Apr 2025 15:42:15 +0200 Subject: [PATCH 026/105] fix path to downstream ci --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bd5c837c0..d75dc860b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -187,7 +187,7 @@ jobs: downstream-ci: name: downstream-ci if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@feature/add_polytope + uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml@feature/add_polytope with: polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} codecov_upload: true @@ -197,7 +197,7 @@ jobs: downstream-ci-hpc: name: downstream-ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/add_polytope + uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/add_polytope with: polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} secrets: inherit From 210e1613e2bc76c9ae79390975a5df6b36b90377 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 28 Apr 2025 11:44:56 +0200 Subject: [PATCH 027/105] update ubuntu version in ci to latest --- .github/workflows/ci.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 24a5887c9..6b1738ac0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -56,17 +56,17 @@ jobs: run: | MATRIX=$(cat << 'EOS' name: - - gnu-11@ubuntu-22.04 - - clang-14@ubuntu-22.04 + - gnu-11@ubuntu-latest + - clang-14@ubuntu-latest include: - - name: gnu-11@ubuntu-22.04 - os: ubuntu-22.04 + - name: gnu-11@ubuntu-latest + os: ubuntu-latest compiler: gnu-11 compiler_cc: gcc-11 compiler_cxx: g++-11 compiler_fc: gfortran-11 - - name: clang-14@ubuntu-22.04 - os: ubuntu-22.04 + - name: clang-14@ubuntu-latest + os: ubuntu-latest compiler: clang-14 compiler_cc: clang-14 compiler_cxx: clang++-14 From 45a2f491ffbcc61332b988645393fde1810c0627 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 28 Apr 2025 11:50:03 +0200 Subject: [PATCH 028/105] finish changing to ubuntu latest in ci --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6b1738ac0..a0c0a0b06 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,7 @@ on: jobs: qa: name: qa - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout Repository uses: actions/checkout@v3 @@ -44,7 +44,7 @@ jobs: run: flake8 . setup: name: setup - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} inputs: ${{ steps.prepare-inputs.outputs.inputs }} @@ -87,7 +87,7 @@ jobs: shell: bash -eux {0} run: | echo inputs=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '.' --output-format json --indent 0 -) >> $GITHUB_OUTPUT - echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-20.04","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT + echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-latest","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT test: name: test needs: From 0922a91cbfadd44ec258f9f8ae3d45ce19650472 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 28 Apr 2025 12:05:06 +0200 Subject: [PATCH 029/105] try to modify ecbuild build options --- .github/workflows/ci.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a0c0a0b06..20e54a235 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -103,11 +103,15 @@ jobs: id: install-dependencies uses: ecmwf-actions/build-package@v2 with: - self_build: false + # self_build: false + cmake: true + cmake_options: -DCMAKE_BUILD_TYPE=Debug dependencies: | ecmwf/ecbuild@develop MathisRosenhauer/libaec@master ecmwf/eccodes@develop + dependency_cmake_options: | + ecmwf/ecbuild: "-DCMAKE_BUILD_TYPE=Debug" - name: Setup Python uses: actions/setup-python@v4 From e7e3e8191ef97aa2703bce5c7cf976367d9055f4 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 28 Apr 2025 12:09:07 +0200 Subject: [PATCH 030/105] use ecmwf instead of ecmwf-actions --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 20e54a235..726877d15 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -101,7 +101,7 @@ jobs: - uses: actions/checkout@v4 - name: Install eccodes and Dependencies id: install-dependencies - uses: ecmwf-actions/build-package@v2 + uses: ecmwf/build-package@v2 with: # self_build: false cmake: true From 781d0421bee5cae0c1226f757cfa452dbb93aa94 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 28 Apr 2025 12:17:53 +0200 Subject: [PATCH 031/105] add specific ubuntu version --- .github/workflows/ci.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 726877d15..c55be68f3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,7 @@ on: jobs: qa: name: qa - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout Repository uses: actions/checkout@v3 @@ -44,7 +44,7 @@ jobs: run: flake8 . setup: name: setup - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} inputs: ${{ steps.prepare-inputs.outputs.inputs }} @@ -56,17 +56,17 @@ jobs: run: | MATRIX=$(cat << 'EOS' name: - - gnu-11@ubuntu-latest - - clang-14@ubuntu-latest + - gnu-11@ubuntu-24.04 + - clang-14@ubuntu-24.04 include: - - name: gnu-11@ubuntu-latest - os: ubuntu-latest + - name: gnu-11@ubuntu-24.04 + os: ubuntu-24.04 compiler: gnu-11 compiler_cc: gcc-11 compiler_cxx: g++-11 compiler_fc: gfortran-11 - - name: clang-14@ubuntu-latest - os: ubuntu-latest + - name: clang-14@ubuntu-24.04 + os: ubuntu-24.04 compiler: clang-14 compiler_cc: clang-14 compiler_cxx: clang++-14 @@ -87,7 +87,7 @@ jobs: shell: bash -eux {0} run: | echo inputs=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '.' --output-format json --indent 0 -) >> $GITHUB_OUTPUT - echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-latest","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT + echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-24.04","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT test: name: test needs: @@ -146,7 +146,7 @@ jobs: needs: test if: ${{ github.event_name == 'release' }} name: Upload to Pypi - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 - name: Set up Python From 78581159ea9365eae22cc5782919c95a9f68d5b3 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 28 Apr 2025 12:24:36 +0200 Subject: [PATCH 032/105] update ci matrix --- .github/workflows/ci.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c55be68f3..9c3342d0c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -61,16 +61,16 @@ jobs: include: - name: gnu-11@ubuntu-24.04 os: ubuntu-24.04 - compiler: gnu-11 - compiler_cc: gcc-11 - compiler_cxx: g++-11 - compiler_fc: gfortran-11 - - name: clang-14@ubuntu-24.04 + compiler: gnu-13 + compiler_cc: gcc + compiler_cxx: g++ + compiler_fc: gfortran + - name: clang-16@ubuntu-24.04 os: ubuntu-24.04 - compiler: clang-14 - compiler_cc: clang-14 - compiler_cxx: clang++-14 - compiler_fc: gfortran-11 + compiler: clang-16 + compiler_cc: clang-16 + compiler_cxx: clang++-16 + compiler_fc: gfortran # Xcode compiler requires empty environment variables, so we pass null (~) here EOS ) From 6477f9d39116e4091efb945086928ec423353547 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 28 Apr 2025 12:30:19 +0200 Subject: [PATCH 033/105] add new matrix for ci --- .github/workflows/ci.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9c3342d0c..fb02f59c7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -93,9 +93,27 @@ jobs: needs: - qa - setup + # strategy: + # fail-fast: false + # matrix: ${{ fromJson(needs.setup.outputs.matrix) }} + # runs-on: ${{ matrix.os }} strategy: fail-fast: false - matrix: ${{ fromJson(needs.setup.outputs.matrix) }} + matrix: + name: + - ubuntu-24.04 + - ubuntu-22.04 + - macos-latest + include: + - name: ubuntu-24.04 + os: ubuntu-24.04 + compiler_fc: gfortran + - name: ubuntu-22.04 + os: ubuntu-22.04 + compiler_fc: gfortran + - name: macos-latest + os: macos-latest + compiler_fc: gfortran-12 runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 From c89494482234799671431c6f3e572b97e3e87e9a Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 28 Apr 2025 12:35:55 +0200 Subject: [PATCH 034/105] change to old ecbuild building --- .github/workflows/ci.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fb02f59c7..8bc25bc22 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -121,15 +121,11 @@ jobs: id: install-dependencies uses: ecmwf/build-package@v2 with: - # self_build: false - cmake: true - cmake_options: -DCMAKE_BUILD_TYPE=Debug + self_build: false dependencies: | ecmwf/ecbuild@develop MathisRosenhauer/libaec@master ecmwf/eccodes@develop - dependency_cmake_options: | - ecmwf/ecbuild: "-DCMAKE_BUILD_TYPE=Debug" - name: Setup Python uses: actions/setup-python@v4 From ef740669ea764f105247de85c8b8d5625663c799 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 28 Apr 2025 12:41:41 +0200 Subject: [PATCH 035/105] remove setup action and ubuntu-24.04 from matrix and add setuptools before testing --- .github/workflows/ci.yaml | 101 +++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8bc25bc22..0e4aaa4e9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,52 +42,52 @@ jobs: - name: Check flake8 run: flake8 . - setup: - name: setup - runs-on: ubuntu-24.04 - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - inputs: ${{ steps.prepare-inputs.outputs.inputs }} - inputs-for-ubuntu: ${{ steps.prepare-inputs.outputs.inputs-for-ubuntu }} - steps: - - name: Set Matrix - id: set-matrix - shell: bash -eux {0} - run: | - MATRIX=$(cat << 'EOS' - name: - - gnu-11@ubuntu-24.04 - - clang-14@ubuntu-24.04 - include: - - name: gnu-11@ubuntu-24.04 - os: ubuntu-24.04 - compiler: gnu-13 - compiler_cc: gcc - compiler_cxx: g++ - compiler_fc: gfortran - - name: clang-16@ubuntu-24.04 - os: ubuntu-24.04 - compiler: clang-16 - compiler_cc: clang-16 - compiler_cxx: clang++-16 - compiler_fc: gfortran - # Xcode compiler requires empty environment variables, so we pass null (~) here - EOS - ) - SKIP_MATRIX_JOBS=$(cat << 'EOS' - ${{ inputs.skip_matrix_jobs }} - EOS - ) - SELECT_NAME_COND="1 != 1" - SELECT_INCLUDE_COND="1 != 1" - for skip_job in $SKIP_MATRIX_JOBS; do SELECT_NAME_COND="$SELECT_NAME_COND or . == \"$skip_job\""; SELECT_INCLUDE_COND="$SELECT_INCLUDE_COND or .name == \"$skip_job\""; done - echo matrix=$(echo "$MATRIX" | yq eval "del(.name[] | select($SELECT_NAME_COND)) | del(.include[] | select($SELECT_INCLUDE_COND))" --output-format json --indent 0 -) >> $GITHUB_OUTPUT - - name: Prepare build-package Inputs - id: prepare-inputs - shell: bash -eux {0} - run: | - echo inputs=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '.' --output-format json --indent 0 -) >> $GITHUB_OUTPUT - echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-24.04","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT + # setup: + # name: setup + # runs-on: ubuntu-24.04 + # outputs: + # matrix: ${{ steps.set-matrix.outputs.matrix }} + # inputs: ${{ steps.prepare-inputs.outputs.inputs }} + # inputs-for-ubuntu: ${{ steps.prepare-inputs.outputs.inputs-for-ubuntu }} + # steps: + # - name: Set Matrix + # id: set-matrix + # shell: bash -eux {0} + # run: | + # MATRIX=$(cat << 'EOS' + # name: + # - gnu-11@ubuntu-24.04 + # - clang-14@ubuntu-24.04 + # include: + # - name: gnu-11@ubuntu-24.04 + # os: ubuntu-24.04 + # compiler: gnu-13 + # compiler_cc: gcc + # compiler_cxx: g++ + # compiler_fc: gfortran + # - name: clang-16@ubuntu-24.04 + # os: ubuntu-24.04 + # compiler: clang-16 + # compiler_cc: clang-16 + # compiler_cxx: clang++-16 + # compiler_fc: gfortran + # # Xcode compiler requires empty environment variables, so we pass null (~) here + # EOS + # ) + # SKIP_MATRIX_JOBS=$(cat << 'EOS' + # ${{ inputs.skip_matrix_jobs }} + # EOS + # ) + # SELECT_NAME_COND="1 != 1" + # SELECT_INCLUDE_COND="1 != 1" + # for skip_job in $SKIP_MATRIX_JOBS; do SELECT_NAME_COND="$SELECT_NAME_COND or . == \"$skip_job\""; SELECT_INCLUDE_COND="$SELECT_INCLUDE_COND or .name == \"$skip_job\""; done + # echo matrix=$(echo "$MATRIX" | yq eval "del(.name[] | select($SELECT_NAME_COND)) | del(.include[] | select($SELECT_INCLUDE_COND))" --output-format json --indent 0 -) >> $GITHUB_OUTPUT + # - name: Prepare build-package Inputs + # id: prepare-inputs + # shell: bash -eux {0} + # run: | + # echo inputs=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '.' --output-format json --indent 0 -) >> $GITHUB_OUTPUT + # echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-24.04","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT test: name: test needs: @@ -101,13 +101,13 @@ jobs: fail-fast: false matrix: name: - - ubuntu-24.04 + # - ubuntu-24.04 - ubuntu-22.04 - macos-latest include: - - name: ubuntu-24.04 - os: ubuntu-24.04 - compiler_fc: gfortran + # - name: ubuntu-24.04 + # os: ubuntu-24.04 + # compiler_fc: gfortran - name: ubuntu-22.04 os: ubuntu-22.04 compiler_fc: gfortran @@ -135,6 +135,7 @@ jobs: - name: Install Python Dependencies run: | python -m pip install --upgrade pip + python -m pip install --upgrade setuptools python -m pip install pytest pytest-cov python -m pip install -r requirements.txt python -m pip install -r ./tests/requirements_test.txt From a522c477f8e781ac83b6bc02ca0dadb41525a09e Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 28 Apr 2025 12:42:46 +0200 Subject: [PATCH 036/105] remove setup action --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0e4aaa4e9..0a090d1ee 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -92,7 +92,7 @@ jobs: name: test needs: - qa - - setup + # - setup # strategy: # fail-fast: false # matrix: ${{ fromJson(needs.setup.outputs.matrix) }} From 4c5b379a53da09881fbfc943c32e7d21af2094f8 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 28 Apr 2025 12:47:06 +0200 Subject: [PATCH 037/105] clean up ci --- .github/workflows/ci.yaml | 55 --------------------------------------- 1 file changed, 55 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0a090d1ee..f48d3ec94 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,72 +42,17 @@ jobs: - name: Check flake8 run: flake8 . - # setup: - # name: setup - # runs-on: ubuntu-24.04 - # outputs: - # matrix: ${{ steps.set-matrix.outputs.matrix }} - # inputs: ${{ steps.prepare-inputs.outputs.inputs }} - # inputs-for-ubuntu: ${{ steps.prepare-inputs.outputs.inputs-for-ubuntu }} - # steps: - # - name: Set Matrix - # id: set-matrix - # shell: bash -eux {0} - # run: | - # MATRIX=$(cat << 'EOS' - # name: - # - gnu-11@ubuntu-24.04 - # - clang-14@ubuntu-24.04 - # include: - # - name: gnu-11@ubuntu-24.04 - # os: ubuntu-24.04 - # compiler: gnu-13 - # compiler_cc: gcc - # compiler_cxx: g++ - # compiler_fc: gfortran - # - name: clang-16@ubuntu-24.04 - # os: ubuntu-24.04 - # compiler: clang-16 - # compiler_cc: clang-16 - # compiler_cxx: clang++-16 - # compiler_fc: gfortran - # # Xcode compiler requires empty environment variables, so we pass null (~) here - # EOS - # ) - # SKIP_MATRIX_JOBS=$(cat << 'EOS' - # ${{ inputs.skip_matrix_jobs }} - # EOS - # ) - # SELECT_NAME_COND="1 != 1" - # SELECT_INCLUDE_COND="1 != 1" - # for skip_job in $SKIP_MATRIX_JOBS; do SELECT_NAME_COND="$SELECT_NAME_COND or . == \"$skip_job\""; SELECT_INCLUDE_COND="$SELECT_INCLUDE_COND or .name == \"$skip_job\""; done - # echo matrix=$(echo "$MATRIX" | yq eval "del(.name[] | select($SELECT_NAME_COND)) | del(.include[] | select($SELECT_INCLUDE_COND))" --output-format json --indent 0 -) >> $GITHUB_OUTPUT - # - name: Prepare build-package Inputs - # id: prepare-inputs - # shell: bash -eux {0} - # run: | - # echo inputs=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '.' --output-format json --indent 0 -) >> $GITHUB_OUTPUT - # echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-24.04","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT test: name: test needs: - qa - # - setup - # strategy: - # fail-fast: false - # matrix: ${{ fromJson(needs.setup.outputs.matrix) }} - # runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: name: - # - ubuntu-24.04 - ubuntu-22.04 - macos-latest include: - # - name: ubuntu-24.04 - # os: ubuntu-24.04 - # compiler_fc: gfortran - name: ubuntu-22.04 os: ubuntu-22.04 compiler_fc: gfortran From 75e9c3246033aa628f0793f4d9cae526913bfa6c Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 14 May 2025 12:02:09 +0200 Subject: [PATCH 038/105] update hash for reduced_ll 3601 grid --- .gitignore | 3 ++- .../datacube_mappers/mapper_types/reduced_ll.py | 2 +- polytope_feature/version.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index a4face77a..a623daae7 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,5 @@ new_polytope_venv venv_python3_11 *.txt tests/data -venv_gj_iterator \ No newline at end of file +venv_gj_iterator +target \ No newline at end of file diff --git a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/reduced_ll.py b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/reduced_ll.py index 5319a4ade..18619dc06 100644 --- a/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/reduced_ll.py +++ b/polytope_feature/datacube/transformations/datacube_mappers/mapper_types/reduced_ll.py @@ -5128,4 +5128,4 @@ def unmap(self, first_val, second_vals): # md5 grid hash in form {resolution : hash} -_md5_hash = {3601: "386742a2dd1201b67f2d19ed421353ea"} +_md5_hash = {3601: "225e56fb2fdee272ca226dc265d08a0a"} diff --git a/polytope_feature/version.py b/polytope_feature/version.py index 6849410aa..a82b376d2 100644 --- a/polytope_feature/version.py +++ b/polytope_feature/version.py @@ -1 +1 @@ -__version__ = "1.1.0" +__version__ = "1.1.1" From 22dfdfed2ad05c6fb1b9be928330760d9ba89749 Mon Sep 17 00:00:00 2001 From: Kai Kratz Date: Mon, 16 Jun 2025 16:19:10 +0200 Subject: [PATCH 039/105] Pin libaec to v1.1.3 --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f48d3ec94..9bd6459f0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -69,7 +69,7 @@ jobs: self_build: false dependencies: | ecmwf/ecbuild@develop - MathisRosenhauer/libaec@master + MathisRosenhauer/libaec@refs/tags/v1.1.3 ecmwf/eccodes@develop - name: Setup Python @@ -123,4 +123,4 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} run: | python -m build - twine upload dist/* \ No newline at end of file + twine upload dist/* From cf57c19b38c41b1562141aed58c50c1d6e1a17ea Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 10:26:20 +0200 Subject: [PATCH 040/105] update gitignore --- .gitignore | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a4face77a..85bccef56 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,10 @@ new_polytope_venv venv_python3_11 *.txt tests/data -venv_gj_iterator \ No newline at end of file +venv_gj_iterator +*.so +rust_deployment_venv +*.lock +**/target +**/build +_version.py \ No newline at end of file From aacc38050ea1a200a52f5e872e3fe5a4ce3b06d9 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 10:39:12 +0200 Subject: [PATCH 041/105] fix libaec to working tag and add conflator dependency --- .github/ci-config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/ci-config.yml b/.github/ci-config.yml index baaa89424..d236546ad 100644 --- a/.github/ci-config.yml +++ b/.github/ci-config.yml @@ -1,11 +1,12 @@ dependencies: | ecmwf/ecbuild - MathisRosenhauer/libaec@master + MathisRosenhauer/libaec@refs/tags/v1.1.3 ecmwf/eccodes ecmwf/eckit@develop ecmwf/metkit ecmwf/fdb ecmwf/gribjump + ecmwf/conflator dependency_cmake_options: | ecmwf/gribjump: "-DENABLE_FDB_BUILD_TOOLS=ON" dependency_branch: develop From 8133cb1d9f4cc3d56790b649a0a245e1cc2923b2 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 10:54:09 +0200 Subject: [PATCH 042/105] fix conflator branch to main --- .github/ci-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ci-config.yml b/.github/ci-config.yml index d236546ad..a27ea56c8 100644 --- a/.github/ci-config.yml +++ b/.github/ci-config.yml @@ -6,7 +6,7 @@ dependencies: | ecmwf/metkit ecmwf/fdb ecmwf/gribjump - ecmwf/conflator + ecmwf/conflator@main dependency_cmake_options: | ecmwf/gribjump: "-DENABLE_FDB_BUILD_TOOLS=ON" dependency_branch: develop From d83a80a467a1f2e4d80ff9894368b1d2f0b5110a Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 10:57:09 +0200 Subject: [PATCH 043/105] remove conflator from ci-config --- .github/ci-config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/ci-config.yml b/.github/ci-config.yml index a27ea56c8..416a96d14 100644 --- a/.github/ci-config.yml +++ b/.github/ci-config.yml @@ -6,7 +6,6 @@ dependencies: | ecmwf/metkit ecmwf/fdb ecmwf/gribjump - ecmwf/conflator@main dependency_cmake_options: | ecmwf/gribjump: "-DENABLE_FDB_BUILD_TOOLS=ON" dependency_branch: develop From 4d18beccfe919da058f2d68944de9748d361628b Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 11:08:07 +0200 Subject: [PATCH 044/105] add requests to requirements for tests --- tests/requirements_test.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index e48f1da1a..dc0df27ff 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -4,4 +4,5 @@ cffi eccodes h5netcdf h5py -earthkit-data \ No newline at end of file +earthkit-data +requests \ No newline at end of file From 5fef3eb291c5675e718129a325f6246b4310e64f Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 11:46:14 +0200 Subject: [PATCH 045/105] add qa --- .github/workflows/ci.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d75dc860b..9bcac441d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -183,6 +183,33 @@ on: types: [labeled] jobs: + qa: + name: qa + runs-on: ubuntu-20.04 + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.ref }} + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python_version }} + + - name: Install Python Dependencies + run: | + python -m pip install --upgrade pip + python -m pip install black flake8 isort + - name: Check isort + run: isort --check . + + - name: Check black + run: black --check . + + - name: Check flake8 + run: flake8 . # Run CI including downstream packages on self-hosted runners downstream-ci: name: downstream-ci From 0740050d52f69dbcb4a6e0e248a6757962ff86f8 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 12:08:44 +0200 Subject: [PATCH 046/105] remove qa --- .github/workflows/ci.yaml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9bcac441d..d75dc860b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -183,33 +183,6 @@ on: types: [labeled] jobs: - qa: - name: qa - runs-on: ubuntu-20.04 - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - with: - repository: ${{ inputs.repository }} - ref: ${{ inputs.ref }} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python_version }} - - - name: Install Python Dependencies - run: | - python -m pip install --upgrade pip - python -m pip install black flake8 isort - - name: Check isort - run: isort --check . - - - name: Check black - run: black --check . - - - name: Check flake8 - run: flake8 . # Run CI including downstream packages on self-hosted runners downstream-ci: name: downstream-ci From a785c4954b627592576a96a05b0afb388f69728e Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 14:11:21 +0200 Subject: [PATCH 047/105] remove hpc downstream ci --- .github/workflows/ci.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d75dc860b..e664e7691 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -193,11 +193,11 @@ jobs: codecov_upload: true secrets: inherit - # Build downstream packages on HPC - downstream-ci-hpc: - name: downstream-ci-hpc - if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/add_polytope - with: - polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} - secrets: inherit + # # Build downstream packages on HPC + # downstream-ci-hpc: + # name: downstream-ci-hpc + # if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + # uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/add_polytope + # with: + # polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} + # secrets: inherit From da67fde3bf0280de80d275889c481248ae938a91 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 15:13:48 +0200 Subject: [PATCH 048/105] add qa and deployment on release --- .github/workflows/ci.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e664e7691..2d8585cda 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -191,6 +191,7 @@ jobs: with: polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} codecov_upload: true + python_qa: true secrets: inherit # # Build downstream packages on HPC @@ -201,3 +202,25 @@ jobs: # with: # polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} # secrets: inherit + + deploy: + if: ${{ github.event_name == 'release' }} + name: Upload to Pypi + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + - name: Build and publish + env: + TWINE_USERNAME: "__token__" + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: | + python -m build + twine upload dist/* From 54af6e1e63f48bfba03cfef3b305074ae29638aa Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 15:42:26 +0200 Subject: [PATCH 049/105] add HTTPError to conftest --- tests/conftest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 1b191ffd7..e1b5ffe52 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,12 @@ import requests import yaml -from .helper_functions import HTTPError + +class HTTPError(Exception): + def __init__(self, status_code, message): + self.status_code = status_code + self.message = message + super().__init__(f"HTTPError {status_code}: {message}") @pytest.fixture From fcbb9640dcbb42f79bf7d1a9701fb2d30372c741 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 15:55:51 +0200 Subject: [PATCH 050/105] add fdb creation to pytest --- tests/test_ecmwf_oper_data_fdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 147396262..0a1c00d76 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -71,7 +71,7 @@ def test_fdb_datacube(self): assert len(result.leaves[0].result) == 3 # @pytest.mark.fdb - def test_fdb_datacube_point(self, downloaded_data_test_files): + def test_fdb_datacube_point(self, downloaded_data_test_files, fdb_store_operational_setup): import pygribjump as gj request = Request( From 890b6b560d39575638cad5be3b85a077012d8510 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 16:00:45 +0200 Subject: [PATCH 051/105] change scope of fdb_path --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index e1b5ffe52..e67b1886f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,7 +21,7 @@ def load_fdb_data_from_nexus(): pass -@pytest.fixture +@pytest.fixture(scope="session") def fdb_path(request) -> pathlib.Path: """ Provides path to test data at '/tests/fdb_data' From ad0cf97134e7a9c0d03f8dbc97c3bc8ca7ce8d03 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 16:10:03 +0200 Subject: [PATCH 052/105] change all fixtures to function scope --- tests/conftest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e67b1886f..c4bdc252d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,7 +21,7 @@ def load_fdb_data_from_nexus(): pass -@pytest.fixture(scope="session") +@pytest.fixture(scope="function") def fdb_path(request) -> pathlib.Path: """ Provides path to test data at '/tests/fdb_data' @@ -31,7 +31,7 @@ def fdb_path(request) -> pathlib.Path: return path -@pytest.fixture(scope="session") +@pytest.fixture(scope="function") def fdb_store_operational_setup(fdb_path, tmp_path, downloaded_data_test_files) -> pathlib.Path: # TODO: once we have the test data in the fdb_data folder, and we have a path to it, load it all in an fdb @@ -70,14 +70,14 @@ def fdb_store_operational_setup(fdb_path, tmp_path, downloaded_data_test_files) return tmp_path -@pytest.fixture(scope="session") +@pytest.fixture(scope="function") def shared_temp_data_dir(tmp_path_factory): # This creates a unique temp dir for the whole test session temp_dir = tmp_path_factory.mktemp("shared_fdb_data") return temp_dir -@pytest.fixture(scope="session") +@pytest.fixture(scope="function") def downloaded_data_test_files(shared_temp_data_dir): files_to_download = [ # ("https://example.com/file1.csv", "file1.csv"), From 22a14f199d874f2a913ad7bc9dc5df699b3595d9 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 16:20:27 +0200 Subject: [PATCH 053/105] add debug of fdb axes --- polytope_feature/datacube/backends/fdb.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/polytope_feature/datacube/backends/fdb.py b/polytope_feature/datacube/backends/fdb.py index b69138c8d..d32812456 100644 --- a/polytope_feature/datacube/backends/fdb.py +++ b/polytope_feature/datacube/backends/fdb.py @@ -47,6 +47,9 @@ def __init__( logging.info("Axes returned from GribJump are: " + str(self.fdb_coordinates)) + print("LOOK HERE") + print(self.fdb_coordinates) + self.fdb_coordinates["values"] = [] for name, values in self.fdb_coordinates.items(): values.sort() From 6cc916dd668ec3a7fab723d1d6a61c02c38fd6bf Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 16:51:50 +0200 Subject: [PATCH 054/105] add new data file to test fdb --- .gitignore | 3 ++- tests/.DS_Store | Bin 8196 -> 0 bytes tests/conftest.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 tests/.DS_Store diff --git a/.gitignore b/.gitignore index 85bccef56..0eaa5567e 100644 --- a/.gitignore +++ b/.gitignore @@ -36,4 +36,5 @@ rust_deployment_venv *.lock **/target **/build -_version.py \ No newline at end of file +_version.py +*.DS_Store \ No newline at end of file diff --git a/tests/.DS_Store b/tests/.DS_Store deleted file mode 100644 index 6669a9c325d6a62f2dbbed0a0af1e2777a9de4a8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHMO>fjN5FM9FRH}q3pimB#BDk+qJ`UVMwLNg=2ZYV;b{BPN%6>@f1?A3v=wIOX zaiVX=4m(bqv_dMXRJ(E}bv(~+o+n8sB_eaY8XXZG5YY}g+r3>J0*$Y8Z?r8p@(k9% zpJ+$}^eK%gYs`)VQGuvHR3Iu46^IJl1qJZV=C;%F-nUodF)9!hxR(m>^C3ZJn>t%s zD~}Ewqy&K7#;`P;qYe@V0Akyy|lb0P# zcxO1aDdL@j**7q(;$O;0kZnV^?lszwGbmfoD6EL&gjVEvXB^)pQ|36XL#hzxg+@y{ z(0sm?4&7{lZyvSm-xzUDB#-#p)q-Y{M`P$4$^_0N&GfNidJ5k{b}Yl_8oL+VAHzAI zF`g)lDfW1VpKF=OE^CxSg>nJ*1)U-ij)vo_XHcbEAV%I>Ppzj4*;k+A-=1K({MS!irz3Hu@pAwthv`>r`v|z@qS?I+bMq*YvXWBUFeA7F; z;bf8cu6Wm$Yd7R`&Hcq0=OMjNJ5lke@)C|Reo1vg1-x1_)W zi+jNJ|4IAz|65u;!bSz60(U_Hv)%vHKL!(5TQ@jyt(~BMM(4)!N^8Xh2Pwy4r5uO- l`wv5$Cz#4Usk5cE7(x5*9|8*eZ57}DEPZ_cvkLdLT>&%nA8G&q diff --git a/tests/conftest.py b/tests/conftest.py index c4bdc252d..de3099d7e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -84,6 +84,7 @@ def downloaded_data_test_files(shared_temp_data_dir): # ("https://example.com/file2.csv", "file2.csv"), # ("https://example.com/file3.csv", "file3.csv"), ("https://get.ecmwf.int/test-data/polytope/test-data/foo.grib", "foo.grib") + ("https://get.ecmwf.int/test-data/polytope/test-data/t2m_jan_3_v2.grib", "t2m_jan_3_v2.grib") ] downloaded_paths = [] From 1c2b1836c91dc1cc1f163e996d45442f6a05469e Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 16:56:47 +0200 Subject: [PATCH 055/105] black --- tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index de3099d7e..07a53d90e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -83,8 +83,8 @@ def downloaded_data_test_files(shared_temp_data_dir): # ("https://example.com/file1.csv", "file1.csv"), # ("https://example.com/file2.csv", "file2.csv"), # ("https://example.com/file3.csv", "file3.csv"), - ("https://get.ecmwf.int/test-data/polytope/test-data/foo.grib", "foo.grib") - ("https://get.ecmwf.int/test-data/polytope/test-data/t2m_jan_3_v2.grib", "t2m_jan_3_v2.grib") + ("https://get.ecmwf.int/test-data/polytope/test-data/foo.grib", "foo.grib"), + ("https://get.ecmwf.int/test-data/polytope/test-data/t2m_jan_3_v2.grib", "t2m_jan_3_v2.grib"), ] downloaded_paths = [] From eb10d9ac49970b6d427b45b4112df656eadc1320 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 17:02:08 +0200 Subject: [PATCH 056/105] clean up --- polytope_feature/datacube/backends/fdb.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/polytope_feature/datacube/backends/fdb.py b/polytope_feature/datacube/backends/fdb.py index d32812456..b69138c8d 100644 --- a/polytope_feature/datacube/backends/fdb.py +++ b/polytope_feature/datacube/backends/fdb.py @@ -47,9 +47,6 @@ def __init__( logging.info("Axes returned from GribJump are: " + str(self.fdb_coordinates)) - print("LOOK HERE") - print(self.fdb_coordinates) - self.fdb_coordinates["values"] = [] for name, values in self.fdb_coordinates.items(): values.sort() From e968f54b3c341f7889719959eec494d73fab2bb3 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 23 Jun 2025 17:03:15 +0200 Subject: [PATCH 057/105] turn on more fdb tests --- tests/test_ecmwf_oper_data_fdb.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 0a1c00d76..5709c3534 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -70,7 +70,6 @@ def test_fdb_datacube(self): assert len(result.leaves) == 3 assert len(result.leaves[0].result) == 3 - # @pytest.mark.fdb def test_fdb_datacube_point(self, downloaded_data_test_files, fdb_store_operational_setup): import pygribjump as gj @@ -99,8 +98,8 @@ def test_fdb_datacube_point(self, downloaded_data_test_files, fdb_store_operatio assert set(result.leaves[0].flatten()["step"]) == set((0, 1)) assert len(result.leaves[0].result) == 4 - @pytest.mark.fdb - def test_fdb_datacube_point_v2(self): + # @pytest.mark.fdb + def test_fdb_datacube_point_v2(self, downloaded_data_test_files, fdb_store_operational_setup): import pygribjump as gj request = Request( @@ -127,8 +126,8 @@ def test_fdb_datacube_point_v2(self): assert len(result.leaves) == 3 assert len(result.leaves[0].result) == 4 - @pytest.mark.fdb - def test_fdb_datacube_point_step_not_compressed(self): + # @pytest.mark.fdb + def test_fdb_datacube_point_step_not_compressed(self, downloaded_data_test_files, fdb_store_operational_setup): import pygribjump as gj self.options = { From c68bc1291b9f944aae7d405fb0215368d3329ebc Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 24 Jun 2025 09:11:14 +0200 Subject: [PATCH 058/105] disable patch coverage --- codecov.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/codecov.yml b/codecov.yml index f911473e5..41335c532 100644 --- a/codecov.yml +++ b/codecov.yml @@ -6,4 +6,10 @@ ignore: - "**/test*" - "**/pb2*" - "**/fdb*" # ignore fdb backend which we can't test in CI yet - - "polytope/datacube/backends/fdb.py" # ignore fdb backend which we can't test in CI yet \ No newline at end of file + - "polytope/datacube/backends/fdb.py" # ignore fdb backend which we can't test in CI yet + +coverage: + status: + patch: + default: + enabled: false # disables patch check \ No newline at end of file From 17c26cef3cba1194cc7c750516badfba48f3038d Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 25 Jun 2025 11:55:46 +0200 Subject: [PATCH 059/105] turn on fdb tests --- tests/test_bad_request_error.py | 2 +- tests/test_cyclic_nearest.py | 2 +- tests/test_date_time_unmerged.py | 2 +- tests/test_ecmwf_oper_data_fdb.py | 2 +- tests/test_fdb_datacube.py | 4 ++-- tests/test_fdb_unmap_tree.py | 2 +- tests/test_incomplete_tree_fdb.py | 4 ++-- tests/test_local_grid_cyclic.py | 4 ++-- tests/test_local_regular_grid.py | 20 ++++++++++---------- tests/test_local_swiss_grid.py | 2 +- tests/test_multiple_param_fdb.py | 2 +- tests/test_override_md5_hash_options.py | 2 +- tests/test_point_nearest.py | 12 ++++++------ tests/test_point_union.py | 6 +++--- tests/test_reduced_ll_grid.py | 2 +- tests/test_regular_grid.py | 4 ++-- tests/test_regular_reduced_grid.py | 2 +- tests/test_shapes.py | 2 +- tests/test_slice_date_range_fdb.py | 12 ++++++------ tests/test_slice_date_range_fdb_v2.py | 2 +- tests/test_slice_fdb_box.py | 2 +- tests/test_tree_protobuf_encoding.py | 2 +- tests/test_tree_protobuf_encoding_fdb.py | 2 +- tests/test_union_gj.py | 6 +++--- tests/test_union_point_box.py | 2 +- tests/test_wave_spectra_data.py | 2 +- 26 files changed, 53 insertions(+), 53 deletions(-) diff --git a/tests/test_bad_request_error.py b/tests/test_bad_request_error.py index b1154cf9c..3224e4f43 100644 --- a/tests/test_bad_request_error.py +++ b/tests/test_bad_request_error.py @@ -51,7 +51,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_cyclic_nearest.py b/tests/test_cyclic_nearest.py index 86e37c69c..544a88b41 100644 --- a/tests/test_cyclic_nearest.py +++ b/tests/test_cyclic_nearest.py @@ -67,7 +67,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points - @pytest.mark.fdb + # @pytest.mark.fdb @pytest.mark.internet def test_regular_grid(self): import pygribjump as gj diff --git a/tests/test_date_time_unmerged.py b/tests/test_date_time_unmerged.py index f675a354e..365972075 100644 --- a/tests/test_date_time_unmerged.py +++ b/tests/test_date_time_unmerged.py @@ -39,7 +39,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 5709c3534..5bbffaac9 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -42,7 +42,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 301ceec30..59023d7c7 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -46,7 +46,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -96,7 +96,7 @@ def test_fdb_datacube(self): # plt.colorbar(label="Temperature") # plt.show() - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_select_grid(self): import pygribjump as gj diff --git a/tests/test_fdb_unmap_tree.py b/tests/test_fdb_unmap_tree.py index 9e2e9b90f..91915b57f 100644 --- a/tests/test_fdb_unmap_tree.py +++ b/tests/test_fdb_unmap_tree.py @@ -43,7 +43,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 2fbfcaf28..854f83735 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -47,7 +47,7 @@ def setup_method(self, method): } @pytest.mark.internet - @pytest.mark.fdb + # @pytest.mark.fdb def test_incomplete_fdb_branch(self): import pygribjump as gj @@ -79,7 +79,7 @@ def test_incomplete_fdb_branch(self): assert result.is_root() @pytest.mark.internet - @pytest.mark.fdb + # @pytest.mark.fdb def test_incomplete_fdb_branch_2(self): import pygribjump as gj diff --git a/tests/test_local_grid_cyclic.py b/tests/test_local_grid_cyclic.py index 3f4bfd36c..82c82c3bc 100644 --- a/tests/test_local_grid_cyclic.py +++ b/tests/test_local_grid_cyclic.py @@ -51,7 +51,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -80,7 +80,7 @@ def test_fdb_datacube(self): assert result.leaves[0].flatten()["latitude"] == (-20,) assert result.leaves[0].flatten()["longitude"] == (-20,) - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_2(self): import pygribjump as gj diff --git a/tests/test_local_regular_grid.py b/tests/test_local_regular_grid.py index 3f8f2d402..6b4e01fee 100644 --- a/tests/test_local_regular_grid.py +++ b/tests/test_local_regular_grid.py @@ -50,7 +50,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -79,7 +79,7 @@ def test_fdb_datacube(self): assert result.leaves[0].flatten()["latitude"] == (0,) assert result.leaves[0].flatten()["longitude"] == (0,) - @pytest.mark.fdb + # @pytest.mark.fdb def test_point_outside_local_region(self): import pygribjump as gj @@ -108,7 +108,7 @@ def test_point_outside_local_region(self): assert result.leaves[0].flatten()["latitude"] == (0,) assert result.leaves[0].flatten()["longitude"] == (60,) - @pytest.mark.fdb + # @pytest.mark.fdb def test_point_outside_local_region_2(self): import pygribjump as gj @@ -137,7 +137,7 @@ def test_point_outside_local_region_2(self): assert result.leaves[0].flatten()["latitude"] == (40,) assert result.leaves[0].flatten()["longitude"] == (1,) - @pytest.mark.fdb + # @pytest.mark.fdb def test_point_outside_local_region_3(self): import pygribjump as gj @@ -165,7 +165,7 @@ def test_point_outside_local_region_3(self): assert len(result.leaves) == 1 assert result.is_root() - @pytest.mark.fdb + # @pytest.mark.fdb def test_point_outside_local_region_4(self): import pygribjump as gj @@ -193,7 +193,7 @@ def test_point_outside_local_region_4(self): assert len(result.leaves) == 1 assert result.is_root() - @pytest.mark.fdb + # @pytest.mark.fdb def test_point_outside_local_region_5(self): import pygribjump as gj @@ -221,7 +221,7 @@ def test_point_outside_local_region_5(self): assert len(result.leaves) == 1 assert result.is_root() - @pytest.mark.fdb + # @pytest.mark.fdb def test_point_outside_local_region_6(self): import pygribjump as gj @@ -249,7 +249,7 @@ def test_point_outside_local_region_6(self): assert len(result.leaves) == 1 assert result.is_root() - @pytest.mark.fdb + # @pytest.mark.fdb def test_point_outside_local_region_7(self): import pygribjump as gj @@ -278,7 +278,7 @@ def test_point_outside_local_region_7(self): assert result.leaves[0].flatten()["latitude"] == (-40,) assert result.leaves[0].flatten()["longitude"] == (1,) - @pytest.mark.fdb + # @pytest.mark.fdb def test_point_outside_local_region_8(self): import pygribjump as gj @@ -307,7 +307,7 @@ def test_point_outside_local_region_8(self): assert result.leaves[0].flatten()["latitude"] == (-30,) assert result.leaves[0].flatten()["longitude"] == (-20,) - @pytest.mark.fdb + # @pytest.mark.fdb def test_point_outside_local_region_9(self): import pygribjump as gj diff --git a/tests/test_local_swiss_grid.py b/tests/test_local_swiss_grid.py index 34019e54f..e4b297876 100644 --- a/tests/test_local_swiss_grid.py +++ b/tests/test_local_swiss_grid.py @@ -39,7 +39,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb @pytest.mark.skip("Non-accessible data") def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_multiple_param_fdb.py b/tests/test_multiple_param_fdb.py index 11285af84..d87c122da 100644 --- a/tests/test_multiple_param_fdb.py +++ b/tests/test_multiple_param_fdb.py @@ -42,7 +42,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_override_md5_hash_options.py b/tests/test_override_md5_hash_options.py index 0233290af..55f762bca 100644 --- a/tests/test_override_md5_hash_options.py +++ b/tests/test_override_md5_hash_options.py @@ -49,7 +49,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py index b9fdc4bed..419029bf0 100644 --- a/tests/test_point_nearest.py +++ b/tests/test_point_nearest.py @@ -43,7 +43,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -69,7 +69,7 @@ def test_fdb_datacube(self): result = self.API.retrieve(request) assert len(result.leaves) == 1 - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_true_point(self): import pygribjump as gj @@ -96,7 +96,7 @@ def test_fdb_datacube_true_point(self): # result.pprint() assert len(result.leaves) == 1 - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_true_point_2(self): import pygribjump as gj @@ -123,7 +123,7 @@ def test_fdb_datacube_true_point_2(self): result.pprint() assert len(result.leaves) == 1 - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_true_point_3(self): import pygribjump as gj @@ -152,7 +152,7 @@ def test_fdb_datacube_true_point_3(self): assert result.leaves[0].values == (359.929906542056,) assert result.leaves[0].axis.name == "longitude" - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_true_point_5(self): import pygribjump as gj @@ -181,7 +181,7 @@ def test_fdb_datacube_true_point_5(self): assert result.leaves[0].values == (359.929906542056,) assert result.leaves[0].axis.name == "longitude" - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_true_point_4(self): import pygribjump as gj diff --git a/tests/test_point_union.py b/tests/test_point_union.py index 32a01272b..36b70ca24 100644 --- a/tests/test_point_union.py +++ b/tests/test_point_union.py @@ -43,7 +43,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -81,7 +81,7 @@ def test_fdb_datacube(self): result.pprint() assert len(result.leaves) == 8 - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_surrounding(self): import pygribjump as gj @@ -117,7 +117,7 @@ def test_fdb_datacube_surrounding(self): tot_leaves += len(leaf.result) assert tot_leaves == 9 - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_axis_order(self): import pygribjump as gj diff --git a/tests/test_reduced_ll_grid.py b/tests/test_reduced_ll_grid.py index b0a3535c4..69fe4c1df 100644 --- a/tests/test_reduced_ll_grid.py +++ b/tests/test_reduced_ll_grid.py @@ -50,7 +50,7 @@ def setup_method(self, method): @pytest.mark.skip(reason="wave data grid packing not supported") @pytest.mark.internet - @pytest.mark.fdb + # @pytest.mark.fdb def test_reduced_ll_grid(self): import pygribjump as gj diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index e094b3bca..aac3eb6a7 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -56,7 +56,7 @@ def setup_method(self, method): ], } - @pytest.mark.fdb + # @pytest.mark.fdb @pytest.mark.internet def test_regular_grid(self): import pygribjump as gj @@ -102,7 +102,7 @@ def test_regular_grid(self): tol = 1e-8 leaves = result.leaves for i in range(len(leaves)): - right_pl_results = leaves[i].result[len(leaves[i].values) :] + right_pl_results = leaves[i].result[len(leaves[i].values):] result_tree = right_pl_results[0] cubepath = leaves[i].flatten() lat = cubepath["latitude"][0] diff --git a/tests/test_regular_reduced_grid.py b/tests/test_regular_reduced_grid.py index 08d64fba6..6673cf8db 100644 --- a/tests/test_regular_reduced_grid.py +++ b/tests/test_regular_reduced_grid.py @@ -51,7 +51,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_shapes.py b/tests/test_shapes.py index ac730905a..09182f04e 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -46,7 +46,7 @@ def test_all_cyclic(self): path = result.leaves[0].flatten() assert path["longitude"] == tuple(range(0, 360)) - @pytest.mark.fdb + # @pytest.mark.fdb def test_all_mapper_cyclic(self): import pygribjump as gj diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index dae81cb22..734bb3034 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -43,7 +43,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -73,7 +73,7 @@ def test_fdb_datacube(self): for i in range(len(result.leaves)): assert len(result.leaves[i].result) == 3 - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_select_non_existing_last(self): import pygribjump as gj @@ -103,7 +103,7 @@ def test_fdb_datacube_select_non_existing_last(self): for i in range(len(result.leaves)): assert len(result.leaves[i].result) == 3 - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_select_non_existing_first(self): import pygribjump as gj @@ -133,7 +133,7 @@ def test_fdb_datacube_select_non_existing_first(self): for i in range(len(result.leaves)): assert len(result.leaves[i].result) == 3 - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_select_completely_non_existing(self): import pygribjump as gj @@ -163,7 +163,7 @@ def test_fdb_datacube_select_completely_non_existing(self): for i in range(len(result.leaves)): assert len(result.leaves[i].result) == 0 - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_disk(self): import pygribjump as gj @@ -195,7 +195,7 @@ def test_fdb_datacube_disk(self): assert len(result.leaves[0].values) == 3 assert len(result.leaves[1].values) == 3 - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_disk_2(self): import pygribjump as gj diff --git a/tests/test_slice_date_range_fdb_v2.py b/tests/test_slice_date_range_fdb_v2.py index fcacf6b7c..7a464138c 100644 --- a/tests/test_slice_date_range_fdb_v2.py +++ b/tests/test_slice_date_range_fdb_v2.py @@ -46,7 +46,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb @pytest.mark.skip(reason="gribjump problem") def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_slice_fdb_box.py b/tests/test_slice_fdb_box.py index a21910dfd..95cf1186b 100644 --- a/tests/test_slice_fdb_box.py +++ b/tests/test_slice_fdb_box.py @@ -48,7 +48,7 @@ def setup_method(self, method): # Testing different shapes @pytest.mark.skip(reason="optimisation test") - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_tree_protobuf_encoding.py b/tests/test_tree_protobuf_encoding.py index b177cbe2b..e62512916 100644 --- a/tests/test_tree_protobuf_encoding.py +++ b/tests/test_tree_protobuf_encoding.py @@ -45,7 +45,7 @@ def setup_method(self): "timedelta_ax": grandchild_ax3, } - @pytest.mark.fdb + # @pytest.mark.fdb def test_encoding(self): import pygribjump as gj diff --git a/tests/test_tree_protobuf_encoding_fdb.py b/tests/test_tree_protobuf_encoding_fdb.py index 2e288f0d6..517566116 100644 --- a/tests/test_tree_protobuf_encoding_fdb.py +++ b/tests/test_tree_protobuf_encoding_fdb.py @@ -8,7 +8,7 @@ class TestEncoder: def setup_method(self): pass - @pytest.mark.fdb + # @pytest.mark.fdb def test_encoding(self): import pygribjump as gj diff --git a/tests/test_union_gj.py b/tests/test_union_gj.py index 9182fc360..2db4c4599 100644 --- a/tests/test_union_gj.py +++ b/tests/test_union_gj.py @@ -43,7 +43,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -84,7 +84,7 @@ def test_fdb_datacube(self): assert total_lons == 16 assert total_vals == 16 - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_complete_overlap(self): import pygribjump as gj @@ -125,7 +125,7 @@ def test_fdb_datacube_complete_overlap(self): assert total_lons == 9 assert total_vals == 9 - @pytest.mark.fdb + # @pytest.mark.fdb def test_fdb_datacube_complete_overlap_v2(self): import pygribjump as gj diff --git a/tests/test_union_point_box.py b/tests/test_union_point_box.py index ba280bc02..91a2061f8 100644 --- a/tests/test_union_point_box.py +++ b/tests/test_union_point_box.py @@ -43,7 +43,7 @@ def setup_method(self, method): } # Testing different shapes - @pytest.mark.fdb + # @pytest.mark.fdb @pytest.mark.skip(reason="point and box are not same dimensions") def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_wave_spectra_data.py b/tests/test_wave_spectra_data.py index 6835fee4c..c5dc52dbb 100644 --- a/tests/test_wave_spectra_data.py +++ b/tests/test_wave_spectra_data.py @@ -44,7 +44,7 @@ def setup_method(self, method): } self.slicer = HullSlicer() - @pytest.mark.fdb + # @pytest.mark.fdb def test_healpix_grid(self): import pygribjump as gj From 37788edda7fbcfd89e1fa0093799eef78cbe9d91 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 25 Jun 2025 13:23:46 +0200 Subject: [PATCH 060/105] change fdb test --- tests/test_ecmwf_oper_data_fdb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 5bbffaac9..75b6ea029 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -49,7 +49,8 @@ def test_fdb_datacube(self): request = Request( Select("step", [0]), Select("levtype", ["sfc"]), - Select("date", [pd.Timestamp("20231102T000000")]), + # Select("date", [pd.Timestamp("20231102T000000")]), + Select("date", [pd.Timestamp("20240103T0000")]), Select("domain", ["g"]), Select("expver", ["0001"]), Select("param", ["167"]), From 0ae6633d9854073986682113decc7ac829c78e31 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 3 Jul 2025 10:50:37 +0200 Subject: [PATCH 061/105] fix more fdb tests --- tests/conftest.py | 2 ++ tests/fdb_data/schema | 22 +++++++++++++++++++++- tests/test_shapes.py | 20 +++++++++++++++----- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 07a53d90e..f32ff9e95 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -85,6 +85,8 @@ def downloaded_data_test_files(shared_temp_data_dir): # ("https://example.com/file3.csv", "file3.csv"), ("https://get.ecmwf.int/test-data/polytope/test-data/foo.grib", "foo.grib"), ("https://get.ecmwf.int/test-data/polytope/test-data/t2m_jan_3_v2.grib", "t2m_jan_3_v2.grib"), + ("https://get.ecmwf.int/test-data/polytope/test-data/wave_spectra.grib", "wave_spectra.grib"), + ("https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib", "era5-levels-members.grib"), ] downloaded_paths = [] diff --git a/tests/fdb_data/schema b/tests/fdb_data/schema index b0e7497db..9d40771cd 100644 --- a/tests/fdb_data/schema +++ b/tests/fdb_data/schema @@ -16,6 +16,12 @@ number: Integer; [ type, levtype [ step, levelist?, param ]] ] + +# dcwv/scwv/wave +[ class, expver, stream=dcwv/scwv/wave, date, time, domain + [ type, levtype + [ step, param, frequency?, direction? ]]] + # enfo [ class, expver, stream=enfo/efov/eefo, date, time, domain [ type, levtype @@ -26,4 +32,18 @@ number: Integer; [ class, expver, stream=waef/weov/weef, date, time, domain [ type, levtype [ step, number?, param, frequency?, direction? ]] -] \ No newline at end of file +] + +# enda +[ class, expver, stream=enda, date, time, domain + + [ type=ef/em/es/ses, levtype + [ step, number?, levelist?, param, channel? ]] + + [ type=ssd + [ step, number, param, ident, instrument, channel ]] + + + [ type, levtype + [ step, number?, levelist?, param ]] +] diff --git a/tests/test_shapes.py b/tests/test_shapes.py index 09182f04e..92ee5fc33 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -66,25 +66,35 @@ def test_all_mapper_cyclic(self): "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"], - "md5_hash": "5ea6378bf5e2904f565ef7221da63a09", + # "md5_hash": "5ea6378bf5e2904f565ef7221da63a09", } ], }, {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, ], - "pre_path": {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"}, + # "pre_path": {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"}, + "pre_path": {"class": "od", "expver": "0001", "levtype": "sfc", "type": "fc", "stream": "oper"}, } self.fdbdatacube = gj.GribJump() self.slicer = HullSlicer() request = Request( - Select("step", [11]), + # Select("step", [11]), + # Select("levtype", ["sfc"]), + # Select("date", [pd.Timestamp("20230710T120000")]), + # Select("domain", ["g"]), + # Select("expver", ["0001"]), + # Select("param", ["151130"]), + # Select("class", ["od"]), + # Select("stream", ["oper"]), + # Select("type", ["fc"]), + Select("step", [0]), Select("levtype", ["sfc"]), - Select("date", [pd.Timestamp("20230710T120000")]), + Select("date", [pd.Timestamp("20240103T0000")]), Select("domain", ["g"]), Select("expver", ["0001"]), - Select("param", ["151130"]), + Select("param", ["167"]), Select("class", ["od"]), Select("stream", ["oper"]), Select("type", ["fc"]), From 420ebf62eee3cefb13ad84c6033b827834bfb953 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 3 Jul 2025 13:56:36 +0200 Subject: [PATCH 062/105] fix more fdb tests --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index f32ff9e95..fa65d8494 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -84,6 +84,7 @@ def downloaded_data_test_files(shared_temp_data_dir): # ("https://example.com/file2.csv", "file2.csv"), # ("https://example.com/file3.csv", "file3.csv"), ("https://get.ecmwf.int/test-data/polytope/test-data/foo.grib", "foo.grib"), + ("https://get.ecmwf.int/test-data/polytope/test-data/local.grib", "local.grib"), ("https://get.ecmwf.int/test-data/polytope/test-data/t2m_jan_3_v2.grib", "t2m_jan_3_v2.grib"), ("https://get.ecmwf.int/test-data/polytope/test-data/wave_spectra.grib", "wave_spectra.grib"), ("https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib", "era5-levels-members.grib"), From af67b6d13e1aebc4678ec6c6280e98796b69e3a7 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 3 Jul 2025 14:04:44 +0200 Subject: [PATCH 063/105] fix more fdb tests --- tests/test_point_nearest.py | 45 +++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py index 419029bf0..320a2737c 100644 --- a/tests/test_point_nearest.py +++ b/tests/test_point_nearest.py @@ -128,15 +128,24 @@ def test_fdb_datacube_true_point_3(self): import pygribjump as gj request = Request( - Select("step", [21]), + # Select("step", [21]), + # Select("levtype", ["sfc"]), + # Select("date", [pd.Timestamp("20231102T000000")]), + # Select("domain", ["g"]), + # Select("expver", ["0001"]), + # Select("param", ["167"]), + # Select("class", ["od"]), + # Select("stream", ["oper"]), + # Select("type", ["fc"]), + Select("step", [0]), Select("levtype", ["sfc"]), - Select("date", [pd.Timestamp("20231102T000000")]), + Select("date", [pd.Timestamp("20230625T120000")]), Select("domain", ["g"]), Select("expver", ["0001"]), Select("param", ["167"]), Select("class", ["od"]), Select("stream", ["oper"]), - Select("type", ["fc"]), + Select("type", ["an"]), Point(["latitude", "longitude"], [[0.035149384216, -0.01]], method="nearest"), ) self.fdbdatacube = gj.GribJump() @@ -157,15 +166,24 @@ def test_fdb_datacube_true_point_5(self): import pygribjump as gj request = Request( - Select("step", [21]), + # Select("step", [21]), + # Select("levtype", ["sfc"]), + # Select("date", [pd.Timestamp("20231102T000000")]), + # Select("domain", ["g"]), + # Select("expver", ["0001"]), + # Select("param", ["167"]), + # Select("class", ["od"]), + # Select("stream", ["oper"]), + # Select("type", ["fc"]), + Select("step", [0]), Select("levtype", ["sfc"]), - Select("date", [pd.Timestamp("20231102T000000")]), + Select("date", [pd.Timestamp("20230625T120000")]), Select("domain", ["g"]), Select("expver", ["0001"]), Select("param", ["167"]), Select("class", ["od"]), Select("stream", ["oper"]), - Select("type", ["fc"]), + Select("type", ["an"]), Point(["latitude", "longitude"], [[0.035149384216, 360 - 0.01]], method="nearest"), ) self.fdbdatacube = gj.GribJump() @@ -186,15 +204,24 @@ def test_fdb_datacube_true_point_4(self): import pygribjump as gj request = Request( - Select("step", [21]), + # Select("step", [21]), + # Select("levtype", ["sfc"]), + # Select("date", [pd.Timestamp("20231102T000000")]), + # Select("domain", ["g"]), + # Select("expver", ["0001"]), + # Select("param", ["167"]), + # Select("class", ["od"]), + # Select("stream", ["oper"]), + # Select("type", ["fc"]), + Select("step", [0]), Select("levtype", ["sfc"]), - Select("date", [pd.Timestamp("20231102T000000")]), + Select("date", [pd.Timestamp("20230625T120000")]), Select("domain", ["g"]), Select("expver", ["0001"]), Select("param", ["167"]), Select("class", ["od"]), Select("stream", ["oper"]), - Select("type", ["fc"]), + Select("type", ["an"]), Point(["latitude", "longitude"], [[0.035149384216, 359.97]], method="nearest"), ) self.fdbdatacube = gj.GribJump() From 9885d5d80a44c4323dd4f368f56e4088c0c4431b Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 3 Jul 2025 15:56:13 +0200 Subject: [PATCH 064/105] fix more tests --- tests/conftest.py | 2 ++ tests/test_regular_reduced_grid.py | 1 + 2 files changed, 3 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index fa65d8494..8290143a4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -85,6 +85,8 @@ def downloaded_data_test_files(shared_temp_data_dir): # ("https://example.com/file3.csv", "file3.csv"), ("https://get.ecmwf.int/test-data/polytope/test-data/foo.grib", "foo.grib"), ("https://get.ecmwf.int/test-data/polytope/test-data/local.grib", "local.grib"), + ("https://get.ecmwf.int/test-data/polytope/test-data/aifs_data_from_fdb.grib", "aifs_data_from_fdb.grib"), + ("https://get.ecmwf.int/test-data/polytope/test-data/wind_gust_and_t2m.grib", "wind_gust_and_t2m.grib"), ("https://get.ecmwf.int/test-data/polytope/test-data/t2m_jan_3_v2.grib", "t2m_jan_3_v2.grib"), ("https://get.ecmwf.int/test-data/polytope/test-data/wave_spectra.grib", "wave_spectra.grib"), ("https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib", "era5-levels-members.grib"), diff --git a/tests/test_regular_reduced_grid.py b/tests/test_regular_reduced_grid.py index 6673cf8db..696753213 100644 --- a/tests/test_regular_reduced_grid.py +++ b/tests/test_regular_reduced_grid.py @@ -1,6 +1,7 @@ import pandas as pd import pytest from eccodes import codes_grib_find_nearest, codes_grib_new_from_file +from helper_functions import download_test_data from polytope_feature.engine.hullslicer import HullSlicer from polytope_feature.polytope import Polytope, Request From d8e751c04c4dd84f0b46674bdd08f564849e09d4 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 7 Jul 2025 09:55:32 +0200 Subject: [PATCH 065/105] make tests work --- polytope_feature/datacube/backends/fdb.py | 1 + tests/test_ecmwf_oper_data_fdb.py | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/polytope_feature/datacube/backends/fdb.py b/polytope_feature/datacube/backends/fdb.py index b69138c8d..93996a344 100644 --- a/polytope_feature/datacube/backends/fdb.py +++ b/polytope_feature/datacube/backends/fdb.py @@ -48,6 +48,7 @@ def __init__( logging.info("Axes returned from GribJump are: " + str(self.fdb_coordinates)) self.fdb_coordinates["values"] = [] + print(self.fdb_coordinates) for name, values in self.fdb_coordinates.items(): values.sort() options = None diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 75b6ea029..2118715a8 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -71,7 +71,8 @@ def test_fdb_datacube(self): assert len(result.leaves) == 3 assert len(result.leaves[0].result) == 3 - def test_fdb_datacube_point(self, downloaded_data_test_files, fdb_store_operational_setup): + # def test_fdb_datacube_point(self, downloaded_data_test_files, fdb_store_operational_setup): + def test_fdb_datacube_point(self): import pygribjump as gj request = Request( @@ -100,7 +101,8 @@ def test_fdb_datacube_point(self, downloaded_data_test_files, fdb_store_operatio assert len(result.leaves[0].result) == 4 # @pytest.mark.fdb - def test_fdb_datacube_point_v2(self, downloaded_data_test_files, fdb_store_operational_setup): + # def test_fdb_datacube_point_v2(self, downloaded_data_test_files, fdb_store_operational_setup): + def test_fdb_datacube_point_v2(self): import pygribjump as gj request = Request( @@ -128,7 +130,8 @@ def test_fdb_datacube_point_v2(self, downloaded_data_test_files, fdb_store_opera assert len(result.leaves[0].result) == 4 # @pytest.mark.fdb - def test_fdb_datacube_point_step_not_compressed(self, downloaded_data_test_files, fdb_store_operational_setup): + # def test_fdb_datacube_point_step_not_compressed(self, downloaded_data_test_files, fdb_store_operational_setup): + def test_fdb_datacube_point_step_not_compressed(self): import pygribjump as gj self.options = { From 25a2360235ea86c0f1a9faab762ae002366267b1 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 7 Jul 2025 10:09:30 +0200 Subject: [PATCH 066/105] WIP: fix tests in CI --- tests/test_ecmwf_oper_data_fdb.py | 9 +++------ tests/test_regular_grid.py | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 2118715a8..75b6ea029 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -71,8 +71,7 @@ def test_fdb_datacube(self): assert len(result.leaves) == 3 assert len(result.leaves[0].result) == 3 - # def test_fdb_datacube_point(self, downloaded_data_test_files, fdb_store_operational_setup): - def test_fdb_datacube_point(self): + def test_fdb_datacube_point(self, downloaded_data_test_files, fdb_store_operational_setup): import pygribjump as gj request = Request( @@ -101,8 +100,7 @@ def test_fdb_datacube_point(self): assert len(result.leaves[0].result) == 4 # @pytest.mark.fdb - # def test_fdb_datacube_point_v2(self, downloaded_data_test_files, fdb_store_operational_setup): - def test_fdb_datacube_point_v2(self): + def test_fdb_datacube_point_v2(self, downloaded_data_test_files, fdb_store_operational_setup): import pygribjump as gj request = Request( @@ -130,8 +128,7 @@ def test_fdb_datacube_point_v2(self): assert len(result.leaves[0].result) == 4 # @pytest.mark.fdb - # def test_fdb_datacube_point_step_not_compressed(self, downloaded_data_test_files, fdb_store_operational_setup): - def test_fdb_datacube_point_step_not_compressed(self): + def test_fdb_datacube_point_step_not_compressed(self, downloaded_data_test_files, fdb_store_operational_setup): import pygribjump as gj self.options = { diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index aac3eb6a7..026496f18 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -102,7 +102,7 @@ def test_regular_grid(self): tol = 1e-8 leaves = result.leaves for i in range(len(leaves)): - right_pl_results = leaves[i].result[len(leaves[i].values):] + right_pl_results = leaves[i].result[len(leaves[i].values) :] result_tree = right_pl_results[0] cubepath = leaves[i].flatten() lat = cubepath["latitude"][0] From 5c9f2ab06d23e3654bd83d339f744b4cf92addc5 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 7 Jul 2025 10:15:46 +0200 Subject: [PATCH 067/105] WIP: fix tests in CI --- tests/test_bad_request_error.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bad_request_error.py b/tests/test_bad_request_error.py index 3224e4f43..277ceead2 100644 --- a/tests/test_bad_request_error.py +++ b/tests/test_bad_request_error.py @@ -52,7 +52,7 @@ def setup_method(self, method): # Testing different shapes # @pytest.mark.fdb - def test_fdb_datacube(self): + def test_fdb_datacube(self, downloaded_data_test_files, fdb_store_operational_setup): import pygribjump as gj with pytest.raises(BadRequestError): From c2c9df60bb6189e04df73d99eb91bfcd6290872a Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 7 Jul 2025 12:04:14 +0200 Subject: [PATCH 068/105] WIP: fix fdb tests --- polytope_feature/datacube/backends/fdb.py | 1 - tests/conftest.py | 244 +++++++++++++++------- tests/test_bad_request_error.py | 2 +- tests/test_cyclic_nearest.py | 6 +- tests/test_ecmwf_oper_data_fdb.py | 7 +- tests/test_point_nearest.py | 18 -- tests/test_regular_reduced_grid.py | 4 +- 7 files changed, 183 insertions(+), 99 deletions(-) diff --git a/polytope_feature/datacube/backends/fdb.py b/polytope_feature/datacube/backends/fdb.py index 93996a344..b69138c8d 100644 --- a/polytope_feature/datacube/backends/fdb.py +++ b/polytope_feature/datacube/backends/fdb.py @@ -48,7 +48,6 @@ def __init__( logging.info("Axes returned from GribJump are: " + str(self.fdb_coordinates)) self.fdb_coordinates["values"] = [] - print(self.fdb_coordinates) for name, values in self.fdb_coordinates.items(): values.sort() options = None diff --git a/tests/conftest.py b/tests/conftest.py index 8290143a4..de2134b40 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,125 @@ +# import os +# import pathlib +# import shutil + +# import pyfdb +# import pytest +# import requests +# import yaml + + +# class HTTPError(Exception): +# def __init__(self, status_code, message): +# self.status_code = status_code +# self.message = message +# super().__init__(f"HTTPError {status_code}: {message}") + + +# @pytest.fixture +# def load_fdb_data_from_nexus(): +# # TODO: load all the relevant data files we want in the fdb from nexus +# pass + + +# @pytest.fixture(scope="function") +# def fdb_path(request) -> pathlib.Path: +# """ +# Provides path to test data at '/tests/fdb_data' +# """ +# path = request.config.rootpath / "tests" / "fdb_data" +# assert path.exists() +# return path + + +# @pytest.fixture(scope="function") +# def fdb_store_operational_setup(fdb_path, tmp_path, downloaded_data_test_files) -> pathlib.Path: +# # TODO: once we have the test data in the fdb_data folder, and we have a path to it, load it all in an fdb + +# # Set up FDB config and schemas +# db_store_path = tmp_path / "db_store" +# db_store_path.mkdir(exist_ok=True) +# schema_path = tmp_path / "schema" +# config = dict( +# type="local", +# engine="toc", +# schema=str(schema_path), +# spaces=[ +# dict( +# handler="Default", +# roots=[ +# {"path": str(db_store_path)}, +# ], +# ) +# ], +# ) +# config_path = tmp_path / "config.yaml" +# config_path.write_text(yaml.dump(config)) +# shutil.copy(fdb_path / "schema", schema_path) +# os.environ["FDB5_CONFIG_FILE"] = str(config_path) + +# # TODO: download data from the internet +# # grib_files = data_path.glob("*.grib") +# # grib_files = [] +# # for file_name in downloaded_data_test_files: +# # # grib_files = [data_path / "synth11.grib"] +# # grib_files.append(file_name) +# fdb = pyfdb.FDB() +# # for f in grib_files: +# for f in downloaded_data_test_files: +# fdb.archive(f.read_bytes()) +# return tmp_path + + +# @pytest.fixture(scope="function") +# def shared_temp_data_dir(tmp_path_factory): +# # This creates a unique temp dir for the whole test session +# temp_dir = tmp_path_factory.mktemp("shared_fdb_data") +# return temp_dir + + +# @pytest.fixture(scope="function") +# def downloaded_data_test_files(shared_temp_data_dir): +# files_to_download = [ +# ("https://get.ecmwf.int/test-data/polytope/test-data/foo.grib", "foo.grib"), +# ("https://get.ecmwf.int/test-data/polytope/test-data/local.grib", "local.grib"), +# ("https://get.ecmwf.int/test-data/polytope/test-data/aifs_data_from_fdb.grib", "aifs_data_from_fdb.grib"), +# ("https://get.ecmwf.int/test-data/polytope/test-data/wind_gust_and_t2m.grib", "wind_gust_and_t2m.grib"), +# ("https://get.ecmwf.int/test-data/polytope/test-data/t2m_jan_3_v2.grib", "t2m_jan_3_v2.grib"), +# ("https://get.ecmwf.int/test-data/polytope/test-data/wave_spectra.grib", "wave_spectra.grib"), +# ("https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib", "era5-levels-members.grib"), +# ] + +# downloaded_paths = [] + +# for url, filename in files_to_download: +# path = shared_temp_data_dir / filename +# if not path.exists(): +# response = requests.get(url) +# if response.status_code != 200: +# raise HTTPError(f"HTTP {response.status_code} - Failed to download data.") +# # path.write_bytes(response.content) +# with open(path, "wb") as f: +# f.write(response.content) +# downloaded_paths.append(path) + +# return downloaded_paths + + +# # @pytest.fixture(scope="session") +# # def downloaded_test_file(shared_temp_data_dir, nexus_url, filename): + +# # local_file_path = shared_temp_data_dir / filename + +# # if not os.path.exists(local_file_path): +# # response = requests.get(nexus_url) +# # if response.status_code != 200: +# # raise HTTPError(f"HTTP {response.status_code} - Failed to download data.") +# # with open(local_file_path, "wb") as f: +# # f.write(response.content) + +# # yield local_file_path + + import os import pathlib import shutil @@ -15,74 +137,21 @@ def __init__(self, status_code, message): super().__init__(f"HTTPError {status_code}: {message}") -@pytest.fixture -def load_fdb_data_from_nexus(): - # TODO: load all the relevant data files we want in the fdb from nexus - pass - - -@pytest.fixture(scope="function") -def fdb_path(request) -> pathlib.Path: +@pytest.fixture(scope="session") +def shared_temp_data_dir(tmp_path_factory): """ - Provides path to test data at '/tests/fdb_data' + Provides a unique temp directory for the entire test session. """ - path = request.config.rootpath / "tests" / "fdb_data" - assert path.exists() - return path - - -@pytest.fixture(scope="function") -def fdb_store_operational_setup(fdb_path, tmp_path, downloaded_data_test_files) -> pathlib.Path: - # TODO: once we have the test data in the fdb_data folder, and we have a path to it, load it all in an fdb - - # Set up FDB config and schemas - db_store_path = tmp_path / "db_store" - db_store_path.mkdir(exist_ok=True) - schema_path = tmp_path / "schema" - config = dict( - type="local", - engine="toc", - schema=str(schema_path), - spaces=[ - dict( - handler="Default", - roots=[ - {"path": str(db_store_path)}, - ], - ) - ], - ) - config_path = tmp_path / "config.yaml" - config_path.write_text(yaml.dump(config)) - shutil.copy(fdb_path / "schema", schema_path) - os.environ["FDB5_CONFIG_FILE"] = str(config_path) - - # TODO: download data from the internet - # grib_files = data_path.glob("*.grib") - # grib_files = [] - # for file_name in downloaded_data_test_files: - # # grib_files = [data_path / "synth11.grib"] - # grib_files.append(file_name) - fdb = pyfdb.FDB() - # for f in grib_files: - for f in downloaded_data_test_files: - fdb.archive(f.read_bytes()) - return tmp_path - - -@pytest.fixture(scope="function") -def shared_temp_data_dir(tmp_path_factory): - # This creates a unique temp dir for the whole test session temp_dir = tmp_path_factory.mktemp("shared_fdb_data") return temp_dir -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def downloaded_data_test_files(shared_temp_data_dir): + """ + Downloads all required GRIB test files once per session. + """ files_to_download = [ - # ("https://example.com/file1.csv", "file1.csv"), - # ("https://example.com/file2.csv", "file2.csv"), - # ("https://example.com/file3.csv", "file3.csv"), ("https://get.ecmwf.int/test-data/polytope/test-data/foo.grib", "foo.grib"), ("https://get.ecmwf.int/test-data/polytope/test-data/local.grib", "local.grib"), ("https://get.ecmwf.int/test-data/polytope/test-data/aifs_data_from_fdb.grib", "aifs_data_from_fdb.grib"), @@ -99,8 +168,7 @@ def downloaded_data_test_files(shared_temp_data_dir): if not path.exists(): response = requests.get(url) if response.status_code != 200: - raise HTTPError(f"HTTP {response.status_code} - Failed to download data.") - # path.write_bytes(response.content) + raise HTTPError(response.status_code, f"Failed to download {url}") with open(path, "wb") as f: f.write(response.content) downloaded_paths.append(path) @@ -108,16 +176,48 @@ def downloaded_data_test_files(shared_temp_data_dir): return downloaded_paths -# @pytest.fixture(scope="session") -# def downloaded_test_file(shared_temp_data_dir, nexus_url, filename): +@pytest.fixture(scope="session") +def fdb_path(request) -> pathlib.Path: + """ + Provides path to test data at '/tests/fdb_data'. + """ + path = request.config.rootpath / "tests" / "fdb_data" + assert path.exists(), f"Expected path {path} to exist." + return path + + +@pytest.fixture(scope="session", autouse=True) +def fdb_store_operational_setup(fdb_path, tmp_path_factory, downloaded_data_test_files) -> pathlib.Path: + """ + Creates an operational FDB store for tests, loading downloaded test files. + """ + tmp_dir = tmp_path_factory.mktemp("shared_path") + db_store_path = tmp_dir / "db_store" + db_store_path.mkdir(exist_ok=True) + + schema_path = tmp_dir / "schema" + config = dict( + type="local", + engine="toc", + schema=str(schema_path), + spaces=[ + dict( + handler="Default", + roots=[ + {"path": str(db_store_path)}, + ], + ) + ], + ) + + config_path = tmp_dir / "config.yaml" + config_path.write_text(yaml.dump(config)) + shutil.copy(fdb_path / "schema", schema_path) -# local_file_path = shared_temp_data_dir / filename + os.environ["FDB5_CONFIG_FILE"] = str(config_path) -# if not os.path.exists(local_file_path): -# response = requests.get(nexus_url) -# if response.status_code != 200: -# raise HTTPError(f"HTTP {response.status_code} - Failed to download data.") -# with open(local_file_path, "wb") as f: -# f.write(response.content) + fdb = pyfdb.FDB() + for f in downloaded_data_test_files: + fdb.archive(f.read_bytes()) -# yield local_file_path + return tmp_dir diff --git a/tests/test_bad_request_error.py b/tests/test_bad_request_error.py index 277ceead2..3224e4f43 100644 --- a/tests/test_bad_request_error.py +++ b/tests/test_bad_request_error.py @@ -52,7 +52,7 @@ def setup_method(self, method): # Testing different shapes # @pytest.mark.fdb - def test_fdb_datacube(self, downloaded_data_test_files, fdb_store_operational_setup): + def test_fdb_datacube(self): import pygribjump as gj with pytest.raises(BadRequestError): diff --git a/tests/test_cyclic_nearest.py b/tests/test_cyclic_nearest.py index 544a88b41..59318e0ed 100644 --- a/tests/test_cyclic_nearest.py +++ b/tests/test_cyclic_nearest.py @@ -75,7 +75,8 @@ def test_regular_grid(self): request = Request( Select("step", [0]), Select("levtype", ["sfc"]), - Select("date", [pd.Timestamp("20231102T000000")]), + # Select("date", [pd.Timestamp("20231102T000000")]), + Select("date", [pd.Timestamp("20240103T0000")]), Select("domain", ["g"]), Select("expver", ["0001"]), Select("param", ["167"]), @@ -99,7 +100,8 @@ def test_regular_grid(self): request = Request( Select("step", [0]), Select("levtype", ["sfc"]), - Select("date", [pd.Timestamp("20231102T000000")]), + # Select("date", [pd.Timestamp("20231102T000000")]), + Select("date", [pd.Timestamp("20240103T0000")]), Select("domain", ["g"]), Select("expver", ["0001"]), Select("param", ["167"]), diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index 75b6ea029..d3b06d436 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -49,7 +49,6 @@ def test_fdb_datacube(self): request = Request( Select("step", [0]), Select("levtype", ["sfc"]), - # Select("date", [pd.Timestamp("20231102T000000")]), Select("date", [pd.Timestamp("20240103T0000")]), Select("domain", ["g"]), Select("expver", ["0001"]), @@ -71,7 +70,7 @@ def test_fdb_datacube(self): assert len(result.leaves) == 3 assert len(result.leaves[0].result) == 3 - def test_fdb_datacube_point(self, downloaded_data_test_files, fdb_store_operational_setup): + def test_fdb_datacube_point(self): import pygribjump as gj request = Request( @@ -100,7 +99,7 @@ def test_fdb_datacube_point(self, downloaded_data_test_files, fdb_store_operatio assert len(result.leaves[0].result) == 4 # @pytest.mark.fdb - def test_fdb_datacube_point_v2(self, downloaded_data_test_files, fdb_store_operational_setup): + def test_fdb_datacube_point_v2(self): import pygribjump as gj request = Request( @@ -128,7 +127,7 @@ def test_fdb_datacube_point_v2(self, downloaded_data_test_files, fdb_store_opera assert len(result.leaves[0].result) == 4 # @pytest.mark.fdb - def test_fdb_datacube_point_step_not_compressed(self, downloaded_data_test_files, fdb_store_operational_setup): + def test_fdb_datacube_point_step_not_compressed(self): import pygribjump as gj self.options = { diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py index 320a2737c..050d4225f 100644 --- a/tests/test_point_nearest.py +++ b/tests/test_point_nearest.py @@ -166,15 +166,6 @@ def test_fdb_datacube_true_point_5(self): import pygribjump as gj request = Request( - # Select("step", [21]), - # Select("levtype", ["sfc"]), - # Select("date", [pd.Timestamp("20231102T000000")]), - # Select("domain", ["g"]), - # Select("expver", ["0001"]), - # Select("param", ["167"]), - # Select("class", ["od"]), - # Select("stream", ["oper"]), - # Select("type", ["fc"]), Select("step", [0]), Select("levtype", ["sfc"]), Select("date", [pd.Timestamp("20230625T120000")]), @@ -204,15 +195,6 @@ def test_fdb_datacube_true_point_4(self): import pygribjump as gj request = Request( - # Select("step", [21]), - # Select("levtype", ["sfc"]), - # Select("date", [pd.Timestamp("20231102T000000")]), - # Select("domain", ["g"]), - # Select("expver", ["0001"]), - # Select("param", ["167"]), - # Select("class", ["od"]), - # Select("stream", ["oper"]), - # Select("type", ["fc"]), Select("step", [0]), Select("levtype", ["sfc"]), Select("date", [pd.Timestamp("20230625T120000")]), diff --git a/tests/test_regular_reduced_grid.py b/tests/test_regular_reduced_grid.py index 696753213..d4ec6119a 100644 --- a/tests/test_regular_reduced_grid.py +++ b/tests/test_regular_reduced_grid.py @@ -59,7 +59,9 @@ def test_fdb_datacube(self): request = Request( Select("step", [0]), Select("levtype", ["sfc"]), - Select("date", [pd.Timestamp("20231102T000000")]), + # Select("date", [pd.Timestamp("20231102T000000")]), + Select("date", [pd.Timestamp("20240103T0000")]), + # Select("date", [pd.Timestamp("20240129T000000")]), Select("domain", ["g"]), Select("expver", ["0001"]), Select("param", ["167"]), From 9015cb35acf9fb1c1508c0a3b4255b874dbf797b Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 7 Jul 2025 13:05:28 +0200 Subject: [PATCH 069/105] WIP: fix tests --- tests/test_regular_reduced_grid.py | 9 +++++++-- tests/test_wave_spectra_data.py | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/test_regular_reduced_grid.py b/tests/test_regular_reduced_grid.py index d4ec6119a..3caee691e 100644 --- a/tests/test_regular_reduced_grid.py +++ b/tests/test_regular_reduced_grid.py @@ -53,7 +53,7 @@ def setup_method(self, method): # Testing different shapes # @pytest.mark.fdb - def test_fdb_datacube(self): + def test_fdb_datacube(self, downloaded_data_test_files): import pygribjump as gj request = Request( @@ -90,7 +90,12 @@ def test_fdb_datacube(self): eccodes_lats = [] eccodes_lons = [] tol = 1e-12 - f = open("tests/data/aifs_data_from_fdb.grib", "rb") + aifs_file = "" + for f in downloaded_data_test_files: + if "aifs_data_from_fdb.grib" in f.name: + aifs_file = f + # f = open("tests/data/aifs_data_from_fdb.grib", "rb") + f = open(aifs_file, "rb") messages = [] message = codes_grib_new_from_file(f) messages.append(message) diff --git a/tests/test_wave_spectra_data.py b/tests/test_wave_spectra_data.py index c5dc52dbb..9f655f616 100644 --- a/tests/test_wave_spectra_data.py +++ b/tests/test_wave_spectra_data.py @@ -45,7 +45,7 @@ def setup_method(self, method): self.slicer = HullSlicer() # @pytest.mark.fdb - def test_healpix_grid(self): + def test_healpix_grid(self, downloaded_data_test_files): import pygribjump as gj self.fdb_datacube = gj.GribJump() @@ -80,6 +80,10 @@ def test_healpix_grid(self): eccodes_lats = [] tol = 1e-8 leaves = result.leaves + wave_file = "" + for f in downloaded_data_test_files: + if "wave_spectra.grib" in f.name: + wave_file = f for i, leaf in enumerate(leaves): cubepath = leaf.flatten() lat = cubepath["latitude"][0] @@ -87,7 +91,8 @@ def test_healpix_grid(self): for j, lon in enumerate(new_lons): lats.append(lat) lons.append(lon) - nearest_points = find_nearest_latlon("./tests/data/wave_spectra.grib", lat, lon) + # nearest_points = find_nearest_latlon("./tests/data/wave_spectra.grib", lat, lon) + nearest_points = find_nearest_latlon(wave_file, lat, lon) eccodes_lat = nearest_points[0][0]["lat"] eccodes_lon = nearest_points[0][0]["lon"] assert eccodes_lat - tol <= lat From f29ec9ba78b8bf4cb9565f284485f01c7a0e0f00 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 7 Jul 2025 13:46:14 +0200 Subject: [PATCH 070/105] clean up --- tests/conftest.py | 122 ----------------------- tests/requirements_test.txt | 3 - tests/test_bad_request_error.py | 6 +- tests/test_cyclic_nearest.py | 4 +- tests/test_cyclic_snapping.py | 2 - tests/test_date_time_unmerged.py | 5 +- tests/test_ecmwf_oper_data_fdb.py | 7 +- tests/test_engine_slicer.py | 6 -- tests/test_fdb_datacube.py | 5 +- tests/test_fdb_unmap_tree.py | 2 +- tests/test_incomplete_tree_fdb.py | 4 +- tests/test_local_grid_cyclic.py | 5 +- tests/test_local_regular_grid.py | 21 ++-- tests/test_local_swiss_grid.py | 3 +- tests/test_multiple_param_fdb.py | 3 +- tests/test_override_md5_hash_options.py | 3 +- tests/test_point_nearest.py | 23 ++--- tests/test_point_shape.py | 1 - tests/test_point_union.py | 7 +- tests/test_reduced_ll_grid.py | 2 +- tests/test_regular_grid.py | 4 +- tests/test_regular_reduced_grid.py | 6 +- tests/test_request_tree.py | 63 ------------ tests/test_shapes.py | 13 +-- tests/test_slice_date_range_fdb.py | 13 ++- tests/test_slice_date_range_fdb_v2.py | 3 +- tests/test_slice_fdb_box.py | 3 +- tests/test_tree_protobuf_encoding.py | 2 +- tests/test_tree_protobuf_encoding_fdb.py | 3 +- tests/test_union_gj.py | 7 +- tests/test_union_point_box.py | 3 +- tests/test_wave_spectra_data.py | 3 +- 32 files changed, 55 insertions(+), 302 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index de2134b40..4d52f0d3d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,125 +1,3 @@ -# import os -# import pathlib -# import shutil - -# import pyfdb -# import pytest -# import requests -# import yaml - - -# class HTTPError(Exception): -# def __init__(self, status_code, message): -# self.status_code = status_code -# self.message = message -# super().__init__(f"HTTPError {status_code}: {message}") - - -# @pytest.fixture -# def load_fdb_data_from_nexus(): -# # TODO: load all the relevant data files we want in the fdb from nexus -# pass - - -# @pytest.fixture(scope="function") -# def fdb_path(request) -> pathlib.Path: -# """ -# Provides path to test data at '/tests/fdb_data' -# """ -# path = request.config.rootpath / "tests" / "fdb_data" -# assert path.exists() -# return path - - -# @pytest.fixture(scope="function") -# def fdb_store_operational_setup(fdb_path, tmp_path, downloaded_data_test_files) -> pathlib.Path: -# # TODO: once we have the test data in the fdb_data folder, and we have a path to it, load it all in an fdb - -# # Set up FDB config and schemas -# db_store_path = tmp_path / "db_store" -# db_store_path.mkdir(exist_ok=True) -# schema_path = tmp_path / "schema" -# config = dict( -# type="local", -# engine="toc", -# schema=str(schema_path), -# spaces=[ -# dict( -# handler="Default", -# roots=[ -# {"path": str(db_store_path)}, -# ], -# ) -# ], -# ) -# config_path = tmp_path / "config.yaml" -# config_path.write_text(yaml.dump(config)) -# shutil.copy(fdb_path / "schema", schema_path) -# os.environ["FDB5_CONFIG_FILE"] = str(config_path) - -# # TODO: download data from the internet -# # grib_files = data_path.glob("*.grib") -# # grib_files = [] -# # for file_name in downloaded_data_test_files: -# # # grib_files = [data_path / "synth11.grib"] -# # grib_files.append(file_name) -# fdb = pyfdb.FDB() -# # for f in grib_files: -# for f in downloaded_data_test_files: -# fdb.archive(f.read_bytes()) -# return tmp_path - - -# @pytest.fixture(scope="function") -# def shared_temp_data_dir(tmp_path_factory): -# # This creates a unique temp dir for the whole test session -# temp_dir = tmp_path_factory.mktemp("shared_fdb_data") -# return temp_dir - - -# @pytest.fixture(scope="function") -# def downloaded_data_test_files(shared_temp_data_dir): -# files_to_download = [ -# ("https://get.ecmwf.int/test-data/polytope/test-data/foo.grib", "foo.grib"), -# ("https://get.ecmwf.int/test-data/polytope/test-data/local.grib", "local.grib"), -# ("https://get.ecmwf.int/test-data/polytope/test-data/aifs_data_from_fdb.grib", "aifs_data_from_fdb.grib"), -# ("https://get.ecmwf.int/test-data/polytope/test-data/wind_gust_and_t2m.grib", "wind_gust_and_t2m.grib"), -# ("https://get.ecmwf.int/test-data/polytope/test-data/t2m_jan_3_v2.grib", "t2m_jan_3_v2.grib"), -# ("https://get.ecmwf.int/test-data/polytope/test-data/wave_spectra.grib", "wave_spectra.grib"), -# ("https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib", "era5-levels-members.grib"), -# ] - -# downloaded_paths = [] - -# for url, filename in files_to_download: -# path = shared_temp_data_dir / filename -# if not path.exists(): -# response = requests.get(url) -# if response.status_code != 200: -# raise HTTPError(f"HTTP {response.status_code} - Failed to download data.") -# # path.write_bytes(response.content) -# with open(path, "wb") as f: -# f.write(response.content) -# downloaded_paths.append(path) - -# return downloaded_paths - - -# # @pytest.fixture(scope="session") -# # def downloaded_test_file(shared_temp_data_dir, nexus_url, filename): - -# # local_file_path = shared_temp_data_dir / filename - -# # if not os.path.exists(local_file_path): -# # response = requests.get(nexus_url) -# # if response.status_code != 200: -# # raise HTTPError(f"HTTP {response.status_code} - Failed to download data.") -# # with open(local_file_path, "wb") as f: -# # f.write(response.content) - -# # yield local_file_path - - import os import pathlib import shutil diff --git a/tests/requirements_test.txt b/tests/requirements_test.txt index 9942b824f..f3408d8ca 100644 --- a/tests/requirements_test.txt +++ b/tests/requirements_test.txt @@ -6,8 +6,5 @@ h5netcdf h5py earthkit-data pyfdb -# pygribjump -# git+ssh://git@github.com/ecmwf/gribjump.git@develop git+https://github.com/ecmwf/gribjump.git@develop - requests \ No newline at end of file diff --git a/tests/test_bad_request_error.py b/tests/test_bad_request_error.py index 3224e4f43..1bff4f308 100644 --- a/tests/test_bad_request_error.py +++ b/tests/test_bad_request_error.py @@ -4,9 +4,6 @@ from polytope_feature.polytope import Polytope from polytope_feature.utility.exceptions import BadRequestError -# import geopandas as gpd -# import matplotlib.pyplot as plt - class TestSlicingFDBDatacube: def setup_method(self, method): @@ -50,8 +47,7 @@ def setup_method(self, method): }, } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_cyclic_nearest.py b/tests/test_cyclic_nearest.py index 59318e0ed..792ce3d43 100644 --- a/tests/test_cyclic_nearest.py +++ b/tests/test_cyclic_nearest.py @@ -67,7 +67,7 @@ def find_nearest_latlon(self, grib_file, target_lat, target_lon): return nearest_points - # @pytest.mark.fdb + @pytest.mark.fdb @pytest.mark.internet def test_regular_grid(self): import pygribjump as gj @@ -75,7 +75,6 @@ def test_regular_grid(self): request = Request( Select("step", [0]), Select("levtype", ["sfc"]), - # Select("date", [pd.Timestamp("20231102T000000")]), Select("date", [pd.Timestamp("20240103T0000")]), Select("domain", ["g"]), Select("expver", ["0001"]), @@ -100,7 +99,6 @@ def test_regular_grid(self): request = Request( Select("step", [0]), Select("levtype", ["sfc"]), - # Select("date", [pd.Timestamp("20231102T000000")]), Select("date", [pd.Timestamp("20240103T0000")]), Select("domain", ["g"]), Select("expver", ["0001"]), diff --git a/tests/test_cyclic_snapping.py b/tests/test_cyclic_snapping.py index 599dc9573..984d3fb96 100644 --- a/tests/test_cyclic_snapping.py +++ b/tests/test_cyclic_snapping.py @@ -23,8 +23,6 @@ def setup_method(self, method): self.slicer = HullSlicer() self.API = Polytope(datacube=array, engine=self.slicer, options=options) - # Testing different shapes - def test_cyclic_float_axis_across_seam(self): request = Request(Select("long", [-0.2], method="surrounding")) result = self.API.retrieve(request) diff --git a/tests/test_date_time_unmerged.py b/tests/test_date_time_unmerged.py index 365972075..5e808b504 100644 --- a/tests/test_date_time_unmerged.py +++ b/tests/test_date_time_unmerged.py @@ -38,17 +38,14 @@ def setup_method(self, method): ], } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj request = Request( Select("step", [0]), Select("levtype", ["sfc"]), - # Select("date", [pd.Timestamp("20240118")]), Select("time", [pd.Timedelta("00:00:00")]), - # Span("time", [pd.Timedelta("00:00:00")]), Span("date", pd.Timestamp("20240118"), pd.Timestamp("20240119")), Select("domain", ["g"]), Select("expver", ["0001"]), diff --git a/tests/test_ecmwf_oper_data_fdb.py b/tests/test_ecmwf_oper_data_fdb.py index d3b06d436..98213a3a1 100644 --- a/tests/test_ecmwf_oper_data_fdb.py +++ b/tests/test_ecmwf_oper_data_fdb.py @@ -41,8 +41,7 @@ def setup_method(self, method): "pre_path": {"class": "od", "expver": "0001", "levtype": "sfc", "type": "fc", "stream": "oper"}, } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -98,7 +97,7 @@ def test_fdb_datacube_point(self): assert set(result.leaves[0].flatten()["step"]) == set((0, 1)) assert len(result.leaves[0].result) == 4 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_point_v2(self): import pygribjump as gj @@ -126,7 +125,7 @@ def test_fdb_datacube_point_v2(self): assert len(result.leaves) == 3 assert len(result.leaves[0].result) == 4 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_point_step_not_compressed(self): import pygribjump as gj diff --git a/tests/test_engine_slicer.py b/tests/test_engine_slicer.py index 3392ccd3a..0c8d1c0d7 100644 --- a/tests/test_engine_slicer.py +++ b/tests/test_engine_slicer.py @@ -46,18 +46,12 @@ def test_triangle(self): result = self.slicer.extract(datacube, triangle) result.pprint() assert len(result.leaves) == 10 - # assert len(result.leaves) == 4 - # total_leaves = 0 - # for leaf in result.leaves: - # total_leaves += len(leaf.values) - # assert total_leaves == 4 + 3 + 2 + 1 def test_reusable(self): datacube = MockDatacube({"x": 100, "y": 100}) polytopes = Polygon(["x", "y"], [[3, 3], [3, 6], [6, 3]]).polytope() result = self.slicer.extract(datacube, polytopes) result.pprint() - # assert len(result.leaves) == 4 assert len(result.leaves) == 10 polytopes = Box(["x", "y"], lower_corner=[3, 3], upper_corner=[6, 6]).polytope() result = self.slicer.extract(datacube, polytopes) diff --git a/tests/test_fdb_datacube.py b/tests/test_fdb_datacube.py index 59023d7c7..eb52af1d6 100644 --- a/tests/test_fdb_datacube.py +++ b/tests/test_fdb_datacube.py @@ -45,8 +45,7 @@ def setup_method(self, method): "pre_path": {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"}, } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -96,7 +95,7 @@ def test_fdb_datacube(self): # plt.colorbar(label="Temperature") # plt.show() - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_select_grid(self): import pygribjump as gj diff --git a/tests/test_fdb_unmap_tree.py b/tests/test_fdb_unmap_tree.py index 91915b57f..9e2e9b90f 100644 --- a/tests/test_fdb_unmap_tree.py +++ b/tests/test_fdb_unmap_tree.py @@ -43,7 +43,7 @@ def setup_method(self, method): } # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 854f83735..2fbfcaf28 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -47,7 +47,7 @@ def setup_method(self, method): } @pytest.mark.internet - # @pytest.mark.fdb + @pytest.mark.fdb def test_incomplete_fdb_branch(self): import pygribjump as gj @@ -79,7 +79,7 @@ def test_incomplete_fdb_branch(self): assert result.is_root() @pytest.mark.internet - # @pytest.mark.fdb + @pytest.mark.fdb def test_incomplete_fdb_branch_2(self): import pygribjump as gj diff --git a/tests/test_local_grid_cyclic.py b/tests/test_local_grid_cyclic.py index 82c82c3bc..63aa38216 100644 --- a/tests/test_local_grid_cyclic.py +++ b/tests/test_local_grid_cyclic.py @@ -50,8 +50,7 @@ def setup_method(self, method): ], } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -80,7 +79,7 @@ def test_fdb_datacube(self): assert result.leaves[0].flatten()["latitude"] == (-20,) assert result.leaves[0].flatten()["longitude"] == (-20,) - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_2(self): import pygribjump as gj diff --git a/tests/test_local_regular_grid.py b/tests/test_local_regular_grid.py index 6b4e01fee..518f12fdf 100644 --- a/tests/test_local_regular_grid.py +++ b/tests/test_local_regular_grid.py @@ -49,8 +49,7 @@ def setup_method(self, method): ], } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -79,7 +78,7 @@ def test_fdb_datacube(self): assert result.leaves[0].flatten()["latitude"] == (0,) assert result.leaves[0].flatten()["longitude"] == (0,) - # @pytest.mark.fdb + @pytest.mark.fdb def test_point_outside_local_region(self): import pygribjump as gj @@ -108,7 +107,7 @@ def test_point_outside_local_region(self): assert result.leaves[0].flatten()["latitude"] == (0,) assert result.leaves[0].flatten()["longitude"] == (60,) - # @pytest.mark.fdb + @pytest.mark.fdb def test_point_outside_local_region_2(self): import pygribjump as gj @@ -137,7 +136,7 @@ def test_point_outside_local_region_2(self): assert result.leaves[0].flatten()["latitude"] == (40,) assert result.leaves[0].flatten()["longitude"] == (1,) - # @pytest.mark.fdb + @pytest.mark.fdb def test_point_outside_local_region_3(self): import pygribjump as gj @@ -165,7 +164,7 @@ def test_point_outside_local_region_3(self): assert len(result.leaves) == 1 assert result.is_root() - # @pytest.mark.fdb + @pytest.mark.fdb def test_point_outside_local_region_4(self): import pygribjump as gj @@ -193,7 +192,7 @@ def test_point_outside_local_region_4(self): assert len(result.leaves) == 1 assert result.is_root() - # @pytest.mark.fdb + @pytest.mark.fdb def test_point_outside_local_region_5(self): import pygribjump as gj @@ -221,7 +220,7 @@ def test_point_outside_local_region_5(self): assert len(result.leaves) == 1 assert result.is_root() - # @pytest.mark.fdb + @pytest.mark.fdb def test_point_outside_local_region_6(self): import pygribjump as gj @@ -249,7 +248,7 @@ def test_point_outside_local_region_6(self): assert len(result.leaves) == 1 assert result.is_root() - # @pytest.mark.fdb + @pytest.mark.fdb def test_point_outside_local_region_7(self): import pygribjump as gj @@ -278,7 +277,7 @@ def test_point_outside_local_region_7(self): assert result.leaves[0].flatten()["latitude"] == (-40,) assert result.leaves[0].flatten()["longitude"] == (1,) - # @pytest.mark.fdb + @pytest.mark.fdb def test_point_outside_local_region_8(self): import pygribjump as gj @@ -307,7 +306,7 @@ def test_point_outside_local_region_8(self): assert result.leaves[0].flatten()["latitude"] == (-30,) assert result.leaves[0].flatten()["longitude"] == (-20,) - # @pytest.mark.fdb + @pytest.mark.fdb def test_point_outside_local_region_9(self): import pygribjump as gj diff --git a/tests/test_local_swiss_grid.py b/tests/test_local_swiss_grid.py index e4b297876..50f88c691 100644 --- a/tests/test_local_swiss_grid.py +++ b/tests/test_local_swiss_grid.py @@ -38,8 +38,7 @@ def setup_method(self, method): "compressed_axes_config": ["longitude", "latitude", "levtype", "levelist", "step", "date", "param"], } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb @pytest.mark.skip("Non-accessible data") def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_multiple_param_fdb.py b/tests/test_multiple_param_fdb.py index d87c122da..774584c93 100644 --- a/tests/test_multiple_param_fdb.py +++ b/tests/test_multiple_param_fdb.py @@ -41,8 +41,7 @@ def setup_method(self, method): ], } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_override_md5_hash_options.py b/tests/test_override_md5_hash_options.py index 55f762bca..58696fe0a 100644 --- a/tests/test_override_md5_hash_options.py +++ b/tests/test_override_md5_hash_options.py @@ -48,8 +48,7 @@ def setup_method(self, method): ], } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_point_nearest.py b/tests/test_point_nearest.py index 050d4225f..78cf8007c 100644 --- a/tests/test_point_nearest.py +++ b/tests/test_point_nearest.py @@ -42,8 +42,7 @@ def setup_method(self, method): ], } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -69,7 +68,7 @@ def test_fdb_datacube(self): result = self.API.retrieve(request) assert len(result.leaves) == 1 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_true_point(self): import pygribjump as gj @@ -96,7 +95,7 @@ def test_fdb_datacube_true_point(self): # result.pprint() assert len(result.leaves) == 1 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_true_point_2(self): import pygribjump as gj @@ -123,20 +122,11 @@ def test_fdb_datacube_true_point_2(self): result.pprint() assert len(result.leaves) == 1 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_true_point_3(self): import pygribjump as gj request = Request( - # Select("step", [21]), - # Select("levtype", ["sfc"]), - # Select("date", [pd.Timestamp("20231102T000000")]), - # Select("domain", ["g"]), - # Select("expver", ["0001"]), - # Select("param", ["167"]), - # Select("class", ["od"]), - # Select("stream", ["oper"]), - # Select("type", ["fc"]), Select("step", [0]), Select("levtype", ["sfc"]), Select("date", [pd.Timestamp("20230625T120000")]), @@ -161,7 +151,7 @@ def test_fdb_datacube_true_point_3(self): assert result.leaves[0].values == (359.929906542056,) assert result.leaves[0].axis.name == "longitude" - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_true_point_5(self): import pygribjump as gj @@ -190,7 +180,7 @@ def test_fdb_datacube_true_point_5(self): assert result.leaves[0].values == (359.929906542056,) assert result.leaves[0].axis.name == "longitude" - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_true_point_4(self): import pygribjump as gj @@ -214,7 +204,6 @@ def test_fdb_datacube_true_point_4(self): options=self.options, ) result = self.API.retrieve(request) - # result.pprint_2() assert len(result.leaves) == 1 assert result.leaves[0].values == (359.929906542056,) assert result.leaves[0].axis.name == "longitude" diff --git a/tests/test_point_shape.py b/tests/test_point_shape.py index 247e4b878..35c803984 100644 --- a/tests/test_point_shape.py +++ b/tests/test_point_shape.py @@ -30,7 +30,6 @@ def test_point(self): assert result.leaves[0].axis.name == "level" def test_multiple_points(self): - # request = Request(Point(["step", "level"], [[3, 10], [3, 12]]), Select("date", ["2000-01-01"])) request = Request( Union(["step", "level"], Point(["step", "level"], [[3, 10]]), Point(["step", "level"], [[3, 12]])), Select("date", ["2000-01-01"]), diff --git a/tests/test_point_union.py b/tests/test_point_union.py index 36b70ca24..b9e8552d5 100644 --- a/tests/test_point_union.py +++ b/tests/test_point_union.py @@ -42,8 +42,7 @@ def setup_method(self, method): ], } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -81,7 +80,7 @@ def test_fdb_datacube(self): result.pprint() assert len(result.leaves) == 8 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_surrounding(self): import pygribjump as gj @@ -117,7 +116,7 @@ def test_fdb_datacube_surrounding(self): tot_leaves += len(leaf.result) assert tot_leaves == 9 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_axis_order(self): import pygribjump as gj diff --git a/tests/test_reduced_ll_grid.py b/tests/test_reduced_ll_grid.py index 69fe4c1df..b0a3535c4 100644 --- a/tests/test_reduced_ll_grid.py +++ b/tests/test_reduced_ll_grid.py @@ -50,7 +50,7 @@ def setup_method(self, method): @pytest.mark.skip(reason="wave data grid packing not supported") @pytest.mark.internet - # @pytest.mark.fdb + @pytest.mark.fdb def test_reduced_ll_grid(self): import pygribjump as gj diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 026496f18..d451638b8 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -56,7 +56,7 @@ def setup_method(self, method): ], } - # @pytest.mark.fdb + @pytest.mark.fdb @pytest.mark.internet def test_regular_grid(self): import pygribjump as gj @@ -102,7 +102,7 @@ def test_regular_grid(self): tol = 1e-8 leaves = result.leaves for i in range(len(leaves)): - right_pl_results = leaves[i].result[len(leaves[i].values) :] + right_pl_results = leaves[i].result[len(leaves[i].values):] result_tree = right_pl_results[0] cubepath = leaves[i].flatten() lat = cubepath["latitude"][0] diff --git a/tests/test_regular_reduced_grid.py b/tests/test_regular_reduced_grid.py index 3caee691e..7de5d8584 100644 --- a/tests/test_regular_reduced_grid.py +++ b/tests/test_regular_reduced_grid.py @@ -1,7 +1,6 @@ import pandas as pd import pytest from eccodes import codes_grib_find_nearest, codes_grib_new_from_file -from helper_functions import download_test_data from polytope_feature.engine.hullslicer import HullSlicer from polytope_feature.polytope import Polytope, Request @@ -52,16 +51,14 @@ def setup_method(self, method): } # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self, downloaded_data_test_files): import pygribjump as gj request = Request( Select("step", [0]), Select("levtype", ["sfc"]), - # Select("date", [pd.Timestamp("20231102T000000")]), Select("date", [pd.Timestamp("20240103T0000")]), - # Select("date", [pd.Timestamp("20240129T000000")]), Select("domain", ["g"]), Select("expver", ["0001"]), Select("param", ["167"]), @@ -94,7 +91,6 @@ def test_fdb_datacube(self, downloaded_data_test_files): for f in downloaded_data_test_files: if "aifs_data_from_fdb.grib" in f.name: aifs_file = f - # f = open("tests/data/aifs_data_from_fdb.grib", "rb") f = open(aifs_file, "rb") messages = [] message = codes_grib_new_from_file(f) diff --git a/tests/test_request_tree.py b/tests/test_request_tree.py index 038599724..99e02eb7c 100644 --- a/tests/test_request_tree.py +++ b/tests/test_request_tree.py @@ -177,69 +177,6 @@ def test_remove_branch(self): child1.remove_branch() assert root_node.children == SortedList([child2]) - # def test_intersect(self): - # axis1 = IntDatacubeAxis() - # axis2 = IntDatacubeAxis() - # axis3 = IntDatacubeAxis() - # axis4 = IntDatacubeAxis() - # axis5 = IntDatacubeAxis() - # axis6 = IntDatacubeAxis() - # axis1.name = "grandchild1" - # axis2.name = "grandchild2" - # axis3.name = "grandchild3" - # axis4.name = "grandchild4" - # axis5.name = "child1" - # axis6.name = "child2" - # grandchild1_1 = TensorIndexTree(axis=axis1) - # grandchild2_1 = TensorIndexTree(axis=axis2) - # grandchild3_1 = TensorIndexTree(axis=axis3) - # grandchild4_1 = TensorIndexTree(axis=axis4) - # child1_1 = TensorIndexTree(axis=axis5) - # child1_1.add_child(grandchild1_1) - # child1_1.add_child(grandchild2_1) - # child2_1 = TensorIndexTree(axis=axis6) - # child2_1.add_child(grandchild3_1) - # child2_1.add_child(grandchild4_1) - # root_node1 = TensorIndexTree() - # root_node1.add_child(child1_1) - # root_node1.add_child(child2_1) - - # axis1 = IntDatacubeAxis() - # axis2 = IntDatacubeAxis() - # axis3 = IntDatacubeAxis() - # axis4 = IntDatacubeAxis() - # axis5 = IntDatacubeAxis() - # axis6 = IntDatacubeAxis() - # axis1.name = "grandchild1" - # axis2.name = "grandchild5" - # axis3.name = "grandchild6" - # axis4.name = "grandchild7" - # axis5.name = "child1" - # axis6.name = "child3" - # grandchild1_2 = TensorIndexTree(axis=axis1) - # grandchild2_2 = TensorIndexTree(axis=axis2) - # grandchild3_2 = TensorIndexTree(axis=axis3) - # grandchild4_2 = TensorIndexTree(axis=axis4) - # child1_2 = TensorIndexTree(axis=axis5) - # child1_2.add_child(grandchild1_2) - # child1_2.add_child(grandchild2_2) - # child2_2 = TensorIndexTree(axis=axis6) - # child2_2.add_child(grandchild3_2) - # child2_2.add_child(grandchild4_2) - # root_node2 = TensorIndexTree() - # root_node2.add_child(child1_2) - # root_node2.add_child(child2_2) - # root_node1.pprint() - # root_node2.pprint() - - # root_node1.intersect(root_node2) - - # root_node1.pprint() - # assert len(root_node1.children) == 1 - # assert list(root_node1.children)[0].axis.name == "child1" - # assert len(list(root_node1.children)[0].children) == 1 - # assert list(list(root_node1.children)[0].children)[0].axis.name == "grandchild1" - def test_flatten(self): axis1 = IntDatacubeAxis() axis2 = IntDatacubeAxis() diff --git a/tests/test_shapes.py b/tests/test_shapes.py index 92ee5fc33..5b5b7b921 100644 --- a/tests/test_shapes.py +++ b/tests/test_shapes.py @@ -46,7 +46,7 @@ def test_all_cyclic(self): path = result.leaves[0].flatten() assert path["longitude"] == tuple(range(0, 360)) - # @pytest.mark.fdb + @pytest.mark.fdb def test_all_mapper_cyclic(self): import pygribjump as gj @@ -66,29 +66,18 @@ def test_all_mapper_cyclic(self): "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"], - # "md5_hash": "5ea6378bf5e2904f565ef7221da63a09", } ], }, {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, ], - # "pre_path": {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"}, "pre_path": {"class": "od", "expver": "0001", "levtype": "sfc", "type": "fc", "stream": "oper"}, } self.fdbdatacube = gj.GribJump() self.slicer = HullSlicer() request = Request( - # Select("step", [11]), - # Select("levtype", ["sfc"]), - # Select("date", [pd.Timestamp("20230710T120000")]), - # Select("domain", ["g"]), - # Select("expver", ["0001"]), - # Select("param", ["151130"]), - # Select("class", ["od"]), - # Select("stream", ["oper"]), - # Select("type", ["fc"]), Select("step", [0]), Select("levtype", ["sfc"]), Select("date", [pd.Timestamp("20240103T0000")]), diff --git a/tests/test_slice_date_range_fdb.py b/tests/test_slice_date_range_fdb.py index 734bb3034..f66695f87 100644 --- a/tests/test_slice_date_range_fdb.py +++ b/tests/test_slice_date_range_fdb.py @@ -42,8 +42,7 @@ def setup_method(self, method): ], } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -73,7 +72,7 @@ def test_fdb_datacube(self): for i in range(len(result.leaves)): assert len(result.leaves[i].result) == 3 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_select_non_existing_last(self): import pygribjump as gj @@ -103,7 +102,7 @@ def test_fdb_datacube_select_non_existing_last(self): for i in range(len(result.leaves)): assert len(result.leaves[i].result) == 3 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_select_non_existing_first(self): import pygribjump as gj @@ -133,7 +132,7 @@ def test_fdb_datacube_select_non_existing_first(self): for i in range(len(result.leaves)): assert len(result.leaves[i].result) == 3 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_select_completely_non_existing(self): import pygribjump as gj @@ -163,7 +162,7 @@ def test_fdb_datacube_select_completely_non_existing(self): for i in range(len(result.leaves)): assert len(result.leaves[i].result) == 0 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_disk(self): import pygribjump as gj @@ -195,7 +194,7 @@ def test_fdb_datacube_disk(self): assert len(result.leaves[0].values) == 3 assert len(result.leaves[1].values) == 3 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_disk_2(self): import pygribjump as gj diff --git a/tests/test_slice_date_range_fdb_v2.py b/tests/test_slice_date_range_fdb_v2.py index 7a464138c..a4454d4e8 100644 --- a/tests/test_slice_date_range_fdb_v2.py +++ b/tests/test_slice_date_range_fdb_v2.py @@ -45,8 +45,7 @@ def setup_method(self, method): ], } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb @pytest.mark.skip(reason="gribjump problem") def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_slice_fdb_box.py b/tests/test_slice_fdb_box.py index 95cf1186b..4918317b1 100644 --- a/tests/test_slice_fdb_box.py +++ b/tests/test_slice_fdb_box.py @@ -46,9 +46,8 @@ def setup_method(self, method): ], } - # Testing different shapes @pytest.mark.skip(reason="optimisation test") - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_tree_protobuf_encoding.py b/tests/test_tree_protobuf_encoding.py index e62512916..b177cbe2b 100644 --- a/tests/test_tree_protobuf_encoding.py +++ b/tests/test_tree_protobuf_encoding.py @@ -45,7 +45,7 @@ def setup_method(self): "timedelta_ax": grandchild_ax3, } - # @pytest.mark.fdb + @pytest.mark.fdb def test_encoding(self): import pygribjump as gj diff --git a/tests/test_tree_protobuf_encoding_fdb.py b/tests/test_tree_protobuf_encoding_fdb.py index 517566116..99b8f169c 100644 --- a/tests/test_tree_protobuf_encoding_fdb.py +++ b/tests/test_tree_protobuf_encoding_fdb.py @@ -8,7 +8,7 @@ class TestEncoder: def setup_method(self): pass - # @pytest.mark.fdb + @pytest.mark.fdb def test_encoding(self): import pygribjump as gj @@ -75,7 +75,6 @@ def test_encoding(self): fdb_datacube = self.API.datacube fdb_datacube.prep_tree_encoding(result) encoded_bytes = encode_tree(result) - # write_encoded_tree_to_file(encoded_bytes) decoded_tree = decode_tree(fdb_datacube, encoded_bytes) decoded_tree.pprint() assert decoded_tree.leaves[0].result_size == [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] diff --git a/tests/test_union_gj.py b/tests/test_union_gj.py index 2db4c4599..98083873b 100644 --- a/tests/test_union_gj.py +++ b/tests/test_union_gj.py @@ -42,8 +42,7 @@ def setup_method(self, method): ], } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube(self): import pygribjump as gj @@ -84,7 +83,7 @@ def test_fdb_datacube(self): assert total_lons == 16 assert total_vals == 16 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_complete_overlap(self): import pygribjump as gj @@ -125,7 +124,7 @@ def test_fdb_datacube_complete_overlap(self): assert total_lons == 9 assert total_vals == 9 - # @pytest.mark.fdb + @pytest.mark.fdb def test_fdb_datacube_complete_overlap_v2(self): import pygribjump as gj diff --git a/tests/test_union_point_box.py b/tests/test_union_point_box.py index 91a2061f8..2057ff91a 100644 --- a/tests/test_union_point_box.py +++ b/tests/test_union_point_box.py @@ -42,8 +42,7 @@ def setup_method(self, method): ], } - # Testing different shapes - # @pytest.mark.fdb + @pytest.mark.fdb @pytest.mark.skip(reason="point and box are not same dimensions") def test_fdb_datacube(self): import pygribjump as gj diff --git a/tests/test_wave_spectra_data.py b/tests/test_wave_spectra_data.py index 9f655f616..2fe4aa208 100644 --- a/tests/test_wave_spectra_data.py +++ b/tests/test_wave_spectra_data.py @@ -44,7 +44,7 @@ def setup_method(self, method): } self.slicer = HullSlicer() - # @pytest.mark.fdb + @pytest.mark.fdb def test_healpix_grid(self, downloaded_data_test_files): import pygribjump as gj @@ -91,7 +91,6 @@ def test_healpix_grid(self, downloaded_data_test_files): for j, lon in enumerate(new_lons): lats.append(lat) lons.append(lon) - # nearest_points = find_nearest_latlon("./tests/data/wave_spectra.grib", lat, lon) nearest_points = find_nearest_latlon(wave_file, lat, lon) eccodes_lat = nearest_points[0][0]["lat"] eccodes_lon = nearest_points[0][0]["lon"] From eb0db198dde83ae5ebb23ac690a8d17c0c9a84ef Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 7 Jul 2025 16:33:33 +0200 Subject: [PATCH 071/105] black --- tests/test_regular_grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index d451638b8..e094b3bca 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -102,7 +102,7 @@ def test_regular_grid(self): tol = 1e-8 leaves = result.leaves for i in range(len(leaves)): - right_pl_results = leaves[i].result[len(leaves[i].values):] + right_pl_results = leaves[i].result[len(leaves[i].values) :] result_tree = right_pl_results[0] cubepath = leaves[i].flatten() lat = cubepath["latitude"][0] From 0e9390ace0dc633ad25762adb823d87ea12bff22 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 7 Jul 2025 17:10:25 +0200 Subject: [PATCH 072/105] clean up ci file --- .github/workflows/ci.yaml | 172 -------------------------------------- 1 file changed, 172 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2d8585cda..c8fffc609 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,166 +1,3 @@ -# name: ci -# on: -# # Trigger the workflow on push to master or develop, except tag creation -# push: -# branches: -# - 'main' -# - 'develop' -# # Trigger the workflow on pull request -# pull_request: ~ -# # Trigger the workflow manually -# workflow_dispatch: ~ -# # Trigger after public PR approved for CI -# pull_request_target: -# types: [labeled] -# release: -# types: [created] -# jobs: -# qa: -# name: qa -# runs-on: ubuntu-20.04 -# steps: -# - name: Checkout Repository -# uses: actions/checkout@v3 -# with: -# repository: ${{ inputs.repository }} -# ref: ${{ inputs.ref }} - -# - name: Setup Python -# uses: actions/setup-python@v4 -# with: -# python-version: ${{ inputs.python_version }} - -# - name: Install Python Dependencies -# run: | -# python -m pip install --upgrade pip -# python -m pip install black flake8 isort -# - name: Check isort -# run: isort --check . - -# - name: Check black -# run: black --check . - -# - name: Check flake8 -# run: flake8 . -# setup: -# name: setup -# runs-on: ubuntu-20.04 -# outputs: -# matrix: ${{ steps.set-matrix.outputs.matrix }} -# inputs: ${{ steps.prepare-inputs.outputs.inputs }} -# inputs-for-ubuntu: ${{ steps.prepare-inputs.outputs.inputs-for-ubuntu }} -# steps: -# - name: Set Matrix -# id: set-matrix -# shell: bash -eux {0} -# run: | -# MATRIX=$(cat << 'EOS' -# name: -# - gnu-11@ubuntu-22.04 -# - clang-14@ubuntu-22.04 -# include: -# - name: gnu-11@ubuntu-22.04 -# os: ubuntu-22.04 -# compiler: gnu-11 -# compiler_cc: gcc-11 -# compiler_cxx: g++-11 -# compiler_fc: gfortran-11 -# - name: clang-14@ubuntu-22.04 -# os: ubuntu-22.04 -# compiler: clang-14 -# compiler_cc: clang-14 -# compiler_cxx: clang++-14 -# compiler_fc: gfortran-11 -# # Xcode compiler requires empty environment variables, so we pass null (~) here -# EOS -# ) -# SKIP_MATRIX_JOBS=$(cat << 'EOS' -# ${{ inputs.skip_matrix_jobs }} -# EOS -# ) -# SELECT_NAME_COND="1 != 1" -# SELECT_INCLUDE_COND="1 != 1" -# for skip_job in $SKIP_MATRIX_JOBS; do SELECT_NAME_COND="$SELECT_NAME_COND or . == \"$skip_job\""; SELECT_INCLUDE_COND="$SELECT_INCLUDE_COND or .name == \"$skip_job\""; done -# echo matrix=$(echo "$MATRIX" | yq eval "del(.name[] | select($SELECT_NAME_COND)) | del(.include[] | select($SELECT_INCLUDE_COND))" --output-format json --indent 0 -) >> $GITHUB_OUTPUT -# - name: Prepare build-package Inputs -# id: prepare-inputs -# shell: bash -eux {0} -# run: | -# echo inputs=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '.' --output-format json --indent 0 -) >> $GITHUB_OUTPUT -# echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-20.04","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT -# test: -# name: test -# needs: -# - qa -# - setup -# strategy: -# fail-fast: false -# matrix: ${{ fromJson(needs.setup.outputs.matrix) }} -# runs-on: ${{ matrix.os }} -# steps: -# - uses: actions/checkout@v4 -# - name: Install eccodes and Dependencies -# id: install-dependencies -# uses: ecmwf-actions/build-package@v2 -# with: -# self_build: false -# dependencies: | -# ecmwf/ecbuild@develop -# MathisRosenhauer/libaec@master -# ecmwf/eccodes@develop - -# - name: Setup Python -# uses: actions/setup-python@v4 -# with: -# python-version: ${{ inputs.python_version }} - -# - name: Install Python Dependencies -# run: | -# python -m pip install --upgrade pip -# python -m pip install pytest pytest-cov -# python -m pip install -r requirements.txt -# python -m pip install -r ./tests/requirements_test.txt - -# - name: Verify Source Distribution -# shell: bash -eux {0} -# run: | -# python setup.py sdist -# python -m pip install dist/* -# - name: Run Tests with Repository Code -# env: -# LD_LIBRARY_PATH: ${{ steps.install-dependencies.outputs.lib_path }} -# shell: bash -eux {0} -# run: | -# DYLD_LIBRARY_PATH=${{ env.LD_LIBRARY_PATH }} python -m pytest -m "not fdb" tests --cov=./ --cov-report=xml -# python -m coverage report - -# - name: Upload coverage to Codecov -# uses: codecov/codecov-action@v4 -# with: -# files: coverage.xml -# deploy: -# needs: test -# if: ${{ github.event_name == 'release' }} -# name: Upload to Pypi -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v3 -# - name: Set up Python -# uses: actions/setup-python@v2 -# with: -# python-version: '3.8' -# - name: Install dependencies -# run: | -# python -m pip install --upgrade pip -# pip install build twine -# - name: Build and publish -# env: -# TWINE_USERNAME: "__token__" -# TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} -# run: | -# python -m build -# twine upload dist/* - name: ci on: @@ -194,15 +31,6 @@ jobs: python_qa: true secrets: inherit - # # Build downstream packages on HPC - # downstream-ci-hpc: - # name: downstream-ci-hpc - # if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - # uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/add_polytope - # with: - # polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} - # secrets: inherit - deploy: if: ${{ github.event_name == 'release' }} name: Upload to Pypi From b76a9dd026f7fe3c0a099090e4e7dacb03f5ae0a Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 7 Jul 2025 17:38:22 +0200 Subject: [PATCH 073/105] remove specific ci branch --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c8fffc609..34114d5ab 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,7 +24,7 @@ jobs: downstream-ci: name: downstream-ci if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml@feature/add_polytope + uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml with: polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} codecov_upload: true From 77ce80f3f9b7b6c0a739b1e2e76baa4e040e9f1c Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 7 Jul 2025 17:42:57 +0200 Subject: [PATCH 074/105] point to downstream ci main branch --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 34114d5ab..fca603554 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,7 +24,7 @@ jobs: downstream-ci: name: downstream-ci if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml + uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml@main with: polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} codecov_upload: true From 7d047538630ad7cee7a8d6a844f268ff7b817ab7 Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 15 Jul 2025 17:01:05 +0200 Subject: [PATCH 075/105] WIP: try to integrate with new qubed develop --- .gitignore | 3 ++- polytope_feature/engine/qubed_slicer.py | 11 +++++++--- tests/test_qubed_extraction_service.py | 29 ++++++++++++++++++------- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 7af8069c4..601078f8c 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,5 @@ rust_deployment_venv polytope_feature/_version.py **/build **.lock -**/target \ No newline at end of file +**/target +rust_venv \ No newline at end of file diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index b71f9f2f2..f72a9c229 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -1,7 +1,7 @@ from qubed import Qube from qubed.value_types import QEnum -from qubed.set_operations import union +# from qubed.set_operations import union from .slicing_tools import slice from ..datacube.backends.qubed import QubedDatacube from .engine import Engine @@ -27,6 +27,9 @@ def find_datacube_vals(): def find_values_between(self, polytope, ax, node, datacube, lower, upper, path=None): if isinstance(ax, UnsliceableDatacubeAxis): + # print(node.values) + # print(lower) + # print(ax.name) return [v for v in node.values if lower <= v <= upper] tol = ax.tol @@ -232,7 +235,8 @@ def _slice_second_grid_axis(self, axis_name, polytopes, datacube, datacube_trans def slice_tree(self, datacube, final_polys): q = datacube.q datacube_transformations = datacube.datacube_transformations - return Qube.root_node(self._slice(q, final_polys, datacube, datacube_transformations)) + # return Qube.root_node(self._slice(q, final_polys, datacube, datacube_transformations)) + return Qube.make_root(self._slice(q, final_polys, datacube, datacube_transformations)) def build_tree(self, polytopes_to_slice, datacube): groups, input_axes = group(polytopes_to_slice) @@ -250,5 +254,6 @@ def build_tree(self, polytopes_to_slice, datacube): final_tree = sub_trees[0] for sub_tree in sub_trees[1:]: - union(final_tree, sub_tree) + # union(final_tree, sub_tree) + final_tree | sub_tree return final_tree diff --git a/tests/test_qubed_extraction_service.py b/tests/test_qubed_extraction_service.py index d303b03d7..3a7a5dd0a 100644 --- a/tests/test_qubed_extraction_service.py +++ b/tests/test_qubed_extraction_service.py @@ -103,6 +103,9 @@ def get_fdb_tree(request): options = { "axis_config": [ {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "expver", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "realization", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "generation", "transformations": [{"name": "type_change", "type": "int"}]}, {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, # { # "axis_name": "date", @@ -139,10 +142,13 @@ def get_fdb_tree(request): "type": "UnsliceableDatacubeAxis", "model": "UnsliceableDatacubeAxis", "stream": "UnsliceableDatacubeAxis", - "realization": "UnsliceableDatacubeAxis", - "expver": "UnsliceableDatacubeAxis", + # "realization": "UnsliceableDatacubeAxis", + "realization": "IntDatacubeAxis", + # "expver": "UnsliceableDatacubeAxis", + "expver": "IntDatacubeAxis", "experiment": "UnsliceableDatacubeAxis", - "generation": "UnsliceableDatacubeAxis", + # "generation": "UnsliceableDatacubeAxis", + "generation": "IntDatacubeAxis", "levtype": "UnsliceableDatacubeAxis", "activity": "UnsliceableDatacubeAxis", "dataset": "UnsliceableDatacubeAxis", @@ -166,27 +172,34 @@ def get_fdb_tree(request): request = Request( # ConvexPolytope(["param"], [["164"]]), - Select("param", ["164"]), + # Select("param", ["164"]), + Select("param", ["165"]), ConvexPolytope(["time"], [[pd.Timedelta(hours=0, minutes=0)], [pd.Timedelta(hours=3, minutes=0)]]), ConvexPolytope(["resolution"], [["high"]]), ConvexPolytope(["type"], [["fc"]]), # ConvexPolytope(["model"], [['ifs-nemo']]), - Select("model", ["ifs-nemo"]), + # Select("model", ["ifs-nemo"]), + Select("model", ["icon"]), Select("stream", ["clte"]), # ConvexPolytope(["stream"], [["clte"]]), ConvexPolytope(["realization"], ["1"]), ConvexPolytope(["expver"], [['0001']]), ConvexPolytope(["experiment"], [['ssp3-7.0']]), - ConvexPolytope(["generation"], [["1"]]), + # ConvexPolytope(["generation"], [["1"]]), + Select("generation", [1]), ConvexPolytope(["levtype"], [["sfc"]]), # ConvexPolytope(["activity"], [["scenariomip"]]), Select("activity", ["scenariomip"]), ConvexPolytope(["dataset"], [["climate-dt"]]), ConvexPolytope(["class"], [["d1"]]), - ConvexPolytope(["date"], [[pd.Timestamp("20220811")]]), + # ConvexPolytope(["date"], [[pd.Timestamp("20220811")]]), + ConvexPolytope(["date"], [[pd.Timestamp("20200908")]]), ConvexPolytope(["latitude", "longitude"], [[0, 0], [5, 5], [0, 5]])) -fdb_tree = get_fdb_tree(request) +# fdb_tree = get_fdb_tree(request) +path_to_qube = "../qubed/" +full_qube_path = path_to_qube+"tests/example_qubes/climate_dt_with_metadata.json" +fdb_tree = Qube.load(full_qube_path) print("HERE WE HAVE THE FDB TREE") print(fdb_tree) From dd540d0457d14623cb7cc033dc4a257e29b0db3a Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 16 Jul 2025 13:01:15 +0200 Subject: [PATCH 076/105] add downstream ci hpc --- .github/workflows/ci.yaml | 8 ++++++++ .gitignore | 3 ++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fca603554..81fb4033d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,6 +31,14 @@ jobs: python_qa: true secrets: inherit + downstream-ci-hpc: + name: downstream-ci-hpc + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main + with: + polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} + secrets: inherit + deploy: if: ${{ github.event_name == 'release' }} name: Upload to Pypi diff --git a/.gitignore b/.gitignore index 0eaa5567e..78e2f722a 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,5 @@ rust_deployment_venv **/target **/build _version.py -*.DS_Store \ No newline at end of file +*.DS_Store +rust_venv \ No newline at end of file From 7c86b92f739a3a88aefa6f983dcd94804fe50be7 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 16 Jul 2025 13:37:44 +0200 Subject: [PATCH 077/105] try to add test cmds to hpc ci --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 81fb4033d..13186d6b6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -34,7 +34,7 @@ jobs: downstream-ci-hpc: name: downstream-ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main + uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/try_to_add_test_cmd_to_hpc_ci with: polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} secrets: inherit From 0ff2252ba8fb29ca5760434e9526bd4ba27d77db Mon Sep 17 00:00:00 2001 From: Iain Russell <40060766+iainrussell@users.noreply.github.com> Date: Wed, 16 Jul 2025 14:01:58 +0100 Subject: [PATCH 078/105] Test ci fixes on hpc --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 13186d6b6..26bcc9192 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,7 +24,7 @@ jobs: downstream-ci: name: downstream-ci if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml@main + uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml@fix-polytope with: polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} codecov_upload: true @@ -34,7 +34,7 @@ jobs: downstream-ci-hpc: name: downstream-ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/try_to_add_test_cmd_to_hpc_ci + uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@fix-polytope with: polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} secrets: inherit From 5d016354cc9164198866a49e126f6f53613149b8 Mon Sep 17 00:00:00 2001 From: Iain Russell <40060766+iainrussell@users.noreply.github.com> Date: Wed, 16 Jul 2025 14:30:47 +0100 Subject: [PATCH 079/105] Add hpc-specific pytest command --- .github/ci-hpc-config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml index 456c4c0f2..98a9a857c 100644 --- a/.github/ci-hpc-config.yml +++ b/.github/ci-hpc-config.yml @@ -16,4 +16,6 @@ build: parallel: 64 env: - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples - - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions \ No newline at end of file + - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions + pytest_cmd: | + python -m pytest -m 'not fdb' tests --cov=./ --cov-report=xml From 16b86b6c2be68922b80dc852528a599ae432762e Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 16 Jul 2025 16:23:09 +0200 Subject: [PATCH 080/105] update branch of ci --- .github/workflows/ci.yaml | 2 +- .gitignore | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 13186d6b6..81fb4033d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -34,7 +34,7 @@ jobs: downstream-ci-hpc: name: downstream-ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/try_to_add_test_cmd_to_hpc_ci + uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main with: polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} secrets: inherit diff --git a/.gitignore b/.gitignore index 78e2f722a..4f156bc34 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,5 @@ rust_deployment_venv **/build _version.py *.DS_Store -rust_venv \ No newline at end of file +rust_venv +test_w_qubed \ No newline at end of file From 7a1d63f4c0e161f8a65d33346fdbf7c24cf63370 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 16 Jul 2025 20:25:32 +0200 Subject: [PATCH 081/105] change downstream ci branch to main --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 458f947ed..81fb4033d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,7 +24,7 @@ jobs: downstream-ci: name: downstream-ci if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml@fix-polytope + uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml@main with: polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }} codecov_upload: true From d9a47910d63ae07e22e62de64da341d174ee056a Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 22 Jul 2025 13:45:26 +0200 Subject: [PATCH 082/105] WIP: add right metadata to constructed qube and flatten metadata at gj request time --- polytope_feature/datacube/backends/qubed.py | 14 ++++++++++++-- polytope_feature/engine/qubed_slicer.py | 2 ++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/polytope_feature/datacube/backends/qubed.py b/polytope_feature/datacube/backends/qubed.py index b471569cb..e42aaa734 100644 --- a/polytope_feature/datacube/backends/qubed.py +++ b/polytope_feature/datacube/backends/qubed.py @@ -149,6 +149,10 @@ def get(self, requests, context=None): complete_list_complete_uncompressed_requests = [] complete_fdb_decoding_info = [] for j, compressed_request in enumerate(fdb_requests): + + compressed_metadata = compressed_request[2] + + # TODO: get uncompressed metadata for each leaf uncompressed_request = {} # Need to determine the possible decompressed requests @@ -196,10 +200,15 @@ def get_fdb_requests( fdb_requests=[], fdb_requests_decoding_info=[], leaf_path=None, + leaf_metadata=None, ): + # TODO: collect leaf metadata from qube here too if leaf_path is None: leaf_path = {} + if leaf_metadata is None: + leaf_metadata = {} + # First when request node is root, go to its children if requests.key == "root": logging.debug("Looking for data for the tree") @@ -225,6 +234,7 @@ def get_fdb_requests( new_vals.append(val[:4] + val[5:7] + val[8:10]) key_value_path[requests.key] = new_vals leaf_path.update(key_value_path) + leaf_metadata.update(requests.metadata) if len(requests.children[0].children[0].children) == 0: # find the fdb_requests and associated nodes to which to add results (path, current_start_idxs, fdb_node_ranges, lat_length) = self.get_2nd_last_values(requests, leaf_path) @@ -233,13 +243,13 @@ def get_fdb_requests( sorted_request_ranges, fdb_node_ranges, ) = self.sort_fdb_request_ranges(current_start_idxs, lat_length, fdb_node_ranges) - fdb_requests.append((path, sorted_request_ranges)) + fdb_requests.append((path, sorted_request_ranges, leaf_metadata)) fdb_requests_decoding_info.append((original_indices, fdb_node_ranges)) # Otherwise remap the path for this key and iterate again over children else: for c in requests.children: - self.get_fdb_requests(c, fdb_requests, fdb_requests_decoding_info, leaf_path) + self.get_fdb_requests(c, fdb_requests, fdb_requests_decoding_info, leaf_path, leaf_metadata) def remove_duplicates_in_request_ranges(self, fdb_node_ranges, current_start_idxs): # seen_indices = set() diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index b71f9f2f2..5aea08d06 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -172,6 +172,8 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations) -> list # find values on child that are within extents found_vals = self.find_values_between(poly, ax, child, datacube, lower, upper) + # TODO: find the indexes of the found_vals wrt child.values, to extract the right metadata that we want to keep inside self.build_branch + if len(found_vals) == 0: continue From 7d5b28118fdde5d062df7e0a3aa8702c0c861abf Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 22 Jul 2025 14:55:13 +0200 Subject: [PATCH 083/105] merge with new qubed develop --- polytope_feature/datacube/backends/qubed.py | 6 ++++-- polytope_feature/engine/qubed_slicer.py | 18 ++++++++-------- tests/test_qubed_extraction_engine.py | 24 +++++++++++---------- tests/test_qubed_extraction_service.py | 2 +- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/polytope_feature/datacube/backends/qubed.py b/polytope_feature/datacube/backends/qubed.py index e42aaa734..7c5f5c4ef 100644 --- a/polytope_feature/datacube/backends/qubed.py +++ b/polytope_feature/datacube/backends/qubed.py @@ -413,8 +413,10 @@ def sort_fdb_request_ranges(self, current_start_idx, lat_length, fdb_node_ranges sorted_list = sorted(enumerate(old_interm_start_idx[j]), key=lambda x: x[1]) original_indices_idx, interm_start_idx = zip(*sorted_list) for interm_fdb_nodes_obj in interm_fdb_nodes[j]: - interm_fdb_nodes_obj.data.values = QEnum(tuple([list(interm_fdb_nodes_obj.values)[k] - for k in original_indices_idx])) + # interm_fdb_nodes_obj.data.values = QEnum(tuple([list(interm_fdb_nodes_obj.values)[k] + # for k in original_indices_idx])) + interm_fdb_nodes_obj.values = QEnum(tuple([list(interm_fdb_nodes_obj.values)[k] + for k in original_indices_idx])) if abs(interm_start_idx[-1] + 1 - interm_start_idx[0]) <= len(interm_start_idx): current_request_ranges = (interm_start_idx[0], interm_start_idx[-1] + 1) interm_request_ranges.append(current_request_ranges) diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index 0827de5ad..f4f6844e9 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -27,9 +27,9 @@ def find_datacube_vals(): def find_values_between(self, polytope, ax, node, datacube, lower, upper, path=None): if isinstance(ax, UnsliceableDatacubeAxis): - # print(node.values) - # print(lower) - # print(ax.name) + print(node.values) + print(lower) + print(ax.name) return [v for v in node.values if lower <= v <= upper] tol = ax.tol @@ -151,10 +151,10 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations) -> list # TODO: when we have an axis that we would like to merge with another, we should skip the node creation here # and instead keep/cache the value to merge with the node from before?? - qube_node = Qube.make(key=grid_axes[0], - values=QEnum([found_val]), - metadata={}, - children=children) + qube_node = Qube.make_node(key=grid_axes[0], + values=QEnum([found_val]), + metadata={}, + children=children) result.append(qube_node) for i, child in enumerate(q.children): @@ -189,7 +189,7 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations) -> list if final_children_and_vals is None: continue - result.extend([Qube.make( + result.extend([Qube.make_node( key=child.key, values=QEnum(new_found_vals), metadata=child.metadata, @@ -226,7 +226,7 @@ def _slice_second_grid_axis(self, axis_name, polytopes, datacube, datacube_trans # NOTE this was the last axis so we do not have children... - result.extend([Qube.make( + result.extend([Qube.make_node( key=axis_name, values=QEnum(new_found_vals), metadata={}, diff --git a/tests/test_qubed_extraction_engine.py b/tests/test_qubed_extraction_engine.py index bf4bb17a6..3d0d98bbb 100644 --- a/tests/test_qubed_extraction_engine.py +++ b/tests/test_qubed_extraction_engine.py @@ -37,7 +37,7 @@ def find_relevant_subcube_from_request(request, qube_url): fdb_tree = Qube.from_json(requests.get( - "https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/climate_dt.json").json()) + "https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/climate-dt.json").json()) combi_polytopes = [ @@ -173,16 +173,16 @@ def find_relevant_subcube_from_request(request, qube_url): # Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), # ) -request = Request(ConvexPolytope(["param"], [["164"]]), +request = Request(ConvexPolytope(["param"], [[164]]), ConvexPolytope(["time"], [[pd.Timedelta(hours=1, minutes=0)], [pd.Timedelta(hours=3, minutes=0)]]), ConvexPolytope(["resolution"], [["high"]]), ConvexPolytope(["type"], [["fc"]]), ConvexPolytope(["model"], [['ifs-nemo']]), ConvexPolytope(["stream"], [["clte"]]), - ConvexPolytope(["realization"], ["1"]), + ConvexPolytope(["realization"], [[1]]), ConvexPolytope(["expver"], [['0001']]), ConvexPolytope(["experiment"], [['ssp3-7.0']]), - ConvexPolytope(["generation"], [["1"]]), + ConvexPolytope(["generation"], [[1]]), ConvexPolytope(["levtype"], [["sfc"]]), ConvexPolytope(["activity"], [["scenariomip"]]), ConvexPolytope(["dataset"], [["climate-dt"]]), @@ -201,7 +201,8 @@ def find_relevant_subcube_from_request(request, qube_url): options=options, ) time1 = time.time() -result = self_API.retrieve(request) +# result = self_API.retrieve(request) +result = self_API.slice(request.polytopes()) time2 = time.time() print(result) @@ -249,11 +250,11 @@ def find_relevant_subcube_from_request(request, qube_url): fdbdatacube = gj.GribJump() slicer = HullSlicer() -self_API = Polytope( - datacube=fdbdatacube, - engine=slicer, - options=options, -) +# self_API = Polytope( +# datacube=fdbdatacube, +# engine=slicer, +# options=options, +# ) request = Request(ConvexPolytope(["param"], [["164"]]), @@ -274,7 +275,8 @@ def find_relevant_subcube_from_request(request, qube_url): ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]])) time3 = time.time() -result = self_API.retrieve(request) +# result = self_API.retrieve(request) +# result = self_API.slice(request.polytopes()) time4 = time.time() print("TIME EXTRACTING USING GJ NORMAL") diff --git a/tests/test_qubed_extraction_service.py b/tests/test_qubed_extraction_service.py index 3a7a5dd0a..3bcb18eeb 100644 --- a/tests/test_qubed_extraction_service.py +++ b/tests/test_qubed_extraction_service.py @@ -44,7 +44,7 @@ def get_fdb_tree(request): fdb_tree = Qube.from_json(requests.get( - "https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/climate_dt.json").json()) + "https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/climate-dt.json").json()) # print(fdb_tree) From 27d73e60b54c21e4b19e93f94d9f5caf2523e17c Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 22 Jul 2025 15:44:15 +0200 Subject: [PATCH 084/105] WIP: collect right metadata in return qube --- polytope_feature/datacube/backends/qubed.py | 9 +++- polytope_feature/engine/qubed_slicer.py | 48 ++++++++++++++------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/polytope_feature/datacube/backends/qubed.py b/polytope_feature/datacube/backends/qubed.py index 7c5f5c4ef..bc0c22b9d 100644 --- a/polytope_feature/datacube/backends/qubed.py +++ b/polytope_feature/datacube/backends/qubed.py @@ -134,7 +134,14 @@ def get_indices(self, path, path_node, axis, lower, upper, method=None): logging.debug(f"For axis {axis.name} between {lower} and {upper}, found indices {idx_between}") - return idx_between + # print("NOW") + # # print(indexes) + # # print(idx_between) + # print(all(idx in indexes for idx in idx_between)) + + indexes = [indexes.index(item) for item in idx_between] + + return (idx_between, indexes) def get(self, requests, context=None): if context is None: diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index f4f6844e9..b41d80991 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -27,17 +27,20 @@ def find_datacube_vals(): def find_values_between(self, polytope, ax, node, datacube, lower, upper, path=None): if isinstance(ax, UnsliceableDatacubeAxis): - print(node.values) - print(lower) - print(ax.name) - return [v for v in node.values if lower <= v <= upper] + # print(node.values) + # print(lower) + # print(ax.name) + values = [v for v in node.values if lower <= v <= upper] + indices = [i for i, v in enumerate(node.values) if lower <= v <= upper] + # return [v for v in node.values if lower <= v <= upper] + return values, indices tol = ax.tol lower = ax.from_float(lower - tol) upper = ax.from_float(upper + tol) method = polytope.method - values = datacube.get_indices(path, node, ax, lower, upper, method) - return values + values, indexes = datacube.get_indices(path, node, ax, lower, upper, method) + return values, indexes def get_sliced_polys(self, found_vals, ax, child_name, poly, slice_axis_idx): sliced_polys = [] @@ -66,7 +69,7 @@ def find_new_vals(self, found_vals, ax): new_found_vals.append(found_val) return new_found_vals - def build_branch(self, real_uncompressed_axis, found_vals, sliced_polys, polytopes, poly, child, datacube, datacube_transformations, ax): + def build_branch(self, real_uncompressed_axis, found_vals, sliced_polys, polytopes, poly, child, datacube, datacube_transformations, ax, idxs=None): final_children_and_vals = [] if real_uncompressed_axis: for i, found_val in enumerate(found_vals): @@ -78,7 +81,13 @@ def build_branch(self, real_uncompressed_axis, found_vals, sliced_polys, polytop continue new_found_vals = self.find_new_vals([found_val], ax) - final_children_and_vals.append((children, new_found_vals)) + + if idxs: + request_child_val = (children, new_found_vals, [idxs[i]]) + else: + request_child_val = (children, new_found_vals) + # final_children_and_vals.append((children, new_found_vals)) + final_children_and_vals.append(request_child_val) else: # if it's compressed, then can add all found values in a single node child_polytopes = self.find_children_polytopes(polytopes, poly, sliced_polys) @@ -89,7 +98,13 @@ def build_branch(self, real_uncompressed_axis, found_vals, sliced_polys, polytop return None new_found_vals = self.find_new_vals(found_vals, ax) - final_children_and_vals.append((children, new_found_vals)) + # final_children_and_vals.append((children, new_found_vals)) + if idxs: + request_child_val = (children, new_found_vals, idxs) + else: + request_child_val = (children, new_found_vals) + final_children_and_vals.append(request_child_val) + if len(final_children_and_vals) == 0: return None return final_children_and_vals @@ -118,7 +133,7 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations) -> list ax = datacube._axes[grid_axes[0]] lower, upper, slice_axis_idx = poly.extents(grid_axes[0]) - found_vals = self.find_values_between(poly, ax, None, datacube, lower, upper) + found_vals, _ = self.find_values_between(poly, ax, None, datacube, lower, upper) if len(found_vals) == 0: continue @@ -173,7 +188,7 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations) -> list lower, upper, slice_axis_idx = poly.extents(child.key) # find values on child that are within extents - found_vals = self.find_values_between(poly, ax, child, datacube, lower, upper) + found_vals, idxs = self.find_values_between(poly, ax, child, datacube, lower, upper) # TODO: find the indexes of the found_vals wrt child.values, to extract the right metadata that we want to keep inside self.build_branch @@ -185,16 +200,19 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations) -> list axis_compressed = (child.key in self.compressed_axes) real_uncompressed_axis = not axis_compressed and len(found_vals) > 1 final_children_and_vals = self.build_branch( - real_uncompressed_axis, found_vals, sliced_polys, polytopes, poly, child, datacube, datacube_transformations, ax) + real_uncompressed_axis, found_vals, sliced_polys, polytopes, poly, child, datacube, datacube_transformations, ax, idxs) if final_children_and_vals is None: continue + print([{k: vs[idxs] for k, vs in child.metadata.items()} + for (children, new_found_vals, idxs) in final_children_and_vals]) result.extend([Qube.make_node( key=child.key, values=QEnum(new_found_vals), - metadata=child.metadata, + # metadata=child.metadata, + metadata={k: vs[idxs] for k, vs in child.metadata.items()}, children=children - ) for (children, new_found_vals) in final_children_and_vals]) + ) for (children, new_found_vals, idxs) in final_children_and_vals]) return result @@ -206,7 +224,7 @@ def _slice_second_grid_axis(self, axis_name, polytopes, datacube, datacube_trans ax = datacube._axes[axis_name] lower, upper, slice_axis_idx = poly.extents(axis_name) - found_vals = self.find_values_between(poly, ax, None, datacube, lower, upper, path) + found_vals, _ = self.find_values_between(poly, ax, None, datacube, lower, upper, path) if len(found_vals) == 0: continue From a684046818def1c99a3dbdd21805234007d3b64a Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 23 Jul 2025 13:06:32 +0200 Subject: [PATCH 085/105] keep right metadata during qubed slicing --- polytope_feature/datacube/backends/qubed.py | 8 +- polytope_feature/engine/engine.py | 5 +- polytope_feature/engine/qubed_slicer.py | 92 ++++-- ...test_qubed_extraction_engine_w_metadata.py | 262 ++++++++++++++++++ 4 files changed, 347 insertions(+), 20 deletions(-) create mode 100644 tests/test_qubed_extraction_engine_w_metadata.py diff --git a/polytope_feature/datacube/backends/qubed.py b/polytope_feature/datacube/backends/qubed.py index bc0c22b9d..f43aabdd9 100644 --- a/polytope_feature/datacube/backends/qubed.py +++ b/polytope_feature/datacube/backends/qubed.py @@ -139,7 +139,13 @@ def get_indices(self, path, path_node, axis, lower, upper, method=None): # # print(idx_between) # print(all(idx in indexes for idx in idx_between)) - indexes = [indexes.index(item) for item in idx_between] + if path_node: + # print(path_node.key) + # print(indexes) + # print(idx_between) + indexes = [indexes.index(item) for item in idx_between] + else: + indexes = None return (idx_between, indexes) diff --git a/polytope_feature/engine/engine.py b/polytope_feature/engine/engine.py index 653a6a2b8..696d44a97 100644 --- a/polytope_feature/engine/engine.py +++ b/polytope_feature/engine/engine.py @@ -80,6 +80,7 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): self.pre_process_polytopes(datacube, polytopes) # assert isinstance(datacube, QubedDatacube) tree = self.build_tree(polytopes, datacube) - print("WHAT DOES THE TREE LOOK LIKE??") - print(tree) + # print("WHAT DOES THE TREE LOOK LIKE??") + # print(tree) + # print([leaf for leaf in tree.leaves_with_metadata()]) return tree diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index b41d80991..7da028448 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -11,6 +11,7 @@ from ..shapes import ConvexPolytope, Product from ..utility.combinatorics import group, tensor_product, find_polytopes_on_axis, find_polytope_combinations from typing import List +import numpy as np from ..datacube.backends.datacube import Datacube @@ -26,6 +27,9 @@ def find_datacube_vals(): pass def find_values_between(self, polytope, ax, node, datacube, lower, upper, path=None): + # print(node.values) + # print(lower) + # print(ax.name) if isinstance(ax, UnsliceableDatacubeAxis): # print(node.values) # print(lower) @@ -69,13 +73,15 @@ def find_new_vals(self, found_vals, ax): new_found_vals.append(found_val) return new_found_vals - def build_branch(self, real_uncompressed_axis, found_vals, sliced_polys, polytopes, poly, child, datacube, datacube_transformations, ax, idxs=None): + def build_branch(self, real_uncompressed_axis, found_vals, sliced_polys, polytopes, poly, child, datacube, datacube_transformations, ax, idxs=None, compressed_idxs=None): final_children_and_vals = [] if real_uncompressed_axis: for i, found_val in enumerate(found_vals): sliced_polys_ = [sliced_polys[i]] child_polytopes = self.find_children_polytopes(polytopes, poly, sliced_polys_) - children = self._slice(child, child_polytopes, datacube, datacube_transformations) + if idxs: + compressed_idxs.append([idxs[i]]) + children = self._slice(child, child_polytopes, datacube, datacube_transformations, compressed_idxs) # If this node used to have children but now has none due to filtering, skip it. if child.children and not children: continue @@ -83,7 +89,8 @@ def build_branch(self, real_uncompressed_axis, found_vals, sliced_polys, polytop new_found_vals = self.find_new_vals([found_val], ax) if idxs: - request_child_val = (children, new_found_vals, [idxs[i]]) + # request_child_val = (children, new_found_vals, [idxs[i]]) + request_child_val = (children, new_found_vals, compressed_idxs) else: request_child_val = (children, new_found_vals) # final_children_and_vals.append((children, new_found_vals)) @@ -92,7 +99,9 @@ def build_branch(self, real_uncompressed_axis, found_vals, sliced_polys, polytop # if it's compressed, then can add all found values in a single node child_polytopes = self.find_children_polytopes(polytopes, poly, sliced_polys) # create children - children = self._slice(child, child_polytopes, datacube, datacube_transformations) + if idxs: + compressed_idxs.append([idxs]) + children = self._slice(child, child_polytopes, datacube, datacube_transformations, compressed_idxs) # If this node used to have children but now has none due to filtering, skip it. if child.children and not children: return None @@ -100,16 +109,21 @@ def build_branch(self, real_uncompressed_axis, found_vals, sliced_polys, polytop new_found_vals = self.find_new_vals(found_vals, ax) # final_children_and_vals.append((children, new_found_vals)) if idxs: - request_child_val = (children, new_found_vals, idxs) + # request_child_val = (children, new_found_vals, idxs) + request_child_val = (children, new_found_vals, compressed_idxs) else: request_child_val = (children, new_found_vals) final_children_and_vals.append(request_child_val) if len(final_children_and_vals) == 0: return None + # print(child.key) + # print(idxs) return final_children_and_vals - def _slice(self, q: Qube, polytopes, datacube, datacube_transformations) -> list[Qube]: + def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compressed_idxs=None) -> list[Qube]: + if compressed_idxs is None: + compressed_idxs = [[[0]]] result = [] if len(q.children) == 0: @@ -172,6 +186,7 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations) -> list children=children) result.append(qube_node) + # combined_idxs = [] for i, child in enumerate(q.children): # find polytopes which are defined on axis child.key polytopes_on_axis = find_polytopes_on_axis(child.key, polytopes) @@ -181,7 +196,6 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations) -> list datacube.add_axes_dynamically(child) # here now first change the values in the polytopes on the axis to reflect the axis type - for poly in polytopes_on_axis: ax = datacube._axes[child.key] # find extents of polytope on child.key @@ -200,19 +214,63 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations) -> list axis_compressed = (child.key in self.compressed_axes) real_uncompressed_axis = not axis_compressed and len(found_vals) > 1 final_children_and_vals = self.build_branch( - real_uncompressed_axis, found_vals, sliced_polys, polytopes, poly, child, datacube, datacube_transformations, ax, idxs) + real_uncompressed_axis, found_vals, sliced_polys, polytopes, poly, child, datacube, datacube_transformations, ax, idxs, compressed_idxs) + # combined_idxs.append(idxs) + print(compressed_idxs) + print([{k: child.metadata[k].shape} for k in child.metadata.keys()]) + # print([(ax.name, idxs) for (children, new_found_vals, idxs) in final_children_and_vals]) if final_children_and_vals is None: continue - print([{k: vs[idxs] for k, vs in child.metadata.items()} - for (children, new_found_vals, idxs) in final_children_and_vals]) - result.extend([Qube.make_node( - key=child.key, - values=QEnum(new_found_vals), - # metadata=child.metadata, - metadata={k: vs[idxs] for k, vs in child.metadata.items()}, - children=children - ) for (children, new_found_vals, idxs) in final_children_and_vals]) + # print(child.metadata.items()) + # print("HERE") + # print(child.key) + # print("\n\n\n\n\n\n") + # print(child.metadata) + # print("\n\n\n\n\n\n") + # print([(ax.name, idxs) for (children, new_found_vals, idxs) in final_children_and_vals]) + # print([{k: vs[idxs] for k, vs in child.metadata.items()} + # for (children, new_found_vals, idxs) in final_children_and_vals if idxs]) + + # result.extend([Qube.make_node( + # key=child.key, + # values=QEnum(new_found_vals), + # metadata=child.metadata, + # # metadata={k: vs[idxs] for k, vs in child.metadata.items()}, + # children=children + # ) for (children, new_found_vals, idxs) in final_children_and_vals]) + + def format_metadata_idxs(idxs): + squeezed_indices = [np.squeeze(i) for i in idxs] + + def format_index(idx): + if isinstance(idx, np.ndarray) and idx.ndim > 0: + return idx.tolist() # for multiple indices + return int(idx) # for scalar index + index_tuple = tuple(format_index(i) for i in squeezed_indices) + return index_tuple + + def find_metadata(metadata_idx): + metadata = {} + for k, vs in child.metadata.items(): + metadata_depth = len(vs) + relevant_metadata_dxs = metadata_idx[:metadata_depth] + metadata[k] = vs[relevant_metadata_dxs] + return metadata + + # print(len(final_children_and_vals)) + + for (children, new_found_vals, idxs) in final_children_and_vals: + metadata_idx = format_metadata_idxs(idxs) + metadata = find_metadata(metadata_idx) + qube_node = Qube.make_node( + key=child.key, + values=QEnum(new_found_vals), + # metadata=child.metadata, + # metadata={k: vs[metadata_idx] for k, vs in child.metadata.items()}, + metadata=metadata, + children=children) + result.append(qube_node) return result diff --git a/tests/test_qubed_extraction_engine_w_metadata.py b/tests/test_qubed_extraction_engine_w_metadata.py new file mode 100644 index 000000000..8b938119a --- /dev/null +++ b/tests/test_qubed_extraction_engine_w_metadata.py @@ -0,0 +1,262 @@ +from polytope_feature.shapes import Box, Select, Span +from polytope_feature.polytope import Polytope, Request +from polytope_feature.engine.qubed_slicer import QubedSlicer +from polytope_feature.datacube.backends.qubed import QubedDatacube +from polytope_feature.datacube.backends.fdb import FDBDatacube +import pytest +from qubed import Qube +import requests +from polytope_feature.datacube.datacube_axis import PandasTimedeltaDatacubeAxis, PandasTimestampDatacubeAxis, UnsliceableDatacubeAxis, FloatDatacubeAxis +# from polytope_feature.datacube.backends.test_qubed_slicing import actual_slice +from polytope_feature.datacube.transformations.datacube_type_change.datacube_type_change import TypeChangeStrToTimestamp, TypeChangeStrToTimedelta +import pandas as pd +from polytope_feature.datacube.transformations.datacube_mappers.mapper_types.healpix_nested import NestedHealpixGridMapper + +from polytope_feature.shapes import ConvexPolytope +import time +import pygribjump as gj +from polytope_feature.engine.hullslicer import HullSlicer + + +def find_relevant_subcube_from_request(request, qube_url): + + # NOTE: final url we want is like: + # "https://qubed.lumi.apps.dte.destination-earth.eu/api/v1/select/climate-dt/?class=d1&dataset=climate-dt" + + for shape in request.shapes: + if isinstance(shape, Select): + qube_url += shape.axis + "=" + for i, val in enumerate(shape.values): + qube_url += str(val) + if i < len(shape.values) - 1: + qube_url += "," + qube_url += "&" + # TODO: remove last unnecessary & + qube_url = qube_url[:-1] + return qube_url + + +fdb_tree = Qube.from_json(requests.get( + "https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/extremes-dt_with_metadata.json").json()) + + +# combi_polytopes = [ +# ConvexPolytope(["param"], [["164"]]), +# ConvexPolytope(["time"], [[pd.Timedelta(hours=0, minutes=0)], [pd.Timedelta(hours=12, minutes=0)]]), +# ConvexPolytope(["resolution"], [["high"]]), +# ConvexPolytope(["type"], [["fc"]]), +# ConvexPolytope(["model"], [['ifs-nemo']]), +# ConvexPolytope(["stream"], [["clte"]]), +# ConvexPolytope(["realization"], ["1"]), +# ConvexPolytope(["expver"], [['0001']]), +# ConvexPolytope(["experiment"], [['ssp3-7.0']]), +# ConvexPolytope(["generation"], [["1"]]), +# ConvexPolytope(["levtype"], [["sfc"]]), +# ConvexPolytope(["activity"], [["scenariomip"]]), +# ConvexPolytope(["dataset"], [["climate-dt"]]), +# ConvexPolytope(["class"], [["d1"]]), +# ConvexPolytope(["date"], [[pd.Timestamp("20220811")], [pd.Timestamp("20220912")]]), +# ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]]) +# ] + +# TODO: add lat and lon axes +datacube_axes = {"param": UnsliceableDatacubeAxis(), + "time": PandasTimedeltaDatacubeAxis(), + "resolution": UnsliceableDatacubeAxis(), + "type": UnsliceableDatacubeAxis(), + "model": UnsliceableDatacubeAxis(), + "stream": UnsliceableDatacubeAxis(), + "realization": UnsliceableDatacubeAxis(), + "expver": UnsliceableDatacubeAxis(), + "experiment": UnsliceableDatacubeAxis(), + "generation": UnsliceableDatacubeAxis(), + "levtype": UnsliceableDatacubeAxis(), + "activity": UnsliceableDatacubeAxis(), + "dataset": UnsliceableDatacubeAxis(), + "class": UnsliceableDatacubeAxis(), + "date": PandasTimestampDatacubeAxis(), + # "latitude": FloatDatacubeAxis(), + # "longitude": FloatDatacubeAxis() + } + +time_val = pd.Timedelta(hours=0, minutes=0) +date_val = pd.Timestamp("20300101T000000") + + +# TODO: add grid axis transformation +datacube_transformations = { + "time": TypeChangeStrToTimedelta("time", time_val), + "date": TypeChangeStrToTimestamp("date", date_val), + "values": NestedHealpixGridMapper("values", ["latitude", "longitude"], 1024) +} + + +options = { + "axis_config": [ + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, + # { + # "axis_name": "date", + # "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + # }, + {"axis_name": "date", "transformations": [{"name": "type_change", "type": "date"}]}, + {"axis_name": "time", "transformations": [{"name": "type_change", "type": "time"}]}, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "octahedral", "resolution": 2560, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ], + "compressed_axes_config": [ + "longitude", + "latitude", + "levtype", + "step", + "date", + "domain", + "expver", + "param", + "class", + "stream", + "type", + ], + "pre_path": {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"}, + "datacube_axes": {"param": "UnsliceableDatacubeAxis", + "time": "PandasTimedeltaDatacubeAxis", + "type": "UnsliceableDatacubeAxis", + "stream": "UnsliceableDatacubeAxis", + "expver": "UnsliceableDatacubeAxis", + "levtype": "UnsliceableDatacubeAxis", + "dataset": "UnsliceableDatacubeAxis", + "class": "UnsliceableDatacubeAxis", + "date": "PandasTimestampDatacubeAxis", + "step": "IntDatacubeAxis", + } +} + +# request = Request( +# Select("step", [0]), +# Select("levtype", ["sfc"]), +# Select("date", [pd.Timestamp("20230625T120000")]), +# Select("domain", ["g"]), +# Select("expver", ["0001"]), +# Select("param", ["167"]), +# Select("class", ["od"]), +# Select("stream", ["oper"]), +# Select("type", ["an"]), +# Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), +# ) + +request = Request(ConvexPolytope(["param"], [["167"]]), + ConvexPolytope(["time"], [[pd.Timedelta(hours=0, minutes=0)], [pd.Timedelta(hours=3, minutes=0)]]), + ConvexPolytope(["type"], [["fc"]]), + ConvexPolytope(["stream"], [["oper"]]), + ConvexPolytope(["expver"], [['0001']]), + ConvexPolytope(["levtype"], [["sfc"]]), + ConvexPolytope(["dataset"], [["extremes-dt"]]), + ConvexPolytope(["class"], [["d1"]]), + ConvexPolytope(["step"], [[1]]), + ConvexPolytope(["date"], [[pd.Timestamp("20240407")]]), + ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]]) + # ConvexPolytope(["latitude", "longitude"], [[0, 0], [-0.5, -0.5], [0, -0.5]]) + ) +# print(fdb_tree) +qubeddatacube = QubedDatacube(fdb_tree, datacube_axes, datacube_transformations) +slicer = QubedSlicer() +self_API = Polytope( + # datacube=qubeddatacube, + datacube=fdb_tree, + engine=slicer, + options=options, +) +time1 = time.time() +# result = self_API.retrieve(request) +result = self_API.slice(request.polytopes()) +time2 = time.time() + +print(result) + +print("TIME EXTRACTING USING QUBED") +print(time2 - time1) + +# USING NORMAL GJ + + +options = { + "axis_config": [ + {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, + {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}, + # { + # "axis_name": "date", + # "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}], + # }, + {"axis_name": "date", "transformations": [{"name": "type_change", "type": "date"}]}, + {"axis_name": "time", "transformations": [{"name": "type_change", "type": "time"}]}, + { + "axis_name": "values", + "transformations": [ + {"name": "mapper", "type": "healpix_nested", "resolution": 1024, "axes": ["latitude", "longitude"]} + ], + }, + {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": True}]}, + {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0, 360]}]}, + ], + "compressed_axes_config": [ + "longitude", + # "latitude", + # "levtype", + # "step", + # "date", + # "domain", + # "expver", + # "param", + # "class", + # "stream", + # "type", + ], + "pre_path": {"class": "d1", "model": "ifs-nemo", "resolution": "high"}, +} + +fdbdatacube = gj.GribJump() +slicer = HullSlicer() +# self_API = Polytope( +# datacube=fdbdatacube, +# engine=slicer, +# options=options, +# ) + + +request = Request(ConvexPolytope(["param"], [["164"]]), + ConvexPolytope(["time"], [[pd.Timedelta(hours=1, minutes=0)], [pd.Timedelta(hours=3, minutes=0)]]), + ConvexPolytope(["resolution"], [["high"]]), + ConvexPolytope(["type"], [["fc"]]), + ConvexPolytope(["model"], [['ifs-nemo']]), + ConvexPolytope(["stream"], [["clte"]]), + ConvexPolytope(["realization"], ["1"]), + ConvexPolytope(["expver"], [['0001']]), + ConvexPolytope(["experiment"], [['ssp3-7.0']]), + ConvexPolytope(["generation"], [["1"]]), + ConvexPolytope(["levtype"], [["sfc"]]), + ConvexPolytope(["activity"], [["scenariomip"]]), + ConvexPolytope(["dataset"], [["climate-dt"]]), + ConvexPolytope(["class"], [["d1"]]), + ConvexPolytope(["date"], [[pd.Timestamp("20220811")]]), + ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]])) + +time3 = time.time() +# result = self_API.retrieve(request) +# result = self_API.slice(request.polytopes()) +time4 = time.time() + +print("TIME EXTRACTING USING GJ NORMAL") +print(time4 - time3) + + +# print(result) + +# print(result.leaves) + +# sliced_tree = actual_slice(fdb_tree, combi_polytopes, datacube_axes, datacube_transformations) From 036f44dcceb8d7855dd0fb7e4d91f3ae920cc319 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 23 Jul 2025 14:37:26 +0200 Subject: [PATCH 086/105] finish adding metadata for gj and pass it all the way to gj --- polytope_feature/datacube/backends/qubed.py | 62 +++++++++++++------ polytope_feature/engine/qubed_slicer.py | 41 ++++++++---- ...test_qubed_extraction_engine_w_metadata.py | 4 +- 3 files changed, 74 insertions(+), 33 deletions(-) diff --git a/polytope_feature/datacube/backends/qubed.py b/polytope_feature/datacube/backends/qubed.py index f43aabdd9..556f1cd7d 100644 --- a/polytope_feature/datacube/backends/qubed.py +++ b/polytope_feature/datacube/backends/qubed.py @@ -164,6 +164,7 @@ def get(self, requests, context=None): for j, compressed_request in enumerate(fdb_requests): compressed_metadata = compressed_request[2] + # print(compressed_metadata) # TODO: get uncompressed metadata for each leaf uncompressed_request = {} @@ -176,12 +177,35 @@ def get(self, requests, context=None): interm_branch_tuple_values.append(compressed_request[0][key]) request_combis = product(*interm_branch_tuple_values) + index_combis = list(product(*[range(len(lst)) for lst in interm_branch_tuple_values])) + # Need to extract the possible requests and add them to the right nodes - for combi in request_combis: + # print("REALLY HERE") + # print(index_combis) + # print([combi for combi in request_combis]) + # print(interm_branch_tuple_values) + + def find_metadata(metadata_idx): + metadata = {} + for k, vs in compressed_metadata.items(): + metadata_depth = len(vs.shape) + # print("LOOK HERE") + # print(metadata_depth) + relevant_metadata_dxs = metadata_idx[:metadata_depth] + metadata[k] = vs[relevant_metadata_dxs] + return metadata + + for i, combi in enumerate(request_combis): + metadata_idxs = index_combis[i] + actual_metadata = find_metadata(metadata_idxs) + # print("WHAT METADATA WILL WE GIVE TO GJ") + # print(actual_metadata) + uncompressed_request = {} for i, key in enumerate(compressed_request[0].keys()): uncompressed_request[key] = combi[i] - complete_uncompressed_request = (uncompressed_request, compressed_request[1], self.grid_md5_hash) + # complete_uncompressed_request = (uncompressed_request, compressed_request[1], self.grid_md5_hash) + complete_uncompressed_request = (actual_metadata, compressed_request[1], self.grid_md5_hash) complete_list_complete_uncompressed_requests.append(complete_uncompressed_request) complete_fdb_decoding_info.append(fdb_requests_decoding_info[j]) @@ -189,23 +213,23 @@ def get(self, requests, context=None): printed_list_to_gj = complete_list_complete_uncompressed_requests[::1000] logging.debug("The requests we give GribJump are: %s", printed_list_to_gj) logging.info("Requests given to GribJump extract for %s", context) - try: - output_values = self.gj.extract(complete_list_complete_uncompressed_requests, context) - except Exception as e: - if "BadValue: Grid hash mismatch" in str(e): - logging.info("Error is: %s", e) - raise BadGridError() - if "Missing JumpInfo" in str(e): - logging.info("Error is: %s", e) - raise GribJumpNoIndexError() - else: - raise e - - logging.info("Requests extracted from GribJump for %s", context) - if logging.root.level <= logging.DEBUG: - printed_output_values = output_values[::1000] - logging.debug("GribJump outputs: %s", printed_output_values) - self.assign_fdb_output_to_nodes(output_values, complete_fdb_decoding_info) + # try: + # output_values = self.gj.extract(complete_list_complete_uncompressed_requests, context) + # except Exception as e: + # if "BadValue: Grid hash mismatch" in str(e): + # logging.info("Error is: %s", e) + # raise BadGridError() + # if "Missing JumpInfo" in str(e): + # logging.info("Error is: %s", e) + # raise GribJumpNoIndexError() + # else: + # raise e + + # logging.info("Requests extracted from GribJump for %s", context) + # if logging.root.level <= logging.DEBUG: + # printed_output_values = output_values[::1000] + # logging.debug("GribJump outputs: %s", printed_output_values) + # self.assign_fdb_output_to_nodes(output_values, complete_fdb_decoding_info) def get_fdb_requests( self, diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index 7da028448..5b19fd6b4 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -217,8 +217,8 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compres real_uncompressed_axis, found_vals, sliced_polys, polytopes, poly, child, datacube, datacube_transformations, ax, idxs, compressed_idxs) # combined_idxs.append(idxs) - print(compressed_idxs) - print([{k: child.metadata[k].shape} for k in child.metadata.keys()]) + # print(compressed_idxs) + # print([{k: child.metadata[k].shape} for k in child.metadata.keys()]) # print([(ax.name, idxs) for (children, new_found_vals, idxs) in final_children_and_vals]) if final_children_and_vals is None: continue @@ -241,28 +241,45 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compres # ) for (children, new_found_vals, idxs) in final_children_and_vals]) def format_metadata_idxs(idxs): - squeezed_indices = [np.squeeze(i) for i in idxs] + # squeezed_indices = [np.squeeze(i) for i in idxs] - def format_index(idx): - if isinstance(idx, np.ndarray) and idx.ndim > 0: - return idx.tolist() # for multiple indices - return int(idx) # for scalar index - index_tuple = tuple(format_index(i) for i in squeezed_indices) - return index_tuple + # def format_index(idx): + # if isinstance(idx, np.ndarray) and idx.ndim > 0: + # return idx.tolist() # for multiple indices + # return int(idx) # for scalar index + # index_tuple = tuple(format_index(i) for i in squeezed_indices) + # return index_tuple + flat_indices = [np.ravel(idx) for idx in idxs] + return flat_indices def find_metadata(metadata_idx): metadata = {} for k, vs in child.metadata.items(): - metadata_depth = len(vs) - relevant_metadata_dxs = metadata_idx[:metadata_depth] - metadata[k] = vs[relevant_metadata_dxs] + metadata_depth = len(vs.shape) + relevant_metadata_idxs = metadata_idx[:metadata_depth] + # print("AND WHAT ABOUT HERE") + # print(relevant_metadata_idxs) + # print(vs.shape) + # print(vs[relevant_metadata_idxs].shape) + # if len(vs[relevant_metadata_idxs].shape) == 0: + # metadata[k] = vs[relevant_metadata_idxs] + + # metadata[k] = vs[relevant_metadata_idxs] + ix = np.ix_(*relevant_metadata_idxs) + metadata[k] = vs[ix] + # print(metadata[k]) return metadata # print(len(final_children_and_vals)) for (children, new_found_vals, idxs) in final_children_and_vals: + # print("WHAT'S THE INDEX LIKE HERE?") + # print(idxs) metadata_idx = format_metadata_idxs(idxs) metadata = find_metadata(metadata_idx) + # print("HERE WHAT'S THE METADATA") + # print(metadata) + # print(metadata_idx) qube_node = Qube.make_node( key=child.key, values=QEnum(new_found_vals), diff --git a/tests/test_qubed_extraction_engine_w_metadata.py b/tests/test_qubed_extraction_engine_w_metadata.py index 8b938119a..8584049e8 100644 --- a/tests/test_qubed_extraction_engine_w_metadata.py +++ b/tests/test_qubed_extraction_engine_w_metadata.py @@ -173,8 +173,8 @@ def find_relevant_subcube_from_request(request, qube_url): options=options, ) time1 = time.time() -# result = self_API.retrieve(request) -result = self_API.slice(request.polytopes()) +result = self_API.retrieve(request) +# result = self_API.slice(request.polytopes()) time2 = time.time() print(result) From 1df4c84ac4707419972cdffa770e200e24ac29c3 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 31 Jul 2025 13:12:57 +0200 Subject: [PATCH 087/105] add metadata as gj expects in qubed backend --- polytope_feature/datacube/backends/qubed.py | 44 ++++++++++++--------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/polytope_feature/datacube/backends/qubed.py b/polytope_feature/datacube/backends/qubed.py index 556f1cd7d..f92f2a86c 100644 --- a/polytope_feature/datacube/backends/qubed.py +++ b/polytope_feature/datacube/backends/qubed.py @@ -201,11 +201,19 @@ def find_metadata(metadata_idx): # print("WHAT METADATA WILL WE GIVE TO GJ") # print(actual_metadata) + path = actual_metadata["path"][0] + scheme = actual_metadata["scheme"] + offset = actual_metadata["offset"][0] + host = actual_metadata["host"][0] + port = actual_metadata["port"] + uncompressed_request = {} for i, key in enumerate(compressed_request[0].keys()): uncompressed_request[key] = combi[i] # complete_uncompressed_request = (uncompressed_request, compressed_request[1], self.grid_md5_hash) - complete_uncompressed_request = (actual_metadata, compressed_request[1], self.grid_md5_hash) + # complete_uncompressed_request = (actual_metadata, compressed_request[1], self.grid_md5_hash) + complete_uncompressed_request = (path, scheme, offset, host, port, + compressed_request[1], self.grid_md5_hash) complete_list_complete_uncompressed_requests.append(complete_uncompressed_request) complete_fdb_decoding_info.append(fdb_requests_decoding_info[j]) @@ -213,23 +221,23 @@ def find_metadata(metadata_idx): printed_list_to_gj = complete_list_complete_uncompressed_requests[::1000] logging.debug("The requests we give GribJump are: %s", printed_list_to_gj) logging.info("Requests given to GribJump extract for %s", context) - # try: - # output_values = self.gj.extract(complete_list_complete_uncompressed_requests, context) - # except Exception as e: - # if "BadValue: Grid hash mismatch" in str(e): - # logging.info("Error is: %s", e) - # raise BadGridError() - # if "Missing JumpInfo" in str(e): - # logging.info("Error is: %s", e) - # raise GribJumpNoIndexError() - # else: - # raise e - - # logging.info("Requests extracted from GribJump for %s", context) - # if logging.root.level <= logging.DEBUG: - # printed_output_values = output_values[::1000] - # logging.debug("GribJump outputs: %s", printed_output_values) - # self.assign_fdb_output_to_nodes(output_values, complete_fdb_decoding_info) + try: + output_values = self.gj.extract_from_paths(complete_list_complete_uncompressed_requests, context) + except Exception as e: + if "BadValue: Grid hash mismatch" in str(e): + logging.info("Error is: %s", e) + raise BadGridError() + if "Missing JumpInfo" in str(e): + logging.info("Error is: %s", e) + raise GribJumpNoIndexError() + else: + raise e + + logging.info("Requests extracted from GribJump for %s", context) + if logging.root.level <= logging.DEBUG: + printed_output_values = output_values[::1000] + logging.debug("GribJump outputs: %s", printed_output_values) + self.assign_fdb_output_to_nodes(output_values, complete_fdb_decoding_info) def get_fdb_requests( self, From 4d21e3fa912348233248a67eebf81b9598931329 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 31 Jul 2025 13:31:26 +0200 Subject: [PATCH 088/105] clean up --- .../datacube/backends/datacube.py | 18 +- polytope_feature/datacube/backends/qubed.py | 57 +++-- .../datacube/backends/qubed_fdb.py | 12 +- polytope_feature/datacube/datacube_axis.py | 13 +- .../datacube_reverse/datacube_reverse.py | 3 +- polytope_feature/engine/engine.py | 10 +- polytope_feature/engine/hullslicer.py | 6 +- polytope_feature/engine/qubed_slicer.py | 154 +++++--------- polytope_feature/options.py | 1 + polytope_feature/polytope.py | 9 +- tests/test_qubed_extraction_engine.py | 196 ++++++++++-------- ...test_qubed_extraction_engine_w_metadata.py | 170 ++++++++------- tests/test_qubed_extraction_service.py | 132 ++++++------ 13 files changed, 396 insertions(+), 385 deletions(-) diff --git a/polytope_feature/datacube/backends/datacube.py b/polytope_feature/datacube/backends/datacube.py index 1d9d94de6..5b149ac1f 100644 --- a/polytope_feature/datacube/backends/datacube.py +++ b/polytope_feature/datacube/backends/datacube.py @@ -170,7 +170,15 @@ def remap_path(self, path: DatacubePath): return path @staticmethod - def create(datacube, config={}, axis_options={}, compressed_axes_options=[], alternative_axes=[], datacube_axes={}, context=None): + def create( + datacube, + config={}, + axis_options={}, + compressed_axes_options=[], + alternative_axes=[], + datacube_axes={}, + context=None, + ): # TODO: get the configs as None for pre-determined value and change them to empty dictionary inside the function if type(datacube).__name__ == "DataArray": from .xarray import XArrayDatacube @@ -185,13 +193,15 @@ def create(datacube, config={}, axis_options={}, compressed_axes_options=[], alt ) return fdbdatacube if type(datacube).__name__ == "Qube": - from .qubed import QubedDatacube from ..datacube_axis import _str_to_axis + from .qubed import QubedDatacube + actual_datacube_axes = {} for key, value in datacube_axes.items(): actual_datacube_axes[key] = _str_to_axis[value] - qubed_datacube = QubedDatacube(datacube, actual_datacube_axes, - config, axis_options, compressed_axes_options, alternative_axes, context) + qubed_datacube = QubedDatacube( + datacube, actual_datacube_axes, config, axis_options, compressed_axes_options, alternative_axes, context + ) return qubed_datacube def check_branching_axes(self, request): diff --git a/polytope_feature/datacube/backends/qubed.py b/polytope_feature/datacube/backends/qubed.py index f92f2a86c..2a4ced348 100644 --- a/polytope_feature/datacube/backends/qubed.py +++ b/polytope_feature/datacube/backends/qubed.py @@ -2,19 +2,26 @@ import operator from copy import deepcopy from itertools import product -from ...utility.exceptions import BadGridError, BadRequestError, GribJumpNoIndexError -from ...utility.geometry import nearest_pt + +import numpy as np import pygribjump as pygj from qubed.value_types import QEnum -import numpy as np +from ...utility.exceptions import BadGridError, GribJumpNoIndexError +from ...utility.geometry import nearest_pt from .datacube import Datacube, TensorIndexTree class QubedDatacube(Datacube): - def __init__( - self, q, datacube_axes, config=None, axis_options=None, compressed_axes_options=[], alternative_axes=[], context=None + self, + q, + datacube_axes, + config=None, + axis_options=None, + compressed_axes_options=[], + alternative_axes=[], + context=None, ): if config is None: config = {} @@ -134,15 +141,7 @@ def get_indices(self, path, path_node, axis, lower, upper, method=None): logging.debug(f"For axis {axis.name} between {lower} and {upper}, found indices {idx_between}") - # print("NOW") - # # print(indexes) - # # print(idx_between) - # print(all(idx in indexes for idx in idx_between)) - if path_node: - # print(path_node.key) - # print(indexes) - # print(idx_between) indexes = [indexes.index(item) for item in idx_between] else: indexes = None @@ -162,9 +161,7 @@ def get(self, requests, context=None): complete_list_complete_uncompressed_requests = [] complete_fdb_decoding_info = [] for j, compressed_request in enumerate(fdb_requests): - compressed_metadata = compressed_request[2] - # print(compressed_metadata) # TODO: get uncompressed metadata for each leaf uncompressed_request = {} @@ -180,17 +177,11 @@ def get(self, requests, context=None): index_combis = list(product(*[range(len(lst)) for lst in interm_branch_tuple_values])) # Need to extract the possible requests and add them to the right nodes - # print("REALLY HERE") - # print(index_combis) - # print([combi for combi in request_combis]) - # print(interm_branch_tuple_values) def find_metadata(metadata_idx): metadata = {} for k, vs in compressed_metadata.items(): metadata_depth = len(vs.shape) - # print("LOOK HERE") - # print(metadata_depth) relevant_metadata_dxs = metadata_idx[:metadata_depth] metadata[k] = vs[relevant_metadata_dxs] return metadata @@ -198,8 +189,6 @@ def find_metadata(metadata_idx): for i, combi in enumerate(request_combis): metadata_idxs = index_combis[i] actual_metadata = find_metadata(metadata_idxs) - # print("WHAT METADATA WILL WE GIVE TO GJ") - # print(actual_metadata) path = actual_metadata["path"][0] scheme = actual_metadata["scheme"] @@ -210,10 +199,15 @@ def find_metadata(metadata_idx): uncompressed_request = {} for i, key in enumerate(compressed_request[0].keys()): uncompressed_request[key] = combi[i] - # complete_uncompressed_request = (uncompressed_request, compressed_request[1], self.grid_md5_hash) - # complete_uncompressed_request = (actual_metadata, compressed_request[1], self.grid_md5_hash) - complete_uncompressed_request = (path, scheme, offset, host, port, - compressed_request[1], self.grid_md5_hash) + complete_uncompressed_request = ( + path, + scheme, + offset, + host, + port, + compressed_request[1], + self.grid_md5_hash, + ) complete_list_complete_uncompressed_requests.append(complete_uncompressed_request) complete_fdb_decoding_info.append(fdb_requests_decoding_info[j]) @@ -271,7 +265,7 @@ def get_fdb_requests( if requests.key == "time": new_vals = [] for val in key_value_path[requests.key]: - new_vals.append(val[7:9]+val[10:12]) + new_vals.append(val[7:9] + val[10:12]) key_value_path[requests.key] = new_vals if requests.key == "date": new_vals = [] @@ -392,8 +386,6 @@ def get_2nd_last_values(self, requests, leaf_path=None): current_start_idx = deepcopy(current_start_idxs[i]) fdb_range_nodes = deepcopy(fdb_node_ranges[i]) key_value_path = {lat_child.key: list(lat_child.values)} - # print("WHAT ARE THE DATACUBE AXES NOW??") - # print(self._axes.keys()) ax = self._axes[lat_child.key] (key_value_path, leaf_path, self.unwanted_path) = ax.unmap_path_key( key_value_path, leaf_path, self.unwanted_path @@ -460,8 +452,9 @@ def sort_fdb_request_ranges(self, current_start_idx, lat_length, fdb_node_ranges for interm_fdb_nodes_obj in interm_fdb_nodes[j]: # interm_fdb_nodes_obj.data.values = QEnum(tuple([list(interm_fdb_nodes_obj.values)[k] # for k in original_indices_idx])) - interm_fdb_nodes_obj.values = QEnum(tuple([list(interm_fdb_nodes_obj.values)[k] - for k in original_indices_idx])) + interm_fdb_nodes_obj.values = QEnum( + tuple([list(interm_fdb_nodes_obj.values)[k] for k in original_indices_idx]) + ) if abs(interm_start_idx[-1] + 1 - interm_start_idx[0]) <= len(interm_start_idx): current_request_ranges = (interm_start_idx[0], interm_start_idx[-1] + 1) interm_request_ranges.append(current_request_ranges) diff --git a/polytope_feature/datacube/backends/qubed_fdb.py b/polytope_feature/datacube/backends/qubed_fdb.py index dcd8b31fa..c3b339b29 100644 --- a/polytope_feature/datacube/backends/qubed_fdb.py +++ b/polytope_feature/datacube/backends/qubed_fdb.py @@ -3,11 +3,12 @@ from copy import deepcopy from itertools import product +import requests +from qubed import Qube + from ...utility.exceptions import BadGridError, BadRequestError from ...utility.geometry import nearest_pt from .datacube import Datacube, TensorIndexTree -from qubed import Qube -import requests class FDBDatacube(Datacube): @@ -30,8 +31,11 @@ def __init__( # Find values in the level 3 FDB datacube self.gj = gj - self.fdb_tree = Qube.from_json(requests.get( - "https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/climate_dt.json").json()) + self.fdb_tree = Qube.from_json( + requests.get( + "https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/climate_dt.json" + ).json() + ) if len(alternative_axes) == 0: logging.info("Find GribJump axes for %s", context) diff --git a/polytope_feature/datacube/datacube_axis.py b/polytope_feature/datacube/datacube_axis.py index 06fbe1953..b2032a5b6 100644 --- a/polytope_feature/datacube/datacube_axis.py +++ b/polytope_feature/datacube/datacube_axis.py @@ -355,9 +355,10 @@ def serialize(self, value): float: FloatDatacubeAxis(), } -_str_to_axis = {"FloatDatacubeAxis": FloatDatacubeAxis(), - "IntDatacubeAxis": IntDatacubeAxis(), - "UnsliceableDatacubeAxis": UnsliceableDatacubeAxis(), - "PandasTimedeltaDatacubeAxis": PandasTimedeltaDatacubeAxis(), - "PandasTimestampDatacubeAxis": PandasTimestampDatacubeAxis(), - } +_str_to_axis = { + "FloatDatacubeAxis": FloatDatacubeAxis(), + "IntDatacubeAxis": IntDatacubeAxis(), + "UnsliceableDatacubeAxis": UnsliceableDatacubeAxis(), + "PandasTimedeltaDatacubeAxis": PandasTimedeltaDatacubeAxis(), + "PandasTimestampDatacubeAxis": PandasTimestampDatacubeAxis(), +} diff --git a/polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py b/polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py index 4d3232910..7a4da9bc4 100644 --- a/polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py +++ b/polytope_feature/datacube/transformations/datacube_reverse/datacube_reverse.py @@ -1,6 +1,7 @@ +import numpy as np + from ....utility.list_tools import bisect_left_cmp, bisect_right_cmp from ..datacube_transformations import DatacubeAxisTransformation -import numpy as np class DatacubeAxisReverse(DatacubeAxisTransformation): diff --git a/polytope_feature/engine/engine.py b/polytope_feature/engine/engine.py index 696d44a97..ad8d84b09 100644 --- a/polytope_feature/engine/engine.py +++ b/polytope_feature/engine/engine.py @@ -1,12 +1,10 @@ +import math from typing import List from ..datacube.backends.datacube import Datacube -from ..datacube.tensor_index_tree import TensorIndexTree -from ..shapes import ConvexPolytope, Product - from ..datacube.datacube_axis import UnsliceableDatacubeAxis +from ..shapes import ConvexPolytope, Product from ..utility.list_tools import unique -import math class Engine: @@ -78,9 +76,5 @@ def extract(self, datacube: Datacube, polytopes: List[ConvexPolytope]): self.find_compressed_axes(datacube, polytopes) self.remove_compressed_axis_in_union(polytopes) self.pre_process_polytopes(datacube, polytopes) - # assert isinstance(datacube, QubedDatacube) tree = self.build_tree(polytopes, datacube) - # print("WHAT DOES THE TREE LOOK LIKE??") - # print(tree) - # print([leaf for leaf in tree.leaves_with_metadata()]) return tree diff --git a/polytope_feature/engine/hullslicer.py b/polytope_feature/engine/hullslicer.py index e326ee536..cb51cbe7c 100644 --- a/polytope_feature/engine/hullslicer.py +++ b/polytope_feature/engine/hullslicer.py @@ -1,11 +1,7 @@ from copy import copy -from typing import List - -from ..datacube.backends.datacube import Datacube from ..datacube.tensor_index_tree import TensorIndexTree -from ..shapes import ConvexPolytope -from ..utility.combinatorics import group, tensor_product, find_polytope_combinations +from ..utility.combinatorics import find_polytope_combinations, group, tensor_product from ..utility.exceptions import UnsliceableShapeError from .engine import Engine from .slicing_tools import slice diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index 5b19fd6b4..69bd05ce9 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -1,19 +1,18 @@ - +import numpy as np +import pandas as pd from qubed import Qube from qubed.value_types import QEnum -# from qubed.set_operations import union -from .slicing_tools import slice -from ..datacube.backends.qubed import QubedDatacube -from .engine import Engine -import pandas as pd + from ..datacube.datacube_axis import UnsliceableDatacubeAxis from ..datacube.transformations.datacube_mappers.datacube_mappers import DatacubeMapper -from ..shapes import ConvexPolytope, Product -from ..utility.combinatorics import group, tensor_product, find_polytopes_on_axis, find_polytope_combinations -from typing import List -import numpy as np - -from ..datacube.backends.datacube import Datacube +from ..utility.combinatorics import ( + find_polytope_combinations, + find_polytopes_on_axis, + group, + tensor_product, +) +from .engine import Engine +from .slicing_tools import slice class QubedSlicer(Engine): @@ -27,16 +26,9 @@ def find_datacube_vals(): pass def find_values_between(self, polytope, ax, node, datacube, lower, upper, path=None): - # print(node.values) - # print(lower) - # print(ax.name) if isinstance(ax, UnsliceableDatacubeAxis): - # print(node.values) - # print(lower) - # print(ax.name) values = [v for v in node.values if lower <= v <= upper] indices = [i for i, v in enumerate(node.values) if lower <= v <= upper] - # return [v for v in node.values if lower <= v <= upper] return values, indices tol = ax.tol @@ -58,8 +50,7 @@ def get_sliced_polys(self, found_vals, ax, child_name, poly, slice_axis_idx): def find_children_polytopes(self, polytopes, poly, sliced_polys): child_polytopes = [p for p in polytopes if p != poly] - child_polytopes.extend( - [sliced_poly_ for sliced_poly_ in sliced_polys if sliced_poly_ is not None]) + child_polytopes.extend([sliced_poly_ for sliced_poly_ in sliced_polys if sliced_poly_ is not None]) return child_polytopes def find_new_vals(self, found_vals, ax): @@ -73,7 +64,20 @@ def find_new_vals(self, found_vals, ax): new_found_vals.append(found_val) return new_found_vals - def build_branch(self, real_uncompressed_axis, found_vals, sliced_polys, polytopes, poly, child, datacube, datacube_transformations, ax, idxs=None, compressed_idxs=None): + def build_branch( + self, + real_uncompressed_axis, + found_vals, + sliced_polys, + polytopes, + poly, + child, + datacube, + datacube_transformations, + ax, + idxs=None, + compressed_idxs=None, + ): final_children_and_vals = [] if real_uncompressed_axis: for i, found_val in enumerate(found_vals): @@ -89,11 +93,9 @@ def build_branch(self, real_uncompressed_axis, found_vals, sliced_polys, polytop new_found_vals = self.find_new_vals([found_val], ax) if idxs: - # request_child_val = (children, new_found_vals, [idxs[i]]) request_child_val = (children, new_found_vals, compressed_idxs) else: request_child_val = (children, new_found_vals) - # final_children_and_vals.append((children, new_found_vals)) final_children_and_vals.append(request_child_val) else: # if it's compressed, then can add all found values in a single node @@ -107,9 +109,7 @@ def build_branch(self, real_uncompressed_axis, found_vals, sliced_polys, polytop return None new_found_vals = self.find_new_vals(found_vals, ax) - # final_children_and_vals.append((children, new_found_vals)) if idxs: - # request_child_val = (children, new_found_vals, idxs) request_child_val = (children, new_found_vals, compressed_idxs) else: request_child_val = (children, new_found_vals) @@ -117,8 +117,6 @@ def build_branch(self, real_uncompressed_axis, found_vals, sliced_polys, polytop if len(final_children_and_vals) == 0: return None - # print(child.key) - # print(idxs) return final_children_and_vals def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compressed_idxs=None) -> list[Qube]: @@ -155,7 +153,6 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compres sliced_polys = self.get_sliced_polys(found_vals, ax, grid_axes[0], poly, slice_axis_idx) # decide if axis should be compressed or not according to polytope # NOTE: actually the first grid axis will never be compressed - # axis_compressed = (grid_axes[0] in self.compressed_axes) # if it's not compressed, need to separate into different nodes to append to the tree for i, found_val in enumerate(found_vals): @@ -168,7 +165,13 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compres flattened_path = {grid_axes[0]: (found_val,)} # get second axis children through slicing children = self._slice_second_grid_axis( - grid_axes[1], child_polytopes, datacube, datacube_transformations, second_axis_vals, flattened_path) + grid_axes[1], + child_polytopes, + datacube, + datacube_transformations, + second_axis_vals, + flattened_path, + ) # If this node used to have children but now has none due to filtering, skip it. if not children: continue @@ -180,13 +183,11 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compres # TODO: when we have an axis that we would like to merge with another, we should skip the node creation here # and instead keep/cache the value to merge with the node from before?? - qube_node = Qube.make_node(key=grid_axes[0], - values=QEnum([found_val]), - metadata={}, - children=children) + qube_node = Qube.make_node( + key=grid_axes[0], values=QEnum([found_val]), metadata={}, children=children + ) result.append(qube_node) - # combined_idxs = [] for i, child in enumerate(q.children): # find polytopes which are defined on axis child.key polytopes_on_axis = find_polytopes_on_axis(child.key, polytopes) @@ -211,44 +212,26 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compres sliced_polys = self.get_sliced_polys(found_vals, ax, child.key, poly, slice_axis_idx) # decide if axis should be compressed or not according to polytope - axis_compressed = (child.key in self.compressed_axes) + axis_compressed = child.key in self.compressed_axes real_uncompressed_axis = not axis_compressed and len(found_vals) > 1 final_children_and_vals = self.build_branch( - real_uncompressed_axis, found_vals, sliced_polys, polytopes, poly, child, datacube, datacube_transformations, ax, idxs, compressed_idxs) + real_uncompressed_axis, + found_vals, + sliced_polys, + polytopes, + poly, + child, + datacube, + datacube_transformations, + ax, + idxs, + compressed_idxs, + ) - # combined_idxs.append(idxs) - # print(compressed_idxs) - # print([{k: child.metadata[k].shape} for k in child.metadata.keys()]) - # print([(ax.name, idxs) for (children, new_found_vals, idxs) in final_children_and_vals]) if final_children_and_vals is None: continue - # print(child.metadata.items()) - # print("HERE") - # print(child.key) - # print("\n\n\n\n\n\n") - # print(child.metadata) - # print("\n\n\n\n\n\n") - # print([(ax.name, idxs) for (children, new_found_vals, idxs) in final_children_and_vals]) - # print([{k: vs[idxs] for k, vs in child.metadata.items()} - # for (children, new_found_vals, idxs) in final_children_and_vals if idxs]) - - # result.extend([Qube.make_node( - # key=child.key, - # values=QEnum(new_found_vals), - # metadata=child.metadata, - # # metadata={k: vs[idxs] for k, vs in child.metadata.items()}, - # children=children - # ) for (children, new_found_vals, idxs) in final_children_and_vals]) def format_metadata_idxs(idxs): - # squeezed_indices = [np.squeeze(i) for i in idxs] - - # def format_index(idx): - # if isinstance(idx, np.ndarray) and idx.ndim > 0: - # return idx.tolist() # for multiple indices - # return int(idx) # for scalar index - # index_tuple = tuple(format_index(i) for i in squeezed_indices) - # return index_tuple flat_indices = [np.ravel(idx) for idx in idxs] return flat_indices @@ -257,41 +240,23 @@ def find_metadata(metadata_idx): for k, vs in child.metadata.items(): metadata_depth = len(vs.shape) relevant_metadata_idxs = metadata_idx[:metadata_depth] - # print("AND WHAT ABOUT HERE") - # print(relevant_metadata_idxs) - # print(vs.shape) - # print(vs[relevant_metadata_idxs].shape) - # if len(vs[relevant_metadata_idxs].shape) == 0: - # metadata[k] = vs[relevant_metadata_idxs] - - # metadata[k] = vs[relevant_metadata_idxs] ix = np.ix_(*relevant_metadata_idxs) metadata[k] = vs[ix] - # print(metadata[k]) return metadata - # print(len(final_children_and_vals)) - - for (children, new_found_vals, idxs) in final_children_and_vals: - # print("WHAT'S THE INDEX LIKE HERE?") - # print(idxs) + for children, new_found_vals, idxs in final_children_and_vals: metadata_idx = format_metadata_idxs(idxs) metadata = find_metadata(metadata_idx) - # print("HERE WHAT'S THE METADATA") - # print(metadata) - # print(metadata_idx) qube_node = Qube.make_node( - key=child.key, - values=QEnum(new_found_vals), - # metadata=child.metadata, - # metadata={k: vs[metadata_idx] for k, vs in child.metadata.items()}, - metadata=metadata, - children=children) + key=child.key, values=QEnum(new_found_vals), metadata=metadata, children=children + ) result.append(qube_node) return result - def _slice_second_grid_axis(self, axis_name, polytopes, datacube, datacube_transformations, second_axis_vals, path) -> list[Qube]: + def _slice_second_grid_axis( + self, axis_name, polytopes, datacube, datacube_transformations, second_axis_vals, path + ) -> list[Qube]: result = [] polytopes_on_axis = find_polytopes_on_axis(axis_name, polytopes) @@ -319,18 +284,12 @@ def _slice_second_grid_axis(self, axis_name, polytopes, datacube, datacube_trans # NOTE this was the last axis so we do not have children... - result.extend([Qube.make_node( - key=axis_name, - values=QEnum(new_found_vals), - metadata={}, - children={} - )]) + result.extend([Qube.make_node(key=axis_name, values=QEnum(new_found_vals), metadata={}, children={})]) return result def slice_tree(self, datacube, final_polys): q = datacube.q datacube_transformations = datacube.datacube_transformations - # return Qube.root_node(self._slice(q, final_polys, datacube, datacube_transformations)) return Qube.make_root(self._slice(q, final_polys, datacube, datacube_transformations)) def build_tree(self, polytopes_to_slice, datacube): @@ -349,6 +308,5 @@ def build_tree(self, polytopes_to_slice, datacube): final_tree = sub_trees[0] for sub_tree in sub_trees[1:]: - # union(final_tree, sub_tree) final_tree | sub_tree return final_tree diff --git a/polytope_feature/options.py b/polytope_feature/options.py index 584dc9b04..020c7007e 100644 --- a/polytope_feature/options.py +++ b/polytope_feature/options.py @@ -3,6 +3,7 @@ from conflator import ConfigModel from pydantic import ConfigDict + from .datacube.datacube_axis import DatacubeAxis diff --git a/polytope_feature/polytope.py b/polytope_feature/polytope.py index 0f806c8c7..ad2cc561a 100644 --- a/polytope_feature/polytope.py +++ b/polytope_feature/polytope.py @@ -46,8 +46,13 @@ def __init__(self, datacube, engine=None, options=None, context=None): if options is None: options = {} - axis_options, compressed_axes_options, config, alternative_axes, datacube_axes = PolytopeOptions.get_polytope_options( - options) + ( + axis_options, + compressed_axes_options, + config, + alternative_axes, + datacube_axes, + ) = PolytopeOptions.get_polytope_options(options) self.context = context self.datacube = Datacube.create( diff --git a/tests/test_qubed_extraction_engine.py b/tests/test_qubed_extraction_engine.py index 3d0d98bbb..a1a6c7440 100644 --- a/tests/test_qubed_extraction_engine.py +++ b/tests/test_qubed_extraction_engine.py @@ -1,25 +1,35 @@ -from polytope_feature.shapes import Box, Select, Span -from polytope_feature.polytope import Polytope, Request -from polytope_feature.engine.qubed_slicer import QubedSlicer -from polytope_feature.datacube.backends.qubed import QubedDatacube -from polytope_feature.datacube.backends.fdb import FDBDatacube +import time + +import pandas as pd +import pygribjump as gj import pytest -from qubed import Qube import requests -from polytope_feature.datacube.datacube_axis import PandasTimedeltaDatacubeAxis, PandasTimestampDatacubeAxis, UnsliceableDatacubeAxis, FloatDatacubeAxis -# from polytope_feature.datacube.backends.test_qubed_slicing import actual_slice -from polytope_feature.datacube.transformations.datacube_type_change.datacube_type_change import TypeChangeStrToTimestamp, TypeChangeStrToTimedelta -import pandas as pd -from polytope_feature.datacube.transformations.datacube_mappers.mapper_types.healpix_nested import NestedHealpixGridMapper +from qubed import Qube -from polytope_feature.shapes import ConvexPolytope -import time -import pygribjump as gj +from polytope_feature.datacube.backends.fdb import FDBDatacube +from polytope_feature.datacube.backends.qubed import QubedDatacube +from polytope_feature.datacube.datacube_axis import ( + FloatDatacubeAxis, + PandasTimedeltaDatacubeAxis, + PandasTimestampDatacubeAxis, + UnsliceableDatacubeAxis, +) +from polytope_feature.datacube.transformations.datacube_mappers.mapper_types.healpix_nested import ( + NestedHealpixGridMapper, +) + +# from polytope_feature.datacube.backends.test_qubed_slicing import actual_slice +from polytope_feature.datacube.transformations.datacube_type_change.datacube_type_change import ( + TypeChangeStrToTimedelta, + TypeChangeStrToTimestamp, +) from polytope_feature.engine.hullslicer import HullSlicer +from polytope_feature.engine.qubed_slicer import QubedSlicer +from polytope_feature.polytope import Polytope, Request +from polytope_feature.shapes import Box, ConvexPolytope, Select, Span def find_relevant_subcube_from_request(request, qube_url): - # NOTE: final url we want is like: # "https://qubed.lumi.apps.dte.destination-earth.eu/api/v1/select/climate-dt/?class=d1&dataset=climate-dt" @@ -36,8 +46,9 @@ def find_relevant_subcube_from_request(request, qube_url): return qube_url -fdb_tree = Qube.from_json(requests.get( - "https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/climate-dt.json").json()) +fdb_tree = Qube.from_json( + requests.get("https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/climate-dt.json").json() +) combi_polytopes = [ @@ -45,39 +56,40 @@ def find_relevant_subcube_from_request(request, qube_url): ConvexPolytope(["time"], [[pd.Timedelta(hours=0, minutes=0)], [pd.Timedelta(hours=12, minutes=0)]]), ConvexPolytope(["resolution"], [["high"]]), ConvexPolytope(["type"], [["fc"]]), - ConvexPolytope(["model"], [['ifs-nemo']]), + ConvexPolytope(["model"], [["ifs-nemo"]]), ConvexPolytope(["stream"], [["clte"]]), ConvexPolytope(["realization"], ["1"]), - ConvexPolytope(["expver"], [['0001']]), - ConvexPolytope(["experiment"], [['ssp3-7.0']]), + ConvexPolytope(["expver"], [["0001"]]), + ConvexPolytope(["experiment"], [["ssp3-7.0"]]), ConvexPolytope(["generation"], [["1"]]), ConvexPolytope(["levtype"], [["sfc"]]), ConvexPolytope(["activity"], [["scenariomip"]]), ConvexPolytope(["dataset"], [["climate-dt"]]), ConvexPolytope(["class"], [["d1"]]), ConvexPolytope(["date"], [[pd.Timestamp("20220811")], [pd.Timestamp("20220912")]]), - ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]]) + ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]]), ] # TODO: add lat and lon axes -datacube_axes = {"param": UnsliceableDatacubeAxis(), - "time": PandasTimedeltaDatacubeAxis(), - "resolution": UnsliceableDatacubeAxis(), - "type": UnsliceableDatacubeAxis(), - "model": UnsliceableDatacubeAxis(), - "stream": UnsliceableDatacubeAxis(), - "realization": UnsliceableDatacubeAxis(), - "expver": UnsliceableDatacubeAxis(), - "experiment": UnsliceableDatacubeAxis(), - "generation": UnsliceableDatacubeAxis(), - "levtype": UnsliceableDatacubeAxis(), - "activity": UnsliceableDatacubeAxis(), - "dataset": UnsliceableDatacubeAxis(), - "class": UnsliceableDatacubeAxis(), - "date": PandasTimestampDatacubeAxis(), - # "latitude": FloatDatacubeAxis(), - # "longitude": FloatDatacubeAxis() - } +datacube_axes = { + "param": UnsliceableDatacubeAxis(), + "time": PandasTimedeltaDatacubeAxis(), + "resolution": UnsliceableDatacubeAxis(), + "type": UnsliceableDatacubeAxis(), + "model": UnsliceableDatacubeAxis(), + "stream": UnsliceableDatacubeAxis(), + "realization": UnsliceableDatacubeAxis(), + "expver": UnsliceableDatacubeAxis(), + "experiment": UnsliceableDatacubeAxis(), + "generation": UnsliceableDatacubeAxis(), + "levtype": UnsliceableDatacubeAxis(), + "activity": UnsliceableDatacubeAxis(), + "dataset": UnsliceableDatacubeAxis(), + "class": UnsliceableDatacubeAxis(), + "date": PandasTimestampDatacubeAxis(), + # "latitude": FloatDatacubeAxis(), + # "longitude": FloatDatacubeAxis() +} time_val = pd.Timedelta(hours=0, minutes=0) date_val = pd.Timestamp("20300101T000000") @@ -87,7 +99,7 @@ def find_relevant_subcube_from_request(request, qube_url): datacube_transformations = { "time": TypeChangeStrToTimedelta("time", time_val), "date": TypeChangeStrToTimestamp("date", date_val), - "values": NestedHealpixGridMapper("values", ["latitude", "longitude"], 1024) + "values": NestedHealpixGridMapper("values", ["latitude", "longitude"], 1024), } @@ -142,22 +154,23 @@ def find_relevant_subcube_from_request(request, qube_url): # # "latitude": FloatDatacubeAxis(), # # "longitude": FloatDatacubeAxis() # } - "datacube_axes": {"param": "UnsliceableDatacubeAxis", - "time": "PandasTimedeltaDatacubeAxis", - "resolution": "UnsliceableDatacubeAxis", - "type": "UnsliceableDatacubeAxis", - "model": "UnsliceableDatacubeAxis", - "stream": "UnsliceableDatacubeAxis", - "realization": "UnsliceableDatacubeAxis", - "expver": "UnsliceableDatacubeAxis", - "experiment": "UnsliceableDatacubeAxis", - "generation": "UnsliceableDatacubeAxis", - "levtype": "UnsliceableDatacubeAxis", - "activity": "UnsliceableDatacubeAxis", - "dataset": "UnsliceableDatacubeAxis", - "class": "UnsliceableDatacubeAxis", - "date": "PandasTimestampDatacubeAxis", - } + "datacube_axes": { + "param": "UnsliceableDatacubeAxis", + "time": "PandasTimedeltaDatacubeAxis", + "resolution": "UnsliceableDatacubeAxis", + "type": "UnsliceableDatacubeAxis", + "model": "UnsliceableDatacubeAxis", + "stream": "UnsliceableDatacubeAxis", + "realization": "UnsliceableDatacubeAxis", + "expver": "UnsliceableDatacubeAxis", + "experiment": "UnsliceableDatacubeAxis", + "generation": "UnsliceableDatacubeAxis", + "levtype": "UnsliceableDatacubeAxis", + "activity": "UnsliceableDatacubeAxis", + "dataset": "UnsliceableDatacubeAxis", + "class": "UnsliceableDatacubeAxis", + "date": "PandasTimestampDatacubeAxis", + }, } # request = Request( @@ -173,24 +186,25 @@ def find_relevant_subcube_from_request(request, qube_url): # Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), # ) -request = Request(ConvexPolytope(["param"], [[164]]), - ConvexPolytope(["time"], [[pd.Timedelta(hours=1, minutes=0)], [pd.Timedelta(hours=3, minutes=0)]]), - ConvexPolytope(["resolution"], [["high"]]), - ConvexPolytope(["type"], [["fc"]]), - ConvexPolytope(["model"], [['ifs-nemo']]), - ConvexPolytope(["stream"], [["clte"]]), - ConvexPolytope(["realization"], [[1]]), - ConvexPolytope(["expver"], [['0001']]), - ConvexPolytope(["experiment"], [['ssp3-7.0']]), - ConvexPolytope(["generation"], [[1]]), - ConvexPolytope(["levtype"], [["sfc"]]), - ConvexPolytope(["activity"], [["scenariomip"]]), - ConvexPolytope(["dataset"], [["climate-dt"]]), - ConvexPolytope(["class"], [["d1"]]), - ConvexPolytope(["date"], [[pd.Timestamp("20220811")]]), - ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]]) - # ConvexPolytope(["latitude", "longitude"], [[0, 0], [-0.5, -0.5], [0, -0.5]]) - ) +request = Request( + ConvexPolytope(["param"], [[164]]), + ConvexPolytope(["time"], [[pd.Timedelta(hours=1, minutes=0)], [pd.Timedelta(hours=3, minutes=0)]]), + ConvexPolytope(["resolution"], [["high"]]), + ConvexPolytope(["type"], [["fc"]]), + ConvexPolytope(["model"], [["ifs-nemo"]]), + ConvexPolytope(["stream"], [["clte"]]), + ConvexPolytope(["realization"], [[1]]), + ConvexPolytope(["expver"], [["0001"]]), + ConvexPolytope(["experiment"], [["ssp3-7.0"]]), + ConvexPolytope(["generation"], [[1]]), + ConvexPolytope(["levtype"], [["sfc"]]), + ConvexPolytope(["activity"], [["scenariomip"]]), + ConvexPolytope(["dataset"], [["climate-dt"]]), + ConvexPolytope(["class"], [["d1"]]), + ConvexPolytope(["date"], [[pd.Timestamp("20220811")]]), + ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]]) + # ConvexPolytope(["latitude", "longitude"], [[0, 0], [-0.5, -0.5], [0, -0.5]]) +) qubeddatacube = QubedDatacube(fdb_tree, datacube_axes, datacube_transformations) slicer = QubedSlicer() @@ -257,22 +271,24 @@ def find_relevant_subcube_from_request(request, qube_url): # ) -request = Request(ConvexPolytope(["param"], [["164"]]), - ConvexPolytope(["time"], [[pd.Timedelta(hours=1, minutes=0)], [pd.Timedelta(hours=3, minutes=0)]]), - ConvexPolytope(["resolution"], [["high"]]), - ConvexPolytope(["type"], [["fc"]]), - ConvexPolytope(["model"], [['ifs-nemo']]), - ConvexPolytope(["stream"], [["clte"]]), - ConvexPolytope(["realization"], ["1"]), - ConvexPolytope(["expver"], [['0001']]), - ConvexPolytope(["experiment"], [['ssp3-7.0']]), - ConvexPolytope(["generation"], [["1"]]), - ConvexPolytope(["levtype"], [["sfc"]]), - ConvexPolytope(["activity"], [["scenariomip"]]), - ConvexPolytope(["dataset"], [["climate-dt"]]), - ConvexPolytope(["class"], [["d1"]]), - ConvexPolytope(["date"], [[pd.Timestamp("20220811")]]), - ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]])) +request = Request( + ConvexPolytope(["param"], [["164"]]), + ConvexPolytope(["time"], [[pd.Timedelta(hours=1, minutes=0)], [pd.Timedelta(hours=3, minutes=0)]]), + ConvexPolytope(["resolution"], [["high"]]), + ConvexPolytope(["type"], [["fc"]]), + ConvexPolytope(["model"], [["ifs-nemo"]]), + ConvexPolytope(["stream"], [["clte"]]), + ConvexPolytope(["realization"], ["1"]), + ConvexPolytope(["expver"], [["0001"]]), + ConvexPolytope(["experiment"], [["ssp3-7.0"]]), + ConvexPolytope(["generation"], [["1"]]), + ConvexPolytope(["levtype"], [["sfc"]]), + ConvexPolytope(["activity"], [["scenariomip"]]), + ConvexPolytope(["dataset"], [["climate-dt"]]), + ConvexPolytope(["class"], [["d1"]]), + ConvexPolytope(["date"], [[pd.Timestamp("20220811")]]), + ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]]), +) time3 = time.time() # result = self_API.retrieve(request) diff --git a/tests/test_qubed_extraction_engine_w_metadata.py b/tests/test_qubed_extraction_engine_w_metadata.py index 8584049e8..7156303df 100644 --- a/tests/test_qubed_extraction_engine_w_metadata.py +++ b/tests/test_qubed_extraction_engine_w_metadata.py @@ -1,25 +1,35 @@ -from polytope_feature.shapes import Box, Select, Span -from polytope_feature.polytope import Polytope, Request -from polytope_feature.engine.qubed_slicer import QubedSlicer -from polytope_feature.datacube.backends.qubed import QubedDatacube -from polytope_feature.datacube.backends.fdb import FDBDatacube +import time + +import pandas as pd +import pygribjump as gj import pytest -from qubed import Qube import requests -from polytope_feature.datacube.datacube_axis import PandasTimedeltaDatacubeAxis, PandasTimestampDatacubeAxis, UnsliceableDatacubeAxis, FloatDatacubeAxis -# from polytope_feature.datacube.backends.test_qubed_slicing import actual_slice -from polytope_feature.datacube.transformations.datacube_type_change.datacube_type_change import TypeChangeStrToTimestamp, TypeChangeStrToTimedelta -import pandas as pd -from polytope_feature.datacube.transformations.datacube_mappers.mapper_types.healpix_nested import NestedHealpixGridMapper +from qubed import Qube -from polytope_feature.shapes import ConvexPolytope -import time -import pygribjump as gj +from polytope_feature.datacube.backends.fdb import FDBDatacube +from polytope_feature.datacube.backends.qubed import QubedDatacube +from polytope_feature.datacube.datacube_axis import ( + FloatDatacubeAxis, + PandasTimedeltaDatacubeAxis, + PandasTimestampDatacubeAxis, + UnsliceableDatacubeAxis, +) +from polytope_feature.datacube.transformations.datacube_mappers.mapper_types.healpix_nested import ( + NestedHealpixGridMapper, +) + +# from polytope_feature.datacube.backends.test_qubed_slicing import actual_slice +from polytope_feature.datacube.transformations.datacube_type_change.datacube_type_change import ( + TypeChangeStrToTimedelta, + TypeChangeStrToTimestamp, +) from polytope_feature.engine.hullslicer import HullSlicer +from polytope_feature.engine.qubed_slicer import QubedSlicer +from polytope_feature.polytope import Polytope, Request +from polytope_feature.shapes import Box, ConvexPolytope, Select, Span def find_relevant_subcube_from_request(request, qube_url): - # NOTE: final url we want is like: # "https://qubed.lumi.apps.dte.destination-earth.eu/api/v1/select/climate-dt/?class=d1&dataset=climate-dt" @@ -36,8 +46,11 @@ def find_relevant_subcube_from_request(request, qube_url): return qube_url -fdb_tree = Qube.from_json(requests.get( - "https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/extremes-dt_with_metadata.json").json()) +fdb_tree = Qube.from_json( + requests.get( + "https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/extremes-dt_with_metadata.json" + ).json() +) # combi_polytopes = [ @@ -60,24 +73,25 @@ def find_relevant_subcube_from_request(request, qube_url): # ] # TODO: add lat and lon axes -datacube_axes = {"param": UnsliceableDatacubeAxis(), - "time": PandasTimedeltaDatacubeAxis(), - "resolution": UnsliceableDatacubeAxis(), - "type": UnsliceableDatacubeAxis(), - "model": UnsliceableDatacubeAxis(), - "stream": UnsliceableDatacubeAxis(), - "realization": UnsliceableDatacubeAxis(), - "expver": UnsliceableDatacubeAxis(), - "experiment": UnsliceableDatacubeAxis(), - "generation": UnsliceableDatacubeAxis(), - "levtype": UnsliceableDatacubeAxis(), - "activity": UnsliceableDatacubeAxis(), - "dataset": UnsliceableDatacubeAxis(), - "class": UnsliceableDatacubeAxis(), - "date": PandasTimestampDatacubeAxis(), - # "latitude": FloatDatacubeAxis(), - # "longitude": FloatDatacubeAxis() - } +datacube_axes = { + "param": UnsliceableDatacubeAxis(), + "time": PandasTimedeltaDatacubeAxis(), + "resolution": UnsliceableDatacubeAxis(), + "type": UnsliceableDatacubeAxis(), + "model": UnsliceableDatacubeAxis(), + "stream": UnsliceableDatacubeAxis(), + "realization": UnsliceableDatacubeAxis(), + "expver": UnsliceableDatacubeAxis(), + "experiment": UnsliceableDatacubeAxis(), + "generation": UnsliceableDatacubeAxis(), + "levtype": UnsliceableDatacubeAxis(), + "activity": UnsliceableDatacubeAxis(), + "dataset": UnsliceableDatacubeAxis(), + "class": UnsliceableDatacubeAxis(), + "date": PandasTimestampDatacubeAxis(), + # "latitude": FloatDatacubeAxis(), + # "longitude": FloatDatacubeAxis() +} time_val = pd.Timedelta(hours=0, minutes=0) date_val = pd.Timestamp("20300101T000000") @@ -87,7 +101,7 @@ def find_relevant_subcube_from_request(request, qube_url): datacube_transformations = { "time": TypeChangeStrToTimedelta("time", time_val), "date": TypeChangeStrToTimestamp("date", date_val), - "values": NestedHealpixGridMapper("values", ["latitude", "longitude"], 1024) + "values": NestedHealpixGridMapper("values", ["latitude", "longitude"], 1024), } @@ -124,17 +138,18 @@ def find_relevant_subcube_from_request(request, qube_url): "type", ], "pre_path": {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"}, - "datacube_axes": {"param": "UnsliceableDatacubeAxis", - "time": "PandasTimedeltaDatacubeAxis", - "type": "UnsliceableDatacubeAxis", - "stream": "UnsliceableDatacubeAxis", - "expver": "UnsliceableDatacubeAxis", - "levtype": "UnsliceableDatacubeAxis", - "dataset": "UnsliceableDatacubeAxis", - "class": "UnsliceableDatacubeAxis", - "date": "PandasTimestampDatacubeAxis", - "step": "IntDatacubeAxis", - } + "datacube_axes": { + "param": "UnsliceableDatacubeAxis", + "time": "PandasTimedeltaDatacubeAxis", + "type": "UnsliceableDatacubeAxis", + "stream": "UnsliceableDatacubeAxis", + "expver": "UnsliceableDatacubeAxis", + "levtype": "UnsliceableDatacubeAxis", + "dataset": "UnsliceableDatacubeAxis", + "class": "UnsliceableDatacubeAxis", + "date": "PandasTimestampDatacubeAxis", + "step": "IntDatacubeAxis", + }, } # request = Request( @@ -150,19 +165,20 @@ def find_relevant_subcube_from_request(request, qube_url): # Box(["latitude", "longitude"], [0, 0], [0.2, 0.2]), # ) -request = Request(ConvexPolytope(["param"], [["167"]]), - ConvexPolytope(["time"], [[pd.Timedelta(hours=0, minutes=0)], [pd.Timedelta(hours=3, minutes=0)]]), - ConvexPolytope(["type"], [["fc"]]), - ConvexPolytope(["stream"], [["oper"]]), - ConvexPolytope(["expver"], [['0001']]), - ConvexPolytope(["levtype"], [["sfc"]]), - ConvexPolytope(["dataset"], [["extremes-dt"]]), - ConvexPolytope(["class"], [["d1"]]), - ConvexPolytope(["step"], [[1]]), - ConvexPolytope(["date"], [[pd.Timestamp("20240407")]]), - ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]]) - # ConvexPolytope(["latitude", "longitude"], [[0, 0], [-0.5, -0.5], [0, -0.5]]) - ) +request = Request( + ConvexPolytope(["param"], [["167"]]), + ConvexPolytope(["time"], [[pd.Timedelta(hours=0, minutes=0)], [pd.Timedelta(hours=3, minutes=0)]]), + ConvexPolytope(["type"], [["fc"]]), + ConvexPolytope(["stream"], [["oper"]]), + ConvexPolytope(["expver"], [["0001"]]), + ConvexPolytope(["levtype"], [["sfc"]]), + ConvexPolytope(["dataset"], [["extremes-dt"]]), + ConvexPolytope(["class"], [["d1"]]), + ConvexPolytope(["step"], [[1]]), + ConvexPolytope(["date"], [[pd.Timestamp("20240407")]]), + ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]]) + # ConvexPolytope(["latitude", "longitude"], [[0, 0], [-0.5, -0.5], [0, -0.5]]) +) # print(fdb_tree) qubeddatacube = QubedDatacube(fdb_tree, datacube_axes, datacube_transformations) slicer = QubedSlicer() @@ -229,22 +245,24 @@ def find_relevant_subcube_from_request(request, qube_url): # ) -request = Request(ConvexPolytope(["param"], [["164"]]), - ConvexPolytope(["time"], [[pd.Timedelta(hours=1, minutes=0)], [pd.Timedelta(hours=3, minutes=0)]]), - ConvexPolytope(["resolution"], [["high"]]), - ConvexPolytope(["type"], [["fc"]]), - ConvexPolytope(["model"], [['ifs-nemo']]), - ConvexPolytope(["stream"], [["clte"]]), - ConvexPolytope(["realization"], ["1"]), - ConvexPolytope(["expver"], [['0001']]), - ConvexPolytope(["experiment"], [['ssp3-7.0']]), - ConvexPolytope(["generation"], [["1"]]), - ConvexPolytope(["levtype"], [["sfc"]]), - ConvexPolytope(["activity"], [["scenariomip"]]), - ConvexPolytope(["dataset"], [["climate-dt"]]), - ConvexPolytope(["class"], [["d1"]]), - ConvexPolytope(["date"], [[pd.Timestamp("20220811")]]), - ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]])) +request = Request( + ConvexPolytope(["param"], [["164"]]), + ConvexPolytope(["time"], [[pd.Timedelta(hours=1, minutes=0)], [pd.Timedelta(hours=3, minutes=0)]]), + ConvexPolytope(["resolution"], [["high"]]), + ConvexPolytope(["type"], [["fc"]]), + ConvexPolytope(["model"], [["ifs-nemo"]]), + ConvexPolytope(["stream"], [["clte"]]), + ConvexPolytope(["realization"], ["1"]), + ConvexPolytope(["expver"], [["0001"]]), + ConvexPolytope(["experiment"], [["ssp3-7.0"]]), + ConvexPolytope(["generation"], [["1"]]), + ConvexPolytope(["levtype"], [["sfc"]]), + ConvexPolytope(["activity"], [["scenariomip"]]), + ConvexPolytope(["dataset"], [["climate-dt"]]), + ConvexPolytope(["class"], [["d1"]]), + ConvexPolytope(["date"], [[pd.Timestamp("20220811")]]), + ConvexPolytope(["latitude", "longitude"], [[0, 0], [0.5, 0.5], [0, 0.5]]), +) time3 = time.time() # result = self_API.retrieve(request) diff --git a/tests/test_qubed_extraction_service.py b/tests/test_qubed_extraction_service.py index 3bcb18eeb..c13f71cc1 100644 --- a/tests/test_qubed_extraction_service.py +++ b/tests/test_qubed_extraction_service.py @@ -1,25 +1,35 @@ -from polytope_feature.shapes import Box, Select, Span -from polytope_feature.polytope import Polytope, Request -from polytope_feature.engine.qubed_slicer import QubedSlicer -from polytope_feature.datacube.backends.qubed import QubedDatacube -from polytope_feature.datacube.backends.fdb import FDBDatacube +import time + +import pandas as pd +import pygribjump as gj import pytest -from qubed import Qube import requests -from polytope_feature.datacube.datacube_axis import PandasTimedeltaDatacubeAxis, PandasTimestampDatacubeAxis, UnsliceableDatacubeAxis, FloatDatacubeAxis -# from polytope_feature.datacube.backends.test_qubed_slicing import actual_slice -from polytope_feature.datacube.transformations.datacube_type_change.datacube_type_change import TypeChangeStrToTimestamp, TypeChangeStrToTimedelta -import pandas as pd -from polytope_feature.datacube.transformations.datacube_mappers.mapper_types.healpix_nested import NestedHealpixGridMapper +from qubed import Qube -from polytope_feature.shapes import ConvexPolytope -import time -import pygribjump as gj +from polytope_feature.datacube.backends.fdb import FDBDatacube +from polytope_feature.datacube.backends.qubed import QubedDatacube +from polytope_feature.datacube.datacube_axis import ( + FloatDatacubeAxis, + PandasTimedeltaDatacubeAxis, + PandasTimestampDatacubeAxis, + UnsliceableDatacubeAxis, +) +from polytope_feature.datacube.transformations.datacube_mappers.mapper_types.healpix_nested import ( + NestedHealpixGridMapper, +) + +# from polytope_feature.datacube.backends.test_qubed_slicing import actual_slice +from polytope_feature.datacube.transformations.datacube_type_change.datacube_type_change import ( + TypeChangeStrToTimedelta, + TypeChangeStrToTimestamp, +) from polytope_feature.engine.hullslicer import HullSlicer +from polytope_feature.engine.qubed_slicer import QubedSlicer +from polytope_feature.polytope import Polytope, Request +from polytope_feature.shapes import Box, ConvexPolytope, Select, Span def find_relevant_subcube_from_request(request, qube_url): - # NOTE: final url we want is like: # "https://qubed.lumi.apps.dte.destination-earth.eu/api/v1/select/climate-dt/?class=d1&dataset=climate-dt" @@ -43,8 +53,9 @@ def get_fdb_tree(request): return fdb_tree -fdb_tree = Qube.from_json(requests.get( - "https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/climate-dt.json").json()) +fdb_tree = Qube.from_json( + requests.get("https://github.com/ecmwf/qubed/raw/refs/heads/main/tests/example_qubes/climate-dt.json").json() +) # print(fdb_tree) @@ -69,24 +80,25 @@ def get_fdb_tree(request): # ] # TODO: add lat and lon axes -datacube_axes = {"param": UnsliceableDatacubeAxis(), - "time": PandasTimedeltaDatacubeAxis(), - "resolution": UnsliceableDatacubeAxis(), - "type": UnsliceableDatacubeAxis(), - "model": UnsliceableDatacubeAxis(), - "stream": UnsliceableDatacubeAxis(), - "realization": UnsliceableDatacubeAxis(), - "expver": UnsliceableDatacubeAxis(), - "experiment": UnsliceableDatacubeAxis(), - "generation": UnsliceableDatacubeAxis(), - "levtype": UnsliceableDatacubeAxis(), - "activity": UnsliceableDatacubeAxis(), - "dataset": UnsliceableDatacubeAxis(), - "class": UnsliceableDatacubeAxis(), - "date": PandasTimestampDatacubeAxis(), - # "latitude": FloatDatacubeAxis(), - # "longitude": FloatDatacubeAxis() - } +datacube_axes = { + "param": UnsliceableDatacubeAxis(), + "time": PandasTimedeltaDatacubeAxis(), + "resolution": UnsliceableDatacubeAxis(), + "type": UnsliceableDatacubeAxis(), + "model": UnsliceableDatacubeAxis(), + "stream": UnsliceableDatacubeAxis(), + "realization": UnsliceableDatacubeAxis(), + "expver": UnsliceableDatacubeAxis(), + "experiment": UnsliceableDatacubeAxis(), + "generation": UnsliceableDatacubeAxis(), + "levtype": UnsliceableDatacubeAxis(), + "activity": UnsliceableDatacubeAxis(), + "dataset": UnsliceableDatacubeAxis(), + "class": UnsliceableDatacubeAxis(), + "date": PandasTimestampDatacubeAxis(), + # "latitude": FloatDatacubeAxis(), + # "longitude": FloatDatacubeAxis() +} time_val = pd.Timedelta(hours=0, minutes=0) date_val = pd.Timestamp("20300101T000000") @@ -96,7 +108,7 @@ def get_fdb_tree(request): datacube_transformations = { "time": TypeChangeStrToTimedelta("time", time_val), "date": TypeChangeStrToTimestamp("date", date_val), - "values": NestedHealpixGridMapper("values", ["latitude", "longitude"], 1024) + "values": NestedHealpixGridMapper("values", ["latitude", "longitude"], 1024), } @@ -136,25 +148,26 @@ def get_fdb_tree(request): "type", ], "pre_path": {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper"}, - "datacube_axes": {"param": "UnsliceableDatacubeAxis", - "time": "PandasTimedeltaDatacubeAxis", - "resolution": "UnsliceableDatacubeAxis", - "type": "UnsliceableDatacubeAxis", - "model": "UnsliceableDatacubeAxis", - "stream": "UnsliceableDatacubeAxis", - # "realization": "UnsliceableDatacubeAxis", - "realization": "IntDatacubeAxis", - # "expver": "UnsliceableDatacubeAxis", - "expver": "IntDatacubeAxis", - "experiment": "UnsliceableDatacubeAxis", - # "generation": "UnsliceableDatacubeAxis", - "generation": "IntDatacubeAxis", - "levtype": "UnsliceableDatacubeAxis", - "activity": "UnsliceableDatacubeAxis", - "dataset": "UnsliceableDatacubeAxis", - "class": "UnsliceableDatacubeAxis", - "date": "PandasTimestampDatacubeAxis", - } + "datacube_axes": { + "param": "UnsliceableDatacubeAxis", + "time": "PandasTimedeltaDatacubeAxis", + "resolution": "UnsliceableDatacubeAxis", + "type": "UnsliceableDatacubeAxis", + "model": "UnsliceableDatacubeAxis", + "stream": "UnsliceableDatacubeAxis", + # "realization": "UnsliceableDatacubeAxis", + "realization": "IntDatacubeAxis", + # "expver": "UnsliceableDatacubeAxis", + "expver": "IntDatacubeAxis", + "experiment": "UnsliceableDatacubeAxis", + # "generation": "UnsliceableDatacubeAxis", + "generation": "IntDatacubeAxis", + "levtype": "UnsliceableDatacubeAxis", + "activity": "UnsliceableDatacubeAxis", + "dataset": "UnsliceableDatacubeAxis", + "class": "UnsliceableDatacubeAxis", + "date": "PandasTimestampDatacubeAxis", + }, } # request = Request( @@ -183,8 +196,8 @@ def get_fdb_tree(request): Select("stream", ["clte"]), # ConvexPolytope(["stream"], [["clte"]]), ConvexPolytope(["realization"], ["1"]), - ConvexPolytope(["expver"], [['0001']]), - ConvexPolytope(["experiment"], [['ssp3-7.0']]), + ConvexPolytope(["expver"], [["0001"]]), + ConvexPolytope(["experiment"], [["ssp3-7.0"]]), # ConvexPolytope(["generation"], [["1"]]), Select("generation", [1]), ConvexPolytope(["levtype"], [["sfc"]]), @@ -194,11 +207,12 @@ def get_fdb_tree(request): ConvexPolytope(["class"], [["d1"]]), # ConvexPolytope(["date"], [[pd.Timestamp("20220811")]]), ConvexPolytope(["date"], [[pd.Timestamp("20200908")]]), - ConvexPolytope(["latitude", "longitude"], [[0, 0], [5, 5], [0, 5]])) + ConvexPolytope(["latitude", "longitude"], [[0, 0], [5, 5], [0, 5]]), +) # fdb_tree = get_fdb_tree(request) path_to_qube = "../qubed/" -full_qube_path = path_to_qube+"tests/example_qubes/climate_dt_with_metadata.json" +full_qube_path = path_to_qube + "tests/example_qubes/climate_dt_with_metadata.json" fdb_tree = Qube.load(full_qube_path) print("HERE WE HAVE THE FDB TREE") From 2fd430685bbaf91b1959d771f6cf3d98030645c0 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 31 Jul 2025 16:28:10 +0200 Subject: [PATCH 089/105] add result to Qube metadata --- polytope_feature/datacube/backends/qubed.py | 12 ++++++------ polytope_feature/engine/qubed_slicer.py | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/polytope_feature/datacube/backends/qubed.py b/polytope_feature/datacube/backends/qubed.py index 2a4ced348..8f0c57838 100644 --- a/polytope_feature/datacube/backends/qubed.py +++ b/polytope_feature/datacube/backends/qubed.py @@ -426,13 +426,13 @@ def assign_fdb_output_to_nodes(self, output_values, fdb_requests_decoding_info): if len(request_output_values.values) == 0: # If we are here, no data was found for this path in the fdb none_array = [None] * len(n.values) - if n.data.metadata.get("result", None) is None: - n.data.metadata["result"] = [] - n.data.metadata["result"].extend(none_array) + if n.metadata.get("result", None) is None: + n.metadata["result"] = [] + n.metadata["result"].extend(none_array) else: - if n.data.metadata.get("result", None) is None: - n.data.metadata["result"] = [] - n.data.metadata["result"].extend(request_output_values.values[i]) + if n.metadata.get("result", None) is None: + n.metadata["result"] = [] + n.metadata["result"].extend(request_output_values.values[i]) def sort_fdb_request_ranges(self, current_start_idx, lat_length, fdb_node_ranges): (new_fdb_node_ranges, new_current_start_idx) = self.remove_duplicates_in_request_ranges( diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index 69bd05ce9..beaf07a97 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -284,7 +284,8 @@ def _slice_second_grid_axis( # NOTE this was the last axis so we do not have children... - result.extend([Qube.make_node(key=axis_name, values=QEnum(new_found_vals), metadata={}, children={})]) + result.extend([Qube.make_node(key=axis_name, values=QEnum( + new_found_vals), metadata={"result": []}, children={})]) return result def slice_tree(self, datacube, final_polys): From ee37224b2a12cbc1b11b05487d5689be926e9bd9 Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 31 Jul 2025 17:28:33 +0200 Subject: [PATCH 090/105] flatten metadata to give to gj --- polytope_feature/datacube/backends/qubed.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/polytope_feature/datacube/backends/qubed.py b/polytope_feature/datacube/backends/qubed.py index 8f0c57838..3db9d445b 100644 --- a/polytope_feature/datacube/backends/qubed.py +++ b/polytope_feature/datacube/backends/qubed.py @@ -190,11 +190,14 @@ def find_metadata(metadata_idx): metadata_idxs = index_combis[i] actual_metadata = find_metadata(metadata_idxs) - path = actual_metadata["path"][0] - scheme = actual_metadata["scheme"] - offset = actual_metadata["offset"][0] - host = actual_metadata["host"][0] - port = actual_metadata["port"] + def flatten_metadata(value): + return value[0] if isinstance(value, np.ndarray) else value + + path = flatten_metadata(actual_metadata["path"]) + scheme = flatten_metadata(actual_metadata["scheme"]) + offset = flatten_metadata(actual_metadata["offset"]) + host = flatten_metadata(actual_metadata["host"]) + port = flatten_metadata(actual_metadata["port"]) uncompressed_request = {} for i, key in enumerate(compressed_request[0].keys()): From 394240ba1d45337ccd1216364ceeea040201f62b Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 31 Jul 2025 17:56:31 +0200 Subject: [PATCH 091/105] handle when there are no sliced polytopes --- polytope_feature/engine/qubed_slicer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index beaf07a97..e83ce6a97 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -81,7 +81,11 @@ def build_branch( final_children_and_vals = [] if real_uncompressed_axis: for i, found_val in enumerate(found_vals): - sliced_polys_ = [sliced_polys[i]] + # sliced_polys_ = [sliced_polys[i]] + if i < len(sliced_polys): + sliced_polys_ = [sliced_polys[i]] + else: + sliced_polys_ = sliced_polys child_polytopes = self.find_children_polytopes(polytopes, poly, sliced_polys_) if idxs: compressed_idxs.append([idxs[i]]) From a07212af78fa813434d5243136c6b8ac2291bcf9 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 4 Aug 2025 09:29:50 +0200 Subject: [PATCH 092/105] fix bug --- polytope_feature/datacube/backends/qubed.py | 10 ++++--- polytope_feature/engine/qubed_slicer.py | 29 ++++++++++++++++----- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/polytope_feature/datacube/backends/qubed.py b/polytope_feature/datacube/backends/qubed.py index 3db9d445b..2d8cb7f30 100644 --- a/polytope_feature/datacube/backends/qubed.py +++ b/polytope_feature/datacube/backends/qubed.py @@ -199,9 +199,10 @@ def flatten_metadata(value): host = flatten_metadata(actual_metadata["host"]) port = flatten_metadata(actual_metadata["port"]) - uncompressed_request = {} - for i, key in enumerate(compressed_request[0].keys()): - uncompressed_request[key] = combi[i] + # uncompressed_request = {} + # for i, key in enumerate(compressed_request[0].keys()): + # uncompressed_request[key] = combi[i] + complete_uncompressed_request = ( path, scheme, @@ -285,7 +286,8 @@ def get_fdb_requests( sorted_request_ranges, fdb_node_ranges, ) = self.sort_fdb_request_ranges(current_start_idxs, lat_length, fdb_node_ranges) - fdb_requests.append((path, sorted_request_ranges, leaf_metadata)) + fdb_requests.append((path, sorted_request_ranges, deepcopy(leaf_metadata))) + # TODO: did we need to deepcopy the leaf metadata?? fdb_requests_decoding_info.append((original_indices, fdb_node_ranges)) # Otherwise remap the path for this key and iterate again over children diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index e83ce6a97..5420c3e21 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -2,6 +2,7 @@ import pandas as pd from qubed import Qube from qubed.value_types import QEnum +from copy import deepcopy from ..datacube.datacube_axis import UnsliceableDatacubeAxis from ..datacube.transformations.datacube_mappers.datacube_mappers import DatacubeMapper @@ -77,6 +78,7 @@ def build_branch( ax, idxs=None, compressed_idxs=None, + metadata_idx_stack=None, ): final_children_and_vals = [] if real_uncompressed_axis: @@ -89,7 +91,11 @@ def build_branch( child_polytopes = self.find_children_polytopes(polytopes, poly, sliced_polys_) if idxs: compressed_idxs.append([idxs[i]]) - children = self._slice(child, child_polytopes, datacube, datacube_transformations, compressed_idxs) + metadata_idx_stack.append([idxs[i]]) + current_metadata_idx_stack = deepcopy(metadata_idx_stack) + children = self._slice(child, child_polytopes, datacube, datacube_transformations, + compressed_idxs, metadata_idx_stack) + metadata_idx_stack.pop() # If this node used to have children but now has none due to filtering, skip it. if child.children and not children: continue @@ -97,7 +103,7 @@ def build_branch( new_found_vals = self.find_new_vals([found_val], ax) if idxs: - request_child_val = (children, new_found_vals, compressed_idxs) + request_child_val = (children, new_found_vals, current_metadata_idx_stack) else: request_child_val = (children, new_found_vals) final_children_and_vals.append(request_child_val) @@ -107,14 +113,18 @@ def build_branch( # create children if idxs: compressed_idxs.append([idxs]) - children = self._slice(child, child_polytopes, datacube, datacube_transformations, compressed_idxs) + metadata_idx_stack.append([idxs]) + current_metadata_idx_stack = deepcopy(metadata_idx_stack) + children = self._slice(child, child_polytopes, datacube, datacube_transformations, + compressed_idxs, metadata_idx_stack) + metadata_idx_stack.pop() # If this node used to have children but now has none due to filtering, skip it. if child.children and not children: return None new_found_vals = self.find_new_vals(found_vals, ax) if idxs: - request_child_val = (children, new_found_vals, compressed_idxs) + request_child_val = (children, new_found_vals, current_metadata_idx_stack) else: request_child_val = (children, new_found_vals) final_children_and_vals.append(request_child_val) @@ -123,11 +133,14 @@ def build_branch( return None return final_children_and_vals - def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compressed_idxs=None) -> list[Qube]: + def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compressed_idxs=None, metadata_idx_stack=None) -> list[Qube]: if compressed_idxs is None: compressed_idxs = [[[0]]] result = [] + if metadata_idx_stack is None: + metadata_idx_stack = [[[0]]] + if len(q.children) == 0: # add "fake" axes and their nodes in order -> what about merged axes?? mapper_transformation = None @@ -230,6 +243,7 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compres ax, idxs, compressed_idxs, + metadata_idx_stack ) if final_children_and_vals is None: @@ -248,8 +262,9 @@ def find_metadata(metadata_idx): metadata[k] = vs[ix] return metadata - for children, new_found_vals, idxs in final_children_and_vals: - metadata_idx = format_metadata_idxs(idxs) + for children, new_found_vals, current_metadata_idxs in final_children_and_vals: + # metadata_idx = format_metadata_idxs(idxs) + metadata_idx = format_metadata_idxs(current_metadata_idxs) metadata = find_metadata(metadata_idx) qube_node = Qube.make_node( key=child.key, values=QEnum(new_found_vals), metadata=metadata, children=children From 35d3286979c35f844ca5aab0ce26992556ce9974 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 4 Aug 2025 10:24:43 +0200 Subject: [PATCH 093/105] clean up --- polytope_feature/engine/qubed_slicer.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index 5420c3e21..82058ef20 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -77,24 +77,21 @@ def build_branch( datacube_transformations, ax, idxs=None, - compressed_idxs=None, metadata_idx_stack=None, ): final_children_and_vals = [] if real_uncompressed_axis: for i, found_val in enumerate(found_vals): - # sliced_polys_ = [sliced_polys[i]] if i < len(sliced_polys): sliced_polys_ = [sliced_polys[i]] else: sliced_polys_ = sliced_polys child_polytopes = self.find_children_polytopes(polytopes, poly, sliced_polys_) if idxs: - compressed_idxs.append([idxs[i]]) metadata_idx_stack.append([idxs[i]]) current_metadata_idx_stack = deepcopy(metadata_idx_stack) children = self._slice(child, child_polytopes, datacube, datacube_transformations, - compressed_idxs, metadata_idx_stack) + metadata_idx_stack) metadata_idx_stack.pop() # If this node used to have children but now has none due to filtering, skip it. if child.children and not children: @@ -112,11 +109,10 @@ def build_branch( child_polytopes = self.find_children_polytopes(polytopes, poly, sliced_polys) # create children if idxs: - compressed_idxs.append([idxs]) metadata_idx_stack.append([idxs]) current_metadata_idx_stack = deepcopy(metadata_idx_stack) children = self._slice(child, child_polytopes, datacube, datacube_transformations, - compressed_idxs, metadata_idx_stack) + metadata_idx_stack) metadata_idx_stack.pop() # If this node used to have children but now has none due to filtering, skip it. if child.children and not children: @@ -133,9 +129,7 @@ def build_branch( return None return final_children_and_vals - def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compressed_idxs=None, metadata_idx_stack=None) -> list[Qube]: - if compressed_idxs is None: - compressed_idxs = [[[0]]] + def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, metadata_idx_stack=None) -> list[Qube]: result = [] if metadata_idx_stack is None: @@ -211,6 +205,8 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compres # TODO: here add the axes to datacube backend with transformations for child.key # TODO: update the datacube axis_options before we dynamically change the axes + + # TODO: this is slow... will need to make it faster and only do this when we need... datacube.add_axes_dynamically(child) # here now first change the values in the polytopes on the axis to reflect the axis type @@ -242,7 +238,6 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, compres datacube_transformations, ax, idxs, - compressed_idxs, metadata_idx_stack ) From 097a67445b2843dde7ff070138edd605525da315 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 4 Aug 2025 13:02:29 +0200 Subject: [PATCH 094/105] WIP: clean up --- polytope_feature/engine/qubed_slicer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index 82058ef20..2a2a981c4 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -258,7 +258,6 @@ def find_metadata(metadata_idx): return metadata for children, new_found_vals, current_metadata_idxs in final_children_and_vals: - # metadata_idx = format_metadata_idxs(idxs) metadata_idx = format_metadata_idxs(current_metadata_idxs) metadata = find_metadata(metadata_idx) qube_node = Qube.make_node( From b1bbf5141c7921cbe106983605ce925f57fad6d5 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 4 Aug 2025 13:11:58 +0200 Subject: [PATCH 095/105] change to sites.ecmwf instead of nexus --- .gitignore | 4 +++- tests/conftest.py | 14 +++++++------- tests/test_cyclic_nearest.py | 2 +- tests/test_datacube_axes_init.py | 2 +- tests/test_healpix_mapper.py | 2 +- tests/test_healpix_nested_grid.py | 2 +- tests/test_incomplete_tree_fdb.py | 2 +- tests/test_merge_octahedral_one_axis.py | 2 +- tests/test_octahedral_grid.py | 2 +- tests/test_reduced_ll_grid.py | 2 +- tests/test_regular_grid.py | 4 ++-- tests/test_slicer_era5.py | 2 +- tests/test_snapping_real_data.py | 2 +- 13 files changed, 22 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 4f156bc34..932d746dd 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,6 @@ rust_deployment_venv _version.py *.DS_Store rust_venv -test_w_qubed \ No newline at end of file +test_w_qubed +*.nc +**/*venv* \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 4d52f0d3d..1ea51bb91 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,13 +30,13 @@ def downloaded_data_test_files(shared_temp_data_dir): Downloads all required GRIB test files once per session. """ files_to_download = [ - ("https://get.ecmwf.int/test-data/polytope/test-data/foo.grib", "foo.grib"), - ("https://get.ecmwf.int/test-data/polytope/test-data/local.grib", "local.grib"), - ("https://get.ecmwf.int/test-data/polytope/test-data/aifs_data_from_fdb.grib", "aifs_data_from_fdb.grib"), - ("https://get.ecmwf.int/test-data/polytope/test-data/wind_gust_and_t2m.grib", "wind_gust_and_t2m.grib"), - ("https://get.ecmwf.int/test-data/polytope/test-data/t2m_jan_3_v2.grib", "t2m_jan_3_v2.grib"), - ("https://get.ecmwf.int/test-data/polytope/test-data/wave_spectra.grib", "wave_spectra.grib"), - ("https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib", "era5-levels-members.grib"), + ("https://sites.ecmwf.int/repository/polytope/test-data/foo.grib", "foo.grib"), + ("https://sites.ecmwf.int/repository/polytope/test-data/local.grib", "local.grib"), + ("https://sites.ecmwf.int/repository/polytope/test-data/aifs_data_from_fdb.grib", "aifs_data_from_fdb.grib"), + ("https://sites.ecmwf.int/repository/polytope/test-data/wind_gust_and_t2m.grib", "wind_gust_and_t2m.grib"), + ("https://sites.ecmwf.int/repository/polytope/test-data/t2m_jan_3_v2.grib", "t2m_jan_3_v2.grib"), + ("https://sites.ecmwf.int/repository/polytope/test-data/wave_spectra.grib", "wave_spectra.grib"), + ("https://sites.ecmwf.int/repository/polytope/test-data/era5-levels-members.grib", "era5-levels-members.grib"), ] downloaded_paths = [] diff --git a/tests/test_cyclic_nearest.py b/tests/test_cyclic_nearest.py index 792ce3d43..d02f3e4d3 100644 --- a/tests/test_cyclic_nearest.py +++ b/tests/test_cyclic_nearest.py @@ -10,7 +10,7 @@ class TestRegularGrid: def setup_method(self, method): - nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" + nexus_url = "https://sites.ecmwf.int/repository/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") self.options = { "axis_config": [ diff --git a/tests/test_datacube_axes_init.py b/tests/test_datacube_axes_init.py index d92a08cf3..e7f40efb2 100644 --- a/tests/test_datacube_axes_init.py +++ b/tests/test_datacube_axes_init.py @@ -10,7 +10,7 @@ class TestInitDatacubeAxes: def setup_method(self, method): - nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" + nexus_url = "https://sites.ecmwf.int/repository/polytope/test-data/foo.grib" download_test_data(nexus_url, "foo.grib") ds = data.from_source("file", "./tests/data/foo.grib") diff --git a/tests/test_healpix_mapper.py b/tests/test_healpix_mapper.py index a0ad6586a..a6d3fc738 100644 --- a/tests/test_healpix_mapper.py +++ b/tests/test_healpix_mapper.py @@ -12,7 +12,7 @@ class TestHealpixGrid: def setup_method(self, method): - nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/healpix.grib" + nexus_url = "https://sites.ecmwf.int/repository/polytope/test-data/healpix.grib" download_test_data(nexus_url, "healpix.grib") ds = data.from_source("file", "./tests/data/healpix.grib") diff --git a/tests/test_healpix_nested_grid.py b/tests/test_healpix_nested_grid.py index 8a929a2fd..8caa68c24 100644 --- a/tests/test_healpix_nested_grid.py +++ b/tests/test_healpix_nested_grid.py @@ -13,7 +13,7 @@ class TestHealpixNestedGrid: def setup_method(self, method): - nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/healpix_nested.grib" + nexus_url = "https://sites.ecmwf.int/repository/polytope/test-data/healpix_nested.grib" download_test_data(nexus_url, "healpix_nested.grib") ds = data.from_source("file", "./tests/data/healpix_nested.grib")[3] diff --git a/tests/test_incomplete_tree_fdb.py b/tests/test_incomplete_tree_fdb.py index 2fbfcaf28..2bc689dcd 100644 --- a/tests/test_incomplete_tree_fdb.py +++ b/tests/test_incomplete_tree_fdb.py @@ -9,7 +9,7 @@ class TestRegularGrid: def setup_method(self, method): - nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" + nexus_url = "https://sites.ecmwf.int/repository/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") self.options = { "axis_config": [ diff --git a/tests/test_merge_octahedral_one_axis.py b/tests/test_merge_octahedral_one_axis.py index 4ddf90582..05f8269af 100644 --- a/tests/test_merge_octahedral_one_axis.py +++ b/tests/test_merge_octahedral_one_axis.py @@ -9,7 +9,7 @@ class TestSlicingMultipleTransformationsOneAxis: def setup_method(self, method): - nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" + nexus_url = "https://sites.ecmwf.int/repository/polytope/test-data/foo.grib" download_test_data(nexus_url, "foo.grib") ds = data.from_source("file", "./tests/data/foo.grib") diff --git a/tests/test_octahedral_grid.py b/tests/test_octahedral_grid.py index f44ccf337..6c3d7fa16 100644 --- a/tests/test_octahedral_grid.py +++ b/tests/test_octahedral_grid.py @@ -9,7 +9,7 @@ class TestOctahedralGrid: def setup_method(self, method): - nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/foo.grib" + nexus_url = "https://sites.ecmwf.int/repository/polytope/test-data/foo.grib" download_test_data(nexus_url, "foo.grib") ds = data.from_source("file", "./tests/data/foo.grib") diff --git a/tests/test_reduced_ll_grid.py b/tests/test_reduced_ll_grid.py index b0a3535c4..3b8f3eb3c 100644 --- a/tests/test_reduced_ll_grid.py +++ b/tests/test_reduced_ll_grid.py @@ -12,7 +12,7 @@ class TestReducedLatLonGrid: def setup_method(self, method): - nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/wave.grib" + nexus_url = "https://sites.ecmwf.int/repository/polytope/test-data/wave.grib" download_test_data(nexus_url, "wave.grib") self.options = { "axis_config": [ diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index e094b3bca..1b0a128a6 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -12,7 +12,7 @@ class TestRegularGrid: def setup_method(self, method): - nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" + nexus_url = "https://sites.ecmwf.int/repository/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") self.options = { "axis_config": [ @@ -102,7 +102,7 @@ def test_regular_grid(self): tol = 1e-8 leaves = result.leaves for i in range(len(leaves)): - right_pl_results = leaves[i].result[len(leaves[i].values) :] + right_pl_results = leaves[i].result[len(leaves[i].values):] result_tree = right_pl_results[0] cubepath = leaves[i].flatten() lat = cubepath["latitude"][0] diff --git a/tests/test_slicer_era5.py b/tests/test_slicer_era5.py index 9f11bcef8..cf5223fbf 100644 --- a/tests/test_slicer_era5.py +++ b/tests/test_slicer_era5.py @@ -10,7 +10,7 @@ class TestSlicingEra5Data: def setup_method(self, method): - nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" + nexus_url = "https://sites.ecmwf.int/repository/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") ds = data.from_source("file", "./tests/data/era5-levels-members.grib") diff --git a/tests/test_snapping_real_data.py b/tests/test_snapping_real_data.py index 9b5950742..aaddd688d 100644 --- a/tests/test_snapping_real_data.py +++ b/tests/test_snapping_real_data.py @@ -13,7 +13,7 @@ class TestSlicingEra5Data: def setup_method(self, method): - nexus_url = "https://get.ecmwf.int/test-data/polytope/test-data/era5-levels-members.grib" + nexus_url = "https://sites.ecmwf.int/repository/polytope/test-data/era5-levels-members.grib" download_test_data(nexus_url, "era5-levels-members.grib") ds = data.from_source("file", "./tests/data/era5-levels-members.grib") From c32c9c514126ae3930f9c2542e0f9ed7876c04f0 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 4 Aug 2025 13:14:45 +0200 Subject: [PATCH 096/105] black --- tests/test_regular_grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_regular_grid.py b/tests/test_regular_grid.py index 1b0a128a6..2c9cc1c78 100644 --- a/tests/test_regular_grid.py +++ b/tests/test_regular_grid.py @@ -102,7 +102,7 @@ def test_regular_grid(self): tol = 1e-8 leaves = result.leaves for i in range(len(leaves)): - right_pl_results = leaves[i].result[len(leaves[i].values):] + right_pl_results = leaves[i].result[len(leaves[i].values) :] result_tree = right_pl_results[0] cubepath = leaves[i].flatten() lat = cubepath["latitude"][0] From 96605eab157c0a72469c334b8b564619df9eb7e2 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 4 Aug 2025 16:13:47 +0200 Subject: [PATCH 097/105] avoid deepcopies --- polytope_feature/engine/qubed_slicer.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index 2a2a981c4..6eaa4777c 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -2,7 +2,6 @@ import pandas as pd from qubed import Qube from qubed.value_types import QEnum -from copy import deepcopy from ..datacube.datacube_axis import UnsliceableDatacubeAxis from ..datacube.transformations.datacube_mappers.datacube_mappers import DatacubeMapper @@ -88,11 +87,9 @@ def build_branch( sliced_polys_ = sliced_polys child_polytopes = self.find_children_polytopes(polytopes, poly, sliced_polys_) if idxs: - metadata_idx_stack.append([idxs[i]]) - current_metadata_idx_stack = deepcopy(metadata_idx_stack) + current_metadata_idx_stack = metadata_idx_stack + [[idxs[i]]] children = self._slice(child, child_polytopes, datacube, datacube_transformations, - metadata_idx_stack) - metadata_idx_stack.pop() + current_metadata_idx_stack) # If this node used to have children but now has none due to filtering, skip it. if child.children and not children: continue @@ -109,11 +106,9 @@ def build_branch( child_polytopes = self.find_children_polytopes(polytopes, poly, sliced_polys) # create children if idxs: - metadata_idx_stack.append([idxs]) - current_metadata_idx_stack = deepcopy(metadata_idx_stack) + current_metadata_idx_stack = metadata_idx_stack + [[idxs]] children = self._slice(child, child_polytopes, datacube, datacube_transformations, metadata_idx_stack) - metadata_idx_stack.pop() # If this node used to have children but now has none due to filtering, skip it. if child.children and not children: return None From acc74c35906eeeea8e2a3af85c721826904849de Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 4 Aug 2025 16:29:06 +0200 Subject: [PATCH 098/105] consistent metadata idxs --- polytope_feature/engine/qubed_slicer.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index 6eaa4777c..eb9573591 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -106,7 +106,7 @@ def build_branch( child_polytopes = self.find_children_polytopes(polytopes, poly, sliced_polys) # create children if idxs: - current_metadata_idx_stack = metadata_idx_stack + [[idxs]] + current_metadata_idx_stack = metadata_idx_stack + [idxs] children = self._slice(child, child_polytopes, datacube, datacube_transformations, metadata_idx_stack) # If this node used to have children but now has none due to filtering, skip it. @@ -128,12 +128,11 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, metadat result = [] if metadata_idx_stack is None: - metadata_idx_stack = [[[0]]] + metadata_idx_stack = [[0]] if len(q.children) == 0: # add "fake" axes and their nodes in order -> what about merged axes?? mapper_transformation = None - # for transformation in list(datacube_transformations.values()): for transformation in datacube_transformations: if isinstance(transformation, DatacubeMapper): mapper_transformation = transformation @@ -239,10 +238,6 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, metadat if final_children_and_vals is None: continue - def format_metadata_idxs(idxs): - flat_indices = [np.ravel(idx) for idx in idxs] - return flat_indices - def find_metadata(metadata_idx): metadata = {} for k, vs in child.metadata.items(): @@ -253,8 +248,7 @@ def find_metadata(metadata_idx): return metadata for children, new_found_vals, current_metadata_idxs in final_children_and_vals: - metadata_idx = format_metadata_idxs(current_metadata_idxs) - metadata = find_metadata(metadata_idx) + metadata = find_metadata(current_metadata_idxs) qube_node = Qube.make_node( key=child.key, values=QEnum(new_found_vals), metadata=metadata, children=children ) From d09d016da3b446253796438b6281f34f00026d50 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 4 Aug 2025 16:36:02 +0200 Subject: [PATCH 099/105] optimise metadata indexing --- polytope_feature/engine/qubed_slicer.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index eb9573591..323198e6c 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -241,10 +241,26 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, metadat def find_metadata(metadata_idx): metadata = {} for k, vs in child.metadata.items(): - metadata_depth = len(vs.shape) - relevant_metadata_idxs = metadata_idx[:metadata_depth] - ix = np.ix_(*relevant_metadata_idxs) - metadata[k] = vs[ix] + depth = len(vs.shape) + idxs = metadata_idx[:depth] + + # Clean and flatten inputs + cleaned_idxs = [] + multi_dim = 0 + for idx in idxs: + if isinstance(idx, np.ndarray): + if idx.ndim > 1: + idx = idx.ravel() + if idx.size > 1: + multi_dim += 1 + cleaned_idxs.append(idx) + + # If multiple axes have multiple elements, use np.ix_ to broadcast properly + if multi_dim > 1: + metadata[k] = vs[np.ix_(*cleaned_idxs)] + else: + metadata[k] = vs[tuple(cleaned_idxs)] + return metadata for children, new_found_vals, current_metadata_idxs in final_children_and_vals: From 29d5c13c1327345534875e60420c6a1eeb4374f4 Mon Sep 17 00:00:00 2001 From: mathleur Date: Mon, 4 Aug 2025 17:24:18 +0200 Subject: [PATCH 100/105] fix compression of axes with qube slicer --- polytope_feature/datacube/backends/qubed.py | 7 ++---- polytope_feature/engine/qubed_slicer.py | 24 ++++----------------- 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/polytope_feature/datacube/backends/qubed.py b/polytope_feature/datacube/backends/qubed.py index 2d8cb7f30..30379b5a9 100644 --- a/polytope_feature/datacube/backends/qubed.py +++ b/polytope_feature/datacube/backends/qubed.py @@ -174,7 +174,8 @@ def get(self, requests, context=None): interm_branch_tuple_values.append(compressed_request[0][key]) request_combis = product(*interm_branch_tuple_values) - index_combis = list(product(*[range(len(lst)) for lst in interm_branch_tuple_values])) + index_combis_raw = list(product(*[range(len(lst)) for lst in interm_branch_tuple_values])) + index_combis = [(0, *comb) for comb in index_combis_raw] # Need to extract the possible requests and add them to the right nodes @@ -199,10 +200,6 @@ def flatten_metadata(value): host = flatten_metadata(actual_metadata["host"]) port = flatten_metadata(actual_metadata["port"]) - # uncompressed_request = {} - # for i, key in enumerate(compressed_request[0].keys()): - # uncompressed_request[key] = combi[i] - complete_uncompressed_request = ( path, scheme, diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index 323198e6c..eb9573591 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -241,26 +241,10 @@ def _slice(self, q: Qube, polytopes, datacube, datacube_transformations, metadat def find_metadata(metadata_idx): metadata = {} for k, vs in child.metadata.items(): - depth = len(vs.shape) - idxs = metadata_idx[:depth] - - # Clean and flatten inputs - cleaned_idxs = [] - multi_dim = 0 - for idx in idxs: - if isinstance(idx, np.ndarray): - if idx.ndim > 1: - idx = idx.ravel() - if idx.size > 1: - multi_dim += 1 - cleaned_idxs.append(idx) - - # If multiple axes have multiple elements, use np.ix_ to broadcast properly - if multi_dim > 1: - metadata[k] = vs[np.ix_(*cleaned_idxs)] - else: - metadata[k] = vs[tuple(cleaned_idxs)] - + metadata_depth = len(vs.shape) + relevant_metadata_idxs = metadata_idx[:metadata_depth] + ix = np.ix_(*relevant_metadata_idxs) + metadata[k] = vs[ix] return metadata for children, new_found_vals, current_metadata_idxs in final_children_and_vals: From a50cf2f1dfd5c870ece58753e746f58fd650cee5 Mon Sep 17 00:00:00 2001 From: mathleur Date: Tue, 5 Aug 2025 15:31:27 +0200 Subject: [PATCH 101/105] WIP:optimise --- .gitignore | 1 + polytope_feature/engine/qubed_slicer.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 85a8d897c..11c4ae298 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ polytope_feature/_version.py rust_venv **/*venv*/ test_w_qubed +performance_plots_qubed \ No newline at end of file diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index eb9573591..0975fcd2d 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -27,8 +27,8 @@ def find_datacube_vals(): def find_values_between(self, polytope, ax, node, datacube, lower, upper, path=None): if isinstance(ax, UnsliceableDatacubeAxis): - values = [v for v in node.values if lower <= v <= upper] - indices = [i for i, v in enumerate(node.values) if lower <= v <= upper] + filtered = [(i, v) for i, v in enumerate(node.values) if lower <= v <= upper] + indices, values = zip(*filtered) if filtered else ([], []) return values, indices tol = ax.tol From 003815b59209c96f5e32bd48c0679a09aee46f01 Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 6 Aug 2025 13:08:34 +0200 Subject: [PATCH 102/105] fix polytope-gj interface for qubed backend --- polytope_feature/datacube/backends/qubed.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/polytope_feature/datacube/backends/qubed.py b/polytope_feature/datacube/backends/qubed.py index 30379b5a9..e3f48de15 100644 --- a/polytope_feature/datacube/backends/qubed.py +++ b/polytope_feature/datacube/backends/qubed.py @@ -200,16 +200,10 @@ def flatten_metadata(value): host = flatten_metadata(actual_metadata["host"]) port = flatten_metadata(actual_metadata["port"]) - complete_uncompressed_request = ( - path, - scheme, - offset, - host, - port, - compressed_request[1], - self.grid_md5_hash, - ) - complete_list_complete_uncompressed_requests.append(complete_uncompressed_request) + gj_extraction_request = pygj.PathExtractionRequest( + path, scheme, offset, host, port, compressed_request[1], self.grid_md5_hash) + + complete_list_complete_uncompressed_requests.append(gj_extraction_request) complete_fdb_decoding_info.append(fdb_requests_decoding_info[j]) if logging.root.level <= logging.DEBUG: From 4221363fa7ce5ee5bb93ccbaa07aa02a9c7cf67f Mon Sep 17 00:00:00 2001 From: mathleur Date: Wed, 6 Aug 2025 17:03:56 +0200 Subject: [PATCH 103/105] think about refactoring gj/qube backend --- polytope_feature/datacube/backends/qubed.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/polytope_feature/datacube/backends/qubed.py b/polytope_feature/datacube/backends/qubed.py index e3f48de15..67aa28dae 100644 --- a/polytope_feature/datacube/backends/qubed.py +++ b/polytope_feature/datacube/backends/qubed.py @@ -149,6 +149,18 @@ def get_indices(self, path, path_node, axis, lower, upper, method=None): return (idx_between, indexes) def get(self, requests, context=None): + """ + We have a compressed tree of requests, which we need to decompress completely with its metadata indexes. + BUT the last two axes, we would like to "ignore" in the decompression and instead, we create grid index ranges from them. + WHILE we decompress, we need to keep some kind of map from decompressed request + grid index ranges to corresponding tree node. + This mapping will map potentially several decompressed request + grid index ranges tuples to the same tree nodes. + + ADDED DIFFICULTY: the grid index ranges MUST be ordered (which is not guaranteed from the tree) so we need to sort them, while also sorting the tree nodes so that they match up. + + UNTIL NOW, we had a map of compressed request + grid index ranges tuples to tree nodes. + We could just add a map of compressed request -> list of decompressed request + associated metadata + """ + if context is None: context = {} if len(requests.children) == 0: @@ -268,6 +280,7 @@ def get_fdb_requests( new_vals.append(val[:4] + val[5:7] + val[8:10]) key_value_path[requests.key] = new_vals leaf_path.update(key_value_path) + # TODO: in the leaf metadata, try to instead store mapping leaf_path -> list(individual metadata dicts for each uncompressed value of tree) leaf_metadata.update(requests.metadata) if len(requests.children[0].children[0].children) == 0: # find the fdb_requests and associated nodes to which to add results From 243d4f9abba6788ba6a1fa720e580f2edd7525ef Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 7 Aug 2025 09:48:34 +0200 Subject: [PATCH 104/105] fix bug --- polytope_feature/engine/qubed_slicer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polytope_feature/engine/qubed_slicer.py b/polytope_feature/engine/qubed_slicer.py index 0975fcd2d..fac8e9611 100644 --- a/polytope_feature/engine/qubed_slicer.py +++ b/polytope_feature/engine/qubed_slicer.py @@ -108,7 +108,7 @@ def build_branch( if idxs: current_metadata_idx_stack = metadata_idx_stack + [idxs] children = self._slice(child, child_polytopes, datacube, datacube_transformations, - metadata_idx_stack) + current_metadata_idx_stack) # If this node used to have children but now has none due to filtering, skip it. if child.children and not children: return None From 22127687c864eaad5bccb7c9eb3d818a3b4614bd Mon Sep 17 00:00:00 2001 From: mathleur Date: Thu, 7 Aug 2025 10:07:43 +0200 Subject: [PATCH 105/105] clean up --- polytope_feature/datacube/backends/datacube.py | 1 - .../transformations/datacube_mappers/datacube_mappers.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/polytope_feature/datacube/backends/datacube.py b/polytope_feature/datacube/backends/datacube.py index 5b149ac1f..8dc7a8ed2 100644 --- a/polytope_feature/datacube/backends/datacube.py +++ b/polytope_feature/datacube/backends/datacube.py @@ -145,7 +145,6 @@ def get_indices(self, path: DatacubePath, axis, lower, upper, method=None): e.g. returns integer discrete points between two floats """ path = self.fit_path(path) - print(path) indexes = axis.find_indexes(path, self) idx_between = axis.find_indices_between(indexes, lower, upper, self, method) diff --git a/polytope_feature/datacube/transformations/datacube_mappers/datacube_mappers.py b/polytope_feature/datacube/transformations/datacube_mappers/datacube_mappers.py index e06c8fd38..80d971188 100644 --- a/polytope_feature/datacube/transformations/datacube_mappers/datacube_mappers.py +++ b/polytope_feature/datacube/transformations/datacube_mappers/datacube_mappers.py @@ -105,9 +105,6 @@ def unmap_path_key(self, key_value_path, leaf_path, unwanted_path, axis): unwanted_path[axis.name] = unwanted_val if axis.name == self._mapped_axes()[1]: first_val = unwanted_path[self._mapped_axes()[0]] - # unmapped_idx = [self.unmap(first_val, (val,)) for val in value] - # print("AND HERE??") - # print(values) unmapped_idx = self.unmap(first_val, values) leaf_path.pop(self._mapped_axes()[0], None) key_value_path.pop(axis.name)