add metadata to qube implementation

.github/ci-config.yml

@@ -0,0 +1,13 @@
+dependencies: |
+  ecmwf/ecbuild
+  MathisRosenhauer/libaec@refs/tags/v1.1.3
+  ecmwf/eccodes
+  ecmwf/eckit@develop
+  ecmwf/metkit
+  ecmwf/fdb
+  ecmwf/gribjump
+dependency_cmake_options: |
+  ecmwf/gribjump: "-DENABLE_FDB_BUILD_TOOLS=ON"
+dependency_branch: develop
+parallelism_factor: 8
+self_build: false

.github/ci-hpc-config.yml

@@ -0,0 +1,21 @@
+build:
+  python: "3.10"
+  modules:
+    - ninja
+    - aec
+  dependencies:
+    - ecmwf/ecbuild@develop
+    - ecmwf/eccodes@develop
+    - ecmwf/eckit@develop
+    - ecmwf/metkit@develop
+    - ecmwf/fdb@develop
+    - ecmwf/gribjump@develop
+  dependency_cmake_options:
+    - "ecmwf/gribjump: '-DENABLE_FDB_BUILD_TOOLS=ON'"
+    - "ecmwf/fdb: '-DENABLE_LUSTRE=OFF'"
+  parallel: 64
+  env:
+    - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
+    - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions
+  pytest_cmd: |
+    python -m pytest -m 'not fdb' tests --cov=./ --cov-report=xml

.github/workflows/cd.yml

@@ -0,0 +1,11 @@
+name: cd
+
+on:
+  push:
+    tags:
+      - '**'
+
+jobs:
+  deploy:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/create-package.yml@v2
+    secrets: inherit

.github/workflows/ci.yaml

@@ -1,145 +1,45 @@
 name: ci
+
 on:
   # Trigger the workflow on push to master or develop, except tag creation
   push:
     branches:
-      - 'main'
-      - 'develop'
+      - "main"
+      - "develop"
+    tags-ignore:
+      - "**"
+
   # Trigger the workflow on pull request
   pull_request: ~
+
   # Trigger the workflow manually
   workflow_dispatch: ~
+
   # Trigger after public PR approved for CI
   pull_request_target:
     types: [labeled]
-  release:
-    types: [created]
-jobs:
-  qa:
-    name: qa
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v3
-        with:
-          repository: ${{ inputs.repository }}
-          ref: ${{ inputs.ref }}
 
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ inputs.python_version }}
-
-      - name: Install Python Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install black flake8 isort
-      - name: Check isort
-        run: isort --check .
-
-      - name: Check black
-        run: black --check .
-
-      - name: Check flake8
-        run: flake8 .
-  setup:
-    name: setup
-    runs-on: ubuntu-20.04
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-      inputs: ${{ steps.prepare-inputs.outputs.inputs }}
-      inputs-for-ubuntu: ${{ steps.prepare-inputs.outputs.inputs-for-ubuntu }}
-    steps:
-      - name: Set Matrix
-        id: set-matrix
-        shell: bash -eux {0}
-        run: |
-          MATRIX=$(cat << 'EOS'
-          name:
-          - gnu-11@ubuntu-22.04
-          - clang-14@ubuntu-22.04
-          include:
-          - name: gnu-11@ubuntu-22.04
-            os: ubuntu-22.04
-            compiler: gnu-11
-            compiler_cc: gcc-11
-            compiler_cxx: g++-11
-            compiler_fc: gfortran-11
-          - name: clang-14@ubuntu-22.04
-            os: ubuntu-22.04
-            compiler: clang-14
-            compiler_cc: clang-14
-            compiler_cxx: clang++-14
-            compiler_fc: gfortran-11
-          # Xcode compiler requires empty environment variables, so we pass null (~) here
-          EOS
-          )
-          SKIP_MATRIX_JOBS=$(cat << 'EOS'
-          ${{ inputs.skip_matrix_jobs }}
-          EOS
-          )
-          SELECT_NAME_COND="1 != 1"
-          SELECT_INCLUDE_COND="1 != 1"
-          for skip_job in $SKIP_MATRIX_JOBS; do SELECT_NAME_COND="$SELECT_NAME_COND or . == \"$skip_job\""; SELECT_INCLUDE_COND="$SELECT_INCLUDE_COND or .name == \"$skip_job\""; done
-          echo matrix=$(echo "$MATRIX" | yq eval "del(.name[] | select($SELECT_NAME_COND)) | del(.include[] | select($SELECT_INCLUDE_COND))" --output-format json --indent 0 -) >> $GITHUB_OUTPUT
-      - name: Prepare build-package Inputs
-        id: prepare-inputs
-        shell: bash -eux {0}
-        run: |
-          echo inputs=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '.' --output-format json --indent 0 -) >> $GITHUB_OUTPUT
-          echo inputs-for-ubuntu=$(echo "${{ inputs.build_package_inputs || '{}' }}" | yq eval '. * {"os":"ubuntu-20.04","compiler":"gnu-10","compiler_cc":"gcc-10","compiler_cxx":"g++-10","compiler_fc":"gfortran-10"}' --output-format json --indent 0 -) >> $GITHUB_OUTPUT
-  test:
-    name: test
-    needs:
-      - qa
-      - setup
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJson(needs.setup.outputs.matrix) }}
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Install eccodes and Dependencies
-        id: install-dependencies
-        uses: ecmwf-actions/build-package@v2
-        with:
-          self_build: false
-          dependencies: |
-            ecmwf/ecbuild@develop
-            MathisRosenhauer/libaec@master
-            ecmwf/eccodes@develop
-
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ inputs.python_version }}
-
-      - name: Install Python Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install pytest pytest-cov
-          python -m pip install -r requirements.txt 
-          python -m pip install -r ./tests/requirements_test.txt
-
-      - name: Verify Source Distribution
-        shell: bash -eux {0}
-        run: |
-          python setup.py sdist
-          python -m pip install dist/*
-      - name: Run Tests with Repository Code
-        env:
-          LD_LIBRARY_PATH: ${{ steps.install-dependencies.outputs.lib_path }}
-        shell: bash -eux {0}
-        run: |
-          DYLD_LIBRARY_PATH=${{ env.LD_LIBRARY_PATH }} python -m pytest -m "not fdb" tests --cov=./ --cov-report=xml
-          python -m coverage report
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
+      python_qa: true
+    secrets: inherit
+
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
+    with:
+      polytope: ecmwf/polytope@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit
 
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v4
-        with:
-          files: coverage.xml
   deploy:
-    needs: test
     if: ${{ github.event_name == 'release' }}
     name: Upload to Pypi
     runs-on: ubuntu-latest
@@ -159,4 +59,4 @@ jobs:
           TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
         run: |
           python -m build
-          twine upload dist/*
+          twine upload dist/*

.github/workflows/label-public-pr.yml

@@ -0,0 +1,10 @@
+# Manage labels of pull requests that originate from forks
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

.gitignore

@@ -29,4 +29,8 @@ polytope_feature/_version.py
 **.lock
 **/target
 **/*venv*/
-test_w_qubed
+test_w_qubed
+performance_plots_qubed
+_version.py
+*.DS_Store
+*.nc

codecov.yml

@@ -6,4 +6,10 @@ ignore:
   - "**/test*"
   - "**/pb2*"
   - "**/fdb*" # ignore fdb backend which we can't test in CI yet
-  - "polytope/datacube/backends/fdb.py" # ignore fdb backend which we can't test in CI yet
+  - "polytope/datacube/backends/fdb.py" # ignore fdb backend which we can't test in CI yet
+
+coverage:
+  status:
+    patch:
+      default:
+        enabled: false  # disables patch check

docs/Service/Examples/examples.md

@@ -12,11 +12,11 @@
 ## Open Data
 
 * <a href="../OpenData/od_timeseries_example">Timeseries</a>
-* <a href="../OpenData/od_polygon_example">Timeseries</a>
-* <a href="../OpenData/od_vertical_profile_example">Timeseries</a>
-* <a href="../OpenData/od_boundingbox_example">Timeseries</a>
-* <a href="../OpenData/od_trajectory_example">Timeseries</a>
-* <a href="../OpenData/od_country_example">Timeseries</a>
+* <a href="../OpenData/od_polygon_example">Polygon</a>
+* <a href="../OpenData/od_vertical_profile_example">Vertical Profile</a>
+* <a href="../OpenData/od_boundingbox_example">Bounding Box</a>
+* <a href="../OpenData/od_trajectory_example">Trajectory</a>
+* <a href="../OpenData/od_country_example">Country Cut-Out</a>
 
 For examples of Polytope Feature Extraction on Destination Earth Digital Twin Data please visit the following Github Repo: https://github.com/destination-earth-digital-twins/polytope-examples
 

polytope_feature/datacube/backends/datacube.py

@@ -145,7 +145,6 @@ def get_indices(self, path: DatacubePath, axis, lower, upper, method=None):
         e.g. returns integer discrete points between two floats
         """
         path = self.fit_path(path)
-        print(path)
         indexes = axis.find_indexes(path, self)
 
         idx_between = axis.find_indices_between(indexes, lower, upper, self, method)
@@ -170,7 +169,15 @@ def remap_path(self, path: DatacubePath):
         return path
 
     @staticmethod
-    def create(datacube, config={}, axis_options={}, compressed_axes_options=[], alternative_axes=[], datacube_axes={}, context=None):
+    def create(
+        datacube,
+        config={},
+        axis_options={},
+        compressed_axes_options=[],
+        alternative_axes=[],
+        datacube_axes={},
+        context=None,
+    ):
         # TODO: get the configs as None for pre-determined value and change them to empty dictionary inside the function
         if type(datacube).__name__ == "DataArray":
             from .xarray import XArrayDatacube
@@ -185,13 +192,15 @@ def create(datacube, config={}, axis_options={}, compressed_axes_options=[], alt
             )
             return fdbdatacube
         if type(datacube).__name__ == "Qube":
-            from .qubed import QubedDatacube
             from ..datacube_axis import _str_to_axis
+            from .qubed import QubedDatacube
+
             actual_datacube_axes = {}
             for key, value in datacube_axes.items():
                 actual_datacube_axes[key] = _str_to_axis[value]
-            qubed_datacube = QubedDatacube(datacube, actual_datacube_axes,
-                                           config, axis_options, compressed_axes_options, alternative_axes, context)
+            qubed_datacube = QubedDatacube(
+                datacube, actual_datacube_axes, config, axis_options, compressed_axes_options, alternative_axes, context
+            )
             return qubed_datacube
 
     def check_branching_axes(self, request):