Gemini Changes

.flake8

@@ -4,7 +4,7 @@ select = A,B,B9,BLK,C,E,F,I,N,S,W
 ignore = E203,W503,E501,B905,E704,C901
 max-complexity = 12
 max-line-length = 80
-application-import-names = operationsgateway_api,test
+application-import-names = operationsgateway_api,util,test
 import-order-style = google
 enable-extensions=G
 per-file-ignores =

.github/ci_config.yml

@@ -6,6 +6,7 @@ app:
   url_prefix: ""
   maintenance_file: operationsgateway_api/maintenance.json.example
   scheduled_maintenance_file: operationsgateway_api/scheduled_maintenance.json.example
+  # use_sub_second_timestamps: set by env
 images:
   thumbnail_size: [50, 50]
   default_colour_map: viridis
@@ -39,6 +40,7 @@ auth:
   refresh_token_validity_days: 7
   fedid_server_url: ldap://fed.cclrc.ac.uk:389
   fedid_server_ldap_realm: FED.CCLRC.AC.UK
+  allow_user_office_login: false
 experiments:
   user_office_rest_api_url: https://devapi.facilities.rl.ac.uk/users-service/v1
   scheduler_wsdl_url: https://devapis.facilities.rl.ac.uk/ws/ScheduleWebService?wsdl

.github/ci_ingest_echo_config.yml

@@ -11,12 +11,11 @@ database:
   connection_uri: mongodb://localhost:27017/opsgateway
   remote_experiments_file_path: /tmp/experiments_for_mongoimport.json
   test_users_file_path: util/users_for_mongoimport.json
-echo:
+source:
   endpoint_url: https://s3.echo.stfc.ac.uk
-  access_key: access_key
-  secret_key: secret_key
-  simulated_data_bucket: og-ci-simulated-data
-  storage_bucket: og-bucket
+  # access_key: set by env
+  # secret_key: set by env
+  # simulated_data_bucket: set by env
   page_size: 400
 api:
   https: false
@@ -27,7 +26,4 @@ api:
   log_config_path: /home/runner/work/operationsgateway-api/operationsgateway-api/operationsgateway_api/logging.ini
   gunicorn_num_workers: "1"
   timeout_seconds: 1000
-observability:
-  # used to view logs and traces in Elasticsearch run by Platforms & Services
-  environment: dev
-  secret_key: test
+  storage_bucket: og-bucket

.github/ci_requirements.txt

@@ -1,2 +1,2 @@
-poetry==2.1.1
+poetry==2.3.3
 nox==2025.2.9

.github/workflows/ci.yml

@@ -1,4 +1,6 @@
 name: CI
+permissions:
+  contents: read
 on:
   workflow_dispatch:
   pull_request:
@@ -7,12 +9,12 @@ on:
       - main
 
 jobs:
-  tests:
+  tests_epac:
     strategy:
       fail-fast: false
     runs-on: ubuntu-24.04
 
-    name: Python 3.11 Tests
+    name: Python 3.11 Tests EPAC
     steps:
       - name: Checkout OperationsGateway API
         uses: actions/checkout@v3
@@ -102,30 +104,153 @@ jobs:
         env:
           SSH_KEY_PUBLIC: ${{secrets.SSH_PUBLIC_KEY_FOR_AUTH_OPENSSH}}
 
-      # Setup steps for Echo ingestion script
-      - name: Configure echo access key
-        run: yq -i ".echo.access_key = \"$ECHO_S3_ACCESS_KEY\"" .github/ci_ingest_echo_config.yml
-        env:
-          ECHO_S3_ACCESS_KEY: ${{secrets.ECHO_S3_ACCESS_KEY}}
-      - name: Configure echo secret key
-        run: yq -i ".echo.secret_key = \"$ECHO_S3_SECRET_KEY\"" .github/ci_ingest_echo_config.yml
-        env:
-          ECHO_S3_SECRET_KEY: ${{secrets.ECHO_S3_SECRET_KEY}}
       - name: Copy config for Echo Ingest script to correct place
         run: cp .github/ci_ingest_echo_config.yml util/realistic_data/config.yml
 
       - name: Run Echo Ingest script
         run: poetry run python util/realistic_data/ingest_echo_data.py
-
-      # Run Nox tests session, saves and uploads a coverage report to codecov
+        env:
+          # App config:
+          APP__USE_SUB_SECOND_TIMESTAMPS: False
+          # Ingest config:
+          SOURCE__ACCESS_KEY: ${{secrets.ECHO_S3_ACCESS_KEY}}
+          SOURCE__SECRET_KEY: ${{secrets.ECHO_S3_SECRET_KEY}}
+          SOURCE__SIMULATED_DATA_BUCKET: og-ci-simulated-data-epac
+
+      - name: Dump logs
+        if: ${{ failure() }}
+        run: cat operationsgateway-api.log
+
+      # Run EPAC tests session, saves and uploads a coverage report to codecov
       - name: Run Nox tests session
         run: nox -s tests -- --cov=operationsgateway_api --cov-report=xml
+        env:
+          # App config:
+          APP__USE_SUB_SECOND_TIMESTAMPS: False
 
       - name: Upload code coverage report
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
+  tests-gemini:
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-24.04
+
+    name: Python 3.11 Tests Gemini
+    steps:
+      - name: Checkout OperationsGateway API
+        uses: actions/checkout@v3
+
+      # Install system dependencies
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y build-essential cmake uuid-dev libsasl2-dev libldap2-dev libssl-dev
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Load Pip cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-3.11-${{ env.pythonLocation }}-${{ hashFiles('.github/ci_requirements.txt') }}
+      - name: Install Poetry & Nox
+        run: pip install -r .github/ci_requirements.txt
+
+      # Install and start MongoDB
+      - name: Start MongoDB
+        uses: supercharge/mongodb-github-action@1.12.1
+        with:
+          mongodb-version: '7.0'
+
+      # Used to install mongoimport when Ubuntu 22.04 is used, identified at https://github.com/actions/runner-images/issues/6626#issuecomment-1327744126
+      - name: Install MongoDB Database Tools
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y wget gnupg
+          wget -qO - https://www.mongodb.org/static/pgp/server-7.0.asc | sudo apt-key add -
+          echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list
+          sudo apt-get update
+          sudo apt-get install -y mongodb-database-tools
+
+      # Read the database name from the config file and store it in an environment variable
+      - name: Get database name from ci_config.yml
+        run: echo "DATABASE_NAME=$(grep database_name .github/ci_config.yml | cut -d ':' -f 2 | tr -d '[:space:]')" >> $GITHUB_ENV
+
+      # Load Poetry virtual environment dependencies and install API dependencies
+      - name: Load Poetry cache
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-3.11-${{ hashFiles('poetry.lock') }}
+      - name: Install dependencies
+        run: poetry install --without simulated-data
+
+      # Setup minio and create bucket
+      - name: Setup minio
+        run: |
+          docker run -d -p 9000:9000 --name minio \
+                     -e "MINIO_ACCESS_KEY=minioadmin" \
+                     -e "MINIO_SECRET_KEY=minioadmin" \
+                     -v /tmp/data:/data \
+                     -v /tmp/config:/root/.minio \
+                     minio/minio server /data
+          export AWS_ACCESS_KEY_ID=minioadmin
+          export AWS_SECRET_ACCESS_KEY=minioadmin
+          export AWS_EC2_METADATA_DISABLED=true
+          aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://og-actions-test
+      - name: Move CI config.yml to correct place
+        run: cp .github/ci_config.yml operationsgateway_api/config.yml
+
+      - name: Setup logging configuration
+        run: cp operationsgateway_api/logging.ini.example operationsgateway_api/logging.ini
+
+      - name: Create log file
+        run: touch "$GITHUB_WORKSPACE/logs.log"
+
+      - name: Create SSH private key file for auth
+        run: 'echo "$SSH_KEY_PRIVATE" > /home/runner/work/operationsgateway-api/id_rsa'
+        shell: bash
+        env:
+          SSH_KEY_PRIVATE: ${{secrets.SSH_PRIVATE_KEY_FOR_AUTH_OPENSSH}}
+
+      - name: Set permissions on private key file
+        run: chmod 600 /home/runner/work/operationsgateway-api/id_rsa
+
+      - name: Create SSH public key file for auth
+        run: 'echo "$SSH_KEY_PUBLIC" > /home/runner/work/operationsgateway-api/id_rsa.pub'
+        shell: bash
+        env:
+          SSH_KEY_PUBLIC: ${{secrets.SSH_PUBLIC_KEY_FOR_AUTH_OPENSSH}}
+
+      - name: Copy config for Echo Ingest script to correct place
+        run: cp .github/ci_ingest_echo_config.yml util/realistic_data/config.yml
+
+      - name: Run Echo Ingest script
+        run: poetry run python util/realistic_data/ingest_echo_data.py
+        env:
+          # App config:
+          APP__USE_SUB_SECOND_TIMESTAMPS: True
+          # Ingest config:
+          SOURCE__ACCESS_KEY: ${{secrets.ECHO_S3_ACCESS_KEY}}
+          SOURCE__SECRET_KEY: ${{secrets.ECHO_S3_SECRET_KEY}}
+          SOURCE__SIMULATED_DATA_BUCKET: og-ci-simulated-data-gemini
+
+      - name: Dump logs
+        if: ${{ failure() }}
+        run: cat operationsgateway-api.log
+
+      # Run Gemini tests session, print coverage but do not upload
+      - name: Run Nox tests session
+        run: nox -s tests -- --cov=operationsgateway_api --cov-report=term-missing
+        env:
+          APP__USE_SUB_SECOND_TIMESTAMPS: True
+
   linting:
     runs-on: ubuntu-latest
     name: Code Linting

.gitignore

@@ -1,3 +1,5 @@
+.tmp
+.venv
 venv/
 .idea/
 *.pyc
@@ -16,6 +18,7 @@ id_rsa
 id_rsa.pub
 logging.ini*
 !logging.ini.example
+pytest*.ini
 util/realistic_data/data
 util/realistic_data/resources
 docs/site

README.md

@@ -142,9 +142,7 @@ export XrdSecSSSKT=/path/to/.keytab
 
 ## Test Data
 
-A script has been created to set up the API with some test data and test users. This script also comes with a configuration file that needs to be reviewed and renamed at `util/realistic_data/config.yml.example`.
-
-Once the appropriate configuration has been entered, run the script using Poetry:
+A script has been created to set up the API with some test data and test users. This script also comes with a configuration file that needs to be reviewed and renamed at `util/realistic_data/config.yml.example`. Note that the example comes pre-populated with example values for `epac` and `gemini` style test data, affecting the source and destination for the test data. The ingest will run automatically as part of `pytest` via the fixtures (see [Testing](#testing)), but it can also be run explicitly using Poetry:
 
 ```bash
 poetry run python util/realistic_data/ingest_echo_data.py
@@ -166,6 +164,18 @@ Alternatively, you can use the Postman requests in this repo. This gives a more
 
 ## Testing
 
+While initially developed for a single facility (EPAC), the API is now designed to support multiple facilities with specific anticipated behaviour. This behaviour is controlled by configuration settings. These settings should be applied both during ingest of test data and while the tests are run. For ease, there is a `"session"` scoped fixture which ingests the test data, but only if the database is empty. This means when developing locally, the first execution of the tests may hang for around 4 minutes while ingestion occurs, but subsequent executions will start testing immediately.
+
+By default, the "usual" location of the configuration at [operationsgateway_api/config.yml](operationsgateway_api/config.yml) will apply (as it does when running the API locally). However, pytest can be configured to override these settings using the `pytest-env` plugin.
+
+### pytest.ini
+
+To support test suites for EPAC and Gemini style data, [test/pytest_epac.ini.example](test/pytest_epac.ini.example) and [test/pytest_gemini.ini.example](test/pytest_gemini.ini.example) should be copied to `test/pytest_epac.ini` and `test/pytest_gemini.ini` respectively. While the majority of the settings can then be used as is, it will also be necessary to choose bucket name(s) (i.e. replace `yourname`) and create a bucket and Mongo database with the corresponding name(s) before the tests will run. As mentioned above, it is not necessary to manually ingest the data if the database starts empty.
+
+Note that because of how [Pydantic parses environment variables](https://pydantic.dev/docs/validation/latest/concepts/pydantic_settings/#parsing-environment-variable-values), settings in the `env` section of `pytest.ini` will always take precedence over the `config.yml`. If one of the env settings is commented out (`;`), or removed, then whatever setting is in `config.yml` will be used as the next highest priority source. Conversely, if a new field is added the env, this will then override the corresponding setting in `config.yml`. There are therefore multiple ways to make config changes for the tests, and the appropriate method to use will depend on the circumstances.
+
+### nox
+
 Like the [DataGateway API](https://github.com/ral-facilities/datagateway-api), this repository contains a [Nox](https://nox.thea.codes) file (`noxfile.py`), which exists at the root level of this repository.
 
 To install Nox, use the following command:
@@ -185,5 +195,7 @@ The following Nox sessions have been created:
 - `black` - This uses Black to format Python code in a pre-defined style.
 - `lint` - This uses [flake8](https://flake8.pycqa.org/en/latest/) with a number of additional plugins to lint the code to keep it Pythonic.
 - `safety` - This uses [safety](https://github.com/pyupio/safety) to check the dependencies (pulled directly from Poetry) for any known vulnerabilities.
-- `tests` - this uses [pytest](https://docs.pytest.org/en/stable/) to execute the automated tests in `test/`.
+- `tests` - This uses [pytest](https://docs.pytest.org/en/stable/) to execute the automated tests in `test/`. Note that this runs without any additional configuration, so will use whatever values are in the locally defined config plus the default `pytest.ini` overrides.
+- `tests_epac` - This uses [pytest](https://docs.pytest.org/en/stable/) to execute the automated tests in `test/`. Note that this expects `pytest_epac.ini` to be defined with appropriate settings (i.e. `APP__USE_SUB_SECOND_TIMESTAMPS=False`)
+- `tests_gemini` - This uses [pytest](https://docs.pytest.org/en/stable/) to execute the automated tests in `test/`. Note that this expects `pytest_gemini.ini` to be defined with appropriate settings (i.e. `APP__USE_SUB_SECOND_TIMESTAMPS=True`)
 

docs/developer/test_data.md

@@ -67,5 +67,9 @@ Depending on which options are enabled, the script will do the following things:
 - Ingest HDF files and a channel manifest file 
 - Stop the API
 
+### Facility specific data
+
+Now that there is facility specific functionality/data formats, the single bucket for simulated data has been replaced with two buckets which are facility specific and contain a smaller number of hdf files to reduce ingest time in the CI/CD workflows. The names for these buckets are defined in [test/pytest.ini.example](test/pytest.ini.example), and will be used by configuring the `pytest.ini` files as specified in the [README](/README.md).
+
 ## Reverting from Simulated Data back to Gemini Data
 As previously stated, there may be cases where you have simulated data loaded into your development environment and you need to reload the Gemini data. As the script used to load Gemini data contains options to wipe the database and Echo, there should be no additional steps required. Clone the test data repo, run the script and check the tests pass!

noxfile.py

@@ -27,6 +27,18 @@ def lint(session):
 def safety(session):
     session.env["POETRY_VIRTUALENVS_CREATE"] = "false"
     session.run("poetry", "install", "--without", "simulated-data", external=True)
+    session.run(
+        "poetry",
+        "run",
+        "python",
+        "-m",
+        "pip",
+        "install",
+        "--upgrade",
+        "pip>=26.0",
+        external=True,
+    )
+
     # Can't fix 70790 because epac data sim uses it
     session.run(
         "poetry",
@@ -46,3 +58,33 @@ def tests(session):
     session.env["POETRY_VIRTUALENVS_CREATE"] = "false"
     session.run("poetry", "install", "--without", "simulated-data", external=True)
     session.run("poetry", "run", "pytest", *args, external=True)
+
+
+@nox.session(python=False)
+def tests_epac(session):
+    args = session.posargs
+    session.env["POETRY_VIRTUALENVS_CREATE"] = "false"
+    session.run("poetry", "install", "--without", "simulated-data", external=True)
+    session.run(
+        "poetry",
+        "run",
+        "pytest",
+        "--config-file=test/pytest_epac.ini",
+        *args,
+        external=True,
+    )
+
+
+@nox.session(python=False)
+def tests_gemini(session):
+    args = session.posargs
+    session.env["POETRY_VIRTUALENVS_CREATE"] = "false"
+    session.run("poetry", "install", "--without", "simulated-data", external=True)
+    session.run(
+        "poetry",
+        "run",
+        "pytest",
+        "--config-file=test/pytest_gemini.ini",
+        *args,
+        external=True,
+    )

operationsgateway_api/config.yml.example

@@ -8,6 +8,7 @@ app:
   url_prefix: ""
   maintenance_file: operationsgateway_api/maintenance.json.example
   scheduled_maintenance_file: operationsgateway_api/scheduled_maintenance.json.example
+  use_sub_second_timestamps: false
 images:
   # Thumbnail sizes should only ever be two element lists, of a x, y resolution
   thumbnail_size: [50, 50]
@@ -49,6 +50,7 @@ auth:
   refresh_token_validity_days: 7
   fedid_server_url: ldap://fed.cclrc.ac.uk:389
   fedid_server_ldap_realm: FED.CCLRC.AC.UK
+  allow_user_office_login: false
   oidc_providers:
     keycloak:
       display_name: A name # This gets displayed on the FE as an auth option