diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 491769a..85f57ab 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13"] name: Run tests for ${{ matrix.python-version }} steps: - uses: actions/checkout@v3 diff --git a/CITATION.cff b/CITATION.cff index bbc5789..2eb9eb0 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -87,6 +87,11 @@ authors: family-names: Schilperoort given-names: Bart orcid: https://orcid.org/0000-0003-4487-9822 + - + affiliation: "TU Delft" + family-names: Melotto + given-names: Mark + orcid: https://orcid.org/0009-0005-2727-660X cff-version: 1.2.0 date-released: 2022-12-13 diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 7504d99..b448bd2 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +# 2.1.0 - 2026-08-07 + +Changes since v2.0.1: + +**Changed:** +- Removed `isort`, `black` & `flake8` in favor of `ruff`. + +**Added:** +- Support for ERA5 & ERA5-Land daily data. + +**Removed:** +- Support for Python 3.9 + # 2.0.1 - 2025-04-04 Changes since v2.0.0: diff --git a/docs/gen_reference_pages.py b/docs/gen_reference_pages.py index 727f7dd..ddd9e75 100644 --- a/docs/gen_reference_pages.py +++ b/docs/gen_reference_pages.py @@ -1,8 +1,10 @@ +import subprocess +from typing import Generator, List + import mkdocs_gen_files -from era5cli import inputref + import era5cli.cli -import subprocess -from typing import List, Generator +from era5cli import inputref def divide_chunks(biglist: List, n: int) -> Generator[List, None, None]: @@ -71,9 +73,7 @@ def add_padding(multiline_string: List[str]): subparsers = ["Hourly", "Monthly"] for subp in subparsers: - with subprocess.Popen( - ["era5cli", subp.lower(), "--help"], stdout=subprocess.PIPE - ) as process: + with subprocess.Popen(["era5cli", subp.lower(), "--help"], stdout=subprocess.PIPE) as process: stdout, stderr = process.communicate() helpstr = stdout.decode("utf-8") helpstr = helpstr[helpstr.index("option") :] diff --git a/docs/general_development.md b/docs/general_development.md index 73ea613..92fe714 100644 --- a/docs/general_development.md +++ b/docs/general_development.md @@ -66,19 +66,17 @@ hatch run coverage This runs tests and prints the results to the command line, as well as storing the result in a `coverage.xml` file (for analysis by, e.g. CodeCov or SonarCloud). ## Running formatters and linters -For linting and code style we use `flake8`, `black` and `isort`. All tools can simply be run by doing: +For linting and code style we use `ruff`. Which can simply be run by doing: ```sh -hatch run lint +ruff check ``` -To easily comply with `black` and `isort`, you can also run: - ```sh hatch run format ``` -This will apply the `black` and `isort` formatting, and then check the code style. +This will apply the `ruff` formatting, and then check the code style. ## Generating the documentation diff --git a/docs/hourly_monthly.md b/docs/hourly_monthly.md index 10c083c..6137f42 100644 --- a/docs/hourly_monthly.md +++ b/docs/hourly_monthly.md @@ -1,6 +1,6 @@ -There are two types of data requests, hourly and monthly. Hourly requests generally have 24 hours of data available for each day, except in the case of forecase ensembles which are available every three hours (i.e. 8 per day). +There are two types of data requests, hourly, daily and monthly. Hourly requests generally have 24 hours of data available for each day, except in the case of forecase ensembles which are available every three hours (i.e. 8 per day). -Hourly and monthly requests mostly have the same variables available, except some of the variables that are only in the hourly datasets. Exceptions on the single level data can be found in table 8 of [ERA5 parameter listings](https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Table8). +Hourly, daily and monthly requests mostly have the same variables available, except some of the variables that are only in the hourly datasets. Exceptions on the single level data can be found in table 8 of [ERA5 parameter listings](https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation#ERA5:datadocumentation-Table8). ## Hourly @@ -15,6 +15,18 @@ More information on the available data and options can be found on the following - [ERA5 hourly pressure levels preliminary back extension download page](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-pressure-levels-preliminary-back-extension). - [ERA5-Land hourly download page](https://cds.climate.copernicus.eu/cdsapp#!/dataset/reanalysis-era5-land). +## Daily + +With the `daily` argument you can fetch daily ERA5 data. +All available arguments in `era5cli` can be seen using `era5cli daily --help`, or by going to the [reference](reference/arguments.md). + +More information on the available data and options can be found on the following pages: + +- [ERA5 single levels daily statistics download page](https://cds.climate.copernicus.eu/datasets/derived-era5-single-levels-daily-statistics?tab=overview) +- [ERA5 daily pressure levels download page](https://cds.climate.copernicus.eu/datasets/derived-era5-pressure-levels-daily-statistics?tab=overview) +- [ERA5-Land daily download page](https://cds.climate.copernicus.eu/datasets/derived-era5-land-daily-statistics?tab=overview) + + ## Monthly With the `monthly` argument you can fetch monthly-means of ERA5 data. diff --git a/docs/reference/arguments.md b/docs/reference/arguments.md index fe6b705..04cf489 100644 --- a/docs/reference/arguments.md +++ b/docs/reference/arguments.md @@ -1,5 +1,5 @@ -All available arguments for the hourly and monthly requests are shown below. This can also be viewed by doing `era5cli hourly --help` and `era5cli monthly --help`. +All available arguments for the hourly, daily and monthly requests are shown below. This can also be viewed by doing `era5cli hourly --help`, `era5cli daily --help` and `era5cli monthly --help`. Note that not all combinations of arguments are compatible, such as `--land` and `--ensemble`. diff --git a/era5cli/__version__.py b/era5cli/__version__.py index b059be2..7eac444 100644 --- a/era5cli/__version__.py +++ b/era5cli/__version__.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- """Author information.""" - # This information is located in its own file so that it can be loaded # without importing the main package when its dependencies are not installed. # See: https://packaging.python.org/guides/single-sourcing-package-version @@ -24,6 +23,7 @@ "Stefan Verhoeven", "Elizaveta Malinina", "Bart Schilperoort", + "Mark Melotto", ) __email__ = "ewatercycle@esciencecenter.nl" -__version__ = "2.0.1" +__version__ = "2.1.0" diff --git a/era5cli/_request_size.py b/era5cli/_request_size.py index 565d179..a0a63c9 100644 --- a/era5cli/_request_size.py +++ b/era5cli/_request_size.py @@ -1,8 +1,8 @@ """Module to compute the size of the CDS request.""" from typing import TYPE_CHECKING -from era5cli import inputref +from era5cli import inputref if TYPE_CHECKING: from era5cli.fetch import Fetch diff --git a/era5cli/args/__init__.py b/era5cli/args/__init__.py index 411041b..dd10746 100644 --- a/era5cli/args/__init__.py +++ b/era5cli/args/__init__.py @@ -1,7 +1,3 @@ -from era5cli.args import common -from era5cli.args import config -from era5cli.args import info -from era5cli.args import periods - +from era5cli.args import common, config, info, periods __all__ = ["common", "config", "periods", "info"] diff --git a/era5cli/args/common.py b/era5cli/args/common.py index e27bf41..63e201b 100644 --- a/era5cli/args/common.py +++ b/era5cli/args/common.py @@ -2,6 +2,7 @@ from argparse import ArgumentParser from datetime import datetime from typing import Union + import era5cli.inputref as ref @@ -36,30 +37,26 @@ def add_common_args(argument_parser: ArgumentParser) -> None: type=str, required=True, nargs="+", - help=textwrap.dedent( - """ + help=textwrap.dedent(""" The variables to download data for. This can be a single variable, or multiple. See the Copernicus Climate Data Store website or run `era5cli info -h` for available variables. - """ - ), + """), ) argument_parser.add_argument( "--startyear", type=int, required=True, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Single year or first year of range for which data should be downloaded. Every year will be downloaded in a separate file by default. Set `--split false` to change this - """ - ), + """), ) argument_parser.add_argument( @@ -67,8 +64,7 @@ def add_common_args(argument_parser: ArgumentParser) -> None: type=int, required=False, default=None, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Last year of range for which data should be downloaded. If only a single year is needed, only @@ -76,8 +72,7 @@ def add_common_args(argument_parser: ArgumentParser) -> None: Every year will be downloaded in a separate file by default. Set `--split false` to change this - """ - ), + """), ) argument_parser.add_argument( @@ -86,8 +81,7 @@ def add_common_args(argument_parser: ArgumentParser) -> None: type=_level_parse, required=False, default=ref.PLEVELS, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Pressure level(s) to download 3D variables for. Default is all available levels. See the Copernicus Climate Data Store website or run `era5cli info -h` @@ -96,21 +90,18 @@ def add_common_args(argument_parser: ArgumentParser) -> None: the single level dataset (previously called orography) - """ - ), + """), ) argument_parser.add_argument( "--outputprefix", type=str, default="era5", - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Prefix to be used for the output filename. Default prefix is `era5` - """ - ), + """), ) argument_parser.add_argument( @@ -118,26 +109,22 @@ def add_common_args(argument_parser: ArgumentParser) -> None: type=str, default="netcdf", choices=["netcdf", "grib"], - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Output file type. Defaults to `netcdf` - """ - ), + """), ) argument_parser.add_argument( "--merge", action="store_true", default=False, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Merge yearly output files. Default is split output files into separate files for every year - """ - ), + """), ) argument_parser.add_argument( @@ -146,61 +133,53 @@ def add_common_args(argument_parser: ArgumentParser) -> None: choices=range(1, 7), required=False, default=1, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Number of parallel threads to use when downloading. Defaults to a single process - """ - ), + """), ) argument_parser.add_argument( "--ensemble", action="store_true", default=False, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Whether to download high resolution realisation (HRES) or a reduced resolution ten member ensemble (EDA). Providing the `--ensemble` argument downloads the reduced resolution ensemble. `--ensemble` is incompatible with `--land` - """ - ), + """), ) argument_parser.add_argument( "--dryrun", action="store_true", default=False, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Whether to print the cdsapi request to the screen, or make the request to start downloading the data. Providing the `--dryrun` argument will print the request to stdout. By default, the data will be downloaded - """ - ), + """), ) argument_parser.add_argument( "--land", action="store_true", default=False, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Whether to download data from the ERA5-Land dataset. Note that the ERA5-Land dataset starts in 1950. `--land` is incompatible with the use of `--ensemble` - """ - ), + """), ) argument_parser.add_argument( @@ -209,8 +188,7 @@ def add_common_args(argument_parser: ArgumentParser) -> None: type=float, metavar=("LAT_MAX", "LON_MIN", "LAT_MIN", "LON_MAX"), required=False, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Coordinates in case extraction of a subregion is requested. Specified as `LAT_MAX LON_MIN LAT_MIN LON_MAX` @@ -221,32 +199,28 @@ def add_common_args(argument_parser: ArgumentParser) -> None: to two decimals. By default, the entire available area will be returned - """ - ), + """), ) argument_parser.add_argument( "--overwrite", action="store_true", default=False, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Whether to overwrite existing files or not. Providing the `--overwrite` argument will make era5cli overwrite existing files. By default, you will be prompted if a file already exists, with the question if you want to overwrite it or not. - """ - ), + """), ) argument_parser.add_argument( "--dashed-varname", action="store_true", default=False, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Whether to use dashed variable names in the output files, or the (default )normal names. Dashed names can allow for easier extraction @@ -255,8 +229,7 @@ def add_common_args(argument_parser: ArgumentParser) -> None: 'era5_temperature-of-snow-layer_1999_hourly.nc' instead: 'era5_temperature_of_snow_layer_1999_hourly.nc' - """ - ), + """), ) @@ -270,13 +243,9 @@ def construct_year_list(args): # check whether correct years have been entered for year in (args.startyear, endyear): if args.land: - assert ( - 1950 <= year <= datetime.now().year - ), "for ERA5-Land, year should be between 1950 and present" + assert 1950 <= year <= datetime.now().year, "for ERA5-Land, year should be between 1950 and present" else: - assert ( - 1940 <= year <= datetime.now().year - ), "year should be between 1940 and present" + assert 1940 <= year <= datetime.now().year, "year should be between 1940 and present" assert endyear >= args.startyear, "endyear should be >= startyear or None" diff --git a/era5cli/args/config.py b/era5cli/args/config.py index b986bab..cfc5965 100644 --- a/era5cli/args/config.py +++ b/era5cli/args/config.py @@ -1,5 +1,6 @@ import argparse import textwrap + from era5cli import key_management @@ -19,8 +20,7 @@ def add_config_args(subparsers: argparse._SubParsersAction) -> None: config = subparsers.add_parser( "config", description="", - prog=textwrap.dedent( - """ + prog=textwrap.dedent(""" Configure the CDS login info for era5cli. This will create a config file in your home directory, in folder named @@ -31,14 +31,11 @@ def add_config_args(subparsers: argparse._SubParsersAction) -> None: right). Use `era5cli config --help` for more information. - """ - ), - help=textwrap.dedent( - """ + """), + help=textwrap.dedent(""" Configure the CDS login info for era5cli. - """ - ), + """), formatter_class=argparse.RawTextHelpFormatter, ) @@ -46,21 +43,17 @@ def add_config_args(subparsers: argparse._SubParsersAction) -> None: "--show", action="store_true", default=False, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Print the stored keys to the screen. - """ - ), + """), ) config.add_argument( "--key", type=str, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Your CDS key, e.g.: "4s215sgs-2dfa-6h34-62h2-1615ad163414" - """ - ), + """), ) config.add_argument( @@ -68,12 +61,10 @@ def add_config_args(subparsers: argparse._SubParsersAction) -> None: type=str, required=False, default=key_management.DEFAULT_CDS_URL, - help=textwrap.dedent( - f""" + help=textwrap.dedent(f""" (optional) URL to the CDS, by default: {key_management.DEFAULT_CDS_URL} - """ - ), + """), ) config.add_argument( @@ -81,11 +72,9 @@ def add_config_args(subparsers: argparse._SubParsersAction) -> None: type=str, required=False, default="", - help=textwrap.dedent( - """ + help=textwrap.dedent(""" DO NOT USE: deprecated due to changes in the CDS API" - """ - ), + """), ) @@ -104,10 +93,7 @@ def run_config(args): args: Arguments collected by argparse """ if len(args.uid) > 0: - msg = ( - "The `uid` argument is deprecated.\n" - "The new CDS API does not use UIDs anymore." - ) + msg = "The `uid` argument is deprecated.\nThe new CDS API does not use UIDs anymore." raise InputError(msg) if args.show and args.key is not None: @@ -116,8 +102,6 @@ def run_config(args): raise InputError("Your CDS API key is a required input.") if args.show: url, key = key_management.load_era5cli_config() - print( - "Contents of .config/era5cli.txt:\n" f" key: {key}\n" f" url: {url}\n" - ) + print(f"Contents of .config/era5cli.txt:\n key: {key}\n url: {url}\n") else: key_management.set_config(args.url, args.key) diff --git a/era5cli/args/info.py b/era5cli/args/info.py index 78266bb..5063a76 100644 --- a/era5cli/args/info.py +++ b/era5cli/args/info.py @@ -1,5 +1,6 @@ import argparse import textwrap + import era5cli.info @@ -11,27 +12,22 @@ def add_info_args(subparsers): info = subparsers.add_parser( "info", description="Show information on available variables and levels.", - prog=textwrap.dedent( - """ + prog=textwrap.dedent(""" Use `era5cli info --help` for more information - """ - ), - help=textwrap.dedent( - """ + """), + help=textwrap.dedent(""" Show information on available variables or levels. Use `era5cli info --help` for more information - """ - ), + """), formatter_class=argparse.RawTextHelpFormatter, ) info.add_argument( "name", type=str, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Enter list name to print info list: \n `levels` for all available pressure levels \n `2Dvars` for all available single level or 2D @@ -43,8 +39,7 @@ def add_info_args(subparsers): or pressure level (e.g. `825`) to show if the variable or level is available, and in which list - """ - ), + """), ) diff --git a/era5cli/args/periods.py b/era5cli/args/periods.py index 092af1b..1420d15 100644 --- a/era5cli/args/periods.py +++ b/era5cli/args/periods.py @@ -1,5 +1,6 @@ import argparse import textwrap + from era5cli import utils @@ -7,7 +8,7 @@ def add_period_args(subparsers, common): """Add period related parsers and arguments. Adds the following parsers: - monthly, hourly. + monthly, daily, hourly. As well as the following arguments (for some of the previously mentioned parsers): @@ -21,14 +22,12 @@ def add_period_args(subparsers, common): required=False, type=int, default=list(range(1, 13)), - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Month(s) to download data for. Defaults to all months. For every year, only these months will be downloaded - """ - ), + """), ) day = argparse.ArgumentParser(add_help=False) @@ -39,14 +38,12 @@ def add_period_args(subparsers, common): required=False, type=int, default=list(range(1, 32)), - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Day(s) to download data for. Defaults to all days. For every year, only these days will be downloaded - """ - ), + """), ) hour = argparse.ArgumentParser(add_help=False) @@ -57,14 +54,12 @@ def add_period_args(subparsers, common): required=False, type=int, default=list(range(24)), - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Time of day in hours to download data for. Defaults to all hours. For every year, only these hours will be downloaded - """ - ), + """), ) splitmonths = argparse.ArgumentParser(add_help=False) @@ -73,66 +68,83 @@ def add_period_args(subparsers, common): "--splitmonths", type=lambda x: bool(utils.strtobool(x)), # type=bool doesn't work. default=True, - help=textwrap.dedent( - """ + help=textwrap.dedent(""" By default when downloading hourly data requests are split by months. To suppress this behavior, use: `--splitmonths False` to have yearly files. - """ - ), + """), ) hourly = subparsers.add_parser( "hourly", parents=[common, mnth, day, hour, splitmonths], description="Execute the data fetch process for hourly data.", - prog=textwrap.dedent( - """ + prog=textwrap.dedent(""" Use `era5cli hourly --help` for more information - """ - ), - help=textwrap.dedent( - """ + """), + help=textwrap.dedent(""" Execute the data fetch process for hourly data. Use `era5cli hourly --help` for more information - """ - ), + """), formatter_class=argparse.RawTextHelpFormatter, ) hourly.add_argument( "--statistics", action="store_true", - help=textwrap.dedent( - """ + help=textwrap.dedent(""" When downloading hourly ensemble data, provide the `--statistics` argument to download statistics (ensemble mean and ensemble spread) - """ - ), + """), + ) + + daily = subparsers.add_parser( + "daily", + parents=[common, mnth, day, splitmonths], + description="Execute the data fetch process for daily data.", + prog=textwrap.dedent(""" + Use `era5cli daily --help` for more information + + """), + help=textwrap.dedent(""" + Execute the data fetch process for daily data. + Use `era5cli daily --help` for more information + + """), + formatter_class=argparse.RawTextHelpFormatter, + ) + + daily.add_argument( + "--statistics", + type=str, + default="daily_mean", + choices=["daily_mean", "daily_minimum", "daily_maximum", "daily_sum"], + help=textwrap.dedent(""" + When downloading daily data, provide + the `--statistics` argument to download statistics + (daily_mean and daily_minimum, daily_maximum, daily_sum) + + """), ) monthly = subparsers.add_parser( "monthly", parents=[common, mnth], description="Execute the data fetch process for monthly data.", - prog=textwrap.dedent( - """ + prog=textwrap.dedent(""" Use `era5cli monthly --help` for more information - """ - ), - help=textwrap.dedent( - """ + """), + help=textwrap.dedent(""" Execute the data fetch process for monthly data. Use `era5cli monthly --help` for more information - """ - ), + """), formatter_class=argparse.RawTextHelpFormatter, ) @@ -141,8 +153,7 @@ def add_period_args(subparsers, common): type=int, default=False, nargs="*", - help=textwrap.dedent( - """ + help=textwrap.dedent(""" Time of day in hours to get the synoptic means (monthly averaged by hour of day) for. For example `--synoptic 0 4 5 6 23`. Give empty option @@ -150,8 +161,7 @@ def add_period_args(subparsers, common): The option defaults to `None` in which case the monthly average of daily means is chosen - """ - ), + """), ) @@ -170,14 +180,20 @@ def set_period_args(args): else: synoptic = True hours = args.synoptic + elif args.command == "daily": + synoptic = None + splitmonths: bool = args.splitmonths + statistics = args.statistics + days = args.days + hours = None + elif args.command == "hourly": synoptic = None splitmonths: bool = args.splitmonths statistics: bool = args.statistics if statistics: assert args.ensemble, ( - "Statistics can only be computed over an ensemble, " - "add --ensemble or remove --statistics." + "Statistics can only be computed over an ensemble, add --ensemble or remove --statistics." ) days = args.days hours = args.hours diff --git a/era5cli/cli.py b/era5cli/cli.py index f4fb09d..076a2e8 100644 --- a/era5cli/cli.py +++ b/era5cli/cli.py @@ -3,6 +3,7 @@ import argparse import sys + import era5cli.fetch as efetch from era5cli import args @@ -45,9 +46,7 @@ def _execute(input_args: argparse.Namespace) -> True: # the fetching subroutines years = args.common.construct_year_list(input_args) - synoptic, statistics, splitmonths, days, hours = args.periods.set_period_args( - input_args - ) + synoptic, statistics, splitmonths, days, hours = args.periods.set_period_args(input_args) # try to build and send download request era5 = efetch.Fetch( diff --git a/era5cli/fetch.py b/era5cli/fetch.py index 2f35e59..d316a7c 100644 --- a/era5cli/fetch.py +++ b/era5cli/fetch.py @@ -4,13 +4,14 @@ import logging import os import sys + import cdsapi from pathos.threading import ThreadPool as Pool + import era5cli.inputref as ref import era5cli.utils from era5cli import key_management -from era5cli._request_size import TooLargeRequestError -from era5cli._request_size import request_too_large +from era5cli._request_size import TooLargeRequestError, request_too_large class Fetch: @@ -131,9 +132,11 @@ def __init__( """list(str): List of zero-padded strings of days (e.g. ['01', '02',..., '31']).""" - self.hours = era5cli.utils._format_hours(hours) - """list(str): List of xx:00 formatted time strings - (e.g. ['00:00', '01:00', ..., '23:00']).""" + self.hours = None if period == "daily" else era5cli.utils._format_hours(hours) + """ + None for daily data, list(str): List of xx:00 formatted time strings otherwise + (e.g. ['00:00', '01:00', ..., '23:00']). + """ self.pressure_levels = pressurelevels """list(any): List of pressure levels (integer), or the indication 'surface', requesting data only from a single-level dataset.""" @@ -236,7 +239,7 @@ def _extension(self): raise ValueError(f"Unknown outputformat: {self.outputformat}") def _process_areaname(self): - (lat_max, lon_min, lat_min, lon_max) = [round(c) for c in self.area] + lat_max, lon_min, lat_min, lon_max = [round(c) for c in self.area] def lon(x): return f"{x}E" if x >= 0 else f"{abs(x)}W" @@ -276,9 +279,7 @@ def _define_outputfilename(self, var, years, month=None): def _split_variable(self): """Split by variable.""" - outputfiles = [ - self._define_outputfilename(var, self.years) for var in self.variables - ] + outputfiles = [self._define_outputfilename(var, self.years) for var in self.variables] if not self.overwrite: era5cli.utils.assert_outputfiles_not_exist(outputfiles) @@ -310,9 +311,7 @@ def _split_variable_yr_month(self): years = [] months = [] - for var, year, month in itertools.product( - self.variables, self.years, self.months - ): + for var, year, month in itertools.product(self.variables, self.years, self.months): outputfiles += [self._define_outputfilename(var, [year, year], month)] variables += [var] years += [year] @@ -326,9 +325,7 @@ def _split_variable_yr_month(self): def _product_type(self): """Construct the product type name from the options.""" - assert not ( - self.land and self.ensemble - ), "ERA5-Land does not contain Ensemble statistics." + assert not (self.land and self.ensemble), "ERA5-Land does not contain Ensemble statistics." if self.period == "hourly" and self.ensemble and self.statistics: # The only configuration to return a list @@ -355,44 +352,47 @@ def _product_type(self): if self.synoptic: producttype += "_by_hour_of_day" + if self.period == "daily": + return None + return producttype def _check_levels(self): """Retrieve pressure level info for request""" if not self.pressure_levels: - raise ValueError( - "Requested 3D variable(s), but no pressure levels specified." - "Aborting." - ) + raise ValueError("Requested 3D variable(s), but no pressure levels specified.Aborting.") if not all(level in ref.PLEVELS for level in self.pressure_levels): - raise ValueError( - f"Invalid pressure levels. Allowed values are: {ref.PLEVELS}" - ) + raise ValueError(f"Invalid pressure levels. Allowed values are: {ref.PLEVELS}") def _check_variable(self, variable): """Check variable available and compatible with other inputs.""" + if self.period == "daily": + if self.land: + if variable not in ref.ERA5_LAND_VARS: + raise ValueError( + f"Variable {variable} is not available in ERA5-Land" + f" daily statistics.\n" + f"Choose from {ref.ERA5_LAND_VARS}" + ) + elif variable not in ref.SLVARS: + raise ValueError(f"Variable {variable} is not available for daily statistics data.") + return # if land then the variable must be in era5 land if self.land: if variable not in ref.ERA5_LAND_VARS: raise ValueError( - f"Variable {variable} is not available in ERA5-Land.\n" - f"Choose from {ref.ERA5_LAND_VARS}" + f"Variable {variable} is not available in ERA5-Land.\nChoose from {ref.ERA5_LAND_VARS}" ) elif variable in ref.PLVARS + ref.SLVARS: if self.period == "monthly" and variable in ref.MISSING_MONTHLY_VARS: - header = ( - "There is no monthly data available for the " - "following variables:\n" - ) - raise ValueError( - era5cli.utils.print_multicolumn(header, ref.MISSING_MONTHLY_VARS) - ) + header = "There is no monthly data available for the following variables:\n" + raise ValueError(era5cli.utils.print_multicolumn(header, ref.MISSING_MONTHLY_VARS)) else: raise ValueError(f"Invalid variable name: {variable}") def _check_area(self): """Confirm that area parameters are correct.""" - (lat_max, lon_min, lat_min, lon_max) = self.area + lat_max, lon_min, lat_min, lon_max = self.area if not ( -90 <= lat_max <= 90 and -90 <= lat_min <= 90 @@ -418,6 +418,11 @@ def _parse_area(self): def _build_name(self, variable): """Build up name of dataset to use""" + if self.period == "daily": + if self.land: + return "derived-era5-land-daily-statistics", variable + return "derived-era5-single-levels-daily-statistics", variable + name = "reanalysis-era5" # report to user in case of ambiguous vars @@ -436,9 +441,7 @@ def _build_name(self, variable): instruction = instruction_surface else: instruction = instruction_pressure - logging.warning( - f"The variable name '{variable}' is ambiguous. {instruction}" - ) + logging.warning(f"The variable name '{variable}' is ambiguous. {instruction}") if self.land: name += "-land" @@ -466,11 +469,9 @@ def _build_request(self, variable, years, months=None): "variable": variable, "year": years, "month": self.months if months is None else months, - "time": self.hours, + **({} if self.period == "daily" else {"time": self.hours}), "data_format": self.outputformat, - "download_format": ( - "unarchived" if self.outputformat.lower() == "netcdf" else "zip" - ), + "download_format": ("unarchived" if self.outputformat.lower() == "netcdf" else "zip"), } if "pressure-levels" in name: @@ -483,9 +484,12 @@ def _build_request(self, variable, years, months=None): if product_type is not None: request["product_type"] = product_type - if self.period == "hourly": + if self.period in ("hourly", "daily"): request["day"] = self.days + if self.period == "daily": + request["daily_statistic"] = self.statistics + return (name, request) def _exit(self): diff --git a/era5cli/key_management.py b/era5cli/key_management.py index 5fb7f62..02c0312 100644 --- a/era5cli/key_management.py +++ b/era5cli/key_management.py @@ -2,10 +2,10 @@ import sys from pathlib import Path from typing import Tuple + import cdsapi from requests.exceptions import ConnectionError # pylint: disable=redefined-builtin - ERA5CLI_CONFIG_PATH = Path.home() / ".config" / "era5cli" / "cds_key.txt" CDSAPI_CONFIG_PATH = Path.home() / ".cdsapirc" DEFAULT_CDS_URL = "https://cds.climate.copernicus.eu/api" @@ -85,12 +85,10 @@ def set_config( try: attempt_cds_login(url, key) write_era5cli_config(url, key) - print( - f"Keys succesfully validated and stored in {ERA5CLI_CONFIG_PATH.resolve()}" - ) + print(f"Keys succesfully validated and stored in {ERA5CLI_CONFIG_PATH.resolve()}") return True except InvalidLoginError: - print("Error: the key is rejected by the CDS. " "Please check and try again.") + print("Error: the key is rejected by the CDS. Please check and try again.") return False @@ -107,10 +105,7 @@ def check_era5cli_config() -> None: else: print("era5cli configuration file not found. Looking for CDSAPI key.") if not valid_cdsapi_config(): - raise InvalidLoginError( - "No valid CDS login found. Please configure your CDS login using: " - "'era5cli config'" - ) + raise InvalidLoginError("No valid CDS login found. Please configure your CDS login using: 'era5cli config'") def valid_cdsapi_config() -> bool: @@ -124,8 +119,7 @@ def valid_cdsapi_config() -> bool: try: if sys.stdin.isatty() and attempt_cds_login(url, key): userinput = input( - "Valid CDS keys found in the .cdsapirc file. Do you want to use " - "these for era5cli? [Y/n]" + "Valid CDS keys found in the .cdsapirc file. Do you want to use these for era5cli? [Y/n]" ) if userinput.lower() in ["y", "yes", ""]: set_config(url, key) diff --git a/era5cli/utils.py b/era5cli/utils.py index d3bf77b..2e673ce 100644 --- a/era5cli/utils.py +++ b/era5cli/utils.py @@ -6,8 +6,10 @@ import textwrap from pathlib import Path from typing import List + import prettytable from netCDF4 import Dataset + import era5cli from era5cli.__version__ import __version__ as era5cliversion @@ -155,9 +157,7 @@ def append_history(name, request, fname): fname: str Filename. """ - dtime = datetime.datetime.now(tz=datetime.timezone.utc).strftime( - "%Y-%m-%d %H:%M:%S %Z" - ) + dtime = datetime.datetime.now(tz=datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z") appendtxt = f"{dtime} by {era5cli.__name__} {era5cliversion}: {name} {request}" extension = Path(fname).suffix if extension == ".nc": @@ -177,11 +177,9 @@ def _append_netcdf_history(ncfile: str, appendtxt: str): # open netCDF file rw and append to history ncfile = Dataset(ncfile, "r+") try: - ncfile.history = textwrap.dedent( - f"""\ + ncfile.history = textwrap.dedent(f"""\ {appendtxt} - {ncfile.history}""" - ) + {ncfile.history}""") except AttributeError: ncfile.history = appendtxt ncfile.close() @@ -203,10 +201,7 @@ def strtobool(value: str) -> bool: return True if value.lower() in falses: return False - raise ValueError( - "Could not convert string to boolean. Valid inputs are:" - f"{trues} and {falses} (case insensitive)." - ) + raise ValueError(f"Could not convert string to boolean. Valid inputs are:{trues} and {falses} (case insensitive).") def assert_outputfiles_not_exist(outputfiles: List[str]) -> None: diff --git a/pyproject.toml b/pyproject.toml index 52e27ec..de85ea3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ name = "era5cli" description = "A command line interface to download ERA5 data from the Copernicus Climate Data Store. https://climate.copernicus.eu/.." readme = "README.md" license = { text = "Apache Software License" } -requires-python = ">=3.9" +requires-python = ">=3.10" authors = [ {name = "Ronald van Haren"}, {name = "Jaro Camphuijsen"}, @@ -31,7 +31,8 @@ authors = [ {name = "Stef Smeets"}, {name = "Stefan Verhoeven"}, {name = "Elizaveta Malinina"}, - {name = "Bart Schilperoort", email = "b.schilperoort@esciencecenter.nl" } + {name = "Bart Schilperoort", email = "b.schilperoort@esciencecenter.nl" }, + {name = "Mark Melotto"} ] keywords = [ "ERA-5", @@ -51,7 +52,6 @@ classifiers = [ "Operating System :: POSIX", "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -72,12 +72,8 @@ era5cli="era5cli.cli:main" dev = [ "hatch", "bump-my-version", - "flake8", - "flake8-pyproject", - "black", - "isort", + "ruff", "pytest", - "pytest-flake8", "pytest-cov", ] docs = [ @@ -94,11 +90,13 @@ features = ["dev",] [tool.hatch.envs.default.scripts] lint = [ - "flake8p .", # flake8p ensures pyproject.toml is used for configuration - "black --check --diff .", - "isort --check-only --diff .", + "ruff check .", + "ruff format --check --diff .", +] +format = [ + "ruff check --fix .", + "ruff format .", ] -format = ["isort .", "black .", "lint",] test = ["pytest",] coverage = ["pytest --cov=era5cli --cov-report term --cov-report xml:cov.xml tests/"] @@ -109,29 +107,6 @@ features = ["docs",] build = ["mkdocs build",] serve = ["mkdocs serve",] -[tool.black] -line-length = 88 -target-version = ['py39', 'py310', 'py311', 'py312', 'py313'] -include = '\.pyi?$' - -[tool.isort] -py_version=39 -skip = [".gitignore"] -skip_glob = ["docs/*"] -force_single_line = true -lines_after_imports = 2 -no_lines_before = ["FUTURE","STDLIB","THIRDPARTY","FIRSTPARTY","LOCALFOLDER"] -known_first_party = ["era5cli"] -src_paths = ["era5cli", "tests"] -line_length = 120 - -[tool.flake8] -max-line-length = 88 -ignore = [ - "E203", # Whitespace before ":". Not PEP8 compliant (https://github.com/psf/black/issues/315) - "W503", # https://peps.python.org/pep-0008/#should-a-line-break-before-or-after-a-binary-operator -] - [tool.coverage.report] exclude_lines = [ "pragma: no cover", @@ -139,3 +114,22 @@ exclude_lines = [ "if TYPE_CHECKING:", "if typing.TYPE_CHECKING:" ] + +[tool.ruff] +line-length = 120 + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.isort] +known-first-party = ["era5cli"] + +[tool.ruff.lint] +select = [ + "E", + "F", + "I", +] +ignore = [ + "E501" +] \ No newline at end of file diff --git a/readthedocs.yml b/readthedocs.yml index e28c1a6..04dbf04 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -1,9 +1,9 @@ version: 2 build: - os: ubuntu-20.04 + os: ubuntu-22.04 tools: - python: "3.9" + python: "3.10" mkdocs: configuration: mkdocs.yml diff --git a/tests/test_cli.py b/tests/test_cli.py index 2e6d7e9..2a2d57c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,11 +1,12 @@ """Tests for era5cli utility functions.""" import unittest.mock as mock + import pytest + import era5cli.args import era5cli.inputref as ref -from era5cli import cli -from era5cli import key_management +from era5cli import cli, key_management def test_parse_args(): @@ -41,6 +42,23 @@ def test_parse_args(): assert not args.area +def test_parse_daily_args(): + argv = [ + "daily", + "--startyear", + "2008", + "--variables", + "total_precipitation", + "--statistics", + "daily_maximum", + ] + args = cli._parse_args(argv) + assert args.command == "daily" + assert args.statistics == "daily_maximum" + assert args.days == list(range(1, 32)) + assert args.months == list(range(1, 13)) + + def test_area_argument(): """Test if area argument is parsed correctly.""" # Test if area arguments are parsed correctly @@ -192,6 +210,20 @@ def test_period_args(): assert era5cli.args.periods.set_period_args(args) +def test_period_daily_args(): + argv = [ + "daily", + "--startyear", + "2008", + "--variables", + "total_precipitation", + ] + args = cli._parse_args(argv) + period_args = era5cli.args.periods.set_period_args(args) + # (synoptic, statistics, splitmonths, days, hours) + assert period_args == (None, "daily_mean", True, list(range(1, 32)), None) + + def test_level_arguments(): """Test if levels are parsed correctly""" argv = [ @@ -282,6 +314,16 @@ def test_main_fetch(fetch): args = cli._parse_args(argv) cli._execute(args) + argv = [ + "daily", + "--startyear", + "2008", + "--variables", + "total_precipitation", + ] + args = cli._parse_args(argv) + assert cli._execute(args) + @mock.patch("era5cli.info.Info", autospec=True) def test_main_info(info): @@ -341,11 +383,7 @@ def test_config_show(self, mock, capsys): args = cli._parse_args(["config", "--show"]) cli._execute(args) - expected = ( - "Contents of .config/era5cli.txt:\n" - " key: abc-def\n" - " url: https://www.test.org/\n" - ) + expected = "Contents of .config/era5cli.txt:\n key: abc-def\n url: https://www.test.org/\n" out, _ = capsys.readouterr() assert expected in out @@ -353,6 +391,8 @@ def test_config_show(self, mock, capsys): "input_args", [ ["config", "--show", "--key", "abc-def"], + ["config", "--uid", "x", "--key", "abc-def"], + ["config"], ], ) def test_config_inputerror(self, input_args): diff --git a/tests/test_config.py b/tests/test_config.py index 2838830..11f206c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,8 +1,9 @@ from unittest.mock import patch + import pytest import requests.exceptions as rex -from era5cli import key_management +from era5cli import key_management CFG_FILE = "url: https://www.github.com/\nkey: abc-def\n" @@ -34,6 +35,22 @@ def valid_path_cds(tmp_path_factory): return fn +@pytest.fixture(scope="function") +def uid_key_path_cds(tmp_path_factory): + fn = tmp_path_factory.mktemp(".config") / "cdsapirc.txt" + with open(fn, mode="w", encoding="utf-8") as f: + f.write("url: https://www.github.com/\nkey: uid:abc-def\n") + return fn + + +@pytest.fixture(scope="function") +def old_url_path_cds(tmp_path_factory): + fn = tmp_path_factory.mktemp(".config") / "cdsapirc.txt" + with open(fn, mode="w", encoding="utf-8") as f: + f.write("url: https://www.github.com/api/v2\nkey: abc-def\n") + return fn + + class TestEra5CliConfig: """Test the functionality for writing and loading the config file.""" @@ -79,9 +96,7 @@ def test_cdsrcfile_user_says_no(self, empty_path_era5, valid_path_cds): mp4 = patch("era5cli.key_management.CDSAPI_CONFIG_PATH", valid_path_cds) mp5 = patch("sys.stdin.isatty", return_value=True) with mp1, mp2, mp3, mp4, mp5: - with pytest.raises( - key_management.InvalidLoginError, match="No valid CDS login found" - ): + with pytest.raises(key_management.InvalidLoginError, match="No valid CDS login found"): key_management.check_era5cli_config() def test_cdsrcfile_user_says_yes(self, empty_path_era5, valid_path_cds): @@ -109,9 +124,7 @@ def test_cdsrcfile_invalid_keys(self, empty_path_era5, valid_path_cds): mp3 = patch("era5cli.key_management.CDSAPI_CONFIG_PATH", valid_path_cds) mp4 = patch("sys.stdin.isatty", return_value=True) with mp1, mp2, mp3, mp4: - with pytest.raises( - key_management.InvalidLoginError, match="No valid CDS login found" - ): + with pytest.raises(key_management.InvalidLoginError, match="No valid CDS login found"): key_management.check_era5cli_config() @@ -138,9 +151,7 @@ def test_connection_fail(self): key_management.InvalidLoginError, match="Authorization with the CDS served failed", ): - key_management.attempt_cds_login( - url="https://www.github.com/", key="abc:def" - ) + key_management.attempt_cds_login(url="https://www.github.com/", key="abc:def") def test_retrieve_fail(self): mp1 = patch("cdsapi.Client.status") @@ -160,3 +171,17 @@ def test_all_pass(self): mp2 = patch("cdsapi.Client.retrieve") with mp1, mp2: assert key_management.attempt_cds_login(url="test", key="abc:def") is True + + +class TestLoadCdsapiConfig: + """Test key_management.load_cdsapi_config directly.""" + + def test_uid_style_key_rejected(self, uid_key_path_cds): + with patch("era5cli.key_management.CDSAPI_CONFIG_PATH", uid_key_path_cds): + with pytest.raises(key_management.InvalidLoginError): + key_management.load_cdsapi_config() + + def test_old_api_url_rejected(self, old_url_path_cds): + with patch("era5cli.key_management.CDSAPI_CONFIG_PATH", old_url_path_cds): + with pytest.raises(key_management.InvalidLoginError): + key_management.load_cdsapi_config() diff --git a/tests/test_fetch.py b/tests/test_fetch.py index 718ee66..3d7ac50 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -2,10 +2,10 @@ import pathlib import unittest.mock as mock + import pytest -from era5cli import _request_size -from era5cli import fetch +from era5cli import _request_size, fetch # fmt: off ALL_HOURS = [ @@ -26,9 +26,7 @@ @pytest.fixture(scope="module", autouse=True) def my_thing_mock(): - with mock.patch( - "era5cli.fetch.key_management.check_era5cli_config", autospec=True - ) as _fixture: + with mock.patch("era5cli.fetch.key_management.check_era5cli_config", autospec=True) as _fixture: yield _fixture @@ -78,9 +76,7 @@ def initialize( ) -@mock.patch( - "era5cli.fetch.key_management.load_era5cli_config", return_value=("url", "key:uid") -) +@mock.patch("era5cli.fetch.key_management.load_era5cli_config", return_value=("url", "key:uid")) def test_init(mockpatch): """Test init function of Fetch class.""" era5 = fetch.Fetch( @@ -118,20 +114,18 @@ def test_init(mockpatch): # initializing hourly variable with days=None should result in ValueError with pytest.raises(TypeError): - era5 = initialize( - variables=["temperature"], period="hourly", days=None, pressurelevels=[1] - ) + era5 = initialize(variables=["temperature"], period="hourly", days=None, pressurelevels=[1]) # initializing monthly variable with days=None returns fetch.Fetch object - era5 = initialize( - variables=["temperature"], period="monthly", days=None, pressurelevels=[1] - ) + era5 = initialize(variables=["temperature"], period="monthly", days=None, pressurelevels=[1]) assert isinstance(era5, fetch.Fetch) with pytest.raises(_request_size.TooLargeRequestError): - initialize( - land=True, variables=["skin_temperature"], ensemble=False, splitmonths=False - ) + initialize(land=True, variables=["skin_temperature"], ensemble=False, splitmonths=False) + + era5 = initialize(period="daily", statistics="daily_mean", ensemble=False) + assert era5.hours is None + assert era5.days == ALL_DAYS @mock.patch("cdsapi.Client", autospec=True) @@ -147,9 +141,7 @@ def test_fetch_nodryrun(cds, era5cli_utilsappend_history): era5 = initialize(outputformat="grib", merge=True, threads=None) assert era5.fetch() is None - era5 = initialize( - outputformat="grib", merge=True, threads=None, ensemble=True, statistics=True - ) + era5 = initialize(outputformat="grib", merge=True, threads=None, ensemble=True, statistics=True) assert era5.fetch() is None era5 = initialize( @@ -182,9 +174,7 @@ def test_fetch_nodryrun(cds, era5cli_utilsappend_history): ) # invalid variable name should raise ValueError - era5 = initialize( - outputformat="grib", merge=True, threads=None, variables=["unknown"] - ) + era5 = initialize(outputformat="grib", merge=True, threads=None, variables=["unknown"]) with pytest.raises(ValueError): assert era5.fetch() @@ -273,6 +263,11 @@ def test_define_outputfilename(): fn = "era5-land_total_precipitation_2008-01_hourly_120E-180E_90S-0N.nc" assert fname == fn + era5 = initialize(period="daily", statistics="daily_mean", ensemble=False, splitmonths=True) + era5._extension() + fname = era5._define_outputfilename("total_precipitation", [2008], month="01") + assert fname == "era5_total_precipitation_2008-01_daily_statistics.nc" + _vars = ["total_precipitation", "runoff"] _years = [2007, 2008, 2009] @@ -289,9 +284,7 @@ def test_define_outputfilename(): (_vars[:1], _years, False, False, False, 1 * 3), ], ) -def test_number_outputfiles( - capsys, variables, years, merge, ensemble, splitmonths, expected -): +def test_number_outputfiles(capsys, variables, years, merge, ensemble, splitmonths, expected): """Test function for the number of outputs.""" # two variables and three years era5 = initialize( @@ -313,6 +306,12 @@ def test_product_type(): """Test _product_type function of Fetch class.""" # Default hourly data era5 = initialize() + + era5.period = "daily" + assert era5._product_type() is None + + era5 = initialize() + producttype = era5._product_type() assert producttype == "ensemble_members" @@ -413,6 +412,29 @@ def test_check_variable(): with pytest.raises(ValueError): era5._check_variable(missing_monthly_var) + era5.period = "daily" + era5.land = True + era5._check_variable("snow_cover") + + # Non-land variable should fail for daily land + with pytest.raises(ValueError): + era5._check_variable("vertical_integral_of_mass_tendency") + + +def test_check_variable_daily_slvars(): + """Non-SLVARS variable should fail for daily, non-land requests.""" + era5 = initialize() + era5.period = "daily" + era5.land = False + with pytest.raises(ValueError): + era5._check_variable("divergence") # PLVARS-only, not in SLVARS + + +def test_exit_is_noop(): + """_exit is an unused no-op stub; confirm it does nothing and doesn't raise.""" + era5 = initialize() + assert era5._exit() is None + def test_build_name(): """Test _build_name function of Fetch class.""" @@ -453,6 +475,14 @@ def test_build_name(): name = era5._build_name("geopotential")[0] assert name == "reanalysis-era5-single-levels" + era5 = initialize(period="daily", statistics="daily_mean", ensemble=False) + name = era5._build_name("total_precipitation")[0] + assert name == "derived-era5-single-levels-daily-statistics" + + era5 = initialize(period="daily", statistics="daily_mean", land=True, ensemble=False) + name = era5._build_name("snow_cover")[0] + assert name == "derived-era5-land-daily-statistics" + def test_build_request(): """Test _build_request function of Fetch class.""" @@ -463,7 +493,7 @@ def test_build_request(): years=[2008], splitmonths=False, ) - (name, request) = era5._build_request("total_precipitation", [2008]) + name, request = era5._build_request("total_precipitation", [2008]) assert name == "reanalysis-era5-single-levels" req = { "variable": "total_precipitation", @@ -479,7 +509,7 @@ def test_build_request(): # monthly data era5 = initialize(period="monthly", variables=["total_precipitation"], years=[2008]) - (name, request) = era5._build_request("total_precipitation", [2008]) + name, request = era5._build_request("total_precipitation", [2008]) assert name == "reanalysis-era5-single-levels-monthly-means" req = { "variable": "total_precipitation", @@ -493,11 +523,9 @@ def test_build_request(): assert request == req # land - era5 = initialize( - period="monthly", variables=["snow_cover"], hours=[0], land=True, ensemble=False - ) + era5 = initialize(period="monthly", variables=["snow_cover"], hours=[0], land=True, ensemble=False) - (name, request) = era5._build_request("snow_cover", [2008]) + name, request = era5._build_request("snow_cover", [2008]) assert name == ("reanalysis-era5-land-monthly-means") req = { "variable": "snow_cover", @@ -514,6 +542,47 @@ def test_build_request(): with pytest.raises(ValueError): era5 = initialize(variables=["temperature"], pressurelevels=None) + era5 = initialize( + period="daily", + variables=["total_precipitation"], + years=[2008], + statistics="daily_mean", + ensemble=False, + ) + name, request = era5._build_request("total_precipitation", [2008]) + assert name == "derived-era5-single-levels-daily-statistics" + req = { + "variable": "total_precipitation", + "year": [2008], + "month": ALL_MONTHS, + "day": ALL_DAYS, + "daily_statistic": "daily_mean", + "data_format": "netcdf", + "download_format": "unarchived", + } + assert request == req + + era5 = initialize( + period="daily", + variables=["snow_cover"], + years=[2008], + statistics="daily_mean", + land=True, + ensemble=False, + ) + name, request = era5._build_request("snow_cover", [2008]) + assert name == "derived-era5-land-daily-statistics" + req = { + "variable": "snow_cover", + "year": [2008], + "month": ALL_MONTHS, + "day": ALL_DAYS, + "daily_statistic": "daily_mean", + "data_format": "netcdf", + "download_format": "unarchived", + } + assert request == req + def test_incompatible_options(): """Test that invalid combinations of arguments don't silently pass.""" @@ -539,13 +608,13 @@ def test_area(): assert era5.area is None era5 = initialize(area=[90, -180, -90, 180]) - (name, request) = era5._build_request("total_precipitation", [2008]) + name, request = era5._build_request("total_precipitation", [2008]) assert era5.area == [90, -180, -90, 180] assert request["area"] == [90, -180, -90, 180] # Decimals are rounded down era5 = initialize(area=[89.9999, -179.90, -90.0000, 179.012]) - (name, request) = era5._build_request("total_precipitation", [2008]) + name, request = era5._build_request("total_precipitation", [2008]) assert request["area"] == [90.0, -179.90, -90.0, 179.01] # lat_max may not be lower than lat_min @@ -555,7 +624,7 @@ def test_area(): # lon_min higher than lon_max should be ok era5 = initialize(area=[90, 120, -90, -120]) - (name, request) = era5._build_request("total_precipitation", [2008]) + name, request = era5._build_request("total_precipitation", [2008]) assert request["area"] == [90.0, 120.0, -90.0, -120.0] # lat_max may not equal lat_min diff --git a/tests/test_info.py b/tests/test_info.py index e25f8f3..d148986 100644 --- a/tests/test_info.py +++ b/tests/test_info.py @@ -1,8 +1,8 @@ """Tests for era5cli Fetch class.""" import pytest -from era5cli import info +from era5cli import info INFO_PARAMS = [ "levels", diff --git a/tests/test_integration.py b/tests/test_integration.py index 5cbb07a..65de3dd 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -3,15 +3,15 @@ import logging from textwrap import dedent from unittest import mock + import pytest + from era5cli.cli import main @pytest.fixture(scope="module", autouse=True) def my_thing_mock(): - with mock.patch( - "era5cli.fetch.key_management.check_era5cli_config", autospec=True - ) as _fixture: + with mock.patch("era5cli.fetch.key_management.check_era5cli_config", autospec=True) as _fixture: yield _fixture @@ -19,13 +19,10 @@ def my_thing_mock(): call_result = [ { # geopotential needs '--levels surface' to be correctly interpreted - "call": dedent( - """\ + "call": dedent("""\ era5cli hourly --variables geopotential --startyear 2008 --dryrun - --splitmonths False --levels surface""" - ), - "result": dedent( - """\ + --splitmonths False --levels surface"""), + "result": dedent("""\ reanalysis-era5-single-levels {'variable': 'geopotential', 'year': 2008, 'month': ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'], 'time': ['00:00', '01:00', '02:00', @@ -37,20 +34,16 @@ def my_thing_mock(): '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31']} - era5_geopotential_2008_hourly.nc""" - ), + era5_geopotential_2008_hourly.nc"""), "warn": "Getting variable from surface level data.", }, { # without --levels surface, geopotential calls pressure level data # Note: only request a single month to avoid TooLargeRequest - "call": dedent( - """\ + "call": dedent("""\ era5cli hourly --variables geopotential --startyear 2008 --months 01 - --dryrun""" - ), - "result": dedent( - """\ + --dryrun"""), + "result": dedent("""\ reanalysis-era5-pressure-levels {'variable': 'geopotential', 'year': 2008, 'month': '01', 'time': ['00:00', '01:00', '02:00', '03:00', '04:00', '05:00', '06:00', '07:00', '08:00', '09:00', @@ -64,26 +57,21 @@ def my_thing_mock(): '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31']} - era5_geopotential_2008-01_hourly.nc""" - ), + era5_geopotential_2008-01_hourly.nc"""), "warn": "Getting variable from pressure level data.", }, { # era5-Land is combined with monthly means - "call": dedent( - """\ + "call": dedent("""\ era5cli monthly --variables snow_cover --startyear 2008 --land - --dryrun""" - ), - "result": dedent( - """\ + --dryrun"""), + "result": dedent("""\ reanalysis-era5-land-monthly-means {'variable': 'snow_cover', 'year': 2008, 'month': ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12'], 'time': ['00:00'], 'data_format': 'netcdf', 'download_format': 'unarchived', 'product_type': 'monthly_averaged_reanalysis'} - era5-land_snow_cover_2008_monthly.nc""" - ), + era5-land_snow_cover_2008_monthly.nc"""), }, ] diff --git a/tests/test_utils.py b/tests/test_utils.py index 2d15344..9a5983e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,6 +2,7 @@ import pytest from netCDF4 import Dataset + import era5cli import era5cli.utils from era5cli.__version__ import __version__ as era5cliversion diff --git a/tests/test_version.py b/tests/test_version.py index ba64918..7973fca 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -1,6 +1,7 @@ """Tests for era5cli __version__ available variales.""" import pytest + import era5cli.__version__ as era5cli