diff --git a/lerna/__init__.py b/lerna/__init__.py index 734e2ca..84027f9 100644 --- a/lerna/__init__.py +++ b/lerna/__init__.py @@ -29,22 +29,22 @@ SweeperManager = _rust.SweeperManager __all__ = [ - "__version__", - "MissingConfigException", - "main", - "utils", - "TaskFunction", - "compose", - "initialize", - "initialize_config_module", - "initialize_config_dir", "CallbackManager", - "JobReturn", "ConfigResult", - "RustFileConfigSource", "ConfigSourceManager", - "RustBasicLauncher", + "JobReturn", "LauncherManager", + "MissingConfigException", + "RustBasicLauncher", "RustBasicSweeper", + "RustFileConfigSource", "SweeperManager", + "TaskFunction", + "__version__", + "compose", + "initialize", + "initialize_config_dir", + "initialize_config_module", + "main", + "utils", ] diff --git a/lerna/_internal/callbacks.py b/lerna/_internal/callbacks.py index fedb6ca..aa5b25e 100644 --- a/lerna/_internal/callbacks.py +++ b/lerna/_internal/callbacks.py @@ -1,6 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import warnings -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any from omegaconf import DictConfig, OmegaConf @@ -21,16 +21,16 @@ class CallbacksCache(metaclass=Singleton): def instance() -> "CallbacksCache": return Singleton.instance(CallbacksCache) # type: ignore - cache: Dict[int, "Callbacks"] + cache: dict[int, "Callbacks"] def __init__(self) -> None: self.cache = {} class Callbacks: - callbacks: List[Any] + callbacks: list[Any] - def __init__(self, config: Optional[DictConfig] = None, check_cache: bool = True) -> None: + def __init__(self, config: DictConfig | None = None, check_cache: bool = True) -> None: if config is None: return cache = CallbacksCache.instance().cache @@ -53,7 +53,7 @@ def _notify(self, function_name: str, reverse: bool = False, **kwargs: Any) -> N for c in callbacks: try: getattr(c, function_name)(**kwargs) - except Exception as e: + except Exception as e: # noqa: BLE001 warnings.warn(f"Callback {type(c).__name__}.{function_name} raised {type(e).__name__}: {e}") def on_run_start(self, config: DictConfig, **kwargs: Any) -> None: @@ -88,8 +88,8 @@ def on_job_end(self, config: DictConfig, job_return: "JobReturn", **kwargs: Any) def on_compose_config( self, config: DictConfig, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], ) -> None: self._notify( function_name="on_compose_config", diff --git a/lerna/_internal/config_loader_impl.py b/lerna/_internal/config_loader_impl.py index b583783..0d345e7 100644 --- a/lerna/_internal/config_loader_impl.py +++ b/lerna/_internal/config_loader_impl.py @@ -4,8 +4,9 @@ import re import sys import warnings +from collections.abc import MutableSequence from textwrap import dedent -from typing import Any, List, MutableSequence, Optional, Tuple, Union +from typing import Any from omegaconf import Container, DictConfig, ListConfig, OmegaConf, flag_override, open_dict from omegaconf.errors import ( @@ -60,7 +61,7 @@ def __init__( @staticmethod def validate_sweep_overrides_legal( - overrides: List[Override], + overrides: list[Override], run_mode: RunMode, from_shell: bool, ) -> None: @@ -90,12 +91,12 @@ def validate_sweep_overrides_legal( else: assert False - def _missing_config_error(self, config_name: Optional[str], msg: str, with_search_path: bool) -> None: + def _missing_config_error(self, config_name: str | None, msg: str, with_search_path: bool) -> None: def add_search_path() -> str: descs = [] for src in self.repository.get_sources(): if src.provider != "schema": - descs.append(f"\t{repr(src)}") + descs.append(f"\t{src!r}") lines = "\n".join(descs) if with_search_path: @@ -108,22 +109,21 @@ def add_search_path() -> str: def ensure_main_config_source_available(self) -> None: for source in self.get_sources(): # if specified, make sure main config search path exists - if source.provider == "main": - if not source.available(): - if source.scheme() == "pkg": - if source.path == "": - msg = "Primary config module is empty.\nPython requires resources to be in a module with an __init__.py file" - else: - msg = f"Primary config module '{source.path}' not found.\nCheck that it's correct and contains an __init__.py file" + if source.provider == "main" and not source.available(): + if source.scheme() == "pkg": + if source.path == "": + msg = "Primary config module is empty.\nPython requires resources to be in a module with an __init__.py file" else: - msg = f"Primary config directory not found.\nCheck that the config directory '{source.path}' exists and readable" + msg = f"Primary config module '{source.path}' not found.\nCheck that it's correct and contains an __init__.py file" + else: + msg = f"Primary config directory not found.\nCheck that the config directory '{source.path}' exists and readable" - self._missing_config_error(config_name=None, msg=msg, with_search_path=False) + self._missing_config_error(config_name=None, msg=msg, with_search_path=False) def load_configuration( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode, from_shell: bool = True, validate_sweep_overrides: bool = True, @@ -141,8 +141,8 @@ def load_configuration( def _process_config_searchpath( self, - config_name: Optional[str], - parsed_overrides: List[Override], + config_name: str | None, + parsed_overrides: list[Override], repo: CachingConfigRepository, ) -> None: if config_name is not None: @@ -204,8 +204,8 @@ def _err() -> None: ) def _parse_overrides_and_create_caching_repo( - self, config_name: Optional[str], overrides: List[str] - ) -> Tuple[List[Override], CachingConfigRepository]: + self, config_name: str | None, overrides: list[str] + ) -> tuple[list[Override], CachingConfigRepository]: parser = OverridesParser.create() parsed_overrides = parser.parse_overrides(overrides=overrides) caching_repo = CachingConfigRepository(self.repository) @@ -214,8 +214,8 @@ def _parse_overrides_and_create_caching_repo( def _load_configuration_impl( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode, from_shell: bool = True, validate_sweep_overrides: bool = True, @@ -289,7 +289,7 @@ def _load_configuration_impl( return cfg - def load_sweep_config(self, master_config: DictConfig, sweep_overrides: List[str]) -> DictConfig: + def load_sweep_config(self, master_config: DictConfig, sweep_overrides: list[str]) -> DictConfig: # Recreate the config for this sweep instance with the appropriate overrides overrides = OmegaConf.to_container(master_config.hydra.overrides.hydra) assert isinstance(overrides, list) @@ -311,7 +311,7 @@ def get_search_path(self) -> ConfigSearchPath: return self.config_search_path @staticmethod - def _apply_overrides_to_config(overrides: List[Override], cfg: DictConfig) -> None: + def _apply_overrides_to_config(overrides: list[Override], cfg: DictConfig) -> None: for override in overrides: if override.package is not None: raise ConfigCompositionException( @@ -336,7 +336,7 @@ def _apply_overrides_to_config(overrides: List[Override], cfg: DictConfig) -> No del cfg[key] else: node = OmegaConf.select(cfg, key[0:last_dot]) - node_key: Union[str, int] = key[last_dot + 1 :] + node_key: str | int = key[last_dot + 1 :] if isinstance(node, ListConfig): node_key = int(node_key) del node[node_key] @@ -498,7 +498,7 @@ def _load_single_config(self, default: ResultDefault, repo: IConfigRepository) - return res @staticmethod - def _embed_result_config(ret: ConfigResult, package_override: Optional[str]) -> ConfigResult: + def _embed_result_config(ret: ConfigResult, package_override: str | None) -> ConfigResult: package = ret.header["package"] if package_override is not None: package = package_override @@ -511,16 +511,16 @@ def _embed_result_config(ret: ConfigResult, package_override: Optional[str]) -> return ret - def list_groups(self, parent_name: str) -> List[str]: + def list_groups(self, parent_name: str) -> list[str]: return self.get_group_options(group_name=parent_name, results_filter=ObjectType.GROUP) def get_group_options( self, group_name: str, - results_filter: Optional[ObjectType] = ObjectType.CONFIG, - config_name: Optional[str] = None, - overrides: Optional[List[str]] = None, - ) -> List[str]: + results_filter: ObjectType | None = ObjectType.CONFIG, + config_name: str | None = None, + overrides: list[str] | None = None, + ) -> list[str]: if overrides is None: overrides = [] _, caching_repo = self._parse_overrides_and_create_caching_repo(config_name, overrides) @@ -528,9 +528,9 @@ def get_group_options( def _try_rust_compose( self, - defaults: List[ResultDefault], + defaults: list[ResultDefault], repo: IConfigRepository, - ) -> Optional[DictConfig]: + ) -> DictConfig | None: """ Compose config using Rust for performance. @@ -680,7 +680,7 @@ def _strip_defaults(cfg: Any) -> None: def _compose_config_from_defaults_list( self, - defaults: List[ResultDefault], + defaults: list[ResultDefault], repo: IConfigRepository, ) -> DictConfig: # Try Rust-accelerated compose first @@ -705,13 +705,13 @@ def _compose_config_from_defaults_list( return cfg - def get_sources(self) -> List[ConfigSource]: + def get_sources(self) -> list[ConfigSource]: return self.repository.get_sources() def compute_defaults_list( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode, ) -> DefaultsList: parsed_overrides, caching_repo = self._parse_overrides_and_create_caching_repo(config_name, overrides) @@ -725,7 +725,7 @@ def compute_defaults_list( return defaults_list -def get_overrides_dirname(overrides: List[Override], exclude_keys: List[str], item_sep: str, kv_sep: str) -> str: +def get_overrides_dirname(overrides: list[Override], exclude_keys: list[str], item_sep: str, kv_sep: str) -> str: lines = [] for override in overrides: if override.key_or_group not in exclude_keys: diff --git a/lerna/_internal/config_repository.py b/lerna/_internal/config_repository.py index a6619fe..507b62f 100644 --- a/lerna/_internal/config_repository.py +++ b/lerna/_internal/config_repository.py @@ -3,7 +3,6 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from textwrap import dedent -from typing import Dict, List, Optional, Tuple from omegaconf import ( Container, @@ -37,7 +36,7 @@ class IConfigRepository(ABC): def get_schema_source(self) -> ConfigSource: ... @abstractmethod - def load_config(self, config_path: str) -> Optional[ConfigResult]: ... + def load_config(self, config_path: str) -> ConfigResult | None: ... @abstractmethod def group_exists(self, config_path: str) -> bool: ... @@ -46,10 +45,10 @@ def group_exists(self, config_path: str) -> bool: ... def config_exists(self, config_path: str) -> bool: ... @abstractmethod - def get_group_options(self, group_name: str, results_filter: Optional[ObjectType] = ObjectType.CONFIG) -> List[str]: ... + def get_group_options(self, group_name: str, results_filter: ObjectType | None = ObjectType.CONFIG) -> list[str]: ... @abstractmethod - def get_sources(self) -> List[ConfigSource]: ... + def get_sources(self) -> list[ConfigSource]: ... @abstractmethod def initialize_sources(self, config_search_path: ConfigSearchPath) -> None: ... @@ -57,7 +56,7 @@ def initialize_sources(self, config_search_path: ConfigSearchPath) -> None: ... class ConfigRepository(IConfigRepository): config_search_path: ConfigSearchPath - sources: List[ConfigSource] + sources: list[ConfigSource] def __init__(self, config_search_path: ConfigSearchPath) -> None: self.initialize_sources(config_search_path) @@ -77,7 +76,7 @@ def get_schema_source(self) -> ConfigSource: assert source.__class__.__name__ == "StructuredConfigSource" and source.provider == "schema", "schema config source must be last" return source - def load_config(self, config_path: str) -> Optional[ConfigResult]: + def load_config(self, config_path: str) -> ConfigResult | None: source = self._find_object_source(config_path=config_path, object_type=ObjectType.CONFIG) ret = None if source is not None: @@ -97,17 +96,17 @@ def group_exists(self, config_path: str) -> bool: def config_exists(self, config_path: str) -> bool: return self._find_object_source(config_path, ObjectType.CONFIG) is not None - def get_group_options(self, group_name: str, results_filter: Optional[ObjectType] = ObjectType.CONFIG) -> List[str]: - options: List[str] = [] + def get_group_options(self, group_name: str, results_filter: ObjectType | None = ObjectType.CONFIG) -> list[str]: + options: list[str] = [] for source in self.sources: if source.is_group(config_path=group_name): options.extend(source.list(config_path=group_name, results_filter=results_filter)) - return sorted(list(set(options))) + return sorted(set(options)) - def get_sources(self) -> List[ConfigSource]: + def get_sources(self) -> list[ConfigSource]: return self.sources - def _find_object_source(self, config_path: str, object_type: Optional[ObjectType]) -> Optional[ConfigSource]: + def _find_object_source(self, config_path: str, object_type: ObjectType | None) -> ConfigSource | None: found_source = None for source in self.sources: if object_type == ObjectType.CONFIG: @@ -139,7 +138,7 @@ def _get_scheme(path: str) -> str: def _split_group( self, group_with_package: str, - ) -> Tuple[str, Optional[str], Optional[str]]: + ) -> tuple[str, str | None, str | None]: idx = group_with_package.find("@") if idx == -1: # group @@ -164,7 +163,7 @@ def _create_defaults_list( self, config_path: str, defaults: ListConfig, - ) -> List[InputDefault]: + ) -> list[InputDefault]: def issue_deprecated_name_warning() -> None: # DEPRECATED: remove in 1.2 url = "https://hydra.cc/docs/1.2/upgrades/1.0_to_1.1/changes_to_package_header" @@ -176,15 +175,14 @@ def issue_deprecated_name_warning() -> None: ), ) - res: List[InputDefault] = [] + res: list[InputDefault] = [] for item in defaults._iter_ex(resolve=False): default: InputDefault if isinstance(item, DictConfig): if not version.base_at_least("1.2"): old_optional = None - if len(item) > 1: - if "optional" in item: - old_optional = item.pop("optional") + if len(item) > 1 and "optional" in item: + old_optional = item.pop("optional") keys = list(item.keys()) if len(keys) > 1: @@ -214,7 +212,7 @@ def issue_deprecated_name_warning() -> None: for v in patch_value: vv = v._value() if not isinstance(vv, str): - raise ValueError( + raise ValueError( # noqa: TRY004 f"Unsupported _patch_ item value in defaults : {type(vv).__name__}, nested list items must be strings" ) operations.append(vv) @@ -229,24 +227,22 @@ def issue_deprecated_name_warning() -> None: keywords = ConfigRepository.Keywords() self._extract_keywords_from_config_group(config_group, keywords) - if not version.base_at_least("1.2"): - if not keywords.optional and old_optional is not None: - keywords.optional = old_optional + if not version.base_at_least("1.2") and not keywords.optional and old_optional is not None: + keywords.optional = old_optional node = item._get_node(key) assert node is not None and isinstance(node, Node) config_value = node._value() - if not version.base_at_least("1.2"): - if old_optional is not None: - msg = dedent( - f""" + if not version.base_at_least("1.2") and old_optional is not None: + msg = dedent( + f""" In {config_path}: 'optional: true' is deprecated. Use 'optional {key}: {config_value}' instead. Support for the old style is removed for Hydra version_base >= 1.2""" - ) + ) - deprecation_warning(msg) + deprecation_warning(msg) if config_value is not None and not isinstance(config_value, (str, list)): raise ValueError(f"Unsupported item value in defaults : {type(config_value).__name__}. Supported: string or list") @@ -256,13 +252,14 @@ def issue_deprecated_name_warning() -> None: for v in config_value: vv = v._value() if not isinstance(vv, str): - raise ValueError(f"Unsupported item value in defaults : {type(vv).__name__}, nested list items must be strings") + raise ValueError( # noqa: TRY004 + f"Unsupported item value in defaults : {type(vv).__name__}, nested list items must be strings" + ) options.append(vv) config_value = options - if not version.base_at_least("1.2"): - if package is not None and "_name_" in package: - issue_deprecated_name_warning() + if not version.base_at_least("1.2") and package is not None and "_name_" in package: + issue_deprecated_name_warning() default = GroupDefault( group=keywords.group, @@ -274,13 +271,12 @@ def issue_deprecated_name_warning() -> None: elif isinstance(item, str): path, package, _package2 = self._split_group(item) - if not version.base_at_least("1.2"): - if package is not None and "_name_" in package: - issue_deprecated_name_warning() + if not version.base_at_least("1.2") and package is not None and "_name_" in package: + issue_deprecated_name_warning() default = ConfigDefault(path=path, package=package) else: - raise ValueError(f"Unsupported type in defaults : {type(item).__name__}") + raise ValueError(f"Unsupported type in defaults : {type(item).__name__}") # noqa: TRY004 res.append(default) return res @@ -289,23 +285,24 @@ def _extract_defaults_list(self, config_path: str, cfg: Container) -> ListConfig if not OmegaConf.is_dict(cfg): return empty assert isinstance(cfg, DictConfig) - with read_write(cfg): - with open_dict(cfg): - if not cfg._is_typed(): - defaults = cfg.pop("defaults", empty) - else: - # If node is a backed by Structured Config, flag it and temporarily keep the defaults list in. - # It will be removed later. - # This is addressing an edge case where the defaults list re-appears once the dataclass is used - # as a prototype during OmegaConf merge. - cfg._set_flag("HYDRA_REMOVE_TOP_LEVEL_DEFAULTS", True) - defaults = cfg.get("defaults", empty) + with read_write(cfg), open_dict(cfg): + if not cfg._is_typed(): + defaults = cfg.pop("defaults", empty) + else: + # If node is a backed by Structured Config, flag it and temporarily keep the defaults list in. + # It will be removed later. + # This is addressing an edge case where the defaults list re-appears once the dataclass is used + # as a prototype during OmegaConf merge. + cfg._set_flag("HYDRA_REMOVE_TOP_LEVEL_DEFAULTS", True) + defaults = cfg.get("defaults", empty) if not isinstance(defaults, ListConfig): if isinstance(defaults, DictConfig): type_str = "mapping" else: type_str = type(defaults).__name__ - raise ValueError(f"Invalid defaults list in '{config_path}', defaults must be a list (got {type_str})") + raise ValueError( # noqa: TRY004 + f"Invalid defaults list in '{config_path}', defaults must be a list (got {type_str})" + ) return defaults @@ -334,7 +331,7 @@ class CachingConfigRepository(IConfigRepository): def __init__(self, delegate: IConfigRepository): # copy the underlying repository to avoid mutating it with initialize_sources() self.delegate = copy.deepcopy(delegate) - self.cache: Dict[str, Optional[ConfigResult]] = {} + self.cache: dict[str, ConfigResult | None] = {} def get_schema_source(self) -> ConfigSource: return self.delegate.get_schema_source() @@ -345,7 +342,7 @@ def initialize_sources(self, config_search_path: ConfigSearchPath) -> None: # For the use case this is used, the only thing in the cache is the primary config # and we want to keep it even though we re-initialized the sources. - def load_config(self, config_path: str) -> Optional[ConfigResult]: + def load_config(self, config_path: str) -> ConfigResult | None: cache_key = f"config_path={config_path}" if cache_key in self.cache: return self.cache[cache_key] @@ -360,8 +357,8 @@ def group_exists(self, config_path: str) -> bool: def config_exists(self, config_path: str) -> bool: return self.delegate.config_exists(config_path=config_path) - def get_group_options(self, group_name: str, results_filter: Optional[ObjectType] = ObjectType.CONFIG) -> List[str]: + def get_group_options(self, group_name: str, results_filter: ObjectType | None = ObjectType.CONFIG) -> list[str]: return self.delegate.get_group_options(group_name=group_name, results_filter=results_filter) - def get_sources(self) -> List[ConfigSource]: + def get_sources(self) -> list[ConfigSource]: return self.delegate.get_sources() diff --git a/lerna/_internal/config_search_path_impl.py b/lerna/_internal/config_search_path_impl.py index 8efa72e..397f8b9 100644 --- a/lerna/_internal/config_search_path_impl.py +++ b/lerna/_internal/config_search_path_impl.py @@ -1,5 +1,5 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -from typing import List, MutableSequence, Optional, Union +from collections.abc import MutableSequence from lerna.core.config_search_path import ( ConfigSearchPath, @@ -9,7 +9,7 @@ class ConfigSearchPathImpl(ConfigSearchPath): - config_search_path: List[SearchPathElement] + config_search_path: list[SearchPathElement] def __init__(self) -> None: self.config_search_path = [] @@ -45,7 +45,7 @@ def find_match(self, reference: SearchPathQuery, reverse: bool) -> int: assert False return -1 - def append(self, provider: str, path: str, anchor: Optional[SearchPathQuery] = None) -> None: + def append(self, provider: str, path: str, anchor: SearchPathQuery | None = None) -> None: if anchor is None: self.config_search_path.append(SearchPathElement(provider, path)) else: @@ -62,7 +62,7 @@ def prepend( self, provider: str, path: str, - anchor: Optional[Union[SearchPathQuery, str]] = None, + anchor: SearchPathQuery | str | None = None, ) -> None: """ Prepends to the search path. diff --git a/lerna/_internal/core_plugins/bash_completion.py b/lerna/_internal/core_plugins/bash_completion.py index 197281d..b9b89b5 100644 --- a/lerna/_internal/core_plugins/bash_completion.py +++ b/lerna/_internal/core_plugins/bash_completion.py @@ -2,7 +2,6 @@ import logging import os import sys -from typing import Optional from lerna.plugins.completion_plugin import CompletionPlugin @@ -68,7 +67,7 @@ def uninstall(self) -> None: def provides() -> str: return "bash" - def query(self, config_name: Optional[str]) -> None: + def query(self, config_name: str | None) -> None: line = os.environ["COMP_LINE"] # key = os.environ["COMP_POINT "] if "COMP_POINT " in os.environ else len(line) diff --git a/lerna/_internal/core_plugins/basic_launcher.py b/lerna/_internal/core_plugins/basic_launcher.py index 3acff14..1577285 100644 --- a/lerna/_internal/core_plugins/basic_launcher.py +++ b/lerna/_internal/core_plugins/basic_launcher.py @@ -1,8 +1,8 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import logging +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path -from typing import List, Optional, Sequence from omegaconf import DictConfig, open_dict @@ -31,9 +31,9 @@ class BasicLauncherConf: class BasicLauncher(Launcher): def __init__(self) -> None: super().__init__() - self.config: Optional[DictConfig] = None - self.task_function: Optional[TaskFunction] = None - self.hydra_context: Optional[HydraContext] = None + self.config: DictConfig | None = None + self.task_function: TaskFunction | None = None + self.hydra_context: HydraContext | None = None def setup( self, @@ -56,7 +56,7 @@ def launch(self, job_overrides: Sequence[Sequence[str]], initial_job_idx: int) - sweep_dir = self.config.hydra.sweep.dir Path(str(sweep_dir)).mkdir(parents=True, exist_ok=True) log.info(f"Launching {len(job_overrides)} jobs locally") - runs: List[JobReturn] = [] + runs: list[JobReturn] = [] for idx, overrides in enumerate(job_overrides): idx = initial_job_idx + idx lst = " ".join(filter_overrides(overrides)) diff --git a/lerna/_internal/core_plugins/basic_sweeper.py b/lerna/_internal/core_plugins/basic_sweeper.py index 2844d74..88a20a4 100644 --- a/lerna/_internal/core_plugins/basic_sweeper.py +++ b/lerna/_internal/core_plugins/basic_sweeper.py @@ -21,9 +21,10 @@ import logging import time from collections import OrderedDict +from collections.abc import Iterable, Sequence from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Sequence +from typing import Any from omegaconf import DictConfig, OmegaConf @@ -40,8 +41,8 @@ @dataclass class BasicSweeperConf: _target_: str = "lerna._internal.core_plugins.basic_sweeper.BasicSweeper" - max_batch_size: Optional[int] = None - params: Optional[Dict[str, str]] = None + max_batch_size: int | None = None + params: dict[str, str] | None = None ConfigStore.instance().store(group="hydra/sweeper", name="basic", node=BasicSweeperConf, provider="hydra") @@ -55,7 +56,7 @@ class BasicSweeper(Sweeper): Basic sweeper """ - def __init__(self, max_batch_size: Optional[int], params: Optional[Dict[str, str]] = None) -> None: + def __init__(self, max_batch_size: int | None, params: dict[str, str] | None = None) -> None: """ Instantiates """ @@ -63,14 +64,14 @@ def __init__(self, max_batch_size: Optional[int], params: Optional[Dict[str, str if params is None: params = {} - self.overrides: Optional[Sequence[Sequence[Sequence[str]]]] = None + self.overrides: Sequence[Sequence[Sequence[str]]] | None = None self.batch_index = 0 self.max_batch_size = max_batch_size self.params = params - self.hydra_context: Optional[HydraContext] = None - self.config: Optional[DictConfig] = None - self.launcher: Optional[Launcher] = None + self.hydra_context: HydraContext | None = None + self.config: DictConfig | None = None + self.launcher: Launcher | None = None def setup( self, @@ -91,7 +92,7 @@ def setup( ) @staticmethod - def split_overrides_to_chunks(lst: List[List[str]], n: Optional[int]) -> Iterable[List[List[str]]]: + def split_overrides_to_chunks(lst: list[list[str]], n: int | None) -> Iterable[list[list[str]]]: if n is None or n == -1: n = len(lst) assert n > 0 @@ -99,7 +100,7 @@ def split_overrides_to_chunks(lst: List[List[str]], n: Optional[int]) -> Iterabl yield lst[i : i + n] @staticmethod - def split_arguments(overrides: List[Override], max_batch_size: Optional[int]) -> List[List[List[str]]]: + def split_arguments(overrides: list[Override], max_batch_size: int | None) -> list[list[list[str]]]: lists = [] final_overrides = OrderedDict() for override in overrides: @@ -116,8 +117,7 @@ def split_arguments(overrides: List[Override], max_batch_size: Optional[int]) -> value = override.get_value_element_as_str() final_overrides[key] = [f"{key}={value}"] - for _, v in final_overrides.items(): - lists.append(v) + lists.extend(final_overrides.values()) all_batches = [list(x) for x in itertools.product(*lists)] assert max_batch_size is None or max_batch_size > 0 @@ -127,13 +127,13 @@ def split_arguments(overrides: List[Override], max_batch_size: Optional[int]) -> chunks_iter = BasicSweeper.split_overrides_to_chunks(all_batches, max_batch_size) return [x for x in chunks_iter] - def _parse_config(self) -> List[str]: + def _parse_config(self) -> list[str]: params_conf = [] for k, v in self.params.items(): params_conf.append(f"{k}={v}") return params_conf - def sweep(self, arguments: List[str]) -> Any: + def sweep(self, arguments: list[str]) -> Any: assert self.config is not None assert self.launcher is not None assert self.hydra_context is not None @@ -154,7 +154,7 @@ def sweep(self, arguments: List[str]) -> Any: overrides = parser.parse_overrides(params_conf) self.overrides = self.split_arguments(overrides, self.max_batch_size) - returns: List[Sequence[JobReturn]] = [] + returns: list[Sequence[JobReturn]] = [] # Save sweep run config in top level sweep working directory sweep_dir = Path(self.config.hydra.sweep.dir) diff --git a/lerna/_internal/core_plugins/file_config_source.py b/lerna/_internal/core_plugins/file_config_source.py index 9277192..d282012 100644 --- a/lerna/_internal/core_plugins/file_config_source.py +++ b/lerna/_internal/core_plugins/file_config_source.py @@ -1,6 +1,5 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import os -from typing import List, Optional import yaml from omegaconf import OmegaConf @@ -49,7 +48,7 @@ def load_config(self, config_path: str) -> ConfigResult: if raw_config is None: raw_config = {} cfg = OmegaConf.create(raw_config) - except Exception: + except Exception: # noqa: BLE001 # Fall back to Python YAML parser on errors f.seek(0) raw = yaml.safe_load(f) @@ -82,8 +81,8 @@ def is_config(self, config_path: str) -> bool: full_path = os.path.realpath(os.path.join(self.path, config_path)) return os.path.isfile(full_path) - def list(self, config_path: str, results_filter: Optional[ObjectType]) -> List[str]: - files: List[str] = [] + def list(self, config_path: str, results_filter: ObjectType | None) -> list[str]: + files: list[str] = [] full_path = os.path.realpath(os.path.join(self.path, config_path)) for file in os.listdir(full_path): file_path = os.path.join(config_path, file) @@ -94,4 +93,4 @@ def list(self, config_path: str, results_filter: Optional[ObjectType]) -> List[s results_filter=results_filter, ) - return sorted(list(set(files))) + return sorted(set(files)) diff --git a/lerna/_internal/core_plugins/fish_completion.py b/lerna/_internal/core_plugins/fish_completion.py index e6dc027..9bad315 100644 --- a/lerna/_internal/core_plugins/fish_completion.py +++ b/lerna/_internal/core_plugins/fish_completion.py @@ -2,7 +2,6 @@ import logging import os import sys -from typing import List, Optional, Tuple from lerna.plugins.completion_plugin import CompletionPlugin @@ -46,7 +45,7 @@ def uninstall(self) -> None: def provides() -> str: return "fish" - def query(self, config_name: Optional[str]) -> None: + def query(self, config_name: str | None) -> None: line = os.environ["COMP_LINE"] line = self.strip_python_or_app_name(line) print("\n".join(self._query(config_name=config_name, line=line))) @@ -57,7 +56,7 @@ def help(command: str) -> str: return f"{{}} -sc {command}=fish | source" @staticmethod - def _get_exec() -> List[Tuple[str, str]]: + def _get_exec() -> list[tuple[str, str]]: # Running as an installed app (setuptools entry point) output = [] # User scenario 1: python script.py diff --git a/lerna/_internal/core_plugins/importlib_resources_config_source.py b/lerna/_internal/core_plugins/importlib_resources_config_source.py index e525753..ace8c30 100644 --- a/lerna/_internal/core_plugins/importlib_resources_config_source.py +++ b/lerna/_internal/core_plugins/importlib_resources_config_source.py @@ -2,7 +2,7 @@ import os import zipfile from importlib import resources -from typing import Any, List, Optional +from typing import Any import yaml from omegaconf import OmegaConf @@ -80,7 +80,7 @@ def _read_config(self, res: Any) -> ConfigResult: if raw_config is None: raw_config = {} cfg = OmegaConf.create(raw_config) - except Exception: + except Exception: # noqa: BLE001 # Fall back to Python YAML parser on errors raw = yaml.safe_load(content_str) if raw is None: @@ -134,8 +134,8 @@ def is_config(self, config_path: str) -> bool: res = files.joinpath(config_path) return self._safe_is_file(res) - def list(self, config_path: str, results_filter: Optional[ObjectType]) -> List[str]: - files: List[str] = [] + def list(self, config_path: str, results_filter: ObjectType | None) -> list[str]: + files: list[str] = [] for file in resources.files(self.path).joinpath(config_path).iterdir(): fname = file.name fpath = os.path.join(config_path, fname) @@ -146,4 +146,4 @@ def list(self, config_path: str, results_filter: Optional[ObjectType]) -> List[s results_filter=results_filter, ) - return sorted(list(set(files))) + return sorted(set(files)) diff --git a/lerna/_internal/core_plugins/pkg_helper.py b/lerna/_internal/core_plugins/pkg_helper.py index ad09252..dba3ca8 100644 --- a/lerna/_internal/core_plugins/pkg_helper.py +++ b/lerna/_internal/core_plugins/pkg_helper.py @@ -6,12 +6,12 @@ """ from importlib import resources -from typing import Any, Dict, List, Optional +from typing import Any import yaml -def load_pkg_config(module_path: str, config_path: str) -> Optional[Dict[str, Any]]: +def load_pkg_config(module_path: str, config_path: str) -> dict[str, Any] | None: """ Load a config file from a Python package. @@ -90,7 +90,7 @@ def pkg_group_exists(module_path: str, group_path: str) -> bool: return False -def pkg_list_options(module_path: str, group_path: str) -> List[str]: +def pkg_list_options(module_path: str, group_path: str) -> list[str]: """ List config options (YAML files) in a package group. @@ -110,7 +110,7 @@ def pkg_list_options(module_path: str, group_path: str) -> List[str]: for item in files.iterdir(): if item.is_file(): name = item.name - if name.endswith(".yaml") or name.endswith(".yml"): + if name.endswith((".yaml", ".yml")): # Remove extension to get option name options.append(name.rsplit(".", 1)[0]) diff --git a/lerna/_internal/core_plugins/structured_config_source.py b/lerna/_internal/core_plugins/structured_config_source.py index aad5d73..439ea9b 100644 --- a/lerna/_internal/core_plugins/structured_config_source.py +++ b/lerna/_internal/core_plugins/structured_config_source.py @@ -1,7 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import importlib import warnings -from typing import List, Optional from lerna.core.config_store import ConfigStore from lerna.core.object_type import ObjectType @@ -17,7 +16,7 @@ def __init__(self, provider: str, path: str) -> None: importlib.import_module(self.path) except Exception as e: warnings.warn(f"Error importing {self.path} : some configs may not be available\n\n\tRoot cause: {e}\n") - raise e + raise @staticmethod def scheme() -> str: @@ -47,8 +46,8 @@ def is_config(self, config_path: str) -> bool: type_ = ConfigStore.instance().get_type(filename) return type_ == ObjectType.CONFIG - def list(self, config_path: str, results_filter: Optional[ObjectType]) -> List[str]: - ret: List[str] = [] + def list(self, config_path: str, results_filter: ObjectType | None) -> list[str]: + ret: list[str] = [] files = ConfigStore.instance().list(config_path) for file in files: @@ -58,4 +57,4 @@ def list(self, config_path: str, results_filter: Optional[ObjectType]) -> List[s file_name=file, results_filter=results_filter, ) - return sorted(list(set(ret))) + return sorted(set(ret)) diff --git a/lerna/_internal/core_plugins/structured_helper.py b/lerna/_internal/core_plugins/structured_helper.py index 5f4ca2e..727ff45 100644 --- a/lerna/_internal/core_plugins/structured_helper.py +++ b/lerna/_internal/core_plugins/structured_helper.py @@ -6,7 +6,7 @@ from Python's ConfigStore singleton. """ -from typing import Any, Dict, List, Optional +from typing import Any from omegaconf import OmegaConf @@ -14,7 +14,7 @@ from lerna.core.object_type import ObjectType -def load_structured_config(config_path: str) -> Optional[Dict[str, Any]]: +def load_structured_config(config_path: str) -> dict[str, Any] | None: """ Load a config from ConfigStore and return as dict. @@ -33,7 +33,7 @@ def load_structured_config(config_path: str) -> Optional[Dict[str, Any]]: config_node = cs.load(path) # Convert DictConfig to plain dict for Rust return OmegaConf.to_container(config_node.node, resolve=False) - except Exception: + except Exception: # noqa: BLE001 return None @@ -55,7 +55,7 @@ def structured_config_exists(config_path: str) -> bool: path = f"{path}.yaml" obj_type = cs.get_type(path) return obj_type == ObjectType.CONFIG - except Exception: + except Exception: # noqa: BLE001 return False @@ -76,11 +76,11 @@ def structured_group_exists(group_path: str) -> bool: return bool(cs.repo) obj_type = cs.get_type(group_path) return obj_type == ObjectType.GROUP - except Exception: + except Exception: # noqa: BLE001 return False -def structured_list_options(group_path: str) -> List[str]: +def structured_list_options(group_path: str) -> list[str]: """ List options (configs and subgroups) in a ConfigStore group. @@ -93,11 +93,11 @@ def structured_list_options(group_path: str) -> List[str]: try: cs = ConfigStore.instance() return cs.list(group_path) - except Exception: + except Exception: # noqa: BLE001 return [] -def get_structured_package(config_path: str) -> Optional[str]: +def get_structured_package(config_path: str) -> str | None: """ Get the package for a structured config. @@ -111,5 +111,5 @@ def get_structured_package(config_path: str) -> Optional[str]: cs = ConfigStore.instance() config_node = cs.load(config_path) return config_node.package - except Exception: + except Exception: # noqa: BLE001 return None diff --git a/lerna/_internal/core_plugins/zsh_completion.py b/lerna/_internal/core_plugins/zsh_completion.py index 73ce94b..fefc81e 100644 --- a/lerna/_internal/core_plugins/zsh_completion.py +++ b/lerna/_internal/core_plugins/zsh_completion.py @@ -1,6 +1,5 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import logging -from typing import Optional from lerna.core.config_loader import ConfigLoader from lerna.plugins.completion_plugin import CompletionPlugin @@ -25,7 +24,7 @@ def uninstall(self) -> None: def provides() -> str: return "zsh" - def query(self, config_name: Optional[str]) -> None: + def query(self, config_name: str | None) -> None: self.delegate.query(config_name) @staticmethod diff --git a/lerna/_internal/defaults_list.py b/lerna/_internal/defaults_list.py index 965ea4b..68cb60f 100644 --- a/lerna/_internal/defaults_list.py +++ b/lerna/_internal/defaults_list.py @@ -3,9 +3,9 @@ import copy import os import warnings +from collections.abc import Callable from dataclasses import dataclass, field from textwrap import dedent -from typing import Callable, Dict, List, Optional, Set, Tuple, Union from omegaconf import DictConfig, OmegaConf @@ -45,33 +45,33 @@ @dataclass class Deletion: - name: Optional[str] + name: str | None used: bool = field(default=False, compare=False) @dataclass class OverrideMetadata: external_override: bool - containing_config_path: Optional[str] = None + containing_config_path: str | None = None used: bool = False - relative_key: Optional[str] = None + relative_key: str | None = None @dataclass class Overrides: - override_choices: Dict[str, Optional[Union[str, List[str]]]] - override_metadata: Dict[str, OverrideMetadata] + override_choices: dict[str, str | list[str] | None] + override_metadata: dict[str, OverrideMetadata] - append_group_defaults: List[GroupDefault] - config_overrides: List[Override] - patch_overrides: List[Override] + append_group_defaults: list[GroupDefault] + config_overrides: list[Override] + patch_overrides: list[Override] - known_choices: Dict[str, Optional[str]] - known_choices_per_group: Dict[str, Set[str]] + known_choices: dict[str, str | None] + known_choices_per_group: dict[str, set[str]] - deletions: Dict[str, Deletion] + deletions: dict[str, Deletion] - def __init__(self, repo: IConfigRepository, overrides_list: List[Override]) -> None: + def __init__(self, repo: IConfigRepository, overrides_list: list[Override]) -> None: self.override_choices = {} self.override_metadata = {} self.append_group_defaults = [] @@ -138,10 +138,10 @@ def _resolve_patch_key(key: str, parent_package: str) -> str: def add_patch_operations( self, - operations: List[str], + operations: list[str], parent_package: str, containing_config_path: str, - package_scope: Optional[str] = None, + package_scope: str | None = None, ) -> None: # If _patch_@pkg is used, scope bare keys to pkg instead of parent_package effective_package = package_scope if package_scope is not None else parent_package @@ -188,10 +188,10 @@ def ensure_overrides_used(self) -> None: for key, meta in self.override_metadata.items(): if not meta.used: group = key.split("@")[0] - choices = self.known_choices_per_group[group] if group in self.known_choices_per_group else set() + choices = self.known_choices_per_group.get(group, set()) if len(choices) > 1: - msg = f"Could not override '{key}'.\nDid you mean to override one of {', '.join(sorted(list(choices)))}?" + msg = f"Could not override '{key}'.\nDid you mean to override one of {', '.join(sorted(choices))}?" elif len(choices) == 1: msg = f"Could not override '{key}'.\nDid you mean to override {copy.copy(choices).pop()}?" elif len(choices) == 0: @@ -251,16 +251,16 @@ def delete(self, default: InputDefault) -> None: @dataclass class DefaultsList: - defaults: List[ResultDefault] + defaults: list[ResultDefault] defaults_tree: DefaultsTreeNode - config_overrides: List[Override] - config_patch_overrides: List[Override] + config_overrides: list[Override] + config_patch_overrides: list[Override] overrides: Overrides def _validate_self( containing_node: InputDefault, - defaults: List[InputDefault], + defaults: list[InputDefault], has_config_content: bool, ) -> bool: # check that self is present only once @@ -306,7 +306,7 @@ def _expand_virtual_root( overrides: Overrides, skip_missing: bool, ) -> DefaultsTreeNode: - children: List[Union[DefaultsTreeNode, InputDefault]] = [] + children: list[DefaultsTreeNode | InputDefault] = [] assert root.children is not None for d in reversed(root.children): assert isinstance(d, InputDefault) @@ -366,7 +366,7 @@ def _check_not_missing( def _create_interpolation_map( overrides: Overrides, - defaults_list: List[InputDefault], + defaults_list: list[InputDefault], self_added: bool, ) -> DictConfig: known_choices = OmegaConf.create(overrides.known_choices) @@ -402,7 +402,7 @@ def _create_defaults_tree( def _update_overrides( - defaults_list: List[InputDefault], + defaults_list: list[InputDefault], overrides: Overrides, parent: InputDefault, interpolated_subtree: bool, @@ -477,7 +477,7 @@ def _has_config_content(cfg: DictConfig) -> bool: if cfg._is_none() or cfg._is_missing(): return False - for key in cfg.keys(): + for key in cfg: if not OmegaConf.is_missing(cfg, key) and key != "defaults": return True return False @@ -492,7 +492,7 @@ def _create_defaults_tree_impl( overrides: Overrides, ) -> DefaultsTreeNode: parent = root.node - children: List[Union[InputDefault, DefaultsTreeNode]] = [] + children: list[InputDefault | DefaultsTreeNode] = [] if parent.is_virtual(): if is_root_config: return _expand_virtual_root(repo, root, overrides, skip_missing) @@ -549,7 +549,7 @@ def _create_defaults_tree_impl( _update_overrides(defaults_list, overrides, parent, interpolated_subtree) def add_child( - child_list: List[Union[InputDefault, DefaultsTreeNode]], + child_list: list[InputDefault | DefaultsTreeNode], new_root_: DefaultsTreeNode, ) -> None: subtree_ = _create_defaults_tree_impl( @@ -641,7 +641,7 @@ def add_child( return root -def _create_result_default(tree: Optional[DefaultsTreeNode], node: InputDefault) -> Optional[ResultDefault]: +def _create_result_default(tree: DefaultsTreeNode | None, node: InputDefault) -> ResultDefault | None: if node.is_virtual(): return None if node.get_name() is None: @@ -677,7 +677,7 @@ def _create_result_default(tree: Optional[DefaultsTreeNode], node: InputDefault) def _dfs_walk( tree: DefaultsTreeNode, - operator: Callable[[Optional[DefaultsTreeNode], InputDefault], None], + operator: Callable[[DefaultsTreeNode | None, InputDefault], None], ) -> None: if tree.children is None or len(tree.children) == 0: operator(tree.parent, tree.node) @@ -692,12 +692,12 @@ def _dfs_walk( def _tree_to_list( tree: DefaultsTreeNode, -) -> List[ResultDefault]: +) -> list[ResultDefault]: class Collector: def __init__(self) -> None: - self.output: List[ResultDefault] = [] + self.output: list[ResultDefault] = [] - def __call__(self, tree_node: Optional[DefaultsTreeNode], node: InputDefault) -> None: + def __call__(self, tree_node: DefaultsTreeNode | None, node: InputDefault) -> None: if node.is_deleted(): return @@ -713,7 +713,7 @@ def __call__(self, tree_node: Optional[DefaultsTreeNode], node: InputDefault) -> return visitor.output -def _create_root(config_name: Optional[str], with_hydra: bool) -> DefaultsTreeNode: +def _create_root(config_name: str | None, with_hydra: bool) -> DefaultsTreeNode: primary: InputDefault if config_name is None: primary = ConfigDefault(path="_dummy_empty_config_", primary=True) @@ -730,7 +730,7 @@ def _create_root(config_name: Optional[str], with_hydra: bool) -> DefaultsTreeNo return root -def ensure_no_duplicates_in_list(result: List[ResultDefault]) -> None: +def ensure_no_duplicates_in_list(result: list[ResultDefault]) -> None: keys = set() for item in result: if not item.is_self: @@ -743,11 +743,11 @@ def ensure_no_duplicates_in_list(result: List[ResultDefault]) -> None: def _create_defaults_list( repo: IConfigRepository, - config_name: Optional[str], + config_name: str | None, overrides: Overrides, prepend_hydra: bool, skip_missing: bool, -) -> Tuple[List[ResultDefault], DefaultsTreeNode]: +) -> tuple[list[ResultDefault], DefaultsTreeNode]: root = _create_root(config_name=config_name, with_hydra=prepend_hydra) defaults_tree = _create_defaults_tree( @@ -766,8 +766,8 @@ def _create_defaults_list( def create_defaults_list( repo: IConfigRepository, - config_name: Optional[str], - overrides_list: List[Override], + config_name: str | None, + overrides_list: list[Override], prepend_hydra: bool, skip_missing: bool, ) -> DefaultsList: @@ -833,7 +833,7 @@ def config_not_found_error(repo: IConfigRepository, tree: DefaultsTreeNode) -> N descs = [] for src in repo.get_sources(): - descs.append(f"\t{repr(src)}") + descs.append(f"\t{src!r}") lines = "\n".join(descs) msg += "\nConfig search path:" + f"\n{lines}" @@ -846,11 +846,11 @@ def config_not_found_error(repo: IConfigRepository, tree: DefaultsTreeNode) -> N def create_defaults_list_rust( repo: IConfigRepository, - config_name: Optional[str], - overrides_list: List[Override], + config_name: str | None, + overrides_list: list[Override], prepend_hydra: bool, skip_missing: bool, -) -> Optional[DefaultsList]: +) -> DefaultsList | None: """ Create defaults list using Rust implementation. diff --git a/lerna/_internal/grammar/functions.py b/lerna/_internal/grammar/functions.py index 1e1aaee..f8775bb 100644 --- a/lerna/_internal/grammar/functions.py +++ b/lerna/_internal/grammar/functions.py @@ -1,7 +1,8 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import inspect +from collections.abc import Callable from dataclasses import dataclass, field -from typing import Any, Callable, Dict, List +from typing import Any from omegaconf._utils import type_str @@ -13,8 +14,8 @@ @dataclass class FunctionCall: name: str - args: List[Any] - kwargs: Dict[str, Any] + args: list[Any] + kwargs: dict[str, Any] @dataclass @@ -38,8 +39,8 @@ class Functions: } ) - definitions: Dict[str, inspect.Signature] = field(default_factory=dict) - functions: Dict[str, Callable[..., Any]] = field(default_factory=dict) + definitions: dict[str, inspect.Signature] = field(default_factory=dict) + functions: dict[str, Callable[..., Any]] = field(default_factory=dict) # Tracks which Rust-native functions have been overridden by user registrations user_overrides: set = field(default_factory=set) diff --git a/lerna/_internal/grammar/grammar_functions.py b/lerna/_internal/grammar/grammar_functions.py index 87fdffb..d71fd44 100644 --- a/lerna/_internal/grammar/grammar_functions.py +++ b/lerna/_internal/grammar/grammar_functions.py @@ -2,8 +2,9 @@ import builtins import json import random +from collections.abc import Callable from copy import copy -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any from lerna._internal.grammar.utils import is_type_matching from lerna.core.override_parser.types import ( @@ -17,18 +18,18 @@ Sweep, ) -ElementType = Union[str, int, bool, float, List[Any], Dict[str, Any]] +ElementType = str | int | bool | float | list[Any] | dict[str, Any] def apply_to_dict_values( # val - value: Dict[Any, Any], + value: dict[Any, Any], # func function: Callable[..., Any], -) -> Dict[Any, Any]: - ret_dict: Dict[str, Any] = {} - for key, value in value.items(): - ret_dict[key] = function(value) +) -> dict[Any, Any]: + ret_dict: dict[str, Any] = {} + for key, item in value.items(): + ret_dict[key] = function(item) return ret_dict @@ -55,18 +56,18 @@ def cast_range(value: RangeSweep, function: Callable[..., Any]) -> RangeSweep: ) -CastType = Union[ParsedElementType, Sweep] +CastType = ParsedElementType | Sweep def _list_to_simple_choice(*args: Any) -> ChoiceSweep: - choices: List[ParsedElementType] = [] + choices: list[ParsedElementType] = [] for arg in args: assert is_type_matching(arg, ParsedElementType) choices.append(arg) return ChoiceSweep(list=builtins.list(choices), simple_form=True) -def _normalize_cast_value(*args: CastType, value: Optional[CastType]) -> CastType: +def _normalize_cast_value(*args: CastType, value: CastType | None) -> CastType: if len(args) > 0 and value is not None: raise TypeError("cannot use both position and named arguments") if value is not None: @@ -80,7 +81,7 @@ def _normalize_cast_value(*args: CastType, value: Optional[CastType]) -> CastTyp assert False -def cast_int(*args: CastType, value: Optional[CastType] = None) -> Any: +def cast_int(*args: CastType, value: CastType | None = None) -> Any: value = _normalize_cast_value(*args, value=value) if isinstance(value, QuotedString): return cast_int(value.text) @@ -98,7 +99,7 @@ def cast_int(*args: CastType, value: Optional[CastType] = None) -> Any: return int(value) -def cast_float(*args: CastType, value: Optional[CastType] = None) -> Any: +def cast_float(*args: CastType, value: CastType | None = None) -> Any: value = _normalize_cast_value(*args, value=value) if isinstance(value, QuotedString): return cast_float(value.text) @@ -116,7 +117,7 @@ def cast_float(*args: CastType, value: Optional[CastType] = None) -> Any: return float(value) -def cast_str(*args: CastType, value: Optional[CastType] = None) -> Any: +def cast_str(*args: CastType, value: CastType | None = None) -> Any: value = _normalize_cast_value(*args, value=value) if isinstance(value, QuotedString): return cast_str(value.text) @@ -129,7 +130,7 @@ def cast_str(*args: CastType, value: Optional[CastType] = None) -> Any: elif isinstance(value, RangeSweep): return cast_range(value, cast_str) elif isinstance(value, IntervalSweep): - raise ValueError("Intervals cannot be cast to str") + raise ValueError("Intervals cannot be cast to str") # noqa: TRY004 assert isinstance(value, (int, float, bool, str)) if isinstance(value, bool): @@ -138,7 +139,7 @@ def cast_str(*args: CastType, value: Optional[CastType] = None) -> Any: return str(value) -def extract_text(*args: Any, value: Optional[Any] = None) -> Any: +def extract_text(*args: Any, value: Any | None = None) -> Any: value = _normalize_cast_value(*args, value=value) if isinstance(value, QuotedString): return value.text @@ -154,7 +155,7 @@ def extract_text(*args: Any, value: Optional[Any] = None) -> Any: return value -def cast_json_str(*args: Any, value: Optional[Any] = None) -> Any: +def cast_json_str(*args: Any, value: Any | None = None) -> Any: value = _normalize_cast_value(*args, value=value) json_val = value if isinstance(value, QuotedString): @@ -170,12 +171,12 @@ def cast_json_str(*args: Any, value: Optional[Any] = None) -> Any: json_range = cast_range(value, extract_text) return cast_range(json_range, json.dumps) elif isinstance(value, IntervalSweep): - raise ValueError("Intervals cannot be cast to json_str") + raise ValueError("Intervals cannot be cast to json_str") # noqa: TRY004 return json.dumps(json_val) -def cast_bool(*args: CastType, value: Optional[CastType] = None) -> Any: +def cast_bool(*args: CastType, value: CastType | None = None) -> Any: value = _normalize_cast_value(*args, value=value) if isinstance(value, QuotedString): return cast_bool(value.text) @@ -188,7 +189,7 @@ def cast_bool(*args: CastType, value: Optional[CastType] = None) -> Any: elif isinstance(value, RangeSweep): return cast_range(value, cast_bool) elif isinstance(value, IntervalSweep): - raise ValueError("Intervals cannot be cast to bool") + raise ValueError("Intervals cannot be cast to bool") # noqa: TRY004 if isinstance(value, str): if value.lower() == "false": @@ -200,7 +201,7 @@ def cast_bool(*args: CastType, value: Optional[CastType] = None) -> Any: return bool(value) -def choice(*args: Union[str, int, float, bool, Dict[Any, Any], List[Any], ChoiceSweep]) -> ChoiceSweep: +def choice(*args: str | float | bool | dict[Any, Any] | list[Any] | ChoiceSweep) -> ChoiceSweep: """ A choice sweep over the specified values """ @@ -219,9 +220,9 @@ def choice(*args: Union[str, int, float, bool, Dict[Any, Any], List[Any], Choice def range( - start: Union[int, float], - stop: Optional[Union[int, float]] = None, - step: Union[int, float] = 1, + start: float, + stop: float | None = None, + step: float = 1, ) -> RangeSweep: """ Range defines a sweep over a range of integer or floating-point values. @@ -238,7 +239,7 @@ def range( return RangeSweep(start=start, stop=stop, step=step) -def interval(start: Union[int, float], end: Union[int, float]) -> IntervalSweep: +def interval(start: float, end: float) -> IntervalSweep: """ A continuous interval between two floating point values. value=interval(x,y) is interpreted as x <= value < y @@ -246,7 +247,7 @@ def interval(start: Union[int, float], end: Union[int, float]) -> IntervalSweep: return IntervalSweep(start=float(start), end=float(end)) -def tag(*args: Union[str, Union[Sweep]], sweep: Optional[Sweep] = None) -> Sweep: +def tag(*args: str | Sweep, sweep: Sweep | None = None) -> Sweep: """ Tags the sweep with a list of string tags. """ @@ -262,19 +263,21 @@ def tag(*args: Union[str, Union[Sweep]], sweep: Optional[Sweep] = None) -> Sweep tags = set() for tag_ in args[0:-1]: if not isinstance(tag_, str): - raise ValueError(f"tag arguments type must be string, got {type(tag_).__name__}") + raise ValueError(f"tag arguments type must be string, got {type(tag_).__name__}") # noqa: TRY004 tags.add(tag_) sweep.tags = tags return sweep else: - raise ValueError(f"Last argument to tag() must be a choice(), range() or interval(), got {type(sweep).__name__}") + raise ValueError( # noqa: TRY004 + f"Last argument to tag() must be a choice(), range() or interval(), got {type(sweep).__name__}" + ) def shuffle( - *args: Union[ElementType, ChoiceSweep, RangeSweep], - sweep: Optional[Union[ChoiceSweep, RangeSweep]] = None, - list: Optional[List[Any]] = None, -) -> Union[List[Any], ChoiceSweep, RangeSweep]: + *args: ElementType | ChoiceSweep | RangeSweep, + sweep: ChoiceSweep | RangeSweep | None = None, + list: list[Any] | None = None, +) -> list[Any] | ChoiceSweep | RangeSweep: """ Shuffle input list or sweep (does not support interval) """ @@ -302,9 +305,9 @@ def shuffle( def sort( - *args: Union[ElementType, ChoiceSweep, RangeSweep], - sweep: Optional[Union[ChoiceSweep, RangeSweep]] = None, - list: Optional[List[Any]] = None, + *args: ElementType | ChoiceSweep | RangeSweep, + sweep: ChoiceSweep | RangeSweep | None = None, + list: list[Any] | None = None, reverse: bool = False, ) -> Any: """ @@ -341,7 +344,7 @@ def sort( return _sort_sweep(cw, reverse) -def _sort_sweep(sweep: Union[ChoiceSweep, RangeSweep], reverse: bool) -> Union[ChoiceSweep, RangeSweep]: +def _sort_sweep(sweep: ChoiceSweep | RangeSweep, reverse: bool) -> ChoiceSweep | RangeSweep: sweep = copy(sweep) if isinstance(sweep, ChoiceSweep): @@ -372,7 +375,7 @@ def _sort_sweep(sweep: Union[ChoiceSweep, RangeSweep], reverse: bool) -> Union[C assert False -def glob(include: Union[List[str], str], exclude: Optional[Union[List[str], str]] = None) -> Glob: +def glob(include: list[str] | str, exclude: list[str] | str | None = None) -> Glob: """ A glob selects from all options in the config group. inputs are in glob format. e.g: *, foo*, *foo. diff --git a/lerna/_internal/hydra.py b/lerna/_internal/hydra.py index e807e9c..c3caa9d 100644 --- a/lerna/_internal/hydra.py +++ b/lerna/_internal/hydra.py @@ -5,7 +5,8 @@ import sys from argparse import ArgumentParser from collections import defaultdict -from typing import Any, Callable, DefaultDict, List, Optional, Sequence, Type, Union +from collections.abc import Callable, Sequence +from typing import Any from omegaconf import Container, DictConfig, OmegaConf, flag_override @@ -34,16 +35,16 @@ from .config_loader_impl import ConfigLoaderImpl from .utils import create_automatic_config_search_path -log: Optional[logging.Logger] = None +log: logging.Logger | None = None class Hydra: @classmethod def create_main_hydra_file_or_module( - cls: Type["Hydra"], - calling_file: Optional[str], - calling_module: Optional[str], - config_path: Optional[str], + cls: type["Hydra"], + calling_file: str | None, + calling_module: str | None, + config_path: str | None, job_name: str, ) -> "Hydra": config_search_path = create_automatic_config_search_path(calling_file, calling_module, config_path) @@ -76,8 +77,8 @@ def __init__(self, task_name: str, config_loader: ConfigLoader) -> None: def get_mode( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], ) -> Any: try: cfg = self.compose_config( @@ -89,14 +90,14 @@ def get_mode( run_callback=False, ) return cfg.hydra.mode - except Exception: + except Exception: # noqa: BLE001 return None def run( self, - config_name: Optional[str], + config_name: str | None, task_function: TaskFunction, - overrides: List[str], + overrides: list[str], with_log_configuration: bool = True, ) -> JobReturn: cfg = self.compose_config( @@ -130,9 +131,9 @@ def run( def multirun( self, - config_name: Optional[str], + config_name: str | None, task_function: TaskFunction, - overrides: List[str], + overrides: list[str], with_log_configuration: bool = True, ) -> Any: cfg = self.compose_config( @@ -179,10 +180,10 @@ def get_sanitized_cfg(self, cfg: DictConfig, cfg_type: str) -> DictConfig: def show_cfg( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], cfg_type: str, - package: Optional[str], + package: str | None, resolve: bool = False, ) -> None: cfg = self.compose_config( @@ -218,8 +219,8 @@ def show_cfg( @staticmethod def get_shell_to_plugin_map( config_loader: ConfigLoader, - ) -> DefaultDict[str, List[CompletionPlugin]]: - shell_to_plugin: DefaultDict[str, List[CompletionPlugin]] = defaultdict(list) + ) -> defaultdict[str, list[CompletionPlugin]]: + shell_to_plugin: defaultdict[str, list[CompletionPlugin]] = defaultdict(list) for clazz in Plugins.instance().discover(CompletionPlugin): assert issubclass(clazz, CompletionPlugin) plugin = clazz(config_loader) @@ -232,7 +233,7 @@ def get_shell_to_plugin_map( return shell_to_plugin - def shell_completion(self, config_name: Optional[str], overrides: List[str]) -> None: + def shell_completion(self, config_name: str | None, overrides: list[str]) -> None: subcommands = ["install", "uninstall", "query"] arguments = OmegaConf.from_dotlist(overrides) num_commands = sum(1 for key in subcommands if key in arguments) @@ -243,7 +244,7 @@ def shell_completion(self, config_name: Optional[str], overrides: List[str]) -> def find_plugin(cmd: str) -> CompletionPlugin: if cmd not in shell_to_plugin: - lst = "\n".join("\t" + x for x in shell_to_plugin.keys()) + lst = "\n".join("\t" + x for x in shell_to_plugin) raise ValueError(f"No completion plugin for '{cmd}' found, available : \n{lst}") return shell_to_plugin[cmd][0] @@ -272,7 +273,7 @@ def format_args_help(args_parser: ArgumentParser) -> str: def list_all_config_groups(self, parent: str = "") -> Sequence[str]: from lerna.core.object_type import ObjectType - groups: List[str] = [] + groups: list[str] = [] for group in self.config_loader.list_groups(parent): if parent == "": group_name = group @@ -324,7 +325,7 @@ def is_not_hydra_group(x: str) -> bool: ) return help_text - def hydra_help(self, config_name: Optional[str], args_parser: ArgumentParser, args: Any) -> None: + def hydra_help(self, config_name: str | None, args_parser: ArgumentParser, args: Any) -> None: cfg = self.compose_config( config_name=None, overrides=args.overrides, @@ -336,7 +337,7 @@ def hydra_help(self, config_name: Optional[str], args_parser: ArgumentParser, ar help_text = self.get_help(help_cfg, cfg, args_parser, resolve=False) print(help_text) - def app_help(self, config_name: Optional[str], args_parser: ArgumentParser, args: Any) -> None: + def app_help(self, config_name: str | None, args_parser: ArgumentParser, args: Any) -> None: cfg = self.compose_config( config_name=config_name, overrides=args.overrides, @@ -379,8 +380,7 @@ def _print_plugins(self) -> None: Hydra._log_header(header=f"{plugin_type.__name__}:", prefix="\t") for plugin in plugins: log.debug(f"\t\t{plugin.__name__}") - if plugin.__name__ in all_plugins: - all_plugins.remove(plugin.__name__) + all_plugins.discard(plugin.__name__) if len(all_plugins) > 0: Hydra._log_header(header="Generic plugins: ", prefix="\t") @@ -389,15 +389,15 @@ def _print_plugins(self) -> None: def _print_search_path( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode = RunMode.RUN, ) -> None: assert log is not None log.debug("") self._log_header(header="Config search path", filler="*") - box: List[List[str]] = [["Provider", "Search path"]] + box: list[list[str]] = [["Provider", "Search path"]] cfg = self.compose_config( config_name=config_name, @@ -439,7 +439,7 @@ def _print_plugins_profiling_info(self, top_n: int) -> None: sorted_items = sorted(filtered, key=lambda x: x[1], reverse=True) top_n = max(len(sorted_items), top_n) - box: List[List[str]] = [["Module", "Sec"]] + box: list[list[str]] = [["Module", "Sec"]] for item in sorted_items[0:top_n]: box.append([item[0], f"{item[1]:.3f}"]) @@ -465,8 +465,8 @@ def _print_plugins_profiling_info(self, top_n: int) -> None: def _print_config_info( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode = RunMode.RUN, ) -> None: assert log is not None @@ -491,8 +491,8 @@ def _print_config_info( def _print_defaults_list( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode = RunMode.RUN, ) -> None: assert log is not None @@ -502,7 +502,7 @@ def _print_defaults_list( run_mode=run_mode, ) - box: List[List[str]] = [ + box: list[list[str]] = [ [ "Config path", "Package", @@ -532,21 +532,14 @@ def _print_defaults_list( self._log_header(header=header, filler="-") for row in box: - log.debug( - "| {} | {} | {} | {} |".format( - row[0].ljust(padding[0]), - row[1].ljust(padding[1]), - row[2].ljust(padding[2]), - row[3].ljust(padding[3]), - ) - ) + log.debug(f"| {row[0].ljust(padding[0])} | {row[1].ljust(padding[1])} | {row[2].ljust(padding[2])} | {row[3].ljust(padding[3])} |") self._log_footer(header=header, filler="-") def _print_debug_info( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode = RunMode.RUN, ) -> None: assert log is not None @@ -555,8 +548,8 @@ def _print_debug_info( def compose_config( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode, with_log_configuration: bool = False, from_shell: bool = True, @@ -598,8 +591,8 @@ def compose_config( def _print_plugins_info( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode = RunMode.RUN, ) -> None: self._print_plugins() @@ -607,8 +600,8 @@ def _print_plugins_info( def _print_all_info( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode = RunMode.RUN, ) -> None: from .. import __version__ @@ -619,7 +612,7 @@ def _print_all_info( def _print_defaults_tree_impl( self, - tree: Union[DefaultsTreeNode, InputDefault], + tree: DefaultsTreeNode | InputDefault, indent: int = 0, ) -> None: assert log is not None @@ -652,8 +645,8 @@ def to_str(node: InputDefault) -> str: def _print_defaults_tree( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode = RunMode.RUN, ) -> None: assert log is not None @@ -669,8 +662,8 @@ def _print_defaults_tree( def show_info( self, info: str, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode = RunMode.RUN, ) -> None: options = { diff --git a/lerna/_internal/instantiate/_instantiate2.py b/lerna/_internal/instantiate/_instantiate2.py index 43d5fa3..999306d 100644 --- a/lerna/_internal/instantiate/_instantiate2.py +++ b/lerna/_internal/instantiate/_instantiate2.py @@ -3,9 +3,10 @@ import copy import functools import os +from collections.abc import Callable, Sequence from enum import Enum from textwrap import dedent -from typing import Any, Callable, Dict, List, Sequence, Tuple, Union +from typing import Any from omegaconf import OmegaConf, SCMode from omegaconf._utils import is_structured_config @@ -43,7 +44,6 @@ "os.chmod", "os.chown", "os.chroot", - "os.fchdir", "os.lchflags", "os.lchmod", "os.lchown", @@ -79,17 +79,14 @@ def _enhance_omegaconf_error(e: OmegaConfBaseException, cfg: Any) -> None: full_key and object_type context in errors. """ # Only enhance if object_type is missing/None - if hasattr(e, "object_type") and e.object_type is None: - if OmegaConf.is_config(cfg): - obj_type = cfg._metadata.object_type - if obj_type is not None: - object.__setattr__(e, "object_type", obj_type) + if hasattr(e, "object_type") and e.object_type is None and OmegaConf.is_config(cfg): + obj_type = cfg._metadata.object_type + if obj_type is not None: + object.__setattr__(e, "object_type", obj_type) # Only enhance if full_key is missing/None - if hasattr(e, "full_key") and e.full_key is None: - # Try to get the key from the error if available - if hasattr(e, "key") and e.key is not None: - object.__setattr__(e, "full_key", str(e.key)) + if hasattr(e, "full_key") and e.full_key is None and hasattr(e, "key") and e.key is not None: + object.__setattr__(e, "full_key", str(e.key)) def _format_enhanced_error_message(e: OmegaConfBaseException) -> None: @@ -139,7 +136,7 @@ def _find_bad_interpolation_key(cfg: Any, prefix: str = "") -> str | None: return None if OmegaConf.is_dict(cfg): - for key in cfg.keys(): + for key in cfg: full_key = f"{prefix}.{key}" if prefix else str(key) try: value = cfg[key] @@ -193,7 +190,7 @@ def _is_target(x: Any) -> bool: return False -def _extract_pos_args(input_args: Any, kwargs: Any) -> Tuple[Any, Any]: +def _extract_pos_args(input_args: Any, kwargs: Any) -> tuple[Any, Any]: config_args = kwargs.pop(_Keys.ARGS, ()) output_args = config_args @@ -209,8 +206,8 @@ def _extract_pos_args(input_args: Any, kwargs: Any) -> Tuple[Any, Any]: def _call_target( _target_: Callable[..., Any], _partial_: bool, - args: Tuple[Any, ...], - kwargs: Dict[str, Any], + args: tuple[Any, ...], + kwargs: dict[str, Any], full_key: str, ) -> Any: """Call target (type) with args and kwargs.""" @@ -226,7 +223,7 @@ def _call_target( if OmegaConf.is_config(v): v._set_parent(None) except Exception as e: - msg = f"Error in collecting args and kwargs for '{_convert_target_to_string(_target_)}':" + f"\n{repr(e)}" + msg = f"Error in collecting args and kwargs for '{_convert_target_to_string(_target_)}':" + f"\n{e!r}" if full_key: msg += f"\nfull_key: {full_key}" @@ -236,7 +233,7 @@ def _call_target( try: return functools.partial(_target_, *args, **kwargs) except Exception as e: - msg = f"Error in creating partial({_convert_target_to_string(_target_)}, ...) object:" + f"\n{repr(e)}" + msg = f"Error in creating partial({_convert_target_to_string(_target_)}, ...) object:" + f"\n{e!r}" if full_key: msg += f"\nfull_key: {full_key}" raise InstantiationException(msg) from e @@ -244,7 +241,7 @@ def _call_target( try: return _target_(*args, **kwargs) except Exception as e: - msg = f"Error in call to target '{_convert_target_to_string(_target_)}':\n{repr(e)}" + msg = f"Error in call to target '{_convert_target_to_string(_target_)}':\n{e!r}" if full_key: msg += f"\nfull_key: {full_key}" raise InstantiationException(msg) from e @@ -257,7 +254,7 @@ def _convert_target_to_string(t: Any) -> Any: return t -def _prepare_input_dict_or_list(d: Union[Dict[Any, Any], List[Any]]) -> Any: +def _prepare_input_dict_or_list(d: dict[Any, Any] | list[Any]) -> Any: res: Any if isinstance(d, dict): res = {} @@ -278,7 +275,7 @@ def _prepare_input_dict_or_list(d: Union[Dict[Any, Any], List[Any]]) -> Any: return res -def _resolve_target(target: Union[str, type, Callable[..., Any]], full_key: str, call: bool = True) -> Union[type, Callable[..., Any], Any]: +def _resolve_target(target: str | type | Callable[..., Any], full_key: str, call: bool = True) -> type | Callable[..., Any] | Any: """Resolve target string, type or callable into type or callable. If call is False, returns the resolved target without requiring it to be callable. @@ -470,7 +467,7 @@ def instantiate( ) -def _convert_node(node: Any, convert: Union[ConvertMode, str]) -> Any: +def _convert_node(node: Any, convert: ConvertMode | str) -> Any: if OmegaConf.is_config(node): if convert == ConvertMode.ALL: node = OmegaConf.to_container(node, resolve=True) @@ -484,7 +481,7 @@ def _convert_node(node: Any, convert: Union[ConvertMode, str]) -> Any: def instantiate_node( node: Any, *args: Any, - convert: Union[str, ConvertMode] = ConvertMode.NONE, + convert: str | ConvertMode = ConvertMode.NONE, recursive: bool = True, partial: bool = False, ) -> Any: @@ -499,9 +496,9 @@ def instantiate_node( if OmegaConf.is_dict(node): # using getitem instead of get(key, default) because OmegaConf will raise an exception # if the key type is incompatible on get. - convert = node[_Keys.CONVERT] if _Keys.CONVERT in node else convert - recursive = node[_Keys.RECURSIVE] if _Keys.RECURSIVE in node else recursive - partial = node[_Keys.PARTIAL] if _Keys.PARTIAL in node else partial + convert = node.get(_Keys.CONVERT, convert) + recursive = node.get(_Keys.RECURSIVE, recursive) + partial = node.get(_Keys.PARTIAL, partial) full_key = node._get_full_key(None) @@ -544,7 +541,7 @@ def instantiate_node( kwargs = {} is_partial = node.get("_partial_", False) or partial - for key in node.keys(): + for key in node: if key not in exclude_keys: if OmegaConf.is_missing(node, key) and is_partial: continue diff --git a/lerna/_internal/sources_registry.py b/lerna/_internal/sources_registry.py index 37e10b9..8190893 100644 --- a/lerna/_internal/sources_registry.py +++ b/lerna/_internal/sources_registry.py @@ -1,17 +1,17 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -from typing import Any, Dict, Type +from typing import Any from lerna.core.singleton import Singleton from lerna.plugins.config_source import ConfigSource class SourcesRegistry(metaclass=Singleton): - types: Dict[str, Type[ConfigSource]] + types: dict[str, type[ConfigSource]] def __init__(self) -> None: self.types = {} - def register(self, type_: Type[ConfigSource]) -> None: + def register(self, type_: type[ConfigSource]) -> None: scheme = type_.scheme() if scheme in self.types: if self.types[scheme].__name__ != type_.__name__: @@ -21,7 +21,7 @@ def register(self, type_: Type[ConfigSource]) -> None: return self.types[scheme] = type_ - def resolve(self, scheme: str) -> Type[ConfigSource]: + def resolve(self, scheme: str) -> type[ConfigSource]: if scheme not in self.types: supported = ", ".join(sorted(self.types.keys())) raise ValueError(f"No config source registered for schema {scheme}, supported types : [{supported}]") diff --git a/lerna/_internal/utils.py b/lerna/_internal/utils.py index 7d9312d..bd18ba8 100644 --- a/lerna/_internal/utils.py +++ b/lerna/_internal/utils.py @@ -6,10 +6,11 @@ import sys import traceback import warnings +from collections.abc import Sequence from dataclasses import dataclass from os.path import dirname, join, normpath, realpath from types import FrameType, TracebackType -from typing import Any, List, Optional, Sequence, Tuple +from typing import Any from omegaconf.errors import OmegaConfBaseException @@ -26,7 +27,7 @@ log = logging.getLogger(__name__) -def _get_module_name_override() -> Optional[str]: +def _get_module_name_override() -> str | None: module_envs = ["HYDRA_MAIN_MODULE", "FB_PAR_MAIN_MODULE", "FB_XAR_MAIN_MODULE"] for module_env in module_envs: if module_env in os.environ: @@ -36,7 +37,7 @@ def _get_module_name_override() -> Optional[str]: def detect_calling_file_or_module_from_task_function( task_function: Any, -) -> Tuple[Optional[str], Optional[str]]: +) -> tuple[str | None, str | None]: # if function is decorated, unwrap it while hasattr(task_function, "__wrapped__"): task_function = task_function.__wrapped__ @@ -46,8 +47,8 @@ def detect_calling_file_or_module_from_task_function( if override is not None: mdl = override - calling_file: Optional[str] - calling_module: Optional[str] + calling_file: str | None + calling_module: str | None if mdl not in (None, "__main__"): calling_file = None calling_module = mdl @@ -63,7 +64,7 @@ def detect_calling_file_or_module_from_task_function( def detect_calling_file_or_module_from_stack_frame( stack_depth: int, -) -> Tuple[Optional[str], Optional[str]]: +) -> tuple[str | None, str | None]: stack = inspect.stack() frame = stack[stack_depth] if is_notebook() and "_dh" in frame[0].f_globals: @@ -99,7 +100,7 @@ def is_notebook() -> bool: return False -def detect_task_name(calling_file: Optional[str], calling_module: Optional[str]) -> str: +def detect_task_name(calling_file: str | None, calling_module: str | None) -> str: if calling_file is not None: target_file = os.path.basename(calling_file) task_name = get_valid_filename(os.path.splitext(target_file)[0]) @@ -116,10 +117,10 @@ def detect_task_name(calling_file: Optional[str], calling_module: Optional[str]) def compute_search_path_dir( - calling_file: Optional[str], - calling_module: Optional[str], - config_path: Optional[str], -) -> Optional[str]: + calling_file: str | None, + calling_module: str | None, + config_path: str | None, +) -> str | None: if config_path is not None: if os.path.isabs(config_path): return config_path @@ -172,9 +173,7 @@ def is_under_debugger() -> bool: frames = inspect.stack() if len(frames) >= 3: filename = frames[-3].filename - if filename.endswith("/pdb.py"): - return True - elif filename.endswith("/pydevd.py"): + if filename.endswith(("/pdb.py", "/pydevd.py")): return True # unknown debugging will sometimes set sys.trace @@ -182,15 +181,15 @@ def is_under_debugger() -> bool: def create_automatic_config_search_path( - calling_file: Optional[str], - calling_module: Optional[str], - config_path: Optional[str], + calling_file: str | None, + calling_module: str | None, + config_path: str | None, ) -> ConfigSearchPath: search_path_dir = compute_search_path_dir(calling_file, calling_module, config_path) return create_config_search_path(search_path_dir) -def create_config_search_path(search_path_dir: Optional[str]) -> ConfigSearchPath: +def create_config_search_path(search_path_dir: str | None) -> ConfigSearchPath: from lerna.core.plugins import Plugins from lerna.plugins.search_path_plugin import SearchPathPlugin @@ -220,7 +219,7 @@ def run_and_report(func: Any) -> Any: return func() except Exception as ex: if _is_env_set("HYDRA_FULL_ERROR") or is_under_debugger(): - raise ex + raise else: try: if isinstance(ex, CompactHydraException): @@ -253,7 +252,7 @@ def run_and_report(func: Any) -> Any: sys.exit(1) # strip OmegaConf frames from bottom of stack - end: Optional[TracebackType] = tb + end: TracebackType | None = tb num_frames = 0 while end is not None: frame = end.tb_frame @@ -267,9 +266,9 @@ def run_and_report(func: Any) -> Any: @dataclass class FakeTracebackType: tb_next: Any = None # Optional["FakeTracebackType"] - tb_frame: Optional[FrameType] = None - tb_lasti: Optional[int] = None - tb_lineno: Optional[int] = None + tb_frame: FrameType | None = None + tb_lasti: int | None = None + tb_lineno: int | None = None iter_tb = tb final_tb = FakeTracebackType() @@ -290,7 +289,7 @@ class FakeTracebackType: traceback.print_exception(None, value=ex, tb=final_tb) # type: ignore sys.stderr.write("\nSet the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.\n") - except Exception as ex2: + except Exception as ex2: # noqa: BLE001 sys.stderr.write("An error occurred during Hydra's exception formatting:" + os.linesep + repr(ex2) + os.linesep) raise ex sys.exit(1) @@ -300,8 +299,8 @@ def _run_hydra( args: argparse.Namespace, args_parser: argparse.ArgumentParser, task_function: TaskFunction, - config_path: Optional[str], - config_name: Optional[str], + config_path: str | None, + config_name: str | None, caller_stack_depth: int = 2, ) -> None: from lerna.core.global_hydra import GlobalHydra @@ -397,11 +396,11 @@ def add_conf_dir() -> None: def _run_app( run: bool, multirun: bool, - mode: Optional[RunMode], + mode: RunMode | None, hydra: Any, - config_name: Optional[str], + config_name: str | None, task_function: TaskFunction, - overrides: List[str], + overrides: list[str], ) -> None: if mode is None: if run: @@ -451,7 +450,7 @@ def _get_completion_help() -> str: from lerna.plugins.completion_plugin import CompletionPlugin completion_plugins = Plugins.instance().discover(CompletionPlugin) - completion_info: List[str] = [] + completion_info: list[str] = [] for plugin_cls in completion_plugins: assert issubclass(plugin_cls, CompletionPlugin) for cmd in ["install", "uninstall"]: @@ -573,15 +572,15 @@ def __contains__(self, item: str) -> bool: return parser -def get_args(args: Optional[Sequence[str]] = None) -> Any: +def get_args(args: Sequence[str] | None = None) -> Any: return get_args_parser().parse_args(args=args) -def get_column_widths(matrix: List[List[str]]) -> List[int]: +def get_column_widths(matrix: list[list[str]]) -> list[int]: num_cols = 0 for row in matrix: num_cols = max(num_cols, len(row)) - widths: List[int] = [0] * num_cols + widths: list[int] = [0] * num_cols for row in matrix: for idx, col in enumerate(row): widths[idx] = max(widths[idx], len(col)) @@ -609,7 +608,7 @@ def _locate(path: str) -> Any: try: obj = import_module(part0) except Exception as exc_import: - raise ImportError(f"Error loading '{path}':\n{repr(exc_import)}" + f"\nAre you sure that module '{part0}' is installed?") from exc_import + raise ImportError(f"Error loading '{path}':\n{exc_import!r}" + f"\nAre you sure that module '{part0}' is installed?") from exc_import for m in range(1, len(parts)): part = parts[m] try: @@ -623,12 +622,12 @@ def _locate(path: str) -> Any: continue except ModuleNotFoundError as exc_import: raise ImportError( - f"Error loading '{path}':\n{repr(exc_import)}" + f"\nAre you sure that '{part}' is importable from module '{parent_dotpath}'?" + f"Error loading '{path}':\n{exc_import!r}" + f"\nAre you sure that '{part}' is importable from module '{parent_dotpath}'?" ) from exc_import except Exception as exc_import: - raise ImportError(f"Error loading '{path}':\n{repr(exc_import)}") from exc_import + raise ImportError(f"Error loading '{path}':\n{exc_import!r}") from exc_import raise ImportError( - f"Error loading '{path}':\n{repr(exc_attr)}" + f"\nAre you sure that '{part}' is an attribute of '{parent_dotpath}'?" + f"Error loading '{path}':\n{exc_attr!r}" + f"\nAre you sure that '{part}' is an attribute of '{parent_dotpath}'?" ) from exc_attr return obj diff --git a/lerna/compose.py b/lerna/compose.py index 58e2581..d03c27f 100644 --- a/lerna/compose.py +++ b/lerna/compose.py @@ -1,6 +1,5 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from textwrap import dedent -from typing import List, Optional from omegaconf import DictConfig, OmegaConf, open_dict @@ -12,10 +11,10 @@ def compose( - config_name: Optional[str] = None, - overrides: Optional[List[str]] = None, + config_name: str | None = None, + overrides: list[str] | None = None, return_hydra_config: bool = False, - strict: Optional[bool] = None, + strict: bool | None = None, ) -> DictConfig: """ :param config_name: the name of the config @@ -42,10 +41,9 @@ def compose( ) assert isinstance(cfg, DictConfig) - if not return_hydra_config: - if "hydra" in cfg: - with open_dict(cfg): - del cfg["hydra"] + if not return_hydra_config and "hydra" in cfg: + with open_dict(cfg): + del cfg["hydra"] if strict is not None: if version.base_at_least("1.2"): diff --git a/lerna/conf/__init__.py b/lerna/conf/__init__.py index f2fdc7b..dcb33b3 100644 --- a/lerna/conf/__init__.py +++ b/lerna/conf/__init__.py @@ -1,6 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import Any from omegaconf import MISSING @@ -36,9 +36,9 @@ class SweepDir: @dataclass class OverridesConf: # Overrides for the hydra configuration - hydra: List[str] = field(default_factory=lambda: []) + hydra: list[str] = field(default_factory=list) # Overrides for the task configuration - task: List[str] = field(default_factory=lambda: []) + task: list[str] = field(default_factory=list) # job runtime information will be populated here @@ -49,7 +49,7 @@ class JobConf: # Change current working dir to the output dir. # Will be non-optional and default to False in Hydra 1.3 - chdir: Optional[bool] = None + chdir: bool | None = None # Populated automatically by Hydra. # Concatenation of job overrides that can be used as a part @@ -64,12 +64,12 @@ class JobConf: num: int = MISSING # The config name used by the job - config_name: Optional[str] = MISSING + config_name: str | None = MISSING # Environment variables to set remotely - env_set: Dict[str, str] = field(default_factory=dict) + env_set: dict[str, str] = field(default_factory=dict) # Environment variables to copy from the launching machine - env_copy: List[str] = field(default_factory=list) + env_copy: list[str] = field(default_factory=list) # Job config @dataclass @@ -79,7 +79,7 @@ class JobConfig: class OverrideDirname: kv_sep: str = "=" item_sep: str = "," - exclude_keys: List[str] = field(default_factory=list) + exclude_keys: list[str] = field(default_factory=list) override_dirname: OverrideDirname = field(default_factory=OverrideDirname) @@ -98,17 +98,17 @@ class RuntimeConf: version: str = MISSING version_base: str = MISSING cwd: str = MISSING - config_sources: List[ConfigSourceInfo] = MISSING + config_sources: list[ConfigSourceInfo] = MISSING output_dir: str = MISSING # Composition choices dictionary # Ideally, the value type would be Union[str, List[str], None] - choices: Dict[str, Any] = field(default_factory=lambda: {}) + choices: dict[str, Any] = field(default_factory=dict) @dataclass class HydraConf: - defaults: List[Any] = field( + defaults: list[Any] = field( default_factory=lambda: [ {"output": "default"}, {"launcher": "basic"}, @@ -123,26 +123,26 @@ class HydraConf: ] ) - mode: Optional[RunMode] = None + mode: RunMode | None = None # Elements to append to the config search path. # Note: This can only be configured in the primary config. - searchpath: List[str] = field(default_factory=list) + searchpath: list[str] = field(default_factory=list) # Normal run output configuration run: RunDir = field(default_factory=RunDir) # Multi-run output configuration sweep: SweepDir = field(default_factory=SweepDir) # Logging configuration for Hydra - hydra_logging: Dict[str, Any] = MISSING + hydra_logging: dict[str, Any] = MISSING # Logging configuration for the job - job_logging: Dict[str, Any] = MISSING + job_logging: dict[str, Any] = MISSING # Sweeper configuration sweeper: Any = MISSING # Launcher configuration launcher: Any = MISSING # Callbacks configuration - callbacks: Dict[str, Any] = field(default_factory=dict) + callbacks: dict[str, Any] = field(default_factory=dict) # Program Help template help: HelpConf = field(default_factory=HelpConf) @@ -153,7 +153,7 @@ class HydraConf: # E.g., hydra.yaml, overrides.yaml will go here. Useful for debugging # and extra context when looking at past runs. # Setting to None will prevent the creation of the output subdir. - output_subdir: Optional[str] = ".hydra" + output_subdir: str | None = ".hydra" # Those lists will contain runtime overrides overrides: OverridesConf = field(default_factory=OverridesConf) diff --git a/lerna/core/config_loader.py b/lerna/core/config_loader.py index b214e5a..c5d01f1 100644 --- a/lerna/core/config_loader.py +++ b/lerna/core/config_loader.py @@ -1,6 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from abc import ABC, abstractmethod -from typing import Any, List, Optional +from typing import Any from omegaconf import DictConfig @@ -18,38 +18,38 @@ class ConfigLoader(ABC): @abstractmethod def load_configuration( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode, from_shell: bool = True, validate_sweep_overrides: bool = True, ) -> DictConfig: ... @abstractmethod - def load_sweep_config(self, master_config: DictConfig, sweep_overrides: List[str]) -> DictConfig: ... + def load_sweep_config(self, master_config: DictConfig, sweep_overrides: list[str]) -> DictConfig: ... @abstractmethod def get_search_path(self) -> ConfigSearchPath: ... @abstractmethod - def get_sources(self) -> List[ConfigSource]: ... + def get_sources(self) -> list[ConfigSource]: ... @abstractmethod - def list_groups(self, parent_name: str) -> List[str]: ... + def list_groups(self, parent_name: str) -> list[str]: ... @abstractmethod def get_group_options( self, group_name: str, - results_filter: Optional[ObjectType] = ObjectType.CONFIG, - config_name: Optional[str] = None, - overrides: Optional[List[str]] = None, - ) -> List[str]: ... + results_filter: ObjectType | None = ObjectType.CONFIG, + config_name: str | None = None, + overrides: list[str] | None = None, + ) -> list[str]: ... @abstractmethod def compute_defaults_list( self, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], run_mode: RunMode, ) -> Any: ... diff --git a/lerna/core/config_search_path.py b/lerna/core/config_search_path.py index bb19285..37d384a 100644 --- a/lerna/core/config_search_path.py +++ b/lerna/core/config_search_path.py @@ -1,7 +1,7 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from abc import ABC, abstractmethod +from collections.abc import MutableSequence from dataclasses import dataclass -from typing import MutableSequence, Optional, Union class SearchPathElement: @@ -22,8 +22,8 @@ class SearchPathQuery: Used in append and prepend API """ - provider: Optional[str] = None - path: Optional[str] = None + provider: str | None = None + path: str | None = None class ConfigSearchPath(ABC): @@ -31,7 +31,7 @@ class ConfigSearchPath(ABC): def get_path(self) -> MutableSequence[SearchPathElement]: ... @abstractmethod - def append(self, provider: str, path: str, anchor: Optional[SearchPathQuery] = None) -> None: + def append(self, provider: str, path: str, anchor: SearchPathQuery | None = None) -> None: """ Appends to the search path. Note, this currently only takes effect if called before the ConfigRepository is instantiated. @@ -42,14 +42,12 @@ def append(self, provider: str, path: str, anchor: Optional[SearchPathQuery] = N :param anchor: Optional anchor query to append after """ - ... - @abstractmethod def prepend( self, provider: str, path: str, - anchor: Optional[Union[SearchPathQuery, str]] = None, + anchor: SearchPathQuery | str | None = None, ) -> None: """ Prepends to the search path. @@ -60,5 +58,3 @@ def prepend( :param path: path element, can be a file system path or a package path (For example pkg://lerna.conf) :param anchor: Optional anchor query to prepend before """ - - ... diff --git a/lerna/core/config_store.py b/lerna/core/config_store.py index d06cbed..d4bb892 100644 --- a/lerna/core/config_store.py +++ b/lerna/core/config_store.py @@ -1,7 +1,7 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import copy from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from typing import Any from omegaconf import DictConfig, OmegaConf @@ -24,28 +24,28 @@ class ConfigStoreWithProvider: def __init__(self, provider: str) -> None: self.provider = provider - def __enter__(self) -> "ConfigStoreWithProvider": + def __enter__(self) -> "ConfigStoreWithProvider": # noqa: PYI034 return self def store( self, name: str, node: Any, - group: Optional[str] = None, - package: Optional[str] = None, + group: str | None = None, + package: str | None = None, ) -> None: ConfigStore.instance().store(group=group, name=name, node=node, package=package, provider=self.provider) - def __exit__(self, exc_type: Any, exc_value: Any, exc_traceback: Any) -> Any: ... + def __exit__(self, exc_type: object, exc_value: object, exc_traceback: object) -> Any: ... @dataclass class ConfigNode: name: str node: DictConfig - group: Optional[str] - package: Optional[str] - provider: Optional[str] + group: str | None + package: str | None + provider: str | None class ConfigStore(metaclass=Singleton): @@ -53,7 +53,7 @@ class ConfigStore(metaclass=Singleton): def instance(*args: Any, **kwargs: Any) -> "ConfigStore": return Singleton.instance(ConfigStore, *args, **kwargs) # type: ignore - repo: Dict[str, Any] + repo: dict[str, Any] _rust_store: Any def __init__(self) -> None: @@ -64,13 +64,13 @@ def __init__(self) -> None: else: self._rust_store = None - def __getstate__(self) -> Dict[str, Any]: + def __getstate__(self) -> dict[str, Any]: # Don't include _rust_store in pickle state state = self.__dict__.copy() state["_rust_store"] = None # Rust store will be recreated return state - def __setstate__(self, state: Dict[str, Any]) -> None: + def __setstate__(self, state: dict[str, Any]) -> None: self.__dict__.update(state) # Recreate Rust store on unpickle if _RUST_AVAILABLE: @@ -82,9 +82,9 @@ def store( self, name: str, node: Any, - group: Optional[str] = None, - package: Optional[str] = None, - provider: Optional[str] = None, + group: str | None = None, + package: str | None = None, + provider: str | None = None, ) -> None: """ Stores a config node into the repository @@ -167,7 +167,7 @@ def get_type(self, path: str) -> ObjectType: else: return ObjectType.CONFIG - def list(self, path: str) -> List[str]: + def list(self, path: str) -> list[str]: d = self._open(path) if d is None: raise OSError(f"Path not found {path}") diff --git a/lerna/core/default_element.py b/lerna/core/default_element.py index 83db5c7..6c68704 100644 --- a/lerna/core/default_element.py +++ b/lerna/core/default_element.py @@ -1,8 +1,9 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import re from dataclasses import dataclass, field +from re import Pattern from textwrap import dedent -from typing import List, Optional, Pattern, Union +from typing import Optional, Union from omegaconf import AnyNode, DictConfig, OmegaConf from omegaconf.errors import InterpolationResolutionError @@ -27,7 +28,7 @@ def _normalize_path(path: str) -> str: return path parts = path.split("/") - result: List[str] = [] + result: list[str] = [] for part in parts: if part == "." or part == "": @@ -46,13 +47,13 @@ def _normalize_path(path: str) -> str: @dataclass class ResultDefault: - config_path: Optional[str] = None - parent: Optional[str] = None - package: Optional[str] = None + config_path: str | None = None + parent: str | None = None + package: str | None = None is_self: bool = False primary: bool = field(default=False, compare=False) - override_key: Optional[str] = field(default=None, compare=False) + override_key: str | None = field(default=None, compare=False) def __repr__(self) -> str: attrs = [] @@ -75,26 +76,25 @@ def __repr__(self) -> str: @dataclass class InputDefault: - package: Optional[str] = None - parent_base_dir: Optional[str] = field(default=None, compare=False, repr=False) - parent_package: Optional[str] = field(default=None, compare=False, repr=False) - package_header: Optional[str] = field(default=None, compare=False) + package: str | None = None + parent_base_dir: str | None = field(default=None, compare=False, repr=False) + parent_package: str | None = field(default=None, compare=False, repr=False) + package_header: str | None = field(default=None, compare=False) primary: bool = field(default=False, compare=False) def is_self(self) -> bool: raise NotImplementedError() - def update_parent(self, parent_base_dir: Optional[str], parent_package: Optional[str]) -> None: + def update_parent(self, parent_base_dir: str | None, parent_package: str | None) -> None: assert self.parent_package is None or self.parent_package == parent_package assert self.parent_base_dir is None or self.parent_base_dir == parent_base_dir self.parent_base_dir = parent_base_dir self.parent_package = parent_package - if self.package is not None: - if "_group_" in self.package: - pkg = self.package - resolved = pkg.replace("_group_", self.get_default_package()) - self.package = f"_global_.{resolved}" + if self.package is not None and "_group_" in self.package: + pkg = self.package + resolved = pkg.replace("_group_", self.get_default_package()) + self.package = f"_global_.{resolved}" def is_optional(self) -> bool: raise NotImplementedError() @@ -118,16 +118,16 @@ def get_final_package(self, default_to_package_header: bool = True) -> str: def _relative_group_path(self) -> str: raise NotImplementedError() - def get_name(self) -> Optional[str]: + def get_name(self) -> str | None: raise NotImplementedError() - def _get_attributes(self) -> List[str]: + def _get_attributes(self) -> list[str]: raise NotImplementedError() - def _get_flags(self) -> List[str]: + def _get_flags(self) -> list[str]: raise NotImplementedError() - def _get_parent_package(self) -> Optional[str]: + def _get_parent_package(self) -> str | None: ret = self.__dict__["parent_package"] assert ret is None or isinstance(ret, str) return ret @@ -141,7 +141,7 @@ def is_deleted(self) -> bool: else: return False - def set_package_header(self, package_header: Optional[str]) -> None: + def set_package_header(self, package_header: str | None) -> None: assert self.__dict__["package_header"] is None if package_header is None: @@ -174,12 +174,12 @@ def set_package_header(self, package_header: Optional[str]) -> None: package_header = package_header.replace("_group_", self.get_default_package()) self.__dict__["package_header"] = package_header - def get_package_header(self) -> Optional[str]: + def get_package_header(self) -> str | None: ret = self.__dict__["package_header"] assert ret is None or isinstance(ret, str) return ret - def get_package(self, default_to_package_header: bool = True) -> Optional[str]: + def get_package(self, default_to_package_header: bool = True) -> str | None: if self.__dict__["package"] is None and default_to_package_header: ret = self.__dict__["package_header"] else: @@ -189,21 +189,18 @@ def get_package(self, default_to_package_header: bool = True) -> Optional[str]: def _get_final_package( self, - parent_package: Optional[str], - package: Optional[str], - name: Optional[str], + parent_package: str | None, + package: str | None, + name: str | None, ) -> str: assert parent_package is not None if package is None: package = self._relative_group_path().replace("/", ".") - if isinstance(name, str): - # name computation should be deferred to after the final config group choice is done - - if not version.base_at_least("1.2"): - if "_name_" in package: - package = package.replace("_name_", name) + # name computation should be deferred to after the final config group choice is done + if isinstance(name, str) and not version.base_at_least("1.2") and "_name_" in package: + package = package.replace("_name_", name) if parent_package == "": ret = package @@ -257,7 +254,7 @@ def is_missing(self) -> bool: def resolve_interpolation(self, known_choices: DictConfig) -> None: raise NotImplementedError() - def _resolve_interpolation_impl(self, known_choices: DictConfig, val: Optional[str]) -> str: + def _resolve_interpolation_impl(self, known_choices: DictConfig, val: str | None) -> str: node = OmegaConf.create({"_dummy_": val}) node._set_parent(known_choices) try: @@ -265,7 +262,7 @@ def _resolve_interpolation_impl(self, known_choices: DictConfig, val: Optional[s assert isinstance(ret, str) return ret except InterpolationResolutionError: - options = [x for x in known_choices.keys() if x != "defaults" and isinstance(x, str)] + options = [x for x in known_choices if x != "defaults" and isinstance(x, str)] if len(options) > 0: options_str = ", ".join(options) msg = f"Error resolving interpolation '{val}', possible interpolation keys: {options_str}" @@ -322,10 +319,10 @@ def get_name(self) -> str: def is_missing(self) -> bool: return False - def _get_attributes(self) -> List[str]: + def _get_attributes(self) -> list[str]: raise NotImplementedError() - def _get_flags(self) -> List[str]: + def _get_flags(self) -> list[str]: raise NotImplementedError() def __repr__(self) -> str: @@ -343,9 +340,9 @@ def is_external_append(self) -> bool: @dataclass(repr=False) class ConfigDefault(InputDefault): - path: Optional[str] = None + path: str | None = None optional: bool = False - deleted: Optional[bool] = None + deleted: bool | None = None def __post_init__(self) -> None: if self.is_self() and self.package is not None: @@ -390,7 +387,7 @@ def get_group_path(self) -> str: # Normalize paths with .. segments (Hydra #2878) return _normalize_path(result) - def get_name(self) -> Optional[str]: + def get_name(self) -> str | None: assert self.path is not None idx = self.path.rfind("/") if idx == -1: @@ -439,10 +436,10 @@ def _relative_group_path(self) -> str: else: return path[0:idx] - def _get_attributes(self) -> List[str]: + def _get_attributes(self) -> list[str]: return ["path", "package", "deleted"] - def _get_flags(self) -> List[str]: + def _get_flags(self) -> list[str]: return ["optional"] def is_interpolation(self) -> bool: @@ -470,13 +467,13 @@ def is_external_append(self) -> bool: @dataclass(repr=False) class GroupDefault(InputDefault): # config group name if present - group: Optional[str] = None + group: str | None = None # config file name - value: Optional[Union[str, List[str]]] = None + value: str | list[str] | None = None optional: bool = False override: bool = False - deleted: Optional[bool] = None + deleted: bool | None = None config_name_overridden: bool = field(default=False, compare=False, repr=False) # True if this item was added using +foo=bar from the external overrides @@ -529,11 +526,11 @@ def is_name(self) -> bool: def is_options(self) -> bool: return isinstance(self.value, list) - def get_name(self) -> Optional[str]: + def get_name(self) -> str | None: assert self.value is None or isinstance(self.value, str) return self.value - def get_options(self) -> List[str]: + def get_options(self) -> list[str]: assert isinstance(self.value, list) return self.value @@ -552,10 +549,10 @@ def _relative_group_path(self) -> str: else: return self.group - def _get_attributes(self) -> List[str]: + def _get_attributes(self) -> list[str]: return ["group", "value", "package", "deleted"] - def _get_flags(self) -> List[str]: + def _get_flags(self) -> list[str]: return ["optional", "override"] def is_interpolation(self) -> bool: @@ -610,8 +607,8 @@ def is_external_append(self) -> bool: @dataclass class PatchDefault(InputDefault): - operations: List[str] = field(default_factory=list) - package_scope: Optional[str] = None + operations: list[str] = field(default_factory=list) + package_scope: str | None = None def is_self(self) -> bool: return False @@ -625,7 +622,7 @@ def get_group_path(self) -> str: def get_config_path(self) -> str: return "_patch_" - def get_name(self) -> Optional[str]: + def get_name(self) -> str | None: return None def get_final_package(self, default_to_package_header: bool = True) -> str: @@ -635,10 +632,10 @@ def get_final_package(self, default_to_package_header: bool = True) -> str: def _relative_group_path(self) -> str: return "" - def _get_attributes(self) -> List[str]: + def _get_attributes(self) -> list[str]: return ["operations", "package_scope"] - def _get_flags(self) -> List[str]: + def _get_flags(self) -> list[str]: return [] def is_interpolation(self) -> bool: @@ -663,7 +660,7 @@ def is_external_append(self) -> bool: @dataclass class DefaultsTreeNode: node: InputDefault - children: Optional[List[Union["DefaultsTreeNode", InputDefault]]] = None + children: list[Union["DefaultsTreeNode", InputDefault]] | None = None parent: Optional["DefaultsTreeNode"] = field( default=None, @@ -671,7 +668,7 @@ class DefaultsTreeNode: compare=False, ) - def parent_node(self) -> Optional[InputDefault]: + def parent_node(self) -> InputDefault | None: if self.parent is None: return None else: diff --git a/lerna/core/global_hydra.py b/lerna/core/global_hydra.py index 7903174..5618899 100644 --- a/lerna/core/global_hydra.py +++ b/lerna/core/global_hydra.py @@ -1,5 +1,5 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -from typing import Any, Optional +from typing import Any from lerna._internal.hydra import Hydra from lerna.core.config_loader import ConfigLoader @@ -8,7 +8,7 @@ class GlobalHydra(metaclass=Singleton): def __init__(self) -> None: - self.hydra: Optional[Hydra] = None + self.hydra: Hydra | None = None def initialize(self, hydra: "Hydra") -> None: assert isinstance(hydra, Hydra), f"Unexpected Hydra type : {type(hydra)}" diff --git a/lerna/core/hydra_config.py b/lerna/core/hydra_config.py index 893998d..e208d5c 100644 --- a/lerna/core/hydra_config.py +++ b/lerna/core/hydra_config.py @@ -1,5 +1,5 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -from typing import Any, Optional +from typing import Any from omegaconf import DictConfig, OmegaConf @@ -9,7 +9,7 @@ class HydraConfig(metaclass=Singleton): def __init__(self) -> None: - self.cfg: Optional[HydraConf] = None + self.cfg: HydraConf | None = None def set_config(self, cfg: DictConfig) -> None: assert cfg is not None diff --git a/lerna/core/override_parser/overrides_parser.py b/lerna/core/override_parser/overrides_parser.py index 313911c..669d583 100644 --- a/lerna/core/override_parser/overrides_parser.py +++ b/lerna/core/override_parser/overrides_parser.py @@ -11,7 +11,7 @@ """ import sys -from typing import Any, List, Optional +from typing import Any from lerna._internal.grammar.functions import Functions from lerna.core.config_loader import ConfigLoader @@ -49,13 +49,13 @@ class OverridesParser: """ _rust_parser: Any = None - _functions: Optional[Functions] = None + _functions: Functions | None = None @classmethod def create( cls, - config_loader: Optional[ConfigLoader] = None, - searchpath: Optional[List[str]] = None, + config_loader: ConfigLoader | None = None, + searchpath: list[str] | None = None, ) -> "OverridesParser": """Create an OverridesParser instance. @@ -68,9 +68,9 @@ def create( def __init__( self, - functions: Optional[Functions] = None, - config_loader: Optional[ConfigLoader] = None, - searchpath: Optional[List[str]] = None, + functions: Functions | None = None, + config_loader: ConfigLoader | None = None, + searchpath: list[str] | None = None, ): self.config_loader = config_loader self.searchpath = searchpath @@ -165,8 +165,8 @@ def parse_override(self, s: str) -> Override: assert isinstance(ret, Override) return ret - def parse_overrides(self, overrides: List[str]) -> List[Override]: - ret: List[Override] = [] + def parse_overrides(self, overrides: list[str]) -> list[Override]: + ret: list[Override] = [] for idx, override in enumerate(overrides): try: parsed = self.parse_rule(override, "override") @@ -270,8 +270,8 @@ def _parse_list_operation(operation_str: str) -> ListOperationType: def _rust_dict_to_override( data: dict, - config_loader: Optional[ConfigLoader] = None, - searchpath: Optional[List[str]] = None, + config_loader: ConfigLoader | None = None, + searchpath: list[str] | None = None, ) -> Override: """Convert Rust parser output dict to Python Override object. @@ -303,7 +303,7 @@ def _rust_dict_to_override( # For DEL overrides without value, value_type should be None if data["value"] is None: - value_type: Optional[ValueType] = None + value_type: ValueType | None = None else: value_type = value_type_map.get(data["value_type"]) @@ -311,8 +311,8 @@ def _rust_dict_to_override( raw_value = _convert_rust_value(data["value"]) # Initialize list operation fields (only used for EXTEND_LIST type) - list_operation: Optional[ListOperationType] = None - list_index: Optional[int] = None + list_operation: ListOperationType | None = None + list_index: int | None = None if raw_value is None: value: Any = None diff --git a/lerna/core/override_parser/types.py b/lerna/core/override_parser/types.py index ae590fa..0545d44 100644 --- a/lerna/core/override_parser/types.py +++ b/lerna/core/override_parser/types.py @@ -1,12 +1,14 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import builtins import decimal import fnmatch +from collections.abc import Callable, Iterator from copy import copy from dataclasses import dataclass, field from enum import Enum from random import shuffle from textwrap import dedent -from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Union, cast +from typing import Any, cast from omegaconf import OmegaConf from omegaconf._utils import is_structured_config @@ -82,23 +84,23 @@ def with_quotes(self) -> str: @dataclass class Sweep: - tags: Set[str] = field(default_factory=set) + tags: set[str] = field(default_factory=set) @dataclass class ChoiceSweep(Sweep): # simple form: a,b,c # explicit form: choices(a,b,c) - list: List["ParsedElementType"] = field(default_factory=list) + list: builtins.list["ParsedElementType"] = field(default_factory=list) simple_form: bool = False shuffle: bool = False @dataclass class FloatRange: - start: Union[decimal.Decimal, float] - stop: Union[decimal.Decimal, float] - step: Union[decimal.Decimal, float] + start: decimal.Decimal | float + stop: decimal.Decimal | float + step: decimal.Decimal | float def __post_init__(self) -> None: self.start = decimal.Decimal(self.start) @@ -136,13 +138,13 @@ class RangeSweep(Sweep): Discrete range of numbers """ - start: Optional[Union[int, float]] = None - stop: Optional[Union[int, float]] = None - step: Union[int, float] = 1 + start: int | float | None = None + stop: int | float | None = None + step: int | float = 1 shuffle: bool = False - def range(self) -> Union[range, FloatRange]: + def range(self) -> range | FloatRange: assert self.start is not None assert self.stop is not None @@ -157,10 +159,10 @@ def range(self) -> Union[range, FloatRange]: @dataclass class IntervalSweep(Sweep): - start: Optional[float] = None - end: Optional[float] = None + start: float | None = None + end: float | None = None - def __eq__(self, other: Any) -> Any: + def __eq__(self, other: object) -> Any: if isinstance(other, IntervalSweep): eq = self.start == other.start and self.end == other.end and self.tags == other.tags @@ -176,8 +178,8 @@ def __eq__(self, other: Any) -> Any: # Ideally we would use List[ElementType] and Dict[str, ElementType] but Python does not seem # to support recursive type definitions. -ElementType = Union[str, int, float, bool, List[Any], Dict[str, Any]] -ParsedElementType = Optional[Union[ElementType, QuotedString]] +ElementType = str | int | float | bool | list[Any] | dict[str, Any] +ParsedElementType = ElementType | QuotedString | None TransformerType = Callable[[ParsedElementType], Any] @@ -213,15 +215,15 @@ class ListOperationType(Enum): class Key: # the config-group or config dot-path key_or_group: str - package: Optional[str] = None + package: str | None = None @dataclass class Glob: - include: List[str] = field(default_factory=list) - exclude: List[str] = field(default_factory=list) + include: list[str] = field(default_factory=list) + exclude: list[str] = field(default_factory=list) - def filter(self, names: List[str]) -> List[str]: + def filter(self, names: list[str]) -> list[str]: """Filter names based on include and exclude patterns.""" if _HAS_RUST: # Use Rust implementation for performance @@ -229,7 +231,7 @@ def filter(self, names: List[str]) -> List[str]: return rust_glob.filter(names) # Fallback to Python implementation - def match(s: str, globs: List[str]) -> bool: + def match(s: str, globs: list[str]) -> bool: for g in globs: if fnmatch.fnmatch(s, g): return True @@ -245,7 +247,7 @@ def match(s: str, globs: List[str]) -> bool: @dataclass class ListExtensionOverrideValue: - values: List["ParsedElementType"] + values: list["ParsedElementType"] class Transformer: @@ -275,29 +277,29 @@ class Override: key_or_group: str # The type of the value, None if there is no value - value_type: Optional[ValueType] + value_type: ValueType | None # The parsed value (component after the =). - _value: Union[ParsedElementType, ChoiceSweep, RangeSweep, IntervalSweep] + _value: ParsedElementType | ChoiceSweep | RangeSweep | IntervalSweep # Optional qualifying package - package: Optional[str] = None + package: str | None = None # Input line used to construct this - input_line: Optional[str] = None + input_line: str | None = None # Configs repo - config_loader: Optional[ConfigLoader] = None + config_loader: ConfigLoader | None = None # For EXTEND_LIST type: the specific list operation (defaults to APPEND) - list_operation: Optional[ListOperationType] = None + list_operation: ListOperationType | None = None # For INSERT and REMOVE_AT operations: the index - list_index: Optional[int] = None + list_index: int | None = None # Optional searchpath for glob sweeps (from hydra.searchpath config) # Used to ensure pkg:// sources are available when enumerating group options - searchpath: Optional[List[str]] = None + searchpath: list[str] | None = None def is_delete(self) -> bool: """ @@ -324,7 +326,7 @@ def is_list_extend(self) -> bool: return self.type == OverrideType.EXTEND_LIST @staticmethod - def _convert_value(value: ParsedElementType) -> Optional[ElementType]: + def _convert_value(value: ParsedElementType) -> ElementType | None: if isinstance(value, list): return [Override._convert_value(x) for x in value] elif isinstance(value, dict): @@ -341,7 +343,7 @@ def _convert_value(value: ParsedElementType) -> Optional[ElementType]: def value( self, - ) -> Optional[Union[ElementType, ChoiceSweep, RangeSweep, IntervalSweep]]: + ) -> ElementType | ChoiceSweep | RangeSweep | IntervalSweep | None: """ :return: the value. replaces Quoted strings by regular strings """ @@ -387,7 +389,7 @@ def sweep_iterator(self, transformer: TransformerType = Transformer.identity) -> # Build searchpath override list if searchpath is set # This ensures pkg:// sources from hydra.searchpath are available for glob sweeps - overrides: Optional[List[str]] = None + overrides: list[str] | None = None if self.searchpath: # Format as hydra.searchpath override import json @@ -434,7 +436,7 @@ def is_interval_sweep(self) -> bool: def is_hydra_override(self) -> bool: kog = self.key_or_group - return kog.startswith("hydra.") or kog.startswith("hydra/") + return kog.startswith(("hydra.", "hydra/")) def get_key_element(self) -> str: def get_key() -> str: @@ -507,13 +509,12 @@ def get_value_element_as_str(self, space_after_sep: bool = False) -> str: return Override._get_value_element_as_str(self._value, space_after_sep=space_after_sep) def validate(self) -> None: - if not version.base_at_least("1.2"): - if self.package is not None and "_name_" in self.package: - url = "https://hydra.cc/docs/1.2/upgrades/1.0_to_1.1/changes_to_package_header" - deprecation_warning( - message=dedent( - f"""\ + if not version.base_at_least("1.2") and self.package is not None and "_name_" in self.package: + url = "https://hydra.cc/docs/1.2/upgrades/1.0_to_1.1/changes_to_package_header" + deprecation_warning( + message=dedent( + f"""\ In override {self.input_line}: _name_ keyword is deprecated in packages, see {url} """ - ), - ) + ), + ) diff --git a/lerna/core/plugins.py b/lerna/core/plugins.py index 9cf2115..93adeff 100644 --- a/lerna/core/plugins.py +++ b/lerna/core/plugins.py @@ -8,8 +8,9 @@ import warnings from collections import defaultdict from dataclasses import dataclass, field +from importlib.metadata import entry_points from timeit import default_timer as timer -from typing import Any, Dict, List, Optional, Tuple, Type +from typing import Any from omegaconf import DictConfig @@ -24,12 +25,7 @@ from lerna.types import HydraContext, TaskFunction from lerna.utils import instantiate -if sys.version_info < (3, 10): - from importlib_metadata import entry_points -else: - from importlib.metadata import entry_points - -PLUGIN_TYPES: List[Type[Plugin]] = [ +PLUGIN_TYPES: list[type[Plugin]] = [ Plugin, ConfigSource, CompletionPlugin, @@ -43,7 +39,7 @@ class ScanStats: total_time: float = 0 total_modules_import_time: float = 0 - modules_import_time: Dict[str, float] = field(default_factory=dict) + modules_import_time: dict[str, float] = field(default_factory=dict) class Plugins(metaclass=Singleton): @@ -54,13 +50,13 @@ def instance(*args: Any, **kwargs: Any) -> "Plugins": return ret def __init__(self) -> None: - self.plugin_type_to_subclass_list: Dict[Type[Plugin], List[Type[Plugin]]] = {} - self.class_name_to_class: Dict[str, Type[Plugin]] = {} - self.stats: Optional[ScanStats] = None + self.plugin_type_to_subclass_list: dict[type[Plugin], list[type[Plugin]]] = {} + self.class_name_to_class: dict[str, type[Plugin]] = {} + self.stats: ScanStats | None = None self._initialize() def _initialize(self) -> None: - top_level: List[Any] = [] + top_level: list[Any] = [] core_plugins = importlib.import_module("lerna._internal.core_plugins") top_level.append(core_plugins) @@ -81,7 +77,7 @@ def _initialize(self) -> None: for clazz in scanned_plugins: self._register(clazz) - def register(self, clazz: Type[Plugin]) -> None: + def register(self, clazz: type[Plugin]) -> None: """ Call Plugins.instance().register(MyPlugin) to manually register a plugin class. """ @@ -89,12 +85,11 @@ def register(self, clazz: Type[Plugin]) -> None: raise ValueError("Not a valid Hydra Plugin") self._register(clazz) - def _register(self, clazz: Type[Plugin]) -> None: + def _register(self, clazz: type[Plugin]) -> None: assert _is_concrete_plugin_type(clazz) for plugin_type in PLUGIN_TYPES: - if issubclass(clazz, plugin_type): - if clazz not in self.plugin_type_to_subclass_list[plugin_type]: - self.plugin_type_to_subclass_list[plugin_type].append(clazz) + if issubclass(clazz, plugin_type) and clazz not in self.plugin_type_to_subclass_list[plugin_type]: + self.plugin_type_to_subclass_list[plugin_type].append(clazz) name = f"{clazz.__module__}.{clazz.__name__}" self.class_name_to_class[name] = clazz if issubclass(clazz, ConfigSource): @@ -113,25 +108,20 @@ def _instantiate(self, config: DictConfig) -> Plugin: # For plugins outside of lerna-core, the approved module is lerna_plugins or hydra_plugins. raise RuntimeError(f"Invalid plugin '{classname}': not in lerna_plugins or hydra_plugins package") - if classname not in self.class_name_to_class.keys(): + if classname not in self.class_name_to_class: raise RuntimeError(f"Unknown plugin class : '{classname}'") clazz = self.class_name_to_class[classname] plugin = instantiate(config=config, _target_=clazz) assert isinstance(plugin, Plugin) except ImportError as e: - raise ImportError(f"Could not instantiate plugin {classname} : {str(e)}\n\n\tIS THE PLUGIN INSTALLED?\n\n") + raise ImportError(f"Could not instantiate plugin {classname} : {e!s}\n\n\tIS THE PLUGIN INSTALLED?\n\n") return plugin @staticmethod def is_in_toplevel_plugins_module(clazz: str) -> bool: - return ( - clazz.startswith("lerna_plugins.") - or clazz.startswith("hydra_plugins.") - or clazz.startswith("lerna._internal.core_plugins.") - or clazz.startswith("lerna._internal.core_plugins.") - ) + return clazz.startswith(("lerna_plugins.", "hydra_plugins.", "lerna._internal.core_plugins.", "lerna._internal.core_plugins.")) def instantiate_sweeper( self, @@ -165,12 +155,12 @@ def instantiate_launcher( @staticmethod def _scan_all_plugins( - modules: List[Any], - ) -> Tuple[List[Type[Plugin]], ScanStats]: + modules: list[Any], + ) -> tuple[list[type[Plugin]], ScanStats]: stats = ScanStats() stats.total_time = timer() - scanned_plugins: List[Type[Plugin]] = [] + scanned_plugins: list[type[Plugin]] = [] for mdl in modules: for importer, modname, ispkg in pkgutil.walk_packages(path=mdl.__path__, prefix=mdl.__name__ + ".", onerror=lambda x: None): @@ -184,21 +174,16 @@ def _scan_all_plugins( import_time = timer() with warnings.catch_warnings(record=True) as recorded_warnings: - if sys.version_info < (3, 10): - m = importer.find_module(modname) # type: ignore - assert m is not None - loaded_mod = m.load_module(modname) + spec = importer.find_spec(modname) # type: ignore[call-arg] + assert spec is not None + if modname in sys.modules: + loaded_mod = sys.modules[modname] else: - spec = importer.find_spec(modname) # type: ignore[call-arg] - assert spec is not None - if modname in sys.modules: - loaded_mod = sys.modules[modname] - else: - loaded_mod = importlib.util.module_from_spec(spec) - if loaded_mod is not None: - assert spec.loader is not None - spec.loader.exec_module(loaded_mod) - sys.modules[modname] = loaded_mod + loaded_mod = importlib.util.module_from_spec(spec) + if loaded_mod is not None: + assert spec.loader is not None + spec.loader.exec_module(loaded_mod) + sys.modules[modname] = loaded_mod import_time = timer() - import_time if len(recorded_warnings) > 0: @@ -235,30 +220,28 @@ def _scan_all_plugins( stats.total_time = timer() - stats.total_time return scanned_plugins, stats - def get_stats(self) -> Optional[ScanStats]: + def get_stats(self) -> ScanStats | None: return self.stats - def discover(self, plugin_type: Optional[Type[Plugin]] = None) -> List[Type[Plugin]]: + def discover(self, plugin_type: type[Plugin] | None = None) -> list[type[Plugin]]: """ :param plugin_type: class of plugin to discover, None for all :return: a list of plugins implementing the plugin type (or all if plugin type is None) """ Plugins.check_usage(self) - ret: List[Type[Plugin]] = [] if plugin_type is None: plugin_type = Plugin assert issubclass(plugin_type, Plugin) if plugin_type not in self.plugin_type_to_subclass_list: return [] - for clazz in self.plugin_type_to_subclass_list[plugin_type]: - ret.append(clazz) - - return ret + return self.plugin_type_to_subclass_list[plugin_type].copy() @staticmethod def check_usage(self_: Any) -> None: if not isinstance(self_, Plugins): - raise ValueError(f"Plugins is now a Singleton. usage: Plugins.instance().{inspect.stack()[1][3]}(...)") + raise ValueError( # noqa: TRY004 + f"Plugins is now a Singleton. usage: Plugins.instance().{inspect.stack()[1][3]}(...)" + ) def _is_concrete_plugin_type(obj: Any) -> bool: @@ -280,7 +263,7 @@ def _is_pkg_path_available(pkg_path: str) -> bool: return False -def _scan_entrypoint_search_path_plugins() -> List[Type[Plugin]]: +def _scan_entrypoint_search_path_plugins() -> list[type[Plugin]]: """ Discover SearchPathPlugin classes registered via the ``hydra.lernaplugins`` and ``lerna.plugins`` entry-point groups so that they are available when @@ -299,9 +282,9 @@ def _scan_entrypoint_search_path_plugins() -> List[Type[Plugin]]: intentionally skipped here because the hydra bridge plugin (``hydra_plugins.lerna.searchpath``) already handles them. """ - scanned_plugins: List[Type[Plugin]] = [] + scanned_plugins: list[type[Plugin]] = [] - discovered: List[Any] = [] + discovered: list[Any] = [] for group in ("hydra.lernaplugins", "lerna.plugins"): try: discovered.extend(entry_points(group=group)) diff --git a/lerna/core/singleton.py b/lerna/core/singleton.py index 2b39451..cc2c44e 100644 --- a/lerna/core/singleton.py +++ b/lerna/core/singleton.py @@ -1,12 +1,12 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from copy import deepcopy -from typing import Any, Dict +from typing import Any, ClassVar from omegaconf.basecontainer import BaseContainer class Singleton(type): - _instances: Dict[type, "Singleton"] = {} + _instances: ClassVar[dict[type, "Singleton"]] = {} def __call__(cls, *args: Any, **kwargs: Any) -> Any: if cls not in cls._instances: diff --git a/lerna/core/utils.py b/lerna/core/utils.py index 5df7a99..0c5669c 100644 --- a/lerna/core/utils.py +++ b/lerna/core/utils.py @@ -4,6 +4,7 @@ import os import re import sys +from collections.abc import Sequence from contextlib import contextmanager from dataclasses import dataclass from datetime import datetime @@ -11,7 +12,7 @@ from os.path import splitext from pathlib import Path from textwrap import dedent -from typing import Any, Dict, Optional, Sequence, Union, cast +from typing import Any, cast from omegaconf import DictConfig, OmegaConf, open_dict, read_write @@ -42,11 +43,11 @@ def simple_stdout_log_config(level: int = logging.INFO) -> None: def configure_log( log_config: DictConfig, - verbose_config: Union[bool, str, Sequence[str]] = False, + verbose_config: bool | str | Sequence[str] = False, ) -> None: assert isinstance(verbose_config, (bool, str)) or OmegaConf.is_list(verbose_config) if log_config is not None: - conf: Dict[str, Any] = OmegaConf.to_container( # type: ignore + conf: dict[str, Any] = OmegaConf.to_container( # type: ignore log_config, resolve=True ) if conf["root"] is not None: @@ -88,7 +89,7 @@ def filter_overrides(overrides: Sequence[str]) -> Sequence[str]: return [x for x in overrides if not x.startswith("hydra.")] -def _check_hydra_context(hydra_context: Optional[HydraContext]) -> None: +def _check_hydra_context(hydra_context: HydraContext | None) -> None: if hydra_context is None: # hydra_context is required as of Hydra 1.2. # We can remove this check in Hydra 1.3. @@ -105,7 +106,7 @@ def run_job( task_function: TaskFunction, config: DictConfig, job_dir_key: str, - job_subdir_key: Optional[str], + job_subdir_key: str | None, hydra_context: HydraContext, configure_logging: bool = True, ) -> "JobReturn": @@ -126,9 +127,8 @@ def run_job( subdir = str(OmegaConf.select(config, job_subdir_key)) output_dir = os.path.join(output_dir, subdir) - with read_write(config.hydra.runtime): - with open_dict(config.hydra.runtime): - config.hydra.runtime.output_dir = os.path.abspath(output_dir) + with read_write(config.hydra.runtime), open_dict(config.hydra.runtime): + config.hydra.runtime.output_dir = os.path.abspath(output_dir) # update Hydra config HydraConfig.instance().set_config(config) @@ -136,9 +136,8 @@ def run_job( try: ret = JobReturn() task_cfg = copy.deepcopy(config) - with read_write(task_cfg): - with open_dict(task_cfg): - del task_cfg["hydra"] + with read_write(task_cfg), open_dict(task_cfg): + del task_cfg["hydra"] ret.cfg = task_cfg hydra_cfg = copy.deepcopy(HydraConfig.instance().cfg) @@ -152,9 +151,8 @@ def run_job( _chdir = hydra_cfg.hydra.job.chdir - if _chdir is None: - if version.base_at_least("1.2"): - _chdir = False + if _chdir is None and version.base_at_least("1.2"): + _chdir = False if _chdir is None: url = "https://hydra.cc/docs/1.2/upgrades/1.1_to_1.2/changes_to_job_working_dir/" @@ -188,7 +186,7 @@ def run_job( try: ret.return_value = task_function(task_cfg) ret.status = JobStatus.COMPLETED - except Exception as e: + except Exception as e: # noqa: BLE001 ret.return_value = e ret.status = JobStatus.FAILED @@ -218,7 +216,7 @@ def setup_globals() -> None: # please add documentation when you add a new resolver OmegaConf.register_new_resolver( "now", - lambda pattern: datetime.now().strftime(pattern), + lambda pattern: datetime.now().strftime(pattern), # noqa: DTZ005 use_cache=True, replace=True, ) @@ -245,11 +243,11 @@ class JobStatus(Enum): @dataclass class JobReturn: - overrides: Optional[Sequence[str]] = None - cfg: Optional[DictConfig] = None - hydra_cfg: Optional[DictConfig] = None - working_dir: Optional[str] = None - task_name: Optional[str] = None + overrides: Sequence[str] | None = None + cfg: DictConfig | None = None + hydra_cfg: DictConfig | None = None + working_dir: str | None = None + task_name: str | None = None status: JobStatus = JobStatus.UNKNOWN _return_value: Any = None @@ -283,7 +281,7 @@ def set(self, key: str, value: Any) -> None: self.conf[key] = value -def validate_config_path(config_path: Optional[str]) -> None: +def validate_config_path(config_path: str | None) -> None: if config_path is not None: split_file = splitext(config_path) if split_file[1] in (".yaml", ".yml"): @@ -297,7 +295,7 @@ def validate_config_path(config_path: Optional[str]) -> None: @contextmanager -def env_override(env: Dict[str, str]) -> Any: +def env_override(env: dict[str, str]) -> Any: """Temporarily set environment variables inside the context manager and fully restore previous environment afterwards """ @@ -319,6 +317,6 @@ def _flush_loggers() -> None: for h_weak_ref in logging._handlerList: # type: ignore try: h_weak_ref().flush() - except Exception: + except Exception: # noqa: BLE001, S110 # ignore exceptions thrown during flushing pass diff --git a/lerna/errors.py b/lerna/errors.py index 8134188..2e79df9 100644 --- a/lerna/errors.py +++ b/lerna/errors.py @@ -1,5 +1,5 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -from typing import Optional, Sequence +from collections.abc import Sequence class HydraException(Exception): ... @@ -28,8 +28,8 @@ class MissingConfigException(IOError, ConfigCompositionException): def __init__( self, message: str, - missing_cfg_file: Optional[str] = None, - options: Optional[Sequence[str]] = None, + missing_cfg_file: str | None = None, + options: Sequence[str] | None = None, ) -> None: super().__init__(message) self.missing_cfg_file = missing_cfg_file diff --git a/lerna/experimental/__init__.py b/lerna/experimental/__init__.py index 3a0a7a8..d34a846 100644 --- a/lerna/experimental/__init__.py +++ b/lerna/experimental/__init__.py @@ -5,6 +5,6 @@ __all__ = [ "compose", "initialize", - "initialize_config_module", "initialize_config_dir", + "initialize_config_module", ] diff --git a/lerna/experimental/callback.py b/lerna/experimental/callback.py index d681b2e..dfc5ea2 100644 --- a/lerna/experimental/callback.py +++ b/lerna/experimental/callback.py @@ -1,6 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import logging -from typing import Any, List, Optional +from typing import Any from omegaconf import DictConfig @@ -17,27 +17,23 @@ def on_run_start(self, config: DictConfig, **kwargs: Any) -> None: Some `hydra.runtime` configs are not populated yet. See hydra.core.utils.run_job for more info. """ - ... def on_run_end(self, config: DictConfig, **kwargs: Any) -> None: """ Called in RUN mode after job/application code returns. """ - ... def on_multirun_start(self, config: DictConfig, **kwargs: Any) -> None: """ Called in MULTIRUN mode before any job starts. When using a launcher, this will be executed on local machine before any Sweeper/Launcher is initialized. """ - ... def on_multirun_end(self, config: DictConfig, **kwargs: Any) -> None: """ Called in MULTIRUN mode after all jobs returns. When using a launcher, this will be executed on local machine. """ - ... def on_job_start(self, config: DictConfig, *, task_function: TaskFunction, **kwargs: Any) -> None: """ @@ -46,7 +42,6 @@ def on_job_start(self, config: DictConfig, *, task_function: TaskFunction, **kwa on the remote server along with your application code. The `task_function` argument is the function decorated with `@hydra.main`. """ - ... def on_job_end(self, config: DictConfig, job_return: JobReturn, **kwargs: Any) -> None: """ @@ -58,16 +53,14 @@ def on_job_end(self, config: DictConfig, job_return: JobReturn, **kwargs: Any) - `job_return` contains info that could be useful for logging or post-processing. See hydra.core.utils.JobReturn for more. """ - ... def on_compose_config( self, config: DictConfig, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], ) -> None: """ Called during the compose phase and before the config is returned to the user. config is the composed config with overrides applied. """ - ... diff --git a/lerna/experimental/callbacks.py b/lerna/experimental/callbacks.py index 577b5fc..e2b1a0a 100644 --- a/lerna/experimental/callbacks.py +++ b/lerna/experimental/callbacks.py @@ -4,7 +4,7 @@ import logging import pickle from pathlib import Path -from typing import Any, List, Optional +from typing import Any from omegaconf import DictConfig, OmegaConf, flag_override @@ -66,8 +66,8 @@ def __init__(self) -> None: def on_compose_config( self, config: DictConfig, - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], ) -> None: gh = GlobalHydra.instance() config_loader = gh.config_loader() @@ -86,7 +86,7 @@ def on_compose_config( non_hydra_defaults = [d.config_path for d in defaults_list.defaults if not d.package.startswith("hydra")] self.log.info( f"""==== -Composed config {config_dir}/{str(config_name)} +Composed config {config_dir}/{config_name!s} {OmegaConf.to_yaml(config)} ---- Includes overrides {overrides} diff --git a/lerna/experimental/compose.py b/lerna/experimental/compose.py index d6d9022..62b06a7 100644 --- a/lerna/experimental/compose.py +++ b/lerna/experimental/compose.py @@ -1,5 +1,4 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -from typing import List, Optional from omegaconf import DictConfig @@ -8,13 +7,15 @@ def compose( - config_name: Optional[str] = None, - overrides: List[str] = [], + config_name: str | None = None, + overrides: list[str] | None = None, return_hydra_config: bool = False, - strict: Optional[bool] = None, + strict: bool | None = None, ) -> DictConfig: from lerna import compose as real_compose + if overrides is None: + overrides = [] message = "hydra.experimental.compose() is no longer experimental. Use hydra.compose()" if version.base_at_least("1.2"): diff --git a/lerna/experimental/initialize.py b/lerna/experimental/initialize.py index e06f650..456613e 100644 --- a/lerna/experimental/initialize.py +++ b/lerna/experimental/initialize.py @@ -1,6 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import copy -from typing import Any, Optional +from typing import Any from lerna import version from lerna._internal.deprecation_warning import deprecation_warning @@ -26,8 +26,8 @@ def restore_gh_from_backup(_gh_backup: Any) -> Any: class initialize: def __init__( self, - config_path: Optional[str] = _UNSPECIFIED_, - job_name: Optional[str] = None, + config_path: str | None = _UNSPECIFIED_, + job_name: str | None = None, caller_stack_depth: int = 1, ) -> None: from lerna import initialize as real_initialize @@ -49,7 +49,7 @@ def __init__( def __enter__(self, *args: Any, **kwargs: Any) -> None: self.delegate.__enter__(*args, **kwargs) - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: self.delegate.__exit__(exc_type, exc_val, exc_tb) def __repr__(self) -> str: @@ -83,7 +83,7 @@ def __init__(self, config_module: str, job_name: str = "app") -> None: def __enter__(self, *args: Any, **kwargs: Any) -> None: self.delegate.__enter__(*args, **kwargs) - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: self.delegate.__exit__(exc_type, exc_val, exc_tb) def __repr__(self) -> str: @@ -118,7 +118,7 @@ def __init__(self, config_dir: str, job_name: str = "app") -> None: def __enter__(self, *args: Any, **kwargs: Any) -> None: self.delegate.__enter__(*args, **kwargs) - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: self.delegate.__exit__(exc_type, exc_val, exc_tb) def __repr__(self) -> str: diff --git a/lerna/extra/pytest_plugin.py b/lerna/extra/pytest_plugin.py index b53928a..b97a500 100644 --- a/lerna/extra/pytest_plugin.py +++ b/lerna/extra/pytest_plugin.py @@ -1,7 +1,7 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import copy +from collections.abc import Callable, Generator from pathlib import Path -from typing import Callable, Generator, List, Optional from pytest import fixture @@ -23,25 +23,25 @@ def hydra_restore_singletons() -> Generator[None, None, None]: @fixture(scope="function") def hydra_sweep_runner() -> Callable[ [ - Optional[str], - Optional[str], - Optional[TaskFunction], - Optional[str], - Optional[str], - Optional[List[str]], - Optional[Path], + str | None, + str | None, + TaskFunction | None, + str | None, + str | None, + list[str] | None, + Path | None, bool, ], SweepTaskFunction, ]: def _( - calling_file: Optional[str], - calling_module: Optional[str], - task_function: Optional[TaskFunction], - config_path: Optional[str], - config_name: Optional[str], - overrides: Optional[List[str]], - temp_dir: Optional[Path] = None, + calling_file: str | None, + calling_module: str | None, + task_function: TaskFunction | None, + config_path: str | None, + config_name: str | None, + overrides: list[str] | None, + temp_dir: Path | None = None, configure_logging: bool = False, ) -> SweepTaskFunction: sweep = SweepTaskFunction() @@ -61,21 +61,21 @@ def _( @fixture(scope="function") def hydra_task_runner() -> Callable[ [ - Optional[str], - Optional[str], - Optional[str], - Optional[str], - Optional[List[str]], + str | None, + str | None, + str | None, + str | None, + list[str] | None, bool, ], TaskTestFunction, ]: def _( - calling_file: Optional[str], - calling_module: Optional[str], - config_path: Optional[str], - config_name: Optional[str], - overrides: Optional[List[str]] = None, + calling_file: str | None, + calling_module: str | None, + config_path: str | None, + config_name: str | None, + overrides: list[str] | None = None, configure_logging: bool = False, ) -> TaskTestFunction: task = TaskTestFunction() diff --git a/lerna/initialize.py b/lerna/initialize.py index a23abef..3681c53 100644 --- a/lerna/initialize.py +++ b/lerna/initialize.py @@ -2,7 +2,7 @@ import copy import os from textwrap import dedent -from typing import Any, Optional +from typing import Any from lerna import version from lerna._internal.deprecation_warning import deprecation_warning @@ -52,10 +52,10 @@ class initialize: def __init__( self, - config_path: Optional[str] = _UNSPECIFIED_, - job_name: Optional[str] = None, + config_path: str | None = _UNSPECIFIED_, + job_name: str | None = None, caller_stack_depth: int = 1, - version_base: Optional[str] = _UNSPECIFIED_, + version_base: str | None = _UNSPECIFIED_, ) -> None: self._gh_backup = get_gh_backup() @@ -93,7 +93,7 @@ def __init__( def __enter__(self, *args: Any, **kwargs: Any) -> None: ... - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: restore_gh_from_backup(self._gh_backup) def __repr__(self) -> str: @@ -112,7 +112,7 @@ def __init__( self, config_module: str, job_name: str = "app", - version_base: Optional[str] = _UNSPECIFIED_, + version_base: str | None = _UNSPECIFIED_, ): self._gh_backup = get_gh_backup() @@ -127,7 +127,7 @@ def __init__( def __enter__(self, *args: Any, **kwargs: Any) -> None: ... - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: restore_gh_from_backup(self._gh_backup) def __repr__(self) -> str: @@ -147,7 +147,7 @@ def __init__( self, config_dir: str, job_name: str = "app", - version_base: Optional[str] = _UNSPECIFIED_, + version_base: str | None = _UNSPECIFIED_, ) -> None: self._gh_backup = get_gh_backup() @@ -163,7 +163,7 @@ def __init__( def __enter__(self, *args: Any, **kwargs: Any) -> None: ... - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: restore_gh_from_backup(self._gh_backup) def __repr__(self) -> str: diff --git a/lerna/main.py b/lerna/main.py index be68d50..767890c 100644 --- a/lerna/main.py +++ b/lerna/main.py @@ -3,9 +3,10 @@ import functools import pickle import warnings +from collections.abc import Callable from pathlib import Path from textwrap import dedent -from typing import Any, Callable, List, Optional +from typing import Any from omegaconf import DictConfig, open_dict, read_write @@ -19,7 +20,7 @@ _UNSPECIFIED_: Any = object() -def _get_rerun_conf(file_path: str, overrides: List[str]) -> DictConfig: +def _get_rerun_conf(file_path: str, overrides: list[str]) -> DictConfig: msg = "Experimental rerun CLI option, other command line args are ignored." warnings.warn(msg, UserWarning) file = Path(file_path) @@ -35,18 +36,17 @@ def _get_rerun_conf(file_path: str, overrides: List[str]) -> DictConfig: configure_log(config.hydra.job_logging, config.hydra.verbose) HydraConfig.instance().set_config(config) task_cfg = copy.deepcopy(config) - with read_write(task_cfg): - with open_dict(task_cfg): - del task_cfg["hydra"] + with read_write(task_cfg), open_dict(task_cfg): + del task_cfg["hydra"] assert isinstance(task_cfg, DictConfig) return task_cfg def main( - config_path: Optional[str] = _UNSPECIFIED_, - config_name: Optional[str] = None, - version_base: Optional[str] = _UNSPECIFIED_, - overrides: Optional[List[str]] = None, + config_path: str | None = _UNSPECIFIED_, + config_name: str | None = None, + version_base: str | None = _UNSPECIFIED_, + overrides: list[str] | None = None, ) -> Callable[[TaskFunction], Any]: """ :param config_path: The config path, a directory where Hydra will search for @@ -82,7 +82,7 @@ def main( def main_decorator(task_function: TaskFunction) -> Callable[[], None]: @functools.wraps(task_function) - def decorated_main(cfg_passthrough: Optional[DictConfig] = None) -> Any: + def decorated_main(cfg_passthrough: DictConfig | None = None) -> Any: if cfg_passthrough is not None: return task_function(cfg_passthrough) else: diff --git a/lerna/plugins/completion_plugin.py b/lerna/plugins/completion_plugin.py index 67d5647..5ce4a1a 100644 --- a/lerna/plugins/completion_plugin.py +++ b/lerna/plugins/completion_plugin.py @@ -9,7 +9,7 @@ import re import sys from abc import abstractmethod -from typing import Any, List, Optional, Tuple +from typing import Any from omegaconf import ( Container, @@ -45,7 +45,7 @@ def provides() -> str: ... @abstractmethod - def query(self, config_name: Optional[str]) -> None: ... + def query(self, config_name: str | None) -> None: ... @staticmethod @abstractmethod @@ -57,7 +57,7 @@ def help(command: str) -> str: ... @staticmethod - def _get_filename(filename: str) -> Tuple[Optional[str], Optional[str]]: + def _get_filename(filename: str) -> tuple[str | None, str | None]: last = filename.rfind("=") if last != -1: key_eq = filename[0 : last + 1] @@ -75,7 +75,7 @@ def _get_filename(filename: str) -> Tuple[Optional[str], Optional[str]]: return None, None @staticmethod - def complete_files(word: str) -> List[str]: + def complete_files(word: str) -> list[str]: if os.path.isdir(word): dirname = word files = os.listdir(word) @@ -94,7 +94,7 @@ def complete_files(word: str) -> List[str]: return ret @staticmethod - def _get_matches(config: Container, word: str) -> List[str]: + def _get_matches(config: Container, word: str) -> list[str]: def str_rep(in_key: Any, in_value: Any) -> str: if OmegaConf.is_config(in_value): return f"{in_key}." @@ -104,8 +104,8 @@ def str_rep(in_key: Any, in_value: Any) -> str: if config is None: return [] elif OmegaConf.is_config(config): - matches: List[str] = [] - if word.endswith(".") or word.endswith("="): + matches: list[str] = [] + if word.endswith((".", "=")): exact_key = word[0:-1] try: conf_node = OmegaConf.select(config, exact_key, throw_on_missing=True) @@ -152,7 +152,7 @@ def str_rep(in_key: Any, in_value: Any) -> str: return matches - def _query_config_groups(self, word: str, config_name: Optional[str], words: List[str]) -> Tuple[List[str], bool]: + def _query_config_groups(self, word: str, config_name: str | None, words: list[str]) -> tuple[list[str], bool]: is_addition = word.startswith("+") is_deletion = word.startswith("~") if is_addition or is_deletion: @@ -178,7 +178,7 @@ def _query_config_groups(self, word: str, config_name: Optional[str], words: Lis config_name=config_name, overrides=words, ) - matched_groups: List[str] = [] + matched_groups: list[str] = [] if results_filter == ObjectType.CONFIG: for match in all_matched_groups: name = f"{parent_group}={match}" if parent_group != "" else match @@ -210,7 +210,7 @@ def _query_config_groups(self, word: str, config_name: Optional[str], words: Lis matched_groups = [f"{prefix}{group}" for group in matched_groups] return matched_groups, exact_match - def _query(self, config_name: Optional[str], line: str) -> List[str]: + def _query(self, config_name: str | None, line: str) -> list[str]: from .._internal.utils import get_args new_word = len(line) == 0 or line[-1] == " " @@ -229,7 +229,7 @@ def _query(self, config_name: Optional[str], line: str) -> List[str]: result = [fname_prefix + file for file in result] else: matched_groups, exact_match = self._query_config_groups(word, config_name=config_name, words=words) - config_matches: List[str] = [] + config_matches: list[str] = [] if not exact_match: run_mode = RunMode.MULTIRUN if parsed_args.multirun else RunMode.RUN config_matches = [] @@ -285,7 +285,7 @@ def uninstall(self) -> None: def provides() -> str: raise NotImplementedError - def query(self, config_name: Optional[str]) -> None: + def query(self, config_name: str | None) -> None: raise NotImplementedError @staticmethod diff --git a/lerna/plugins/config_source.py b/lerna/plugins/config_source.py index 30fb2c0..addee4b 100644 --- a/lerna/plugins/config_source.py +++ b/lerna/plugins/config_source.py @@ -1,8 +1,8 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import builtins import re from abc import abstractmethod from dataclasses import dataclass -from typing import Dict, List, Optional from omegaconf import Container @@ -26,8 +26,8 @@ class ConfigResult: provider: str path: str config: Container - header: Dict[str, Optional[str]] - defaults_list: Optional[List[InputDefault]] = None + header: dict[str, str | None] + defaults_list: list[InputDefault] | None = None is_schema_source: bool = False @@ -74,7 +74,7 @@ def available(self) -> bool: ... @abstractmethod - def list(self, config_path: str, results_filter: Optional[ObjectType]) -> List[str]: + def list(self, config_path: str, results_filter: ObjectType | None) -> list[str]: """ List items under the specified config path :param config_path: config path to list items in, examples: "", "foo", "foo/bar" @@ -91,10 +91,10 @@ def __repr__(self) -> str: def _list_add_result( self, - files: List[str], + files: builtins.list[str], file_path: str, file_name: str, - results_filter: Optional[ObjectType], + results_filter: ObjectType | None, ) -> None: filtered = ["__pycache__", "__init__.py"] is_group = self.is_group(file_path) @@ -132,8 +132,8 @@ def _normalize_file_name(filename: str) -> str: return filename @staticmethod - def _get_header_dict(config_text: str) -> Dict[str, Optional[str]]: - res: Dict[str, Optional[str]] = {} + def _get_header_dict(config_text: str) -> dict[str, str | None]: + res: dict[str, str | None] = {} for line in config_text.splitlines(): line = line.strip() if len(line) == 0: diff --git a/lerna/plugins/launcher.py b/lerna/plugins/launcher.py index 995604c..030ef54 100644 --- a/lerna/plugins/launcher.py +++ b/lerna/plugins/launcher.py @@ -4,7 +4,7 @@ """ from abc import abstractmethod -from typing import Sequence +from collections.abc import Sequence from omegaconf import DictConfig diff --git a/lerna/plugins/sweeper.py b/lerna/plugins/sweeper.py index 8ec1cec..7260785 100644 --- a/lerna/plugins/sweeper.py +++ b/lerna/plugins/sweeper.py @@ -4,7 +4,8 @@ """ from abc import abstractmethod -from typing import Any, List, Optional, Sequence +from collections.abc import Sequence +from typing import Any from omegaconf import DictConfig @@ -21,9 +22,9 @@ class Sweeper(Plugin): (where each job typically takes a different command line arguments) """ - hydra_context: Optional[HydraContext] - config: Optional[DictConfig] - launcher: Optional[Launcher] + hydra_context: HydraContext | None + config: DictConfig | None + launcher: Launcher | None @abstractmethod def setup( @@ -36,7 +37,7 @@ def setup( raise NotImplementedError() @abstractmethod - def sweep(self, arguments: List[str]) -> Any: + def sweep(self, arguments: list[str]) -> Any: """ Execute a sweep :param arguments: list of strings describing what this sweeper should do. diff --git a/lerna/test_utils/config_source_common_tests.py b/lerna/test_utils/config_source_common_tests.py index 906aa3a..04941c3 100644 --- a/lerna/test_utils/config_source_common_tests.py +++ b/lerna/test_utils/config_source_common_tests.py @@ -1,5 +1,5 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -from typing import Any, List, Optional, Type +from typing import Any from pytest import mark, param, raises, skip @@ -22,7 +22,7 @@ def skip_overlap_config_path_name(self) -> bool: """ return False - def test_not_available(self, type_: Type[ConfigSource], path: str) -> None: + def test_not_available(self, type_: type[ConfigSource], path: str) -> None: scheme = type_(provider="foo", path=path).scheme() # Test is meaningless for StructuredConfigSource if scheme == "structured": @@ -48,7 +48,7 @@ def test_not_available(self, type_: Type[ConfigSource], path: str) -> None: param("not_found", False, id="not_found"), ], ) - def test_is_group(self, type_: Type[ConfigSource], path: str, config_path: str, expected: bool) -> None: + def test_is_group(self, type_: type[ConfigSource], path: str, config_path: str, expected: bool) -> None: src = type_(provider="foo", path=path) ret = src.is_group(config_path=config_path) assert ret == expected @@ -69,7 +69,7 @@ def test_is_group(self, type_: Type[ConfigSource], path: str, config_path: str, ("not_found", False), ], ) - def test_is_config(self, type_: Type[ConfigSource], path: str, config_path: str, expected: bool) -> None: + def test_is_config(self, type_: type[ConfigSource], path: str, config_path: str, expected: bool) -> None: src = type_(provider="foo", path=path) ret = src.is_config(config_path=config_path) assert ret == expected @@ -80,7 +80,7 @@ def test_is_config(self, type_: Type[ConfigSource], path: str, config_path: str, ("dataset", True), ], ) - def test_is_config_with_overlap_name(self, type_: Type[ConfigSource], path: str, config_path: str, expected: bool) -> None: + def test_is_config_with_overlap_name(self, type_: type[ConfigSource], path: str, config_path: str, expected: bool) -> None: if self.skip_overlap_config_path_name(): skip(f"ConfigSourcePlugin {type_.__name__} does not support config objects and config groups with overlapping names.") src = type_(provider="foo", path=path) @@ -108,16 +108,15 @@ def test_is_config_with_overlap_name(self, type_: Type[ConfigSource], path: str, ("optimizer", None, ["adam", "nesterov"]), ("level1", None, ["level2"]), ("level1/level2", None, ["nested1", "nested2"]), - ("", None, ["config_without_group", "dataset", "level1", "optimizer"]), ], ) def test_list( self, - type_: Type[ConfigSource], + type_: type[ConfigSource], path: str, config_path: str, - results_filter: Optional[ObjectType], - expected: List[str], + results_filter: ObjectType | None, + expected: list[str], ) -> None: src = type_(provider="foo", path=path) ret = src.list(config_path=config_path, results_filter=results_filter) @@ -134,11 +133,11 @@ def test_list( ) def test_list_with_overlap_name( self, - type_: Type[ConfigSource], + type_: type[ConfigSource], path: str, config_path: str, - results_filter: Optional[ObjectType], - expected: List[str], + results_filter: ObjectType | None, + expected: list[str], ) -> None: if self.skip_overlap_config_path_name(): skip(f"ConfigSourcePlugin {type_.__name__} does not support config objects and config groups with overlapping names.") @@ -234,10 +233,10 @@ def test_list_with_overlap_name( ) def test_source_load_config( self, - type_: Type[ConfigSource], + type_: type[ConfigSource], path: str, config_path: str, - expected_defaults_list: List[InputDefault], + expected_defaults_list: list[InputDefault], expected_package: Any, expected_config: Any, recwarn: Any, @@ -271,7 +270,7 @@ def test_source_load_config( ) def test_package_behavior( self, - type_: Type[ConfigSource], + type_: type[ConfigSource], path: str, config_path: str, expected_result: Any, @@ -282,12 +281,12 @@ def test_package_behavior( assert cfg.header["package"] == expected_package assert cfg.config == expected_result - def test_default_package_for_primary_config(self, type_: Type[ConfigSource], path: str) -> None: + def test_default_package_for_primary_config(self, type_: type[ConfigSource], path: str) -> None: src = type_(provider="foo", path=path) cfg = src.load_config(config_path="primary_config") assert cfg.header["package"] is None - def test_primary_config_with_non_global_package(self, type_: Type[ConfigSource], path: str) -> None: + def test_primary_config_with_non_global_package(self, type_: type[ConfigSource], path: str) -> None: src = type_(provider="foo", path=path) cfg = src.load_config(config_path="primary_config_with_non_global_package") assert cfg.header["package"] == "foo" diff --git a/lerna/test_utils/launcher_common_tests.py b/lerna/test_utils/launcher_common_tests.py index be357a7..17d7e57 100644 --- a/lerna/test_utils/launcher_common_tests.py +++ b/lerna/test_utils/launcher_common_tests.py @@ -6,8 +6,9 @@ import copy import os import re +from collections.abc import Callable from pathlib import Path -from typing import Any, Callable, List, Optional, Set +from typing import Any from omegaconf import DictConfig, OmegaConf from pytest import mark, param, raises @@ -23,7 +24,7 @@ @mark.usefixtures("hydra_restore_singletons") class LauncherTestSuite: - def get_task_function(self) -> Optional[Callable[[Any], Any]]: + def get_task_function(self) -> Callable[[Any], Any] | None: def task_func(_: DictConfig) -> Any: return 100 @@ -33,7 +34,7 @@ def test_sweep_1_job( self, hydra_sweep_runner: TSweepRunner, launcher_name: str, - overrides: List[str], + overrides: list[str], tmpdir: Path, ) -> None: sweep_1_job( @@ -47,7 +48,7 @@ def test_sweep_2_jobs( self, hydra_sweep_runner: TSweepRunner, launcher_name: str, - overrides: List[str], + overrides: list[str], tmpdir: Path, ) -> None: sweep_2_jobs( @@ -61,14 +62,15 @@ def test_not_sweeping_hydra_overrides( self, hydra_sweep_runner: TSweepRunner, launcher_name: str, - overrides: List[str], + overrides: list[str], tmpdir: Path, ) -> None: - with raises( - HydraException, - match=re.escape("Sweeping over Hydra's configuration is not supported : 'hydra.verbose=true,false'"), - ): - with hydra_sweep_runner( + with ( + raises( + HydraException, + match=re.escape("Sweeping over Hydra's configuration is not supported : 'hydra.verbose=true,false'"), + ), + hydra_sweep_runner( calling_file=None, calling_module="lerna.test_utils.a_module", task_function=None, @@ -81,14 +83,15 @@ def test_not_sweeping_hydra_overrides( "hydra.verbose=true,false", ], temp_dir=tmpdir, - ): - pass + ), + ): + pass def test_sweep_1_job_strict( self, hydra_sweep_runner: TSweepRunner, launcher_name: str, - overrides: List[str], + overrides: list[str], tmpdir: Path, ) -> None: sweep_1_job( @@ -102,12 +105,12 @@ def test_sweep_1_job_strict_and_bad_key( self, hydra_sweep_runner: TSweepRunner, launcher_name: str, - overrides: List[str], + overrides: list[str], tmpdir: Path, ) -> None: # Ideally this would be KeyError, This can't be more specific because some launcher plugins # like submitit raises a different exception on job failure and not the underlying exception. - with raises(Exception): + with raises(Exception): # noqa: B017 sweep_1_job( hydra_sweep_runner, overrides=["hydra/launcher=" + launcher_name, "boo=bar"] + overrides, @@ -119,7 +122,7 @@ def test_sweep_2_optimizers( self, hydra_sweep_runner: TSweepRunner, launcher_name: str, - overrides: List[str], + overrides: list[str], tmpdir: Path, ) -> None: sweep_two_config_groups( @@ -133,7 +136,7 @@ def test_sweep_over_unspecified_mandatory_default( self, hydra_sweep_runner: TSweepRunner, launcher_name: str, - overrides: List[str], + overrides: list[str], tmpdir: Path, ) -> None: base_overrides = ["hydra/launcher=" + launcher_name, "group1=file1,file2"] @@ -161,7 +164,7 @@ def test_sweep_and_override( self, hydra_sweep_runner: TSweepRunner, launcher_name: str, - overrides: List[str], + overrides: list[str], tmpdir: Path, ) -> None: """ @@ -211,7 +214,7 @@ def test_sweep_with_custom_resolver( self, hydra_sweep_runner: TSweepRunner, launcher_name: str, - overrides: List[str], + overrides: list[str], tmpdir: Path, ) -> None: overrides1 = ["hydra/launcher=" + launcher_name] + overrides @@ -246,7 +249,7 @@ def test_sweep_2_jobs_2_batches( self, hydra_sweep_runner: TSweepRunner, launcher_name: str, - overrides: List[str], + overrides: list[str], tmpdir: Path, ) -> None: job_overrides = ["group1=file1,file2", "bar=100,200,300"] @@ -280,7 +283,7 @@ def test_sweep_2_jobs_2_batches( {"foo": 20, "bar": 300}, ] - dirs: Set[str] = set() + dirs: set[str] = set() with sweep: temp_dir = sweep.temp_dir assert temp_dir is not None @@ -307,8 +310,8 @@ def test_sweep_2_jobs_2_batches( def sweep_1_job( hydra_sweep_runner: TSweepRunner, - overrides: List[str], - task_function: Optional[TaskFunction], + overrides: list[str], + task_function: TaskFunction | None, temp_dir: Path, ) -> None: """ @@ -338,8 +341,8 @@ def sweep_1_job( def sweep_2_jobs( hydra_sweep_runner: TSweepRunner, - overrides: List[str], - task_function: Optional[TaskFunction], + overrides: list[str], + task_function: TaskFunction | None, temp_dir: Path, ) -> None: """ @@ -388,8 +391,8 @@ def sweep_2_jobs( def sweep_two_config_groups( hydra_sweep_runner: TSweepRunner, - overrides: List[str], - task_function: Optional[TaskFunction], + overrides: list[str], + task_function: TaskFunction | None, temp_dir: Path, ) -> None: """ @@ -423,7 +426,7 @@ def sweep_two_config_groups( @mark.usefixtures("hydra_restore_singletons") class IntegrationTestSuite: - def get_test_app_working_dir(self) -> Optional[Path]: + def get_test_app_working_dir(self) -> Path | None: """ By default test applications working dir is tmpdir, override this method if that's not the case. This could be helpful when the tests kick off applications on remote machines. @@ -437,13 +440,13 @@ def get_test_scratch_dir(self, tmpdir: Path) -> Path: """ return tmpdir - def generate_custom_cmd(self) -> Callable[..., List[str]]: + def generate_custom_cmd(self) -> Callable[..., list[str]]: """ By default this does nothing, but it allows custom execution commands. Useful if the tests are not kicked off by python """ - def fun(cmd: List[str], filename: str) -> List[str]: + def fun(cmd: list[str], filename: str) -> list[str]: """ param cmd: old python commands in list of strings param filename: file name to be executed as main hydra module @@ -484,11 +487,11 @@ def test_custom_task_name( self, tmpdir: Path, task_config: DictConfig, - overrides: List[str], + overrides: list[str], filename: str, expected_name: str, task_launcher_cfg: DictConfig, - extra_flags: List[str], + extra_flags: list[str], ) -> None: overrides = extra_flags + overrides task_launcher_cfg = OmegaConf.create(task_launcher_cfg or {}) @@ -594,10 +597,10 @@ def test_custom_sweeper_run_workdir( self, tmpdir: Path, task_config: str, - overrides: List[str], + overrides: list[str], expected_dir: str, task_launcher_cfg: DictConfig, - extra_flags: List[str], + extra_flags: list[str], ) -> None: overrides = extra_flags + overrides task_launcher_cfg = OmegaConf.create(task_launcher_cfg or {}) @@ -618,7 +621,7 @@ def test_custom_sweeper_run_workdir( generate_custom_cmd=self.generate_custom_cmd(), ) - def test_get_orig_dir_multirun(self, tmpdir: Path, task_launcher_cfg: DictConfig, extra_flags: List[str]) -> None: + def test_get_orig_dir_multirun(self, tmpdir: Path, task_launcher_cfg: DictConfig, extra_flags: list[str]) -> None: overrides = extra_flags task_launcher_cfg = OmegaConf.create(task_launcher_cfg or {}) task_config = OmegaConf.create() @@ -634,7 +637,7 @@ def test_get_orig_dir_multirun(self, tmpdir: Path, task_launcher_cfg: DictConfig generate_custom_cmd=self.generate_custom_cmd(), ) - def test_to_absolute_path_multirun(self, tmpdir: Path, task_launcher_cfg: DictConfig, extra_flags: List[str]) -> None: + def test_to_absolute_path_multirun(self, tmpdir: Path, task_launcher_cfg: DictConfig, extra_flags: list[str]) -> None: expected_dir = "cli_dir/cli_dir_0" overrides = extra_flags + [ "hydra.job.chdir=True", diff --git a/lerna/test_utils/test_utils.py b/lerna/test_utils/test_utils.py index 12d62a2..866cf71 100644 --- a/lerna/test_utils/test_utils.py +++ b/lerna/test_utils/test_utils.py @@ -12,11 +12,12 @@ import subprocess import sys import tempfile +from collections.abc import Callable, Iterator from contextlib import contextmanager from difflib import unified_diff from pathlib import Path from subprocess import PIPE, Popen -from typing import Any, Callable, Dict, Iterator, List, Optional, Protocol, Tuple, Union +from typing import Any, Protocol from omegaconf import Container, DictConfig, OmegaConf @@ -27,7 +28,7 @@ from lerna.types import TaskFunction -def normalize_path_for_override(path: Union[str, Path]) -> str: +def normalize_path_for_override(path: str | Path) -> str: """ Normalize a path for use in Hydra overrides. @@ -49,14 +50,14 @@ class TaskTestFunction: """ def __init__(self) -> None: - self.temp_dir: Optional[str] = None - self.overrides: Optional[List[str]] = None - self.calling_file: Optional[str] = None - self.calling_module: Optional[str] = None - self.config_path: Optional[str] = None - self.config_name: Optional[str] = None - self.hydra: Optional[Hydra] = None - self.job_ret: Optional[JobReturn] = None + self.temp_dir: str | None = None + self.overrides: list[str] | None = None + self.calling_file: str | None = None + self.calling_module: str | None = None + self.config_path: str | None = None + self.config_name: str | None = None + self.hydra: Hydra | None = None + self.job_ret: JobReturn | None = None self.configure_logging: bool = False def __call__(self, cfg: DictConfig) -> Any: @@ -66,7 +67,7 @@ def __call__(self, cfg: DictConfig) -> Any: return 100 - def __enter__(self) -> "TaskTestFunction": + def __enter__(self) -> "TaskTestFunction": # noqa: PYI034 try: validate_config_path(self.config_path) @@ -92,7 +93,7 @@ def __enter__(self) -> "TaskTestFunction": finally: GlobalHydra().clear() - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: # release log file handles. if self.configure_logging: logging.shutdown() @@ -103,11 +104,11 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: class TTaskRunner(Protocol): def __call__( self, - calling_file: Optional[str], - calling_module: Optional[str], - config_path: Optional[str], - config_name: Optional[str], - overrides: Optional[List[str]] = None, + calling_file: str | None, + calling_module: str | None, + config_path: str | None, + config_name: str | None, + overrides: list[str] | None = None, configure_logging: bool = False, ) -> TaskTestFunction: ... @@ -121,13 +122,13 @@ def __init__(self) -> None: """ if sweep_dir is None, we use a temp dir, else we will create dir with the path from sweep_dir. """ - self.temp_dir: Optional[str] = None - self.overrides: Optional[List[str]] = None - self.calling_file: Optional[str] = None - self.calling_module: Optional[str] = None - self.task_function: Optional[TaskFunction] = None - self.config_path: Optional[str] = None - self.config_name: Optional[str] = None + self.temp_dir: str | None = None + self.overrides: list[str] | None = None + self.calling_file: str | None = None + self.calling_module: str | None = None + self.task_function: TaskFunction | None = None + self.config_path: str | None = None + self.config_name: str | None = None self.sweeps = None self.returns = None self.configure_logging: bool = False @@ -140,7 +141,7 @@ def __call__(self, cfg: DictConfig) -> Any: return self.task_function(cfg) return 100 - def __enter__(self) -> "SweepTaskFunction": + def __enter__(self) -> "SweepTaskFunction": # noqa: PYI034 overrides = copy.deepcopy(self.overrides) assert overrides is not None if self.temp_dir: @@ -171,7 +172,7 @@ def __enter__(self) -> "SweepTaskFunction": return self - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + def __exit__(self, exc_type: object, exc_val: object, exc_tb: object) -> None: if self.configure_logging: logging.shutdown() assert self.temp_dir is not None @@ -179,21 +180,21 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: class TSweepRunner(Protocol): - returns: List[List[JobReturn]] + returns: list[list[JobReturn]] def __call__( self, - calling_file: Optional[str], - calling_module: Optional[str], - task_function: Optional[TaskFunction], - config_path: Optional[str], - config_name: Optional[str], - overrides: Optional[List[str]], - temp_dir: Optional[Path] = None, + calling_file: str | None, + calling_module: str | None, + task_function: TaskFunction | None, + config_path: str | None, + config_name: str | None, + overrides: list[str] | None, + temp_dir: Path | None = None, ) -> SweepTaskFunction: ... -def chdir_hydra_root(subdir: Optional[str] = None) -> None: +def chdir_hydra_root(subdir: str | None = None) -> None: """ Change the cwd to the root of the hydra project. used from unit tests to make them runnable from anywhere in the tree. @@ -226,7 +227,7 @@ def find_parent_dir_containing(target: str, max_up: int = 6, initial_dir: str = return cur -def verify_dir_outputs(job_return: JobReturn, overrides: Optional[List[str]] = None) -> None: +def verify_dir_outputs(job_return: JobReturn, overrides: list[str] | None = None) -> None: """ Verify that directory output makes sense """ @@ -242,7 +243,7 @@ def verify_dir_outputs(job_return: JobReturn, overrides: Optional[List[str]] = N assert OmegaConf.load(os.path.join(hydra_dir, "overrides.yaml")) == OmegaConf.create(overrides or []) -def _get_statements(indent: str, statements: Union[None, str, List[str]]) -> str: +def _get_statements(indent: str, statements: None | str | list[str]) -> str: if isinstance(statements, str): statements = [statements] @@ -258,14 +259,14 @@ def _get_statements(indent: str, statements: Union[None, str, List[str]]) -> str def integration_test( tmpdir: Path, task_config: Any, - overrides: List[str], - prints: Union[str, List[str]], - expected_outputs: Union[str, List[str]], - prolog: Union[None, str, List[str]] = None, + overrides: list[str], + prints: str | list[str], + expected_outputs: str | list[str], + prolog: None | str | list[str] = None, filename: str = "task.py", - env_override: Optional[Dict[str, str]] = None, + env_override: dict[str, str] | None = None, clean_environment: bool = False, - generate_custom_cmd: Callable[..., List[str]] = lambda cmd, *args, **kwargs: cmd, + generate_custom_cmd: Callable[..., list[str]] = lambda cmd, *args, **kwargs: cmd, ) -> str: Path(tmpdir).mkdir(parents=True, exist_ok=True) if isinstance(expected_outputs, str): @@ -355,7 +356,7 @@ def run_python_script( allow_warnings: bool = False, print_error: bool = True, raise_exception: bool = True, -) -> Tuple[str, str]: +) -> tuple[str, str]: if allow_warnings: cmd = [sys.executable] + cmd else: @@ -368,8 +369,8 @@ def run_process( env: Any = None, print_error: bool = True, raise_exception: bool = True, - timeout: Optional[float] = None, -) -> Tuple[str, str]: + timeout: float | None = None, +) -> tuple[str, str]: try: process = subprocess.Popen( args=cmd, @@ -387,11 +388,11 @@ def run_process( if raise_exception: raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd) return stdout, stderr - except Exception as e: + except Exception: if print_error: cmd = " ".join(cmd) sys.stderr.write(f"=== Error executing:\n{cmd}\n===================") - raise e + raise def normalize_newlines(s: str) -> str: diff --git a/lerna/tests/conftest.py b/lerna/tests/conftest.py index 01f8340..39f36d9 100644 --- a/lerna/tests/conftest.py +++ b/lerna/tests/conftest.py @@ -25,7 +25,7 @@ _initial_state_without_structured = copy.deepcopy(Singleton.get_state()) # Import the structured config test module to register its configs -import lerna.tests.test_apps.config_source_test.structured # noqa: F401, E402 +import lerna.tests.test_apps.config_source_test.structured # noqa: F401 # Store the state with the structured configs registered _initial_state_with_configs = copy.deepcopy(Singleton.get_state()) diff --git a/lerna/tests/defaults_list/__init__.py b/lerna/tests/defaults_list/__init__.py index bfe1d7c..6d58454 100644 --- a/lerna/tests/defaults_list/__init__.py +++ b/lerna/tests/defaults_list/__init__.py @@ -1,5 +1,5 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -from typing import Any, List, Optional +from typing import Any from lerna._internal.config_repository import ConfigRepository, IConfigRepository from lerna._internal.config_search_path_impl import ConfigSearchPathImpl @@ -21,12 +21,12 @@ def create_repo() -> IConfigRepository: def _test_defaults_tree_impl( - config_name: Optional[str], - input_overrides: List[str], + config_name: str | None, + input_overrides: list[str], expected: Any, prepend_hydra: bool = False, skip_missing: bool = False, -) -> Optional[DefaultsList]: +) -> DefaultsList | None: parser = OverridesParser.create() repo = create_repo() root = _create_root(config_name=config_name, with_hydra=prepend_hydra) diff --git a/lerna/tests/defaults_list/test_defaults_list.py b/lerna/tests/defaults_list/test_defaults_list.py index f55be4d..5bcca33 100644 --- a/lerna/tests/defaults_list/test_defaults_list.py +++ b/lerna/tests/defaults_list/test_defaults_list.py @@ -1,7 +1,7 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import re from textwrap import dedent -from typing import Any, List, Optional +from typing import Any from pytest import mark, param, raises, warns @@ -62,7 +62,7 @@ ), ], ) -def test_loaded_defaults_list(config_path: str, expected_list: List[InputDefault]) -> None: +def test_loaded_defaults_list(config_path: str, expected_list: list[InputDefault]) -> None: repo = create_repo() result = repo.load_config(config_path=config_path) assert result is not None @@ -83,7 +83,7 @@ class TestDeprecatedOptional: def test_version_base_1_1( self, config_path: str, - expected_list: List[InputDefault], + expected_list: list[InputDefault], hydra_restore_singletons: Any, ) -> None: version.setbase("1.1") @@ -106,8 +106,8 @@ def test_version_base_1_1( def test_version_base_1_2( self, config_path: str, - expected_list: List[InputDefault], - version_base: Optional[str], + expected_list: list[InputDefault], + version_base: str | None, hydra_restore_singletons: Any, ) -> None: version.setbase(version_base) @@ -121,8 +121,8 @@ def test_version_base_1_2( def _test_defaults_list_impl( - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], expected: Any, prepend_hydra: bool = False, skip_missing: bool = False, @@ -424,7 +424,7 @@ def test_get_final_package(default: InputDefault, parent_package: str, parent_ba ), ], ) -def test_simple_defaults_list_cases(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_simple_defaults_list_cases(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl(config_name=config_name, overrides=overrides, expected=expected) @@ -488,7 +488,7 @@ def test_simple_defaults_list_cases(config_name: str, overrides: List[str], expe ), ], ) -def test_override_package_in_defaults_list(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_override_package_in_defaults_list(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl(config_name=config_name, overrides=overrides, expected=expected) @@ -535,7 +535,7 @@ def test_override_package_in_defaults_list(config_name: str, overrides: List[str ), ], ) -def test_include_nested_group_pkg2(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_include_nested_group_pkg2(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl(config_name=config_name, overrides=overrides, expected=expected) @@ -586,7 +586,7 @@ def test_include_nested_group_pkg2(config_name: str, overrides: List[str], expec ), ], ) -def test_group_default_pkg1(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_group_default_pkg1(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl(config_name=config_name, overrides=overrides, expected=expected) @@ -641,7 +641,7 @@ def test_group_default_pkg1(config_name: str, overrides: List[str], expected: Li ), ], ) -def test_include_nested_group_global(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_include_nested_group_global(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl(config_name=config_name, overrides=overrides, expected=expected) @@ -711,7 +711,7 @@ def test_include_nested_group_global(config_name: str, overrides: List[str], exp ), ], ) -def test_group_global(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_group_global(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl(config_name=config_name, overrides=overrides, expected=expected) @@ -766,7 +766,7 @@ def test_group_global(config_name: str, overrides: List[str], expected: List[Res ), ], ) -def test_include_nested_group_global_foo(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_include_nested_group_global_foo(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl(config_name=config_name, overrides=overrides, expected=expected) @@ -850,8 +850,8 @@ def test_include_nested_group_global_foo(config_name: str, overrides: List[str], ) def test_include_nested_group_name_( config_name: str, - overrides: List[str], - expected: List[ResultDefault], + overrides: list[str], + expected: list[ResultDefault], warning_file: str, ) -> None: url = "https://hydra.cc/docs/1.2/upgrades/1.0_to_1.1/changes_to_package_header" @@ -910,7 +910,7 @@ def test_include_nested_group_name_( ), ], ) -def test_primary_cfg_pkg_header_foo(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_primary_cfg_pkg_header_foo(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl(config_name=config_name, overrides=overrides, expected=expected) @@ -1011,7 +1011,7 @@ def test_primary_cfg_pkg_header_foo(config_name: str, overrides: List[str], expe ), ], ) -def test_include_nested_group_pkg_header_foo(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_include_nested_group_pkg_header_foo(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl(config_name=config_name, overrides=overrides, expected=expected) @@ -1053,7 +1053,7 @@ def test_include_nested_group_pkg_header_foo(config_name: str, overrides: List[s ), ], ) -def test_nested_package_header_is_absolute(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_nested_package_header_is_absolute(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl(config_name=config_name, overrides=overrides, expected=expected) @@ -1137,7 +1137,7 @@ def test_nested_package_header_is_absolute(config_name: str, overrides: List[str ), ], ) -def test_overriding_package_header_from_defaults_list(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_overriding_package_header_from_defaults_list(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl(config_name=config_name, overrides=overrides, expected=expected) @@ -1163,10 +1163,10 @@ def test_overriding_package_header_from_defaults_list(config_name: str, override @mark.parametrize("version_base", ["1.2", None]) def test_legacy_override_hydra_version_base_1_2( config_name: str, - overrides: List[str], - expected: List[ResultDefault], + overrides: list[str], + expected: list[ResultDefault], recwarn: Any, # Testing deprecated behavior - version_base: Optional[str], + version_base: str | None, hydra_restore_singletons: Any, ) -> None: version.setbase(version_base) @@ -1216,8 +1216,8 @@ def test_legacy_override_hydra_version_base_1_2( ) def test_legacy_override_hydra_version_base_1_1( config_name: str, - overrides: List[str], - expected: List[ResultDefault], + overrides: list[str], + expected: list[ResultDefault], recwarn: Any, # Testing deprecated behavior hydra_restore_singletons: Any, ) -> None: @@ -1292,8 +1292,8 @@ def test_legacy_override_hydra_version_base_1_1( ) def test_with_hydra_config( config_name: str, - overrides: List[str], - expected: List[ResultDefault], + overrides: list[str], + expected: list[ResultDefault], recwarn: Any, # Testing deprecated behavior ) -> None: _test_defaults_list_impl( @@ -1329,7 +1329,7 @@ def test_with_hydra_config( ), ], ) -def test_experiment_use_case(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_experiment_use_case(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl( config_name=config_name, overrides=overrides, @@ -1370,7 +1370,7 @@ def test_experiment_use_case(config_name: str, overrides: List[str], expected: L ), ], ) -def test_as_as_primary(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_as_as_primary(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl( config_name=config_name, overrides=overrides, @@ -1432,7 +1432,7 @@ def test_as_as_primary(config_name: str, overrides: List[str], expected: List[Re ), ], ) -def test_placeholder(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_placeholder(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl( config_name=config_name, overrides=overrides, @@ -1501,7 +1501,7 @@ def test_placeholder(config_name: str, overrides: List[str], expected: List[Resu ), ], ) -def test_interpolation_simple(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_interpolation_simple(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl( config_name=config_name, overrides=overrides, @@ -1522,7 +1522,7 @@ def test_interpolation_simple(config_name: str, overrides: List[str], expected: ), ], ) -def test_deletion(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_deletion(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl( config_name=config_name, overrides=overrides, @@ -1553,7 +1553,7 @@ def test_deletion(config_name: str, overrides: List[str], expected: List[ResultD ), ], ) -def test_duplicate_items(config_name: str, overrides: List[str], expected: List[ResultDefault]) -> None: +def test_duplicate_items(config_name: str, overrides: list[str], expected: list[ResultDefault]) -> None: _test_defaults_list_impl( config_name=config_name, overrides=overrides, @@ -1647,9 +1647,9 @@ def test_duplicate_items(config_name: str, overrides: List[str], expected: List[ @mark.parametrize("version_base", ["1.2", None]) def test_name_collision( config_name: str, - overrides: List[str], - expected: List[ResultDefault], - version_base: Optional[str], + overrides: list[str], + expected: list[ResultDefault], + version_base: str | None, hydra_restore_singletons: Any, ) -> None: version.setbase(version_base) @@ -1708,7 +1708,7 @@ def test_name_collision( ), ], ) -def test_load_group_header(config_name: str, overrides: List[str], expected: List[ResultDefault], recwarn: Any) -> None: +def test_load_group_header(config_name: str, overrides: list[str], expected: list[ResultDefault], recwarn: Any) -> None: _test_defaults_list_impl( config_name=config_name, overrides=overrides, @@ -1741,8 +1741,8 @@ def test_load_group_header(config_name: str, overrides: List[str], expected: Lis ) def test_with_none_primary( config_name: str, - overrides: List[str], - expected: List[ResultDefault], + overrides: list[str], + expected: list[ResultDefault], ) -> None: _test_defaults_list_impl( config_name=config_name, @@ -1809,8 +1809,8 @@ def test_with_none_primary( ) def test_with_none_primary_with_hydra( config_name: str, - overrides: List[str], - expected: List[ResultDefault], + overrides: list[str], + expected: list[ResultDefault], ) -> None: _test_defaults_list_impl( config_name=config_name, @@ -1845,8 +1845,8 @@ def test_with_none_primary_with_hydra( ) def test_two_config_items( config_name: str, - overrides: List[str], - expected: List[ResultDefault], + overrides: list[str], + expected: list[ResultDefault], ) -> None: _test_defaults_list_impl( config_name=config_name, @@ -1898,9 +1898,9 @@ def test_two_config_items( ) def test_with_missing_config( config_name: str, - overrides: List[str], + overrides: list[str], skip_missing: bool, - expected: List[ResultDefault], + expected: list[ResultDefault], ) -> None: _test_defaults_list_impl( config_name=config_name, @@ -1991,9 +1991,9 @@ def test_set_package_header_with_parent_pkg(default: InputDefault, package_heade ) def test_select_multi_pkg( config_name: str, - overrides: List[str], + overrides: list[str], skip_missing: bool, - expected: List[ResultDefault], + expected: list[ResultDefault], ) -> None: _test_defaults_list_impl( config_name=config_name, diff --git a/lerna/tests/defaults_list/test_defaults_tree.py b/lerna/tests/defaults_list/test_defaults_tree.py index 87d3733..07a39d1 100644 --- a/lerna/tests/defaults_list/test_defaults_tree.py +++ b/lerna/tests/defaults_list/test_defaults_tree.py @@ -1,7 +1,7 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import re from textwrap import dedent -from typing import Any, Dict, List, Optional +from typing import Any from pytest import mark, param, raises, warns @@ -129,7 +129,7 @@ ) def test_simple_defaults_tree_cases( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl(config_name=config_name, input_overrides=overrides, expected=expected) @@ -213,7 +213,7 @@ def test_simple_defaults_tree_cases( ) def test_tree_with_append_override( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl(config_name=config_name, input_overrides=overrides, expected=expected) @@ -286,7 +286,7 @@ def test_tree_with_append_override( ) def test_simple_group_override( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl(config_name=config_name, input_overrides=overrides, expected=expected) @@ -332,7 +332,7 @@ def test_simple_group_override( ) def test_misc_errors( config_name: str, - overrides: List[str], + overrides: list[str], expected: Any, ) -> None: _test_defaults_tree_impl(config_name=config_name, input_overrides=overrides, expected=expected) @@ -405,7 +405,7 @@ def test_misc_errors( ) def test_defaults_tree_with_package_overrides( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl(config_name=config_name, input_overrides=overrides, expected=expected) @@ -480,7 +480,7 @@ def test_defaults_tree_with_package_overrides( ) def test_defaults_tree_with_package_overrides__group_override( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl(config_name=config_name, input_overrides=overrides, expected=expected) @@ -616,7 +616,7 @@ def test_defaults_tree_with_package_overrides__group_override( ) def test_override_option_from_defaults_list( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl(config_name=config_name, input_overrides=overrides, expected=expected) @@ -681,7 +681,7 @@ def test_override_option_from_defaults_list( ) def test_two_group_defaults_different_pkgs( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl(config_name=config_name, input_overrides=overrides, expected=expected) @@ -765,7 +765,7 @@ def test_two_group_defaults_different_pkgs( ) def test_hydra_overrides_from_primary_config( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -826,9 +826,9 @@ def test_hydra_overrides_from_primary_config( @mark.parametrize("version_base", ["1.2", None]) def test_legacy_override_hydra_version_base_1_2( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, - version_base: Optional[str], + version_base: str | None, hydra_restore_singletons: Any, ) -> None: version.setbase(version_base) @@ -916,7 +916,7 @@ def test_legacy_override_hydra_version_base_1_2( ) def test_legacy_override_hydra_version_base_1_1( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, hydra_restore_singletons: Any, ) -> None: @@ -965,7 +965,7 @@ def test_legacy_override_hydra_version_base_1_1( ), ], ) -def test_legacy_hydra_overrides_from_primary_config_2(config_name: str, overrides: List[str], expected: DefaultsTreeNode, recwarn: Any) -> None: +def test_legacy_hydra_overrides_from_primary_config_2(config_name: str, overrides: list[str], expected: DefaultsTreeNode, recwarn: Any) -> None: """ Override two Hydra config groups using legacy notation """ @@ -1015,7 +1015,7 @@ def test_legacy_hydra_overrides_from_primary_config_2(config_name: str, override ) def test_group_default_with_explicit_experiment( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -1058,7 +1058,7 @@ def test_group_default_with_explicit_experiment( ) def test_group_default_with_appended_experiment( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -1101,7 +1101,7 @@ def test_group_default_with_appended_experiment( ) def test_experiment_where_primary_config_has_override( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -1146,9 +1146,9 @@ def test_experiment_where_primary_config_has_override( @mark.parametrize("version_base", ["1.2", None]) def test_use_of_custom_subgroup_of_hydra( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, - version_base: Optional[str], + version_base: str | None, hydra_restore_singletons: Any, ) -> None: version.setbase(version_base) @@ -1205,7 +1205,7 @@ def test_use_of_custom_subgroup_of_hydra( ) def test_experiment_include_absolute_config( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl(config_name=config_name, input_overrides=overrides, expected=expected) @@ -1270,7 +1270,7 @@ def test_experiment_include_absolute_config( ) def test_experiment_overriding_hydra_group( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -1314,7 +1314,7 @@ def test_experiment_overriding_hydra_group( ) def test_experiment_overriding_global_group( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -1351,7 +1351,7 @@ def test_experiment_overriding_global_group( ) def test_experiment_as_primary_config( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -1441,7 +1441,7 @@ def test_experiment_as_primary_config( ) def test_extension_use_cases( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -1518,7 +1518,7 @@ def test_extension_use_cases( ) def test_name_collision( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -1660,7 +1660,7 @@ def test_name_collision( ) def test_with_missing( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -1689,7 +1689,7 @@ def test_with_missing( ) def test_with_missing_and_skip_missing_flag( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -1767,7 +1767,7 @@ def test_with_missing_and_skip_missing_flag( ) def test_placeholder( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -2014,7 +2014,7 @@ def test_placeholder( ) def test_interpolation( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -2059,7 +2059,7 @@ def test_interpolation( ) def test_legacy_interpolation( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, hydra_restore_singletons: Any, ) -> None: @@ -2128,7 +2128,7 @@ def test_legacy_interpolation( ) def test_override_nested_to_null( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -2253,7 +2253,7 @@ def test_override_nested_to_null( ) def test_deletion( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -2297,7 +2297,7 @@ def test_deletion( ) def test_delete_non_existing( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -2328,20 +2328,11 @@ def test_delete_non_existing( ), id="missing_included_config", ), - param( - "empty", - ["+group1=not_found"], - raises( - ConfigCompositionException, - match="^In 'empty': Could not find 'group1/not_found'\n\nAvailable options in 'group1':", - ), - id="missing_included_config", - ), ], ) def test_missing_config_errors( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -2448,7 +2439,7 @@ def test_missing_config_errors( ) def test_override_errors( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -2478,7 +2469,7 @@ def test_override_errors( ) def test_load_missing_optional( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -2519,7 +2510,7 @@ def test_load_missing_optional( ) def test_overriding_group_file_with_global_header( config_name: str, - overrides: List[str], + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -2553,8 +2544,8 @@ def test_overriding_group_file_with_global_header( ], ) def test_none_config( - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -2614,8 +2605,8 @@ def test_none_config( ], ) def test_none_config_with_hydra( - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -2673,8 +2664,8 @@ def test_none_config_with_hydra( ], ) def test_defaults_with_overrides_only( - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -2728,8 +2719,8 @@ def test_defaults_with_overrides_only( ], ) def test_group_with_keyword_names( - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -2816,10 +2807,10 @@ def test_group_with_keyword_names( ], ) def test_choices( - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], expected: DefaultsTreeNode, - expected_choices: Dict[str, str], + expected_choices: dict[str, str], ) -> None: res = _test_defaults_tree_impl( config_name=config_name, @@ -2864,8 +2855,8 @@ def test_choices( ], ) def test_deprecated_package_header_keywords( - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], package_header: str, expected: DefaultsTreeNode, hydra_restore_singletons: Any, @@ -3101,8 +3092,8 @@ def test_deprecated_package_header_keywords( ], ) def test_select_multi( - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], with_hydra: bool, expected: DefaultsTreeNode, ) -> None: @@ -3207,8 +3198,8 @@ def test_select_multi( ], ) def test_select_multi_pkg( - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( @@ -3246,8 +3237,8 @@ def test_select_multi_pkg( ], ) def test_nested_override_errors( - config_name: Optional[str], - overrides: List[str], + config_name: str | None, + overrides: list[str], expected: DefaultsTreeNode, ) -> None: _test_defaults_tree_impl( diff --git a/lerna/tests/instantiate/__init__.py b/lerna/tests/instantiate/__init__.py index 65e8f64..29c053e 100644 --- a/lerna/tests/instantiate/__init__.py +++ b/lerna/tests/instantiate/__init__.py @@ -3,7 +3,7 @@ import collections.abc from dataclasses import dataclass, field from functools import partial -from typing import Any, Dict, List, NoReturn, Optional, Tuple +from typing import Any, NoReturn from omegaconf import MISSING, DictConfig, ListConfig @@ -33,7 +33,7 @@ def partial_equal(obj1: Any, obj2: Any) -> bool: if isinstance(obj1, dict): if len(obj1) != len(obj2): return False - for i in obj1.keys(): + for i in obj1: if not partial_equal(obj1[i], obj2[i]): return False return True @@ -43,7 +43,7 @@ def partial_equal(obj1: Any, obj2: Any) -> bool: return all(partial_equal(o1, o2) for o1, o2 in zip(obj1, obj2)) if not (isinstance(obj1, partial) and isinstance(obj2, partial)): return False - return all([partial_equal(getattr(obj1, attr), getattr(obj2, attr)) for attr in ["func", "args", "keywords"]]) + return all(partial_equal(getattr(obj1, attr), getattr(obj2, attr)) for attr in ["func", "args", "keywords"]) class ArgsClass: @@ -56,7 +56,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: def __repr__(self) -> str: return f"self.args={self.args},self.kwarg={self.kwargs}" - def __eq__(self, other: Any) -> Any: + def __eq__(self, other: object) -> Any: if isinstance(other, ArgsClass): return self.args == other.args and self.kwargs == other.kwargs else: @@ -141,7 +141,7 @@ class UntypedPassthroughClass: # Type not legal in a config class IllegalType: - def __eq__(self, other: Any) -> Any: + def __eq__(self, other: object) -> Any: return isinstance(other, IllegalType) @@ -161,15 +161,15 @@ def static_method(z: int) -> int: class Parameters: - def __init__(self, params: List[float]): + def __init__(self, params: list[float]): self.params = params - def __eq__(self, other: Any) -> Any: + def __eq__(self, other: object) -> Any: if isinstance(other, Parameters): return self.params == other.params return False - def __deepcopy__(self, memodict: Any = {}) -> Any: + def __deepcopy__(self, memodict: Any = None) -> Any: raise NotImplementedError("Pytorch parameters does not support deepcopy") @@ -177,7 +177,7 @@ def __deepcopy__(self, memodict: Any = {}) -> Any: class Adam: params: Parameters lr: float = 0.001 - betas: Tuple[float, ...] = (0.9, 0.999) + betas: tuple[float, ...] = (0.9, 0.999) eps: float = 1e-08 weight_decay: int = 0 amsgrad: bool = False @@ -193,7 +193,7 @@ class NestingClass: class ClassWithMissingModule: def __init__(self) -> None: - import some_missing_module # type: ignore # noqa: F401 + import some_missing_module # type: ignore self.x = 1 @@ -202,7 +202,7 @@ def __init__(self) -> None: class AdamConf: _target_: str = "lerna.tests.instantiate.Adam" lr: float = 0.001 - betas: Tuple[float, ...] = (0.9, 0.999) + betas: tuple[float, ...] = (0.9, 0.999) eps: float = 1e-08 weight_decay: int = 0 amsgrad: bool = False @@ -223,7 +223,7 @@ class User: @dataclass class UserGroup: name: str = MISSING - users: List[User] = MISSING + users: list[User] = MISSING # RECURSIVE @@ -235,7 +235,7 @@ class CenterCrop(Transform): def __init__(self, size: int): self.size = size - def __eq__(self, other: Any) -> Any: + def __eq__(self, other: object) -> Any: if isinstance(other, type(self)): return self.size == other.size else: @@ -246,7 +246,7 @@ class Rotation(Transform): def __init__(self, degrees: int): self.degrees = degrees - def __eq__(self, other: Any) -> Any: + def __eq__(self, other: object) -> Any: if isinstance(other, type(self)): return self.degrees == other.degrees else: @@ -254,12 +254,12 @@ def __eq__(self, other: Any) -> Any: class Compose: - transforms: List[Transform] + transforms: list[Transform] - def __init__(self, transforms: List[Transform]): + def __init__(self, transforms: list[Transform]): self.transforms = transforms - def __eq__(self, other: Any) -> Any: + def __eq__(self, other: object) -> Any: return partial_equal(self.transforms, other.transforms) @@ -274,7 +274,7 @@ def __init__(self, value: Any, left: Any = None, right: Any = None) -> None: self.left = left self.right = right - def __eq__(self, other: Any) -> Any: + def __eq__(self, other: object) -> Any: if isinstance(other, type(self)): return partial_equal(self.value, other.value) and partial_equal(self.left, other.left) and partial_equal(self.right, other.right) @@ -286,14 +286,14 @@ def __repr__(self) -> str: class Mapping: - dictionary: Optional[Dict[str, "Mapping"]] = None + dictionary: dict[str, "Mapping"] | None = None value: Any = None - def __init__(self, value: Any = None, dictionary: Optional[Dict[str, "Mapping"]] = None) -> None: + def __init__(self, value: Any = None, dictionary: dict[str, "Mapping"] | None = None) -> None: self.dictionary = dictionary self.value = value - def __eq__(self, other: Any) -> Any: + def __eq__(self, other: object) -> Any: if isinstance(other, type(self)): return partial_equal(self.dictionary, other.dictionary) and partial_equal(self.value, other.value) else: @@ -325,15 +325,15 @@ class RotationConf(TransformConf): class ComposeConf: _target_: str = "lerna.tests.instantiate.Compose" _partial_: bool = False - transforms: List[TransformConf] = MISSING + transforms: list[TransformConf] = MISSING @dataclass class TreeConf: _target_: str = "lerna.tests.instantiate.Tree" _partial_: bool = False - left: Optional["TreeConf"] = None - right: Optional["TreeConf"] = None + left: "TreeConf | None" = None + right: "TreeConf | None" = None value: Any = MISSING @@ -341,11 +341,11 @@ class TreeConf: class MappingConf: _target_: str = "lerna.tests.instantiate.Mapping" _partial_: bool = False - dictionary: Optional[Dict[str, "MappingConf"]] = None + dictionary: dict[str, "MappingConf"] | None = None def __init__( self, - dictionary: Optional[Dict[str, "MappingConf"]] = None, + dictionary: dict[str, "MappingConf"] | None = None, _partial_: bool = False, ): self.dictionary = dictionary @@ -366,13 +366,13 @@ def __init__(self, a: Any, b: Any) -> None: self.a = a self.b = b - def __eq__(self, other: Any) -> Any: + def __eq__(self, other: object) -> Any: if isinstance(other, SimpleClass): return self.a == other.a and self.b == other.b return False @property - def _fields(self) -> List[str]: + def _fields(self) -> list[str]: return ["a", "b"] @@ -407,13 +407,13 @@ class NestedConf: class TargetWithInstantiateInInit: - def __init__(self, user_config: Optional[DictConfig], user: Optional[User] = None) -> None: + def __init__(self, user_config: DictConfig | None, user: User | None = None) -> None: if user: self.user = user else: self.user = instantiate(user_config) - def __eq__(self, other: Any) -> bool: + def __eq__(self, other: object) -> bool: return self.user.__eq__(other.user) diff --git a/lerna/tests/instantiate/positional_only.py b/lerna/tests/instantiate/positional_only.py index 65e58fd..9679555 100644 --- a/lerna/tests/instantiate/positional_only.py +++ b/lerna/tests/instantiate/positional_only.py @@ -12,7 +12,7 @@ def __init__(self, a: Any, b: Any, /, **kwargs: Any) -> None: def __repr__(self) -> str: return f"{self.a=},{self.b},{self.kwargs=}" - def __eq__(self, other: Any) -> Any: + def __eq__(self, other: object) -> Any: if isinstance(other, PosOnlyArgsClass): return self.a == other.a and self.b == other.b and self.kwargs == other.kwargs else: diff --git a/lerna/tests/instantiate/test_instantiate.py b/lerna/tests/instantiate/test_instantiate.py index 3157ff6..8ac0b13 100644 --- a/lerna/tests/instantiate/test_instantiate.py +++ b/lerna/tests/instantiate/test_instantiate.py @@ -2,10 +2,11 @@ import copy import pickle import re +from collections.abc import Callable from dataclasses import dataclass from functools import partial from textwrap import dedent -from typing import Any, Callable, Dict, List, Optional, Tuple +from typing import Any from omegaconf import MISSING, DictConfig, ListConfig, MissingMandatoryValue, OmegaConf from pytest import fixture, mark, param, raises, warns @@ -413,7 +414,7 @@ def config(request: Any, src: Any) -> Any: def test_class_instantiate( instantiate_func: Any, config: Any, - passthrough: Dict[str, Any], + passthrough: dict[str, Any], expected: Any, recursive: bool, ) -> Any: @@ -597,7 +598,7 @@ def test_none_cases( def test_interpolation_accessing_parent( instantiate_func: Any, input_conf: Any, - passthrough: Dict[str, Any], + passthrough: dict[str, Any], expected: Any, convert_to_list: bool, skip_deepcopy: bool, @@ -1071,7 +1072,7 @@ def test_instantiate_with_callable_target_keyword(instantiate_func: Any, target: def test_recursive_instantiation( instantiate_func: Any, config: Any, - passthrough: Dict[str, Any], + passthrough: dict[str, Any], expected: Any, ) -> None: obj = instantiate_func(config, **passthrough) @@ -1258,7 +1259,7 @@ def test_recursive_instantiation( def test_partial_instantiate( instantiate_func: Any, config: Any, - passthrough: Dict[str, Any], + passthrough: dict[str, Any], expected: Any, ) -> None: obj = instantiate_func(config, **passthrough) @@ -1657,7 +1658,7 @@ def test_allowlist_works(instantiate_func: Any, monkeypatch: Any) -> None: ) def test_convert_params_override( instantiate_func: Any, - primitive: Optional[bool], + primitive: bool | None, expected_primitive: bool, input_: Any, expected: Any, @@ -1884,7 +1885,7 @@ def test_convert_and_recursive_node(instantiate_func: Any, nested_recursive: boo ), ], ) -def test_instantiate_convert_dataclasses(instantiate_func: Any, config: Any, expected: Tuple[Any, Any, Any, Any]) -> None: +def test_instantiate_convert_dataclasses(instantiate_func: Any, config: Any, expected: tuple[Any, Any, Any, Any]) -> None: """Instantiate on nested dataclass + dataclass.""" modes = [ConvertMode.NONE, ConvertMode.PARTIAL, ConvertMode.OBJECT, ConvertMode.ALL] assert len(modes) == len(expected) @@ -2102,12 +2103,12 @@ def test_nested_dataclass_with_partial_convert(instantiate_func: Any) -> None: class DictValues: - def __init__(self, d: Dict[str, User]): + def __init__(self, d: dict[str, User]): self.d = d class ListValues: - def __init__(self, d: List[User]): + def __init__(self, d: list[User]): self.d = d @@ -2115,7 +2116,7 @@ def test_dict_with_structured_config(instantiate_func: Any) -> None: @dataclass class DictValuesConf: _target_: str = "lerna.tests.instantiate.test_instantiate.DictValues" - d: Dict[str, User] = MISSING + d: dict[str, User] = MISSING schema = OmegaConf.structured(DictValuesConf) cfg = OmegaConf.merge(schema, {"d": {"007": {"name": "Bond", "age": 7}}}) @@ -2136,7 +2137,7 @@ def test_list_with_structured_config(instantiate_func: Any) -> None: @dataclass class ListValuesConf: _target_: str = "lerna.tests.instantiate.test_instantiate.ListValues" - d: List[User] = MISSING + d: list[User] = MISSING schema = OmegaConf.structured(ListValuesConf) cfg = OmegaConf.merge(schema, {"d": [{"name": "Bond", "age": 7}]}) @@ -2158,7 +2159,7 @@ def test_list_as_none(instantiate_func: Any) -> None: @dataclass class ListValuesConf: _target_: str = "lerna.tests.instantiate.test_instantiate.ListValues" - d: Optional[List[User]] = None + d: list[User] | None = None cfg = OmegaConf.structured(ListValuesConf) obj = instantiate_func(config=cfg) @@ -2169,7 +2170,7 @@ def test_dict_as_none(instantiate_func: Any) -> None: @dataclass class DictValuesConf: _target_: str = "lerna.tests.instantiate.test_instantiate.DictValues" - d: Optional[Dict[str, User]] = None + d: dict[str, User] | None = None cfg = OmegaConf.structured(DictValuesConf) obj = instantiate_func(config=cfg) diff --git a/lerna/tests/standalone_apps/initialization_test_app/setup.py b/lerna/tests/standalone_apps/initialization_test_app/setup.py index 74480d9..3a18123 100644 --- a/lerna/tests/standalone_apps/initialization_test_app/setup.py +++ b/lerna/tests/standalone_apps/initialization_test_app/setup.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from setuptools import find_packages, setup diff --git a/lerna/tests/standalone_apps/initialization_test_app/tests/test_app.py b/lerna/tests/standalone_apps/initialization_test_app/tests/test_app.py index d0f1e31..cb3c24d 100644 --- a/lerna/tests/standalone_apps/initialization_test_app/tests/test_app.py +++ b/lerna/tests/standalone_apps/initialization_test_app/tests/test_app.py @@ -1,7 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import subprocess import sys -from typing import List from pytest import mark, param @@ -20,5 +19,5 @@ ), ], ) -def test_initialization_full_app_installed(run_cmd: List[str]) -> None: +def test_initialization_full_app_installed(run_cmd: list[str]) -> None: subprocess.check_call(run_cmd) diff --git a/lerna/tests/test_apps/app_exception/my_app.py b/lerna/tests/test_apps/app_exception/my_app.py index e040c0d..2c0573f 100644 --- a/lerna/tests/test_apps/app_exception/my_app.py +++ b/lerna/tests/test_apps/app_exception/my_app.py @@ -1,4 +1,4 @@ -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved # ruff: noqa: B018 from omegaconf import DictConfig import lerna diff --git a/lerna/tests/test_apps/app_with_callbacks/app_with_log_compose_callback/my_app.py b/lerna/tests/test_apps/app_with_callbacks/app_with_log_compose_callback/my_app.py index 59b8829..16ff9d2 100644 --- a/lerna/tests/test_apps/app_with_callbacks/app_with_log_compose_callback/my_app.py +++ b/lerna/tests/test_apps/app_with_callbacks/app_with_log_compose_callback/my_app.py @@ -1,6 +1,5 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from dataclasses import dataclass -from typing import Dict from omegaconf import MISSING @@ -13,7 +12,7 @@ class Config: age: int = MISSING name: str = MISSING - group: Dict[str, str] = MISSING + group: dict[str, str] = MISSING ConfigStore.instance().store(name="config_schema", node=Config) diff --git a/lerna/tests/test_apps/defaults_in_schema_missing/my_app.py b/lerna/tests/test_apps/defaults_in_schema_missing/my_app.py index fa3152a..e92db33 100644 --- a/lerna/tests/test_apps/defaults_in_schema_missing/my_app.py +++ b/lerna/tests/test_apps/defaults_in_schema_missing/my_app.py @@ -1,6 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from dataclasses import dataclass -from typing import Any, List +from typing import Any from omegaconf import MISSING, DictConfig, OmegaConf @@ -25,7 +25,7 @@ class MySQLConfig(DBConfig): @dataclass class Config: - defaults: List[Any] = MISSING + defaults: list[Any] = MISSING db: DBConfig = MISSING diff --git a/lerna/tests/test_apps/schema_overrides_hydra/my_app.py b/lerna/tests/test_apps/schema_overrides_hydra/my_app.py index fcdf1fc..998a77f 100644 --- a/lerna/tests/test_apps/schema_overrides_hydra/my_app.py +++ b/lerna/tests/test_apps/schema_overrides_hydra/my_app.py @@ -1,6 +1,5 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from dataclasses import dataclass -from typing import Dict from omegaconf import MISSING @@ -13,7 +12,7 @@ class Config: age: int = MISSING name: str = MISSING - group: Dict[str, str] = MISSING + group: dict[str, str] = MISSING ConfigStore.instance().store(name="config_schema", node=Config) diff --git a/lerna/tests/test_apps/structured_with_none_list/my_app.py b/lerna/tests/test_apps/structured_with_none_list/my_app.py index 8a01d92..6594d4a 100644 --- a/lerna/tests/test_apps/structured_with_none_list/my_app.py +++ b/lerna/tests/test_apps/structured_with_none_list/my_app.py @@ -1,6 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import builtins from dataclasses import dataclass -from typing import List, Optional from omegaconf import DictConfig @@ -10,7 +10,7 @@ @dataclass class Config: - list: Optional[List[int]] = None + list: builtins.list[int] | None = None cs = ConfigStore.instance() diff --git a/lerna/tests/test_basic_launcher.py b/lerna/tests/test_basic_launcher.py index 23da6dd..76cf208 100644 --- a/lerna/tests/test_basic_launcher.py +++ b/lerna/tests/test_basic_launcher.py @@ -33,8 +33,6 @@ class TestBasicLauncherIntegration(IntegrationTestSuite): Run this launcher through the integration test suite. """ - pass - @mark.parametrize( "launcher_name, overrides", diff --git a/lerna/tests/test_basic_sweeper.py b/lerna/tests/test_basic_sweeper.py index 626e5ed..0934219 100644 --- a/lerna/tests/test_basic_sweeper.py +++ b/lerna/tests/test_basic_sweeper.py @@ -2,7 +2,7 @@ import re import sys from textwrap import dedent -from typing import Any, List, Optional +from typing import Any from pytest import mark, param @@ -50,7 +50,7 @@ param(["a=range(3)"], None, [[["a=0"], ["a=1"], ["a=2"]]], id="range_no_start"), ], ) -def test_split(args: List[str], max_batch_size: Optional[int], expected: List[List[List[str]]]) -> None: +def test_split(args: list[str], max_batch_size: int | None, expected: list[list[list[str]]]) -> None: parser = OverridesParser.create() ret = BasicSweeper.split_arguments(parser.parse_overrides(args), max_batch_size=max_batch_size) lret = [list(x) for x in ret] @@ -66,7 +66,7 @@ def test_partial_failure( "lerna/tests/test_apps/app_can_fail/my_app.py", "--multirun", "+divisor=1,0", - f'hydra.run.dir="{str(tmpdir)}"', + f'hydra.run.dir="{tmpdir!s}"', "hydra.job.chdir=True", "hydra.hydra_logging.formatters.simple.format='[HYDRA] %(message)s'", ] diff --git a/lerna/tests/test_callbacks.py b/lerna/tests/test_callbacks.py index 2ae16fe..53e8242 100644 --- a/lerna/tests/test_callbacks.py +++ b/lerna/tests/test_callbacks.py @@ -5,7 +5,7 @@ import sys from pathlib import Path from textwrap import dedent -from typing import Any, List +from typing import Any from omegaconf import open_dict, read_write from pytest import mark, param @@ -115,7 +115,7 @@ def test_app_with_callbacks( tmpdir: Path, app_path: str, - args: List[str], + args: list[str], expected: str, ) -> None: cmd = [ @@ -161,9 +161,8 @@ def load_pickle(path: Path) -> Any: job_return_on_job_end: JobReturn = load_pickle(callback_output / "job_return.pickle") task_cfg_from_callback = copy.deepcopy(config_on_job_start) - with read_write(task_cfg_from_callback): - with open_dict(task_cfg_from_callback): - del task_cfg_from_callback["hydra"] + with read_write(task_cfg_from_callback), open_dict(task_cfg_from_callback): + del task_cfg_from_callback["hydra"] # load pickles generated from the application app_output_dir = tmpdir / "0" if multirun else tmpdir @@ -215,7 +214,7 @@ def test_save_job_return_callback(tmpdir: Path, multirun: bool) -> None: ("Config overrides are not supported as of now", ["+x=1"]), ], ) -def test_experimental_rerun(tmpdir: Path, warning_msg: str, overrides: List[str]) -> None: +def test_experimental_rerun(tmpdir: Path, warning_msg: str, overrides: list[str]) -> None: app_path = "lerna/tests/test_apps/app_with_pickle_job_info_callback/my_app.py" cmd = [ @@ -247,7 +246,7 @@ def test_experimental_rerun(tmpdir: Path, warning_msg: str, overrides: List[str] str(config_file), ] cmd.extend(overrides) - result, err = run_python_script(cmd, allow_warnings=True) + _result, err = run_python_script(cmd, allow_warnings=True) assert warning_msg in err with open(log_file) as file: diff --git a/lerna/tests/test_completion.py b/lerna/tests/test_completion.py index 9296497..64cf0d8 100644 --- a/lerna/tests/test_completion.py +++ b/lerna/tests/test_completion.py @@ -5,7 +5,6 @@ import subprocess import sys from pathlib import Path -from typing import List from packaging import version from pytest import mark, param, skip, xfail @@ -27,7 +26,7 @@ def is_fish_supported() -> bool: if shutil.which("fish") is None: return False - proc = subprocess.run(["fish", "--version"], stdout=subprocess.PIPE, encoding="utf-8") + proc = subprocess.run(["fish", "--version"], stdout=subprocess.PIPE, encoding="utf-8", check=False) matches = re.match(r".*version\s+(\d\.\d\.\d)(.*)", proc.stdout) if not matches: return False @@ -35,19 +34,14 @@ def is_fish_supported() -> bool: fish_version, git_version = matches.groups() # Release after 3.1.2 or git build after 3.1.2 contain space fix. - if version.parse(fish_version) > version.parse("3.1.2"): - return True - elif version.parse(fish_version) >= version.parse("3.1.2") and git_version: - return True - else: - return False + return bool(version.parse(fish_version) > version.parse("3.1.2") or version.parse(fish_version) >= version.parse("3.1.2") and git_version) def is_zsh_supported() -> bool: if shutil.which("zsh") is None: return False - proc = subprocess.run(["zsh", "--version"], stdout=subprocess.PIPE, encoding="utf-8") + proc = subprocess.run(["zsh", "--version"], stdout=subprocess.PIPE, encoding="utf-8", check=False) matches = re.match(r"zsh\s+(\d\.\d(\.\d)?)", proc.stdout) if not matches: return False @@ -55,10 +49,7 @@ def is_zsh_supported() -> bool: zsh_version = matches.groups()[0] # Support for Bash completion functions introduced in Zsh 4.2 - if version.parse(zsh_version) > version.parse("4.2"): - return True - else: - return False + return version.parse(zsh_version) > version.parse("4.2") def create_config_loader() -> ConfigLoaderImpl: @@ -79,10 +70,9 @@ def test_bash_completion_with_dot_in_path() -> None: stdout, stderr = process.communicate() assert stderr == b"" assert stdout == b"TRUE\n" - return -base_completion_list: List[str] = [ +base_completion_list: list[str] = [ "dict.", "dict_prefix=", "group=", @@ -150,7 +140,6 @@ def test_bash_completion_with_dot_in_path() -> None: param("group=", 2, ["group=dict", "group=list"], id="group"), param("group=dict group.dict=", 2, ["group.dict=true"], id="group"), param("group=dict group=", 2, ["group=dict", "group=list"], id="group"), - param("group=dict group=", 2, ["group=dict", "group=list"], id="group"), param("+", 2, ["+group=", "+hydra", "+test_hydra/"], id="bare_plus"), param("+gro", 2, ["+group="], id="append_group_partial"), param("+group=di", 2, ["+group=dict"], id="append_group_partial_option"), @@ -190,7 +179,7 @@ def test_bash_completion_with_dot_in_path() -> None: ], ) class TestRunCompletion: - def test_completion_plugin(self, line_prefix: str, num_tabs: int, line: str, expected: List[str]) -> None: + def test_completion_plugin(self, line_prefix: str, num_tabs: int, line: str, expected: list[str]) -> None: config_loader = create_config_loader() bc = DefaultCompletionPlugin(config_loader) ret = bc._query(config_name="config.yaml", line=line_prefix + line) @@ -217,11 +206,11 @@ def test_completion_plugin(self, line_prefix: str, num_tabs: int, line: str, exp def test_shell_integration( self, shell: str, - prog: List[str], + prog: list[str], num_tabs: int, line_prefix: str, line: str, - expected: List[str], + expected: list[str], ) -> None: if shell == "fish" and not is_fish_supported(): skip("fish is not installed or the version is too old") @@ -286,7 +275,7 @@ def test_shell_integration( ], ) class TestMultirunCompletion: - def test_completion_plugin_multirun(self, line: str, expected: List[str]) -> None: + def test_completion_plugin_multirun(self, line: str, expected: list[str]) -> None: config_loader = create_config_loader() bc = DefaultCompletionPlugin(config_loader) ret = bc._query(config_name="config.yaml", line="--multirun " + line) @@ -301,7 +290,7 @@ def test_completion_plugin_multirun(self, line: str, expected: List[str]) -> Non ("-c all ", base_completion_list), ], ) -def test_with_flags(line: str, expected: List[str]) -> None: +def test_with_flags(line: str, expected: list[str]) -> None: config_loader = create_config_loader() bc = DefaultCompletionPlugin(config_loader) ret = bc._query(config_name="config.yaml", line=line) @@ -321,7 +310,7 @@ def test_with_flags(line: str, expected: List[str]) -> None: ("group=dict toys.", ["toys.andy=", "toys.list.", "toys.slinky="]), ], ) -def test_missing_default_value(line: str, expected: List[str]) -> None: +def test_missing_default_value(line: str, expected: list[str]) -> None: config_loader = create_config_loader() bc = DefaultCompletionPlugin(config_loader) ret = bc._query(config_name="missing_default", line=line) @@ -354,7 +343,7 @@ def test_missing_default_value(line: str, expected: List[str]) -> None: ), ], ) -def test_searchpath_addition(line: str, expected: List[str]) -> None: +def test_searchpath_addition(line: str, expected: list[str]) -> None: config_loader = create_config_loader() bc = DefaultCompletionPlugin(config_loader) ret = bc._query(config_name="additional_searchpath", line=line) @@ -376,14 +365,14 @@ def test_searchpath_addition(line: str, expected: List[str]) -> None: ) def test_file_completion( tmpdir: Path, - files: List[str], + files: list[str], line_prefix: str, key_eq: str, fname_prefix: str, - expected: List[str], + expected: list[str], relative: bool, ) -> None: - def create_files(in_files: List[str]) -> None: + def create_files(in_files: list[str]) -> None: for f in in_files: path = Path(f) dirname = path.parent @@ -425,7 +414,6 @@ def create_files(in_files: List[str]) -> None: "f_o-o1=2.par", "python foo.py", "python tutorials/hydra_app/example/hydra_app/main.py", - "python foo.py", ], ) @mark.parametrize( diff --git a/lerna/tests/test_compose.py b/lerna/tests/test_compose.py index ba94840..1c17c67 100644 --- a/lerna/tests/test_compose.py +++ b/lerna/tests/test_compose.py @@ -6,7 +6,7 @@ from enum import Enum from pathlib import Path from textwrap import dedent -from typing import Any, Dict, List, Optional +from typing import Any from omegaconf import MISSING, OmegaConf from pytest import fixture, mark, param, raises, warns @@ -34,7 +34,7 @@ @fixture -def initialize_hydra(config_path: Optional[str]) -> Any: +def initialize_hydra(config_path: str | None) -> Any: init = None try: init = initialize(version_base=None, config_path=config_path) @@ -138,13 +138,13 @@ class TestCompose: def test_compose_config( self, config_file: str, - overrides: List[str], + overrides: list[str], expected: Any, ) -> None: cfg = compose(config_file, overrides) assert cfg == expected - def test_strict_failure_global_strict(self, config_file: str, overrides: List[str], expected: Any) -> None: + def test_strict_failure_global_strict(self, config_file: str, overrides: list[str], expected: Any) -> None: # default strict True, call is unspecified overrides.append("fooooooooo=bar") with raises(HydraException): @@ -231,7 +231,7 @@ def test_top_level_config_is_list() -> None: ], ) class TestComposeInits: - def test_initialize_ctx(self, config_file: str, overrides: List[str], expected: Any) -> None: + def test_initialize_ctx(self, config_file: str, overrides: list[str], expected: Any) -> None: with initialize( version_base=None, config_path="../../examples/jupyter_notebooks/cloud_app/conf", @@ -239,20 +239,22 @@ def test_initialize_ctx(self, config_file: str, overrides: List[str], expected: ret = compose(config_file, overrides) assert ret == expected - def test_initialize_config_dir_ctx_with_relative_dir(self, config_file: str, overrides: List[str], expected: Any) -> None: - with raises( - HydraException, - match=re.escape("initialize_config_dir() requires an absolute config_dir as input"), - ): - with initialize_config_dir( + def test_initialize_config_dir_ctx_with_relative_dir(self, config_file: str, overrides: list[str], expected: Any) -> None: + with ( + raises( + HydraException, + match=re.escape("initialize_config_dir() requires an absolute config_dir as input"), + ), + initialize_config_dir( config_dir="../../examples/jupyter_notebooks/cloud_app/conf", version_base=None, job_name="job_name", - ): - ret = compose(config_file, overrides) - assert ret == expected + ), + ): + ret = compose(config_file, overrides) + assert ret == expected - def test_initialize_config_module_ctx(self, config_file: str, overrides: List[str], expected: Any) -> None: + def test_initialize_config_module_ctx(self, config_file: str, overrides: list[str], expected: Any) -> None: with initialize_config_module( config_module="examples.jupyter_notebooks.cloud_app.conf", version_base=None, @@ -263,9 +265,11 @@ def test_initialize_config_module_ctx(self, config_file: str, overrides: List[st def test_initialize_ctx_with_absolute_dir(hydra_restore_singletons: Any, tmpdir: Any) -> None: - with raises(HydraException, match=re.escape("config_path in initialize() must be relative")): - with initialize(version_base=None, config_path=str(tmpdir)): - compose(overrides=["+test_group=test"]) + with ( + raises(HydraException, match=re.escape("config_path in initialize() must be relative")), + initialize(version_base=None, config_path=str(tmpdir)), + ): + compose(overrides=["+test_group=test"]) def test_initialize_config_dir_ctx_with_absolute_dir(hydra_restore_singletons: Any, tmpdir: Any) -> None: @@ -286,7 +290,7 @@ def test_initialize_config_dir_ctx_with_absolute_dir(hydra_restore_singletons: A @mark.parametrize("job_name,expected", [(None, "test_compose"), ("test_job", "test_job")]) -def test_jobname_override_initialize_ctx(hydra_restore_singletons: Any, job_name: Optional[str], expected: str) -> None: +def test_jobname_override_initialize_ctx(hydra_restore_singletons: Any, job_name: str | None, expected: str) -> None: with initialize( version_base=None, config_path="../../examples/jupyter_notebooks/cloud_app/conf", @@ -330,14 +334,16 @@ def test_initialize_config_module_ctx(hydra_restore_singletons: Any) -> None: def test_missing_init_py_error(hydra_restore_singletons: Any) -> None: expected = "Primary config module 'lerna.test_utils.configs.missing_init_py' not found.\nCheck that it's correct and contains an __init__.py file" - with raises(Exception, match=re.escape(expected)): - with initialize_config_module( + with ( + raises(Exception, match=re.escape(expected)), + initialize_config_module( config_module="lerna.test_utils.configs.missing_init_py", version_base=None, - ): - hydra = GlobalHydra.instance().hydra - assert hydra is not None - compose(config_name="test.yaml", overrides=[]) + ), + ): + hydra = GlobalHydra.instance().hydra + assert hydra is not None + compose(config_name="test.yaml", overrides=[]) def test_missing_bad_config_dir_error(hydra_restore_singletons: Any) -> None: @@ -349,14 +355,16 @@ def test_missing_bad_config_dir_error(hydra_restore_singletons: Any) -> None: expected = f"Primary config directory not found.\nCheck that the config directory '{bad_dir}' exists and readable" - with raises(Exception, match=re.escape(expected)): - with initialize_config_dir( + with ( + raises(Exception, match=re.escape(expected)), + initialize_config_dir( config_dir=bad_dir, version_base=None, - ): - hydra = GlobalHydra.instance().hydra - assert hydra is not None - compose(config_name="test.yaml", overrides=[]) + ), + ): + hydra = GlobalHydra.instance().hydra + assert hydra is not None + compose(config_name="test.yaml", overrides=[]) def test_initialize_with_module(hydra_restore_singletons: Any) -> None: @@ -390,10 +398,10 @@ def test_initialization_root_module(monkeypatch: Any) -> None: param(["map.foo=bar"], raises(ConfigCompositionException), id="add_no_plus"), ], ) -def test_adding_to_sc_dict(hydra_restore_singletons: Any, overrides: List[str], expected: Any) -> None: +def test_adding_to_sc_dict(hydra_restore_singletons: Any, overrides: list[str], expected: Any) -> None: @dataclass class Config: - map: Dict[str, str] = field(default_factory=dict) + map: dict[str, str] = field(default_factory=dict) ConfigStore.instance().store(name="config", node=Config) @@ -431,7 +439,7 @@ class Config: ), ], ) -def test_extending_list(hydra_restore_singletons: Any, overrides: List[str], expected: Any) -> None: +def test_extending_list(hydra_restore_singletons: Any, overrides: list[str], expected: Any) -> None: @dataclass class Config: list_key: Any = field(default_factory=lambda: ["a", "b", "c"]) @@ -555,7 +563,7 @@ def test_searchpath_in_primary_config( self, init_configs: Any, config_name: str, - overrides: List[str], + overrides: list[str], expected: Any, ) -> None: cfg = compose(config_name=config_name, overrides=overrides) @@ -615,7 +623,7 @@ def test_searchpath_config_errors( self, init_configs: Any, config_name: str, - overrides: List[str], + overrides: list[str], expected: Any, ) -> None: with expected: @@ -640,19 +648,23 @@ def test_deprecated_compose(hydra_restore_singletons: Any) -> None: msg = "hydra.experimental.compose() is no longer experimental. Use hydra.compose()" - with initialize(version_base="1.1"): - with warns( + with ( + initialize(version_base="1.1"), + warns( expected_warning=UserWarning, match=re.escape(msg), - ): - assert expr_compose() == {} + ), + ): + assert expr_compose() == {} - with initialize(version_base="1.2"): - with raises( + with ( + initialize(version_base="1.2"), + raises( ImportError, match=re.escape(msg), - ): - assert expr_compose() == {} + ), + ): + assert expr_compose() == {} def test_deprecated_initialize(hydra_restore_singletons: Any) -> None: @@ -661,14 +673,12 @@ def test_deprecated_initialize(hydra_restore_singletons: Any) -> None: msg = "hydra.experimental.initialize() is no longer experimental. Use hydra.initialize()" version.setbase("1.1") - with warns(expected_warning=UserWarning, match=re.escape(msg)): - with expr_initialize(): - assert compose() == {} + with warns(expected_warning=UserWarning, match=re.escape(msg)), expr_initialize(): + assert compose() == {} version.setbase("1.2") - with raises(ImportError, match=re.escape(msg)): - with expr_initialize(): - assert compose() == {} + with raises(ImportError, match=re.escape(msg)), expr_initialize(): + assert compose() == {} def test_deprecated_initialize_config_dir(hydra_restore_singletons: Any) -> None: @@ -677,24 +687,28 @@ def test_deprecated_initialize_config_dir(hydra_restore_singletons: Any) -> None msg = "hydra.experimental.initialize_config_dir() is no longer experimental. Use hydra.initialize_config_dir()" version.setbase("1.1") - with warns( - expected_warning=UserWarning, - match=re.escape(msg), - ): - with expr_initialize_config_dir( + with ( + warns( + expected_warning=UserWarning, + match=re.escape(msg), + ), + expr_initialize_config_dir( config_dir=str(Path(".").absolute()), - ): - assert compose() == {} + ), + ): + assert compose() == {} version.setbase("1.2") - with raises( - ImportError, - match=re.escape(msg), - ): - with expr_initialize_config_dir( + with ( + raises( + ImportError, + match=re.escape(msg), + ), + expr_initialize_config_dir( config_dir=str(Path(".").absolute()), - ): - assert compose() == {} + ), + ): + assert compose() == {} def test_deprecated_initialize_config_module(hydra_restore_singletons: Any) -> None: @@ -705,18 +719,22 @@ def test_deprecated_initialize_config_module(hydra_restore_singletons: Any) -> N msg = "hydra.experimental.initialize_config_module() is no longer experimental. Use hydra.initialize_config_module()" version.setbase("1.1") - with warns(expected_warning=UserWarning, match=re.escape(msg)): - with expr_initialize_config_module( + with ( + warns(expected_warning=UserWarning, match=re.escape(msg)), + expr_initialize_config_module( config_module="examples.jupyter_notebooks.cloud_app.conf", - ): - assert compose() == {} + ), + ): + assert compose() == {} version.setbase("1.2") - with raises(ImportError, match=re.escape(msg)): - with expr_initialize_config_module( + with ( + raises(ImportError, match=re.escape(msg)), + expr_initialize_config_module( config_module="examples.jupyter_notebooks.cloud_app.conf", - ): - assert compose() == {} + ), + ): + assert compose() == {} def test_initialize_without_config_path(tmpdir: Path) -> None: @@ -731,9 +749,8 @@ def test_initialize_without_config_path(tmpdir: Path) -> None: config_path is not specified in hydra.initialize(). See https://hydra.cc/docs/1.2/upgrades/1.0_to_1.1/changes_to_hydra_main_config_path for more information.""" ) - with warns(expected_warning=UserWarning) as record: - with initialize(): - pass + with warns(expected_warning=UserWarning) as record, initialize(): + pass assert len(record) == 2 assert str(record[0].message) == expected0 assert str(record[1].message) == expected1 @@ -761,7 +778,7 @@ def test_initialize_without_config_path(tmpdir: Path) -> None: ), ], ) -def test_error_assigning_null_to_logging_config(hydra_restore_singletons: Any, overrides: List[str], expected: Any) -> None: +def test_error_assigning_null_to_logging_config(hydra_restore_singletons: Any, overrides: list[str], expected: Any) -> None: with expected: compose(overrides=overrides) @@ -793,12 +810,12 @@ def test_deprecated_compose_strict_flag(strict: bool, hydra_restore_singletons: def test_missing_node_with_defaults_list(hydra_restore_singletons: Any) -> None: @dataclass class Reducer: - defaults: List[Any] = field(default_factory=lambda: []) + defaults: list[Any] = field(default_factory=list) @dataclass class Trainer: reducer: Reducer = MISSING - defaults: List[Any] = field(default_factory=lambda: [{"/reducer": "base_reducer"}]) + defaults: list[Any] = field(default_factory=lambda: [{"/reducer": "base_reducer"}]) cs = ConfigStore.instance() cs.store(name="base_trainer", node=Trainer(), group="trainer") @@ -817,9 +834,9 @@ class Category(Enum): @dataclass class Conf: - enum_dict: Dict[Category, str] = field(default_factory=dict) - int_dict: Dict[int, str] = field(default_factory=dict) - str_dict: Dict[str, str] = field(default_factory=dict) + enum_dict: dict[Category, str] = field(default_factory=dict) + int_dict: dict[int, str] = field(default_factory=dict) + str_dict: dict[str, str] = field(default_factory=dict) cs = ConfigStore.instance() cs.store(name="conf", node=Conf) diff --git a/lerna/tests/test_config_loader.py b/lerna/tests/test_config_loader.py index 8f6ca51..af54f15 100644 --- a/lerna/tests/test_config_loader.py +++ b/lerna/tests/test_config_loader.py @@ -2,7 +2,7 @@ import re from dataclasses import dataclass, field from textwrap import dedent -from typing import Any, List +from typing import Any from omegaconf import MISSING, OmegaConf, ValidationError, open_dict from pytest import mark, param, raises, warns @@ -95,7 +95,7 @@ def test_load_with_optional_default(self, path: str) -> None: ), ], ) - def test_override_compose_two_package_one_group(self, path: str, overrides: List[str], expected: Any) -> None: + def test_override_compose_two_package_one_group(self, path: str, overrides: list[str], expected: Any) -> None: config_loader = ConfigLoaderImpl(config_search_path=create_config_search_path(f"{path}/package_tests")) cfg = config_loader.load_configuration( config_name="two_packages_one_group", @@ -154,7 +154,7 @@ def test_load_strict(self, path: str) -> None: # Test that accessing a key that is not there will fail with raises(AttributeError): # noinspection PyStatementEffect - cfg.not_here + _ = cfg.not_here # Test that bad overrides triggers the KeyError with raises(HydraException): @@ -248,7 +248,7 @@ def test_load_config_file_with_schema_validation(self, hydra_restore_singletons: dedent( """ This behavior is deprecated in Hydra 1.1 and will be removed in Hydra 1.2. - See https://hydra.cc/docs/1.2/upgrades/1.0_to_1.1/automatic_schema_matching for migration instructions.""" # noqa: E501 line too long + See https://hydra.cc/docs/1.2/upgrades/1.0_to_1.1/automatic_schema_matching for migration instructions.""" ) ) with warns(UserWarning, match=msg): @@ -343,13 +343,13 @@ def test_sweep_config_cache(self, hydra_restore_singletons: Any, path: str, monk assert isinstance(master_cfg.test_uncached, str) # "2nd" master_cfg_cache = OmegaConf.get_cache(master_cfg) - assert "now" in master_cfg_cache.keys() + assert "now" in master_cfg_cache # oc.env is not cached as of OmegaConf 2.1 - assert "oc.env" not in master_cfg_cache.keys() + assert "oc.env" not in master_cfg_cache assert master_cfg.test_env == "test_env" - assert "cached" in master_cfg_cache.keys() + assert "cached" in master_cfg_cache assert master_cfg.test_cached == "1st" # use cached value - assert "uncached" not in master_cfg_cache.keys() + assert "uncached" not in master_cfg_cache assert master_cfg.test_uncached == "3rd" # use `next` value sweep_cfg = config_loader.load_sweep_config( @@ -364,10 +364,10 @@ def test_sweep_config_cache(self, hydra_restore_singletons: Any, path: str, monk sweep_cfg_cache = OmegaConf.get_cache(sweep_cfg) assert len(sweep_cfg_cache.keys()) == 2 # "now", and "cached" - assert "now" in sweep_cfg_cache.keys() - assert "oc.env" not in sweep_cfg_cache.keys() - assert "cached" in sweep_cfg_cache.keys() - assert "uncached" not in sweep_cfg_cache.keys() + assert "now" in sweep_cfg_cache + assert "oc.env" not in sweep_cfg_cache + assert "cached" in sweep_cfg_cache + assert "uncached" not in sweep_cfg_cache assert sweep_cfg_cache["now"] == master_cfg_cache["now"] assert sweep_cfg_cache["cached"] == master_cfg_cache["cached"] monkeypatch.setenv("TEST_ENV", "test_env2") @@ -578,7 +578,6 @@ def test_complex_defaults(overrides: Any, expected: Any) -> None: ), # override param({"x": 20}, ["x=10"], {"x": 10}, id="override"), - param({"x": 20}, ["x=10"], {"x": 10}, id="override"), param({"x": None}, ["x=[1,2,3]"], {"x": [1, 2, 3]}, id="override:list"), param({"x": 20}, ["x=null"], {"x": None}, id="override_with_null"), param({"x": {"a": 10}}, ["x={a:20}"], {"x": {"a": 20}}, id="merge_dict"), @@ -627,11 +626,6 @@ def test_complex_defaults(overrides: Any, expected: Any) -> None: param({"x": {"y": 10}}, ["~x"], {}, id="delete"), param({"x": {"y": 10}}, ["~x.y"], {"x": {}}, id="delete"), param({"x": {"y": 10}}, ["~x.y=10"], {"x": {}}, id="delete_strict"), - param({"x": 20}, ["~x"], {}, id="delete"), - param({"x": 20}, ["~x=20"], {}, id="delete_strict"), - param({"x": {"y": 10}}, ["~x"], {}, id="delete"), - param({"x": {"y": 10}}, ["~x.y"], {"x": {}}, id="delete"), - param({"x": {"y": 10}}, ["~x.y=10"], {"x": {}}, id="delete_strict"), param({"x": [1, 2, 3]}, ["~x"], {}, id="delete:list"), param({"x": [1, 2, 3]}, ["~x=[1,2,3]"], {}, id="delete:list"), param({"x": [1, 2, 3]}, ["~x.0"], {"x": [2, 3]}, id="delete:list_item"), @@ -665,7 +659,7 @@ def test_complex_defaults(overrides: Any, expected: Any) -> None: ), ], ) -def test_apply_overrides_to_config(input_cfg: Any, overrides: List[str], expected: Any) -> None: +def test_apply_overrides_to_config(input_cfg: Any, overrides: list[str], expected: Any) -> None: cfg = OmegaConf.create(input_cfg) OmegaConf.set_struct(cfg, True) parser = OverridesParser.create() diff --git a/lerna/tests/test_config_repository.py b/lerna/tests/test_config_repository.py index 5831b05..44922e2 100644 --- a/lerna/tests/test_config_repository.py +++ b/lerna/tests/test_config_repository.py @@ -2,7 +2,7 @@ import copy import re import zipfile -from typing import Any, List +from typing import Any from pytest import mark, param, raises @@ -117,7 +117,7 @@ def test_config_repository_list( hydra_restore_singletons: Any, path: str, config_path: str, - expected: List[InputDefault], + expected: list[InputDefault], ) -> None: Plugins.instance() config_search_path = create_config_search_path(path) diff --git a/lerna/tests/test_config_search_path.py b/lerna/tests/test_config_search_path.py index a6bfbad..b53d87c 100644 --- a/lerna/tests/test_config_search_path.py +++ b/lerna/tests/test_config_search_path.py @@ -1,7 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import os from os.path import realpath -from typing import List, Optional, Tuple from pytest import mark @@ -10,7 +9,7 @@ from lerna.core.config_search_path import SearchPathElement, SearchPathQuery -def create_search_path(base_list: List[Tuple[str, str]]) -> ConfigSearchPathImpl: +def create_search_path(base_list: list[tuple[str, str]]) -> ConfigSearchPathImpl: csp = ConfigSearchPathImpl() csp.config_search_path = [SearchPathElement(x[0], x[1]) for x in base_list] return csp @@ -18,7 +17,7 @@ def create_search_path(base_list: List[Tuple[str, str]]) -> ConfigSearchPathImpl def to_tuples_list( search_path: ConfigSearchPathImpl, -) -> List[Tuple[Optional[str], Optional[str]]]: +) -> list[tuple[str | None, str | None]]: return [(x.provider, x.path) for x in search_path.config_search_path] @@ -32,7 +31,7 @@ def to_tuples_list( ([("a", "10"), ("b", "20"), ("a", "30")], ("a", "10"), 0), ], ) -def test_find_last_match(input_list: List[Tuple[str, str]], reference: str, expected_idx: int) -> None: +def test_find_last_match(input_list: list[tuple[str, str]], reference: str, expected_idx: int) -> None: csp = create_search_path(input_list) assert csp.find_last_match(SearchPathQuery(reference[0], reference[1])) == expected_idx @@ -47,7 +46,7 @@ def test_find_last_match(input_list: List[Tuple[str, str]], reference: str, expe ([("a", "10"), ("b", "20"), ("a", "30")], ("a", "10"), 0), ], ) -def test_find_first_match(input_list: List[Tuple[str, str]], reference: str, expected_idx: int) -> None: +def test_find_first_match(input_list: list[tuple[str, str]], reference: str, expected_idx: int) -> None: csp = create_search_path(input_list) sp = SearchPathQuery(reference[0], reference[1]) assert csp.find_first_match(sp) == expected_idx @@ -87,11 +86,11 @@ def test_find_first_match(input_list: List[Tuple[str, str]], reference: str, exp ], ) def test_append( - base_list: List[Tuple[str, str]], + base_list: list[tuple[str, str]], provider: str, path: str, anchor_provider: SearchPathQuery, - result_list: List[Tuple[str, str]], + result_list: list[tuple[str, str]], ) -> None: csp = create_search_path(base_list) csp.append(provider=provider, path=path, anchor=anchor_provider) @@ -132,11 +131,11 @@ def test_append( ], ) def test_prepend( - base_list: List[Tuple[str, str]], + base_list: list[tuple[str, str]], provider: str, path: str, anchor_provider: SearchPathQuery, - result_list: List[Tuple[str, str]], + result_list: list[tuple[str, str]], ) -> None: csp = create_search_path(base_list) csp.prepend(provider=provider, path=path, anchor=anchor_provider) diff --git a/lerna/tests/test_config_source.py b/lerna/tests/test_config_source.py index 18f7d73..bee9302 100644 --- a/lerna/tests/test_config_source.py +++ b/lerna/tests/test_config_source.py @@ -334,7 +334,7 @@ def test_load_invalid_yaml(self): result = source.load_config("bad.yaml") # If it returns, check it's usable assert result is None or isinstance(result, ConfigResult) - except Exception: + except Exception: # noqa: BLE001, S110 # Expected - invalid YAML should raise pass diff --git a/lerna/tests/test_env_defaults.py b/lerna/tests/test_env_defaults.py index 79227c0..3a93839 100644 --- a/lerna/tests/test_env_defaults.py +++ b/lerna/tests/test_env_defaults.py @@ -9,7 +9,7 @@ def test_env_defaults(tmpdir: Path) -> None: cmd = [ "lerna/tests/test_apps/custom_env_defaults/my_app.py", - f'hydra.run.dir="{str(tmpdir)}"', + f'hydra.run.dir="{tmpdir!s}"', "hydra.job.chdir=True", ] run_python_script(cmd) diff --git a/lerna/tests/test_examples/test_advanced_config_search_path.py b/lerna/tests/test_examples/test_advanced_config_search_path.py index 4208b51..f9d92cc 100644 --- a/lerna/tests/test_examples/test_advanced_config_search_path.py +++ b/lerna/tests/test_examples/test_advanced_config_search_path.py @@ -1,7 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import re from pathlib import Path -from typing import List, Optional from omegaconf import OmegaConf from pytest import mark @@ -32,7 +31,7 @@ ), ], ) -def test_config_search_path(args: List[str], expected: str, tmpdir: Path, error: Optional[str]) -> None: +def test_config_search_path(args: list[str], expected: str, tmpdir: Path, error: str | None) -> None: cmd = [ "examples/advanced/config_search_path/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', diff --git a/lerna/tests/test_examples/test_basic_sweep.py b/lerna/tests/test_examples/test_basic_sweep.py index 3f1d126..13ebb3e 100644 --- a/lerna/tests/test_examples/test_basic_sweep.py +++ b/lerna/tests/test_examples/test_basic_sweep.py @@ -1,7 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from pathlib import Path from textwrap import dedent -from typing import List from pytest import mark @@ -63,7 +62,7 @@ ) def test_basic_sweep_example( tmpdir: Path, - args: List[str], + args: list[str], expected: str, ) -> None: app_path = "examples/tutorials/basic/running_your_hydra_app/5_basic_sweep/my_app.py" diff --git a/lerna/tests/test_examples/test_instantiate_examples.py b/lerna/tests/test_examples/test_instantiate_examples.py index ec80aa6..f36d35e 100644 --- a/lerna/tests/test_examples/test_instantiate_examples.py +++ b/lerna/tests/test_examples/test_instantiate_examples.py @@ -1,7 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from pathlib import Path from textwrap import dedent -from typing import List from pytest import mark, param @@ -22,7 +21,7 @@ (["db=postgresql"], "PostgreSQL connecting to localhost"), ], ) -def test_instantiate_object(tmpdir: Path, overrides: List[str], output: str) -> None: +def test_instantiate_object(tmpdir: Path, overrides: list[str], output: str) -> None: cmd = [ "examples/instantiate/object/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -42,7 +41,7 @@ def test_instantiate_object(tmpdir: Path, overrides: List[str], output: str) -> ), ], ) -def test_instantiate_object_recursive(tmpdir: Path, overrides: List[str], output: str) -> None: +def test_instantiate_object_recursive(tmpdir: Path, overrides: list[str], output: str) -> None: cmd = [ "examples/instantiate/object_recursive/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -69,7 +68,7 @@ def test_instantiate_object_partial(tmpdir: Path) -> None: (["db=postgresql"], "PostgreSQL connecting to localhost"), ], ) -def test_instantiate_schema(tmpdir: Path, overrides: List[str], output: str) -> None: +def test_instantiate_schema(tmpdir: Path, overrides: list[str], output: str) -> None: cmd = [ "examples/instantiate/schema/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -96,7 +95,7 @@ def test_instantiate_schema(tmpdir: Path, overrides: List[str], output: str) -> ), ], ) -def test_instantiate_schema_recursive(tmpdir: Path, overrides: List[str], expected: str) -> None: +def test_instantiate_schema_recursive(tmpdir: Path, overrides: list[str], expected: str) -> None: cmd = [ "examples/instantiate/schema_recursive/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -133,7 +132,7 @@ def test_instantiate_schema_recursive(tmpdir: Path, overrides: List[str], expect ), ], ) -def test_instantiate_docs_example(tmpdir: Path, overrides: List[str], expected: str) -> None: +def test_instantiate_docs_example(tmpdir: Path, overrides: list[str], expected: str) -> None: cmd = [ "examples/instantiate/docs_example/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', diff --git a/lerna/tests/test_examples/test_patterns.py b/lerna/tests/test_examples/test_patterns.py index 3e1e342..6c40ddd 100644 --- a/lerna/tests/test_examples/test_patterns.py +++ b/lerna/tests/test_examples/test_patterns.py @@ -1,7 +1,7 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from pathlib import Path from textwrap import dedent -from typing import Any, List +from typing import Any from omegaconf import OmegaConf from pytest import mark, param @@ -28,10 +28,10 @@ def test_specializing_config_example(hydra_restore_singletons: Any, hydra_task_r overrides=["dataset=cifar10"], configure_logging=True, ) as task: - assert task.job_ret is not None and task.job_ret.cfg == dict( - dataset=dict(name="cifar10", path="/datasets/cifar10"), - model=dict(num_layers=5, type="alexnet"), - ) + assert task.job_ret is not None and task.job_ret.cfg == { + "dataset": {"name": "cifar10", "path": "/datasets/cifar10"}, + "model": {"num_layers": 5, "type": "alexnet"}, + } verify_dir_outputs(task.job_ret, overrides=task.overrides) @@ -64,7 +64,7 @@ def test_write_protect_config_node(tmpdir: Any) -> None: param(["db=mysql_extending_from_another_group"], id="from_different_group"), ], ) -def test_extending_configs(monkeypatch: Any, tmpdir: Path, overrides: List[str]) -> None: +def test_extending_configs(monkeypatch: Any, tmpdir: Path, overrides: list[str]) -> None: monkeypatch.chdir("examples/patterns/extending_configs") cmd = [ "my_app.py", @@ -103,7 +103,7 @@ def test_extending_configs(monkeypatch: Any, tmpdir: Path, overrides: List[str]) ), ], ) -def test_configuring_experiments(monkeypatch: Any, tmpdir: Path, overrides: List[str], expected: Any) -> None: +def test_configuring_experiments(monkeypatch: Any, tmpdir: Path, overrides: list[str], expected: Any) -> None: monkeypatch.chdir("examples/patterns/configuring_experiments") cmd = [ "my_app.py", @@ -172,7 +172,7 @@ def test_configuring_experiments(monkeypatch: Any, tmpdir: Path, overrides: List ), ], ) -def test_multi_select(monkeypatch: Any, tmpdir: Path, overrides: List[str], expected: Any) -> None: +def test_multi_select(monkeypatch: Any, tmpdir: Path, overrides: list[str], expected: Any) -> None: monkeypatch.chdir("examples/patterns/multi-select") cmd = [ "my_app.py", diff --git a/lerna/tests/test_examples/test_tutorials_basic.py b/lerna/tests/test_examples/test_tutorials_basic.py index 828d643..2711fab 100644 --- a/lerna/tests/test_examples/test_tutorials_basic.py +++ b/lerna/tests/test_examples/test_tutorials_basic.py @@ -4,7 +4,7 @@ import subprocess from pathlib import Path from textwrap import dedent -from typing import Any, List +from typing import Any try: from _pytest.python_api import RaisesContext @@ -35,7 +35,7 @@ ), ], ) -def test_tutorial_simple_cli_app(tmpdir: Path, args: List[str], output_conf: DictConfig) -> None: +def test_tutorial_simple_cli_app(tmpdir: Path, args: list[str], output_conf: DictConfig) -> None: cmd = [ "examples/tutorials/basic/your_first_hydra_app/1_simple_cli/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -53,7 +53,7 @@ def test_tutorial_working_directory(tmpdir: Path) -> None: "hydra.job.chdir=True", ] result, _err = run_python_script(cmd) - working_directory, output_directory = result.split("\n") + working_directory, _output_directory = result.split("\n") assert working_directory == f"Working directory : {tmpdir}" @@ -84,7 +84,7 @@ def test_tutorial_working_directory_original_cwd(tmpdir: Path) -> None: (["hydra.verbose=[__main__]"], ["Info level message", "Debug level message"]), ], ) -def test_tutorial_logging(tmpdir: Path, args: List[str], expected: List[str]) -> None: +def test_tutorial_logging(tmpdir: Path, args: list[str], expected: list[str]) -> None: cmd = [ "examples/tutorials/basic/running_your_hydra_app/4_logging/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -107,7 +107,7 @@ def test_tutorial_logging(tmpdir: Path, args: List[str], expected: List[str]) -> ) ], ) -def test_tutorial_config_file(tmpdir: Path, args: List[str], output_conf: Any) -> None: +def test_tutorial_config_file(tmpdir: Path, args: list[str], output_conf: Any) -> None: cmd = [ "examples/tutorials/basic/your_first_hydra_app/2_config_file/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -128,7 +128,7 @@ def test_tutorial_config_file(tmpdir: Path, args: List[str], output_conf: Any) - (["dataset.path=abc"], raises(subprocess.CalledProcessError)), ], ) -def test_tutorial_config_file_bad_key(tmpdir: Path, args: List[str], expected: Any) -> None: +def test_tutorial_config_file_bad_key(tmpdir: Path, args: list[str], expected: Any) -> None: """Similar to the previous test, but also tests exception values""" cmd = [ @@ -164,7 +164,7 @@ def test_tutorial_config_file_bad_key(tmpdir: Path, args: List[str], expected: A ), ], ) -def test_tutorial_config_groups(tmpdir: Path, args: List[str], output_conf: DictConfig) -> None: +def test_tutorial_config_groups(tmpdir: Path, args: list[str], output_conf: DictConfig) -> None: cmd = [ "examples/tutorials/basic/your_first_hydra_app/4_config_groups/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -203,7 +203,7 @@ def test_tutorial_config_groups(tmpdir: Path, args: List[str], output_conf: Dict ), ], ) -def test_tutorial_defaults(tmpdir: Path, args: List[str], expected: DictConfig) -> None: +def test_tutorial_defaults(tmpdir: Path, args: list[str], expected: DictConfig) -> None: cmd = [ "examples/tutorials/basic/your_first_hydra_app/5_defaults/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -298,7 +298,7 @@ def test_sweeping_example(hydra_restore_singletons: Any, hydra_sweep_runner: TSw ) ], ) -def test_advanced_ad_hoc_composition(monkeypatch: Any, tmpdir: Path, args: List[str], expected: Any) -> None: +def test_advanced_ad_hoc_composition(monkeypatch: Any, tmpdir: Path, args: list[str], expected: Any) -> None: monkeypatch.setenv("USER", "test_user") cmd = [ "examples/advanced/ad_hoc_composition/hydra_compose_example.py", diff --git a/lerna/tests/test_hydra.py b/lerna/tests/test_hydra.py index 47d404a..300a77d 100644 --- a/lerna/tests/test_hydra.py +++ b/lerna/tests/test_hydra.py @@ -6,7 +6,7 @@ from logging import getLogger from pathlib import Path from textwrap import dedent -from typing import Any, List, Optional, Set +from typing import Any from omegaconf import DictConfig, OmegaConf from pytest import mark, param, raises @@ -37,14 +37,16 @@ def test_missing_conf_dir( calling_file: str, calling_module: str, ) -> None: - with raises(MissingConfigException): - with hydra_task_runner( + with ( + raises(MissingConfigException), + hydra_task_runner( calling_file=calling_file, calling_module=calling_module, config_path="dir_not_found", config_name=None, - ): - pass + ), + ): + pass @mark.parametrize( @@ -60,14 +62,16 @@ def test_missing_conf_file( calling_file: str, calling_module: str, ) -> None: - with raises(MissingConfigException): - with hydra_task_runner( + with ( + raises(MissingConfigException), + hydra_task_runner( calling_file=calling_file, calling_module=calling_module, config_path=None, config_name="not_found.yaml", - ): - pass + ), + ): + pass def test_run_dir() -> None: @@ -141,7 +145,7 @@ def test_app_without_config__with_append( overrides=["+abc=123", "+a.b=1", "+a.a=2"], configure_logging=True, ) as task: - assert task.job_ret is not None and task.job_ret.cfg == dict(abc=123, a=dict(b=1, a=2)) + assert task.job_ret is not None and task.job_ret.cfg == {"abc": 123, "a": {"b": 1, "a": 2}} verify_dir_outputs(task.job_ret, task.overrides) @@ -225,7 +229,7 @@ def test_app_with_config_file__with_override( overrides=["dataset.path=/datasets/imagenet2"], configure_logging=True, ) as task: - assert task.job_ret is not None and task.job_ret.cfg == dict(dataset=dict(name="imagenet", path="/datasets/imagenet2")) + assert task.job_ret is not None and task.job_ret.cfg == {"dataset": {"name": "imagenet", "path": "/datasets/imagenet2"}} verify_dir_outputs(task.job_ret, task.overrides) @@ -249,7 +253,7 @@ def test_app_with_config_file__with_decorators( config_name="config.yaml", configure_logging=True, ) as task: - assert task.job_ret is not None and task.job_ret.cfg == dict(dataset=dict(name="imagenet", path="/datasets/imagenet")) + assert task.job_ret is not None and task.job_ret.cfg == {"dataset": {"name": "imagenet", "path": "/datasets/imagenet"}} verify_dir_outputs(task.job_ret) @@ -273,10 +277,10 @@ def test_app_with_split_config( config_name="config.yaml", configure_logging=True, ) as task: - assert task.job_ret is not None and task.job_ret.cfg == dict( - dataset=dict(name="imagenet", path="/datasets/imagenet"), - optimizer=dict(lr=0.001, type="nesterov"), - ) + assert task.job_ret is not None and task.job_ret.cfg == { + "dataset": {"name": "imagenet", "path": "/datasets/imagenet"}, + "optimizer": {"lr": 0.001, "type": "nesterov"}, + } verify_dir_outputs(task.job_ret) @@ -293,15 +297,17 @@ def test_app_with_config_groups__override_dataset__wrong( calling_file: str, calling_module: str, ) -> None: - with raises(MissingConfigException) as ex: - with hydra_task_runner( + with ( + raises(MissingConfigException) as ex, + hydra_task_runner( calling_file=calling_file, calling_module=calling_module, config_path="conf", config_name=None, overrides=["+optimizer=wrong_name"], - ): - pass + ), + ): + pass assert sorted(ex.value.options) == sorted(["adam", "nesterov"]) # type: ignore @@ -326,7 +332,7 @@ def test_app_with_config_groups__override_all_configs( overrides=["+optimizer=adam", "optimizer.lr=10"], configure_logging=True, ) as task: - assert task.job_ret is not None and task.job_ret.cfg == dict(optimizer=dict(type="adam", lr=10, beta=0.01)) + assert task.job_ret is not None and task.job_ret.cfg == {"optimizer": {"type": "adam", "lr": 10, "beta": 0.01}} verify_dir_outputs(task.job_ret, overrides=task.overrides) @@ -400,7 +406,7 @@ def test_module_env_override(tmpdir: Path, env_name: str) -> None: "flag,expected_keys", [("--cfg=all", ["db", "hydra"]), ("--cfg=hydra", ["hydra"]), ("--cfg=job", ["db"])], ) -def test_cfg(tmpdir: Path, flag: str, resolve: bool, expected_keys: List[str]) -> None: +def test_cfg(tmpdir: Path, flag: str, resolve: bool, expected_keys: list[str]) -> None: cmd = [ "examples/tutorials/basic/your_first_hydra_app/5_defaults/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -458,7 +464,7 @@ def test_cfg(tmpdir: Path, flag: str, resolve: bool, expected_keys: List[str]) - param(["--cfg=job", "--package=db.driver"], "mysql\n", id="package=db.driver"), ], ) -def test_cfg_with_package(tmpdir: Path, flags: List[str], resolve: bool, expected: str) -> None: +def test_cfg_with_package(tmpdir: Path, flags: list[str], resolve: bool, expected: str) -> None: cmd = [ "examples/tutorials/basic/your_first_hydra_app/5_defaults/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -514,7 +520,7 @@ def test_cfg_with_package(tmpdir: Path, flags: List[str], resolve: bool, expecte ), ], ) -def test_cfg_resolve_interpolation(tmpdir: Path, script: str, resolve: bool, flags: List[str], expected: str) -> None: +def test_cfg_resolve_interpolation(tmpdir: Path, script: str, resolve: bool, flags: list[str], expected: str) -> None: cmd = [ script, f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -558,7 +564,7 @@ def test_pass_callable_class_to_hydra_main(tmpdir: Path, script: str, expected: "other_flag", [None, "--run", "--multirun", "--info", "--shell-completion", "--hydra-help"], ) -def test_resolve_flag_errmsg(tmpdir: Path, other_flag: Optional[str]) -> None: +def test_resolve_flag_errmsg(tmpdir: Path, other_flag: str | None) -> None: cmd = [ "examples/tutorials/basic/your_first_hydra_app/3_using_config/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -585,7 +591,7 @@ def test_multirun_with_free_override( hydra_sweep_runner: TSweepRunner, calling_file: str, calling_module: str, - overrides: List[str], + overrides: list[str], ) -> None: sweep = hydra_sweep_runner( calling_file=calling_file, @@ -805,7 +811,7 @@ def test_sweep_complex_defaults( ), ], ) -def test_help(tmpdir: Path, script: str, flags: List[str], overrides: List[str], expected: Any) -> None: +def test_help(tmpdir: Path, script: str, flags: list[str], overrides: list[str], expected: Any) -> None: cmd = [script, f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', "hydra.job.chdir=True"] cmd.extend(overrides) cmd.extend(flags) @@ -851,7 +857,7 @@ def test_shell_completion_help(tmpdir: Path) -> None: ), ], ) -def test_searchpath_config(tmpdir: Path, overrides: List[str], expected: str) -> None: +def test_searchpath_config(tmpdir: Path, overrides: list[str], expected: str) -> None: cmd = ["examples/advanced/config_search_path/my_app.py"] cmd.extend(overrides) cmd.extend([f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', "hydra.job.chdir=True"]) @@ -893,7 +899,7 @@ def test_sys_exit(tmpdir: Path) -> None: f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', "hydra.job.chdir=True", ] - assert subprocess.run(cmd).returncode == 42 + assert subprocess.run(cmd, check=False).returncode == 42 @mark.parametrize( @@ -924,7 +930,7 @@ def test_sys_exit(tmpdir: Path) -> None: ), ], ) -def test_local_run_workdir(tmpdir: Path, task_config: DictConfig, overrides: List[str], expected_dir: str) -> None: +def test_local_run_workdir(tmpdir: Path, task_config: DictConfig, overrides: list[str], expected_dir: str) -> None: cfg = OmegaConf.create(task_config) assert isinstance(cfg, DictConfig) expected_dir1 = tmpdir / expected_dir @@ -993,8 +999,8 @@ def test_hydra_env_set_with_override(tmpdir: Path) -> None: def test_override_with_invalid_group_choice( hydra_restore_singletons: Any, hydra_task_runner: TTaskRunner, - calling_file: Optional[str], - calling_module: Optional[str], + calling_file: str | None, + calling_module: str | None, override: str, ) -> None: # Lerna uses 'db' (without trailing slash) for empty override, 'db/xyz' for non-empty @@ -1009,15 +1015,17 @@ def test_override_with_invalid_group_choice( """ ) - with raises(MissingConfigException) as e: - with hydra_task_runner( + with ( + raises(MissingConfigException) as e, + hydra_task_runner( calling_file=calling_file, calling_module=calling_module, config_path="configs", config_name="db_conf", overrides=[f"db={override}"], - ): - pass + ), + ): + pass # assert_text_same(from_line=msg, to_line=str(e.value)) assert re.search(msg, str(e.value)) is not None @@ -1066,8 +1074,8 @@ def test_hydra_output_dir( hydra_task_runner: TTaskRunner, calling_file: str, calling_module: str, - overrides: List[str], - expected_files: Set[str], + overrides: list[str], + expected_files: set[str], ) -> None: with hydra_task_runner( calling_file=calling_file, @@ -1096,7 +1104,7 @@ def test_hydra_output_dir( ("lerna/tests/test_apps/run_as_module_4", "module/my_app.py", "module.my_app", None), ], ) -def test_module_run(tmpdir: Any, directory: str, file: str, module: str, error: Optional[str]) -> None: +def test_module_run(tmpdir: Any, directory: str, file: str, module: str, error: str | None) -> None: cmd = [ directory + "/" + file, f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -1149,7 +1157,7 @@ def test_module_run(tmpdir: Any, directory: str, file: str, module: str, error: param(["test.param=1,2", "-m"], False, "1\n2", id="multirun:choice_sweep"), ], ) -def test_multirun_structured_conflict(tmpdir: Any, overrides: List[str], error: bool, expected: Any) -> None: +def test_multirun_structured_conflict(tmpdir: Any, overrides: list[str], error: bool, expected: Any) -> None: cmd = [ "lerna/tests/test_apps/multirun_structured_conflict/my_app.py", "hydra.sweep.dir=" + normalize_path_for_override(tmpdir), @@ -1182,7 +1190,7 @@ class TestVariousRuns: param(True, id="sweep"), ], ) - def test_run_with_missing_default(self, cmd_base: List[str], tmpdir: Any, sweep: bool) -> None: + def test_run_with_missing_default(self, cmd_base: list[str], tmpdir: Any, sweep: bool) -> None: cmd = cmd_base + [ "hydra.sweep.dir=" + normalize_path_for_override(tmpdir), "hydra.job.chdir=True", @@ -1199,7 +1207,7 @@ def test_run_with_missing_default(self, cmd_base: List[str], tmpdir: Any, sweep: ret = run_with_error(cmd) assert re.search(re.escape(expected), ret) is not None - def test_command_line_interpolations_evaluated_lazily(self, cmd_base: List[str], tmpdir: Any) -> None: + def test_command_line_interpolations_evaluated_lazily(self, cmd_base: list[str], tmpdir: Any) -> None: cmd = cmd_base + [ "hydra.sweep.dir=" + normalize_path_for_override(tmpdir), "hydra.job.chdir=True", @@ -1215,7 +1223,7 @@ def test_command_line_interpolations_evaluated_lazily(self, cmd_base: List[str], ret, _err = run_python_script(cmd) assert normalize_newlines(ret) == normalize_newlines(expected) - def test_multirun_config_overrides_evaluated_lazily(self, cmd_base: List[str], tmpdir: Any) -> None: + def test_multirun_config_overrides_evaluated_lazily(self, cmd_base: list[str], tmpdir: Any) -> None: cmd = cmd_base + [ "hydra.sweep.dir=" + normalize_path_for_override(tmpdir), "hydra.job.chdir=True", @@ -1231,7 +1239,7 @@ def test_multirun_config_overrides_evaluated_lazily(self, cmd_base: List[str], t ret, _err = run_python_script(cmd) assert normalize_newlines(ret) == normalize_newlines(expected) - def test_multirun_defaults_override(self, cmd_base: List[str], tmpdir: Any) -> None: + def test_multirun_defaults_override(self, cmd_base: list[str], tmpdir: Any) -> None: cmd = cmd_base + [ "hydra.sweep.dir=" + normalize_path_for_override(tmpdir), "hydra.job.chdir=True", @@ -1248,7 +1256,7 @@ def test_multirun_defaults_override(self, cmd_base: List[str], tmpdir: Any) -> N ret, _err = run_python_script(cmd) assert normalize_newlines(ret) == normalize_newlines(expected) - def test_run_pass_list(self, cmd_base: List[str], tmpdir: Any) -> None: + def test_run_pass_list(self, cmd_base: list[str], tmpdir: Any) -> None: cmd = cmd_base + [ "hydra.sweep.dir=" + normalize_path_for_override(tmpdir), "hydra.job.chdir=True", @@ -1329,7 +1337,7 @@ def test_hydra_to_job_config_interpolation(tmpdir: Any) -> Any: ), ], ) -def test_config_dir_argument(monkeypatch: Any, tmpdir: Path, overrides: List[str], expected: DictConfig) -> None: +def test_config_dir_argument(monkeypatch: Any, tmpdir: Path, overrides: list[str], expected: DictConfig) -> None: monkeypatch.chdir("lerna/tests/test_apps/user-config-dir") cmd = [ "my_app.py", @@ -1491,7 +1499,7 @@ def test_job_chdir_not_specified(tmpdir: Path) -> None: "lerna/tests/test_apps/app_with_no_chdir_override/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', ] - out, err = run_python_script(cmd, allow_warnings=True) + _out, err = run_python_script(cmd, allow_warnings=True) expected = dedent( """ @@ -1530,7 +1538,7 @@ def test_app_with_unicode_config(tmpdir: Path) -> None: (["--help"], "frozen is powered by Hydra."), ], ) -def test_frozen_primary_config(tmpdir: Path, overrides: List[str], expected: str) -> None: +def test_frozen_primary_config(tmpdir: Path, overrides: list[str], expected: str) -> None: cmd = [ "examples/patterns/write_protect_config_node/frozen.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -1588,7 +1596,7 @@ def test_hydra_deprecation_warning(env_deprecation_err: bool, expected: str, tmp (True, ["0/my_app.log", "0/.hydra/config.yaml", "multirun.yaml"]), ], ) -def test_disable_chdir(tmpdir: Path, multirun: bool, expected: List[str]) -> None: +def test_disable_chdir(tmpdir: Path, multirun: bool, expected: list[str]) -> None: cmd = [ "examples/tutorials/basic/running_your_hydra_app/3_working_directory/my_app.py", f'hydra.run.dir="{normalize_path_for_override(tmpdir)}"', @@ -1787,11 +1795,11 @@ def test_hydra_resolver_in_output_dir(tmpdir: Path, multirun: bool) -> None: ) def test_hydra_mode( tmpdir: Path, - overrides: List[str], + overrides: list[str], expected_output: str, error: bool, warning: bool, - warning_msg: Optional[str], + warning_msg: str | None, ) -> None: cmd = [ "lerna/tests/test_apps/app_print_hydra_mode/my_app.py", diff --git a/lerna/tests/test_hydra_bugfixes.py b/lerna/tests/test_hydra_bugfixes.py index 0eb2f42..d5bfde8 100644 --- a/lerna/tests/test_hydra_bugfixes.py +++ b/lerna/tests/test_hydra_bugfixes.py @@ -798,9 +798,11 @@ def test_patch_sweep_override_rejected(self, patch_sweep_config_dir): GlobalHydra.instance().clear() try: - with initialize_config_dir(config_dir=str(patch_sweep_config_dir), version_base=None): - with pytest.raises(ConfigCompositionException, match="_patch_ does not support sweep"): - compose(config_name="config") + with ( + initialize_config_dir(config_dir=str(patch_sweep_config_dir), version_base=None), + pytest.raises(ConfigCompositionException, match="_patch_ does not support sweep"), + ): + compose(config_name="config") finally: GlobalHydra.instance().clear() @@ -812,9 +814,11 @@ def test_patch_delete_nonexistent_key(self, patch_nonexistent_delete_dir): GlobalHydra.instance().clear() try: - with initialize_config_dir(config_dir=str(patch_nonexistent_delete_dir), version_base=None): - with pytest.raises(ConfigCompositionException, match="does not exist"): - compose(config_name="config") + with ( + initialize_config_dir(config_dir=str(patch_nonexistent_delete_dir), version_base=None), + pytest.raises(ConfigCompositionException, match="does not exist"), + ): + compose(config_name="config") finally: GlobalHydra.instance().clear() @@ -825,9 +829,11 @@ def test_patch_empty_package_scope_rejected(self, patch_empty_scope_dir): GlobalHydra.instance().clear() try: - with initialize_config_dir(config_dir=str(patch_empty_scope_dir), version_base=None): - with pytest.raises(Exception, match="_patch_@ requires a package name"): - compose(config_name="config") + with ( + initialize_config_dir(config_dir=str(patch_empty_scope_dir), version_base=None), + pytest.raises(ValueError, match="_patch_@ requires a package name"), + ): + compose(config_name="config") finally: GlobalHydra.instance().clear() diff --git a/lerna/tests/test_hydra_cli_errors.py b/lerna/tests/test_hydra_cli_errors.py index 4464a96..3253880 100644 --- a/lerna/tests/test_hydra_cli_errors.py +++ b/lerna/tests/test_hydra_cli_errors.py @@ -1,6 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved from pathlib import Path -from typing import Any, List +from typing import Any from pytest import mark, param @@ -88,7 +88,7 @@ def test_cli_error( tmpdir: Any, monkeypatch: Any, override: Any, - expected_substrings: List[str], + expected_substrings: list[str], ) -> None: monkeypatch.chdir("lerna/tests/test_apps/app_without_config/") if isinstance(override, str): diff --git a/lerna/tests/test_hydra_context_warnings.py b/lerna/tests/test_hydra_context_warnings.py index 10756fb..db54488 100644 --- a/lerna/tests/test_hydra_context_warnings.py +++ b/lerna/tests/test_hydra_context_warnings.py @@ -1,7 +1,8 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved import re +from collections.abc import Sequence from textwrap import dedent -from typing import Any, List, Sequence, Union +from typing import Any from unittest.mock import Mock from omegaconf import DictConfig, OmegaConf @@ -34,7 +35,7 @@ def setup( # type: ignore ) -> None: pass - def sweep(self, arguments: List[str]) -> Any: + def sweep(self, arguments: list[str]) -> Any: pass @@ -63,7 +64,7 @@ def launch( # type: ignore[empty-body] (IncompatibleSweeper(), OmegaConf.create({"hydra": {"sweeper": {}}})), ], ) -def test_setup_plugins(monkeypatch: Any, plugin: Union[Launcher, Sweeper], config: DictConfig) -> None: +def test_setup_plugins(monkeypatch: Any, plugin: Launcher | Sweeper, config: DictConfig) -> None: task_function = Mock(spec=TaskFunction) config_loader = ConfigLoaderImpl(config_search_path=create_config_search_path(None)) hydra_context = HydraContext(config_loader=config_loader, callbacks=Callbacks()) diff --git a/lerna/tests/test_internal_utils.py b/lerna/tests/test_internal_utils.py index 84e2c26..6cb72ad 100644 --- a/lerna/tests/test_internal_utils.py +++ b/lerna/tests/test_internal_utils.py @@ -1,5 +1,6 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -from typing import Any, Callable, Optional +from collections.abc import Callable +from typing import Any from omegaconf import DictConfig, OmegaConf from pytest import mark, param @@ -16,9 +17,6 @@ ([["a", "bb"], ["aa", "b"]], [2, 2]), ([["a"], ["aa", "b"]], [2, 1]), ([["a", "aa"], ["bb"]], [2, 2]), - ([["a"]], [1]), - ([["a"]], [1]), - ([["a"]], [1]), ], ) def test_get_column_widths(matrix: Any, expected: Any) -> None: @@ -47,8 +45,8 @@ def test_get_class_name(config: DictConfig, expected: Any) -> None: ) def test_detect_calling_file_or_module_from_task_function( task_function: Callable[..., None], - expected_file: Optional[str], - expected_module: Optional[str], + expected_file: str | None, + expected_module: str | None, ) -> None: file, module = utils.detect_calling_file_or_module_from_task_function(task_function) assert file == expected_file diff --git a/lerna/tests/test_list_operations.py b/lerna/tests/test_list_operations.py index 4c8b5c8..bcf91ca 100644 --- a/lerna/tests/test_list_operations.py +++ b/lerna/tests/test_list_operations.py @@ -240,12 +240,10 @@ def config_dir(self, tmp_path): def test_append_to_non_list_fails(self, config_dir): """Cannot append to a non-list value.""" - with initialize_config_dir(version_base=None, config_dir=config_dir): - with pytest.raises(Exception, match="not a list"): - compose(config_name="config", overrides=["name=append(new)"]) + with initialize_config_dir(version_base=None, config_dir=config_dir), pytest.raises(Exception, match="not a list"): + compose(config_name="config", overrides=["name=append(new)"]) def test_remove_at_out_of_bounds(self, config_dir): """Remove at out-of-bounds index should fail.""" - with initialize_config_dir(version_base=None, config_dir=config_dir): - with pytest.raises(Exception, match="Cannot remove item"): - compose(config_name="config", overrides=["tags=remove_at(10)"]) + with initialize_config_dir(version_base=None, config_dir=config_dir), pytest.raises(Exception, match="Cannot remove item"): + compose(config_name="config", overrides=["tags=remove_at(10)"]) diff --git a/lerna/tests/test_overrides_parser.py b/lerna/tests/test_overrides_parser.py index 1f2a472..a78b4eb 100644 --- a/lerna/tests/test_overrides_parser.py +++ b/lerna/tests/test_overrides_parser.py @@ -2,8 +2,9 @@ import builtins import math import re +from collections.abc import Callable from dataclasses import dataclass -from typing import Any, Callable, Dict, List, Union +from typing import Any try: from _pytest.python_api import RaisesContext @@ -312,7 +313,7 @@ def test_shuffle_sequence(value: str, expected: Any) -> None: param("{3.14: 0, 1e3: 1}", {3.14: 0, 1000.0: 1}, id="dict_float_key"), param("{true: 1, fAlSe: 0}", {True: 1, False: 0}, id="dict_bool_key"), param( - "{%s: 1}" % UNQUOTED_SPECIAL, + f"{{{UNQUOTED_SPECIAL}: 1}}", {UNQUOTED_SPECIAL: 1}, id="dict_unquoted_char_key", ), @@ -654,9 +655,6 @@ def test_key(value: str, expected: Any) -> None: param("-inf", -math.inf, id="primitive:float:inf:neg"), param("nan", math.nan, id="primitive:float:nan"), param("NaN", math.nan, id="primitive:float:nan"), - # bool - param("true", True, id="primitive:bool"), - param("false", False, id="primitive:bool"), # interpolations: param("${a}", "${a}", id="primitive:interpolation"), param("${a.b.c}", "${a.b.c}", id="primitive:interpolation"), @@ -780,11 +778,6 @@ def test_primitive(value: str, expected: Any) -> None: QuotedString(text="inf", quote=Quote.single), id="float:constant", ), - param( - "'nan'", - QuotedString(text="nan", quote=Quote.single), - id="float:constant", - ), param( "'true'", QuotedString(text="true", quote=Quote.single), @@ -1189,9 +1182,9 @@ def test_get_key_element(override: str, expected: str) -> None: param("key={a:10,b:20}", "{a: 10, b: 20}", True, id="dict"), param("key={a:10,b:[1,2,3]}", "{a: 10, b: [1, 2, 3]}", True, id="dict"), param( - "key={%s: 1}" % UNQUOTED_SPECIAL, + f"key={{{UNQUOTED_SPECIAL}: 1}}", # Note that \ gets escaped. - "{%s: 1}" % UNQUOTED_SPECIAL.replace("\\", "\\\\"), + "{{{}: 1}}".format(UNQUOTED_SPECIAL.replace("\\", "\\\\")), True, id="dict_unquoted_key_special", ), @@ -1224,18 +1217,14 @@ def test_override_get_value_element_method(override: str, expected: str, space_a param("key=3.1415", 3.1415, id="float"), param("key=[]", [], id="list"), param("key=[1,2,3]", [1, 2, 3], id="list"), - param("key=[1,2,3]", [1, 2, 3], id="list"), - param("key=['a b', 2, 3]", ["a b", 2, 3], id="list"), param("key=['a b', 2, 3]", ["a b", 2, 3], id="list"), param("key={}", {}, id="dict"), param("key={a:10}", {"a": 10}, id="dict"), - param("key={a:10}", {"a": 10}, id="dict"), - param("key={a:10,b:20}", {"a": 10, "b": 20}, id="dict"), param("key={a:10,b:20}", {"a": 10, "b": 20}, id="dict"), param("key={a:10,b:[1,2,3]}", {"a": 10, "b": [1, 2, 3]}, id="dict"), param("key={123id: 0}", {"123id": 0}, id="dict_key_int_plus_id"), param( - "key={%s: 0}" % UNQUOTED_SPECIAL, + f"key={{{UNQUOTED_SPECIAL}: 0}}", {UNQUOTED_SPECIAL: 0}, id="dict_key_noquote", ), @@ -1263,13 +1252,12 @@ def test_override_value_method(override: str, expected: str) -> None: # up param(0, 2, 1, [0.0, 1.0], id="FloatRange:up"), param(0, 2, 0.5, [0.0, 0.5, 1.0, 1.5], id="FloatRange:up"), - param(0, 2, 1, [0.0, 1.0], id="FloatRange:up"), # down param(2, 0, -1, [2.0, 1.0], id="FloatRange:down"), param(10.0, 5.0, -2, [10.0, 8.0, 6.0], id="FloatRange:down"), ], ) -def test_float_range(start: float, stop: float, step: float, expected: List[float]) -> None: +def test_float_range(start: float, stop: float, step: float, expected: list[float]) -> None: res = list(FloatRange(start, stop, step)) assert len(res) == len(expected) for i in range(len(res)): @@ -1450,39 +1438,11 @@ def test_sweep_shuffle(value: str, expected: str) -> None: @dataclass class CastResults: - json_str: Union[ - str, - Sweep, - RaisesContext[HydraException], - ] - int: Union[ - int, - List[Union[int, List[int]]], - Dict[str, Any], - Sweep, - RaisesContext[HydraException], - ] - float: Union[ - float, - List[Union[float, List[float]]], - Dict[str, Any], - Sweep, - RaisesContext[HydraException], - ] - bool: Union[ - bool, - List[Union[bool, List[bool]]], - Dict[str, Any], - Sweep, - RaisesContext[HydraException], - ] - str: Union[ - str, - List[Union[str, List[str]]], - Dict[str, Any], - Sweep, - RaisesContext[HydraException], - ] + json_str: str | Sweep | RaisesContext[HydraException] + int: int | list[int | list[int]] | dict[str, Any] | Sweep | RaisesContext[HydraException] + float: float | list[float | list[float]] | dict[str, Any] | Sweep | RaisesContext[HydraException] + bool: bool | list[bool | list[bool]] | dict[str, Any] | Sweep | RaisesContext[HydraException] + str: str | list[str | list[str]] | dict[str, Any] | Sweep | RaisesContext[HydraException] @staticmethod def error(msg: builtins.str) -> Any: @@ -1847,11 +1807,11 @@ def test_function(value: Any, expected_value: Any) -> None: class F: @staticmethod def foo1(value: int) -> str: - return f"{type(value).__name__}:{str(value)}" + return f"{type(value).__name__}:{value!s}" @staticmethod - def foo2(x: Union[int, str], y: Union[int, str]) -> str: - return f"{type(x).__name__}:{str(x)},{type(y).__name__}:{str(y)}" + def foo2(x: int | str, y: int | str) -> str: + return f"{type(x).__name__}:{x!s},{type(y).__name__}:{y!s}" @staticmethod def range(start: int, stop: int, step: int = 1) -> str: @@ -1866,11 +1826,11 @@ def sum(*args: int) -> int: return sum(args, 0) @staticmethod - def sort(*args: int, reverse: bool = False) -> List[int]: + def sort(*args: int, reverse: bool = False) -> list[int]: if reverse: - return list(reversed(sorted(args))) + return sorted(args, reverse=True) else: - return list(sorted(args)) + return sorted(args) @mark.parametrize( @@ -2070,7 +2030,7 @@ def test_glob(value: str, expected: Any) -> None: param(["t*"], [], ["the", "the"], id="=*"), ], ) -def test_glob_filter(include: List[str], exclude: List[str], expected: List[str]) -> None: +def test_glob_filter(include: list[str], exclude: list[str], expected: list[str]) -> None: strings = ["the", "quick", "brown", "fox", "jumped", "under", "the", "lazy", "dog"] assert Glob(include=include, exclude=exclude).filter(strings) == expected @@ -2179,8 +2139,8 @@ def test_whitespaces(value: str, expected_key: str, expected_value: Any, expecte ) def test_sweep_iterators( value: str, - expected_sweep_string_list: List[str], - expected_sweep_encoded_list: List[Any], + expected_sweep_string_list: list[str], + expected_sweep_encoded_list: list[Any], ) -> None: ret = parser.parse_override(value) actual_sweep_string_list = [x for x in ret.sweep_string_iterator()] diff --git a/lerna/tests/test_plugin_interface.py b/lerna/tests/test_plugin_interface.py index a05cd45..ad1b5fc 100644 --- a/lerna/tests/test_plugin_interface.py +++ b/lerna/tests/test_plugin_interface.py @@ -1,5 +1,4 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved -from typing import List, Type from pytest import mark, raises @@ -15,7 +14,7 @@ # Individual plugins are responsible to test that they are discoverable. launchers = ["lerna._internal.core_plugins.basic_launcher.BasicLauncher"] sweepers = ["lerna._internal.core_plugins.basic_sweeper.BasicSweeper"] -search_path_plugins: List[str] = [] +search_path_plugins: list[str] = [] @mark.parametrize( @@ -27,7 +26,7 @@ (Plugin, launchers + sweepers + search_path_plugins), ], ) -def test_discover(plugin_type: Type[Plugin], expected: List[str]) -> None: +def test_discover(plugin_type: type[Plugin], expected: list[str]) -> None: plugins = Plugins.instance().discover(plugin_type) expected_classes = [get_class(c) for c in expected] for ex in expected_classes: diff --git a/lerna/tests/test_rust_defaults_list.py b/lerna/tests/test_rust_defaults_list.py index 20e49a9..09e54d8 100644 --- a/lerna/tests/test_rust_defaults_list.py +++ b/lerna/tests/test_rust_defaults_list.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python """Test the Rust defaults list helper.""" from typing import Any diff --git a/lerna/tests/test_rust_integration.py b/lerna/tests/test_rust_integration.py index 591d4fb..9ae7271 100644 --- a/lerna/tests/test_rust_integration.py +++ b/lerna/tests/test_rust_integration.py @@ -805,7 +805,7 @@ def test_env_resolver_get_required(self): import lerna.lerna as rs resolver = rs.env.EnvResolver() - with pytest.raises(Exception): + with pytest.raises(ValueError): resolver.get_required("NONEXISTENT_VAR_12345") def test_env_resolver_with_override(self): diff --git a/lerna/tests/test_rust_omegaconf.py b/lerna/tests/test_rust_omegaconf.py index 3bb7dc0..b4e96ab 100644 --- a/lerna/tests/test_rust_omegaconf.py +++ b/lerna/tests/test_rust_omegaconf.py @@ -24,8 +24,8 @@ def test_create_empty(self) -> None: def test_create_with_dict(self) -> None: cfg = DictConfig({"a": 1, "b": 2}) assert len(cfg) == 2 - assert "a" in cfg.keys() - assert "b" in cfg.keys() + assert "a" in cfg + assert "b" in cfg def test_getitem(self) -> None: cfg = DictConfig({"a": 1, "b": "hello", "c": True}) @@ -43,8 +43,8 @@ def test_setitem(self) -> None: def test_delitem(self) -> None: cfg = DictConfig({"a": 1, "b": 2}) del cfg["a"] - assert "a" not in cfg.keys() - assert "b" in cfg.keys() + assert "a" not in cfg + assert "b" in cfg def test_contains(self) -> None: cfg = DictConfig({"a": 1}) diff --git a/lerna/tests/test_utils.py b/lerna/tests/test_utils.py index 8b848a2..39701af 100644 --- a/lerna/tests/test_utils.py +++ b/lerna/tests/test_utils.py @@ -5,7 +5,7 @@ import re from pathlib import Path from textwrap import dedent -from typing import Any, NoReturn, Optional +from typing import Any, NoReturn from unittest.mock import patch from omegaconf import DictConfig, OmegaConf @@ -59,7 +59,6 @@ def test_to_absolute_path(hydra_restore_singletons: Any, orig_cwd: str, path: st @mark.parametrize( "path, expected", [ - ("foo/bar", f"{os.getcwd()}/foo/bar"), ("foo/bar", f"{os.getcwd()}/foo/bar"), ("/foo/bar", os.path.abspath("/foo/bar")), ], @@ -104,7 +103,7 @@ def test_to_hydra_override_value_str_roundtrip(hydra_restore_singletons: Any, ob param("1", True, id="env_set"), ], ) -def test_deprecation_warning(monkeypatch: Any, env_setting: Optional[str], expected_error: bool) -> None: +def test_deprecation_warning(monkeypatch: Any, env_setting: str | None, expected_error: bool) -> None: msg = "Feature FooBar is deprecated" if env_setting is not None: monkeypatch.setenv("HYDRA_DEPRECATION_WARNINGS_AS_ERRORS", env_setting) @@ -237,9 +236,12 @@ def test_success(self) -> None: ) def test_failure(self, demo_func: Any, expected_traceback_regex: str) -> None: mock_stderr = io.StringIO() - with patch("lerna._internal.utils.is_under_debugger", return_value=False): - with raises(SystemExit, match="1"), patch("sys.stderr", new=mock_stderr): - run_and_report(demo_func) + with ( + patch("lerna._internal.utils.is_under_debugger", return_value=False), + raises(SystemExit, match="1"), + patch("sys.stderr", new=mock_stderr), + ): + run_and_report(demo_func) mock_stderr.seek(0) stderr_output = mock_stderr.read() assert_multiline_regex_search(expected_traceback_regex, stderr_output) @@ -262,14 +264,15 @@ def test_simplified_traceback_with_no_module(self) -> None: """ ) mock_stderr = io.StringIO() - with patch("lerna._internal.utils.is_under_debugger", return_value=False): - with raises(SystemExit, match="1"), patch("sys.stderr", new=mock_stderr): - # Patch `inspect.getmodule` so that it will return None. This simulates a - # situation where a python module cannot be identified from a traceback - # stack frame. This can occur when python extension modules or - # multithreading are involved. - with patch("inspect.getmodule", new=lambda *args: None): - run_and_report(demo_func) + with ( + patch("lerna._internal.utils.is_under_debugger", return_value=False), + raises(SystemExit, match="1"), + patch("sys.stderr", new=mock_stderr), + patch("inspect.getmodule", new=lambda *args: None), + ): + # Simulate a frame from a Python extension module or multithreaded code + # where the Python module cannot be identified. + run_and_report(demo_func) mock_stderr.seek(0) stderr_output = mock_stderr.read() assert_regex_match(expected_traceback_regex, stderr_output) @@ -291,15 +294,13 @@ def throws(*args: Any, **kwargs: Any) -> NoReturn: """ ) mock_stderr = io.StringIO() - with patch("lerna._internal.utils.is_under_debugger", return_value=False): - with ( - raises(AssertionError, match="nested_err"), - patch("sys.stderr", new=mock_stderr), - ): - # patch `traceback.print_exception` so that an exception will occur - # in the simplified traceback logic: - with patch("traceback.print_exception", new=throws): - run_and_report(demo_func) + with ( + patch("lerna._internal.utils.is_under_debugger", return_value=False), + raises(AssertionError, match="nested_err"), + patch("sys.stderr", new=mock_stderr), + patch("traceback.print_exception", new=throws), + ): + run_and_report(demo_func) mock_stderr.seek(0) stderr_output = mock_stderr.read() assert_regex_match(expected_traceback_regex, stderr_output) diff --git a/lerna/types.py b/lerna/types.py index d71fd5c..523735c 100644 --- a/lerna/types.py +++ b/lerna/types.py @@ -1,7 +1,8 @@ # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +from collections.abc import Callable from dataclasses import dataclass from enum import Enum -from typing import TYPE_CHECKING, Any, Callable +from typing import TYPE_CHECKING, Any from omegaconf import MISSING @@ -81,7 +82,7 @@ class ConvertMode(Enum): # Fully convert the OmegaConf config to primitive containers (dict, list and primitives). ALL = "all" - def __eq__(self, other: Any) -> Any: + def __eq__(self, other: object) -> Any: if isinstance(other, ConvertMode): return other.value == self.value elif isinstance(other, str): diff --git a/lerna/utils.py b/lerna/utils.py index d78d50d..4eea4e1 100644 --- a/lerna/utils.py +++ b/lerna/utils.py @@ -3,8 +3,9 @@ import json import logging.config import os +from collections.abc import Callable from pathlib import Path -from typing import Any, Callable +from typing import Any import lerna._internal.instantiate._instantiate2 import lerna.types @@ -31,11 +32,13 @@ def get_class(path: str) -> type: try: cls = _locate(path) if not isinstance(cls, type): - raise ValueError(f"Located non-class of type '{type(cls).__name__}'" + f" while loading '{path}'") + raise ValueError( # noqa: TRY004 + f"Located non-class of type '{type(cls).__name__}'" + f" while loading '{path}'" + ) return cls except Exception as e: log.error(f"Error getting class at {path}: {e}") - raise e + raise def get_method(path: str) -> Callable[..., Any]: @@ -50,12 +53,14 @@ def get_method(path: str) -> Callable[..., Any]: try: obj = _locate(path) if not callable(obj): - raise ValueError(f"Located non-callable of type '{type(obj).__name__}'" + f" while loading '{path}'") + raise ValueError( # noqa: TRY004 + f"Located non-callable of type '{type(obj).__name__}'" + f" while loading '{path}'" + ) cl: Callable[..., Any] = obj return cl except Exception as e: log.error(f"Error getting callable at {path} : {e}") - raise e + raise # Alias for get_method @@ -76,7 +81,7 @@ def get_object(path: str) -> Any: return obj except Exception as e: log.error(f"Error getting object at {path} : {e}") - raise e + raise def get_original_cwd() -> str: diff --git a/lerna/version.py b/lerna/version.py index f3d9032..32bd34d 100644 --- a/lerna/version.py +++ b/lerna/version.py @@ -3,7 +3,7 @@ # Source of truth for Hydra's version from textwrap import dedent -from typing import Any, Optional +from typing import Any from packaging.version import Version @@ -19,13 +19,13 @@ class VersionBase(metaclass=Singleton): def __init__(self) -> None: - self.version_base: Optional[Version] = _UNSPECIFIED_ + self.version_base: Version | None = _UNSPECIFIED_ def setbase(self, version: "Version") -> None: assert isinstance(version, Version), f"Unexpected Version type : {type(version)}" self.version_base = version - def getbase(self) -> Optional[Version]: + def getbase(self) -> Version | None: return self.version_base @staticmethod @@ -53,7 +53,7 @@ def base_at_least(ver: str) -> bool: return _version_base >= _get_version(ver) -def getbase() -> Optional[Version]: +def getbase() -> Version | None: return VersionBase.instance().getbase()