diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 47ca0c24..191185bf 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -26,9 +26,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r test_requirements.txt - - name: Full integration test + pip install pytest pyyaml optuna + pip install -e . --no-deps + - name: Unit tests run: | - # `./component-library` was removed in the repo reorganization; - # run_tests.py is at the repo root. - python ./run_tests.py + pytest tests/unit/test_iterate2.py -v diff --git a/docs/iterate2.md b/docs/iterate2.md index 0a84e261..62c4ed6c 100644 --- a/docs/iterate2.md +++ b/docs/iterate2.md @@ -234,13 +234,38 @@ The **last** occurrence of the pattern `: ` or ` dict: logger.info("Static params: %d key(s): %s", len(static), list(static.keys())) return static -def load_metrics(data: dict, fallback: str = "score") -> List[str]: +_MINIMIZE_KEYWORDS = ("loss", "error", "err", "mse", "mae", "rmse") + +def _default_direction(metric_name: str) -> str: + """Heuristic: metrics whose name contains a loss/error keyword are minimized.""" + lower = metric_name.lower() + if any(kw in lower for kw in _MINIMIZE_KEYWORDS): + return "minimize" + return "maximize" + +def load_metrics(data: dict, fallback: str = "score") -> tuple[List[str], List[str]]: + """Return (metric_names, directions). + + The YAML ``metrics:`` section accepts two forms: + + Simple (direction inferred from name):: + + metrics: + - val_loss # → minimize (contains "loss") + - accuracy # → maximize + + Extended (direction explicit):: + + metrics: + - name: val_loss + direction: minimize + - name: accuracy + direction: maximize + + A mix of both forms is allowed. + """ raw = data.get("metrics", None) if raw is None: logger.warning("No 'metrics:' key in YAML – defaulting to '%s'", fallback) - return [fallback] - if isinstance(raw, list): - return [str(m).strip() for m in raw] - return [m.strip() for m in str(raw).split(",")] + return [fallback], [_default_direction(fallback)] + + if not isinstance(raw, list): + # comma-separated string fallback + names = [m.strip() for m in str(raw).split(",")] + directions = [_default_direction(n) for n in names] + return names, directions + + names: List[str] = [] + directions: List[str] = [] + for item in raw: + if isinstance(item, dict): + name = str(item["name"]).strip() + direction = str(item.get("direction", _default_direction(name))).strip().lower() + if direction not in ("minimize", "maximize"): + raise ValueError( + f"Invalid direction '{direction}' for metric '{name}'. " + "Must be 'minimize' or 'maximize'." + ) + else: + name = str(item).strip() + direction = _default_direction(name) + names.append(name) + directions.append(direction) + return names, directions # ─── OPTUNA PARAM SAMPLING ─────────────────────────────────────────────────── @@ -216,12 +273,12 @@ def main(): args.optuna_study_name, args.optuna_db_path, args.optuna_n_trials, args.parallelism) - data = load_yaml(args.hpo_yaml) - hpo_space = load_hpo_space(data) - static = load_static(data) - metrics = load_metrics(data) - directions = ["maximize"] * len(metrics) + data = load_yaml(args.hpo_yaml) + hpo_space = load_hpo_space(data) + static = load_static(data) + metrics, directions = load_metrics(data) logger.info("Metrics: %s", metrics) + logger.info("Directions: %s", directions) storage = resolve_storage(args.optuna_db_path) study = optuna.create_study(