From ac6cb2f1f27182e6aba346f04a4dbecb44a03482 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 6 Apr 2026 20:56:31 -0400 Subject: [PATCH 1/7] Migration to peprs --- pephub/const.py | 8 +- pephub/helpers.py | 20 +-- pephub/main.py | 6 +- pephub/routers/api/v1/helpers.py | 54 +++----- pephub/routers/api/v1/namespace.py | 27 ++-- pephub/routers/api/v1/project.py | 56 ++++---- pephub/routers/eido/eido.py | 130 +++++++++++------- pephub/routers/models.py | 18 +-- pephub/routers/views/eido.py | 17 ++- requirements/requirements-all.txt | 3 +- tests/conftest.py | 2 +- tests/test_validation.py | 14 +- web/src/api/namespace.ts | 4 +- web/src/api/project.ts | 2 +- web/src/api/server.ts | 2 +- web/src/components/layout/page-layout.tsx | 2 +- .../project/project-info-footer.tsx | 2 +- .../components/project/project-interface.tsx | 6 +- .../project/project-page-description.tsx | 2 +- web/types.ts | 6 +- 20 files changed, 212 insertions(+), 169 deletions(-) diff --git a/pephub/const.py b/pephub/const.py index 739fdbba..fc03c944 100644 --- a/pephub/const.py +++ b/pephub/const.py @@ -6,18 +6,20 @@ import pandas as pd from fastapi import __version__ as fastapi_version from pepdbagent import __version__ as pepdbagent_version -from peppy import __version__ as peppy_version -from peppy.const import PEP_LATEST_VERSION from ._version import __version__ as pephub_version +# peprs has no __version__ attribute and no PEP_LATEST_VERSION constant. +PEP_LATEST_VERSION = "2.1.0" +peprs_version = "unknown" + PKG_NAME = "pephub" DATA_REPO = "https://github.com/pepkit/data.pephub.git" ALL_VERSIONS = { "pephub_version": pephub_version, - "peppy_version": peppy_version, + "peprs_version": peprs_version, "python_version": python_version(), "fastapi_version": fastapi_version, "pepdbagent_version": pepdbagent_version, diff --git a/pephub/helpers.py b/pephub/helpers.py index a1f13613..9247b4aa 100644 --- a/pephub/helpers.py +++ b/pephub/helpers.py @@ -9,16 +9,18 @@ import json from fastapi import Response, UploadFile from fastapi.exceptions import HTTPException -from peppy.const import ( - CFG_SAMPLE_TABLE_KEY, - CFG_SUBSAMPLE_TABLE_KEY, +from peprs.const import ( CONFIG_KEY, - NAME_KEY, SAMPLE_RAW_DICT_KEY, - SUBSAMPLE_RAW_LIST_KEY, + SUBSAMPLE_RAW_DICT_KEY, ) from .const import JWT_EXPIRATION, JWT_SECRET +# peprs.const does not export these — they are PEP config schema strings. +CFG_SAMPLE_TABLE_KEY = "sample_table" +CFG_SUBSAMPLE_TABLE_KEY = "subsample_table" +NAME_KEY = "name" + def jwt_encode_user_data(user_data: dict, exp: datetime = None) -> str: """ @@ -53,15 +55,15 @@ def zip_pep(project: Dict[str, Any]) -> Response: project[SAMPLE_RAW_DICT_KEY] ).to_csv(index=False) - if project[SUBSAMPLE_RAW_LIST_KEY] is not None: - if not isinstance(project[SUBSAMPLE_RAW_LIST_KEY], list): + if project[SUBSAMPLE_RAW_DICT_KEY] is not None: + if not isinstance(project[SUBSAMPLE_RAW_DICT_KEY], list): config[CFG_SUBSAMPLE_TABLE_KEY] = ["subsample_table1.csv"] content_to_zip["subsample_table1.csv"] = pd.DataFrame( - project[SUBSAMPLE_RAW_LIST_KEY] + project[SUBSAMPLE_RAW_DICT_KEY] ).to_csv(index=False) else: config[CFG_SUBSAMPLE_TABLE_KEY] = [] - for number, file in enumerate(project[SUBSAMPLE_RAW_LIST_KEY]): + for number, file in enumerate(project[SUBSAMPLE_RAW_DICT_KEY]): file_name = f"subsample_table{number + 1}.csv" config[CFG_SUBSAMPLE_TABLE_KEY].append(file_name) content_to_zip[file_name] = pd.DataFrame(file).to_csv(index=False) diff --git a/pephub/main.py b/pephub/main.py index 7b9bd491..6dc91cc1 100644 --- a/pephub/main.py +++ b/pephub/main.py @@ -28,12 +28,12 @@ fmt="[%(levelname)s] [%(asctime)s] [PEPDBAGENT] %(message)s", ) -_LOGGER_PEPPY = logging.getLogger("peppy") +_LOGGER_PEPRS = logging.getLogger("peprs") coloredlogs.install( - logger=_LOGGER_PEPPY, + logger=_LOGGER_PEPRS, level=logging.ERROR, datefmt="%b %d %Y %H:%M:%S", - fmt="[%(levelname)s] [%(asctime)s] [PEPPY] %(message)s", + fmt="[%(levelname)s] [%(asctime)s] [PEPRS] %(message)s", ) _LOGGER_PEPHUB = logging.getLogger("uvicorn.access") diff --git a/pephub/routers/api/v1/helpers.py b/pephub/routers/api/v1/helpers.py index 1610a249..70eae68a 100644 --- a/pephub/routers/api/v1/helpers.py +++ b/pephub/routers/api/v1/helpers.py @@ -1,16 +1,14 @@ import logging -import eido -from eido.validation import validate_config -from eido.exceptions import EidoValidationError -import peppy +import peprs import yaml from fastapi.exceptions import HTTPException -from peppy import Project -from peppy.const import ( +from peprs import Project +from peprs.eido import EidoValidationError, validate_config, validate_project +from peprs.const import ( CONFIG_KEY, SAMPLE_RAW_DICT_KEY, - SUBSAMPLE_RAW_LIST_KEY, + SUBSAMPLE_RAW_DICT_KEY, ) from ....dependencies import ( get_db, @@ -22,7 +20,7 @@ DEFAULT_SCHEMA_VERSION = "2.1.0" -async def verify_updated_project(updated_project) -> peppy.Project: +async def verify_updated_project(updated_project) -> peprs.Project: new_raw_project = {} agent = get_db() @@ -37,43 +35,23 @@ async def verify_updated_project(updated_project) -> peppy.Project: status_code=400, detail="Please provide a sample table and project config yaml to update project", ) - try: - validate_config( - yaml.safe_load(updated_project.project_config_yaml), default_schema - ) - except EidoValidationError as e: - raise HTTPException( - status_code=400, - detail=f"Config structure error: {', '.join(list(e.errors_by_type.keys()))}. Please check schema definition and try again.", - ) - # sample table update - new_raw_project[SAMPLE_RAW_DICT_KEY] = updated_project.sample_table try: yaml_dict = yaml.safe_load(updated_project.project_config_yaml) - new_raw_project[CONFIG_KEY] = yaml_dict except yaml.scanner.ScannerError as e: raise HTTPException( status_code=400, detail=f"Could not parse provided yaml. Error: {e}", ) - # sample_table_index_col = yaml_dict.get( - # SAMPLE_TABLE_INDEX_KEY, SAMPLE_NAME_ATTR # default to sample_name - # ) - - # await check_sample_names( - # new_raw_project[SAMPLE_RAW_DICT_KEY], sample_table_index_col - # ) + new_raw_project[CONFIG_KEY] = yaml_dict + new_raw_project[SAMPLE_RAW_DICT_KEY] = updated_project.sample_table # subsample table update if updated_project.subsample_tables is not None: subsamples = list(updated_project.subsample_tables[0][0].values()) - new_raw_project[SUBSAMPLE_RAW_LIST_KEY] = ( - updated_project.subsample_tables - if len(subsamples) > 0 and subsamples[0] - else None - ) + if len(subsamples) > 0 and subsamples[0]: + new_raw_project[SUBSAMPLE_RAW_DICT_KEY] = updated_project.subsample_tables try: new_project = Project.from_dict(new_raw_project) @@ -83,9 +61,19 @@ async def verify_updated_project(updated_project) -> peppy.Project: detail=f"Could not create PEP from provided data. Error: {e}", ) + # peprs.eido.validate_config takes a Project (not a raw dict like eido did), + # so we validate after constructing the Project. + try: + validate_config(new_project, default_schema) + except EidoValidationError as e: + raise HTTPException( + status_code=400, + detail=f"Config structure error: {', '.join(list(e.errors_by_type.keys()))}. Please check schema definition and try again.", + ) + try: # validate project (it will also validate samples) - eido.validate_project(new_project, default_schema) + validate_project(new_project, default_schema) except Exception as _: raise HTTPException( status_code=400, diff --git a/pephub/routers/api/v1/namespace.py b/pephub/routers/api/v1/namespace.py index ac357566..022b60ca 100644 --- a/pephub/routers/api/v1/namespace.py +++ b/pephub/routers/api/v1/namespace.py @@ -3,7 +3,7 @@ from typing import List, Literal, Optional, Union import os -import peppy +import peprs from dotenv import load_dotenv from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, Request from fastapi.responses import JSONResponse @@ -22,10 +22,13 @@ NamespaceStats, TarNamespaceModelReturn, ) -from peppy import Project -from peppy.const import DESC_KEY, NAME_KEY +from peprs import Project from typing_extensions import Annotated +# peprs.const does not export these — they are PEP config schema strings. +NAME_KEY = "name" +DESC_KEY = "description" + from ....const import ( DEFAULT_TAG, ARCHIVE_URL_PATH, @@ -213,7 +216,7 @@ async def create_pep( }, status_code=202, ) - # create a blank peppy.Project object with fake files + # create a blank peprs.Project object with fake files else: raise HTTPException( detail="Project files were not provided", @@ -258,21 +261,23 @@ async def upload_raw_pep( # This configurations needed due to Issue #124 Should be removed in the future project_dict = ProjectRawModel(**project_from_json.pep_dict.dict()) ff = project_dict.model_dump(by_alias=True) - p_project = peppy.Project().from_dict(ff) + p_project = peprs.Project.from_dict(ff) - p_project.namespace = name + # peprs.Project has no `namespace` attribute, so we set the registry name + # in the config and pass `name` separately to agent.project.create. + p_project.name = name p_project.description = description except Exception as e: raise HTTPException( - detail=f"Incorrect raw project was provided. Couldn't initiate peppy object: {e}", + detail=f"Incorrect raw project was provided. Couldn't initiate peprs object: {e}", status_code=417, ) try: agent.project.create( p_project, namespace=namespace, - name=p_project.namespace, + name=name, tag=tag, description=description, is_private=is_private, @@ -282,15 +287,15 @@ async def upload_raw_pep( ) except ProjectUniqueNameError: raise HTTPException( - detail=f"Project '{namespace}/{p_project.namespace}:{tag}' already exists in namespace", + detail=f"Project '{namespace}/{name}:{tag}' already exists in namespace", status_code=400, ) return JSONResponse( content={ "namespace": namespace, - "name": p_project.namespace, + "name": name, "tag": tag, - "registry_path": f"{namespace}/{p_project.namespace}:{tag}", + "registry_path": f"{namespace}/{name}:{tag}", }, status_code=202, ) diff --git a/pephub/routers/api/v1/project.py b/pephub/routers/api/v1/project.py index ea181b92..7ac5d8f6 100644 --- a/pephub/routers/api/v1/project.py +++ b/pephub/routers/api/v1/project.py @@ -1,10 +1,9 @@ import logging from typing import Annotated, Any, Literal, Dict, List, Optional, Union -import eido import numpy as np import pandas as pd -import peppy +import peprs import yaml from dotenv import load_dotenv from fastapi import APIRouter, Body, Depends, Query @@ -29,7 +28,7 @@ ProjectViews, HistoryAnnotationModel, ) -from peppy.const import SAMPLE_RAW_DICT_KEY +from peprs.const import SAMPLE_RAW_DICT_KEY # from ....const import SAMPLE_CONVERSION_FUNCTIONS from ....dependencies import ( @@ -262,25 +261,23 @@ async def get_pep_samples( ) if isinstance(proj, dict): - if len(proj["_sample_dict"]) > MAX_PROCESSED_PROJECT_SIZE: + if len(proj[SAMPLE_RAW_DICT_KEY]) > MAX_PROCESSED_PROJECT_SIZE: raise HTTPException( status_code=400, detail=f"Project is too large. View raw samples, or create a view. Limit is {MAX_PROCESSED_PROJECT_SIZE} samples.", ) - proj = peppy.Project.from_dict(proj) + proj = peprs.Project.from_dict(proj) if format == "json": return { "samples": [sample.to_dict() for sample in proj.samples], } elif format == "csv": - return PlainTextResponse(eido.convert_project(proj, "csv")["samples"]) + return PlainTextResponse(proj.to_csv_string()) elif format == "yaml": - return PlainTextResponse( - eido.convert_project(proj, "yaml-samples")["samples"] - ) + return PlainTextResponse(proj.to_yaml_string()) elif format == "basic": - return eido.convert_project(proj, "basic") + return proj.to_dict() if raw: df = pd.DataFrame(proj[SAMPLE_RAW_DICT_KEY]) @@ -289,12 +286,12 @@ async def get_pep_samples( items=df.replace({np.nan: None}).to_dict(orient="records"), ) if isinstance(proj, dict): - if len(proj["_sample_dict"]) > MAX_PROCESSED_PROJECT_SIZE: + if len(proj[SAMPLE_RAW_DICT_KEY]) > MAX_PROCESSED_PROJECT_SIZE: raise HTTPException( status_code=400, detail=f"Project is too large. View raw samples, or create a view. Limit is {MAX_PROCESSED_PROJECT_SIZE} samples.", ) - proj = peppy.Project.from_dict(proj) + proj = peprs.Project.from_dict(proj) return [sample.to_dict() for sample in proj.samples] @@ -500,7 +497,7 @@ async def delete_sample( @project.get("/subsamples", response_model=SamplesResponseModel) async def get_subsamples_endpoint( - subsamples: peppy.Project = Depends(get_subsamples), + subsamples: list = Depends(get_subsamples), download: bool = False, ): """ @@ -543,11 +540,8 @@ async def convert_pep( format: Optional[str] = "plain", ): """ - Convert a PEP to a specific format, f. For a list of available formats/filters, - see /eido/filters. - - See, http://eido.databio.org/en/latest/filters/#convert-a-pep-into-an-alternative-format-with-a-filter - for more information. + Convert a PEP to a specific format. Supported filters are: basic, csv, yaml, + yaml-samples, json. Don't have a namespace, or project? @@ -559,18 +553,26 @@ async def convert_pep( """ # default to basic if filter is None: - filter = "basic" # default to basic + filter = "basic" - # validate filter exists - filter_list = eido.get_available_pep_filters() - if filter not in filter_list: + # eido filter infrastructure is not in peprs; emulate the previously supported + # filters using peprs Project conversion methods. + available_filters = ["basic", "csv", "yaml", "yaml-samples", "json"] + if filter not in available_filters: raise HTTPException( - 400, f"Unknown filter '{filter}'. Available filters: {filter_list}" + 400, f"Unknown filter '{filter}'. Available filters: {available_filters}" ) - # generate result - peppy_project = peppy.Project.from_dict(proj) - conv_result = eido.run_filter(peppy_project, filter, verbose=False) + peprs_project = peprs.Project.from_dict(proj) + + if filter == "basic": + conv_result = {"project_config.yaml": peprs_project.to_yaml_string()} + elif filter == "csv": + conv_result = {"sample_table.csv": peprs_project.to_csv_string()} + elif filter in ("yaml", "yaml-samples"): + conv_result = {"sample_table.yaml": peprs_project.to_yaml_string()} + else: # json + conv_result = {"project.json": peprs_project.to_json_string()} if format == "plain": return_str = "\n".join([conv_result[k] for k in conv_result]) @@ -996,7 +998,7 @@ def get_project_history_by_id( with_id=True, ) # convert the config to a yaml string - project_at_history["_config"] = yaml.dump(project_at_history["_config"]) + project_at_history["config"] = yaml.dump(project_at_history["config"]) return project_at_history except ProjectNotFoundError: diff --git a/pephub/routers/eido/eido.py b/pephub/routers/eido/eido.py index 248b29a1..d1136ef0 100644 --- a/pephub/routers/eido/eido.py +++ b/pephub/routers/eido/eido.py @@ -2,8 +2,7 @@ import tempfile from typing import List, Optional, Tuple -import eido -import peppy +import peprs import requests import yaml from fastapi import APIRouter, Depends, Form, UploadFile @@ -13,6 +12,7 @@ from pepdbagent.exceptions import SchemaDoesNotExistError from pepdbagent.utils import schema_path_converter from pepdbagent.const import LATEST_SCHEMA_VERSION +from peprs.eido import EidoValidationError, validate_project from starlette.requests import Request from starlette.responses import JSONResponse @@ -31,6 +31,20 @@ async def status(): return JSONResponse(schemas_to_test) +def _read_schema(path_or_url: str) -> dict: + """ + Fetch and parse an eido YAML schema from a local file path or URL. + peprs.eido does not yet expose a read_schema helper, so we inline a + minimal implementation here. + """ + if path_or_url.startswith(("http://", "https://")): + resp = requests.get(path_or_url) + resp.raise_for_status() + return yaml.safe_load(resp.text) + with open(path_or_url) as f: + return yaml.safe_load(f) + + @router.get("/schemas/{namespace}/{project}") async def get_schema(request: Request, namespace: str, project: str): """ @@ -41,10 +55,10 @@ async def get_schema(request: Request, namespace: str, project: str): # like pipelines/ProseqPEP.yaml try: - schema = eido.read_schema( + schema = _read_schema( f"https://schema.databio.org/{namespace}/{project}.yaml" - )[0] - except IndexError: + ) + except Exception: raise HTTPException(status_code=404, detail="Schema not found") return schema @@ -90,12 +104,12 @@ async def validate( pep_annot = agent.annotation.get(namespace=namespace, name=name, tag=tag) - if pep_annot.results[0].number_of_samples > MAX_PROCESSED_PROJECT_SIZE: - return { - "valid": False, - "error_type": "Project size", - "errors": ["Project is too large. Can't validate."], - } + # if pep_annot.results[0].number_of_samples > MAX_PROCESSED_PROJECT_SIZE: + # return { + # "valid": False, + # "error_type": "Project size", + # "errors": ["Project is too large. Can't validate."], + # } p = agent.project.get(namespace, name, tag, raw=False) else: @@ -115,7 +129,7 @@ async def validate( with open(f"{dirpath}/{upload_file.filename}", "wb") as local_tmpf: shutil.copyfileobj(upload_file.file, local_tmpf) - p = peppy.Project(f"{dirpath}/{init_file.filename}") + p = peprs.Project(f"{dirpath}/{init_file.filename}") if schema is None and schema_registry is None and schema_file is None: raise HTTPException( @@ -156,28 +170,28 @@ async def validate( contents = schema_file.file.read() schema_dict = yaml.safe_load(contents) else: - # save schema string to temp file, then read in with eido - with tempfile.NamedTemporaryFile(mode="w") as schema_file: - schema_file.write(schema) - schema_file.flush() - try: - schema_dict = eido.read_schema(schema_file.name)[0] - except eido.exceptions.EidoSchemaInvalidError as e: - raise HTTPException( - status_code=200, - detail={"error": f"Schema is invalid: {str(e)}"}, - ) + # parse the schema string directly; peprs.eido has no read_schema / + # EidoSchemaInvalidError, so any parse failure is reported as a schema error. + try: + schema_dict = yaml.safe_load(schema) + if not isinstance(schema_dict, dict): + raise ValueError("Schema must parse to a YAML mapping") + except Exception as e: + raise HTTPException( + status_code=200, + detail={"error": f"Schema is invalid: {str(e)}"}, + ) # validate project try: - eido.validate_project( + validate_project( p, schema_dict, ) # while we catch this, its still a 200 response since we want to # return the validation errors - except eido.exceptions.EidoValidationError as e: + except EidoValidationError as e: error_type, property_names = await eido_error_string_converter(e) return {"valid": False, "error_type": error_type, "errors": property_names} @@ -193,33 +207,57 @@ async def validate( async def eido_error_string_converter( - e: eido.exceptions.EidoValidationError, + e: EidoValidationError, ) -> Tuple[str, List[str]]: """ Convert eido error into nice modified string - :param e: eido Validation error + peprs.eido.EidoValidationError.errors_by_type has the shape: + { + "": [ + {"path": ..., "message": ..., "sample_names": [...optional]}, + ... + ], + ... + } + An item without "sample_names" is a project-level error; otherwise it + is a sample-level error affecting the listed samples. + + :param e: peprs.eido Validation error :return: error_type, property_names """ - property_names = [] - error_type_list = [] - for item_list in e.errors_by_type.values(): - property_type = item_list[0]["type"] - property_name_list = [] + property_names: List[str] = [] + error_type_set = set() + messages = [] + + for error_type_key, item_list in e.errors_by_type.items(): + sample_names_all: List[str] = [] + has_project_level = False for item in item_list: - if item["sample_name"] == "project": - error_type_list.append("Project") - break + sample_names = item.get("sample_names") or [] + if sample_names: + sample_names_all.extend(sample_names) + messages.append(item.get("message")) + else: + has_project_level = True + + if sample_names_all: + error_type_set.add("Samples") + if len(sample_names_all) > 20: + property_names.append( + f"{error_type_key}: more than 20 samples have encountered errors." + ) else: - error_type_list.append("Samples") - if len(item_list) > 20: - property_names = ["More than 20 samples have encountered errors."] - else: - property_name_list.append(item["sample_name"]) - - if len(property_name_list) > 0: - property_names.append(f"{property_type} ({', '.join(property_name_list)})") - else: - property_names.append(f"{property_type} in the project") - error_type = " and ".join(set(error_type_list)) + property_names.append( + f"{error_type_key} ({', '.join(sample_names_all)})" + ) + if not sample_names_all: + property_names = messages + + if has_project_level: + error_type_set.add("Project") + property_names.append(f"{error_type_key} in the project") + + error_type = " and ".join(sorted(error_type_set)) + return error_type, property_names diff --git a/pephub/routers/models.py b/pephub/routers/models.py index 8e52579b..f649a545 100644 --- a/pephub/routers/models.py +++ b/pephub/routers/models.py @@ -74,25 +74,25 @@ class JWTDeviceTokenResponse(BaseModel): class ProjectRawModel(BaseModel): - config: dict = Field(alias="_config") - subsample_list: Optional[list] = Field(alias="_subsample_list", default=None) - sample_list: list[dict] = Field(alias="_sample_dict") + config: dict + subsamples: Optional[list] = None + samples: list[dict] model_config = ConfigDict(populate_by_name=True) class ProjectHistoryResponse(BaseModel): - config: str = Field(alias="_config") - subsample_list: Optional[list] = Field(alias="_subsample_list", default=None) - sample_list: list[dict] = Field(alias="_sample_dict") + config: str + subsamples: Optional[list] = None + samples: list[dict] model_config = ConfigDict(populate_by_name=True) class ProjectRawRequest(BaseModel): config: dict - subsample_list: Optional[List[List[dict]]] = None - sample_list: List[dict] + subsamples: Optional[List[List[dict]]] = None + samples: List[dict] model_config = ConfigDict(populate_by_name=True, extra="allow") @@ -120,7 +120,7 @@ class DeveloperKey(BaseModel): class VersionResponseModel(BaseModel): pephub_version: str - peppy_version: str + peprs_version: str python_version: str fastapi_version: str pepdbagent_version: str diff --git a/pephub/routers/views/eido.py b/pephub/routers/views/eido.py index d57fff81..7c3bc55c 100644 --- a/pephub/routers/views/eido.py +++ b/pephub/routers/views/eido.py @@ -1,15 +1,16 @@ from platform import python_version -import eido import jinja2 +import requests +import yaml from dotenv import load_dotenv from fastapi import APIRouter, Request +from fastapi.exceptions import HTTPException from fastapi.responses import HTMLResponse -from peppy import __version__ as peppy_version from starlette.templating import Jinja2Templates from ..._version import __version__ as pephub_version -from ...const import EIDO_TEMPLATES_PATH +from ...const import EIDO_TEMPLATES_PATH, peprs_version load_dotenv() @@ -18,7 +19,7 @@ ALL_VERSIONS = { "pephub_version": pephub_version, - "peppy_version": peppy_version, + "peprs_version": peprs_version, "python_version": python_version(), "api_version": 1, } @@ -35,7 +36,13 @@ async def get_schema(request: Request, namespace: str, project: str): # endpoint to schema.databio.org/... # like pipelines/ProseqPEP.yaml - schema = eido.read_schema(f"http://schema.databio.org/{namespace}/{project}") + # peprs.eido has no read_schema helper, so fetch and parse the YAML directly. + try: + resp = requests.get(f"http://schema.databio.org/{namespace}/{project}") + resp.raise_for_status() + schema = yaml.safe_load(resp.text) + except Exception: + raise HTTPException(status_code=404, detail="Schema not found") return templates.TemplateResponse( "schema.html", diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index b1b12340..c553d90e 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -2,8 +2,7 @@ fastapi>=0.108.0 psycopg>=3.1.15 pepdbagent>=0.12.4 # pepdbagent @ git+https://github.com/pepkit/pepdbagent.git@dev#egg=pepdbagent -peppy>=0.40.7 -eido>=0.2.4 +peprs jinja2>=3.1.2 python-multipart>=0.0.5 uvicorn diff --git a/tests/conftest.py b/tests/conftest.py index 2ab8d763..8cf53b2e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,7 @@ import pytest import requests from pepdbagent import PEPDatabaseAgent -from peppy import Project +from peprs import Project @pytest.fixture diff --git a/tests/test_validation.py b/tests/test_validation.py index b2b79f43..3b0c8667 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -1,4 +1,4 @@ -import eido +from peprs import eido import pytest from pephub.dependencies import * @@ -13,7 +13,7 @@ def test_file_file_validate_valid(project_object_file, schema_file_path): # test project file on schema file validation failure def test_file_file_validate_invalid(project_object_file, schema_file_path_invalid): - with pytest.raises(eido.exceptions.EidoValidationError): + with pytest.raises(eido.EidoValidationError): eido.validate_project( project=project_object_file, schema=schema_file_path_invalid ) @@ -26,7 +26,7 @@ def test_file_string_validate_valid(project_object_file, schema_paste): # test project file on schema string validation failure def test_file_string_validate_invalid(project_object_file, schema_paste_invalid): - with pytest.raises(eido.exceptions.EidoValidationError): + with pytest.raises(eido.EidoValidationError): eido.validate_project(project=project_object_file, schema=schema_paste_invalid) @@ -37,7 +37,7 @@ def test_file_url_validate_valid(project_object_file, schema_from_url_valid): # test project file on schema url validation failure def test_file_url_validate_invalid(project_object_file, schema_from_url_invalid): - with pytest.raises(eido.exceptions.EidoValidationError): + with pytest.raises(eido.EidoValidationError): eido.validate_project( project=project_object_file, schema=schema_from_url_invalid ) @@ -50,7 +50,7 @@ def test_registry_paste_valid(db, schema_paste): def test_registry_paste_invalid(db, schema_paste_invalid): p = db.project.get("ayobi", "new-project-test12345", "default") - with pytest.raises(eido.exceptions.EidoValidationError): + with pytest.raises(eido.EidoValidationError): eido.validate_project(project=p, schema=schema_paste_invalid) @@ -61,7 +61,7 @@ def test_registry_file_valid(db, schema_file_path): def test_registry_file_invalid(db, schema_file_path_invalid): p = db.project.get("ayobi", "new-project-test12345", "default") - with pytest.raises(eido.exceptions.EidoValidationError): + with pytest.raises(eido.EidoValidationError): eido.validate_project(project=p, schema=schema_file_path_invalid) @@ -72,5 +72,5 @@ def test_registry_url_valid(db, schema_from_url_valid): def test_registry_url_invalid(db, schema_from_url_invalid): p = db.project.get("ayobi", "new-project-test12345", "default") - with pytest.raises(eido.exceptions.EidoValidationError): + with pytest.raises(eido.EidoValidationError): eido.validate_project(project=p, schema=schema_from_url_invalid) diff --git a/web/src/api/namespace.ts b/web/src/api/namespace.ts index 933c2dff..663d0476 100644 --- a/web/src/api/namespace.ts +++ b/web/src/api/namespace.ts @@ -219,7 +219,7 @@ export const submitProjectJSON = ( { pep_dict: { config: config_json, - sample_list: sample_table, + samples: sample_table, }, description: description || '', name: name, @@ -269,7 +269,7 @@ export const submitPop = ( { pep_dict: { config: config_json, - sample_list: peps, + samples: peps, pep_schema: pep_schema, }, description: description || '', diff --git a/web/src/api/project.ts b/web/src/api/project.ts index 540c1dde..6793f24b 100644 --- a/web/src/api/project.ts +++ b/web/src/api/project.ts @@ -25,7 +25,7 @@ type ProjectUpdateMetadata = ProjectUpdateItems & { sample_table?: Sample[] | null; project_config_yaml?: string | null; description?: string | null; - subsample_list?: string[] | null; + subsample_tables?: any[][] | null; }; export type SampleTableResponse = { count: number; diff --git a/web/src/api/server.ts b/web/src/api/server.ts index ef3ab976..19e756cf 100644 --- a/web/src/api/server.ts +++ b/web/src/api/server.ts @@ -2,7 +2,7 @@ import axios from 'axios'; export interface ApiBase { pephub_version: string; - peppy_version: string; + peprs_version: string; python_version: string; fastapi_version: string; pepdbagent_version: string; diff --git a/web/src/components/layout/page-layout.tsx b/web/src/components/layout/page-layout.tsx index 9351a0e0..f2a80baa 100644 --- a/web/src/components/layout/page-layout.tsx +++ b/web/src/components/layout/page-layout.tsx @@ -24,7 +24,7 @@ const Footer: FC = () => {
pephub {data?.pephub_version || ''} - peppy {data?.peppy_version || ''} + peprs {data?.peprs_version || ''} Python {data?.python_version || ''} pepdbagent {data?.pepdbagent_version || ''}
diff --git a/web/src/components/project/project-info-footer.tsx b/web/src/components/project/project-info-footer.tsx index 1f564601..4a8144e9 100644 --- a/web/src/components/project/project-info-footer.tsx +++ b/web/src/components/project/project-info-footer.tsx @@ -49,7 +49,7 @@ export const ProjectInfoFooter = () => { {projectInfo?.pop ? 'Project' : 'Sample'} Count: {currentHistoryId !== null - ? projectHistoryQuery.data?._sample_dict.length + ? projectHistoryQuery.data?.samples.length : projectInfo?.number_of_samples}
diff --git a/web/src/components/project/project-interface.tsx b/web/src/components/project/project-interface.tsx index 35378dc1..e9183dc1 100644 --- a/web/src/components/project/project-interface.tsx +++ b/web/src/components/project/project-interface.tsx @@ -253,7 +253,7 @@ export const ProjectInterface = (props: Props) => { view !== undefined ? sampleListToArrays(viewSamples) : currentHistoryId - ? sampleListToArrays(historyData?._sample_dict || []) + ? sampleListToArrays(historyData?.samples || []) : newSamples } // height={window.innerHeight - 15 - (projectDataRef.current?.offsetTop || 300)} @@ -272,7 +272,7 @@ export const ProjectInterface = (props: Props) => { onChange={(subsamples) => { onChange(subsamples); }} - data={currentHistoryId ? sampleListToArrays(historyData?._subsample_list[0] || []) : newSubsamples} + data={currentHistoryId ? sampleListToArrays(historyData?.subsamples?.[0] || []) : newSubsamples} // height={window.innerHeight - 15 - (projectDataRef.current?.offsetTop || 300)} readOnly={!userCanEdit} /> @@ -285,7 +285,7 @@ export const ProjectInterface = (props: Props) => { name="config" render={({ field: { onChange } }) => ( { onChange(val); }} diff --git a/web/src/components/project/project-page-description.tsx b/web/src/components/project/project-page-description.tsx index f1e6505f..c3559b94 100644 --- a/web/src/components/project/project-page-description.tsx +++ b/web/src/components/project/project-page-description.tsx @@ -35,7 +35,7 @@ export const ProjectDescription = () => {
{currentHistoryId !== null - ? YAML.parse(projectHistoryQuery.data?._config || '')?.description || 'No description' + ? YAML.parse(projectHistoryQuery.data?.config || '')?.description || 'No description' : projectInfo?.description || 'No description'}
diff --git a/web/types.ts b/web/types.ts index f31c230b..b80debc0 100644 --- a/web/types.ts +++ b/web/types.ts @@ -127,9 +127,9 @@ export type ProjectAllHistory = { }; export type ProjectHistory = { - _config: string; - _subsample_list: any[]; - _sample_dict: Sample[]; + config: string; + subsamples: any[]; + samples: Sample[]; }; export interface PaginationResult { From 5f559986644edf84cbe38a94fc053eb0dfd1ec03 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 8 Apr 2026 16:30:09 -0400 Subject: [PATCH 2/7] Added eido as wasm bindings --- pephub/routers/eido/eido.py | 3 + web/package.json | 1 + web/src/components/forms/validator-form.tsx | 165 ++++++++---------- .../components/modals/validation-result.tsx | 23 ++- .../components/project/project-interface.tsx | 45 ++++- .../project-validation-and-edit-buttons.tsx | 31 ++-- .../project/validation/validation-result.tsx | 19 +- web/src/hooks/queries/useValidation.ts | 57 ------ web/vite.config.ts | 9 + 9 files changed, 164 insertions(+), 189 deletions(-) delete mode 100644 web/src/hooks/queries/useValidation.ts diff --git a/pephub/routers/eido/eido.py b/pephub/routers/eido/eido.py index d1136ef0..5b0061c2 100644 --- a/pephub/routers/eido/eido.py +++ b/pephub/routers/eido/eido.py @@ -64,6 +64,9 @@ async def get_schema(request: Request, namespace: str, project: str): return schema +# Note: The pephub web UI now runs validation client-side via the +# @pepkit/peprs WASM bindings. This server-side endpoint is kept for +# programmatic/API consumers. See validation_wasm_plan.md. @router.post("/validate") async def validate( # accept both pep_registry and pep_files, both should be optional diff --git a/web/package.json b/web/package.json index c0a240fe..90c0c591 100644 --- a/web/package.json +++ b/web/package.json @@ -15,6 +15,7 @@ "@hookform/error-message": "^2.0.1", "@mdx-js/react": "^3.0.0", "@monaco-editor/react": "^4.4.6", + "@pepkit/peprs": "file:../../peprs/peprs-wasm/pkg", "@popperjs/core": "^2.11.6", "@tanstack/react-query": "^5.0.0", "@tanstack/react-query-devtools": "^5.0.0", diff --git a/web/src/components/forms/validator-form.tsx b/web/src/components/forms/validator-form.tsx index bbfb2508..6ee2d189 100644 --- a/web/src/components/forms/validator-form.tsx +++ b/web/src/components/forms/validator-form.tsx @@ -6,8 +6,12 @@ import Select from 'react-select'; import { useSession } from '../../contexts/session-context'; import { useNamespaceProjects } from '../../hooks/queries/useNamespaceProjects'; -import { ValidationParams } from '../../hooks/queries/useValidation'; -import { useValidation } from '../../hooks/queries/useValidation'; +import { PepValidationOutcome, validatePep } from '../../utils/validate-pep'; +import { + prepareSchema, + preparePepFromFiles, + preparePepFromRegistry, +} from '../../utils/validator-form-helpers'; import { popFileFromFileList } from '../../utils/dragndrop'; import { FileDropZone } from './components/file-dropzone'; import { SchemaDropdown } from './components/schemas-databio-dropdown'; @@ -35,7 +39,6 @@ export const ValidatorForm: FC = ({ defaultPepRegistryPath, const { user, login } = useSession(); const { data: projects } = useNamespaceProjects(user?.login, {}); - // instantiate form const { reset: resetForm, setValue: setFormValue, @@ -45,16 +48,10 @@ export const ValidatorForm: FC = ({ defaultPepRegistryPath, } = useForm({ defaultValues: { pepRegistryPath: defaultPepRegistryPath - ? { - label: defaultPepRegistryPath || '', - value: defaultPepRegistryPath || '', - } + ? { label: defaultPepRegistryPath, value: defaultPepRegistryPath } : null, schemaRegistryPath: defaultSchemaRegistryPath - ? { - label: defaultSchemaRegistryPath || '', - value: defaultSchemaRegistryPath || '', - } + ? { label: defaultSchemaRegistryPath, value: defaultSchemaRegistryPath } : null, }, }); @@ -64,42 +61,15 @@ export const ValidatorForm: FC = ({ defaultPepRegistryPath, const [useExistingPEP, setUseExistingPEP] = useState(true); const [useExistingSchema, setUseExistingSchema] = useState(true); - // watch the form data so we can use it const pepFiles = watch('pepFiles'); const pepRegistryPath = watch('pepRegistryPath'); const schemaFile = watch('schemaFile'); const schemaRegistryPath = watch('schemaRegistryPath'); const schemaPasteValue = watch('schemaPaste'); - // validation params for the useValidation hook - let params = { - enabled: false, - } as ValidationParams; - - // populate params based on form data for the PEP - if (useExistingPEP) { - params.pepRegistry = pepRegistryPath?.value; - params.pepFiles = undefined; - } else { - params.pepRegistry = undefined; - params.pepFiles = pepFiles; - } - - // populate params based on form data for the schema - if (useExistingSchema) { - params.schema_registry = schemaRegistryPath?.value; - params.schema = undefined; - } else if (schemaPasteValue) { - params.schema_registry = undefined; - params.schema = schemaPasteValue; - } else { - params.schema_registry = undefined; - // just take the first file they give - params.schema_file = schemaFile && schemaFile.length > 0 ? schemaFile[0] : undefined; - } - - // validator hook - const { data: result, error, isFetching: isValidating, refetch } = useValidation(params); + const [isValidating, setIsValidating] = useState(false); + const [result, setResult] = useState(); + const [runError, setRunError] = useState(); const resetValidator = () => { resetForm({ @@ -109,29 +79,41 @@ export const ValidatorForm: FC = ({ defaultPepRegistryPath, schemaRegistryPath: null, schemaPaste: undefined, }); + setResult(undefined); + setRunError(undefined); }; - const runValidation = () => { - refetch(); + const runValidation = async () => { + setIsValidating(true); + setResult(undefined); + setRunError(undefined); + try { + const pep = useExistingPEP + ? await preparePepFromRegistry(pepRegistryPath?.value || '') + : await preparePepFromFiles(pepFiles as FileList); + + const schema = await prepareSchema({ + registryPath: useExistingSchema ? schemaRegistryPath?.value : undefined, + file: !useExistingSchema && schemaFile && schemaFile.length > 0 ? schemaFile[0] : undefined, + pasted: !useExistingSchema && !schemaFile && schemaPasteValue ? schemaPasteValue : undefined, + }); + + const outcome = await validatePep({ + configYaml: pep.configYaml, + samples: pep.samples, + subsamples: pep.subsamples, + schema, + }); + setResult(outcome); + } catch (e) { + setRunError(e instanceof Error ? e.message : String(e)); + } finally { + setIsValidating(false); + } }; return ( <> - {/* Only in development mode */} - {/* render the params */} - {/* @ts-ignore */} - {process.env.NODE_ENV === 'development' && ( -
- 
-            {JSON.stringify(params, null, 2)}
-
- 
-            Use existing PEP: {JSON.stringify(useExistingPEP)}
-            

-            Use existing schema: {JSON.stringify(useExistingSchema)}
-
-
- )}
@@ -243,10 +225,7 @@ export const ValidatorForm: FC = ({ defaultPepRegistryPath, { - setFormValue('schemaRegistryPath', { - value: schema, - label: schema, - }); + setFormValue('schemaRegistryPath', { value: schema, label: schema }); }} /> )} @@ -298,8 +277,13 @@ export const ValidatorForm: FC = ({ defaultPepRegistryPath,
-
- ) : error ? ( + ) : runError ? (
- 
-              {JSON.stringify(error, null, 2)}
-
+ 
{runError}
) : result ? ( - <> - {result.valid ? ( -
-

PEP is valid!

-
- ) : ( - <> -
-

- {result.error_type === 'Schema' ? 'Schema is invalid, found issue with:' : 'PEP is invalid!'} -

-

{result.error_type !== 'Schema' && <>Errors found in {result.error_type} }

- - {result.errors.map((e) => ( - 
-                        
-                        {`${e}`}
-
- ))} - -
- - )} - + result.state === 'valid' ? ( +
+

PEP is valid!

+
+ ) : result.state === 'error' ? ( +
+

Validation could not run:

+ 
{result.message}
+
+ ) : ( +
+

PEP is invalid!

+

Errors found in {result.errorType}

+ + {result.errors.map((e, i) => ( + 
+                    
+                    {e}
+
+ ))} + +
+ ) ) : null}
diff --git a/web/src/components/modals/validation-result.tsx b/web/src/components/modals/validation-result.tsx index 975aeb2a..8be86bf8 100644 --- a/web/src/components/modals/validation-result.tsx +++ b/web/src/components/modals/validation-result.tsx @@ -4,13 +4,13 @@ import { Controller, useForm } from 'react-hook-form'; import { useProjectPage } from '../../contexts/project-page-context'; import { useEditProjectMetaMutation } from '../../hooks/mutations/useEditProjectMetaMutation'; -import { useValidation } from '../../hooks/queries/useValidation'; +import { PepValidationOutcome } from '../../utils/validate-pep'; import { SchemaDropdown } from '../forms/components/schemas-databio-dropdown'; type Props = { show: boolean; onHide: () => void; - validationResult: ReturnType['data']; + validationResult: PepValidationOutcome | undefined; currentSchema: string | undefined; }; @@ -58,7 +58,7 @@ export const ValidationResultModal = (props: Props) => {

{currentSchema ? ( <> - {validationResult?.valid ? ( + {validationResult?.state === 'valid' ? ( Validation Passed @@ -78,16 +78,25 @@ export const ValidationResultModal = (props: Props) => { {currentSchema && ( <> - {validationResult?.valid ? ( + {validationResult?.state === 'valid' ? (

Your PEP is valid against the schema.

- ) : ( + ) : validationResult?.state === 'invalid' ? (

Your PEP is invalid against the schema.

-

Validation result:

+

Errors found in {validationResult.errorType}:

+ 
+                  {validationResult.errors.join('\n')}
+
+ + ) : validationResult?.state === 'error' ? ( + +

Validation could not run:

-                  {JSON.stringify(validationResult, null, 2)}
+                  {validationResult.message}
+ ) : ( +

Validating...

)} )} diff --git a/web/src/components/project/project-interface.tsx b/web/src/components/project/project-interface.tsx index e9183dc1..915a7639 100644 --- a/web/src/components/project/project-interface.tsx +++ b/web/src/components/project/project-interface.tsx @@ -25,6 +25,8 @@ import { ProjectValidationAndEditButtons } from './project-validation-and-edit-b import { StandardizeMetadataModal } from '../modals/standardize-metadata'; import { useStandardizeModalStore } from '../../hooks/stores/useStandardizeModalStore' import { useSchemaVersions } from '../../hooks/queries/useSchemaVersions'; +import { useSchemaJson } from '../../hooks/queries/useSchemaJson'; +import { useClientSidePepValidation } from '../../hooks/useClientSidePepValidation'; type Props = { projectConfig: ReturnType['data']; @@ -90,6 +92,37 @@ export const ProjectInterface = (props: Props) => { const newSubsamples = projectUpdates.watch('subsamples'); const newConfig = projectUpdates.watch('config'); + // Client-side live validation via peprs WASM. + const { data: schemaJson } = useSchemaJson(projectInfo?.pep_schema); + + const liveValidationInput = (() => { + try { + const samples = arraysToSampleList(newSamples || [], 'Sample') as unknown as Record[]; + const subsamplesParsed = newSubsamples && newSubsamples.length > 1 + ? [arraysToSampleList(newSubsamples, 'Subsample') as unknown as Record[]] + : undefined; + return { samples, subsamples: subsamplesParsed, parseError: undefined as string | undefined }; + } catch (e) { + return { + samples: undefined, + subsamples: undefined, + parseError: e instanceof Error ? e.message : String(e), + }; + } + })(); + + const { isValidating, result: validationResult } = useClientSidePepValidation({ + configYaml: newConfig, + samples: liveValidationInput.samples, + subsamples: liveValidationInput.subsamples, + schema: schemaJson, + enabled: !!schemaJson && !liveValidationInput.parseError, + }); + + const effectiveValidationResult = liveValidationInput.parseError + ? { state: 'error' as const, message: liveValidationInput.parseError } + : validationResult; + const userCanEdit = projectInfo && canEdit(user, projectInfo); const { isPending: isSubmitting, submit } = useTotalProjectChangeMutation(namespace, projectName, tag); @@ -175,11 +208,9 @@ export const ProjectInterface = (props: Props) => { ctrlKey = e.ctrlKey; break; } - // SAVE (ctrl + s) + // SAVE (ctrl + s) — only when the form is actually dirty. if (ctrlKey && e.key === 's') { - if (true && !isSubmitting) { - // TODO: why does this not work in production? - // if (projectUpdates.formState.isDirty && !isSubmitting) { + if (projectUpdates.formState.isDirty && !isSubmitting) { e.preventDefault(); handleSubmit(); } @@ -229,13 +260,13 @@ export const ProjectInterface = (props: Props) => {
diff --git a/web/src/components/project/project-validation-and-edit-buttons.tsx b/web/src/components/project/project-validation-and-edit-buttons.tsx index 1a8f2e72..c1007efa 100644 --- a/web/src/components/project/project-validation-and-edit-buttons.tsx +++ b/web/src/components/project/project-validation-and-edit-buttons.tsx @@ -1,12 +1,10 @@ import { Fragment } from 'react'; -import { OverlayTrigger, Tooltip } from 'react-bootstrap'; import { useProjectPage } from '../../contexts/project-page-context'; import { useSession } from '../../contexts/session-context'; import { useProjectAnnotation } from '../../hooks/queries/useProjectAnnotation'; -import { useValidation } from '../../hooks/queries/useValidation'; +import { PepValidationOutcome } from '../../utils/validate-pep'; import { canEdit } from '../../utils/permissions'; -import { StatusIcon } from '../badges/status-icons'; import { ProjectDataNav } from '../layout/project-data-nav'; import { ValidationResult } from './validation/validation-result'; @@ -16,26 +14,26 @@ type ProjectValidationAndEditButtonsProps = { reset: () => void; handleSubmit: () => void; filteredSamples: string[]; + validationResult: PepValidationOutcome | undefined; + isValidating: boolean; }; -const MAX_SAMPLES_FOR_VALIDATION = 5000; - export const ProjectValidationAndEditButtons = (props: ProjectValidationAndEditButtonsProps) => { - const { isDirty, isUpdatingProject, reset, handleSubmit, filteredSamples } = props; + const { + isDirty, + isUpdatingProject, + reset, + handleSubmit, + filteredSamples, + validationResult, + isValidating, + } = props; const { user } = useSession(); const { namespace, projectName, tag } = useProjectPage(); const { data: projectInfo } = useProjectAnnotation(namespace, projectName, tag); - const shouldValidate: boolean = (projectInfo?.number_of_samples || 0) > MAX_SAMPLES_FOR_VALIDATION; - - const projectValidationQuery = useValidation({ - pepRegistry: `${namespace}/${projectName}:${tag}`, - schema_registry: projectInfo?.pep_schema || 'pep/2.0.0', - enabled: !!projectInfo?.pep_schema, - }); - const validationResult = projectValidationQuery.data; const projectSchema = projectInfo?.pep_schema; const userHasOwnership = user && projectInfo && canEdit(user, projectInfo); @@ -50,14 +48,14 @@ export const ProjectValidationAndEditButtons = (props: ProjectValidationAndEditB