From 5fb35d58d92ed8b5e14060f819eb3b58affd0923 Mon Sep 17 00:00:00 2001 From: Daniel Danis Date: Fri, 10 Sep 2021 21:48:33 -0400 Subject: [PATCH 01/37] Simplify pom.xml files --- mvnw | 0 pom.xml | 23 ++++------------- validator-cli/pom.xml | 14 ----------- validator-core/pom.xml | 57 ------------------------------------------ 4 files changed, 5 insertions(+), 89 deletions(-) mode change 100644 => 100755 mvnw diff --git a/mvnw b/mvnw old mode 100644 new mode 100755 diff --git a/pom.xml b/pom.xml index da1fe05..f478ff9 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,10 @@ UTF-8 11 + + 3.14.0 + 2.0.0 + 5.7.1 @@ -89,29 +93,12 @@ org.slf4j slf4j-api - - ch.qos.logback - logback-classic - test - - - - - - org.junit.jupiter - junit-jupiter-api - ${junit.jupiter.version} + junit-jupiter test - - org.junit.jupiter - junit-jupiter-engine - ${junit.jupiter.version} - test - org.hamcrest hamcrest-core diff --git a/validator-cli/pom.xml b/validator-cli/pom.xml index 1b1d521..afd8867 100644 --- a/validator-cli/pom.xml +++ b/validator-cli/pom.xml @@ -11,18 +11,10 @@ validator-cli - 0.1.1-SNAPSHOT - jar validator-cli Command-line utility for validating phenopackets - - UTF-8 - UTF-8 - 1.8 - - org.phenopackets.phenopacket-schema @@ -45,12 +37,6 @@ - - - src/main/resources - true - - org.springframework.boot diff --git a/validator-core/pom.xml b/validator-core/pom.xml index 0a0999f..d24da07 100644 --- a/validator-core/pom.xml +++ b/validator-core/pom.xml @@ -11,17 +11,10 @@ validator-core - 0.1.1-SNAPSHOT - jar validator-core Validator utilities for phenopackets - - 3.14.0 - 2.0.0 - - org.phenopackets @@ -57,54 +50,4 @@ - - - - - maven-resources-plugin - 3.2.0 - - - copy-resources - validate - - copy-resources - - - ${project.build.directory}/resources - - - src/main/resources - - true - - schema - log4j2.xml - - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 11 - 11 - - - - - - - - \ No newline at end of file From da7f7ccf16237ea80add25627aab44cc0df24879 Mon Sep 17 00:00:00 2001 From: Daniel Danis Date: Sat, 11 Sep 2021 00:06:59 -0400 Subject: [PATCH 02/37] Revert resource filtering --- validator-cli/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/validator-cli/pom.xml b/validator-cli/pom.xml index afd8867..e085c3a 100644 --- a/validator-cli/pom.xml +++ b/validator-cli/pom.xml @@ -37,6 +37,12 @@ + + + src/main/resources + true + + org.springframework.boot From c5259e1683c85376f34f17a96edb26311d98d244 Mon Sep 17 00:00:00 2001 From: Daniel Danis Date: Sat, 11 Sep 2021 00:07:34 -0400 Subject: [PATCH 03/37] Use `slf4j-simple` for logging in tests --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index f478ff9..0303396 100644 --- a/pom.xml +++ b/pom.xml @@ -104,5 +104,10 @@ hamcrest-core test + + org.slf4j + slf4j-simple + test + \ No newline at end of file From e643cc32f5c34f9d0ea4b2f63bd626c169bf4cf2 Mon Sep 17 00:00:00 2001 From: Daniel Danis Date: Mon, 13 Sep 2021 13:30:51 -0400 Subject: [PATCH 04/37] Change `groupId`s, add `validator-jsonschema` module, update POMs --- pom.xml | 3 +- validator-cli/pom.xml | 6 +- validator-core/pom.xml | 18 +----- validator-jsonschema/pom.xml | 60 +++++++++++++++++++ .../src/test/resources/logback-test.xml | 14 +++++ 5 files changed, 80 insertions(+), 21 deletions(-) create mode 100644 validator-jsonschema/pom.xml create mode 100644 validator-jsonschema/src/test/resources/logback-test.xml diff --git a/pom.xml b/pom.xml index 0303396..30fea66 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.phenopackets.phenopacket-schema + org.phenopackets.validator validator 0.1.1-SNAPSHOT @@ -13,6 +13,7 @@ validator-core validator-cli + validator-jsonschema diff --git a/validator-cli/pom.xml b/validator-cli/pom.xml index e085c3a..8109704 100644 --- a/validator-cli/pom.xml +++ b/validator-cli/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - org.phenopackets.phenopacket-schema + org.phenopackets.validator validator 0.1.1-SNAPSHOT @@ -17,8 +17,8 @@ - org.phenopackets.phenopacket-schema - validator-core + org.phenopackets.validator + validator-jsonschema ${project.parent.version} diff --git a/validator-core/pom.xml b/validator-core/pom.xml index d24da07..c655b37 100644 --- a/validator-core/pom.xml +++ b/validator-core/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - org.phenopackets.phenopacket-schema + org.phenopackets.validator validator 0.1.1-SNAPSHOT @@ -16,12 +16,6 @@ Validator utilities for phenopackets - - org.phenopackets - phenopacket-schema - ${phenopacket-schema.version} - - com.networknt json-schema-validator @@ -38,16 +32,6 @@ curie-util 0.0.2 - - com.google.protobuf - protobuf-java - ${protobuf.version} - - - com.google.protobuf - protobuf-java-util - ${protobuf.version} - \ No newline at end of file diff --git a/validator-jsonschema/pom.xml b/validator-jsonschema/pom.xml new file mode 100644 index 0000000..f67193c --- /dev/null +++ b/validator-jsonschema/pom.xml @@ -0,0 +1,60 @@ + + + + validator + org.phenopackets.validator + 0.1.1-SNAPSHOT + + 4.0.0 + + validator-jsonschema + + + + org.phenopackets.validator + validator-core + ${project.parent.version} + + + + com.networknt + json-schema-validator + 1.0.42 + + + + + org.prefixcommons + curie-util + 0.0.2 + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/validator-jsonschema/src/test/resources/logback-test.xml b/validator-jsonschema/src/test/resources/logback-test.xml new file mode 100644 index 0000000..0f78182 --- /dev/null +++ b/validator-jsonschema/src/test/resources/logback-test.xml @@ -0,0 +1,14 @@ + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + \ No newline at end of file From 3a6957e917bdc63c7f41bdad621e3d33085bb79d Mon Sep 17 00:00:00 2001 From: Daniel Danis Date: Mon, 13 Sep 2021 13:37:38 -0400 Subject: [PATCH 05/37] Outline validation protocol, reorganize packages, implement JSONschema validator --- .../schema/validator/cli/ValidateCommand.java | 60 -------- .../validator/cli/ValidatorApplication.java | 27 ---- .../validator/cli/PhenopacketValidation.java | 134 ++++++++++++++++++ .../validator/cli/ValidateCommand.java | 88 ++++++++++++ .../validator/cli/ValidatorApplication.java | 32 +++++ .../cli/ValidatorApplicationRunner.java | 4 +- .../validator/core/PhenopacketValidator.java | 103 -------------- .../PhenopacketValidatorRuntimeException.java | 7 - ...AdditionalJsonFileJsonSchemaValidator.java | 27 ---- .../HpoRareDiseaseJsonSchemaValidator.java | 38 ----- .../core/jsonschema/JsonSchemaValidator.java | 83 ----------- .../core/jsonschema/JsonValidationError.java | 45 ------ .../core/validation/JsonValidator.java | 13 -- .../core/validation/ValidationItem.java | 8 -- .../validator/core/DefaultValidationInfo.java | 60 ++++++++ .../core}/ErrorType.java | 4 +- .../validator/core/PhenopacketValidator.java | 35 +++++ .../core/PhenopacketValidatorFactory.java | 15 ++ .../validator/core/ValidationAspect.java | 18 +++ .../validator/core/ValidationItem.java | 24 ++++ .../validator/core/ValidatorInfo.java | 30 ++++ .../PhenopacketValidatorRuntimeException.java | 24 ++++ .../ClasspathJsonSchemaValidatorFactory.java | 64 +++++++++ .../jsonschema/JsonSchemaValidator.java | 105 ++++++++++++++ .../jsonschema/JsonValidationError.java | 61 ++++++++ .../jsonschema/JsonSchemaValidatorTest.java | 46 +++--- .../json/bethlehamMyopathyExample.json | 0 .../json/bethlehamMyopathyInvalidExample.json | 0 .../json/invalidSimplePhenopacket.json | 0 .../json/validSimplePhenopacket.json | 0 30 files changed, 722 insertions(+), 433 deletions(-) delete mode 100644 validator-cli/src/main/java/org/phenopackets/schema/validator/cli/ValidateCommand.java delete mode 100644 validator-cli/src/main/java/org/phenopackets/schema/validator/cli/ValidatorApplication.java create mode 100644 validator-cli/src/main/java/org/phenopackets/validator/cli/PhenopacketValidation.java create mode 100644 validator-cli/src/main/java/org/phenopackets/validator/cli/ValidateCommand.java create mode 100644 validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java rename validator-cli/src/main/java/org/phenopackets/{schema => }/validator/cli/ValidatorApplicationRunner.java (83%) delete mode 100644 validator-core/src/main/java/org/phenopackets/schema/validator/core/PhenopacketValidator.java delete mode 100644 validator-core/src/main/java/org/phenopackets/schema/validator/core/except/PhenopacketValidatorRuntimeException.java delete mode 100644 validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/AdditionalJsonFileJsonSchemaValidator.java delete mode 100644 validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/HpoRareDiseaseJsonSchemaValidator.java delete mode 100644 validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/JsonSchemaValidator.java delete mode 100644 validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/JsonValidationError.java delete mode 100644 validator-core/src/main/java/org/phenopackets/schema/validator/core/validation/JsonValidator.java delete mode 100644 validator-core/src/main/java/org/phenopackets/schema/validator/core/validation/ValidationItem.java create mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidationInfo.java rename validator-core/src/main/java/org/phenopackets/{schema/validator/core/validation => validator/core}/ErrorType.java (88%) create mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidator.java create mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorFactory.java create mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/ValidationAspect.java create mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/ValidationItem.java create mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/ValidatorInfo.java create mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/except/PhenopacketValidatorRuntimeException.java create mode 100644 validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/ClasspathJsonSchemaValidatorFactory.java create mode 100644 validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java create mode 100644 validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationError.java rename {validator-core/src/test/java/org/phenopackets/schema/validator/core => validator-jsonschema/src/test/java/org/phenopackets/validator}/jsonschema/JsonSchemaValidatorTest.java (65%) rename {validator-core => validator-jsonschema}/src/test/resources/json/bethlehamMyopathyExample.json (100%) rename {validator-core => validator-jsonschema}/src/test/resources/json/bethlehamMyopathyInvalidExample.json (100%) rename {validator-core => validator-jsonschema}/src/test/resources/json/invalidSimplePhenopacket.json (100%) rename {validator-core => validator-jsonschema}/src/test/resources/json/validSimplePhenopacket.json (100%) diff --git a/validator-cli/src/main/java/org/phenopackets/schema/validator/cli/ValidateCommand.java b/validator-cli/src/main/java/org/phenopackets/schema/validator/cli/ValidateCommand.java deleted file mode 100644 index a1e2e94..0000000 --- a/validator-cli/src/main/java/org/phenopackets/schema/validator/cli/ValidateCommand.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.phenopackets.schema.validator.cli; - - -import org.phenopackets.schema.validator.core.PhenopacketValidator; -import org.phenopackets.schema.validator.core.jsonschema.JsonSchemaValidator; -import org.phenopackets.schema.validator.core.validation.ValidationItem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; -import picocli.CommandLine.Parameters; - -import java.util.List; -import java.util.concurrent.Callable; - -@Component -@Command(name = "validate", - mixinStandardHelpOptions = true) -public class ValidateCommand implements Callable { - private static Logger LOG = LoggerFactory - .getLogger(ValidateCommand.class); - @Option(names = {"-p","--phenopacket"}, description = "Path to phenopacket", required = true) - private String phenopacketPath; - - @Option(names = "--rare", description = "apply HPO rare-disease constraints") - private boolean rareHpoConstraints = false; - - @Parameters(description = "one or more phenopacket files") - private List positionals; - - - - - @Override - public Integer call() { - System.out.printf("mycommand was called with --rare=%s and positionals: %s%n", rareHpoConstraints, positionals); - LOG.info("EXECUTING : command line runner"); - PhenopacketValidator validator; - if (rareHpoConstraints) { - validator = new PhenopacketValidator(phenopacketPath, PhenopacketValidator.ValidationType.RARE_DISEASE_VALIDATION); - } else if (positionals.size() > 0) { - String[] itemsArray = new String[positionals.size()]; - itemsArray = positionals.toArray(itemsArray); - validator = new PhenopacketValidator(phenopacketPath, itemsArray); - } else { - validator = new PhenopacketValidator(phenopacketPath); - } - List errors = validator.getValidationErrors(); - if (errors.isEmpty()) { - System.out.println("\t no errors found"); - } else { - for (ValidationItem ve : errors) { - System.out.println("\t(" + ve.errorType() + ") " + ve.message()); - } - } - return 0; - } - -} \ No newline at end of file diff --git a/validator-cli/src/main/java/org/phenopackets/schema/validator/cli/ValidatorApplication.java b/validator-cli/src/main/java/org/phenopackets/schema/validator/cli/ValidatorApplication.java deleted file mode 100644 index a4757ff..0000000 --- a/validator-cli/src/main/java/org/phenopackets/schema/validator/cli/ValidatorApplication.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.phenopackets.schema.validator.cli; - -import org.phenopackets.schema.validator.core.jsonschema.JsonSchemaValidator; -import org.phenopackets.schema.validator.core.validation.ValidationItem; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; - - -/** - * @author Jules Jacobsen - */ -@SpringBootApplication -public class ValidatorApplication { - - private static Logger LOG = LoggerFactory - .getLogger(ValidatorApplication.class); - - public static void main(String[] args) { - LOG.info("STARTING THE APPLICATION"); - SpringApplication.run(ValidatorApplication.class, args); - LOG.info("APPLICATION FINISHED"); - } - - -} diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/PhenopacketValidation.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/PhenopacketValidation.java new file mode 100644 index 0000000..75bea59 --- /dev/null +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/PhenopacketValidation.java @@ -0,0 +1,134 @@ +package org.phenopackets.validator.cli; + +import org.phenopackets.validator.core.PhenopacketValidator; +import org.phenopackets.validator.core.PhenopacketValidatorFactory; +import org.phenopackets.validator.core.ValidationItem; +import org.phenopackets.validator.core.ValidatorInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.util.LinkedList; +import java.util.List; +import java.util.function.Function; + +/** + * Entry point for the library. Performs JSON schema validation. If an additional config file is provided, it performs + * additional validation. Several default validation files are provided that can be used with command line flags. + * For instance, adding the --rare flag will additionally apply the {@code hpo-rare-disease-schema.json} specification. + * @author Peter N Robinson + */ +public class PhenopacketValidation { + + private static final Logger LOGGER = LoggerFactory.getLogger(PhenopacketValidation.class); + + private final PhenopacketValidatorFactory validatorFactory; + + public PhenopacketValidation(PhenopacketValidatorFactory validatorFactory) { + this.validatorFactory = validatorFactory; + } + +// List validationErrors; + +// +// /** Can be used to validate a Phenopacket against the built-in additional JSON-Schema files, +// * including HPO-rare disease TODO add another two examples. +// * @param phenopacketPath Path to the phenopacket file +// * @param validationType One of the built-in types, see {@link ValidationType}. +// */ +// public PhenopacketValidation(String phenopacketPath, ValidationType validationType) { +// phenopacket = initPhenopacketFile(phenopacketPath); +// validationErrors = genericValidation(); +// +// switch (validationType) { +// case RARE_DISEASE_VALIDATION: +// List specErrors = hpoRareDiseaseValidation(); +// validationErrors.addAll(specErrors); +// break; +// } +// } +// +// /** +// * This constructor can be used with one or more user defined JSON spec files that are passed from the command line +// * @param phenopacketPath Path to the phenopacket file +// * @param jsonSchemPaths Paths to user-defined JSON Schema files +// */ +// public PhenopacketValidation(String phenopacketPath, String... jsonSchemPaths) { +// phenopacket = initPhenopacketFile(phenopacketPath); +// validationErrors = genericValidation(); +// for (String jsonPath : jsonSchemPaths) { +// List specErrors = additionalValidation(jsonPath); +// validationErrors.addAll(specErrors); +// } +// } +// +// private List additionalValidation(String jsonPath) { +// File jsonFile = new File(jsonPath); +// if (! jsonFile.isFile()) { +// throw new PhenopacketValidatorRuntimeException("Could not find input json file \"" + jsonFile.getAbsolutePath() + "\""); +// } +// JsonSchemaValidator validator = new AdditionalJsonFileJsonSchemaValidator(phenopacket, jsonFile); +// return validator.validate(); +// } + + public List validate(InputStream inputStream, ValidatorInfo... validations) { + List items = new LinkedList<>(); + try { + byte[] content = inputStream.readAllBytes(); + for (ValidatorInfo validationType : validations) { + validatorFactory.getValidatorForType(validationType) + .map(validate(content)) + .ifPresent(items::addAll); + } + return List.copyOf(items); + + } catch (IOException e) { + LOGGER.warn("Error occurred during validation {}", e.getMessage(), e); + return List.of(); + } + } + + private static Function> validate(byte[] content) { + return validator -> { + try (InputStream is = new ByteArrayInputStream(content)) { + return validator.validate(is); + } catch (IOException e) { + LOGGER.warn("Error occurred during validation {}", e.getMessage(), e); + return List.of(); + } + }; + } + +// TODO - not relevant here +// /** +// * Create a file object and check that the file exists +// * @param phenopacketPath String to the input phenopacket file +// * @return File object representing the input phenopacket +// */ +// File initPhenopacketFile(String phenopacketPath) { +// File f = new File(phenopacketPath); +// if (! f.isFile()) { +// throw new PhenopacketValidatorRuntimeException("Could not find file \"" + phenopacketPath + "\""); +// } +// return f; +// } +// +// /** +// * Validate the Phenopacket file against the generic JSON Schema +// */ +// private List genericValidation() { +// JsonSchemaValidator jsonSchemaValidator = new JsonSchemaValidator(phenopacket); +// return jsonSchemaValidator.validate(); +// } +// /** +// * Validate the Phenopacket file for HPO-rare disease specific requirements. +// */ +// private List hpoRareDiseaseValidation() { +// JsonSchemaValidator hpoRareDiseaseValidator = new HpoRareDiseaseJsonSchemaValidator(phenopacket); +// return hpoRareDiseaseValidator.validate(); +// } +// +// public List getValidationErrors() { +// return validationErrors; +// } +} diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidateCommand.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidateCommand.java new file mode 100644 index 0000000..c9cdc4e --- /dev/null +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidateCommand.java @@ -0,0 +1,88 @@ +package org.phenopackets.validator.cli; + + +import org.phenopackets.validator.core.ValidationItem; +import org.phenopackets.validator.core.ValidatorInfo; +import org.phenopackets.validator.core.PhenopacketValidatorFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.Callable; + +@Component +@Command(name = "validate", + mixinStandardHelpOptions = true) +public class ValidateCommand implements Callable { + + private static final Logger LOGGER = LoggerFactory.getLogger(ValidateCommand.class); + + // TODO - do we need this as we accept 1..* phenopacket as positional arguments? +// @Option(names = {"-p","--phenopacket"}, description = "Path to phenopacket", required = true) +// public String phenopacketPath; + + @Option(names = "--rare", description = "apply HPO rare-disease constraints") + public boolean rareHpoConstraints = false; + + @Parameters(arity = "1..*", description = "one or more phenopacket files") + public List phenopackets; + + private final PhenopacketValidatorFactory phenopacketValidatorFactory; + + public ValidateCommand(PhenopacketValidatorFactory phenopacketValidatorFactory) { + this.phenopacketValidatorFactory = phenopacketValidatorFactory; + } + + @Override + public Integer call() { + // What type of validation do we run? + List validationTypes = new LinkedList<>(); + validationTypes.add(ValidatorInfo.generic()); // we run this by default + if (rareHpoConstraints) { + LOGGER.info("Validating with HPO rare-disease constraints"); + validationTypes.add(ValidatorInfo.rareDiseaseValidation()); + } + + LOGGER.info("Validating {} phenopackets", phenopackets.size()); + + + PhenopacketValidation validator = new PhenopacketValidation(phenopacketValidatorFactory); + + for (File phenopacket : phenopackets) { + LOGGER.info("Validating phenopacket at `{}`", phenopacket.getAbsolutePath()); + + try (InputStream in = Files.newInputStream(phenopacket.toPath())) { + + List validationItems = validator.validate(in, validationTypes.toArray(ValidatorInfo[]::new)); + if (validationItems.isEmpty()) { + LOGGER.info("No errors found"); + } else { + LOGGER.info("Found {} errors:", validationItems.size()); + for (ValidationItem item : validationItems) { + LOGGER.info("({}) {}", item.errorType(), item.message()); + } + } + + } catch (IOException e) { + LOGGER.warn("Error opening the phenopacket", e); + } + + // poor man's formatting + LOGGER.info(""); + LOGGER.info("--------------------------------------------------------------------------------"); + LOGGER.info(""); + } + + return 0; + } + +} \ No newline at end of file diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java new file mode 100644 index 0000000..bca5e5e --- /dev/null +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java @@ -0,0 +1,32 @@ +package org.phenopackets.validator.cli; + +import org.phenopackets.validator.core.PhenopacketValidatorFactory; +import org.phenopackets.validator.jsonschema.ClasspathJsonSchemaValidatorFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.Bean; + + +/** + * @author Jules Jacobsen + */ +@SpringBootApplication +public class ValidatorApplication { + + private static final Logger LOGGER = LoggerFactory.getLogger(ValidatorApplication.class); + + public static void main(String[] args) { + LOGGER.info("STARTING THE APPLICATION"); + SpringApplication.run(ValidatorApplication.class, args); + LOGGER.info("APPLICATION FINISHED"); + } + + @Bean + public PhenopacketValidatorFactory phenopacketValidatorFactory() { + // TODO - for development only... + return ClasspathJsonSchemaValidatorFactory.defaultValidators(); + } + +} diff --git a/validator-cli/src/main/java/org/phenopackets/schema/validator/cli/ValidatorApplicationRunner.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplicationRunner.java similarity index 83% rename from validator-cli/src/main/java/org/phenopackets/schema/validator/cli/ValidatorApplicationRunner.java rename to validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplicationRunner.java index 1cecee4..46e9005 100644 --- a/validator-cli/src/main/java/org/phenopackets/schema/validator/cli/ValidatorApplicationRunner.java +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplicationRunner.java @@ -1,7 +1,5 @@ -package org.phenopackets.schema.validator.cli; +package org.phenopackets.validator.cli; -import org.phenopackets.schema.validator.core.jsonschema.JsonSchemaValidator; -import org.phenopackets.schema.validator.core.validation.ValidationItem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; diff --git a/validator-core/src/main/java/org/phenopackets/schema/validator/core/PhenopacketValidator.java b/validator-core/src/main/java/org/phenopackets/schema/validator/core/PhenopacketValidator.java deleted file mode 100644 index 335a0d5..0000000 --- a/validator-core/src/main/java/org/phenopackets/schema/validator/core/PhenopacketValidator.java +++ /dev/null @@ -1,103 +0,0 @@ -package org.phenopackets.schema.validator.core; - -import org.phenopackets.schema.validator.core.except.PhenopacketValidatorRuntimeException; -import org.phenopackets.schema.validator.core.jsonschema.AdditionalJsonFileJsonSchemaValidator; -import org.phenopackets.schema.validator.core.jsonschema.HpoRareDiseaseJsonSchemaValidator; -import org.phenopackets.schema.validator.core.jsonschema.JsonSchemaValidator; -import org.phenopackets.schema.validator.core.validation.ValidationItem; - -import java.io.File; -import java.util.List; - -/** - * Entry point for the library. Performs JSON schema validation. If an additional config file is provided, it performs - * additional validation. Several default validation files are provided that can be used with command line flags. - * For instance, adding the --rare flag will additionally apply the {@code hpo-rare-disease-schema.json} specification. - * @author Peter N Robinson - */ -public class PhenopacketValidator { - - List validationErrors; - - public enum ValidationType { GENERIC, RARE_DISEASE_VALIDATION} - - private final File phenopacket; - - public PhenopacketValidator(String phenopacketPath) { - phenopacket = initPhenopacketFile(phenopacketPath); - validationErrors = genericValidation(); - - } - - /** Can be used to validate a Phenopacket against the built-in additional JSON-Schema files, - * including HPO-rare disease TODO add another two examples. - * @param phenopacketPath Path to the phenopacket file - * @param validationType One of the built-in types, see {@link ValidationType}. - */ - public PhenopacketValidator(String phenopacketPath, ValidationType validationType) { - phenopacket = initPhenopacketFile(phenopacketPath); - validationErrors = genericValidation(); - - switch (validationType) { - case RARE_DISEASE_VALIDATION: - List specErrors = hpoRareDiseaseValidation(); - validationErrors.addAll(specErrors); - break; - } - } - - /** - * This constructor can be used with one or more user defined JSON spec files that are passed from the command line - * @param phenopacketPath Path to the phenopacket file - * @param jsonSchemPaths Paths to user-defined JSON Schema files - */ - public PhenopacketValidator(String phenopacketPath, String... jsonSchemPaths) { - phenopacket = initPhenopacketFile(phenopacketPath); - validationErrors = genericValidation(); - for (String jsonPath : jsonSchemPaths) { - List specErrors = additionalValidation(jsonPath); - validationErrors.addAll(specErrors); - } - } - - private List additionalValidation(String jsonPath) { - File jsonFile = new File(jsonPath); - if (! jsonFile.isFile()) { - throw new PhenopacketValidatorRuntimeException("Could not find input json file \"" + jsonFile.getAbsolutePath() + "\""); - } - JsonSchemaValidator validator = new AdditionalJsonFileJsonSchemaValidator(phenopacket, jsonFile); - return validator.validate(); - } - - /** - * Create a file object and check that the file exists - * @param phenopacketPath String to the input phenopacket file - * @return File object representing the input phenopacket - */ - File initPhenopacketFile(String phenopacketPath) { - File f = new File(phenopacketPath); - if (! f.isFile()) { - throw new PhenopacketValidatorRuntimeException("Could not find file \"" + phenopacketPath + "\""); - } - return f; - } - - /** - * Validate the Phenopacket file against the generic JSON Schema - */ - private List genericValidation() { - JsonSchemaValidator jsonSchemaValidator = new JsonSchemaValidator(phenopacket); - return jsonSchemaValidator.validate(); - } - /** - * Validate the Phenopacket file for HPO-rare disease specific requirements. - */ - private List hpoRareDiseaseValidation() { - JsonSchemaValidator hpoRareDiseaseValidator = new HpoRareDiseaseJsonSchemaValidator(phenopacket); - return hpoRareDiseaseValidator.validate(); - } - - public List getValidationErrors() { - return validationErrors; - } -} diff --git a/validator-core/src/main/java/org/phenopackets/schema/validator/core/except/PhenopacketValidatorRuntimeException.java b/validator-core/src/main/java/org/phenopackets/schema/validator/core/except/PhenopacketValidatorRuntimeException.java deleted file mode 100644 index 72093d6..0000000 --- a/validator-core/src/main/java/org/phenopackets/schema/validator/core/except/PhenopacketValidatorRuntimeException.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.phenopackets.schema.validator.core.except; - -public class PhenopacketValidatorRuntimeException extends RuntimeException { - - public PhenopacketValidatorRuntimeException() { super(); } - public PhenopacketValidatorRuntimeException(String msg) { super(msg); } -} diff --git a/validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/AdditionalJsonFileJsonSchemaValidator.java b/validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/AdditionalJsonFileJsonSchemaValidator.java deleted file mode 100644 index 4ffd0c3..0000000 --- a/validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/AdditionalJsonFileJsonSchemaValidator.java +++ /dev/null @@ -1,27 +0,0 @@ -package org.phenopackets.schema.validator.core.jsonschema; - -import com.networknt.schema.JsonSchemaFactory; -import org.phenopackets.schema.validator.core.except.PhenopacketValidatorRuntimeException; - -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; - -public class AdditionalJsonFileJsonSchemaValidator extends JsonSchemaValidator{ - - public AdditionalJsonFileJsonSchemaValidator(File phenopacketFile, File jsonFile) { - this.jsonFile = phenopacketFile; - if (! phenopacketFile.isFile()) { - throw new PhenopacketValidatorRuntimeException("Could not open file at \"" + phenopacketFile.getAbsolutePath() + "\""); - } - JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(VERSION_FLAG); - try { - InputStream targetStream = new FileInputStream(jsonFile); - this.jsonSchema = schemaFactory.getSchema(targetStream); - } catch (Exception e) { - e.printStackTrace(); - } - } - - -} diff --git a/validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/HpoRareDiseaseJsonSchemaValidator.java b/validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/HpoRareDiseaseJsonSchemaValidator.java deleted file mode 100644 index 6d60c68..0000000 --- a/validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/HpoRareDiseaseJsonSchemaValidator.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.phenopackets.schema.validator.core.jsonschema; - - -import com.networknt.schema.JsonSchemaFactory; - -import org.phenopackets.schema.validator.core.except.PhenopacketValidatorRuntimeException; - - -import java.io.File; -import java.io.InputStream; - -/** - * This class implements additional validation of a phenopacket that is intended to be used - * for HPO rare disease phenotyping. By assumption, the phenopacket will have been first - * checked agoinst the generic specification. This class performs validation with the - * file {@code hpo-rare-disease-schema.json}. - * @author Peter N Robinson - */ -public class HpoRareDiseaseJsonSchemaValidator extends JsonSchemaValidator { - - - public HpoRareDiseaseJsonSchemaValidator(File f) { - this.jsonFile = f; - if (! f.isFile()) { - throw new PhenopacketValidatorRuntimeException("Could not open file at \"" + f.getAbsolutePath() + "\""); - } - JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(VERSION_FLAG); - try { - InputStream baseSchemaStream = inputStreamFromClasspath("schema/hpo-rare-disease-schema.json"); - this.jsonSchema = schemaFactory.getSchema(baseSchemaStream); - } catch (Exception e) { - e.printStackTrace(); - } - } - - - -} diff --git a/validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/JsonSchemaValidator.java b/validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/JsonSchemaValidator.java deleted file mode 100644 index b75f9c2..0000000 --- a/validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/JsonSchemaValidator.java +++ /dev/null @@ -1,83 +0,0 @@ -package org.phenopackets.schema.validator.core.jsonschema; - - - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; - -import com.networknt.schema.JsonSchema; -import com.networknt.schema.JsonSchemaFactory; -import com.networknt.schema.SpecVersion; -import com.networknt.schema.ValidationMessage; -import org.phenopackets.schema.validator.core.except.PhenopacketValidatorRuntimeException; -import org.phenopackets.schema.validator.core.validation.JsonValidator; -import org.phenopackets.schema.validator.core.validation.ValidationItem; - -import java.io.File; -import java.io.FileInputStream; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; - - -public class JsonSchemaValidator implements JsonValidator { - - protected JsonSchema jsonSchema; - protected ObjectMapper objectMapper = new ObjectMapper(); - /** The latest version of the spec that is supported by our JSON SCHEMA library is 2019/09. */ - protected static final SpecVersion.VersionFlag VERSION_FLAG = SpecVersion.VersionFlag.V201909; - protected File jsonFile; - - /** - * Constructor for validation using JSON Schema - * @param f a JSON file. - */ - public JsonSchemaValidator(File f) { - this.jsonFile = f; - if (! f.isFile()) { - throw new PhenopacketValidatorRuntimeException("Could not open file at \"" + f.getAbsolutePath() + "\""); - } - JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(VERSION_FLAG); - try { - InputStream baseSchemaStream = inputStreamFromClasspath("schema/phenopacket-general-schema.json"); - this.jsonSchema = schemaFactory.getSchema(baseSchemaStream); - } catch (Exception e) { - e.printStackTrace(); - } - } - - public JsonSchemaValidator() { - - } - - - /** - * Get a resource from the maven src/main/resources directory - * @param path Path to a file resource - * @return corresponding input stream - */ - protected static InputStream inputStreamFromClasspath(String path) { - return Thread.currentThread().getContextClassLoader().getResourceAsStream(path); - } - - /** Validate the file at fpath (assumed to be a Phenopacket formated in JSON) using the - * base validation schema - * @return List of {@link JsonValidationError} objects (empty list if there were no errors) - */ - @Override - public List validate() { - List errors = new ArrayList<>(); - try { - InputStream jsonStream = new FileInputStream(jsonFile); - JsonNode json = objectMapper.readTree(jsonStream); - Set validationResult = jsonSchema.validate(json); - if (! validationResult.isEmpty()) { - validationResult.forEach(e -> errors.add(new JsonValidationError(e))); - } - } catch (Exception e) { - e.printStackTrace(); - } - return errors; - } -} diff --git a/validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/JsonValidationError.java b/validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/JsonValidationError.java deleted file mode 100644 index 8e4b80c..0000000 --- a/validator-core/src/main/java/org/phenopackets/schema/validator/core/jsonschema/JsonValidationError.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.phenopackets.schema.validator.core.jsonschema; -import com.networknt.schema.ValidationMessage; -import org.phenopackets.schema.validator.core.validation.ErrorType; -import org.phenopackets.schema.validator.core.validation.ValidationItem; - -import java.util.Objects; - -/** - * POJO to represent errors identified by JSON Schema validation. - * @author Peter N Robinson - */ -public final class JsonValidationError implements ValidationItem { - - private final ErrorType errorType; - private final String message; - - public JsonValidationError(ValidationMessage validationMessage) { - this.errorType = ErrorType.stringToErrorType(validationMessage.getType()); - this.message = validationMessage.getMessage(); - } - - - @Override - public String message() { - return message; - } - - @Override - public ErrorType errorType() { - return this.errorType; - } - - @Override - public int hashCode() { - return Objects.hash(message, errorType); - } - - @Override - public boolean equals(Object obj) { - if (! (obj instanceof JsonValidationError)) return false; - JsonValidationError that = (JsonValidationError) obj; - return this.message.equals(that.message) && this.errorType.equals(that.errorType); - } - -} diff --git a/validator-core/src/main/java/org/phenopackets/schema/validator/core/validation/JsonValidator.java b/validator-core/src/main/java/org/phenopackets/schema/validator/core/validation/JsonValidator.java deleted file mode 100644 index 61c9e9c..0000000 --- a/validator-core/src/main/java/org/phenopackets/schema/validator/core/validation/JsonValidator.java +++ /dev/null @@ -1,13 +0,0 @@ -package org.phenopackets.schema.validator.core.validation; - -import java.util.List; - -/** - * Common interface for validation by JSON schema. - * @author Peter N Robinson - */ -public interface JsonValidator { - - List validate(); - -} diff --git a/validator-core/src/main/java/org/phenopackets/schema/validator/core/validation/ValidationItem.java b/validator-core/src/main/java/org/phenopackets/schema/validator/core/validation/ValidationItem.java deleted file mode 100644 index f22109d..0000000 --- a/validator-core/src/main/java/org/phenopackets/schema/validator/core/validation/ValidationItem.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.phenopackets.schema.validator.core.validation; - -public interface ValidationItem { - - ErrorType errorType(); - String message(); - -} diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidationInfo.java b/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidationInfo.java new file mode 100644 index 0000000..76ae74f --- /dev/null +++ b/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidationInfo.java @@ -0,0 +1,60 @@ +package org.phenopackets.validator.core; + +import java.util.Objects; + +class DefaultValidationInfo implements ValidatorInfo { + + private static final DefaultValidationInfo GENERIC = of("GENERIC", "Validation of a generic Phenopacket"); + private static final DefaultValidationInfo RARE_DISEASE_VALIDATOR = of("RARE_DISEASE_VALIDATOR", "Validation of rare disease Phenopacket constraints"); + + static ValidatorInfo generic() { + return GENERIC; + } + + static ValidatorInfo rareDiseaseValidator() { + return RARE_DISEASE_VALIDATOR; + } + + private final String validatorId; + private final String validatorName; + + static DefaultValidationInfo of(String validatorId, String validatorName) { + return new DefaultValidationInfo(validatorId, validatorName); + } + + private DefaultValidationInfo(String validationId, String validatorName) { + this.validatorId = validationId; + this.validatorName = validatorName; + } + + @Override + public String validatorId() { + return validatorId; + } + + @Override + public String validatorName() { + return validatorName; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DefaultValidationInfo that = (DefaultValidationInfo) o; + return Objects.equals(validatorId, that.validatorId) && Objects.equals(validatorName, that.validatorName); + } + + @Override + public int hashCode() { + return Objects.hash(validatorId, validatorName); + } + + @Override + public String toString() { + return "DefaultValidationInfo{" + + "validatorId='" + validatorId + '\'' + + ", validatorName='" + validatorName + '\'' + + '}'; + } +} diff --git a/validator-core/src/main/java/org/phenopackets/schema/validator/core/validation/ErrorType.java b/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java similarity index 88% rename from validator-core/src/main/java/org/phenopackets/schema/validator/core/validation/ErrorType.java rename to validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java index 30a469b..e401e05 100644 --- a/validator-core/src/main/java/org/phenopackets/schema/validator/core/validation/ErrorType.java +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java @@ -1,6 +1,6 @@ -package org.phenopackets.schema.validator.core.validation; +package org.phenopackets.validator.core; -import org.phenopackets.schema.validator.core.except.PhenopacketValidatorRuntimeException; +import org.phenopackets.validator.core.except.PhenopacketValidatorRuntimeException; public enum ErrorType { /** JSON schema error meaning that the JSON code contained a property not present in the schema. */ diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidator.java b/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidator.java new file mode 100644 index 0000000..4d7914e --- /dev/null +++ b/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidator.java @@ -0,0 +1,35 @@ +package org.phenopackets.validator.core; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.List; + +/** + * Phenopacket validator applies rules to check that the provided data meets the requirements for a valid Phenopacket. + *

+ * @author Daniel Danis + * @author Peter N Robinson + */ +public interface PhenopacketValidator { + + ValidatorInfo info(); + + List validate(InputStream inputStream); + + // ----------------------------------------------------------------------------------------------------------------- + + default List validate(File phenopacket) throws IOException { + try (InputStream inputStream = Files.newInputStream(phenopacket.toPath())) { + return validate(inputStream); + } + } + + default List validate(String content) { + return validate(new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))); + } + +} diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorFactory.java b/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorFactory.java new file mode 100644 index 0000000..c079ccb --- /dev/null +++ b/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorFactory.java @@ -0,0 +1,15 @@ +package org.phenopackets.validator.core; + +import java.util.Optional; + +/** + * The phenopacket validator factory provides phenopacket validators designed to perform specific {@link ValidatorInfo}. + *

+ * @author Daniel Danis + * @author Peter N Robinson + */ +public interface PhenopacketValidatorFactory { + + Optional getValidatorForType(ValidatorInfo type); + +} diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationAspect.java b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationAspect.java new file mode 100644 index 0000000..a5942c7 --- /dev/null +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationAspect.java @@ -0,0 +1,18 @@ +package org.phenopackets.validator.core; + + +public enum ValidationAspect { + + // This enum is supposed to be used instead of the ErrorType + // TODO - elaborate the categories + + // general syntax inconsistency, e.g. bad JSON format + GENERAL, + + // e.g. missing at least one phenotypic feature, age, etc. + MISSING_PROPERTY, + + // e.g. invalid ontology used to represent an info + INVALID_VALUE, + +} diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItem.java b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItem.java new file mode 100644 index 0000000..66c3548 --- /dev/null +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItem.java @@ -0,0 +1,24 @@ +package org.phenopackets.validator.core; + +/** + * The interface represents a single issue found during validation. + *

+ * @author Daniel Danis + * @author Peter N Robinson + */ +public interface ValidationItem { + + /** + * @return basic description of the validator that produced this issue. + */ + ValidatorInfo validatorInfo(); + + // TODO - decide which enum to use here - either ErrorType or ValidationAspect + ErrorType errorType(); + + /** + * @return string with description of the issue intended for human consumption. + */ + String message(); + +} diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ValidatorInfo.java b/validator-core/src/main/java/org/phenopackets/validator/core/ValidatorInfo.java new file mode 100644 index 0000000..64915d3 --- /dev/null +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ValidatorInfo.java @@ -0,0 +1,30 @@ +package org.phenopackets.validator.core; + +public interface ValidatorInfo { + + static ValidatorInfo generic() { + return DefaultValidationInfo.generic(); + } + + /** + * This class implements additional validation of a phenopacket that is intended to be used + * for HPO rare disease phenotyping. By assumption, the phenopacket will have been first + * checked against the {@link ValidatorInfo#generic()} specification. This class performs validation with the + * file {@code hpo-rare-disease-schema.json}. + */ + static ValidatorInfo rareDiseaseValidation() { + return DefaultValidationInfo.rareDiseaseValidator(); + } + + static ValidatorInfo of(String validatorId, String validatorName) { + return DefaultValidationInfo.of(validatorId, validatorName); + } + + String validatorId(); + + String validatorName(); + + int hashCode(); + + boolean equals(Object o); +} diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/except/PhenopacketValidatorRuntimeException.java b/validator-core/src/main/java/org/phenopackets/validator/core/except/PhenopacketValidatorRuntimeException.java new file mode 100644 index 0000000..f640744 --- /dev/null +++ b/validator-core/src/main/java/org/phenopackets/validator/core/except/PhenopacketValidatorRuntimeException.java @@ -0,0 +1,24 @@ +package org.phenopackets.validator.core.except; + +public class PhenopacketValidatorRuntimeException extends RuntimeException { + + public PhenopacketValidatorRuntimeException() { + super(); + } + + public PhenopacketValidatorRuntimeException(String message) { + super(message); + } + + public PhenopacketValidatorRuntimeException(String message, Throwable cause) { + super(message, cause); + } + + public PhenopacketValidatorRuntimeException(Throwable cause) { + super(cause); + } + + protected PhenopacketValidatorRuntimeException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { + super(message, cause, enableSuppression, writableStackTrace); + } +} diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/ClasspathJsonSchemaValidatorFactory.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/ClasspathJsonSchemaValidatorFactory.java new file mode 100644 index 0000000..6db9516 --- /dev/null +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/ClasspathJsonSchemaValidatorFactory.java @@ -0,0 +1,64 @@ +package org.phenopackets.validator.jsonschema; + +import org.phenopackets.validator.core.PhenopacketValidatorFactory; +import org.phenopackets.validator.core.ValidatorInfo; +import org.phenopackets.validator.core.except.PhenopacketValidatorRuntimeException; + +import java.io.InputStream; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Validator factory that uses JSON schema definitions that are bundled within the application (on classpath). + *

+ * @author Daniel Danis + * @author Peter N Robinson + */ +public class ClasspathJsonSchemaValidatorFactory implements PhenopacketValidatorFactory { + + private final Map validatorMap; + + public static ClasspathJsonSchemaValidatorFactory defaultValidators() { + Map validatorMap = makeValidatorMap(); + return new ClasspathJsonSchemaValidatorFactory(validatorMap); + } + + private static Map makeValidatorMap() { + return Map.of( + ValidatorInfo.generic(), makeJsonValidator("/schema/phenopacket-general-schema.json", ValidatorInfo.generic()), + ValidatorInfo.rareDiseaseValidation(), makeJsonValidator("/schema/hpo-rare-disease-schema.json", ValidatorInfo.rareDiseaseValidation()) + ); + } + + private static JsonSchemaValidator makeJsonValidator(String schemaPath, ValidatorInfo validationName) { + InputStream inputStream = ClasspathJsonSchemaValidatorFactory.class.getResourceAsStream(schemaPath); + if (inputStream == null) + throw new PhenopacketValidatorRuntimeException("Invalid JSON schema path `" + schemaPath + '`'); + + return JsonSchemaValidator.of(inputStream, validationName); + } + + private ClasspathJsonSchemaValidatorFactory(Map validatorMap) { + this.validatorMap = validatorMap; + } + + @Override + public Optional getValidatorForType(ValidatorInfo type) { + return Optional.ofNullable(validatorMap.get(type)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClasspathJsonSchemaValidatorFactory that = (ClasspathJsonSchemaValidatorFactory) o; + return Objects.equals(validatorMap, that.validatorMap); + } + + @Override + public int hashCode() { + return Objects.hash(validatorMap); + } + +} diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java new file mode 100644 index 0000000..be4e90b --- /dev/null +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java @@ -0,0 +1,105 @@ +package org.phenopackets.validator.jsonschema; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import com.networknt.schema.JsonSchema; +import com.networknt.schema.JsonSchemaFactory; +import com.networknt.schema.SpecVersion; +import org.phenopackets.validator.core.PhenopacketValidator; +import org.phenopackets.validator.core.ValidatorInfo; +import org.phenopackets.validator.core.except.PhenopacketValidatorRuntimeException; +import org.phenopackets.validator.core.ValidationItem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + + +public class JsonSchemaValidator implements PhenopacketValidator { + + private static final Logger LOGGER = LoggerFactory.getLogger(JsonSchemaValidator.class); + + /** + * The latest version of the spec that is supported by our JSON SCHEMA library is 2019/09. + */ + private static final SpecVersion.VersionFlag VERSION_FLAG = SpecVersion.VersionFlag.V201909; + private final JsonSchema jsonSchema; + private final ObjectMapper objectMapper = new ObjectMapper(); + private final ValidatorInfo validatorInfo; + + + /** + * @param jsonSchema path to JSON schema specification file + * @throws PhenopacketValidatorRuntimeException if jsonSchema is not a valid file or if the file is + * not a valid JSON schema specification + */ + public static JsonSchemaValidator of(File jsonSchema, ValidatorInfo validatorInfo) { + if (!jsonSchema.isFile()) { + throw new PhenopacketValidatorRuntimeException("Could not open file at \"" + jsonSchema.getAbsolutePath() + "\""); + } + LOGGER.debug("Reading JSON schema from `{}`", jsonSchema.getAbsolutePath()); + try (InputStream inputStream = Files.newInputStream(jsonSchema.toPath())) { + return of(inputStream, validatorInfo); + } catch (Exception e) { + throw new PhenopacketValidatorRuntimeException("Invalid JSON schema specification: " + e.getMessage()); + } + } + + public static JsonSchemaValidator of(InputStream inputStream, ValidatorInfo validatorInfo) { + JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(VERSION_FLAG); + return new JsonSchemaValidator(schemaFactory.getSchema(inputStream), validatorInfo); + } + + private JsonSchemaValidator(JsonSchema jsonSchema, ValidatorInfo validatorInfo) { + this.jsonSchema = jsonSchema; + this.validatorInfo = validatorInfo; + } + + @Override + public ValidatorInfo info() { + return validatorInfo; + } + + /** + * Validate the {@code inputStream} content (assumed to be a Phenopacket formated in JSON) + * + * @return List of {@link ValidationItem} objects (empty list if there were no errors) + */ + @Override + public List validate(InputStream inputStream) { + List errors = new ArrayList<>(); + try { + JsonNode json = objectMapper.readTree(inputStream); + jsonSchema.validate(json) + .forEach(e -> errors.add(new JsonValidationError(validatorInfo, e))); + + return errors; + } catch (IOException e) { + LOGGER.warn("Error while decoding JSON content: {}", e.getMessage(), e); + return List.of(); + } catch (RuntimeException e) { + LOGGER.warn("Error while validating "); + return List.of(); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + JsonSchemaValidator that = (JsonSchemaValidator) o; + return Objects.equals(jsonSchema, that.jsonSchema) && Objects.equals(objectMapper, that.objectMapper) && Objects.equals(validatorInfo, that.validatorInfo); + } + + @Override + public int hashCode() { + return Objects.hash(jsonSchema, objectMapper, validatorInfo); + } +} diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationError.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationError.java new file mode 100644 index 0000000..5a8b3a0 --- /dev/null +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationError.java @@ -0,0 +1,61 @@ +package org.phenopackets.validator.jsonschema; +import com.networknt.schema.ValidationMessage; +import org.phenopackets.validator.core.ErrorType; +import org.phenopackets.validator.core.ValidationItem; +import org.phenopackets.validator.core.ValidatorInfo; + +import java.util.Objects; + +/** + * POJO to represent errors identified by JSON Schema validation. + * @author Peter N Robinson + */ +public final class JsonValidationError implements ValidationItem { + + private final ValidatorInfo validatorInfo; + private final ErrorType errorType; + private final String message; + + public JsonValidationError(ValidatorInfo validatorInfo, ValidationMessage validationMessage) { + this.validatorInfo = validatorInfo; + this.errorType = ErrorType.stringToErrorType(validationMessage.getType()); + this.message = validationMessage.getMessage(); + } + + @Override + public ValidatorInfo validatorInfo() { + return validatorInfo; + } + + @Override + public ErrorType errorType() { + return this.errorType; + } + + @Override + public String message() { + return message; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + JsonValidationError that = (JsonValidationError) o; + return Objects.equals(validatorInfo, that.validatorInfo) && errorType == that.errorType && Objects.equals(message, that.message); + } + + @Override + public int hashCode() { + return Objects.hash(validatorInfo, errorType, message); + } + + @Override + public String toString() { + return "JsonValidationError{" + + "validatorInfo='" + validatorInfo + '\'' + + ", errorType=" + errorType + + ", message='" + message + '\'' + + '}'; + } +} diff --git a/validator-core/src/test/java/org/phenopackets/schema/validator/core/jsonschema/JsonSchemaValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java similarity index 65% rename from validator-core/src/test/java/org/phenopackets/schema/validator/core/jsonschema/JsonSchemaValidatorTest.java rename to validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java index 252b38e..8fb6982 100644 --- a/validator-core/src/test/java/org/phenopackets/schema/validator/core/jsonschema/JsonSchemaValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java @@ -1,28 +1,33 @@ -package org.phenopackets.schema.validator.core.jsonschema; +package org.phenopackets.validator.jsonschema; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.phenopackets.schema.validator.core.validation.ErrorType; -import org.phenopackets.schema.validator.core.validation.ValidationItem; +import org.phenopackets.validator.core.ErrorType; +import org.phenopackets.validator.core.ValidationItem; +import org.phenopackets.validator.core.ValidatorInfo; import java.io.File; import java.io.IOException; import java.util.List; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; public class JsonSchemaValidatorTest { + private static final ClasspathJsonSchemaValidatorFactory FACTORY = ClasspathJsonSchemaValidatorFactory.defaultValidators(); + private static File fileFromClasspath(String path) { String fname = Thread.currentThread().getContextClassLoader().getResource(path).getPath(); return new File(fname); } @Test - public void testValidationOfSimpleValidPhenopacket() throws IOException { + public void testValidationOfSimpleValidPhenopacket() throws Exception { + JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.generic()).get(); + File validSimplePhenopacket = fileFromClasspath("json/validSimplePhenopacket.json"); - JsonSchemaValidator validator = new JsonSchemaValidator(validSimplePhenopacket); - List errors = validator.validate(); + List errors = validator.validate(validSimplePhenopacket); + assertTrue(errors.isEmpty()); } @@ -31,10 +36,13 @@ public void testValidationOfSimpleValidPhenopacket() throws IOException { * It does not contain an id or a metaData element and thus should fail. */ @Test - public void testValidationOfSimpleInValidPhenopacket() { + @Disabled // TODO - we should rework the testing strategy to invalidate a valid phenopacket and check that it raises the expected error + public void testValidationOfSimpleInValidPhenopacket() throws Exception { + JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.generic()).get(); + File invalidSimplePhenopacket = fileFromClasspath("json/invalidSimplePhenopacket.json"); - JsonSchemaValidator validator = new JsonSchemaValidator(invalidSimplePhenopacket); - List errors = validator.validate(); + List errors = validator.validate(invalidSimplePhenopacket); + assertEquals(3, errors.size()); ValidationItem error = errors.get(0); assertEquals(ErrorType.JSON_REQUIRED, error.errorType()); @@ -48,18 +56,22 @@ public void testValidationOfSimpleInValidPhenopacket() { } @Test - public void testRareDiseaseBethlemahmValidPhenopacket() throws IOException { + public void testRareDiseaseBethlemahmValidPhenopacket() throws Exception { + JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.rareDiseaseValidation()).get(); + File myopathyPhenopacket = fileFromClasspath("json/bethlehamMyopathyExample.json"); - JsonSchemaValidator validator = new JsonSchemaValidator(myopathyPhenopacket); - List errors = validator.validate(); + List errors = validator.validate(myopathyPhenopacket); + assertTrue(errors.isEmpty()); } @Test + @Disabled // TODO - we should rework the testing strategy to invalidate a valid phenopacket and check that it raises the expected error public void testRareDiseaseBethlemahmInvalidValidPhenopacket() throws IOException { + JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.rareDiseaseValidation()).get(); + File invalidMyopathyPhenopacket = fileFromClasspath("json/bethlehamMyopathyInvalidExample.json"); - JsonSchemaValidator validator = new JsonSchemaValidator(invalidMyopathyPhenopacket); - List errors = validator.validate(); + List errors = validator.validate(invalidMyopathyPhenopacket); // for (ValidationError ve : errors) { // System.out.println(ve.getMessage()); // } @@ -70,4 +82,4 @@ public void testRareDiseaseBethlemahmInvalidValidPhenopacket() throws IOExceptio } -} +} \ No newline at end of file diff --git a/validator-core/src/test/resources/json/bethlehamMyopathyExample.json b/validator-jsonschema/src/test/resources/json/bethlehamMyopathyExample.json similarity index 100% rename from validator-core/src/test/resources/json/bethlehamMyopathyExample.json rename to validator-jsonschema/src/test/resources/json/bethlehamMyopathyExample.json diff --git a/validator-core/src/test/resources/json/bethlehamMyopathyInvalidExample.json b/validator-jsonschema/src/test/resources/json/bethlehamMyopathyInvalidExample.json similarity index 100% rename from validator-core/src/test/resources/json/bethlehamMyopathyInvalidExample.json rename to validator-jsonschema/src/test/resources/json/bethlehamMyopathyInvalidExample.json diff --git a/validator-core/src/test/resources/json/invalidSimplePhenopacket.json b/validator-jsonschema/src/test/resources/json/invalidSimplePhenopacket.json similarity index 100% rename from validator-core/src/test/resources/json/invalidSimplePhenopacket.json rename to validator-jsonschema/src/test/resources/json/invalidSimplePhenopacket.json diff --git a/validator-core/src/test/resources/json/validSimplePhenopacket.json b/validator-jsonschema/src/test/resources/json/validSimplePhenopacket.json similarity index 100% rename from validator-core/src/test/resources/json/validSimplePhenopacket.json rename to validator-jsonschema/src/test/resources/json/validSimplePhenopacket.json From c1fefde51888a1315535f8abc88b51152f47f8d6 Mon Sep 17 00:00:00 2001 From: Daniel Danis Date: Mon, 13 Sep 2021 13:57:52 -0400 Subject: [PATCH 06/37] Remove `json-schema-validator` from `validator-core` --- validator-core/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/validator-core/pom.xml b/validator-core/pom.xml index c655b37..58e0948 100644 --- a/validator-core/pom.xml +++ b/validator-core/pom.xml @@ -16,12 +16,6 @@ Validator utilities for phenopackets - - com.networknt - json-schema-validator - 1.0.42 - - - - - - - - - - - - - - - - - - - - + + org.phenopackets + phenopacket-schema + ${phenopacket-schema.version} + test + + + com.google.protobuf + protobuf-java + ${protobuf.version} + test + + + com.google.protobuf + protobuf-java-util + ${protobuf.version} + test + \ No newline at end of file diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/SimplePhenopacket.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/SimplePhenopacket.java new file mode 100644 index 0000000..b846f65 --- /dev/null +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/SimplePhenopacket.java @@ -0,0 +1,29 @@ +package org.phenopackets.validator.builder; + +import org.phenopackets.schema.v2.Phenopacket; +import org.phenopackets.schema.v2.core.MetaData; +import org.phenopackets.validator.builder.util.MetaDataUtil; + + +/** + * Build the simplest possible phenopacket for validation + */ +public class SimplePhenopacket { + + private final Phenopacket phenopacket; + + public SimplePhenopacket() { + MetaData simplMetaData = MetaDataUtil.simpleHpoMetaData(); + phenopacket = Phenopacket.newBuilder() + .setId("hello world") + .setMetaData(simplMetaData) + .build(); + } + + + public Phenopacket getPhenopacket() { + return phenopacket; + } + + +} diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/MetaDataUtil.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/MetaDataUtil.java new file mode 100644 index 0000000..e065822 --- /dev/null +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/MetaDataUtil.java @@ -0,0 +1,133 @@ +package org.phenopackets.validator.builder.util; + +import com.google.protobuf.Timestamp; +import org.phenopackets.schema.v2.core.ExternalReference; +import org.phenopackets.schema.v2.core.MetaData; +import org.phenopackets.schema.v2.core.Resource; +import org.phenopackets.schema.v2.core.Update; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +import static org.phenopackets.validator.builder.util.PhenopacketUtil.externalReference; +import static org.phenopackets.validator.builder.util.PhenopacketUtil.resource;; + +public class MetaDataUtil { + + private final static String DEFAULT_PMID = "PMID:33264411"; + private final static String DEFAULT_TITLE = "The Human Phenotype Ontology in 2021"; + private final static ExternalReference DEFAULT_EXTERNAL_REFERENCE = + externalReference(DEFAULT_PMID, DEFAULT_TITLE ); + private final static Resource HPO_RESOURCE = resource("hp", "Human Phenotype Ontology", "HP", "http://purl.obolibrary.org/obo/HP_","http://purl.obolibrary.org/obo/hp.owl","2018-03-08"); + private final static Resource GENO_RESOURCE = resource("geno", "Genotype Ontology", "GENO", "http://purl.obolibrary.org/obo/GENO_", "http://purl.obolibrary.org/obo/geno.owl", "19-03-2018"); + + private final static LocalDateTime timeNow = LocalDate.of(2021, 5, 14).atTime(10, 35); + /** + * Time when the MetaData was constructed. + */ + private final static Timestamp timestamp =Timestamp.newBuilder() + .setSeconds(timeNow.toEpochSecond(ZoneOffset.UTC)) + .build(); + + private final MetaData metaData; + + private MetaDataUtil() { + this(DEFAULT_EXTERNAL_REFERENCE); + } + + private MetaDataUtil(ExternalReference citation) { + this.metaData = MetaData.newBuilder() + .addResources(HPO_RESOURCE) + .addResources(GENO_RESOURCE) + .setCreatedBy("Peter R.") + .setCreated(timestamp) + .setPhenopacketSchemaVersion("2.0") + .addExternalReferences(citation) + .build(); + } + + public static MetaData simpleHpoMetaData() { + return MetaData.newBuilder() + .addResources(HPO_RESOURCE) + .setCreatedBy("Peter R.") + .setCreated(timestamp) + .setPhenopacketSchemaVersion("2.0") + .build(); + } + + + + public static MetaData defaultRareDiseaseMetaData(ExternalReference externalReference) { + MetaDataUtil mdatautil = new MetaDataUtil(externalReference); + return mdatautil.metaData; + } + + + public static Timestamp parseTimestamp(String times) { + try { + return com.google.protobuf.util.Timestamps.parse(times); + } catch (java.text.ParseException pe) { + throw new RuntimeException(pe.getMessage()); + } + } + + + public static MetaData defaultCancerMetadata() { + return MetaData.newBuilder() + .setPhenopacketSchemaVersion(PhenopacketUtil.SCHEMA_VERSION) + .setCreated(parseTimestamp("2016-06-29T12:03:03.240Z")) + .addUpdates(Update.newBuilder().setTimestamp(parseTimestamp("2018-06-10T10:59:06.784Z"))) + .addResources(Resource.newBuilder() + .setId("pato") + .setName("PhenotypicFeature And Trait Ontology") + .setNamespacePrefix("PATO") + .setUrl("http://purl.obolibrary.org/obo/pato.owl") + .setIriPrefix("http://purl.obolibrary.org/obo/PATO_") + .setVersion("2018-03-28") + .build()) + .addResources(Resource.newBuilder() + .setId("efo") + .setName("Experimental Factor Ontology") + .setNamespacePrefix("EFO") + .setUrl("http://www.ebi.ac.uk/efo/efo.owl") + .setIriPrefix("http://purl.obolibrary.org/obo/EFO_") + .setVersion("2.97") + .build()) + .addResources(Resource.newBuilder() + .setId("cl") + .setName("Cell Ontology") + .setNamespacePrefix("CL") + .setUrl("http://purl.obolibrary.org/obo/cl.owl") + .setIriPrefix("http://purl.obolibrary.org/obo/CL_") + .setVersion("2017-12-11") + .build()) + .addResources(Resource.newBuilder() + .setId("bto") + .setName("BRENDA tissue / enzyme source") + .setNamespacePrefix("BTO") + .setUrl("http://purl.obolibrary.org/obo/bto") + .setIriPrefix("http://purl.obolibrary.org/obo/BTO_") + .setVersion("2016-05-05") + .build()) + .addResources(Resource.newBuilder() + .setId("uberon") + .setName("Uber-anatomy ontology") + .setNamespacePrefix("UBERON") + .setUrl("http://purl.obolibrary.org/obo/uberon.owl") + .setIriPrefix("http://purl.obolibrary.org/obo/UBERON_") + .setVersion("2018-05-14") + .build()) + .addResources(Resource.newBuilder() + .setId("ncbitaxon") + .setName("NCBI organismal classification") + .setNamespacePrefix("NCBITaxon") + .setUrl("http://purl.obolibrary.org/obo/ncbitaxon.owl") + .setIriPrefix("http://purl.obolibrary.org/obo/NCBITaxon_") + .setVersion("2018-03-02") + .build()) + .setCreatedBy("Anonymous biocurator") + .build(); + } + +} \ No newline at end of file diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/PhenopacketUtil.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/PhenopacketUtil.java new file mode 100644 index 0000000..26e00bd --- /dev/null +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/PhenopacketUtil.java @@ -0,0 +1,28 @@ +package org.phenopackets.validator.builder.util; + +import org.phenopackets.schema.v2.core.Resource; + + + +import org.phenopackets.schema.v2.core.ExternalReference; + + +public class PhenopacketUtil { + public static final String SCHEMA_VERSION = "2.0"; + + public static ExternalReference externalReference(String id, String description) { + return ExternalReference.newBuilder().setId(id).setDescription(description).build(); + } + + + public static Resource resource(String id, String name, String nsPrefix, String iriPrefix, String url, String version) { + return Resource.newBuilder() + .setId(id) + .setName(name) + .setNamespacePrefix(nsPrefix) + .setIriPrefix(iriPrefix) + .setUrl(url) + .setVersion(version) + .build(); + } +} diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java index 8fb6982..03b9cfd 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java @@ -2,6 +2,9 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.DisplayNameGenerator.Simple; +import org.phenopackets.schema.v2.Phenopacket; +import org.phenopackets.validator.builder.SimplePhenopacket; import org.phenopackets.validator.core.ErrorType; import org.phenopackets.validator.core.ValidationItem; import org.phenopackets.validator.core.ValidatorInfo; @@ -9,6 +12,9 @@ import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Objects; + +import com.google.protobuf.util.JsonFormat; import static org.junit.jupiter.api.Assertions.*; @@ -16,19 +22,28 @@ public class JsonSchemaValidatorTest { private static final ClasspathJsonSchemaValidatorFactory FACTORY = ClasspathJsonSchemaValidatorFactory.defaultValidators(); + private static final SimplePhenopacket simplePhenopacket = new SimplePhenopacket(); + private static File fileFromClasspath(String path) { - String fname = Thread.currentThread().getContextClassLoader().getResource(path).getPath(); + String fname = Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResource(path)).getPath(); return new File(fname); } @Test public void testValidationOfSimpleValidPhenopacket() throws Exception { JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.generic()).get(); - - File validSimplePhenopacket = fileFromClasspath("json/validSimplePhenopacket.json"); - List errors = validator.validate(validSimplePhenopacket); - + Phenopacket phenopacket = simplePhenopacket.getPhenopacket(); + String json = JsonFormat.printer().print(phenopacket); + List errors = validator.validate(json); assertTrue(errors.isEmpty()); + // the Phenopacket is not valid if we remove the id + phenopacket = Phenopacket.newBuilder(phenopacket).clearId().build(); + json = JsonFormat.printer().print(phenopacket); + errors = validator.validate(json); + assertEquals(1, errors.size()); + ValidationItem error = errors.get(0); + assertEquals(ErrorType.JSON_REQUIRED, error.errorType()); + assertEquals("$.id: is missing but it is required", error.message()); } /** @@ -36,12 +51,12 @@ public void testValidationOfSimpleValidPhenopacket() throws Exception { * It does not contain an id or a metaData element and thus should fail. */ @Test - @Disabled // TODO - we should rework the testing strategy to invalidate a valid phenopacket and check that it raises the expected error public void testValidationOfSimpleInValidPhenopacket() throws Exception { JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.generic()).get(); - File invalidSimplePhenopacket = fileFromClasspath("json/invalidSimplePhenopacket.json"); - List errors = validator.validate(invalidSimplePhenopacket); + String invalidPhenopacketJson = "{\"disney\" : \"donald\"}"; + + List errors = validator.validate(invalidPhenopacketJson); assertEquals(3, errors.size()); ValidationItem error = errors.get(0); From 808fe37f5d23a28decb6132a3992a12ac011d751 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Thu, 16 Sep 2021 10:57:28 -0400 Subject: [PATCH 08/37] datagen for testing --- .../validator/builder/SimplePhenopacket.java | 29 -- .../validator/builder/util/MetaDataUtil.java | 133 -------- .../builder/util/PhenopacketUtil.java | 28 -- .../jsonschema/JsonSchemaValidatorTest.java | 13 +- .../testdatagen/PhenopacketUtil.java | 310 ++++++++++++++++++ .../testdatagen/RareDiseasePhenopacket.java | 64 ++++ .../testdatagen/SimplePhenopacket.java | 35 ++ 7 files changed, 418 insertions(+), 194 deletions(-) delete mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/builder/SimplePhenopacket.java delete mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/MetaDataUtil.java delete mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/PhenopacketUtil.java create mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/PhenopacketUtil.java create mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java create mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/SimplePhenopacket.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/SimplePhenopacket.java deleted file mode 100644 index b846f65..0000000 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/SimplePhenopacket.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.phenopackets.validator.builder; - -import org.phenopackets.schema.v2.Phenopacket; -import org.phenopackets.schema.v2.core.MetaData; -import org.phenopackets.validator.builder.util.MetaDataUtil; - - -/** - * Build the simplest possible phenopacket for validation - */ -public class SimplePhenopacket { - - private final Phenopacket phenopacket; - - public SimplePhenopacket() { - MetaData simplMetaData = MetaDataUtil.simpleHpoMetaData(); - phenopacket = Phenopacket.newBuilder() - .setId("hello world") - .setMetaData(simplMetaData) - .build(); - } - - - public Phenopacket getPhenopacket() { - return phenopacket; - } - - -} diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/MetaDataUtil.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/MetaDataUtil.java deleted file mode 100644 index e065822..0000000 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/MetaDataUtil.java +++ /dev/null @@ -1,133 +0,0 @@ -package org.phenopackets.validator.builder.util; - -import com.google.protobuf.Timestamp; -import org.phenopackets.schema.v2.core.ExternalReference; -import org.phenopackets.schema.v2.core.MetaData; -import org.phenopackets.schema.v2.core.Resource; -import org.phenopackets.schema.v2.core.Update; - -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.ZoneOffset; - -import static org.phenopackets.validator.builder.util.PhenopacketUtil.externalReference; -import static org.phenopackets.validator.builder.util.PhenopacketUtil.resource;; - -public class MetaDataUtil { - - private final static String DEFAULT_PMID = "PMID:33264411"; - private final static String DEFAULT_TITLE = "The Human Phenotype Ontology in 2021"; - private final static ExternalReference DEFAULT_EXTERNAL_REFERENCE = - externalReference(DEFAULT_PMID, DEFAULT_TITLE ); - private final static Resource HPO_RESOURCE = resource("hp", "Human Phenotype Ontology", "HP", "http://purl.obolibrary.org/obo/HP_","http://purl.obolibrary.org/obo/hp.owl","2018-03-08"); - private final static Resource GENO_RESOURCE = resource("geno", "Genotype Ontology", "GENO", "http://purl.obolibrary.org/obo/GENO_", "http://purl.obolibrary.org/obo/geno.owl", "19-03-2018"); - - private final static LocalDateTime timeNow = LocalDate.of(2021, 5, 14).atTime(10, 35); - /** - * Time when the MetaData was constructed. - */ - private final static Timestamp timestamp =Timestamp.newBuilder() - .setSeconds(timeNow.toEpochSecond(ZoneOffset.UTC)) - .build(); - - private final MetaData metaData; - - private MetaDataUtil() { - this(DEFAULT_EXTERNAL_REFERENCE); - } - - private MetaDataUtil(ExternalReference citation) { - this.metaData = MetaData.newBuilder() - .addResources(HPO_RESOURCE) - .addResources(GENO_RESOURCE) - .setCreatedBy("Peter R.") - .setCreated(timestamp) - .setPhenopacketSchemaVersion("2.0") - .addExternalReferences(citation) - .build(); - } - - public static MetaData simpleHpoMetaData() { - return MetaData.newBuilder() - .addResources(HPO_RESOURCE) - .setCreatedBy("Peter R.") - .setCreated(timestamp) - .setPhenopacketSchemaVersion("2.0") - .build(); - } - - - - public static MetaData defaultRareDiseaseMetaData(ExternalReference externalReference) { - MetaDataUtil mdatautil = new MetaDataUtil(externalReference); - return mdatautil.metaData; - } - - - public static Timestamp parseTimestamp(String times) { - try { - return com.google.protobuf.util.Timestamps.parse(times); - } catch (java.text.ParseException pe) { - throw new RuntimeException(pe.getMessage()); - } - } - - - public static MetaData defaultCancerMetadata() { - return MetaData.newBuilder() - .setPhenopacketSchemaVersion(PhenopacketUtil.SCHEMA_VERSION) - .setCreated(parseTimestamp("2016-06-29T12:03:03.240Z")) - .addUpdates(Update.newBuilder().setTimestamp(parseTimestamp("2018-06-10T10:59:06.784Z"))) - .addResources(Resource.newBuilder() - .setId("pato") - .setName("PhenotypicFeature And Trait Ontology") - .setNamespacePrefix("PATO") - .setUrl("http://purl.obolibrary.org/obo/pato.owl") - .setIriPrefix("http://purl.obolibrary.org/obo/PATO_") - .setVersion("2018-03-28") - .build()) - .addResources(Resource.newBuilder() - .setId("efo") - .setName("Experimental Factor Ontology") - .setNamespacePrefix("EFO") - .setUrl("http://www.ebi.ac.uk/efo/efo.owl") - .setIriPrefix("http://purl.obolibrary.org/obo/EFO_") - .setVersion("2.97") - .build()) - .addResources(Resource.newBuilder() - .setId("cl") - .setName("Cell Ontology") - .setNamespacePrefix("CL") - .setUrl("http://purl.obolibrary.org/obo/cl.owl") - .setIriPrefix("http://purl.obolibrary.org/obo/CL_") - .setVersion("2017-12-11") - .build()) - .addResources(Resource.newBuilder() - .setId("bto") - .setName("BRENDA tissue / enzyme source") - .setNamespacePrefix("BTO") - .setUrl("http://purl.obolibrary.org/obo/bto") - .setIriPrefix("http://purl.obolibrary.org/obo/BTO_") - .setVersion("2016-05-05") - .build()) - .addResources(Resource.newBuilder() - .setId("uberon") - .setName("Uber-anatomy ontology") - .setNamespacePrefix("UBERON") - .setUrl("http://purl.obolibrary.org/obo/uberon.owl") - .setIriPrefix("http://purl.obolibrary.org/obo/UBERON_") - .setVersion("2018-05-14") - .build()) - .addResources(Resource.newBuilder() - .setId("ncbitaxon") - .setName("NCBI organismal classification") - .setNamespacePrefix("NCBITaxon") - .setUrl("http://purl.obolibrary.org/obo/ncbitaxon.owl") - .setIriPrefix("http://purl.obolibrary.org/obo/NCBITaxon_") - .setVersion("2018-03-02") - .build()) - .setCreatedBy("Anonymous biocurator") - .build(); - } - -} \ No newline at end of file diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/PhenopacketUtil.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/PhenopacketUtil.java deleted file mode 100644 index 26e00bd..0000000 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/builder/util/PhenopacketUtil.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.phenopackets.validator.builder.util; - -import org.phenopackets.schema.v2.core.Resource; - - - -import org.phenopackets.schema.v2.core.ExternalReference; - - -public class PhenopacketUtil { - public static final String SCHEMA_VERSION = "2.0"; - - public static ExternalReference externalReference(String id, String description) { - return ExternalReference.newBuilder().setId(id).setDescription(description).build(); - } - - - public static Resource resource(String id, String name, String nsPrefix, String iriPrefix, String url, String version) { - return Resource.newBuilder() - .setId(id) - .setName(name) - .setNamespacePrefix(nsPrefix) - .setIriPrefix(iriPrefix) - .setUrl(url) - .setVersion(version) - .build(); - } -} diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java index 03b9cfd..5bdb878 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java @@ -2,9 +2,9 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.DisplayNameGenerator.Simple; import org.phenopackets.schema.v2.Phenopacket; -import org.phenopackets.validator.builder.SimplePhenopacket; +import org.phenopackets.validator.testdatagen.RareDiseasePhenopacket; +import org.phenopackets.validator.testdatagen.SimplePhenopacket; import org.phenopackets.validator.core.ErrorType; import org.phenopackets.validator.core.ValidationItem; import org.phenopackets.validator.core.ValidatorInfo; @@ -24,6 +24,8 @@ public class JsonSchemaValidatorTest { private static final SimplePhenopacket simplePhenopacket = new SimplePhenopacket(); + private static final RareDiseasePhenopacket rareDiseasePhenopacket = new RareDiseasePhenopacket(); + private static File fileFromClasspath(String path) { String fname = Objects.requireNonNull(Thread.currentThread().getContextClassLoader().getResource(path)).getPath(); return new File(fname); @@ -74,8 +76,11 @@ public void testValidationOfSimpleInValidPhenopacket() throws Exception { public void testRareDiseaseBethlemahmValidPhenopacket() throws Exception { JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.rareDiseaseValidation()).get(); - File myopathyPhenopacket = fileFromClasspath("json/bethlehamMyopathyExample.json"); - List errors = validator.validate(myopathyPhenopacket); + // File myopathyPhenopacket = fileFromClasspath("json/bethlehamMyopathyExample.json"); + + Phenopacket bethlehamMyopathy = rareDiseasePhenopacket.getPhenopacket(); + String json = JsonFormat.printer().print(bethlehamMyopathy); + List errors = validator.validate(json); assertTrue(errors.isEmpty()); } diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/PhenopacketUtil.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/PhenopacketUtil.java new file mode 100644 index 0000000..9bd6cda --- /dev/null +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/PhenopacketUtil.java @@ -0,0 +1,310 @@ +package org.phenopackets.validator.testdatagen; + + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.List; + +import com.google.protobuf.Timestamp; +import com.google.protobuf.util.Timestamps; + + +import org.phenopackets.schema.v2.core.Resource; +import org.phenopackets.schema.v2.core.Sex; +import org.phenopackets.schema.v2.core.TimeElement; +import org.phenopackets.schema.v2.core.Update; +import org.phenopackets.schema.v2.core.VitalStatus; +import org.phenopackets.schema.v2.core.Individual; +import org.phenopackets.schema.v2.core.Age; +import org.phenopackets.schema.v2.core.ExternalReference; +import org.phenopackets.schema.v2.core.File; +import org.phenopackets.schema.v2.core.MetaData; +import org.phenopackets.schema.v2.core.OntologyClass; +import org.phenopackets.schema.v2.core.PhenotypicFeature; + + +public class PhenopacketUtil { + public static final String SCHEMA_VERSION = "2.0"; + public static final OntologyClass CONGENITAL_ONSET = ontologyClass("HP:0003577", "Congenital onset"); + public static final OntologyClass CHILDHOOD_ONSET = ontologyClass("HP:0011463", "Childhood onset"); + public static final OntologyClass ADULT_ONSET = ontologyClass("HP:0003581", "Adult onset"); + public static final OntologyClass SEVERE = ontologyClass("HP:0012828", "Severe"); + + public static ExternalReference externalReference(String id, String description) { + return ExternalReference.newBuilder().setId(id).setDescription(description).build(); + } + + public static OntologyClass ontologyClass(String termid, String label) { + return OntologyClass.newBuilder() + .setId(termid) + .setLabel(label) + .build(); + } + + + public static Resource resource(String id, String name, String nsPrefix, String iriPrefix, String url, String version) { + return Resource.newBuilder() + .setId(id) + .setName(name) + .setNamespacePrefix(nsPrefix) + .setIriPrefix(iriPrefix) + .setUrl(url) + .setVersion(version) + .build(); + } + + public static PhenotypicFeature phenotypicFeatureCongenital(String termid, String label) { + return PhenotypicFeature.newBuilder() + .setType(ontologyClass(termid, label)) + .setType(CONGENITAL_ONSET) + .build(); + } + + public static PhenotypicFeature phenotypicFeatureChildhood(String termid, String label) { + return PhenotypicFeature.newBuilder() + .setType(ontologyClass(termid, label)) + .setType(CHILDHOOD_ONSET) + .build(); + } + + + + /** + * This has convenience methods for building PhenotypicFeature messages with some + * commonly used options. + * @author Peter N Robinson + */ + public static class PhenotypicFeatureBuilder { + private final String termid; + private final String label; + private OntologyClass onset = null; + private OntologyClass severity = null; + private List modifiers = new ArrayList<>(); + + public PhenotypicFeatureBuilder(String id, String label) { + this.termid = id; + this.label = label; + } + + public PhenotypicFeatureBuilder onset(String id, String label) { + onset = ontologyClass(id, label); + return this; + } + + public PhenotypicFeatureBuilder congenitalOnset() { + onset = CONGENITAL_ONSET; + return this; + } + + public PhenotypicFeatureBuilder childhoodOnset() { + onset = CHILDHOOD_ONSET; + return this; + } + + public PhenotypicFeatureBuilder adultOnset() { + onset = ADULT_ONSET; + return this; + } + + public PhenotypicFeatureBuilder severity(String id, String label) { + severity = ontologyClass(id, label); + return this; + } + + public PhenotypicFeatureBuilder severe() { + severity = SEVERE; + return this; + } + + public PhenotypicFeature build() { + OntologyClass hpoTerm = ontologyClass(termid, label); + PhenotypicFeature.Builder builder = PhenotypicFeature.newBuilder().setType(hpoTerm); + if (this.onset != null) { + builder.mergeFrom(builder.build()).setOnset(TimeElement.newBuilder().setOntologyClass(onset)); + } + if (this.severity != null) { + builder.mergeFrom(builder.build()).setSeverity(severity); + } + if (! this.modifiers.isEmpty()) { + for (OntologyClass clz : this.modifiers) { + builder.mergeFrom(builder.build()).addModifiers(clz); + } + } + + return builder.build(); + } + + public static PhenotypicFeatureBuilder create(String id, String label) { + return new PhenotypicFeatureBuilder(id, label); + } + } + + + + public static Resource hpoResource(String version) { + return resource("hp", "Human Phenotype Ontology", "HP", "http://purl.obolibrary.org/obo/HP_","http://purl.obolibrary.org/obo/hp.owl",version); + } + + public static Resource mondoResource(String version) { + return resource("mondo", "Mondo Disease Ontology", "Mondo", "http://purl.obolibrary.org/obo/mondo_","http://purl.obolibrary.org/obo/mondo.owl",version); + } + + /** + * Parse a google protobuf timestamp object from a string in RFC 3339 format (e.g., 2019-10-12T07:20:50.52Z) + * @param tstamp RFC 3339 string + * @return corresponding protobuf timestamp object + */ + public static Timestamp fromRFC3339(String tstamp) { + try { + return Timestamps.parse(tstamp); + } catch (Exception e) { + LocalDateTime timeNow = LocalDateTime.now(); + return Timestamp.newBuilder() + .setSeconds(timeNow.toEpochSecond(ZoneOffset.UTC)) + .build(); + } + } + + + public static class MetaDataBuilder { + + + private MetaData.Builder builder; + + + public MetaDataBuilder(String created, String createdBy) { + builder = MetaData.newBuilder() + .setCreated(fromRFC3339(created)) + .setCreatedBy(createdBy) + .setPhenopacketSchemaVersion(SCHEMA_VERSION); // only one option for schema version! + } + + + public static MetaDataBuilder create(String created, String createdBy) { + return new MetaDataBuilder(created, createdBy); + } + + public MetaDataBuilder submittedBy(String submitter) { + builder = builder.mergeFrom(builder.build()).setSubmittedBy(submitter); + return this; + } + + public MetaDataBuilder addResource(Resource r) { + builder = builder.mergeFrom(builder.build()).addResources(r); + return this; + } + + public MetaDataBuilder addUpdate(Update u) { + builder = builder.mergeFrom(builder.build()).addUpdates(u); + return this; + } + + public MetaDataBuilder addExternalReference(ExternalReference er) { + builder = builder.mergeFrom(builder.build()).addExternalReferences(er); + return this; + } + + public MetaDataBuilder addExternalReference(String id, String description) { + ExternalReference er = ExternalReference.newBuilder().setId(id).setDescription(description).build(); + builder = builder.mergeFrom(builder.build()).addExternalReferences(er); + return this; + } + + public MetaData build() { + return builder.build(); + } + + } + + + public static class FileBuilder { + + private final File.Builder builder; + + public FileBuilder(String uri) { + builder = File.newBuilder().setUri(uri); + } + + public FileBuilder addFileAttribute(String k, String v) { + builder.mergeFrom(builder.build()).putFileAttributes(k, v); + return this; + } + + public FileBuilder addIndividualToFileIdentifiers(String individual, String fileIdentifier) { + builder.mergeFrom(builder.build()).putIndividualToFileIdentifiers(individual, fileIdentifier); + return this; + } + + public File build() { + return builder.build(); + } + + public static FileBuilder create(String uri) { + return new FileBuilder(uri); + } + + public static FileBuilder hg38vcf(String uri) { + FileBuilder fb = new FileBuilder(uri); + fb.addFileAttribute("genomeAssembly", "GRCh38"); + fb.addFileAttribute("fileFormat", "vcf"); + return fb; + } + } + + public static class IndividualBuilder { + + private Individual.Builder builder; + + public IndividualBuilder(String id) { + builder = Individual.newBuilder().setId(id); + } + + public IndividualBuilder dateOfBirth(String dobirth) { + Timestamp dob = fromRFC3339(dobirth); + builder = builder.mergeFrom(builder.build()).setDateOfBirth(dob); + return this; + } + + + public IndividualBuilder timeAtLastEncounter(String iso8601) { + TimeElement t = TimeElement.newBuilder().setAge(Age.newBuilder().setIso8601Duration(iso8601)).build(); + builder = builder.mergeFrom(builder.build()).setTimeAtLastEncounter(t); + return this; + } + + public IndividualBuilder alive() { + VitalStatus status = VitalStatus.newBuilder().setStatus(VitalStatus.Status.ALIVE).build(); + builder = builder.mergeFrom(builder.build()).setVitalStatus(status); + return this; + } + + public IndividualBuilder deceased() { + VitalStatus status = VitalStatus.newBuilder().setStatus(VitalStatus.Status.DECEASED).build(); + builder = builder.mergeFrom(builder.build()).setVitalStatus(status); + return this; + } + + public IndividualBuilder male() { + builder = builder.mergeFrom(builder.build()).setSex(Sex.MALE); + return this; + } + + public IndividualBuilder female() { + builder = builder.mergeFrom(builder.build()).setSex(Sex.FEMALE); + return this; + } + + + public Individual build() { + return builder.build(); + } + + public static IndividualBuilder create(String id) { + return new IndividualBuilder(id); + } + + } + + +} diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java new file mode 100644 index 0000000..2726300 --- /dev/null +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java @@ -0,0 +1,64 @@ +package org.phenopackets.validator.testdatagen; + +import org.phenopackets.schema.v2.Phenopacket; +import org.phenopackets.schema.v2.core.File; +import org.phenopackets.schema.v2.core.Individual; +import org.phenopackets.schema.v2.core.MetaData; +import org.phenopackets.schema.v2.core.Resource; +import org.phenopackets.validator.testdatagen.PhenopacketUtil.PhenotypicFeatureBuilder; +import static org.phenopackets.validator.testdatagen.PhenopacketUtil.*; + +public class RareDiseasePhenopacket { + + + private final Phenopacket phenopacket; + + public RareDiseasePhenopacket() { + + Individual proband = IndividualBuilder.create("patient 1") + .dateOfBirth("1998-01-01T00:00:00Z") + .timeAtLastEncounter("P3Y") + .male() + .build(); + var syndactyly = PhenotypicFeatureBuilder.create("HP:0001159", "Syndactyly") + .congenitalOnset().build(); + var pneumonia = PhenotypicFeatureBuilder.create("HP:0002090", "Pneumonia") + .childhoodOnset().build(); + var cryptorchidism = PhenotypicFeatureBuilder.create("HP:0000028", "Cryptorchidism") + .congenitalOnset().build(); + var sinusitis = PhenotypicFeatureBuilder.create("HP:0011109", "Chronic sinusitis") + .severe().adultOnset().build(); + + Resource hpo = hpoResource("2021-08-02"); + Resource mondo = mondoResource("2021-09-01"); + MetaData meta = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") + .submittedBy("anonymous submitter") + .addResource(hpo) + .addResource(mondo) + .addExternalReference("PMID:20842687", + "Severe dystonic encephalopathy without hyperphenylalaninemia associated with an 18-bp deletion within the proximal GCH1 promoter") + .build(); + File vcf = FileBuilder.hg38vcf("file://data/file.vcf.gz") + .addIndividualToFileIdentifiers("kindred 1A", "SAME000234") + .build(); + + this.phenopacket = Phenopacket.newBuilder() + .setId("phenopacket-id-1") + .setSubject(proband) + .addPhenotypicFeatures(syndactyly) + .addPhenotypicFeatures(pneumonia) + .addPhenotypicFeatures(cryptorchidism) + .addPhenotypicFeatures(sinusitis) + .addFiles(vcf) + .setMetaData(meta) + .build(); + + } + + public Phenopacket getPhenopacket() { + return phenopacket; + } + + + +} diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java new file mode 100644 index 0000000..03c0f1b --- /dev/null +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java @@ -0,0 +1,35 @@ +package org.phenopackets.validator.testdatagen; + +import org.phenopackets.schema.v2.Phenopacket; +import org.phenopackets.schema.v2.core.MetaData; +import org.phenopackets.schema.v2.core.Resource; + +import static org.phenopackets.validator.testdatagen.PhenopacketUtil.hpoResource; + + +/** + * Build the simplest possible phenopacket for validation + */ +public class SimplePhenopacket { + + private final Phenopacket phenopacket; + + public SimplePhenopacket() { + Resource hpo = hpoResource("2021-08-02"); + MetaData meta = PhenopacketUtil.MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") + .submittedBy("anonymous submitter") + .addResource(hpo) + .build(); + phenopacket = Phenopacket.newBuilder() + .setId("hello world") + .setMetaData(meta) + .build(); + } + + + public Phenopacket getPhenopacket() { + return phenopacket; + } + + +} From 045c258a13f0037effc03f3179e4915c293905ba Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Thu, 16 Sep 2021 20:59:23 -0400 Subject: [PATCH 09/37] adding 'type' error type --- .../main/java/org/phenopackets/validator/core/ErrorType.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java b/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java index e401e05..0b45d0f 100644 --- a/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java @@ -7,6 +7,8 @@ public enum ErrorType { JSON_ADDITIONAL_PROPERTIES("additionalProperties"), /** JSON schema error meaning that the JSON code failed to contain a property required by the schema. */ JSON_REQUIRED("required"), + /** The type of an object is not as required by the schema, e.g., we get a string instead of an array. */ + JSON_TYPE("type"), PHENOPACKET_SUBJECT_LACKS_AGE("phenopacket subject lacks age"), PHENOPACKET_LACKS_SUBJECT("phenopacket lacks subject"), INVALID_ONTOLOGY("invalid ontology"), @@ -29,6 +31,7 @@ public static ErrorType stringToErrorType(String error) { switch (error) { case "additionalProperties": return JSON_ADDITIONAL_PROPERTIES; case "required": return JSON_REQUIRED; + case "type": return JSON_TYPE; default: throw new PhenopacketValidatorRuntimeException("Did not recognize error type: \"" + error + "\""); From 3a8b168a7acde5f51be0636d1d2dbf9424ef1055 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Thu, 16 Sep 2021 20:59:42 -0400 Subject: [PATCH 10/37] adding tests for disease element --- .../jsonschema/JsonSchemaValidator.java | 2 +- .../JsonSchemaDiseaseValidatorTest.java | 89 +++++++++++++++++++ .../json/invalidSimplePhenopacket.json | 1 - .../json/validSimplePhenopacket.json | 16 ---- 4 files changed, 90 insertions(+), 18 deletions(-) create mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java delete mode 100644 validator-jsonschema/src/test/resources/json/invalidSimplePhenopacket.json delete mode 100644 validator-jsonschema/src/test/resources/json/validSimplePhenopacket.json diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java index be4e90b..2471457 100644 --- a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java @@ -85,7 +85,7 @@ public List validate(InputStream inputStream) { LOGGER.warn("Error while decoding JSON content: {}", e.getMessage(), e); return List.of(); } catch (RuntimeException e) { - LOGGER.warn("Error while validating "); + LOGGER.warn("Error while validating: {}", e.getMessage()); return List.of(); } } diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java new file mode 100644 index 0000000..d850f9b --- /dev/null +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java @@ -0,0 +1,89 @@ +package org.phenopackets.validator.jsonschema; + + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import org.junit.jupiter.api.Test; +import org.phenopackets.schema.v2.Phenopacket; +import org.phenopackets.schema.v2.core.Disease; +import org.phenopackets.schema.v2.core.MetaData; +import org.phenopackets.schema.v2.core.Resource; +import org.phenopackets.schema.v2.core.TimeElement; +import org.phenopackets.validator.core.ErrorType; +import org.phenopackets.validator.core.ValidationItem; +import org.phenopackets.validator.core.ValidatorInfo; +import org.phenopackets.validator.testdatagen.PhenopacketUtil; + +import java.io.IOException; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.phenopackets.validator.testdatagen.PhenopacketUtil.*; + +/** + * This class creates a simple phenopacket with a Disease object and creates some variations with + * validation errors. + * @author Peter N Robinson + */ +public class JsonSchemaDiseaseValidatorTest { + + private static final ClasspathJsonSchemaValidatorFactory FACTORY = ClasspathJsonSchemaValidatorFactory.defaultValidators(); + + + private static Disease mondoDisease() { + var chagas = ontologyClass("MONDO:0005491", "Chagas cardiomyopathy"); + var nyha3 = ontologyClass("NCIT:C66907", "New York Heart Association Class III"); + var childhood = TimeElement.newBuilder().setOntologyClass(CHILDHOOD_ONSET).build(); + return Disease.newBuilder() + .setTerm(chagas) + .setOnset(childhood) + .addDiseaseStage(nyha3) + .build(); + } + + private static Phenopacket phenopacketWithDisease() { + Resource hpo = hpoResource("2021-08-02"); + Resource mondo = mondoResource("2021-09-01"); + MetaData meta = PhenopacketUtil.MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") + .submittedBy("anonymous submitter") + .addResource(hpo) + .addResource(mondo) + .addExternalReference("PMID:20842687", + "Severe dystonic encephalopathy without hyperphenylalaninemia associated with an 18-bp deletion within the proximal GCH1 promoter") + .build(); + Disease chagasCardiomyopathy = mondoDisease(); + return Phenopacket.newBuilder() + .setId("A") + .addDiseases(chagasCardiomyopathy) + .setMetaData(meta) + .build(); + } + + private static final Phenopacket phenopacket = phenopacketWithDisease(); + + @Test + public void testPhenopacketValidity() throws InvalidProtocolBufferException { + JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.generic()).get(); + String json = JsonFormat.printer().print(phenopacket); + List errors = validator.validate(json); + assertTrue(errors.isEmpty()); + } + + @Test + public void testLacksId() throws InvalidProtocolBufferException { + JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.generic()).get(); + // the Phenopacket is not valid if we remove the id + Phenopacket p1 = Phenopacket.newBuilder(phenopacket).clearId().build(); + String json = JsonFormat.printer().print(p1); + System.out.println(json); + List errors = validator.validate(json); + assertEquals(1, errors.size()); + ValidationItem error = errors.get(0); + assertEquals(ErrorType.JSON_REQUIRED, error.errorType()); + assertEquals("$.id: is missing but it is required", error.message()); + } + + + +} diff --git a/validator-jsonschema/src/test/resources/json/invalidSimplePhenopacket.json b/validator-jsonschema/src/test/resources/json/invalidSimplePhenopacket.json deleted file mode 100644 index 8258e4e..0000000 --- a/validator-jsonschema/src/test/resources/json/invalidSimplePhenopacket.json +++ /dev/null @@ -1 +0,0 @@ -{"disney" : "donald"} \ No newline at end of file diff --git a/validator-jsonschema/src/test/resources/json/validSimplePhenopacket.json b/validator-jsonschema/src/test/resources/json/validSimplePhenopacket.json deleted file mode 100644 index eb6c3bb..0000000 --- a/validator-jsonschema/src/test/resources/json/validSimplePhenopacket.json +++ /dev/null @@ -1,16 +0,0 @@ -{"id" : "world", - "metaData": { - "created": "2021-07-01T19:32:35Z", - "createdBy": "Hpo Case Annotator : 1.0.15-SNAPSHOT", - "submittedBy": "HPO:probinson", - "resources": [{ - "id": "hp", - "name": "human phenotype ontology", - "url": "http://purl.obolibrary.org/obo/hp.owl", - "version": "2018-03-08", - "namespacePrefix": "HP", - "iriPrefix": "http://purl.obolibrary.org/obo/HP_" - }], - "phenopacketSchemaVersion": "2.0" - } -} \ No newline at end of file From 1602ee1c2df69f1888e1f3eeae70163345a9394c Mon Sep 17 00:00:00 2001 From: Daniel Danis Date: Fri, 17 Sep 2021 11:52:52 -0400 Subject: [PATCH 11/37] Clean up commented code --- .../validator/jsonschema/JsonSchemaDiseaseValidatorTest.java | 1 - .../validator/jsonschema/JsonSchemaValidatorTest.java | 2 -- 2 files changed, 3 deletions(-) diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java index d850f9b..dc94360 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java @@ -14,7 +14,6 @@ import org.phenopackets.validator.core.ValidatorInfo; import org.phenopackets.validator.testdatagen.PhenopacketUtil; -import java.io.IOException; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java index 5bdb878..efa1bf3 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java @@ -76,8 +76,6 @@ public void testValidationOfSimpleInValidPhenopacket() throws Exception { public void testRareDiseaseBethlemahmValidPhenopacket() throws Exception { JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.rareDiseaseValidation()).get(); - // File myopathyPhenopacket = fileFromClasspath("json/bethlehamMyopathyExample.json"); - Phenopacket bethlehamMyopathy = rareDiseasePhenopacket.getPhenopacket(); String json = JsonFormat.printer().print(bethlehamMyopathy); List errors = validator.validate(json); From 4de85cac43e0c9e8771341717d23cf36318d99bd Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Wed, 22 Sep 2021 15:24:51 -0400 Subject: [PATCH 12/37] correcting params --- .../validator/cli/ValidateCommand.java | 63 ++++++++++--------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidateCommand.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidateCommand.java index c9cdc4e..09310f5 100644 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidateCommand.java +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidateCommand.java @@ -26,15 +26,14 @@ public class ValidateCommand implements Callable { private static final Logger LOGGER = LoggerFactory.getLogger(ValidateCommand.class); - // TODO - do we need this as we accept 1..* phenopacket as positional arguments? -// @Option(names = {"-p","--phenopacket"}, description = "Path to phenopacket", required = true) -// public String phenopacketPath; - @Option(names = "--rare", description = "apply HPO rare-disease constraints") public boolean rareHpoConstraints = false; - @Parameters(arity = "1..*", description = "one or more phenopacket files") - public List phenopackets; + @Parameters(arity = "0..*", description = "one or more JSON Schema configuration files") + public List jsonSchemaFiles = List.of(); + + @Option(names = {"-p", "--phenopacket"}, required = true, description = "Phenopacket file to be validated") + public String phenopacket; private final PhenopacketValidatorFactory phenopacketValidatorFactory; @@ -51,37 +50,41 @@ public Integer call() { LOGGER.info("Validating with HPO rare-disease constraints"); validationTypes.add(ValidatorInfo.rareDiseaseValidation()); } + File phenopacketFile = new File(phenopacket); + LOGGER.info("Validating {} phenopacket", phenopacketFile); + // TODO -- adapt PhenopacketValidatorFactory to accept multiple JSON Schema files + for (File jsonSchema : jsonSchemaFiles) { + LOGGER.info("Adding configuration file at `{}`", jsonSchema.getAbsolutePath()); + } - LOGGER.info("Validating {} phenopackets", phenopackets.size()); - - + // poor man's formatting + LOGGER.info(""); + LOGGER.info("--------------------------------------------------------------------------------"); + LOGGER.info(""); PhenopacketValidation validator = new PhenopacketValidation(phenopacketValidatorFactory); - - for (File phenopacket : phenopackets) { - LOGGER.info("Validating phenopacket at `{}`", phenopacket.getAbsolutePath()); - - try (InputStream in = Files.newInputStream(phenopacket.toPath())) { - - List validationItems = validator.validate(in, validationTypes.toArray(ValidatorInfo[]::new)); - if (validationItems.isEmpty()) { - LOGGER.info("No errors found"); - } else { - LOGGER.info("Found {} errors:", validationItems.size()); - for (ValidationItem item : validationItems) { - LOGGER.info("({}) {}", item.errorType(), item.message()); - } + try (InputStream in = Files.newInputStream(phenopacketFile.toPath())) { + + List validationItems = validator.validate(in, validationTypes.toArray(ValidatorInfo[]::new)); + if (validationItems.isEmpty()) { + LOGGER.info("No errors found"); + } else { + LOGGER.info("Found {} errors:", validationItems.size()); + for (ValidationItem item : validationItems) { + LOGGER.info("({}) {}", item.errorType(), item.message()); } - - } catch (IOException e) { - LOGGER.warn("Error opening the phenopacket", e); } - // poor man's formatting - LOGGER.info(""); - LOGGER.info("--------------------------------------------------------------------------------"); - LOGGER.info(""); + } catch (IOException e) { + LOGGER.warn("Error opening the phenopacket", e); } + // poor man's formatting + LOGGER.info(""); + LOGGER.info("--------------------------------------------------------------------------------"); + LOGGER.info(""); + + + return 0; } From b4176cdbf403804a0d08117dfa855a556666d9bb Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Wed, 22 Sep 2021 15:35:47 -0400 Subject: [PATCH 13/37] fix JSON schema for vitalStatus --- .../org/phenopackets/validator/core/ErrorType.java | 3 ++- .../resources/schema/phenopacket-general-schema.json | 10 +++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java b/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java index 0b45d0f..e170b68 100644 --- a/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java @@ -9,6 +9,7 @@ public enum ErrorType { JSON_REQUIRED("required"), /** The type of an object is not as required by the schema, e.g., we get a string instead of an array. */ JSON_TYPE("type"), + JSON_ENUM("todo"), PHENOPACKET_SUBJECT_LACKS_AGE("phenopacket subject lacks age"), PHENOPACKET_LACKS_SUBJECT("phenopacket lacks subject"), INVALID_ONTOLOGY("invalid ontology"), @@ -32,7 +33,7 @@ public static ErrorType stringToErrorType(String error) { case "additionalProperties": return JSON_ADDITIONAL_PROPERTIES; case "required": return JSON_REQUIRED; case "type": return JSON_TYPE; - + case "enum": return JSON_TYPE; default: throw new PhenopacketValidatorRuntimeException("Did not recognize error type: \"" + error + "\""); } diff --git a/validator-core/src/main/resources/schema/phenopacket-general-schema.json b/validator-core/src/main/resources/schema/phenopacket-general-schema.json index 834d59b..b0df498 100644 --- a/validator-core/src/main/resources/schema/phenopacket-general-schema.json +++ b/validator-core/src/main/resources/schema/phenopacket-general-schema.json @@ -32,7 +32,15 @@ "$ref": "#/definitions/timeElement" }, "vitalStatus": { - "enum": ["UNKNOWN_STATUS", "ALIVE", "DECEASED"] + "type": "object", + "properties": { + "status": {"enum": ["UNKNOWN_STATUS", "ALIVE", "DECEASED"]}, + "timeOfDeath": {"$ref": "#/definitions/timeElement"}, + "causeOfDeath": {"$ref": "#/definitions/ontologyClass"}, + "survivalTimeInDays": "integer" + }, + "required": [ "status"], + "additionalProperties": false }, "sex": { "enum": ["UNKNOWN_SEX", "FEMALE", "MALE"] From 7d5140341235ef008bb5114ef41cfd9b9a838a70 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Wed, 22 Sep 2021 16:17:17 -0400 Subject: [PATCH 14/37] fix JSON schema for timeElement --- .../main/resources/schema/phenopacket-general-schema.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/validator-core/src/main/resources/schema/phenopacket-general-schema.json b/validator-core/src/main/resources/schema/phenopacket-general-schema.json index b0df498..1f66fd7 100644 --- a/validator-core/src/main/resources/schema/phenopacket-general-schema.json +++ b/validator-core/src/main/resources/schema/phenopacket-general-schema.json @@ -187,20 +187,22 @@ } }, "timeElement": { + "type": "object", "properties": { "gestationalAge": {"$ref": "#/definitions/gestationalAge"}, "age" : {"$ref": "#/definitions/age"}, "ageRange": { "$ref": "#/definitions/ageRange"}, "ontologyClass": { "$ref": "#/definitions/ontologyClass"}, "timestamp": { "type": "string", "format": "date-time"}, - "timeInterval": {"$ref": "#/definitions/timeInterval"} + "interval": {"$ref": "#/definitions/timeInterval"} }, + "additionalProperties": false, "oneOf": [ { "required": [ "gestationalAge" ]}, { "required": [ "age" ]}, { "required": [ "ageRange"]}, { "required": [ "ontologyClass" ]}, { "required": [ "timestamp" ]}, - { "required": [ "timeInterval" ]} + { "required": [ "interval" ]} ] }, "update": { From 89e64bfcfb9f059e0f6ea59c8608ea30c59d5ae4 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Sun, 26 Sep 2021 12:47:59 -0400 Subject: [PATCH 15/37] fixed disease-stage definition --- .../phenopackets/validator/core/DefaultValidationInfo.java | 2 +- .../org/phenopackets/validator/core/ValidationAspect.java | 4 ++++ .../src}/resources/schema/hpo-rare-disease-schema.json | 0 .../src/resources/schema/phenopacket-generic.json | 5 ++++- 4 files changed, 9 insertions(+), 2 deletions(-) rename {validator-core/src/main => validator-jsonschema/src}/resources/schema/hpo-rare-disease-schema.json (100%) rename validator-core/src/main/resources/schema/phenopacket-general-schema.json => validator-jsonschema/src/resources/schema/phenopacket-generic.json (99%) diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidationInfo.java b/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidationInfo.java index 76ae74f..15d8285 100644 --- a/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidationInfo.java +++ b/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidationInfo.java @@ -4,7 +4,7 @@ class DefaultValidationInfo implements ValidatorInfo { - private static final DefaultValidationInfo GENERIC = of("GENERIC", "Validation of a generic Phenopacket"); + private static final DefaultValidationInfo GENERIC = of("GENERIC", "validation with generic JSON Schema"); private static final DefaultValidationInfo RARE_DISEASE_VALIDATOR = of("RARE_DISEASE_VALIDATOR", "Validation of rare disease Phenopacket constraints"); static ValidatorInfo generic() { diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationAspect.java b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationAspect.java index a5942c7..2c25c9b 100644 --- a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationAspect.java +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationAspect.java @@ -1,6 +1,10 @@ package org.phenopackets.validator.core; +/* +Probably this should go into error type, they have the same purpose + */ +@Deprecated public enum ValidationAspect { // This enum is supposed to be used instead of the ErrorType diff --git a/validator-core/src/main/resources/schema/hpo-rare-disease-schema.json b/validator-jsonschema/src/resources/schema/hpo-rare-disease-schema.json similarity index 100% rename from validator-core/src/main/resources/schema/hpo-rare-disease-schema.json rename to validator-jsonschema/src/resources/schema/hpo-rare-disease-schema.json diff --git a/validator-core/src/main/resources/schema/phenopacket-general-schema.json b/validator-jsonschema/src/resources/schema/phenopacket-generic.json similarity index 99% rename from validator-core/src/main/resources/schema/phenopacket-general-schema.json rename to validator-jsonschema/src/resources/schema/phenopacket-generic.json index 1f66fd7..24aeb8b 100644 --- a/validator-core/src/main/resources/schema/phenopacket-general-schema.json +++ b/validator-jsonschema/src/resources/schema/phenopacket-generic.json @@ -477,7 +477,10 @@ "$ref": "#/definitions/timeElement" }, "diseaseStage": { - "$ref": "#/definitions/ontologyClass" + "type": "array", + "items": { + "$ref": "#/definitions/ontologyClass" + } }, "clinicalTnmFinding": { "$ref": "#/definitions/ontologyClass" From 52b6c5ed383d479ecaacb6ac89d83a7061bef901 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Sun, 26 Sep 2021 12:48:27 -0400 Subject: [PATCH 16/37] prototype cli app --- validator-cli/pom.xml | 17 +-- .../validator/cli/ValidatorApplication.java | 103 +++++++++++++++--- .../results/ValidationItemTsvVisualizer.java | 23 ++++ .../cli/results/ValidationTsvVisualizer.java | 50 +++++++++ .../ClasspathJsonSchemaValidatorFactory.java | 15 ++- .../JsonSchemaDiseaseValidatorTest.java | 15 ++- .../jsonschema/JsonSchemaValidatorTest.java | 33 +++--- 7 files changed, 209 insertions(+), 47 deletions(-) create mode 100644 validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationItemTsvVisualizer.java create mode 100644 validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationTsvVisualizer.java diff --git a/validator-cli/pom.xml b/validator-cli/pom.xml index 8109704..b47f307 100644 --- a/validator-cli/pom.xml +++ b/validator-cli/pom.xml @@ -21,19 +21,12 @@ validator-jsonschema ${project.parent.version} - - org.springframework.boot - spring-boot-autoconfigure - - - ch.qos.logback - logback-classic - info.picocli - picocli-spring-boot-starter + picocli 4.6.1 + @@ -43,12 +36,6 @@ true - - - org.springframework.boot - spring-boot-maven-plugin - - diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java index bca5e5e..3afc5f2 100644 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java @@ -1,32 +1,107 @@ package org.phenopackets.validator.cli; -import org.phenopackets.validator.core.PhenopacketValidatorFactory; +import org.phenopackets.validator.cli.results.ValidationItemTsvVisualizer; +import org.phenopackets.validator.cli.results.ValidationTsvVisualizer; +import org.phenopackets.validator.core.PhenopacketValidator; +import org.phenopackets.validator.core.ValidationItem; +import org.phenopackets.validator.core.ValidatorInfo; import org.phenopackets.validator.jsonschema.ClasspathJsonSchemaValidatorFactory; +import org.phenopackets.validator.jsonschema.JsonSchemaValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.annotation.Bean; +import picocli.CommandLine; +import java.io.*; +import java.nio.file.Files; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; -/** - * @author Jules Jacobsen - */ -@SpringBootApplication -public class ValidatorApplication { +@CommandLine.Command(name = "PhenopacketValidator", version = "0.1.0", mixinStandardHelpOptions = true) +public class ValidatorApplication implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(ValidatorApplication.class); + @CommandLine.Option(names = "--rare", description = "apply HPO rare-disease constraints") + public boolean rareHpoConstraints = false; + + @CommandLine.Parameters(arity = "0..*", description = "one or more JSON Schema configuration files") + public List jsonSchemaFiles = List.of(); + + @CommandLine.Option(names = {"-p", "--phenopacket"}, required = true, description = "Phenopacket file to be validated") + public String phenopacket; + + @CommandLine.Option(names={"-o", "--out"}, description = "name of output file (default ${DEFAULT_VALUE})") + public String outfileName = "phenopacket-validation.tsv"; + + private static final String CUSTOM_JSON_VALIDATOR_TYPE = "Custom JSON Schema validation"; + + public static void main(String[] args) { LOGGER.info("STARTING THE APPLICATION"); - SpringApplication.run(ValidatorApplication.class, args); + int exitCode = new CommandLine(new ValidatorApplication()).execute(args); + System.exit(exitCode); LOGGER.info("APPLICATION FINISHED"); } - @Bean - public PhenopacketValidatorFactory phenopacketValidatorFactory() { - // TODO - for development only... - return ClasspathJsonSchemaValidatorFactory.defaultValidators(); + + + @Override + public void run() { + List validationTypes = new LinkedList<>(); + validationTypes.add(ValidatorInfo.generic()); // we run this by default + if (rareHpoConstraints) { + LOGGER.info("Validating with HPO rare-disease constraints"); + validationTypes.add(ValidatorInfo.rareDiseaseValidation()); + } + File phenopacketFile = new File(phenopacket); + LOGGER.info("Validating {} phenopacket", phenopacketFile); + // TODO -- adapt PhenopacketValidatorFactory to accept multiple JSON Schema files + Map jsonValidatorMap = ClasspathJsonSchemaValidatorFactory.genericValidator(); + for (File jsonSchema : jsonSchemaFiles) { + // we will create ValidatorInfo objects based on the names and paths of the files. + String baseName = jsonSchema.getName(); + ValidatorInfo vinfo = ValidatorInfo.of(CUSTOM_JSON_VALIDATOR_TYPE, baseName); + JsonSchemaValidator jvalid = JsonSchemaValidator.of(jsonSchema, vinfo); + jsonValidatorMap.put(vinfo, jvalid); + LOGGER.info("Adding configuration file at `{}`", vinfo); + } + // poor man's formatting + LOGGER.info(""); + LOGGER.info("--------------------------------------------------------------------------------"); + LOGGER.info(""); + ValidationTsvVisualizer resultVisualizer = new ValidationTsvVisualizer(); + try (InputStream in = Files.newInputStream(phenopacketFile.toPath())) { + for (var e : jsonValidatorMap.entrySet()) { + ValidatorInfo vinfo = e.getKey(); + PhenopacketValidator validator = e.getValue(); + List validationItems = validator.validate(phenopacketFile); + if (validationItems.isEmpty()) { + LOGGER.info("No errors found"); + resultVisualizer.errorFree(vinfo); + } else { + LOGGER.info("Found {} errors:", validationItems.size()); + for (ValidationItem item : validationItems) { + LOGGER.info("({}) {}", item.errorType(), item.message()); + resultVisualizer.error(vinfo, new ValidationItemTsvVisualizer(item)); + } + } + } + } catch (IOException e) { + LOGGER.warn("Error opening the phenopacket", e); + } + + // poor man's formatting + LOGGER.info(""); + LOGGER.info("--------------------------------------------------------------------------------"); + LOGGER.info(""); + + // TSV output + try (BufferedWriter bw = new BufferedWriter(new FileWriter(outfileName))) { + resultVisualizer.write(bw); + } catch (IOException e) { + LOGGER.warn("Error writing the validation results", e); + } } } diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationItemTsvVisualizer.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationItemTsvVisualizer.java new file mode 100644 index 0000000..5454aa4 --- /dev/null +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationItemTsvVisualizer.java @@ -0,0 +1,23 @@ +package org.phenopackets.validator.cli.results; + +import org.phenopackets.validator.core.ValidationItem; + +import java.util.List; + +/** + * Visualizer for validationItems + */ +public class ValidationItemTsvVisualizer { + private final ValidationItem error; + public ValidationItemTsvVisualizer(ValidationItem item) { + this.error = item; + } + + /** + * Get a list of fields for display + * @return + */ + public List getFields() { + return List.of(error.errorType().name(), error.message(), error.validatorInfo().validatorId()); + } +} diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationTsvVisualizer.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationTsvVisualizer.java new file mode 100644 index 0000000..1aebad2 --- /dev/null +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationTsvVisualizer.java @@ -0,0 +1,50 @@ +package org.phenopackets.validator.cli.results; + +import org.phenopackets.validator.core.ValidatorInfo; + +import java.io.IOException; +import java.io.Writer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ValidationTsvVisualizer { + + private final List errorFreeValidations; + private final Map errors; + + + public ValidationTsvVisualizer() { + errorFreeValidations = new ArrayList<>(); + errors = new HashMap<>(); + } + + /** + * Add a ValidationInfo (i.e., validation type) for which no errors were found. + * @param vinfo + */ + public void errorFree(ValidatorInfo vinfo) { + errorFreeValidations.add(vinfo); + } + + public void error(ValidatorInfo vinfo, ValidationItemTsvVisualizer result) { + errors.put(vinfo, result); + } + + + + private static String tsvHeader() { + return String.join("\t", List.of("Error Type", "Message", "Validator Id")); + } + + + public void write(Writer writer) throws IOException { + writer.write(tsvHeader() + "\n"); + for (var result : this.errors.values()) { + writer.write(String.join("\t",result.getFields()) + "\n"); + } + } + + +} diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/ClasspathJsonSchemaValidatorFactory.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/ClasspathJsonSchemaValidatorFactory.java index 6db9516..22e2896 100644 --- a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/ClasspathJsonSchemaValidatorFactory.java +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/ClasspathJsonSchemaValidatorFactory.java @@ -19,6 +19,7 @@ public class ClasspathJsonSchemaValidatorFactory implements PhenopacketValidator private final Map validatorMap; + /* public static ClasspathJsonSchemaValidatorFactory defaultValidators() { Map validatorMap = makeValidatorMap(); return new ClasspathJsonSchemaValidatorFactory(validatorMap); @@ -26,9 +27,21 @@ public static ClasspathJsonSchemaValidatorFactory defaultValidators() { private static Map makeValidatorMap() { return Map.of( - ValidatorInfo.generic(), makeJsonValidator("/schema/phenopacket-general-schema.json", ValidatorInfo.generic()), + ValidatorInfo.generic(), makeJsonValidator("/schema/phenopacket-generic.json", ValidatorInfo.generic()), ValidatorInfo.rareDiseaseValidation(), makeJsonValidator("/schema/hpo-rare-disease-schema.json", ValidatorInfo.rareDiseaseValidation()) ); + } */ + + public static Map genericValidator() { + return Map.of( + ValidatorInfo.generic(), makeJsonValidator("/schema/phenopacket-generic.json", ValidatorInfo.generic()) + ); + } + + public static Map rareHpoValidator() { + return Map.of( + ValidatorInfo.generic(), makeJsonValidator("/schema/hpo-rare-disease-schema.json", ValidatorInfo.generic()) + ); } private static JsonSchemaValidator makeJsonValidator(String schemaPath, ValidatorInfo validationName) { diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java index dc94360..391cf00 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java @@ -15,6 +15,7 @@ import org.phenopackets.validator.testdatagen.PhenopacketUtil; import java.util.List; +import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -27,8 +28,7 @@ */ public class JsonSchemaDiseaseValidatorTest { - private static final ClasspathJsonSchemaValidatorFactory FACTORY = ClasspathJsonSchemaValidatorFactory.defaultValidators(); - + private static final Map jsonValidatorMap = ClasspathJsonSchemaValidatorFactory.genericValidator(); private static Disease mondoDisease() { var chagas = ontologyClass("MONDO:0005491", "Chagas cardiomyopathy"); @@ -63,7 +63,9 @@ private static Phenopacket phenopacketWithDisease() { @Test public void testPhenopacketValidity() throws InvalidProtocolBufferException { - JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.generic()).get(); + JsonSchemaValidator validator = jsonValidatorMap.values().stream() + .findFirst() + .get(); String json = JsonFormat.printer().print(phenopacket); List errors = validator.validate(json); assertTrue(errors.isEmpty()); @@ -71,12 +73,17 @@ public void testPhenopacketValidity() throws InvalidProtocolBufferException { @Test public void testLacksId() throws InvalidProtocolBufferException { - JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.generic()).get(); + JsonSchemaValidator validator = jsonValidatorMap.values().stream() + .findFirst() + .get(); // the Phenopacket is not valid if we remove the id Phenopacket p1 = Phenopacket.newBuilder(phenopacket).clearId().build(); String json = JsonFormat.printer().print(p1); System.out.println(json); List errors = validator.validate(json); + for (var e: errors) { + System.out.println(e); + } assertEquals(1, errors.size()); ValidationItem error = errors.get(0); assertEquals(ErrorType.JSON_REQUIRED, error.errorType()); diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java index efa1bf3..561d960 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java @@ -1,6 +1,5 @@ package org.phenopackets.validator.jsonschema; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.phenopackets.schema.v2.Phenopacket; import org.phenopackets.validator.testdatagen.RareDiseasePhenopacket; @@ -12,6 +11,7 @@ import java.io.File; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Objects; import com.google.protobuf.util.JsonFormat; @@ -20,7 +20,8 @@ public class JsonSchemaValidatorTest { - private static final ClasspathJsonSchemaValidatorFactory FACTORY = ClasspathJsonSchemaValidatorFactory.defaultValidators(); + private static final Map genericValidatorMap = ClasspathJsonSchemaValidatorFactory.genericValidator(); + private static final Map rareHpoValidatorMap = ClasspathJsonSchemaValidatorFactory.rareHpoValidator(); private static final SimplePhenopacket simplePhenopacket = new SimplePhenopacket(); @@ -33,7 +34,9 @@ private static File fileFromClasspath(String path) { @Test public void testValidationOfSimpleValidPhenopacket() throws Exception { - JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.generic()).get(); + JsonSchemaValidator validator = genericValidatorMap.values().stream() + .findFirst() + .get(); Phenopacket phenopacket = simplePhenopacket.getPhenopacket(); String json = JsonFormat.printer().print(phenopacket); List errors = validator.validate(json); @@ -54,7 +57,9 @@ public void testValidationOfSimpleValidPhenopacket() throws Exception { */ @Test public void testValidationOfSimpleInValidPhenopacket() throws Exception { - JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.generic()).get(); + JsonSchemaValidator validator = genericValidatorMap.values().stream() + .findFirst() + .get(); String invalidPhenopacketJson = "{\"disney\" : \"donald\"}"; @@ -74,7 +79,9 @@ public void testValidationOfSimpleInValidPhenopacket() throws Exception { @Test public void testRareDiseaseBethlemahmValidPhenopacket() throws Exception { - JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.rareDiseaseValidation()).get(); + JsonSchemaValidator validator = rareHpoValidatorMap.values().stream() + .findFirst() + .get(); Phenopacket bethlehamMyopathy = rareDiseasePhenopacket.getPhenopacket(); String json = JsonFormat.printer().print(bethlehamMyopathy); @@ -84,19 +91,19 @@ public void testRareDiseaseBethlemahmValidPhenopacket() throws Exception { } @Test - @Disabled // TODO - we should rework the testing strategy to invalidate a valid phenopacket and check that it raises the expected error public void testRareDiseaseBethlemahmInvalidValidPhenopacket() throws IOException { - JsonSchemaValidator validator = FACTORY.getValidatorForType(ValidatorInfo.rareDiseaseValidation()).get(); - + JsonSchemaValidator validator = rareHpoValidatorMap.values().stream() + .findFirst() + .get(); File invalidMyopathyPhenopacket = fileFromClasspath("json/bethlehamMyopathyInvalidExample.json"); List errors = validator.validate(invalidMyopathyPhenopacket); -// for (ValidationError ve : errors) { -// System.out.println(ve.getMessage()); -// } + for (ValidationItem ve : errors) { + System.out.println(ve.message()); + } assertEquals(1, errors.size()); ValidationItem error = errors.get(0); - assertEquals(ErrorType.JSON_ADDITIONAL_PROPERTIES, error.errorType()); - assertEquals("$.phenotypicFeaturesMALFORMED: is not defined in the schema and the schema does not allow additional properties", error.message()); + assertEquals(ErrorType.JSON_REQUIRED, error.errorType()); + assertEquals("$.phenotypicFeatures: is missing but it is required", error.message()); } From 7a13c78c2c5a6972ae29a648f4edde23351179ed Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Sun, 26 Sep 2021 15:10:30 -0400 Subject: [PATCH 17/37] prototype OntologyValidator --- pom.xml | 1 + .../validator/cli/PhenopacketValidation.java | 134 ----------------- .../validator/cli/ValidateCommand.java | 91 ------------ .../cli/ValidatorApplicationRunner.java | 38 ----- .../validator/core/ErrorType.java | 2 + .../jsonschema/JsonSchemaValidator.java | 2 +- ...tionError.java => JsonValidationItem.java} | 6 +- validator-ontology/pom.xml | 140 ++++++++++++++++++ .../validator/ontology/HpoValidator.java | 105 +++++++++++++ .../ontology/OntologyValidationItem.java | 44 ++++++ .../validator/ontology/OntologyValidator.java | 29 ++++ .../ontology/OntologyValidatorFactory.java | 35 +++++ 12 files changed, 360 insertions(+), 267 deletions(-) delete mode 100644 validator-cli/src/main/java/org/phenopackets/validator/cli/PhenopacketValidation.java delete mode 100644 validator-cli/src/main/java/org/phenopackets/validator/cli/ValidateCommand.java delete mode 100644 validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplicationRunner.java rename validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/{JsonValidationError.java => JsonValidationItem.java} (87%) create mode 100644 validator-ontology/pom.xml create mode 100644 validator-ontology/src/main/java/org/phenopackets/validator/ontology/HpoValidator.java create mode 100644 validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItem.java create mode 100644 validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidator.java create mode 100644 validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidatorFactory.java diff --git a/pom.xml b/pom.xml index 30fea66..f790f70 100644 --- a/pom.xml +++ b/pom.xml @@ -14,6 +14,7 @@ validator-core validator-cli validator-jsonschema + validator-ontology diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/PhenopacketValidation.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/PhenopacketValidation.java deleted file mode 100644 index 75bea59..0000000 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/PhenopacketValidation.java +++ /dev/null @@ -1,134 +0,0 @@ -package org.phenopackets.validator.cli; - -import org.phenopackets.validator.core.PhenopacketValidator; -import org.phenopackets.validator.core.PhenopacketValidatorFactory; -import org.phenopackets.validator.core.ValidationItem; -import org.phenopackets.validator.core.ValidatorInfo; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.util.LinkedList; -import java.util.List; -import java.util.function.Function; - -/** - * Entry point for the library. Performs JSON schema validation. If an additional config file is provided, it performs - * additional validation. Several default validation files are provided that can be used with command line flags. - * For instance, adding the --rare flag will additionally apply the {@code hpo-rare-disease-schema.json} specification. - * @author Peter N Robinson - */ -public class PhenopacketValidation { - - private static final Logger LOGGER = LoggerFactory.getLogger(PhenopacketValidation.class); - - private final PhenopacketValidatorFactory validatorFactory; - - public PhenopacketValidation(PhenopacketValidatorFactory validatorFactory) { - this.validatorFactory = validatorFactory; - } - -// List validationErrors; - -// -// /** Can be used to validate a Phenopacket against the built-in additional JSON-Schema files, -// * including HPO-rare disease TODO add another two examples. -// * @param phenopacketPath Path to the phenopacket file -// * @param validationType One of the built-in types, see {@link ValidationType}. -// */ -// public PhenopacketValidation(String phenopacketPath, ValidationType validationType) { -// phenopacket = initPhenopacketFile(phenopacketPath); -// validationErrors = genericValidation(); -// -// switch (validationType) { -// case RARE_DISEASE_VALIDATION: -// List specErrors = hpoRareDiseaseValidation(); -// validationErrors.addAll(specErrors); -// break; -// } -// } -// -// /** -// * This constructor can be used with one or more user defined JSON spec files that are passed from the command line -// * @param phenopacketPath Path to the phenopacket file -// * @param jsonSchemPaths Paths to user-defined JSON Schema files -// */ -// public PhenopacketValidation(String phenopacketPath, String... jsonSchemPaths) { -// phenopacket = initPhenopacketFile(phenopacketPath); -// validationErrors = genericValidation(); -// for (String jsonPath : jsonSchemPaths) { -// List specErrors = additionalValidation(jsonPath); -// validationErrors.addAll(specErrors); -// } -// } -// -// private List additionalValidation(String jsonPath) { -// File jsonFile = new File(jsonPath); -// if (! jsonFile.isFile()) { -// throw new PhenopacketValidatorRuntimeException("Could not find input json file \"" + jsonFile.getAbsolutePath() + "\""); -// } -// JsonSchemaValidator validator = new AdditionalJsonFileJsonSchemaValidator(phenopacket, jsonFile); -// return validator.validate(); -// } - - public List validate(InputStream inputStream, ValidatorInfo... validations) { - List items = new LinkedList<>(); - try { - byte[] content = inputStream.readAllBytes(); - for (ValidatorInfo validationType : validations) { - validatorFactory.getValidatorForType(validationType) - .map(validate(content)) - .ifPresent(items::addAll); - } - return List.copyOf(items); - - } catch (IOException e) { - LOGGER.warn("Error occurred during validation {}", e.getMessage(), e); - return List.of(); - } - } - - private static Function> validate(byte[] content) { - return validator -> { - try (InputStream is = new ByteArrayInputStream(content)) { - return validator.validate(is); - } catch (IOException e) { - LOGGER.warn("Error occurred during validation {}", e.getMessage(), e); - return List.of(); - } - }; - } - -// TODO - not relevant here -// /** -// * Create a file object and check that the file exists -// * @param phenopacketPath String to the input phenopacket file -// * @return File object representing the input phenopacket -// */ -// File initPhenopacketFile(String phenopacketPath) { -// File f = new File(phenopacketPath); -// if (! f.isFile()) { -// throw new PhenopacketValidatorRuntimeException("Could not find file \"" + phenopacketPath + "\""); -// } -// return f; -// } -// -// /** -// * Validate the Phenopacket file against the generic JSON Schema -// */ -// private List genericValidation() { -// JsonSchemaValidator jsonSchemaValidator = new JsonSchemaValidator(phenopacket); -// return jsonSchemaValidator.validate(); -// } -// /** -// * Validate the Phenopacket file for HPO-rare disease specific requirements. -// */ -// private List hpoRareDiseaseValidation() { -// JsonSchemaValidator hpoRareDiseaseValidator = new HpoRareDiseaseJsonSchemaValidator(phenopacket); -// return hpoRareDiseaseValidator.validate(); -// } -// -// public List getValidationErrors() { -// return validationErrors; -// } -} diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidateCommand.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidateCommand.java deleted file mode 100644 index 09310f5..0000000 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidateCommand.java +++ /dev/null @@ -1,91 +0,0 @@ -package org.phenopackets.validator.cli; - - -import org.phenopackets.validator.core.ValidationItem; -import org.phenopackets.validator.core.ValidatorInfo; -import org.phenopackets.validator.core.PhenopacketValidatorFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; -import picocli.CommandLine.Parameters; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.Callable; - -@Component -@Command(name = "validate", - mixinStandardHelpOptions = true) -public class ValidateCommand implements Callable { - - private static final Logger LOGGER = LoggerFactory.getLogger(ValidateCommand.class); - - @Option(names = "--rare", description = "apply HPO rare-disease constraints") - public boolean rareHpoConstraints = false; - - @Parameters(arity = "0..*", description = "one or more JSON Schema configuration files") - public List jsonSchemaFiles = List.of(); - - @Option(names = {"-p", "--phenopacket"}, required = true, description = "Phenopacket file to be validated") - public String phenopacket; - - private final PhenopacketValidatorFactory phenopacketValidatorFactory; - - public ValidateCommand(PhenopacketValidatorFactory phenopacketValidatorFactory) { - this.phenopacketValidatorFactory = phenopacketValidatorFactory; - } - - @Override - public Integer call() { - // What type of validation do we run? - List validationTypes = new LinkedList<>(); - validationTypes.add(ValidatorInfo.generic()); // we run this by default - if (rareHpoConstraints) { - LOGGER.info("Validating with HPO rare-disease constraints"); - validationTypes.add(ValidatorInfo.rareDiseaseValidation()); - } - File phenopacketFile = new File(phenopacket); - LOGGER.info("Validating {} phenopacket", phenopacketFile); - // TODO -- adapt PhenopacketValidatorFactory to accept multiple JSON Schema files - for (File jsonSchema : jsonSchemaFiles) { - LOGGER.info("Adding configuration file at `{}`", jsonSchema.getAbsolutePath()); - } - - // poor man's formatting - LOGGER.info(""); - LOGGER.info("--------------------------------------------------------------------------------"); - LOGGER.info(""); - PhenopacketValidation validator = new PhenopacketValidation(phenopacketValidatorFactory); - try (InputStream in = Files.newInputStream(phenopacketFile.toPath())) { - - List validationItems = validator.validate(in, validationTypes.toArray(ValidatorInfo[]::new)); - if (validationItems.isEmpty()) { - LOGGER.info("No errors found"); - } else { - LOGGER.info("Found {} errors:", validationItems.size()); - for (ValidationItem item : validationItems) { - LOGGER.info("({}) {}", item.errorType(), item.message()); - } - } - - } catch (IOException e) { - LOGGER.warn("Error opening the phenopacket", e); - } - - // poor man's formatting - LOGGER.info(""); - LOGGER.info("--------------------------------------------------------------------------------"); - LOGGER.info(""); - - - - return 0; - } - -} \ No newline at end of file diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplicationRunner.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplicationRunner.java deleted file mode 100644 index 46e9005..0000000 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplicationRunner.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.phenopackets.validator.cli; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import picocli.CommandLine; -import picocli.CommandLine.IFactory; - -import org.springframework.boot.CommandLineRunner; -import org.springframework.boot.ExitCodeGenerator; -import org.springframework.stereotype.Component; - -@Component -public class ValidatorApplicationRunner implements CommandLineRunner, ExitCodeGenerator { - private static Logger LOG = LoggerFactory - .getLogger(ValidatorApplicationRunner.class); - private final ValidateCommand myCommand; - - private final IFactory factory; // auto-configured to inject PicocliSpringFactory - - private int exitCode; - - public ValidatorApplicationRunner(ValidateCommand myCommand, IFactory factory) { - this.myCommand = myCommand; - this.factory = factory; - } - - - @Override - public void run(String... args) { - exitCode = new CommandLine(myCommand, factory).execute(args); - } - - @Override - public int getExitCode() { - return exitCode; - } -} - diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java b/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java index e170b68..fadea93 100644 --- a/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java @@ -10,6 +10,8 @@ public enum ErrorType { /** The type of an object is not as required by the schema, e.g., we get a string instead of an array. */ JSON_TYPE("type"), JSON_ENUM("todo"), + ONTOLOGY_INVALID_ID("Term id does not exist in ontology"), + ONTOLOGY_TERM_WITH_ALTERNATE_ID("Term id is not the primary id for this term"), PHENOPACKET_SUBJECT_LACKS_AGE("phenopacket subject lacks age"), PHENOPACKET_LACKS_SUBJECT("phenopacket lacks subject"), INVALID_ONTOLOGY("invalid ontology"), diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java index 2471457..5b12b3a 100644 --- a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java @@ -78,7 +78,7 @@ public List validate(InputStream inputStream) { try { JsonNode json = objectMapper.readTree(inputStream); jsonSchema.validate(json) - .forEach(e -> errors.add(new JsonValidationError(validatorInfo, e))); + .forEach(e -> errors.add(new JsonValidationItem(validatorInfo, e))); return errors; } catch (IOException e) { diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationError.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java similarity index 87% rename from validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationError.java rename to validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java index 5a8b3a0..0bad30f 100644 --- a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationError.java +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java @@ -10,13 +10,13 @@ * POJO to represent errors identified by JSON Schema validation. * @author Peter N Robinson */ -public final class JsonValidationError implements ValidationItem { +public final class JsonValidationItem implements ValidationItem { private final ValidatorInfo validatorInfo; private final ErrorType errorType; private final String message; - public JsonValidationError(ValidatorInfo validatorInfo, ValidationMessage validationMessage) { + public JsonValidationItem(ValidatorInfo validatorInfo, ValidationMessage validationMessage) { this.validatorInfo = validatorInfo; this.errorType = ErrorType.stringToErrorType(validationMessage.getType()); this.message = validationMessage.getMessage(); @@ -41,7 +41,7 @@ public String message() { public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - JsonValidationError that = (JsonValidationError) o; + JsonValidationItem that = (JsonValidationItem) o; return Objects.equals(validatorInfo, that.validatorInfo) && errorType == that.errorType && Objects.equals(message, that.message); } diff --git a/validator-ontology/pom.xml b/validator-ontology/pom.xml new file mode 100644 index 0000000..a34ed71 --- /dev/null +++ b/validator-ontology/pom.xml @@ -0,0 +1,140 @@ + + + + validator + org.phenopackets.validator + 0.1.1-SNAPSHOT + + 4.0.0 + + 1.6.3 + 2.0.0 + + + validator-ontology + + + + org.phenopackets.validator + validator-core + ${project.parent.version} + + + + org.monarchinitiative.phenol + phenol-core + ${phenol.version} + + + + org.monarchinitiative.phenol + phenol-io + ${phenol.version} + + + + com.googlecode.json-simple + json-simple + 1.1.1 + + + + com.google.protobuf + protobuf-java + 3.11.4 + + + com.google.protobuf + protobuf-java-util + 3.11.4 + + + com.google.guava + guava + + + + + + org.phenopackets + phenopacket-schema + ${phenopacket.version} + compile + + + com.google.protobuf + protobuf-java + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + com.fasterxml.jackson.dataformat + jackson-dataformat-protobuf + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + org.yaml + snakeyaml + + + com.google.guava + guava + + + com.google.errorprone + error_prone_annotations + + + org.slf4j + slf4j-api + + + org.apache.logging.log4j + log4j-slf4j18-impl + + + commons-collections + commons-collections + + + commons-logging + commons-logging + + + -com.google.protobuf + protobuf-java-util + + + + + + + + org.prefixcommons + curie-util + 0.0.2 + + + + + + \ No newline at end of file diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/HpoValidator.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/HpoValidator.java new file mode 100644 index 0000000..f287963 --- /dev/null +++ b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/HpoValidator.java @@ -0,0 +1,105 @@ +package org.phenopackets.validator.ontology; + +import com.google.protobuf.util.JsonFormat; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.monarchinitiative.phenol.base.PhenolRuntimeException; +import org.monarchinitiative.phenol.ontology.data.TermId; +import org.phenopackets.schema.v2.Phenopacket; +import org.phenopackets.validator.core.ValidationItem; +import org.phenopackets.validator.core.ValidatorInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + + + + +public class HpoValidator extends OntologyValidator { + private static final Logger LOGGER = LoggerFactory.getLogger(HpoValidator.class); + + private static final String HPO_PREFIX = "HP"; + + protected HpoValidator(File jsonFile, ValidatorInfo vinfo) { + super(jsonFile, vinfo); + } + + @Override + public ValidatorInfo info() { + return ValidatorInfo.of(ONTOLOGY_VALIDATOR, "Human Phenotype Ontology"); + } + + + + /** + * The logic of this validator is to check the Phenotypic features for HPO terms. If HPO terms + * are used there, the we check that the HPO terms are included in the ontology (thus, we check for + * invalid HPO ids, and also check if the id used is the latest or if it is an alternate id). If there + * are both observed and excluded terms, we check that there is no logical inconsistency, e.g., + * NOT Abnormal liver morphology but OBSERVED Hepatic fibrosis. The latter is not possible because + * Hepatic fibrosis implies Abnormal liver morphology + * @param inputStream -- stream from the phenopacket. + * @return + */ + @Override + public List validate(InputStream inputStream) { + List errors = new ArrayList<>(); + JSONParser parser = new JSONParser(); + Set observedFeatures = new HashSet<>(); + Set excludedFeatures = new HashSet<>(); + try { + Object obj = parser.parse(new InputStreamReader(inputStream)); + JSONObject jsonObject = (JSONObject) obj; + String phenopacketJsonString = jsonObject.toJSONString(); + Phenopacket.Builder phenoPacketBuilder = Phenopacket.newBuilder(); + JsonFormat.parser().merge(phenopacketJsonString, phenoPacketBuilder); + Phenopacket phenopacket = phenoPacketBuilder.build(); + //return new PhenopacketImporter(phenopacket,ontology); + for (var pf : phenopacket.getPhenotypicFeaturesList()) { + String id = pf.getType().getId(); + try { + TermId tid = TermId.of(id); + if (!tid.getPrefix().equals(HPO_PREFIX)) { + continue; // only check HPO terms + } + if (pf.getExcluded()) { + excludedFeatures.add(tid); + } else { + observedFeatures.add(tid); + } + } catch (PhenolRuntimeException pe) { + LOGGER.error("Invalid Ontology term id: " + id); + } + } + // 1 check that all terms are in the ontology and use the primary id + for (TermId tid: observedFeatures) { + if (!ontology.containsTerm(tid)) { + String message = String.format("Could not find term id: %s", tid.getValue()); + ValidationItem vitem = OntologyValidationItem.invalidTermId(validatorInfo,message); + errors.add(vitem); + } else { + TermId primaryId = ontology.getPrimaryTermId(tid); + if (! primaryId.equals(tid)) { + String message = String.format("Using alternate (obsolete) id (%s) instead of primary id (%s)", + tid.getValue(), primaryId); + ValidationItem vitem = OntologyValidationItem.invalidTermId(validatorInfo,message); + errors.add(vitem); + } + } + } + return errors; + } catch (IOException | RuntimeException | ParseException e) { + LOGGER.warn("Error while validating: {}", e.getMessage()); + } + return errors; + } +} diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItem.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItem.java new file mode 100644 index 0000000..f6d8d2c --- /dev/null +++ b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItem.java @@ -0,0 +1,44 @@ +package org.phenopackets.validator.ontology; + +import org.phenopackets.validator.core.ErrorType; +import org.phenopackets.validator.core.ValidationItem; +import org.phenopackets.validator.core.ValidatorInfo; + +public class OntologyValidationItem implements ValidationItem { + + private final ValidatorInfo validatorInfo; + private final ErrorType errorType; + private final String message; + + public OntologyValidationItem(ValidatorInfo validatorInfo, ErrorType errorType, String validationMessage) { + this.validatorInfo = validatorInfo; + this.errorType = errorType; + this.message = validationMessage; + } + + + @Override + public ValidatorInfo validatorInfo() { + return validatorInfo; + } + + @Override + public ErrorType errorType() { + return errorType; + } + + @Override + public String message() { + return message; + } + + public static OntologyValidationItem invalidTermId(ValidatorInfo validatorInfo,String validationMessage) { + return new OntologyValidationItem(validatorInfo, ErrorType.ONTOLOGY_INVALID_ID, validationMessage); + } + + public static OntologyValidationItem alternateId(ValidatorInfo validatorInfo,String validationMessage) { + return new OntologyValidationItem(validatorInfo, ErrorType.ONTOLOGY_TERM_WITH_ALTERNATE_ID, validationMessage); + } + + +} diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidator.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidator.java new file mode 100644 index 0000000..cbaff46 --- /dev/null +++ b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidator.java @@ -0,0 +1,29 @@ +package org.phenopackets.validator.ontology; + +import org.monarchinitiative.phenol.ontology.data.Ontology; +import org.phenopackets.validator.core.PhenopacketValidator; + +import org.phenopackets.validator.core.ValidatorInfo; +import org.monarchinitiative.phenol.io.OntologyLoader; + + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; + +public abstract class OntologyValidator implements PhenopacketValidator { + private static final Logger LOGGER = LoggerFactory.getLogger(OntologyValidator.class); + + protected final ValidatorInfo validatorInfo; + protected final Ontology ontology; + + protected final static String ONTOLOGY_VALIDATOR = "Otology validation"; + + protected OntologyValidator(File jsonFile, ValidatorInfo vinfo) { + validatorInfo = vinfo; + ontology = OntologyLoader.loadOntology(jsonFile); + } + +} diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidatorFactory.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidatorFactory.java new file mode 100644 index 0000000..9df442b --- /dev/null +++ b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidatorFactory.java @@ -0,0 +1,35 @@ +package org.phenopackets.validator.ontology; + +import org.phenopackets.validator.core.PhenopacketValidator; +import org.phenopackets.validator.core.PhenopacketValidatorFactory; +import org.phenopackets.validator.core.ValidatorInfo; +import org.phenopackets.validator.core.except.PhenopacketValidatorRuntimeException; + +import java.io.File; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +public class OntologyValidatorFactory implements PhenopacketValidatorFactory { + + private final Map validatorMap; + + private static final String ONTOLOGY_VALIDATOR_TYPE = "Semantic validation"; + + public OntologyValidatorFactory(File... ontologyJsonFiles) { + validatorMap = new HashMap<>(); + for (var jsonFile: ontologyJsonFiles) { + + } + } + + + @Override + public Optional getValidatorForType(ValidatorInfo type) { + if (validatorMap.containsKey(type)) { + return Optional.of(validatorMap.get(type)); + } + return Optional.empty(); + } +} From 0cf1996829a83a2ef7f704a63379fafdb3da99c2 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Mon, 27 Sep 2021 12:08:25 -0400 Subject: [PATCH 18/37] refactor ValidationInfoType --- validator-cli/pom.xml | 5 ++ .../validator/cli/ValidatorApplication.java | 24 ++++++++- .../results/ValidationItemTsvVisualizer.java | 4 +- validator-cli/src/main/resources/log4j2.xml | 35 ++++++++++++ .../src/main/resources/logback-spring.xml | 24 --------- .../validator/core/ErrorType.java | 43 --------------- .../validator/core/ValidationAspect.java | 22 -------- .../validator/core/ValidationItem.java | 2 +- .../validator/core/ValidationItemType.java | 8 +++ .../jsonschema/JsonValidationItem.java | 18 ++++--- .../jsonschema/JsonValidationItemType.java | 54 +++++++++++++++++++ .../schema/hpo-rare-disease-schema.json | 0 .../resources/schema/phenopacket-generic.json | 0 .../JsonSchemaDiseaseValidatorTest.java | 4 +- .../jsonschema/JsonSchemaValidatorTest.java | 13 ++--- .../validator/ontology/HpoValidator.java | 9 ++-- .../ontology/OntologyValidationItem.java | 19 ++++--- .../ontology/OntologyValidationItemType.java | 45 ++++++++++++++++ 18 files changed, 207 insertions(+), 122 deletions(-) create mode 100644 validator-cli/src/main/resources/log4j2.xml delete mode 100644 validator-cli/src/main/resources/logback-spring.xml delete mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java delete mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/ValidationAspect.java create mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemType.java create mode 100644 validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemType.java rename validator-jsonschema/src/{ => main}/resources/schema/hpo-rare-disease-schema.json (100%) rename validator-jsonschema/src/{ => main}/resources/schema/phenopacket-generic.json (100%) create mode 100644 validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemType.java diff --git a/validator-cli/pom.xml b/validator-cli/pom.xml index b47f307..f3fc4af 100644 --- a/validator-cli/pom.xml +++ b/validator-cli/pom.xml @@ -21,6 +21,11 @@ validator-jsonschema ${project.parent.version} + + org.phenopackets.validator + validator-ontology + ${project.parent.version} + info.picocli picocli diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java index 3afc5f2..cac3260 100644 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java @@ -7,6 +7,8 @@ import org.phenopackets.validator.core.ValidatorInfo; import org.phenopackets.validator.jsonschema.ClasspathJsonSchemaValidatorFactory; import org.phenopackets.validator.jsonschema.JsonSchemaValidator; +import org.phenopackets.validator.ontology.HpoValidator; +import org.phenopackets.validator.ontology.OntologyValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; @@ -28,6 +30,9 @@ public class ValidatorApplication implements Runnable { @CommandLine.Parameters(arity = "0..*", description = "one or more JSON Schema configuration files") public List jsonSchemaFiles = List.of(); + @CommandLine.Option(names = "--hp", required = true, description = "check with HPO (hp.json)") + public String hpoJsonPath; + @CommandLine.Option(names = {"-p", "--phenopacket"}, required = true, description = "Phenopacket file to be validated") public String phenopacket; @@ -78,19 +83,36 @@ public void run() { List validationItems = validator.validate(phenopacketFile); if (validationItems.isEmpty()) { LOGGER.info("No errors found"); + System.out.println("JSON: No errors found"); resultVisualizer.errorFree(vinfo); } else { LOGGER.info("Found {} errors:", validationItems.size()); for (ValidationItem item : validationItems) { - LOGGER.info("({}) {}", item.errorType(), item.message()); + LOGGER.info("({}) {}", item.type(), item.message()); resultVisualizer.error(vinfo, new ValidationItemTsvVisualizer(item)); } } } + // Now check with HPO + OntologyValidator hpoValidator = new HpoValidator(new File(hpoJsonPath)); + ValidatorInfo hpoInfo = hpoValidator.info(); + List validationItems = hpoValidator.validate(phenopacketFile); + if (validationItems.isEmpty()) { + LOGGER.info("HPO: No errors found"); + System.out.println("HPO: No errors found"); + resultVisualizer.errorFree(hpoInfo); + } else { + LOGGER.info("Found {} errors:", validationItems.size()); + for (ValidationItem item : validationItems) { + LOGGER.info("({}) {}", item.type(), item.message()); + resultVisualizer.error(hpoInfo, new ValidationItemTsvVisualizer(item)); + } + } } catch (IOException e) { LOGGER.warn("Error opening the phenopacket", e); } + // poor man's formatting LOGGER.info(""); LOGGER.info("--------------------------------------------------------------------------------"); diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationItemTsvVisualizer.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationItemTsvVisualizer.java index 5454aa4..d72c961 100644 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationItemTsvVisualizer.java +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationItemTsvVisualizer.java @@ -15,9 +15,9 @@ public ValidationItemTsvVisualizer(ValidationItem item) { /** * Get a list of fields for display - * @return + * @return A list of Strings that can be used to create a row of a TSV or CSV file */ public List getFields() { - return List.of(error.errorType().name(), error.message(), error.validatorInfo().validatorId()); + return List.of(error.type().itemType(), error.message(), error.validatorInfo().validatorId()); } } diff --git a/validator-cli/src/main/resources/log4j2.xml b/validator-cli/src/main/resources/log4j2.xml new file mode 100644 index 0000000..2c48de3 --- /dev/null +++ b/validator-cli/src/main/resources/log4j2.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/validator-cli/src/main/resources/logback-spring.xml b/validator-cli/src/main/resources/logback-spring.xml deleted file mode 100644 index 3a6c413..0000000 --- a/validator-cli/src/main/resources/logback-spring.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java b/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java deleted file mode 100644 index fadea93..0000000 --- a/validator-core/src/main/java/org/phenopackets/validator/core/ErrorType.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.phenopackets.validator.core; - -import org.phenopackets.validator.core.except.PhenopacketValidatorRuntimeException; - -public enum ErrorType { - /** JSON schema error meaning that the JSON code contained a property not present in the schema. */ - JSON_ADDITIONAL_PROPERTIES("additionalProperties"), - /** JSON schema error meaning that the JSON code failed to contain a property required by the schema. */ - JSON_REQUIRED("required"), - /** The type of an object is not as required by the schema, e.g., we get a string instead of an array. */ - JSON_TYPE("type"), - JSON_ENUM("todo"), - ONTOLOGY_INVALID_ID("Term id does not exist in ontology"), - ONTOLOGY_TERM_WITH_ALTERNATE_ID("Term id is not the primary id for this term"), - PHENOPACKET_SUBJECT_LACKS_AGE("phenopacket subject lacks age"), - PHENOPACKET_LACKS_SUBJECT("phenopacket lacks subject"), - INVALID_ONTOLOGY("invalid ontology"), - PHENOPACKET_LACKS_PHENOTYPIC_FEATURE("phenopacket lacks phenotypic feature"); - - private final String name; - - - ErrorType(String value) { - this.name = value; - } - - @Override - public String toString() { - return this.name; - } - - - public static ErrorType stringToErrorType(String error) { - switch (error) { - case "additionalProperties": return JSON_ADDITIONAL_PROPERTIES; - case "required": return JSON_REQUIRED; - case "type": return JSON_TYPE; - case "enum": return JSON_TYPE; - default: - throw new PhenopacketValidatorRuntimeException("Did not recognize error type: \"" + error + "\""); - } - } -} diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationAspect.java b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationAspect.java deleted file mode 100644 index 2c25c9b..0000000 --- a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationAspect.java +++ /dev/null @@ -1,22 +0,0 @@ -package org.phenopackets.validator.core; - - -/* -Probably this should go into error type, they have the same purpose - */ -@Deprecated -public enum ValidationAspect { - - // This enum is supposed to be used instead of the ErrorType - // TODO - elaborate the categories - - // general syntax inconsistency, e.g. bad JSON format - GENERAL, - - // e.g. missing at least one phenotypic feature, age, etc. - MISSING_PROPERTY, - - // e.g. invalid ontology used to represent an info - INVALID_VALUE, - -} diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItem.java b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItem.java index 66c3548..a6c67af 100644 --- a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItem.java +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItem.java @@ -14,7 +14,7 @@ public interface ValidationItem { ValidatorInfo validatorInfo(); // TODO - decide which enum to use here - either ErrorType or ValidationAspect - ErrorType errorType(); + ValidationItemType type(); /** * @return string with description of the issue intended for human consumption. diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemType.java b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemType.java new file mode 100644 index 0000000..cd4f775 --- /dev/null +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemType.java @@ -0,0 +1,8 @@ +package org.phenopackets.validator.core; + +public interface ValidationItemType { + + String itemType(); + String itemDescription(); + +} diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java index 0bad30f..78081d4 100644 --- a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java @@ -1,7 +1,8 @@ package org.phenopackets.validator.jsonschema; import com.networknt.schema.ValidationMessage; -import org.phenopackets.validator.core.ErrorType; + import org.phenopackets.validator.core.ValidationItem; +import org.phenopackets.validator.core.ValidationItemType; import org.phenopackets.validator.core.ValidatorInfo; import java.util.Objects; @@ -13,12 +14,12 @@ public final class JsonValidationItem implements ValidationItem { private final ValidatorInfo validatorInfo; - private final ErrorType errorType; + private final ValidationItemType itemType; private final String message; public JsonValidationItem(ValidatorInfo validatorInfo, ValidationMessage validationMessage) { this.validatorInfo = validatorInfo; - this.errorType = ErrorType.stringToErrorType(validationMessage.getType()); + this.itemType = JsonValidationItemType.stringToErrorType(validationMessage.getType()); this.message = validationMessage.getMessage(); } @@ -28,8 +29,8 @@ public ValidatorInfo validatorInfo() { } @Override - public ErrorType errorType() { - return this.errorType; + public ValidationItemType type() { + return this.itemType; } @Override @@ -42,20 +43,21 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; JsonValidationItem that = (JsonValidationItem) o; - return Objects.equals(validatorInfo, that.validatorInfo) && errorType == that.errorType && Objects.equals(message, that.message); + return Objects.equals(validatorInfo, that.validatorInfo) && itemType == that.itemType && Objects.equals(message, that.message); } @Override public int hashCode() { - return Objects.hash(validatorInfo, errorType, message); + return Objects.hash(validatorInfo, itemType, message); } @Override public String toString() { return "JsonValidationError{" + "validatorInfo='" + validatorInfo + '\'' + - ", errorType=" + errorType + + ", errorType=" + itemType + ", message='" + message + '\'' + '}'; } + } diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemType.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemType.java new file mode 100644 index 0000000..2511241 --- /dev/null +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemType.java @@ -0,0 +1,54 @@ +package org.phenopackets.validator.jsonschema; + +import org.phenopackets.validator.core.ValidationItemType; +import org.phenopackets.validator.core.except.PhenopacketValidatorRuntimeException; + +public class JsonValidationItemType implements ValidationItemType { + + private final String itemType; + private final String description; + + /** + * JSON schema error meaning that the JSON code contained a property not present in the schema. + */ + public final static JsonValidationItemType JSON_ADDITIONAL_PROPERTIES = + new JsonValidationItemType("additionalProperties", "Use of a property not present in the schema"); + /** JSON schema error meaning that the JSON code failed to contain a property required by the schema. */ + public final static JsonValidationItemType JSON_REQUIRED = + new JsonValidationItemType("required", "failure to contain a property required by the schema"); + /** The type of an object is not as required by the schema, e.g., we get a string instead of an array. */ + public final static JsonValidationItemType JSON_TYPE = + new JsonValidationItemType("type", "incorrect data type of object"); + public final static JsonValidationItemType JSON_ENUM= + new JsonValidationItemType("enum", "TODO"); + + + + + + public JsonValidationItemType(String type, String desc) { + itemType = type; + description = desc; + } + + @Override + public String itemType() { + return itemType; + } + + @Override + public String itemDescription() { + return description; + } + + public static JsonValidationItemType stringToErrorType(String error) { + switch (error) { + case "additionalProperties": return JSON_ADDITIONAL_PROPERTIES; + case "required": return JSON_REQUIRED; + case "type": return JSON_TYPE; + case "enum": return JSON_TYPE; + default: + throw new PhenopacketValidatorRuntimeException("Did not recognize JSON error type: \"" + error + "\""); + } + } +} diff --git a/validator-jsonschema/src/resources/schema/hpo-rare-disease-schema.json b/validator-jsonschema/src/main/resources/schema/hpo-rare-disease-schema.json similarity index 100% rename from validator-jsonschema/src/resources/schema/hpo-rare-disease-schema.json rename to validator-jsonschema/src/main/resources/schema/hpo-rare-disease-schema.json diff --git a/validator-jsonschema/src/resources/schema/phenopacket-generic.json b/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json similarity index 100% rename from validator-jsonschema/src/resources/schema/phenopacket-generic.json rename to validator-jsonschema/src/main/resources/schema/phenopacket-generic.json diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java index 391cf00..fac7ef5 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java @@ -9,7 +9,6 @@ import org.phenopackets.schema.v2.core.MetaData; import org.phenopackets.schema.v2.core.Resource; import org.phenopackets.schema.v2.core.TimeElement; -import org.phenopackets.validator.core.ErrorType; import org.phenopackets.validator.core.ValidationItem; import org.phenopackets.validator.core.ValidatorInfo; import org.phenopackets.validator.testdatagen.PhenopacketUtil; @@ -19,6 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.phenopackets.validator.jsonschema.JsonValidationItemType.JSON_REQUIRED; import static org.phenopackets.validator.testdatagen.PhenopacketUtil.*; /** @@ -86,7 +86,7 @@ public void testLacksId() throws InvalidProtocolBufferException { } assertEquals(1, errors.size()); ValidationItem error = errors.get(0); - assertEquals(ErrorType.JSON_REQUIRED, error.errorType()); + assertEquals(JSON_REQUIRED, error.type()); assertEquals("$.id: is missing but it is required", error.message()); } diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java index 561d960..94fe13b 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java @@ -4,7 +4,6 @@ import org.phenopackets.schema.v2.Phenopacket; import org.phenopackets.validator.testdatagen.RareDiseasePhenopacket; import org.phenopackets.validator.testdatagen.SimplePhenopacket; -import org.phenopackets.validator.core.ErrorType; import org.phenopackets.validator.core.ValidationItem; import org.phenopackets.validator.core.ValidatorInfo; @@ -17,6 +16,8 @@ import com.google.protobuf.util.JsonFormat; import static org.junit.jupiter.api.Assertions.*; +import static org.phenopackets.validator.jsonschema.JsonValidationItemType.JSON_ADDITIONAL_PROPERTIES; +import static org.phenopackets.validator.jsonschema.JsonValidationItemType.JSON_REQUIRED; public class JsonSchemaValidatorTest { @@ -47,7 +48,7 @@ public void testValidationOfSimpleValidPhenopacket() throws Exception { errors = validator.validate(json); assertEquals(1, errors.size()); ValidationItem error = errors.get(0); - assertEquals(ErrorType.JSON_REQUIRED, error.errorType()); + assertEquals(JSON_REQUIRED, error.type()); assertEquals("$.id: is missing but it is required", error.message()); } @@ -67,13 +68,13 @@ public void testValidationOfSimpleInValidPhenopacket() throws Exception { assertEquals(3, errors.size()); ValidationItem error = errors.get(0); - assertEquals(ErrorType.JSON_REQUIRED, error.errorType()); + assertEquals(JSON_REQUIRED, error.type()); assertEquals("$.id: is missing but it is required", error.message()); error = errors.get(1); - assertEquals(ErrorType.JSON_REQUIRED, error.errorType()); + assertEquals(JSON_REQUIRED, error.type()); assertEquals("$.metaData: is missing but it is required", error.message()); error = errors.get(2); - assertEquals(ErrorType.JSON_ADDITIONAL_PROPERTIES, error.errorType()); + assertEquals(JSON_ADDITIONAL_PROPERTIES, error.type()); assertEquals("$.disney: is not defined in the schema and the schema does not allow additional properties", error.message()); } @@ -102,7 +103,7 @@ public void testRareDiseaseBethlemahmInvalidValidPhenopacket() throws IOExceptio } assertEquals(1, errors.size()); ValidationItem error = errors.get(0); - assertEquals(ErrorType.JSON_REQUIRED, error.errorType()); + assertEquals(JSON_REQUIRED, error.type()); assertEquals("$.phenotypicFeatures: is missing but it is required", error.message()); } diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/HpoValidator.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/HpoValidator.java index f287963..e431d3e 100644 --- a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/HpoValidator.java +++ b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/HpoValidator.java @@ -29,13 +29,13 @@ public class HpoValidator extends OntologyValidator { private static final String HPO_PREFIX = "HP"; - protected HpoValidator(File jsonFile, ValidatorInfo vinfo) { - super(jsonFile, vinfo); + public HpoValidator(File jsonFile) { + super(jsonFile, ValidatorInfo.of(ONTOLOGY_VALIDATOR, "Human Phenotype Ontology")); } @Override public ValidatorInfo info() { - return ValidatorInfo.of(ONTOLOGY_VALIDATOR, "Human Phenotype Ontology"); + return validatorInfo; } @@ -48,7 +48,7 @@ public ValidatorInfo info() { * NOT Abnormal liver morphology but OBSERVED Hepatic fibrosis. The latter is not possible because * Hepatic fibrosis implies Abnormal liver morphology * @param inputStream -- stream from the phenopacket. - * @return + * @return list of validation errors */ @Override public List validate(InputStream inputStream) { @@ -63,7 +63,6 @@ public List validate(InputStream inputStream) { Phenopacket.Builder phenoPacketBuilder = Phenopacket.newBuilder(); JsonFormat.parser().merge(phenopacketJsonString, phenoPacketBuilder); Phenopacket phenopacket = phenoPacketBuilder.build(); - //return new PhenopacketImporter(phenopacket,ontology); for (var pf : phenopacket.getPhenotypicFeaturesList()) { String id = pf.getType().getId(); try { diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItem.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItem.java index f6d8d2c..4803d58 100644 --- a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItem.java +++ b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItem.java @@ -1,18 +1,21 @@ package org.phenopackets.validator.ontology; -import org.phenopackets.validator.core.ErrorType; import org.phenopackets.validator.core.ValidationItem; +import org.phenopackets.validator.core.ValidationItemType; import org.phenopackets.validator.core.ValidatorInfo; +import static org.phenopackets.validator.ontology.OntologyValidationItemType.ONTOLOGY_INVALID_ID; +import static org.phenopackets.validator.ontology.OntologyValidationItemType.ONTOLOGY_TERM_WITH_ALTERNATE_ID; + public class OntologyValidationItem implements ValidationItem { private final ValidatorInfo validatorInfo; - private final ErrorType errorType; + private final ValidationItemType itemType; private final String message; - public OntologyValidationItem(ValidatorInfo validatorInfo, ErrorType errorType, String validationMessage) { + public OntologyValidationItem(ValidatorInfo validatorInfo, ValidationItemType errorType, String validationMessage) { this.validatorInfo = validatorInfo; - this.errorType = errorType; + this.itemType = errorType; this.message = validationMessage; } @@ -23,8 +26,8 @@ public ValidatorInfo validatorInfo() { } @Override - public ErrorType errorType() { - return errorType; + public ValidationItemType type() { + return itemType; } @Override @@ -33,11 +36,11 @@ public String message() { } public static OntologyValidationItem invalidTermId(ValidatorInfo validatorInfo,String validationMessage) { - return new OntologyValidationItem(validatorInfo, ErrorType.ONTOLOGY_INVALID_ID, validationMessage); + return new OntologyValidationItem(validatorInfo, ONTOLOGY_INVALID_ID, validationMessage); } public static OntologyValidationItem alternateId(ValidatorInfo validatorInfo,String validationMessage) { - return new OntologyValidationItem(validatorInfo, ErrorType.ONTOLOGY_TERM_WITH_ALTERNATE_ID, validationMessage); + return new OntologyValidationItem(validatorInfo, ONTOLOGY_TERM_WITH_ALTERNATE_ID, validationMessage); } diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemType.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemType.java new file mode 100644 index 0000000..3c5ed73 --- /dev/null +++ b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemType.java @@ -0,0 +1,45 @@ +package org.phenopackets.validator.ontology; + +import org.phenopackets.validator.core.ValidationItemType; + +public class OntologyValidationItemType implements ValidationItemType { + + + private final String itemType; + private final String description; + + /** + * This error type refers to the use of a term ID, e.g., HP:1234567, which is syntactically correct but which + * does not existing in the actual ontology. + */ + public final static OntologyValidationItemType ONTOLOGY_INVALID_ID = + new OntologyValidationItemType("Invalid Ontology Term ID", "Term id does not exist in ontology"); + + /** + * This error type refers to the use of a term ID that is an alternate_id, i.e., an outdated ID that should + * be replaced by the primary ID of the corresponding Ontology term. + */ + public final static OntologyValidationItemType ONTOLOGY_TERM_WITH_ALTERNATE_ID = + new OntologyValidationItemType("Use of outdated Ontology Term ID", "Term id is not the primary id for this term"); + + + OntologyValidationItemType(String type, String desc) { + itemType = type; + description = desc; + } + + + @Override + public String itemType() { + return itemType; + } + + @Override + public String itemDescription() { + return description; + } + + + + +} From f96488037292330849790e2a7a9687116249996d Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Mon, 27 Sep 2021 19:47:34 -0400 Subject: [PATCH 19/37] cleanup --- .gitignore | 1 + .../validator/cli/ValidatorApplication.java | 65 ++++++---------- .../cli/results/ValidationTsvVisualizer.java | 5 +- .../core/DefaultValidatorRegistry.java | 25 ++++++ .../core/PhenopacketValidatorFactory.java | 15 ---- .../core/PhenopacketValidatorRegistry.java | 22 ++++++ .../ClasspathJsonSchemaValidatorFactory.java | 77 ------------------- .../jsonschema/JsonSchemaValidator.java | 4 +- .../JsonSchemaValidatorFactory.java | 39 ++++++++++ .../jsonschema/JsonValidationItem.java | 1 + .../jsonschema/JsonValidationItemType.java | 10 +-- .../JsonSchemaDiseaseValidatorTest.java | 7 +- .../jsonschema/JsonSchemaValidatorTest.java | 15 ++-- .../testdatagen/RareDiseasePhenopacket.java | 2 +- .../ontology/OntologyValidationItemType.java | 4 +- .../ontology/OntologyValidatorFactory.java | 35 --------- 16 files changed, 138 insertions(+), 189 deletions(-) create mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidatorRegistry.java delete mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorFactory.java create mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorRegistry.java delete mode 100644 validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/ClasspathJsonSchemaValidatorFactory.java create mode 100644 validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorFactory.java delete mode 100644 validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidatorFactory.java diff --git a/.gitignore b/.gitignore index 28ae4e6..7e1893a 100644 --- a/.gitignore +++ b/.gitignore @@ -125,3 +125,4 @@ nb-configuration.xml .nb-gradle/ # Created by .ignore support plugin (hsz.mobi) +/phenopacket-validation.tsv diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java index cac3260..2e8970a 100644 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java @@ -3,9 +3,10 @@ import org.phenopackets.validator.cli.results.ValidationItemTsvVisualizer; import org.phenopackets.validator.cli.results.ValidationTsvVisualizer; import org.phenopackets.validator.core.PhenopacketValidator; +import org.phenopackets.validator.core.PhenopacketValidatorRegistry; import org.phenopackets.validator.core.ValidationItem; import org.phenopackets.validator.core.ValidatorInfo; -import org.phenopackets.validator.jsonschema.ClasspathJsonSchemaValidatorFactory; +import org.phenopackets.validator.jsonschema.JsonSchemaValidatorFactory; import org.phenopackets.validator.jsonschema.JsonSchemaValidator; import org.phenopackets.validator.ontology.HpoValidator; import org.phenopackets.validator.ontology.OntologyValidator; @@ -15,13 +16,11 @@ import java.io.*; import java.nio.file.Files; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; @CommandLine.Command(name = "PhenopacketValidator", version = "0.1.0", mixinStandardHelpOptions = true) -public class ValidatorApplication implements Runnable { +public class ValidatorApplication implements Runnable { private static final Logger LOGGER = LoggerFactory.getLogger(ValidatorApplication.class); @CommandLine.Option(names = "--rare", description = "apply HPO rare-disease constraints") @@ -36,7 +35,7 @@ public class ValidatorApplication implements Runnable { @CommandLine.Option(names = {"-p", "--phenopacket"}, required = true, description = "Phenopacket file to be validated") public String phenopacket; - @CommandLine.Option(names={"-o", "--out"}, description = "name of output file (default ${DEFAULT_VALUE})") + @CommandLine.Option(names = {"-o", "--out"}, description = "name of output file (default ${DEFAULT_VALUE})") public String outfileName = "phenopacket-validation.tsv"; private static final String CUSTOM_JSON_VALIDATOR_TYPE = "Custom JSON Schema validation"; @@ -50,7 +49,6 @@ public static void main(String[] args) { } - @Override public void run() { List validationTypes = new LinkedList<>(); @@ -61,58 +59,43 @@ public void run() { } File phenopacketFile = new File(phenopacket); LOGGER.info("Validating {} phenopacket", phenopacketFile); - // TODO -- adapt PhenopacketValidatorFactory to accept multiple JSON Schema files - Map jsonValidatorMap = ClasspathJsonSchemaValidatorFactory.genericValidator(); + Map validatorMap = new HashMap<>(JsonSchemaValidatorFactory.genericValidator()); for (File jsonSchema : jsonSchemaFiles) { // we will create ValidatorInfo objects based on the names and paths of the files. String baseName = jsonSchema.getName(); ValidatorInfo vinfo = ValidatorInfo.of(CUSTOM_JSON_VALIDATOR_TYPE, baseName); - JsonSchemaValidator jvalid = JsonSchemaValidator.of(jsonSchema, vinfo); - jsonValidatorMap.put(vinfo, jvalid); + PhenopacketValidator jvalid = JsonSchemaValidator.of(jsonSchema, vinfo); + validatorMap.put(vinfo, jvalid); LOGGER.info("Adding configuration file at `{}`", vinfo); } + OntologyValidator hpoValidator = new HpoValidator(new File(hpoJsonPath)); + validatorMap.put(hpoValidator.info(), hpoValidator); + PhenopacketValidatorRegistry registry = PhenopacketValidatorRegistry.of(validatorMap); + // poor man's formatting LOGGER.info(""); LOGGER.info("--------------------------------------------------------------------------------"); LOGGER.info(""); ValidationTsvVisualizer resultVisualizer = new ValidationTsvVisualizer(); try (InputStream in = Files.newInputStream(phenopacketFile.toPath())) { - for (var e : jsonValidatorMap.entrySet()) { - ValidatorInfo vinfo = e.getKey(); - PhenopacketValidator validator = e.getValue(); - List validationItems = validator.validate(phenopacketFile); - if (validationItems.isEmpty()) { - LOGGER.info("No errors found"); - System.out.println("JSON: No errors found"); - resultVisualizer.errorFree(vinfo); - } else { - LOGGER.info("Found {} errors:", validationItems.size()); - for (ValidationItem item : validationItems) { - LOGGER.info("({}) {}", item.type(), item.message()); - resultVisualizer.error(vinfo, new ValidationItemTsvVisualizer(item)); + Set validatorInfoSet = registry.getValidationTypeSet(); + for (var vinfo : validatorInfoSet) { + var opt = registry.getValidatorForType(vinfo); + if (opt.isPresent()) { + PhenopacketValidator validator = opt.get(); + List validationItems = validator.validate(phenopacketFile); + if (validationItems.isEmpty()) { + resultVisualizer.errorFree(vinfo); + } else { + for (var item : validationItems) { + resultVisualizer.error(vinfo, new ValidationItemTsvVisualizer(item)); + } } } } - // Now check with HPO - OntologyValidator hpoValidator = new HpoValidator(new File(hpoJsonPath)); - ValidatorInfo hpoInfo = hpoValidator.info(); - List validationItems = hpoValidator.validate(phenopacketFile); - if (validationItems.isEmpty()) { - LOGGER.info("HPO: No errors found"); - System.out.println("HPO: No errors found"); - resultVisualizer.errorFree(hpoInfo); - } else { - LOGGER.info("Found {} errors:", validationItems.size()); - for (ValidationItem item : validationItems) { - LOGGER.info("({}) {}", item.type(), item.message()); - resultVisualizer.error(hpoInfo, new ValidationItemTsvVisualizer(item)); - } - } } catch (IOException e) { LOGGER.warn("Error opening the phenopacket", e); } - - // poor man's formatting LOGGER.info(""); LOGGER.info("--------------------------------------------------------------------------------"); diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationTsvVisualizer.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationTsvVisualizer.java index 1aebad2..09e74a0 100644 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationTsvVisualizer.java +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationTsvVisualizer.java @@ -22,7 +22,7 @@ public ValidationTsvVisualizer() { /** * Add a ValidationInfo (i.e., validation type) for which no errors were found. - * @param vinfo + * @param vinfo validation info for a test that was error free */ public void errorFree(ValidatorInfo vinfo) { errorFreeValidations.add(vinfo); @@ -44,6 +44,9 @@ public void write(Writer writer) throws IOException { for (var result : this.errors.values()) { writer.write(String.join("\t",result.getFields()) + "\n"); } + for (var vinfo : this.errorFreeValidations) { + writer.write(String.join("\t",List.of(vinfo.validatorName().toString(), "no errors", vinfo.validatorId().toString())) + "\n"); + } } diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidatorRegistry.java b/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidatorRegistry.java new file mode 100644 index 0000000..ef01ad4 --- /dev/null +++ b/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidatorRegistry.java @@ -0,0 +1,25 @@ +package org.phenopackets.validator.core; + + +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public class DefaultValidatorRegistry implements PhenopacketValidatorRegistry { + + private final Map validatorMap; + + public DefaultValidatorRegistry(Map validMap) { + validatorMap = Map.copyOf(validMap); + } + + @Override + public Optional getValidatorForType(ValidatorInfo type) { + return Optional.ofNullable(validatorMap.get(type)); + } + + @Override + public Set getValidationTypeSet() { + return validatorMap.keySet(); + } +} diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorFactory.java b/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorFactory.java deleted file mode 100644 index c079ccb..0000000 --- a/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorFactory.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.phenopackets.validator.core; - -import java.util.Optional; - -/** - * The phenopacket validator factory provides phenopacket validators designed to perform specific {@link ValidatorInfo}. - *

- * @author Daniel Danis - * @author Peter N Robinson - */ -public interface PhenopacketValidatorFactory { - - Optional getValidatorForType(ValidatorInfo type); - -} diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorRegistry.java b/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorRegistry.java new file mode 100644 index 0000000..99f0443 --- /dev/null +++ b/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorRegistry.java @@ -0,0 +1,22 @@ +package org.phenopackets.validator.core; + +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * The phenopacket validator registry stores available Validators designed to perform specific {@link ValidatorInfo}. + *

+ * @author Daniel Danis + * @author Peter N Robinson + */ +public interface PhenopacketValidatorRegistry { + Optional getValidatorForType(ValidatorInfo type); + + Set getValidationTypeSet(); + + static PhenopacketValidatorRegistry of(Map validMap) { + return new DefaultValidatorRegistry(validMap); + } +} + diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/ClasspathJsonSchemaValidatorFactory.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/ClasspathJsonSchemaValidatorFactory.java deleted file mode 100644 index 22e2896..0000000 --- a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/ClasspathJsonSchemaValidatorFactory.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.phenopackets.validator.jsonschema; - -import org.phenopackets.validator.core.PhenopacketValidatorFactory; -import org.phenopackets.validator.core.ValidatorInfo; -import org.phenopackets.validator.core.except.PhenopacketValidatorRuntimeException; - -import java.io.InputStream; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; - -/** - * Validator factory that uses JSON schema definitions that are bundled within the application (on classpath). - *

- * @author Daniel Danis - * @author Peter N Robinson - */ -public class ClasspathJsonSchemaValidatorFactory implements PhenopacketValidatorFactory { - - private final Map validatorMap; - - /* - public static ClasspathJsonSchemaValidatorFactory defaultValidators() { - Map validatorMap = makeValidatorMap(); - return new ClasspathJsonSchemaValidatorFactory(validatorMap); - } - - private static Map makeValidatorMap() { - return Map.of( - ValidatorInfo.generic(), makeJsonValidator("/schema/phenopacket-generic.json", ValidatorInfo.generic()), - ValidatorInfo.rareDiseaseValidation(), makeJsonValidator("/schema/hpo-rare-disease-schema.json", ValidatorInfo.rareDiseaseValidation()) - ); - } */ - - public static Map genericValidator() { - return Map.of( - ValidatorInfo.generic(), makeJsonValidator("/schema/phenopacket-generic.json", ValidatorInfo.generic()) - ); - } - - public static Map rareHpoValidator() { - return Map.of( - ValidatorInfo.generic(), makeJsonValidator("/schema/hpo-rare-disease-schema.json", ValidatorInfo.generic()) - ); - } - - private static JsonSchemaValidator makeJsonValidator(String schemaPath, ValidatorInfo validationName) { - InputStream inputStream = ClasspathJsonSchemaValidatorFactory.class.getResourceAsStream(schemaPath); - if (inputStream == null) - throw new PhenopacketValidatorRuntimeException("Invalid JSON schema path `" + schemaPath + '`'); - - return JsonSchemaValidator.of(inputStream, validationName); - } - - private ClasspathJsonSchemaValidatorFactory(Map validatorMap) { - this.validatorMap = validatorMap; - } - - @Override - public Optional getValidatorForType(ValidatorInfo type) { - return Optional.ofNullable(validatorMap.get(type)); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - ClasspathJsonSchemaValidatorFactory that = (ClasspathJsonSchemaValidatorFactory) o; - return Objects.equals(validatorMap, that.validatorMap); - } - - @Override - public int hashCode() { - return Objects.hash(validatorMap); - } - -} diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java index 5b12b3a..791f8d9 100644 --- a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java @@ -40,7 +40,7 @@ public class JsonSchemaValidator implements PhenopacketValidator { * @throws PhenopacketValidatorRuntimeException if jsonSchema is not a valid file or if the file is * not a valid JSON schema specification */ - public static JsonSchemaValidator of(File jsonSchema, ValidatorInfo validatorInfo) { + public static PhenopacketValidator of(File jsonSchema, ValidatorInfo validatorInfo) { if (!jsonSchema.isFile()) { throw new PhenopacketValidatorRuntimeException("Could not open file at \"" + jsonSchema.getAbsolutePath() + "\""); } @@ -52,7 +52,7 @@ public static JsonSchemaValidator of(File jsonSchema, ValidatorInfo validatorInf } } - public static JsonSchemaValidator of(InputStream inputStream, ValidatorInfo validatorInfo) { + public static PhenopacketValidator of(InputStream inputStream, ValidatorInfo validatorInfo) { JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(VERSION_FLAG); return new JsonSchemaValidator(schemaFactory.getSchema(inputStream), validatorInfo); } diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorFactory.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorFactory.java new file mode 100644 index 0000000..87105a0 --- /dev/null +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorFactory.java @@ -0,0 +1,39 @@ +package org.phenopackets.validator.jsonschema; + +import org.phenopackets.validator.core.PhenopacketValidator; +import org.phenopackets.validator.core.ValidatorInfo; +import org.phenopackets.validator.core.except.PhenopacketValidatorRuntimeException; + +import java.io.InputStream; +import java.util.Map; + +/** + * Validator factory that uses JSON schema definitions that are bundled within the application (on classpath). + *

+ * @author Daniel Danis + * @author Peter N Robinson + */ +public class JsonSchemaValidatorFactory { + + public static Map genericValidator() { + return Map.of( + ValidatorInfo.generic(), makeJsonValidator("/schema/phenopacket-generic.json", ValidatorInfo.generic()) + ); + } + + public static Map rareHpoValidator() { + return Map.of( + ValidatorInfo.generic(), makeJsonValidator("/schema/hpo-rare-disease-schema.json", ValidatorInfo.generic()) + ); + } + + private static PhenopacketValidator makeJsonValidator(String schemaPath, ValidatorInfo validationName) { + InputStream inputStream = JsonSchemaValidatorFactory.class.getResourceAsStream(schemaPath); + if (inputStream == null) + throw new PhenopacketValidatorRuntimeException("Invalid JSON schema path `" + schemaPath + '`'); + + return JsonSchemaValidator.of(inputStream, validationName); + } + + +} diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java index 78081d4..c5dc1e1 100644 --- a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java @@ -17,6 +17,7 @@ public final class JsonValidationItem implements ValidationItem { private final ValidationItemType itemType; private final String message; + public JsonValidationItem(ValidatorInfo validatorInfo, ValidationMessage validationMessage) { this.validatorInfo = validatorInfo; this.itemType = JsonValidationItemType.stringToErrorType(validationMessage.getType()); diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemType.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemType.java index 2511241..4c77752 100644 --- a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemType.java +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemType.java @@ -11,15 +11,15 @@ public class JsonValidationItemType implements ValidationItemType { /** * JSON schema error meaning that the JSON code contained a property not present in the schema. */ - public final static JsonValidationItemType JSON_ADDITIONAL_PROPERTIES = + public final static ValidationItemType JSON_ADDITIONAL_PROPERTIES = new JsonValidationItemType("additionalProperties", "Use of a property not present in the schema"); /** JSON schema error meaning that the JSON code failed to contain a property required by the schema. */ - public final static JsonValidationItemType JSON_REQUIRED = + public final static ValidationItemType JSON_REQUIRED = new JsonValidationItemType("required", "failure to contain a property required by the schema"); /** The type of an object is not as required by the schema, e.g., we get a string instead of an array. */ - public final static JsonValidationItemType JSON_TYPE = + public final static ValidationItemType JSON_TYPE = new JsonValidationItemType("type", "incorrect data type of object"); - public final static JsonValidationItemType JSON_ENUM= + public final static ValidationItemType JSON_ENUM= new JsonValidationItemType("enum", "TODO"); @@ -41,7 +41,7 @@ public String itemDescription() { return description; } - public static JsonValidationItemType stringToErrorType(String error) { + public static ValidationItemType stringToErrorType(String error) { switch (error) { case "additionalProperties": return JSON_ADDITIONAL_PROPERTIES; case "required": return JSON_REQUIRED; diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java index fac7ef5..7e8cf65 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java @@ -9,6 +9,7 @@ import org.phenopackets.schema.v2.core.MetaData; import org.phenopackets.schema.v2.core.Resource; import org.phenopackets.schema.v2.core.TimeElement; +import org.phenopackets.validator.core.PhenopacketValidator; import org.phenopackets.validator.core.ValidationItem; import org.phenopackets.validator.core.ValidatorInfo; import org.phenopackets.validator.testdatagen.PhenopacketUtil; @@ -28,7 +29,7 @@ */ public class JsonSchemaDiseaseValidatorTest { - private static final Map jsonValidatorMap = ClasspathJsonSchemaValidatorFactory.genericValidator(); + private static final Map jsonValidatorMap = JsonSchemaValidatorFactory.genericValidator(); private static Disease mondoDisease() { var chagas = ontologyClass("MONDO:0005491", "Chagas cardiomyopathy"); @@ -63,7 +64,7 @@ private static Phenopacket phenopacketWithDisease() { @Test public void testPhenopacketValidity() throws InvalidProtocolBufferException { - JsonSchemaValidator validator = jsonValidatorMap.values().stream() + PhenopacketValidator validator = jsonValidatorMap.values().stream() .findFirst() .get(); String json = JsonFormat.printer().print(phenopacket); @@ -73,7 +74,7 @@ public void testPhenopacketValidity() throws InvalidProtocolBufferException { @Test public void testLacksId() throws InvalidProtocolBufferException { - JsonSchemaValidator validator = jsonValidatorMap.values().stream() + PhenopacketValidator validator = jsonValidatorMap.values().stream() .findFirst() .get(); // the Phenopacket is not valid if we remove the id diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java index 94fe13b..0cde29d 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java @@ -2,6 +2,7 @@ import org.junit.jupiter.api.Test; import org.phenopackets.schema.v2.Phenopacket; +import org.phenopackets.validator.core.PhenopacketValidator; import org.phenopackets.validator.testdatagen.RareDiseasePhenopacket; import org.phenopackets.validator.testdatagen.SimplePhenopacket; import org.phenopackets.validator.core.ValidationItem; @@ -21,8 +22,8 @@ public class JsonSchemaValidatorTest { - private static final Map genericValidatorMap = ClasspathJsonSchemaValidatorFactory.genericValidator(); - private static final Map rareHpoValidatorMap = ClasspathJsonSchemaValidatorFactory.rareHpoValidator(); + private static final Map genericValidatorMap = JsonSchemaValidatorFactory.genericValidator(); + private static final Map rareHpoValidatorMap = JsonSchemaValidatorFactory.rareHpoValidator(); private static final SimplePhenopacket simplePhenopacket = new SimplePhenopacket(); @@ -35,7 +36,7 @@ private static File fileFromClasspath(String path) { @Test public void testValidationOfSimpleValidPhenopacket() throws Exception { - JsonSchemaValidator validator = genericValidatorMap.values().stream() + PhenopacketValidator validator = genericValidatorMap.values().stream() .findFirst() .get(); Phenopacket phenopacket = simplePhenopacket.getPhenopacket(); @@ -57,8 +58,8 @@ public void testValidationOfSimpleValidPhenopacket() throws Exception { * It does not contain an id or a metaData element and thus should fail. */ @Test - public void testValidationOfSimpleInValidPhenopacket() throws Exception { - JsonSchemaValidator validator = genericValidatorMap.values().stream() + public void testValidationOfSimpleInValidPhenopacket() { + PhenopacketValidator validator = genericValidatorMap.values().stream() .findFirst() .get(); @@ -80,7 +81,7 @@ public void testValidationOfSimpleInValidPhenopacket() throws Exception { @Test public void testRareDiseaseBethlemahmValidPhenopacket() throws Exception { - JsonSchemaValidator validator = rareHpoValidatorMap.values().stream() + PhenopacketValidator validator = rareHpoValidatorMap.values().stream() .findFirst() .get(); @@ -93,7 +94,7 @@ public void testRareDiseaseBethlemahmValidPhenopacket() throws Exception { @Test public void testRareDiseaseBethlemahmInvalidValidPhenopacket() throws IOException { - JsonSchemaValidator validator = rareHpoValidatorMap.values().stream() + PhenopacketValidator validator = rareHpoValidatorMap.values().stream() .findFirst() .get(); File invalidMyopathyPhenopacket = fileFromClasspath("json/bethlehamMyopathyInvalidExample.json"); diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java index 2726300..abb3d3f 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java @@ -5,7 +5,7 @@ import org.phenopackets.schema.v2.core.Individual; import org.phenopackets.schema.v2.core.MetaData; import org.phenopackets.schema.v2.core.Resource; -import org.phenopackets.validator.testdatagen.PhenopacketUtil.PhenotypicFeatureBuilder; + import static org.phenopackets.validator.testdatagen.PhenopacketUtil.*; public class RareDiseasePhenopacket { diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemType.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemType.java index 3c5ed73..4246ec7 100644 --- a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemType.java +++ b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemType.java @@ -12,14 +12,14 @@ public class OntologyValidationItemType implements ValidationItemType { * This error type refers to the use of a term ID, e.g., HP:1234567, which is syntactically correct but which * does not existing in the actual ontology. */ - public final static OntologyValidationItemType ONTOLOGY_INVALID_ID = + public final static ValidationItemType ONTOLOGY_INVALID_ID = new OntologyValidationItemType("Invalid Ontology Term ID", "Term id does not exist in ontology"); /** * This error type refers to the use of a term ID that is an alternate_id, i.e., an outdated ID that should * be replaced by the primary ID of the corresponding Ontology term. */ - public final static OntologyValidationItemType ONTOLOGY_TERM_WITH_ALTERNATE_ID = + public final static ValidationItemType ONTOLOGY_TERM_WITH_ALTERNATE_ID = new OntologyValidationItemType("Use of outdated Ontology Term ID", "Term id is not the primary id for this term"); diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidatorFactory.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidatorFactory.java deleted file mode 100644 index 9df442b..0000000 --- a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidatorFactory.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.phenopackets.validator.ontology; - -import org.phenopackets.validator.core.PhenopacketValidator; -import org.phenopackets.validator.core.PhenopacketValidatorFactory; -import org.phenopackets.validator.core.ValidatorInfo; -import org.phenopackets.validator.core.except.PhenopacketValidatorRuntimeException; - -import java.io.File; -import java.io.InputStream; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -public class OntologyValidatorFactory implements PhenopacketValidatorFactory { - - private final Map validatorMap; - - private static final String ONTOLOGY_VALIDATOR_TYPE = "Semantic validation"; - - public OntologyValidatorFactory(File... ontologyJsonFiles) { - validatorMap = new HashMap<>(); - for (var jsonFile: ontologyJsonFiles) { - - } - } - - - @Override - public Optional getValidatorForType(ValidatorInfo type) { - if (validatorMap.containsKey(type)) { - return Optional.of(validatorMap.get(type)); - } - return Optional.empty(); - } -} From 75d3fb626eeb5a93f93a2b111501b28e879b00ed Mon Sep 17 00:00:00 2001 From: Daniel Danis Date: Thu, 30 Sep 2021 16:51:25 -0400 Subject: [PATCH 20/37] Work on `validator-ontology` - extend `HpoValidator` checks + add tests - move `properties` into top-level `pom.xml` - remove `phenol-io` as the `validator-ontology` is not responsible for preparing the ontology - remove `json-simple` as it does not really bring any value - add `dependencyManagement` --- pom.xml | 60 +++++- validator-cli/pom.xml | 5 +- .../validator/cli/ValidatorApplication.java | 7 +- .../validator/core/ValidationItem.java | 8 +- .../validator/core/ValidationItemDefault.java | 3 + .../validator/core/ValidationItemType.java | 5 + .../core/ValidationItemTypeDefault.java | 3 + .../validator/core/ValidationItemTypes.java | 13 ++ validator-jsonschema/pom.xml | 21 --- validator-ontology/pom.xml | 112 +---------- .../validator/ontology/HpoValidator.java | 130 +++++++------ .../ontology/OntologyValidationItem.java | 47 ----- .../ontology/OntologyValidationItemType.java | 45 ----- .../ontology/OntologyValidationItemTypes.java | 29 +++ .../validator/ontology/OntologyValidator.java | 85 ++++++++- .../validator/ontology/HpoValidatorTest.java | 178 ++++++++++++++++++ 16 files changed, 452 insertions(+), 299 deletions(-) create mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemDefault.java create mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemTypeDefault.java create mode 100644 validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemTypes.java delete mode 100644 validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItem.java delete mode 100644 validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemType.java create mode 100644 validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemTypes.java create mode 100644 validator-ontology/src/test/java/org/phenopackets/validator/ontology/HpoValidatorTest.java diff --git a/pom.xml b/pom.xml index f790f70..6f2d031 100644 --- a/pom.xml +++ b/pom.xml @@ -64,6 +64,7 @@ UTF-8 11 + 1.6.3 3.14.0 2.0.0 @@ -89,6 +90,50 @@ + + + + org.monarchinitiative.phenol + phenol-core + ${phenol.version} + + + org.monarchinitiative.phenol + phenol-io + ${phenol.version} + + + org.phenopackets + phenopacket-schema + ${phenopacket-schema.version} + + + com.google.protobuf + protobuf-java-util + ${protobuf.version} + + + com.google.guava + guava + + + + + + com.networknt + json-schema-validator + 1.0.42 + + + + + org.mockito + mockito-core + 3.12.4 + + + + @@ -101,11 +146,16 @@ junit-jupiter test - - org.hamcrest - hamcrest-core - test - + + org.hamcrest + hamcrest-core + test + + + org.mockito + mockito-core + test + org.slf4j slf4j-simple diff --git a/validator-cli/pom.xml b/validator-cli/pom.xml index f3fc4af..03e0715 100644 --- a/validator-cli/pom.xml +++ b/validator-cli/pom.xml @@ -26,12 +26,15 @@ validator-ontology ${project.parent.version} + + org.monarchinitiative.phenol + phenol-io + info.picocli picocli 4.6.1 - diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java index 2e8970a..8cc1df2 100644 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java @@ -1,5 +1,7 @@ package org.phenopackets.validator.cli; +import org.monarchinitiative.phenol.io.OntologyLoader; +import org.monarchinitiative.phenol.ontology.data.Ontology; import org.phenopackets.validator.cli.results.ValidationItemTsvVisualizer; import org.phenopackets.validator.cli.results.ValidationTsvVisualizer; import org.phenopackets.validator.core.PhenopacketValidator; @@ -30,7 +32,7 @@ public class ValidatorApplication implements Runnable { public List jsonSchemaFiles = List.of(); @CommandLine.Option(names = "--hp", required = true, description = "check with HPO (hp.json)") - public String hpoJsonPath; + public File hpoJsonPath; @CommandLine.Option(names = {"-p", "--phenopacket"}, required = true, description = "Phenopacket file to be validated") public String phenopacket; @@ -68,7 +70,8 @@ public void run() { validatorMap.put(vinfo, jvalid); LOGGER.info("Adding configuration file at `{}`", vinfo); } - OntologyValidator hpoValidator = new HpoValidator(new File(hpoJsonPath)); + Ontology hpoOntology = OntologyLoader.loadOntology(hpoJsonPath); + OntologyValidator hpoValidator = new HpoValidator(hpoOntology); validatorMap.put(hpoValidator.info(), hpoValidator); PhenopacketValidatorRegistry registry = PhenopacketValidatorRegistry.of(validatorMap); diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItem.java b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItem.java index a6c67af..44efe0f 100644 --- a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItem.java +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItem.java @@ -8,12 +8,18 @@ */ public interface ValidationItem { + static ValidationItem of(ValidatorInfo validatorInfo, ValidationItemType type, String message) { + return new ValidationItemDefault(validatorInfo, type, message); + } + /** * @return basic description of the validator that produced this issue. */ ValidatorInfo validatorInfo(); - // TODO - decide which enum to use here - either ErrorType or ValidationAspect + /** + * @return description of the issue in a standard way intended for both grouping of the issues and human consumption. + */ ValidationItemType type(); /** diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemDefault.java b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemDefault.java new file mode 100644 index 0000000..34f8860 --- /dev/null +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemDefault.java @@ -0,0 +1,3 @@ +package org.phenopackets.validator.core; + +record ValidationItemDefault(ValidatorInfo validatorInfo, ValidationItemType type, String message) implements ValidationItem {} diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemType.java b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemType.java index cd4f775..ac8489d 100644 --- a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemType.java +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemType.java @@ -2,7 +2,12 @@ public interface ValidationItemType { + static ValidationItemType of(String itemType, String itemDescription) { + return new ValidationItemTypeDefault(itemType, itemDescription); + } + String itemType(); + String itemDescription(); } diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemTypeDefault.java b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemTypeDefault.java new file mode 100644 index 0000000..5f30f99 --- /dev/null +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemTypeDefault.java @@ -0,0 +1,3 @@ +package org.phenopackets.validator.core; + +record ValidationItemTypeDefault(String itemType, String itemDescription) implements ValidationItemType {} diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemTypes.java b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemTypes.java new file mode 100644 index 0000000..2499929 --- /dev/null +++ b/validator-core/src/main/java/org/phenopackets/validator/core/ValidationItemTypes.java @@ -0,0 +1,13 @@ +package org.phenopackets.validator.core; + +public class ValidationItemTypes { + + private static final ValidationItemType SYNTAX_ERROR = ValidationItemType.of("Syntax error", "The input does not conform the with phenopacket syntax"); + + private ValidationItemTypes() {} + + public static ValidationItemType syntaxError() { + return SYNTAX_ERROR; + } + +} diff --git a/validator-jsonschema/pom.xml b/validator-jsonschema/pom.xml index 23b1675..416c586 100644 --- a/validator-jsonschema/pom.xml +++ b/validator-jsonschema/pom.xml @@ -21,38 +21,17 @@ com.networknt json-schema-validator - 1.0.42 - - - org.prefixcommons - curie-util - 0.0.2 - - - org.phenopackets phenopacket-schema - ${phenopacket-schema.version} - test - - - com.google.protobuf - protobuf-java - ${protobuf.version} test com.google.protobuf protobuf-java-util - ${protobuf.version} test diff --git a/validator-ontology/pom.xml b/validator-ontology/pom.xml index a34ed71..3bd16a5 100644 --- a/validator-ontology/pom.xml +++ b/validator-ontology/pom.xml @@ -8,10 +8,6 @@ 0.1.1-SNAPSHOT 4.0.0 - - 1.6.3 - 2.0.0 - validator-ontology @@ -21,120 +17,20 @@ validator-core ${project.parent.version} - org.monarchinitiative.phenol phenol-core - ${phenol.version} - - - - org.monarchinitiative.phenol - phenol-io - ${phenol.version} - - - - com.googlecode.json-simple - json-simple - 1.1.1 - - - - com.google.protobuf - protobuf-java - 3.11.4 - - - com.google.protobuf - protobuf-java-util - 3.11.4 - - - com.google.guava - guava - - - org.phenopackets phenopacket-schema - ${phenopacket.version} - compile - - - com.google.protobuf - protobuf-java - - - com.fasterxml.jackson.core - jackson-databind - - - com.fasterxml.jackson.core - jackson-core - - - com.fasterxml.jackson.core - jackson-annotations - - - com.fasterxml.jackson.dataformat - jackson-dataformat-protobuf - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - org.yaml - snakeyaml - - - com.google.guava - guava - - - com.google.errorprone - error_prone_annotations - - - org.slf4j - slf4j-api - - - org.apache.logging.log4j - log4j-slf4j18-impl - - - commons-collections - commons-collections - - - commons-logging - commons-logging - - - -com.google.protobuf - protobuf-java-util - - - - - + - org.prefixcommons - curie-util - 0.0.2 + com.google.protobuf + protobuf-java-util - - \ No newline at end of file diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/HpoValidator.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/HpoValidator.java index e431d3e..b4d4dd4 100644 --- a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/HpoValidator.java +++ b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/HpoValidator.java @@ -1,36 +1,53 @@ package org.phenopackets.validator.ontology; import com.google.protobuf.util.JsonFormat; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; import org.monarchinitiative.phenol.base.PhenolRuntimeException; +import org.monarchinitiative.phenol.ontology.data.Ontology; import org.monarchinitiative.phenol.ontology.data.TermId; import org.phenopackets.schema.v2.Phenopacket; +import org.phenopackets.schema.v2.core.PhenotypicFeature; import org.phenopackets.validator.core.ValidationItem; +import org.phenopackets.validator.core.ValidationItemTypes; import org.phenopackets.validator.core.ValidatorInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; - - +/** + * The validator checks the Phenotypic features for HPO terms. If HPO terms + * are used there, then we check that the HPO terms are included in the ontology (thus, we check for + * invalid HPO ids, and also check if the id used is the latest or if it is an alternate id). + *

+ * The validator performs logical checks too. Due to the annotation propagation rule, where presence of a child term + * implies presence of all its ancestors, it is an error to specify both a child term and its parent, and only the + * child term must be used. The rule is checked for both observed and excluded terms. + *

+ * In addition, if there are both observed and excluded terms, we check that there is no logical inconsistency, e.g., + * NOT Abnormal liver morphology but OBSERVED Hepatic fibrosis. The latter is not possible because + * Hepatic fibrosis implies Abnormal liver morphology. + */ public class HpoValidator extends OntologyValidator { private static final Logger LOGGER = LoggerFactory.getLogger(HpoValidator.class); private static final String HPO_PREFIX = "HP"; - public HpoValidator(File jsonFile) { - super(jsonFile, ValidatorInfo.of(ONTOLOGY_VALIDATOR, "Human Phenotype Ontology")); + public HpoValidator(Ontology hpoOntology) { + super(hpoOntology, ValidatorInfo.of(ONTOLOGY_VALIDATOR, "Human Phenotype Ontology")); + } + + private static Phenopacket readPhenopacket(InputStream inputStream) throws IOException { + String phenopacketJsonString = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); + Phenopacket.Builder phenoPacketBuilder = Phenopacket.newBuilder(); + JsonFormat.parser().merge(phenopacketJsonString, phenoPacketBuilder); + return phenoPacketBuilder.build(); } @Override @@ -38,67 +55,56 @@ public ValidatorInfo info() { return validatorInfo; } - - - /** - * The logic of this validator is to check the Phenotypic features for HPO terms. If HPO terms - * are used there, the we check that the HPO terms are included in the ontology (thus, we check for - * invalid HPO ids, and also check if the id used is the latest or if it is an alternate id). If there - * are both observed and excluded terms, we check that there is no logical inconsistency, e.g., - * NOT Abnormal liver morphology but OBSERVED Hepatic fibrosis. The latter is not possible because - * Hepatic fibrosis implies Abnormal liver morphology - * @param inputStream -- stream from the phenopacket. - * @return list of validation errors - */ @Override public List validate(InputStream inputStream) { + Phenopacket phenopacket; + try { + phenopacket = readPhenopacket(inputStream); + } catch (IOException e) { + LOGGER.warn("Error while validating: {}", e.getMessage(), e); + return List.of(ValidationItem.of(validatorInfo, ValidationItemTypes.syntaxError(), e.getMessage())); + } + List errors = new ArrayList<>(); - JSONParser parser = new JSONParser(); Set observedFeatures = new HashSet<>(); Set excludedFeatures = new HashSet<>(); - try { - Object obj = parser.parse(new InputStreamReader(inputStream)); - JSONObject jsonObject = (JSONObject) obj; - String phenopacketJsonString = jsonObject.toJSONString(); - Phenopacket.Builder phenoPacketBuilder = Phenopacket.newBuilder(); - JsonFormat.parser().merge(phenopacketJsonString, phenoPacketBuilder); - Phenopacket phenopacket = phenoPacketBuilder.build(); - for (var pf : phenopacket.getPhenotypicFeaturesList()) { - String id = pf.getType().getId(); - try { - TermId tid = TermId.of(id); - if (!tid.getPrefix().equals(HPO_PREFIX)) { - continue; // only check HPO terms - } - if (pf.getExcluded()) { - excludedFeatures.add(tid); - } else { - observedFeatures.add(tid); - } - } catch (PhenolRuntimeException pe) { - LOGGER.error("Invalid Ontology term id: " + id); + + partitionTermsToObservedAndExcluded(phenopacket.getPhenotypicFeaturesList(), errors, observedFeatures, excludedFeatures); + + checkTermsArePresentInOntology(observedFeatures, errors); + checkTermsUsePrimaryId(observedFeatures, errors); + + checkTermsArePresentInOntology(excludedFeatures, errors); + checkTermsUsePrimaryId(excludedFeatures, errors); + + checkAncestry(observedFeatures, excludedFeatures, errors); + + return errors; + } + + private void partitionTermsToObservedAndExcluded(Iterable phenotypicFeatures, + List errors, + Set observedFeatures, + Set excludedFeatures) { + for (var pf : phenotypicFeatures) { + TermId tid; + String id = pf.getType().getId(); + try { + tid = TermId.of(id); + if (!tid.getPrefix().equals(HPO_PREFIX)) { + continue; // only check HPO terms } + } catch (PhenolRuntimeException pe) { + String message = String.format("Invalid ontology term id: %s", id); + errors.add(ValidationItem.of(validatorInfo, OntologyValidationItemTypes.ONTOLOGY_INVALID_ID, message)); + continue; } - // 1 check that all terms are in the ontology and use the primary id - for (TermId tid: observedFeatures) { - if (!ontology.containsTerm(tid)) { - String message = String.format("Could not find term id: %s", tid.getValue()); - ValidationItem vitem = OntologyValidationItem.invalidTermId(validatorInfo,message); - errors.add(vitem); - } else { - TermId primaryId = ontology.getPrimaryTermId(tid); - if (! primaryId.equals(tid)) { - String message = String.format("Using alternate (obsolete) id (%s) instead of primary id (%s)", - tid.getValue(), primaryId); - ValidationItem vitem = OntologyValidationItem.invalidTermId(validatorInfo,message); - errors.add(vitem); - } - } + + if (pf.getExcluded()) { + excludedFeatures.add(tid); + } else { + observedFeatures.add(tid); } - return errors; - } catch (IOException | RuntimeException | ParseException e) { - LOGGER.warn("Error while validating: {}", e.getMessage()); } - return errors; } } diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItem.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItem.java deleted file mode 100644 index 4803d58..0000000 --- a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItem.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.phenopackets.validator.ontology; - -import org.phenopackets.validator.core.ValidationItem; -import org.phenopackets.validator.core.ValidationItemType; -import org.phenopackets.validator.core.ValidatorInfo; - -import static org.phenopackets.validator.ontology.OntologyValidationItemType.ONTOLOGY_INVALID_ID; -import static org.phenopackets.validator.ontology.OntologyValidationItemType.ONTOLOGY_TERM_WITH_ALTERNATE_ID; - -public class OntologyValidationItem implements ValidationItem { - - private final ValidatorInfo validatorInfo; - private final ValidationItemType itemType; - private final String message; - - public OntologyValidationItem(ValidatorInfo validatorInfo, ValidationItemType errorType, String validationMessage) { - this.validatorInfo = validatorInfo; - this.itemType = errorType; - this.message = validationMessage; - } - - - @Override - public ValidatorInfo validatorInfo() { - return validatorInfo; - } - - @Override - public ValidationItemType type() { - return itemType; - } - - @Override - public String message() { - return message; - } - - public static OntologyValidationItem invalidTermId(ValidatorInfo validatorInfo,String validationMessage) { - return new OntologyValidationItem(validatorInfo, ONTOLOGY_INVALID_ID, validationMessage); - } - - public static OntologyValidationItem alternateId(ValidatorInfo validatorInfo,String validationMessage) { - return new OntologyValidationItem(validatorInfo, ONTOLOGY_TERM_WITH_ALTERNATE_ID, validationMessage); - } - - -} diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemType.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemType.java deleted file mode 100644 index 4246ec7..0000000 --- a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemType.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.phenopackets.validator.ontology; - -import org.phenopackets.validator.core.ValidationItemType; - -public class OntologyValidationItemType implements ValidationItemType { - - - private final String itemType; - private final String description; - - /** - * This error type refers to the use of a term ID, e.g., HP:1234567, which is syntactically correct but which - * does not existing in the actual ontology. - */ - public final static ValidationItemType ONTOLOGY_INVALID_ID = - new OntologyValidationItemType("Invalid Ontology Term ID", "Term id does not exist in ontology"); - - /** - * This error type refers to the use of a term ID that is an alternate_id, i.e., an outdated ID that should - * be replaced by the primary ID of the corresponding Ontology term. - */ - public final static ValidationItemType ONTOLOGY_TERM_WITH_ALTERNATE_ID = - new OntologyValidationItemType("Use of outdated Ontology Term ID", "Term id is not the primary id for this term"); - - - OntologyValidationItemType(String type, String desc) { - itemType = type; - description = desc; - } - - - @Override - public String itemType() { - return itemType; - } - - @Override - public String itemDescription() { - return description; - } - - - - -} diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemTypes.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemTypes.java new file mode 100644 index 0000000..45cc3b0 --- /dev/null +++ b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidationItemTypes.java @@ -0,0 +1,29 @@ +package org.phenopackets.validator.ontology; + +import org.phenopackets.validator.core.ValidationItemType; + +class OntologyValidationItemTypes { + + /** + * This error type refers to the use of a term ID, e.g., HP:1234567, which is syntactically correct but which + * does not existing in the actual ontology. + */ + static final ValidationItemType ONTOLOGY_INVALID_ID = + ValidationItemType.of("Invalid Ontology Term ID", "Term id does not exist in ontology"); + + /** + * This error type refers to the use of a term ID that is an alternate_id, i.e., an outdated ID that should + * be replaced by the primary ID of the corresponding Ontology term. + */ + static final ValidationItemType ONTOLOGY_TERM_WITH_ALTERNATE_ID = + ValidationItemType.of("Use of outdated Ontology Term ID", "Term id is not the primary id for this term"); + + static final ValidationItemType TERM_AND_ANCESTOR_ARE_USED = + ValidationItemType.of("Term and ancestor are used", "Only the child/specific term should be used"); + + static final ValidationItemType TERM_IS_USED_AND_ANCESTOR_IS_EXCLUDED = + ValidationItemType.of("Term is used while ancestor is excluded", "The term is used despite the ancestor term has been excluded"); + + private OntologyValidationItemTypes() {} + +} diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidator.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidator.java index cbaff46..d779fa6 100644 --- a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidator.java +++ b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidator.java @@ -1,29 +1,100 @@ package org.phenopackets.validator.ontology; import org.monarchinitiative.phenol.ontology.data.Ontology; +import org.monarchinitiative.phenol.ontology.data.TermId; import org.phenopackets.validator.core.PhenopacketValidator; +import org.phenopackets.validator.core.ValidationItem; import org.phenopackets.validator.core.ValidatorInfo; -import org.monarchinitiative.phenol.io.OntologyLoader; - - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; +import java.util.List; +import java.util.Set; public abstract class OntologyValidator implements PhenopacketValidator { private static final Logger LOGGER = LoggerFactory.getLogger(OntologyValidator.class); + protected final static String ONTOLOGY_VALIDATOR = "Ontology validation"; + protected final ValidatorInfo validatorInfo; protected final Ontology ontology; protected final static String ONTOLOGY_VALIDATOR = "Otology validation"; - protected OntologyValidator(File jsonFile, ValidatorInfo vinfo) { - validatorInfo = vinfo; - ontology = OntologyLoader.loadOntology(jsonFile); + protected OntologyValidator(Ontology ontology, ValidatorInfo vinfo) { + this.ontology = ontology; + this.validatorInfo = vinfo; + } + + protected void checkTermsArePresentInOntology(Iterable termIds, List errors) { + for (TermId termId : termIds) { + if (!ontology.containsTerm(termId)) { + String message = String.format("Could not find term id: %s", termId.getValue()); + ValidationItem item = ValidationItem.of(validatorInfo, OntologyValidationItemTypes.ONTOLOGY_INVALID_ID, message); + errors.add(item); + } + } + } + + protected void checkTermsUsePrimaryId(Iterable termIds, List errors) { + for (TermId termId : termIds) { + TermId primaryId = ontology.getPrimaryTermId(termId); + if (primaryId == null) + // not in ontology, this is checked in `checkTermsArePresentInOntology` + continue; + if (!primaryId.equals(termId)) { + String message = String.format("Using alternate (obsolete) id (%s) instead of primary id (%s)", termId.getValue(), primaryId.getValue()); + ValidationItem item = ValidationItem.of(validatorInfo, OntologyValidationItemTypes.ONTOLOGY_TERM_WITH_ALTERNATE_ID, message); + errors.add(item); + } + } } + protected void checkAncestry(Set observedTerms, Set excludedTerms, List errors) { + /* + OBSERVED TERMS: + It is an error to: + - provide term and its ancestor, + - provide term while the ancestor is excluded. + */ + for (TermId term : observedTerms) { + Set ancestors = ontology.getAncestorTermIds(term); + + for (TermId ancestor : ancestors) { + if (ancestor.equals(term)) + continue; + + if (observedTerms.contains(ancestor)) { + String message = String.format("Using both term %s and its ancestor %s", term.getValue(), ancestor.getValue()); + errors.add(ValidationItem.of(validatorInfo, OntologyValidationItemTypes.TERM_AND_ANCESTOR_ARE_USED, message)); + } + + if (excludedTerms.contains(ancestor)) { + String message = String.format("Term %s is present while its ancestor %s is excluded", term.getValue(), ancestor.getValue()); + errors.add(ValidationItem.of(validatorInfo, OntologyValidationItemTypes.TERM_IS_USED_AND_ANCESTOR_IS_EXCLUDED, message)); + } + } + } + + /* + EXCLUDED TERMS: + It is an error to provide term and its ancestor. It is, however, OK to provide term while the ancestor is present, + i.e. `Arachnodactyly` is excluded, while `Long fingers` are present. + */ + for (TermId term : excludedTerms) { + Set ancestors = ontology.getAncestorTermIds(term); + + for (TermId ancestor : ancestors) { + if (ancestor.equals(term)) + continue; + + if (excludedTerms.contains(ancestor)) { + String message = String.format("Using both term %s and its ancestor %s", term.getValue(), ancestor.getValue()); + errors.add(ValidationItem.of(validatorInfo, OntologyValidationItemTypes.TERM_AND_ANCESTOR_ARE_USED, message)); + } + } + } + } } diff --git a/validator-ontology/src/test/java/org/phenopackets/validator/ontology/HpoValidatorTest.java b/validator-ontology/src/test/java/org/phenopackets/validator/ontology/HpoValidatorTest.java new file mode 100644 index 0000000..909cad4 --- /dev/null +++ b/validator-ontology/src/test/java/org/phenopackets/validator/ontology/HpoValidatorTest.java @@ -0,0 +1,178 @@ +package org.phenopackets.validator.ontology; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.monarchinitiative.phenol.ontology.data.Ontology; +import org.monarchinitiative.phenol.ontology.data.TermId; +import org.phenopackets.schema.v2.Phenopacket; +import org.phenopackets.schema.v2.core.OntologyClass; +import org.phenopackets.schema.v2.core.PhenotypicFeature; +import org.phenopackets.validator.core.ValidationItem; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Set; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class HpoValidatorTest { + + public Ontology ontology; + + public HpoValidator validator; + + private static byte[] wrapInPhenopacketAndGetJsonBytes(List phenotypicFeatures) throws InvalidProtocolBufferException { + Phenopacket phenopacket = Phenopacket.newBuilder() + .addAllPhenotypicFeatures(phenotypicFeatures) + .build(); + return JsonFormat.printer().print(phenopacket).getBytes(StandardCharsets.UTF_8); + } + + private static PhenotypicFeature makePhenotypicFeature(String termId, boolean excluded) { + return PhenotypicFeature.newBuilder() + .setType(OntologyClass.newBuilder().setId(termId).build()) + .setExcluded(excluded) + .build(); + } + + // set up the Ontology mocks. There is no better way to test this unless we stash ~2mb hp.json.gz in the repo + private static Ontology prepareMiniOntology() { + Ontology ontology = mock(Ontology.class); + TermId one = TermId.of("HP:0000001"); // the root + TermId two = TermId.of("HP:0000002"); // one's child + TermId three = TermId.of("HP:0000003"); // one's child + TermId four = TermId.of("HP:0000004"); // three's child + TermId five = TermId.of("HP:0000005"); // three's child + when(ontology.containsTerm(one)).thenReturn(true); + when(ontology.getPrimaryTermId(one)).thenReturn(one); + when(ontology.containsTerm(two)).thenReturn(true); + when(ontology.getPrimaryTermId(two)).thenReturn(two); + when(ontology.containsTerm(three)).thenReturn(true); + when(ontology.getPrimaryTermId(three)).thenReturn(three); + when(ontology.containsTerm(four)).thenReturn(true); + when(ontology.getPrimaryTermId(four)).thenReturn(four); + when(ontology.containsTerm(five)).thenReturn(true); + when(ontology.getPrimaryTermId(five)).thenReturn(five); + + when(ontology.getAncestorTermIds(one)).thenReturn(Set.of(one)); + when(ontology.getAncestorTermIds(two)).thenReturn(Set.of(two, one)); + when(ontology.getAncestorTermIds(three)).thenReturn(Set.of(three, one)); + when(ontology.getAncestorTermIds(four)).thenReturn(Set.of(four, three, one)); + when(ontology.getAncestorTermIds(five)).thenReturn(Set.of(five, three, one)); + + return ontology; + } + + @BeforeEach + public void setUp() { + ontology = prepareMiniOntology(); + validator = new HpoValidator(ontology); + } + + @Nested + public class NoErrorsInValidation { + + @Test + public void disjointTerms() throws Exception { + List phenotypicFeatures = List.of( + makePhenotypicFeature("HP:0000002", false), + makePhenotypicFeature("HP:0000005", false)); + byte[] bytes = wrapInPhenopacketAndGetJsonBytes(phenotypicFeatures); + + List items = validator.validate(new ByteArrayInputStream(bytes)); + + assertThat(items, hasSize(0)); + } + + @Test + public void parentIsPresentWhileChildrenAreExcluded() throws Exception { + List phenotypicFeatures = List.of( + makePhenotypicFeature("HP:0000003", false), + makePhenotypicFeature("HP:0000004", true), + makePhenotypicFeature("HP:0000005", true) + ); + byte[] bytes = wrapInPhenopacketAndGetJsonBytes(phenotypicFeatures); + + List items = validator.validate(new ByteArrayInputStream(bytes)); + + assertThat(items, hasSize(0)); + } + + } + + @Nested + public class ValidationFails { + + @Test + public void termIdIsAbsentFromOntology() throws Exception { + List phenotypicFeatures = List.of(makePhenotypicFeature("HP:9999999", false)); + byte[] bytes = wrapInPhenopacketAndGetJsonBytes(phenotypicFeatures); + + List items = validator.validate(new ByteArrayInputStream(bytes)); + + assertThat(items, hasSize(1)); + ValidationItem expected = ValidationItem.of(validator.validatorInfo, OntologyValidationItemTypes.ONTOLOGY_INVALID_ID, "Could not find term id: HP:9999999"); + assertThat(items.get(0), equalTo(expected)); + } + + @Test + public void nonPrimaryTermIdIsUsed() throws Exception { + List phenotypicFeatures = List.of(makePhenotypicFeature("HP:9999999", false)); + byte[] bytes = wrapInPhenopacketAndGetJsonBytes(phenotypicFeatures); + + when(ontology.containsTerm(TermId.of("HP:9999999"))).thenReturn(true); + when(ontology.getPrimaryTermId(TermId.of("HP:9999999"))).thenReturn(TermId.of("HP:0000001")); + + List items = validator.validate(new ByteArrayInputStream(bytes)); + + assertThat(items, hasSize(1)); + ValidationItem expected = ValidationItem.of(validator.validatorInfo, + OntologyValidationItemTypes.ONTOLOGY_TERM_WITH_ALTERNATE_ID, + "Using alternate (obsolete) id (HP:9999999) instead of primary id (HP:0000001)"); + assertThat(items.get(0), equalTo(expected)); + } + + @Test + public void usingTermAlongWithItsAncestor() throws Exception { + List phenotypicFeatures = List.of( + makePhenotypicFeature("HP:0000003", false), + makePhenotypicFeature("HP:0000004", false)); + byte[] bytes = wrapInPhenopacketAndGetJsonBytes(phenotypicFeatures); + + + List items = validator.validate(new ByteArrayInputStream(bytes)); + + assertThat(items, hasSize(1)); + ValidationItem expected = ValidationItem.of(validator.validatorInfo, + OntologyValidationItemTypes.TERM_AND_ANCESTOR_ARE_USED, + "Using both term HP:0000004 and its ancestor HP:0000003"); + assertThat(items.get(0), equalTo(expected)); + } + + @Test + public void usingTermWhileParentTermIsExcluded() throws Exception { + List phenotypicFeatures = List.of( + makePhenotypicFeature("HP:0000003", true), + makePhenotypicFeature("HP:0000004", false)); + byte[] bytes = wrapInPhenopacketAndGetJsonBytes(phenotypicFeatures); + + + List items = validator.validate(new ByteArrayInputStream(bytes)); + + assertThat(items, hasSize(1)); + ValidationItem expected = ValidationItem.of(validator.validatorInfo, + OntologyValidationItemTypes.TERM_IS_USED_AND_ANCESTOR_IS_EXCLUDED, + "Term HP:0000004 is present while its ancestor HP:0000003 is excluded"); + assertThat(items.get(0), equalTo(expected)); + } + } + +} \ No newline at end of file From a7d8f0fe04181a424eb237888d0c9f704082f3aa Mon Sep 17 00:00:00 2001 From: Daniel Danis Date: Thu, 30 Sep 2021 16:53:01 -0400 Subject: [PATCH 21/37] Use Java 16, simplify POMs, add description --- pom.xml | 4 +++- validator-core/pom.xml | 12 ------------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/pom.xml b/pom.xml index 6f2d031..33dfa05 100644 --- a/pom.xml +++ b/pom.xml @@ -9,6 +9,8 @@ 0.1.1-SNAPSHOT pom + Phenopacket validator + Library and tools to help validate phenopackets validator-core @@ -62,7 +64,7 @@ UTF-8 - 11 + 16 1.6.3 3.14.0 diff --git a/validator-core/pom.xml b/validator-core/pom.xml index 58e0948..e61b377 100644 --- a/validator-core/pom.xml +++ b/validator-core/pom.xml @@ -15,17 +15,5 @@ validator-core Validator utilities for phenopackets - - - - org.prefixcommons - curie-util - 0.0.2 - - \ No newline at end of file From 563dd1571f43e80d21b6d5b7c676a3552a6e72a8 Mon Sep 17 00:00:00 2001 From: Daniel Danis Date: Thu, 30 Sep 2021 16:53:47 -0400 Subject: [PATCH 22/37] Remove redundant POM lines --- validator-cli/pom.xml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/validator-cli/pom.xml b/validator-cli/pom.xml index 03e0715..69831cf 100644 --- a/validator-cli/pom.xml +++ b/validator-cli/pom.xml @@ -37,13 +37,4 @@ - - - - src/main/resources - true - - - - From a45e1b4f59ecf0629a24c83567fd3b8b8a5885e2 Mon Sep 17 00:00:00 2001 From: Daniel Danis Date: Thu, 30 Sep 2021 17:12:12 -0400 Subject: [PATCH 23/37] Simplify and standardize JSONschema validator. --- .../validator/cli/ValidatorApplication.java | 4 +- .../core/DefaultValidatorRegistry.java | 18 +++++- .../core/PhenopacketValidatorRegistry.java | 7 +- .../jsonschema/JsonSchemaValidator.java | 40 +++++++----- ...Factory.java => JsonSchemaValidators.java} | 7 +- .../jsonschema/JsonValidationItem.java | 64 ------------------- .../jsonschema/JsonValidationItemType.java | 54 ---------------- .../jsonschema/JsonValidationItemTypes.java | 26 ++++++++ .../JsonSchemaDiseaseValidatorTest.java | 4 +- .../jsonschema/JsonSchemaValidatorTest.java | 8 +-- 10 files changed, 82 insertions(+), 150 deletions(-) rename validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/{JsonSchemaValidatorFactory.java => JsonSchemaValidators.java} (81%) delete mode 100644 validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java delete mode 100644 validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemType.java create mode 100644 validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemTypes.java diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java index 8cc1df2..207008b 100644 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java @@ -8,7 +8,7 @@ import org.phenopackets.validator.core.PhenopacketValidatorRegistry; import org.phenopackets.validator.core.ValidationItem; import org.phenopackets.validator.core.ValidatorInfo; -import org.phenopackets.validator.jsonschema.JsonSchemaValidatorFactory; +import org.phenopackets.validator.jsonschema.JsonSchemaValidators; import org.phenopackets.validator.jsonschema.JsonSchemaValidator; import org.phenopackets.validator.ontology.HpoValidator; import org.phenopackets.validator.ontology.OntologyValidator; @@ -61,7 +61,7 @@ public void run() { } File phenopacketFile = new File(phenopacket); LOGGER.info("Validating {} phenopacket", phenopacketFile); - Map validatorMap = new HashMap<>(JsonSchemaValidatorFactory.genericValidator()); + Map validatorMap = new HashMap<>(JsonSchemaValidators.genericValidator()); for (File jsonSchema : jsonSchemaFiles) { // we will create ValidatorInfo objects based on the names and paths of the files. String baseName = jsonSchema.getName(); diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidatorRegistry.java b/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidatorRegistry.java index ef01ad4..b987b75 100644 --- a/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidatorRegistry.java +++ b/validator-core/src/main/java/org/phenopackets/validator/core/DefaultValidatorRegistry.java @@ -2,14 +2,15 @@ import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; -public class DefaultValidatorRegistry implements PhenopacketValidatorRegistry { +class DefaultValidatorRegistry implements PhenopacketValidatorRegistry { private final Map validatorMap; - public DefaultValidatorRegistry(Map validMap) { + DefaultValidatorRegistry(Map validMap) { validatorMap = Map.copyOf(validMap); } @@ -22,4 +23,17 @@ public Optional getValidatorForType(ValidatorInf public Set getValidationTypeSet() { return validatorMap.keySet(); } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DefaultValidatorRegistry that = (DefaultValidatorRegistry) o; + return Objects.equals(validatorMap, that.validatorMap); + } + + @Override + public int hashCode() { + return Objects.hash(validatorMap); + } } diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorRegistry.java b/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorRegistry.java index 99f0443..c72fbd8 100644 --- a/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorRegistry.java +++ b/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorRegistry.java @@ -11,12 +11,13 @@ * @author Peter N Robinson */ public interface PhenopacketValidatorRegistry { - Optional getValidatorForType(ValidatorInfo type); - - Set getValidationTypeSet(); static PhenopacketValidatorRegistry of(Map validMap) { return new DefaultValidatorRegistry(validMap); } + + Optional getValidatorForType(ValidatorInfo type); + + Set getValidationTypeSet(); } diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java index 791f8d9..17565c5 100644 --- a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidator.java @@ -2,14 +2,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; - import com.networknt.schema.JsonSchema; import com.networknt.schema.JsonSchemaFactory; import com.networknt.schema.SpecVersion; -import org.phenopackets.validator.core.PhenopacketValidator; -import org.phenopackets.validator.core.ValidatorInfo; +import org.phenopackets.validator.core.*; import org.phenopackets.validator.core.except.PhenopacketValidatorRuntimeException; -import org.phenopackets.validator.core.ValidationItem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,12 +32,17 @@ public class JsonSchemaValidator implements PhenopacketValidator { private final ValidatorInfo validatorInfo; + private JsonSchemaValidator(JsonSchema jsonSchema, ValidatorInfo validatorInfo) { + this.jsonSchema = jsonSchema; + this.validatorInfo = validatorInfo; + } + /** * @param jsonSchema path to JSON schema specification file * @throws PhenopacketValidatorRuntimeException if jsonSchema is not a valid file or if the file is * not a valid JSON schema specification */ - public static PhenopacketValidator of(File jsonSchema, ValidatorInfo validatorInfo) { + public static JsonSchemaValidator of(File jsonSchema, ValidatorInfo validatorInfo) { if (!jsonSchema.isFile()) { throw new PhenopacketValidatorRuntimeException("Could not open file at \"" + jsonSchema.getAbsolutePath() + "\""); } @@ -52,14 +54,18 @@ public static PhenopacketValidator of(File jsonSchema, ValidatorInfo validatorIn } } - public static PhenopacketValidator of(InputStream inputStream, ValidatorInfo validatorInfo) { + public static JsonSchemaValidator of(InputStream inputStream, ValidatorInfo validatorInfo) { JsonSchemaFactory schemaFactory = JsonSchemaFactory.getInstance(VERSION_FLAG); return new JsonSchemaValidator(schemaFactory.getSchema(inputStream), validatorInfo); } - private JsonSchemaValidator(JsonSchema jsonSchema, ValidatorInfo validatorInfo) { - this.jsonSchema = jsonSchema; - this.validatorInfo = validatorInfo; + private static ValidationItemType stringToErrorType(String error) { + return switch (error) { + case "additionalProperties" -> JsonValidationItemTypes.JSON_ADDITIONAL_PROPERTIES; + case "required" -> JsonValidationItemTypes.JSON_REQUIRED; + case "type", "enum" -> JsonValidationItemTypes.JSON_TYPE; + default -> throw new PhenopacketValidatorRuntimeException("Did not recognize JSON error type: \"" + error + "\""); + }; } @Override @@ -78,15 +84,17 @@ public List validate(InputStream inputStream) { try { JsonNode json = objectMapper.readTree(inputStream); jsonSchema.validate(json) - .forEach(e -> errors.add(new JsonValidationItem(validatorInfo, e))); + .forEach(e -> errors.add(ValidationItem.of(validatorInfo, stringToErrorType(e.getType()), e.getMessage()))); return errors; - } catch (IOException e) { - LOGGER.warn("Error while decoding JSON content: {}", e.getMessage(), e); - return List.of(); - } catch (RuntimeException e) { - LOGGER.warn("Error while validating: {}", e.getMessage()); - return List.of(); + } catch (Exception e) { + if (e instanceof IOException) { + LOGGER.warn("Error while decoding JSON content: {}", e.getMessage(), e); + } else { + LOGGER.warn("Error while validating: {}", e.getMessage(), e); + } + + return List.of(ValidationItem.of(validatorInfo, ValidationItemTypes.syntaxError(), e.getMessage())); } } diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorFactory.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidators.java similarity index 81% rename from validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorFactory.java rename to validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidators.java index 87105a0..9c75490 100644 --- a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorFactory.java +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonSchemaValidators.java @@ -8,12 +8,12 @@ import java.util.Map; /** - * Validator factory that uses JSON schema definitions that are bundled within the application (on classpath). + * Utility class that uses JSON schema definitions that are bundled within the application (on classpath). *

* @author Daniel Danis * @author Peter N Robinson */ -public class JsonSchemaValidatorFactory { +public class JsonSchemaValidators { public static Map genericValidator() { return Map.of( @@ -28,12 +28,13 @@ public static Map rareHpoValidator() { } private static PhenopacketValidator makeJsonValidator(String schemaPath, ValidatorInfo validationName) { - InputStream inputStream = JsonSchemaValidatorFactory.class.getResourceAsStream(schemaPath); + InputStream inputStream = JsonSchemaValidators.class.getResourceAsStream(schemaPath); if (inputStream == null) throw new PhenopacketValidatorRuntimeException("Invalid JSON schema path `" + schemaPath + '`'); return JsonSchemaValidator.of(inputStream, validationName); } + private JsonSchemaValidators() {} } diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java deleted file mode 100644 index c5dc1e1..0000000 --- a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItem.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.phenopackets.validator.jsonschema; -import com.networknt.schema.ValidationMessage; - -import org.phenopackets.validator.core.ValidationItem; -import org.phenopackets.validator.core.ValidationItemType; -import org.phenopackets.validator.core.ValidatorInfo; - -import java.util.Objects; - -/** - * POJO to represent errors identified by JSON Schema validation. - * @author Peter N Robinson - */ -public final class JsonValidationItem implements ValidationItem { - - private final ValidatorInfo validatorInfo; - private final ValidationItemType itemType; - private final String message; - - - public JsonValidationItem(ValidatorInfo validatorInfo, ValidationMessage validationMessage) { - this.validatorInfo = validatorInfo; - this.itemType = JsonValidationItemType.stringToErrorType(validationMessage.getType()); - this.message = validationMessage.getMessage(); - } - - @Override - public ValidatorInfo validatorInfo() { - return validatorInfo; - } - - @Override - public ValidationItemType type() { - return this.itemType; - } - - @Override - public String message() { - return message; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - JsonValidationItem that = (JsonValidationItem) o; - return Objects.equals(validatorInfo, that.validatorInfo) && itemType == that.itemType && Objects.equals(message, that.message); - } - - @Override - public int hashCode() { - return Objects.hash(validatorInfo, itemType, message); - } - - @Override - public String toString() { - return "JsonValidationError{" + - "validatorInfo='" + validatorInfo + '\'' + - ", errorType=" + itemType + - ", message='" + message + '\'' + - '}'; - } - -} diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemType.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemType.java deleted file mode 100644 index 4c77752..0000000 --- a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemType.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.phenopackets.validator.jsonschema; - -import org.phenopackets.validator.core.ValidationItemType; -import org.phenopackets.validator.core.except.PhenopacketValidatorRuntimeException; - -public class JsonValidationItemType implements ValidationItemType { - - private final String itemType; - private final String description; - - /** - * JSON schema error meaning that the JSON code contained a property not present in the schema. - */ - public final static ValidationItemType JSON_ADDITIONAL_PROPERTIES = - new JsonValidationItemType("additionalProperties", "Use of a property not present in the schema"); - /** JSON schema error meaning that the JSON code failed to contain a property required by the schema. */ - public final static ValidationItemType JSON_REQUIRED = - new JsonValidationItemType("required", "failure to contain a property required by the schema"); - /** The type of an object is not as required by the schema, e.g., we get a string instead of an array. */ - public final static ValidationItemType JSON_TYPE = - new JsonValidationItemType("type", "incorrect data type of object"); - public final static ValidationItemType JSON_ENUM= - new JsonValidationItemType("enum", "TODO"); - - - - - - public JsonValidationItemType(String type, String desc) { - itemType = type; - description = desc; - } - - @Override - public String itemType() { - return itemType; - } - - @Override - public String itemDescription() { - return description; - } - - public static ValidationItemType stringToErrorType(String error) { - switch (error) { - case "additionalProperties": return JSON_ADDITIONAL_PROPERTIES; - case "required": return JSON_REQUIRED; - case "type": return JSON_TYPE; - case "enum": return JSON_TYPE; - default: - throw new PhenopacketValidatorRuntimeException("Did not recognize JSON error type: \"" + error + "\""); - } - } -} diff --git a/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemTypes.java b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemTypes.java new file mode 100644 index 0000000..5c1d621 --- /dev/null +++ b/validator-jsonschema/src/main/java/org/phenopackets/validator/jsonschema/JsonValidationItemTypes.java @@ -0,0 +1,26 @@ +package org.phenopackets.validator.jsonschema; + +import org.phenopackets.validator.core.ValidationItemType; + +public class JsonValidationItemTypes { + + /** + * JSON schema error meaning that the JSON code contained a property not present in the schema. + */ + public final static ValidationItemType JSON_ADDITIONAL_PROPERTIES = ValidationItemType.of("additionalProperties", "Use of a property not present in the schema"); + + /** + * JSON schema error meaning that the JSON code failed to contain a property required by the schema. + */ + public final static ValidationItemType JSON_REQUIRED = ValidationItemType.of("required", "failure to contain a property required by the schema"); + + /** + * The type of an object is not as required by the schema, e.g., we get a string instead of an array. + */ + public final static ValidationItemType JSON_TYPE = ValidationItemType.of("type", "incorrect data type of object"); + + // TODO - add item description + public final static ValidationItemType JSON_ENUM = ValidationItemType.of("enum", "TODO"); + + private JsonValidationItemTypes() {} +} diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java index 7e8cf65..54c53bd 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java @@ -19,7 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.phenopackets.validator.jsonschema.JsonValidationItemType.JSON_REQUIRED; +import static org.phenopackets.validator.jsonschema.JsonValidationItemTypes.*; import static org.phenopackets.validator.testdatagen.PhenopacketUtil.*; /** @@ -29,7 +29,7 @@ */ public class JsonSchemaDiseaseValidatorTest { - private static final Map jsonValidatorMap = JsonSchemaValidatorFactory.genericValidator(); + private static final Map jsonValidatorMap = JsonSchemaValidators.genericValidator(); private static Disease mondoDisease() { var chagas = ontologyClass("MONDO:0005491", "Chagas cardiomyopathy"); diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java index 0cde29d..dc5ebc9 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java @@ -17,13 +17,13 @@ import com.google.protobuf.util.JsonFormat; import static org.junit.jupiter.api.Assertions.*; -import static org.phenopackets.validator.jsonschema.JsonValidationItemType.JSON_ADDITIONAL_PROPERTIES; -import static org.phenopackets.validator.jsonschema.JsonValidationItemType.JSON_REQUIRED; +import static org.phenopackets.validator.jsonschema.JsonValidationItemTypes.JSON_ADDITIONAL_PROPERTIES; +import static org.phenopackets.validator.jsonschema.JsonValidationItemTypes.JSON_REQUIRED; public class JsonSchemaValidatorTest { - private static final Map genericValidatorMap = JsonSchemaValidatorFactory.genericValidator(); - private static final Map rareHpoValidatorMap = JsonSchemaValidatorFactory.rareHpoValidator(); + private static final Map genericValidatorMap = JsonSchemaValidators.genericValidator(); + private static final Map rareHpoValidatorMap = JsonSchemaValidators.rareHpoValidator(); private static final SimplePhenopacket simplePhenopacket = new SimplePhenopacket(); From fb8c52c66337ce85f3438f30272bc91ab818b410 Mon Sep 17 00:00:00 2001 From: Daniel Danis Date: Thu, 30 Sep 2021 17:12:45 -0400 Subject: [PATCH 24/37] Refactor HpoValidatorTest --- .../validator/ontology/OntologyValidator.java | 2 -- .../phenopackets/validator/ontology/HpoValidatorTest.java | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidator.java b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidator.java index d779fa6..28ecf2f 100644 --- a/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidator.java +++ b/validator-ontology/src/main/java/org/phenopackets/validator/ontology/OntologyValidator.java @@ -21,8 +21,6 @@ public abstract class OntologyValidator implements PhenopacketValidator { protected final ValidatorInfo validatorInfo; protected final Ontology ontology; - protected final static String ONTOLOGY_VALIDATOR = "Otology validation"; - protected OntologyValidator(Ontology ontology, ValidatorInfo vinfo) { this.ontology = ontology; this.validatorInfo = vinfo; diff --git a/validator-ontology/src/test/java/org/phenopackets/validator/ontology/HpoValidatorTest.java b/validator-ontology/src/test/java/org/phenopackets/validator/ontology/HpoValidatorTest.java index 909cad4..3d6cd96 100644 --- a/validator-ontology/src/test/java/org/phenopackets/validator/ontology/HpoValidatorTest.java +++ b/validator-ontology/src/test/java/org/phenopackets/validator/ontology/HpoValidatorTest.java @@ -119,7 +119,7 @@ public void termIdIsAbsentFromOntology() throws Exception { List items = validator.validate(new ByteArrayInputStream(bytes)); assertThat(items, hasSize(1)); - ValidationItem expected = ValidationItem.of(validator.validatorInfo, OntologyValidationItemTypes.ONTOLOGY_INVALID_ID, "Could not find term id: HP:9999999"); + ValidationItem expected = ValidationItem.of(validator.info(), OntologyValidationItemTypes.ONTOLOGY_INVALID_ID, "Could not find term id: HP:9999999"); assertThat(items.get(0), equalTo(expected)); } @@ -134,7 +134,7 @@ public void nonPrimaryTermIdIsUsed() throws Exception { List items = validator.validate(new ByteArrayInputStream(bytes)); assertThat(items, hasSize(1)); - ValidationItem expected = ValidationItem.of(validator.validatorInfo, + ValidationItem expected = ValidationItem.of(validator.info(), OntologyValidationItemTypes.ONTOLOGY_TERM_WITH_ALTERNATE_ID, "Using alternate (obsolete) id (HP:9999999) instead of primary id (HP:0000001)"); assertThat(items.get(0), equalTo(expected)); @@ -151,7 +151,7 @@ public void usingTermAlongWithItsAncestor() throws Exception { List items = validator.validate(new ByteArrayInputStream(bytes)); assertThat(items, hasSize(1)); - ValidationItem expected = ValidationItem.of(validator.validatorInfo, + ValidationItem expected = ValidationItem.of(validator.info(), OntologyValidationItemTypes.TERM_AND_ANCESTOR_ARE_USED, "Using both term HP:0000004 and its ancestor HP:0000003"); assertThat(items.get(0), equalTo(expected)); @@ -168,7 +168,7 @@ public void usingTermWhileParentTermIsExcluded() throws Exception { List items = validator.validate(new ByteArrayInputStream(bytes)); assertThat(items, hasSize(1)); - ValidationItem expected = ValidationItem.of(validator.validatorInfo, + ValidationItem expected = ValidationItem.of(validator.info(), OntologyValidationItemTypes.TERM_IS_USED_AND_ANCESTOR_IS_EXCLUDED, "Term HP:0000004 is present while its ancestor HP:0000003 is excluded"); assertThat(items.get(0), equalTo(expected)); From c42b7c737de2401248bd940fab8f9cb4d5450232 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Thu, 7 Oct 2021 09:51:44 -0400 Subject: [PATCH 25/37] update to use phenotools for creating examples for testing. --- README.md | 5 + validator-jsonschema/pom.xml | 7 + .../JsonSchemaDiseaseValidatorTest.java | 16 +- .../testdatagen/PhenopacketUtil.java | 310 ------------------ .../testdatagen/RareDiseasePhenopacket.java | 52 +-- .../testdatagen/SimplePhenopacket.java | 10 +- .../validator/testdatagen/TestBase.java | 21 ++ 7 files changed, 71 insertions(+), 350 deletions(-) delete mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/PhenopacketUtil.java create mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/TestBase.java diff --git a/README.md b/README.md index 7b08824..72b4f52 100644 --- a/README.md +++ b/README.md @@ -9,3 +9,8 @@ Library and tools to help validate phenopackets This app uses version 2.0.0 (branch v2) of [phenopacket-schema](https://github.com/phenopackets/phenopacket-schema). To build this app, clone phenopacket-schema and ``mvn install`` the ``v2`` branch locally. The you should be able to build and run this app with standard maven/java. Note the app requires Java 11 or higher. + +It also uses version 0.0.1-SNAPSHOT of [phenopacket-tools](https://github.com/phenopackets/phenopacket-tools) +to create the test cases. It also needs to be install locally using maven. + +We plan to put all requirements into maven central shortly to simplify the build. \ No newline at end of file diff --git a/validator-jsonschema/pom.xml b/validator-jsonschema/pom.xml index 416c586..8fffb20 100644 --- a/validator-jsonschema/pom.xml +++ b/validator-jsonschema/pom.xml @@ -34,6 +34,13 @@ protobuf-java-util test + + + org.phenopackets.phenotools + phenotools-builder + 0.0.1-SNAPSHOT + test + \ No newline at end of file diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java index 54c53bd..999b44c 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java @@ -4,6 +4,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import org.junit.jupiter.api.Test; +import org.phenopackets.phenotools.builder.builders.MetaDataBuilder; import org.phenopackets.schema.v2.Phenopacket; import org.phenopackets.schema.v2.core.Disease; import org.phenopackets.schema.v2.core.MetaData; @@ -12,15 +13,16 @@ import org.phenopackets.validator.core.PhenopacketValidator; import org.phenopackets.validator.core.ValidationItem; import org.phenopackets.validator.core.ValidatorInfo; -import org.phenopackets.validator.testdatagen.PhenopacketUtil; +import org.phenopackets.validator.testdatagen.TestBase; import java.util.List; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.phenopackets.phenotools.builder.builders.OntologyClassBuilder.ontologyClass; import static org.phenopackets.validator.jsonschema.JsonValidationItemTypes.*; -import static org.phenopackets.validator.testdatagen.PhenopacketUtil.*; +import static org.phenopackets.validator.testdatagen.TestBase.*; /** * This class creates a simple phenopacket with a Disease object and creates some variations with @@ -43,13 +45,11 @@ private static Disease mondoDisease() { } private static Phenopacket phenopacketWithDisease() { - Resource hpo = hpoResource("2021-08-02"); - Resource mondo = mondoResource("2021-09-01"); - MetaData meta = PhenopacketUtil.MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") + MetaData meta = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") .submittedBy("anonymous submitter") - .addResource(hpo) - .addResource(mondo) - .addExternalReference("PMID:20842687", + .hpWithVersion("2021-08-02") + .mondoWithVersion("2021-09-01") + .externalReference("PMID:20842687", "Severe dystonic encephalopathy without hyperphenylalaninemia associated with an 18-bp deletion within the proximal GCH1 promoter") .build(); Disease chagasCardiomyopathy = mondoDisease(); diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/PhenopacketUtil.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/PhenopacketUtil.java deleted file mode 100644 index 9bd6cda..0000000 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/PhenopacketUtil.java +++ /dev/null @@ -1,310 +0,0 @@ -package org.phenopackets.validator.testdatagen; - - -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.List; - -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; - - -import org.phenopackets.schema.v2.core.Resource; -import org.phenopackets.schema.v2.core.Sex; -import org.phenopackets.schema.v2.core.TimeElement; -import org.phenopackets.schema.v2.core.Update; -import org.phenopackets.schema.v2.core.VitalStatus; -import org.phenopackets.schema.v2.core.Individual; -import org.phenopackets.schema.v2.core.Age; -import org.phenopackets.schema.v2.core.ExternalReference; -import org.phenopackets.schema.v2.core.File; -import org.phenopackets.schema.v2.core.MetaData; -import org.phenopackets.schema.v2.core.OntologyClass; -import org.phenopackets.schema.v2.core.PhenotypicFeature; - - -public class PhenopacketUtil { - public static final String SCHEMA_VERSION = "2.0"; - public static final OntologyClass CONGENITAL_ONSET = ontologyClass("HP:0003577", "Congenital onset"); - public static final OntologyClass CHILDHOOD_ONSET = ontologyClass("HP:0011463", "Childhood onset"); - public static final OntologyClass ADULT_ONSET = ontologyClass("HP:0003581", "Adult onset"); - public static final OntologyClass SEVERE = ontologyClass("HP:0012828", "Severe"); - - public static ExternalReference externalReference(String id, String description) { - return ExternalReference.newBuilder().setId(id).setDescription(description).build(); - } - - public static OntologyClass ontologyClass(String termid, String label) { - return OntologyClass.newBuilder() - .setId(termid) - .setLabel(label) - .build(); - } - - - public static Resource resource(String id, String name, String nsPrefix, String iriPrefix, String url, String version) { - return Resource.newBuilder() - .setId(id) - .setName(name) - .setNamespacePrefix(nsPrefix) - .setIriPrefix(iriPrefix) - .setUrl(url) - .setVersion(version) - .build(); - } - - public static PhenotypicFeature phenotypicFeatureCongenital(String termid, String label) { - return PhenotypicFeature.newBuilder() - .setType(ontologyClass(termid, label)) - .setType(CONGENITAL_ONSET) - .build(); - } - - public static PhenotypicFeature phenotypicFeatureChildhood(String termid, String label) { - return PhenotypicFeature.newBuilder() - .setType(ontologyClass(termid, label)) - .setType(CHILDHOOD_ONSET) - .build(); - } - - - - /** - * This has convenience methods for building PhenotypicFeature messages with some - * commonly used options. - * @author Peter N Robinson - */ - public static class PhenotypicFeatureBuilder { - private final String termid; - private final String label; - private OntologyClass onset = null; - private OntologyClass severity = null; - private List modifiers = new ArrayList<>(); - - public PhenotypicFeatureBuilder(String id, String label) { - this.termid = id; - this.label = label; - } - - public PhenotypicFeatureBuilder onset(String id, String label) { - onset = ontologyClass(id, label); - return this; - } - - public PhenotypicFeatureBuilder congenitalOnset() { - onset = CONGENITAL_ONSET; - return this; - } - - public PhenotypicFeatureBuilder childhoodOnset() { - onset = CHILDHOOD_ONSET; - return this; - } - - public PhenotypicFeatureBuilder adultOnset() { - onset = ADULT_ONSET; - return this; - } - - public PhenotypicFeatureBuilder severity(String id, String label) { - severity = ontologyClass(id, label); - return this; - } - - public PhenotypicFeatureBuilder severe() { - severity = SEVERE; - return this; - } - - public PhenotypicFeature build() { - OntologyClass hpoTerm = ontologyClass(termid, label); - PhenotypicFeature.Builder builder = PhenotypicFeature.newBuilder().setType(hpoTerm); - if (this.onset != null) { - builder.mergeFrom(builder.build()).setOnset(TimeElement.newBuilder().setOntologyClass(onset)); - } - if (this.severity != null) { - builder.mergeFrom(builder.build()).setSeverity(severity); - } - if (! this.modifiers.isEmpty()) { - for (OntologyClass clz : this.modifiers) { - builder.mergeFrom(builder.build()).addModifiers(clz); - } - } - - return builder.build(); - } - - public static PhenotypicFeatureBuilder create(String id, String label) { - return new PhenotypicFeatureBuilder(id, label); - } - } - - - - public static Resource hpoResource(String version) { - return resource("hp", "Human Phenotype Ontology", "HP", "http://purl.obolibrary.org/obo/HP_","http://purl.obolibrary.org/obo/hp.owl",version); - } - - public static Resource mondoResource(String version) { - return resource("mondo", "Mondo Disease Ontology", "Mondo", "http://purl.obolibrary.org/obo/mondo_","http://purl.obolibrary.org/obo/mondo.owl",version); - } - - /** - * Parse a google protobuf timestamp object from a string in RFC 3339 format (e.g., 2019-10-12T07:20:50.52Z) - * @param tstamp RFC 3339 string - * @return corresponding protobuf timestamp object - */ - public static Timestamp fromRFC3339(String tstamp) { - try { - return Timestamps.parse(tstamp); - } catch (Exception e) { - LocalDateTime timeNow = LocalDateTime.now(); - return Timestamp.newBuilder() - .setSeconds(timeNow.toEpochSecond(ZoneOffset.UTC)) - .build(); - } - } - - - public static class MetaDataBuilder { - - - private MetaData.Builder builder; - - - public MetaDataBuilder(String created, String createdBy) { - builder = MetaData.newBuilder() - .setCreated(fromRFC3339(created)) - .setCreatedBy(createdBy) - .setPhenopacketSchemaVersion(SCHEMA_VERSION); // only one option for schema version! - } - - - public static MetaDataBuilder create(String created, String createdBy) { - return new MetaDataBuilder(created, createdBy); - } - - public MetaDataBuilder submittedBy(String submitter) { - builder = builder.mergeFrom(builder.build()).setSubmittedBy(submitter); - return this; - } - - public MetaDataBuilder addResource(Resource r) { - builder = builder.mergeFrom(builder.build()).addResources(r); - return this; - } - - public MetaDataBuilder addUpdate(Update u) { - builder = builder.mergeFrom(builder.build()).addUpdates(u); - return this; - } - - public MetaDataBuilder addExternalReference(ExternalReference er) { - builder = builder.mergeFrom(builder.build()).addExternalReferences(er); - return this; - } - - public MetaDataBuilder addExternalReference(String id, String description) { - ExternalReference er = ExternalReference.newBuilder().setId(id).setDescription(description).build(); - builder = builder.mergeFrom(builder.build()).addExternalReferences(er); - return this; - } - - public MetaData build() { - return builder.build(); - } - - } - - - public static class FileBuilder { - - private final File.Builder builder; - - public FileBuilder(String uri) { - builder = File.newBuilder().setUri(uri); - } - - public FileBuilder addFileAttribute(String k, String v) { - builder.mergeFrom(builder.build()).putFileAttributes(k, v); - return this; - } - - public FileBuilder addIndividualToFileIdentifiers(String individual, String fileIdentifier) { - builder.mergeFrom(builder.build()).putIndividualToFileIdentifiers(individual, fileIdentifier); - return this; - } - - public File build() { - return builder.build(); - } - - public static FileBuilder create(String uri) { - return new FileBuilder(uri); - } - - public static FileBuilder hg38vcf(String uri) { - FileBuilder fb = new FileBuilder(uri); - fb.addFileAttribute("genomeAssembly", "GRCh38"); - fb.addFileAttribute("fileFormat", "vcf"); - return fb; - } - } - - public static class IndividualBuilder { - - private Individual.Builder builder; - - public IndividualBuilder(String id) { - builder = Individual.newBuilder().setId(id); - } - - public IndividualBuilder dateOfBirth(String dobirth) { - Timestamp dob = fromRFC3339(dobirth); - builder = builder.mergeFrom(builder.build()).setDateOfBirth(dob); - return this; - } - - - public IndividualBuilder timeAtLastEncounter(String iso8601) { - TimeElement t = TimeElement.newBuilder().setAge(Age.newBuilder().setIso8601Duration(iso8601)).build(); - builder = builder.mergeFrom(builder.build()).setTimeAtLastEncounter(t); - return this; - } - - public IndividualBuilder alive() { - VitalStatus status = VitalStatus.newBuilder().setStatus(VitalStatus.Status.ALIVE).build(); - builder = builder.mergeFrom(builder.build()).setVitalStatus(status); - return this; - } - - public IndividualBuilder deceased() { - VitalStatus status = VitalStatus.newBuilder().setStatus(VitalStatus.Status.DECEASED).build(); - builder = builder.mergeFrom(builder.build()).setVitalStatus(status); - return this; - } - - public IndividualBuilder male() { - builder = builder.mergeFrom(builder.build()).setSex(Sex.MALE); - return this; - } - - public IndividualBuilder female() { - builder = builder.mergeFrom(builder.build()).setSex(Sex.FEMALE); - return this; - } - - - public Individual build() { - return builder.build(); - } - - public static IndividualBuilder create(String id) { - return new IndividualBuilder(id); - } - - } - - -} diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java index abb3d3f..39a00ff 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java @@ -1,25 +1,29 @@ package org.phenopackets.validator.testdatagen; +import org.phenopackets.phenotools.builder.PhenopacketBuilder; +import org.phenopackets.phenotools.builder.builders.FileBuilder; +import org.phenopackets.phenotools.builder.builders.IndividualBuilder; +import org.phenopackets.phenotools.builder.builders.MetaDataBuilder; +import org.phenopackets.phenotools.builder.builders.PhenotypicFeatureBuilder; import org.phenopackets.schema.v2.Phenopacket; import org.phenopackets.schema.v2.core.File; import org.phenopackets.schema.v2.core.Individual; import org.phenopackets.schema.v2.core.MetaData; import org.phenopackets.schema.v2.core.Resource; -import static org.phenopackets.validator.testdatagen.PhenopacketUtil.*; - -public class RareDiseasePhenopacket { +public class RareDiseasePhenopacket extends TestBase{ private final Phenopacket phenopacket; public RareDiseasePhenopacket() { - Individual proband = IndividualBuilder.create("patient 1") - .dateOfBirth("1998-01-01T00:00:00Z") - .timeAtLastEncounter("P3Y") - .male() - .build(); + .dateOfBirth("1998-01-01T00:00:00Z") + .ageAtLastEncounter("P3Y") + .male() + .build(); + + var syndactyly = PhenotypicFeatureBuilder.create("HP:0001159", "Syndactyly") .congenitalOnset().build(); var pneumonia = PhenotypicFeatureBuilder.create("HP:0002090", "Pneumonia") @@ -29,28 +33,24 @@ public RareDiseasePhenopacket() { var sinusitis = PhenotypicFeatureBuilder.create("HP:0011109", "Chronic sinusitis") .severe().adultOnset().build(); - Resource hpo = hpoResource("2021-08-02"); - Resource mondo = mondoResource("2021-09-01"); MetaData meta = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") - .submittedBy("anonymous submitter") - .addResource(hpo) - .addResource(mondo) - .addExternalReference("PMID:20842687", - "Severe dystonic encephalopathy without hyperphenylalaninemia associated with an 18-bp deletion within the proximal GCH1 promoter") - .build(); + .submittedBy("anonymous submitter") + .hpWithVersion("2021-08-02") + .mondoWithVersion("2021-09-01") + .externalReference("PMID:20842687", + "Severe dystonic encephalopathy without hyperphenylalaninemia associated with an 18-bp deletion within the proximal GCH1 promoter") + .build(); File vcf = FileBuilder.hg38vcf("file://data/file.vcf.gz") - .addIndividualToFileIdentifiers("kindred 1A", "SAME000234") + .individualToFileIdentifier("kindred 1A", "SAME000234") .build(); - this.phenopacket = Phenopacket.newBuilder() - .setId("phenopacket-id-1") - .setSubject(proband) - .addPhenotypicFeatures(syndactyly) - .addPhenotypicFeatures(pneumonia) - .addPhenotypicFeatures(cryptorchidism) - .addPhenotypicFeatures(sinusitis) - .addFiles(vcf) - .setMetaData(meta) + this.phenopacket = PhenopacketBuilder.create("phenopacket-id-1", meta) + .individual(proband) + .phenotypicFeature(syndactyly) + .phenotypicFeature(pneumonia) + .phenotypicFeature(cryptorchidism) + .phenotypicFeature(sinusitis) + .file(vcf) .build(); } diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java index 03c0f1b..bbda509 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java @@ -1,24 +1,22 @@ package org.phenopackets.validator.testdatagen; +import org.phenopackets.phenotools.builder.builders.MetaDataBuilder; import org.phenopackets.schema.v2.Phenopacket; import org.phenopackets.schema.v2.core.MetaData; -import org.phenopackets.schema.v2.core.Resource; -import static org.phenopackets.validator.testdatagen.PhenopacketUtil.hpoResource; /** * Build the simplest possible phenopacket for validation */ -public class SimplePhenopacket { +public class SimplePhenopacket extends TestBase { private final Phenopacket phenopacket; public SimplePhenopacket() { - Resource hpo = hpoResource("2021-08-02"); - MetaData meta = PhenopacketUtil.MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") + MetaData meta = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") .submittedBy("anonymous submitter") - .addResource(hpo) + .hpWithVersion("2021-08-02") .build(); phenopacket = Phenopacket.newBuilder() .setId("hello world") diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/TestBase.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/TestBase.java new file mode 100644 index 0000000..c747c58 --- /dev/null +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/TestBase.java @@ -0,0 +1,21 @@ +package org.phenopackets.validator.testdatagen; + + +import org.phenopackets.schema.v2.core.OntologyClass; + + +import static org.phenopackets.phenotools.builder.builders.OntologyClassBuilder.ontologyClass; + + +public class TestBase { + public static final String SCHEMA_VERSION = "2.0"; + public static final OntologyClass CONGENITAL_ONSET = ontologyClass("HP:0003577", "Congenital onset"); + public static final OntologyClass CHILDHOOD_ONSET = ontologyClass("HP:0011463", "Childhood onset"); + public static final OntologyClass ADULT_ONSET = ontologyClass("HP:0003581", "Adult onset"); + public static final OntologyClass SEVERE = ontologyClass("HP:0012828", "Severe"); + + + + + +} From 089d289e084bbbc32491eb3370d79095d23467de Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Thu, 7 Oct 2021 20:06:39 -0400 Subject: [PATCH 26/37] fixed bug (disease requires at least one ontology class (term) --- .../resources/schema/phenopacket-generic.json | 4 +- .../JsonSchemaDiseaseValidatorTest.java | 43 +++++++++++++------ 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json b/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json index 24aeb8b..db5652f 100644 --- a/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json +++ b/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json @@ -491,7 +491,9 @@ "laterality": { "$ref": "#/definitions/ontologyClass" } - } + }, + "required": ["term"], + "additionalProperties": false }, "interpretation": { "type": "object", diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java index 999b44c..90b2fe4 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java @@ -4,6 +4,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import org.junit.jupiter.api.Test; +import org.phenopackets.phenotools.builder.builders.DiseaseBuilder; import org.phenopackets.phenotools.builder.builders.MetaDataBuilder; import org.phenopackets.schema.v2.Phenopacket; import org.phenopackets.schema.v2.core.Disease; @@ -33,17 +34,6 @@ public class JsonSchemaDiseaseValidatorTest { private static final Map jsonValidatorMap = JsonSchemaValidators.genericValidator(); - private static Disease mondoDisease() { - var chagas = ontologyClass("MONDO:0005491", "Chagas cardiomyopathy"); - var nyha3 = ontologyClass("NCIT:C66907", "New York Heart Association Class III"); - var childhood = TimeElement.newBuilder().setOntologyClass(CHILDHOOD_ONSET).build(); - return Disease.newBuilder() - .setTerm(chagas) - .setOnset(childhood) - .addDiseaseStage(nyha3) - .build(); - } - private static Phenopacket phenopacketWithDisease() { MetaData meta = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") .submittedBy("anonymous submitter") @@ -52,7 +42,12 @@ private static Phenopacket phenopacketWithDisease() { .externalReference("PMID:20842687", "Severe dystonic encephalopathy without hyperphenylalaninemia associated with an 18-bp deletion within the proximal GCH1 promoter") .build(); - Disease chagasCardiomyopathy = mondoDisease(); + var nyha3 = ontologyClass("NCIT:C66907", "New York Heart Association Class III"); + var childhood = TimeElement.newBuilder().setOntologyClass(CHILDHOOD_ONSET).build(); + var chagasCardiomyopathy = DiseaseBuilder.create("MONDO:0005491", "Chagas cardiomyopathy") + .diseaseStage(nyha3) + .onset(childhood) + .build(); return Phenopacket.newBuilder() .setId("A") .addDiseases(chagasCardiomyopathy) @@ -80,7 +75,6 @@ public void testLacksId() throws InvalidProtocolBufferException { // the Phenopacket is not valid if we remove the id Phenopacket p1 = Phenopacket.newBuilder(phenopacket).clearId().build(); String json = JsonFormat.printer().print(p1); - System.out.println(json); List errors = validator.validate(json); for (var e: errors) { System.out.println(e); @@ -91,6 +85,29 @@ public void testLacksId() throws InvalidProtocolBufferException { assertEquals("$.id: is missing but it is required", error.message()); } + @Test + public void testDiseaseLacksOntologyTerm() throws InvalidProtocolBufferException { + // Disease must have an ontology term + PhenopacketValidator validator = jsonValidatorMap.values().stream() + .findFirst() + .get(); + // the phenopacket has a single Disease message + Disease disease = phenopacket.getDiseases(0); + // remove the Ontology Term from the disease + disease = Disease.newBuilder(disease).clearTerm().build(); + // replace the original disease object with the new disease object + Phenopacket p1 = Phenopacket.newBuilder(phenopacket) + .setDiseases(0, disease).build(); + String json = JsonFormat.printer().print(p1); + //System.out.println(json); + List errors = validator.validate(json); + assertEquals(1, errors.size()); + ValidationItem error = errors.get(0); + assertEquals(JSON_REQUIRED, error.type()); + // this error means that the first disease lacks a term + assertEquals("$.diseases[0].term: is missing but it is required", error.message()); + } + } From 7217939ddfdbf721a0c82c0636dca966ac5cb1f4 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Thu, 7 Oct 2021 21:22:52 -0400 Subject: [PATCH 27/37] adding definitions for measurement --- .../resources/schema/phenopacket-generic.json | 12 +++ .../jsonschema/ComplexValueValidatorTest.java | 100 ++++++++++++++++++ ...torTest.java => DiseaseValidatorTest.java} | 2 +- ...atorTest.java => SimpleValidatorTest.java} | 3 +- 4 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java rename validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/{JsonSchemaDiseaseValidatorTest.java => DiseaseValidatorTest.java} (99%) rename validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/{JsonSchemaValidatorTest.java => SimpleValidatorTest.java} (99%) diff --git a/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json b/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json index db5652f..1d9e712 100644 --- a/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json +++ b/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json @@ -314,6 +314,18 @@ "type": "null" }, "measurement": { + "type": "object", + "properties": { + "description": "string", + "assay": { "$ref": "#/definitions/ontologyClass" }, + "value": {"$ref": "#/definitions/value" }, + "complexValue": { "$ref": "#/definitions/complexValue" }, + "timeObserved": { "$ref": "#/definitions/timeElement"}, + "procedure": { "$ref": "#/definitions/procedure"} + }, + "required": ["assay"], + "additionalProperties": false, + "oneOf": [ { "required": [ "value" ]}, { "required": [ "complexValue" ]}] }, "age": { "type": "object", diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java new file mode 100644 index 0000000..f66f74e --- /dev/null +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java @@ -0,0 +1,100 @@ +package org.phenopackets.validator.jsonschema; + + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import org.junit.jupiter.api.Test; +import org.phenopackets.phenotools.builder.PhenopacketBuilder; +import org.phenopackets.phenotools.builder.builders.*; +import org.phenopackets.schema.v2.Phenopacket; +import org.phenopackets.schema.v2.core.*; +import org.phenopackets.validator.core.PhenopacketValidator; +import org.phenopackets.validator.core.ValidationItem; +import org.phenopackets.validator.core.ValidatorInfo; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.phenopackets.phenotools.builder.builders.OntologyClassBuilder.ontologyClass; +import static org.phenopackets.validator.testdatagen.TestBase.CHILDHOOD_ONSET; + +/** + * Test a few errors in the Value and ComplexValue messages + * @author Peter N Robinson + */ +public class ComplexValueValidatorTest { + private static final Map jsonValidatorMap = JsonSchemaValidators.genericValidator(); + + private static Phenopacket phenopacketWithValueAndComplexValue() { + MetaData meta = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") + .submittedBy("anonymous submitter") + .hpWithVersion("2021-08-02") + .mondoWithVersion("2021-09-01") + .build(); + // Simple value for blood platelets + OntologyClass cellsPerMl = ontologyClass("UO:0000316", "cells per microliter"); + ReferenceRange rrange = ReferenceRangeCreator.create(cellsPerMl, 150_000, 450_000); + Quantity quantity = QuantityBuilder.create(cellsPerMl,24000) + .referenceRange(rrange) + .build(); + Value value = ValueBuilder.create(quantity).build(); + // Complex value for blood pressure + OntologyClass systolic = ontologyClass("NCIT:C25298", "Systolic Blood Pressure"); + OntologyClass diastolic = ontologyClass("NCIT:C25299", "Diastolic Blood Pressure"); + OntologyClass mmHg = ontologyClass("NCIT:C49670", "Millimeter of Mercury"); + ComplexValue complexValue = ComplexValueBuilder.create() + .typedQuantity(systolic, QuantityBuilder.create(mmHg, 120).build()) + .typedQuantity(diastolic, QuantityBuilder.create(mmHg, 70).build()) + .build(); + OntologyClass plateletAssay = ontologyClass( "LOINC:26515-7", "Platelets [#/volume] in Blood"); + OntologyClass bpAssay = ontologyClass("CMO:0000003", "blood pressure measurement"); + Measurement plateletM = MeasurementBuilder.value(plateletAssay, value).build(); + Measurement bpM = MeasurementBuilder.complexValue(bpAssay, complexValue).build(); + return PhenopacketBuilder.create("id:A", meta) + .measurement(plateletM) + .measurement(bpM) + .build(); + } + + private static final Phenopacket phenopacket = phenopacketWithValueAndComplexValue(); + + /** + * First test that the unaltered phenopacket is OK + */ + @Test + public void validatePhenopacket() throws InvalidProtocolBufferException { + PhenopacketValidator validator = jsonValidatorMap.values().stream() + .findFirst() + .get(); + String json = JsonFormat.printer().print(phenopacket); + System.out.println(json); + List errors = validator.validate(json); + for (var e : errors) { + System.out.println(e.message()); + } + assertTrue(errors.isEmpty()); + } + + /** + * Each measurement requires an ontology term for the assay + */ + @Test + public void testMissingAssay() throws InvalidProtocolBufferException { + PhenopacketValidator validator = jsonValidatorMap.values().stream() + .findFirst() + .get(); + // Alter the first measurement, which is a Value + Measurement m1 = phenopacket.getMeasurements(0); + assertTrue(m1.hasValue()); + m1 = Measurement.newBuilder(m1).clearAssay().build(); + Phenopacket p1 = Phenopacket.newBuilder(phenopacket).setMeasurements(0, m1).build(); + String json = JsonFormat.printer().print(p1); + List errors = validator.validate(json); + for (var e : errors) { + System.out.println(e); + } + } + + +} diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/DiseaseValidatorTest.java similarity index 99% rename from validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java rename to validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/DiseaseValidatorTest.java index 90b2fe4..e3c3490 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaDiseaseValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/DiseaseValidatorTest.java @@ -30,7 +30,7 @@ * validation errors. * @author Peter N Robinson */ -public class JsonSchemaDiseaseValidatorTest { +public class DiseaseValidatorTest { private static final Map jsonValidatorMap = JsonSchemaValidators.genericValidator(); diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/SimpleValidatorTest.java similarity index 99% rename from validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java rename to validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/SimpleValidatorTest.java index dc5ebc9..2233efe 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/SimpleValidatorTest.java @@ -20,7 +20,7 @@ import static org.phenopackets.validator.jsonschema.JsonValidationItemTypes.JSON_ADDITIONAL_PROPERTIES; import static org.phenopackets.validator.jsonschema.JsonValidationItemTypes.JSON_REQUIRED; -public class JsonSchemaValidatorTest { +public class SimpleValidatorTest { private static final Map genericValidatorMap = JsonSchemaValidators.genericValidator(); private static final Map rareHpoValidatorMap = JsonSchemaValidators.rareHpoValidator(); @@ -88,7 +88,6 @@ public void testRareDiseaseBethlemahmValidPhenopacket() throws Exception { Phenopacket bethlehamMyopathy = rareDiseasePhenopacket.getPhenopacket(); String json = JsonFormat.printer().print(bethlehamMyopathy); List errors = validator.validate(json); - assertTrue(errors.isEmpty()); } From d18cbe4370f5ad2dc13bb2d53398a3a09204a485 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Fri, 8 Oct 2021 17:20:32 -0400 Subject: [PATCH 28/37] adding tests for measurement (Value) --- .../jsonschema/ComplexValueValidatorTest.java | 10 -- .../jsonschema/ValueValidatorTest.java | 137 ++++++++++++++++++ 2 files changed, 137 insertions(+), 10 deletions(-) create mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ValueValidatorTest.java diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java index f66f74e..f2fe119 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java @@ -17,7 +17,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.phenopackets.phenotools.builder.builders.OntologyClassBuilder.ontologyClass; -import static org.phenopackets.validator.testdatagen.TestBase.CHILDHOOD_ONSET; /** * Test a few errors in the Value and ComplexValue messages @@ -33,12 +32,6 @@ private static Phenopacket phenopacketWithValueAndComplexValue() { .mondoWithVersion("2021-09-01") .build(); // Simple value for blood platelets - OntologyClass cellsPerMl = ontologyClass("UO:0000316", "cells per microliter"); - ReferenceRange rrange = ReferenceRangeCreator.create(cellsPerMl, 150_000, 450_000); - Quantity quantity = QuantityBuilder.create(cellsPerMl,24000) - .referenceRange(rrange) - .build(); - Value value = ValueBuilder.create(quantity).build(); // Complex value for blood pressure OntologyClass systolic = ontologyClass("NCIT:C25298", "Systolic Blood Pressure"); OntologyClass diastolic = ontologyClass("NCIT:C25299", "Diastolic Blood Pressure"); @@ -47,12 +40,9 @@ private static Phenopacket phenopacketWithValueAndComplexValue() { .typedQuantity(systolic, QuantityBuilder.create(mmHg, 120).build()) .typedQuantity(diastolic, QuantityBuilder.create(mmHg, 70).build()) .build(); - OntologyClass plateletAssay = ontologyClass( "LOINC:26515-7", "Platelets [#/volume] in Blood"); OntologyClass bpAssay = ontologyClass("CMO:0000003", "blood pressure measurement"); - Measurement plateletM = MeasurementBuilder.value(plateletAssay, value).build(); Measurement bpM = MeasurementBuilder.complexValue(bpAssay, complexValue).build(); return PhenopacketBuilder.create("id:A", meta) - .measurement(plateletM) .measurement(bpM) .build(); } diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ValueValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ValueValidatorTest.java new file mode 100644 index 0000000..62da27f --- /dev/null +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ValueValidatorTest.java @@ -0,0 +1,137 @@ +package org.phenopackets.validator.jsonschema; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import org.junit.jupiter.api.Test; +import org.phenopackets.phenotools.builder.PhenopacketBuilder; +import org.phenopackets.phenotools.builder.builders.*; +import org.phenopackets.schema.v2.Phenopacket; +import org.phenopackets.schema.v2.core.*; +import org.phenopackets.validator.core.PhenopacketValidator; +import org.phenopackets.validator.core.ValidationItem; +import org.phenopackets.validator.core.ValidatorInfo; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.phenopackets.phenotools.builder.builders.OntologyClassBuilder.ontologyClass; + +/** + * + * Test a few errors in the Value messages + * @author Peter N Robinson + */ +public class ValueValidatorTest { + private static final Map jsonValidatorMap = JsonSchemaValidators.genericValidator(); + + private static Phenopacket phenopacketWithValueAndComplexValue() { + MetaData meta = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") + .submittedBy("anonymous submitter") + .hpWithVersion("2021-08-02") + .mondoWithVersion("2021-09-01") + .build(); + // Simple value for blood platelets + OntologyClass cellsPerMl = ontologyClass("UO:0000316", "cells per microliter"); + ReferenceRange rrange = ReferenceRangeCreator.create(cellsPerMl, 150_000, 450_000); + Quantity quantity = QuantityBuilder.create(cellsPerMl,24000) + .referenceRange(rrange) + .build(); + Value value = ValueBuilder.create(quantity).build(); + OntologyClass plateletAssay = ontologyClass( "LOINC:26515-7", "Platelets [#/volume] in Blood"); + Measurement plateletM = MeasurementBuilder.value(plateletAssay, value).build(); + return PhenopacketBuilder.create("id:A", meta) + .measurement(plateletM) + .build(); + } + + private static final Phenopacket phenopacket = phenopacketWithValueAndComplexValue(); + + /** + * First test that the unaltered phenopacket is OK + */ + @Test + public void validatePhenopacket() throws InvalidProtocolBufferException { + PhenopacketValidator validator = jsonValidatorMap.values().stream() + .findFirst() + .get(); + String json = JsonFormat.printer().print(phenopacket); + //System.out.println(json); + List errors = validator.validate(json); + for (var e : errors) { + System.out.println(e.message()); + } + assertTrue(errors.isEmpty()); + } + + /** + * Test what happens if we remove the ontology class from the Measurement + * Note that the validation tool reports two errors + * $.measurements[0].value: is missing but it is required + * $.measurements[0].complexValue: is missing but it is required + * The second of which is incorrect but I think this is more a limitation of JSON Schema validation + * of oneOf elements + */ + @Test + public void testMissingOntologyTerm() throws InvalidProtocolBufferException { + PhenopacketValidator validator = jsonValidatorMap.values().stream() + .findFirst() + .get(); + Measurement m1 = Phenopacket.newBuilder(phenopacket).getMeasurements(0); + m1 = Measurement.newBuilder(m1).clearValue().build(); + Phenopacket p1 = Phenopacket.newBuilder(phenopacket).setMeasurements(0, m1).build(); + String json = JsonFormat.printer().print(p1); + System.out.println(json); + List errors = validator.validate(json); + for (var e : errors) { + System.out.println(e.message()); + } + assertFalse(errors.isEmpty()); + String expectedErrorMsg = "$.measurements[0].value: is missing but it is required"; + boolean foundError = errors.stream().map(ValidationItem::message).anyMatch(m -> m.equals(expectedErrorMsg)); + assertTrue(foundError, "was expecting to find \"$.measurements[0].value: is missing but it is required\" but did not"); + } + + @Test + public void referenceRangeMissingLowValue() throws InvalidProtocolBufferException { + PhenopacketValidator validator = jsonValidatorMap.values().stream() + .findFirst() + .get(); + Measurement m1 = Phenopacket.newBuilder(phenopacket).getMeasurements(0); + Value v1 = m1.getValue(); + ReferenceRange rrange = v1.getQuantity().getReferenceRange(); + rrange = ReferenceRange.newBuilder(rrange).clearLow().build(); + Quantity q1= Quantity.newBuilder(v1.getQuantity()).setReferenceRange(rrange).build(); + v1 = Value.newBuilder(v1).setQuantity(q1).build(); + m1 = Measurement.newBuilder(m1).setValue(v1).build(); + Phenopacket p1 = Phenopacket.newBuilder(phenopacket).setMeasurements(0, m1).build(); + String json = JsonFormat.printer().print(p1); + //System.out.println(json); + List errors = validator.validate(json); + assertEquals(1, errors.size()); + String expectedErrorMsg = "$.measurements[0].value.quantity.referenceRange.low: is missing but it is required"; + assertEquals(expectedErrorMsg, errors.get(0).message()); + } + + @Test + public void measurementMissingAssay() throws InvalidProtocolBufferException { + PhenopacketValidator validator = jsonValidatorMap.values().stream() + .findFirst() + .get(); + Measurement m1 = Phenopacket.newBuilder(phenopacket).getMeasurements(0); + m1 = Measurement.newBuilder(m1).clearAssay().build(); + Phenopacket p1 = Phenopacket.newBuilder(phenopacket).setMeasurements(0, m1).build(); + String json = JsonFormat.printer().print(p1); + //System.out.println(json); + List errors = validator.validate(json); +// for (var e : errors) { +// System.out.println(e.message()); +// } + assertEquals(1, errors.size()); + String expectedErrorMsg = "$.measurements[0].assay: is missing but it is required"; + assertEquals(expectedErrorMsg, errors.get(0).message()); + } + + + +} From 7327e42444822d85e8b06cf5561904b42c19393f Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Fri, 8 Oct 2021 17:20:51 -0400 Subject: [PATCH 29/37] fixed JSON Schema for Value (Measurement) --- .../resources/schema/phenopacket-generic.json | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json b/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json index 1d9e712..d70855b 100644 --- a/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json +++ b/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json @@ -301,17 +301,35 @@ "additionalProperties": false }, "value": { - "oneOf": [ - { + "type": "object", + "properties": { + "quantity": { + "$ref": "#/definitions/quantity" + }, + "ontologyClass": { + "$ref": "#/definitions/ontologyClass" + } + }, + "additionalProperties": false, + "oneOf": [ { "required": [ "quantity" ]}, + { "required": [ "ontologyClass" ]} + ] + }, + "typedQuantity": { + "type": "object", + "properties": { + "type": { "$ref": "#/definitions/ontologyClass" }, - { + "quantity": { "$ref": "#/definitions/quantity" } - ] + }, + "required": ["type", "quantity"], + "additionalProperties": false }, "complexValue": { - "type": "null" + "type": "object" }, "measurement": { "type": "object", @@ -319,7 +337,7 @@ "description": "string", "assay": { "$ref": "#/definitions/ontologyClass" }, "value": {"$ref": "#/definitions/value" }, - "complexValue": { "$ref": "#/definitions/complexValue" }, + "complexValue": { "$ref": "#/definitions/complexValue" }, "timeObserved": { "$ref": "#/definitions/timeElement"}, "procedure": { "$ref": "#/definitions/procedure"} }, From 6297fdf18407d4f162d58866cb8f196d8e1959e2 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Fri, 8 Oct 2021 18:04:01 -0400 Subject: [PATCH 30/37] fixed JSON Schema for ComplexValue (Measurement) --- .../main/resources/schema/phenopacket-generic.json | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json b/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json index d70855b..63a2bc7 100644 --- a/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json +++ b/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json @@ -329,7 +329,19 @@ "additionalProperties": false }, "complexValue": { - "type": "object" + "type": "object", + "properties": { + "typedquantities": { + "type": "array", + "items": { + "$ref": "#/definitions/typedQuantity" + }, + "minItems": 1, + "uniqueItems": true + } + }, + "required": ["typedquantities"], + "additionalProperties": false }, "measurement": { "type": "object", From 3dece6ba7d4e1e14715c02b5b06cd977fdbb83de Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Fri, 8 Oct 2021 18:04:16 -0400 Subject: [PATCH 31/37] ComplexValue (Measurement) tests --- .../jsonschema/ComplexValueValidatorTest.java | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java index f2fe119..75e847a 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Map; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.phenopackets.phenotools.builder.builders.OntologyClassBuilder.ontologyClass; @@ -74,9 +75,9 @@ public void testMissingAssay() throws InvalidProtocolBufferException { PhenopacketValidator validator = jsonValidatorMap.values().stream() .findFirst() .get(); - // Alter the first measurement, which is a Value + // Alter the first measurement, which is a ComplexValue Measurement m1 = phenopacket.getMeasurements(0); - assertTrue(m1.hasValue()); + assertTrue(m1.hasComplexValue()); m1 = Measurement.newBuilder(m1).clearAssay().build(); Phenopacket p1 = Phenopacket.newBuilder(phenopacket).setMeasurements(0, m1).build(); String json = JsonFormat.printer().print(p1); @@ -86,5 +87,31 @@ public void testMissingAssay() throws InvalidProtocolBufferException { } } + /** + * Each measurement requires an ontology term for the assay + */ + @Test + public void testComplexValueHasNoTypedValue() throws InvalidProtocolBufferException { + PhenopacketValidator validator = jsonValidatorMap.values().stream() + .findFirst() + .get(); + // Alter the first measurement, which is a ComplexValue + Measurement m1 = phenopacket.getMeasurements(0); + assertTrue(m1.hasComplexValue()); + ComplexValue cvale = m1.getComplexValue(); + cvale = ComplexValue.newBuilder(cvale).clearTypedQuantities().build(); + m1 = Measurement.newBuilder(m1).setComplexValue(cvale).build(); + Phenopacket p1 = Phenopacket.newBuilder(phenopacket).setMeasurements(0, m1).build(); + String json = JsonFormat.printer().print(p1); + System.out.println(json); + List errors = validator.validate(json); + for (var e : errors) { + System.out.println(e); + } + assertEquals(1, errors.size()); + String expectedErrorMsg = "$.measurements[0].complexValue.typedquantities: is missing but it is required"; + assertEquals(expectedErrorMsg, errors.get(0).message()); + } + } From 127a851de064e60419da7150a00550f4bb239b30 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Sat, 9 Oct 2021 07:35:55 -0400 Subject: [PATCH 32/37] fixed definition of Evidence --- .../main/resources/schema/hpo-rare-disease-schema.json | 1 - .../src/main/resources/schema/phenopacket-generic.json | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/validator-jsonschema/src/main/resources/schema/hpo-rare-disease-schema.json b/validator-jsonschema/src/main/resources/schema/hpo-rare-disease-schema.json index b3c2450..fc7325b 100644 --- a/validator-jsonschema/src/main/resources/schema/hpo-rare-disease-schema.json +++ b/validator-jsonschema/src/main/resources/schema/hpo-rare-disease-schema.json @@ -3,7 +3,6 @@ "$id": "https://www.ga4gh.org/phenopackets", "title": "HPO Rare Disease Phenopacket Schema", "description": "HPO Rare Disease Schema for GGA4GH Phenopacket", - "_comment": "Here we require the phenopacket to have the following elements that are not required by the default schema 1. subject (proband being investigated) 2. at least one phenotypicFeature element 3. time_at_last encounter (subelement of subject), representing the age of the proband. In addition, we require that Human Phenotype Ontology (HPO) terms are used to represent phenotypicFeature", "type": "object", "properties": { "subject": { diff --git a/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json b/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json index 63a2bc7..00aa0f5 100644 --- a/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json +++ b/validator-jsonschema/src/main/resources/schema/phenopacket-generic.json @@ -184,7 +184,9 @@ "reference": { "$ref": "#/definitions/externalReference" } - } + }, + "required": ["evidenceCode"], + "additionalProperties": false }, "timeElement": { "type": "object", @@ -331,7 +333,7 @@ "complexValue": { "type": "object", "properties": { - "typedquantities": { + "typedQuantities": { "type": "array", "items": { "$ref": "#/definitions/typedQuantity" @@ -340,7 +342,7 @@ "uniqueItems": true } }, - "required": ["typedquantities"], + "required": ["typedQuantities"], "additionalProperties": false }, "measurement": { From 7e56aec539f74cc08725a0eb2e4e81d0e405d3a2 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Sat, 9 Oct 2021 07:41:09 -0400 Subject: [PATCH 33/37] new unit tests --- .../validator/cli/ValidatorApplication.java | 6 -- .../jsonschema/ComplexValueValidatorTest.java | 28 ++----- .../jsonschema/DiseaseValidatorTest.java | 36 ++------- .../JsonSchemaValidatorTestBase.java | 33 ++++++++ .../PhenotypicFeatureValidatorTest.java | 78 +++++++++++++++++++ .../jsonschema/SimpleValidatorTest.java | 32 +------- .../jsonschema/ValueValidatorTest.java | 22 +----- .../{TestBase.java => DatagenBase.java} | 2 +- .../testdatagen/RareDiseasePhenopacket.java | 3 +- .../testdatagen/SimplePhenopacket.java | 2 +- 10 files changed, 129 insertions(+), 113 deletions(-) create mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTestBase.java create mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/PhenotypicFeatureValidatorTest.java rename validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/{TestBase.java => DatagenBase.java} (96%) diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java index 207008b..19d83b2 100644 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java @@ -53,12 +53,6 @@ public static void main(String[] args) { @Override public void run() { - List validationTypes = new LinkedList<>(); - validationTypes.add(ValidatorInfo.generic()); // we run this by default - if (rareHpoConstraints) { - LOGGER.info("Validating with HPO rare-disease constraints"); - validationTypes.add(ValidatorInfo.rareDiseaseValidation()); - } File phenopacketFile = new File(phenopacket); LOGGER.info("Validating {} phenopacket", phenopacketFile); Map validatorMap = new HashMap<>(JsonSchemaValidators.genericValidator()); diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java index 75e847a..2d88586 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ComplexValueValidatorTest.java @@ -8,12 +8,9 @@ import org.phenopackets.phenotools.builder.builders.*; import org.phenopackets.schema.v2.Phenopacket; import org.phenopackets.schema.v2.core.*; -import org.phenopackets.validator.core.PhenopacketValidator; import org.phenopackets.validator.core.ValidationItem; -import org.phenopackets.validator.core.ValidatorInfo; import java.util.List; -import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -23,15 +20,9 @@ * Test a few errors in the Value and ComplexValue messages * @author Peter N Robinson */ -public class ComplexValueValidatorTest { - private static final Map jsonValidatorMap = JsonSchemaValidators.genericValidator(); +public class ComplexValueValidatorTest extends JsonSchemaValidatorTestBase{ private static Phenopacket phenopacketWithValueAndComplexValue() { - MetaData meta = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") - .submittedBy("anonymous submitter") - .hpWithVersion("2021-08-02") - .mondoWithVersion("2021-09-01") - .build(); // Simple value for blood platelets // Complex value for blood pressure OntologyClass systolic = ontologyClass("NCIT:C25298", "Systolic Blood Pressure"); @@ -43,7 +34,7 @@ private static Phenopacket phenopacketWithValueAndComplexValue() { .build(); OntologyClass bpAssay = ontologyClass("CMO:0000003", "blood pressure measurement"); Measurement bpM = MeasurementBuilder.complexValue(bpAssay, complexValue).build(); - return PhenopacketBuilder.create("id:A", meta) + return PhenopacketBuilder.create("id:A", metadataWithHpo) .measurement(bpM) .build(); } @@ -55,11 +46,8 @@ private static Phenopacket phenopacketWithValueAndComplexValue() { */ @Test public void validatePhenopacket() throws InvalidProtocolBufferException { - PhenopacketValidator validator = jsonValidatorMap.values().stream() - .findFirst() - .get(); String json = JsonFormat.printer().print(phenopacket); - System.out.println(json); + //System.out.println(json); List errors = validator.validate(json); for (var e : errors) { System.out.println(e.message()); @@ -72,9 +60,6 @@ public void validatePhenopacket() throws InvalidProtocolBufferException { */ @Test public void testMissingAssay() throws InvalidProtocolBufferException { - PhenopacketValidator validator = jsonValidatorMap.values().stream() - .findFirst() - .get(); // Alter the first measurement, which is a ComplexValue Measurement m1 = phenopacket.getMeasurements(0); assertTrue(m1.hasComplexValue()); @@ -92,9 +77,6 @@ public void testMissingAssay() throws InvalidProtocolBufferException { */ @Test public void testComplexValueHasNoTypedValue() throws InvalidProtocolBufferException { - PhenopacketValidator validator = jsonValidatorMap.values().stream() - .findFirst() - .get(); // Alter the first measurement, which is a ComplexValue Measurement m1 = phenopacket.getMeasurements(0); assertTrue(m1.hasComplexValue()); @@ -103,13 +85,13 @@ public void testComplexValueHasNoTypedValue() throws InvalidProtocolBufferExcept m1 = Measurement.newBuilder(m1).setComplexValue(cvale).build(); Phenopacket p1 = Phenopacket.newBuilder(phenopacket).setMeasurements(0, m1).build(); String json = JsonFormat.printer().print(p1); - System.out.println(json); + //System.out.println(json); List errors = validator.validate(json); for (var e : errors) { System.out.println(e); } assertEquals(1, errors.size()); - String expectedErrorMsg = "$.measurements[0].complexValue.typedquantities: is missing but it is required"; + String expectedErrorMsg = "$.measurements[0].complexValue.typedQuantities: is missing but it is required"; assertEquals(expectedErrorMsg, errors.get(0).message()); } diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/DiseaseValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/DiseaseValidatorTest.java index e3c3490..d48a469 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/DiseaseValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/DiseaseValidatorTest.java @@ -4,54 +4,37 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import org.junit.jupiter.api.Test; +import org.phenopackets.phenotools.builder.PhenopacketBuilder; import org.phenopackets.phenotools.builder.builders.DiseaseBuilder; -import org.phenopackets.phenotools.builder.builders.MetaDataBuilder; import org.phenopackets.schema.v2.Phenopacket; import org.phenopackets.schema.v2.core.Disease; -import org.phenopackets.schema.v2.core.MetaData; -import org.phenopackets.schema.v2.core.Resource; import org.phenopackets.schema.v2.core.TimeElement; -import org.phenopackets.validator.core.PhenopacketValidator; import org.phenopackets.validator.core.ValidationItem; -import org.phenopackets.validator.core.ValidatorInfo; -import org.phenopackets.validator.testdatagen.TestBase; import java.util.List; -import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.phenopackets.phenotools.builder.builders.OntologyClassBuilder.ontologyClass; import static org.phenopackets.validator.jsonschema.JsonValidationItemTypes.*; -import static org.phenopackets.validator.testdatagen.TestBase.*; +import static org.phenopackets.validator.testdatagen.DatagenBase.*; /** * This class creates a simple phenopacket with a Disease object and creates some variations with * validation errors. * @author Peter N Robinson */ -public class DiseaseValidatorTest { - - private static final Map jsonValidatorMap = JsonSchemaValidators.genericValidator(); +public class DiseaseValidatorTest extends JsonSchemaValidatorTestBase { private static Phenopacket phenopacketWithDisease() { - MetaData meta = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") - .submittedBy("anonymous submitter") - .hpWithVersion("2021-08-02") - .mondoWithVersion("2021-09-01") - .externalReference("PMID:20842687", - "Severe dystonic encephalopathy without hyperphenylalaninemia associated with an 18-bp deletion within the proximal GCH1 promoter") - .build(); var nyha3 = ontologyClass("NCIT:C66907", "New York Heart Association Class III"); var childhood = TimeElement.newBuilder().setOntologyClass(CHILDHOOD_ONSET).build(); var chagasCardiomyopathy = DiseaseBuilder.create("MONDO:0005491", "Chagas cardiomyopathy") .diseaseStage(nyha3) .onset(childhood) .build(); - return Phenopacket.newBuilder() - .setId("A") - .addDiseases(chagasCardiomyopathy) - .setMetaData(meta) + return PhenopacketBuilder.create("id:A",metadataWithHpo) + .disease(chagasCardiomyopathy) .build(); } @@ -59,9 +42,6 @@ private static Phenopacket phenopacketWithDisease() { @Test public void testPhenopacketValidity() throws InvalidProtocolBufferException { - PhenopacketValidator validator = jsonValidatorMap.values().stream() - .findFirst() - .get(); String json = JsonFormat.printer().print(phenopacket); List errors = validator.validate(json); assertTrue(errors.isEmpty()); @@ -69,9 +49,6 @@ public void testPhenopacketValidity() throws InvalidProtocolBufferException { @Test public void testLacksId() throws InvalidProtocolBufferException { - PhenopacketValidator validator = jsonValidatorMap.values().stream() - .findFirst() - .get(); // the Phenopacket is not valid if we remove the id Phenopacket p1 = Phenopacket.newBuilder(phenopacket).clearId().build(); String json = JsonFormat.printer().print(p1); @@ -88,9 +65,6 @@ public void testLacksId() throws InvalidProtocolBufferException { @Test public void testDiseaseLacksOntologyTerm() throws InvalidProtocolBufferException { // Disease must have an ontology term - PhenopacketValidator validator = jsonValidatorMap.values().stream() - .findFirst() - .get(); // the phenopacket has a single Disease message Disease disease = phenopacket.getDiseases(0); // remove the Ontology Term from the disease diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTestBase.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTestBase.java new file mode 100644 index 0000000..56744f5 --- /dev/null +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTestBase.java @@ -0,0 +1,33 @@ +package org.phenopackets.validator.jsonschema; + +import org.phenopackets.phenotools.builder.builders.MetaDataBuilder; +import org.phenopackets.phenotools.builder.exceptions.PhenotoolsRuntimeException; +import org.phenopackets.schema.v2.core.MetaData; +import org.phenopackets.validator.core.PhenopacketValidator; +import org.phenopackets.validator.core.ValidatorInfo; + +import java.util.Map; + +public class JsonSchemaValidatorTestBase { + private static final Map genericValidatorMap = JsonSchemaValidators.genericValidator(); + private static final Map rareHpoValidatorMap = JsonSchemaValidators.rareHpoValidator(); + + protected final PhenopacketValidator validator = genericValidatorMap.values().stream() + .findFirst() + .orElseThrow(() -> new PhenotoolsRuntimeException("Could not retrieve generic validator")); + + protected final PhenopacketValidator rareDiseaseValidator = rareHpoValidatorMap.values().stream() + .findFirst() + .orElseThrow(() -> new PhenotoolsRuntimeException("Could not retrieve rare disease validator")); + + protected static final MetaData metadataWithHpo = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") + .submittedBy("anonymous submitter") + .hpWithVersion("2021-08-02") + .mondoWithVersion("2021-09-01") + .externalReference("PMID:20842687", + "Severe dystonic encephalopathy without hyperphenylalaninemia associated with an 18-bp deletion within the proximal GCH1 promoter") + .build(); + + + +} diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/PhenotypicFeatureValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/PhenotypicFeatureValidatorTest.java new file mode 100644 index 0000000..a47b909 --- /dev/null +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/PhenotypicFeatureValidatorTest.java @@ -0,0 +1,78 @@ +package org.phenopackets.validator.jsonschema; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import org.junit.jupiter.api.Test; +import org.phenopackets.phenotools.builder.PhenopacketBuilder; +import org.phenopackets.phenotools.builder.builders.EvidenceBuilder; +import org.phenopackets.phenotools.builder.builders.PhenotypicFeatureBuilder; +import org.phenopackets.schema.v2.Phenopacket; +import org.phenopackets.schema.v2.core.Evidence; +import org.phenopackets.schema.v2.core.PhenotypicFeature; +import org.phenopackets.validator.core.ValidationItem; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.phenopackets.validator.jsonschema.JsonValidationItemTypes.JSON_REQUIRED; + + +public class PhenotypicFeatureValidatorTest extends JsonSchemaValidatorTestBase { + private static final String PMID = "PMID:30808312"; + private static final String publication = "COL6A1 mutation leading to Bethlem myopathy with recurrent hematuria: a case report"; + + private static Phenopacket phenopacketWithOnePhenotypicFeature() { + var authorAssertion = EvidenceBuilder.authorStatementEvidence(PMID, publication); + var VSD = + PhenotypicFeatureBuilder.create("HP:0001629","Ventricular septal defect") + .congenitalOnset() + .evidence(authorAssertion) + .build(); + return PhenopacketBuilder.create("ID:A", metadataWithHpo) + .phenotypicFeature(VSD) + .build(); + } + + private final Phenopacket phenopacket = phenopacketWithOnePhenotypicFeature(); + + @Test + public void testPhenotypicFeatureWithoutOntologyTerm() throws InvalidProtocolBufferException { + assertTrue(phenopacket.getPhenotypicFeaturesCount() > 0); + PhenotypicFeature pf = phenopacket.getPhenotypicFeatures(0); + pf = PhenotypicFeature.newBuilder(pf).clearType().build(); + Phenopacket p1 = Phenopacket.newBuilder(phenopacket).setPhenotypicFeatures(0, pf).build(); + String json = JsonFormat.printer().print(p1); + List errors = validator.validate(json); + assertEquals(1, errors.size()); + ValidationItem error = errors.get(0); + assertEquals(JSON_REQUIRED, error.type()); + String expectedErrorMsg = "$.phenotypicFeatures[0].type: is missing but it is required"; + assertEquals(expectedErrorMsg, error.message()); + } + + /** + * This Phenopacket is invalid, because the PhenotypicFeature has an Evidence element that is lacking + * evidenceCode + */ + @Test + public void testPhenotypicFeatureWithInvalidEvidence() throws InvalidProtocolBufferException { + assertTrue(phenopacket.getPhenotypicFeaturesCount() > 0); + PhenotypicFeature pf = phenopacket.getPhenotypicFeatures(0); + assertTrue(pf.getEvidenceCount() > 0); + Evidence evi = pf.getEvidence(0); + evi = Evidence.newBuilder(evi).clearEvidenceCode().build(); + pf = PhenotypicFeature.newBuilder(pf).setEvidence(0, evi).build(); + Phenopacket p1 = Phenopacket.newBuilder(phenopacket).setPhenotypicFeatures(0, pf).build(); + String json = JsonFormat.printer().print(p1); + // System.out.println(json); + List errors = validator.validate(json); + assertEquals(1, errors.size()); + ValidationItem error = errors.get(0); + assertEquals(JSON_REQUIRED, error.type()); + String expectedErrorMsg = "$.phenotypicFeatures[0].evidence[0].evidenceCode: is missing but it is required"; + assertEquals(expectedErrorMsg, error.message()); + } + + +} diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/SimpleValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/SimpleValidatorTest.java index 2233efe..e194245 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/SimpleValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/SimpleValidatorTest.java @@ -2,16 +2,13 @@ import org.junit.jupiter.api.Test; import org.phenopackets.schema.v2.Phenopacket; -import org.phenopackets.validator.core.PhenopacketValidator; import org.phenopackets.validator.testdatagen.RareDiseasePhenopacket; import org.phenopackets.validator.testdatagen.SimplePhenopacket; import org.phenopackets.validator.core.ValidationItem; -import org.phenopackets.validator.core.ValidatorInfo; import java.io.File; import java.io.IOException; import java.util.List; -import java.util.Map; import java.util.Objects; import com.google.protobuf.util.JsonFormat; @@ -20,10 +17,7 @@ import static org.phenopackets.validator.jsonschema.JsonValidationItemTypes.JSON_ADDITIONAL_PROPERTIES; import static org.phenopackets.validator.jsonschema.JsonValidationItemTypes.JSON_REQUIRED; -public class SimpleValidatorTest { - - private static final Map genericValidatorMap = JsonSchemaValidators.genericValidator(); - private static final Map rareHpoValidatorMap = JsonSchemaValidators.rareHpoValidator(); +public class SimpleValidatorTest extends JsonSchemaValidatorTestBase { private static final SimplePhenopacket simplePhenopacket = new SimplePhenopacket(); @@ -36,9 +30,6 @@ private static File fileFromClasspath(String path) { @Test public void testValidationOfSimpleValidPhenopacket() throws Exception { - PhenopacketValidator validator = genericValidatorMap.values().stream() - .findFirst() - .get(); Phenopacket phenopacket = simplePhenopacket.getPhenopacket(); String json = JsonFormat.printer().print(phenopacket); List errors = validator.validate(json); @@ -59,14 +50,8 @@ public void testValidationOfSimpleValidPhenopacket() throws Exception { */ @Test public void testValidationOfSimpleInValidPhenopacket() { - PhenopacketValidator validator = genericValidatorMap.values().stream() - .findFirst() - .get(); - String invalidPhenopacketJson = "{\"disney\" : \"donald\"}"; - List errors = validator.validate(invalidPhenopacketJson); - assertEquals(3, errors.size()); ValidationItem error = errors.get(0); assertEquals(JSON_REQUIRED, error.type()); @@ -81,10 +66,6 @@ public void testValidationOfSimpleInValidPhenopacket() { @Test public void testRareDiseaseBethlemahmValidPhenopacket() throws Exception { - PhenopacketValidator validator = rareHpoValidatorMap.values().stream() - .findFirst() - .get(); - Phenopacket bethlehamMyopathy = rareDiseasePhenopacket.getPhenopacket(); String json = JsonFormat.printer().print(bethlehamMyopathy); List errors = validator.validate(json); @@ -93,18 +74,13 @@ public void testRareDiseaseBethlemahmValidPhenopacket() throws Exception { @Test public void testRareDiseaseBethlemahmInvalidValidPhenopacket() throws IOException { - PhenopacketValidator validator = rareHpoValidatorMap.values().stream() - .findFirst() - .get(); File invalidMyopathyPhenopacket = fileFromClasspath("json/bethlehamMyopathyInvalidExample.json"); List errors = validator.validate(invalidMyopathyPhenopacket); - for (ValidationItem ve : errors) { - System.out.println(ve.message()); - } assertEquals(1, errors.size()); ValidationItem error = errors.get(0); - assertEquals(JSON_REQUIRED, error.type()); - assertEquals("$.phenotypicFeatures: is missing but it is required", error.message()); + assertEquals(JSON_ADDITIONAL_PROPERTIES, error.type()); + String expectedErrorMsg = "$.phenotypicFeaturesMALFORMED: is not defined in the schema and the schema does not allow additional properties"; + assertEquals(expectedErrorMsg, error.message()); } diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ValueValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ValueValidatorTest.java index 62da27f..e7bf1e3 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ValueValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ValueValidatorTest.java @@ -7,12 +7,9 @@ import org.phenopackets.phenotools.builder.builders.*; import org.phenopackets.schema.v2.Phenopacket; import org.phenopackets.schema.v2.core.*; -import org.phenopackets.validator.core.PhenopacketValidator; import org.phenopackets.validator.core.ValidationItem; -import org.phenopackets.validator.core.ValidatorInfo; import java.util.List; -import java.util.Map; import static org.junit.jupiter.api.Assertions.*; import static org.phenopackets.phenotools.builder.builders.OntologyClassBuilder.ontologyClass; @@ -22,8 +19,7 @@ * Test a few errors in the Value messages * @author Peter N Robinson */ -public class ValueValidatorTest { - private static final Map jsonValidatorMap = JsonSchemaValidators.genericValidator(); +public class ValueValidatorTest extends JsonSchemaValidatorTestBase { private static Phenopacket phenopacketWithValueAndComplexValue() { MetaData meta = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") @@ -52,9 +48,6 @@ private static Phenopacket phenopacketWithValueAndComplexValue() { */ @Test public void validatePhenopacket() throws InvalidProtocolBufferException { - PhenopacketValidator validator = jsonValidatorMap.values().stream() - .findFirst() - .get(); String json = JsonFormat.printer().print(phenopacket); //System.out.println(json); List errors = validator.validate(json); @@ -74,9 +67,6 @@ public void validatePhenopacket() throws InvalidProtocolBufferException { */ @Test public void testMissingOntologyTerm() throws InvalidProtocolBufferException { - PhenopacketValidator validator = jsonValidatorMap.values().stream() - .findFirst() - .get(); Measurement m1 = Phenopacket.newBuilder(phenopacket).getMeasurements(0); m1 = Measurement.newBuilder(m1).clearValue().build(); Phenopacket p1 = Phenopacket.newBuilder(phenopacket).setMeasurements(0, m1).build(); @@ -94,9 +84,6 @@ public void testMissingOntologyTerm() throws InvalidProtocolBufferException { @Test public void referenceRangeMissingLowValue() throws InvalidProtocolBufferException { - PhenopacketValidator validator = jsonValidatorMap.values().stream() - .findFirst() - .get(); Measurement m1 = Phenopacket.newBuilder(phenopacket).getMeasurements(0); Value v1 = m1.getValue(); ReferenceRange rrange = v1.getQuantity().getReferenceRange(); @@ -115,18 +102,11 @@ public void referenceRangeMissingLowValue() throws InvalidProtocolBufferExceptio @Test public void measurementMissingAssay() throws InvalidProtocolBufferException { - PhenopacketValidator validator = jsonValidatorMap.values().stream() - .findFirst() - .get(); Measurement m1 = Phenopacket.newBuilder(phenopacket).getMeasurements(0); m1 = Measurement.newBuilder(m1).clearAssay().build(); Phenopacket p1 = Phenopacket.newBuilder(phenopacket).setMeasurements(0, m1).build(); String json = JsonFormat.printer().print(p1); - //System.out.println(json); List errors = validator.validate(json); -// for (var e : errors) { -// System.out.println(e.message()); -// } assertEquals(1, errors.size()); String expectedErrorMsg = "$.measurements[0].assay: is missing but it is required"; assertEquals(expectedErrorMsg, errors.get(0).message()); diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/TestBase.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/DatagenBase.java similarity index 96% rename from validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/TestBase.java rename to validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/DatagenBase.java index c747c58..48db3de 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/TestBase.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/DatagenBase.java @@ -7,7 +7,7 @@ import static org.phenopackets.phenotools.builder.builders.OntologyClassBuilder.ontologyClass; -public class TestBase { +public class DatagenBase { public static final String SCHEMA_VERSION = "2.0"; public static final OntologyClass CONGENITAL_ONSET = ontologyClass("HP:0003577", "Congenital onset"); public static final OntologyClass CHILDHOOD_ONSET = ontologyClass("HP:0011463", "Childhood onset"); diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java index 39a00ff..648333e 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java @@ -9,9 +9,8 @@ import org.phenopackets.schema.v2.core.File; import org.phenopackets.schema.v2.core.Individual; import org.phenopackets.schema.v2.core.MetaData; -import org.phenopackets.schema.v2.core.Resource; -public class RareDiseasePhenopacket extends TestBase{ +public class RareDiseasePhenopacket extends DatagenBase { private final Phenopacket phenopacket; diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java index bbda509..b2acece 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java @@ -9,7 +9,7 @@ /** * Build the simplest possible phenopacket for validation */ -public class SimplePhenopacket extends TestBase { +public class SimplePhenopacket extends DatagenBase { private final Phenopacket phenopacket; From ddb774f7c69138ecf1be328c35a56d7c16d60b7f Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Mon, 11 Oct 2021 11:52:42 -0400 Subject: [PATCH 34/37] biosample validation --- .../jsonschema/BiosampleValidatorTest.java | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/BiosampleValidatorTest.java diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/BiosampleValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/BiosampleValidatorTest.java new file mode 100644 index 0000000..e8e03a3 --- /dev/null +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/BiosampleValidatorTest.java @@ -0,0 +1,81 @@ +package org.phenopackets.validator.jsonschema; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import org.junit.jupiter.api.Test; +import org.phenopackets.phenotools.builder.PhenopacketBuilder; +import org.phenopackets.phenotools.builder.builders.*; +import org.phenopackets.schema.v2.Phenopacket; +import org.phenopackets.schema.v2.core.Biosample; +import org.phenopackets.schema.v2.core.MetaData; +import org.phenopackets.schema.v2.core.OntologyClass; +import org.phenopackets.validator.core.ValidationItem; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.phenopackets.phenotools.builder.builders.OntologyClassBuilder.ontologyClass; + +public class BiosampleValidatorTest extends JsonSchemaValidatorTestBase { + private static final String PHENOPACKET_ID = "arbitrary.id"; + private static final String PROBAND_ID = "proband A"; + private static final OntologyClass BIOPSY = ontologyClass("NCIT:C15189", "Biopsy"); + private static MetaData metadata = MetaDataBuilder.create("2021-05-14T10:35:00Z", "anonymous biocurator") + .ncitWithVersion("21.05d") + .efoWithVersion("3.34.0") + .uberonWithVersion("2021-07-27") + .ncbiTaxonWithVersion(" 2021-06-10") + .build(); + private static final Biosample lymphNodeBiopsy = BiosampleBuilder.create("biosample 2") + .individualId(PROBAND_ID) + .timeOfCollection(TimeElementBuilder.create().age("P48Y3M").build()) + .sampledTissue(ontologyClass("NCIT:C139196", "Esophageal Lymph Node")) + .tumorProgression(ontologyClass("NCIT:C84509", "Primary Malignant Neoplasm")) + .histologicalDiagnosis(ontologyClass("NCIT:C4024", "Esophageal Squamous Cell Carcinoma")) + .diagnosticMarker(ontologyClass("NCIT:C131711", "Human Papillomavirus-18 Positive")) + .procedure(ProcedureBuilder.create(BIOPSY).build()) + .build(); + + private static final Biosample lungBiopsy = BiosampleBuilder.create("biosample 3") + .individualId(PROBAND_ID) + .timeOfCollection(TimeElementBuilder.create().age("P50Y7M").build()) + .sampledTissue(ontologyClass("NCIT:C12468", "Lung")) + .tumorProgression(ontologyClass("NCIT:C3261", "Metastatic Neoplasm")) + .procedure(ProcedureBuilder.create(BIOPSY).build()) + .build(); + + private static Phenopacket phenopacketWithBiosample() { + return PhenopacketBuilder.create(PHENOPACKET_ID, metadata) + .biosample(lymphNodeBiopsy) + .build(); + } + + private static final Phenopacket phenopacket = phenopacketWithBiosample(); + + @Test + public void testValidityOfBiosample() throws InvalidProtocolBufferException { + String json = JsonFormat.printer().print(phenopacket); + List errors = validator.validate(json); + assertTrue(errors.isEmpty()); + } + + /** + * The only required element in a Biosample is the ID + */ + @Test + public void testBiosampleLacksId() throws InvalidProtocolBufferException { + assertTrue(phenopacket.getBiosamplesCount() == 1); + Biosample biosample = phenopacket.getBiosamples(0); + biosample = Biosample.newBuilder(biosample).clearId().build(); + Phenopacket p1 = Phenopacket.newBuilder(phenopacket).setBiosamples(0, biosample).build(); + String json = JsonFormat.printer().print(p1); + List errors = validator.validate(json); + assertEquals(1, errors.size()); + String expectedErrorMsg = "$.biosamples[0].id: is missing but it is required"; + assertEquals(expectedErrorMsg, errors.get(0).message()); + } + + + +} From f244cae9e18dbfceba2cec8e550b0f63d1d0aee1 Mon Sep 17 00:00:00 2001 From: pnrobinson Date: Mon, 11 Oct 2021 16:38:44 -0400 Subject: [PATCH 35/37] refactoring --- .../org/phenopackets/validator/cli/ValidatorApplication.java | 4 +--- .../org/phenopackets/validator/core/PhenopacketValidator.java | 3 ++- .../validator/core/PhenopacketValidatorRegistry.java | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java index 19d83b2..56ac9d6 100644 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java @@ -10,8 +10,6 @@ import org.phenopackets.validator.core.ValidatorInfo; import org.phenopackets.validator.jsonschema.JsonSchemaValidators; import org.phenopackets.validator.jsonschema.JsonSchemaValidator; -import org.phenopackets.validator.ontology.HpoValidator; -import org.phenopackets.validator.ontology.OntologyValidator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine; @@ -65,7 +63,7 @@ public void run() { LOGGER.info("Adding configuration file at `{}`", vinfo); } Ontology hpoOntology = OntologyLoader.loadOntology(hpoJsonPath); - OntologyValidator hpoValidator = new HpoValidator(hpoOntology); + PhenopacketValidator hpoValidator = new HpoValidator(hpoOntology); validatorMap.put(hpoValidator.info(), hpoValidator); PhenopacketValidatorRegistry registry = PhenopacketValidatorRegistry.of(validatorMap); diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidator.java b/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidator.java index 4d7914e..51c13c4 100644 --- a/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidator.java +++ b/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidator.java @@ -24,7 +24,8 @@ public interface PhenopacketValidator { default List validate(File phenopacket) throws IOException { try (InputStream inputStream = Files.newInputStream(phenopacket.toPath())) { - return validate(inputStream); + byte[] bytes = inputStream.readAllBytes(); + return validate(bytes); } } diff --git a/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorRegistry.java b/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorRegistry.java index c72fbd8..b48b36c 100644 --- a/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorRegistry.java +++ b/validator-core/src/main/java/org/phenopackets/validator/core/PhenopacketValidatorRegistry.java @@ -12,7 +12,7 @@ */ public interface PhenopacketValidatorRegistry { - static PhenopacketValidatorRegistry of(Map validMap) { + static PhenopacketValidatorRegistry of(Map validMap) { return new DefaultValidatorRegistry(validMap); } From 15e3f1824e72de38761924d7cadd08c2257ed819 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Mon, 11 Oct 2021 16:49:41 -0400 Subject: [PATCH 36/37] updating for new phenopacket-tools API --- .../cli/results/ValidationTsvVisualizer.java | 2 +- .../jsonschema/BiosampleValidatorTest.java | 16 ++++++++-------- .../jsonschema/JsonSchemaValidatorTestBase.java | 5 +++-- .../validator/jsonschema/ValueValidatorTest.java | 4 ++-- .../testdatagen/RareDiseasePhenopacket.java | 9 +++------ .../validator/testdatagen/SimplePhenopacket.java | 3 ++- 6 files changed, 19 insertions(+), 20 deletions(-) diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationTsvVisualizer.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationTsvVisualizer.java index 09e74a0..2bcf80a 100644 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationTsvVisualizer.java +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/results/ValidationTsvVisualizer.java @@ -45,7 +45,7 @@ public void write(Writer writer) throws IOException { writer.write(String.join("\t",result.getFields()) + "\n"); } for (var vinfo : this.errorFreeValidations) { - writer.write(String.join("\t",List.of(vinfo.validatorName().toString(), "no errors", vinfo.validatorId().toString())) + "\n"); + writer.write(String.join("\t",List.of(vinfo.validatorName(), "no errors", vinfo.validatorId())) + "\n"); } } diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/BiosampleValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/BiosampleValidatorTest.java index e8e03a3..963ea74 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/BiosampleValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/BiosampleValidatorTest.java @@ -21,15 +21,15 @@ public class BiosampleValidatorTest extends JsonSchemaValidatorTestBase { private static final String PHENOPACKET_ID = "arbitrary.id"; private static final String PROBAND_ID = "proband A"; private static final OntologyClass BIOPSY = ontologyClass("NCIT:C15189", "Biopsy"); - private static MetaData metadata = MetaDataBuilder.create("2021-05-14T10:35:00Z", "anonymous biocurator") - .ncitWithVersion("21.05d") - .efoWithVersion("3.34.0") - .uberonWithVersion("2021-07-27") - .ncbiTaxonWithVersion(" 2021-06-10") + private static final MetaData metadata = MetaDataBuilder.create("2021-05-14T10:35:00Z", "anonymous biocurator") + .resource(Resources.ncitVersion("21.05d")) + .resource(Resources.efoVersion("3.34.0")) + .resource(Resources.uberonVersion("2021-07-27")) + .resource(Resources.ncbiTaxonVersion("2021-06-10")) .build(); private static final Biosample lymphNodeBiopsy = BiosampleBuilder.create("biosample 2") .individualId(PROBAND_ID) - .timeOfCollection(TimeElementBuilder.create().age("P48Y3M").build()) + .timeOfCollection(TimeElements.age("P48Y3M")) .sampledTissue(ontologyClass("NCIT:C139196", "Esophageal Lymph Node")) .tumorProgression(ontologyClass("NCIT:C84509", "Primary Malignant Neoplasm")) .histologicalDiagnosis(ontologyClass("NCIT:C4024", "Esophageal Squamous Cell Carcinoma")) @@ -39,7 +39,7 @@ public class BiosampleValidatorTest extends JsonSchemaValidatorTestBase { private static final Biosample lungBiopsy = BiosampleBuilder.create("biosample 3") .individualId(PROBAND_ID) - .timeOfCollection(TimeElementBuilder.create().age("P50Y7M").build()) + .timeOfCollection(TimeElements.age("P50Y7M")) .sampledTissue(ontologyClass("NCIT:C12468", "Lung")) .tumorProgression(ontologyClass("NCIT:C3261", "Metastatic Neoplasm")) .procedure(ProcedureBuilder.create(BIOPSY).build()) @@ -65,7 +65,7 @@ public void testValidityOfBiosample() throws InvalidProtocolBufferException { */ @Test public void testBiosampleLacksId() throws InvalidProtocolBufferException { - assertTrue(phenopacket.getBiosamplesCount() == 1); + assertEquals(1, phenopacket.getBiosamplesCount()); Biosample biosample = phenopacket.getBiosamples(0); biosample = Biosample.newBuilder(biosample).clearId().build(); Phenopacket p1 = Phenopacket.newBuilder(phenopacket).setBiosamples(0, biosample).build(); diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTestBase.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTestBase.java index 56744f5..83216ae 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTestBase.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/JsonSchemaValidatorTestBase.java @@ -1,6 +1,7 @@ package org.phenopackets.validator.jsonschema; import org.phenopackets.phenotools.builder.builders.MetaDataBuilder; +import org.phenopackets.phenotools.builder.builders.Resources; import org.phenopackets.phenotools.builder.exceptions.PhenotoolsRuntimeException; import org.phenopackets.schema.v2.core.MetaData; import org.phenopackets.validator.core.PhenopacketValidator; @@ -22,8 +23,8 @@ public class JsonSchemaValidatorTestBase { protected static final MetaData metadataWithHpo = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") .submittedBy("anonymous submitter") - .hpWithVersion("2021-08-02") - .mondoWithVersion("2021-09-01") + .resource(Resources.hpoVersion("2021-08-02")) + .resource(Resources.mondoVersion("2021-09-01")) .externalReference("PMID:20842687", "Severe dystonic encephalopathy without hyperphenylalaninemia associated with an 18-bp deletion within the proximal GCH1 promoter") .build(); diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ValueValidatorTest.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ValueValidatorTest.java index e7bf1e3..7197a10 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ValueValidatorTest.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/jsonschema/ValueValidatorTest.java @@ -24,8 +24,8 @@ public class ValueValidatorTest extends JsonSchemaValidatorTestBase { private static Phenopacket phenopacketWithValueAndComplexValue() { MetaData meta = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") .submittedBy("anonymous submitter") - .hpWithVersion("2021-08-02") - .mondoWithVersion("2021-09-01") + .resource(Resources.hpoVersion("2021-08-02")) + .resource(Resources.mondoVersion("2021-09-01")) .build(); // Simple value for blood platelets OntologyClass cellsPerMl = ontologyClass("UO:0000316", "cells per microliter"); diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java index 648333e..9f65081 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/RareDiseasePhenopacket.java @@ -1,10 +1,7 @@ package org.phenopackets.validator.testdatagen; import org.phenopackets.phenotools.builder.PhenopacketBuilder; -import org.phenopackets.phenotools.builder.builders.FileBuilder; -import org.phenopackets.phenotools.builder.builders.IndividualBuilder; -import org.phenopackets.phenotools.builder.builders.MetaDataBuilder; -import org.phenopackets.phenotools.builder.builders.PhenotypicFeatureBuilder; +import org.phenopackets.phenotools.builder.builders.*; import org.phenopackets.schema.v2.Phenopacket; import org.phenopackets.schema.v2.core.File; import org.phenopackets.schema.v2.core.Individual; @@ -34,8 +31,8 @@ public RareDiseasePhenopacket() { MetaData meta = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") .submittedBy("anonymous submitter") - .hpWithVersion("2021-08-02") - .mondoWithVersion("2021-09-01") + .resource(Resources.hpoVersion("2021-08-02")) + .resource(Resources.mondoVersion("2021-09-01")) .externalReference("PMID:20842687", "Severe dystonic encephalopathy without hyperphenylalaninemia associated with an 18-bp deletion within the proximal GCH1 promoter") .build(); diff --git a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java index b2acece..20735de 100644 --- a/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java +++ b/validator-jsonschema/src/test/java/org/phenopackets/validator/testdatagen/SimplePhenopacket.java @@ -1,6 +1,7 @@ package org.phenopackets.validator.testdatagen; import org.phenopackets.phenotools.builder.builders.MetaDataBuilder; +import org.phenopackets.phenotools.builder.builders.Resources; import org.phenopackets.schema.v2.Phenopacket; import org.phenopackets.schema.v2.core.MetaData; @@ -16,7 +17,7 @@ public class SimplePhenopacket extends DatagenBase { public SimplePhenopacket() { MetaData meta = MetaDataBuilder.create("2021-07-01T19:32:35Z", "anonymous biocurator") .submittedBy("anonymous submitter") - .hpWithVersion("2021-08-02") + .resource(Resources.hpoVersion("2021-08-02")) .build(); phenopacket = Phenopacket.newBuilder() .setId("hello world") From 9eda567d8fdd5ba9bfcf5833b0e59ee8e660b0f4 Mon Sep 17 00:00:00 2001 From: Peter Robinson Date: Mon, 11 Oct 2021 16:58:14 -0400 Subject: [PATCH 37/37] refactor to use JSON string --- .../validator/cli/ValidatorApplication.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java index 19d83b2..f22a9a7 100644 --- a/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java +++ b/validator-cli/src/main/java/org/phenopackets/validator/cli/ValidatorApplication.java @@ -8,6 +8,7 @@ import org.phenopackets.validator.core.PhenopacketValidatorRegistry; import org.phenopackets.validator.core.ValidationItem; import org.phenopackets.validator.core.ValidatorInfo; +import org.phenopackets.validator.core.except.PhenopacketValidatorRuntimeException; import org.phenopackets.validator.jsonschema.JsonSchemaValidators; import org.phenopackets.validator.jsonschema.JsonSchemaValidator; import org.phenopackets.validator.ontology.HpoValidator; @@ -19,6 +20,7 @@ import java.io.*; import java.nio.file.Files; import java.util.*; +import java.io.IOException; @CommandLine.Command(name = "PhenopacketValidator", version = "0.1.0", mixinStandardHelpOptions = true) @@ -35,7 +37,7 @@ public class ValidatorApplication implements Runnable { public File hpoJsonPath; @CommandLine.Option(names = {"-p", "--phenopacket"}, required = true, description = "Phenopacket file to be validated") - public String phenopacket; + public String phenopacketFilePath; @CommandLine.Option(names = {"-o", "--out"}, description = "name of output file (default ${DEFAULT_VALUE})") public String outfileName = "phenopacket-validation.tsv"; @@ -52,8 +54,8 @@ public static void main(String[] args) { @Override - public void run() { - File phenopacketFile = new File(phenopacket); + public void run() { + File phenopacketFile = new File(phenopacketFilePath); LOGGER.info("Validating {} phenopacket", phenopacketFile); Map validatorMap = new HashMap<>(JsonSchemaValidators.genericValidator()); for (File jsonSchema : jsonSchemaFiles) { @@ -74,13 +76,14 @@ public void run() { LOGGER.info("--------------------------------------------------------------------------------"); LOGGER.info(""); ValidationTsvVisualizer resultVisualizer = new ValidationTsvVisualizer(); - try (InputStream in = Files.newInputStream(phenopacketFile.toPath())) { + try { + String phenopacketJson = Files.readString(phenopacketFile.toPath()); Set validatorInfoSet = registry.getValidationTypeSet(); for (var vinfo : validatorInfoSet) { var opt = registry.getValidatorForType(vinfo); if (opt.isPresent()) { PhenopacketValidator validator = opt.get(); - List validationItems = validator.validate(phenopacketFile); + List validationItems = validator.validate(phenopacketJson); if (validationItems.isEmpty()) { resultVisualizer.errorFree(vinfo); } else {