From 92816460ac7b93fd649bba50f3c04fb6e8be6d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Mon, 20 Oct 2025 12:29:15 +0100 Subject: [PATCH 1/5] models: Update proto variant model and converter #TASK-7555 --- .../main/proto/protobuf/opencb/common.proto | 3 + .../main/proto/protobuf/opencb/evidence.proto | 29 ++- .../main/proto/protobuf/opencb/variant.proto | 75 +++++-- .../protobuf/opencb/variant_annotation.proto | 201 ++++++++++++----- .../proto/AvroToProtoConverter.java | 208 ++++++++++++++++++ .../VariantAvroToVariantProtoConverter.java | 167 +++++--------- ...VariantContextToVariantProtoConverter.java | 10 +- ...ariantAvroToVariantProtoConverterTest.java | 100 +++++++++ 8 files changed, 601 insertions(+), 192 deletions(-) create mode 100644 biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/AvroToProtoConverter.java create mode 100644 biodata-tools/src/test/java/org/opencb/biodata/tools/variant/converters/proto/VariantAvroToVariantProtoConverterTest.java diff --git a/biodata-models/src/main/proto/protobuf/opencb/common.proto b/biodata-models/src/main/proto/protobuf/opencb/common.proto index 6e3292f93..e7a60d349 100644 --- a/biodata-models/src/main/proto/protobuf/opencb/common.proto +++ b/biodata-models/src/main/proto/protobuf/opencb/common.proto @@ -30,4 +30,7 @@ message GeneDrugInteraction { string source = 3; string study_type = 4; string type = 5; + string interaction_type = 6; + string chembl_id = 7; + repeated string publications = 8; } diff --git a/biodata-models/src/main/proto/protobuf/opencb/evidence.proto b/biodata-models/src/main/proto/protobuf/opencb/evidence.proto index ecd21886b..bd78ddd8e 100644 --- a/biodata-models/src/main/proto/protobuf/opencb/evidence.proto +++ b/biodata-models/src/main/proto/protobuf/opencb/evidence.proto @@ -185,15 +185,26 @@ Pharmacogenomics drug response variant classification * `efficacy` : a variant that affects the efficacy of the treatment */ enum DrugResponseClassification { - responsive = 0; - resistant = 1; - toxicity = 2; - indication = 3; - contraindication = 4; - dosing = 5; - increased_monitoring = 6; - efficacy = 7; -} + responsive = 0; // DEPRECATED: kept just for retrocompatibility purposes + altered_sensitivity = 1; + reduced_sensitivity = 2; + increased_sensitivity = 3; + altered_resistance = 4; + increased_resistance = 5; + reduced_resistance = 6; + increased_risk_of_toxicity = 7; + reduced_risk_of_toxicity = 8; + altered_toxicity = 9; + adverse_drug_reaction = 10; + indication = 11; + contraindication = 12; + dosing_alteration = 13; + increased_dose = 14; + reduced_dose = 15; + increased_monitoring = 16; + increased_efficacy = 17; + reduced_efficacy = 18; + altered_efficacy = 19;} /** Association of variants to a given trait. diff --git a/biodata-models/src/main/proto/protobuf/opencb/variant.proto b/biodata-models/src/main/proto/protobuf/opencb/variant.proto index 52950ccc4..676c34d80 100644 --- a/biodata-models/src/main/proto/protobuf/opencb/variant.proto +++ b/biodata-models/src/main/proto/protobuf/opencb/variant.proto @@ -207,6 +207,47 @@ message SampleEntry { repeated string data = 3; } +message IssueEntry { + + enum IssueType { + DUPLICATION = 0; + DISCREPANCY = 1; + MENDELIAN_ERROR = 2; + DE_NOVO = 3; + COMPOUND_HETEROZYGOUS = 4; + } + + IssueType type = 1; + + SampleEntry sample = 2; + + map data = 3; +} + +message VariantScore { + +/** + * Variant score ID. + */ + string id = 1; +/** + * Main cohort used for calculating the score. + */ + string cohort1 = 2; +/** + * Optional secondary cohort used for calculating the score. + */ + string cohort2 = 3; +/** + * Score value + */ + float score = 4; +/** + * Score p value + */ + float pValue = 5; +} + message StudyEntry { string studyId = 1; repeated FileEntry files = 2; @@ -217,22 +258,11 @@ message StudyEntry { repeated string sampleDataKeys = 4; repeated SampleEntry samples = 5; repeated VariantStats stats = 6; + repeated IssueEntry issues = 7; + repeated VariantScore scores = 8; } -///** -// * Type of structural variation -// * -// */ -//enum StructuralVariantType { -// unused = 0; // SO:0001742 -//// COPY_NUMBER_GAIN = 0; // SO:0001742 -//// COPY_NUMBER_LOSS = 1; // SO:0001743 -//// TANDEM_DUPLICATION = 2; // SO:1000173 -//} + /* @@ -263,6 +293,21 @@ message Breakend { message StructuralVariation { + /** + * Type of structural variation + * + */ + enum StructuralVariantType { +// unused = 0; // SO:0001742 + COPY_NUMBER_GAIN = 0; // SO:0001742 + COPY_NUMBER_LOSS = 1; // SO:0001743 + TANDEM_DUPLICATION = 2; // SO:1000173 + } + int32 ciStartLeft = 1; int32 ciStartRight = 2; int32 ciEndLeft = 3; @@ -283,7 +328,7 @@ message StructuralVariation { /** * Structural variation type: COPY_NUMBER_GAIN, COPY_NUMBER_LOSS, TANDEM_DUPLICATION, ... */ -// StructuralVariantType type = 8; + StructuralVariantType type = 8; Breakend breakend = 9; } diff --git a/biodata-models/src/main/proto/protobuf/opencb/variant_annotation.proto b/biodata-models/src/main/proto/protobuf/opencb/variant_annotation.proto index 35d22b870..cc9458333 100644 --- a/biodata-models/src/main/proto/protobuf/opencb/variant_annotation.proto +++ b/biodata-models/src/main/proto/protobuf/opencb/variant_annotation.proto @@ -11,54 +11,6 @@ import "protobuf/opencb/common.proto"; import "protobuf/opencb/gene.proto"; import "protobuf/opencb/evidence.proto"; -// enum ExpressionCall in common.proto - -message ClinVar { - string accession = 1; - string clinical_significance = 2; - repeated string traits = 3; - repeated string geneNames = 4; - string review_status = 5; -} - -message Gwas { - string snpIdCurrent = 1; - repeated string traits = 2; - double risk_allele_frequency = 3; - string reported_genes = 4; -} - -message Cosmic { - string mutation_id = 1; - string primary_site = 2; - string site_subtype = 3; - string primary_histology = 4; - string histology_subtype = 5; - string sample_source = 6; - string tumour_origin = 7; - string gene_name = 8; - string mutation_somatic_status = 9; -} - -message VariantTraitAssociation { - repeated ClinVar clinvar = 1; - repeated Gwas gwas = 2; - repeated Cosmic cosmic = 3; -} - -// message GeneDrugInteraction in common.proto -// message Expression in common.proto - -//message GeneTraitAssociation { -// string id = 1; -// string name = 2; -// string hpo = 3; -// float score = 4; -// int32 number_of_pubmeds = 5; -// repeated string association_types = 6; -// repeated string sources = 7; -// string source = 8; -//} message PopulationFrequency { string study = 1; @@ -70,6 +22,12 @@ message PopulationFrequency { float ref_hom_genotype_freq = 7; float het_genotype_freq = 8; float alt_hom_genotype_freq = 9; + + int32 ref_allele_count = 10; + int32 alt_allele_count = 11; + int32 ref_hom_genotype_count = 12; + int32 het_genotype_count = 13; + int32 alt_hom_genotype_count = 14; } message SequenceOntologyTerm { @@ -95,7 +53,7 @@ message ProteinVariantAnnotation { string uniprot_accession = 1; string uniprot_name = 2; int32 position = 3; - //string aaChange = 4; + string proteinId = 4; // Ensembl or RefSeq protein ID string reference = 5; string alternate = 6; string uniprot_variant_id = 7; @@ -110,6 +68,12 @@ message ExonOverlap { float percentage = 2; } + +message SpliceScores { + string source = 1; + map scores = 2; +} + message ConsequenceType { string gene_name = 1; string ensembl_gene_id = 2; @@ -117,12 +81,18 @@ message ConsequenceType { string strand = 4; string biotype = 5; repeated ExonOverlap exon_overlap = 6; - repeated string transcript_annotation_flags = 7; - int32 c_dna_position = 8; + int32 cdna_position = 8; int32 cds_position = 9; string codon = 10; ProteinVariantAnnotation protein_variant_annotation = 11; repeated SequenceOntologyTerm sequence_ontology_terms = 12; + + string gene_id = 13; + string transcript_id = 14; + repeated string hgvs = 15; + repeated SpliceScores spliceScores = 16; + repeated string transcriptFlags = 17; + string source = 18; } message Repeat { @@ -157,6 +127,127 @@ message Drug { repeated string bibliography = 7; } +message Constraint { + string source = 1; + string method = 2; + string name = 3; + double value = 4; +} + +message GeneMirnaTarget { + string id = 1; + string name = 2; + string biotype = 3; +} + +message GeneCancerAssociation { + string id = 1; + string source = 2; + string tier = 3; + bool somatic = 4; + bool germline = 5; + repeated string somaticTumourTypes = 6; + repeated string germlineTumourTypes = 7; + repeated string syndromes = 8; + repeated string tissues = 9; + repeated string modeOfInheritance = 10; + repeated string roleInCancer = 11; + repeated string mutationTypes = 12; + repeated string translocationPartners = 13; +} + +message PharmacogenomicsAlleles { + string allele = 1; + string annotation = 2; + string description = 3; +} + +message PharmacogenomicsClinicalAnnotation { + string variantId = 1; + repeated string geneNames = 2; + repeated string phenotypes = 3; + repeated string phenotypeTypes = 4; + string confidence = 5; + string score = 6; + string url = 7; + string summary = 8; + repeated string pubmed = 9; + repeated PharmacogenomicsAlleles alleles = 10; +} + +message Pharmacogenomics { + string id = 1; + string name = 2; + string source = 3; + repeated string types = 4; + string smiles = 5; + string inChI = 6; + repeated PharmacogenomicsClinicalAnnotation annotations = 7; +} + +message GwasAssociationStudyTraitScores { + double pValue = 1; + double pValueMlog = 2; + string pValueText = 3; + double orBeta = 4; + string percentCI = 5; +} + +message OntologyTermAnnotation { + string id = 1; + string name = 2; + string description = 3; + string source = 4; + string url = 5; + map attributes = 6; +} + +message GwasAssociationStudyTrait { + string diseaseTrait = 1; + repeated string strongestSnpRiskAllele = 2; + repeated OntologyTermAnnotation ontologies = 3; + repeated GwasAssociationStudyTraitScores scores = 4; +} + +message GwasAssociationStudy { + string pubmedid = 1; + string study = 2; + string studyAccession = 3; + string initialSampleSizeDescription = 4; + int32 initialSampleSize = 5; + string platform = 6; + string genotypingTechnology = 7; + repeated GwasAssociationStudyTrait traits = 8; +} + +message GwasAssociation { + string source = 1; + string region = 2; + string snpId = 3; + string riskAllele = 4; + double riskAlleleFrequency = 5; + repeated GwasAssociationStudy studies = 6; +} + +message CancerHotspotAlternateAnnotation { + string aminoacidAlternate = 1; + int32 count = 2; + map sampleCount = 3; +} + +message CancerHotspotVariantAnnotation { + string geneName = 1; + string proteinId = 2; + int32 aminoacidPosition = 3; + string aminoacidReference = 4; + string cancerType = 5; + map scores = 6; + map cancerTypeCount = 7; + map organCount = 8; + repeated CancerHotspotAlternateAnnotation variants = 9; + string source = 10; +} + message VariantAnnotation { string chromosome = 1; int32 start = 2; @@ -181,13 +272,19 @@ message VariantAnnotation { repeated Expression gene_expression = 16; repeated GeneTraitAssociation gene_trait_association = 17; repeated GeneDrugInteraction gene_drug_interaction = 18; - VariantTraitAssociation variant_trait_association = 19; + repeated Constraint gene_constraints = 26; repeated EvidenceEntry trait_association = 24; repeated Score functional_score = 20; repeated Repeat repeat = 21; repeated Cytoband cytoband = 22; repeated Drug drugs = 25; + repeated GeneMirnaTarget gene_mirna_targets = 27; + repeated GeneCancerAssociation gene_cancer_associations = 28; + repeated Pharmacogenomics pharmacogenomics = 29; + repeated GwasAssociation gwas = 30; + repeated CancerHotspotVariantAnnotation cancer_hotspots = 31; + message AdditionalAttribute { map attribute = 1; } diff --git a/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/AvroToProtoConverter.java b/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/AvroToProtoConverter.java new file mode 100644 index 000000000..b2283c1e2 --- /dev/null +++ b/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/AvroToProtoConverter.java @@ -0,0 +1,208 @@ +package org.opencb.biodata.tools.variant.converters.proto; + +import com.google.protobuf.Descriptors; +import com.google.protobuf.MapEntry; +import com.google.protobuf.Message; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; + +public class AvroToProtoConverter { + private final Map builders = new HashMap<>(); + private final Logger logger = LoggerFactory.getLogger(getClass()); + + protected Builder toProto(GenericRecord record, Builder builder) { + return toProto(record, builder, record.getSchema().getName()); + } + + protected Builder toProto(GenericRecord record, Builder builder, String context) { + Descriptors.Descriptor descriptor = builder.getDescriptorForType(); + Map protoFieldsMap = new HashMap<>(); + for (Descriptors.FieldDescriptor fieldDescriptor : descriptor.getFields()) { + protoFieldsMap.put(fieldDescriptor.getName(), fieldDescriptor); + protoFieldsMap.put(fieldDescriptor.getJsonName(), fieldDescriptor); + } + + if (descriptor.getFields().size() != record.getSchema().getFields().size()) { + List missingInAvro = new ArrayList<>(); + List missingInProto = new ArrayList<>(); + for (Schema.Field avroField : record.getSchema().getFields()) { + if (!protoFieldsMap.containsKey(avroField.name())) { + missingInProto.add(avroField.name()); + } + } + for (Descriptors.FieldDescriptor protoField : descriptor.getFields()) { + if (record.getSchema().getField(protoField.getName()) == null + && record.getSchema().getField(protoField.getJsonName()) == null) { + missingInAvro.add(protoField.getName()); + } + } + logger.warn("Number of fields mismatch at " + context + ": Avro fields=" + + record.getSchema().getFields().size() + ", Proto fields=" + descriptor.getFields().size() + + ", missing avro fields: " + missingInAvro + ", missing proto fields: " + missingInProto); + throw new IllegalArgumentException("Number of fields mismatch at " + context + ": Avro fields=" + + record.getSchema().getFields().size() + ", Proto fields=" + descriptor.getFields().size()); + } + + for (Schema.Field avroField : record.getSchema().getFields()) { + Descriptors.FieldDescriptor protoField = protoFieldsMap.get(avroField.name()); + Object protoValue; + if (protoField == null) { + throw new IllegalArgumentException("Field " + context + "." + avroField.name() + " not found in Proto message " + + descriptor.getFullName()); + } + Object avroValue = record.get(avroField.pos()); + switch (avroField.schema().getType()) { + case RECORD: + protoValue = toProto((GenericRecord) avroValue, protoField, context + "." + avroField.name()); + break; + case ENUM: + protoValue = protoField.getEnumType().findValueByName(avroValue.toString()); + break; + case ARRAY: + Collection avroCollection = (Collection) avroValue; + List protoList = new ArrayList<>(avroCollection.size()); + for (Object avroElement : avroCollection) { + Object protoElement = toProto(avroElement, protoField, context + "." + avroField.name()); + protoList.add(protoElement); + } + protoValue = protoList; + break; + case MAP: + Map avroMap = (Map) avroValue; + Map protoMap = new HashMap<>(); + for (Map.Entry entry : avroMap.entrySet()) { + Object protoElement = toProto(entry.getValue(), protoField, context + "." + avroField.name()); + protoMap.put(entry.getKey().toString(), protoElement); + } + protoValue = protoMap; + break; + case UNION: + protoValue = toProto(avroValue, protoField, context + "." + avroField.name()); + break; + case FIXED: + throw new UnsupportedOperationException("FIXED type not supported yet"); + case STRING: + case BYTES: + case INT: + case LONG: + case FLOAT: + case DOUBLE: + case BOOLEAN: + case NULL: + protoValue = avroValue; // Nothing to do + break; + default: + throw new IllegalArgumentException("Unsupported Avro type " + avroField.schema().getType() + + " for field " + context + "." + avroField.name()); + } + + if (protoValue != null) { + if (protoValue instanceof Message.Builder) { + protoValue = ((Message.Builder) protoValue).build(); + } + try { + if (protoField.isMapField()) { + Map mapValue = (Map) protoValue; + MapEntry defaultInstance = MapEntry.newDefaultInstance(protoField.getMessageType(), + protoField.getMessageType().getFields().get(0).getLiteType(), "", + protoField.getMessageType().getFields().get(1).getLiteType(), null + ); + for (Map.Entry entry : mapValue.entrySet()) { + builder.addRepeatedField(protoField, defaultInstance.newBuilderForType() + .setKey(entry.getKey()).setValue(entry.getValue()).build()); + } + } else if (protoField.isRepeated()) { + Collection c = protoValue instanceof Collection ? ((Collection) protoValue) : Collections.singletonList(protoValue); + for (Object o1 : c) { + builder.addRepeatedField(protoField, o1); + } + } else { + builder.setField(protoField, protoValue); + } + } catch (RuntimeException e) { + Descriptors.GenericDescriptor d; + if (protoField.getJavaType() == Descriptors.FieldDescriptor.JavaType.MESSAGE) { + d = protoField.getMessageType(); + } else if (protoField.getJavaType() == Descriptors.FieldDescriptor.JavaType.ENUM ) { + d = protoField.getEnumType(); + } else { + logger.warn("Type = " + protoField.getJavaType()); + d = descriptor; + } + logger.warn("Error adding field '" + context + "." + protoField.getName() + "' type: " + d.toProto() + " value: " + protoValue, e); + throw e; + } + } + } + + return builder; + } + + private Object toProto(Object o, Descriptors.FieldDescriptor fieldDescriptor, String context) { + if (o instanceof GenericRecord) { + return toProto((GenericRecord) o, fieldDescriptor, context).build(); + } else if (o instanceof Collection) { + List list = new ArrayList<>(((Collection) o).size()); + for (Object o1 : ((Collection) o)) { + Object proto = toProto(o1, fieldDescriptor, context); + list.add(proto); + } + return list; + } else if (o instanceof Map) { + Map map = new HashMap<>(); + Descriptors.FieldDescriptor valueDescriptor = fieldDescriptor.getMessageType().getFields().get(1); + for (Map.Entry entry : ((Map) o).entrySet()) { + map.put(entry.getKey().toString(), toProto(entry.getValue(), valueDescriptor, context)); + } + return map; + } else if (o instanceof Enum) { + return fieldDescriptor.getEnumType().findValueByName(o.toString()); + } else { + return o; + } + } + + private Message.Builder toProto(GenericRecord record, Descriptors.FieldDescriptor fieldDescriptor, String context) { + Descriptors.Descriptor messageType = fieldDescriptor.getMessageType(); + Message.Builder builder = builders.computeIfAbsent(messageType.getFullName(), (key) -> newBuilder(messageType)) + .getDefaultInstanceForType().newBuilderForType(); + return toProto(record, builder, context); + } + + private static Message.Builder newBuilder(Descriptors.Descriptor messageType) { + String clazzName = messageType.getFullName().replace("protobuf.opencb.", ""); + clazzName = clazzName.replace(".", "$"); + clazzName = messageType.getFile().toProto().getOptions().getJavaPackage() + "." + messageType.getFile().toProto().getOptions().getJavaOuterClassname() + "$" + clazzName; + Message.Builder builder; + try { + Class aClass = Class.forName(clazzName); + // Invoke static method "newBuilder" + builder = (Message.Builder) aClass.getMethod("newBuilder").invoke(null); + } catch (Exception e) { + throw new RuntimeException(e); + } + return builder; + } + + protected void set(Supplier source, Consumer target) { + T t = source.get(); + if (Objects.nonNull(t)) { + target.accept(t); + } + } + + protected void set(Supplier source, Consumer target, Function mapper) { + TA a = source.get(); + if (Objects.nonNull(a)) { + TP p = mapper.apply(a); + target.accept(p); + } + } +} diff --git a/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/VariantAvroToVariantProtoConverter.java b/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/VariantAvroToVariantProtoConverter.java index 624c26085..528df9d29 100644 --- a/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/VariantAvroToVariantProtoConverter.java +++ b/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/VariantAvroToVariantProtoConverter.java @@ -19,35 +19,19 @@ package org.opencb.biodata.tools.variant.converters.proto; -import com.google.protobuf.Descriptors; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.Message; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.opencb.biodata.models.variant.StudyEntry; import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.VariantBuilder; -import org.opencb.biodata.models.variant.avro.FileEntry; -import org.opencb.biodata.models.variant.avro.SampleEntry; +import org.opencb.biodata.models.variant.avro.*; import org.opencb.biodata.models.variant.protobuf.VariantAnnotationProto; import org.opencb.biodata.models.variant.protobuf.VariantProto; -import org.opencb.biodata.models.variant.stats.VariantStats; import org.opencb.biodata.tools.commons.Converter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; -import java.util.function.Consumer; -import java.util.function.Supplier; -import java.util.stream.Collectors; /** * Created on 10/01/17. * * @author Jacobo Coll <jacobo167@gmail.com> */ -public class VariantAvroToVariantProtoConverter implements Converter { - private final Logger logger = LoggerFactory.getLogger(getClass()); +public class VariantAvroToVariantProtoConverter extends AvroToProtoConverter implements Converter { @Override public VariantProto.Variant convert(Variant variant) { @@ -61,30 +45,33 @@ public VariantProto.Variant convert(Variant variant) { set(variant::getLength, builder::setLength); set(variant::getReference, builder::setReference); set(variant::getAlternate, builder::setAlternate); + set(variant::getStrand, builder::setStrand); set(variant::getType, t -> builder.setType(VariantBuilder.getProtoVariantType(t))); + set(variant::getSv, builder::setSv, sv -> toProto(sv, VariantProto.StructuralVariation.newBuilder(), "variant.sv")); if (variant.getStudies() != null) { - for (StudyEntry study : variant.getStudies()) { - VariantProto.StudyEntry.Builder studyBuilder = toProto(study); + for (org.opencb.biodata.models.variant.avro.StudyEntry studyEntry : variant.getImpl().getStudies()) { + VariantProto.StudyEntry.Builder studyBuilder = toProto(studyEntry); builder.addStudies(studyBuilder); } } if (variant.getAnnotation() != null) { - builder.setAnnotation(toProto(variant.getAnnotation(), VariantAnnotationProto.VariantAnnotation.newBuilder())); + builder.setAnnotation(toProto(variant.getAnnotation(), VariantAnnotationProto.VariantAnnotation.newBuilder(), "variant.annotation")); } return builder.build(); } - private VariantProto.StudyEntry.Builder toProto(StudyEntry study) { + private VariantProto.StudyEntry.Builder toProto(org.opencb.biodata.models.variant.avro.StudyEntry study) { VariantProto.StudyEntry.Builder studyBuilder = VariantProto.StudyEntry.newBuilder(); studyBuilder.setStudyId(study.getStudyId()); set(study::getStudyId, studyBuilder::setStudyId); set(study::getSampleDataKeys, studyBuilder::addAllSampleDataKeys); for (SampleEntry sampleEntry : study.getSamples()) { - studyBuilder.addSamples(VariantProto.SampleEntry.newBuilder().addAllData(sampleEntry.getData())); + studyBuilder.addSamples(toProto(sampleEntry) + ); } - for (VariantStats stats : study.getStats()) { + for (org.opencb.biodata.models.variant.avro.VariantStats stats : study.getStats()) { VariantProto.VariantStats.Builder variantStats = toProto(stats); studyBuilder.addStats(variantStats.build()); } @@ -92,9 +79,50 @@ private VariantProto.StudyEntry.Builder toProto(StudyEntry study) { VariantProto.FileEntry.Builder fileBuilder = toProto(fileEntry); studyBuilder.addFiles(fileBuilder); } + if (study.getSecondaryAlternates() != null) { + for (AlternateCoordinate secondaryAlternate : study.getSecondaryAlternates()) { + VariantProto.AlternateCoordinate.Builder altBuilder = VariantProto.AlternateCoordinate.newBuilder(); + set(secondaryAlternate::getChromosome, altBuilder::setChromosome); + set(secondaryAlternate::getStart, altBuilder::setStart); + set(secondaryAlternate::getEnd, altBuilder::setEnd); + set(secondaryAlternate::getReference, altBuilder::setReference); + set(secondaryAlternate::getAlternate, altBuilder::setAlternate); + set(secondaryAlternate::getType, t -> altBuilder.setType(VariantBuilder.getProtoVariantType(t))); + studyBuilder.addSecondaryAlternates(altBuilder); + } + } + if (study.getScores() != null) { + for (VariantScore score : study.getScores()) { + VariantProto.VariantScore.Builder scoreBuilder = VariantProto.VariantScore.newBuilder(); + set(score::getId, scoreBuilder::setId); + set(score::getScore, scoreBuilder::setScore); + set(score::getPValue, scoreBuilder::setPValue); + set(score::getCohort1, scoreBuilder::setCohort1); + set(score::getCohort2, scoreBuilder::setCohort2); + studyBuilder.addScores(scoreBuilder); + } + } + if (study.getIssues() != null) { + for (IssueEntry issue : study.getIssues()) { + VariantProto.IssueEntry.Builder issueBuilder = VariantProto.IssueEntry.newBuilder(); + set(issue::getType, issueType -> issueBuilder.setType(VariantProto.IssueEntry.IssueType.valueOf(issueType.name()))); + set(issue::getData, issueBuilder::putAllData); + set(issue::getSample, sample -> issueBuilder.setSample(toProto(sample))); + + studyBuilder.addIssues(issueBuilder); + } + } return studyBuilder; } + private static VariantProto.SampleEntry toProto(SampleEntry sampleEntry) { + return VariantProto.SampleEntry.newBuilder() + .setSampleId(sampleEntry.getSampleId()) + .addAllData(sampleEntry.getData()) + .setFileIndex(sampleEntry.getFileIndex()) + .build(); + } + private VariantProto.FileEntry.Builder toProto(FileEntry fileEntry) { VariantProto.FileEntry.Builder fileBuilder = VariantProto.FileEntry.newBuilder(); set(fileEntry::getFileId, fileBuilder::setFileId); @@ -107,7 +135,7 @@ private VariantProto.FileEntry.Builder toProto(FileEntry fileEntry) { return fileBuilder; } - private VariantProto.VariantStats.Builder toProto(VariantStats stats) { + private VariantProto.VariantStats.Builder toProto(org.opencb.biodata.models.variant.avro.VariantStats stats) { VariantProto.VariantStats.Builder statsBuilder = VariantProto.VariantStats.newBuilder(); set(stats::getAlleleCount, statsBuilder::setAlleleCount); set(stats::getRefAlleleCount, statsBuilder::setRefAlleleCount); @@ -122,88 +150,13 @@ private VariantProto.VariantStats.Builder toProto(VariantStats stats) { set(stats::getMgf, statsBuilder::setMgf); set(stats::getMafAllele, statsBuilder::setMafAllele); set(stats::getMgfGenotype, statsBuilder::setMgfGenotype); + set(stats::getFilterCount, map -> map.forEach(statsBuilder::putFilterCount)); + set(stats::getFilterFreq, map -> map.forEach(statsBuilder::putFilterFreq)); + set(stats::getQualityAvg, statsBuilder::setQualityAvg); + set(stats::getCohortId, statsBuilder::setCohortId); + set(stats::getSampleCount, statsBuilder::setSampleCount); + set(stats::getFileCount, statsBuilder::setFileCount); + set(stats::getQualityCount, statsBuilder::setQualityCount); return statsBuilder; } - - private Object toProto(Object o, Descriptors.FieldDescriptor fieldDescriptor) { - if (o instanceof GenericRecord) { - return toProto((GenericRecord) o, fieldDescriptor).build(); - } else if (o instanceof Collection) { - return ((Collection) o).stream().map(o1 -> toProto(o1, fieldDescriptor)).collect(Collectors.toList()); - } else if (o instanceof Map) { - Map map = new HashMap<>(); - ((Map) o).forEach((k, v) -> map.put(k.toString(), toProto(v, fieldDescriptor))); - return map; - } else { - return o; - } - } - - private Message.Builder toProto(GenericRecord record, Descriptors.FieldDescriptor fieldDescriptor) { - Descriptors.Descriptor messageType = fieldDescriptor.getMessageType(); - DynamicMessage.Builder builder = DynamicMessage.newBuilder(messageType); - return toProto(record, builder); - } - - private Builder toProto(GenericRecord record, Builder builder) { - - Descriptors.Descriptor descriptor = builder.getDescriptorForType(); - Map map = new HashMap<>(); - for (Descriptors.FieldDescriptor fieldDescriptor : descriptor.getFields()) { - map.put(fieldDescriptor.getName(), fieldDescriptor); - map.put(fieldDescriptor.getJsonName(), fieldDescriptor); - } - - for (Schema.Field field : record.getSchema().getFields()) { -// Descriptors.FieldDescriptor fieldDescriptor = descriptor.findFieldByName(field.name()); - Descriptors.FieldDescriptor fieldDescriptor = map.get(field.name()); - Object o = record.get(field.pos()); - if (fieldDescriptor == null) { -// logger.warn("Field " + field.name() + " not found! "); - continue; - } - if (fieldDescriptor.getJavaType() == Descriptors.FieldDescriptor.JavaType.MESSAGE) { - o = toProto(o, fieldDescriptor); - } - if (o != null) { - try { - if (fieldDescriptor.isRepeated()) { - Collection c = o instanceof Collection ? ((Collection) o) : Collections.singletonList(o); - for (Object o1 : c) { - if (fieldDescriptor.getJavaType() == Descriptors.FieldDescriptor.JavaType.ENUM) { - o1 = fieldDescriptor.getEnumType().findValueByName(String.valueOf(o1)); - } - builder.addRepeatedField(fieldDescriptor, o1); - } - } else { - if (fieldDescriptor.getJavaType() == Descriptors.FieldDescriptor.JavaType.ENUM) { - o = fieldDescriptor.getEnumType().findValueByName(String.valueOf(o)); - } - builder.setField(fieldDescriptor, o); - } - } catch (RuntimeException e) { - Descriptors.GenericDescriptor d; - if (fieldDescriptor.getJavaType() == Descriptors.FieldDescriptor.JavaType.MESSAGE) { - d = fieldDescriptor.getMessageType(); - } else if (fieldDescriptor.getJavaType() == Descriptors.FieldDescriptor.JavaType.ENUM ) { - d = fieldDescriptor.getEnumType(); - } else { - logger.warn("Type = " + fieldDescriptor.getJavaType()); - d = descriptor; - } - logger.warn("Error adding field '" + fieldDescriptor.getName() + "' type: " + d.toProto() + " value: " + o); - // throw e; - } - } - } - - return builder; - } - - private void set(Supplier source, Consumer target) { - T t = source.get(); - if (Objects.nonNull(t)) { - target.accept(t); - } - } } diff --git a/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/VariantContextToVariantProtoConverter.java b/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/VariantContextToVariantProtoConverter.java index 5c2053680..1abbbcf70 100644 --- a/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/VariantContextToVariantProtoConverter.java +++ b/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/VariantContextToVariantProtoConverter.java @@ -226,7 +226,7 @@ private List setConsequenceTypeParams(){ consequenceType.setEnsemblTranscriptId(null); consequenceType.setStrand(null); consequenceType.setBiotype(null); - consequenceType.setCDnaPosition(0); + consequenceType.setCdnaPosition(0); consequenceType.setCdsPosition(0); consequenceType.setCodon(null); @@ -329,14 +329,6 @@ private VariantAnnotationProto.VariantAnnotation setVaraintAnnotationParams(){ */ variantAnnotation.setChromosome(null); - /* - * set Clinical map type parameter - */ - VariantAnnotationProto.VariantTraitAssociation.Builder variantTraitAssociation = VariantAnnotationProto.VariantTraitAssociation.newBuilder(); - variantTraitAssociation.addAllClinvar(Arrays.asList()); - variantTraitAssociation.addAllCosmic(Arrays.asList()); - variantTraitAssociation.addAllGwas(Arrays.asList()); - variantAnnotation.setVariantTraitAssociation(variantTraitAssociation); /* * set ConsequenceTypes list type parameter diff --git a/biodata-tools/src/test/java/org/opencb/biodata/tools/variant/converters/proto/VariantAvroToVariantProtoConverterTest.java b/biodata-tools/src/test/java/org/opencb/biodata/tools/variant/converters/proto/VariantAvroToVariantProtoConverterTest.java new file mode 100644 index 000000000..cf27bc9e0 --- /dev/null +++ b/biodata-tools/src/test/java/org/opencb/biodata/tools/variant/converters/proto/VariantAvroToVariantProtoConverterTest.java @@ -0,0 +1,100 @@ +package org.opencb.biodata.tools.variant.converters.proto; + +import junit.framework.TestCase; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.RandomUtils; +import org.opencb.biodata.models.variant.Variant; +import org.opencb.biodata.models.variant.avro.VariantAvro; +import org.opencb.biodata.models.variant.protobuf.VariantProto; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +public class VariantAvroToVariantProtoConverterTest extends TestCase { + + public void testConvert() throws Exception { + for (int r = 0; r < 100; r++) { + VariantAvro record = getRandomValue(VariantAvro.getClassSchema()); + + AvroToProtoConverter plainConverter = new VariantAvroToVariantProtoConverter(); + VariantAvroToVariantProtoConverter converter = new VariantAvroToVariantProtoConverter(); + + VariantProto.Variant proto1 = converter.convert(new Variant(record)); + VariantProto.Variant proto2 = plainConverter.toProto(record, VariantProto.Variant.newBuilder()).build(); + + String[] proto1Lines = proto1.toString().split("\n"); + String[] proto2Lines = proto2.toString().split("\n"); + assertEquals(proto1Lines.length, proto2Lines.length); + for (int i = 0; i < proto1Lines.length; i++) { + assertEquals(proto1Lines[i], proto2Lines[i]); + } + assertEquals(proto1, proto2); + } + } + + private static T getRandomValue(Schema schema) throws Exception { + GenericRecord record = (GenericRecord) Class.forName(schema.getFullName()).newInstance(); + + for (Schema.Field subField : schema.getFields()) { + Object value = getRandomValue(subField); + record.put(subField.pos(), value); + } + return (T) record; + } + + private static Object getRandomValue(Schema.Field field) throws Exception { + switch (field.schema().getType()) { + case RECORD: + Schema schema = field.schema(); + GenericRecord record = (GenericRecord) Class.forName(schema.getFullName()).newInstance(); + + for (Schema.Field subField : schema.getFields()) { + Object value = getRandomValue(subField); + record.put(subField.pos(), value); + } + return record; + case ARRAY: + List list = new ArrayList<>(3); + for (int i = 0; i < 3; i++) { + list.add(getRandomValue(field.schema().getElementType())); + } + return list; + case MAP: + Map map = new LinkedHashMap<>(); + map.put("v1", getRandomValue(field.schema().getValueType())); + map.put("v2", getRandomValue(field.schema().getValueType())); + return map; + case UNION: + for (Schema s : field.schema().getTypes()) { + if (s.getType() != Schema.Type.NULL) { + return getRandomValue(s); + } + } + return null; + case STRING: + return RandomStringUtils.randomAlphanumeric(10); + case INT: + return RandomUtils.nextInt(); + case LONG: + return RandomUtils.nextLong(); + case FLOAT: + return RandomUtils.nextFloat(); + case DOUBLE: + return RandomUtils.nextDouble(); + case BOOLEAN: + return RandomUtils.nextBoolean(); + case NULL: + return null; + case ENUM: + Class aClass = (Class) Class.forName(field.schema().getFullName()); + Object[] constants = aClass.getEnumConstants(); + return constants[RandomUtils.nextInt(0, constants.length)]; + default: + throw new IllegalArgumentException("Unsupported type " + field.schema().getType()); + } + } +} \ No newline at end of file From 17c8efd0c11098b93379fcf28fbc5427c6ebd9cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Mon, 20 Oct 2025 13:36:35 +0100 Subject: [PATCH 2/5] tools: Fix VariantAvroToVariantProtoConverterTest. #TASK-7555 --- ...ariantAvroToVariantProtoConverterTest.java | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/biodata-tools/src/test/java/org/opencb/biodata/tools/variant/converters/proto/VariantAvroToVariantProtoConverterTest.java b/biodata-tools/src/test/java/org/opencb/biodata/tools/variant/converters/proto/VariantAvroToVariantProtoConverterTest.java index cf27bc9e0..0a6835e00 100644 --- a/biodata-tools/src/test/java/org/opencb/biodata/tools/variant/converters/proto/VariantAvroToVariantProtoConverterTest.java +++ b/biodata-tools/src/test/java/org/opencb/biodata/tools/variant/converters/proto/VariantAvroToVariantProtoConverterTest.java @@ -18,7 +18,7 @@ public class VariantAvroToVariantProtoConverterTest extends TestCase { public void testConvert() throws Exception { for (int r = 0; r < 100; r++) { - VariantAvro record = getRandomValue(VariantAvro.getClassSchema()); + VariantAvro record = (VariantAvro) getRandomValue(VariantAvro.getClassSchema()); AvroToProtoConverter plainConverter = new VariantAvroToVariantProtoConverter(); VariantAvroToVariantProtoConverter converter = new VariantAvroToVariantProtoConverter(); @@ -36,40 +36,29 @@ public void testConvert() throws Exception { } } - private static T getRandomValue(Schema schema) throws Exception { - GenericRecord record = (GenericRecord) Class.forName(schema.getFullName()).newInstance(); - - for (Schema.Field subField : schema.getFields()) { - Object value = getRandomValue(subField); - record.put(subField.pos(), value); - } - return (T) record; - } - - private static Object getRandomValue(Schema.Field field) throws Exception { - switch (field.schema().getType()) { + private static Object getRandomValue(Schema schema) throws Exception { + switch (schema.getType()) { case RECORD: - Schema schema = field.schema(); GenericRecord record = (GenericRecord) Class.forName(schema.getFullName()).newInstance(); for (Schema.Field subField : schema.getFields()) { - Object value = getRandomValue(subField); + Object value = getRandomValue(subField.schema()); record.put(subField.pos(), value); } return record; case ARRAY: List list = new ArrayList<>(3); for (int i = 0; i < 3; i++) { - list.add(getRandomValue(field.schema().getElementType())); + list.add(getRandomValue(schema.getElementType())); } return list; case MAP: Map map = new LinkedHashMap<>(); - map.put("v1", getRandomValue(field.schema().getValueType())); - map.put("v2", getRandomValue(field.schema().getValueType())); + map.put("v1", getRandomValue(schema.getValueType())); + map.put("v2", getRandomValue(schema.getValueType())); return map; case UNION: - for (Schema s : field.schema().getTypes()) { + for (Schema s : schema.getTypes()) { if (s.getType() != Schema.Type.NULL) { return getRandomValue(s); } @@ -90,11 +79,11 @@ private static Object getRandomValue(Schema.Field field) throws Exception { case NULL: return null; case ENUM: - Class aClass = (Class) Class.forName(field.schema().getFullName()); + Class aClass = (Class) Class.forName(schema.getFullName()); Object[] constants = aClass.getEnumConstants(); return constants[RandomUtils.nextInt(0, constants.length)]; default: - throw new IllegalArgumentException("Unsupported type " + field.schema().getType()); + throw new IllegalArgumentException("Unsupported type " + schema.getType()); } } } \ No newline at end of file From 7ddc9696af6c24161a534c0a582d7f18f7dec7b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 12 Nov 2025 13:00:34 +0000 Subject: [PATCH 3/5] models: Add missing fields to variant_annotation.proto #TASK-7555 --- .../main/proto/protobuf/opencb/variant_annotation.proto | 9 +++++++++ .../variant/converters/proto/AvroToProtoConverter.java | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/biodata-models/src/main/proto/protobuf/opencb/variant_annotation.proto b/biodata-models/src/main/proto/protobuf/opencb/variant_annotation.proto index cc9458333..65c71be27 100644 --- a/biodata-models/src/main/proto/protobuf/opencb/variant_annotation.proto +++ b/biodata-models/src/main/proto/protobuf/opencb/variant_annotation.proto @@ -93,6 +93,7 @@ message ConsequenceType { repeated SpliceScores spliceScores = 16; repeated string transcriptFlags = 17; string source = 18; + repeated string acmg = 19; } message Repeat { @@ -248,6 +249,12 @@ message CancerHotspotVariantAnnotation { string source = 10; } +message GenomicSequenceContext { + int32 start = 1; + int32 end = 2; + string sequence = 3; +} + message VariantAnnotation { string chromosome = 1; int32 start = 2; @@ -285,6 +292,8 @@ message VariantAnnotation { repeated GwasAssociation gwas = 30; repeated CancerHotspotVariantAnnotation cancer_hotspots = 31; + GenomicSequenceContext genomic_sequence_context = 32; + message AdditionalAttribute { map attribute = 1; } diff --git a/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/AvroToProtoConverter.java b/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/AvroToProtoConverter.java index b2283c1e2..ae5b93146 100644 --- a/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/AvroToProtoConverter.java +++ b/biodata-tools/src/main/java/org/opencb/biodata/tools/variant/converters/proto/AvroToProtoConverter.java @@ -12,6 +12,7 @@ import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.Collectors; public class AvroToProtoConverter { private final Map builders = new HashMap<>(); @@ -46,6 +47,10 @@ protected Builder toProto(GenericRecord record logger.warn("Number of fields mismatch at " + context + ": Avro fields=" + record.getSchema().getFields().size() + ", Proto fields=" + descriptor.getFields().size() + ", missing avro fields: " + missingInAvro + ", missing proto fields: " + missingInProto); + logger.warn("Proto fields: " + + descriptor.getFields().stream().map(Descriptors.FieldDescriptor::getJsonName).sorted().collect(Collectors.toList())); + logger.warn("Avro fields: " + + record.getSchema().getFields().stream().map(Schema.Field::name).sorted().collect(Collectors.toList())); throw new IllegalArgumentException("Number of fields mismatch at " + context + ": Avro fields=" + record.getSchema().getFields().size() + ", Proto fields=" + descriptor.getFields().size()); } From 05ac65bf0431322a915cccded3631d91b347c4b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 13 Nov 2025 08:46:33 +0000 Subject: [PATCH 4/5] cicd: Fix get-xetabase-branch.sh call #TASK-7555 --- .github/workflows/pull-request-approved.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index 43fd65432..dea8a94b8 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -25,7 +25,7 @@ jobs: chmod +x ./.github/workflows/scripts/get-xetabase-branch.sh echo "github.event.pull_request.base.ref: ${{ github.event.pull_request.base.ref }}" echo "github.event.pull_request.head.ref: ${{ github.event.pull_request.head.ref }}" - xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }}) + xetabase_branch=$(./.github/workflows/scripts/get-xetabase-branch.sh ${{ github.event.pull_request.base.ref }} ${{ github.event.pull_request.head.ref }}) echo "__Xetabase ref:__ \"${xetabase_branch}\"" | tee -a ${GITHUB_STEP_SUMMARY} echo "xetabase_branch=${xetabase_branch}" >> $GITHUB_OUTPUT env: From 0485caf5ddbf1c5dd1424c69e65785f4d95d158a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 13 Nov 2025 10:38:23 +0000 Subject: [PATCH 5/5] cicd: Fix step "Clone java-common-libs" #TASK-7555 --- .github/workflows/pull-request-approved.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull-request-approved.yml b/.github/workflows/pull-request-approved.yml index dea8a94b8..e7268507d 100644 --- a/.github/workflows/pull-request-approved.yml +++ b/.github/workflows/pull-request-approved.yml @@ -16,6 +16,8 @@ jobs: - name: Clone java-common-libs uses: actions/checkout@v4 with: + repository: opencb/java-common-libs + ref: develop fetch-depth: '10' ## This is important to avoid the error in the next step: "fatal: repository 'https://github.com/zetta-genomics/opencga-enterprise.git/' not found" persist-credentials: false