From 0bad43c5a57b68761495642b1c1385fa03aa1275 Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 2 Nov 2023 22:58:33 -0400 Subject: [PATCH] [BI-1910] Adding unit tests --- .env.template | 5 +- .../geno/SampleSubmissionController.java | 2 +- .../sample/SampleSubmissionImport.java | 8 +- .../importer/services/FileImportService.java | 2 +- .../processors/SampleSubmissionProcessor.java | 31 +- .../services/SampleSubmissionService.java | 30 +- src/main/resources/application.yml | 4 +- .../V1.16.0__create_sampleimport_mapping.sql | 4 +- ...leSubmissionControllerIntegrationTest.java | 502 +++++++++++++++ .../v2/ListControllerIntegrationTest.java | 2 +- .../importer/ExperimentFileImportTest.java | 71 +-- .../brapps/importer/ImportTestUtils.java | 2 +- .../SampleSubmissionFileImportTest.java | 592 ++++++++++++++++++ 13 files changed, 1176 insertions(+), 79 deletions(-) create mode 100644 src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java create mode 100644 src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java diff --git a/.env.template b/.env.template index 283b97128..0f100a7db 100644 --- a/.env.template +++ b/.env.template @@ -54,4 +54,7 @@ AWS_REGION= AWS_ACCESS_KEY_ID= AWS_SECRET_KEY= AWS_GENO_BUCKET= -AWS_S3_ENDPOINT= \ No newline at end of file +AWS_S3_ENDPOINT= + +BRAPI_VENDOR_SUBMISSION_ENABLED=false #can a submission be sent to a vendor via BrAPI +BRAPI_VENDOR_CHECK_FREQUENCY=1d #how often to check for vendor updates for sample submissions \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java index 60d39d7ec..43923c71a 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java @@ -63,7 +63,7 @@ public class SampleSubmissionController { private final Gson gson; @Inject - public SampleSubmissionController(@Property(name = "brapi.vendors.submission-enabled") boolean brapiSubmissionEnabled, SampleSubmissionService sampleSubmissionService, ProgramService programService, SecurityService securityService, UserService userService) { + public SampleSubmissionController(@Property(name = "brapi.vendor-submission-enabled") boolean brapiSubmissionEnabled, SampleSubmissionService sampleSubmissionService, ProgramService programService, SecurityService securityService, UserService userService) { this.brapiSubmissionEnabled = brapiSubmissionEnabled; this.sampleSubmissionService = sampleSubmissionService; this.programService = programService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java index 242b5e4a7..65a6be3bc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java @@ -84,8 +84,8 @@ public class SampleSubmissionImport implements BrAPIImport { private String tissue; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "comment", name = Columns.COMMENTS, description = "Generic comments about this sample for the vendor") - private String comment; + @ImportFieldMetadata(id = "comments", name = Columns.COMMENTS, description = "Generic comments about this sample for the vendor") + private String comments; @ImportFieldType(type= ImportFieldTypeEnum.TEXT, collectTime = ImportCollectTimeEnum.UPLOAD) @ImportMappingRequired @@ -99,7 +99,7 @@ public static final class Columns { public static final String ORGANISM = "Organism"; public static final String SPECIES = "Species"; public static final String GERMPLASM_NAME = "Germplasm Name"; - public static final String GERMPLASM_GID = "Germplasm GID"; + public static final String GERMPLASM_GID = "GID"; public static final String TISSUE = "Tissue"; public static final String COMMENTS = "Comments"; public static final String OBS_UNIT_ID = "ObsUnitID"; @@ -146,7 +146,7 @@ public BrAPISample constructBrAPISample(boolean commit, Program program, User us .row(row) .column(Integer.valueOf(column)) .tissueType(tissue) - .sampleDescription(comment); + .sampleDescription(comments); if (ou != null) { brAPISample diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 0d8f7d971..05c48601d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -454,7 +454,7 @@ private void processFile(List finalBrAPIImportList, Table data, Pro progress.setUpdatedBy(actingUser.getId()); importDAO.update(upload); }catch (ValidatorException e) { - log.info("Validation errors", e); + log.info("Validation errors: \n" + e); ImportProgress progress = upload.getProgress(); progress.setStatuscode((short) HttpStatus.UNPROCESSABLE_ENTITY.getCode()); progress.setMessage("Multiple Errors"); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java index 531df2a09..d92ebb300 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java @@ -20,6 +20,7 @@ import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.HttpStatus; +import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; @@ -73,10 +74,6 @@ public class SampleSubmissionProcessor implements Processor { private final BrAPIGermplasmDAO germplasmDAO; private final BrAPIObservationUnitDAO observationUnitDAO; private final SampleSubmissionService sampleSubmissionService; - private final DSLContext dsl; - private final BrAPIPlateDAO plateDAO; - private final BrAPISampleDAO sampleDAO; - private SampleSubmission submission; private Map germplasmByGID = new HashMap<>(); private Map germplasmByDbId = new HashMap<>(); @@ -88,17 +85,11 @@ public class SampleSubmissionProcessor implements Processor { public SampleSubmissionProcessor(@Property(name = "brapi.server.reference-source") String referenceSource, BrAPIGermplasmDAO germplasmDAO, BrAPIObservationUnitDAO observationUnitDAO, - SampleSubmissionService sampleSubmissionService, - DSLContext dsl, - BrAPIPlateDAO plateDAO, - BrAPISampleDAO sampleDAO) { + SampleSubmissionService sampleSubmissionService) { this.referenceSource = referenceSource; this.germplasmDAO = germplasmDAO; this.observationUnitDAO = observationUnitDAO; this.sampleSubmissionService = sampleSubmissionService; - this.dsl = dsl; - this.plateDAO = plateDAO; - this.sampleDAO = sampleDAO; } @Override @@ -260,15 +251,21 @@ public void validateDependencies(Map mappedBrAPIImport) @Override public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) throws ValidatorException { - dsl.transaction(() -> { - List platesToSave = plateById.values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toList()); - List samplesToSave = mappedBrAPIImport.values().stream().map(row -> row.getSample().getBrAPIObject()).collect(Collectors.toList()); + List platesToSave = plateById.values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toList()); + List samplesToSave = mappedBrAPIImport.values().stream().map(row -> row.getSample().getBrAPIObject()).collect(Collectors.toList()); - submission.setPlates(platesToSave); - submission.setSamples(samplesToSave); + submission.setPlates(platesToSave); + submission.setSamples(samplesToSave); + try { sampleSubmissionService.createSubmission(submission, program, upload); - }); + } catch (ApiException e) { + log.error("Error saving sample submission import: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving sample submission import", e); + } catch (Exception e) { + log.error("Error saving sample submission import", e); + throw new InternalServerException(e.getMessage(), e); + } } @Override diff --git a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java index 7c117e34f..2c629da55 100644 --- a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java +++ b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java @@ -67,10 +67,11 @@ public class SampleSubmissionService { private static final String VENDOR_NOT_SUBMITTED_STATUS = "NOT SUBMITTED"; private static final String VENDOR_SUBMITTED_STATUS = "SUBMITTED"; private final String referenceSource; - private String dartBrapiUrl; - private String dartClientId; - private String dartToken; - private Duration requestTimeout; + private final String dartBrapiUrl; + private final String dartClientId; + private final String dartToken; + private final Duration requestTimeout; + private final boolean brapiSubmissionEnabled; private final SampleSubmissionDAO submissionDAO; private final BrAPIPlateDAO plateDAO; @@ -85,6 +86,7 @@ public SampleSubmissionService(@Property(name = "brapi.server.reference-source") @Property(name = "brapi.vendors.dart.client-id") String dartClientId, @Property(name = "brapi.vendors.dart.token") String dartToken, @Value(value = "${brapi.read-timeout:5m}") Duration requestTimeout, + @Property(name = "brapi.vendor-submission-enabled") boolean brapiSubmissionEnabled, SampleSubmissionDAO submissionDAO, BrAPIPlateDAO plateDAO, BrAPISampleDAO sampleDAO, @@ -96,6 +98,7 @@ public SampleSubmissionService(@Property(name = "brapi.server.reference-source") this.dartClientId = dartClientId; this.dartToken = dartToken; this.requestTimeout = requestTimeout; + this.brapiSubmissionEnabled = brapiSubmissionEnabled; this.submissionDAO = submissionDAO; this.plateDAO = plateDAO; this.sampleDAO = sampleDAO; @@ -106,6 +109,9 @@ public SampleSubmissionService(@Property(name = "brapi.server.reference-source") @Scheduled(fixedDelay = "${brapi.vendor-check-frequency}", initialDelay = "10s") void checkSubmissionStatuses() { + if(!brapiSubmissionEnabled) { + return; + } log.trace("checking vendor order statuses"); List submittedAndNotCompleted = submissionDAO.getSubmittedAndNotComplete(); log.trace(submittedAndNotCompleted.size() + " orders to check"); @@ -127,15 +133,17 @@ public SampleSubmission createSubmission(SampleSubmission submission, Program pr submission.setUpdatedByUser(upload.getUpdatedByUser()); submission.setUpdatedBy(upload.getUpdatedBy()); - submissionDAO.insert(submission); + dsl.transaction(() -> { + submissionDAO.insert(submission); - List savedPlates = plateDAO.createPlates(program, submission.getPlates(), upload); - submission.setPlates(savedPlates); - Map plateNameToDbId = savedPlates.stream().collect(Collectors.toMap(BrAPIPlate::getPlateName, BrAPIPlate::getPlateDbId)); + List savedPlates = plateDAO.createPlates(program, submission.getPlates(), upload); + submission.setPlates(savedPlates); + Map plateNameToDbId = savedPlates.stream().collect(Collectors.toMap(BrAPIPlate::getPlateName, BrAPIPlate::getPlateDbId)); - List samplesToSave = submission.getSamples().stream().map(sample -> sample.plateDbId(plateNameToDbId.get(sample.getPlateName()))).collect(Collectors.toList()); - List savedSamples = sampleDAO.createSamples(program, samplesToSave, upload); - submission.setSamples(savedSamples); + List samplesToSave = submission.getSamples().stream().map(sample -> sample.plateDbId(plateNameToDbId.get(sample.getPlateName()))).collect(Collectors.toList()); + List savedSamples = sampleDAO.createSamples(program, samplesToSave, upload); + submission.setSamples(savedSamples); + }); return submission; } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 13e35457b..aec211b82 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -160,8 +160,9 @@ brapi: pheno-url: ${brapi.server.default-url} geno-url: ${brapi.server.default-url} reference-source: ${BRAPI_REFERENCE_SOURCE:breedinginsight.org} + vendor-submission-enabled: ${BRAPI_VENDOR_SUBMISSION_ENABLED:false} + vendor-check-frequency: ${BRAPI_VENDOR_CHECK_FREQUENCY:1d} vendors: - submission-enabled: ${BRAPI_VENDOR_SUBMISSION_ENABLED:false} dart: url: ${DART_VENDOR_URL:`https://test-server.brapi.org`} client-id: ${DART_CLIENT_ID:potato-salad} @@ -171,7 +172,6 @@ brapi: search: wait-time: 1000 post-group-size: ${POST_CHUNK_SIZE:1000} - vendor-check-frequency: ${VENDOR_CHECK_FREQUENCY:30s} email: relay-server: diff --git a/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql b/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql index eb88ec05d..50bac1903 100644 --- a/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql +++ b/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql @@ -93,9 +93,9 @@ $$ { "id": "de7fed3a-ec44-4139-83cb-c773e09237bd", "value": { - "fileFieldName": "Comment" + "fileFieldName": "Comments" }, - "objectId": "comment" + "objectId": "comments" } ]', '[]', false, diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java new file mode 100644 index 000000000..d41874783 --- /dev/null +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -0,0 +1,502 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.v1.controller; + +import com.eclipsesource.json.Json; +import com.google.gson.*; +import com.google.gson.reflect.TypeToken; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import org.apache.commons.lang3.tuple.Pair; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.ImportTestUtils; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport; +import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport.Columns; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.dao.db.tables.pojos.ProgramBreedingMethodEntity; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.parsers.ParsingException; +import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.utilities.FileUtil; +import org.breedinginsight.utilities.Utilities; +import org.jetbrains.annotations.NotNull; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; +import org.testcontainers.containers.localstack.LocalStackContainer; +import tech.tablesaw.api.Table; + +import javax.inject.Inject; +import java.io.*; +import java.time.OffsetDateTime; +import java.util.*; + +import static io.micronaut.http.HttpRequest.*; +import static org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields.SUBMISSION_NAME; +import static org.junit.jupiter.api.Assertions.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class SampleSubmissionControllerIntegrationTest extends BrAPITest { + + private Program program; + + private ImportTestUtils importTestUtils; + + private String experimentId; + private List envIds = new ArrayList<>(); + private final List> rows = new ArrayList<>(); + private final List columns = ExperimentFileColumns.getOrderedColumns(); + private List traits; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Inject + private DSLContext dsl; + @Inject + private UserDAO userDAO; + @Inject + private SpeciesDAO speciesDAO; + @Inject + private OntologyService ontologyService; + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + +// private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) +// (json, type, context) -> OffsetDateTime.parse(json.getAsString())) +// .create(); + + private final Gson gson = new BrAPIClient().getJSON().getGson(); + + @BeforeAll + void setup() throws Exception { + importTestUtils = new ImportTestUtils(); + FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); + + // Test User + User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Species + super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + + // Test Program + ProgramRequest programRequest = ProgramRequest.builder() + .name("Test Program") + .abbreviation("Test") + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key("TEST") + .build(); + program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + List germplasm = createGermplasm(96); + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); + } + + @NotNull + @Override + public Map getProperties() { + Map properties = super.getProperties(); + + properties.put("brapi.vendor-submission-enabled", "true"); + + Integer containerPort = getBrapiContainer().getMappedPort(8080); + String containerIp = getBrapiContainer().getContainerIpAddress(); + properties.put("brapi.vendors.dart.url", String.format("http://%s:%s/", containerIp, containerPort)); + + return properties; + } + + /* + Tests + - fetch all sample submissions + - fetch individual sample submission + - manual update submission status + - brapi submit + - check vendor status + + */ + + @Test + public void testFetchProgramSubmissions() throws IOException, InterruptedException { + List submissionIds = new ArrayList<>(); + submissionIds.add(createSubmission(program).getLeft().getId()); + submissionIds.add(createSubmission(program).getLeft().getId()); + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/submissions", program.getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + List submissions = gson.fromJson(JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").getAsJsonArray("data"), new TypeToken>(){}.getType()); + assertTrue(submissions.size() >= 2); + + + List returnedSubmissionIds = new ArrayList<>(); + for(var submission : submissions) { + if(submissionIds.contains(submission.getId())) { + returnedSubmissionIds.add(submission.getId()); + } + } + assertEquals(submissionIds.size(), returnedSubmissionIds.size()); + } + + @Test + public void testFetchIndividualSubmissions() throws IOException, InterruptedException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=true", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + SampleSubmission retrievedSubmission = gson.fromJson(JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + + assertNotNull(retrievedSubmission); + assertEquals(uploadedSubmission.getLeft().getId(), retrievedSubmission.getId()); + assertEquals(uploadedSubmission.getLeft().getName(), retrievedSubmission.getName()); + assertEquals(1, retrievedSubmission.getPlates().size()); + assertEquals(96, retrievedSubmission.getSamples().size()); + } + + @Test + public void testManualUpdateSubmissions() throws IOException, InterruptedException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> putCall = client.exchange( + PUT(String.format("/programs/%s/submissions/%s/status", program.getId(), uploadedSubmission.getLeft().getId()), "{status:\"SUBMITTED\"}").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse putResponse = putCall.blockingFirst(); + assertNotNull(putResponse.body()); + + Flowable> fetchCall = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=true", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchResponse = fetchCall.blockingFirst(); + SampleSubmission retrievedSubmission = gson.fromJson(JsonParser.parseString(fetchResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + + assertNotNull(retrievedSubmission); + assertEquals("SUBMITTED", retrievedSubmission.getVendorStatus()); + assertNull(retrievedSubmission.getVendorOrderId()); + assertNull(retrievedSubmission.getVendorStatusLastCheck()); + } + + @Test + public void testSubmitViaBrAPI() throws IOException, InterruptedException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/submissions/%s/submit?vendor=dart", program.getId(), uploadedSubmission.getLeft().getId()), null).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse postResponse = postCall.blockingFirst(); + assertNotNull(postResponse.body()); + + Flowable> fetchCall = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=true", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchResponse = fetchCall.blockingFirst(); + SampleSubmission retrievedSubmission = gson.fromJson(JsonParser.parseString(fetchResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + + assertNotNull(retrievedSubmission); + assertEquals("SUBMITTED", retrievedSubmission.getVendorStatus()); + assertNotNull(retrievedSubmission.getVendorOrderId()); + assertNull(retrievedSubmission.getVendorStatusLastCheck()); + } + + @Test + public void testCheckVendorStatus() throws IOException, InterruptedException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/submissions/%s/submit?vendor=dart", program.getId(), uploadedSubmission.getLeft().getId()), null).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse postResponse = postCall.blockingFirst(); + assertNotNull(postResponse.body()); + + Flowable> fetchCall = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=false", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchResponse = fetchCall.blockingFirst(); + SampleSubmission retrievedSubmission = gson.fromJson(JsonParser.parseString(fetchResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + + assertNotNull(retrievedSubmission); + assertNotNull(retrievedSubmission.getVendorOrderId()); + assertNull(retrievedSubmission.getVendorStatusLastCheck()); + + Flowable> fetchStatus = client.exchange( + GET(String.format("/programs/%s/submissions/%s/status", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchStatusResponse = fetchStatus.blockingFirst(); + assertNotNull(fetchStatusResponse.body()); + + Flowable> fetchSubmissionAfterStatus = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=false", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchSubmissionAfterStatusResponse = fetchSubmissionAfterStatus.blockingFirst(); + SampleSubmission updatedStatusResponse = gson.fromJson(JsonParser.parseString(fetchSubmissionAfterStatusResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + assertNotNull(updatedStatusResponse.getVendorStatusLastCheck()); + + Thread.sleep(1000); + + fetchStatus = client.exchange( + GET(String.format("/programs/%s/submissions/%s/status", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + fetchStatusResponse = fetchStatus.blockingFirst(); + assertNotNull(fetchStatusResponse.body()); + + fetchSubmissionAfterStatus = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=false", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + fetchSubmissionAfterStatusResponse = fetchSubmissionAfterStatus.blockingFirst(); + SampleSubmission updatedStatusResponse2 = gson.fromJson(JsonParser.parseString(fetchSubmissionAfterStatusResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + assertTrue(updatedStatusResponse2.getVendorStatusLastCheck().isAfter(updatedStatusResponse.getVendorStatusLastCheck())); + } + + @Test + public void testGenerateDArTFile() throws IOException, InterruptedException, ParsingException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/submissions/%s/dart", + program.getId().toString(), uploadedSubmission.getLeft().getId())) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse response = call.blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(response.body())); + Table lookupTable = FileUtil.parseTableFromCsv(bodyStream); + assertEquals(8, lookupTable.columnCount()); + assertEquals(Columns.PLATE_ID, lookupTable.column(0).name()); + assertEquals(Columns.ROW, lookupTable.column(1).name()); + assertEquals(Columns.COLUMN, lookupTable.column(2).name()); + assertEquals(Columns.ORGANISM, lookupTable.column(3).name()); + assertEquals(Columns.SPECIES, lookupTable.column(4).name()); + assertEquals("Genotype", lookupTable.column(5).name()); + assertEquals(Columns.TISSUE, lookupTable.column(6).name()); + assertEquals(Columns.COMMENTS, lookupTable.column(7).name()); + } + + @Test + public void testGenerateLookupFile() throws IOException, InterruptedException, ParsingException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/submissions/%s/lookup", + program.getId().toString(), uploadedSubmission.getLeft().getId())) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse response = call.blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(response.body())); + Table lookupTable = FileUtil.parseTableFromCsv(bodyStream); + assertEquals(3, lookupTable.columnCount()); + assertEquals("Genotype", lookupTable.column(0).name()); + assertEquals("Germplasm Name", lookupTable.column(1).name()); + assertEquals("GID", lookupTable.column(2).name()); + } + + + private Pair>> createSubmission(Program program) throws IOException, InterruptedException { + Flowable> call = client.exchange( + GET("/import/mappings?importName=SampleImport").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String sampleMappingId = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + var submissionData = makeSubmission(); + var submission = new SampleSubmission(); + submission.setName("test-"+UUID.randomUUID()); + + JsonObject importResult = importTestUtils.uploadAndFetch( + writeSubmissionToFile(submissionData), + Map.of(SUBMISSION_NAME, submission.getName()), + true, + client, + program, + sampleMappingId); + JsonArray previewRows = importResult.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(96, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + BrAPISample sample = new Gson().fromJson(row.getAsJsonObject("sample").getAsJsonObject("brAPIObject"), BrAPISample.class); + BrAPIExternalReference xref = Utilities.getExternalReference(sample.getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.PLATE_SUBMISSIONS)).get(); + submission.setId(UUID.fromString(xref.getReferenceId())); + + return Pair.of(submission, submissionData); + } + + private List> makeSubmission() { + List> validFile = new ArrayList<>(); + int germGidCounter = 1; + for(int i = 0; i < 8; i++) { + for(int j = 0; j < 12; j++) { + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A' + i)); + validRow.put(Columns.COLUMN, j+1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, germGidCounter++); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + } + } + + return validFile; + } + + private String createExperiment(Program program) throws IOException, InterruptedException { + Flowable> call = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String expMappingId = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + JsonObject importResult = importTestUtils.uploadAndFetch( + importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null), + null, + true, + client, + program, + expMappingId); + return importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + } + + private Map makeExpImportRow(String environment) { + Map row = new HashMap<>(); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); + row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); + row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); + row.put(ExperimentObservation.Columns.ENV, environment); + row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); + row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, "a-1"); + row.put(ExperimentObservation.Columns.REP_NUM, "1"); + row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); + row.put(ExperimentObservation.Columns.ROW, "1"); + row.put(ExperimentObservation.Columns.COLUMN, "1"); + return row; + } + + public File writeSubmissionToFile(List> data) throws IOException { + File file = File.createTempFile("test", ".csv"); + + List columns = new ArrayList<>(); + columns.add(Column.builder().value(Columns.PLATE_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.ROW).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.COLUMN).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.ORGANISM).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.SPECIES).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_NAME).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_GID).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.OBS_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.TISSUE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.COMMENTS).dataType(Column.ColumnDataType.STRING).build()); + + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index ccdf04e19..fdabb53db 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -142,7 +142,7 @@ public void setup() { newExp.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetch( - importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId + importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId ); } diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index ac8520a68..748299daf 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -18,7 +18,6 @@ package org.breedinginsight.brapps.importer; import com.google.gson.*; -import com.google.gson.reflect.TypeToken; import io.kowalski.fannypack.FannyPack; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; @@ -50,7 +49,6 @@ import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; import org.breedinginsight.daos.ProgramDAO; @@ -64,7 +62,6 @@ import org.breedinginsight.services.exceptions.BadRequestException; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.ValidatorException; -import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import org.junit.jupiter.api.*; @@ -74,9 +71,7 @@ import org.opentest4j.AssertionFailedError; import javax.inject.Inject; -import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.time.OffsetDateTime; import java.util.*; @@ -199,7 +194,7 @@ public void importNewExpNewLocNoObsSuccess() { validRow.put(Columns.COLUMN, "1"); validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeDataToFile(List.of(validRow), null), null, true, client, program, mappingId); + Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); @@ -257,7 +252,7 @@ public void importNewExpMultiNewEnvNoObsSuccess() { secondEnv.put(Columns.COLUMN, "1"); secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); + Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); @@ -304,7 +299,7 @@ public void importNewEnvExistingExpNoObsSuccess() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map newEnv = new HashMap<>(); newEnv.put(Columns.GERMPLASM_GID, "1"); @@ -321,7 +316,7 @@ public void importNewEnvExistingExpNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -355,43 +350,43 @@ public void verifyMissingDataThrowsError(boolean commit) { Map noGID = new HashMap<>(base); noGID.remove(Columns.GERMPLASM_GID); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); Map noExpTitle = new HashMap<>(base); noExpTitle.remove(Columns.EXP_TITLE); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); Map noExpUnit = new HashMap<>(base); noExpUnit.remove(Columns.EXP_UNIT); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); Map noExpType = new HashMap<>(base); noExpType.remove(Columns.EXP_TYPE); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); Map noEnv = new HashMap<>(base); noEnv.remove(Columns.ENV); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noEnv), null), Columns.ENV, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); Map noEnvLoc = new HashMap<>(base); noEnvLoc.remove(Columns.ENV_LOCATION); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); Map noExpUnitId = new HashMap<>(base); noExpUnitId.remove(Columns.EXP_UNIT_ID); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); Map noExpRep = new HashMap<>(base); noExpRep.remove(Columns.REP_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); Map noExpBlock = new HashMap<>(base); noExpBlock.remove(Columns.BLOCK_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); Map noEnvYear = new HashMap<>(base); noEnvYear.remove(Columns.ENV_YEAR); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); } @Test @@ -415,7 +410,7 @@ public void importNewExpWithObsVar() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -465,7 +460,7 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(rows, null), Columns.ENV_YEAR, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); } @ParameterizedTest @@ -503,7 +498,7 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(rows, null), Columns.ENV_LOCATION, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); } @ParameterizedTest @@ -528,7 +523,7 @@ public void importNewExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -568,7 +563,7 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "Red"); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); } @ParameterizedTest @@ -591,14 +586,14 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map newOU = new HashMap<>(newExp); newOU.put(Columns.EXP_UNIT_ID, "a-2"); newOU.put(Columns.ROW, "1"); newOU.put(Columns.COLUMN, "2"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); + Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); @@ -632,7 +627,7 @@ public void importNewObsVarExisingOu() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -660,7 +655,7 @@ public void importNewObsVarExisingOu() { newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -697,7 +692,7 @@ public void importNewObsExisingOu(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -725,7 +720,7 @@ public void importNewObsExisingOu(boolean commit) { newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -764,7 +759,7 @@ public void verifyFailureImportNewObsExisingOuWithExistingObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -792,7 +787,7 @@ public void verifyFailureImportNewObsExisingOuWithExistingObs(boolean commit) { newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit); } /* @@ -822,7 +817,7 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsA.size()); @@ -850,7 +845,7 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsB.size()); @@ -891,7 +886,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -920,7 +915,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -965,7 +960,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -996,7 +991,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), ""); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index b8b00374d..3a521aa55 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -196,7 +196,7 @@ public List createTraits(int numToCreate) { return traits; } - public File writeDataToFile(List> data, List traits) throws IOException { + public File writeExperimentDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); List columns = new ArrayList<>(); diff --git a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java new file mode 100644 index 000000000..aa446cc55 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java @@ -0,0 +1,592 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer; + +import com.google.gson.*; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import lombok.SneakyThrows; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.typeAdapters.PaginationTypeAdapter; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.BrAPIPagination; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.daos.*; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport.Columns; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.Column; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.SampleSubmission; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.*; +import org.breedinginsight.services.exceptions.BadRequestException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.platform.commons.util.StringUtils; + +import javax.inject.Inject; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.*; + +import static io.micronaut.http.HttpRequest.GET; +import static org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields.SUBMISSION_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class SampleSubmissionFileImportTest extends BrAPITest { + + private FannyPack securityFp; + private String mappingId; + private BiUserEntity testUser; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Property(name = "brapi.server.core-url") + private String BRAPI_URL; + + @Inject + private SpeciesService speciesService; + @Inject + private UserDAO userDAO; + @Inject + private DSLContext dsl; + + @Inject + private SpeciesDAO speciesDAO; + + @Inject + private ProgramService programService; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private ImportTestUtils importTestUtils; + + @Inject + private OntologyService ontologyService; + + @Inject + private BrAPITrialDAO brAPITrialDAO; + + @Inject + private BrAPIStudyDAO brAPIStudyDAO; + + @Inject + private BrAPIObservationUnitDAO ouDAO; + + @Inject + private ProgramLocationService locationService; + + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + private BrAPIObservationDAO observationDAO; + + @Inject + private BrAPISeasonDAO seasonDAO; + + @Inject + private SampleSubmissionService sampleSubmissionService; + + private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) + .create(); + + @BeforeAll + public void setup() { + importTestUtils = new ImportTestUtils(); + Map setupObjects = importTestUtils.setup(client, gson, dsl, speciesService, userDAO, super.getBrapiDsl(), "SampleImport"); + mappingId = (String) setupObjects.get("mappingId"); + testUser = (BiUserEntity) setupObjects.get("testUser"); + securityFp = (FannyPack) setupObjects.get("securityFp"); + + } + + /* + Tests + - valid submission GID + - valid submission ObsUnitID + - missing columns error + - conflicting wells error + - bad GID error + - bad ObsUnitID error + */ + + @Test + @SneakyThrows + public void importGIDSuccess() { + Program program = createProgram("Import GID Success", "GIDS", "GIDS", BRAPI_REFERENCE_SOURCE, createGermplasm(96), null); + List> validFile = new ArrayList<>(); + + int germGidCounter = 1; + for(int i = 0; i < 8; i++) { + for(int j = 0; j < 12; j++) { + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A' + i)); + validRow.put(Columns.COLUMN, j+1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, germGidCounter++); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + } + } + + Flowable> call = importTestUtils.uploadDataFile(writeDataToFile(validFile), Map.of(SUBMISSION_NAME, "test-"+UUID.randomUUID()), true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(96, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", row.getAsJsonObject("plate").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("sample").get("state").getAsString()); + + BrAPISample sample = new Gson().fromJson(row.getAsJsonObject("sample").getAsJsonObject("brAPIObject"), BrAPISample.class); + BrAPIExternalReference xref = Utilities.getExternalReference(sample.getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.PLATE_SUBMISSIONS)).get(); + + assertFileSaved(validFile, program, UUID.fromString(xref.getReferenceId())); + } + + @Test + @SneakyThrows + public void importObsUnitIdSuccess() { + Program program = createProgram("Import ObsUnitID success", "OBSID", "OBSID", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + + var experimentId = createExperiment(program); + + BrAPITrial trial = brAPITrialDAO.getTrialById(program.getId(), UUID.fromString(experimentId)).get(); + + List ous = ouDAO.getObservationUnitsForTrialDbId(program.getId(), trial.getTrialDbId()); + BrAPIExternalReference obsUnitId = Utilities.getExternalReference(ous.get(0).getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS)).get(); + + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, "A"); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, ""); + validRow.put(Columns.OBS_UNIT_ID, obsUnitId.getReferenceId()); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + Flowable> call = importTestUtils.uploadDataFile(writeDataToFile(validFile), Map.of(SUBMISSION_NAME, "test-"+UUID.randomUUID()), true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", row.getAsJsonObject("plate").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("sample").get("state").getAsString()); + + BrAPISample sample = new Gson().fromJson(row.getAsJsonObject("sample").getAsJsonObject("brAPIObject"), BrAPISample.class); + BrAPIExternalReference xref = Utilities.getExternalReference(sample.getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.PLATE_SUBMISSIONS)).get(); + + assertFileSaved(validFile, program, UUID.fromString(xref.getReferenceId())); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void importMissingGIDAndObsUnitIdFailure(boolean commit) { + Program program = createProgram("Missing GID/ObsUnit ID " + (commit ? "C" : "P"), "MGIOB"+ (commit ? "C" : "P"), "MGIOB"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, ""); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + uploadAndVerifyFailure(program, writeDataToFile(validFile), Columns.GERMPLASM_GID, commit); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void verifyMissingDataThrowsError(boolean commit) { + Program program = createProgram("Missing Req Cols "+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(96), null); + Map base = new HashMap<>(); + base.put(Columns.PLATE_ID, "valid_1"); + base.put(Columns.ROW, "A"); + base.put(Columns.COLUMN, 1); + base.put(Columns.ORGANISM, "TEST"); + base.put(Columns.SPECIES, "TEST"); + base.put(Columns.GERMPLASM_NAME, ""); + base.put(Columns.GERMPLASM_GID, "1"); + base.put(Columns.OBS_UNIT_ID, ""); + base.put(Columns.TISSUE, "TEST"); + base.put(Columns.COMMENTS, "Test sample"); + + createUploadAndVerifyFailure(program, base, Columns.PLATE_ID, commit); + createUploadAndVerifyFailure(program, base, Columns.ROW, commit); + createUploadAndVerifyFailure(program, base, Columns.COLUMN, commit); + createUploadAndVerifyFailure(program, base, Columns.ORGANISM, commit); + createUploadAndVerifyFailure(program, base, Columns.TISSUE, commit); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void importInvalidGIDFailure(boolean commit) { + Program program = createProgram("Invalid GID " + (commit ? "C" : "P"), "INGID"+ (commit ? "C" : "P"), "INGID"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, ""); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + uploadAndVerifyFailure(program, writeDataToFile(validFile), Columns.GERMPLASM_GID, commit); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void importInvalidObsUnitIdFailure(boolean commit) { + Program program = createProgram("Invalid ObsUnit ID " + (commit ? "C" : "P"), "INOBS"+ (commit ? "C" : "P"), "INOBS"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, ""); + validRow.put(Columns.OBS_UNIT_ID, "hgfhgfhg"); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + uploadAndVerifyFailure(program, writeDataToFile(validFile), Columns.OBS_UNIT_ID, commit); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void importConflictingWellsFailure(boolean commit) { + Program program = createProgram("Conflicting Wells " + (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(2), null); + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, 1); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, 2); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + uploadAndVerifyFailure(program, writeDataToFile(validFile), Columns.ROW + "/" + Columns.COLUMN, commit); + } + + private void assertFileSaved(List> validFile, Program program, UUID submissionId) throws ApiException { + Optional submission = sampleSubmissionService.getSampleSubmission(program, submissionId, true); + assertTrue(submission.isPresent(), "Could not find sample submission by ID: " + submissionId); + for(var row : validFile) { + assertRowSaved(row, program, submission.get()); + } + } + + private Map assertRowSaved(Map expected, Program program, SampleSubmission submission) { + Map ret = new HashMap<>(); + + Optional plate = submission.getPlates().stream().filter(p -> p.getPlateName().equals(expected.get(Columns.PLATE_ID))).findFirst(); + assertTrue(plate.isPresent(), "plate not found"); + + Optional sample = submission.getSamples().stream().filter(s -> s.getPlateName().equals(expected.get(Columns.PLATE_ID)) + && s.getRow().equals(expected.get(Columns.ROW)) + && s.getColumn().equals(expected.get(Columns.COLUMN))) + .findFirst(); + assertTrue(sample.isPresent(), String.format("sample %s%s not found", expected.get(Columns.ROW), expected.get(Columns.COLUMN))); + + assertEquals(expected.get(Columns.ORGANISM), sample.get().getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_ORGANISM).getAsString()); + assertEquals(expected.get(Columns.SPECIES), sample.get().getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_SPECIES).getAsString()); + assertTrue(sample.get().getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_NAME)); + assertEquals(expected.get(Columns.TISSUE), sample.get().getTissueType()); + assertEquals(expected.get(Columns.COMMENTS), sample.get().getSampleDescription()); + + if(StringUtils.isNotBlank((String)expected.get(Columns.OBS_UNIT_ID))) { + assertTrue(sample.get().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBS_UNIT_ID)); + assertEquals(expected.get(Columns.OBS_UNIT_ID), sample.get().getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBS_UNIT_ID).getAsString()); + } else { + assertEquals(String.valueOf(expected.get(Columns.GERMPLASM_GID)), sample.get().getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID).getAsString()); + } + + return ret; + } + + private Program createProgram(String name, String abbv, String key, String referenceSource, List germplasm, List traits) throws ApiException, DoesNotExistException, ValidatorException, BadRequestException { + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + ProgramRequest programRequest1 = ProgramRequest.builder() + .name(name) + .abbreviation(abbv) + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key(key) + .build(); + + + TestUtils.insertAndFetchTestProgram(gson, client, programRequest1); + + // Get main program + Program program = programService.getByKey(key).get(); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId().toString()); + + if(germplasm != null && !germplasm.isEmpty()) { + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", referenceSource)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); + } + + if(traits != null && !traits.isEmpty()) { + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + try { + ontologyService.createTraits(program.getId(), traits, user, false); + } catch (ValidatorException e) { + System.err.println(e.getErrors()); + throw e; + } + } + + return program; + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } + + private void createUploadAndVerifyFailure(Program program, Map base, String columnToRemove, boolean commit) throws IOException, InterruptedException { + Map invalidRow = new HashMap<>(base); + invalidRow.remove(columnToRemove); + uploadAndVerifyFailure(program, writeDataToFile(List.of(invalidRow)), columnToRemove, commit); + } + + private JsonObject uploadAndVerifyFailure(Program program, File file, String expectedColumnError, boolean commit) throws InterruptedException, IOException { + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(SUBMISSION_NAME, "test-"+UUID.randomUUID()), true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); + assertEquals(1, rowErrors.size()); + JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); + assertEquals(1, fieldErrors.size()); + JsonObject error = fieldErrors.get(0).getAsJsonObject(); + assertEquals(expectedColumnError, error.get("field").getAsString()); + assertEquals(422, error.get("httpStatusCode").getAsInt()); + + return result; + } + + public File writeDataToFile(List> data) throws IOException { + File file = File.createTempFile("test", ".csv"); + + List columns = new ArrayList<>(); + columns.add(Column.builder().value(Columns.PLATE_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.ROW).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.COLUMN).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.ORGANISM).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.SPECIES).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_NAME).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_GID).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.OBS_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.TISSUE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.COMMENTS).dataType(Column.ColumnDataType.STRING).build()); + + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + + private String createExperiment(Program program) throws IOException, InterruptedException { + Flowable> call = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String expMappingId = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + JsonObject importResult = importTestUtils.uploadAndFetch( + importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null), + null, + true, + client, + program, + expMappingId); + return importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + } + + private Map makeExpImportRow(String environment) { + Map row = new HashMap<>(); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); + row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); + row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); + row.put(ExperimentObservation.Columns.ENV, environment); + row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); + row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, "a-1"); + row.put(ExperimentObservation.Columns.REP_NUM, "1"); + row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); + row.put(ExperimentObservation.Columns.ROW, "1"); + row.put(ExperimentObservation.Columns.COLUMN, "1"); + return row; + } + +}