diff --git a/.env.template b/.env.template index 0e768adcd..73ff2d89e 100644 --- a/.env.template +++ b/.env.template @@ -79,3 +79,5 @@ TRIAL_START_DELAY=15s TRAIT_START_DELAY=20s OBSERVATION_START_DELAY=25s OBSERVATION_UNIT_START_DELAY=30s + +DATA_TABLE_MAX_SIZE=200 \ No newline at end of file diff --git a/pom.xml b/pom.xml index 3afa66cab..1a24e322f 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ 31.0.1-jre 4.9.3 4.3.1 - 2.2-SNAPSHOT + 2.2.2-SNAPSHOT 2.11.0 2.2.1 diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index 988402889..878a9441d 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -46,7 +46,6 @@ @Secured(SecurityRule.IS_AUTHENTICATED) public class ExperimentController { private final BrAPITrialService experimentService; - private final ExperimentQueryMapper experimentQueryMapper; private final ProgramService programService; private final ExperimentalCollaboratorService experimentalCollaboratorService; private final SecurityService securityService; @@ -54,9 +53,8 @@ public class ExperimentController { private final RoleService roleService; @Inject - public ExperimentController(BrAPITrialService experimentService, ExperimentQueryMapper experimentQueryMapper, ProgramService programService, ExperimentalCollaboratorService experimentalCollaboratorService, SecurityService securityService, ProgramUserService programUserService, RoleService roleService) { + public ExperimentController(BrAPITrialService experimentService, ProgramService programService, ExperimentalCollaboratorService experimentalCollaboratorService, SecurityService securityService, ProgramUserService programUserService, RoleService roleService) { this.experimentService = experimentService; - this.experimentQueryMapper = experimentQueryMapper; this.programService = programService; this.experimentalCollaboratorService = experimentalCollaboratorService; this.securityService = securityService; diff --git a/src/main/java/org/breedinginsight/brapi/v1/model/request/query/BrapiQuery.java b/src/main/java/org/breedinginsight/brapi/v1/model/request/query/BrapiQuery.java index f51bcf317..1b4f9234e 100644 --- a/src/main/java/org/breedinginsight/brapi/v1/model/request/query/BrapiQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v1/model/request/query/BrapiQuery.java @@ -26,6 +26,7 @@ import javax.validation.constraints.Positive; import javax.validation.constraints.PositiveOrZero; +import java.util.Map; @Getter @Setter @@ -42,7 +43,7 @@ public class BrapiQuery implements PaginationParams { private Integer page; private String sortField; - private SortOrder sortOrder; + private SortOrder sortOrder = SortOrder.ASC; public Integer getDefaultPage() { return DEFAULT_PAGE; @@ -58,4 +59,26 @@ protected FilterRequest constructFilterRequest(String field, String value) { .value(value) .build(); } + + /** + * For BrapiQuery objects that have implemented this method, this map is used by the BrAPIDAOUtil + * to iterate through all valid filter columns and their associated values that are submitted from the fe. + * + * See @ExperimentQuery for example implementation + */ + public Map getFilterValuesByBrAPIColumnName() { + throw new UnsupportedOperationException( + String.format("Server side BrAPI filtering not implemented for class [%s]", this.getClass().getSimpleName()) + ); + } + + /** + * For BrapiQuery objects that have implemented this method, this map is used by the BrAPIDAOUtil + * to match submitted sortField column names to the associated brapi column name for sorting. + * + * See @ExperimentQuery for example implementation + */ + public Map getBrAPIColumnNamesByBiColumnName() { + throw new UnsupportedOperationException(String.format("Server side BrAPI sorting not implemented for class [%s]", this.getClass().getSimpleName())); + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java index 7bcf71de6..319a1e7de 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java @@ -1,6 +1,5 @@ package org.breedinginsight.brapi.v2; -import com.drew.lang.annotations.Nullable; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpHeaders; import io.micronaut.http.HttpResponse; @@ -27,7 +26,6 @@ import org.brapi.v2.model.germ.response.BrAPIGermplasmProgenyResponse; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; -import org.breedinginsight.api.model.v1.request.query.SearchRequest; import org.breedinginsight.api.model.v1.response.DataResponse; import org.breedinginsight.api.model.v1.response.Response; import org.breedinginsight.api.model.v1.validators.QueryValid; @@ -108,17 +106,8 @@ public HttpResponse searchGermplasm( return HttpResponse.notFound(); } - // Can't use BrAPI server programDbId filtering, think germplasm are linked to program through observation - // units and doesn't work if don't have any loaded, use external refs instead for now - // Just use DeltaBreed program UUID - String extRefId = program.get().getId().toString(); - body.externalReferenceIds(List.of(extRefId)); - - // convert request filter dbIds from DeltaBreed UUID to BrAPI service dbIds - if (body.getGermplasmDbIds() != null && !body.getGermplasmDbIds().isEmpty()) { - List convertedDbIds = germplasmService.getGermplasmDbIdsForUUIDs(program.get().getId(), body.getGermplasmDbIds()); - body.setGermplasmDbIds(convertedDbIds); - } + String brapiProgramDbId = programService.getBrAPIProgramDbId(program.get().getId()); + body.setProgramDbIds(List.of(brapiProgramDbId)); ApiResponse, Optional>> brapiGermplasm; brapiGermplasm = brAPIEndpointProvider @@ -179,10 +168,6 @@ private void batchProcessGermplasm(List germplasmList, String pr // Prepare a regex pattern for program key removal Pattern programKeyPattern = Utilities.getRegexPatternMatchAllProgramKeysAnyAccession(programKey); germplasmList.parallelStream().forEach(germplasm -> { - // Set dbId - germplasm.germplasmDbId(Utilities.getExternalReference(germplasm.getExternalReferences(), "breedinginsight.org") - .orElseThrow(() -> new IllegalStateException("No BI external reference found")) - .getReferenceId()); // Process synonyms if (germplasm.getSynonyms() != null) { germplasm.getSynonyms().forEach(synonym -> { @@ -199,7 +184,7 @@ private void batchProcessGermplasm(List germplasmList, String pr @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/germplasm{?queryParams*}") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) - public HttpResponse>>> getGermplasm( + public HttpResponse>> getGermplasm( @PathVariable("programId") UUID programId, @QueryValue @QueryValid(using = GermplasmQueryMapper.class) @Valid GermplasmQuery queryParams) { try { @@ -212,15 +197,19 @@ public HttpResponse>>> getGermplasm( } // Fetch all germplasm in the program unless a list id is supplied to return only germplasm in that collection - List germplasm = queryParams.getListDbId() == null ? germplasmService.getGermplasm(programId) : germplasmService.getGermplasmByList(programId, queryParams.getListDbId()); - SearchRequest searchRequest = queryParams.constructSearchRequest(); - return ResponseUtils.getBrapiQueryResponse(germplasm, germplasmQueryMapper, queryParams, searchRequest); + BrAPIGermplasmListResponse brapiResponse = queryParams.getListDbId() == null ? germplasmService.searchGermplasm(programId, queryParams) : germplasmService.getGermplasmByList(programId, queryParams); + + List foundGermplasm = brapiResponse.getResult().getData(); + return ResponseUtils.getBrapiQueryResponse(foundGermplasm, brapiResponse); } catch (ApiException e) { log.info(e.getMessage(), e); return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving germplasm"); } catch (IllegalArgumentException e) { log.info(e.getMessage(), e); return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, "Error parsing requested date format"); + } catch (DoesNotExistException e) { + log.info(e.getMessage(), e); + return HttpResponse.status(HttpStatus.NOT_FOUND, "Supplied programId does not exist"); } } @@ -322,8 +311,7 @@ public HttpResponse getGermplasmPedigreeInfo( metadata.setPagination(pagination); response = new BrAPIGermplasmPedigreeResponse(); } else { - BrAPIGermplasm germplasm = germplasmService.getGermplasmByDBID(programId, germplasmId) - .orElseThrow(() -> new DoesNotExistException("DBID for this germplasm does not exist")); + BrAPIGermplasm germplasm = germplasmService.getGermplasmByUUID(programId, germplasmId); //Forward the pedigree call to the backing BrAPI system of the program passing the germplasmDbId that came in the request GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), GermplasmApi.class); diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java index ed2ef1884..65fa2c278 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java @@ -31,7 +31,6 @@ import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; import org.brapi.v2.model.BrAPIIndexPagination; import org.brapi.v2.model.BrAPIMetadata; -import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponse; @@ -39,7 +38,6 @@ import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.brapi.v1.controller.BrapiVersion; -import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.BrAPIConstants; @@ -65,15 +63,13 @@ public class BrAPIObservationLevelsController { private final ProgramDAO programDAO; private final ProgramService programService; private final BrAPITrialDAO trialDAO; - private final BrAPIStudyDAO studyDAO; @Inject - public BrAPIObservationLevelsController(BrAPIEndpointProvider brAPIEndpointProvider, ProgramDAO programDAO, ProgramService programService, BrAPITrialDAO trialDAO, BrAPIStudyDAO studyDAO) { + public BrAPIObservationLevelsController(BrAPIEndpointProvider brAPIEndpointProvider, ProgramDAO programDAO, ProgramService programService, BrAPITrialDAO trialDAO) { this.brAPIEndpointProvider = brAPIEndpointProvider; this.programDAO = programDAO; this.programService = programService; this.trialDAO = trialDAO; - this.studyDAO = studyDAO; } @Get("/observationlevels") @@ -93,25 +89,13 @@ public HttpResponse observationlevelsGet(@Pat } String programDbId = program.get().getBrapiProgram().getProgramDbId(); - String studyDbId = null; + String studyDbId = environmentId; String trialDbId = null; - if(environmentId != null) { - try { - Optional study = studyDAO.getStudyByEnvironmentId(UUID.fromString(environmentId), program.get()); - if(study.isPresent()) { - studyDbId = study.get().getStudyDbId(); - } else { - studyDbId = environmentId; - } - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e), "Error fetching environment"); - return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error finding observation levels"); - } - } else if(experimentId != null) { + if(environmentId == null && experimentId != null) { try { List trial = trialDAO.getTrialsByExperimentIds(List.of(UUID.fromString(experimentId)), program.get()); - if(trial.size() == 1) { + if (trial.size() == 1) { trialDbId = trial.get(0).getTrialDbId(); } else { trialDbId = experimentId; diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java index ca2b470f8..5e652d326 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java @@ -149,7 +149,7 @@ public HttpResponse observationunitsGet(@PathV @Get("/observationunits/{observationUnitDbId}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) public HttpResponse observationunitsObservationUnitDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("observationUnitDbId") String observationUnitDbId) { - log.debug("observationunitsObservationUnitDbIdGet: fetching ou by externalReferenceId: " + observationUnitDbId); + log.debug("observationunitsObservationUnitDbIdGet: fetching ou by dbId: " + observationUnitDbId); Optional program = programService.getById(programId); if(program.isEmpty()) { log.warn("Program id: " + programId + " not found"); @@ -158,7 +158,7 @@ public HttpResponse observationunitsObservat try { List ous = observationUnitDAO.getObservationUnitsById(List.of(observationUnitDbId), program.get()); if(ous.size() != 1) { - log.warn("did not find a single ou with externalReferenceId: " + observationUnitDbId); + log.warn("did not find a single ou with dbId: " + observationUnitDbId); return HttpResponse.notFound(); } setDbIds(ous.get(0)); @@ -220,18 +220,10 @@ public HttpResponse observationunitsTableGet(@PathVariable("programId") UUID } private void setDbIds(BrAPIObservationUnit ou) { - ou.observationUnitDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)) - .orElseThrow(() -> new IllegalStateException("No BI external reference found")) - .getReferenceID()); - ou.studyDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES)) - .orElseThrow(() -> new IllegalStateException("No BI external reference found")) - .getReferenceID()); - ou.trialDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) - .orElseThrow(() -> new IllegalStateException("No BI external reference found")) - .getReferenceID()); ou.programDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) .orElseThrow(() -> new IllegalStateException("No BI external reference found")) .getReferenceID()); + // TODO: Remove this as part of [BI-3006] if (ou.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_UUID)) { ou.setGermplasmDbId(ou.getAdditionalInfo() .get(BrAPIAdditionalInfoFields.GERMPLASM_UUID) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java index 9727ad1be..ad1bed214 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java @@ -150,23 +150,25 @@ public HttpResponse variablesGet(@PathVari public HttpResponse variablesObservationVariableDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("observationVariableDbId") String observationVariableDbId) { log.debug("fetching variable: " + observationVariableDbId); - UUID traitId; try { - traitId = UUID.fromString(observationVariableDbId); + // Check if submitted ID is valid UUID + UUID.fromString(observationVariableDbId); } catch (IllegalArgumentException e) { return HttpResponse.notFound(); } try { - Optional trait = traitService.getById(programId, traitId); + List programTraits = traitService.getByProgramId(programId, true); - if(trait.isEmpty()) { + if(programTraits.isEmpty()) { return HttpResponse.notFound(); } + BrAPIObservationVariable observationVariable = observationVariableService.filterVariableById(programTraits, observationVariableDbId); + BrAPIObservationVariableSingleResponse response = new BrAPIObservationVariableSingleResponse() .metadata(new BrAPIMetadata()) - .result(observationVariableService.convertToBrAPI(trait.get())); + .result(observationVariable); return HttpResponse.ok(response); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java index 27ac4e34c..094880e4d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java @@ -17,7 +17,6 @@ package org.breedinginsight.brapi.v2; - import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import io.micronaut.http.annotation.*; @@ -28,13 +27,12 @@ import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationsApi; -import org.brapi.v2.model.BrAPIIndexPagination; import org.brapi.v2.model.BrAPIMetadata; import org.brapi.v2.model.BrAPIStatus; import org.brapi.v2.model.BrAPIWSMIMEDataTypes; -import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.response.*; +import org.brapi.v2.model.pheno.response.BrAPIObservationListResponse; +import org.brapi.v2.model.pheno.response.BrAPIObservationTableResponse; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.brapi.v1.controller.BrapiVersion; @@ -46,7 +44,7 @@ import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.utilities.Utilities; - +import org.breedinginsight.utilities.response.ResponseUtils; import javax.annotation.Nullable; import javax.inject.Inject; @@ -61,20 +59,17 @@ public class BrAPIObservationsController { private final ProgramService programService; private final ProgramDAO programDAO; - private final BrAPIStudyDAO brAPIStudyDAO; private final BrAPIEndpointProvider brAPIEndpointProvider; private final BrAPIObservationDAO observationDAO; @Inject public BrAPIObservationsController(ProgramService programService, - ProgramDAO programDAO, ProgramDAO programDAO1, BrAPIStudyDAO brAPIStudyDAO, BrAPIEndpointProvider brAPIEndpointProvider, BrAPIObservationDAO brAPIObservationDAO) { this.programService = programService; this.programDAO = programDAO1; - this.brAPIStudyDAO = brAPIStudyDAO; this.brAPIEndpointProvider = brAPIEndpointProvider; this.observationDAO = brAPIObservationDAO; } @@ -147,53 +142,37 @@ public HttpResponse observationsGet(@PathVariable("programId") UUID programId, ); } - // Get a filtered list of observations. - List observations = observationDAO.getObservationsByFilters(program.get(), studyDbId); - - // If page is not provided, set it to the default value 0. - if (page == null) page = 0; - // If pageSize is not provided, set it to the default value 1000. - if (pageSize == null) pageSize = 1000; - - // Total number of records in the unpaged super set. - int totalCount = observations.size(); - // The least of pageSize and totalCount, unless pageSize is zero, in which case use totalCount. - int requestedPageSize = pageSize > 0 ? Math.min(pageSize, totalCount) : totalCount; - // Integer division and round up. - int totalPages = totalCount / requestedPageSize + ((totalCount % requestedPageSize == 0) ? 0 : 1); - log.info("(Pagination) totalCount: " + totalCount + " page (0-indexed): " + page + " requestedPageSize: " + requestedPageSize + " totalPages: " + totalPages); + boolean pageSizeValid = pageSize == null || pageSize > 0; + boolean pageValid = page == null || page >= 0; - // Determine validity of pagination query parameters. - boolean pageSizeValid = pageSize > 0; - boolean pageValid = page >= 0 && page < totalPages; - - // Only paginate if valid pagination values were sent. - if (pageSizeValid && pageValid) { - int start = page * requestedPageSize; - // Account for last page, which may have fewer than requestedPageSize items, or exactly requestedPageSize items. - int end = (page == (totalPages - 1) && totalCount % requestedPageSize != 0) ? (start + (totalCount % requestedPageSize)) : Math.min(((page + 1) * requestedPageSize), totalCount); - log.info("(Pagination) start " + start + " end " + end); - // Sort observations so that paging is consistent and coherent. - observations.sort(Comparator.comparing(BrAPIObservation::getObservationDbId)); - // Paginate response. - observations = observations.subList(start, end); - } else { + if (!pageSizeValid || !pageValid) { String errorMessage = "Invalid query parameters: page, pageSize"; - return HttpResponse.badRequest(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) - .message(errorMessage))))); + return HttpResponse.badRequest(new BrAPIObservationListResponse() + .metadata(new BrAPIMetadata() + .status(List.of(new BrAPIStatus() + .messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message(errorMessage))))); } - return HttpResponse.ok(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().pagination(new BrAPIIndexPagination() - .currentPage(page) - .totalPages(totalPages) - .pageSize(observations.size()) - .totalCount(totalCount))) - .result(new BrAPIObservationListResponseResult().data(observations))); + BrAPIObservationListResponse brAPIResponse = observationDAO.getObservationsByFilters(program.get(), studyDbId, page, pageSize); + + List observations = brAPIResponse.getResult().getData(); + return ResponseUtils.getBrapiQueryResponse(observations, brAPIResponse); } catch (ApiException e) { log.error(Utilities.generateApiExceptionLogMessage(e), e); - return HttpResponse.serverError(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) - .message("Error fetching observations"))))); + + BrAPIObservationListResponse errorResponse = new BrAPIObservationListResponse() + .metadata(new BrAPIMetadata() + .status(List.of(new BrAPIStatus() + .messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message("Error fetching observations")))); + + if (e.getCode() == HttpStatus.BAD_REQUEST.getCode()) { + return HttpResponse.badRequest(errorResponse); + } + + return HttpResponse.serverError(errorResponse); } catch (Exception e) { log.error("Error fetching Observations", e); return HttpResponse.serverError(new BrAPIObservationListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) @@ -253,16 +232,6 @@ public HttpResponse observationsTableGet( } try { - // Translate studyDbId if provided. - if (queryParams.getStudyDbId() != null) { - Optional study = brAPIStudyDAO.getStudyByEnvironmentId(UUID.fromString(queryParams.getStudyDbId()), program.get()); - if (study.isEmpty()) { - return HttpResponse.notFound(); - } - queryParams.setStudyDbId(study.get().getStudyDbId()); - } - // TODO: Translate other DbIds if provided as well (but studyDbId is sufficient for Mr. Bean). - ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); ApiResponse response = api.observationsTableGet(BrAPIWSMIMEDataTypes.APPLICATION_JSON, queryParams.toBrAPIQueryParams()); diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java index daadea133..88f7095c4 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java @@ -17,7 +17,6 @@ package org.breedinginsight.brapi.v2; -import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; @@ -29,22 +28,23 @@ import org.brapi.v2.model.BrAPIMetadata; import org.brapi.v2.model.BrAPIStatus; import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.response.BrAPIStudyListResponse; import org.brapi.v2.model.core.response.BrAPIStudySingleResponse; -import org.breedinginsight.api.auth.*; -import org.breedinginsight.api.model.v1.request.query.SearchRequest; +import org.breedinginsight.api.auth.ProgramSecured; +import org.breedinginsight.api.auth.ProgramSecuredRole; +import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; +import org.breedinginsight.api.auth.SecurityService; import org.breedinginsight.api.model.v1.response.DataResponse; import org.breedinginsight.api.model.v1.response.Response; import org.breedinginsight.api.model.v1.validators.QueryValid; import org.breedinginsight.brapi.v1.controller.BrapiVersion; import org.breedinginsight.brapi.v2.model.request.query.StudyQuery; import org.breedinginsight.brapi.v2.services.BrAPIStudyService; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.Program; import org.breedinginsight.model.ProgramUser; import org.breedinginsight.services.ExperimentalCollaboratorService; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.ProgramUserService; -import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.Utilities; import org.breedinginsight.utilities.response.ResponseUtils; import org.breedinginsight.utilities.response.mappers.StudyQueryMapper; @@ -54,14 +54,12 @@ import java.util.List; import java.util.Optional; import java.util.UUID; -import java.util.stream.Collectors; @Slf4j @Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) @Secured(SecurityRule.IS_AUTHENTICATED) public class BrAPIStudiesController { - private final String referenceSource; private final BrAPIStudyService studyService; private final StudyQueryMapper studyQueryMapper; private final ProgramService programService; @@ -73,14 +71,12 @@ public class BrAPIStudiesController { @Inject public BrAPIStudiesController(BrAPIStudyService studyService, StudyQueryMapper studyQueryMapper, - @Property(name = "brapi.server.reference-source") String referenceSource, ProgramService programService, SecurityService securityService, ProgramUserService programUserService, ExperimentalCollaboratorService experimentalCollaboratorService) { this.studyService = studyService; this.studyQueryMapper = studyQueryMapper; - this.referenceSource = referenceSource; this.programService = programService; this.securityService = securityService; this.programUserService = programUserService; @@ -91,34 +87,40 @@ public BrAPIStudiesController(BrAPIStudyService studyService, @Produces(MediaType.APPLICATION_JSON) @ProgramSecured( roles = {ProgramSecuredRole.SYSTEM_ADMIN, ProgramSecuredRole.READ_ONLY, ProgramSecuredRole.PROGRAM_ADMIN ,ProgramSecuredRole.EXPERIMENTAL_COLLABORATOR} ) - public HttpResponse>>> getStudies( + public HttpResponse>> getStudies( @PathVariable("programId") UUID programId, @QueryValue @QueryValid(using = StudyQueryMapper.class) @Valid StudyQuery queryParams) { try { Optional program = programService.getById(programId); if (program.isEmpty()) { return HttpResponse.notFound(); } - queryParams.setSortField(studyQueryMapper.getDefaultSortField()); - queryParams.setSortOrder(studyQueryMapper.getDefaultSortOrder()); - SearchRequest searchRequest = queryParams.constructSearchRequest(); + if (queryParams.getSortField() == null) { + queryParams.setSortField(studyQueryMapper.getDefaultSortField()); + } + if (queryParams.getSortOrder() == null) { + queryParams.setSortOrder(studyQueryMapper.getDefaultSortOrder()); + } log.debug("fetching studies for program: " + programId); // If the program user is an experimental collaborator, filter results for only authorized studies. Optional experimentalCollaborator = programUserService.getIfExperimentalCollaborator(programId, securityService.getUser().getId()); if (experimentalCollaborator.isPresent()) { List authorizedExperimentIds = experimentalCollaboratorService.getAuthorizedExperimentIds(experimentalCollaborator.get().getId()); - List authorizedStudies = studyService.getStudiesByExperimentIds(program.get(), authorizedExperimentIds) - .stream() - .peek(this::setDbIds) - .collect(Collectors.toList()); - return ResponseUtils.getBrapiQueryResponse(authorizedStudies, studyQueryMapper, queryParams, searchRequest); + if (authorizedExperimentIds.isEmpty()) { + return ResponseUtils.getEmptyBrapiQueryResponse(); + } + + BrAPIStudyListResponse brapiResponse = studyService.searchStudies( + program.get(), + authorizedExperimentIds, + queryParams); + List foundStudies = brapiResponse.getResult().getData(); + return ResponseUtils.getBrapiQueryResponse(foundStudies, brapiResponse); } - List studies = studyService.getStudies(programId) - .stream() - .peek(this::setDbIds) - .collect(Collectors.toList()); - return ResponseUtils.getBrapiQueryResponse(studies, studyQueryMapper, queryParams, searchRequest); + BrAPIStudyListResponse brapiResponse = studyService.searchStudies(program.get(), queryParams); + List foundStudies = brapiResponse.getResult().getData(); + return ResponseUtils.getBrapiQueryResponse(foundStudies, brapiResponse); } catch (ApiException e) { log.info(e.getMessage(), e); return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving study"); @@ -137,25 +139,31 @@ public HttpResponse studiesPost(@PathVariable("programId") UUID programId, @Body @Get("/studies/{studyDbId}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.PROGRAM_SCOPED_ROLES}) - public HttpResponse studiesStudyDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("studyDbId") String environmentId) { + public HttpResponse studiesStudyDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("studyDbId") String studyDbId) { Optional program = programService.getById(programId); - if(program.isEmpty()) { + if (program.isEmpty()) { log.warn("program id: " + programId + " not found"); return HttpResponse.notFound(); } + try { - Optional study = studyService.getStudyByEnvironmentId(program.get(), UUID.fromString(environmentId)); - if(study.isPresent()) { - setDbIds(study.get()); - return HttpResponse.ok(new BrAPIStudySingleResponse().result(study.get())); + Optional study = studyService.getStudyByDbId(program.get(), studyDbId); + + if (study.isPresent()) { + return HttpResponse.ok( + new BrAPIStudySingleResponse() + .result(study.get())); } else { - log.warn("studyDbId: " + environmentId + " not found"); + log.warn("studyDbId: " + studyDbId + " not found"); return HttpResponse.notFound(); } } catch (ApiException e) { log.error(Utilities.generateApiExceptionLogMessage(e), e); - return HttpResponse.serverError(new BrAPIStudySingleResponse().metadata(new BrAPIMetadata().addStatusItem(new BrAPIStatus().message("Error fetching study") - .messageType(BrAPIStatus.MessageTypeEnum.ERROR)))); + return HttpResponse.serverError(new BrAPIStudySingleResponse() + .metadata(new BrAPIMetadata() + .addStatusItem(new BrAPIStatus() + .message("Error fetching study") + .messageType(BrAPIStatus.MessageTypeEnum.ERROR)))); } } @@ -167,15 +175,4 @@ public HttpResponse studiesStudyDbIdPut(@PathVariable("programId") UUID programI return HttpResponse.notFound(); } - private void setDbIds(BrAPIStudy study) { - study.studyDbId(Utilities.getExternalReference(study.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES)) - .orElseThrow(() -> new IllegalStateException("No BI external reference found")) - .getReferenceID()); - - study.trialDbId(Utilities.getExternalReference(study.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) - .orElseThrow(() -> new IllegalStateException("No BI external reference found")) - .getReferenceID()); - - //TODO update locationDbId - } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java index 1ad489d91..4dab93f16 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java @@ -27,6 +27,7 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPITrialListResponse; import org.brapi.v2.model.core.response.BrAPITrialSingleResponse; import org.breedinginsight.api.auth.*; import org.breedinginsight.api.model.v1.request.query.SearchRequest; @@ -42,8 +43,6 @@ import org.breedinginsight.services.ExperimentalCollaboratorService; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.ProgramUserService; -import org.breedinginsight.model.ProgramUser; -import org.breedinginsight.model.Role; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.Utilities; import org.breedinginsight.utilities.response.ResponseUtils; @@ -64,7 +63,6 @@ public class BrAPITrialsController { private final String referenceSource; private final BrAPITrialService experimentService; - private final ExperimentQueryMapper experimentQueryMapper; private final SecurityService securityService; private final ProgramUserService programUserService; private final ExperimentalCollaboratorService experimentalCollaboratorService; @@ -72,14 +70,12 @@ public class BrAPITrialsController { @Inject public BrAPITrialsController(BrAPITrialService experimentService, - ExperimentQueryMapper experimentQueryMapper, @Property(name = "brapi.server.reference-source") String referenceSource, SecurityService securityService, ProgramUserService programUserService, ExperimentalCollaboratorService experimentalCollaboratorService, ProgramService programService) { this.experimentService = experimentService; - this.experimentQueryMapper = experimentQueryMapper; this.referenceSource = referenceSource; this.securityService = securityService; this.programUserService = programUserService; @@ -91,26 +87,39 @@ public BrAPITrialsController(BrAPITrialService experimentService, @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roles = {ProgramSecuredRole.SYSTEM_ADMIN, ProgramSecuredRole.READ_ONLY, ProgramSecuredRole.PROGRAM_ADMIN ,ProgramSecuredRole.EXPERIMENTAL_COLLABORATOR }) - public HttpResponse>>> getExperiments( + public HttpResponse>> getExperiments( @PathVariable("programId") UUID programId, @QueryValue @QueryValid(using = ExperimentQueryMapper.class) @Valid ExperimentQuery queryParams) { try { Optional program = programService.getById(programId); if (program.isEmpty()) { return HttpResponse.notFound(); } - SearchRequest searchRequest = queryParams.constructSearchRequest(); log.debug("fetching trials for program: " + programId); // If the program user is an experimental collaborator, filter results for only authorized experiments. Optional experimentalCollaborator = programUserService.getIfExperimentalCollaborator(programId, securityService.getUser().getId()); if (experimentalCollaborator.isPresent()) { - List experimentIds = experimentalCollaboratorService.getAuthorizedExperimentIds(experimentalCollaborator.get().getId()); - List authorizedExperiments = experimentService.getTrialsByExperimentIds(program.get(), experimentIds).stream().peek(this::setDbIds).collect(Collectors.toList()); - return ResponseUtils.getBrapiQueryResponse(authorizedExperiments, experimentQueryMapper, queryParams, searchRequest); + List authorizedExperimentIds = experimentalCollaboratorService.getAuthorizedExperimentIds(experimentalCollaborator.get().getId()); + + if (authorizedExperimentIds.isEmpty()) { + // User is not authorized for any experiments, return empty response. + return ResponseUtils.getEmptyBrapiQueryResponse(); + } else { + // User is authorized. Grab the experiment data from BrAPI. + BrAPITrialListResponse brapiResponse = experimentService.searchTrials(program.get(), authorizedExperimentIds, queryParams); + + List foundTrials = brapiResponse.getResult().getData(); + foundTrials.forEach(this::setDbIds); + + return ResponseUtils.getBrapiQueryResponse(foundTrials, brapiResponse); + } } - List experiments = experimentService.getExperiments(programId).stream().peek(this::setDbIds).collect(Collectors.toList()); - return ResponseUtils.getBrapiQueryResponse(experiments, experimentQueryMapper, queryParams, searchRequest); + BrAPITrialListResponse brapiResponse = experimentService.searchTrials(program.get(), queryParams); + + List foundTrials = brapiResponse.getResult().getData(); + foundTrials.forEach(this::setDbIds); + return ResponseUtils.getBrapiQueryResponse(foundTrials, brapiResponse); } catch (ApiException e) { log.info(e.getMessage(), e); return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving experiments"); @@ -158,10 +167,8 @@ public HttpResponse trialsTrialDbIdPut(@PathVariable("programId") UUID progra return HttpResponse.notFound(); } + // TODO: Verify that the programDbId does not need to be reset and that the BrAPI programDbId can be used instead [BI-2931] private void setDbIds(BrAPITrial trial) { - trial.trialDbId(Utilities.getExternalReference(trial.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) - .orElseThrow(() -> new IllegalStateException("No BI external reference found")) - .getReferenceID()); trial.programDbId(Utilities.getExternalReference(trial.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) .orElseThrow(() -> new IllegalStateException("No BI external reference found")) .getReferenceID()); diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPICachedDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPICachedDAO.java deleted file mode 100644 index 2d623b02b..000000000 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPICachedDAO.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.breedinginsight.brapi.v2.dao; - -import org.breedinginsight.daos.cache.ProgramCache; - -import java.util.UUID; - -public abstract class BrAPICachedDAO { - - protected ProgramCache programCache; - - public void repopulateCache(UUID programId) { - this.programCache.invalidate(programId); - this.programCache.populate(programId); - } -} diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java index 58910396b..89d57a72d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAO.java @@ -17,27 +17,31 @@ package org.breedinginsight.brapi.v2.dao; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.JsonObject; -import io.micronaut.context.annotation.Context; import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpStatus; import io.micronaut.http.server.exceptions.InternalServerException; -import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.model.queryParams.germplasm.GermplasmQueryParams; import org.brapi.client.v2.modules.germplasm.GermplasmApi; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIListItemWithFullItem; +import org.brapi.v2.model.core.BrAPIProgram; +import org.brapi.v2.model.core.response.BrAPIListItemsListResponse; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; import org.brapi.v2.model.germ.request.BrAPIGermplasmSearchRequest; +import org.brapi.v2.model.germ.response.BrAPIGermplasmListResponse; import org.brapi.v2.model.germ.response.BrAPIGermplasmSingleResponse; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.model.request.query.GermplasmQuery; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; -import org.breedinginsight.daos.cache.ProgramCache; -import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.services.exceptions.DoesNotExistException; @@ -45,53 +49,46 @@ import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; +import javax.inject.Named; import javax.inject.Singleton; import java.util.*; -import java.util.concurrent.Callable; import java.util.stream.Collectors; @Slf4j @Singleton -@Context public class BrAPIGermplasmDAO { private final ProgramDAO programDAO; private final ImportDAO importDAO; private final BrAPIDAOUtil brAPIDAOUtil; + private final ObjectMapper objectMapper; + private final BrAPIListDAO brAPIListDAO; @Property(name = "brapi.server.reference-source") private String referenceSource; - @Property(name = "micronaut.bi.api.run-scheduled-tasks") - private boolean runScheduledTasks; - - @Property(name = "brapi.paginate.germplasm") - private boolean paginateGermplasm; - - private final ProgramCache programGermplasmCache; + @Property(name = "data-table.max-size") + private int dataTableMaxSize; private final BrAPIEndpointProvider brAPIEndpointProvider; + private final int brapiMaxPageSize; + @Inject - public BrAPIGermplasmDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, ProgramCacheProvider programCacheProvider, BrAPIEndpointProvider brAPIEndpointProvider) { + public BrAPIGermplasmDAO(ProgramDAO programDAO, + ImportDAO importDAO, + BrAPIDAOUtil brAPIDAOUtil, + BrAPIEndpointProvider brAPIEndpointProvider, + @Property(name = "brapi.cache.fetch-page-size") int brapiFetchPageSize, + ObjectMapper objectMapper, + BrAPIListDAO brAPIListDAO) { this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; - this.programGermplasmCache = programCacheProvider.getProgramCache(this::fetchProgramGermplasm, BrAPIGermplasm.class); this.brAPIEndpointProvider = brAPIEndpointProvider; - } - - @Scheduled(initialDelay = "${startup.delay.germplasm}") - public void setup() { - if(!runScheduledTasks) { - return; - } - // Populate germplasm cache for all programs on startup - log.debug("populating germplasm cache"); - List programs = programDAO.getActive(); - if(programs != null) { - programGermplasmCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); - } + this.brapiMaxPageSize = brapiFetchPageSize; + this.objectMapper = objectMapper; + this.brAPIListDAO = brAPIListDAO; } /** @@ -101,7 +98,29 @@ public void setup() { * @throws ApiException */ public List getGermplasm(UUID programId) throws ApiException { - return new ArrayList<>(programGermplasmCache.get(programId).values()); + Program program = programDAO.get(programId) + .stream() + .findFirst() + .orElseThrow(); + + if (program.getId() == null) { + throw new InternalServerException("BI-API Program or Program ID is null"); + } + + String brapiProgramDbId = Optional.of(program) + .map(Program::getBrapiProgram) + .map(BrAPIProgram::getProgramDbId) + .orElse(null); + + if (brapiProgramDbId == null) { + brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); + } + + GermplasmQueryParams germplasmQueryParams = GermplasmQueryParams.builder() + .programDbId(brapiProgramDbId) + .build(); + + return getBrAPIGermplasmUsingBrAPIProgramId(germplasmQueryParams, program); } /** @@ -111,9 +130,10 @@ public List getGermplasm(UUID programId) throws ApiException { * @throws ApiException */ public List getRawGermplasm(UUID programId) throws ApiException { + // TODO: This method is used for checking if germplasm already exist in the system. Once a generalized BrAPI exists check is made, we should update this method [BI-2938] Program program = new Program(programDAO.fetchOneById(programId)); - List cacheList = new ArrayList<>(programGermplasmCache.get(programId).values()); - return cacheList.stream().map(germplasm -> { + List programGermplasm = getGermplasm(programId); + return programGermplasm.stream().map(germplasm -> { germplasm.setGermplasmName(Utilities.appendProgramKey(germplasm.getDefaultDisplayName(), program.getKey(), germplasm.getAccessionNumber())); if(germplasm.getAdditionalInfo() != null && germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_RAW_PEDIGREE) && !(germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_RAW_PEDIGREE).isJsonNull())) { @@ -124,59 +144,17 @@ public List getRawGermplasm(UUID programId) throws ApiException }).collect(Collectors.toList()); } - - /** - * Fetch formatted germplasm for this program - * @param programId - * @return Map - * @throws ApiException - */ - private Map fetchProgramGermplasm(UUID programId) throws ApiException { - GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), GermplasmApi.class); - // Get the program key - List programs = programDAO.get(programId); - if (programs.size() != 1) { - throw new InternalServerException("Program was not found for given key"); - } - Program program = programs.get(0); - - // Set query params and make call - BrAPIGermplasmSearchRequest germplasmSearch = new BrAPIGermplasmSearchRequest(); - germplasmSearch.externalReferenceIDs(List.of(programId.toString())); - germplasmSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); - - if (paginateGermplasm) { - log.debug("Fetching germplasm with pagination to BrAPI"); - return processGermplasmForDisplay(brAPIDAOUtil.search( - api::searchGermplasmPost, - api::searchGermplasmSearchResultsDbIdGet, - germplasmSearch), - program.getKey()); - } else { - log.debug("Fetching germplasm without pagination to BrAPI"); - return processGermplasmForDisplay(brAPIDAOUtil.searchNoPaging( - api::searchGermplasmPost, - api::searchGermplasmSearchResultsDbIdGet, - germplasmSearch), - program.getKey()); - } - } - - public void repopulateGermplasmCacheForProgram(UUID programId) { - programGermplasmCache.populate(programId); - } - /** * Process germplasm into a format for display - * @param programGermplasm + * @param germplasmToProcess * @return Map * @throws ApiException */ - private Map processGermplasmForDisplay(List programGermplasm, String programKey) { + private List processGermplasmForDisplay(List germplasmToProcess, + Program program) throws ApiException { // Process the germplasm - Map programGermplasmMap = new HashMap<>(); - log.trace("processing germ for display: " + programGermplasm); - for (BrAPIGermplasm germplasm: programGermplasm) { + log.trace("processing germ for display: " + germplasmToProcess); + for (BrAPIGermplasm germplasm: germplasmToProcess) { JsonObject additionalInfo = germplasm.getAdditionalInfo(); if (additionalInfo != null && additionalInfo.has(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD_ID)) { @@ -190,14 +168,34 @@ private Map processGermplasmForDisplay(Listmale/femaleParentUUID. (See GermplasmProcessor.constructPedigreeString() for assignment of this data) + // To avoid completely re-working the importer to create pedigree nodes and associated germplasm referenced first, this "hack" relates the bi-generated exrefs + // to the brapi-generated germplasmDbId via an extra lookup and processing to the database. + // This is necessary because we need to overwrite bi-generated ids in the pedigree string to use the brapi germplasmDbIds so that front end reference links work properly. + // This should always work because processGermplasmForDisplay is only ever called once Germplasm data has been created in the database, and provided we improve lookups to not fetch + // all program germplasm at once, the performance hit should be negligible. + // TODO: This hack can be removed once/if we implement [BI-2588/BI-2452] + Map pedigreeBrAPIGermplasmDbIdByBICreatedExRef = null; + + // This should ease concerns for extra time pulling this data. The intended usage is for the Germplasm data table, + // but any usage under 200 germs (current data table max size) should not be too large of a hit to the server or database. + boolean pedigreeExRefMutation = germplasmToProcess.size() <= dataTableMaxSize; + + if (pedigreeExRefMutation) { + // Only perform this operation for callers that require it + pedigreeBrAPIGermplasmDbIdByBICreatedExRef = getPedigreeGermplasmDbIdByBICreatedExRef(germplasmToProcess, program); + } + // Update pedigree string - for (BrAPIGermplasm germplasm: programGermplasm) { + for (BrAPIGermplasm germplasm: germplasmToProcess) { JsonObject additionalInfo = germplasm.getAdditionalInfo(); if(additionalInfo == null) { additionalInfo = new JsonObject(); @@ -222,7 +220,7 @@ private Map processGermplasmForDisplay(List processGermplasmForDisplay(List processGermplasmForDisplay(List processGermplasmForDisplay(List getPedigreeGermplasmDbIdByBICreatedExRef(List brAPIGermplasm, Program program) throws ApiException { + Map germplasmDbIdByGeneratedExRef = new HashMap<>(); + + // First, utilize initialized map to store all related parent bi-generated exref IDs. + for (BrAPIGermplasm germplasm : brAPIGermplasm) { + if (germplasm.getAdditionalInfo() == null) { + continue; + } + if (germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_UUID)) { + try { + UUID.fromString(germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_UUID).getAsString()); + } catch (IllegalArgumentException e) { + throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.getCode(), String.format("Germplasm with BrAPI dbId [%s] has an additionalInfo.femaleParentUUID that is not a UUID", germplasm.getGermplasmDbId())); + } + germplasmDbIdByGeneratedExRef.put(germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_FEMALE_PARENT_UUID).getAsString(), null); + } + if (germplasm.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_UUID)) { + try { + UUID.fromString(germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_UUID).getAsString()); + } catch (IllegalArgumentException e) { + throw new ApiException(HttpStatus.INTERNAL_SERVER_ERROR.getCode(), String.format("Germplasm with BrAPI dbId [%s] has an additionalInfo.maleParentUUID that is not a UUID", germplasm.getGermplasmDbId())); + } + germplasmDbIdByGeneratedExRef.put(germplasm.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_MALE_PARENT_UUID).getAsString(), null); + } + } + + if (germplasmDbIdByGeneratedExRef.isEmpty()) { + // If there are none, no reason to proceed with a BrAPI search. + return germplasmDbIdByGeneratedExRef; + } + + // Now grab all male/female exrefs and search for them to get their germplasmDbIds + List exRefIds = new ArrayList<>(germplasmDbIdByGeneratedExRef.keySet()); - BrAPIExternalReference extRef = germplasm.getExternalReferences().stream().filter(reference -> referenceSource.equals(reference.getReferenceSource())).findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); - String germplasmId = extRef.getReferenceID(); - programGermplasmMap.put(germplasmId, germplasm); + BrAPIGermplasmSearchRequest searchRequest = new BrAPIGermplasmSearchRequest(); + searchRequest.setExternalReferenceIds(exRefIds); + searchRequest.setExternalReferenceSources(List.of(referenceSource)); + + // Reuse some code to properly set searchRequest with brapiProgramDbId, and proper paging + searchRequest = buildSearchRequest(program, searchRequest, null); + + GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), GermplasmApi.class); + + + // This will use CACHE_BRAPI_FETCH_PAGE_SIZE env variable to lookup at maximum 65k pedigree germplasm at once. + // This should not be normal though, as we should try not to exceed 1k lookups at once using this code. + // If there is a need to look up a large amount of germplasm records at once, alternative methods should be used. + List germplasmSearchedWithExRef = brAPIDAOUtil.searchNoPaging( + api::searchGermplasmPost, + api::searchGermplasmSearchResultsDbIdGet, + searchRequest); + + for (BrAPIGermplasm germplasm : germplasmSearchedWithExRef) { + Optional exRef = Utilities.getExternalReference(germplasm.getExternalReferences(), referenceSource); + + if (exRef.isPresent()) { + germplasmDbIdByGeneratedExRef.put(exRef.get().getReferenceId(), germplasm.getGermplasmDbId()); + } else { + throw new IllegalStateException("External references wasn't found for germplasm (dbid): " + germplasm.getGermplasmDbId()); + } } - return programGermplasmMap; + return germplasmDbIdByGeneratedExRef; + } + + /** + * This method requires a BI-API program. If the BrAPIProgram inside this data model is not set, + * this method will retrieve it. + */ + private List getBrAPIGermplasmUsingBrAPIProgramId(GermplasmQueryParams germplasmQueryParams, Program program) throws ApiException { + if (germplasmQueryParams.page() == null) { + germplasmQueryParams.setPage(0); + } + + if (germplasmQueryParams.pageSize() == null) { + germplasmQueryParams.setPageSize(brapiMaxPageSize); + } + + GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), GermplasmApi.class); + + List result = brAPIDAOUtil.get(api::germplasmGet, germplasmQueryParams); + + // TODO: Once cache is removed for this class, fix processGermplasmForDisplay to return List [BI-2906] + return processGermplasmForDisplay(result, program); } public List createBrAPIGermplasm(List postBrAPIGermplasmList, UUID programId, ImportUpload upload) { GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), GermplasmApi.class); - var program = programDAO.fetchOneById(programId); + var program = new Program(programDAO.fetchOneById(programId)); try { if (!postBrAPIGermplasmList.isEmpty()) { List postResponse = brAPIDAOUtil.post(postBrAPIGermplasmList, upload, api::germplasmPost, importDAO::update); - return new ArrayList<>(processGermplasmForDisplay(postResponse, program.getKey()).values()); + return processGermplasmForDisplay(postResponse, program); } return new ArrayList<>(); } catch (Exception e) { @@ -296,14 +388,11 @@ public List createBrAPIGermplasm(List postBrAPIG public List updateBrAPIGermplasm(List putBrAPIGermplasmList, UUID programId, ImportUpload upload) { GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), GermplasmApi.class); - var program = programDAO.fetchOneById(programId); + var program = new Program(programDAO.fetchOneById(programId)); try { if (!putBrAPIGermplasmList.isEmpty()) { - Callable> postFunction = () -> { - List putResponse = putGermplasm(putBrAPIGermplasmList, api); - return processGermplasmForDisplay(putResponse, program.getKey()); - }; - return programGermplasmCache.post(programId, postFunction); + List putResponse = putGermplasm(putBrAPIGermplasmList, api); + return processGermplasmForDisplay(putResponse, program); } return new ArrayList<>(); } catch (Exception e) { @@ -313,58 +402,109 @@ public List updateBrAPIGermplasm(List putBrAPIGe public List getGermplasmByRawName(List germplasmNames, UUID programId) throws ApiException { Program program = new Program(programDAO.fetchOneById(programId)); - return getGermplasm(programId) - .stream() - .filter(brAPIGermplasm -> germplasmNames.contains(Utilities.appendProgramKey(brAPIGermplasm.getGermplasmName(),program.getKey(),brAPIGermplasm.getAccessionNumber()))) - .collect(Collectors.toList()); - } - public BrAPIGermplasm getGermplasmByUUID(String germplasmId, UUID programId) throws ApiException, DoesNotExistException { - Map cache = programGermplasmCache.get(programId); - BrAPIGermplasm germplasm = null; - if (cache != null) { - germplasm = cache.get(germplasmId); - } - if (germplasm == null) { - throw new DoesNotExistException("UUID for this germplasm does not exist"); + if (germplasmNames.isEmpty()) { + return new ArrayList<>(); } - return germplasm; + + BrAPIGermplasmSearchRequest searchRequest = new BrAPIGermplasmSearchRequest(); + searchRequest.setGermplasmNames(germplasmNames); + + searchRequest = buildSearchRequest(program, searchRequest, null); + + GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), GermplasmApi.class); + + List germplasm = brAPIDAOUtil.search( + api::searchGermplasmPost, + api::searchGermplasmSearchResultsDbIdGet, + searchRequest); + + return processGermplasmForDisplay(germplasm, program); } - public List getGermplasmDbIdsForUUIDs(List germplasmUUIDs, UUID programId) throws ApiException, DoesNotExistException { - Map cache = programGermplasmCache.get(programId); - List germplasmList = new ArrayList<>(); - if (cache != null) { - // not using streams because want to throw checked exception - for (String germplasmUUID : germplasmUUIDs) { - BrAPIGermplasm germplasm = cache.get(germplasmUUID); - if (germplasm == null) { - throw new DoesNotExistException("UUID for this germplasm does not exist: " + germplasmUUID); - } - germplasmList.add(germplasm.getGermplasmDbId()); - } + public BrAPIListItemsListResponse searchGermplasmByList(String listDbId, UUID programId, GermplasmQuery germplasmQuery) throws ApiException { + Program program = new Program(programDAO.fetchOneById(programId)); + + BrAPIGermplasmSearchRequest searchRequest = buildSearchRequest(program, null, germplasmQuery); + + // Because the list item search endpoint is generalized, a typed BrAPISearchRequest won't work, so convert it to JsonNode. + // The search request type is derived by the client/prod server based off of the type of the submitted list dbId. + JsonNode listRequestBody = objectMapper.valueToTree(searchRequest); + + BrAPIListItemsListResponse response = brAPIListDAO.searchBrAPIListItemsByList(listDbId, programId, listRequestBody); + + processGermplasmForDisplay(response.getResult().getData().stream().map(BrAPIListItemWithFullItem::getItem).collect(Collectors.toList()), program); + + return response; + } + + public List brapiGermplasmSearchReturnList(Program program, + BrAPIGermplasmSearchRequest searchRequest) throws ApiException { + return brapiGermplasmSearchReturnResponse(program, searchRequest, null).getResult().getData(); + } + + public BrAPIGermplasmListResponse brapiGermplasmSearchReturnResponse(Program program, + BrAPIGermplasmSearchRequest searchRequest, + GermplasmQuery germplasmQuery) throws ApiException { + + GermplasmApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), GermplasmApi.class); + + BrAPIGermplasmSearchRequest brAPIGermplasmSearchRequest = buildSearchRequest(program, searchRequest, germplasmQuery); + + BrAPIGermplasmListResponse brAPIResponse = + brAPIDAOUtil.simpleSearch( + api::searchGermplasmPost, + brAPIGermplasmSearchRequest + ); + + List processedGermplasm = processGermplasmForDisplay(brAPIDAOUtil.getListResult(brAPIResponse), program); + + brAPIResponse.getResult().setData(processedGermplasm); + + return brAPIResponse; + } + + private BrAPIGermplasmSearchRequest buildSearchRequest(Program program, BrAPIGermplasmSearchRequest searchRequestPassThru, GermplasmQuery germplasmQuery) throws ApiException { + BrAPIGermplasmSearchRequest searchRequest; + + if (searchRequestPassThru == null) { + searchRequest = new BrAPIGermplasmSearchRequest(); + } else { + searchRequest = searchRequestPassThru; } - return germplasmList; + + searchRequest.programDbIds(List.of(brAPIDAOUtil.getBrAPIProgramDbId(program.getId()))); + + brAPIDAOUtil.setGenericSearchParameters(searchRequest, germplasmQuery); + + return searchRequest; } - public Optional getGermplasmByDBID(String germplasmDbId, UUID programId) throws ApiException { - Map cache = programGermplasmCache.get(programId); - //key is UUID, want to filter by DBID - BrAPIGermplasm germplasm = null; - if (cache != null) { - germplasm = cache.values().stream().filter(x -> x.getGermplasmDbId().equals(germplasmDbId)).collect(Collectors.toList()).get(0); + public BrAPIGermplasm getGermplasmByUUID(String germplasmId, UUID programId) throws ApiException, DoesNotExistException { + Program program = new Program(programDAO.fetchOneById(programId)); + + BrAPIGermplasmSearchRequest searchRequest = new BrAPIGermplasmSearchRequest(); + searchRequest.setGermplasmDbIds(List.of(germplasmId)); + + List result = brapiGermplasmSearchReturnList(program, searchRequest); + + if (result.size() > 1) { + throw new ApiException(String.format("Multiple germplasms found for germplasm with ID: [%s]", germplasmId)); + } else if (result.isEmpty()) { + throw new DoesNotExistException(String.format("Germplasm with ID: [%s] does not exist", germplasmId)); } - return Optional.ofNullable(germplasm); + + return result.get(0); } public List getGermplasmsByDBID(Collection germplasmDbIds, UUID programId) throws ApiException { - Map cache = programGermplasmCache.get(programId); - //key is UUID, want to filter by DBID - List germplasm = new ArrayList<>(); - if (cache != null) { - germplasm = cache.values().stream().filter(x -> germplasmDbIds.contains(x.getGermplasmDbId())).collect(Collectors.toList()); - } - return germplasm; + // TODO: This method is mainly used by the download experiment export tool. This method will fail until we address the parameter limit for async Germplasm requests for germplasmDbIds > 4. Return to this use case during [BI-3021] + Program program = new Program(programDAO.fetchOneById(programId)); + + BrAPIGermplasmSearchRequest searchRequest = new BrAPIGermplasmSearchRequest(); + searchRequest.setGermplasmDbIds(new ArrayList<>(germplasmDbIds)); + + return brapiGermplasmSearchReturnList(program, searchRequest); } public List putGermplasm(List germplasmList, GermplasmApi api) throws ApiException { diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java index 4f6cfdec7..be9b3fcd3 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapi.v2.dao; +import com.fasterxml.jackson.databind.JsonNode; import io.micronaut.http.HttpResponse; import lombok.extern.slf4j.Slf4j; import okhttp3.HttpUrl; @@ -27,14 +28,13 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.BrAPIResponse; import org.brapi.v2.model.BrAPIResponseResult; +import org.brapi.v2.model.core.BrAPIListItemWithFullItem; import org.brapi.v2.model.core.BrAPIListSummary; import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.request.BrAPIListSearchRequest; -import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.brapi.v2.model.core.response.BrAPIListsListResponse; -import org.brapi.v2.model.core.response.BrAPIListsListResponseResult; -import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; +import org.brapi.v2.model.core.response.*; +import org.brapi.v2.model.germ.BrAPIGermplasm; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.daos.ProgramDAO; @@ -71,7 +71,7 @@ public BrAPIListDAO(ProgramDAO programDAO, } public List getListsByName(List listNames, UUID programId) throws ApiException { - if(listNames.isEmpty()) { + if (listNames.isEmpty()) { return Collections.emptyList(); } @@ -122,11 +122,11 @@ private List processListsForProgram(List pro // check that all lists were created via the BI UI in case the BrAPI service silently ignores the search by // externalReference source and ID List filteredLists = new ArrayList<>(); - for (BrAPIListSummary list: programLists) { + for (BrAPIListSummary list : programLists) { if (list.getExternalReferences() == null || list.getExternalReferences().size() == 0) { continue; } else { - for (BrAPIExternalReference ref: list.getExternalReferences()) { + for (BrAPIExternalReference ref : list.getExternalReferences()) { if (ref.getReferenceID().equals(externalReferenceId.toString()) && ref.getReferenceSource().equals(externalReferenceSource)) { filteredLists.add(list); @@ -136,6 +136,7 @@ private List processListsForProgram(List pro } return filteredLists; } + public List updateBrAPIList(String brAPIListDbId, BrAPIListDetails mutatedList, UUID programId) throws ApiException { ListsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ListsApi.class); BrAPIListNewRequest request = new BrAPIListNewRequest(); @@ -160,7 +161,7 @@ public List updateBrAPIList(String brAPIListDbId, BrAPIListDetails mutat log.error(Utilities.generateApiExceptionLogMessage(e)); throw e; } - if(response != null) { + if (response != null) { BrAPIListsSingleResponse body = response.getBody(); if (body == null) { throw new ApiException("Response is missing body", 0, response.getHeaders(), null); @@ -188,7 +189,7 @@ public List createBrAPILists(List brapiLi log.warn(Utilities.generateApiExceptionLogMessage(e)); throw e; } - if(response != null) { + if (response != null) { BrAPIResponse body = response.getBody(); if (body == null) { throw new ApiException("Response is missing body", 0, response.getHeaders(), null); @@ -220,4 +221,32 @@ public HttpResponse deleteBrAPIList(String listDbId, UUID programId, boo return brAPIDAOUtil.makeCall(brapiRequest); } + public BrAPIListItemsListResponse searchBrAPIListItemsByList(String listDbId, UUID programId, JsonNode requestBody) throws ApiException { + ListsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ListsApi.class); + // Do manually, it doesn't like List to List for some reason + ApiResponse> response; + try { + response = api.searchListItemsByList(listDbId, requestBody); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw e; + } + + if (response != null) { + BrAPIResponse> body = response.getBody(); + if (body == null) { + throw new ApiException("Response is missing body", 0, response.getHeaders(), null); + } + BrAPIResponseResult> result = body.getResult(); + if (result == null) { + throw new ApiException("Response body is missing result", 0, response.getHeaders(), response.getBody().toString()); + } + if (result.getData() == null) { + throw new ApiException("Response result is missing data", 0, response.getHeaders(), response.getBody().toString()); + } + return response.getBody(); + } + + throw new ApiException("No response for BrAPI list item search"); + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index 10048564c..2026f2ac5 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -18,129 +18,59 @@ import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; -import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.model.queryParams.phenotype.ObservationQueryParams; import org.brapi.client.v2.modules.phenotype.ObservationsApi; -import org.brapi.v2.model.BrAPIAcceptedSearchResponse; -import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIProgram; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.request.BrAPIObservationSearchRequest; import org.brapi.v2.model.pheno.response.BrAPIObservationListResponse; -import org.brapi.v2.model.pheno.response.BrAPIObservationSingleResponse; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; -import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; -import org.breedinginsight.model.Trait; import org.breedinginsight.services.TraitService; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.BrAPIDAOUtil; import org.breedinginsight.utilities.Utilities; -import org.jetbrains.annotations.NotNull; import javax.inject.Inject; import javax.inject.Singleton; import java.util.*; -import java.util.concurrent.Callable; import java.util.stream.Collectors; -import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; - @Singleton @Slf4j -public class BrAPIObservationDAO extends BrAPICachedDAO { +public class BrAPIObservationDAO { private ProgramDAO programDAO; private ImportDAO importDAO; private BrAPIObservationUnitDAO observationUnitDAO; private final BrAPIDAOUtil brAPIDAOUtil; private final BrAPIEndpointProvider brAPIEndpointProvider; - private final String referenceSource; - private boolean runScheduledTasks; private final TraitService traitService; + private final int brapiMaxPageSize; + @Inject public BrAPIObservationDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIObservationUnitDAO observationUnitDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider, - @Property(name = "brapi.server.reference-source") String referenceSource, - @Property(name = "micronaut.bi.api.run-scheduled-tasks") boolean runScheduledTasks, - ProgramCacheProvider programCacheProvider, TraitService traitService) { + @Property(name = "brapi.cache.fetch-page-size") int brapiFetchPageSize, + TraitService traitService) { this.programDAO = programDAO; this.importDAO = importDAO; this.observationUnitDAO = observationUnitDAO; this.brAPIDAOUtil = brAPIDAOUtil; this.brAPIEndpointProvider = brAPIEndpointProvider; - this.referenceSource = referenceSource; - this.runScheduledTasks = runScheduledTasks; this.traitService = traitService; - this.programCache = programCacheProvider.getProgramCache(this::fetchProgramObservations, BrAPIObservation.class); - } - - @Scheduled(initialDelay = "${startup.delay.observation}") - public void setup() { - if(!runScheduledTasks) { - return; - } - // Populate the observation cache for all programs on startup. - log.debug("populating observation cache"); - List programs = programDAO.getActive(); - if (programs != null) { - programCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); - } - } - - /** - * Fetch formatted observations for this program. - */ - private Map fetchProgramObservations(UUID programId) throws ApiException { - ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); - // Get the program. - List programs = programDAO.get(programId); - if (programs.size() != 1) { - throw new InternalServerException("Program was not found for given id"); - } - Program program = programs.get(0); - - // Set query params and make call. - BrAPIObservationSearchRequest observationSearch = new BrAPIObservationSearchRequest(); - observationSearch.externalReferenceIds(List.of(programId.toString())); - observationSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); - return processObservationsForCache(brAPIDAOUtil.search( - api::searchObservationsPost, - api::searchObservationsSearchResultsDbIdGet, - observationSearch - ), program.getKey()); - } - - /** - * Process a list of observations for insertion into the cache. - */ - private Map processObservationsForCache(List programObservations, String programKey) { - // Process programObservations in place (strip program key, etc.). - processObservations(programKey, programObservations); - // Build map. - Map programObservationsMap = new HashMap<>(); - log.trace("processing observationUnits for cache: " + programObservations); - for (BrAPIObservation observation: programObservations) { - BrAPIExternalReference xref = observation - .getExternalReferences() - .stream() - .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATIONS.getName()).equals(reference.getReferenceSource())) - .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); - programObservationsMap.put(xref.getReferenceId(), observation); - } - return programObservationsMap; + this.brapiMaxPageSize = brapiFetchPageSize; } /** @@ -162,14 +92,6 @@ private void processObservations(String programKey, List obser } } - /** - * Get all observations for a program from the cache. - */ - private Map getProgramObservations(UUID programId) throws ApiException { - return programCache.get(programId); - } - - // Note: not using cache, because unique studyName (with "[ProgramKey-ExtraInfo]") is not stored directly on Observation. public List getObservationsByStudyName(List studyNames, Program program) throws ApiException { if(studyNames.isEmpty()) { return Collections.emptyList(); @@ -181,11 +103,25 @@ public List getObservationsByStudyName(List studyNames ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationsApi.class); return brAPIDAOUtil.search( api::searchObservationsPost, - (brAPIWSMIMEDataTypes, searchResultsDbId, page, pageSize) -> searchObservationsSearchResultsDbIdGet(program.getId(), searchResultsDbId, page, pageSize), + api::searchObservationsSearchResultsDbIdGet, observationSearchRequest ); } + private List searchBrapiObservations(BrAPIObservationSearchRequest observationSearchRequest, Program program) throws ApiException { + ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationsApi.class); + + List observations = brAPIDAOUtil.search( + api::searchObservationsPost, + api::searchObservationsSearchResultsDbIdGet, + observationSearchRequest + ); + + processObservations(program.getKey(), observations); + + return observations; + } + /** * Retrieves a list of observations based on their database IDs and a specific program. * @@ -196,97 +132,80 @@ public List getObservationsByStudyName(List studyNames */ public List getObservationsByDbIds(List dbIds, Program program) throws ApiException { // Check if the dbIds list is empty and return an empty list if so - if(dbIds.isEmpty()) { + if (dbIds.isEmpty()) { return Collections.emptyList(); } - // Filter the observations based on the provided program ID and the provided list of dbIds - // Collect the filtered observations into a List and return the result - return getProgramObservations(program.getId()).values().stream() - .filter(o -> dbIds.contains(o.getObservationDbId())) - .collect(Collectors.toList()); + BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest() + .programDbIds(List.of(program.getBrapiProgram().getProgramDbId())) + .observationDbIds(new ArrayList<>(dbIds)); + + return searchBrapiObservations(observationSearchRequest, program); } public List getObservationsByTrialDbId(List trialDbIds, Program program) throws ApiException { - if(trialDbIds.isEmpty()) { + if (trialDbIds.isEmpty()) { return Collections.emptyList(); } // First, get all ObservationUnits for the given trialDbIds. List observationUnitDbIds = observationUnitDAO.getObservationUnitsForTrialDbIds(program.getId(), trialDbIds) .stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()); - // Finally, return all Observations for those ObservationUnits (Observations are linked to Trial through ObservationUnits). - return getProgramObservations(program.getId()).values().stream() - .filter(o -> observationUnitDbIds.contains(o.getObservationUnitDbId())) - .collect(Collectors.toList()); + + if (observationUnitDbIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest() + .programDbIds(List.of(program.getBrapiProgram().getProgramDbId())) + .observationUnitDbIds(observationUnitDbIds); + + return searchBrapiObservations(observationSearchRequest, program); } public List getObservationsByObservationUnitsAndVariables(Collection ouDbIds, Collection variableDbIds, Program program) throws ApiException { - if(ouDbIds.isEmpty() || variableDbIds.isEmpty()) { + if (ouDbIds.isEmpty() || variableDbIds.isEmpty()) { return Collections.emptyList(); } - return getProgramObservations(program.getId()).values().stream() - .filter(o -> ouDbIds.contains(o.getObservationUnitDbId()) && variableDbIds.contains(o.getObservationVariableDbId())) - .collect(Collectors.toList()); + + BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest() + .programDbIds(List.of(program.getBrapiProgram().getProgramDbId())) + .observationUnitDbIds(new ArrayList<>(ouDbIds)) + .observationVariableDbIds(new ArrayList<>(variableDbIds)); + + return searchBrapiObservations(observationSearchRequest, program); } public List getObservationsByObservationUnits(Collection ouDbIds, Program program) throws ApiException { - if(ouDbIds.isEmpty()) { + + if (ouDbIds.isEmpty()) { return Collections.emptyList(); } - return getProgramObservations(program.getId()).values().stream() - .filter(o -> ouDbIds.contains(o.getObservationUnitDbId())) - .collect(Collectors.toList()); + + BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest() + .programDbIds(List.of(program.getBrapiProgram().getProgramDbId())) + .observationUnitDbIds(new ArrayList<>(ouDbIds)); + + return searchBrapiObservations(observationSearchRequest, program); } - public List getObservationsByObservationUnitsAndStudies(Collection ouDbIds, Collection studyDbIds, Program program) throws ApiException { - if(ouDbIds.isEmpty()) { - return Collections.emptyList(); + public BrAPIObservationListResponse getObservationsByFilters(Program program, String studyDbId, Integer page, Integer pageSize) throws ApiException, DoesNotExistException { + BrAPIObservationSearchRequest observationSearchRequest = new BrAPIObservationSearchRequest() + .programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); + + if (studyDbId != null) { + observationSearchRequest.setStudyDbIds(List.of(studyDbId)); } - return getProgramObservations(program.getId()).values().stream() - .filter(o -> ouDbIds.contains(o.getObservationUnitDbId()) && studyDbIds.contains(o.getStudyDbId())) - .collect(Collectors.toList()); - } + observationSearchRequest.setPage(page == null ? 0 : page); + observationSearchRequest.setPageSize(pageSize == null ? 1000 : pageSize); - // TODO: implement other filters in BI-2506. - public List getObservationsByFilters(Program program, String studyDbId) throws ApiException, DoesNotExistException { - - String studySource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES); - String observationUnitSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS); - String observationSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATIONS); - - // Get all observations for the program. - Collection observations = getProgramObservations(program.getId()).values(); - // Build a hashmap of traits for fast lookup. The key is ObservationVariableDbId, the value is the Trait Id. - HashMap traitIdsByObservationVariableDbId = traitService.getIdsByObservationVariableDbIds(program.getId(), observations.stream().map(BrAPIObservation::getObservationVariableDbId).collect(Collectors.toList())); - - // Lookup studyDbId. - return observations.stream() - .filter(o -> { - // Short circuit if filter is null. - if (studyDbId == null) return true; - Optional xref = Utilities.getExternalReference(o.getExternalReferences(), studySource); - return xref.filter(brAPIExternalReference -> studyDbId.equals(brAPIExternalReference.getReferenceId())).isPresent(); - }) - .peek(o -> { - // Translate ObservationVariableDbId. - o.setObservationVariableDbId(traitIdsByObservationVariableDbId.get(o.getObservationVariableDbId())); - // Translate ObservationUnitDbId. - o.setObservationUnitDbId(Utilities.getExternalReference(o.getExternalReferences(), observationUnitSource) - .orElseThrow(() -> new RuntimeException("observationUnit xref not found on observation")).getReferenceId()); - // Translate ObservationId. - o.setObservationDbId(Utilities.getExternalReference(o.getExternalReferences(), observationSource) - .orElseThrow(() -> new RuntimeException("observation xref not found on observation")).getReferenceId()); - // Translate StudyDbId. - o.setStudyDbId(Utilities.getExternalReference(o.getExternalReferences(), studySource) - .orElseThrow(() -> new RuntimeException("study xref not found on observation")).getReferenceId()); - // TODO: consider translating germplasmDbId in BI-2506. - }).collect(Collectors.toList()); - } + ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationsApi.class); - @NotNull - private ApiResponse, Optional>> searchObservationsSearchResultsDbIdGet(UUID programId, String searchResultsDbId, Integer page, Integer pageSize) throws ApiException { - ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); - return api.searchObservationsSearchResultsDbIdGet(APPLICATION_JSON, searchResultsDbId, page, pageSize); + BrAPIObservationListResponse brAPIResponse = brAPIDAOUtil.simpleSearch(api::searchObservationsPost, observationSearchRequest); + + List observations = brAPIDAOUtil.getListResult(brAPIResponse); + processObservations(program.getKey(), observations); + + return brAPIResponse; } public List createBrAPIObservations(List brAPIObservationList, UUID programId, ImportUpload upload) throws ApiException { @@ -294,11 +213,9 @@ public List createBrAPIObservations(List brA var program = programDAO.fetchOneById(programId); try { if (!brAPIObservationList.isEmpty()) { - Callable> postFunction = () -> { List postResponse = brAPIDAOUtil.post(brAPIObservationList, upload, api::observationsPost, importDAO::update); - return processObservationsForCache(postResponse, program.getKey()); - }; - return programCache.post(programId, postFunction); + processObservations(program.getKey(), postResponse); + return postResponse; } return new ArrayList<>(); } catch (Exception e) { @@ -306,44 +223,44 @@ public List createBrAPIObservations(List brA } } - public BrAPIObservation updateBrAPIObservation(String dbId, BrAPIObservation observation, UUID programId) throws ApiException { - ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); - var program = programDAO.fetchOneById(programId); - try { - Callable> postFunction = () -> { - ApiResponse response = api.observationsObservationDbIdPut(dbId, observation); - if (response == null) - { - throw new ApiException("Response is null", 0, null, null); - } - BrAPIObservationSingleResponse body = response.getBody(); - if (body == null) { - throw new ApiException("Response is missing body", 0, response.getHeaders(), null); - } - BrAPIObservation updatedObservation = body.getResult(); - if (updatedObservation == null) { - throw new ApiException("Response body is missing result", 0, response.getHeaders(), response.getBody().toString()); - } - return processObservationsForCache(List.of(updatedObservation), program.getKey()); - }; - return programCache.post(programId, postFunction).get(0); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e)); - throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); - } catch (Exception e) { - throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); + private List getBrAPIObservationsUsingBrAPIProgramId(Program program) throws ApiException { + + if (program == null || program.getId() == null) { + throw new InternalServerException("BI-API Program or Program ID is null"); } + + String brapiProgramDbId = Optional.of(program) + .map(Program::getBrapiProgram) + .map(BrAPIProgram::getProgramDbId) + .orElse(null); + + if (brapiProgramDbId == null) { + brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); + } + + ObservationQueryParams observationQueryParams = + ObservationQueryParams.builder() + .programDbId(brapiProgramDbId) + .pageSize(brapiMaxPageSize) + .page(0) + .build(); + + ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationsApi.class); + + List result = brAPIDAOUtil.get(api::observationsGet, observationQueryParams); + + processObservations(program.getKey(), result); + + return result; } - // This method overloads updateBrAPIObservation(String dbId, BrAPIObservation observation, UUID programId) - // It was added to increase efficiency. It insures that ProgramCache.populate() is called only once - // not once per observation. public List updateBrAPIObservation(Map mutatedObservationByDbId, UUID programId) throws ApiException { ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); var program = programDAO.fetchOneById(programId); List updatedObservations = new ArrayList<>(); try { + // TODO: Instead of a for loop, utilize BrAPI Observations put to do all updates in one request. [BI-2969] for (Map.Entry entry : mutatedObservationByDbId.entrySet()) { String dbId = entry.getKey(); BrAPIObservation observation = entry.getValue(); @@ -351,23 +268,13 @@ public List updateBrAPIObservation(Map response = null; + BrAPIObservation updatedObservation; + try { - response = api.observationsObservationDbIdPut(dbId, observation); + updatedObservation = brAPIDAOUtil.put(dbId, observation, api::observationsObservationDbIdPut); } catch (ApiException e) { throw new RuntimeException(e); } - if (response == null) { - throw new ApiException("Response is null", 0, null, null); - } - BrAPIObservationSingleResponse body = response.getBody(); - if (body == null) { - throw new ApiException("Response is missing body", 0, response.getHeaders(), null); - } - BrAPIObservation updatedObservation = body.getResult(); - if (updatedObservation == null) { - throw new ApiException("Response body is missing result", 0, response.getHeaders(), response.getBody().toString()); - } updatedObservations.add(updatedObservation); if (!Objects.equals(observation.getValue(), updatedObservation.getValue()) @@ -381,8 +288,8 @@ public List updateBrAPIObservation(Map processedObservations = processObservationsForCache(updatedObservations, program.getKey()); - return programCache.postThese(programId,processedObservations); + processObservations(program.getKey(), updatedObservations); + return updatedObservations; } catch (ApiException e) { log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); throw e; diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index 76acb8076..1d1bab246 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -17,31 +17,27 @@ package org.breedinginsight.brapi.v2.dao; -import com.google.gson.Gson; -import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import com.google.gson.reflect.TypeToken; import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; -import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.model.queryParams.phenotype.ObservationUnitQueryParams; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIProgram; import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.BrAPIObservationTreatment; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; +import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; -import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; @@ -52,15 +48,13 @@ import javax.inject.Inject; import javax.inject.Singleton; import javax.validation.constraints.NotNull; -import java.lang.reflect.Type; import java.util.*; -import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @Slf4j @Singleton -public class BrAPIObservationUnitDAO extends BrAPICachedDAO { +public class BrAPIObservationUnitDAO { private final ProgramDAO programDAO; private final ImportDAO importDAO; private final BrAPIDAOUtil brAPIDAOUtil; @@ -69,10 +63,8 @@ public class BrAPIObservationUnitDAO extends BrAPICachedDAO>(){}.getType(); + private final int brapiMaxPageSize; @Inject public BrAPIObservationUnitDAO(ProgramDAO programDAO, @@ -82,88 +74,80 @@ public BrAPIObservationUnitDAO(ProgramDAO programDAO, BrAPIGermplasmService germplasmService, ProgramService programService, @Property(name = "brapi.server.reference-source") String referenceSource, - @Property(name = "micronaut.bi.api.run-scheduled-tasks") boolean runScheduledTasks, - ProgramCacheProvider programCacheProvider) { + @Property(name = "brapi.cache.fetch-page-size") int brapiFetchPageSize) { this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; this.brAPIEndpointProvider = brAPIEndpointProvider; this.referenceSource = referenceSource; - this.runScheduledTasks = runScheduledTasks; this.programService = programService; this.germplasmService = germplasmService; - this.programCache = programCacheProvider.getProgramCache(this::fetchProgramObservationUnits, BrAPIObservationUnit.class); + this.brapiMaxPageSize = brapiFetchPageSize; } - @Scheduled(initialDelay = "${startup.delay.observation_unit}") - public void setup() { - if(!runScheduledTasks) { - return; + /** + * Get all observation units for a program from the cache. + */ + public List getProgramObservationUnits(UUID programId) throws ApiException { + Program program = programDAO.get(programId) + .stream() + .findFirst() + .orElseThrow(); + + if (program.getId() == null) { + throw new InternalServerException("BI-API Program or Program ID is null"); } - // Populate observation unit cache for all programs on startup. - log.debug("populating observation unit cache"); - List programs = programDAO.getActive(); - if(programs != null) { - programCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); + + String brapiProgramDbId = Optional.of(program) + .map(Program::getBrapiProgram) + .map(BrAPIProgram::getProgramDbId) + .orElse(null); + + if (brapiProgramDbId == null) { + brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); } + + ObservationUnitQueryParams observationUnitQueryParams = ObservationUnitQueryParams.builder() + .programDbId(brapiProgramDbId) + .build(); + + return getBrAPIObservationUnitsUsingQueryParams(observationUnitQueryParams, program); } - /** - * Fetch formatted observation units for this program. - */ - private Map fetchProgramObservationUnits(UUID programId) throws ApiException { - ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationUnitsApi.class); - // Get the program. - List programs = programDAO.get(programId); - if (programs.size() != 1) { - throw new InternalServerException("Program was not found for given id"); + private List getBrAPIObservationUnitsUsingQueryParams(ObservationUnitQueryParams observationUnitQueryParams, + Program program) throws ApiException { + + if (observationUnitQueryParams.page() == null) { + observationUnitQueryParams.setPage(0); } - Program program = programs.get(0); - - // Set query params and make call. - BrAPIObservationUnitSearchRequest observationUnitSearch = new BrAPIObservationUnitSearchRequest(); - observationUnitSearch.externalReferenceIds(List.of(programId.toString())); - observationUnitSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); - return processObservationUnitsForCache(brAPIDAOUtil.search( - api::searchObservationunitsPost, - api::searchObservationunitsSearchResultsDbIdGet, - observationUnitSearch - ), program, true); - } - /** - * Process a list of observation units for insertion into the cache. - */ - private Map processObservationUnitsForCache(List programObservationUnits, Program program, boolean withGID) throws ApiException { - // Process programObservationUnits in place (strip program key, etc.). - processObservationUnits(program, programObservationUnits, withGID); - // Build map. - Map programObservationUnitsMap = new HashMap<>(); - for (BrAPIObservationUnit observationUnit: programObservationUnits) { - BrAPIExternalReference xref = observationUnit - .getExternalReferences() - .stream() - .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName()).equals(reference.getReferenceSource())) - .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); - programObservationUnitsMap.put(xref.getReferenceId(), observationUnit); + if (observationUnitQueryParams.pageSize() == null) { + observationUnitQueryParams.setPageSize(brapiMaxPageSize); } - return programObservationUnitsMap; - } - /** - * Get all observation units for a program from the cache. - */ - private Map getProgramObservationUnits(UUID programId) throws ApiException { - return programCache.get(programId); + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationUnitsApi.class); + + List result = brAPIDAOUtil.get(api::observationunitsGet, observationUnitQueryParams); + + processObservationUnits(program, result, true); + + return result; } - public List getObservationUnitByName(List observationUnitNames, Program program) throws ApiException { - if(observationUnitNames.isEmpty()) { - return Collections.emptyList(); - } - return getProgramObservationUnits(program.getId()).values().stream() - .filter(ou -> observationUnitNames.contains(ou.getObservationUnitName())) - .collect(Collectors.toList()); + private List searchBrapiObservationUnits(BrAPIObservationUnitSearchRequest observationUnitSearchRequest, + Program program) throws ApiException { + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationUnitsApi.class); + + BrAPIObservationUnitListResponse brAPIResponse = + brAPIDAOUtil.simpleSearch( + api::searchObservationunitsPost, + observationUnitSearchRequest + ); + + List observationUnits = brAPIDAOUtil.getListResult(brAPIResponse); + processObservationUnits(program, observationUnits, false); + + return observationUnits; } /** @@ -175,11 +159,9 @@ public List createBrAPIObservationUnits(List> postFunction = () -> { - List ous = brAPIDAOUtil.post(brAPIObservationUnitList, upload, api::observationunitsPost, importDAO::update); - return processObservationUnitsForCache(ous, program, false); - }; - return programCache.post(programId, postFunction); + List ous = brAPIDAOUtil.post(brAPIObservationUnitList, upload, api::observationunitsPost, importDAO::update); + processObservationUnits(program, ous, false); + return ous; } return new ArrayList<>(); } catch (Exception e) { @@ -196,11 +178,9 @@ public List createBrAPIObservationUnits(List> postFunction = () -> { - List ous = brAPIDAOUtil.post(brAPIObservationUnitList, api::observationunitsPost); - return processObservationUnitsForCache(ous, program, false); - }; - return programCache.post(programId, postFunction); + List ous = brAPIDAOUtil.post(brAPIObservationUnitList, api::observationunitsPost); + processObservationUnits(program, ous, false); + return ous; } return new ArrayList<>(); } catch (Exception e) { @@ -208,40 +188,43 @@ public List createBrAPIObservationUnits(List getObservationUnitsById(Collection observationUnitExternalIds, Program program) throws ApiException { - if(observationUnitExternalIds.isEmpty()) { + public List getObservationUnitsById(Collection observationUnitDbIds, Program program) throws ApiException { + if (observationUnitDbIds.isEmpty()) { return Collections.emptyList(); } - return getProgramObservationUnits(program.getId()).entrySet().stream() - .filter(entry -> observationUnitExternalIds.contains(entry.getKey())) - .map(Map.Entry::getValue) + + // TODO: Optimize and change to search/get on observationUnitDbId. This will cause sample submission tests to fail until we have a better solution for exists checks for tabular errors. [BI-2987] + return getProgramObservationUnits(program.getId()).stream() + .filter(ou -> observationUnitDbIds.contains(ou.getObservationUnitDbId())) .collect(Collectors.toList()); } public List getObservationUnitsForStudyDbId(@NotNull String studyDbId, Program program) throws ApiException { - return getProgramObservationUnits(program.getId()).values().stream() - .filter(ou -> ou.getStudyDbId().equals(studyDbId)) - .collect(Collectors.toList()); + BrAPIObservationUnitSearchRequest observationUnitSearchRequest = buildSearchRequest(program, null); + observationUnitSearchRequest.setStudyDbIds(List.of(studyDbId)); + return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, true); } public List getObservationUnitsForTrialDbIds(@NotNull UUID programId, List trialDbIds) throws ApiException { if (trialDbIds.isEmpty()) { return Collections.emptyList(); } - return getProgramObservationUnits(programId).values().stream() - .filter(ou -> trialDbIds.contains(ou.getTrialDbId())) - .collect(Collectors.toList()); + + Program program = programDAO.get(programId).stream().findFirst().orElseThrow(); + + BrAPIObservationUnitSearchRequest observationUnitSearchRequest = buildSearchRequest(program, null); + observationUnitSearchRequest.setTrialDbIds(trialDbIds); + return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, true); } public List getObservationUnitsForTrialDbId(@NotNull UUID programId, @NotNull String trialDbId) throws ApiException { - return getProgramObservationUnits(programId).values().stream() - .filter(ou -> ou.getTrialDbId().equals(trialDbId)) - .collect(Collectors.toList()); + return getObservationUnitsForTrialDbIds(programId, List.of(trialDbId)); } public List getObservationUnitsForDataset(@NotNull String datasetId, @NotNull Program program) throws ApiException { String datasetReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.DATASET); - return getProgramObservationUnits(program.getId()).values().stream() + // TODO: Optimize and change this to search/get on dataset id once a solution is in place to relate ous to datasets [BI-2961] + return getProgramObservationUnits(program.getId()).stream() .filter(ou -> { Optional exRef = Utilities.getExternalReference(ou.getExternalReferences(), datasetReferenceSource); return exRef.map(brAPIExternalReference -> brAPIExternalReference.getReferenceId().equals(datasetId)).orElse(false); @@ -251,17 +234,16 @@ public List getObservationUnitsForDataset(@NotNull String public List getObservationUnitsForDatasetAndEnvs(@NotNull String datasetId, Collection envIds, @NotNull Program program) throws ApiException { String datasetReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.DATASET); - String studyReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES); - return getProgramObservationUnits(program.getId()).values().stream() + + return getProgramObservationUnits(program.getId()) + .stream() .filter(ou -> { - Optional datasetExRef = Utilities.getExternalReference(ou.getExternalReferences(), datasetReferenceSource); - Optional studyExRef = Utilities.getExternalReference(ou.getExternalReferences(), studyReferenceSource); - return Boolean.logicalAnd( - datasetExRef.map(x -> x.getReferenceId().equals(datasetId)).orElse(false), - studyExRef.map(x -> envIds.contains(x.getReferenceId())).orElse(false) - ); - }) - .collect(Collectors.toList()); + Optional datasetExRef = + Utilities.getExternalReference(ou.getExternalReferences(), datasetReferenceSource); + + return Boolean.logicalAnd(datasetExRef.map(x -> x.getReferenceId().equals(datasetId)).orElse(false), + envIds.contains(ou.getStudyDbId())); + }).collect(Collectors.toList()); } // Note: does not use cache, impractical to implement all search parameters client-side. @@ -288,61 +270,31 @@ public List getObservationUnits(Program program, BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram() .getProgramDbId())); - //TODO add pagination support + //TODO add pagination support: This should be easy to implement with BrAPIDAOUtil.simpleSearch() // .page(page) // .pageSize(pageSize); - List xrefIds = new ArrayList<>(); - List xrefSources = new ArrayList<>(); + BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); AtomicBoolean levelFilter = new AtomicBoolean(false); BrAPIObservationUnitLevelRelationship relationship = new BrAPIObservationUnitLevelRelationship(); AtomicBoolean relationshipFilter = new AtomicBoolean(false); - observationUnitId.ifPresent(ouId -> addXRefFilter(ouId, ExternalReferenceSource.OBSERVATION_UNITS, xrefIds, xrefSources)); + observationUnitId.ifPresent(dbid -> observationUnitSearchRequest.setObservationUnitDbIds(List.of(dbid))); observationUnitName.ifPresent(name -> observationUnitSearchRequest.setObservationUnitNames(List.of(Utilities.appendProgramKey(name, program.getKey())))); locationDbId.ifPresent(dbid -> observationUnitSearchRequest.setLocationDbIds(List.of(dbid))); seasonDbId.ifPresent(dbid -> observationUnitSearchRequest.setSeasonDbIds(List.of(dbid))); + experimentId.ifPresent(dbId -> observationUnitSearchRequest.setTrialDbIds(List.of(dbId))); includeObservations.ifPresent(observationUnitSearchRequest::includeObservations); + // TODO: Are level filters being used here at all? Should they be? addLevelFilter(observationUnitLevelName, observationUnitLevelOrder, observationUnitLevelCode, level, levelFilter); addLevelFilter(observationUnitLevelRelationshipName, observationUnitLevelRelationshipOrder, observationUnitLevelRelationshipCode, relationship, relationshipFilter); - experimentId.ifPresent(expId -> addXRefFilter(expId, ExternalReferenceSource.TRIALS, xrefIds, xrefSources)); - environmentId.ifPresent(envId -> addXRefFilter(envId, ExternalReferenceSource.STUDIES, xrefIds, xrefSources)); -// germplasmId.ifPresent(germId -> { -// xrefIds.add(germId); -// xrefSources.add(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.)); -// }); - - if(!xrefIds.isEmpty()) { - observationUnitSearchRequest.externalReferenceIDs(xrefIds); - } - if(!xrefSources.isEmpty()) { - observationUnitSearchRequest.externalReferenceSources(xrefSources); - } - return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, true).stream().filter(ou -> { - //xref search does an OR, so we need to convert the searching for ouId/expId/envId to be an AND - boolean matches = observationUnitId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)) - .get() - .getReferenceId())) - .orElse(true); - matches = matches && experimentId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) - .get() - .getReferenceId())) - .orElse(true); - matches = matches && environmentId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES)) - .get() - .getReferenceId())) - .orElse(true); - - //adding filter for germplasmDbId because we can't easily search that in the stored data object - return matches && germplasmId.map(id -> id.equals(ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_UUID).getAsString())).orElse(true); - }).collect(Collectors.toList()); - } + environmentId.ifPresent(envId -> observationUnitSearchRequest.setStudyDbIds(List.of(envId))); - private void addXRefFilter(String ouId, ExternalReferenceSource externalReferenceSource, List xrefIds, List xrefSources) { - xrefIds.add(ouId); - xrefSources.add(Utilities.generateReferenceSource(referenceSource, externalReferenceSource)); + return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, true).stream() + // TODO: Add search on germplasmDbId directly in the search request [BI-3006] + .filter(ou -> germplasmId.map(id -> id.equals(ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_UUID).getAsString())).orElse(true)).collect(Collectors.toList()); } private void addLevelFilter(Optional observationUnitLevelName, Optional observationUnitLevelOrder, Optional observationUnitLevelCode, BrAPIObservationUnitLevelRelationship level, AtomicBoolean levelFilter) { @@ -376,12 +328,32 @@ private List searchObservationUnitsAndProcess(BrAPIObserva return brapiObservationUnits; } + private BrAPIObservationUnitSearchRequest buildSearchRequest(Program program, List brapiOUDbIds) { + BrAPIProgram brAPIProgram = programDAO.getProgramBrAPI(program); + + if (brAPIProgram == null || brAPIProgram.getProgramDbId() == null) { + throw new InternalServerException(String.format("BI program with id [%s] not found in BrAPI db", program.getId())); + } + + BrAPIObservationUnitSearchRequest searchRequest = new BrAPIObservationUnitSearchRequest(); + + searchRequest.programDbIds(List.of(brAPIProgram.getProgramDbId())); + + if (brapiOUDbIds != null && !brapiOUDbIds.isEmpty()) { + searchRequest.setObservationUnitDbIds(brapiOUDbIds); + } + + // TODO: Utilize a search query for filtering/pagination on ous/datasets [BI-2961] + brAPIDAOUtil.setGenericSearchParameters(searchRequest, null); + + return searchRequest; + } + private void processObservationUnits(Program program, List brapiObservationUnits, boolean withGID) throws ApiException { HashMap germplasmByDbId = new HashMap<>(); if( withGID ){ - // Load germplasm for program into map. - // TODO: if we use redis search, that may be more efficient than loading all germplasm for the program. + // TODO: Optimize this to use germplasm information directly in BrAPIObservationUnit by searching on ou.germplasmDbIds in a GermplasmSearchRequest [BI-3006] this.germplasmService.getGermplasm(program.getId()).forEach((germplasm -> germplasmByDbId.put(germplasm.getGermplasmDbId(), germplasm))); } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationVariableDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationVariableDAO.java index 0de5c19ed..87b2158ca 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationVariableDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationVariableDAO.java @@ -16,13 +16,19 @@ */ package org.breedinginsight.brapi.v2.dao; +import io.micronaut.http.server.exceptions.InternalServerException; +import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationVariablesApi; import org.brapi.v2.model.pheno.BrAPIObservationVariable; import org.brapi.v2.model.pheno.request.BrAPIObservationVariableSearchRequest; +import org.brapi.v2.model.pheno.response.BrAPIObservationVariableSingleResponse; import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java index 3a164c07d..b89a28e7e 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIPedigreeDAO.java @@ -85,7 +85,7 @@ public List getPedigree( PedigreeQueryParams pedigreeRequest = new PedigreeQueryParams(); - // TODO: Issue with BrAPI server programDbId filtering, think germplasm are linked to program through observation + // TODO: Issue with BrAPI server programDbId filtering, think germplasm are linked to program through observation [BI- // units and doesn't work if don't have any loaded // use external refs instead for now //pedigreeSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java index 217d10216..424a17339 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java @@ -19,18 +19,20 @@ import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; -import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.model.queryParams.core.StudyQueryParams; import org.brapi.client.v2.modules.core.StudiesApi; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIProgram; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.request.BrAPIStudySearchRequest; +import org.brapi.v2.model.core.response.BrAPIStudyListResponse; +import org.breedinginsight.brapi.v2.model.request.query.StudyQuery; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; -import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.utilities.BrAPIDAOUtil; @@ -40,16 +42,16 @@ import javax.inject.Singleton; import javax.validation.constraints.NotNull; import java.util.*; -import java.util.concurrent.Callable; import java.util.stream.Collectors; @Slf4j @Singleton -public class BrAPIStudyDAO extends BrAPICachedDAO { +public class BrAPIStudyDAO { @Property(name = "brapi.server.reference-source") private String referenceSource; - @Property(name = "micronaut.bi.api.run-scheduled-tasks") - private boolean runScheduledTasks; + + @Property(name = "brapi.cache.fetch-page-size") + private int brapiMaxPageSize; private ProgramDAO programDAO; private ImportDAO importDAO; @@ -57,52 +59,15 @@ public class BrAPIStudyDAO extends BrAPICachedDAO { private final BrAPIEndpointProvider brAPIEndpointProvider; @Inject - public BrAPIStudyDAO(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, BrAPIEndpointProvider brAPIEndpointProvider, ProgramCacheProvider programCacheProvider) { + public BrAPIStudyDAO( + ProgramDAO programDAO, + ImportDAO importDAO, + BrAPIDAOUtil brAPIDAOUtil, + BrAPIEndpointProvider brAPIEndpointProvider) { this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; this.brAPIEndpointProvider = brAPIEndpointProvider; - this.programCache = programCacheProvider.getProgramCache(this::fetchProgramStudy, BrAPIStudy.class); - } - - @Scheduled(initialDelay = "${startup.delay.study}") - public void setup() { - if(!runScheduledTasks) { - return; - } - // Populate study cache for all programs on startup - log.debug("populating study cache"); - List programs = programDAO.getActive(); - if(programs != null) { - programCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); - } - } - - - /** - * Fetch formatted study for this program - * @param programId - * @return Map - Key = string representing study UUID, value = formatted BrAPIStudy - * @throws ApiException - */ - private Map fetchProgramStudy(UUID programId) throws ApiException { - StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), StudiesApi.class); - // Get the program key - List programs = programDAO.get(programId); - if (programs.size() != 1) { - throw new InternalServerException("Program was not found for given key"); - } - Program program = programs.get(0); - - // Set query params and make call - BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); - studySearch.externalReferenceIDs(List.of(programId.toString())); - studySearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))); - return processStudyForDisplay(brAPIDAOUtil.search( - api::searchStudiesPost, - api::searchStudiesSearchResultsDbIdGet, - studySearch - ), program.getKey()); } /** @@ -112,12 +77,78 @@ private Map fetchProgramStudy(UUID programId) throws ApiExce * @throws ApiException */ public List getStudies(UUID programId) throws ApiException { - return new ArrayList<>(programCache.get(programId).values()); + Program program = programDAO.get(programId) + .stream() + .findFirst() + .orElseThrow(); + + return getBrAPIStudiesUsingBrAPIProgramId(program); } - public Optional getStudyByName(String studyName, Program program) throws ApiException { - List studies = getStudiesByName(List.of(studyName), program); - return Utilities.getSingleOptional(studies); + public BrAPIStudyListResponse brapiStudySearch(Program program, StudyQuery studyQuery) throws ApiException { + return brapiStudySearch(program, Collections.emptyList(), studyQuery); + } + + public BrAPIStudyListResponse brapiStudySearch(Program program, List brapiTrialIds, + StudyQuery studyQuery) throws ApiException { + StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), StudiesApi.class); + + BrAPIStudySearchRequest brAPIStudySearchRequest = buildSearchRequest(program, brapiTrialIds, studyQuery); + + BrAPIStudyListResponse brAPIResponse = brAPIDAOUtil.simpleSearch(api::searchStudiesPost, brAPIStudySearchRequest); + + List processedStudies = processStudyForDisplay(brAPIDAOUtil.getListResult(brAPIResponse), program.getKey()); + + brAPIResponse.getResult().setData(processedStudies); + + return brAPIResponse; + } + + private BrAPIStudySearchRequest buildSearchRequest(Program program, List brapiTrialIds, StudyQuery studyQuery) { + BrAPIProgram brAPIProgram = programDAO.getProgramBrAPI(program); + + if (brAPIProgram == null || brAPIProgram.getProgramDbId() == null) { + throw new InternalServerException(String.format("BI program with id [%s] not found in BrAPI db", program.getId())); + } + + BrAPIStudySearchRequest searchRequest = new BrAPIStudySearchRequest(); + + searchRequest.programDbIds(List.of(brAPIProgram.getProgramDbId())); + + if (brapiTrialIds != null && !brapiTrialIds.isEmpty()) { + searchRequest.setTrialDbIds(brapiTrialIds.stream().map(UUID::toString).collect(Collectors.toList())); + } + + brAPIDAOUtil.setGenericSearchParameters(searchRequest, studyQuery); + + return searchRequest; + } + + private List getBrAPIStudiesUsingBrAPIProgramId(Program program) throws ApiException { + if (program == null || program.getId() == null) { + throw new InternalServerException("BI-API Program or Program ID is null"); + } + + String brapiProgramDbId = Optional.of(program) + .map(Program::getBrapiProgram) + .map(BrAPIProgram::getProgramDbId) + .orElse(null); + + if (brapiProgramDbId == null) { + brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); + } + + StudyQueryParams studyQueryParams = StudyQueryParams.builder() + .programDbId(brapiProgramDbId) + .page(0) + .pageSize(brapiMaxPageSize) + .build(); + + StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), StudiesApi.class); + + List result = brAPIDAOUtil.get(api::studiesGet, studyQueryParams); + + return processStudyForDisplay(result, program.getKey()); } public List getStudiesByName(List studyNames, Program program) throws ApiException { @@ -136,10 +167,11 @@ public List getStudiesByName(List studyNames, Program progra ); } - public List getStudiesByExperimentID(@NotNull UUID experimentId, Program program) throws ApiException { + public List getStudiesByBrAPITrialExRefId(@NotNull UUID brapiTrialExRefId, Program program) throws ApiException { + // TODO: If external references are removed for trial for studies, this method should look up on trialDbId, and the trialDbId should be passed through. [BI-2933] BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); studySearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - studySearch.addExternalReferenceIdsItem(experimentId.toString()); + studySearch.addExternalReferenceIdsItem(brapiTrialExRefId.toString()); studySearch.addExternalReferenceSourcesItem(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)); StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), StudiesApi.class); return brAPIDAOUtil.search( @@ -149,15 +181,6 @@ public List getStudiesByExperimentID(@NotNull UUID experimentId, Pro ); } - public List getStudiesByEnvironmentIds(@NotNull Collection environmentIds, Program program) throws ApiException { - return programCache.get(program.getId()) - .entrySet() - .stream() - .filter(entry -> environmentIds.contains(UUID.fromString(entry.getKey()))) - .map(Map.Entry::getValue) - .collect(Collectors.toList()); - } - /** * Get a list of studies by a list of BI-assigned experiment UUIDs within a program. * @param experimentIds a list of BI-assigned experiment UUIDs. @@ -167,54 +190,34 @@ public List getStudiesByEnvironmentIds(@NotNull Collection env public List getStudiesByExperimentIds(@NotNull Collection experimentIds, Program program) throws ApiException { BrAPIStudySearchRequest studySearch = new BrAPIStudySearchRequest(); studySearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - // Add all experimentIds as xref ID search terms. - for (UUID experimentId : experimentIds) { - studySearch.addExternalReferenceIdsItem(experimentId.toString()); - } - studySearch.addExternalReferenceSourcesItem(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)); + studySearch.trialDbIds(experimentIds.stream().map(UUID::toString).collect(Collectors.toList())); StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), StudiesApi.class); - return new ArrayList<>(processStudyForDisplay(brAPIDAOUtil.search( + return processStudyForDisplay(brAPIDAOUtil.search( api::searchStudiesPost, api::searchStudiesSearchResultsDbIdGet, studySearch - ), program.getKey()).values()); + ), program.getKey()); } public List createBrAPIStudies(List brAPIStudyList, UUID programId, ImportUpload upload) throws ApiException { StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), StudiesApi.class); - List createdStudies = new ArrayList<>(); + try { if (!brAPIStudyList.isEmpty()) { - Callable> postCallback = () -> { - List postedStudies = brAPIDAOUtil - .post(brAPIStudyList, upload, api::studiesPost, importDAO::update); - return environmentById(postedStudies); - }; - createdStudies.addAll(programCache.post(programId, postCallback)); + //Create studies directly through BrAPI. + List postedStudies = brAPIDAOUtil.post(brAPIStudyList, upload, api::studiesPost, importDAO::update); + + // Return the BrAPI response without updating Redis. + return postedStudies; } - return createdStudies; + // Preserve the existing empty-input behavior. + return new ArrayList<>(); } catch (Exception e) { throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); } } - /** - * @return Map - Key = BI external reference ID, Value = BrAPIStudy - * */ - private Map environmentById(List studies) { - Map environmentById = new HashMap<>(); - for (BrAPIStudy environment: studies) { - BrAPIExternalReference xref = environment - .getExternalReferences() - .stream() - .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.STUDIES.getName()).equals(reference.getReferenceSource())) - .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); - environmentById.put(xref.getReferenceID(), environment); - } - return environmentById; - } - public List getStudiesByStudyDbId(Collection studyDbIds, Program program) throws ApiException { if(studyDbIds.isEmpty()) { return Collections.emptyList(); @@ -237,26 +240,24 @@ public Optional getStudyByDbId(String studyDbId, Program program) th return Utilities.getSingleOptional(studies); } - public Optional getStudyByEnvironmentId(UUID environmentId, Program program) throws ApiException { - List studies = getStudiesByEnvironmentIds(List.of(environmentId), program); - - return Utilities.getSingleOptional(studies); - } - - /** - * Process study into a format for display - * @param programStudy - * @return Map - Key = string representing study UUID, value = formatted BrAPIStudy + * Process studies into a format for display while preserving their order. + * + * @param programStudy studies returned by BrAPI + * @param programKey the program key to remove from display fields + * @return formatted studies in their original order */ - private Map processStudyForDisplay(List programStudy, String programKey) { - Map programStudyMap = new HashMap<>(); + private List processStudyForDisplay(List programStudy, String programKey) { + List processedStudies = new ArrayList<>(programStudy.size()); + log.trace("processing study for display: " + programStudy); - for (BrAPIStudy study: programStudy) { + + for (BrAPIStudy study : programStudy) { // Remove program key from studyName, trialName and locationName. if (study.getStudyName() != null) { // Study name is appended with experiment sequence number in addition to program key. - study.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey)); + study.setStudyName( + Utilities.removeProgramKeyAndUnknownAdditionalData(study.getStudyName(), programKey)); } if (study.getTrialName() != null) { study.setTrialName(Utilities.removeProgramKey(study.getTrialName(), programKey)); @@ -264,24 +265,13 @@ private Map processStudyForDisplay(List programSt if (study.getLocationName() != null) { study.setLocationName(Utilities.removeProgramKey(study.getLocationName(), programKey)); } - } - - String refSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES); - // Add to map. - for (BrAPIStudy study: programStudy) { - JsonObject additionalInfo = study.getAdditionalInfo(); - if(additionalInfo == null) { - additionalInfo = new JsonObject(); - study.setAdditionalInfo(additionalInfo); + if (study.getAdditionalInfo() == null) { + study.setAdditionalInfo(new JsonObject()); } - BrAPIExternalReference extRef = study.getExternalReferences().stream() - .filter(reference -> reference.getReferenceSource().equals(refSource)) - .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); - String studyId = extRef.getReferenceID(); - programStudyMap.put(studyId, study); + processedStudies.add(study); } - return programStudyMap; + return processedStudies; } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPITrialDAO.java index 85298c158..1a6a4424b 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPITrialDAO.java @@ -19,6 +19,9 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.request.BrAPITrialSearchRequest; +import org.brapi.v2.model.core.response.BrAPITrialListResponse; +import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.DoesNotExistException; @@ -40,13 +43,14 @@ List createBrAPITrials(List brAPITrialList, UUID program Optional getTrialById(UUID programId, UUID trialId) throws ApiException, DoesNotExistException; - Optional getTrialByDbId(String trialDbId, Program program) throws ApiException; - List getTrialsByDbIds(Collection trialDbIds, Program program) throws ApiException; List getTrialsByExperimentIds(Collection experimentIds, Program program) throws ApiException; void deleteBrAPITrial(Program program, BrAPITrial trial, boolean hard) throws ApiException; - void repopulateCache(UUID programId); + BrAPITrialListResponse brapiTrialSearch(Program program, ExperimentQuery experimentQuery) throws ApiException; + + BrAPITrialListResponse brapiTrialSearch(Program program, List brapiTrialIds, ExperimentQuery experimentQuery) throws ApiException; + } diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java b/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java index b0c771f6d..96d7e9ff7 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java @@ -17,26 +17,28 @@ package org.breedinginsight.brapi.v2.dao.impl; import io.micronaut.context.annotation.Context; -import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; -import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; import okhttp3.HttpUrl; import okhttp3.Request; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.model.queryParams.core.TrialQueryParams; import org.brapi.client.v2.modules.core.TrialsApi; -import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.BrAPIFilterBy; +import org.brapi.v2.model.BrAPISortBy; +import org.brapi.v2.model.core.BrAPIProgram; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPITrialSearchRequest; +import org.brapi.v2.model.core.response.BrAPITrialListResponse; import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; +import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; +import org.breedinginsight.brapi.v2.utilities.BrAPITrialsMapper; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; -import org.breedinginsight.daos.cache.ProgramCache; -import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.Program; -import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.BrAPIDAOUtil; @@ -45,127 +47,77 @@ import javax.inject.Inject; import javax.inject.Singleton; import java.util.*; -import java.util.concurrent.Callable; import java.util.stream.Collectors; @Slf4j @Context @Singleton public class BrAPITrialDAOImpl implements BrAPITrialDAO { - private final ProgramCache programExperimentCache; private final ProgramDAO programDAO; private final ImportDAO importDAO; private final BrAPIDAOUtil brAPIDAOUtil; - private final ProgramService programService; private final BrAPIEndpointProvider brAPIEndpointProvider; - private final String referenceSource; - private final boolean runScheduledTasks; + private final BrAPITrialsMapper brAPITrialsMapper; + + + private static final String BI_FILTER_COLUMN_NAME = "name"; + private static final String BI_FILTER_COLUMN_ACTIVE = "active"; + private static final String BI_FILTER_COLUMN_CREATED_DATE = "createdDate"; + private static final String BI_FILTER_COLUMN_CREATED_BY = "createdBy"; @Inject - public BrAPITrialDAOImpl(ProgramCacheProvider programCacheProvider, - ProgramDAO programDAO, + public BrAPITrialDAOImpl(ProgramDAO programDAO, ImportDAO importDAO, BrAPIDAOUtil brAPIDAOUtil, - ProgramService programService, - @Property(name = "brapi.server.reference-source") String referenceSource, BrAPIEndpointProvider brAPIEndpointProvider, - @Property(name = "micronaut.bi.api.run-scheduled-tasks") boolean runScheduledTasks) { - this.programExperimentCache = programCacheProvider.getProgramCache(this::fetchProgramExperiments, BrAPITrial.class); + BrAPITrialsMapper brAPITrialsMapper) { this.programDAO = programDAO; this.importDAO = importDAO; this.brAPIDAOUtil = brAPIDAOUtil; - this.programService = programService; - this.referenceSource = referenceSource; this.brAPIEndpointProvider = brAPIEndpointProvider; - this.runScheduledTasks = runScheduledTasks; + this.brAPITrialsMapper = brAPITrialsMapper; } + /** + * This method requires a BI-API program. If the BrAPIProgram inside this data model is not set, + * this method will retrieve it. + */ + private List getBrAPITrialsUsingBrAPIProgramId(Program program) throws ApiException { - @Scheduled(initialDelay = "${startup.delay.trial}") - public void setup() { - if(!runScheduledTasks) { - return; + if (program == null || program.getId() == null) { + throw new InternalServerException("BI-API Program or Program ID is null"); } - // Populate the experiment cache for all programs on startup - log.debug("populating experiment cache"); - List programs = programDAO.getActive(); - if (programs != null) { - programExperimentCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); - } - } + String brapiProgramDbId = Optional.of(program) + .map(Program::getBrapiProgram) + .map(BrAPIProgram::getProgramDbId) + .orElse(null); - private Map fetchProgramExperiments(UUID programId) throws ApiException { - TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); - - // Get the program - List programs = programDAO.get(programId); - if (programs.size() != 1) { - throw new InternalServerException("Program was not found for given key"); + if (brapiProgramDbId == null) { + brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); } - Program program = programs.get(0); - // Get the program experiments - BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); - trialSearch.externalReferenceIDs(List.of(programId.toString())); - trialSearch.externalReferenceSources( - List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) - ); - List programExperiments = brAPIDAOUtil.search( - api::searchTrialsPost, - api::searchTrialsSearchResultsDbIdGet, - trialSearch - ); + TrialQueryParams trialQueryParams = + TrialQueryParams.builder() + .programDbId(brapiProgramDbId) + .build(); - return experimentById(processExperimentsForDisplay(programExperiments, program.getKey())); - } + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); - private Map experimentById(List trials) { - Map experimentById = new HashMap<>(); - for (BrAPITrial experiment: trials) { - BrAPIExternalReference xref = experiment - .getExternalReferences() - .stream() - .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.TRIALS.getName()).equals(reference.getReferenceSource())) - .findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found")); - experimentById.put(xref.getReferenceID(), experiment); - } - return experimentById; + List result = brAPIDAOUtil.get(api::trialsGet, trialQueryParams); + + return processExperimentsForDisplay(result, program.getKey()); } @Override public List getTrialsByName(List trialNames, Program program) throws ApiException { - Map cache = programExperimentCache.get(program.getId()); - List trials = new ArrayList<>(); - if (cache != null) { + List allTrialsForProgram = getBrAPITrialsUsingBrAPIProgramId(program); - // TODO: replace with more performant cache search, e.g. RediSearch - trials.addAll(cache - .values() - .stream() - .filter(t -> trialNames.contains(t.getTrialName())) - .collect(Collectors.toList())); - } - - return trials; - } - - private List getTrialsByExRef(String referenceSource, String referenceId, Program program) throws ApiException { - Map cache = programExperimentCache.get(program.getId()); List trials = new ArrayList<>(); - if (cache != null) { - trials.addAll(cache - .values() + if (allTrialsForProgram != null) { + trials.addAll(allTrialsForProgram .stream() - .filter(t -> { - BrAPIExternalReference xref = t - .getExternalReferences() - .stream() - .filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.TRIALS) - .equalsIgnoreCase(reference.getReferenceSource())) - .findFirst().orElseThrow(() -> new IllegalStateException("No BI trial external reference found")); - return referenceId.equals(xref.getReferenceID()); - }) + .filter(t -> trialNames.contains(t.getTrialName())) .collect(Collectors.toList())); } @@ -173,43 +125,20 @@ private List getTrialsByExRef(String referenceSource, String referen } @Override - public List createBrAPITrials(List brAPITrialList, UUID programId, ImportUpload upload) - throws ApiException { + public List createBrAPITrials(List brAPITrialList, UUID programId, ImportUpload upload) { TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); - List createdTrials = new ArrayList<>(); try { - if (!brAPITrialList.isEmpty()) { - Callable> postCallback = () -> { - List postedTrials = brAPIDAOUtil - .post(brAPITrialList, upload, api::trialsPost, importDAO::update); - return experimentById(postedTrials); - }; - createdTrials.addAll(programExperimentCache.post(programId, postCallback)); - } - - return createdTrials; + return brAPIDAOUtil.post(brAPITrialList, upload, api::trialsPost, importDAO::update); } catch (Exception e) { throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); } } + @Override - public BrAPITrial updateBrAPITrial(String trialDbId, BrAPITrial trial, UUID programId) throws ApiException { + public BrAPITrial updateBrAPITrial(String trialDbId, BrAPITrial trial, UUID programId) { TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); - BrAPITrial updatedTrial = null; try { - if (trial != null && !trialDbId.isBlank()) { - Callable> putCallback = () -> { - BrAPITrial putTrial = brAPIDAOUtil.put(trialDbId, trial, api::trialsTrialDbIdPut); - return experimentById(List.of(putTrial)); - }; - List cachedUpdates = programExperimentCache.post(programId, putCallback); - if (cachedUpdates.isEmpty()) { - throw new Exception(); - } - updatedTrial = cachedUpdates.get(0); - } - - return updatedTrial; + return brAPIDAOUtil.put(trialDbId, trial, api::trialsTrialDbIdPut); } catch (Exception e) { throw new InternalServerException("Unknown error has occurred: " + e.getMessage(), e); } @@ -217,7 +146,10 @@ public BrAPITrial updateBrAPITrial(String trialDbId, BrAPITrial trial, UUID prog @Override public List getTrials(UUID programId) throws ApiException { - return new ArrayList<>(programExperimentCache.get(programId).values()); + Program program = programDAO.get(programId).get(0); + program.setBrapiProgram(programDAO.getProgramBrAPI(program)); + + return getBrAPITrialsUsingBrAPIProgramId(program); } //Removes program key from trial name @@ -234,44 +166,45 @@ private List processExperimentsForDisplay( @Override public Optional getTrialById(UUID programId, UUID trialId) throws ApiException, DoesNotExistException { - Map cache = programExperimentCache.get(programId); - BrAPITrial trial = null; - if (cache != null) { - trial = cache.get(trialId.toString()); + Program program = programDAO.get(programId).get(0); + + TrialQueryParams params = TrialQueryParams.builder() + .trialDbId(trialId.toString()) + .build(); + + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), TrialsApi.class); + + List result = brAPIDAOUtil.get(api::trialsGet, params); + + if (result.isEmpty()) { + throw new DoesNotExistException(String.format("Trial with ID [%s] does not exist", trialId)); + } else if (result.size() > 1) { + throw new InternalServerException(String.format("Trial with ID [%s] found multiple times in database. This should not be possible.", trialId)); } - return Optional.ofNullable(trial); - } + processExperimentsForDisplay(result, program.getKey()); - @Override - public Optional getTrialByDbId(String trialDbId, Program program) throws ApiException { - List trials = getTrialsByDbIds(List.of(trialDbId), program); - return Utilities.getSingleOptional(trials); + return result.stream().findFirst(); } @Override public List getTrialsByDbIds(Collection trialDbIds, Program program) throws ApiException { - Map cache = programExperimentCache.get(program.getId()); - List trials = new ArrayList<>(); - if (cache != null) { - trials.addAll(cache - .values() - .stream() - .filter(t -> trialDbIds.contains(t.getTrialDbId())) - .collect(Collectors.toList())); - } + Collection trialDbUUIDs = trialDbIds.stream().map(UUID::fromString).collect(Collectors.toList()); - return trials; + return getTrialsByExperimentIds(trialDbUUIDs, program); } + @Override public List getTrialsByExperimentIds(Collection experimentIds, Program program) throws ApiException { if(experimentIds.isEmpty()) { return Collections.emptyList(); } + + List experimentIdsAsStrings = experimentIds.stream().map(UUID::toString).collect(Collectors.toList()); + BrAPITrialSearchRequest trialSearch = new BrAPITrialSearchRequest(); trialSearch.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); - trialSearch.externalReferenceSources(List.of(referenceSource + "/" + ExternalReferenceSource.TRIALS.getName())); - trialSearch.externalReferenceIds(experimentIds.stream().map(UUID::toString).collect(Collectors.toList())); + trialSearch.trialDbIds(experimentIdsAsStrings); TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); return processExperimentsForDisplay(brAPIDAOUtil.search( api::searchTrialsPost, @@ -280,9 +213,35 @@ public List getTrialsByExperimentIds(Collection experimentIds, ), program.getKey()); } + @Override + public BrAPITrialListResponse brapiTrialSearch(Program program, ExperimentQuery experimentQuery) throws ApiException { + return brapiTrialSearch(program, Collections.emptyList(), experimentQuery); + } + + @Override + public BrAPITrialListResponse brapiTrialSearch(Program program, + List brapiTrialIds, + ExperimentQuery experimentQuery) throws ApiException { + TrialsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), TrialsApi.class); + + BrAPITrialSearchRequest brAPITrialSearchRequest = buildSearchRequest(program, brapiTrialIds, experimentQuery); + + BrAPITrialListResponse brAPIResponse = + brAPIDAOUtil.simpleSearch( + api::searchTrialsPost, + brAPITrialSearchRequest + ); + + List processedTrials = + processExperimentsForDisplay(brAPIDAOUtil.getListResult(brAPIResponse), program.getKey()); + + brAPIResponse.getResult().setData(processedTrials); + + return brAPIResponse; + } + @Override public void deleteBrAPITrial(Program program, BrAPITrial trial, boolean hard) throws ApiException { - // TODO: Switch to using the TrialsApi from the BrAPI client library once the delete endpoints are merged into it. var programBrAPIBaseUrl = brAPIDAOUtil.getProgramBrAPIBaseUrl(program.getId()); var requestUrl = HttpUrl.parse(programBrAPIBaseUrl + "/trials/" + trial.getTrialDbId()).newBuilder(); requestUrl.addQueryParameter("hardDelete", Boolean.toString(hard)); @@ -295,9 +254,24 @@ public void deleteBrAPITrial(Program program, BrAPITrial trial, boolean hard) th brAPIDAOUtil.makeCall(brapiRequest); } - @Override - public void repopulateCache(UUID programId) { - this.programExperimentCache.invalidate(programId); - this.programExperimentCache.populate(programId); + + private BrAPITrialSearchRequest buildSearchRequest(Program program, List brapiTrialIds, ExperimentQuery experimentQuery) { + BrAPIProgram brAPIProgram = programDAO.getProgramBrAPI(program); + + if (brAPIProgram == null || brAPIProgram.getProgramDbId() == null) { + throw new InternalServerException(String.format("BI program with id [%s] not found in BrAPI db", program.getId())); + } + + BrAPITrialSearchRequest searchRequest = new BrAPITrialSearchRequest(); + + searchRequest.programDbIds(List.of(brAPIProgram.getProgramDbId())); + + if (brapiTrialIds != null && !brapiTrialIds.isEmpty()) { + searchRequest.setTrialDbIds(brapiTrialIds.stream().map(UUID::toString).collect(Collectors.toList())); + } + + brAPIDAOUtil.setGenericSearchParameters(searchRequest, experimentQuery); + + return searchRequest; } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentQuery.java index b4c08c31f..02f025968 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/ExperimentQuery.java @@ -2,13 +2,10 @@ import io.micronaut.core.annotation.Introspected; import lombok.Getter; -import org.breedinginsight.api.model.v1.request.query.FilterRequest; -import org.breedinginsight.api.model.v1.request.query.SearchRequest; import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; -import org.jooq.tools.StringUtils; -import java.util.ArrayList; -import java.util.List; +import java.util.HashMap; +import java.util.Map; @Getter @Introspected @@ -18,20 +15,27 @@ public class ExperimentQuery extends BrapiQuery { private String createdBy; private String createdDate; - public SearchRequest constructSearchRequest() { - List filters = new ArrayList<>(); - if (!StringUtils.isBlank(getName())) { - filters.add(constructFilterRequest("name", getName())); - } - if (!StringUtils.isBlank(getActive())) { - filters.add(constructFilterRequest("active", getActive())); - } - if (!StringUtils.isBlank(getCreatedBy())) { - filters.add(constructFilterRequest("createdBy", getCreatedBy())); - } - if (!StringUtils.isBlank(getCreatedDate())) { - filters.add(constructFilterRequest("createdDate", getCreatedDate())); - } - return new SearchRequest(filters); + @Override + public Map getFilterValuesByBrAPIColumnName() { + Map filterValuesByBrAPIColumnName = new HashMap<>(); + + filterValuesByBrAPIColumnName.put("trialName", getName()); + filterValuesByBrAPIColumnName.put("active", getActive()); + filterValuesByBrAPIColumnName.put("createdBy", getCreatedBy()); + filterValuesByBrAPIColumnName.put("createdDate", getCreatedDate()); + + return filterValuesByBrAPIColumnName; + } + + @Override + public Map getBrAPIColumnNamesByBiColumnName() { + Map brAPIColumnNamesByBiColumnName = new HashMap<>(); + + brAPIColumnNamesByBiColumnName.put("name", "trialName"); + brAPIColumnNamesByBiColumnName.put("active", "active"); + brAPIColumnNamesByBiColumnName.put("createdBy", "createdBy"); + brAPIColumnNamesByBiColumnName.put("createdDate", "createdDate"); + + return brAPIColumnNamesByBiColumnName; } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/GermplasmQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/GermplasmQuery.java index 464634d54..567159613 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/GermplasmQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/GermplasmQuery.java @@ -8,11 +8,14 @@ import org.jooq.tools.StringUtils; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; @Getter @Introspected public class GermplasmQuery extends BrapiQuery { + // Actual DB entry number stored in each germplasm.additional_info private String importEntryNumber; private String accessionNumber; private String defaultDisplayName; @@ -22,56 +25,62 @@ public class GermplasmQuery extends BrapiQuery { private String femaleParentGID; private String maleParentGID; private String createdDate; + // TODO: externalUUID is broken until support on brapi server appears private String externalUID; private String createdByUserName; private String synonym; + // This entry number depicts the sequential order of any given germplasm list. + // To be used only in the use case where listDbId is present and a search request is occurring on a germplasm list. + private String listEntryNumber; // This is a meta-parameter, it describes the display format of any date fields. private String dateDisplayFormat; // The list id used to get a collection of germplasm private String listDbId; - public SearchRequest constructSearchRequest() { - List filters = new ArrayList<>(); - if (!StringUtils.isBlank(getImportEntryNumber())) { - filters.add(constructFilterRequest("importEntryNumber", getImportEntryNumber())); - } - if (!StringUtils.isBlank(getAccessionNumber())) { - filters.add(constructFilterRequest("accessionNumber", getAccessionNumber())); - } - if (!StringUtils.isBlank(getDefaultDisplayName())) { - filters.add(constructFilterRequest("defaultDisplayName", getDefaultDisplayName())); - } - if (!StringUtils.isBlank(getBreedingMethod())) { - filters.add(constructFilterRequest("breedingMethod", getBreedingMethod())); - } - if (!StringUtils.isBlank(getSeedSource())) { - filters.add(constructFilterRequest("seedSource", getSeedSource())); - } - if (!StringUtils.isBlank(getPedigree())) { - filters.add(constructFilterRequest("pedigree", getPedigree())); - } - if (!StringUtils.isBlank(getFemaleParentGID())) { - filters.add(constructFilterRequest("femaleParentGID", getFemaleParentGID())); - } - if (!StringUtils.isBlank(getMaleParentGID())) { - filters.add(constructFilterRequest("maleParentGID", getMaleParentGID())); - } + @Override + public Map getFilterValuesByBrAPIColumnName() { + Map filterValuesByBrAPIColumnName = new HashMap<>(); - if (!StringUtils.isBlank(getCreatedDate())) { - filters.add(constructFilterRequest("createdDate", getCreatedDate())); - } + filterValuesByBrAPIColumnName.put("importEntryNumber", getImportEntryNumber()); + filterValuesByBrAPIColumnName.put("accessionNumber", getAccessionNumber()); + filterValuesByBrAPIColumnName.put("defaultDisplayName", getDefaultDisplayName()); + filterValuesByBrAPIColumnName.put("breedingMethod", getBreedingMethod()); + filterValuesByBrAPIColumnName.put("seedSource", getSeedSource()); + filterValuesByBrAPIColumnName.put("pedigree", getPedigree()); + filterValuesByBrAPIColumnName.put("femaleParentGID", getFemaleParentGID()); + filterValuesByBrAPIColumnName.put("maleParentGID", getMaleParentGID()); + filterValuesByBrAPIColumnName.put("createdDate", getCreatedDate()); + filterValuesByBrAPIColumnName.put("createdBy", getCreatedByUserName()); + filterValuesByBrAPIColumnName.put("synonyms", getSynonym()); - if (!StringUtils.isBlank(getExternalUID())) { - filters.add(constructFilterRequest("externalUID", getExternalUID())); + if (!StringUtils.isBlank(listDbId)) { + filterValuesByBrAPIColumnName.put("listItemPosition", getListEntryNumber()); } - if (!StringUtils.isBlank(getCreatedByUserName())) { - filters.add(constructFilterRequest("createdByUserName", getCreatedByUserName())); - } - if (!StringUtils.isBlank(getSynonym())) { - filters.add(constructFilterRequest("synonyms", getSynonym())); + return filterValuesByBrAPIColumnName; + } + + @Override + public Map getBrAPIColumnNamesByBiColumnName() { + Map brAPIColumnNamesByBiColumnName = new HashMap<>(); + + brAPIColumnNamesByBiColumnName.put("importEntryNumber", "importEntryNumber"); + brAPIColumnNamesByBiColumnName.put("accessionNumber", "accessionNumber"); + brAPIColumnNamesByBiColumnName.put("defaultDisplayName", "defaultDisplayName"); + brAPIColumnNamesByBiColumnName.put("breedingMethod", "breedingMethod"); + brAPIColumnNamesByBiColumnName.put("seedSource", "seedSource"); + brAPIColumnNamesByBiColumnName.put("pedigree", "pedigree"); + brAPIColumnNamesByBiColumnName.put("femaleParentGID", "femaleParentGID"); + brAPIColumnNamesByBiColumnName.put("maleParentGID", "maleParentGID"); + brAPIColumnNamesByBiColumnName.put("createdDate", "createdDate"); + brAPIColumnNamesByBiColumnName.put("createdByUserName", "createdBy"); + brAPIColumnNamesByBiColumnName.put("synonyms", "synonyms"); + + if (!StringUtils.isBlank(listDbId)) { + brAPIColumnNamesByBiColumnName.put("listEntryNumber", "listItemPosition"); } - return new SearchRequest(filters); + + return brAPIColumnNamesByBiColumnName; } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java index 5c3d6bfeb..edd228be1 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java +++ b/src/main/java/org/breedinginsight/brapi/v2/model/request/query/StudyQuery.java @@ -8,7 +8,9 @@ import org.jooq.tools.StringUtils; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; @Getter @Introspected @@ -24,38 +26,38 @@ public class StudyQuery extends BrapiQuery { private String externalReferenceSource; private String externalReferenceId; - public SearchRequest constructSearchRequest() { - List filters = new ArrayList<>(); - if (!StringUtils.isBlank(getStudyType())) { - filters.add(constructFilterRequest("studyType", getStudyType())); - } - if (!StringUtils.isBlank(getLocationDbId())) { - filters.add(constructFilterRequest("locationDbId", getLocationDbId())); - } - if (!StringUtils.isBlank(getStudyCode())) { - filters.add(constructFilterRequest("studyCode", getStudyCode())); - } - if (!StringUtils.isBlank(getStudyPUI())) { - filters.add(constructFilterRequest("studyPUI", getStudyPUI())); - } - if (!StringUtils.isBlank(getCommonCropName())) { - filters.add(constructFilterRequest("commonCropName", getCommonCropName())); - } - if (!StringUtils.isBlank(getTrialDbId())) { - filters.add(constructFilterRequest("trialDbId", getTrialDbId())); - } - if (!StringUtils.isBlank(getStudyDbId())) { - filters.add(constructFilterRequest("studyDbId", getStudyDbId())); - } - if (!StringUtils.isBlank(getStudyName())) { - filters.add(constructFilterRequest("studyName", getStudyName())); - } - if (!StringUtils.isBlank(getExternalReferenceSource())) { - filters.add(constructFilterRequest("externalReferenceSource", getExternalReferenceSource())); - } - if (!StringUtils.isBlank(getExternalReferenceId())) { - filters.add(constructFilterRequest("externalReferenceId", getExternalReferenceId())); - } - return new SearchRequest(filters); + @Override + public Map getFilterValuesByBrAPIColumnName() { + Map filterValuesByBrAPIColumnName = new HashMap<>(); + + filterValuesByBrAPIColumnName.put("studyType", getStudyType()); + filterValuesByBrAPIColumnName.put("locationDbId", getLocationDbId()); + filterValuesByBrAPIColumnName.put("studyCode", getStudyCode()); + filterValuesByBrAPIColumnName.put("studyPUI", getStudyPUI()); + filterValuesByBrAPIColumnName.put("commonCropName", getCommonCropName()); + filterValuesByBrAPIColumnName.put("trialDbId", getTrialDbId()); + filterValuesByBrAPIColumnName.put("studyDbId", getStudyDbId()); + filterValuesByBrAPIColumnName.put("studyName", getStudyName()); + filterValuesByBrAPIColumnName.put("externalReferenceSource", getExternalReferenceSource()); + filterValuesByBrAPIColumnName.put("externalReferenceId", getExternalReferenceId()); + + return filterValuesByBrAPIColumnName; } + + @Override + public Map getBrAPIColumnNamesByBiColumnName() { + Map brAPIColumnNamesByBiColumnName = new HashMap<>(); + + brAPIColumnNamesByBiColumnName.put("studyType", "studyType"); + brAPIColumnNamesByBiColumnName.put("locationDbId", "locationDbId"); + brAPIColumnNamesByBiColumnName.put("studyCode", "studyCode"); + brAPIColumnNamesByBiColumnName.put("studyPUI", "studyPUI"); + brAPIColumnNamesByBiColumnName.put("commonCropName", "commonCropName"); + brAPIColumnNamesByBiColumnName.put("trialDbId", "trialDbId"); + brAPIColumnNamesByBiColumnName.put("studyDbId", "studyDbId"); + brAPIColumnNamesByBiColumnName.put("studyName", "studyName"); + + return brAPIColumnNamesByBiColumnName; + } + } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index b7ba4a5eb..320f4d57f 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -7,13 +7,18 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.core.BrAPIListItemWithFullItem; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.core.response.BrAPIListItemsListResponse; import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; +import org.brapi.v2.model.germ.response.BrAPIGermplasmListResponse; +import org.brapi.v2.model.germ.response.BrAPIGermplasmListResponseResult; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapi.v2.model.request.query.GermplasmQuery; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.model.Column; import org.breedinginsight.model.DownloadFile; @@ -70,18 +75,6 @@ public BrAPIGermplasm getGermplasmByUUID(UUID programId, String germplasmId) thr } } - public List getGermplasmDbIdsForUUIDs(UUID programId, List germplasmUUIDs) throws DoesNotExistException { - try { - return germplasmDAO.getGermplasmDbIdsForUUIDs(germplasmUUIDs, programId); - } catch (ApiException e) { - throw new InternalServerException(e.getMessage(), e); - } - } - - public Optional getGermplasmByDBID(UUID programId, String germplasmId) throws ApiException { - return germplasmDAO.getGermplasmByDBID(germplasmId, programId); - } - public List> processListData(List germplasm, List listData, Program program){ Map germplasmByName = new HashMap<>(); for (BrAPIGermplasm g: germplasm) { @@ -189,49 +182,46 @@ public List> processListData(List germplasm, return processedData; } - public List getGermplasmByList(UUID programId, String listDbId) throws ApiException { + public BrAPIGermplasmListResponse getGermplasmByList(UUID programId, GermplasmQuery germplasmQuery) throws ApiException, DoesNotExistException { // get list germplasm names - BrAPIListsSingleResponse listResponse = brAPIListDAO.getListById(listDbId, programId); + BrAPIListsSingleResponse listResponse = brAPIListDAO.getListById(germplasmQuery.getListDbId(), programId); if(Objects.nonNull(listResponse) && Objects.nonNull(listResponse.getResult())) { // get the list ID stored in the list external references - UUID listId = getGermplasmListId(listResponse.getResult()); - - // get list BrAPI germplasm variables - List germplasmNames = listResponse.getResult().getData(); - List germplasm = germplasmDAO.getGermplasmByRawName(germplasmNames, programId); - Map germplasmByGid = new HashMap<>(); - - for (BrAPIGermplasm g : germplasm) { - // set the list ID in the germplasm additional info - g.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ID, listId); - // Add to map. - germplasmByGid.put(g.getAccessionNumber(), g); - } + // TODO: See if this can be removed/replaced with actual BrAPI list db id + UUID biGermplasmListId = getGermplasmListId(listResponse.getResult()); + + // Required for the actual data search in BrAPI + String brapiListDbId = listResponse.getResult().getListDbId(); + + BrAPIListItemsListResponse listSearchResponse = germplasmDAO.searchGermplasmByList(brapiListDbId, programId, germplasmQuery); - // Extract gids from list names - List gids = germplasmNames.stream().map(Utilities::extractGid).collect(Collectors.toList()); - - // Build list from BrAPI list that preserves ordering and duplicates and assigns sequential entry numbers. - List germplasmList = new ArrayList<>(); - int entryNumber = 0; - for (String gid : gids) { - ++entryNumber; - BrAPIGermplasm listEntry = cloneBrAPIGermplasm(germplasmByGid.get(gid)); - // Set entry number. - listEntry.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER, entryNumber); - germplasmList.add(listEntry); + List> listItemsWithFullItem = listSearchResponse.getResult().getData(); + + listItemsWithFullItem.forEach(li -> { + li.getItem().putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_LIST_ID, biGermplasmListId); + // listItem.position start of index is 0 in BrAPI, but in DeltaBreed, it starts at 1. increment result data to account for this. + // The actual sort/filter performs the search on a formula column created on the prod server, so no need to worry about that in the request above. + li.getItem().putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER, li.getListItem().getPosition() + 1); + }); + + List germplasm = listItemsWithFullItem.stream().map(BrAPIListItemWithFullItem::getItem).collect(Collectors.toList()); + + BrAPIGermplasmListResponse germplasmSearchResponse = new BrAPIGermplasmListResponse(); + BrAPIGermplasmListResponseResult germplasmListResponseResult = new BrAPIGermplasmListResponseResult(); + + germplasmSearchResponse.setMetadata(listSearchResponse.getMetadata()); + germplasmListResponseResult.setData(germplasm); + germplasmSearchResponse.setResult(germplasmListResponseResult); + + if (germplasm.isEmpty()) { + return germplasmSearchResponse; } - return germplasmList; + return germplasmSearchResponse; } else throw new ApiException(); } - private BrAPIGermplasm cloneBrAPIGermplasm(BrAPIGermplasm germplasm) { - // Serialize then deserialize to deep copy. - return (BrAPIGermplasm) gson.fromJson(gson.toJson(germplasm), BrAPIGermplasm.class); - } - public DownloadFile exportGermplasm(UUID programId, FileType fileExtension) throws IllegalArgumentException, ApiException, IOException, DoesNotExistException { List columns = GermplasmFileColumns.getOrderedColumns(); @@ -375,4 +365,11 @@ public List getGermplasmByDisplayName(List germplasmDisp } return matchingGermplasm; } + + public BrAPIGermplasmListResponse searchGermplasm(UUID programId, + GermplasmQuery germplasmQuery) throws ApiException, DoesNotExistException { + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Could not find program: " + programId)); + + return germplasmDAO.brapiGermplasmSearchReturnResponse(program, null, germplasmQuery); + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java index 96a21cd14..4968695a4 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java @@ -17,7 +17,7 @@ package org.breedinginsight.brapi.v2.services; -import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -26,13 +26,15 @@ import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.pheno.*; +import org.brapi.v2.model.pheno.response.BrAPIObservationVariableSingleResponse; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.model.*; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationVariableDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.DatasetUtil; -import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; import javax.inject.Inject; @@ -45,15 +47,16 @@ public class BrAPIObservationVariableService { private final ProgramService programService; private final BrAPITrialService trialService; - private final String referenceSource; + private final BrAPIObservationVariableDAO observationVariableDAO; + private final OntologyService ontologyService; @Inject public BrAPIObservationVariableService( - ProgramService programService, BrAPITrialService trialService, - @Property(name = "brapi.server.reference-source") String referenceSource) { + ProgramService programService, BrAPITrialService trialService, BrAPIObservationVariableDAO observationVariableDAO, OntologyService ontologyService){ this.programService = programService; this.trialService = trialService; - this.referenceSource = referenceSource; + this.observationVariableDAO = observationVariableDAO; + this.ontologyService = ontologyService; } // TODO: support sub-entity datasets. @@ -73,11 +76,9 @@ public List getBrAPIObservationVariablesForExperiment( if(experimentId.isPresent()) { expId = UUID.fromString(experimentId.get()); } else { - UUID envId = UUID.fromString(environmentId.orElseThrow(() -> new IllegalStateException("no environment id found"))); - BrAPIStudy environment = trialService.getEnvironment(program.get(), envId); - expId = UUID.fromString(Utilities.getExternalReference(environment.getExternalReferences(), - Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) - .orElseThrow(() -> new IllegalStateException("no external reference found")).getReferenceId()); + String studyDbId = environmentId.orElseThrow(() -> new IllegalStateException("no study db id found")); + BrAPIStudy environment = trialService.getEnvironment(program.get(), studyDbId); + expId = UUID.fromString(environment.getTrialDbId()); } BrAPITrial experiment = trialService.getExperiment(program.get(), expId); @@ -91,6 +92,31 @@ public List getBrAPIObservationVariablesForExperiment( return new ArrayList<>(); } + @NotNull + public BrAPIObservationVariable filterVariableById(List programTraits, + String observationVariableDbId) throws DoesNotExistException { + List filterResult = filterVariables(programTraits, + Optional.of(observationVariableDbId), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty() + ); + + if (filterResult.isEmpty()) { + throw new DoesNotExistException(String.format("No observation variables found for brapi observation variable dbId [%s]", observationVariableDbId)); + } else if (filterResult.size() > 1) { + throw new InternalServerException(String.format("Multiple observation variables found for brapi observation variable db id [%s]. This should not be possible considering this id is unique.", observationVariableDbId)); + } + + return filterResult.get(0); + } + @NotNull public List filterVariables(List programTraits, Optional observationVariableDbId, @@ -122,14 +148,12 @@ public List filterVariables(List programTraits, Map, String>> filterParams = new HashMap<>(); filterParams.put("observationVariableDbId", Pair.of(observationVariableDbId, - trait.getId() - .toString())); + trait.getObservationVariableDbId())); filterParams.put("observationVariableName", Pair.of(observationVariableName, trait.getObservationVariableName())); filterParams.put("traitClass", Pair.of(traitClass, trait.getTraitClass())); filterParams.put("methodDbId", Pair.of(methodDbId, - trait.getMethodId() - .toString())); + trait.getMethod().getBrapiMethodDbId())); filterParams.put("methodName", Pair.of(methodName, trait.getMethod() @@ -137,16 +161,14 @@ public List filterVariables(List programTraits, filterParams.put("scaleDbId", Pair.of(scaleDbId, trait.getScale() - .getId() - .toString())); + .getBrapiScaleDbId())); filterParams.put("scaleName", Pair.of(scaleName, trait.getScale() .getScaleName())); filterParams.put("traitDbId", Pair.of(traitDbId, - trait.getId() - .toString())); + trait.getBrapiTraitDbId())); filterParams.put("traitName", Pair.of(traitName, trait.getObservationVariableName())); filterParams.put("ontologyDbId", Pair.of(ontologyDbId, @@ -174,29 +196,31 @@ public List filterVariables(List programTraits, } public BrAPIObservationVariable convertToBrAPI(Trait trait) { + // This data is not stored in BrAPI by DeltaBreed, which is why it is being set with the DeltaBreed dbIds instead. + // They can be filtered upon in BrAPIObservationVariableController.variablesGet() BrAPIOntologyReference brAPIOntologyReference = new BrAPIOntologyReference().ontologyDbId(trait.getProgramOntologyId() .toString()); String status = trait.getActive() ? "active" : "inactive"; List synonyms = prepSynonyms(trait); - return new BrAPIObservationVariable().observationVariableDbId(trait.getId().toString()) + return new BrAPIObservationVariable().observationVariableDbId(trait.getObservationVariableDbId()) .observationVariableName(trait.getObservationVariableName()) .defaultValue(trait.getDefaultValue()) .status(status) .synonyms(synonyms) .trait(new BrAPITrait().ontologyReference(brAPIOntologyReference) .traitName(trait.getObservationVariableName()) - .traitDbId(trait.getId().toString()) + .traitDbId(trait.getBrapiTraitDbId()) .entity(trait.getEntity()) .attribute(trait.getAttribute()) .status(status) .synonyms(synonyms)) .method(new BrAPIMethod().ontologyReference(brAPIOntologyReference) - .methodDbId(trait.getMethod().getId().toString()) + .methodDbId(trait.getMethod().getBrapiMethodDbId()) .methodClass(trait.getMethod().getMethodClass()) .description(trait.getMethod().getDescription()) .formula(trait.getMethod().getFormula())) .scale(new BrAPIScale().ontologyReference(brAPIOntologyReference) - .scaleDbId(trait.getScale().getId().toString()) + .scaleDbId(trait.getScale().getBrapiScaleDbId()) .scaleName(trait.getScale().getScaleName()) .dataType(BrAPITraitDataType.fromValue(trait.getScale().getDataType().getLiteral())) .decimalPlaces(trait.getScale().getDecimalPlaces()) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java index 104664c49..fea126380 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java @@ -20,7 +20,9 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.response.BrAPIStudyListResponse; import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapi.v2.model.request.query.StudyQuery; import org.breedinginsight.model.Program; import javax.inject.Inject; @@ -40,12 +42,21 @@ public BrAPIStudyService(BrAPIStudyDAO studyDAO) { this.studyDAO = studyDAO; } + public BrAPIStudyListResponse searchStudies(Program program, StudyQuery studyQuery) throws ApiException { + return studyDAO.brapiStudySearch(program, studyQuery); + } + + public BrAPIStudyListResponse searchStudies(Program program, List authorizedExperimentIds, + StudyQuery studyQuery) throws ApiException { + return studyDAO.brapiStudySearch(program, authorizedExperimentIds, studyQuery); + } + public List getStudies(UUID programId) throws ApiException { return studyDAO.getStudies(programId); } - public Optional getStudyByEnvironmentId(Program program, UUID environmentId) throws ApiException { - return studyDAO.getStudyByEnvironmentId(environmentId, program); + public Optional getStudyByDbId(Program program, String studyDbId) throws ApiException { + return studyDAO.getStudyByDbId(studyDbId, program); } /** diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index bc1ce1ce0..f6a78c695 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -1,9 +1,9 @@ package org.breedinginsight.brapi.v2.services; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; import com.github.filosganga.geogson.model.Coordinates; import com.github.filosganga.geogson.model.positions.SinglePosition; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; import io.micronaut.http.MediaType; @@ -15,13 +15,14 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.*; import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; +import org.brapi.v2.model.core.response.BrAPITrialListResponse; import org.brapi.v2.model.germ.BrAPIGermplasm; - import org.brapi.v2.model.pheno.*; import org.breedinginsight.api.model.v1.request.SubEntityDatasetRequest; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; +import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; @@ -29,21 +30,20 @@ import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; import org.breedinginsight.dao.db.enums.DataType; -import org.breedinginsight.model.BrAPIConstants; -import org.breedinginsight.model.Column; -import org.breedinginsight.model.DownloadFile; -import org.breedinginsight.model.Program; import org.breedinginsight.model.*; import org.breedinginsight.model.delta.DeltaEntityFactory; import org.breedinginsight.model.delta.Experiment; import org.breedinginsight.services.TraitService; import org.breedinginsight.services.exceptions.AlreadyExistsException; -import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.CreationBusyException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.lock.DistributedLockService; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; -import org.breedinginsight.utilities.*; +import org.breedinginsight.utilities.DatasetUtil; +import org.breedinginsight.utilities.FileUtil; +import org.breedinginsight.utilities.IntOrderComparator; +import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; -import org.breedinginsight.services.lock.DistributedLockService; import javax.inject.Inject; import javax.inject.Singleton; @@ -54,11 +54,11 @@ import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.*; +import java.util.concurrent.TimeoutException; import java.util.function.Function; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import java.util.concurrent.TimeoutException; import static org.breedinginsight.brapps.importer.services.processors.experiment.model.ExpImportProcessConstants.OBSERVATION_UNIT_ID_SUFFIX; @@ -120,7 +120,18 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin this.observationLevelService = observationLevelService; } - public List getExperiments(UUID programId) throws ApiException, DoesNotExistException { + public BrAPITrialListResponse searchTrials(Program program, + ExperimentQuery experimentQuery) throws ApiException, DoesNotExistException { + return trialDAO.brapiTrialSearch(program, experimentQuery); + } + + public BrAPITrialListResponse searchTrials(Program program, + List brapiTrialIds, + ExperimentQuery experimentQuery) throws ApiException, DoesNotExistException { + return trialDAO.brapiTrialSearch(program, brapiTrialIds, experimentQuery); + } + + public List getTrialsByProgramId(UUID programId) throws ApiException { return trialDAO.getTrials(programId); } @@ -185,14 +196,22 @@ public DownloadFile exportObservations( // add columns for requested dataset obsvars and timestamps log.debug(logHash + ": fetching experiment for export"); BrAPITrial experiment = getExperiment(program, experimentId); + // TODO: Replace this lookup and usages others with the brapiTrialDbId [BI-2933] + String expExRefId = Utilities.getExternalReference(experiment.getExternalReferences(), this.referenceSource, ExternalReferenceSource.TRIALS) + .map(BrAPIExternalReference::getReferenceId) + .orElse(null); + + if (expExRefId == null) { + throw new InternalServerException(String.format("Trials exref does not exist for experiment [%s]", experimentId)); + } // get requested environments for the experiment log.debug(logHash + ": fetching environments for export"); - List expStudies = studyDAO.getStudiesByExperimentID(experimentId, program); + List expStudies = studyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(expExRefId), program); if (!requestedEnvIds.isEmpty()) { expStudies = expStudies .stream() - .filter(study -> requestedEnvIds.contains(getStudyId(study))) + .filter(study -> requestedEnvIds.contains(study.getStudyDbId())) .collect(Collectors.toList()); } expStudies.forEach(study -> studyByDbId.putIfAbsent(study.getStudyDbId(), study)); @@ -273,7 +292,7 @@ public DownloadFile exportObservations( // make export rows for OUs without observations if (rowByOUId.size() < ous.size()) { for (BrAPIObservationUnit ou: ous) { - String ouId = getOUId(ou); + String ouId = ou.getObservationUnitDbId(); // Map Observation Unit to the Study it belongs to. studyDbIdByOUId.put(ouId, ou.getStudyDbId()); if (!rowByOUId.containsKey(ouId)) { @@ -660,8 +679,7 @@ public BrAPIObservationUnit createSubObservationUnit( // ObservationLevelRelationships for top-level Exp Unit linking. BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); expUnitLevel.setLevelNameDbId(expUnit.getObservationUnitPosition().getObservationLevel().getLevelNameDbId()); - String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId(); - expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnitUUID, program.getKey(), seqVal)); + expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnit.getObservationUnitDbId(), program.getKey(), seqVal)); levelRelationships.add(expUnitLevel); position.setObservationLevelRelationships(levelRelationships); @@ -692,7 +710,7 @@ private void addBrAPIObsToRecords( // get observation unit for observation BrAPIObservationUnit ou = ouByOUDbId.get(obs.getObservationUnitDbId()); - String ouId = getOUId(ou); + String ouId = ou.getObservationUnitDbId(); // get observation variable for BrAPI observation Trait var = varByDbId.get(obs.getObservationVariableDbId()); @@ -713,24 +731,6 @@ private void addBrAPIObsToRecords( } } - private String getOUId(BrAPIObservationUnit ou) { - BrAPIExternalReference ouXref = Utilities.getExternalReference( - ou.getExternalReferences(), - String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName())) - .orElseThrow(() -> new RuntimeException("observation unit id not found")); - return ouXref.getReferenceID(); - } - - private String getStudyId(BrAPIStudy study) { - // HACK: avoid null reference exceptions. - if (study == null) return null; - BrAPIExternalReference studyXref = Utilities.getExternalReference( - study.getExternalReferences(), - String.format("%s/%s", referenceSource, ExternalReferenceSource.STUDIES.getName())) - .orElseThrow(() -> new RuntimeException("study id not found")); - return studyXref.getReferenceID(); - } - private void addObsVarDataToRow( Map row, BrAPIObservation obs, @@ -788,28 +788,32 @@ public int deleteExperiment(Program program, UUID experimentId, boolean hard) th } BrAPITrial trial = trials.get(0); + UUID exRefId = Utilities.getExternalReference(trial.getExternalReferences(), referenceSource, ExternalReferenceSource.TRIALS) + .map(BrAPIExternalReference::getReferenceId) + .map(UUID::fromString) + .orElse(null); + + if (exRefId == null) { + throw new InternalServerException(String.format("Experiment with UUID [%s] does not have TRIALS exref", experimentId)); + } + List existingObservations = observationDAO.getObservationsByTrialDbId(List.of(trial.getTrialDbId()), program); // If there are no observations or a soft delete is requested, proceed. if (existingObservations.isEmpty() || !hard) { // Make request to delete experiment. trialDAO.deleteBrAPITrial(program, trial, hard); // Get all lists for the trial. + // TODO: Get lists by trialDbId if trials get decoupled from datasets. [BI-2993] List lists = listDAO .getListsByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, program.getId(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS), - experimentId); + exRefId); // TODO: replace with a single call to a batch delete method if that becomes available. // Iterate over lists, delete each by listDbId. for (BrAPIListSummary list : lists) { listDAO.deleteBrAPIList(list.getListDbId(), program.getId(), hard); } - // TODO: if performance is poor, implement more precise invalidation, possibly using hierarchical cache keys. - // Invalidate and repopulate cache for Trial, Study, Observation, ObservationUnit. - trialDAO.repopulateCache(program.getId()); - studyDAO.repopulateCache(program.getId()); - observationDAO.repopulateCache(program.getId()); - observationUnitDAO.repopulateCache(program.getId()); } // Successful or not, return the number of observations in this experiment. @@ -838,12 +842,7 @@ private Map createExportRow( boolean isSubEntity) throws ApiException, DoesNotExistException { HashMap row = new HashMap<>(); - // get OU id, germplasm, and study - BrAPIExternalReference ouXref = Utilities.getExternalReference( - ou.getExternalReferences(), - String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName())) - .orElseThrow(() -> new RuntimeException("observation unit id not found")); - String ouId = ouXref.getReferenceID(); + String ouId = ou.getObservationUnitDbId(); BrAPIGermplasm germplasm = Optional.ofNullable(programGermplasmByDbId.get(ou.getGermplasmDbId())) .orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service")); BrAPIStudy study = Optional.ofNullable(studyByDbId.get(ou.getStudyDbId())) @@ -1090,16 +1089,6 @@ private String makeZipFileName(BrAPITrial experiment, Program program, String da return Utilities.makePortableFilename(unsafeName); } - private List filterDatasetByEnvironment( - List dataset, - List envIds, - Map studyByDbId) { - return dataset - .stream() - .filter(obs -> envIds.contains(getStudyId(studyByDbId.get(obs.getStudyDbId())))) - .collect(Collectors.toList()); - } - private boolean isSubEntityDataset(List ous){ return (ous.get(0).getObservationUnitPosition().getObservationLevelRelationships().size() > 2); } @@ -1128,12 +1117,8 @@ private void sortDefaultForExportRows(@NotNull List> exportR exportRows.sort(envComparator.thenComparing(expUnitIdComparator).thenComparing(subUnitIdComparator)); } - public BrAPIStudy getEnvironment(Program program, UUID envId) throws ApiException { - List environments = studyDAO.getStudiesByEnvironmentIds(List.of(envId), program); - if (environments.isEmpty()) { - throw new RuntimeException("A study with given experiment id was not returned"); + public BrAPIStudy getEnvironment(Program program, String studyDbId) throws ApiException { + return studyDAO.getStudyByDbId(studyDbId, program) + .orElseThrow(() -> new RuntimeException("A study with given study db id was not returned")); } - - return environments.get(0); - } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/utilities/BrAPISortFilterMapper.java b/src/main/java/org/breedinginsight/brapi/v2/utilities/BrAPISortFilterMapper.java new file mode 100644 index 000000000..388a481cd --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/utilities/BrAPISortFilterMapper.java @@ -0,0 +1,28 @@ +package org.breedinginsight.brapi.v2.utilities; + +import java.util.Map; + +public abstract class BrAPISortFilterMapper { + + private final Map fields; + + protected BrAPISortFilterMapper(Map fields) { + this.fields = Map.copyOf(fields); + } + + public boolean exists(String fieldName) { + return fields.containsKey(fieldName); + } + + public String getBrAPIName(String biFieldName) throws NullPointerException { + String value = fields.get(biFieldName); + + if (value == null) { + throw new IllegalArgumentException( + "No BrAPI mapping exists for field: " + biFieldName + ); + } + + return value; + } +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/utilities/BrAPITrialsMapper.java b/src/main/java/org/breedinginsight/brapi/v2/utilities/BrAPITrialsMapper.java new file mode 100644 index 000000000..0107efe86 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/utilities/BrAPITrialsMapper.java @@ -0,0 +1,16 @@ +package org.breedinginsight.brapi.v2.utilities; + +import javax.inject.Singleton; +import java.util.Map; + +@Singleton +public class BrAPITrialsMapper extends BrAPISortFilterMapper { + public BrAPITrialsMapper() { + super(Map.of( + "name", "trialName", + "active", "active", + "createdBy", "createdBy", + "createdDate", "createdDate" + )); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java index 775ecc860..bb1c900c8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/base/Germplasm.java @@ -365,12 +365,20 @@ private void setBrAPIGermplasmCommitFields(BrAPIGermplasm germplasm, String prog } } - public BrAPIGermplasm constructBrAPIGermplasm(Program program, ProgramBreedingMethodEntity breedingMethod, User user, boolean commit, String referenceSource, Supplier nextVal, UUID listId) { + public BrAPIGermplasm constructBrAPIGermplasm(Program program, + ProgramBreedingMethodEntity breedingMethod, + User user, + boolean commit, + String referenceSource, + Supplier nextVal, + UUID listId, + String brapiProgramDbId) { BrAPIGermplasm germplasm = constructBrAPIGermplasm(breedingMethod, user, listId); if (commit) { setBrAPIGermplasmCommitFields(germplasm, program.getKey(), referenceSource, nextVal); } germplasm.setCommonCropName(program.getBrapiProgram().getCommonCropName()); + germplasm.setProgramDbId(brapiProgramDbId); // Set program id in external references BrAPIExternalReference newReference = new BrAPIExternalReference(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java index 6ae9fc2ff..8c18bfba5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java @@ -151,9 +151,7 @@ public BrAPISample constructBrAPISample(boolean commit, Program program, User us if (ou != null) { brAPISample .putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBS_UNIT_ID, - Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)) - .get() - .getReferenceId()) + ou.getObservationUnitDbId()) .observationUnitDbId(ou.getObservationUnitDbId()); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/response/PendingImportObject.java b/src/main/java/org/breedinginsight/brapps/importer/model/response/PendingImportObject.java index 9b0027e27..934d1a565 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/response/PendingImportObject.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/response/PendingImportObject.java @@ -33,6 +33,7 @@ public class PendingImportObject { public PendingImportObject(ImportObjectState state, T brAPIObject, UUID id) { this.state = state; this.brAPIObject = brAPIObject; + // TODO: With 1.5 work, this id is the BI-API generated ID that is set for external references in BrAPI. It is different than the BrAPI DB ID of an entity. We may want to consider changing this at some point [BI-2933] this.id = id; } public PendingImportObject(ImportObjectState state, T brAPIObject) { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java deleted file mode 100644 index 6f1e6723a..000000000 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.breedinginsight.brapps.importer.services.processors; - -import io.micronaut.context.annotation.Prototype; -import io.micronaut.http.HttpStatus; -import io.micronaut.http.exceptions.HttpStatusException; -import io.micronaut.http.server.exceptions.InternalServerException; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.base.ObservationUnit; -import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; -import org.breedinginsight.brapps.importer.model.imports.PendingImport; -import org.breedinginsight.brapps.importer.model.response.ImportObjectState; -import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; -import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.ValidatorException; -import tech.tablesaw.api.Table; - -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; - -@Prototype -public class ObservationUnitProcessor implements Processor { - - private static final String NAME = "Observation unit"; - - private BrAPIObservationUnitDAO brAPIObservationUnitDAO; - private Map> observationUnitByName = new HashMap<>(); - private Set studyNames = new HashSet(); - private static final Set ALLOWED_LEVELS = Set.of("plot", "plant"); - - @Inject - public ObservationUnitProcessor(BrAPIObservationUnitDAO brAPIObservationUnitDAO) { - this.brAPIObservationUnitDAO = brAPIObservationUnitDAO; - } - - public void getExistingBrapiData(List importRows, Program program) { - - // get unique observation unit names - List uniqueObservationUnitNames = importRows.stream() - .map(studyImport -> studyImport.getObservationUnit().getObservationUnitName()) - .distinct() - .collect(Collectors.toList()); - - // check for existing observation units. Don't want to update existing, just create new. - // TODO: do we allow adding observations to existing studies? yes, but not updating - // ignore all data for observation units existing in system - - List existingObservationUnits; - - try { - existingObservationUnits = brAPIObservationUnitDAO.getObservationUnitByName(uniqueObservationUnitNames, program); - existingObservationUnits.forEach(existingObservationUnit -> { - - // update mapped brapi import, does in process - observationUnitByName.put(existingObservationUnit.getObservationUnitName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingObservationUnit)); - }); - } catch (ApiException e) { - // We shouldn't get an error back from our services. If we do, nothing the user can do about it - throw new InternalServerException(e.toString(), e); - } - } - - private void getExistingStudyObjects() { - - } - - @Override - public Map process(ImportUpload upload, List importRows, - Map mappedBrAPIImport, Table data, - Program program, User user, boolean commit) { - - for (int i = 0; i < importRows.size(); i++) { - BrAPIImport brapiImport = importRows.get(i); - PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(i, new PendingImport()); - - ObservationUnit observationUnit = brapiImport.getObservationUnit(); - if (observationUnit == null) { - throw new IllegalArgumentException("Import profile must have observation unit for observation unit processor"); - } - - BrAPIObservationUnit brapiObservationUnit = observationUnit.constructBrAPIObservationUnit(); - - String levelName = brapiObservationUnit.getObservationUnitPosition().getObservationLevel().getLevelName().toLowerCase(); - if (!ALLOWED_LEVELS.contains(levelName)) { - throw new HttpStatusException(HttpStatus.BAD_REQUEST, "Observation unit level name not allowed"); - } - brapiObservationUnit.getObservationUnitPosition().getObservationLevel().setLevelName(levelName); - - if (!observationUnitByName.containsKey(observationUnit.getObservationUnitName())) { - observationUnitByName.put(brapiObservationUnit.getObservationUnitName(), new PendingImportObject<>(ImportObjectState.NEW, brapiObservationUnit)); - mappedImportRow.setObservationUnit(new PendingImportObject<>(ImportObjectState.NEW, brapiObservationUnit)); - } - mappedImportRow.setObservationUnit(observationUnitByName.get(observationUnit.getObservationUnitName())); - mappedBrAPIImport.put(i, mappedImportRow); - } - - ImportPreviewStatistics stats = ImportPreviewStatistics.builder() - .newObjectCount(ProcessorData.getNumNewObjects(observationUnitByName)) - .ignoredObjectCount(ProcessorData.getNumExistingObjects(observationUnitByName)) - .build(); - - return Map.of(NAME, stats); - } - - @Override - public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { - // TODO: check if have studies, etc. - } - - - @Override - public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) { - - // check shared data object for dependency data and update observation units - updateDependencyValues(mappedBrAPIImport); - - List observationUnits = ProcessorData.getNewObjects(observationUnitByName); - - List createdObservationUnits = new ArrayList<>(); - try { - createdObservationUnits.addAll(brAPIObservationUnitDAO.createBrAPIObservationUnits(observationUnits, program.getId(), upload)); - } catch (ApiException e) { - throw new InternalServerException(e.toString(), e); - } catch (DoesNotExistException e) { - throw new InternalServerException(e.toString(), e); - } - - // Update our records - createdObservationUnits.forEach(observationUnit -> { - // update mapped brapi import, will have to do either way? no - PendingImportObject preview = observationUnitByName.get(observationUnit.getObservationUnitName()); - preview.setBrAPIObject(observationUnit); - updateObservationUnitDbIds(mappedBrAPIImport); - }); - - } - - private void updateObservationUnitDbIds(Map mappedBrAPIImport) { - observationUnitByName.values().stream() - .filter(preview -> preview != null && preview.getState() == ImportObjectState.NEW) - .map(PendingImportObject::getBrAPIObject) - .forEach(p -> updateObservationUnitDbId(mappedBrAPIImport, p)); - } - - private void updateObservationUnitDbId(Map mappedBrAPIImport, BrAPIObservationUnit observationUnit) { - mappedBrAPIImport.values().stream() - .filter(obsUnit -> obsUnit.getObservationUnit() - .getBrAPIObject().getObservationUnitName().equals(observationUnit.getObservationUnitName())) - .forEach(obsUnit -> obsUnit.getObservationUnit().getBrAPIObject().setObservationUnitDbId(observationUnit.getObservationUnitDbId())); - } - - private void updateDependencyValues(Map mappedBrAPIImport) { - - // update study DbIds - mappedBrAPIImport.values().stream() - .map(PendingImport::getStudy) - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(this::updateStudyDbId); - - // update germplasm DbIds - mappedBrAPIImport.values().stream() - .map(PendingImport::getGermplasm) - .filter(Objects::nonNull) - .distinct() - .map(PendingImportObject::getBrAPIObject) - .forEach(this::updateGermplasmDbId); - - } - - private void updateStudyDbId(BrAPIStudy study) { - observationUnitByName.values().stream() - .filter(obsUnit -> obsUnit.getBrAPIObject().getStudyName().equals(study.getStudyName())) - .forEach(obsUnit -> obsUnit.getBrAPIObject().setStudyDbId(study.getStudyDbId())); - } - - private void updateGermplasmDbId(BrAPIGermplasm germplasm) { - observationUnitByName.values().stream() - .filter(obsUnit -> obsUnit.getBrAPIObject().getGermplasmName() != null && - obsUnit.getBrAPIObject().getGermplasmName().equals(germplasm.getGermplasmName())) - .forEach(obsUnit -> obsUnit.getBrAPIObject().setGermplasmDbId(germplasm.getGermplasmDbId())); - } - - @Override - public String getName() { - return NAME; - } - - -} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java index 8ded93d89..d946d9175 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java @@ -107,11 +107,8 @@ public void getExistingBrapiData(List importRows, Program program) List observationUnits = observationUnitDAO.getObservationUnitsById(obsUnitIds, program); Set germDbIds = new HashSet<>(); - String ouRefSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS); observationUnits.forEach(ou -> { - observationUnitsById.put(Utilities.getExternalReference(ou.getExternalReferences(), ouRefSource) - .get() - .getReferenceId(), ou); + observationUnitsById.put(ou.getObservationUnitDbId(), ou); germDbIds.add(ou.getGermplasmDbId()); }); germplasm.stream() diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java index f9ec45a2b..589fc5f59 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java @@ -213,8 +213,9 @@ public void initializeWorkflow(List members) { // Construct pending import objects from the units List> pendingUnits = members.stream().map(u -> (BrAPIObservationUnit) u).map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); - // Construct a hashmap to look up the pending unit by ID - Map> pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + // Construct a hashmap to look up the pending unit by brapiOUDbId + Map> pendingUnitById = pendingUnits.stream() + .collect(Collectors.toMap(pio -> pio.getBrAPIObject().getObservationUnitDbId(), pio -> pio)); // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed Map> pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), importContext.getProgram()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 9aa92fca7..9c0f304c5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -26,7 +26,6 @@ import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; @@ -38,7 +37,6 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.ProcessedDataFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; @@ -158,14 +156,8 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext Set varBrapiDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouBrapiDbIds, varBrapiDbIds, program); - // OU Exref Ids are what are displayed to users in the Obs Unit Id columns in experiments in DeltaBreed. - Map OuExRefIdByObsBrAPIDbId = new HashMap<>(); - - for (BrAPIObservation observation : observations) { - Optional ouExref = Utilities.getExternalReference(observation.getExternalReferences(), brapiReferenceSource, ExternalReferenceSource.OBSERVATION_UNITS); - ouExref.ifPresent(exRef -> OuExRefIdByObsBrAPIDbId.put(observation.getObservationDbId(), exRef.getReferenceId().toString())); - } - + // OU BrAPI DB Ids are what are displayed to users in the Obs Unit Id columns in experiments in DeltaBreed. + Map OuBrAPIDbIdByObsBrAPIDbId = observations.stream().collect(Collectors.toMap(BrAPIObservation::getObservationDbId, BrAPIObservation::getObservationUnitDbId)); // Construct helper lookup tables to use for hashing stored observation data Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); Map studyNameByDbId = context.getAppendOverwriteWorkflowContext().getStudyByNameNoScope().values().stream() @@ -175,7 +167,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext // Hash stored observation data using a signature of unit, variable, and study names Map observationByObsHash = observations.stream().collect(Collectors.toMap(o->{ - return observationService.getObservationHash(OuExRefIdByObsBrAPIDbId.get(o.getObservationDbId()), + return observationService.getObservationHash(OuBrAPIDbIdByObsBrAPIDbId.get(o.getObservationDbId()), variableNameByDbId.get(o.getObservationVariableDbId()), studyNameByDbId.get(o.getStudyDbId())); }, o->o)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java index bdcc8956f..d4f3e2198 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/CommitPendingImportObjectsStep.java @@ -208,9 +208,14 @@ public void process(ProcessContext processContext, ProcessedData processedData) // set the DbId to the for each newly created trial for (BrAPITrial createdTrial : createdTrials) { String createdTrialName = Utilities.removeProgramKey(createdTrial.getTrialName(), program.getKey()); + + trialByNameNoScope.get(createdTrialName) .getBrAPIObject() .setTrialDbId(createdTrial.getTrialDbId()); + // TODO: May need to set the main ID for pending objects with the BrAPI ID for downstream usage once cache is removed. + trialByNameNoScope.get(createdTrialName) + .setId(UUID.fromString(createdTrial.getTrialDbId())); } List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java index f30512d2b..e4d6d93cc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateExistingPendingImportObjectsStep.java @@ -198,10 +198,22 @@ private Map> initializeStudyByNameNoScop List existingStudies; Optional> trial = getTrialPIO(experimentImportRows, trialByNameNoScope); + String expExRefId = Utilities.getExternalReference(trial.get() + .getBrAPIObject() + .getExternalReferences(), BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS) + .map(BrAPIExternalReference::getReferenceId) + .orElse(null); + + if (expExRefId == null) { + String logMessage = "No exref found in trial to link trial to study"; + log.error(logMessage); + throw new InternalServerException(logMessage); + } + try { // the 'trial' variable will never be "null". - UUID experimentId = trial.get().getId(); - existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); + UUID experimentId = UUID.fromString(expExRefId); + existingStudies = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(experimentId, program); for (BrAPIStudy existingStudy : existingStudies) { experimentStudyService.processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java index f2b6ba133..a02054a69 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/create/workflow/steps/PopulateNewPendingImportObjectsStep.java @@ -1,4 +1,4 @@ - /* +/* * See the NOTICE file distributed with this work for additional information * regarding copyright ownership. * @@ -45,17 +45,12 @@ import org.breedinginsight.brapps.importer.services.processors.experiment.create.model.ProcessedPhenotypeData; import org.breedinginsight.brapps.importer.services.processors.experiment.service.DatasetService; import org.breedinginsight.brapps.importer.services.processors.experiment.services.ExperimentSeasonService; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.ProgramLocation; -import org.breedinginsight.model.User; +import org.breedinginsight.model.*; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.Utilities; -import org.breedinginsight.model.DatasetMetadata; -import org.breedinginsight.model.DatasetLevel; import org.jooq.DSLContext; -import org.breedinginsight.model.Trait; import tech.tablesaw.columns.Column; import javax.inject.Inject; @@ -635,8 +630,6 @@ private void fetchOrCreateObservationPIO(ProcessedPhenotypeData phenotypeData, newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); } - newObservation.setStudyDbId(studyPIO.getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); observationByHash.put(key, pio); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index 3c387f061..b219164f3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -70,12 +70,7 @@ public List getObservationUnitsById(Set obsUnitIds Set missingIds = new HashSet<>(obsUnitIds); // Calculate missing IDs based on retrieved BrAPI units - //missingIds.removeAll(brapiUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toSet())); - missingIds.removeAll(brapiUnits.stream() - .map(unit -> Utilities.getExternalReference(unit.getExternalReferences(), BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS)) - .filter(Optional::isPresent) - .map(Optional::get) - .map(BrAPIExternalReference::getReferenceId).collect(Collectors.toSet())); + missingIds.removeAll(brapiUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toSet())); // Throw exception with missing IDs information throw new EntityNotFoundException(missingIds); @@ -114,31 +109,6 @@ public PendingImportObject constructPIOFromBrapiUnit(BrAPI return pio[0]; } - /** - * Maps pending observation units by their reference IDs. - * This function takes a list of pending import objects representing BrAPI observation units - * and constructs a map where the key is the external reference ID of the observation unit - * and the value is the pending import object itself. - * - * @param pios List of pending import objects for BrAPI observation units - * @return A map of pending observation units keyed by their external reference ID - */ - public Map> mapPendingUnitById(List> pios) { - Map> pendingUnitById = new HashMap<>(); - - // Construct the DeltaBreed observation unit source for external references - String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - - for (PendingImportObject pio : pios) { - - // Get external reference for the Observation Unit - Optional xref = Utilities.getExternalReference(pio.getBrAPIObject().getExternalReferences(), deltaBreedOUSource); - pendingUnitById.put(xref.get().getReferenceId(),pio); - } - - return pendingUnitById; - } - /** * This method takes a list of PendingImportObject objects and a Program object as input * and maps the PendingImportObject objects by their observation unit name without the program scope. @@ -167,30 +137,4 @@ public Map> mapPendingUnitByNa return pendingUnitByNameNoScope; } - - /** - * Collects missing Observation Unit IDs from a set of reference IDs and a list of existing Observation Units. - * - * This function takes a Set of reference IDs and a List of existing Observation Units, filters out the Observation Units - * that have external references matching a specific source, and returns a List of missing Observation Unit IDs that are - * present in the reference IDs but not found in the existing Observation Units. - * - * @param referenceIds The Set of reference IDs representing all possible Observation Unit IDs to match against. - * @param existingUnits The List of existing Observation Units to compare against the reference IDs. - * @return A List of Observation Unit IDs that are missing from the existing Observation Units but present in the reference IDs. - */ - public List collectMissingOUIds(Set referenceIds, List existingUnits) { - List missingIds = new ArrayList<>(referenceIds); - - // Construct the DeltaBreed observation unit source for external references - String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - - Set fetchedIds = existingUnits.stream() - .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) - .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) - .collect(Collectors.toSet()); - missingIds.removeAll(fetchedIds); - - return missingIds; - } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java index 1a9a7f02f..a8e1d0adf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/TrialService.java @@ -273,11 +273,7 @@ private void processAndCacheTrial( Program program, Map> trialByNameNoScope) { - //get TrialId from existingTrial - BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) - .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); - UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + UUID experimentId = UUID.fromString(existingTrial.getTrialDbId()); trialByNameNoScope.put( Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), @@ -288,20 +284,14 @@ private void processAndCacheTrial( * Constructs a PendingImportObject containing a BrAPITrial object based on the input BrAPITrial. * * This function takes a BrAPITrial object as input and constructs a PendingImportObject which - * encapsulates the trial along with its associated experiment ID. The experiment ID is retrieved - * from the external references of the trial object using utility method getExternalReference. - * If the experiment ID is not found in the external references, an InternalServerException is thrown. + * encapsulates the trial along with its associated experiment ID, which is the trialDbId from the BrAPI DB. * * @param trial the BrAPITrial object for which the PendingImportObject is to be constructed * @return a PendingImportObject containing the BrAPITrial object and its associated experiment ID - * @throws InternalServerException if the experiment ID is not found in the external references of the trial */ - public PendingImportObject constructPIOFromBrapiTrial(BrAPITrial trial) throws InternalServerException { - PendingImportObject pio = null; - BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(trial.getExternalReferences(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) - .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); - UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + public PendingImportObject constructPIOFromBrapiTrial(BrAPITrial trial) { + PendingImportObject pio; + UUID experimentId = UUID.fromString(trial.getTrialDbId()); pio = new PendingImportObject<>(ImportObjectState.EXISTING, trial, experimentId); return pio; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java index 2e3a2ec33..2be781952 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/services/ExperimentTrialService.java @@ -149,11 +149,7 @@ private void processAndCacheTrial( Program program, Map> trialByNameNoScope) { - //get TrialId from existingTrial - BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) - .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); - UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + UUID experimentId = UUID.fromString(existingTrial.getTrialDbId()); trialByNameNoScope.put( Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java index 899924f4b..4ea4d7963 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/germplasm/GermplasmProcessor.java @@ -43,8 +43,10 @@ import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.brapps.importer.services.processors.Processor; +import org.breedinginsight.dao.db.tables.daos.ProgramDao; import org.breedinginsight.dao.db.tables.pojos.ProgramBreedingMethodEntity; import org.breedinginsight.daos.BreedingMethodDAO; +import org.breedinginsight.daos.impl.ProgramDAOImpl; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; import org.breedinginsight.services.exceptions.ValidatorException; @@ -64,6 +66,7 @@ public class GermplasmProcessor implements Processor { private static final String NAME = "Germplasm"; + private final ProgramDAOImpl programDAOImpl; @Property(name = "brapi.server.reference-source") private String BRAPI_REFERENCE_SOURCE; @@ -109,13 +112,14 @@ public class GermplasmProcessor implements Processor { }; @Inject - public GermplasmProcessor(BrAPIGermplasmService brAPIGermplasmService, DSLContext dsl, BreedingMethodDAO breedingMethodDAO, BrAPIListDAO brAPIListDAO, BrAPIGermplasmDAO brAPIGermplasmDAO) { + public GermplasmProcessor(BrAPIGermplasmService brAPIGermplasmService, DSLContext dsl, BreedingMethodDAO breedingMethodDAO, BrAPIListDAO brAPIListDAO, BrAPIGermplasmDAO brAPIGermplasmDAO, ProgramDAOImpl programDAOImpl) { this.brAPIGermplasmService = brAPIGermplasmService; this.dsl = dsl; this.breedingMethodDAO = breedingMethodDAO; this.brAPIGermplasmDAO = brAPIGermplasmDAO; this.brAPIListDAO = brAPIListDAO; this.brAPIGermplasmService = brAPIGermplasmService; + this.programDAOImpl = programDAOImpl; } public void getExistingBrapiData(List importRows, Program program) throws ApiException { @@ -289,6 +293,8 @@ public Map process(ImportUpload upload, List
userProvidedEntryNumbers = new ArrayList<>(); ValidationErrors validationErrors = new ValidationErrors(); + String brapiProgramDbId = programDAOImpl.getProgramBrAPI(program).getProgramDbId(); + for (int i = 0; i < importRows.size(); i++) { log.debug("processing germplasm row: {}", i + 1); BrAPIImport brapiImport = importRows.get(i); @@ -315,7 +321,7 @@ public Map process(ImportUpload upload, List
process(ImportUpload upload, List
breedingMethods, List badBreedingMethods, - Program program, UUID importListId, boolean commit, PendingImport mappedImportRow, int i, User user, Supplier nextVal) { + Program program, UUID importListId, boolean commit, PendingImport mappedImportRow, int i, User user, Supplier nextVal, + String brapiProgramDbId) { germplasm = removeBreedingMethodBlanks(germplasm); //Validating if Breeding Method exists for valid parent entries validateGermplasmBreedingMethod(germplasm, i + 2, validationErrors); @@ -361,7 +368,7 @@ private void processNewGermplasm(Germplasm germplasm, ValidationErrors validatio numNewPedigreeConnections++; } - BrAPIGermplasm newGermplasm = germplasm.constructBrAPIGermplasm(program, breedingMethod, user, commit, BRAPI_REFERENCE_SOURCE, nextVal, importListId); + BrAPIGermplasm newGermplasm = germplasm.constructBrAPIGermplasm(program, breedingMethod, user, commit, BRAPI_REFERENCE_SOURCE, nextVal, importListId, brapiProgramDbId); newGermplasmList.add(newGermplasm); // Assign status of the germplasm @@ -717,8 +724,6 @@ public void postBrapiData(Map mappedBrAPIImport, Program try { // Create germplasm list brAPIListDAO.createBrAPILists(List.of(importList), program.getId(), upload); - // Now that we have finished uploading, fetch all the data posted to BrAPI to the cache so it is up-to-date. - brAPIGermplasmDAO.repopulateGermplasmCacheForProgram(program.getId()); } catch (ApiException e) { throw new InternalServerException(e.toString(), e); } diff --git a/src/main/java/org/breedinginsight/daos/TraitDAO.java b/src/main/java/org/breedinginsight/daos/TraitDAO.java index 4a14f94f0..a80e8ad1f 100644 --- a/src/main/java/org/breedinginsight/daos/TraitDAO.java +++ b/src/main/java/org/breedinginsight/daos/TraitDAO.java @@ -7,6 +7,7 @@ import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.DoesNotExistException; import org.jooq.DAO; import java.util.List; @@ -24,7 +25,7 @@ public interface TraitDAO extends DAO { List getTraitsById(UUID... traitIds); - Optional getTraitFull(UUID programId, UUID traitId); + Optional getTraitFull(UUID programId, UUID traitId) throws DoesNotExistException; // could be more efficient to do a single get instead of search in saved search case but less code this way // and search stuff is working in breedbase diff --git a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java index 618d1239d..3b810cfe0 100644 --- a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java @@ -23,7 +23,6 @@ import com.google.gson.JsonDeserializer; import io.micronaut.context.annotation.Property; import io.micronaut.http.server.exceptions.InternalServerException; -import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.BrAPIClient; @@ -42,12 +41,11 @@ import org.breedinginsight.daos.ObservationDAO; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.TraitDAO; -import org.breedinginsight.daos.cache.ProgramCache; -import org.breedinginsight.daos.cache.ProgramCacheProvider; import org.breedinginsight.model.*; import org.breedinginsight.model.User; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.services.brapi.BrAPIProvider; +import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.BrAPIDAOUtil; import org.breedinginsight.utilities.Utilities; import org.jooq.*; @@ -61,7 +59,6 @@ import java.util.stream.Stream; import static org.breedinginsight.dao.db.Tables.*; -import static org.jooq.impl.DSL.lower; @Singleton @Slf4j @@ -73,7 +70,6 @@ public class TraitDAOImpl extends TraitDao implements TraitDAO { private final Boolean runScheduledTasks; private final ObservationDAO observationDao; private final BrAPIDAOUtil brAPIDAOUtil; - private final ProgramCache cache; private final ProgramDAO programDAO; private final BrAPIEndpointProvider brAPIEndpointProvider; private final Gson gson; @@ -88,7 +84,6 @@ public TraitDAOImpl(Configuration config, ObservationDAO observationDao, BrAPIDAOUtil brAPIDAOUtil, ProgramDAO programDAO, - ProgramCacheProvider programCacheProvider, BrAPIEndpointProvider brAPIEndpointProvider, @Property(name = "brapi.server.reference-source") String referenceSource, @Property(name = "micronaut.bi.api.run-scheduled-tasks") Boolean runScheduledTasks) { @@ -97,7 +92,6 @@ public TraitDAOImpl(Configuration config, this.brAPIProvider = brAPIProvider; this.observationDao = observationDao; this.brAPIDAOUtil = brAPIDAOUtil; - this.cache = programCacheProvider.getProgramCache(this::populateCache, Trait.class); this.programDAO = programDAO; this.brAPIEndpointProvider = brAPIEndpointProvider; this.gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) @@ -109,29 +103,6 @@ public TraitDAOImpl(Configuration config, this.runScheduledTasks = runScheduledTasks; } - @Scheduled(initialDelay = "${startup.delay.trait}") - public void setup() { - if(!runScheduledTasks) { - return; - } - // Populate trait cache for all programs on startup - log.debug("Populate traits cache"); - List programs = programDAO.getActive(); - if(programs != null) { - cache.populate(programs.stream().map(Program::getId).collect(Collectors.toList())); - } - } - - private Map populateCache(UUID programId) { - List programTraits = fetchTraitsFullByProgramId(programId); - Map traits = new HashMap<>(); - if (!programTraits.isEmpty()) { - programTraits.forEach(trait -> traits.put(trait.getId().toString(), trait)); - } - - return traits; - } - @Override public List getTraitsFullByProgramId(UUID programId) { List programIds = new ArrayList<>(); @@ -143,15 +114,7 @@ public List getTraitsFullByProgramId(UUID programId) { public List getTraitsFullByProgramIds(List programIds) { List saturatedTraits = new ArrayList<>(); for(UUID id : programIds) { - Map programTraits = null; - try { - programTraits = cache.get(id); - } catch (ApiException e) { - throw new RuntimeException(e); - } - if(programTraits != null) { - saturatedTraits.addAll(programTraits.values()); - } + saturatedTraits.addAll(fetchTraitsFullByProgramId(id)); } return saturatedTraits; @@ -165,9 +128,17 @@ private List fetchTraitsFullByProgramId(UUID programId) { return new ArrayList<>(); } + String brapiProgramDbId; + + try { + brapiProgramDbId = brAPIDAOUtil.getBrAPIProgramDbId(programId); + } catch (ApiException e) { + throw new InternalServerException(String.format("While fetching traits, unable to retrieve brapi programDbId using bi program ID [%s]", programId), e); + } + // Get brapi variables VariableQueryParams variablesRequest = new VariableQueryParams(); - variablesRequest.externalReferenceSource(referenceSource); + variablesRequest.programDbId(brapiProgramDbId); variablesRequest.pageSize(10000); ApiResponse brApiVariables; @@ -207,6 +178,40 @@ private List fetchTraitsFullByProgramId(UUID programId) { return saturatedTraits; } + private Trait fetchTraitFullByBiTraitAndProgramId(UUID traitId, UUID programId) throws DoesNotExistException { + // Get our db trait (equivalent to brapi variables) + Optional biTraitOpt = getTrait(programId, traitId); + if (biTraitOpt.isEmpty()) { + throw new DoesNotExistException(String.format("Unable to find BI trait with id [%s]", traitId)); + } + + Trait trait = biTraitOpt.get(); + + String brapiProgramDbId; + + try { + brapiProgramDbId = brAPIDAOUtil.getBrAPIProgramDbId(programId); + } catch (ApiException e) { + throw new InternalServerException(String.format("While fetching an individual trait in full, unable to retrieve brapi programDbId using bi program ID [%s]", programId), e); + } + + // Get brapi variables + VariableQueryParams variablesRequest = new VariableQueryParams(); + variablesRequest.programDbId(brapiProgramDbId); + variablesRequest.pageSize(1); + + variablesRequest.setExternalReferenceId(traitId.toString()); + variablesRequest.externalReferenceSource(referenceSource); + + ObservationVariablesApi variablesApi = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationVariablesApi.class); + + BrAPIObservationVariable brAPIObservationVariable = getBrAPIVariable(variablesApi, traitId, programId); + + saturateTrait(trait, brAPIObservationVariable); + + return trait; + } + @Override public List getTraitsByProgramId(UUID programId) { return getTraitsByProgramIds(programId); @@ -251,14 +256,8 @@ public List getTraitsById(UUID... traitIds){ } @Override - public Optional getTraitFull(UUID programId, UUID traitId){ - - try { - return Optional.ofNullable(cache.get(programId).get(traitId.toString())); - } catch (ApiException e) { - log.warn(Utilities.generateApiExceptionLogMessage(e)); - throw new InternalServerException("Error making BrAPI call", e); - } + public Optional getTraitFull(UUID programId, UUID traitId) throws DoesNotExistException { + return Optional.of(fetchTraitFullByBiTraitAndProgramId(traitId, programId)); } // could be more efficient to do a single get instead of search in saved search case but less code this way @@ -354,6 +353,14 @@ public List createTraitsBrAPI(List traits, User actingUser, Progra //TODO: Pass ontology reference + String brapiProgramDbId; + + try { + brapiProgramDbId = brAPIDAOUtil.getBrAPIProgramDbId(program.getId()); + } catch (ApiException e) { + throw new InternalServerException(String.format("While creating traits, unable to fetch brapi programDbId using bi program with id [%s]", program.getId()), e); + } + // Convert our traits into BrAPI traits List brApiVariables = new ArrayList<>(); for (Trait trait: traits) { @@ -419,7 +426,8 @@ public List createTraitsBrAPI(List traits, User actingUser, Progra .defaultValue(trait.getDefaultValue()) .synonyms(trait.getSynonyms()) .institution(program.getName()) - .commonCropName(program.getSpecies().getCommonName()); + .commonCropName(program.getSpecies().getCommonName()) + .programDbId(brapiProgramDbId); if (trait.getTags() != null) brApiVariable.putAdditionalInfoItem(TAGS_KEY, trait.getTags()); if (trait.getFullName() != null) brApiVariable.putAdditionalInfoItem(FULLNAME_KEY, trait.getFullName()); @@ -466,8 +474,6 @@ public List createTraitsBrAPI(List traits, User actingUser, Progra if (brApiExternalReference.getReferenceSource().equals(referenceSource) && brApiExternalReference.getReferenceID().equals(trait.getId().toString())){ saturateTrait(trait, variable); - - cache.set(program.getId(), trait.getId().toString(), trait); } } } @@ -484,7 +490,7 @@ public Trait updateTraitBrAPI(Trait trait, Program program) { List variablesAPIS = brAPIProvider.getAllUniqueVariablesAPI(); for (ObservationVariablesApi variablesAPI: variablesAPIS){ // GET brapi trait - BrAPIObservationVariable existingVariable = getBrAPIVariable(variablesAPI, trait.getId()); + BrAPIObservationVariable existingVariable = getBrAPIVariable(variablesAPI, trait.getId(), program.getId()); // Change method existingVariable.getMethod().setMethodName(constructMethodName(trait, program)); @@ -530,9 +536,6 @@ public Trait updateTraitBrAPI(Trait trait, Program program) { // Retrieve our update trait from the db updatedTrait = getTrait(program.getId(), trait.getId()).get(); saturateTrait(updatedTrait, updatedVariable); - - //update cache - cache.set(program.getId(), updatedTrait.getId().toString(), updatedTrait); } return updatedTrait; @@ -544,11 +547,17 @@ private String constructMethodName(Trait trait, Program program) { String.format("%s [%s]", trait.getMethod().getMethodClass(), program.getKey()); } - private BrAPIObservationVariable getBrAPIVariable(ObservationVariablesApi variablesAPI, UUID traitId) { + private BrAPIObservationVariable getBrAPIVariable(ObservationVariablesApi variablesAPI, + UUID traitId, + UUID biProgramId) { BrAPIObservationVariable existingVariable; try { - VariableQueryParams queryParams = new VariableQueryParams(); + + String brapiProgramDbId = brAPIDAOUtil.getBrAPIProgramDbId(biProgramId); + + VariableQueryParams queryParams = new VariableQueryParams() + .programDbId(brapiProgramDbId); queryParams.externalReferenceID(traitId.toString()); queryParams.externalReferenceSource(referenceSource); ApiResponse existingVariableResponse = diff --git a/src/main/java/org/breedinginsight/model/Method.java b/src/main/java/org/breedinginsight/model/Method.java index b300e96f5..4ce50dd90 100644 --- a/src/main/java/org/breedinginsight/model/Method.java +++ b/src/main/java/org/breedinginsight/model/Method.java @@ -17,6 +17,7 @@ package org.breedinginsight.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import io.micronaut.core.annotation.Introspected; import lombok.Getter; @@ -48,6 +49,9 @@ public class Method extends MethodEntity { private ProgramOntology programOntology; // BrAPI Properties + @JsonIgnore + // brapiMethodDbId is only used to convert the bi method id to brapiMethodDbId for the BrAPIObservationVariableController GET endpoints + private String brapiMethodDbId; private String methodClass; private String description; private String formula; @@ -76,5 +80,6 @@ public void setBrAPIProperties(BrAPIMethod brApiMethod){ this.setMethodClass(brApiMethod.getMethodClass()); this.setDescription(brApiMethod.getDescription()); this.setFormula(brApiMethod.getFormula()); + this.setBrapiMethodDbId(brApiMethod.getMethodDbId()); } } diff --git a/src/main/java/org/breedinginsight/model/Scale.java b/src/main/java/org/breedinginsight/model/Scale.java index b2458e80b..e504ce6c0 100644 --- a/src/main/java/org/breedinginsight/model/Scale.java +++ b/src/main/java/org/breedinginsight/model/Scale.java @@ -17,6 +17,7 @@ package org.breedinginsight.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import io.micronaut.core.annotation.Introspected; import lombok.Getter; @@ -50,6 +51,9 @@ public class Scale extends ScaleEntity { private ProgramOntology programOntology; // BrAPI properties + @JsonIgnore + // brapiScaleDbId is only used to convert the bi scale id to brapiScaleDbId for the BrAPIObservationVariableController GET endpoints + private String brapiScaleDbId; private Integer validValueMax; private Integer validValueMin; private Integer decimalPlaces; @@ -89,6 +93,7 @@ public void setBrAPIProperties(BrAPIScale brApiScale) { this.setValidValueMin(validValues.getMin()); this.setCategories(validValues.getCategories()); } + this.setBrapiScaleDbId(brApiScale.getScaleDbId()); } } diff --git a/src/main/java/org/breedinginsight/model/Trait.java b/src/main/java/org/breedinginsight/model/Trait.java index 0dae446bf..f476aac66 100644 --- a/src/main/java/org/breedinginsight/model/Trait.java +++ b/src/main/java/org/breedinginsight/model/Trait.java @@ -29,12 +29,10 @@ import lombok.experimental.SuperBuilder; import lombok.extern.jackson.Jacksonized; import org.brapi.v2.model.pheno.BrAPIObservationVariable; -import org.breedinginsight.api.deserializer.ArrayOfStringDeserializer; import org.breedinginsight.api.deserializer.ListOfStringDeserializer; import org.breedinginsight.dao.db.tables.pojos.TraitEntity; import org.jooq.Record; -import javax.validation.Valid; import java.util.List; import static org.breedinginsight.dao.db.Tables.TRAIT; @@ -64,6 +62,9 @@ public class Trait extends TraitEntity { // Properties from brapi private String observationVariableDbId; + // brapiTraitDbId is only used to convert the bi trait id to brapiTraitDbId for the BrAPIObservationVariableController GET endpoints + @JsonIgnore + private String brapiTraitDbId; private String traitClass; private String traitDescription; private String attribute; @@ -122,6 +123,7 @@ public void setBrAPIProperties(BrAPIObservationVariable brApiVariable) { this.setEntity(brApiVariable.getTrait().getEntity()); this.setMainAbbreviation(brApiVariable.getTrait().getMainAbbreviation()); this.setSynonyms(brApiVariable.getTrait().getSynonyms()); + this.setBrapiTraitDbId(brApiVariable.getTrait().getTraitDbId()); } this.setObservationVariableDbId(brApiVariable.getObservationVariableDbId()); this.setDefaultValue(brApiVariable.getDefaultValue()); diff --git a/src/main/java/org/breedinginsight/services/ProgramService.java b/src/main/java/org/breedinginsight/services/ProgramService.java index d7b35801b..aea07a9fa 100644 --- a/src/main/java/org/breedinginsight/services/ProgramService.java +++ b/src/main/java/org/breedinginsight/services/ProgramService.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIProgram; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.auth.SecurityService; @@ -316,5 +317,21 @@ public ArrayList getKeyValidationErrors(String key) { return keyErrors; } + public String getBrAPIProgramDbId(UUID biProgramId) throws ApiException { + Optional programOpt = getById(biProgramId); + + if (programOpt.isEmpty()) { + throw new ApiException("Program does not exist"); + } + + Program program = programOpt.get(); + + if (program.getBrapiProgram() == null) { + throw new ApiException("Program does not exist in BrAPI"); + } + + return program.getBrapiProgram().getProgramDbId(); + } + } diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index f48d3bf80..07d012b06 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -32,8 +32,10 @@ import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.model.queryParams.core.BrAPIQueryParams; import org.brapi.v2.model.*; import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.model.ProgramBrAPIEndpoints; import org.breedinginsight.services.ProgramService; @@ -43,10 +45,7 @@ import javax.inject.Singleton; import java.io.IOException; import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.UUID; +import java.util.*; import java.util.concurrent.TimeUnit; import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; @@ -77,6 +76,99 @@ public BrAPIDAOUtil(@Property(name = "brapi.search.wait-time") int searchWaitTim this.programService = programService; } + public List get(Function> getMethod, + U queryParams) throws ApiException { + T brapiResponseResult; + + boolean paginationSet = queryParams.page() != null && queryParams.pageSize() != null; + + if (!paginationSet) { + queryParams.setPage(0); + queryParams.setPageSize(pageSize); + } + + try { + brapiResponseResult = Optional.ofNullable(getMethod.apply(queryParams)) + .map(ApiResponse::getBody) + .orElseThrow(); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw e; + } catch (NoSuchElementException e) { + log.debug("Unable to retrieve BrAPIResponse from POST search request", e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.debug("error", e); + throw new InternalServerException(e.toString(), e); + } + + BrAPIPagination responsePagination = Optional.of(brapiResponseResult) + .map(BrAPIResponse::getMetadata) + .map(BrAPIMetadata::getPagination) + .orElse(null); + + if (responsePagination == null) { + throw new InternalServerException("Expected pagination metadata in BrAPI get response not present"); + } + + if (!paginationSet && responsePagination.getTotalPages() > 1) { + // Size of BrAPI entity GET data can not exceed configured pageSize. + // If it does, it is recommended the GET method not be used for this purpose. + // If you are trying to utilize this method to get a large amount of data for an entity, you can use + // searchInternal for now. In the future, we need a more robust way of transmitting this data using + // compression client techniques to reduce strain on the server and avoid keeping all the database data + // in java memory. + throw new InternalServerException(String.format("More than one page found, meaning there is more data to be returned " + + "with pageSize = [%s]", queryParams.pageSize())); + } + + return getListResult(brapiResponseResult); + } + + // Performs a POST brapi search on an entity without looping paging logic, utilizing filters and pagination provided in the searchBody. + // Also verifies response paging matches requested paging + public T simpleSearch(Function, Optional>>> searchMethod, + U searchBody) throws ApiException { + T brapiResponseResult; + + try { + // Traverse response with an optional to allow for free null checking with .map() + brapiResponseResult = Optional.ofNullable(searchMethod.apply(searchBody)) + .map(ApiResponse::getBody) + .map(Pair::getLeft) + // Deal with wrapped Optionals + .flatMap(optional -> optional) + .orElseThrow(); + } catch (ApiException e) { + log.warn(Utilities.generateApiExceptionLogMessage(e)); + throw e; + } catch (NoSuchElementException e) { + log.debug("Unable to retrieve BrAPIResponse from POST search request", e); + throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.debug("error", e); + throw new InternalServerException(e.toString(), e); + } + + BrAPIPagination responsePagination = Optional.of(brapiResponseResult) + .map(BrAPIResponse::getMetadata) + .map(BrAPIMetadata::getPagination) + .orElse(null); + + if (responsePagination == null) { + throw new InternalServerException("Expected pagination metadata in BrAPI search response not present"); + } + + var requestPageNumber = searchBody.getPage(); + var requestPageSize = searchBody.getPageSize(); + + if (!responsePagination.getCurrentPage().equals(requestPageNumber) || !responsePagination.getPageSize().equals(requestPageSize)) { + throw new InternalServerException("Page number and/or page size do not match between BrAPI search request and response"); + } + + return brapiResponseResult; + } + public List search(Function, Optional>>> searchMethod, Function3, Optional>>> searchGetMethod, U searchBody @@ -301,12 +393,12 @@ private List getListResult(ApiResponse, Optional(); } - // TODO: write generic put code - public List put(List brapiObjects, - ImportUpload upload, - Function, ApiResponse> putMethod, - Consumer progressUpdateMethod) throws ApiException { - throw new UnsupportedOperationException(); + public List getListResult(T response) { + return Optional.ofNullable(response) + .map(BrAPIResponse::getResult) + .map(result -> (BrAPIResponseResult) result) + .map(BrAPIResponseResult::getData) + .orElseThrow(); } public List post(List brapiObjects, @@ -454,4 +546,88 @@ public String getProgramBrAPIBaseUrl(UUID programId) { programBrAPIBaseUrl = programBrAPIBaseUrl.endsWith("/") ? programBrAPIBaseUrl.substring(0, programBrAPIBaseUrl.length() - 1) : programBrAPIBaseUrl; return programBrAPIBaseUrl.endsWith(BrapiVersion.BRAPI_V2) ? programBrAPIBaseUrl : programBrAPIBaseUrl + BrapiVersion.BRAPI_V2; } + + private BrAPIFilterBy constructFilterBy(String filterOn, String value) { + var filterBy = new BrAPIFilterBy(); + + filterBy.setFilterOn(filterOn); + filterBy.setValue(value); + return filterBy; + } + + private BrAPISortBy constructSortBy(String sortOn, String sortOrder) { + var sortBy = new BrAPISortBy(); + sortBy.setSortedOn(sortOn); + + BrAPISortOrder sortOrderEnum = BrAPISortOrder.valueOf(sortOrder.toUpperCase()); + + if (sortOrderEnum == null) { + sortOrderEnum = BrAPISortOrder.ASC; + } + + sortBy.setSortOrder(sortOrderEnum); + + return sortBy; + } + + /** + * Sets three generic search parameters for an outgoing BrAPI Search Query: + * - Sorting + * - Filtering + * - Pagination + */ + public void setGenericSearchParameters(T brapiSearchRequest, U biSearchQuery) { + + if (biSearchQuery == null) { + // If the search query is not available, assume maximum size fetch is required and break out of this method. + brapiSearchRequest.setPage(0); + brapiSearchRequest.setPageSize(brapiFetchPageSize); + return; + } + + // Set SortBy + List brAPISortBy = new ArrayList<>(); + + Map brapiColNameByBiColName = biSearchQuery.getBrAPIColumnNamesByBiColumnName(); + + if (StringUtils.isNotBlank(biSearchQuery.getSortField()) && brapiColNameByBiColName.containsKey(biSearchQuery.getSortField())) { + brAPISortBy.add(constructSortBy(brapiColNameByBiColName.get(biSearchQuery.getSortField()), biSearchQuery.getSortOrder().toString())); + } + + if (!brAPISortBy.isEmpty()) { + brapiSearchRequest.setSortBy(brAPISortBy); + } + + // Set FilterBy + List brAPIFilterBy = new ArrayList<>(); + + Map filterValuesByBrAPIColName = biSearchQuery.getFilterValuesByBrAPIColumnName(); + + filterValuesByBrAPIColName.forEach((brapiColumnName, value) -> { + if (StringUtils.isNotBlank(value)) { + brAPIFilterBy.add(constructFilterBy(brapiColumnName, value)); + } + }); + + if (!brAPIFilterBy.isEmpty()) { + brapiSearchRequest.setFilterBy(brAPIFilterBy); + } + + // Set Pagination + if (biSearchQuery.getPage() == null) { + brapiSearchRequest.setPage(biSearchQuery.getDefaultPage()); + } else { + brapiSearchRequest.setPage(biSearchQuery.getPage()); + } + + if (biSearchQuery.getPageSize() == null) { + brapiSearchRequest.setPageSize(biSearchQuery.getDefaultPageSize()); + } else { + brapiSearchRequest.setPageSize(biSearchQuery.getPageSize()); + } + } + + public String getBrAPIProgramDbId(UUID biProgramId) throws ApiException { + return programService.getBrAPIProgramDbId(biProgramId); + } } diff --git a/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java b/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java index cb8791fae..19650eb8a 100644 --- a/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java +++ b/src/main/java/org/breedinginsight/utilities/response/ResponseUtils.java @@ -24,6 +24,9 @@ import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.server.types.files.StreamedFile; import org.apache.commons.lang3.tuple.Pair; +import org.brapi.client.v2.model.queryParams.core.BrAPIQueryParams; +import org.brapi.v2.model.BrAPIPagination; +import org.brapi.v2.model.BrAPIResponse; import org.breedinginsight.api.model.v1.request.query.PaginationParams; import org.breedinginsight.api.model.v1.request.query.QueryParams; import org.breedinginsight.api.model.v1.request.query.SearchRequest; @@ -129,6 +132,38 @@ private static HttpResponse>> processBrapiResponse( return HttpResponse.ok(new Response(metadata, new DataResponse(paginationResult.getLeft()))); } + public static HttpResponse>> getBrapiQueryResponse(List data, + BrAPIResponse brAPIResponse) { + Pagination pagination = new Pagination(); + + // We want to use the pagination results from BrAPI here. + BrAPIPagination brAPIPagination = brAPIResponse.getMetadata().getPagination(); + + pagination.setTotalCount(brAPIPagination.getTotalCount()); + pagination.setCurrentPage(brAPIPagination.getCurrentPage()); + pagination.setPageSize(brAPIPagination.getPageSize()); + pagination.setTotalPages((int) Math.ceil(pagination.getTotalCount() / (double) pagination.getPageSize())); + + Metadata metadata = constructMetadata(new Metadata(), pagination); + + return HttpResponse.ok(new Response<>(metadata, new DataResponse<>(data))); + } + + public static HttpResponse>> getEmptyBrapiQueryResponse() { + Pagination pagination = new Pagination(); + Metadata metadata; + + // Caller expects empty BrAPI response set. Set with empty defaults + pagination.setTotalCount(0); + pagination.setCurrentPage(0); + pagination.setPageSize(0); + pagination.setTotalPages(0); + + metadata = constructMetadata(new Metadata(), pagination); + + return HttpResponse.ok(new Response<>(metadata, new DataResponse<>(new ArrayList<>()))); + } + private static HttpResponse> processUploadSearchResponse( ProgramUpload upload, SearchRequest searchRequest, QueryParams queryParams, AbstractQueryMapper mapper, Metadata metadata) { diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index e236ded35..838c1b992 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -237,3 +237,5 @@ startup: trait: ${TRAIT_START_DELAY:2s} observation: ${OBSERVATION_START_DELAY:3s} observation_unit: ${OBSERVATION_UNIT_START_DELAY:3s} +data-table: + max-size: ${DATA_TABLE_MAX_SIZE:200} diff --git a/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java index c3ba499c4..cb379596e 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/BreedingMethodControllerIntegrationTest.java @@ -355,6 +355,7 @@ public void deleteProgramMethod() { @Test public void createGermplasmWithProgramMethod() throws ApiException { Program program = createProgram("createGermProgBM", "CGBM", "CGERBM"); + String brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); ProgramBreedingMethodEntity method = ProgramBreedingMethodEntity.builder() .programId(program.getId()) @@ -401,6 +402,8 @@ public void createGermplasmWithProgramMethod() throws ApiException { .externalReferences(List.of(programRef, germIdRef)) .accessionNumber(accessionNum); + germplasm.setProgramDbId(brapiProgramDbId); + assertDoesNotThrow(() -> germplasmService.createBrAPIGermplasm(List.of(germplasm), program.getId(), null)); String germplasmUrl = String.format("/programs/%s/brapi/v2/germplasm", program.getId()); @@ -427,6 +430,7 @@ public void createGermplasmWithProgramMethod() throws ApiException { @Test public void tryDeleteProgramMethodInUse() throws ApiException { Program program = createProgram("tryDeleteProgramBM", "TDBM", "TRYDBM"); + String brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); ProgramBreedingMethodEntity method = ProgramBreedingMethodEntity.builder() .programId(program.getId()) @@ -472,6 +476,7 @@ public void tryDeleteProgramMethodInUse() throws ApiException { .putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_DATE, formatter.format(now)) .externalReferences(List.of(programRef, germIdRef)) .accessionNumber(accessionNum); + germplasm.setProgramDbId(brapiProgramDbId); assertDoesNotThrow(() -> germplasmService.createBrAPIGermplasm(List.of(germplasm), program.getId(), null)); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java index 506c89cef..07747c80e 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/ExperimentControllerIntegrationTest.java @@ -16,6 +16,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.breedinginsight.BrAPITest; @@ -24,6 +25,7 @@ import org.breedinginsight.api.model.v1.request.ProgramRequest; import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.brapps.importer.ImportTestUtils; import org.breedinginsight.brapps.importer.model.exports.FileType; @@ -33,6 +35,7 @@ import org.breedinginsight.dao.db.tables.pojos.ProgramUserRoleEntity; import org.breedinginsight.dao.db.tables.pojos.RoleEntity; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.ProgramUserDAO; import org.breedinginsight.daos.SpeciesDAO; import org.breedinginsight.daos.UserDAO; @@ -45,19 +48,24 @@ import org.breedinginsight.utilities.DatasetUtil; import org.breedinginsight.utilities.FileUtil; import org.jooq.DSLContext; -import org.junit.jupiter.api.*; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import tech.tablesaw.api.ColumnType; import tech.tablesaw.api.Row; import tech.tablesaw.api.Table; + import javax.inject.Inject; import java.io.*; import java.math.BigDecimal; import java.time.OffsetDateTime; import java.util.*; import java.util.stream.Collectors; + import static io.micronaut.http.HttpRequest.*; import static org.junit.Assert.assertNotEquals; import static org.junit.jupiter.api.Assertions.*; @@ -95,6 +103,10 @@ public class ExperimentControllerIntegrationTest extends BrAPITest { private ProgramUserDAO programUserDAO; @Inject private RoleDao roleDao; + @Inject + private BrAPITrialService brAPITrialService; + @Inject + private BrAPIStudyDAO brAPIStudyDAO; @Inject @Client("/${micronaut.bi.api.version}") @@ -104,6 +116,8 @@ public class ExperimentControllerIntegrationTest extends BrAPITest { private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); + @Inject + private ProgramDAO programDAO; @BeforeAll void setup() throws Exception { @@ -163,8 +177,10 @@ void setup() throws Exception { throw e; } + String brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); + // Add germplasm to program - List germplasm = createGermplasm(1); + List germplasm = createGermplasm(1, brapiProgramDbId); BrAPIExternalReference newReference = new BrAPIExternalReference(); newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); newReference.setReferenceID(program.getId().toString()); @@ -193,7 +209,7 @@ void setup() throws Exception { rows.add(row2); // Import test experiment, environments, and any observations - JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( + importTestUtils.uploadAndFetchWorkflow( writeDataToFile(rows, traits), null, true, @@ -201,15 +217,16 @@ void setup() throws Exception { program, mappingId, newExperimentWorkflowId); - experimentId = importResult - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(0).getAsJsonObject() - .get("trial").getAsJsonObject() - .get("id").getAsString(); - // Add environmentIds. - envIds.add(getEnvId(importResult, 0)); - envIds.add(getEnvId(importResult, 1)); + + BrAPITrial trial = brAPITrialService.getTrialsByProgramId(program.getId()).get(0); + + experimentId = trial.getTrialDbId(); + + envIds.clear(); + brAPIStudyDAO.getStudies(program.getId()) + .stream() + .sorted(Comparator.comparing(BrAPIStudy::getStudyName)) + .forEach(study -> envIds.add(study.getStudyDbId())); } // Create an experiment with no observations. @@ -234,7 +251,7 @@ private String uploadExperimentWithoutObs(Program targetProgram, String title, S expRows.add(row2); // Import test experiment, environments. - JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( + importTestUtils.uploadAndFetchWorkflow( writeDataToFile(expRows, null), null, true, @@ -242,14 +259,14 @@ private String uploadExperimentWithoutObs(Program targetProgram, String title, S targetProgram, mappingId, newExperimentWorkflowId); - String expId = importResult - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(0).getAsJsonObject() - .get("trial").getAsJsonObject() - .get("id").getAsString(); - - return expId; + + BrAPITrial trial = brAPITrialService.getTrialsByProgramId(targetProgram.getId()) + .stream() + .filter(t -> t.getTrialName().equals(title)) + .findFirst() + .orElseThrow(); + + return trial.getTrialDbId(); } /** @@ -959,6 +976,28 @@ public void deleteExperimentSuccess(boolean hardDelete, boolean withObservations } + @Test + @SneakyThrows + public void hardDeleteExperimentIgnoresObservationsFromOtherExperiment() { + Program deleteProgram = createSeededProgram("Delete Experiment Isolation"); + + String observedExperimentTitle = "Observed Experiment " + UUID.randomUUID(); + String emptyExperimentTitle = "Empty Experiment " + UUID.randomUUID(); + + uploadExperimentWithObs(deleteProgram, observedExperimentTitle, buildObservedRows(observedExperimentTitle)); + + String emptyExperimentId = uploadExperimentWithoutObs(deleteProgram, emptyExperimentTitle,"Plot"); + + Flowable> deleteCall = client.exchange(DELETE(String.format("/programs/%s/experiments/%s?hard=true", deleteProgram.getId(), emptyExperimentId)).cookie(new NettyCookie("phylo-token","test-registered-user")),String.class); + + HttpResponse deleteResponse = deleteCall.blockingFirst(); + + assertEquals(HttpStatus.NO_CONTENT, deleteResponse.getStatus()); + + JsonArray remainingTrials = getProgramTrials(deleteProgram.getId().toString()); + assertEquals(1, remainingTrials.size()); + } + private List> buildObservedRows(String title) { List> observedRows = new ArrayList<>(); String envBase = title.replaceAll("\\s+", ""); @@ -977,7 +1016,7 @@ private List> buildObservedRows(String title) { private String uploadExperimentWithObs(Program targetProgram, String title, List> expRows) throws Exception { ImportTestUtils importTestUtils = new ImportTestUtils(); - JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( + importTestUtils.uploadAndFetchWorkflow( importTestUtils.writeExperimentDataToFile(expRows, traits, false, false, null), null, true, @@ -986,12 +1025,13 @@ private String uploadExperimentWithObs(Program targetProgram, String title, List mappingId, newExperimentWorkflowId); - return importResult - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(0).getAsJsonObject() - .get("trial").getAsJsonObject() - .get("id").getAsString(); + BrAPITrial trial = brAPITrialService.getTrialsByProgramId(targetProgram.getId()) + .stream() + .filter(t -> t.getTrialName().equals(title)) + .findFirst() + .orElseThrow(); + + return trial.getTrialDbId(); } private List> buildSubEntityRows(List> topLevelRows, String entityName, int repeatedMeasures) { @@ -1088,7 +1128,9 @@ private Program createSeededProgram(String prefix) throws Exception { AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); ontologyService.createTraits(seededProgram.getId(), createTraits(2), user, false); - List germplasm = createGermplasm(1); + String brapiProgramDbId = programDAO.getProgramBrAPI(seededProgram).getProgramDbId(); + + List germplasm = createGermplasm(1, brapiProgramDbId); BrAPIExternalReference newReference = new BrAPIExternalReference(); newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); newReference.setReferenceID(seededProgram.getId().toString()); @@ -1154,7 +1196,7 @@ private List createTraits(int numToCreate) { return traits; } - private List createGermplasm(int numToCreate) { + private List createGermplasm(int numToCreate, String brapiProgramDbId) { List germplasm = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { String gid = ""+(i+1); @@ -1173,6 +1215,7 @@ private List createGermplasm(int numToCreate) { testReference.setReferenceID(UUID.randomUUID().toString()); externalRef.add(testReference); testGermplasm.setExternalReferences(externalRef); + testGermplasm.setProgramDbId(brapiProgramDbId); germplasm.add(testGermplasm); } @@ -1336,18 +1379,6 @@ private BigDecimal toBigDecimal(Object value) { return new BigDecimal(value.toString()); } - private String getEnvId(JsonObject result, int index) { - return result - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(index).getAsJsonObject() - .get("study").getAsJsonObject() - .get("brAPIObject").getAsJsonObject() - .get("externalReferences").getAsJsonArray() - .get(2).getAsJsonObject() - .get("referenceId").getAsString(); - } - private JsonArray getProgramTrials(String programId) { Flowable> getCall = client.exchange( GET(String.format("/programs/%s/brapi/v2/trials", programId)) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java index e9ea45e26..1998fb142 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -44,6 +44,7 @@ import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport.Columns; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.GenotypeImportDAO; import org.breedinginsight.daos.SpeciesDAO; import org.breedinginsight.daos.UserDAO; @@ -100,6 +101,8 @@ public class SampleSubmissionControllerIntegrationTest extends BrAPITest { private String newExperimentWorkflowId; private final Gson gson = new BrAPIClient().getJSON().getGson(); + @Inject + private ProgramDAO programDao; @BeforeAll void setup() throws Exception { @@ -135,7 +138,9 @@ void setup() throws Exception { dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); - List germplasm = createGermplasm(96); + String brapiProgramDbId = programDao.getProgramBrAPI(program).getProgramDbId(); + + List germplasm = createGermplasm(96, brapiProgramDbId); BrAPIExternalReference newReference = new BrAPIExternalReference(); newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); newReference.setReferenceID(program.getId().toString()); @@ -579,7 +584,7 @@ public File writeSubmissionToFile(List> data) throws IOExcep return file; } - private List createGermplasm(int numToCreate) { + private List createGermplasm(int numToCreate, String brapiProgramDbId) { List germplasm = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { String gid = ""+(i+1); @@ -598,6 +603,7 @@ private List createGermplasm(int numToCreate) { testReference.setReferenceID(UUID.randomUUID().toString()); externalRef.add(testReference); testGermplasm.setExternalReferences(externalRef); + testGermplasm.setProgramDbId(brapiProgramDbId); germplasm.add(testGermplasm); } diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java index 01a478bdf..1679da0fc 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java @@ -30,18 +30,16 @@ import io.reactivex.Flowable; import junit.framework.AssertionFailedError; import org.breedinginsight.BrAPITest; -import org.breedinginsight.DatabaseTest; import org.breedinginsight.TestUtils; import org.breedinginsight.api.model.v1.request.ProgramRequest; import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.api.model.v1.request.query.FilterRequest; import org.breedinginsight.api.model.v1.request.query.SearchRequest; import org.breedinginsight.api.v1.controller.metadata.SortOrder; +import org.breedinginsight.brapi.v2.dao.BrAPIProgramDAO; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.enums.TermType; import org.breedinginsight.dao.db.tables.daos.ProgramDao; -import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; -import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; import org.breedinginsight.daos.TraitDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.Program; @@ -59,7 +57,6 @@ import java.util.stream.Collectors; import static io.micronaut.http.HttpRequest.*; -import static org.breedinginsight.TestUtils.insertAndFetchTestProgram; import static org.junit.jupiter.api.Assertions.*; @MicronautTest @@ -79,6 +76,8 @@ public class TraitUploadControllerIntegrationTest extends BrAPITest { private TraitDAO traitDAO; @Inject private SpeciesService speciesService; + @Inject + private BrAPIProgramDAO programDAO; private Species validSpecies; @@ -135,7 +134,6 @@ public void setup() { //dsl.execute(fp.get("InsertProgramOntology")); dsl.execute(fp.get("InsertMethod")); dsl.execute(fp.get("InsertScale")); - dsl.execute(fp.get("InsertTrait")); } diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java index 4f15c553d..60b08d1b8 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java @@ -37,10 +37,12 @@ import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapps.importer.ImportTestUtils; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.SpeciesDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.*; @@ -60,8 +62,7 @@ import java.util.*; import static io.micronaut.http.HttpRequest.GET; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -87,6 +88,10 @@ public class BrAPIObservationLevelsControllerIntegrationTest extends BrAPITest { private OntologyService ontologyService; @Inject private BrAPIGermplasmDAO germplasmDAO; + @Inject + private ProgramDAO programDAO; + @Inject + private BrAPIStudyDAO brAPIStudyDAO; @Inject @Client("/${micronaut.bi.api.version}") @@ -127,6 +132,7 @@ void setup() throws Exception { .key("TEST") .build(); program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + String brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); @@ -154,7 +160,7 @@ void setup() throws Exception { } // Add germplasm to program - List germplasm = createGermplasm(1); + List germplasm = createGermplasm(1, brapiProgramDbId); BrAPIExternalReference newReference = new BrAPIExternalReference(); newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); newReference.setReferenceID(program.getId().toString()); @@ -199,9 +205,10 @@ void setup() throws Exception { .get(0).getAsJsonObject() .get("trial").getAsJsonObject() .get("id").getAsString(); - // Add environmentIds. - envIds.add(getEnvId(importResult, 0)); - envIds.add(getEnvId(importResult, 1)); + + envIds.clear(); + brAPIStudyDAO.getStudies(program.getId()) + .forEach(study -> envIds.add(study.getStudyDbId())); } @Test @@ -225,6 +232,25 @@ public void testGetObservationLevels() { assertTrue(levelNames.contains("plot")); } + @Test + public void testGetObservationLevelsByStudyDbId() { + Flowable> call = client.exchange( + GET(String.format( + "/programs/%s/brapi/v2/observationlevels?studyDbId=%s", + program.getId(), envIds.get(0))) + .bearerAuth("test-registered-user"), String.class); + + HttpResponse response = call.blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson( + response.body(), + JsonObject.class); + + assertNotNull(responseObj.getAsJsonObject("result")); + } + private File writeDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); @@ -243,19 +269,7 @@ private File writeDataToFile(List> data, List traits) return file; } - private String getEnvId(JsonObject result, int index) { - return result - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(index).getAsJsonObject() - .get("study").getAsJsonObject() - .get("brAPIObject").getAsJsonObject() - .get("externalReferences").getAsJsonArray() - .get(2).getAsJsonObject() - .get("referenceId").getAsString(); - } - - private List createGermplasm(int numToCreate) { + private List createGermplasm(int numToCreate, String brapiProgramDbId) { List germplasm = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { String gid = ""+(i+1); @@ -274,6 +288,7 @@ private List createGermplasm(int numToCreate) { testReference.setReferenceID(UUID.randomUUID().toString()); externalRef.add(testReference); testGermplasm.setExternalReferences(externalRef); + testGermplasm.setProgramDbId(brapiProgramDbId); germplasm.add(testGermplasm); } diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java index 44792ea64..fb47a3364 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java @@ -31,6 +31,7 @@ import io.reactivex.Flowable; import lombok.SneakyThrows; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.response.BrAPIObservationUnitSingleResponse; @@ -41,10 +42,13 @@ import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.ImportTestUtils; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.SpeciesDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.*; @@ -91,6 +95,12 @@ public class BrAPIObservationUnitControllerIntegrationTest extends BrAPITest { private OntologyService ontologyService; @Inject private BrAPIGermplasmDAO germplasmDAO; + @Inject + private BrAPITrialDAO brapiTrialDAO; + @Inject + private ProgramDAO programDAO; + @Inject + private BrAPIStudyDAO brAPIStudyDAO; @Inject @Client("/${micronaut.bi.api.version}") @@ -131,6 +141,7 @@ void setup() throws Exception { .key("TEST") .build(); program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + String brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); @@ -158,7 +169,7 @@ void setup() throws Exception { } // Add germplasm to program - List germplasm = createGermplasm(1); + List germplasm = createGermplasm(1, brapiProgramDbId); BrAPIExternalReference newReference = new BrAPIExternalReference(); newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); newReference.setReferenceID(program.getId().toString()); @@ -187,7 +198,7 @@ void setup() throws Exception { rows.add(row2); // Import test experiment, environments, and any observations - JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( + importTestUtils.uploadAndFetchWorkflow( writeDataToFile(rows, traits), null, true, @@ -195,15 +206,12 @@ void setup() throws Exception { program, mappingId, newExperimentWorkflowId); - experimentId = importResult - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(0).getAsJsonObject() - .get("trial").getAsJsonObject() - .get("id").getAsString(); - // Add environmentIds. - envIds.add(getEnvId(importResult, 0)); - envIds.add(getEnvId(importResult, 1)); + List trials = brapiTrialDAO.getTrials(program.getId()); + experimentId = trials.get(0).getTrialDbId(); + + envIds.clear(); + brAPIStudyDAO.getStudies(program.getId()) + .forEach(study -> envIds.add(study.getStudyDbId())); } @Test @@ -357,6 +365,44 @@ public void testGetOUById() { assertEquals(ou.get("observationUnitDbId").getAsString(), brAPIObservationUnitSingleResponse.get().getResult().getObservationUnitDbId()); } + @Test + public void testGetOUListByStudyDbId() { + String studyDbId = envIds.get(0); + + Flowable> call = + client.exchange( + GET(String.format( + "/programs/%s/brapi/v2/observationunits?studyDbId=%s", + program.getId(), + studyDbId)) + .bearerAuth("test-registered-user"), + String.class); + + HttpResponse response = call.blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = + gson.fromJson( + response.body(), + JsonObject.class); + + JsonArray observationUnits = + responseObj + .getAsJsonObject("result") + .getAsJsonArray("data"); + + assertEquals(1, observationUnits.size()); + + assertEquals( + studyDbId, + observationUnits + .get(0) + .getAsJsonObject() + .get("studyDbId") + .getAsString()); // NEW: returned ID remains real studyDbId + } + private File writeDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); @@ -375,19 +421,7 @@ private File writeDataToFile(List> data, List traits) return file; } - private String getEnvId(JsonObject result, int index) { - return result - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(index).getAsJsonObject() - .get("study").getAsJsonObject() - .get("brAPIObject").getAsJsonObject() - .get("externalReferences").getAsJsonArray() - .get(2).getAsJsonObject() - .get("referenceId").getAsString(); - } - - private List createGermplasm(int numToCreate) { + private List createGermplasm(int numToCreate, String brapiProgramDbId) { List germplasm = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { String gid = ""+(i+1); @@ -406,6 +440,7 @@ private List createGermplasm(int numToCreate) { testReference.setReferenceID(UUID.randomUUID().toString()); externalRef.add(testReference); testGermplasm.setExternalReferences(externalRef); + testGermplasm.setProgramDbId(brapiProgramDbId); germplasm.add(testGermplasm); } diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java index 59cd33630..8ba8580b0 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java @@ -40,10 +40,12 @@ import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapps.importer.ImportTestUtils; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.SpeciesDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.*; @@ -92,6 +94,10 @@ public class BrAPIObservationsControllerIntegrationTest extends BrAPITest { private OntologyService ontologyService; @Inject private BrAPIGermplasmDAO germplasmDAO; + @Inject + private ProgramDAO programDAO; + @Inject + private BrAPIStudyDAO brAPIStudyDAO; @Inject @Client("/${micronaut.bi.api.version}") @@ -132,6 +138,7 @@ void setup() throws Exception { .key("TEST") .build(); program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + String brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); @@ -159,7 +166,7 @@ void setup() throws Exception { } // Add germplasm to program - List germplasm = createGermplasm(1); + List germplasm = createGermplasm(1, brapiProgramDbId); BrAPIExternalReference newReference = new BrAPIExternalReference(); newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); newReference.setReferenceID(program.getId().toString()); @@ -199,9 +206,10 @@ void setup() throws Exception { .get(0).getAsJsonObject() .get("trial").getAsJsonObject() .get("id").getAsString(); - // Add environmentIds. - envIds.add(getEnvId(importResult, 0)); - envIds.add(getEnvId(importResult, 1)); + + envIds.clear(); + brAPIStudyDAO.getStudies(program.getId()) + .forEach(study -> envIds.add(study.getStudyDbId())); } @Test @@ -329,6 +337,10 @@ public void testGetObsByStudyDbId() { JsonArray observations = responseObj.getAsJsonObject("result").getAsJsonArray("data"); assertEquals(2, observations.size()); + for (JsonElement observation : observations) { + assertEquals(envIds.get(0), observation.getAsJsonObject().get("studyDbId").getAsString()); + } + // Check the observation values, keep in mind the order of results is not guaranteed. Float value1 = observations.get(0).getAsJsonObject().get("value").getAsFloat(); Float value2 = observations.get(1).getAsJsonObject().get("value").getAsFloat(); @@ -356,6 +368,58 @@ public void testGetObsPagination() { checkPagination(1, 0, HttpStatus.BAD_REQUEST, null, null, null); checkPagination(10, 100, HttpStatus.BAD_REQUEST, null, null, null); + // Invalid negative values. + checkPagination(-1, 2, HttpStatus.BAD_REQUEST, null, null, null); + checkPagination(0, -1, HttpStatus.BAD_REQUEST, null, null, null); + + // Invalid boundary when the final page is partial. + checkPagination(2, 3, HttpStatus.BAD_REQUEST, null, null, null); + } + + @Test + public void testGetObsUnknownStudyReturnsEmptyFirstPage() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observations" + "?studyDbId=%s&page=0&pageSize=2", program.getId(), UUID.randomUUID())) + .bearerAuth("test-registered-user"), String.class); + + HttpResponse response = call.blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonArray observations = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + JsonObject pagination = responseObj.getAsJsonObject("metadata").getAsJsonObject("pagination"); + + assertEquals(0, observations.size()); + assertEquals(0, pagination.get("currentPage").getAsInt()); + assertEquals(2, pagination.get("pageSize").getAsInt()); + assertEquals(0, pagination.get("totalCount").getAsInt()); + assertEquals(0, pagination.get("totalPages").getAsInt()); + } + + @Test + public void testGetObsByStudyDbIdWithPagination() { + Flowable> call = client.exchange( + GET(String.format( + "/programs/%s/brapi/v2/observations?studyDbId=%s&page=0&pageSize=1", + program.getId(), + envIds.get(0))) + .bearerAuth("test-registered-user"), String.class); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonArray observations = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + JsonObject pagination = responseObj.getAsJsonObject("metadata").getAsJsonObject("pagination"); + + assertEquals(1, observations.size()); + assertEquals(envIds.get(0), observations.get(0).getAsJsonObject().get("studyDbId").getAsString()); + + assertEquals(0, pagination.get("currentPage").getAsInt()); + assertEquals(1, pagination.get("pageSize").getAsInt()); + assertEquals(2, pagination.get("totalCount").getAsInt()); + assertEquals(2, pagination.get("totalPages").getAsInt()); } private void checkPagination(Integer page, Integer pageSize, HttpStatus expectedStatus, Integer expectedSize, Integer expectedTotalCount, Integer expectedTotalPages) { @@ -388,7 +452,8 @@ private void checkPagination(Integer page, Integer pageSize, HttpStatus expected JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); // Get metadata. JsonObject pagination = responseObj.getAsJsonObject("metadata").getAsJsonObject("pagination"); - assertEquals(expectedSize, pagination.get("pageSize").getAsInt()); + int expectedPageSize = pageSize == null ? 1000 : pageSize; + assertEquals(expectedPageSize, pagination.get("pageSize").getAsInt()); int expectedPage = page == null ? 0 : page; assertEquals(expectedPage, pagination.get("currentPage").getAsInt()); assertEquals(expectedTotalPages, pagination.get("totalPages").getAsInt()); @@ -422,19 +487,7 @@ private File writeDataToFile(List> data, List traits) return file; } - private String getEnvId(JsonObject result, int index) { - return result - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(index).getAsJsonObject() - .get("study").getAsJsonObject() - .get("brAPIObject").getAsJsonObject() - .get("externalReferences").getAsJsonArray() - .get(2).getAsJsonObject() - .get("referenceId").getAsString(); - } - - private List createGermplasm(int numToCreate) { + private List createGermplasm(int numToCreate, String brapiProgramDbId) { List germplasm = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { String gid = ""+(i+1); @@ -453,6 +506,7 @@ private List createGermplasm(int numToCreate) { testReference.setReferenceID(UUID.randomUUID().toString()); externalRef.add(testReference); testGermplasm.setExternalReferences(externalRef); + testGermplasm.setProgramDbId(brapiProgramDbId); germplasm.add(testGermplasm); } diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java index fdd0df29b..94db5dd6c 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java @@ -95,6 +95,48 @@ public void testGetStudiesAsExperimentalCollaborator() { assertTrue(studyNames.contains("Env4")); } + @Test + public void testGetStudiesAsExperimentalCollaboratorWithStudyFilter() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/studies?studyName=Env3", program.getId())) + .bearerAuth("other-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonArray studies = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + assertEquals(1, studies.size()); + assertEquals("Env3", studies.get(0).getAsJsonObject().get("studyName").getAsString()); + } + + @Test + public void testGetStudiesUsesProdServerSortingAndPagination() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/studies?sortField=studyName&sortOrder=DESC&page=1&pageSize=2", + program.getId())) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonArray studies = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + assertEquals(2, studies.size()); + assertEquals("Env2", studies.get(0).getAsJsonObject().get("studyName").getAsString()); + assertEquals("Env1", studies.get(1).getAsJsonObject().get("studyName").getAsString()); + + JsonObject pagination = responseObj.getAsJsonObject("metadata").getAsJsonObject("pagination"); + assertEquals(1, pagination.get("currentPage").getAsInt()); + assertEquals(2, pagination.get("pageSize").getAsInt()); + assertEquals(4, pagination.get("totalCount").getAsInt()); + assertEquals(2, pagination.get("totalPages").getAsInt()); + } + @Test @SneakyThrows public void testPostGetStudiesNotFound() { @@ -181,6 +223,21 @@ public void testGetStudyById() { assertEquals(2, studies.size()); JsonObject study = studies.get(0).getAsJsonObject(); + String studyExternalReferenceId = null; + + for (JsonElement reference : study.getAsJsonArray("externalReferences")) { + JsonObject externalReference = reference.getAsJsonObject(); + + if (externalReference.get("referenceSource").getAsString().endsWith("/studies")) { + studyExternalReferenceId = externalReference.get("referenceId").getAsString(); + break; + } + } + + assertNotNull(studyExternalReferenceId); + + assertNotEquals(studyExternalReferenceId, study.get("studyDbId").getAsString()); + Flowable> studyCall = client.exchange( GET(String.format("/programs/%s/brapi/v2/studies/%s", program.getId(), study.get("studyDbId").getAsString())) .bearerAuth("test-registered-user"), diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerUnitTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerUnitTest.java new file mode 100644 index 000000000..2a9991db7 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerUnitTest.java @@ -0,0 +1,200 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import lombok.SneakyThrows; +import org.brapi.v2.model.BrAPIMetadata; +import org.brapi.v2.model.BrAPIPagination; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.response.BrAPIStudyListResponse; +import org.brapi.v2.model.core.response.BrAPIStudyListResponseResult; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.auth.SecurityService; +import org.breedinginsight.api.model.v1.response.DataResponse; +import org.breedinginsight.api.model.v1.response.Response; +import org.breedinginsight.api.v1.controller.metadata.SortOrder; +import org.breedinginsight.brapi.v2.model.request.query.StudyQuery; +import org.breedinginsight.brapi.v2.services.BrAPIStudyService; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.ProgramUser; +import org.breedinginsight.services.ExperimentalCollaboratorService; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.ProgramUserService; +import org.breedinginsight.utilities.response.mappers.StudyQueryMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.*; + +class BrAPIStudiesControllerUnitTest { + + private BrAPIStudyService studyService; + private ProgramService programService; + private SecurityService securityService; + private ProgramUserService programUserService; + private ExperimentalCollaboratorService experimentalCollaboratorService; + private BrAPIStudiesController controller; + private Program program; + private UUID programId; + private UUID userId; + + @BeforeEach + void setup() { + studyService = mock(BrAPIStudyService.class); + programService = mock(ProgramService.class); + securityService = mock(SecurityService.class); + programUserService = mock(ProgramUserService.class); + experimentalCollaboratorService = mock(ExperimentalCollaboratorService.class); + + controller = new BrAPIStudiesController( + studyService, + new StudyQueryMapper(), + programService, + securityService, + programUserService, + experimentalCollaboratorService); + + programId = UUID.randomUUID(); + userId = UUID.randomUUID(); + program = new Program(); + program.setId(programId); + + AuthenticatedUser user = new AuthenticatedUser("test-user", List.of(), userId, List.of()); + when(programService.getById(programId)).thenReturn(Optional.of(program)); + when(securityService.getUser()).thenReturn(user); + when(programUserService.getIfExperimentalCollaborator(programId, userId)).thenReturn(Optional.empty()); + } + + @Test + @SneakyThrows + void getStudiesDelegatesQueryToProdServerAndUsesReturnedPagination() { + StudyQuery query = new StudyQuery(); + query.setSortField("studyCode"); + query.setSortOrder(SortOrder.DESC); + query.setPage(1); + query.setPageSize(2); + + List studies = List.of( + new BrAPIStudy().studyName("Environment 2"), + new BrAPIStudy().studyName("Environment 1")); + BrAPIStudyListResponse brAPIResponse = studyResponse(studies, 1, 2, 5); + when(studyService.searchStudies(same(program), same(query))) + .thenReturn(brAPIResponse); + + HttpResponse>> response = controller.getStudies(programId, query); + + assertEquals(HttpStatus.OK, response.getStatus()); + assertEquals(studies, response.body().getResult().getData()); + assertEquals(1, response.body().getMetadata().getPagination().getCurrentPage()); + assertEquals(2, response.body().getMetadata().getPagination().getPageSize()); + assertEquals(5, response.body().getMetadata().getPagination().getTotalCount()); + assertEquals(3, response.body().getMetadata().getPagination().getTotalPages()); + assertEquals("studyCode", query.getSortField()); + assertEquals(SortOrder.DESC, query.getSortOrder()); + + verify(studyService).searchStudies(same(program), same(query)); + verify(studyService, never()).getStudies(any(UUID.class)); + } + + @Test + @SneakyThrows + void getStudiesAppliesDefaultSortOnlyWhenSortIsMissing() { + StudyQuery query = new StudyQuery(); + query.setSortField(null); + query.setSortOrder(null); + BrAPIStudyListResponse brAPIResponse = studyResponse(List.of(), 0, 50, 0); + when(studyService.searchStudies(same(program), same(query))) + .thenReturn(brAPIResponse); + + controller.getStudies(programId, query); + + assertEquals("studyName", query.getSortField()); + assertEquals(SortOrder.ASC, query.getSortOrder()); + verify(studyService).searchStudies(same(program), same(query)); + } + + @Test + @SneakyThrows + void getStudiesLimitsCollaboratorSearchToAuthorizedExperiments() { + StudyQuery query = new StudyQuery(); + ProgramUser collaborator = new ProgramUser(); + UUID programUserRoleId = UUID.randomUUID(); + collaborator.setId(programUserRoleId); + List authorizedExperimentIds = List.of(UUID.randomUUID(), UUID.randomUUID()); + List studies = List.of(new BrAPIStudy().studyName("Authorized Environment")); + + when(programUserService.getIfExperimentalCollaborator(programId, userId)) + .thenReturn(Optional.of(collaborator)); + when(experimentalCollaboratorService.getAuthorizedExperimentIds(programUserRoleId)) + .thenReturn(authorizedExperimentIds); + BrAPIStudyListResponse brAPIResponse = studyResponse(studies, 0, 50, 1); + when(studyService.searchStudies(same(program), same(authorizedExperimentIds), same(query))) + .thenReturn(brAPIResponse); + + HttpResponse>> response = controller.getStudies(programId, query); + + assertEquals(HttpStatus.OK, response.getStatus()); + assertEquals(studies, response.body().getResult().getData()); + verify(studyService).searchStudies(same(program), same(authorizedExperimentIds), same(query)); + verify(studyService, never()).searchStudies(any(Program.class), any(StudyQuery.class)); + } + + @Test + void getStudiesReturnsEmptyResponseWithoutCallingProdServerWhenCollaboratorHasNoAuthorizedExperiments() { + StudyQuery query = new StudyQuery(); + ProgramUser collaborator = new ProgramUser(); + UUID programUserRoleId = UUID.randomUUID(); + collaborator.setId(programUserRoleId); + + when(programUserService.getIfExperimentalCollaborator(programId, userId)) + .thenReturn(Optional.of(collaborator)); + when(experimentalCollaboratorService.getAuthorizedExperimentIds(programUserRoleId)) + .thenReturn(List.of()); + + HttpResponse>> response = controller.getStudies(programId, query); + + assertEquals(HttpStatus.OK, response.getStatus()); + assertTrue(response.body().getResult().getData().isEmpty()); + assertEquals(0, response.body().getMetadata().getPagination().getTotalCount()); + assertEquals(0, response.body().getMetadata().getPagination().getTotalPages()); + verifyNoInteractions(studyService); + } + + private BrAPIStudyListResponse studyResponse(List studies, + int currentPage, + int pageSize, + int totalCount) { + BrAPIPagination pagination = mock(BrAPIPagination.class); + when(pagination.getCurrentPage()).thenReturn(currentPage); + when(pagination.getPageSize()).thenReturn(pageSize); + when(pagination.getTotalCount()).thenReturn(totalCount); + + return new BrAPIStudyListResponse() + .metadata(new BrAPIMetadata().pagination(pagination)) + .result(new BrAPIStudyListResponseResult().data(studies)); + } +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java index 936bd401c..756d66db3 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPITestUtils.java @@ -30,6 +30,7 @@ import io.micronaut.http.netty.cookies.NettyCookie; import io.reactivex.Flowable; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.breedinginsight.TestUtils; import org.breedinginsight.api.auth.AuthenticatedUser; @@ -37,10 +38,12 @@ import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.brapps.importer.ImportTestUtils; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.SpeciesDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.*; @@ -56,6 +59,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.util.*; +import java.util.stream.Collectors; import static io.micronaut.http.HttpRequest.GET; @@ -75,6 +79,10 @@ public class BrAPITestUtils { private OntologyService ontologyService; @Inject private BrAPIGermplasmDAO germplasmDAO; + @Inject + private BrAPITrialService brAPITrialService; + @Inject + ProgramDAO programDAO; @Inject @Client("/${micronaut.bi.api.version}") @@ -116,6 +124,7 @@ public Tuple2> setupTestProgram(DSLContext brAPIDslContext .key("TEST") .build(); Program program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + String brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); // Add Program Administrator user. dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); @@ -147,7 +156,7 @@ public Tuple2> setupTestProgram(DSLContext brAPIDslContext } // Add germplasm to program - List germplasm = createGermplasm(1, BRAPI_REFERENCE_SOURCE); + List germplasm = createGermplasm(1, BRAPI_REFERENCE_SOURCE, brapiProgramDbId); BrAPIExternalReference newReference = new BrAPIExternalReference(); newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); newReference.setReferenceId(program.getId().toString()); @@ -184,12 +193,6 @@ public Tuple2> setupTestProgram(DSLContext brAPIDslContext program, mappingId, newExperimentWorkflowId); - String experiment1Id = importResult - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(0).getAsJsonObject() - .get("trial").getAsJsonObject() - .get("id").getAsString(); // Import test experiment, environments, and any observations importResult = importTestUtils.uploadAndFetchWorkflow( @@ -200,12 +203,15 @@ public Tuple2> setupTestProgram(DSLContext brAPIDslContext program, mappingId, newExperimentWorkflowId); - String experiment2Id = importResult - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(0).getAsJsonObject() - .get("trial").getAsJsonObject() - .get("id").getAsString(); + + List trials = brAPITrialService.getTrialsByProgramId(program.getId()) + .stream() + // Ensure trial with title xyz is always second + .sorted(Comparator.comparing(BrAPITrial::getTrialName)) + .collect(Collectors.toList()); + + String experiment1Id = trials.get(0).getTrialDbId(); + String experiment2Id = trials.get(1).getTrialDbId(); // Refetch collaborator, since we updated program_user_roles. collaborator = userDAO.getUserByOAuthId(TestTokenValidator.OTHER_TEST_USER_ORCID).orElseThrow(Exception::new); @@ -246,7 +252,7 @@ public String getEnvId(JsonObject result, int index) { .get("referenceId").getAsString(); } - public List createGermplasm(int numToCreate, String referenceSource) { + public List createGermplasm(int numToCreate, String referenceSource, String brapiProgramDbId) { List germplasm = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { String gid = ""+(i+1); @@ -265,6 +271,7 @@ public List createGermplasm(int numToCreate, String referenceSou testReference.setReferenceID(UUID.randomUUID().toString()); externalRef.add(testReference); testGermplasm.setExternalReferences(externalRef); + testGermplasm.setProgramDbId(brapiProgramDbId); germplasm.add(testGermplasm); } diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPITrialsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPITrialsControllerIntegrationTest.java index dafa9b7a9..17888d69e 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPITrialsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPITrialsControllerIntegrationTest.java @@ -28,6 +28,7 @@ import io.reactivex.Flowable; import org.brapi.v2.model.core.BrAPITrial; import org.breedinginsight.BrAPITest; +import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.*; import org.breedinginsight.utilities.Utilities; @@ -59,6 +60,9 @@ public class BrAPITrialsControllerIntegrationTest extends BrAPITest { @Inject private BrAPITestUtils brAPITestUtils; + @Inject + private BrAPITrialService brAPIITrialService; + private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .create(); @@ -98,16 +102,17 @@ public void testGetTrials() { // Check names and experimentIds (the BI-assigned UUID stored as an xref). List expNames = new ArrayList<>(); - List expIds = new ArrayList<>(); + List trialExRefIds = new ArrayList<>(); for (BrAPITrial trial : trials) { String experimentId = Utilities.getExternalReference(trial.getExternalReferences(), source).get().getReferenceId(); - expIds.add(experimentId); + trialExRefIds.add(experimentId); expNames.add(trial.getTrialName()); } assert(expNames.contains("xyz")); assert(expNames.contains("Test Exp")); - assert(expIds.contains(experiment1Id)); - assert(expIds.contains(experiment2Id)); + // Verify two different bi-api ex ref ids were generated for each trial + assertEquals(2 , trialExRefIds.size()); + assertNotEquals(trialExRefIds.get(0), trialExRefIds.get(1)); } /** @@ -129,9 +134,7 @@ public void testGetTrialsAsExperimentalCollaborator() { assertEquals(1, trials.size()); BrAPITrial trial = gson.fromJson(trials.get(0).getAsJsonObject(), BrAPITrial.class); assertEquals("xyz", trial.getTrialName()); - String source = Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS); - String experimentId = Utilities.getExternalReference(trial.getExternalReferences(), source).get().getReferenceId(); - assertEquals(this.experiment2Id, experimentId); + assertEquals(this.experiment2Id, trial.getTrialDbId()); } } diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java index 9fc0aac90..9f4419bee 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java @@ -31,6 +31,7 @@ import io.reactivex.Flowable; import lombok.SneakyThrows; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationVariable; import org.breedinginsight.BrAPITest; @@ -40,10 +41,14 @@ import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; +import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.brapps.importer.ImportTestUtils; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.SpeciesDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.*; @@ -88,6 +93,12 @@ public class BrAPIV2ObservationVariableControllerIntegrationTest extends BrAPITe private OntologyService ontologyService; @Inject private BrAPIGermplasmDAO germplasmDAO; + @Inject + ProgramDAO programDAO; + @Inject + private BrAPITrialService brAPITrialService; + @Inject + private BrAPIStudyDAO brAPIStudyDAO; @Inject @Client("/${micronaut.bi.api.version}") @@ -129,6 +140,7 @@ void setup() throws Exception { .key("TEST") .build(); program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + String brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); @@ -149,14 +161,19 @@ void setup() throws Exception { traits = createTraits(4); AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); try { - ontologyService.createTraits(program.getId(), traits, user, false); + // Re-assign to get obtain all brapi related data as well + traits = ontologyService.createTraits(program.getId(), traits, user, false); } catch (ValidatorException e) { System.err.println(e.getErrors()); throw e; } + // Re-assignment returns the data in an indeterminate order. Sort on variable name, which should be determinate. + // Tests below depend on the order being correct. + traits.sort(Comparator.comparing(Trait::getObservationVariableName)); + // Add germplasm to program - List germplasm = createGermplasm(1); + List germplasm = createGermplasm(1, brapiProgramDbId); BrAPIExternalReference newReference = new BrAPIExternalReference(); newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); newReference.setReferenceId(program.getId().toString()); @@ -187,7 +204,7 @@ void setup() throws Exception { rows.add(row2); // Import test experiment, environments, and any observations - JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( + importTestUtils.uploadAndFetchWorkflow( writeDataToFile(rows, expTraits), null, true, @@ -195,15 +212,17 @@ void setup() throws Exception { program, mappingId, newExperimentWorkflowId); - experimentId = importResult - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(0).getAsJsonObject() - .get("trial").getAsJsonObject() - .get("id").getAsString(); - // Add environmentIds. - envIds.add(getEnvId(importResult, 0)); - envIds.add(getEnvId(importResult, 1)); + + BrAPITrial trial = brAPITrialService.getTrialsByProgramId(program.getId()) + .stream() + .findFirst() + .orElseThrow(); + + experimentId = trial.getTrialDbId(); + + envIds.clear(); + brAPIStudyDAO.getStudies(program.getId()) + .forEach(study -> envIds.add(study.getStudyDbId())); } @Test @@ -291,7 +310,7 @@ public void testGetVariablesByExpId() { @Test public void testGetVariableById() { Flowable> call = client.exchange( - GET(String.format("/programs/%s/brapi/v2/variables/%s", program.getId(), traits.get(0).getId())) + GET(String.format("/programs/%s/brapi/v2/variables/%s", program.getId(), traits.get(0).getObservationVariableDbId())) .bearerAuth("test-registered-user"), String.class ); @@ -302,7 +321,7 @@ public void testGetVariableById() { JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); JsonObject variable = responseObj.getAsJsonObject("result"); assertNotNull(variable); - assertEquals(traits.get(0).getId().toString(), variable.get("observationVariableDbId").getAsString()); + assertEquals(traits.get(0).getObservationVariableDbId(), variable.get("observationVariableDbId").getAsString()); assertEquals(traits.get(0).getObservationVariableName(), variable.get("observationVariableName").getAsString()); JsonObject trait = variable.getAsJsonObject("trait"); @@ -321,6 +340,25 @@ public void testGetVariableById() { assertEquals(traits.get(0).getScale().getValidValueMax(), scale.getAsJsonObject("validValues").get("max").getAsInt()); } + @Test + public void testGetVariablesByStudyDbId() { + Flowable> call = client.exchange( + GET(String.format( + "/programs/%s/brapi/v2/variables?studyDbId=%s", + program.getId(), envIds.get(0))) + .bearerAuth("test-registered-user"), String.class); + + HttpResponse response = call.blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + + JsonArray variables = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + + assertEquals(2, variables.size()); + } + private File writeDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); @@ -339,19 +377,7 @@ private File writeDataToFile(List> data, List traits) return file; } - private String getEnvId(JsonObject result, int index) { - return result - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(index).getAsJsonObject() - .get("study").getAsJsonObject() - .get("brAPIObject").getAsJsonObject() - .get("externalReferences").getAsJsonArray() - .get(2).getAsJsonObject() - .get("referenceId").getAsString(); - } - - private List createGermplasm(int numToCreate) { + private List createGermplasm(int numToCreate, String brapiProgramDbId) { List germplasm = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { String gid = ""+(i+1); @@ -370,6 +396,7 @@ private List createGermplasm(int numToCreate) { testReference.setReferenceId(UUID.randomUUID().toString()); externalRef.add(testReference); testGermplasm.setExternalReferences(externalRef); + testGermplasm.setProgramDbId(brapiProgramDbId); germplasm.add(testGermplasm); } @@ -382,6 +409,7 @@ private List createTraits(int numToCreate) { String varName = "tt_test_" + (i + 1); traits.add(Trait.builder() .observationVariableName(varName) + .observationVariableDbId(UUID.randomUUID().toString()) .fullName(varName) .entity("Plant " + i) .attribute("height " + i) diff --git a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java index bcd796d30..c5a96f7bc 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java @@ -411,6 +411,7 @@ public void searchGermplasmOneMatchOneNoMatch() { @Test @SneakyThrows public void filterGermplasmCreatedDateSuccess() { + // TODO: This test is going to continue to fail until date data is migrated to the proper format [BI-3057] // Case 1: createdDate="/". // If createdDate is filtered in the default format, "dd/MM/yyyy HH:mm:ss", "/" will match all, test fails. // If createdDate is filtered after formatting with "yyyy-MM-dd", "/" will match none, test succeeds. diff --git a/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java index d958d0b44..c0b3dd029 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/SubEntityDatasetLockIntegrationTest.java @@ -1,7 +1,6 @@ package org.breedinginsight.brapi.v2; import com.google.gson.*; -import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpRequest; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; @@ -10,7 +9,10 @@ import io.micronaut.http.client.annotation.Client; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; +import org.brapi.v2.model.core.BrAPITrial; import org.breedinginsight.BrAPITest; +import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; +import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.model.Program; import org.junit.jupiter.api.*; @@ -20,7 +22,6 @@ import java.util.Objects; import java.util.UUID; import java.util.concurrent.*; -import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; @@ -34,6 +35,9 @@ public class SubEntityDatasetLockIntegrationTest extends BrAPITest { @Inject private BrAPITestUtils brAPITestUtils; + @Inject + BrAPITrialService brAPIITrialService; + @Inject @Client("/${micronaut.bi.api.version}") private RxHttpClient client; @@ -46,7 +50,10 @@ public class SubEntityDatasetLockIntegrationTest extends BrAPITest { void setup() throws Exception { var setup = brAPITestUtils.setupTestProgram(super.getBrapiDsl(), gson); program = setup.getV1(); - experimentId = setup.getV2().get(0); + experimentId = brAPIITrialService.getTrialsByProgramId(program.getId()).stream() + .map(BrAPITrial::getTrialDbId) + .findFirst() + .orElseThrow(); } @Test diff --git a/src/test/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAOUnitTest.java b/src/test/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAOUnitTest.java new file mode 100644 index 000000000..038fdbfaf --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/dao/BrAPIGermplasmDAOUnitTest.java @@ -0,0 +1,111 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapi.v2.dao; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.reactivex.functions.Function; +import io.reactivex.functions.Function3; +import lombok.SneakyThrows; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.client.v2.model.queryParams.germplasm.GermplasmQueryParams; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.germ.request.BrAPIGermplasmSearchRequest; +import org.breedinginsight.brapps.importer.daos.ImportDAO; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.util.List; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +public class BrAPIGermplasmDAOUnitTest { + + private static final String BRAPI_PROGRAM_DB_ID = "brapi-program-1"; + + private BrAPIGermplasmDAO germplasmDAO; + private BrAPIDAOUtil brAPIDAOUtil; + private Program program; + + @BeforeEach + @SneakyThrows + void setup() { + UUID programId = UUID.randomUUID(); + + program = new Program(); + program.setId(programId); + program.setKey("TEST"); + + ProgramDAO programDAO = mock(ProgramDAO.class); + brAPIDAOUtil = mock(BrAPIDAOUtil.class); + + when(programDAO.fetchOneById(programId)).thenReturn(program); + when(programDAO.getCoreClient(programId)).thenReturn(mock(BrAPIClient.class)); + when(brAPIDAOUtil.getBrAPIProgramDbId(programId)).thenReturn(BRAPI_PROGRAM_DB_ID); + + germplasmDAO = new BrAPIGermplasmDAO(programDAO, mock(ImportDAO.class), + brAPIDAOUtil, + new BrAPIEndpointProvider(), + 65000, + new ObjectMapper(), + mock(BrAPIListDAO.class) + ); + } + + @Test + @SneakyThrows + void getGermplasmByRawNameUsesBrAPISearch() { + List germplasmNames = List.of("Germplasm A [TEST-1]", "Germplasm B [TEST-2]"); + + BrAPIGermplasm germplasm = new BrAPIGermplasm(); + germplasm.setGermplasmName("Germplasm A [TEST-1]"); + germplasm.setDefaultDisplayName("Germplasm A"); + germplasm.setAccessionNumber("1"); + when(brAPIDAOUtil.search(any(Function.class), any(Function3.class), any(BrAPIGermplasmSearchRequest.class))).thenReturn(List.of(germplasm)); + + List result = germplasmDAO.getGermplasmByRawName(germplasmNames, program.getId()); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(BrAPIGermplasmSearchRequest.class); + + verify(brAPIDAOUtil).search(any(Function.class), any(Function3.class), requestCaptor.capture()); + verify(brAPIDAOUtil, never()).get(any(Function.class), any(GermplasmQueryParams.class)); + + BrAPIGermplasmSearchRequest request = requestCaptor.getValue(); + + assertEquals(List.of(BRAPI_PROGRAM_DB_ID), request.getProgramDbIds()); + assertEquals(germplasmNames, request.getGermplasmNames()); + + assertEquals(1, result.size()); + assertEquals("Germplasm A", result.get(0).getGermplasmName()); + } + + @Test + @SneakyThrows + void getGermplasmByRawNameWithEmptyNamesDoesNotSearch() { + assertTrue(germplasmDAO.getGermplasmByRawName(List.of(), program.getId()).isEmpty()); + + verify(brAPIDAOUtil, never()).search(any(Function.class), any(Function3.class), any(BrAPIGermplasmSearchRequest.class)); + } +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAOUnitTest.java b/src/test/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAOUnitTest.java new file mode 100644 index 000000000..ab4cbd794 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAOUnitTest.java @@ -0,0 +1,274 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapi.v2.dao; + +import io.reactivex.functions.Function; +import io.reactivex.functions.Function4; +import lombok.SneakyThrows; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.v2.model.core.BrAPIProgram; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.request.BrAPIObservationSearchRequest; +import org.breedinginsight.brapps.importer.daos.ImportDAO; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.TraitService; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.util.List; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; +import org.brapi.v2.model.pheno.response.BrAPIObservationListResponse; +import org.brapi.v2.model.pheno.response.BrAPIObservationListResponseResult; +import java.util.HashMap; + +public class BrAPIObservationDAOUnitTest { + + private static final String BRAPI_PROGRAM_DB_ID = "brapi-program-1"; + + private BrAPIObservationDAO observationDAO; + private BrAPIObservationUnitDAO observationUnitDAO; + private BrAPIDAOUtil brAPIDAOUtil; + private Program program; + private TraitService traitService; + + @BeforeEach + void setup() { + UUID programId = UUID.randomUUID(); + + BrAPIProgram brapiProgram = new BrAPIProgram().programDbId(BRAPI_PROGRAM_DB_ID); + + program = new Program(); + program.setId(programId); + program.setKey("TEST"); + program.setBrapiProgram(brapiProgram); + + ProgramDAO programDAO = mock(ProgramDAO.class); + observationUnitDAO = mock(BrAPIObservationUnitDAO.class); + brAPIDAOUtil = mock(BrAPIDAOUtil.class); + traitService = mock(TraitService.class); + + when(programDAO.getCoreClient(programId)).thenReturn(mock(BrAPIClient.class)); + + observationDAO = new BrAPIObservationDAO( + programDAO, + mock(ImportDAO.class), + observationUnitDAO, + brAPIDAOUtil, + new BrAPIEndpointProvider(), + 1000, + traitService + ); + } + + @Test + @SneakyThrows + void getObservationsByDbIdsUsesObservationSearch() { + List observationDbIds = List.of("observation-1", "observation-2"); + + BrAPIObservation observation = new BrAPIObservation().observationDbId("observation-1"); + + when(brAPIDAOUtil.search(any(Function.class), any(Function4.class), any(BrAPIObservationSearchRequest.class))).thenReturn(List.of(observation)); + + List result = observationDAO.getObservationsByDbIds(observationDbIds, program); + + assertEquals(List.of(observation), result); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(BrAPIObservationSearchRequest.class); + + verify(brAPIDAOUtil).search(any(Function.class), any(Function4.class), requestCaptor.capture()); + + BrAPIObservationSearchRequest request = requestCaptor.getValue(); + + assertEquals(List.of(BRAPI_PROGRAM_DB_ID), request.getProgramDbIds()); + + assertEquals(observationDbIds, request.getObservationDbIds()); + } + + @Test + @SneakyThrows + void getObservationsByTrialDbIdUsesObservationSearch() { + String trialDbId = "trial-1"; + String observationUnitDbId = "ou-1"; + + BrAPIObservationUnit observationUnit = new BrAPIObservationUnit().observationUnitDbId(observationUnitDbId); + + BrAPIObservation observation = new BrAPIObservation().observationDbId("observation-1").observationUnitDbId(observationUnitDbId); + + when(observationUnitDAO.getObservationUnitsForTrialDbIds(program.getId(), List.of(trialDbId))).thenReturn(List.of(observationUnit)); + + when(brAPIDAOUtil.search(any(Function.class),any(Function4.class),any(BrAPIObservationSearchRequest.class))).thenReturn(List.of(observation)); + + List result = observationDAO.getObservationsByTrialDbId(List.of(trialDbId), program); + + assertEquals(List.of(observation), result); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(BrAPIObservationSearchRequest.class); + + verify(brAPIDAOUtil).search(any(Function.class),any(Function4.class),requestCaptor.capture()); + + BrAPIObservationSearchRequest request = requestCaptor.getValue(); + + assertEquals(List.of(BRAPI_PROGRAM_DB_ID), request.getProgramDbIds()); + + assertEquals(List.of(observationUnitDbId), request.getObservationUnitDbIds()); + } + + @Test + @SneakyThrows + void getObservationsByTrialDbIdReturnsEmptyWhenNoUnitsExist() { + String trialDbId = "trial-without-units"; + + when(observationUnitDAO.getObservationUnitsForTrialDbIds(program.getId(), List.of(trialDbId))).thenReturn(List.of()); + + List result = observationDAO.getObservationsByTrialDbId(List.of(trialDbId), program); + + assertTrue(result.isEmpty()); + + verify(brAPIDAOUtil, never()).search(any(Function.class),any(Function4.class),any(BrAPIObservationSearchRequest.class)); + } + + @Test + @SneakyThrows + void getObservationsByObservationUnitsUsesObservationSearch() { + List observationUnitDbIds = List.of("ou-1", "ou-2"); + + when(brAPIDAOUtil.search(any(Function.class),any(Function4.class),any(BrAPIObservationSearchRequest.class))).thenReturn(List.of()); + + observationDAO.getObservationsByObservationUnits(observationUnitDbIds, program); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(BrAPIObservationSearchRequest.class); + + verify(brAPIDAOUtil).search(any(Function.class),any(Function4.class),requestCaptor.capture()); + + BrAPIObservationSearchRequest request = requestCaptor.getValue(); + + assertEquals(List.of(BRAPI_PROGRAM_DB_ID), request.getProgramDbIds()); + assertEquals(observationUnitDbIds, request.getObservationUnitDbIds()); + } + + @Test + @SneakyThrows + void getObservationsByObservationUnitsAndVariablesUsesBothFilters() { + List observationUnitDbIds = List.of("ou-1", "ou-2"); + List observationVariableDbIds = List.of("variable-1", "variable-2"); + + when(brAPIDAOUtil.search(any(Function.class), any(Function4.class),any(BrAPIObservationSearchRequest.class))).thenReturn(List.of()); + + observationDAO.getObservationsByObservationUnitsAndVariables(observationUnitDbIds, observationVariableDbIds, program); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(BrAPIObservationSearchRequest.class); + + verify(brAPIDAOUtil).search(any(Function.class),any(Function4.class),requestCaptor.capture()); + + BrAPIObservationSearchRequest request = requestCaptor.getValue(); + + assertEquals(List.of(BRAPI_PROGRAM_DB_ID), request.getProgramDbIds()); + assertEquals(observationUnitDbIds, request.getObservationUnitDbIds()); + assertEquals(observationVariableDbIds, request.getObservationVariableDbIds()); + } + + @Test + @SneakyThrows + void observationSearchIsSkippedForEmptyFilters() { + assertTrue(observationDAO.getObservationsByTrialDbId(List.of(), program).isEmpty()); + + assertTrue(observationDAO.getObservationsByDbIds(List.of(),program).isEmpty()); + + assertTrue(observationDAO.getObservationsByObservationUnits(List.of(), program).isEmpty()); + + assertTrue(observationDAO.getObservationsByObservationUnitsAndVariables(List.of("ou-1"), List.of(), program).isEmpty()); + + verifyNoInteractions(observationUnitDAO); + verify(brAPIDAOUtil, never()).search(any(Function.class),any(Function4.class),any(BrAPIObservationSearchRequest.class)); + } + + @Test + @SneakyThrows + void getObservationsByFiltersUsesDefaultsWithoutStudyFilter() { + List observations = List.of(); + + BrAPIObservationListResponse brAPIResponse = new BrAPIObservationListResponse().result(new BrAPIObservationListResponseResult().data(observations)); + + when(brAPIDAOUtil.simpleSearch(any(Function.class), any(BrAPIObservationSearchRequest.class))).thenReturn(brAPIResponse); + + doReturn(observations).when(brAPIDAOUtil).getListResult(brAPIResponse); + + when(traitService.getIdsByObservationVariableDbIds(program.getId(), List.of())).thenReturn(new HashMap<>()); + + observationDAO.getObservationsByFilters(program, null, null, null); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(BrAPIObservationSearchRequest.class); + + verify(brAPIDAOUtil).simpleSearch(any(Function.class), requestCaptor.capture()); + + BrAPIObservationSearchRequest request = requestCaptor.getValue(); + + assertEquals(List.of(BRAPI_PROGRAM_DB_ID), request.getProgramDbIds()); + assertEquals(null, request.getStudyDbIds()); + assertEquals(0, request.getPage()); + assertEquals(1000, request.getPageSize()); + } + + @Test + @SneakyThrows + void getObservationsByFiltersUsesBrAPISearchPagination() { + String brapiVariableDbId = "brapi-variable-1"; + String traitId = UUID.randomUUID().toString(); + + BrAPIObservation observation = new BrAPIObservation() + .observationDbId("observation-1") + .studyDbId("study-1") + .observationVariableDbId(brapiVariableDbId); + + List brapiObservations = List.of(observation); + BrAPIObservationListResponse brAPIResponse = new BrAPIObservationListResponse().result(new BrAPIObservationListResponseResult().data(brapiObservations)); + + when(brAPIDAOUtil.simpleSearch(any(Function.class), any(BrAPIObservationSearchRequest.class))).thenReturn(brAPIResponse); + + doReturn(brapiObservations).when(brAPIDAOUtil).getListResult(brAPIResponse); + + HashMap traitIdsByObservationVariableDbId = new HashMap<>(); + traitIdsByObservationVariableDbId.put(brapiVariableDbId, traitId); + + when(traitService.getIdsByObservationVariableDbIds(program.getId(), List.of(brapiVariableDbId))) + .thenReturn(traitIdsByObservationVariableDbId); + + BrAPIObservationListResponse result = observationDAO.getObservationsByFilters(program, "study-1", 2, 25); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(BrAPIObservationSearchRequest.class); + + verify(brAPIDAOUtil).simpleSearch(any(Function.class), requestCaptor.capture()); + + BrAPIObservationSearchRequest request = requestCaptor.getValue(); + + assertEquals(List.of(BRAPI_PROGRAM_DB_ID), request.getProgramDbIds()); + assertEquals(List.of("study-1"), request.getStudyDbIds()); + assertEquals(2, request.getPage()); + assertEquals(25, request.getPageSize()); + assertEquals(brapiVariableDbId, result.getResult().getData().get(0).getObservationVariableDbId()); + } +} \ No newline at end of file diff --git a/src/test/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAOUnitTest.java b/src/test/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAOUnitTest.java new file mode 100644 index 000000000..6e70bcfd5 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAOUnitTest.java @@ -0,0 +1,142 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapi.v2.dao; + +import io.reactivex.functions.Function; +import io.reactivex.functions.Function3; +import lombok.SneakyThrows; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.v2.model.core.BrAPIProgram; +import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; +import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; +import org.breedinginsight.brapps.importer.daos.ImportDAO; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.util.List; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +public class BrAPIObservationUnitDAOUnitTest { + + private static final String BRAPI_PROGRAM_DB_ID = "brapi-program-1"; + + private BrAPIObservationUnitDAO observationUnitDAO; + private ProgramDAO programDAO; + private BrAPIDAOUtil brAPIDAOUtil; + private Program program; + + @BeforeEach + @SneakyThrows + void setup() { + UUID programId = UUID.randomUUID(); + + BrAPIProgram brapiProgram = new BrAPIProgram().programDbId(BRAPI_PROGRAM_DB_ID); + + program = new Program(); + program.setId(programId); + program.setKey("TEST"); + program.setBrapiProgram(brapiProgram); + + programDAO = mock(ProgramDAO.class); + brAPIDAOUtil = mock(BrAPIDAOUtil.class); + BrAPIGermplasmService germplasmService = mock(BrAPIGermplasmService.class); + + when(programDAO.get(programId)).thenReturn(List.of(program)); + when(programDAO.getProgramBrAPI(program)).thenReturn(brapiProgram); + when(programDAO.getCoreClient(programId)).thenReturn(mock(BrAPIClient.class)); + when(germplasmService.getGermplasm(programId)).thenReturn(List.of()); + + when(brAPIDAOUtil.search(any(Function.class), any(Function3.class), any(BrAPIObservationUnitSearchRequest.class))).thenReturn(List.of()); + + observationUnitDAO = new BrAPIObservationUnitDAO(programDAO, mock(ImportDAO.class), + brAPIDAOUtil, + new BrAPIEndpointProvider(), germplasmService, mock(ProgramService.class), + "test-reference-source", + 1000 + ); + } + + @Test + @SneakyThrows + void getObservationUnitsForStudyDbIdUsesBrAPISearch() { + String studyDbId = "study-1"; + + observationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(BrAPIObservationUnitSearchRequest.class); + + verify(brAPIDAOUtil).search(any(Function.class), any(Function3.class), requestCaptor.capture()); + + BrAPIObservationUnitSearchRequest request = requestCaptor.getValue(); + + assertEquals(List.of(BRAPI_PROGRAM_DB_ID), request.getProgramDbIds()); + assertEquals(List.of(studyDbId), request.getStudyDbIds()); + } + + @Test + @SneakyThrows + void getObservationUnitsForTrialDbIdsUsesBrAPISearch() { + List trialDbIds = List.of("trial-1", "trial-2"); + + observationUnitDAO.getObservationUnitsForTrialDbIds(program.getId(), trialDbIds); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(BrAPIObservationUnitSearchRequest.class); + + verify(brAPIDAOUtil).search(any(Function.class), any(Function3.class), requestCaptor.capture()); + + BrAPIObservationUnitSearchRequest request = requestCaptor.getValue(); + + assertEquals(List.of(BRAPI_PROGRAM_DB_ID), request.getProgramDbIds()); + assertEquals(trialDbIds, request.getTrialDbIds()); + } + + @Test + @SneakyThrows + void getObservationUnitsForTrialDbIdUsesBrAPISearch() { + String trialDbId = "trial-1"; + + observationUnitDAO.getObservationUnitsForTrialDbId(program.getId(), trialDbId); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(BrAPIObservationUnitSearchRequest.class); + + verify(brAPIDAOUtil).search(any(Function.class), any(Function3.class), requestCaptor.capture()); + + BrAPIObservationUnitSearchRequest request = requestCaptor.getValue(); + + assertEquals(List.of(BRAPI_PROGRAM_DB_ID), request.getProgramDbIds()); + assertEquals(List.of(trialDbId), request.getTrialDbIds()); + } + + @Test + @SneakyThrows + void getObservationUnitsForEmptyTrialDbIdsDoesNotSearch() { + assertTrue(observationUnitDAO.getObservationUnitsForTrialDbIds(program.getId(), List.of()).isEmpty()); + + verify(brAPIDAOUtil, never()).search(any(Function.class), any(Function3.class), any(BrAPIObservationUnitSearchRequest.class)); + } +} \ No newline at end of file diff --git a/src/test/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAOUnitTest.java b/src/test/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAOUnitTest.java new file mode 100644 index 000000000..58806202e --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAOUnitTest.java @@ -0,0 +1,263 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.breedinginsight.brapi.v2.dao; + +import io.reactivex.functions.Consumer; +import io.reactivex.functions.Function; +import io.reactivex.functions.Function3; +import lombok.SneakyThrows; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.client.v2.model.queryParams.core.StudyQueryParams; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.BrAPIMetadata; +import org.brapi.v2.model.BrAPIPagination; +import org.brapi.v2.model.BrAPISortOrder; +import org.brapi.v2.model.core.BrAPIProgram; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.request.BrAPIStudySearchRequest; +import org.brapi.v2.model.core.response.BrAPIStudyListResponse; +import org.brapi.v2.model.core.response.BrAPIStudyListResponseResult; +import org.breedinginsight.api.v1.controller.metadata.SortOrder; +import org.breedinginsight.brapi.v2.model.request.query.StudyQuery; +import org.breedinginsight.brapps.importer.daos.ImportDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.lang.reflect.Field; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +public class BrAPIStudyDAOUnitTest { + + private static final String REFERENCE_SOURCE = "breedinginsight.org"; + + private BrAPIStudyDAO studyDAO; + private ProgramDAO programDAO; + private BrAPIDAOUtil brAPIDAOUtil; + private Program program; + private UUID programId; + private UUID environmentId; + + @BeforeEach + @SneakyThrows + void setup() { + programId = UUID.randomUUID(); + environmentId = UUID.randomUUID(); + + program = new Program(); + program.setId(programId); + program.setKey("TEST"); + BrAPIProgram brapiProgram = new BrAPIProgram() + .programDbId("brapi-program-1"); + program.setBrapiProgram(brapiProgram); + + programDAO = mock(ProgramDAO.class); + brAPIDAOUtil = mock(BrAPIDAOUtil.class); + + when(programDAO.get(programId)).thenReturn(List.of(program)); + when(programDAO.getCoreClient(programId)).thenReturn(mock(BrAPIClient.class)); + when(programDAO.getProgramBrAPI(program)).thenReturn(brapiProgram); + + studyDAO = new BrAPIStudyDAO( + programDAO, + mock(ImportDAO.class), + brAPIDAOUtil, + new BrAPIEndpointProvider() + ); + + Field referenceSource = BrAPIStudyDAO.class.getDeclaredField("referenceSource"); + referenceSource.setAccessible(true); + referenceSource.set(studyDAO, REFERENCE_SOURCE); + + Field brapiMaxPageSize = BrAPIStudyDAO.class.getDeclaredField("brapiMaxPageSize"); + brapiMaxPageSize.setAccessible(true); + brapiMaxPageSize.set(studyDAO, 1000); + } + + @Test + @SneakyThrows + void getStudiesUsesDirectBrAPIGetInsteadOfProgramCache() { + BrAPIStudy study = study(environmentId, "Env1 [TEST-1]"); + + when(brAPIDAOUtil.get(any(Function.class), any(StudyQueryParams.class))) + .thenReturn(List.of(study)); + + List result = studyDAO.getStudies(programId); + + assertEquals(1, result.size()); + assertEquals("Env1", result.get(0).getStudyName()); + ArgumentCaptor queryParamsCaptor = ArgumentCaptor.forClass(StudyQueryParams.class); + verify(brAPIDAOUtil).get(any(Function.class), queryParamsCaptor.capture()); + assertEquals("brapi-program-1", queryParamsCaptor.getValue().programDbId()); + assertEquals(0, queryParamsCaptor.getValue().page()); + assertEquals(1000, queryParamsCaptor.getValue().pageSize()); + } + + @Test + @SneakyThrows + void getStudiesByStudyDbIdUsesDirectBrAPISearch() { + String studyDbId = UUID.randomUUID().toString(); + BrAPIStudy study = new BrAPIStudy().studyDbId(studyDbId).studyName("Env1"); + + when(brAPIDAOUtil.search(any(Function.class), any(Function3.class), any(BrAPIStudySearchRequest.class))) + .thenReturn(List.of(study)); + + List result = studyDAO.getStudiesByStudyDbId( + List.of(studyDbId), program); + + assertEquals(1, result.size()); + assertEquals(studyDbId, result.get(0).getStudyDbId()); + + ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(BrAPIStudySearchRequest.class); + + verify(brAPIDAOUtil).search(any(Function.class), any(Function3.class), requestCaptor.capture()); + + assertEquals(List.of("brapi-program-1"), requestCaptor.getValue().getProgramDbIds()); + assertEquals(List.of(studyDbId), requestCaptor.getValue().getStudyDbIds()); + } + + @Test + @SneakyThrows + void getStudyByDbIdReturnsEmptyWhenBrAPIDoesNotFindStudy() { + when(brAPIDAOUtil.search(any(Function.class), any(Function3.class), any(BrAPIStudySearchRequest.class))) + .thenReturn(List.of()); + + Optional result = studyDAO.getStudyByDbId(UUID.randomUUID().toString(), program); + + assertTrue(result.isEmpty()); + } + + @Test + @SneakyThrows + void brapiStudySearchSendsServerSideQueryAndPreservesPagedResponseOrder() { + UUID authorizedTrialId = UUID.randomUUID(); + StudyQuery studyQuery = new StudyQuery(); + setStudyQueryField(studyQuery, "studyName", "Environment"); + studyQuery.setSortField("studyName"); + studyQuery.setSortOrder(SortOrder.DESC); + studyQuery.setPage(1); + studyQuery.setPageSize(2); + + BrAPIDAOUtil genericSearchParameterUtil = new BrAPIDAOUtil(1, Duration.ofSeconds(1), 50, 50, 1000, mock(ProgramService.class)); + doAnswer(invocation -> { + genericSearchParameterUtil.setGenericSearchParameters( + invocation.getArgument(0), + invocation.getArgument(1)); + return null; + }).when(brAPIDAOUtil).setGenericSearchParameters(any(BrAPIStudySearchRequest.class), same(studyQuery)); + + BrAPIPagination pagination = mock(BrAPIPagination.class); + when(pagination.getCurrentPage()).thenReturn(1); + when(pagination.getPageSize()).thenReturn(2); + when(pagination.getTotalCount()).thenReturn(4); + when(pagination.getTotalPages()).thenReturn(2); + + BrAPIStudyListResponse brAPIResponse = new BrAPIStudyListResponse() + .metadata(new BrAPIMetadata().pagination(pagination)) + .result(new BrAPIStudyListResponseResult()); + List serverOrderedStudies = List.of( + study("2", "Environment 2 [TEST-2]"), + study("1", "Environment 1 [TEST-1]")); + + when(brAPIDAOUtil.simpleSearch(any(Function.class), any(BrAPIStudySearchRequest.class))).thenReturn(brAPIResponse); + doReturn(serverOrderedStudies).when(brAPIDAOUtil).getListResult(brAPIResponse); + + BrAPIStudyListResponse result = studyDAO.brapiStudySearch(program, List.of(authorizedTrialId), studyQuery); + + ArgumentCaptor requestCaptor = + ArgumentCaptor.forClass(BrAPIStudySearchRequest.class); + verify(brAPIDAOUtil).simpleSearch(any(Function.class), requestCaptor.capture()); + + BrAPIStudySearchRequest request = requestCaptor.getValue(); + assertEquals(List.of("brapi-program-1"), request.getProgramDbIds()); + assertEquals(List.of(authorizedTrialId.toString()), request.getTrialDbIds()); + assertEquals(1, request.getPage()); + assertEquals(2, request.getPageSize()); + assertEquals(1, request.getSortBy().size()); + assertEquals("studyName", request.getSortBy().get(0).getSortedOn()); + assertEquals(BrAPISortOrder.DESC, request.getSortBy().get(0).getSortOrder()); + + Map filters = request.getFilterBy().stream() + .collect(Collectors.toMap(filter -> filter.getFilterOn(), filter -> filter.getValue())); + assertEquals(Map.of("studyName", "Environment"), filters); + + assertSame(brAPIResponse, result); + assertSame(pagination, result.getMetadata().getPagination()); + assertEquals(List.of("Environment 2", "Environment 1"), result.getResult().getData().stream() + .map(BrAPIStudy::getStudyName) + .collect(Collectors.toList())); + } + + @Test + @SneakyThrows + void createBrAPIStudiesReturnsDirectBrAPIPostResponse() { + + BrAPIStudy requestedStudy = new BrAPIStudy().studyName("Env1 [TEST-1]"); + List requestedStudies = List.of(requestedStudy); + + BrAPIStudy createdStudy = new BrAPIStudy().studyDbId("study-db-id").studyName("Env1 [TEST-1]"); + ImportUpload upload = mock(ImportUpload.class); + + doReturn(List.of(createdStudy)) + .when(brAPIDAOUtil) + .post(eq(requestedStudies),eq(upload),any(Function.class),any(Consumer.class)); + + List result = studyDAO.createBrAPIStudies(requestedStudies,programId,upload); + + assertEquals(1, result.size()); + assertEquals("study-db-id",result.get(0).getStudyDbId()); + + verify(brAPIDAOUtil).post(eq(requestedStudies),eq(upload),any(Function.class),any(Consumer.class)); + } + + private BrAPIStudy study(UUID environmentId, String studyName) { + return study(environmentId.toString(), studyName); + } + + private BrAPIStudy study(String environmentId, String studyName) { + BrAPIExternalReference studyReference = new BrAPIExternalReference() + .referenceSource(Utilities.generateReferenceSource(REFERENCE_SOURCE, ExternalReferenceSource.STUDIES)) + .referenceId(environmentId); + + return new BrAPIStudy() + .studyName(studyName) + .externalReferences(List.of(studyReference)); + } + + private void setStudyQueryField(StudyQuery studyQuery, String fieldName, String value) throws Exception { + Field field = StudyQuery.class.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(studyQuery, value); + } +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/model/request/query/StudyQueryUnitTest.java b/src/test/java/org/breedinginsight/brapi/v2/model/request/query/StudyQueryUnitTest.java new file mode 100644 index 000000000..71b969db1 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/model/request/query/StudyQueryUnitTest.java @@ -0,0 +1,76 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2.model.request.query; + +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class StudyQueryUnitTest { + + @Test + void exposesAllSupportedStudyFiltersUsingBrAPIColumnNames() throws Exception { + StudyQuery query = new StudyQuery(); + setField(query, "studyType", "Phenotyping"); + setField(query, "locationDbId", "location-1"); + setField(query, "studyCode", "study-code"); + setField(query, "studyPUI", "study-pui"); + setField(query, "commonCropName", "maize"); + setField(query, "trialDbId", "trial-1"); + setField(query, "studyDbId", "study-1"); + setField(query, "studyName", "Environment 1"); + setField(query, "externalReferenceSource", "breedinginsight.org/studies"); + setField(query, "externalReferenceId", "environment-1"); + + assertEquals(Map.ofEntries( + Map.entry("studyType", "Phenotyping"), + Map.entry("locationDbId", "location-1"), + Map.entry("studyCode", "study-code"), + Map.entry("studyPUI", "study-pui"), + Map.entry("commonCropName", "maize"), + Map.entry("trialDbId", "trial-1"), + Map.entry("studyDbId", "study-1"), + Map.entry("studyName", "Environment 1"), + Map.entry("externalReferenceSource", "breedinginsight.org/studies"), + Map.entry("externalReferenceId", "environment-1")), + query.getFilterValuesByBrAPIColumnName()); + } + + @Test + void mapsAllSupportedBiSortFieldsToBrAPIColumnNames() { + assertEquals(Map.ofEntries( + Map.entry("studyType", "studyType"), + Map.entry("locationDbId", "locationDbId"), + Map.entry("studyCode", "studyCode"), + Map.entry("studyPUI", "studyPUI"), + Map.entry("commonCropName", "commonCropName"), + Map.entry("trialDbId", "trialDbId"), + Map.entry("studyDbId", "studyDbId"), + Map.entry("studyName", "studyName")), + new StudyQuery().getBrAPIColumnNamesByBiColumnName()); + } + + private void setField(StudyQuery query, String fieldName, String value) throws Exception { + Field field = StudyQuery.class.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(query, value); + } +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java b/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java index 5a055bdd0..a4afcedb1 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/services/BrAPITrialServiceUnitTest.java @@ -35,6 +35,7 @@ import org.breedinginsight.services.TraitService; import org.breedinginsight.services.lock.DistributedLockService; import org.breedinginsight.utilities.FileUtil; +import org.breedinginsight.utilities.Utilities; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import tech.tablesaw.api.Table; @@ -121,6 +122,12 @@ void setup() { experimentInfo.addProperty(BrAPIAdditionalInfoFields.EXPERIMENT_TYPE, "Phenotyping"); experiment.setAdditionalInfo(experimentInfo); + BrAPIExternalReference externalReference = new BrAPIExternalReference(); + externalReference.setReferenceSource(Utilities.generateReferenceSource(REFERENCE_SOURCE, ExternalReferenceSource.TRIALS)); + externalReference.setReferenceId("11111111-1111-1111-1111-111111111111"); + + experiment.setExternalReferences(List.of(externalReference)); + study = new BrAPIStudy(); study.setStudyDbId("study-1"); study.setStudyName("Environment 1"); @@ -164,7 +171,7 @@ void exportObservationsFetchesSeasonOncePerDistinctSeasonAndWritesEnvYears() thr when(trialDAO.getTrialsByExperimentIds(eq(List.of(UUID.fromString("11111111-1111-1111-1111-111111111111"))), eq(program))) .thenReturn(List.of(experiment)); - when(studyDAO.getStudiesByExperimentID(eq(UUID.fromString("11111111-1111-1111-1111-111111111111")), eq(program))) + when(studyDAO.getStudiesByBrAPITrialExRefId(eq(UUID.fromString("11111111-1111-1111-1111-111111111111")), eq(program))) .thenReturn(List.of(study, secondStudy)); when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); when(seasonDAO.getSeasonById("season-2", program.getId())).thenReturn(secondSeason); @@ -240,6 +247,53 @@ void getDatasetDataThrowsWhenSeasonYearIsNull() throws Exception { assertEquals("Env Year not found for Study DbId = 'study-1'.", exception.getMessage()); } + @Test + void exportObservationsFiltersByStudyDbId() throws Exception { + ExperimentExportQuery params = exportQuery(EXPORT_DATASET_ID); + + setField(params, "environments", "study-1"); + + BrAPIStudy secondStudy = new BrAPIStudy(); + secondStudy.setStudyDbId("study-2"); + secondStudy.setStudyName("Environment 2"); + secondStudy.setLocationName("Location 2"); + secondStudy.setSeasons(List.of("season-2")); + + BrAPIObservationUnit requestedObservationUnit = createObservationUnit("ou-db-1", "plot-1"); + + when(trialDAO.getTrialsByExperimentIds(anyCollection(), eq(program))) + .thenReturn(List.of(experiment)); + + when(studyDAO.getStudiesByBrAPITrialExRefId(any(UUID.class), eq(program))) + .thenReturn(List.of(study, secondStudy)); + + when(observationUnitDAO.getObservationUnitsForDatasetAndEnvs(EXPORT_DATASET_ID, List.of("study-1"), program)) + .thenReturn(List.of(requestedObservationUnit)); + + when(seasonDAO.getSeasonById("season-1", program.getId())).thenReturn(season); + + when(listDAO.getListsByTypeAndExternalRef(any(), eq(program.getId()), any(), any())) + .thenReturn(Collections.emptyList()); + + when(observationDAO.getObservationsByObservationUnits(anyCollection(), eq(program))) + .thenReturn(Collections.emptyList()); + + when(germplasmDAO.getGermplasmsByDBID(anyList(), eq(program.getId()))) + .thenReturn(List.of(germplasm)); + + DownloadFile downloadFile = service.exportObservations(program, UUID.fromString("11111111-1111-1111-1111-111111111111"), params); + + Table exportTable = FileUtil.parseTableFromCsv(new ByteArrayInputStream(downloadFile.getStreamedFile() + .getInputStream() + .readAllBytes())); + + assertEquals(1, exportTable.rowCount()); + + assertEquals("Environment 1", exportTable.stringColumn(Columns.ENV).get(0)); + + verify(observationUnitDAO).getObservationUnitsForDatasetAndEnvs(EXPORT_DATASET_ID, List.of("study-1"), program); + } + private ExperimentExportQuery exportQuery(String datasetId) throws Exception { ExperimentExportQuery params = new ExperimentExportQuery(); setField(params, "datasetId", datasetId); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 802bf6889..30240f5fb 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -204,7 +204,7 @@ public void appendExperimentWithObsVarFromPriorDataset() { // Create a plot-level dataset that includes observation variable tt_test_1 List traits = importTestUtils.createTraits(1); - Program program = createProgram("Append Exp with Prior Observations Vars", "EXPPRI", "EXPPRI", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("Append Exp with Prior Observations Vars", "EXPPRI", "EXPPRI", BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -310,7 +310,7 @@ public void appendExperimentMultipleDatasets() { // Create a plot-level dataset List traits = importTestUtils.createTraits(1); - Program program = createProgram("Append Exp with Multiple Datasets", "MULSET", "MULSET", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("Append Exp with Multiple Datasets", "MULSET", "MULSET", BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -327,12 +327,7 @@ public void appendExperimentMultipleDatasets() { newExp.put(Columns.COLUMN, "1"); JsonObject importResponse = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newExp), null, false, false, null), null, true, client, program, mappingId, newExperimentWorkflowId); - String expId = importResponse - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(0).getAsJsonObject() - .get("trial").getAsJsonObject() - .get("id").getAsString(); + String expId = brAPITrialDAO.getTrialsByName(List.of(newExp.get(Columns.EXP_TITLE).toString()), program).get(0).getTrialDbId(); // Create two sub-entity datasets that have two different sub entity units Flowable> sub1PostCall = client.exchange( @@ -446,7 +441,7 @@ public void appendExperimentMultipleDatasets() { @SneakyThrows public void importNewExpNewLocNoObsSuccess() { log.debug("importNewExpNewLocNoObsSuccess"); - Program program = createProgram("New Exp and Loc", "NEXPL", "NEXPL", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Program program = createProgram("New Exp and Loc", "NEXPL", "NEXPL", BRAPI_REFERENCE_SOURCE, 1, null); Map validRow = new HashMap<>(); validRow.put(Columns.GERMPLASM_GID, "1"); validRow.put(Columns.TEST_CHECK, "T"); @@ -481,7 +476,7 @@ public void importNewExpNewLocNoObsSuccess() { @SneakyThrows public void importNewExpMultiNewEnvSuccess() { log.debug("importNewExpMultiNewEnvSucces"); - Program program = createProgram("New Exp and Multi New Env", "MULENV", "MULENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Program program = createProgram("New Exp and Multi New Env", "MULENV", "MULENV", BRAPI_REFERENCE_SOURCE, 1, null); Map firstEnv = new HashMap<>(); firstEnv.put(Columns.GERMPLASM_GID, "1"); firstEnv.put(Columns.TEST_CHECK, "T"); @@ -541,7 +536,7 @@ public void importNewExpMultiNewEnvSuccess() { @SneakyThrows public void importExistingExpAndEnvErrorMessage() { log.debug("importExistingExpAndEnvErrorMessage"); - Program program = createProgram("New Env Existing Exp", "DUPENV", "DUPENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Program program = createProgram("New Env Existing Exp", "DUPENV", "DUPENV", BRAPI_REFERENCE_SOURCE, 1, null); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -584,7 +579,7 @@ public void importExistingExpAndEnvErrorMessage() { @SneakyThrows public void importNewEnvNoObsSuccess() { log.debug("importNewEnvNoObsSuccess"); - Program program = createProgram("New Env", "NEWENV", "NEWENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Program program = createProgram("New Env", "NEWENV", "NEWENV", BRAPI_REFERENCE_SOURCE, 1, null); Map newEnv = new HashMap<>(); newEnv.put(Columns.GERMPLASM_GID, "1"); @@ -619,7 +614,7 @@ public void importNewEnvNoObsSuccess() { @SneakyThrows public void verifyMissingDataThrowsError(boolean commit) { log.debug("verifyMissingDataThrowsError"); - Program program = createProgram("Missing Req Cols "+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Program program = createProgram("Missing Req Cols "+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 1, null); Map base = new HashMap<>(); base.put(Columns.GERMPLASM_GID, "1"); @@ -690,7 +685,7 @@ public void verifyMissingDataThrowsError(boolean commit) { public void importNewExpWithObsVar() { log.debug("importNewExpWithObsVar"); List traits = importTestUtils.createTraits(1); - Program program = createProgram("New Exp with Observations Vars", "EXPVRR", "EXPVRR", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("New Exp with Observations Vars", "EXPVRR", "EXPVRR", BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -728,7 +723,7 @@ public void importNewExpWithObsVar() { @SneakyThrows public void verifyDiffYearSameEnvThrowsError(boolean commit) { log.debug("verifyDiffYEarSameEnvThrowsError"); - Program program = createProgram("Diff Years "+(commit ? "C" : "P"), "YEARS"+(commit ? "C" : "P"), "YEARS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(2), null); + Program program = createProgram("Diff Years "+(commit ? "C" : "P"), "YEARS"+(commit ? "C" : "P"), "YEARS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 2, null); List> rows = new ArrayList<>(); Map row = new HashMap<>(); @@ -768,7 +763,7 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { @SneakyThrows public void verifyDiffLocSameEnvThrowsError(boolean commit) { log.debug("verifyDiffLocSameEnvThrowsError"); - Program program = createProgram("Diff Locations "+(commit ? "C" : "P"), "LOCS"+(commit ? "C" : "P"), "LOCS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(2), null); + Program program = createProgram("Diff Locations "+(commit ? "C" : "P"), "LOCS"+(commit ? "C" : "P"), "LOCS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 2, null); List> rows = new ArrayList<>(); Map row = new HashMap<>(); @@ -808,7 +803,7 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { public void importNewExpWithObs(boolean commit) { log.debug("importNewExpWithObs"); List traits = importTestUtils.createTraits(1); - Program program = createProgram("New Exp with Observations "+(commit ? "C" : "P"), "NEXOB"+(commit ? "C" : "P"), "NEXOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("New Exp with Observations "+(commit ? "C" : "P"), "NEXOB"+(commit ? "C" : "P"), "NEXOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -849,7 +844,7 @@ public void importNewExpWithObs(boolean commit) { public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { log.debug("verifyFailureImportNewExpWithInvalidObs"); List traits = importTestUtils.createTraits(1); - Program program = createProgram("Invalid Observations "+(commit ? "C" : "P"), "INVOB"+(commit ? "C" : "P"), "INVOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("Invalid Observations "+(commit ? "C" : "P"), "INVOB"+(commit ? "C" : "P"), "INVOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -875,7 +870,7 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { @SneakyThrows public void verifyFailureNewOuExistingEnv(boolean commit) { log.debug("verifyFailureNewOuExistingEnv"); - Program program = createProgram("New OU Existing Env "+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Program program = createProgram("New OU Existing Env "+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 1, null); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -909,7 +904,7 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { public void importNewObsVarExistingOu() { log.debug("importNewObsVarExistingOu"); List traits = importTestUtils.createTraits(2); - Program program = createProgram("New ObsVar Existing OU", "OUVAR", "OUVAR", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("New ObsVar Existing OU", "OUVAR", "OUVAR", BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -931,11 +926,9 @@ public void importNewObsVarExistingOu() { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObsVar = new HashMap<>(); newObsVar.put(Columns.GERMPLASM_GID, "1"); @@ -951,7 +944,7 @@ public void importNewObsVarExistingOu() { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), null, true, client, program, mappingId, appendOverwriteWorkflowId); @@ -975,7 +968,7 @@ public void importNewObsVarExistingOu() { public void importNewObsVarByObsUnitId() { log.debug("importNewObsVarByObsUnitId"); List traits = importTestUtils.createTraits(2); - Program program = createProgram("New ObsVar Referring to OU by ID", "OUVAR", "VAROU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("New ObsVar Referring to OU by ID", "OUVAR", "VAROU", BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -997,14 +990,12 @@ public void importNewObsVarByObsUnitId() { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObsVar = new HashMap<>(); - newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), null, true, client, program, mappingId, appendOverwriteWorkflowId); @@ -1029,7 +1020,7 @@ public void importNewObsVarByObsUnitId() { public void importNewObservationDataByObsUnitId(boolean commit) { log.debug("importNewObservationDataByObsUnitId"); List traits = importTestUtils.createTraits(1); - Program program = createProgram("New Observation Referring to OU by ID"+(commit ? "C" : "P"), "OUDAT"+(commit ? "C" : "P"), "DATOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("New Observation Referring to OU by ID"+(commit ? "C" : "P"), "OUDAT"+(commit ? "C" : "P"), "DATOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -1051,11 +1042,9 @@ public void importNewObservationDataByObsUnitId(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObsVar = new HashMap<>(); newObsVar.put(Columns.GERMPLASM_GID, "1"); @@ -1071,7 +1060,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObsVar.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), null, commit, client, program, mappingId, appendOverwriteWorkflowId); @@ -1105,7 +1094,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { @SneakyThrows public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { List traits = importTestUtils.createTraits(1); - Program program = createProgram("Overwrite Attempt With Blank Obs"+(commit ? "C" : "P"), "NOOP"+(commit ? "C" : "P"), "NOOP"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("Overwrite Attempt With Blank Obs"+(commit ? "C" : "P"), "NOOP"+(commit ? "C" : "P"), "NOOP"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -1128,10 +1117,8 @@ public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); assertRowSaved(newExp, program, traits); @@ -1149,7 +1136,7 @@ public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); // Indicates this is an overwrite. + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); // Indicates this is an overwrite. newObsVar.put(traits.get(0).getObservationVariableName(), ""); // Empty string should be no op. Map requestBody = new HashMap<>(); @@ -1176,7 +1163,7 @@ public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { public void importNewObsExistingOu(boolean commit) { log.debug("importNewObsExistingOu"); List traits = importTestUtils.createTraits(1); - Program program = createProgram("New Obs Existing OU "+(commit ? "C" : "P"), "OUOBS"+(commit ? "C" : "P"), "OUOBS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("New Obs Existing OU "+(commit ? "C" : "P"), "OUOBS"+(commit ? "C" : "P"), "OUOBS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -1197,11 +1184,9 @@ public void importNewObsExistingOu(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObservation = new HashMap<>(); newObservation.put(Columns.GERMPLASM_GID, "1"); @@ -1217,7 +1202,7 @@ public void importNewObsExistingOu(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), null, commit, client, program, mappingId, appendOverwriteWorkflowId); @@ -1243,7 +1228,7 @@ public void importNewObsExistingOu(boolean commit) { public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { log.debug("verifyFailureImportNewObsExistingOuWithExistingObs"); List traits = importTestUtils.createTraits(1); - Program program = createProgram("New Obs Existing Obs "+(commit ? "C" : "P"), "FEXOB"+(commit ? "C" : "P"), "FEXOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("New Obs Existing Obs "+(commit ? "C" : "P"), "FEXOB"+(commit ? "C" : "P"), "FEXOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -1265,11 +1250,9 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObservation = new HashMap<>(); newObservation.put(Columns.GERMPLASM_GID, "1"); @@ -1285,7 +1268,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); uploadAndVerifyWorkflowFailureNonTabular(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); @@ -1302,7 +1285,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { public void importSecondExpAfterFirstExpWithObs() { log.debug("importSecondExpAfterFirstExpWithObs"); List traits = importTestUtils.createTraits(1); - Program program = createProgram("New Exp After First", "NEAF", "NEAF", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("New Exp After First", "NEAF", "NEAF", BRAPI_REFERENCE_SOURCE, 1, traits); Map newExpA = new HashMap<>(); newExpA.put(Columns.GERMPLASM_GID, "1"); newExpA.put(Columns.TEST_CHECK, "T"); @@ -1372,7 +1355,7 @@ public void importSecondExpAfterFirstExpWithObs() { public void importNewObsAfterFirstExpWithObs(boolean commit) { log.debug("importNewObsAfterFirstExpWithObs"); List traits = importTestUtils.createTraits(2); - Program program = createProgram("Exp with additional Uploads "+(commit ? "C" : "P"), "EXAU"+(commit ? "C" : "P"), "EXAU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("Exp with additional Uploads "+(commit ? "C" : "P"), "EXAU"+(commit ? "C" : "P"), "EXAU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -1394,11 +1377,9 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObservation = new HashMap<>(); newObservation.put(Columns.GERMPLASM_GID, "1"); @@ -1414,7 +1395,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); @@ -1446,7 +1427,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { public void importNewObsAfterFirstExpWithObsAndTimestamps() { log.debug("importNewObsAfterFirstExpWithObsAndTimestamps"); List traits = importTestUtils.createTraits(2); - Program program = createProgram("Exp with TS and additional Uploads ", "EXTSAU", "EXTSAU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("Exp with TS and additional Uploads ", "EXTSAU", "EXTSAU", BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -1471,11 +1452,9 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObservation = new HashMap<>(); newObservation.put(Columns.GERMPLASM_GID, "1"); @@ -1491,7 +1470,7 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "1"); @@ -1523,7 +1502,7 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { log.debug("importNewObsAfterFirstExpWithObs_blank"); List traits = importTestUtils.createTraits(3); - Program program = createProgram("Exp with additional Uploads (blank) "+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("Exp with additional Uploads (blank) "+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), "EXAUB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 1, traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -1547,11 +1526,9 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); assertRowSaved(newExp, program, traits); @@ -1569,7 +1546,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObservation.put(traits.get(0).getObservationVariableName(), ""); // This blank value should not overwrite. newObservation.put(traits.get(1).getObservationVariableName(), "3"); // This valid value should overwrite. newObservation.put(traits.get(2).getObservationVariableName(), "4"); // This valid new observation should be appended. @@ -1644,7 +1621,7 @@ private Map assertRowSaved(Map expected, Program Optional trialIdXref = Utilities.getExternalReference(trial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - List studies = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program); + List studies = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program); assertFalse(studies.isEmpty()); BrAPIStudy study = null; for(BrAPIStudy s : studies) { @@ -1882,7 +1859,7 @@ private String yearToSeasonDbId(String year, UUID programId) throws ApiException return null; } - private Program createProgram(String name, String abbv, String key, String referenceSource, List germplasm, List traits) throws ApiException, DoesNotExistException, ValidatorException, BadRequestException { + private Program createProgram(String name, String abbv, String key, String referenceSource, int numGermplasmToCreate, List traits) throws ApiException, DoesNotExistException, ValidatorException, BadRequestException { SpeciesEntity validSpecies = speciesDAO.findAll().get(0); SpeciesRequest speciesRequest = SpeciesRequest.builder() .commonName(validSpecies.getCommonName()) @@ -1902,10 +1879,14 @@ private Program createProgram(String name, String abbv, String key, String refer // Get main program Program program = programService.getByKey(key).get(); + String brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId().toString()); - if(germplasm != null && !germplasm.isEmpty()) { + if(numGermplasmToCreate > 0) { + + List germplasm = createGermplasm(numGermplasmToCreate, brapiProgramDbId); + BrAPIExternalReference newReference = new BrAPIExternalReference(); newReference.setReferenceSource(String.format("%s/programs", referenceSource)); newReference.setReferenceID(program.getId().toString()); @@ -1928,7 +1909,7 @@ private Program createProgram(String name, String abbv, String key, String refer return program; } - private List createGermplasm(int numToCreate) { + private List createGermplasm(int numToCreate, String brapiProgramDbId) { List germplasm = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { String gid = ""+(i+1); @@ -1947,6 +1928,7 @@ private List createGermplasm(int numToCreate) { testReference.setReferenceID(UUID.randomUUID().toString()); externalRef.add(testReference); testGermplasm.setExternalReferences(externalRef); + testGermplasm.setProgramDbId(brapiProgramDbId); germplasm.add(testGermplasm); } diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index 92ed87e57..c406821aa 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -27,6 +27,7 @@ import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; import org.breedinginsight.dao.db.tables.pojos.ProgramBreedingMethodEntity; import org.breedinginsight.daos.BreedingMethodDAO; +import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.Program; import org.breedinginsight.services.SpeciesService; @@ -75,6 +76,8 @@ public class GermplasmFileImportTest extends BrAPITest { private DSLContext dsl; @Inject private BrAPIGermplasmDAO germplasmDAO; + @Inject + private ProgramDAO programDAO; private ImportTestUtils importTestUtils; @@ -1255,11 +1258,13 @@ private String getPreviewState(JsonObject result, int index) { } private void seedExistingGermplasm(String accessionNumber, String displayName, String source, String breedingMethodCode) { + String brapiProgramDbId = programDAO.getProgramBrAPI(validProgram).getProgramDbId(); BrAPIGermplasm germplasm = new BrAPIGermplasm(); germplasm.setAccessionNumber(accessionNumber); germplasm.setDefaultDisplayName(displayName); germplasm.setGermplasmName(String.format("%s [%s-%s]", displayName, validProgram.getKey(), accessionNumber)); germplasm.setSeedSource(source); + germplasm.setProgramDbId(brapiProgramDbId); JsonObject additionalInfo = new JsonObject(); additionalInfo.addProperty(BrAPIAdditionalInfoFields.GERMPLASM_IMPORT_ENTRY_NUMBER, accessionNumber); @@ -1290,8 +1295,5 @@ private void seedExistingGermplasm(String accessionNumber, String displayName, S germplasm.setExternalReferences(externalReferences); germplasmDAO.createBrAPIGermplasm(List.of(germplasm), validProgram.getId(), null); - - //Refresh cache so importer lookup sees the seeded germplasm immediately. - germplasmDAO.repopulateGermplasmCacheForProgram(validProgram.getId()); } } diff --git a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java index f19803127..7217b5542 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java @@ -45,12 +45,13 @@ import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; -import org.breedinginsight.brapps.importer.daos.*; +import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport.Columns; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.SpeciesDAO; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.Column; @@ -143,6 +144,12 @@ public class SampleSubmissionFileImportTest extends BrAPITest { @Inject private SampleSubmissionService sampleSubmissionService; + @Inject + BrAPITrialService brAPITrialService; + + @Inject + private ProgramDAO programDAO; + private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) (json, type, context) -> OffsetDateTime.parse(json.getAsString())) .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) @@ -174,7 +181,7 @@ public void setup() { @SneakyThrows public void importGIDSuccess() { log.debug("importGIDSuccess"); - Program program = createProgram("Import GID Success", "GIDS", "GIDS", BRAPI_REFERENCE_SOURCE, createGermplasm(96), null); + Program program = createProgram("Import GID Success", "GIDS", "GIDS", BRAPI_REFERENCE_SOURCE, 96, null); List> validFile = new ArrayList<>(); int germGidCounter = 1; @@ -221,14 +228,13 @@ public void importGIDSuccess() { @SneakyThrows public void importObsUnitIdSuccess() { log.debug("importObsUnitIdSuccess"); - Program program = createProgram("Import ObsUnitID success", "OBSID", "OBSID", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Program program = createProgram("Import ObsUnitID success", "OBSID", "OBSID", BRAPI_REFERENCE_SOURCE, 1, null); var experimentId = createExperiment(program); BrAPITrial trial = brAPITrialDAO.getTrialById(program.getId(), UUID.fromString(experimentId)).get(); - List ous = ouDAO.getObservationUnitsForTrialDbId(program.getId(), trial.getTrialDbId()); - BrAPIExternalReference obsUnitId = Utilities.getExternalReference(ous.get(0).getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS)).get(); + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForTrialDbId(program.getId(), trial.getTrialDbId()).get(0); List> validFile = new ArrayList<>(); @@ -240,7 +246,7 @@ public void importObsUnitIdSuccess() { validRow.put(Columns.SPECIES, "TEST"); validRow.put(Columns.GERMPLASM_NAME, ""); validRow.put(Columns.GERMPLASM_GID, ""); - validRow.put(Columns.OBS_UNIT_ID, obsUnitId.getReferenceId()); + validRow.put(Columns.OBS_UNIT_ID, ou.getObservationUnitDbId()); validRow.put(Columns.TISSUE, "TEST"); validRow.put(Columns.COMMENTS, "Test sample"); validFile.add(validRow); @@ -296,7 +302,7 @@ public void importMissingGIDAndObsUnitIdFailure(boolean commit) { @SneakyThrows public void verifyMissingDataThrowsError(boolean commit) { log.debug("verifyMissingDataThrowsError"); - Program program = createProgram("Missing Req Cols "+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(96), null); + Program program = createProgram("Missing Req Cols "+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 96, null); Map base = new HashMap<>(); base.put(Columns.PLATE_ID, "valid_1"); base.put(Columns.ROW, "A"); @@ -369,7 +375,7 @@ public void importInvalidObsUnitIdFailure(boolean commit) { @SneakyThrows public void importConflictingWellsFailure(boolean commit) { log.debug("importConflictingWellsFailure"); - Program program = createProgram("Conflicting Wells " + (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(2), null); + Program program = createProgram("Conflicting Wells " + (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, 2, null); List> validFile = new ArrayList<>(); Map validRow = new HashMap<>(); @@ -436,7 +442,7 @@ private Map assertRowSaved(Map expected, Program return ret; } - private Program createProgram(String name, String abbv, String key, String referenceSource, List germplasm, List traits) throws ApiException, DoesNotExistException, ValidatorException, BadRequestException { + private Program createProgram(String name, String abbv, String key, String referenceSource, Integer numGermplasmToCreate, List traits) throws ApiException, DoesNotExistException, ValidatorException, BadRequestException { SpeciesEntity validSpecies = speciesDAO.findAll().get(0); SpeciesRequest speciesRequest = SpeciesRequest.builder() .commonName(validSpecies.getCommonName()) @@ -456,10 +462,13 @@ private Program createProgram(String name, String abbv, String key, String refer // Get main program Program program = programService.getByKey(key).get(); + String brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId().toString()); - if(germplasm != null && !germplasm.isEmpty()) { + if(numGermplasmToCreate != null && numGermplasmToCreate > 0) { + List germplasm = createGermplasm(numGermplasmToCreate, brapiProgramDbId); + BrAPIExternalReference newReference = new BrAPIExternalReference(); newReference.setReferenceSource(String.format("%s/programs", referenceSource)); newReference.setReferenceID(program.getId().toString()); @@ -482,7 +491,7 @@ private Program createProgram(String name, String abbv, String key, String refer return program; } - private List createGermplasm(int numToCreate) { + private List createGermplasm(int numToCreate, String brapiProgramDbId) { List germplasm = new ArrayList<>(); for (int i = 0; i < numToCreate; i++) { String gid = ""+(i+1); @@ -501,6 +510,7 @@ private List createGermplasm(int numToCreate) { testReference.setReferenceID(UUID.randomUUID().toString()); externalRef.add(testReference); testGermplasm.setExternalReferences(externalRef); + testGermplasm.setProgramDbId(brapiProgramDbId); germplasm.add(testGermplasm); } @@ -567,7 +577,7 @@ private String createExperiment(Program program) throws IOException, Interrupted .getAsJsonArray("data") .get(0).getAsJsonObject().get("id").getAsString(); - JsonObject importResult = importTestUtils.uploadAndFetchWorkflow( + importTestUtils.uploadAndFetchWorkflow( importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null, false, false, null), null, true, @@ -575,12 +585,23 @@ private String createExperiment(Program program) throws IOException, Interrupted program, expMappingId, newExperimentWorkflowId); - return importResult - .get("preview").getAsJsonObject() - .get("rows").getAsJsonArray() - .get(0).getAsJsonObject() - .get("trial").getAsJsonObject() - .get("id").getAsString(); + + + String result; + + try { + // Assumes only one trial/experiment exists + BrAPITrial trial = brAPITrialService.getTrialsByProgramId(program.getId()) + .stream() + .findFirst() + .orElseThrow(); + + result = trial.getTrialDbId(); + } catch (Exception e) { + throw new RuntimeException(e); + } + + return result; } private Map makeExpImportRow(String environment) { diff --git a/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java b/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java index bd2048b61..eb071b652 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java @@ -143,7 +143,7 @@ public void testCreateObservationUnitAdditionalInfoSingleTreatmentFactor() { @SneakyThrows @Order(2) public void testGetObservationUnitAdditionalInfoSingleTreatmentFactor() { - List createdOus = obsUnitDAO.getObservationUnitByName(List.of("test1"), validProgram); + List createdOus = obsUnitDAO.getProgramObservationUnits(validProgram.getId()); singleTreatmentAsserts(createdOus, testTreatment); } diff --git a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java index b957a3f88..aff9d6c34 100644 --- a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java +++ b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java @@ -1,11 +1,13 @@ package org.breedinginsight.services; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.JsonNull; import com.google.gson.JsonObject; import io.reactivex.functions.Function; import io.reactivex.functions.Function3; import lombok.SneakyThrows; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIProgram; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.brapi.v2.model.germ.BrAPIGermplasm; @@ -50,7 +52,6 @@ public class BrAPIGermplasmServiceUnitTest extends DatabaseTest { private BrAPIGermplasmService germplasmService; private BrAPIDAOUtil brAPIDAOUtil; private String referenceSource; - private ProgramCacheProvider cacheProvider; @SneakyThrows @BeforeEach @@ -60,8 +61,7 @@ void setup() { listDAO = mock(BrAPIListDAO.class); programDAO = mock(ProgramDAO.class); brAPIDAOUtil = mock(BrAPIDAOUtil.class); - cacheProvider = new ProgramCacheProvider(super.getRedisConnection()); - germplasmDAO = new BrAPIGermplasmDAO(programDAO, mock(ImportDAO.class), brAPIDAOUtil, cacheProvider, new BrAPIEndpointProvider()); + germplasmDAO = new BrAPIGermplasmDAO(programDAO, mock(ImportDAO.class), brAPIDAOUtil, new BrAPIEndpointProvider(), 65000, new ObjectMapper(), listDAO); programService = mock(ProgramService.class); Field externalReferenceSource = BrAPIGermplasmDAO.class.getDeclaredField("referenceSource"); @@ -79,6 +79,14 @@ public void getGermplasmListExport() { testProgram.setKey("TEST"); testProgram.setId(testProgramId); + + BrAPIProgram brapiProgram = new BrAPIProgram(); + String brapiProgramDbId = UUID.randomUUID().toString(); + brapiProgram.setProgramDbId(brapiProgramDbId); + brapiProgram.setProgramName("Test Program [TEST]"); + + testProgram.setBrapiProgram(brapiProgram); + //Create List BrAPIListsSingleResponse listResponse = new BrAPIListsSingleResponse(); String listId = "1"; @@ -101,6 +109,7 @@ public void getGermplasmListExport() { testGermplasm.setSeedSource("Wild"); testGermplasm.setAccessionNumber("1"); testGermplasm.setDefaultDisplayName("Germplasm A"); + testGermplasm.setProgramDbId(brapiProgramDbId); JsonObject additionalInfo = new JsonObject(); additionalInfo.addProperty(GERMPLASM_IMPORT_ENTRY_NUMBER, "2"); additionalInfo.addProperty(GERMPLASM_BREEDING_METHOD, "Allopolyploid"); @@ -138,6 +147,7 @@ public void getGermplasmListExport() { externalUidReference2.setReferenceID("UID-2"); externalRef.add(externalUidReference2); testGermplasm.setExternalReferences(externalRef); + testGermplasm.setProgramDbId(brapiProgramDbId); germplasm.add(testGermplasm); //Stub out the spies @@ -147,17 +157,12 @@ public void getGermplasmListExport() { doReturn(Optional.of(testProgram)).when(programSpy).getById(testProgramId); //Stub out the mocks - when(programDAO.getAll()).thenReturn(Arrays.asList(Program.builder().id(testProgramId).name("Test Program").active(true).build())); when(programDAO.fetchOneById(any(UUID.class))).thenReturn(testProgram); when(programDAO.get(any(UUID.class))).thenReturn(Arrays.asList(testProgram)); - when(brAPIDAOUtil.searchNoPaging(any(Function.class), - any(Function3.class), + when(programDAO.getProgramBrAPI(any())).thenReturn(brapiProgram); + when(brAPIDAOUtil.search(any(Function.class), any(Function3.class), any(BrAPIGermplasmSearchRequest.class))).thenReturn(germplasm); - - //Create germplasm cache of stub data - Method setupMethod = BrAPIGermplasmDAO.class.getDeclaredMethod("setup"); - setupMethod.setAccessible(true); - setupMethod.invoke(germplasmDAO); + when(brAPIDAOUtil.getBrAPIProgramDbId(any())).thenReturn(brapiProgramDbId); //Create test instance of service, injecting spy- and mock-dependencies germplasmService = new BrAPIGermplasmService(brAPIListSpy, programSpy, germplasmDAO);