[JN-TBD] Refactoring pre-enroll surveys

core/src/main/java/bio/terra/pearl/core/dao/study/AttachSurvey.java

@@ -0,0 +1,7 @@
+package bio.terra.pearl.core.dao.study;
+
+
+public enum AttachSurvey {
+    WITH_CONTENT, // include the content json of the survey
+    WITHOUT_CONTENT  // exclude the content from the retrieval
+}

core/src/main/java/bio/terra/pearl/core/dao/study/StudyDao.java

@@ -84,9 +84,9 @@ public Optional<Study> findOneFullLoad(UUID id) {
      * So, for example, if a portal has two studies, this might return the 'sandbox' environment for
      * both studies
      */
-    public List<Study> findWithPreregContent(String portalShortcode, EnvironmentName envName) {
+    public List<Study> findWithPreEnrollContent(String portalShortcode, EnvironmentName envName) {
         List<Study> studies = findByPortal(portalShortcode);
-        List<StudyEnvironment> studyEnvs =  studyEnvironmentDao.findWithPreregContent(portalShortcode, envName);
+        List<StudyEnvironment> studyEnvs =  studyEnvironmentDao.findWithPreEnrollContent(portalShortcode, envName);
         for (Study study : studies) {
             Optional<StudyEnvironment> studyEnvOpt = studyEnvs.stream()
                     .filter(studyEnv -> studyEnv.getStudyId().equals(study.getId())).findFirst();

core/src/main/java/bio/terra/pearl/core/dao/study/StudyEnvironmentDao.java

@@ -6,6 +6,7 @@
 import bio.terra.pearl.core.model.EnvironmentName;
 import bio.terra.pearl.core.model.notification.Trigger;
 import bio.terra.pearl.core.model.study.StudyEnvironment;
+import bio.terra.pearl.core.model.survey.SurveyType;
 import bio.terra.pearl.core.service.survey.SurveyService;
 import java.util.List;
 import java.util.Optional;
@@ -74,15 +75,16 @@ public List<StudyEnvironment> findAllByPortalAndEnvironment(UUID portalId, Envir
      * returns all the studies associated with the given portal for the given environment
      * So, for example, if a portal has two studies, this might return the 'sandbox' environment for
      * both studies
+     * this attaches the pre-enroll survey so that the "join" process can be done without a separate load
      */
-    public List<StudyEnvironment> findWithPreregContent(String portalShortcode, EnvironmentName envName) {
-        List<String> primaryCols = getQueryColumns.stream().map(col -> "a." + col)
-                .collect(Collectors.toList());
+    public List<StudyEnvironment> findWithPreEnrollContent(String portalShortcode, EnvironmentName envName) {
         List<StudyEnvironment> studyEnvs = jdbi.withHandle(handle ->
-                handle.createQuery("select " + StringUtils.join(primaryCols, ", ") + " from " + tableName
-                                + " a join portal_study on a.study_id = portal_study.study_id "
-                                + " join portal on portal_study.portal_id = portal.id"
-                                + " where portal.shortcode = :portalShortcode and a.environment_name = :environmentName")
+                handle.createQuery("""
+                                    select a.* from %s a 
+                                    join portal_study on a.study_id = portal_study.study_id
+                                    join portal on portal_study.portal_id = portal.id
+                                    where portal.shortcode = :portalShortcode and a.environment_name = :environmentName
+                                    """.formatted(tableName))
                         .bind("portalShortcode", portalShortcode)
                         .bind("environmentName", envName)
                         .mapTo(clazz)
@@ -91,9 +93,8 @@ public List<StudyEnvironment> findWithPreregContent(String portalShortcode, Envi
         for (StudyEnvironment studyEnv : studyEnvs) {
             studyEnv.setStudyEnvironmentConfig(studyEnvironmentConfigDao
                     .find(studyEnv.getStudyEnvironmentConfigId()).get());
-            if (studyEnv.getPreEnrollSurveyId() != null) {
-                studyEnv.setPreEnrollSurvey(surveyDao.find(studyEnv.getPreEnrollSurveyId()).get());
-            }
+            studyEnv.setConfiguredSurveys(studyEnvironmentSurveyDao
+                    .findAllByType(List.of(studyEnv.getId()), SurveyType.PRE_ENROLL, true, AttachSurvey.WITH_CONTENT));
         };
         return studyEnvs;
     }
@@ -103,9 +104,6 @@ public StudyEnvironment attachAllContent(StudyEnvironment studyEnv) {
         UUID studyEnvId = studyEnv.getId();
         studyEnv.setStudyEnvironmentConfig(studyEnvironmentConfigDao.find(studyEnv.getStudyEnvironmentConfigId()).get());
         studyEnv.setConfiguredSurveys(studyEnvironmentSurveyDao.findAllWithSurvey(studyEnvId, true));
-        if (studyEnv.getPreEnrollSurveyId() != null) {
-            studyEnv.setPreEnrollSurvey(surveyService.find(studyEnv.getPreEnrollSurveyId()).get());
-        }
         List<Trigger> triggers = triggerDao.findByStudyEnvironmentId(studyEnvId, true);
         triggerDao.attachTemplates(triggers);
         studyEnv.setTriggers(triggers);

core/src/main/java/bio/terra/pearl/core/dao/study/StudyEnvironmentSurveyDao.java

@@ -5,6 +5,7 @@
 import bio.terra.pearl.core.dao.survey.SurveyDao;
 import bio.terra.pearl.core.model.survey.StudyEnvironmentSurvey;
 import bio.terra.pearl.core.model.survey.Survey;
+import bio.terra.pearl.core.model.survey.SurveyType;
 import org.jdbi.v3.core.Jdbi;
 import org.jdbi.v3.core.statement.Query;
 import org.springframework.context.annotation.Lazy;
@@ -88,19 +89,19 @@ where a.study_environment_id IN (<studyEnvIds>)
     /** gets all the study environment surveys and attaches the relevant survey objects in a batch */
     public List<StudyEnvironmentSurvey> findAllWithSurvey(UUID studyEnvId, Boolean active) {
         List<StudyEnvironmentSurvey> studyEnvSurvs = findAll(List.of(studyEnvId), null, active);
-        attachSurveys(studyEnvSurvs, ATTACH_SURVEY.WITH_CONTENT);
+        attachSurveys(studyEnvSurvs, AttachSurvey.WITH_CONTENT);
         return studyEnvSurvs;
     }
 
-    public List<StudyEnvironmentSurvey> findAllWithSurveyNoContent(List<UUID> studyEnvironmentIds, String stableId, Boolean active) {
+    public List<StudyEnvironmentSurvey>  findAllWithSurveyNoContent(List<UUID> studyEnvironmentIds, String stableId, Boolean active) {
         List<StudyEnvironmentSurvey> studyEnvSurveys = findAll(studyEnvironmentIds, stableId, active);
-        attachSurveys(studyEnvSurveys, ATTACH_SURVEY.WITHOUT_CONTENT);
+        attachSurveys(studyEnvSurveys, AttachSurvey.WITHOUT_CONTENT);
         return studyEnvSurveys;
     }
 
-    protected void attachSurveys(List<StudyEnvironmentSurvey> studyEnvSurveys, ATTACH_SURVEY attach) {
+    protected void attachSurveys(List<StudyEnvironmentSurvey> studyEnvSurveys, AttachSurvey attach) {
         List<UUID> surveyIds = studyEnvSurveys.stream().map(ses -> ses.getSurveyId()).collect(Collectors.toList());
-        List<Survey> surveys = attach.equals(ATTACH_SURVEY.WITH_CONTENT) ? surveyDao.findAll(surveyIds) : surveyDao.findAllNoContent(surveyIds);
+        List<Survey> surveys = attach.equals(AttachSurvey.WITH_CONTENT) ? surveyDao.findAll(surveyIds) : surveyDao.findAllNoContent(surveyIds);
         for (StudyEnvironmentSurvey ses : studyEnvSurveys) {
             ses.setSurvey(surveys.stream().filter(survey -> survey.getId().equals(ses.getSurveyId()))
                     .findFirst().orElseThrow());
@@ -126,8 +127,31 @@ select count(*) from %s a
                         .one()) > 0;
     }
 
-    protected enum ATTACH_SURVEY {
-        WITH_CONTENT, // include the content json of the survey
-        WITHOUT_CONTENT  // exclude the content from the retrieval
+    public List<StudyEnvironmentSurvey> findAllByType(List<UUID> studyEnvIds, SurveyType surveyType, Boolean active, AttachSurvey attach) {
+        List<StudyEnvironmentSurvey> studyEnvSurveys = jdbi.withHandle(handle -> {
+            Query query = handle.createQuery("""
+                                select a.* from %s a
+                                    join survey on survey.id = a.survey_id
+                                    where a.study_environment_id IN (<studyEnvIds>)
+                                    %s
+                                    %s
+                                    order by survey.stable_id asc, survey_order asc;
+                                """.formatted(
+                            tableName,
+                            surveyType != null ? " and survey.survey_type = :surveyType" : "",
+                            active != null ? " and a.active = :active" : ""))
+                    .bind("surveyType", surveyType)
+                    .bindList("studyEnvIds", studyEnvIds);
+            if (active != null) {
+                query = query.bind("active", active);
+            }
+            if (surveyType != null) {
+                query = query.bind("surveyType", surveyType);
+            }
+            return query.mapTo(clazz)
+                    .list();
+        });
+        attachSurveys(studyEnvSurveys, attach);
+        return studyEnvSurveys;
     }
 }

core/src/main/java/bio/terra/pearl/core/model/survey/SurveyType.java

@@ -1,6 +1,7 @@
 package bio.terra.pearl.core.model.survey;
 
 public enum SurveyType {
+    PRE_ENROLL, // for prenrollment surveys
     CONSENT, // for consent forms
     RESEARCH, // for surveys intended to be included in the research dataset of a study
     OUTREACH, // for surveys intended for outreach purposes, e.g. to collect marketing/feedback information

core/src/main/java/bio/terra/pearl/core/service/export/EnrolleeExportService.java

@@ -163,27 +163,14 @@ public List<ModuleFormatter> generateModuleInfos(ExportOptions exportOptions, UU
         return moduleFormatters;
     }
 
-    List<SurveyType> SURVEY_TYPE_EXPORT_ORDER = List.of(SurveyType.CONSENT, SurveyType.RESEARCH, SurveyType.DOCUMENT_REQUEST, SurveyType.OUTREACH);
+    List<SurveyType> SURVEY_TYPE_EXPORT_ORDER = List.of(SurveyType.PRE_ENROLL, SurveyType.CONSENT, SurveyType.RESEARCH, SurveyType.DOCUMENT_REQUEST, SurveyType.OUTREACH);
 
     /**
      * returns a ModuleExportInfo for each unique survey stableId that has ever been attached to the studyEnvironment
      * If multiple versions of a survey have been attached, those will be consolidated into a single ModuleExportInfo
      */
     protected List<SurveyFormatter> generateSurveyModules(ExportOptions exportOptions, UUID studyEnvironmentId, List<EnrolleeExportData> enrolleeExportData) {
         List<SurveyFormatter> moduleFormatters = new ArrayList<>();
-        // now add the pre-enrollment survey (if it exists)
-        StudyEnvironment studyEnvironment = studyEnvironmentService.find(studyEnvironmentId).orElseThrow();
-        if (studyEnvironment.getPreEnrollSurveyId() != null) {
-            Survey preEnrollSurvey = surveyService.find(studyEnvironment.getPreEnrollSurveyId()).orElseThrow();
-            List<SurveyQuestionDefinition> preEnrollSurveyQuestionDefinitions = surveyQuestionDefinitionDao.findAllBySurveyIds(List.of(preEnrollSurvey.getId()));
-            moduleFormatters.add(new SurveyFormatter(
-                    exportOptions,
-                    preEnrollSurvey.getStableId(),
-                    List.of(preEnrollSurvey),
-                    preEnrollSurveyQuestionDefinitions,
-                    enrolleeExportData,
-                    objectMapper));
-        }
 
         // get all surveys that have ever been attached to the StudyEnvironment, including inactive ones
         List<StudyEnvironmentSurvey> configuredSurveys = studyEnvironmentSurveyService.findAllByStudyEnvIdWithSurvey(studyEnvironmentId, null);

core/src/main/java/bio/terra/pearl/core/service/portal/PortalService.java

@@ -126,7 +126,7 @@ public Optional<Portal> loadWithParticipantSiteContent(
             Optional<PortalEnvironment> portalEnv = portalEnvironmentService
                     .loadWithParticipantSiteContent(portal.getShortcode(), environmentName, language);
             portal.getPortalEnvironments().add(portalEnv.get());
-            List<Study> studies = studyService.findWithPreregContent(portal.getShortcode(), environmentName);
+            List<Study> studies = studyService.findWithPreEnrollContent(portal.getShortcode(), environmentName);
             for (Study study : studies) {
                 portal.getPortalStudies().add(
                         PortalStudy.builder().study(study).build()

core/src/main/java/bio/terra/pearl/core/service/study/StudyEnvironmentSurveyService.java

@@ -1,5 +1,6 @@
 package bio.terra.pearl.core.service.study;
 
+import bio.terra.pearl.core.dao.study.AttachSurvey;
 import bio.terra.pearl.core.dao.study.StudyEnvironmentSurveyDao;
 import bio.terra.pearl.core.model.portal.PortalEnvironment;
 import bio.terra.pearl.core.model.publishing.ListChange;
@@ -8,6 +9,7 @@
 import bio.terra.pearl.core.model.study.StudyEnvironment;
 import bio.terra.pearl.core.model.survey.StudyEnvironmentSurvey;
 import bio.terra.pearl.core.model.survey.Survey;
+import bio.terra.pearl.core.model.survey.SurveyType;
 import bio.terra.pearl.core.service.CrudService;
 import java.util.List;
 import java.util.Optional;
@@ -93,6 +95,10 @@ public List<StudyEnvironmentSurvey> findAllWithSurveyNoContent(List<UUID> studyE
         return dao.findAllWithSurveyNoContent(studyEnvIds, stableId, active);
     }
 
+    public List<StudyEnvironmentSurvey> findAllByType(List<UUID> studyEnvIds, SurveyType type, Boolean active, AttachSurvey attachSurvey) {
+        return dao.findAllByType(studyEnvIds, type, active, attachSurvey);
+    }
+
     public void deleteBySurveyId(UUID surveyId) {
         dao.deleteBySurveyId(surveyId);
     }

core/src/main/java/bio/terra/pearl/core/service/study/StudyService.java

@@ -65,8 +65,8 @@ public Study create(Study study) {
         }
     }
 
-    public List<Study> findWithPreregContent(String portalShortcode, EnvironmentName envName) {
-        List<Study> studies = dao.findWithPreregContent(portalShortcode, envName);
+    public List<Study> findWithPreEnrollContent(String portalShortcode, EnvironmentName envName) {
+        List<Study> studies = dao.findWithPreEnrollContent(portalShortcode, envName);
         studies.forEach(this::attachStudyEnvironmentKitTypes);
 
         return studies;

core/src/main/java/bio/terra/pearl/core/service/survey/SurveyTaskDispatcher.java

@@ -111,6 +111,7 @@ protected TaskType getTaskType(SurveyTaskConfigDto taskDto) {
             SurveyType.RESEARCH, TaskType.SURVEY,
             SurveyType.OUTREACH, TaskType.OUTREACH,
             SurveyType.ADMIN, TaskType.ADMIN_FORM,
+            SurveyType.PRE_ENROLL, TaskType.SURVEY, // pre-enroll surveys are NOT typically assigned as tasks
             SurveyType.DOCUMENT_REQUEST, TaskType.DOCUMENT_REQUEST
     );
 

core/src/main/java/bio/terra/pearl/core/service/workflow/EnrollmentService.java

@@ -1,5 +1,6 @@
 package bio.terra.pearl.core.service.workflow;
 
+import bio.terra.pearl.core.dao.study.AttachSurvey;
 import bio.terra.pearl.core.dao.survey.AnswerMappingDao;
 import bio.terra.pearl.core.dao.survey.PreEnrollmentResponseDao;
 import bio.terra.pearl.core.model.EnvironmentName;
@@ -17,6 +18,7 @@
 import bio.terra.pearl.core.service.participant.PortalParticipantUserService;
 import bio.terra.pearl.core.service.study.StudyEnvironmentConfigService;
 import bio.terra.pearl.core.service.study.StudyEnvironmentService;
+import bio.terra.pearl.core.service.study.StudyEnvironmentSurveyService;
 import bio.terra.pearl.core.service.study.exception.StudyEnvConfigMissing;
 import bio.terra.pearl.core.service.survey.AnswerProcessingService;
 import bio.terra.pearl.core.service.survey.SurveyParseUtils;
@@ -54,6 +56,7 @@ public class EnrollmentService {
     private final AnswerMappingDao answerMappingDao;
     private final SurveyResponseService surveyResponseService;
     private final AnswerProcessingService answerProcessingService;
+    private final StudyEnvironmentSurveyService studyEnvironmentSurveyService;
 
     public EnrollmentService(SurveyService surveyService,
                              PreEnrollmentResponseDao preEnrollmentResponseDao,
@@ -67,7 +70,8 @@ public EnrollmentService(SurveyService surveyService,
                              ParticipantUserService participantUserService,
                              AnswerMappingDao answerMappingDao,
                              SurveyResponseService surveyResponseService,
-                             AnswerProcessingService answerProcessingService) {
+                             AnswerProcessingService answerProcessingService,
+                             StudyEnvironmentSurveyService studyEnvironmentSurveyService) {
         this.surveyService = surveyService;
         this.preEnrollmentResponseDao = preEnrollmentResponseDao;
         this.studyEnvironmentService = studyEnvironmentService;
@@ -82,6 +86,7 @@ public EnrollmentService(SurveyService surveyService,
         this.answerMappingDao = answerMappingDao;
         this.surveyResponseService = surveyResponseService;
         this.answerProcessingService = answerProcessingService;
+        this.studyEnvironmentSurveyService = studyEnvironmentSurveyService;
     }
 
     /**
@@ -370,11 +375,13 @@ private PreEnrollmentResponse validatePreEnrollResponse(PortalParticipantUser op
                                                             UUID preEnrollResponseId,
                                                             UUID participantUserId,
                                                             boolean isSubject) {
-        if (studyEnv.getPreEnrollSurveyId() == null) {
-            // no pre-enroll required
-            return null;
-        }
         if (preEnrollResponseId == null) {
+            List<StudyEnvironmentSurvey> preEnrollSurveys = studyEnvironmentSurveyService.findAllByType(
+                    List.of(studyEnv.getId()), SurveyType.PRE_ENROLL, true, AttachSurvey.WITHOUT_CONTENT);
+            if (preEnrollSurveys.isEmpty()) {
+                // no pre-enroll required
+                return null;
+            }
             if (isSubject) {
                 log.warn("Could not match enrollee to pre-enrollment survey results; user {}", participantUserId);
             }

core/src/main/resources/db/changelog/changesets/2025_01_27_prenroll_type.yaml

@@ -0,0 +1,13 @@
+databaseChangeLog:
+  - changeSet:
+      id: "preenroll_type"
+      author: dbush
+      changes:
+        - sql:
+            sql:
+              insert into study_environment_survey (study_environment_id, created_at, last_updated_at, survey_id, survey_order, active)
+              select id, now(), now(), pre_enroll_survey_id as survey_id, 1, true from study_environment where pre_enroll_survey_id is not null;
+        - sql:
+            sql:
+              update survey set survey_type = 'PRE_ENROLL', auto_assign = false where stable_id in (select 
+              stable_id from survey join study_environment on pre_enroll_survey_id = survey.id);

core/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -392,6 +392,9 @@ databaseChangeLog:
   - include:
       file: changesets/2025_02_05_create_relation_indices.yaml
       relativeToChangelogFile: true
+  - include:
+      file: changesets/2025_01_27_prenroll_type.yaml
+      relativeToChangelogFile: true
 
 
 # README: it is a best practice to put each DDL statement in its own change set. DDL statements