Merge pull request #67 from MeasureAuthoringTool/MAT-6916

MAT-6916 Fetch And Udpate All Code Systems

src/main/java/gov/cms/madie/terminology/controller/VsacFhirTerminologyController.java

@@ -4,14 +4,19 @@
 import gov.cms.madie.terminology.dto.QdmValueSet;
 import gov.cms.madie.terminology.dto.ValueSetsSearchCriteria;
 import gov.cms.madie.terminology.exceptions.VsacUnauthorizedException;
+import gov.cms.madie.terminology.models.CodeSystem;
 import gov.cms.madie.terminology.models.UmlsUser;
 import gov.cms.madie.terminology.service.FhirTerminologyService;
 import gov.cms.madie.terminology.service.VsacService;
+import jakarta.servlet.http.HttpServletRequest;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
+import org.springframework.security.access.prepost.PreAuthorize;
 
 import java.security.Principal;
 import java.util.List;
@@ -59,4 +64,26 @@ public ResponseEntity<List<QdmValueSet>> getValueSetsExpansions(
         username);
     throw new VsacUnauthorizedException("Please login to UMLS before proceeding");
   }
+
+  @GetMapping(path = "/update-code-systems", produces = MediaType.APPLICATION_JSON_VALUE)
+  @PreAuthorize("#request.getHeader('api-key') == #apiKey")
+  public ResponseEntity<List<CodeSystem>> retrieveAndUpdateCodeSystems(
+          Principal principal,
+          HttpServletRequest request,
+          @Value("${admin-api-key}") String apiKey,
+          @RequestHeader("Authorization") String accessToken) {
+    final String username = principal.getName();
+    Optional<UmlsUser> umlsUser = vsacService.findByHarpId(username);
+
+    if (umlsUser.isPresent() && !StringUtils.isBlank(umlsUser.get().getApiKey())) {
+      return ResponseEntity.ok()
+          .body(fhirTerminologyService.retrieveAllCodeSystems(umlsUser.get()));
+    } else {
+      log.error(
+          "Unable to Retrieve List of code systems, "
+              + "UMLS Authentication Key Not found for user : [{}}]",
+          username);
+      throw new VsacUnauthorizedException("Please login to UMLS before proceeding");
+    }
+  }
 }

src/main/java/gov/cms/madie/terminology/models/CodeSystem.java

@@ -0,0 +1,28 @@
+package gov.cms.madie.terminology.models;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.time.Instant;
+import java.util.Date;
+
+import org.springframework.data.annotation.Id;
+import org.springframework.data.mongodb.core.mapping.Document;
+
+@Data
+@AllArgsConstructor
+@Builder(toBuilder = true)
+@NoArgsConstructor
+@Document
+public class CodeSystem {
+  @Id String id; // title + version (both required fields)
+  private String title;
+  private String name;
+  private String version;
+  private String versionId;
+  private String oid; // identifier[0].value oid of identifier List
+  private Instant lastUpdated; // when queried
+  private Date lastUpdatedUpstream; // when was resource last updated on vsac end
+}

src/main/java/gov/cms/madie/terminology/repositories/CodeSystemRepository.java

@@ -0,0 +1,10 @@
+package gov.cms.madie.terminology.repositories;
+
+import gov.cms.madie.terminology.models.CodeSystem;
+import org.springframework.data.mongodb.repository.MongoRepository;
+import java.util.Optional;
+
+public interface CodeSystemRepository extends MongoRepository<CodeSystem, String> {
+    Optional<CodeSystem> findById(String id);
+
+}

src/main/java/gov/cms/madie/terminology/service/FhirTerminologyService.java

@@ -6,7 +6,9 @@
 import gov.cms.madie.models.measure.ManifestExpansion;
 import gov.cms.madie.terminology.dto.QdmValueSet;
 import gov.cms.madie.terminology.dto.ValueSetsSearchCriteria;
+import gov.cms.madie.terminology.models.CodeSystem;
 import gov.cms.madie.terminology.models.UmlsUser;
+import gov.cms.madie.terminology.repositories.CodeSystemRepository;
 import gov.cms.madie.terminology.util.TerminologyServiceUtil;
 import gov.cms.madie.terminology.webclient.FhirTerminologyServiceWebClient;
 import lombok.RequiredArgsConstructor;
@@ -15,7 +17,9 @@
 import org.hl7.fhir.r4.model.ValueSet;
 import org.springframework.cache.annotation.Cacheable;
 import org.springframework.stereotype.Service;
+import org.springframework.web.util.UriComponentsBuilder;
 
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
@@ -27,6 +31,7 @@ public class FhirTerminologyService {
   private final FhirContext fhirContext;
   private final FhirTerminologyServiceWebClient fhirTerminologyServiceWebClient;
   private final MappingService mappingService;
+  private final CodeSystemRepository codeSystemRepository;
 
   @Cacheable("manifest-list")
   public List<ManifestExpansion> getManifests(UmlsUser umlsUser) {
@@ -108,4 +113,74 @@ private List<QdmValueSet.Concept> getValueSetConcepts(
     log.info("No Expansion codes are found for the valueSet oid : [{}]", valueSet.getId());
     return List.of();
   }
+
+
+  public List<CodeSystem> retrieveAllCodeSystems(UmlsUser umlsUser) {
+      List<CodeSystem> allCodeSystems = new ArrayList<>();
+
+      recursiveRetrieveCodeSystems(umlsUser, 0, 50, allCodeSystems);
+      // Once we have all codeSystems, update DB using mongo
+      updateOrInsertAllCodeSystems(allCodeSystems);
+      return allCodeSystems;
+  }
+    private void recursiveRetrieveCodeSystems(UmlsUser umlsUser, Integer offset, Integer count, List<CodeSystem> allCodeSystems) {
+        log.info("requesting page offset: {} count: {}", offset, count);
+        Bundle codeSystemBundle = retrieveCodeSystemsPage(umlsUser, offset, count);
+        List<CodeSystem> codeSystemsPage = new ArrayList<>(); // build small list
+        codeSystemBundle.getEntry().forEach(entry -> {
+            var codeSystem = (org.hl7.fhir.r4.model.CodeSystem) entry.getResource();
+            String codeSystemValue = "";
+            for (org.hl7.fhir.r4.model.Identifier identifier : codeSystem.getIdentifier()) {
+                if (identifier.getValue() != null && !identifier.getValue().isEmpty()) {
+                    codeSystemValue = identifier.getValue();
+                    break;
+                }
+            }
+            codeSystemsPage.add(
+                    CodeSystem.builder()
+                            .id(codeSystem.getTitle() + codeSystem.getVersion())
+                            .title(codeSystem.getTitle())
+                            .name(codeSystem.getName())
+                            .version(codeSystem.getVersion())
+                            .versionId(codeSystem.getMeta().getVersionId())
+                            .oid(codeSystemValue)
+                            .lastUpdated(Instant.now())
+                            .lastUpdatedUpstream(codeSystem.getMeta().getLastUpdated())
+                            .build());
+        });
+        allCodeSystems.addAll(codeSystemsPage); // update big list
+        var links = codeSystemBundle.getLink();
+        links.forEach((l) -> {
+            if (l.getRelation().equals("next")){
+                // if next, call self and continue until fail.
+                UriComponentsBuilder builder = UriComponentsBuilder.fromHttpUrl(l.getUrl());
+                String newOffset = builder.build().getQueryParams().getFirst("_offset");
+                String newCount = builder.build().getQueryParams().getFirst("_count");
+                assert newOffset != null;
+                assert newCount != null;
+                recursiveRetrieveCodeSystems(umlsUser, Integer.parseInt(newOffset), Integer.parseInt(newCount), allCodeSystems);
+            }
+        });
+    }
+    // one to call only, one to mutate and build
+    private Bundle retrieveCodeSystemsPage(UmlsUser umlsUser, Integer offset, Integer count) {
+        IParser parser = fhirContext.newJsonParser();
+        String responseString =
+                fhirTerminologyServiceWebClient.getCodeSystemsPage(offset, count, umlsUser.getApiKey());
+        return parser.parseResource(
+                Bundle.class, responseString);
+    }
+
+    private void updateOrInsertAllCodeSystems(List<CodeSystem> codeSystemList){
+        for (CodeSystem codeSystem : codeSystemList) {
+            var id = codeSystem.getTitle() + codeSystem.getVersion();
+            Optional<CodeSystem> existingCodeSystemOptional = codeSystemRepository.findById(id);
+            if (existingCodeSystemOptional.isEmpty()) {
+                // Insert new CodeSystem
+                codeSystemRepository.save(codeSystem);
+                log.info("New CodeSystem inserted: {}", codeSystem);
+            }
+        }
+    }
+
 }

src/main/java/gov/cms/madie/terminology/util/TerminologyServiceUtil.java

@@ -63,6 +63,16 @@ public static URI buildRetrieveCodeUri(String baseUrl, String codePath) {
         .toUri();
   }
 
+  public static URI buildRetrieveCodeSystemsUri(String baseUrl, Integer offset, Integer count) {
+    //    http://uat-cts.nlm.nih.gov/fhir/res/CodeSystem?_offset=100&_count=100
+    return UriComponentsBuilder.fromUriString(baseUrl)
+        .queryParam("_offset", Integer.toString(offset))
+        .queryParam("_count", Integer.toString(count))
+        .buildAndExpand()
+        .encode()
+        .toUri();
+  }
+
   public static String buildCodePath(
       String codeSystemName, String codeSystemVersion, String codeId) {
     // "/CodeSystem/LOINC22/Version/2.67/Code/21112-8/Info";

src/main/java/gov/cms/madie/terminology/webclient/FhirTerminologyServiceWebClient.java

@@ -1,8 +1,8 @@
 package gov.cms.madie.terminology.webclient;
 
+import gov.cms.madie.terminology.util.TerminologyServiceUtil;
 import gov.cms.madie.models.measure.ManifestExpansion;
 import gov.cms.madie.terminology.dto.ValueSetsSearchCriteria;
-import gov.cms.madie.terminology.util.TerminologyServiceUtil;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
@@ -22,18 +22,23 @@ public class FhirTerminologyServiceWebClient {
 
   private final WebClient fhirTerminologyWebClient;
   private final String manifestPath;
+  private final String codeSystemPath;
   private final String defaultProfile;
 
   public FhirTerminologyServiceWebClient(
       @Value("${client.fhir-terminology-service.base-url}") String fhirTerminologyServiceBaseUrl,
       @Value("${client.fhir-terminology-service.manifests-urn}") String manifestUrn,
+      @Value("${client.fhir-terminology-service.code-system-urn}") String codeSystemUrn,
       @Value("${client.default_profile}") String defaultProfile) {
     fhirTerminologyWebClient =
         WebClient.builder()
             .baseUrl(fhirTerminologyServiceBaseUrl)
             .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
+            .codecs(
+                clientCodecConfigurer -> clientCodecConfigurer.defaultCodecs().maxInMemorySize(-1))
             .build();
     this.manifestPath = manifestUrn;
+    this.codeSystemPath = codeSystemUrn;
     this.defaultProfile = defaultProfile;
   }
 
@@ -55,6 +60,27 @@ public String getManifestBundle(String apiKey) {
         .block();
   }
 
+  public String getCodeSystemsPage(Integer offset, Integer count, String apiKey) {
+    //  https://uat-cts.nlm.nih.gov/fhir/res/CodeSystem?_offset=0&_count=100
+    URI codeUri = TerminologyServiceUtil.buildRetrieveCodeSystemsUri(codeSystemPath, offset, count);
+    log.debug("Retrieving codeSystems at {}, offset {}, count {}", codeSystemPath, offset, count);
+    return fhirTerminologyWebClient
+        .get()
+        .uri(codeUri.toString())
+        .headers(headers -> headers.setBasicAuth("apikey", apiKey))
+        .exchangeToMono(
+            clientResponse -> {
+              if (clientResponse.statusCode().equals(HttpStatus.BAD_REQUEST)
+                  || clientResponse.statusCode().equals(HttpStatus.OK)) {
+                return clientResponse.bodyToMono(String.class);
+              } else {
+                log.debug("Received NON-OK response while retrieving codePath");
+                return clientResponse.createException().flatMap(Mono::error);
+              }
+            })
+        .block();
+  }
+
   public String getValueSetResource(
       String apiKey,
       ValueSetsSearchCriteria.ValueSetParams valueSetParams,

src/main/resources/application.yml

@@ -18,13 +18,16 @@ okta:
     issuer: ${OKTA_ISSUER:https://dev-18092578.okta.com/oauth2/default}
     audience: ${OKTA_AUDIENCE:api://default}
 
+admin-api-key: ${ADMIN_API_KEY:0a51991c}
+
 client:
   vsac_base_url: https://vsac.nlm.nih.gov/vsac
   valueset_endpoint: /svs/RetrieveMultipleValueSets?id={oid}&profile={profile}&includeDraft={includeDraft}
   default_profile: ${DEFAULT_VSAC_PROFILE:Most Recent Code System Versions in VSAC}
   fhir-terminology-service:
     base-url: ${FHIR_TERMINOLOGY_BASE_URL:https://uat-cts.nlm.nih.gov/fhir}
     manifests-urn: /Library
+    code-system-urn: /CodeSystem
 
 spring:
   session:
@@ -37,4 +40,4 @@ spring:
 
 mapping:
   data:
-    code-system-entry-url: ${CODE_SYSTEM_ENTRY_URL:https://madie-dev-static.s3.amazonaws.com/mappings/code-system-entry.json}
+    code-system-entry-url: ${CODE_SYSTEM_ENTRY_URL:https://madie-dev-static.s3.amazonaws.com/mappings/code-system-entry.json}

src/test/java/gov/cms/madie/terminology/controller/VsacFhirTerminologyControllerMvcTest.java

@@ -1,8 +1,10 @@
 package gov.cms.madie.terminology.controller;
 
 import gov.cms.madie.models.measure.ManifestExpansion;
+import gov.cms.madie.models.measure.Measure;
 import gov.cms.madie.terminology.dto.QdmValueSet;
 import gov.cms.madie.terminology.dto.ValueSetsSearchCriteria;
+import gov.cms.madie.terminology.models.CodeSystem;
 import gov.cms.madie.terminology.models.UmlsUser;
 import gov.cms.madie.terminology.service.FhirTerminologyService;
 import gov.cms.madie.terminology.service.VsacService;
@@ -23,12 +25,14 @@
 
 import static org.hamcrest.CoreMatchers.*;
 import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.*;
 import static org.mockito.Mockito.when;
 import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
 import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.user;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 @WebMvcTest(VsacFhirTerminologyController.class)
@@ -48,6 +52,8 @@ class VsacFhirTerminologyControllerMvcTest {
   private final List<QdmValueSet> mockQdmValueSets = new ArrayList<>();
   private static final String TEST_USER = "test.user";
   private static final String TEST_API_KEY = "te$tKey";
+  private static final String ADMIN_TEST_API_KEY_HEADER = "api-key";
+  private static final String ADMIN_TEST_API_KEY_HEADER_VALUE = "0a51991c";
 
   @BeforeEach
   public void setup() {
@@ -190,4 +196,35 @@ void testUnAuthorizedUmlsUserWhileGetValueSetsExpansionsMvc() throws Exception {
             .andReturn();
     assertThat(result.getResponse().getStatus(), is(equalTo(401)));
   }
+  @Test
+  public void testRetrieveAndUpdateCodeSystemsSuccessfully() throws Exception {
+    Principal principal = mock(Principal.class);
+    when(principal.getName()).thenReturn(TEST_USER);
+    when(vsacService.findByHarpId(anyString())).thenReturn(Optional.empty());
+    MvcResult result =
+            mockMvc
+            .perform(
+                    MockMvcRequestBuilders.get("/terminology/update-code-systems")
+                            .with(csrf())
+                            .with(user(TEST_USR))
+                            .header(ADMIN_TEST_API_KEY_HEADER, ADMIN_TEST_API_KEY_HEADER_VALUE)
+                            .header("Authorization", "test-okta"))
+            .andExpect(status().isUnauthorized())
+            .andReturn();
+    assertThat(result.getResponse().getStatus(), is(equalTo(401)));
+  }
+  @Test
+  public void testRetrieveAndUpdateCodeSystemsUnauthorized() throws Exception {
+    Principal principal = mock(Principal.class);
+    when(principal.getName()).thenReturn(TEST_USER);
+    when(vsacService.findByHarpId(anyString())).thenReturn(Optional.ofNullable(umlsUser));
+    mockMvc
+            .perform(
+                    MockMvcRequestBuilders.get("/terminology/update-code-systems")
+                            .with(csrf())
+                            .with(user(TEST_USR))
+                            .header(ADMIN_TEST_API_KEY_HEADER, ADMIN_TEST_API_KEY_HEADER_VALUE)
+                            .header("Authorization", "test-okta"))
+            .andExpect(status().isOk());
+  }
 }