Skip to content

Commit

Permalink
Refresh IG Group file POST addition & messaging improvements (#506)
Browse files Browse the repository at this point in the history 
* Refresh IG POST expanded to include other resources such as group files.
Duplicate bundle files during refresh fixed.
-x added as possible argument during Refresh IG to show expanded reporting for errors, warnings, and information messages.
Bundle Test Cases enhanced with multithreading optimizations.
General enhancements to message output for readability.

* Tracking POST tasks via IBaseResource as key rather than resourceID to avoid possible duplicate keys

* Corrected formatting of cql processing summary.

* Code cleanup.

* users

* Adjusted over to exception handling rather than boolean return for various methods during bundling process, to relay exception messages to uusers

* Safeguarding lists against concurrency issues.

* Adjusted some Maps in IOUtils to be concurrent safe, clarified POST summary during refresh.

* Moved start/end messages for bundle process from igbundleprocessor to abstract class. Finished message won't appear if no resources exist for bundling.

* Consistent titling of sections in console (ie [Bundling Measures]

* Group file creation bug resolved.

* Added Group file verification during Refresh IG test.

* Added failed test tracking so we don't have to delete any test files and can cleanly report to measure developers the issues they have with tests.

* Moving some hard coded strings to constants

* Added failed POST logging so users can retain a log of what files fail post. Added ability to ignore certain tests during refresh.

* Sorting all summary lists at end of bundle process. Moved summary message generation to its own method.

* applying a timestamp to the httpfail log filename

* Sorting the test case refresh summary for readability.

* Using SLF4J provider ch.qos.logback.classic.spi.LogbackServiceProvider so that warnings from ca.uhn.fhir.parser.LenientErrorHandler can be supressed during various operations. We already handle the translator error list.

* Using logger over System.out

---------

Co-authored-by: Evan Chicoine <[email protected]>
Co-authored-by: JP <[email protected]>
  • Loading branch information
@echicoine-icf @JPercival
3 people authored Mar 18, 2024
1 parent d395b04 commit dca1a3d
Show file tree
Hide file tree
Showing 61 changed files with 5,601 additions and 1,494 deletions.
12 changes: 6 additions & 6 deletions tooling-cli/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@
<description>CQF Tooling CLI</description>

<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<scope>compile</scope>
</dependency>

<dependency>
<groupId>org.opencds.cqf</groupId>
<artifactId>tooling</artifactId>
Expand All @@ -32,12 +38,6 @@
<artifactId>model-jackson</artifactId>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
<scope>compile</scope>
</dependency>

<dependency>
<groupId>org.reflections</groupId>
<artifactId>reflections</artifactId>
Expand Down
5 changes: 5 additions & 0 deletions tooling-cli/src/main/java/org/opencds/cqf/tooling/cli/Main.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,8 @@
//import org.opencds.cqf.tooling.operations.ExecutableOperation;
//import org.opencds.cqf.tooling.operations.Operation;
//import org.reflections.Reflections;

import org.opencds.cqf.tooling.common.ThreadUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -284,6 +286,9 @@ public class Main {
// }

public static void main(String[] args) {
//ensure any and all executors are shutdown cleanly when app is shutdown:
Runtime.getRuntime().addShutdownHook(new Thread(ThreadUtils::shutdownRunningExecutors));

if (args.length == 0) {
System.err.println("cqf-tooling version: " + Main.class.getPackage().getImplementationVersion());
System.err.println("Requests must include which operation to run as a command line argument. See docs for examples on how to use this project.");
Expand Down
45 changes: 37 additions & 8 deletions tooling/src/main/java/org/opencds/cqf/tooling/common/ThreadUtils.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.opencds.cqf.tooling.common;

import org.opencds.cqf.tooling.utilities.LogUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -14,6 +13,9 @@

public class ThreadUtils {
protected static final Logger logger = LoggerFactory.getLogger(ThreadUtils.class);

private static List<ExecutorService> runningExecutors = new ArrayList<>();

/**
* Executes a list of tasks concurrently using a thread pool.
* <p>
Expand All @@ -23,32 +25,59 @@ public class ThreadUtils {
*
* @param tasks A list of Callable tasks to execute concurrently.
*/
public static void executeTasks(List<Callable<Void>> tasks) {
if (tasks == null || tasks.isEmpty()){
public static void executeTasks(List<Callable<Void>> tasks, ExecutorService executor) {
if (tasks == null || tasks.isEmpty()) {
return;
}

runningExecutors.add(executor);

List<Callable<Void>> retryTasks = new ArrayList<>();

//let OS handle threading:
ExecutorService executorService = Executors.newCachedThreadPool();// Submit tasks and obtain futures
try {
List<Future<Void>> futures = new ArrayList<>();
for (Callable<Void> task : tasks) {
futures.add(executorService.submit(task));
try {
futures.add(executor.submit(task));
} catch (OutOfMemoryError e) {
retryTasks.add(task);
}
}

// Wait for all tasks to complete
for (Future<Void> future : futures) {
future.get();
}
} catch (Exception e) {
logger.error("ThreadUtils.executeTasks", e);
logger.error("ThreadUtils.executeTasks: ", e);
} finally {
executorService.shutdown();
if (retryTasks.isEmpty()) {
runningExecutors.remove(executor);
executor.shutdown();
}else{
executeTasks(retryTasks, executor);
}
}
}

public static void executeTasks(List<Callable<Void>> tasks) {
executeTasks(tasks, Executors.newCachedThreadPool());
}

public static void executeTasks(Queue<Callable<Void>> callables) {
executeTasks(new ArrayList<>(callables), Executors.newCachedThreadPool());
}

executeTasks(new ArrayList<>(callables));
public static void shutdownRunningExecutors() {
try {
if (runningExecutors.isEmpty()) return;
for (ExecutorService es : runningExecutors) {
es.shutdownNow();
}
runningExecutors = new ArrayList<>();
}catch (Exception e){
//fail silently, shutting down anyways
}
}
}
38 changes: 0 additions & 38 deletions tooling/src/main/java/org/opencds/cqf/tooling/cql/exception/CQLTranslatorException.java
View file

This file was deleted.

46 changes: 46 additions & 0 deletions tooling/src/main/java/org/opencds/cqf/tooling/cql/exception/CqlTranslatorException.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package org.opencds.cqf.tooling.cql.exception;

import org.cqframework.cql.cql2elm.CqlCompilerException;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;

/**
* Custom exception to pass the list of errors returned by the translator to calling methods.
*/
public class CqlTranslatorException extends Exception implements Serializable {
private static final long serialVersionUID = 20600L;

/**
* Using Set to avoid duplicate entries.
*/
private final transient List<CqlCompilerException> errors = new ArrayList<>();

public CqlTranslatorException(Exception e) {
super("CQL Translation Error(s): " + e.getMessage());
}

public CqlTranslatorException(List<CqlCompilerException> errors) {
super("CQL Translation Error(s)");
this.errors.addAll(errors);
}

public CqlTranslatorException(List<String> errorsInput, CqlCompilerException.ErrorSeverity errorSeverity) {
super("CQL Translation Error(s)");
for (String error : errorsInput){
errors.add(new CqlCompilerException(error, errorSeverity));
}
}

public CqlTranslatorException(String message) {
super("CQL Translation Error(s): " + message);
}

public List<CqlCompilerException> getErrors() {
if (errors.isEmpty()) {
errors.add(new CqlCompilerException(this.getMessage()));
}
return errors;
}
}
83 changes: 35 additions & 48 deletions tooling/src/main/java/org/opencds/cqf/tooling/library/LibraryProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,21 +48,23 @@
public class LibraryProcessor extends BaseProcessor {
private static final Logger logger = LoggerFactory.getLogger(LibraryProcessor.class);
public static final String ResourcePrefix = "library-";

public static String getId(String baseId) {
return ResourcePrefix + baseId;
}

private static Pattern pattern;

private static Pattern getPattern() {
if(pattern == null) {
if (pattern == null) {
String regex = "^[a-zA-Z]+[a-zA-Z0-9_\\-\\.]*";
pattern = Pattern.compile(regex);
}
return pattern;
}

public static void validateIdAlphaNumeric(String id) {
if(!getPattern().matcher(id).find()) {
if (!getPattern().matcher(id).find()) {
throw new RuntimeException("The library id format is invalid.");
}
}
Expand All @@ -76,8 +78,7 @@ public List<String> refreshIgLibraryContent(BaseProcessor parentContext, Encodin
}

public List<String> refreshIgLibraryContent(BaseProcessor parentContext, Encoding outputEncoding, String libraryPath, String libraryOutputDirectory, Boolean versioned, FhirContext fhirContext, Boolean shouldApplySoftwareSystemStamp) {
System.out.println("Refreshing libraries...");
// ArrayList<String> refreshedLibraryNames = new ArrayList<String>();
logger.info("[Refreshing Libraries]");

LibraryProcessor libraryProcessor;
switch (fhirContext.getVersion().getVersion()) {
Expand All @@ -93,9 +94,8 @@ public List<String> refreshIgLibraryContent(BaseProcessor parentContext, Encodin
}

if (libraryPath == null) {
libraryPath = FilenameUtils.concat(parentContext.getRootDir(), IGProcessor.libraryPathElement);
}
else if (!Utilities.isAbsoluteFileName(libraryPath)) {
libraryPath = FilenameUtils.concat(parentContext.getRootDir(), IGProcessor.LIBRARY_PATH_ELEMENT);
} else if (!Utilities.isAbsoluteFileName(libraryPath)) {
libraryPath = FilenameUtils.concat(parentContext.getRootDir(), libraryPath);
}
RefreshLibraryParameters params = new RefreshLibraryParameters();
Expand All @@ -117,64 +117,51 @@ else if (!Utilities.isAbsoluteFileName(libraryPath)) {
* Bundles library dependencies for a given FHIR library file and populates the provided resource map.
* This method executes asynchronously by invoking the associated task queue.
*
* @param path The path to the FHIR library file.
* @param fhirContext The FHIR context to use for processing resources.
* @param resources The map to populate with library resources.
* @param encoding The encoding to use for reading and processing resources.
* @param versioned A boolean indicating whether to consider versioned resources.
* @return True if the bundling of library dependencies is successful; false otherwise.
* @param path The path to the FHIR library file.
* @param fhirContext The FHIR context to use for processing resources.
* @param resources The map to populate with library resources.
* @param encoding The encoding to use for reading and processing resources.
* @param versioned A boolean indicating whether to consider versioned resources.
*/
public Boolean bundleLibraryDependencies(String path, FhirContext fhirContext, Map<String, IBaseResource> resources,
Encoding encoding, boolean versioned) {
try{
Queue<Callable<Void>> bundleLibraryDependenciesTasks = bundleLibraryDependenciesTasks(path, fhirContext, resources, encoding, versioned);
ThreadUtils.executeTasks(bundleLibraryDependenciesTasks);
return true;
}catch (Exception e){
return false;
}

public void bundleLibraryDependencies(String path, FhirContext fhirContext, Map<String, IBaseResource> resources,
Encoding encoding, boolean versioned) throws Exception {
Queue<Callable<Void>> bundleLibraryDependenciesTasks = bundleLibraryDependenciesTasks(path, fhirContext, resources, encoding, versioned);
ThreadUtils.executeTasks(bundleLibraryDependenciesTasks);
}

/**
* Recursively bundles library dependencies for a given FHIR library file and populates the provided resource map.
* Each dependency is added as a Callable task to be executed asynchronously.
*
* @param path The path to the FHIR library file.
* @param fhirContext The FHIR context to use for processing resources.
* @param resources The map to populate with library resources.
* @param encoding The encoding to use for reading and processing resources.
* @param versioned A boolean indicating whether to consider versioned resources.
* @param path The path to the FHIR library file.
* @param fhirContext The FHIR context to use for processing resources.
* @param resources The map to populate with library resources.
* @param encoding The encoding to use for reading and processing resources.
* @param versioned A boolean indicating whether to consider versioned resources.
* @return A queue of Callable tasks, each representing the bundling of a library dependency.
* The Callable returns null (Void) and is meant for asynchronous execution.
* The Callable returns null (Void) and is meant for asynchronous execution.
*/
public Queue<Callable<Void>> bundleLibraryDependenciesTasks(String path, FhirContext fhirContext, Map<String, IBaseResource> resources,
Encoding encoding, boolean versioned) {
Encoding encoding, boolean versioned) throws Exception {

Queue<Callable<Void>> returnTasks = new ConcurrentLinkedQueue<>();

String fileName = FilenameUtils.getName(path);
boolean prefixed = fileName.toLowerCase().startsWith("library-");
try {
Map<String, IBaseResource> dependencies = ResourceUtils.getDepLibraryResources(path, fhirContext, encoding, versioned, logger);
// String currentResourceID = IOUtils.getTypeQualifiedResourceId(path, fhirContext);
for (IBaseResource resource : dependencies.values()) {
returnTasks.add(() -> {
resources.putIfAbsent(resource.getIdElement().getIdPart(), resource);
Map<String, IBaseResource> dependencies = ResourceUtils.getDepLibraryResources(path, fhirContext, encoding, versioned, logger);
// String currentResourceID = IOUtils.getTypeQualifiedResourceId(path, fhirContext);
for (IBaseResource resource : dependencies.values()) {
returnTasks.add(() -> {
resources.putIfAbsent(resource.getIdElement().getIdPart(), resource);

// NOTE: Assuming dependency library will be in directory of dependent.
String dependencyPath = IOUtils.getResourceFileName(IOUtils.getResourceDirectory(path), resource, encoding, fhirContext, versioned, prefixed);
// NOTE: Assuming dependency library will be in directory of dependent.
String dependencyPath = IOUtils.getResourceFileName(IOUtils.getResourceDirectory(path), resource, encoding, fhirContext, versioned, prefixed);

returnTasks.addAll(bundleLibraryDependenciesTasks(dependencyPath, fhirContext, resources, encoding, versioned));
returnTasks.addAll(bundleLibraryDependenciesTasks(dependencyPath, fhirContext, resources, encoding, versioned));

//return statement needed for Callable<Void>
return null;
});
}
} catch (Exception e) {
logger.error(path, e);
//purposely break addAll:
return null;
//return statement needed for Callable<Void>
return null;
});
}
return returnTasks;
}
Expand Down Expand Up @@ -259,7 +246,7 @@ protected void setTranslatorOptions(Library sourceLibrary, CqlTranslatorOptions
optionsReferenceValue = "#options";
optionsReference.setReference(optionsReferenceValue);
}
Parameters optionsParameters = (Parameters)sourceLibrary.getContained(optionsReferenceValue);
Parameters optionsParameters = (Parameters) sourceLibrary.getContained(optionsReferenceValue);
if (optionsParameters == null) {
optionsParameters = new Parameters();
optionsParameters.setId(optionsReferenceValue.substring(1));
Expand Down
Loading

0 comments on commit dca1a3d

Please sign in to comment.