Merge pull request #75 from genepi/fixes/security-updates

Fixes/security updates

files/imputationserver-beagle.yaml

@@ -1,7 +1,7 @@
 id: imputationserver-beagle
 name: Genotype Imputation supporting Beagle (Minimac4)
 description: This is the new Michigan Imputation Server Pipeline using <a href="https://github.com/statgen/Minimac4">Minimac4</a>. Documentation can be found <a href="http://imputationserver.readthedocs.io/en/latest/">here</a>.<br><br>If your input data is <b>GRCh37/hg19</b> please ensure chromosomes are encoded without prefix  (e.g. <b>20</b>).<br>If your input data is <b>GRCh38hg38</b> please ensure chromosomes are encoded with prefix 'chr' (e.g. <b>chr20</b>).
-version: 1.6.1
+version: 1.6.1
 website: https://imputationserver.readthedocs.io
 category:
 

files/imputationserver-pgs.yaml

@@ -1,7 +1,7 @@
 id: imputationserver-pgs
 name: Genotype Imputation (PGS Calc Integration)
 description: This is the new Michigan Imputation Server Pipeline using <a href="https://github.com/statgen/Minimac4">Minimac4</a>. Documentation can be found <a href="http://imputationserver.readthedocs.io/en/latest/">here</a>.<br><br>If your input data is <b>GRCh37/hg19</b> please ensure chromosomes are encoded without prefix  (e.g. <b>20</b>).<br>If your input data is <b>GRCh38hg38</b> please ensure chromosomes are encoded with prefix 'chr' (e.g. <b>chr20</b>).
-version: 1.6.1
+version: 1.6.2-rc1
 website: https://imputationserver.readthedocs.io
 category:
 

files/minimac4.yaml

@@ -1,7 +1,7 @@
 id: imputationserver
 name: Genotype Imputation (Minimac4)
-description: This is the new Michigan Imputation Server Pipeline using <a href="https://github.com/statgen/Minimac4">Minimac4</a>. Documentation can be found <a href="http://imputationserver.readthedocs.io/en/latest/">here</a>.<br><br>If your input data is <b>GRCh37/hg19</b> please ensure chromosomes are encoded without prefix  (e.g. <b>20</b>).<br>If your input data is <b>GRCh38hg38</b> please ensure chromosomes are encoded with prefix 'chr' (e.g. <b>chr20</b>).
-version: 1.6.1
+description: This is the new Michigan Imputation Server Pipeline using <a href="https://github.com/statgen/Minimac4">Minimac4</a>. Documentation can be found <a href="http://imputationserver.readthedocs.io/en/latest/">here</a>.<br><br>If your input data is <b>GRCh37/hg19</b> please ensure chromosomes are encoded without prefix  (e.g. <b>20</b>).<br>If your input data is <b>GRCh38hg38</b> please ensure chromosomes are encoded with prefix 'chr' (e.g. <b>chr20</b>).
+version: 1.6.1
 website: https://imputationserver.readthedocs.io
 category:
 
@@ -10,7 +10,7 @@ installation:
   - import:
       source: ${app_local_folder}/bin
       target: ${app_hdfs_folder}/bin
- 
+
 workflow:
 
   setups:
@@ -31,14 +31,14 @@ workflow:
       params: $mafFile
       output: ${qcreport}.html
       generates: $qcreport
-      
-#end            
+
+#end
 
   onFailure:
     name: Send Notification on Failure
     jar: imputationserver.jar
     classname: genepi.imputationserver.steps.FailureNotification
-    
+
   steps:
 
 #if($mode == "imputation" || $mode == "phasing")
@@ -70,8 +70,8 @@ workflow:
        value: hg19
        values:
          hg19: GRCh37/hg19
-         hg38: GRCh38/hg38     
-       details: Please note that the final SNP coordinates always match the reference build.        
+         hg38: GRCh38/hg38
+       details: Please note that the final SNP coordinates always match the reference build.
 
      - id: r2Filter
        description: rsq Filter
@@ -91,21 +91,21 @@ workflow:
        values:
          eagle: Eagle v2.4 (phased output)
          no_phasing: No phasing
-  
+
      - id: population
        description: Population
        type: list
        values:
           bind: refpanel
           property: populations
           category: RefPanel
-                
+
      - id: mode
        description: Mode
        type: list
        value: imputation
        values:
-          qconly: Quality Control Only 
+          qconly: Quality Control Only
           imputation: Quality Control & Imputation
           phasing: Quality Control & Phasing Only
 
@@ -118,15 +118,15 @@ workflow:
          true: yes
          false: no
        visible: true
-       
+
      - id: meta
        description: Generate Meta-imputation file
        type: checkbox
        value: no
        values:
          true: yes
          false: no
-       visible: true   
+       visible: true
 
      - id: myseparator
        type: separator
@@ -140,12 +140,12 @@ workflow:
      - id: check1
        description: I will not attempt to re-identify or contact research participants.
        type: terms_checkbox
-       
+
      - id: check2
        description: I will report any inadvertent data release, security breach or other data management incident of which I become aware.
-       type: terms_checkbox   
+       type: terms_checkbox
+
 
-
   outputs:
 
     - id: qcreport
@@ -163,7 +163,7 @@ workflow:
       mergeOutput: true
       removeHeader: false
       zip: false
-       
+
     - id: chunksDir
       description: Actual chunk files
       type: local-folder
@@ -201,13 +201,13 @@ workflow:
       type: local-folder
       download: true
       temp: true
-             
+
     - id: logfile
       description: Logs
       type: local-folder
       download: true
       temp: false
-           
+
     - id: hadooplogs
       description: Hadoop Task Logs
       type: local-folder

pom.xml

@@ -5,7 +5,9 @@
 
 	<groupId>genepi</groupId>
 	<artifactId>imputationserver</artifactId>
-	<version>1.6.1</version>
+
+	<version>1.6.2-rc1</version>
+
 	<packaging>jar</packaging>
 
 	<name>University of Michigan Imputation Server</name>
@@ -79,7 +81,8 @@
 									<include>org.codehaus.plexus:plexus-classworlds:*:*</include>
 									<include>org.codehaus.groovy:groovy-all:*:*</include>
 									<include>lukfor:pgs-calc:*:*</include>
-									<include>lukfor:magic-progress:*:*</include>
+									<include>com.github.lukfor:magic-progress:*:*</include>
+									<include>com.github.lukfor:magic-reports:*:*</include>
 									<include>com.samskivert:jmustache:*:*</include>
 									<include>com.google.code.gson:gson:*:*</include>
 									<include>com.github.tommyettinger:blazingchain:*:*</include>
@@ -195,28 +198,28 @@
 		<dependency>
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>
-			<version>4.13.1</version>
+			<version>4.13.2</version>
 			<scope>test</scope>
 		</dependency>
 
 		<dependency>
 			<groupId>net.lingala.zip4j</groupId>
 			<artifactId>zip4j</artifactId>
-			<version>2.5.2</version>
+			<version>2.9.1</version>
 		</dependency>
 
 		<dependency>
 			<groupId>com.esotericsoftware.yamlbeans</groupId>
 			<artifactId>yamlbeans</artifactId>
-			<version>1.14</version>
+			<version>1.15</version>
 		</dependency>
 
 		<dependency>
 			<groupId>genepi</groupId>
 			<artifactId>genepi-io</artifactId>
-			<version>1.0.12</version>
+			<version>1.2.0</version>
 		</dependency>
-
+		
 		<dependency>
 			<groupId>genepi</groupId>
 			<artifactId>genepi-hadoop</artifactId>
@@ -226,7 +229,7 @@
 		<dependency>
 			<groupId>com.github.samtools</groupId>
 			<artifactId>htsjdk</artifactId>
-			<version>2.21.3</version>
+			<version>2.24.1</version>
 		</dependency>
 
 		<dependency>
@@ -289,14 +292,14 @@
 
 		<dependency>
 			<groupId>org.codehaus.groovy</groupId>
-			<artifactId>groovy-all</artifactId>
-			<version>2.4.14</version>
+			<artifactId>groovy-templates</artifactId>
+			<version>3.0.9</version>
 		</dependency>
-
+		
 		<dependency>
 			<groupId>lukfor</groupId>
 			<artifactId>pgs-calc</artifactId>
-			<version>0.9.2</version>
+			<version>0.9.12</version>
 		</dependency>
 
 	</dependencies>

src/main/java/genepi/imputationserver/steps/CompressionEncryption.java

@@ -304,12 +304,6 @@ public boolean run(WorkflowContext context) {
 				MetaFile metaFile = MetaFile.load(FileUtil.path(folder, "pgs-catalog.json"));
 				report.mergeWithMeta(metaFile);
 
-				CreateHtmlReportTask htmlReport = new CreateHtmlReportTask();
-				htmlReport.setReport(report);
-				htmlReport.setData(mergeScore.getResult());
-				htmlReport.setOutput(outputFileHtml);
-				TaskService.run(htmlReport);
-
 				String fileName = "scores.zip";
 				String filePath = FileUtil.path(localOutput, fileName);
 				File file = new File(filePath);

src/main/java/genepi/imputationserver/steps/imputation/ImputationPipeline.java

@@ -3,6 +3,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.io.FileUtils;
@@ -13,17 +14,16 @@
 import genepi.imputationserver.steps.vcf.VcfChunkOutput;
 import genepi.io.FileUtil;
 import genepi.riskscore.io.Chunk;
-import genepi.riskscore.io.OutputFile;
 import genepi.riskscore.io.PGSCatalog;
-import genepi.riskscore.io.ReportFile;
 import genepi.riskscore.tasks.ApplyScoreTask;
 import groovy.text.SimpleTemplateEngine;
 import htsjdk.samtools.util.StopWatch;
 import lukfor.progress.TaskService;
+import lukfor.progress.tasks.Task;
 
 public class ImputationPipeline {
 
-	public static final String PIPELINE_VERSION = "michigan-imputationserver-1.6.1";
+	public static final String PIPELINE_VERSION = "michigan-imputationserver-1.6.2-rc1";
 
 	public static final String IMPUTATION_VERSION = "minimac4-1.0.2";
 
@@ -171,7 +171,7 @@ public boolean execute(VcfChunk chunk, VcfChunkOutput output) throws Interrupted
 			return false;
 		}
 
-		if (scores != null && !scores.equals("no_score")) {
+		if (scores != null && scores.length >= 0) {
 
 			System.out.println("  Starting PGS calculation '" + scores + "'...");
 
@@ -344,15 +344,19 @@ private boolean runPgsCalc(VcfChunkOutput output) {
 			task.setVcfFilename(output.getImputedVcfFilename());
 			task.setChunk(scoreChunk);
 			task.setRiskScoreFilenames(scores);
+			task.setOutputReportFilename(output.getScoreFilename() + ".json");
+			task.setOutput(output.getScoreFilename());
 
 			TaskService.setAnsiSupport(false);
-			TaskService.run(task);
+			List<Task> runningTasks = TaskService.run(task);
 
-			OutputFile outputFile = new OutputFile(task.getRiskScores(), task.getSummaries());
-			outputFile.save(output.getScoreFilename());
-
-			ReportFile reportFile = new ReportFile(task.getSummaries());
-			reportFile.save(output.getScoreFilename() + ".json");
+			for (Task runningTask : runningTasks) {
+				if (!runningTask.getStatus().isSuccess()) {
+					System.out.println("PGS-Calc failed: " + runningTask.getStatus().getThrowable());
+					runningTask.getStatus().getThrowable().printStackTrace();
+					return false;
+				}
+			}
 
 			return true;
 

src/test/java/genepi/imputationserver/steps/ImputationTest.java

@@ -757,7 +757,7 @@ public void testPipelineWithEagleAndScores() throws IOException, ZipException {
 		}
 		readerExpected.close();
 		readerActual.close();
-		// FileUtil.deleteDirectory("test-data/tmp");
+		FileUtil.deleteDirectory("test-data/tmp");
 
 	}