Add links to nextflow.log, trace and report

genepi · Sep 9, 2023 · cba304f · cba304f
1 parent c1795bf
commit cba304f
Show file tree

Hide file tree

Showing 6 changed files with 53 additions and 5 deletions.
diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java
@@ -104,6 +104,8 @@ abstract public class AbstractJob extends PriorityRunnable {
 
 	protected List<CloudgeneParameterOutput> outputParams = new Vector<CloudgeneParameterOutput>();
 
+	protected CloudgeneParameterOutput logParam = new CloudgeneParameterOutput();
+
 	protected List<CloudgeneStep> steps = new Vector<CloudgeneStep>();
 
 	protected BufferedOutputStream stdOutStream;
@@ -255,6 +257,10 @@ public void setOutputParams(List<CloudgeneParameterOutput> outputParams) {
 		this.outputParams = outputParams;
 	}
 
+	public CloudgeneParameterOutput getLogParam() {
+		return logParam;
+	}
+
 	public void setPositionInQueue(int positionInQueue) {
 		this.positionInQueue = positionInQueue;
 	}

diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java
@@ -88,6 +88,13 @@ public CloudgeneJob(User user, String id, WdlApp app, Map<String, String> params
 			outputParams.add(newOutput);
 		}
 
+		logParam = new CloudgeneParameterOutput();
+		logParam.setAdminOnly(true);
+		logParam.setDownload(true);
+		logParam.setDescription("Logs");
+		logParam.setType(WdlParameterOutputType.LOCAL_FOLDER);
+		logParam.setJob(this);
+
 	}
 
 	@Override
@@ -118,19 +125,20 @@ public boolean setup() {
 			switch (param.getType()) {
 			case HDFS_FILE:
 			case HDFS_FOLDER:
-				log.info("[Job {}] Setting output param '{}' failed. HDFS support was removed in Cloudgene 3'", getId(), param.getName());
+				log.info("[Job {}] Setting output param '{}' failed. HDFS support was removed in Cloudgene 3'", getId(),
+						param.getName());
 				throw new RuntimeException("HDFS support was removed in Cloudgene 3");
 
-			case LOCAL_FILE:				
+			case LOCAL_FILE:
 				String filename = workspace.createFile(param.getName(), param.getName());
 				param.setValue(filename);
 				log.info("[Job {}] Set output file '{}' to '{}'", getId(), param.getName(), param.getValue());
 				break;
 
 			case LOCAL_FOLDER:
 				String folder = workspace.createFolder(param.getName());
-				log.info("[Job {}] Set output folder '{}' to '{}'", getId(), param.getName(), param.getValue());
 				param.setValue(folder);
+				log.info("[Job {}] Set output folder '{}' to '{}'", getId(), param.getName(), param.getValue());
 				break;
 			}
 
@@ -142,7 +150,7 @@ public boolean setup() {
 
 	@Override
 	public boolean execute() {
-		
+
 		try {
 
 			// evaluate WDL and replace all variables (e.g. ${job_id})
@@ -224,7 +232,7 @@ public boolean executeFailureStep(WdlStep failedStep) {
 	public boolean cleanUp() {
 
 		log.info("[Job {}] Cleaning up...", getId());
-		
+
 		try {
 			workspace.cleanup(getId());
 		} catch (IOException e) {
@@ -249,6 +257,13 @@ public boolean after() {
 			}
 		}
 
+		log.info("[Job {}] Export logs...", getId());
+		List<Download> logs = workspace.getLogs();
+		for (Download log: logs){
+			log.setCount(-1);
+		}
+		logParam.setFiles(logs);
+
 		return true;
 	}
 

diff --git a/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java b/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java
@@ -79,6 +79,16 @@ protected void jobCompleted(AbstractJob job) {
 
 		}
 
+		if (job.getLogParam().getFiles() != null) {
+
+			for (Download download : job.getLogParam().getFiles()) {
+				download.setParameterId(job.getLogParam().getId());
+				download.setParameter(job.getLogParam());
+				downloadDao.insert(download);
+			}
+
+		}
+
 		if (job.getSteps() != null) {
 			StepDao dao2 = new StepDao(database);
 			for (CloudgeneStep step : job.getSteps()) {
@@ -142,6 +152,9 @@ protected void jobSubmitted(AbstractJob job) {
 			parameter.setJobId(job.getId());
 			dao.insert(parameter);
 		}
+
+
+		dao.insert(job.getLogParam());
 	}
 
 	@Override

diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java
@@ -35,6 +35,8 @@ public interface IWorkspace {
 
 	public List<Download> getDownloads(String url);
 
+	public List<Download> getLogs();
+
 	public void cleanup(String job) throws IOException;
 
 	public boolean exists(String path) throws IOException;

diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java
@@ -221,4 +221,10 @@ protected String absolute(String path) {
 		return new File(path).getAbsolutePath();
 	}
 
+	@Override
+	public List<Download> getLogs() {
+		String location = FileUtil.path(workspace, LOGS_DIRECTORY);
+		return getDownloads(location);
+	}
+
 }
diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java
@@ -245,5 +245,11 @@ public List<Download> getDownloads(String url) {
 
 		return downloads;
 	}
+
+	@Override
+	public List<Download> getLogs() {
+		String url = location + "/" + job + "/" + LOGS_DIRECTORY;
+		return getDownloads(url);
+	}
 
 }