From cba304f9d9ce840c0ccb35451c6093556e72207e Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 9 Sep 2023 16:53:34 +0200 Subject: [PATCH] Add links to nextflow.log, trace and report --- .../cloudgene/mapred/jobs/AbstractJob.java | 6 +++++ .../cloudgene/mapred/jobs/CloudgeneJob.java | 25 +++++++++++++++---- .../mapred/jobs/PersistentWorkflowEngine.java | 13 ++++++++++ .../mapred/jobs/workspace/IWorkspace.java | 2 ++ .../mapred/jobs/workspace/LocalWorkspace.java | 6 +++++ .../mapred/jobs/workspace/S3Workspace.java | 6 +++++ 6 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java index 40afc595..1a4af4a1 100644 --- a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java +++ b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java @@ -104,6 +104,8 @@ abstract public class AbstractJob extends PriorityRunnable { protected List outputParams = new Vector(); + protected CloudgeneParameterOutput logParam = new CloudgeneParameterOutput(); + protected List steps = new Vector(); protected BufferedOutputStream stdOutStream; @@ -255,6 +257,10 @@ public void setOutputParams(List outputParams) { this.outputParams = outputParams; } + public CloudgeneParameterOutput getLogParam() { + return logParam; + } + public void setPositionInQueue(int positionInQueue) { this.positionInQueue = positionInQueue; } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java index 71f514d4..7123f756 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java @@ -88,6 +88,13 @@ public CloudgeneJob(User user, String id, WdlApp app, Map params outputParams.add(newOutput); } + logParam = new CloudgeneParameterOutput(); + logParam.setAdminOnly(true); + logParam.setDownload(true); + logParam.setDescription("Logs"); + logParam.setType(WdlParameterOutputType.LOCAL_FOLDER); + logParam.setJob(this); + } @Override @@ -118,10 +125,11 @@ public boolean setup() { switch (param.getType()) { case HDFS_FILE: case HDFS_FOLDER: - log.info("[Job {}] Setting output param '{}' failed. HDFS support was removed in Cloudgene 3'", getId(), param.getName()); + log.info("[Job {}] Setting output param '{}' failed. HDFS support was removed in Cloudgene 3'", getId(), + param.getName()); throw new RuntimeException("HDFS support was removed in Cloudgene 3"); - case LOCAL_FILE: + case LOCAL_FILE: String filename = workspace.createFile(param.getName(), param.getName()); param.setValue(filename); log.info("[Job {}] Set output file '{}' to '{}'", getId(), param.getName(), param.getValue()); @@ -129,8 +137,8 @@ public boolean setup() { case LOCAL_FOLDER: String folder = workspace.createFolder(param.getName()); - log.info("[Job {}] Set output folder '{}' to '{}'", getId(), param.getName(), param.getValue()); param.setValue(folder); + log.info("[Job {}] Set output folder '{}' to '{}'", getId(), param.getName(), param.getValue()); break; } @@ -142,7 +150,7 @@ public boolean setup() { @Override public boolean execute() { - + try { // evaluate WDL and replace all variables (e.g. ${job_id}) @@ -224,7 +232,7 @@ public boolean executeFailureStep(WdlStep failedStep) { public boolean cleanUp() { log.info("[Job {}] Cleaning up...", getId()); - + try { workspace.cleanup(getId()); } catch (IOException e) { @@ -249,6 +257,13 @@ public boolean after() { } } + log.info("[Job {}] Export logs...", getId()); + List logs = workspace.getLogs(); + for (Download log: logs){ + log.setCount(-1); + } + logParam.setFiles(logs); + return true; } diff --git a/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java b/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java index 5506598c..fb410915 100644 --- a/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java +++ b/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java @@ -79,6 +79,16 @@ protected void jobCompleted(AbstractJob job) { } + if (job.getLogParam().getFiles() != null) { + + for (Download download : job.getLogParam().getFiles()) { + download.setParameterId(job.getLogParam().getId()); + download.setParameter(job.getLogParam()); + downloadDao.insert(download); + } + + } + if (job.getSteps() != null) { StepDao dao2 = new StepDao(database); for (CloudgeneStep step : job.getSteps()) { @@ -142,6 +152,9 @@ protected void jobSubmitted(AbstractJob job) { parameter.setJobId(job.getId()); dao.insert(parameter); } + + + dao.insert(job.getLogParam()); } @Override diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java index 2667ab82..701a6fc1 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java @@ -35,6 +35,8 @@ public interface IWorkspace { public List getDownloads(String url); + public List getLogs(); + public void cleanup(String job) throws IOException; public boolean exists(String path) throws IOException; diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java index 057690d4..ff1f8301 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java @@ -221,4 +221,10 @@ protected String absolute(String path) { return new File(path).getAbsolutePath(); } + @Override + public List getLogs() { + String location = FileUtil.path(workspace, LOGS_DIRECTORY); + return getDownloads(location); + } + } diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java index c9b3e091..ec8ff6e4 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java @@ -245,5 +245,11 @@ public List getDownloads(String url) { return downloads; } + + @Override + public List getLogs() { + String url = location + "/" + job + "/" + LOGS_DIRECTORY; + return getDownloads(url); + } }