From ab12ce54aa5bb4333e1edd7cac074e0ceff5ec62 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Wed, 23 Aug 2023 12:25:25 +0200 Subject: [PATCH 01/35] Remove haddop support and refactor workspace --- pom.xml | 88 +---- .../mapred/CommandLineInterface.java | 2 - .../mapred/apps/ApplicationInstaller.java | 146 ------- .../mapred/apps/ApplicationRepository.java | 2 +- .../mapred/cli/CloneApplications.java | 2 +- .../cloudgene/mapred/cli/CommandLineUtil.java | 26 +- .../cloudgene/mapred/cli/RunApplication.java | 54 +-- .../cloudgene/mapred/cli/StartServer.java | 21 - .../cloudgene/mapred/cli/VerifyCluster.java | 52 --- .../cloudgene/mapred/jobs/AbstractJob.java | 88 +++-- .../mapred/jobs/CloudgeneContext.java | 80 +--- .../cloudgene/mapred/jobs/CloudgeneJob.java | 363 +++--------------- .../mapred/jobs/CloudgeneStepFactory.java | 17 +- .../cloudgene/mapred/jobs/Environment.java | 19 +- .../mapred/jobs/engine/graph/GraphNode.java | 6 +- .../engine/plugins/ParameterValueInput.java | 22 -- .../engine/plugins/ParameterValueOutput.java | 22 -- .../workspace/ExternalWorkspaceFactory.java | 42 +- .../jobs/workspace/IExternalWorkspace.java | 38 ++ .../mapred/jobs/workspace/LocalWorkspace.java | 200 ++++++++++ .../mapred/jobs/workspace/S3Workspace.java | 136 +++++-- .../jobs/workspace/WorkspaceWrapper.java | 47 +++ .../mapred/plugins/PluginManager.java | 2 - .../mapred/plugins/docker/DockerBinary.java | 2 +- .../mapred/plugins/docker/DockerStep.java | 21 +- .../mapred/plugins/hadoop/HadoopPlugin.java | 76 ---- .../plugins/nextflow/NextflowBinary.java | 2 +- .../mapred/plugins/nextflow/NextflowStep.java | 9 +- .../mapred/plugins/nextflow/NextflowTask.java | 1 - .../mapred/plugins/rscript/RScriptBinary.java | 2 +- .../server/controller/AppController.java | 4 - .../server/controller/DownloadController.java | 15 +- .../server/responses/ApplicationResponse.java | 21 +- .../server/services/ApplicationService.java | 35 -- .../server/services/DownloadService.java | 29 +- .../server/services/JobCleanUpService.java | 33 +- .../mapred/server/services/JobService.java | 147 +++---- .../mapred/server/tasks/ServerTasks.java | 43 --- .../mapred/steps/BashCommandStep.java | 38 +- .../cloudgene/mapred/steps/GroovyStep.java | 38 +- .../mapred/steps/HadoopMapReduceStep.java | 281 -------------- .../cloudgene/mapred/steps/HadoopPigStep.java | 63 --- .../mapred/steps/HadoopSparkStep.java | 62 --- .../mapred/steps/HdfsImporterStep.java | 97 ----- .../steps/JavaInternalStepDeprecrated.java | 187 --------- .../mapred/steps/RMarkdownLocalStep.java | 35 +- .../java/cloudgene/mapred/steps/SftpStep.java | 111 ------ .../java/cloudgene/mapred/util/S3Util.java | 184 +++++++++ .../java/cloudgene/mapred/util/Settings.java | 17 - .../mapred/util/command/Command.java | 204 ++++++++++ .../util/command/CommandStreamHandler.java | 73 ++++ .../cloudgene/mapred/TestApplication.java | 51 +-- .../api/v2/jobs/DownloadResultsTest.java | 34 -- .../mapred/api/v2/jobs/SubmitJobTest.java | 55 --- .../jobs/PriorityThreadPoolExecutorTest.java | 10 +- .../mapred/jobs/WorkflowEngineTest.java | 137 +------ .../mapred/jobs/WrongWorkspaceTest.java | 9 +- .../mapred/jobs/steps/CheckHdfsInputs.java | 50 --- .../mapred/jobs/steps/PrintDataset.java | 4 +- .../steps/WriteFilesToHdfsFolderStep.java | 51 --- .../jobs/steps/WriteTextToHdfsFileStep.java | 28 -- .../cloudgene/mapred/steps/TestCommand.java | 9 +- .../cloudgene/mapred/util/TestCluster.java | 70 ---- .../cloudgene/mapred/util/TestMailServer.java | 9 +- test-data/all-possible-inputs-hdfs.yaml | 41 -- test-data/app-installation-child.yaml | 13 - test-data/app-installation.yaml | 56 --- test-data/app-installation2.yaml | 12 - test-data/app-installation3.yaml | 11 - test-data/sftp-import.yaml | 27 -- test-data/write-files-to-hdfs-folder.yaml | 28 -- test-data/write-text-to-hdfs-file.yaml | 28 -- 72 files changed, 1177 insertions(+), 2861 deletions(-) delete mode 100644 src/main/java/cloudgene/mapred/apps/ApplicationInstaller.java delete mode 100644 src/main/java/cloudgene/mapred/cli/VerifyCluster.java create mode 100644 src/main/java/cloudgene/mapred/jobs/workspace/IExternalWorkspace.java create mode 100644 src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java create mode 100644 src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceWrapper.java delete mode 100644 src/main/java/cloudgene/mapred/plugins/hadoop/HadoopPlugin.java delete mode 100644 src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java delete mode 100644 src/main/java/cloudgene/mapred/steps/HadoopPigStep.java delete mode 100644 src/main/java/cloudgene/mapred/steps/HadoopSparkStep.java delete mode 100644 src/main/java/cloudgene/mapred/steps/HdfsImporterStep.java delete mode 100644 src/main/java/cloudgene/mapred/steps/JavaInternalStepDeprecrated.java delete mode 100644 src/main/java/cloudgene/mapred/steps/SftpStep.java create mode 100644 src/main/java/cloudgene/mapred/util/S3Util.java create mode 100644 src/main/java/cloudgene/mapred/util/command/Command.java create mode 100644 src/main/java/cloudgene/mapred/util/command/CommandStreamHandler.java delete mode 100644 src/test/java/cloudgene/mapred/jobs/steps/CheckHdfsInputs.java delete mode 100644 src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToHdfsFolderStep.java delete mode 100644 src/test/java/cloudgene/mapred/jobs/steps/WriteTextToHdfsFileStep.java delete mode 100644 src/test/java/cloudgene/mapred/util/TestCluster.java delete mode 100644 test-data/all-possible-inputs-hdfs.yaml delete mode 100644 test-data/app-installation-child.yaml delete mode 100644 test-data/app-installation.yaml delete mode 100644 test-data/app-installation2.yaml delete mode 100644 test-data/app-installation3.yaml delete mode 100644 test-data/sftp-import.yaml delete mode 100644 test-data/write-files-to-hdfs-folder.yaml delete mode 100644 test-data/write-text-to-hdfs-file.yaml diff --git a/pom.xml b/pom.xml index 0b0825f9..a842ceca 100644 --- a/pom.xml +++ b/pom.xml @@ -67,15 +67,9 @@ - jfrog-genepi-maven - jfrog-genepi-maven - https://genepi.jfrog.io/artifactory/maven/ - - - - cloudera - Cloudera repository - https://repository.cloudera.com/artifactory/cloudera-repos/ + genepi-maven + genepi-maven + https://genepi.i-med.ac.at/maven @@ -106,26 +100,6 @@ - - genepi - genepi-hadoop - mr1-1.4.1 - - - org.apache.hadoop - hadoop-client - - - org.apache.httpcomponents - httpcore - - - org.apache.httpcomponents - httpclient - - - - commons-dbutils commons-dbutils @@ -217,62 +191,6 @@ 5.8.3 - - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - test - test-jar - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - test - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - test - - - org.apache.httpcomponents - httpcore - - - org.slf4j - slf4j-reload4j - - - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - test - test-jar - - - org.slf4j - slf4j-reload4j - - - - - - org.apache.hadoop - hadoop-test - 2.6.0-mr1-cdh5.16.1 - test - - org.junit.jupiter junit-jupiter diff --git a/src/main/java/cloudgene/mapred/CommandLineInterface.java b/src/main/java/cloudgene/mapred/CommandLineInterface.java index b25d2fec..4f183faf 100644 --- a/src/main/java/cloudgene/mapred/CommandLineInterface.java +++ b/src/main/java/cloudgene/mapred/CommandLineInterface.java @@ -10,7 +10,6 @@ import cloudgene.mapred.cli.ShowVersion; import cloudgene.mapred.cli.StartServer; import cloudgene.mapred.cli.ValidateApplication; -import cloudgene.mapred.cli.VerifyCluster; import cloudgene.mapred.server.Application; import cloudgene.mapred.util.BuildUtil; import genepi.base.Toolbox; @@ -42,7 +41,6 @@ public static void main(String[] args) throws Exception { toolbox.addTool("remove", RemoveApplication.class); toolbox.addTool("server", StartServer.class); toolbox.addTool("validate", ValidateApplication.class); - toolbox.addTool("verify-cluster", VerifyCluster.class); toolbox.addTool("plugins", ShowPlugins.class); toolbox.addTool("version", ShowVersion.class); toolbox.start(); diff --git a/src/main/java/cloudgene/mapred/apps/ApplicationInstaller.java b/src/main/java/cloudgene/mapred/apps/ApplicationInstaller.java deleted file mode 100644 index 7da435e1..00000000 --- a/src/main/java/cloudgene/mapred/apps/ApplicationInstaller.java +++ /dev/null @@ -1,146 +0,0 @@ -package cloudgene.mapred.apps; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.util.List; -import java.util.Map; - -import org.apache.commons.io.FileUtils; - -import cloudgene.mapred.jobs.Environment; -import cloudgene.mapred.plugins.PluginManager; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; -import cloudgene.mapred.util.Settings; -import cloudgene.mapred.wdl.WdlApp; -import genepi.hadoop.HdfsUtil; -import genepi.hadoop.S3Util; -import genepi.hadoop.io.HdfsLineWriter; -import genepi.io.FileUtil; - -public class ApplicationInstaller { - - public static boolean isInstalled(WdlApp app, Settings settings) { - - PluginManager manager = PluginManager.getInstance(); - //skip hdfs installation when Hadoop plugin is not activated - if (!manager.isEnabled(HadoopPlugin.ID)) { - return true; - } - - Map environment = Environment.getApplicationVariables(app, settings); - String target = environment.get("hdfs_app_folder"); - String installationFile = HdfsUtil.path(target, "installed"); - return HdfsUtil.exists(installationFile); - } - - public static void uninstall(WdlApp app, Settings settings) throws IOException { - - PluginManager manager = PluginManager.getInstance(); - if (!manager.isEnabled(HadoopPlugin.ID)) { - return; - } - - Map environment = Environment.getApplicationVariables(app, settings); - String target = environment.get("hdfs_app_folder"); - HdfsUtil.delete(target); - } - - public static void install(WdlApp app, Settings settings) throws IOException { - - PluginManager manager = PluginManager.getInstance(); - if (!manager.isEnabled(HadoopPlugin.ID)) { - return; - } - - Map environment = Environment.getApplicationVariables(app, settings); - String target = environment.get("hdfs_app_folder"); - String installationFile = HdfsUtil.path(target, "installed"); - HdfsUtil.delete(target); - ApplicationInstaller.runCommands(app.getInstallation(), environment); - HdfsLineWriter lineWriter = new HdfsLineWriter(installationFile); - lineWriter.write(System.currentTimeMillis() + ""); - lineWriter.close(); - } - - private static void runCommands(List> commands, Map environment) - throws IOException { - - for (Map commandObject : commands) { - runCommand(commandObject, environment); - } - } - - private static void runCommand(Map commandObject, Map environment) - throws IOException { - if (commandObject.keySet().size() == 1) { - for (String command : commandObject.keySet()) { - Object parametersObject = commandObject.get(command); - Map parameters = (Map) parametersObject; - runCommand(command, parameters, environment); - } - } else { - throw new IOException("Unknown command structure."); - } - } - - private static void runCommand(String command, Map parameters, Map environment) - throws IOException { - switch (command) { - case "import": - String source = Environment.env((String) parameters.get("source"), environment); - String target = Environment.env((String) parameters.get("target"), environment); - System.out.println("Import data from " + source + " to " + target + "..."); - runImportCommand(source, target); - break; - default: - throw new IOException("Unknown command '" + command + "'"); - } - } - - private static void runImportCommand(String source, String target) throws IOException { - String tempFilename = FileUtil.path("temp", "hdfs_import.tempfile"); - if (source.startsWith("http://") || source.startsWith("https://")) { - // download - FileUtils.copyURLToFile(new URL(source), new File(tempFilename)); - - if (source.endsWith(".zip")) { - HdfsUtil.createDirectory(target); - HdfsUtil.putZip(tempFilename, target); - } else if (source.endsWith(".gz")) { - HdfsUtil.createDirectory(target); - HdfsUtil.putTarGz(tempFilename, target); - } else { - HdfsUtil.put(tempFilename, target); - } - - } else if (source.startsWith("s3://")) { - - S3Util.copyToFile(source, new File(tempFilename)); - - if (source.endsWith(".zip")) { - HdfsUtil.createDirectory(target); - HdfsUtil.putZip(tempFilename, target); - } else if (source.endsWith(".gz")) { - HdfsUtil.createDirectory(target); - HdfsUtil.putTarGz(tempFilename, target); - } else { - HdfsUtil.put(tempFilename, target); - } - - } else { - if (source.endsWith(".zip")) { - HdfsUtil.createDirectory(target); - HdfsUtil.putZip(source, target); - } else if (source.endsWith(".gz")) { - HdfsUtil.createDirectory(target); - HdfsUtil.putTarGz(source, target); - } else { - HdfsUtil.put(source, target); - } - } - - FileUtil.deleteFile(tempFilename); - } - -} diff --git a/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java b/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java index bc2cc641..2f979201 100644 --- a/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java +++ b/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java @@ -22,8 +22,8 @@ import cloudgene.mapred.util.GitHubException; import cloudgene.mapred.util.GitHubUtil; import cloudgene.mapred.util.GitHubUtil.Repository; +import cloudgene.mapred.util.S3Util; import cloudgene.mapred.wdl.WdlApp; -import genepi.hadoop.S3Util; import genepi.io.FileUtil; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; diff --git a/src/main/java/cloudgene/mapred/cli/CloneApplications.java b/src/main/java/cloudgene/mapred/cli/CloneApplications.java index e2cdae85..5250b7c8 100644 --- a/src/main/java/cloudgene/mapred/cli/CloneApplications.java +++ b/src/main/java/cloudgene/mapred/cli/CloneApplications.java @@ -10,7 +10,7 @@ import com.esotericsoftware.yamlbeans.YamlReader; import cloudgene.mapred.apps.Application; -import genepi.hadoop.S3Util; +import cloudgene.mapred.util.S3Util; public class CloneApplications extends BaseTool { diff --git a/src/main/java/cloudgene/mapred/cli/CommandLineUtil.java b/src/main/java/cloudgene/mapred/cli/CommandLineUtil.java index 3cb21a9d..0a530bcf 100644 --- a/src/main/java/cloudgene/mapred/cli/CommandLineUtil.java +++ b/src/main/java/cloudgene/mapred/cli/CommandLineUtil.java @@ -12,8 +12,6 @@ import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlParameterInput; import cloudgene.mapred.wdl.WdlParameterInputType; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; public class CommandLineUtil { @@ -79,29 +77,7 @@ public static Map createParams(WdlApp app, CommandLine line, Str String entryName = tmpFile.getName(); if (input.isHdfs()) { - String targetPath = HdfsUtil.path(hdfs, "input", input.getId()); - if (tmpFile.isDirectory()) { - String[] files = FileUtil.getFiles(value, ""); - for (String sourceFile : files) { - if (!new File(sourceFile).isDirectory()) { - String name = FileUtil.getFilename(sourceFile); - String target = HdfsUtil.path(targetPath, name); - HdfsUtil.put(sourceFile, target); - } - - } - } else { - String target = HdfsUtil.path(targetPath, entryName); - HdfsUtil.put(value, target); - } - - if (input.isFolder()) { - // folder - props.put(input.getId(), HdfsUtil.path(hdfs, "input", input.getId())); - } else { - // file - props.put(input.getId(), HdfsUtil.path(hdfs, "input", input.getId(), entryName)); - } + throw new RuntimeException("HDFS not supported in CG3"); } else { props.put(input.getId(), value); diff --git a/src/main/java/cloudgene/mapred/cli/RunApplication.java b/src/main/java/cloudgene/mapred/cli/RunApplication.java index cd325e1a..738a17a3 100644 --- a/src/main/java/cloudgene/mapred/cli/RunApplication.java +++ b/src/main/java/cloudgene/mapred/cli/RunApplication.java @@ -27,8 +27,6 @@ import cloudgene.mapred.wdl.WdlParameterInputType; import cloudgene.mapred.wdl.WdlParameterOutput; import cloudgene.mapred.wdl.WdlReader; -import genepi.hadoop.HadoopCluster; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; public class RunApplication extends BaseTool { @@ -41,8 +39,6 @@ public class RunApplication extends BaseTool { private boolean logging = false; - private boolean force = false; - public RunApplication(String[] args) { super(args); } @@ -138,28 +134,11 @@ public int start() { noOutputOption.setRequired(false); options.addOption(noOutputOption); - // add general options: run on docker - Option forceOption = new Option(null, "force", false, - "Force Cloudgene to reinstall application in HDFS even if it already installed."); - forceOption.setRequired(false); - options.addOption(forceOption); - - // add general options: hadoop configuration - Option confOption = new Option(null, "conf", true, "Hadoop configuration folder"); - confOption.setRequired(false); - options.addOption(confOption); - // add general options: output folder Option outputFolderOption = new Option(null, "output", true, "Output folder"); outputFolderOption.setRequired(false); options.addOption(outputFolderOption); - // add general options: hadoop user - Option usernameOption = new Option(null, "user", true, - "Hadoop username [default: " + DEFAULT_HADOOP_USER + "]"); - usernameOption.setRequired(false); - options.addOption(usernameOption); - // parse the command line arguments CommandLine line = null; try { @@ -182,24 +161,6 @@ public int start() { output = true; } - if (line.hasOption("force")) { - force = true; - } - - if (line.hasOption("conf")) { - - String conf = line.getOptionValue("conf"); - String username = line.getOptionValue("user", null); - printText(0, spaces("[INFO]", 8) + "Use Haddop configuration folder " + conf - + (username != null ? " with username " + username : "")); - HadoopCluster.setConfPath("Unknown", conf, username); - - } else { - if (settings.getCluster() == null) { - // printText(0, spaces("[INFO]", 8) + "No external Haddop cluster set."); - } - } - // show supported plugins PluginManager manager = PluginManager.getInstance(); manager.initPlugins(settings); @@ -221,12 +182,6 @@ public int start() { SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd-HHmmss"); String id = "job-" + sdf.format(new Date()); - String hdfs = ""; - try { - hdfs = HdfsUtil.path("cloudgene-cli", id); - } catch (NoClassDefFoundError e) { - - } String local = FileUtil.path(id); if (line.hasOption("output")) { @@ -237,7 +192,7 @@ public int start() { // file params with values from cmdline try { - Map params = CommandLineUtil.createParams(app, line, local, hdfs); + Map params = CommandLineUtil.createParams(app, line, local, ""); // dummy user User user = new User(); @@ -308,16 +263,11 @@ public void writeLog(String line) { job.setId(id); job.setName(id); job.setLocalWorkspace(local); - job.setHdfsWorkspace(hdfs); + //job.setHdfsWorkspace(hdfs); job.setSettings(settings); - job.setRemoveHdfsWorkspace(true); job.setApplication(app.getName() + " " + app.getVersion()); job.setApplicationId(app.getId()); - job.forceInstallation(force); - if (force) { - printText(0, spaces("[INFO]", 8) + "Force Cloudgene to reinstall application in HDFS"); - } printText(0, spaces("[INFO]", 8) + "Submit job " + id + "..."); diff --git a/src/main/java/cloudgene/mapred/cli/StartServer.java b/src/main/java/cloudgene/mapred/cli/StartServer.java index a055d90c..d3c493b8 100644 --- a/src/main/java/cloudgene/mapred/cli/StartServer.java +++ b/src/main/java/cloudgene/mapred/cli/StartServer.java @@ -16,7 +16,6 @@ import cloudgene.mapred.util.Config; import cloudgene.mapred.util.Settings; import genepi.base.Tool; -import genepi.hadoop.HadoopCluster; import io.micronaut.context.env.Environment; import io.micronaut.runtime.Micronaut; @@ -35,9 +34,7 @@ public StartServer(String[] args) { @Override public void createParameters() { - addOptionalParameter("user", "Hadoop username [default: " + DEFAULT_HADOOP_USER + "]", Tool.STRING); addOptionalParameter("port", "running webinterface on this port [default: 8082]", Tool.STRING); - addOptionalParameter("conf", "Hadoop configuration folder", Tool.STRING); } @Override @@ -49,24 +46,6 @@ public int run() { System.setProperty(ContextInitializer.CONFIG_FILE_PROPERTY, "logback-dev.xml"); } - - if (getValue("conf") != null) { - - String conf = getValue("conf").toString(); - - String username = null; - if (getValue("user") != null) { - username = getValue("user").toString(); - } - System.out.println( - "Use Haddop configuration folder " + conf + (username != null ? " with username " + username : "")); - HadoopCluster.setConfPath("Unknown", conf, username); - - } else { - // if (settings.getCluster() == null) { - // System.out.println("No external Haddop cluster set."); - // } - } try { diff --git a/src/main/java/cloudgene/mapred/cli/VerifyCluster.java b/src/main/java/cloudgene/mapred/cli/VerifyCluster.java deleted file mode 100644 index 84bf5f94..00000000 --- a/src/main/java/cloudgene/mapred/cli/VerifyCluster.java +++ /dev/null @@ -1,52 +0,0 @@ -package cloudgene.mapred.cli; - -import genepi.hadoop.HadoopCluster; -import genepi.hadoop.HadoopUtil; - -public class VerifyCluster extends BaseTool { - - public VerifyCluster(String[] args) { - super(args); - } - - @Override - public void createParameters() { - } - - @Override - public int run() { - - try { - HadoopCluster.verifyCluster(); - - StringBuffer state = new StringBuffer(); - state.append("Mode: " + (HadoopUtil.getInstance().isInSafeMode() ? "Safe Mode" : "Running")); - state.append("JobTracker: " + HadoopCluster.getJobTracker() + "\n"); - state.append("Default FS: " + HadoopCluster.getDefaultFS() + "\n"); - state.append("State: " + HadoopCluster.getJobTrackerStatus().toString() + "\n"); - state.append("MapTask: " + HadoopCluster.getMaxMapTasks() + "\n"); - state.append("ReduceTask: " + HadoopCluster.getMaxReduceTasks() + "\n"); - state.append("Nodes\n"); - for (String tracker : HadoopCluster.getActiveTrackerNames()) { - state.append(" " + tracker + "\n"); - } - state.append("Blacklist:\n"); - for (String tracker : HadoopCluster.getBlacklistedTrackerNames()) { - state.append(" " + tracker + "\n"); - } - System.out.println(state.toString()); - System.out.println(); - printlnInGreen("Hadoop cluster is ready to use."); - System.out.println(); - - return 0; - - } catch (Exception e) { - System.out.println(); - printlnInRed(e.getMessage()); - System.out.println(); - return 1; - } - - } -} \ No newline at end of file diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java index 6fbcff2c..88eceba1 100644 --- a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java +++ b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java @@ -11,15 +11,20 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.Vector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import cloudgene.mapred.apps.Application; +import cloudgene.mapred.apps.ApplicationRepository; import cloudgene.mapred.core.User; import cloudgene.mapred.jobs.queue.PriorityRunnable; -import cloudgene.mapred.steps.ErrorStep; +import cloudgene.mapred.jobs.workspace.IExternalWorkspace; import cloudgene.mapred.util.Settings; +import cloudgene.mapred.wdl.WdlApp; +import cloudgene.mapred.wdl.WdlParameterInputType; import genepi.io.FileUtil; abstract public class AbstractJob extends PriorityRunnable { @@ -110,15 +115,11 @@ abstract public class AbstractJob extends PriorityRunnable { private String logs; - private boolean removeHdfsWorkspace; - private String localWorkspace; - private String hdfsWorkspace; - private boolean canceld = false; - private boolean forceInstallation = false; + protected IExternalWorkspace externalWorkspace; private String workspaceSize = null; @@ -229,7 +230,7 @@ public long getDeletedOn() { public void setUserAgent(String userAgent) { this.userAgent = userAgent; } - + public String getUserAgent() { return userAgent; } @@ -291,22 +292,47 @@ public boolean afterSubmission() { public void runSetupSteps() { - log.info("Job " + getId() + ": executing installation..."); - writeLog("Executing Job installation...."); + Settings settings = getSettings(); + ApplicationRepository repository = settings.getApplicationRepository(); - // execute installation + // resolve application links + for (CloudgeneParameterInput input : getInputParams()) { + if (input.getType() == WdlParameterInputType.APP_LIST) { + String value = input.getValue(); + String linkedAppId = value; + if (value.startsWith("apps@")) { + linkedAppId = value.replaceAll("apps@", ""); + } - boolean result = executeInstallation(forceInstallation); + if (!value.isEmpty()) { + Application linkedApp = repository.getByIdAndUser(linkedAppId, getUser()); + if (linkedApp != null) { + // update environment variables + Map envApp = Environment.getApplicationVariables(linkedApp.getWdlApp(), + settings); + Map envJob = Environment.getJobVariables(context); + Map properties = linkedApp.getWdlApp().getProperties(); + for (String property : properties.keySet()) { + Object propertyValue = properties.get(property); + if (propertyValue instanceof String) { + propertyValue = Environment.env(propertyValue.toString(), envApp); + propertyValue = Environment.env(propertyValue.toString(), envJob); + } + properties.put(property, propertyValue); + } - if (result == false || state == AbstractJob.STATE_CANCELED || state == AbstractJob.STATE_FAILED) { - ErrorStep errorStep = new ErrorStep(getError() != null ? getError() : "Error"); - errorStep.setJob(this); - errorStep.setName("Job Setup failed: " + getError()); - getSteps().add(errorStep); - setState(AbstractJob.STATE_FAILED); - onFailure(); - setSetupComplete(false); - return; + getContext().setData(input.getName(), properties); + } else { + String error = "Application " + linkedAppId + " is not installed or wrong permissions."; + log.info(error); + writeOutput(error); + setError(error); + setSetupComplete(false); + setState(AbstractJob.STATE_FAILED); + return; + } + } + } } // execute setup steps @@ -709,14 +735,6 @@ public void setSettings(Settings settings) { this.settings = settings; } - public boolean isRemoveHdfsWorkspace() { - return removeHdfsWorkspace; - } - - public void setRemoveHdfsWorkspace(boolean removeHdfsWorkspace) { - this.removeHdfsWorkspace = removeHdfsWorkspace; - } - public String getLocalWorkspace() { return localWorkspace; } @@ -725,12 +743,12 @@ public void setLocalWorkspace(String localWorkspace) { this.localWorkspace = localWorkspace; } - public String getHdfsWorkspace() { - return hdfsWorkspace; + public void setExternalWorkspace(IExternalWorkspace externalWorkspace) { + this.externalWorkspace = externalWorkspace; } - public void setHdfsWorkspace(String hdfsWorkspace) { - this.hdfsWorkspace = hdfsWorkspace; + public IExternalWorkspace getExternalWorkspace() { + return externalWorkspace; } public void setApplication(String application) { @@ -769,8 +787,6 @@ public boolean isRunning() { abstract public boolean executeSetupSteps(); - abstract public boolean executeInstallation(boolean forceInstallation); - abstract public boolean setup(); abstract public boolean before(); @@ -793,10 +809,6 @@ public void kill() { } - public void forceInstallation(boolean forceInstallation) { - this.forceInstallation = forceInstallation; - } - public long getCurrentTime() { return System.currentTimeMillis(); } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java index fb6163fd..dc4eb8d7 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java @@ -14,27 +14,16 @@ import cloudgene.mapred.util.MailUtil; import cloudgene.mapred.util.Settings; import cloudgene.sdk.internal.WorkflowContext; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; public class CloudgeneContext extends WorkflowContext { - private String hdfsTemp; - private String localTemp; - private String localInput; - - private String hdfsOutput; - - private String hdfsInput; - private String localOutput; private String workingDirectory; - private String workspace; - private Settings settings; private CloudgeneStep step; @@ -69,22 +58,9 @@ public CloudgeneContext(CloudgeneJob job) { setData("cloudgene.user.mail", user.getMail()); setData("cloudgene.user.name", user.getFullName()); - workspace = job.getHdfsWorkspace(); - - try { - hdfsTemp = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace, "temp")); - hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace)); - hdfsInput = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace)); - } catch (Error e) { - log("No hdfs folders created."); - } - localOutput = new File(job.getLocalWorkspace()).getAbsolutePath(); - localTemp = new File(FileUtil.path(job.getLocalWorkspace(), "temp")).getAbsolutePath(); - localInput = new File(FileUtil.path(job.getLocalWorkspace(), "input")).getAbsolutePath(); - inputParameters = new HashMap(); for (CloudgeneParameterInput param : job.getInputParams()) { inputParameters.put(param.getName(), param); @@ -109,18 +85,7 @@ public CloudgeneStep getCurrentStep() { return step; } - public void setupOutputParameters(boolean hasHdfsOutputs) { - - // cleanup temp directories - if (hasHdfsOutputs) { - try { - HdfsUtil.delete(getHdfsTemp()); - } catch (Exception e) { - System.out.println("Warning: problems during hdfs init."); - } catch (Error e) { - System.out.println("Warning: problems during hdfs init."); - } - } + /*public void setupOutputParameters() throws Exception { FileUtil.deleteDirectory(getLocalTemp()); @@ -135,25 +100,7 @@ public void setupOutputParameters(boolean hasHdfsOutputs) { case HDFS_FILE: case HDFS_FOLDER: - String value = ""; - - if (param.isDownload()) { - value = HdfsUtil.path(getHdfsOutput(), param.getName()); - } else { - value = HdfsUtil.path(getHdfsTemp(), param.getName()); - } - - if (!HdfsUtil.isAbsolute(value)) { - value = HdfsUtil.makeAbsolute(value); - } - // delete (needed for restart) - try { - HdfsUtil.delete(value); - } catch (Exception e) { - System.out.println("Warning: problems during hdfs init."); - } - param.setValue(value); - break; + throw new Exception("HDFS support was removed in Cloudgene 3"); case LOCAL_FILE: String parent = getLocalOutput(); @@ -184,7 +131,7 @@ public void setupOutputParameters(boolean hasHdfsOutputs) { } - } + }*/ public String getInput(String param) { @@ -250,26 +197,10 @@ public Settings getSettings() { return settings; } - public String getHdfsTemp() { - return hdfsTemp; - } - public String getLocalTemp() { return localTemp; } - public String getLocalInput() { - return localInput; - } - - public String getHdfsOutput() { - return hdfsOutput; - } - - public String getHdfsInput() { - return hdfsInput; - } - public String getLocalOutput() { return localOutput; } @@ -516,4 +447,9 @@ public void addFile(String filename) { message(chunkFilename, 27); } + @Override + public String getHdfsTemp() { + throw new RuntimeException("Not support in cg3"); + } + } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java index 6d688fd4..8cc61ed5 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java @@ -1,35 +1,26 @@ package cloudgene.mapred.jobs; -import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Vector; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import cloudgene.mapred.apps.Application; -import cloudgene.mapred.apps.ApplicationInstaller; -import cloudgene.mapred.apps.ApplicationRepository; import cloudgene.mapred.core.User; import cloudgene.mapred.jobs.engine.Executor; import cloudgene.mapred.jobs.engine.Planner; import cloudgene.mapred.jobs.engine.graph.Graph; import cloudgene.mapred.jobs.engine.graph.GraphNode; -import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; -import cloudgene.mapred.util.HashUtil; -import cloudgene.mapred.util.Settings; +import cloudgene.mapred.jobs.workspace.IExternalWorkspace; +import cloudgene.mapred.jobs.workspace.WorkspaceWrapper; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlParameterInput; -import cloudgene.mapred.wdl.WdlParameterInputType; import cloudgene.mapred.wdl.WdlParameterOutput; import cloudgene.mapred.wdl.WdlParameterOutputType; import cloudgene.mapred.wdl.WdlStep; -import cloudgene.sdk.internal.IExternalWorkspace; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; public class CloudgeneJob extends AbstractJob { @@ -44,10 +35,6 @@ public class CloudgeneJob extends AbstractJob { private static Logger log = LoggerFactory.getLogger(CloudgeneJob.class); - private String externalOutput = null; - - private IExternalWorkspace externalWorkspace; - public CloudgeneJob() { super(); } @@ -84,8 +71,6 @@ public CloudgeneJob(User user, String id, WdlApp app, Map params setUser(user); workingDirectory = app.getPath(); - externalOutput = params.get("external-output"); - // init parameters inputParams = new Vector(); for (WdlParameterInput input : app.getWorkflow().getInputs()) { @@ -112,8 +97,44 @@ public CloudgeneJob(User user, String id, WdlApp app, Map params public boolean setup() { context = new CloudgeneContext(this); - // context.updateInputParameters(); - context.setupOutputParameters(app.getWorkflow().hasHdfsOutputs()); + + FileUtil.deleteDirectory(context.getLocalTemp()); + + // create output directories + FileUtil.createDirectory(context.getLocalTemp()); + + try { + context.log("Setup External Workspace on " + externalWorkspace.getName()); + externalWorkspace.setup(this.getId()); + context.setExternalWorkspace(new WorkspaceWrapper(externalWorkspace)); + } catch (Exception e) { + writeLog(e.toString()); + log.info("Error setup external workspace", e); + setError(e.toString()); + return false; + } + + // create output directories + for (CloudgeneParameterOutput param : outputParams) { + + switch (param.getType()) { + case HDFS_FILE: + case HDFS_FOLDER: + + throw new RuntimeException("HDFS support was removed in Cloudgene 3"); + + case LOCAL_FILE: + String filename = externalWorkspace.createFile(param.getName(), param.getName()); + param.setValue(filename); + break; + + case LOCAL_FOLDER: + String folder = externalWorkspace.createFolder(param.getName()); + param.setValue(folder); + break; + } + + } return true; @@ -209,119 +230,6 @@ public boolean executeSetupSteps() { } - @Override - public boolean executeInstallation(boolean forceInstallation) { - - try { - - Settings settings = getSettings(); - ApplicationRepository repository = settings.getApplicationRepository(); - - // find dependencies - List applications = new Vector<>(); - - // install application - applications.add(app); - - for (CloudgeneParameterInput input : getInputParams()) { - if (input.getType() == WdlParameterInputType.APP_LIST) { - String value = input.getValue(); - String linkedAppId = value; - if (value.startsWith("apps@")) { - linkedAppId = value.replaceAll("apps@", ""); - } - - if (!value.isEmpty()) { - Application linkedApp = repository.getByIdAndUser(linkedAppId, getUser()); - if (linkedApp != null) { - applications.add(linkedApp.getWdlApp()); - // update environment variables - Map envApp = Environment.getApplicationVariables(linkedApp.getWdlApp(), - settings); - Map envJob = Environment.getJobVariables(context); - Map properties = linkedApp.getWdlApp().getProperties(); - for (String property : properties.keySet()) { - Object propertyValue = properties.get(property); - if (propertyValue instanceof String) { - propertyValue = Environment.env(propertyValue.toString(), envApp); - propertyValue = Environment.env(propertyValue.toString(), envJob); - } - properties.put(property, propertyValue); - } - - getContext().setData(input.getName(), properties); - } else { - String error = "Application " + linkedAppId + " is not installed or wrong permissions."; - log.info(error); - writeOutput(error); - setError(error); - return false; - } - } - } - } - - for (WdlApp app : applications) { - - log.info("Job " + getId() + ": executing installation for " + app.getId() + "..."); - - if (app.needsInstallation()) { - - writeLog(" Preparing Application '" + app.getName() + "'..."); - boolean installed = ApplicationInstaller.isInstalled(app, settings); - if (!installed || forceInstallation) { - try { - writeLog(" Installing Application " + app.getId() + "..."); - ApplicationInstaller.install(app, settings); - log.info("Installation of application " + app.getId() + " finished."); - writeLog(" Installation finished."); - } catch (IOException e) { - log.info("Installation of application " + app.getId() + " failed.", e); - writeLog(" Installation failed."); - writeLog(e.getMessage()); - setError(e.getMessage()); - return false; - } - } else { - String info = "Application '" + app.getName() + "'is already installed."; - log.info(info); - writeLog(" " + info); - } - } - - } - - // if default location is set, override user defined - if (!settings.getExternalWorkspaceLocation().isEmpty()) { - externalOutput = settings.getExternalWorkspaceLocation(); - } - - externalWorkspace = ExternalWorkspaceFactory.get(settings.getExternalWorkspaceType(), externalOutput); - - if (externalWorkspace != null) { - - try { - context.log("Setup External Workspace on " + externalWorkspace.getName()); - externalWorkspace.setup(this.getId()); - context.setExternalWorkspace(externalWorkspace); - } catch (Exception e) { - writeLog(e.toString()); - log.info("Error setup external workspace", e); - setError(e.toString()); - return false; - } - - } - - return true; - } catch (Exception e) { - writeLog("Installation of Application " + getApplicationId() + " failed."); - log.info("Installation of application " + app.getId() + " failed.", e); - setError(e.getMessage()); - return false; - } - } - @Override public boolean before() { @@ -384,34 +292,13 @@ public boolean executeFailureStep(WdlStep failedStep) { @Override public boolean cleanUp() { - // delete local temp folders - - writeLog("Cleaning up uploaded local files..."); - FileUtil.deleteDirectory(context.getLocalInput()); - - writeLog("Cleaning up temporary local files..."); - FileUtil.deleteDirectory(context.getLocalTemp()); - - if (app.getWorkflow().hasHdfsOutputs()) { - - try { - // delete hdfs temp folders - writeLog("Cleaning up temporary hdfs files..."); - HdfsUtil.delete(context.getHdfsTemp()); - - // delete hdfs workspace - if (isRemoveHdfsWorkspace()) { - if (context.getHdfsOutput() != null) { - writeLog("Cleaning up hdfs files..."); - HdfsUtil.delete(context.getHdfsOutput()); - HdfsUtil.delete(context.getHdfsInput()); - } - } - } catch (Exception e) { - log.warn("Warning: problems during hdfs cleanup."); - } catch (Error e) { - log.warn("Warning: problems during hdfs cleanup."); - } + try { + externalWorkspace.cleanup(getId()); + } catch (IOException e) { + writeLog("Cleanup failed."); + writeLog(e.getMessage()); + setError(e.getMessage()); + return false; } return true; @@ -438,118 +325,26 @@ public boolean exportParameter(CloudgeneParameterOutput out) { writeLog(" Exporting parameter " + out.getName() + "..."); - String localOutput = context.getLocalOutput(); - String workspace = getHdfsWorkspace(); - - if (out.getType() == WdlParameterOutputType.HDFS_FOLDER) { - - String localOutputDirectory = FileUtil.path(localOutput, out.getName()); - - FileUtil.createDirectory(localOutputDirectory); - - String filename = context.getOutput(out.getName()); - String hdfsPath = null; - if (filename.startsWith("hdfs://") || filename.startsWith("file:/")) { - hdfsPath = filename; - - } else { - - hdfsPath = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace, filename)); - } - - if (HdfsUtil.exists(hdfsPath)) { - - if (out.isZip()) { - - String zipName = FileUtil.path(localOutputDirectory, out.getName() + ".zip"); - - if (out.isMergeOutput()) { - - HdfsUtil.compressAndMerge(zipName, hdfsPath, out.isRemoveHeader()); - - } else { - - HdfsUtil.compress(zipName, hdfsPath); - - } - - } else { - - if (out.isMergeOutput()) { - - HdfsUtil.exportDirectoryAndMerge(localOutputDirectory, out.getName(), hdfsPath, - out.isRemoveHeader()); - - } else { - - HdfsUtil.exportDirectory(localOutputDirectory, out.getName(), hdfsPath); - - } - - } - - } - - } - - if (out.getType() == WdlParameterOutputType.HDFS_FILE) { + if (out.getType() == WdlParameterOutputType.HDFS_FOLDER || out.getType() == WdlParameterOutputType.HDFS_FILE) { - String localOutputDirectory = FileUtil.path(localOutput, out.getName()); - - FileUtil.createDirectory(localOutputDirectory); - - String filename = context.getOutput(out.getName()); - String hdfsPath = null; - if (filename.startsWith("hdfs://") || filename.startsWith("file:/")) { - - hdfsPath = filename; - - } else { - - hdfsPath = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace, filename)); - } - - if (!out.isZip()) { - - if (HdfsUtil.exists(hdfsPath)) { - HdfsUtil.exportFile(localOutputDirectory, hdfsPath); - } - - } else { - if (HdfsUtil.exists(hdfsPath)) { - HdfsUtil.compressFile(localOutputDirectory, hdfsPath); - } - } + throw new RuntimeException("HDFS support was removed in Cloudgene 3"); } out.setJobId(getId()); - String name = out.getName(); - String n = FileUtil.path(localOutput, name); - - File f = new File(n); - - List downloads = new Vector(); - - if (f.exists() && f.isDirectory()) { - - try { - exportFolder(out, "", name, f, downloads); - } catch (Exception e) { - - writeLog("Export paramater '" + out.getName() + "' failed:" + e); - log.info("Export paramater '" + out.getName() + "' failed:" + e); - setError("Export paramater '" + out.getName() + "' failed:" + e); - return false; - } - writeLog(" Added " + downloads.size() + " downloads."); + List downloads = externalWorkspace.getDownloads(out.getValue()); + for (Download download : downloads) { + download.setParameter(out); + download.setCount(MAX_DOWNLOAD); } + writeLog(" Added " + downloads.size() + " downloads."); List customDownloads = context.getDownloads(out.getName()); if (customDownloads != null) { for (Download download : customDownloads) { download.setParameter(out); + download.setCount(MAX_DOWNLOAD); } writeLog(" Added " + customDownloads.size() + " custom downloads."); downloads.addAll(customDownloads); @@ -561,50 +356,6 @@ public boolean exportParameter(CloudgeneParameterOutput out) { return true; } - private void exportFolder(CloudgeneParameterOutput out, String prefix, String name, File folder, - List downloads) throws Exception { - if (folder.exists() && folder.isDirectory()) { - - File[] files = folder.listFiles(); - - for (int i = 0; i < files.length; i++) { - if (!files[i].isDirectory()) { - - String filename = prefix.equals("") ? files[i].getName() : prefix + "/" + files[i].getName(); - String id = name + "/" + files[i].getName(); - String size = FileUtils.byteCountToDisplaySize(files[i].length()); - String hash = HashUtil.getSha256(filename + id + size + getId() + (Math.random() * 100000)); - Download download = new Download(); - download.setName(filename); - download.setPath(FileUtil.path(getId(), id)); - download.setSize(size); - download.setHash(hash); - download.setParameter(out); - download.setCount(MAX_DOWNLOAD); - downloads.add(download); - - if (externalWorkspace != null) { - // upload to s3 bucket, update path and delete local file - try { - context.log(" Uploading file " + files[i].getAbsolutePath() + " to external workspace"); - String url = externalWorkspace.upload(name, files[i]); - context.log(" Uploaded file to " + url + "."); - download.setPath(url); - files[i].delete(); - } catch (Exception e) { - throw new Exception("Error uploading output '" + files[i].getAbsolutePath() + "'. " + e); - } - - } - - } else { - exportFolder(out, prefix.equals("") ? files[i].getName() : prefix + "/" + files[i].getName(), - name + "/" + files[i].getName(), files[i], downloads); - } - } - } - } - public String getWorkingDirectory() { return workingDirectory; } @@ -633,4 +384,8 @@ public void updateProgress() { } + public IExternalWorkspace getExternalWorkspace() { + return externalWorkspace; + } + } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneStepFactory.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneStepFactory.java index 017e391e..cc2e6aa2 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneStepFactory.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneStepFactory.java @@ -5,13 +5,9 @@ import cloudgene.mapred.steps.BashCommandStep; import cloudgene.mapred.steps.GroovyStep; -import cloudgene.mapred.steps.HadoopMapReduceStep; -import cloudgene.mapred.steps.HadoopPigStep; -import cloudgene.mapred.steps.HadoopSparkStep; import cloudgene.mapred.steps.HtmlWidgetStep; import cloudgene.mapred.steps.JavaExternalStep; import cloudgene.mapred.steps.RMarkdownDockerStep; -import cloudgene.mapred.steps.RMarkdownLocalStep; import cloudgene.mapred.steps.RMarkdownStep; import cloudgene.mapred.wdl.WdlStep; @@ -60,16 +56,10 @@ public String getClassname(WdlStep step) { } if (step.get("pig") != null) { - - // pig script - return HadoopPigStep.class.getName(); - + throw new RuntimeException("Hadoop support was removed in Cloudgene 3"); } if (step.get("spark") != null) { - - // spark - return HadoopSparkStep.class.getName(); - + throw new RuntimeException("Hadoop support was removed in Cloudgene 3"); } else if (step.get("rmd") != null) { // rscript @@ -93,8 +83,7 @@ public String getClassname(WdlStep step) { } else { String runtime = step.get("runtime"); if (runtime == null || runtime.isEmpty() || runtime.toLowerCase().equals("hadoop")) { - // mapreduce - return HadoopMapReduceStep.class.getName(); + throw new RuntimeException("Hadoop support was removed in Cloudgene 3"); } else if (runtime != null && runtime.toLowerCase().equals("java")) { // normal java when no Hadoop suppport return JavaExternalStep.class.getName(); diff --git a/src/main/java/cloudgene/mapred/jobs/Environment.java b/src/main/java/cloudgene/mapred/jobs/Environment.java index 61f9ccd3..26f4bf3c 100644 --- a/src/main/java/cloudgene/mapred/jobs/Environment.java +++ b/src/main/java/cloudgene/mapred/jobs/Environment.java @@ -6,38 +6,25 @@ import cloudgene.mapred.plugins.IPlugin; import cloudgene.mapred.plugins.PluginManager; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; -import genepi.io.FileUtil; public class Environment { public static Map getApplicationVariables(WdlApp application, Settings settings) { String localFolder = application.getPath(); - String hdfsFolder = ""; - - // set hdfsFolder to localFolder when hadoop plugin is not activated - PluginManager manager = PluginManager.getInstance(); - if (manager.isEnabled(HadoopPlugin.ID)) { - String hdfsAppFolder = settings.getHdfsAppWorkspace(); - hdfsFolder = FileUtil.path(hdfsAppFolder, application.getId().split(":")[0], application.getVersion()); - } else { - hdfsFolder = localFolder; - } HashMap environment = new HashMap(); environment.put("app_id", application.getId()); environment.put("app_name", application.getName()); environment.put("app_version", application.getVersion()); - environment.put("app_hdfs_folder", hdfsFolder); environment.put("app_local_folder", localFolder); // Deprecated - environment.put("hdfs_app_folder", hdfsFolder); environment.put("local_app_folder", localFolder); + environment.put("app_hdfs_folder", localFolder); // Technologies - + PluginManager manager = PluginManager.getInstance(); for (IPlugin plugin : manager.getPlugins()) { environment.put(plugin.getId() + "_installed", manager.isEnabled(plugin) ? "true" : "false"); } @@ -49,9 +36,7 @@ public static Map getJobVariables(CloudgeneContext context) { Map environment = new HashMap(); environment.put("job_id", context.getJobId()); environment.put("job_local_temp", context.getLocalTemp()); - environment.put("job_hdfs_temp", context.getHdfsTemp()); environment.put("job_local_output", context.getLocalOutput()); - environment.put("job_hdfs_output", context.getHdfsOutput()); environment.put("user_username", context.getUser().getUsername()); environment.put("user_mail", context.getUser().getMail()); // Deprecated diff --git a/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphNode.java b/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphNode.java index 540cd0d5..7d0fceb1 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphNode.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphNode.java @@ -21,7 +21,6 @@ import cloudgene.mapred.plugins.PluginManager; import cloudgene.mapred.steps.ErrorStep; import cloudgene.mapred.steps.JavaInternalStep; -import cloudgene.mapred.steps.JavaInternalStepDeprecrated; import cloudgene.mapred.wdl.WdlStep; import cloudgene.sdk.internal.WorkflowStep; import genepi.io.FileUtil; @@ -99,10 +98,7 @@ private void instance() if (object instanceof CloudgeneStep) { instance = (CloudgeneStep) object; } else if (object instanceof WorkflowStep) { - instance = new JavaInternalStep((WorkflowStep) object); - //old genepi-hadoop support! this is deprecreated! - } else if (object instanceof genepi.hadoop.common.WorkflowStep) { - instance = new JavaInternalStepDeprecrated((genepi.hadoop.common.WorkflowStep) object); + instance = new JavaInternalStep((WorkflowStep) object); } else { instance = new ErrorStep("Error during initialization: class " + step.getClassname() + " ( " + object.getClass().getSuperclass().getCanonicalName() + ") " diff --git a/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueInput.java b/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueInput.java index 64a316dc..203a2bc7 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueInput.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueInput.java @@ -1,12 +1,9 @@ package cloudgene.mapred.jobs.engine.plugins; import java.io.File; -import java.io.IOException; -import java.util.List; import cloudgene.mapred.wdl.WdlParameterInput; import cloudgene.mapred.wdl.WdlParameterInputType; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; import genepi.io.WildCardFileFilter; @@ -27,25 +24,6 @@ public String toString() { public String[] listFiles(String ext) { - if (parameter.getTypeAsEnum() == WdlParameterInputType.HDFS_FOLDER) { - List files = null; - try { - - files = HdfsUtil.getFiles(value, ext); - String[] result = new String[files.size()]; - for (int i = 0; i < files.size(); i++) { - result[i] = FileUtil.getFilename(files.get(i)); - } - return result; - } catch (IOException e) { - e.printStackTrace(); - String[] result = new String[1]; - result[0] = FileUtil.getFilename(value); - ; - return result; - } - } - if (parameter.getTypeAsEnum() == WdlParameterInputType.LOCAL_FOLDER) { return getFiles(value, ext); diff --git a/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueOutput.java b/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueOutput.java index fbf55356..63f5ec77 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueOutput.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueOutput.java @@ -1,12 +1,9 @@ package cloudgene.mapred.jobs.engine.plugins; import java.io.File; -import java.io.IOException; -import java.util.List; import cloudgene.mapred.wdl.WdlParameterOutput; import cloudgene.mapred.wdl.WdlParameterOutputType; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; import genepi.io.WildCardFileFilter; @@ -27,25 +24,6 @@ public String toString() { public String[] listFiles(String ext) { - if (parameter.getTypeAsEnum() == WdlParameterOutputType.HDFS_FOLDER) { - List files = null; - try { - - files = HdfsUtil.getFiles(value, ext); - String[] result = new String[files.size()]; - for (int i = 0; i < files.size(); i++) { - result[i] = FileUtil.getFilename(files.get(i)); - } - return result; - } catch (IOException e) { - e.printStackTrace(); - String[] result = new String[1]; - result[0] = FileUtil.getFilename(value); - ; - return result; - } - } - if (parameter.getTypeAsEnum() == WdlParameterOutputType.LOCAL_FOLDER) { return getFiles(value, ext); diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/ExternalWorkspaceFactory.java b/src/main/java/cloudgene/mapred/jobs/workspace/ExternalWorkspaceFactory.java index b9f743b8..58342468 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/ExternalWorkspaceFactory.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/ExternalWorkspaceFactory.java @@ -1,33 +1,55 @@ package cloudgene.mapred.jobs.workspace; -import cloudgene.sdk.internal.IExternalWorkspace; +import cloudgene.mapred.jobs.AbstractJob; +import cloudgene.mapred.server.Application; +import cloudgene.mapred.util.Settings; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; +@Singleton public class ExternalWorkspaceFactory { - public static IExternalWorkspace get(String type, String location) { + @Inject + protected Application application; + + public IExternalWorkspace getDefault() { + + Settings settings = application.getSettings(); + + String type = settings.getExternalWorkspaceType(); if (type == null) { - return null; + return new LocalWorkspace(settings.getLocalWorkspace()); } if (type.equalsIgnoreCase("S3")) { - return new S3Workspace(location); + String bucket = settings.getExternalWorkspaceLocation(); + return new S3Workspace(bucket); } - return null; + return new LocalWorkspace(settings.getLocalWorkspace()); + } - public static IExternalWorkspace get(String url) { + public IExternalWorkspace getByUrl(String url) { - if (url == null) { - return null; + Settings settings = application.getSettings(); + + if (url == null || url.isEmpty()) { + throw new RuntimeException("Workspace type could not determined for empty url."); } if (url.startsWith("s3://")) { - return new S3Workspace(""); + String bucket = settings.getExternalWorkspaceLocation(); + return new S3Workspace(bucket); } - return null; + return new LocalWorkspace(settings.getLocalWorkspace()); + + } + + public IExternalWorkspace getByJob(AbstractJob job) { + return getDefault(); } } diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/IExternalWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/IExternalWorkspace.java new file mode 100644 index 00000000..b7cf0cd6 --- /dev/null +++ b/src/main/java/cloudgene/mapred/jobs/workspace/IExternalWorkspace.java @@ -0,0 +1,38 @@ +package cloudgene.mapred.jobs.workspace; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import cloudgene.mapred.jobs.Download; + +public interface IExternalWorkspace { + + public void setup(String job) throws IOException; + + public String upload(String id, File file) throws IOException; + + public String uploadInput(String id, File file) throws IOException; + + public InputStream download(String url) throws IOException; + + public void delete(String job) throws IOException; + + public String getName(); + + public String createPublicLink(String url); + + public String getParent(String url); + + public String createFolder(String id); + + public String createFile(String name, String name2); + + public String createTempFolder(String string); + + public List getDownloads(String url); + + public void cleanup(String job) throws IOException; + +} diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java new file mode 100644 index 00000000..31de2b81 --- /dev/null +++ b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java @@ -0,0 +1,200 @@ +package cloudgene.mapred.jobs.workspace; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Vector; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import cloudgene.mapred.jobs.Download; +import cloudgene.mapred.util.HashUtil; +import genepi.io.FileUtil; + +public class LocalWorkspace implements IExternalWorkspace { + + private static final String INPUT_DIRECTORY = "input"; + + private static final String TEMP_DIRECTORY = "temp"; + + private static final Logger log = LoggerFactory.getLogger(LocalWorkspace.class); + + private String location; + + private String workspace; + + public LocalWorkspace(String location) { + this.location = absolute(location); + } + + @Override + public String getName() { + return "Local Workspace"; + } + + @Override + public void setup(String job) throws IOException { + workspace = FileUtil.path(location, job); + log.info("Init workspace " + workspace); + FileUtil.createDirectory(workspace); + } + + @Override + public String upload(String id, File file) throws IOException { + String folder = FileUtil.path(workspace, id); + FileUtil.createDirectory(folder); + String target = FileUtil.path(folder, file.getName()); + log.info("Copy file " + file.getAbsolutePath() + " to " + target); + FileUtil.copy(file.getAbsolutePath(), target); + return target; + } + + @Override + public String uploadInput(String id, File file) throws IOException { + return upload(FileUtil.path(INPUT_DIRECTORY, id), file); + } + + @Override + public InputStream download(String path) throws IOException { + String absolutePath = FileUtil.path(location, path); + File file = new File(absolutePath); + if (file.exists()) { + return new FileInputStream(file); + } else { + throw new IOException("File '" + path + "' not found in workspace."); + } + } + + @Override + public void delete(String job) throws IOException { + + try { + log.debug("Deleting " + job + " on local workspace..."); + String workspace = FileUtil.path(location, job); + FileUtil.deleteDirectory(workspace); + + log.debug("Deleted all files on local workspace for job " + job + "."); + + } catch (Exception e) { + log.error("Deleting " + job + " failed.", e); + throw new IOException("Deleting " + job + " failed.", e); + } + + } + + @Override + public void cleanup(String job) throws IOException { + + //TODO: add flag to disable cleanup (e.g. debugging) + + try { + log.debug("Cleanup " + job + " on local workspace..."); + String temp = FileUtil.path(location, job, TEMP_DIRECTORY); + FileUtil.deleteDirectory(temp); + + String inputs = FileUtil.path(location, job, INPUT_DIRECTORY); + FileUtil.deleteDirectory(inputs); + + log.debug("Deleted all files on local workspace for job " + job + "."); + + } catch (Exception e) { + log.error("Deleting " + job + " failed.", e); + throw new IOException("Deleting " + job + " failed.", e); + } + + } + + @Override + public String createPublicLink(String url) { + return null; + } + + @Override + public String getParent(String url) { + return new File(url).getParent(); + } + + @Override + public String createFolder(String id) { + String folder = FileUtil.path(workspace, id); + FileUtil.createDirectory(folder); + return folder; + } + + @Override + public String createFile(String parent, String id) { + String folder = FileUtil.path(workspace, parent); + FileUtil.createDirectory(folder); + return FileUtil.path(folder, id); + } + + @Override + public String createTempFolder(String id) { + String folder = FileUtil.path(workspace, TEMP_DIRECTORY, id); + FileUtil.createDirectory(folder); + return folder; + } + + @Override + public List getDownloads(String url) { + File folder = new File(url); + List downloads = new Vector(); + exportFolder("", folder, downloads); + return downloads; + } + + private void exportFolder(String prefix, File folder, List downloads) { + + if (!folder.exists()) { + return; + } + + if (folder.isFile()) { + Download download = createDownload(prefix, folder); + downloads.add(download); + return; + } + + File[] files = folder.listFiles(); + + for (File file : files) { + if (folder.isFile()) { + Download download = createDownload(prefix, file); + downloads.add(download); + } else { + exportFolder(prefix.equals("") ? file.getName() : prefix + "/" + file.getName(), file, downloads); + } + } + + } + + protected Download createDownload(String prefix, File file) { + String filename = prefix.equals("") ? file.getName() : prefix + "/" + file.getName(); + String size = FileUtils.byteCountToDisplaySize(file.length()); + String hash = HashUtil.getSha256(filename + size + (Math.random() * 100000)); + Download download = new Download(); + download.setName(filename); + download.setPath(relative(file.getAbsolutePath())); + download.setSize(size); + download.setHash(hash); + return download; + } + + protected String relative(String absolute) { + Path pathAbsolute = Paths.get(absolute); + Path pathRoot = Paths.get(location); + Path pathRelative = pathRoot.relativize(pathAbsolute); + return pathRelative.toString(); + } + + protected String absolute(String path) { + return new File(path).getAbsolutePath(); + } + +} diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java index 265a6a12..9059a988 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java @@ -4,26 +4,36 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.util.List; +import java.util.Vector; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.amazonaws.HttpMethod; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.amazonaws.services.s3.model.S3ObjectSummary; -import cloudgene.sdk.internal.IExternalWorkspace; -import genepi.hadoop.S3Util; +import cloudgene.mapred.jobs.Download; +import cloudgene.mapred.util.HashUtil; +import cloudgene.mapred.util.S3Util; +import genepi.io.FileUtil; public class S3Workspace implements IExternalWorkspace { + private static final String OUTPUT_DIRECTORY = "outputs"; + public static long EXPIRATION_MS = 1000 * 60 * 60; + private static final String INPUT_DIRECTORY = "input"; + + private static final String TEMP_DIRECTORY = "temp"; + private static final Logger log = LoggerFactory.getLogger(S3Workspace.class); private String location; @@ -55,7 +65,8 @@ public void setup(String job) throws IOException { try { S3Util.copyToS3(job, location + "/" + job + "/version.txt"); } catch (Exception e) { - throw new IOException("Output Url '" + location + "' is not writable."); + log.error("Copy file to '" + location + "/" + job + "/version.txt' failed.", e); + throw new IOException("Output Url '" + location + "' is not writable.", e); } } @@ -63,10 +74,16 @@ public void setup(String job) throws IOException { @Override public String upload(String id, File file) throws IOException { String target = location + "/" + job + "/" + id + "/" + file.getName(); + log.info("Copy file " + file.getAbsolutePath() + " to " + target); S3Util.copyToS3(file, target); return target; } + @Override + public String uploadInput(String id, File file) throws IOException { + return upload(FileUtil.path(INPUT_DIRECTORY, id), file); + } + @Override public InputStream download(String url) throws IOException { @@ -88,36 +105,44 @@ public void delete(String job) throws IOException { throw new IOException("Output Url '" + location + "' is not a valid S3 bucket."); } + String url = location + "/" + job; + try { - String url = location + "/" + job; - String bucket = S3Util.getBucket(url); - String key = S3Util.getKey(url); + log.info("Deleting " + job + " on S3 workspace: '" + url + "'..."); - AmazonS3 s3 = S3Util.getAmazonS3(); + S3Util.deleteFolder(url); - log.debug("Deleting " + job + " on S3 workspace..."); + log.info("Deleted all files on S3 for job " + job + "."); - ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(key); + } catch (Exception e) { + throw new IOException("Folder '" + url + "' could not be deleted.", e); + } - ObjectListing objectListing = s3.listObjects(listObjectsRequest); + } - while (true) { - for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { - log.debug(" Deleting file " + bucket + " " + objectSummary.getKey() + " ..."); - s3.deleteObject(bucket, objectSummary.getKey()); - } - if (objectListing.isTruncated()) { - objectListing = s3.listNextBatchOfObjects(objectListing); - } else { - break; - } - } + @Override + public void cleanup(String job) throws IOException { + if (!S3Util.isValidS3Url(location)) { + throw new IOException("Output Url '" + location + "' is not a valid S3 bucket."); + } - log.debug("Deleted all files on S3 for job " + job + "."); + String temp = location + "/" + job + "/" + TEMP_DIRECTORY; + try { + log.info("Deleting temp directory for " + job + " on S3 workspace: '" + temp + "'..."); + S3Util.deleteFolder(temp); + log.info("Deleted all files on S3 for job " + job + "."); + } catch (Exception e) { + throw new IOException("Folder '" + temp + "' could not be deleted.", e); + } + String input = location + "/" + job + "/" + INPUT_DIRECTORY; + try { + log.info("Deleting input directory for " + input + " on S3 workspace: '" + input + "'..."); + S3Util.deleteFolder(input); + log.info("Deleted all files on S3 for job " + job + "."); } catch (Exception e) { - throw new IOException("Output Url '" + location + "' is not writable."); + throw new IOException("Folder '" + input + "' could not be deleted.", e); } } @@ -144,4 +169,67 @@ public String createPublicLink(String url) { return publicUrl.toString(); } + @Override + public String getParent(String url) { + if (url.startsWith("s3://")) { + int index = url.lastIndexOf('/'); + if (index > 0) { + return url.substring(0, index); + } + return null; + } else { + return null; + } + } + + @Override + public String createFolder(String id) { + return location + "/" + job + "/" + OUTPUT_DIRECTORY + "/" + id; + } + + @Override + public String createFile(String folder, String id) { + return location + "/" + job + "/" + OUTPUT_DIRECTORY + "/" + folder + "/" + id; + } + + @Override + public String createTempFolder(String id) { + return location + "/" + job + "/" + TEMP_DIRECTORY + "/" + id; + } + + @Override + public List getDownloads(String url) { + List downloads = new Vector(); + ObjectListing listing; + try { + listing = S3Util.listObjects(url); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + return downloads; + } + + String baseKey = S3Util.getKey(url); + + for (S3ObjectSummary summary : listing.getObjectSummaries()) { + + if (summary.getKey().endsWith("/")) { + continue; + } + + String filename = summary.getKey().replaceAll(baseKey + "/", ""); + String size = FileUtils.byteCountToDisplaySize(summary.getSize()); + String hash = HashUtil.getSha256(filename + size + (Math.random() * 100000)); + Download download = new Download(); + download.setName(filename); + download.setPath("s3://" + summary.getBucketName() + "/" + summary.getKey()); + download.setSize(size); + download.setHash(hash); + downloads.add(download); + + } + + return downloads; + } + } diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceWrapper.java b/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceWrapper.java new file mode 100644 index 00000000..1d5f84c8 --- /dev/null +++ b/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceWrapper.java @@ -0,0 +1,47 @@ +package cloudgene.mapred.jobs.workspace; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; + +import cloudgene.sdk.internal.IExternalWorkspace; + +public class WorkspaceWrapper implements IExternalWorkspace { + + private cloudgene.mapred.jobs.workspace.IExternalWorkspace newWorkspace; + + public WorkspaceWrapper(cloudgene.mapred.jobs.workspace.IExternalWorkspace newWorkspace) { + this.newWorkspace = newWorkspace; + } + + @Override + public String createPublicLink(String arg0) { + return newWorkspace.createPublicLink(arg0); + } + + @Override + public void delete(String arg0) throws IOException { + newWorkspace.delete(arg0); + } + + @Override + public InputStream download(String arg0) throws IOException { + return newWorkspace.download(arg0); + } + + @Override + public String getName() { + return newWorkspace.getName(); + } + + @Override + public void setup(String arg0) throws IOException { + newWorkspace.setup(arg0); + } + + @Override + public String upload(String arg0, File arg1) throws IOException { + return newWorkspace.upload(arg0, arg1); + } + +} diff --git a/src/main/java/cloudgene/mapred/plugins/PluginManager.java b/src/main/java/cloudgene/mapred/plugins/PluginManager.java index 34dd7148..1be23a36 100644 --- a/src/main/java/cloudgene/mapred/plugins/PluginManager.java +++ b/src/main/java/cloudgene/mapred/plugins/PluginManager.java @@ -9,7 +9,6 @@ import org.slf4j.LoggerFactory; import cloudgene.mapred.plugins.docker.DockerPlugin; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; import cloudgene.mapred.plugins.nextflow.NextflowPlugin; import cloudgene.mapred.plugins.rscript.RMarkdownPlugin; import cloudgene.mapred.plugins.rscript.RScriptPlugin; @@ -34,7 +33,6 @@ public static PluginManager getInstance() { private PluginManager() { plugins = new Vector(); - plugins.add(new HadoopPlugin()); plugins.add(new DockerPlugin()); plugins.add(new RScriptPlugin()); plugins.add(new RMarkdownPlugin()); diff --git a/src/main/java/cloudgene/mapred/plugins/docker/DockerBinary.java b/src/main/java/cloudgene/mapred/plugins/docker/DockerBinary.java index 55e561d0..4f0f469f 100644 --- a/src/main/java/cloudgene/mapred/plugins/docker/DockerBinary.java +++ b/src/main/java/cloudgene/mapred/plugins/docker/DockerBinary.java @@ -4,7 +4,7 @@ import cloudgene.mapred.util.BinaryFinder; import cloudgene.mapred.util.Settings; -import genepi.hadoop.command.Command; +import cloudgene.mapred.util.command.Command; import genepi.io.FileUtil; public class DockerBinary { diff --git a/src/main/java/cloudgene/mapred/plugins/docker/DockerStep.java b/src/main/java/cloudgene/mapred/plugins/docker/DockerStep.java index c382a517..6ae40ad3 100644 --- a/src/main/java/cloudgene/mapred/plugins/docker/DockerStep.java +++ b/src/main/java/cloudgene/mapred/plugins/docker/DockerStep.java @@ -1,7 +1,6 @@ package cloudgene.mapred.plugins.docker; import java.io.File; -import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -9,8 +8,6 @@ import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; import cloudgene.mapred.wdl.WdlStep; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; public class DockerStep extends CloudgeneStep { @@ -64,23 +61,7 @@ protected boolean runInDockerContainer(CloudgeneContext context, String image, S String[] newParams = new String[cmd.length]; for (int i = 0; i < newParams.length; i++) { String param = cmd[i]; - - // checkout hdfs file - if (param.startsWith("hdfs://")) { - String name = FileUtil.getFilename(param); - String localFile = FileUtil.path(((CloudgeneContext) context).getLocalTemp(), "local_" + name); - try { - HdfsUtil.checkOut(param, localFile); - String localFilename = new File(localFile).getAbsolutePath(); - newParams[i] = localFilename; - } catch (IOException e) { - context.log(e.getMessage()); - newParams[i] = param.replaceAll(localWorkspace, DOCKER_WORKSPACE); - } - - } else { - newParams[i] = param.replaceAll(localWorkspace, DOCKER_WORKSPACE); - } + newParams[i] = param.replaceAll(localWorkspace, DOCKER_WORKSPACE); } if (!image.contains(":")) { diff --git a/src/main/java/cloudgene/mapred/plugins/hadoop/HadoopPlugin.java b/src/main/java/cloudgene/mapred/plugins/hadoop/HadoopPlugin.java deleted file mode 100644 index 41b1507a..00000000 --- a/src/main/java/cloudgene/mapred/plugins/hadoop/HadoopPlugin.java +++ /dev/null @@ -1,76 +0,0 @@ -package cloudgene.mapred.plugins.hadoop; - -import cloudgene.mapred.plugins.IPlugin; -import cloudgene.mapred.util.Settings; -import genepi.hadoop.HadoopCluster; - -public class HadoopPlugin implements IPlugin { - - public static final String ID = "hadoop"; - - @Override - public String getId() { - return ID; - } - - @Override - public String getName() { - return "Hadoop Cluster"; - } - - @Override - public boolean isInstalled() { - try { - if (HadoopCluster.verifyCluster()) { - return true; - } else { - return false; - } - } catch (Exception e) { - return false; - } - - } - - @Override - public String getDetails() { - StringBuffer state = new StringBuffer(); - state.append("JobTracker: " + HadoopCluster.getJobTracker() + "\n"); - state.append("Default FS: " + HadoopCluster.getDefaultFS() + "\n"); - state.append("State: " + HadoopCluster.getJobTrackerStatus().toString() + "\n"); - state.append("MapTask: " + HadoopCluster.getMaxMapTasks() + "\n"); - state.append("ReduceTask: " + HadoopCluster.getMaxReduceTasks() + "\n"); - state.append("Nodes\n"); - for (String tracker : HadoopCluster.getActiveTrackerNames()) { - state.append(" " + tracker + "\n"); - } - state.append("Blacklist:\n"); - for (String tracker : HadoopCluster.getBlacklistedTrackerNames()) { - state.append(" " + tracker + "\n"); - } - return state.toString(); - } - - @Override - public void configure(Settings settings) { - // TODO Auto-generated method stub - } - - @Override - public String getStatus() { - if (isInstalled()) { - int nodes = HadoopCluster.getActiveTrackerNames().size(); - int mappTasks = HadoopCluster.getMaxMapTasks(); - int reduceTasks = HadoopCluster.getMaxReduceTasks(); - return "Cluster has " + nodes + " nodes, " + mappTasks + " map tasks and " + reduceTasks + " reduce tasks"; - } else { - try { - HadoopCluster.verifyCluster(); - return "Hadoop support disabled."; - } catch (Exception e) { - return "Hadoop support disabled. " + e.getMessage(); - } - } - } - -} diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java index 352c22e7..4126d276 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java @@ -4,7 +4,7 @@ import cloudgene.mapred.util.BinaryFinder; import cloudgene.mapred.util.Settings; -import genepi.hadoop.command.Command; +import cloudgene.mapred.util.command.Command; import genepi.io.FileUtil; public class NextflowBinary { diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java index 4ea90fe3..70ea0ea7 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java @@ -11,19 +11,23 @@ import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.CloudgeneContext; +import cloudgene.mapred.jobs.CloudgeneJob; import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; +import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; +import cloudgene.mapred.jobs.workspace.IExternalWorkspace; import cloudgene.mapred.util.HashUtil; import cloudgene.mapred.wdl.WdlStep; import genepi.io.FileUtil; import groovy.json.JsonOutput; +import jakarta.inject.Inject; public class NextflowStep extends CloudgeneStep { private CloudgeneContext context; private Map messages = new HashMap(); - + @Override public boolean run(WdlStep step, CloudgeneContext context) { @@ -84,7 +88,8 @@ public boolean run(WdlStep step, CloudgeneContext context) { nextflowCommand.add("-w"); nextflowCommand.add(work); } else { - String workDir = FileUtil.path(context.getLocalOutput(), "nextflow"); + IExternalWorkspace externalWorkspace = job.getExternalWorkspace(); + String workDir = externalWorkspace.createTempFolder("nextflow"); FileUtil.createDirectory(workDir); nextflowCommand.add("-w"); nextflowCommand.add(workDir); diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java index d3646053..8bc6ba79 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java @@ -34,7 +34,6 @@ public void update(Map trace) throws IOException { if (status.equals("COMPLETED") || status.equals("FAILED")) { String workDir = (String) trace.get("workdir"); String logFilename = FileUtil.path(workDir, "cloudgene.log"); - // TODO: implement s3 support. How to handle other cloud providers? if (new File(logFilename).exists()) { diff --git a/src/main/java/cloudgene/mapred/plugins/rscript/RScriptBinary.java b/src/main/java/cloudgene/mapred/plugins/rscript/RScriptBinary.java index c1d31198..9dbda6de 100644 --- a/src/main/java/cloudgene/mapred/plugins/rscript/RScriptBinary.java +++ b/src/main/java/cloudgene/mapred/plugins/rscript/RScriptBinary.java @@ -4,7 +4,7 @@ import cloudgene.mapred.util.BinaryFinder; import cloudgene.mapred.util.Settings; -import genepi.hadoop.command.Command; +import cloudgene.mapred.util.command.Command; import genepi.io.FileUtil; public class RScriptBinary { diff --git a/src/main/java/cloudgene/mapred/server/controller/AppController.java b/src/main/java/cloudgene/mapred/server/controller/AppController.java index 01bea502..ea44465b 100644 --- a/src/main/java/cloudgene/mapred/server/controller/AppController.java +++ b/src/main/java/cloudgene/mapred/server/controller/AppController.java @@ -84,10 +84,6 @@ public ApplicationResponse updateApp(String appId, @Nullable Boolean enabled, @N applicationService.updatePermissions(app, permission); // update config applicationService.updateConfig(app, config); - // reinstall application - if (reinstall != null) { - applicationService.reinstallApp(app, reinstall); - } app.checkForChanges(); diff --git a/src/main/java/cloudgene/mapred/server/controller/DownloadController.java b/src/main/java/cloudgene/mapred/server/controller/DownloadController.java index 14f7d0c6..9f76d9dd 100644 --- a/src/main/java/cloudgene/mapred/server/controller/DownloadController.java +++ b/src/main/java/cloudgene/mapred/server/controller/DownloadController.java @@ -1,6 +1,8 @@ package cloudgene.mapred.server.controller; import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.net.URISyntaxException; import java.util.List; @@ -22,6 +24,7 @@ import genepi.io.FileUtil; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; +import io.micronaut.http.MutableHttpResponse; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; import io.micronaut.security.annotation.Secured; @@ -46,10 +49,10 @@ public class DownloadController { @Inject protected JobService jobService; - @Get("/downloads/{jobId}/{hash}/{filename}") + @Get("/downloads/{jobId}/{hash}/{filename:.+}") @Secured(SecurityRule.IS_ANONYMOUS) - public HttpResponse downloadExternalResults(String jobId, String hash, String filename) - throws URISyntaxException { + public MutableHttpResponse downloadExternalResults(String jobId, String hash, String filename) + throws URISyntaxException, IOException { AbstractJob job = jobService.getById(jobId); @@ -75,10 +78,10 @@ public HttpResponse downloadExternalResults(String jobId, String hash, Str } - @Get("/share/results/{hash}/{filename}") + @Get("/share/results/{hash}/{filename:.+}") @Secured(SecurityRule.IS_ANONYMOUS) - public HttpResponse downloadPublicLink(String hash, String filename) throws URISyntaxException { - + public MutableHttpResponse downloadPublicLink(String hash, String filename) throws URISyntaxException, IOException { +System.out.println("------> " + filename); DownloadDao dao = new DownloadDao(application.getDatabase()); Download download = dao.findByHash(hash); diff --git a/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java b/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java index 790d68ef..13746079 100644 --- a/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java @@ -6,7 +6,6 @@ import java.util.Vector; import cloudgene.mapred.apps.Application; -import cloudgene.mapred.apps.ApplicationInstaller; import cloudgene.mapred.apps.ApplicationRepository; import cloudgene.mapred.jobs.Environment; import cloudgene.mapred.util.Settings; @@ -86,24 +85,8 @@ public static List buildWithDetails(List appli } private static String updateState(Application app, Settings settings) { - WdlApp wdlApp = app.getWdlApp(); - if (wdlApp != null) { - if (wdlApp.needsInstallation()) { - try { - boolean installed = ApplicationInstaller.isInstalled(wdlApp, settings); - if (installed) { - return "completed"; - } else { - return "on demand"; - } - } catch (NoClassDefFoundError e) { - // TODO: handle exception - } - } else { - return "n/a"; - } - } - return ""; + //TODO: remove + return "completed"; } public String getId() { diff --git a/src/main/java/cloudgene/mapred/server/services/ApplicationService.java b/src/main/java/cloudgene/mapred/server/services/ApplicationService.java index aacba423..5235ae24 100644 --- a/src/main/java/cloudgene/mapred/server/services/ApplicationService.java +++ b/src/main/java/cloudgene/mapred/server/services/ApplicationService.java @@ -1,6 +1,5 @@ package cloudgene.mapred.server.services; -import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -10,11 +9,8 @@ import org.slf4j.LoggerFactory; import cloudgene.mapred.apps.Application; -import cloudgene.mapred.apps.ApplicationInstaller; import cloudgene.mapred.apps.ApplicationRepository; import cloudgene.mapred.core.User; -import cloudgene.mapred.plugins.PluginManager; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; import cloudgene.mapred.server.exceptions.JsonHttpStatusException; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; @@ -79,14 +75,6 @@ public void checkRequirements(Application app) { String.format(APPLICATION_IS_DATA_PACKAGE, app.getId())); } - if (wdlApp.getWorkflow().hasHdfsInputs()) { - - PluginManager manager = PluginManager.getInstance(); - if (!manager.isEnabled(HadoopPlugin.ID)) { - throw new JsonHttpStatusException(HttpStatus.SERVICE_UNAVAILABLE, HADOOP_CLUSTER_UNREACHABLE); - } - } - } public Application removeApp(String appId) { @@ -151,29 +139,6 @@ public void updateConfig(Application app, Map config) { } - public void reinstallApp(Application app, boolean reinstall) { - - WdlApp wdlApp = app.getWdlApp(); - - if (!reinstall) { - return; - } - - boolean installed = ApplicationInstaller.isInstalled(wdlApp, this.application.getSettings()); - if (!installed) { - return; - } - - try { - ApplicationInstaller.uninstall(wdlApp, this.application.getSettings()); - } catch (IOException e) { - e.printStackTrace(); - throw new JsonHttpStatusException(HttpStatus.BAD_REQUEST, - String.format(APPLICATION_NOT_UPDATED, e.getMessage())); - } - - } - public List listApps(boolean reload) { ApplicationRepository repository = application.getSettings().getApplicationRepository(); diff --git a/src/main/java/cloudgene/mapred/server/services/DownloadService.java b/src/main/java/cloudgene/mapred/server/services/DownloadService.java index fd4dc4b2..60a6d1b9 100644 --- a/src/main/java/cloudgene/mapred/server/services/DownloadService.java +++ b/src/main/java/cloudgene/mapred/server/services/DownloadService.java @@ -1,18 +1,19 @@ package cloudgene.mapred.server.services; -import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import cloudgene.mapred.database.DownloadDao; import cloudgene.mapred.jobs.Download; import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; +import cloudgene.mapred.jobs.workspace.IExternalWorkspace; import cloudgene.mapred.server.Application; import cloudgene.mapred.server.exceptions.JsonHttpStatusException; -import cloudgene.sdk.internal.IExternalWorkspace; -import genepi.io.FileUtil; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; +import io.micronaut.http.MutableHttpResponse; import jakarta.inject.Inject; public class DownloadService { @@ -20,7 +21,10 @@ public class DownloadService { @Inject protected Application application; - public HttpResponse download(Download download) throws URISyntaxException { + @Inject + protected ExternalWorkspaceFactory workspaceFactory; + + public MutableHttpResponse download(Download download) throws URISyntaxException, IOException { DownloadDao dao = new DownloadDao(application.getDatabase()); @@ -37,20 +41,17 @@ public HttpResponse download(Download download) throws URISyntaxException download.decCount(); dao.update(download); } - - IExternalWorkspace externalWorkspace = ExternalWorkspaceFactory.get(download.getPath()); - if (externalWorkspace != null) { - // external workspace found, use link method and create redirect response - String publicUrl = externalWorkspace.createPublicLink(download.getPath()); + + IExternalWorkspace externalWorkspace = workspaceFactory.getByUrl(download.getPath()); + + // external workspace found, use link method and create redirect response + String publicUrl = externalWorkspace.createPublicLink(download.getPath()); + if (publicUrl != null) { URI location = new URI(publicUrl); return HttpResponse.redirect(location); } else { - // no external workspace found, use local workspace - String localWorkspace = application.getSettings().getLocalWorkspace(); - String resultFile = FileUtil.path(localWorkspace, download.getPath()); - return HttpResponse.ok(new File(resultFile)); + return HttpResponse.ok(externalWorkspace.download(download.getPath())); } - } } diff --git a/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java b/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java index 0158ef98..9443b7ae 100644 --- a/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java +++ b/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java @@ -10,11 +10,10 @@ import cloudgene.mapred.database.util.Database; import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; +import cloudgene.mapred.jobs.workspace.IExternalWorkspace; import cloudgene.mapred.server.Application; import cloudgene.mapred.util.MailUtil; import cloudgene.mapred.util.Settings; -import cloudgene.sdk.internal.IExternalWorkspace; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -27,6 +26,9 @@ public class JobCleanUpService { @Inject protected Application application; + @Inject + protected ExternalWorkspaceFactory workspaceFactory; + public int executeRetire() { Database database = application.getDatabase(); @@ -38,39 +40,26 @@ public int executeRetire() { int deleted = 0; - IExternalWorkspace externalWorkspace = null; - if (!settings.getExternalWorkspaceLocation().isEmpty()) { - String externalOutput = settings.getExternalWorkspaceLocation(); - externalWorkspace = ExternalWorkspaceFactory.get(settings.getExternalWorkspaceType(), externalOutput); - } - for (AbstractJob job : oldJobs) { if (job.getDeletedOn() < System.currentTimeMillis()) { - // delete local directory and hdfs directory + // delete local directory String localOutput = FileUtil.path(settings.getLocalWorkspace(), job.getId()); FileUtil.deleteDirectory(localOutput); - try { - String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(settings.getHdfsWorkspace(), job.getId())); - HdfsUtil.delete(hdfsOutput); - } catch (NoClassDefFoundError e) { - // TODO: handle exception - } - job.setState(AbstractJob.STATE_RETIRED); dao.update(job); log.info("Job " + job.getId() + " retired."); deleted++; - if (externalWorkspace != null) { - try { - externalWorkspace.delete(job.getId()); - } catch (Exception e) { - log.error("Retire " + job.getId() + " failed.", e); - } + IExternalWorkspace externalWorkspace = workspaceFactory.getByJob(job); + + try { + externalWorkspace.delete(job.getId()); + } catch (Exception e) { + log.error("Retire " + job.getId() + " failed.", e); } } diff --git a/src/main/java/cloudgene/mapred/server/services/JobService.java b/src/main/java/cloudgene/mapred/server/services/JobService.java index b8c4266a..2a3d3af2 100644 --- a/src/main/java/cloudgene/mapred/server/services/JobService.java +++ b/src/main/java/cloudgene/mapred/server/services/JobService.java @@ -9,6 +9,8 @@ import java.util.Vector; import org.apache.commons.lang.StringEscapeUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import cloudgene.mapred.apps.ApplicationRepository; import cloudgene.mapred.core.User; @@ -20,6 +22,8 @@ import cloudgene.mapred.jobs.Download; import cloudgene.mapred.jobs.WorkflowEngine; import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; +import cloudgene.mapred.jobs.workspace.IExternalWorkspace; +import cloudgene.mapred.jobs.workspace.LocalWorkspace; import cloudgene.mapred.server.Application; import cloudgene.mapred.server.exceptions.JsonHttpStatusException; import cloudgene.mapred.util.FormUtil.Parameter; @@ -28,9 +32,6 @@ import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlParameterInput; import cloudgene.mapred.wdl.WdlParameterInputType; -import cloudgene.sdk.internal.IExternalWorkspace; -import genepi.hadoop.HdfsUtil; -import genepi.hadoop.importer.ImporterFactory; import genepi.io.FileUtil; import io.micronaut.http.HttpStatus; import jakarta.inject.Inject; @@ -39,9 +40,14 @@ @Singleton public class JobService { + private static final Logger log = LoggerFactory.getLogger(JobService.class); + @Inject protected Application application; + @Inject + protected ExternalWorkspaceFactory workspaceFactory; + private static final String PARAM_JOB_NAME = "job-name"; public AbstractJob getById(String id) { @@ -113,20 +119,18 @@ public AbstractJob submitJob(String appId, List form, User user) { String id = createId(); - String hdfsWorkspace = ""; + Map inputParams = null; + + IExternalWorkspace externalWorkspace = workspaceFactory.getDefault(); + try { - hdfsWorkspace = HdfsUtil.path(settings.getHdfsWorkspace(), id); - } catch (NoClassDefFoundError e) { - // Ignore HDFS exceptions to work also without Hadoop; - } - String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), id); - FileUtil.createDirectory(localWorkspace); + // setup workspace + externalWorkspace.setup(id); - Map inputParams = null; + // parse input params + inputParams = parseAndUpdateInputParams(form, app, externalWorkspace); - try { - inputParams = parseAndUpdateInputParams(form, app, hdfsWorkspace, localWorkspace); } catch (Exception e) { throw new JsonHttpStatusException(HttpStatus.BAD_REQUEST, e.getMessage()); } @@ -137,13 +141,15 @@ public AbstractJob submitJob(String appId, List form, User user) { name = jobName; } + String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), id); + FileUtil.createDirectory(localWorkspace); + CloudgeneJob job = new CloudgeneJob(user, id, app, inputParams); job.setId(id); job.setName(name); job.setLocalWorkspace(localWorkspace); - job.setHdfsWorkspace(hdfsWorkspace); + job.setExternalWorkspace(externalWorkspace); job.setSettings(settings); - job.setRemoveHdfsWorkspace(settings.isRemoveHdfsWorkspace()); job.setApplication(app.getName() + " " + app.getVersion()); job.setApplicationId(appId); @@ -211,17 +217,10 @@ public Page getAllByUserAndPage(User user, Integer page, int pageSi public AbstractJob delete(AbstractJob job) { Settings settings = application.getSettings(); - // delete local directory and hdfs directory + // delete local directory String localOutput = FileUtil.path(settings.getLocalWorkspace(), job.getId()); FileUtil.deleteDirectory(localOutput); - try { - String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(settings.getHdfsWorkspace(), job.getId())); - HdfsUtil.delete(hdfsOutput); - } catch (NoClassDefFoundError e) { - // ignore HDFS exception to work also without hadoop. - } - // delete job from database job.setState(AbstractJob.STATE_DELETED); @@ -229,16 +228,14 @@ public AbstractJob delete(AbstractJob job) { dao.update(job); // delete all results that are stored on external workspaces - IExternalWorkspace externalWorkspace = null; - if (!settings.getExternalWorkspaceLocation().isEmpty()) { - String externalOutput = settings.getExternalWorkspaceLocation(); - externalWorkspace = ExternalWorkspaceFactory.get(settings.getExternalWorkspaceType(), externalOutput); - try { - externalWorkspace.delete(job.getId()); - } catch (Exception e) { - e.printStackTrace(); - } + + IExternalWorkspace externalWorkspace = workspaceFactory.getByJob(job); + try { + externalWorkspace.delete(job.getId()); + } catch (Exception e) { + log.error("Deleteting " + job.getId() + " form workspace failed.", e); } + return job; } @@ -255,18 +252,10 @@ public AbstractJob restart(AbstractJob job) { throw new JsonHttpStatusException(HttpStatus.BAD_REQUEST, "Job " + job.getId() + " is not pending."); } - String hdfsWorkspace = ""; - try { - hdfsWorkspace = HdfsUtil.path(settings.getHdfsWorkspace(), job.getId()); - } catch (NoClassDefFoundError e) { - // ignore HDFS exception to work also without hadoop. - } String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), job.getId()); job.setLocalWorkspace(localWorkspace); - job.setHdfsWorkspace(hdfsWorkspace); job.setSettings(settings); - job.setRemoveHdfsWorkspace(settings.isRemoveHdfsWorkspace()); String appId = job.getApplicationId(); @@ -323,29 +312,19 @@ public String archive(AbstractJob job) { try { - IExternalWorkspace externalWorkspace = null; - if (!settings.getExternalWorkspaceLocation().isEmpty()) { - String externalOutput = settings.getExternalWorkspaceLocation(); - externalWorkspace = ExternalWorkspaceFactory.get(settings.getExternalWorkspaceType(), externalOutput); - } - // delete local directory and hdfs directory String localOutput = FileUtil.path(settings.getLocalWorkspace(), job.getId()); FileUtil.deleteDirectory(localOutput); - try { - String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(settings.getHdfsWorkspace(), job.getId())); - HdfsUtil.delete(hdfsOutput); - } catch (NoClassDefFoundError e) { - } + job.setState(AbstractJob.STATE_RETIRED); dao.update(job); - if (externalWorkspace != null) { - try { - externalWorkspace.delete(job.getId()); - } catch (Exception e) { - e.printStackTrace(); - } + IExternalWorkspace externalWorkspace = workspaceFactory.getByJob(job); + + try { + externalWorkspace.delete(job.getId()); + } catch (Exception e) { + log.error("Deleteting " + job.getId() + " from workspace failed.", e); } return "Retired job " + job.getId(); @@ -388,8 +367,8 @@ public String createId() { // TODO: refactore and combine this method with CommandLineUtil.parseArgs... - private Map parseAndUpdateInputParams(List form, WdlApp app, String hdfsWorkspace, - String localWorkspace) throws Exception { + private Map parseAndUpdateInputParams(List form, WdlApp app, + IExternalWorkspace externalWorkspace) throws Exception { Map props = new HashMap(); Map params = new HashMap(); @@ -406,8 +385,6 @@ private Map parseAndUpdateInputParams(List form, WdlA try { - String entryName = inputFile.getName(); - // remove upload indentification! String fieldName = name.replace("-upload", "").replace("input-", ""); @@ -420,43 +397,18 @@ private Map parseAndUpdateInputParams(List form, WdlA if (inputParam.isHdfs()) { - String targetPath = HdfsUtil.path(hdfsWorkspace, fieldName); + throw new Exception("HDFS support was removed in Cloudgene 3"); - String target = HdfsUtil.path(targetPath, entryName); - - HdfsUtil.put(inputFile.getAbsolutePath(), target); + } - if (inputParam.isFolder()) { - // folder - props.put(fieldName, HdfsUtil.makeAbsolute(HdfsUtil.path(hdfsWorkspace, fieldName))); - } else { - // file - props.put(fieldName, - HdfsUtil.makeAbsolute(HdfsUtil.path(hdfsWorkspace, fieldName, entryName))); - } + // copy to workspace in input directory + String target = externalWorkspace.uploadInput(fieldName, inputFile); + if (inputParam.isFolder()) { + props.put(fieldName, externalWorkspace.getParent(target)); } else { - - // copy to workspace in input directory - String targetPath = FileUtil.path(localWorkspace, "input", fieldName); - FileUtil.createDirectory(targetPath); - - String target = FileUtil.path(targetPath, entryName); - - FileUtil.copy(inputFile.getAbsolutePath(), target); - - if (inputParam.isFolder()) { - // folder - if (inputParam.getPattern() != null && !inputParam.getPattern().isEmpty()) { - props.put(fieldName, new File(targetPath).getAbsolutePath()); - } else { - props.put(fieldName, new File(targetPath).getAbsolutePath()); - } - } else { - // file - props.put(fieldName, new File(target).getAbsolutePath()); - } - + // file + props.put(fieldName, target); } // deletes temporary file @@ -482,7 +434,7 @@ private Map parseAndUpdateInputParams(List form, WdlA String cleanedValue = StringEscapeUtils.escapeHtml(value.toString()); - if (input != null && input.isFileOrFolder() && ImporterFactory.needsImport(cleanedValue)) { + if (input != null && input.isFileOrFolder() && needsImport(cleanedValue)) { throw new Exception("Parameter '" + input.getId() + "': URL-based uploads are no longer supported. Please use direct file uploads instead."); } @@ -584,4 +536,9 @@ public List getJobs(String state) { return jobs; } + public boolean needsImport(String url) { + return url.startsWith("sftp://") || url.startsWith("http://") || url.startsWith("https://") + || url.startsWith("ftp://") || url.startsWith("s3://"); + } + } diff --git a/src/main/java/cloudgene/mapred/server/tasks/ServerTasks.java b/src/main/java/cloudgene/mapred/server/tasks/ServerTasks.java index 073175a0..e2f6c76a 100644 --- a/src/main/java/cloudgene/mapred/server/tasks/ServerTasks.java +++ b/src/main/java/cloudgene/mapred/server/tasks/ServerTasks.java @@ -10,8 +10,6 @@ import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.WorkflowEngine; import cloudgene.mapred.server.Application; -import cloudgene.mapred.util.MailUtil; -import genepi.hadoop.HadoopUtil; import io.micronaut.scheduling.annotation.Scheduled; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -22,47 +20,6 @@ public class ServerTasks { @Inject protected Application application; - @Scheduled(fixedDelay = "1m") - public void checkHadoopCluster() { - - // check namenode state - - try { - boolean safemode = HadoopUtil.getInstance().isInSafeMode(); - - if (safemode) { - - if (application.getWorkflowEngine().isRunning()) { - - try { - - MailUtil.notifyAdmin(application.getSettings(), - "[" + application.getSettings().getName() + "] Problems with your Hadoop cluster", - "Hi,\n\n" + "This is a notification sent by Cloudgene.\n\n" - + "Your Hadoop cluster is in Safemode. " - + "Don't worry, we blocked the queue for you!"); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - application.getWorkflowEngine().block(); - - } - - } - } catch (NoClassDefFoundError e) { - // TODO: handle exception - } - - // if (cluster.getJobTrackerStatus() == ) - - // block queue if tasktracker is in safemode - - // send notification to admins - - } - @Scheduled(fixedDelay = "5m") public void writeStatistics() { diff --git a/src/main/java/cloudgene/mapred/steps/BashCommandStep.java b/src/main/java/cloudgene/mapred/steps/BashCommandStep.java index c725516a..13e95973 100644 --- a/src/main/java/cloudgene/mapred/steps/BashCommandStep.java +++ b/src/main/java/cloudgene/mapred/steps/BashCommandStep.java @@ -1,7 +1,6 @@ package cloudgene.mapred.steps; import java.io.File; -import java.io.IOException; import java.util.List; import java.util.Vector; @@ -9,8 +8,6 @@ import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; import cloudgene.mapred.wdl.WdlStep; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; public class BashCommandStep extends CloudgeneStep { @@ -62,35 +59,10 @@ public boolean run(WdlStep step, CloudgeneContext context) { String bashCommand = ""; for (String param : params) { - // checkout hdfs file - if (param.startsWith("hdfs://")) { - String name = FileUtil.getFilename(param); - String localFile = FileUtil.path(((CloudgeneContext) context).getLocalTemp(), "local_" + name); - try { - HdfsUtil.checkOut(param, localFile); - String localFilename = new File(localFile).getAbsolutePath(); - if (useBash) { - bashCommand += localFilename + " "; - } else { - command.add(localFilename); - } - - } catch (IOException e) { - context.log(e.getMessage()); - if (useBash) { - bashCommand += param + " "; - } else { - command.add(param); - } - } - + if (useBash) { + bashCommand += param + " "; } else { - if (useBash) { - bashCommand += param + " "; - } else { - command.add(param); - } - + command.add(param); } } @@ -117,7 +89,9 @@ public boolean run(WdlStep step, CloudgeneContext context) { if (streamStdout) { context.endTask(output.toString(), Message.ERROR); } else { - context.endTask("Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", Message.ERROR); + context.endTask( + "Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", + Message.ERROR); } return false; } diff --git a/src/main/java/cloudgene/mapred/steps/GroovyStep.java b/src/main/java/cloudgene/mapred/steps/GroovyStep.java index 9e0ec43a..ac4c691e 100644 --- a/src/main/java/cloudgene/mapred/steps/GroovyStep.java +++ b/src/main/java/cloudgene/mapred/steps/GroovyStep.java @@ -10,6 +10,7 @@ import cloudgene.sdk.internal.WorkflowContext; import genepi.io.FileUtil; import groovy.util.GroovyScriptEngine; + public class GroovyStep extends CloudgeneStep { @Override @@ -26,27 +27,12 @@ public boolean run(WdlStep step, CloudgeneContext context) { Class scriptClass = new GroovyScriptEngine(".", getClass().getClassLoader()).loadScriptByName(filename); Object scriptInstance = scriptClass.newInstance(); - try { - Method method = scriptClass.getDeclaredMethod("run", new Class[] { WorkflowContext.class }); - Object result = method.invoke(scriptInstance, new Object[] { context }); - if (result instanceof Boolean) { - return (Boolean) result; - } else { - return true; - } - } catch (NoSuchMethodException e) { - - // old dependency to genepi.hadoop - Method method = scriptClass.getDeclaredMethod("run", - new Class[] { genepi.hadoop.common.WorkflowContext.class }); - Object result = method.invoke(scriptInstance, - new Object[] { JavaInternalStepDeprecrated.adapter(context) }); - if (result instanceof Boolean) { - return (Boolean) result; - } else { - return true; - } - + Method method = scriptClass.getDeclaredMethod("run", new Class[] { WorkflowContext.class }); + Object result = method.invoke(scriptInstance, new Object[] { context }); + if (result instanceof Boolean) { + return (Boolean) result; + } else { + return true; } } catch (Exception e) { @@ -61,9 +47,9 @@ public boolean run(WdlStep step, CloudgeneContext context) { } public static String getStackTraceAsString(Throwable throwable) { - StringWriter stringWriter = new StringWriter(); - throwable.printStackTrace(new PrintWriter(stringWriter)); - return stringWriter.toString(); - } - + StringWriter stringWriter = new StringWriter(); + throwable.printStackTrace(new PrintWriter(stringWriter)); + return stringWriter.toString(); + } + } diff --git a/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java b/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java deleted file mode 100644 index d2e45e0b..00000000 --- a/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java +++ /dev/null @@ -1,281 +0,0 @@ -package cloudgene.mapred.steps; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.List; -import java.util.Vector; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.jobs.CloudgeneStep; -import cloudgene.mapred.jobs.Message; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; -import cloudgene.mapred.wdl.WdlStep; -import cloudgene.sdk.internal.WorkflowContext; -import genepi.hadoop.HadoopUtil; -import genepi.io.FileUtil; - -public class HadoopMapReduceStep extends CloudgeneStep { - - private String jobId; - - private int map = 0; - - private int reduce = 0; - - @Override - public boolean run(WdlStep step, CloudgeneContext context) { - - String hadoopPath = context.getSettings().getHadoopPath(); - - File path = new File(hadoopPath); - - if (!path.exists()) { - context.error("Hadoop Binary was not found. Please set the correct path in the admin panel."); - return false; - } - - String hadoop = ""; - - if (path.isDirectory()) { - hadoop = FileUtil.path(hadoopPath, "bin", "hadoop"); - } else { - hadoop = hadoopPath; - } - - File file = new File(hadoop); - - if (!file.exists()) { - context.error("Hadoop Binary was not found. Please set the correct path in the admin panel."); - return false; - } - - if (!file.canExecute()) { - context.error( - "Hadoop Binary was found (" + hadoop + ") but can not be executed. Please check the permissions."); - return false; - } - - String streamingJar = context.getSettings().getStreamingJar(); - - // params - String paramsString = step.get("params"); - String[] params = new String[] {}; - if (paramsString != null) { - params = paramsString.split(" "); - } - - // hadoop jar or streaming - List command = new Vector(); - - command.add(hadoop); - - // -fs and -jt and -config are supported by generic tool runner. - - command.add("jar"); - - if (step.getJar() != null) { - - // classical - command.add(step.getJar()); - - } else { - - // streaming - - if (context.getSettings().isStreaming()) { - - command.add(streamingJar); - - } else { - - context.error( - "Streaming mode is disabled.\nPlease specify the streaming-jar file in config/settings.yaml to run this job.."); - return false; - - } - - } - - for (String tile : params) { - command.add(tile.trim()); - } - - // mapper and reducer - - if (step.getJar() == null) { - - if (step.get("mapper") != null) { - - String tiles[] = step.get("mapper").split(" ", 2); - String filename = tiles[0]; - - command.add("-mapper"); - - if (tiles.length > 1) { - String params2 = tiles[1]; - command.add(filename + " " + params2); - } else { - command.add(filename); - } - - } - - if (step.get("reducer") != null) { - - String tiles[] = step.get("reducer").split(" ", 2); - String filename = tiles[0]; - - command.add("-reducer"); - - if (tiles.length > 1) { - String params2 = tiles[1]; - command.add(filename + " " + params2); - } else { - command.add(filename); - } - - } - - } - - try { - context.beginTask("Running Hadoop Job..."); - boolean successful = executeCommand(command, context); - if (successful) { - context.endTask("Execution successful.", Message.OK); - return true; - } else { - context.endTask("Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", Message.ERROR); - return false; - } - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - } - - @Override - protected boolean executeCommand(List command, WorkflowContext context) - throws IOException, InterruptedException { - // set global variables - for (int j = 0; j < command.size(); j++) { - - String cmd = command.get(j).replaceAll("\\$job_id", context.getJobId()); - command.set(j, cmd); - } - - context.log("Command: " + command); - context.log("Working Directory: " + new File(context.getWorkingDirectory()).getAbsolutePath()); - - ProcessBuilder builder = new ProcessBuilder(command); - builder.directory(new File(context.getWorkingDirectory())); - builder.redirectErrorStream(true); - Process process = builder.start(); - InputStream is = process.getInputStream(); - InputStreamReader isr = new InputStreamReader(is); - BufferedReader br = new BufferedReader(isr); - String line = null; - - // Find job id and write output into file - Pattern pattern = Pattern.compile("Running job: (.*)"); - Pattern pattern2 = Pattern.compile("HadoopJobId: (.*)"); - - while ((line = br.readLine()) != null) { - - Matcher matcher = pattern.matcher(line); - if (matcher.find()) { - // write statistics from old job - jobId = matcher.group(1).trim(); - context.log("Job " + context.getJobId() + " -> HadoopJob " + jobId); - } else { - Matcher matcher2 = pattern2.matcher(line); - if (matcher2.find()) { - jobId = matcher2.group(1).trim(); - context.log("Job " + context.getJobId() + " -> HadoopJob " + jobId); - } - } - - context.println(line); - } - - br.close(); - isr.close(); - is.close(); - - process.waitFor(); - context.log("Exit Code: " + process.exitValue()); - - if (process.exitValue() != 0) { - return false; - } else { - process.destroy(); - } - return true; - } - - @Override - public void updateProgress() { - - /*RunningJob job = HadoopUtil.getInstance().getJob(jobId); - if (job != null) { - - try { - - if (job.setupProgress() >= 1) { - map = ((int) (job.mapProgress() * 100)); - reduce = ((int) (job.reduceProgress() * 100)); - } else { - map = 0; - reduce = 0; - } - - } catch (Exception e) { - map = 0; - reduce = 0; - } - - } else { - map = 0; - reduce = 0; - }*/ - - } - - @Override - public int getProgress() { - return map / 2 + reduce / 2; - } - - @Override - public void kill() { - - try { - - if (jobId != null) { - - context.log(" Cancel Job " + jobId); - - HadoopUtil.getInstance().kill(jobId); - - } - - } catch (IOException e) { - - context.log(" Cancel Job failed: " + e); - - } - - } - - @Override - public String[] getRequirements() { - return new String[] { HadoopPlugin.ID }; - } - -} diff --git a/src/main/java/cloudgene/mapred/steps/HadoopPigStep.java b/src/main/java/cloudgene/mapred/steps/HadoopPigStep.java deleted file mode 100644 index da78fe12..00000000 --- a/src/main/java/cloudgene/mapred/steps/HadoopPigStep.java +++ /dev/null @@ -1,63 +0,0 @@ -package cloudgene.mapred.steps; - -import java.util.List; -import java.util.Vector; - -import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.jobs.CloudgeneStep; -import cloudgene.mapred.jobs.Message; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; -import cloudgene.mapred.wdl.WdlStep; -import genepi.io.FileUtil; - -public class HadoopPigStep extends CloudgeneStep { - - @Override - public boolean run(WdlStep step, CloudgeneContext context) { - - String pigPath = context.getSettings().getPigPath(); - String pig = FileUtil.path(pigPath, "bin", "pig"); - - // params - String paramsString = step.get("params"); - String[] params = new String[] {}; - if (paramsString != null) { - params = paramsString.split(" "); - } - - // pig script - List command = new Vector(); - - command.add(pig); - command.add("-f"); - command.add(step.get("pig")); - - // params - for (String tile : params) { - command.add(tile.trim()); - } - - try { - context.beginTask("Running Pig Script..."); - boolean successful = executeCommand(command, context); - if (successful) { - context.endTask("Execution successful.", Message.OK); - return true; - } else { - context.endTask("Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", - Message.ERROR); - return false; - } - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - } - - @Override - public String[] getRequirements() { - return new String[] { HadoopPlugin.ID }; - } - -} diff --git a/src/main/java/cloudgene/mapred/steps/HadoopSparkStep.java b/src/main/java/cloudgene/mapred/steps/HadoopSparkStep.java deleted file mode 100644 index 65362518..00000000 --- a/src/main/java/cloudgene/mapred/steps/HadoopSparkStep.java +++ /dev/null @@ -1,62 +0,0 @@ -package cloudgene.mapred.steps; - -import java.util.List; -import java.util.Vector; - -import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.jobs.CloudgeneStep; -import cloudgene.mapred.jobs.Message; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; -import cloudgene.mapred.wdl.WdlStep; - -public class HadoopSparkStep extends CloudgeneStep { - - @Override - public boolean run(WdlStep step, CloudgeneContext context) { - - String pigPath = context.getSettings().getSparkPath(); - - // params - String paramsString = step.get("params"); - String[] params = new String[] {}; - if (paramsString != null) { - params = paramsString.split(" "); - } - - // spark script - List command = new Vector(); - - command.add(pigPath); - command.add("--class"); - command.add(step.get("main_class")); - command.add("--master"); - command.add("yarn"); - command.add(step.get("spark")); - - // params - for (String tile : params) { - command.add(tile.trim()); - } - - try { - context.beginTask("Running Spark Script..."); - boolean successful = executeCommand(command, context); - if (successful) { - context.endTask("Execution successful.", Message.OK); - return true; - } else { - context.endTask("Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", Message.ERROR); - return false; - } - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - } - - @Override - public String[] getRequirements() { - return new String[] { HadoopPlugin.ID }; - } -} diff --git a/src/main/java/cloudgene/mapred/steps/HdfsImporterStep.java b/src/main/java/cloudgene/mapred/steps/HdfsImporterStep.java deleted file mode 100644 index 4b589529..00000000 --- a/src/main/java/cloudgene/mapred/steps/HdfsImporterStep.java +++ /dev/null @@ -1,97 +0,0 @@ -package cloudgene.mapred.steps; - -import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.jobs.CloudgeneStep; -import cloudgene.mapred.jobs.Message; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; -import cloudgene.mapred.wdl.WdlStep; -import genepi.hadoop.HdfsUtil; -import genepi.hadoop.importer.IImporter; -import genepi.hadoop.importer.ImporterFactory; - -public class HdfsImporterStep extends CloudgeneStep { - - @Override - public boolean run(WdlStep step, CloudgeneContext context) { - - try { - - for (String input : context.getInputs()) { - - if (ImporterFactory.needsImport(context.get(input))) { - - String[] urlList = context.get(input).split(";")[0].split("\\s+"); - - String username = ""; - if (context.get(input).split(";").length > 1) { - username = context.get(input).split(";")[1]; - } - - String password = ""; - if (context.get(input).split(";").length > 2) { - password = context.get(input).split(";")[2]; - } - - for (String url2 : urlList) { - - String url = url2 + ";" + username + ";" + password; - - String target = HdfsUtil.path(((CloudgeneContext) context).getHdfsTemp(), "importer", input); - - try { - - context.beginTask("Import File(s) " + url2 + "..."); - - IImporter importer = ImporterFactory.createImporter(url, target); - - if (importer != null) { - - boolean successful = importer.importFiles(); - - if (successful) { - - context.setInput(input, target); - - context.endTask("Import File(s) " + url2 + " successful.", Message.OK); - - } else { - - context.endTask("Import File(s) " + url2 + " failed: " + importer.getErrorMessage(), - Message.ERROR); - - return false; - - } - - } else { - - context.endTask("Import File(s) " + url2 + " failed: Protocol not supported", - Message.ERROR); - - return false; - - } - - } catch (Exception e) { - context.endTask("Import File(s) " + url2 + " failed: " + e.toString(), Message.ERROR); - return false; - } - - } - - } - } - return true; - - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - } - - @Override - public String[] getRequirements() { - return new String[] { HadoopPlugin.ID }; - } -} diff --git a/src/main/java/cloudgene/mapred/steps/JavaInternalStepDeprecrated.java b/src/main/java/cloudgene/mapred/steps/JavaInternalStepDeprecrated.java deleted file mode 100644 index 58bb32aa..00000000 --- a/src/main/java/cloudgene/mapred/steps/JavaInternalStepDeprecrated.java +++ /dev/null @@ -1,187 +0,0 @@ -package cloudgene.mapred.steps; - -import java.util.Map; -import java.util.Set; - -import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.jobs.CloudgeneStep; -import cloudgene.mapred.wdl.WdlStep; -import genepi.hadoop.common.WorkflowContext; -import genepi.hadoop.common.WorkflowStep; - -public class JavaInternalStepDeprecrated extends CloudgeneStep { - - private WorkflowStep workflowStep; - - public JavaInternalStepDeprecrated(WorkflowStep step) { - this.workflowStep = step; - } - - @Override - public void setup(CloudgeneContext context) { - super.setup(context); - workflowStep.setup(adapter(context)); - } - - @Override - public void kill() { - workflowStep.kill(); - } - - @Override - public void updateProgress() { - workflowStep.updateProgress(); - } - - @Override - public boolean run(WdlStep step, CloudgeneContext context) { - context.setConfig(step); - return workflowStep.run(adapter(context)); - } - - public static genepi.hadoop.common.WorkflowContext adapter(CloudgeneContext context) { - return new WorkflowContext() { - - @Override - public void updateTask(String name, int type) { - context.updateTask(name, type); - } - - @Override - public void submitCounter(String name) { - context.submitCounter(name); - } - - @Override - public void setOutput(String input, String value) { - context.setOutput(input, value); - } - - @Override - public void setInput(String input, String value) { - context.setInput(input, value); - } - - @Override - public void setConfig(Map config) { - context.setConfig(config); - } - - @Override - public boolean sendNotification(String body) throws Exception { - return context.sendNotification(body); - } - - @Override - public boolean sendMail(String to, String subject, String body) throws Exception { - return context.sendMail(to, subject, body); - } - - @Override - public boolean sendMail(String subject, String body) throws Exception { - return context.sendMail(subject, body); - } - - @Override - public void println(String line) { - context.println(line); - } - - @Override - public void message(String message, int type) { - context.message(message, type); - } - - @Override - public void log(String line) { - context.log(line); - } - - @Override - public void incCounter(String name, int value) { - context.incCounter(name, value); - } - - @Override - public String getWorkingDirectory() { - return context.getWorkingDirectory(); - } - - @Override - public String getOutput(String param) { - return context.getOutput(param); - } - - @Override - public String getLocalTemp() { - return context.getLocalTemp(); - } - - @Override - public String getJobName() { - return context.getJobName(); - } - - @Override - public String getJobId() { - return context.getJobId(); - } - - @Override - public Set getInputs() { - return context.getInputs(); - } - - @Override - public String getInput(String param) { - return context.getInput(param); - } - - @Override - public String getHdfsTemp() { - return context.getHdfsTemp(); - } - - @Override - public Object getData(String key) { - return context.getData(key); - } - - @Override - public Map getCounters() { - return context.getCounters(); - } - - @Override - public String getConfig(String param) { - return context.getConfig(param); - } - - @Override - public String get(String param) { - return context.get(param); - } - - @Override - public void endTask(String message, int type) { - context.endTask(message, type); - } - - @Override - public String createLinkToFile(String id) { - return context.createLinkToFile(id); - } - - @Override - public String createLinkToFile(String id, String filename) { - return context.createLinkToFile(id, filename); - } - - @Override - public void beginTask(String name) { - context.beginTask(name); - } - }; - } - -} diff --git a/src/main/java/cloudgene/mapred/steps/RMarkdownLocalStep.java b/src/main/java/cloudgene/mapred/steps/RMarkdownLocalStep.java index e1780304..1380eac6 100644 --- a/src/main/java/cloudgene/mapred/steps/RMarkdownLocalStep.java +++ b/src/main/java/cloudgene/mapred/steps/RMarkdownLocalStep.java @@ -1,7 +1,6 @@ package cloudgene.mapred.steps; import java.io.File; -import java.io.IOException; import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.CloudgeneStep; @@ -10,9 +9,8 @@ import cloudgene.mapred.plugins.rscript.RScriptBinary; import cloudgene.mapred.plugins.rscript.RScriptFile; import cloudgene.mapred.plugins.rscript.RScriptPlugin; +import cloudgene.mapred.util.command.Command; import cloudgene.mapred.wdl.WdlStep; -import genepi.hadoop.HdfsUtil; -import genepi.hadoop.command.Command; import genepi.io.FileUtil; public class RMarkdownLocalStep extends CloudgeneStep { @@ -63,7 +61,9 @@ public boolean run(WdlStep step, CloudgeneContext context) { } return true; } else { - context.endTask("Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", Message.ERROR); + context.endTask( + "Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", + Message.ERROR); return false; } @@ -96,32 +96,7 @@ public int convert(String rmdScript, String outputHtml, String[] args, Cloudgene argsForScript[0] = scriptFilename; // argsForScript[1] = "--args"; for (int i = 0; i < args.length; i++) { - - // checkout hdfs file - if (args[i].startsWith("hdfs://")) { - - String localFile = FileUtil.path(folder, "local_file_" + i); - context.log("Check out file " + args[i] + "..."); - try { - HdfsUtil.checkOut(args[i], localFile); - argsForScript[i + 1] = localFile; - } catch (IOException e) { - context.log(e.getMessage()); - argsForScript[i + 1] = args[i]; - } - - try { - context.log("Number of lines: " + FileUtil.getLineCount(localFile)); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - } else { - - argsForScript[i + 1] = args[i]; - - } + argsForScript[i + 1] = args[i]; } rScript.setParams(argsForScript); diff --git a/src/main/java/cloudgene/mapred/steps/SftpStep.java b/src/main/java/cloudgene/mapred/steps/SftpStep.java deleted file mode 100644 index 21438be4..00000000 --- a/src/main/java/cloudgene/mapred/steps/SftpStep.java +++ /dev/null @@ -1,111 +0,0 @@ -package cloudgene.mapred.steps; - -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; -import genepi.hadoop.importer.IImporter; -import genepi.hadoop.importer.ImporterFactory; -import genepi.io.FileUtil; - -public class SftpStep extends WorkflowStep { - - @Override - public boolean run(WorkflowContext context) { - - try { - Thread.sleep(5000); - return importVcfFiles(context); - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - } - - private boolean importVcfFiles(WorkflowContext context) { - try { - for (String input : context.getInputs()) { - - if (ImporterFactory.needsImport(context.get(input))) { - - context.beginTask("Importing files..."); - - String[] urlList = context.get(input).split(";")[0].split("\\s+"); - - String username = ""; - if (context.get(input).split(";").length > 1) { - username = context.get(input).split(";")[1]; - } - - String password = ""; - if (context.get(input).split(";").length > 2) { - password = context.get(input).split(";")[2]; - } - - for (String url2 : urlList) { - - String url = url2 + ";" + username + ";" + password; - String target = FileUtil.path(context.getLocalTemp(), "importer", input); - FileUtil.createDirectory(target); - context.log("Import to local workspace " + target + "..."); - - try { - - context.updateTask("Import " + url2 + "...", WorkflowContext.RUNNING); - - IImporter importer = ImporterFactory.createImporter(url, target); - - if (importer != null) { - - boolean successful = importer.importFiles("vcf.gz"); - - if (successful) { - - context.setInput(input, target); - - } else { - - context.updateTask("Import " + url2 + " failed: " + importer.getErrorMessage(), - WorkflowContext.ERROR); - - return false; - - } - - } else { - - context.updateTask("Import " + url2 + " failed: Protocol not supported", - WorkflowContext.ERROR); - - return false; - - } - - } catch (Exception e) { - context.updateTask("Import File(s) " + url2 + " failed: " + e.toString(), - WorkflowContext.ERROR); - e.printStackTrace(); - - return false; - } - - } - - context.updateTask("File Import successful. ", WorkflowContext.OK); - - } - - } - - return true; - } catch (Exception e) { - // context.updateTask("Import File(s) " + url2 + " failed: " + - // e.toString(), - // WorkflowContext.ERROR); - e.printStackTrace(); - - return false; - } - - } - -} diff --git a/src/main/java/cloudgene/mapred/util/S3Util.java b/src/main/java/cloudgene/mapred/util/S3Util.java new file mode 100644 index 00000000..1116d577 --- /dev/null +++ b/src/main/java/cloudgene/mapred/util/S3Util.java @@ -0,0 +1,184 @@ +package cloudgene.mapred.util; + +import java.io.File; +import java.io.IOException; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.transfer.Download; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import com.amazonaws.services.s3.transfer.Upload; + +public class S3Util { + + private static AmazonS3 s3; + + private static TransferManager tm; + + public static AmazonS3 getAmazonS3() { + if (s3 == null) { + s3 = AmazonS3ClientBuilder.defaultClient(); + } + return s3; + } + + public static TransferManager getTransferManager() { + if (tm == null) { + s3 = getAmazonS3(); + tm = TransferManagerBuilder.standard().withS3Client(s3).build(); + } + return tm; + } + + public static boolean isValidS3Url(String url) { + if (url.startsWith("s3://")) { + String temp = url.replaceAll("s3://", ""); + String[] tiles = temp.split("/", 2); + if (tiles.length == 2) { + return true; + } else { + return false; + } + } else { + return false; + } + + } + + public static String getBucket(String url) { + if (url.startsWith("s3://")) { + String temp = url.replaceAll("s3://", ""); + String[] tiles = temp.split("/", 2); + if (tiles.length == 2) { + return tiles[0]; + } else { + return null; + } + } else { + return null; + } + } + + public static String getKey(String url) { + if (url.startsWith("s3://")) { + String temp = url.replaceAll("s3://", ""); + String[] tiles = temp.split("/", 2); + if (tiles.length == 2) { + return tiles[1]; + } else { + return null; + } + } else { + return null; + } + + } + + public static void copyToFile(String url, File file) throws IOException { + if (isValidS3Url(url)) { + String bucket = getBucket(url); + String key = getKey(url); + copyToFile(bucket, key, file); + } else { + throw new IOException("Url '" + url + "' is not a valid S3 bucket."); + } + } + + public static void copyToFile(String bucket, String key, File file) throws IOException { + + TransferManager tm = getTransferManager(); + Download download = tm.download(bucket, key, file); + try { + download.waitForCompletion(); + } catch (InterruptedException e) { + throw new IOException(e); + } + + } + + public static void copyToS3(File file, String url) throws IOException { + if (isValidS3Url(url)) { + String bucket = getBucket(url); + String key = getKey(url); + copyToS3(file, bucket, key); + } else { + throw new IOException("Url '" + url + "' is not a valid S3 bucket."); + } + } + + public static void copyToS3(String content, String url) throws IOException { + if (isValidS3Url(url)) { + String bucket = getBucket(url); + String key = getKey(url); + copyToS3(content, bucket, key); + } else { + throw new IOException("Url '" + url + "' is not a valid S3 bucket."); + } + } + + public static void copyToS3(File file, String bucket, String key) throws IOException { + + TransferManager tm = getTransferManager(); + Upload upload = tm.upload(bucket, key, file); + try { + upload.waitForCompletion(); + } catch (InterruptedException e) { + throw new IOException(e); + } + + } + + public static void copyToS3(String content, String bucket, String key) throws IOException { + + AmazonS3 s3 = getAmazonS3(); + s3.putObject(bucket, key, content); + + } + + public static ObjectListing listObjects(String url) throws IOException { + + AmazonS3 s3 = getAmazonS3(); + + if (isValidS3Url(url)) { + + String bucket = getBucket(url); + String key = getKey(url); + + ObjectListing objects = s3.listObjects(bucket, key); + + return objects; + } else { + throw new IOException("Url '" + url + "' is not a valid S3 bucket."); + } + + } + + public static void deleteFolder(String url) { + + String bucket = S3Util.getBucket(url); + String key = S3Util.getKey(url); + + AmazonS3 s3 = S3Util.getAmazonS3(); + + ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(key); + + ObjectListing objectListing = s3.listObjects(listObjectsRequest); + + while (true) { + for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { + s3.deleteObject(bucket, objectSummary.getKey()); + } + if (objectListing.isTruncated()) { + objectListing = s3.listNextBatchOfObjects(objectListing); + } else { + break; + } + } + + } + +} \ No newline at end of file diff --git a/src/main/java/cloudgene/mapred/util/Settings.java b/src/main/java/cloudgene/mapred/util/Settings.java index e675aad8..093e9aee 100644 --- a/src/main/java/cloudgene/mapred/util/Settings.java +++ b/src/main/java/cloudgene/mapred/util/Settings.java @@ -19,7 +19,6 @@ import cloudgene.mapred.apps.Application; import cloudgene.mapred.apps.ApplicationRepository; -import genepi.hadoop.HadoopCluster; import genepi.io.FileUtil; public class Settings { @@ -198,22 +197,6 @@ public static Settings load(Config config) throws FileNotFoundException, YamlExc log.info("Notify user after " + settings.notificationAfter + " days."); log.info("Write statistics: " + settings.writeStatistics); - if (settings.cluster != null) { - String conf = settings.cluster.get("conf"); - String username = settings.cluster.get("user"); - String name = settings.cluster.get("name"); - if (conf != null) { - log.info("Use Haddop configuration folder '" + conf + "'" - + (username != null ? " with username " + username : "")); - try { - HadoopCluster.setConfPath(name, conf, username); - } catch (NoClassDefFoundError e) { - } - } - } - - settings.config = config; - // workspace in config has higher priority if (config.getWorkspace() != null) { settings.setLocalWorkspace(config.getWorkspace()); diff --git a/src/main/java/cloudgene/mapred/util/command/Command.java b/src/main/java/cloudgene/mapred/util/command/Command.java new file mode 100644 index 00000000..5d6df9e8 --- /dev/null +++ b/src/main/java/cloudgene/mapred/util/command/Command.java @@ -0,0 +1,204 @@ +package cloudgene.mapred.util.command; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Vector; + +import org.apache.commons.codec.digest.DigestUtils; + +public class Command { + + protected String cmd; + + private String[] params; + + private boolean silent = false; + + private boolean deleteInput = false; + + private String directory = null; + + private StringBuffer stout = new StringBuffer(); + + private String stdoutFileName = null; + + private String stderrFileName = null; + + private List inputs = new Vector(); + + private List outputs = new Vector(); + + public Command(String cmd, String... params) { + this.cmd = cmd; + this.params = params; + } + + public Command(String cmd) { + this.cmd = cmd; + } + + public void setParams(String... params) { + this.params = params; + } + + public void setParams(List params) { + this.params = new String[params.size()]; + for (int i = 0; i < params.size(); i++) { + this.params[i] = params.get(i); + } + } + + public void saveStdOut(String filename) { + this.stdoutFileName = filename; + } + + public void saveStdErr(String filename) { + this.stderrFileName = filename; + } + + public int execute() { + + List command = new ArrayList(); + + command.add(cmd); + + if (params != null) { + for (String param : params) { + command.add(param); + } + } + + try { + + ProcessBuilder builder = new ProcessBuilder(command); + // builder.redirectErrorStream(true); + if (directory != null) { + builder.directory(new File(directory)); + } + + Process process = builder.start(); + CommandStreamHandler handler = new CommandStreamHandler( + process.getInputStream(), stdoutFileName); + handler.setSilent(silent); + Thread inputStreamHandler = new Thread(handler); + + CommandStreamHandler handler2 = new CommandStreamHandler( + process.getErrorStream(), stderrFileName); + handler2.setSilent(silent); + Thread errorStreamHandler = new Thread(handler2); + + inputStreamHandler.start(); + errorStreamHandler.start(); + + process.waitFor(); + + inputStreamHandler.interrupt(); + errorStreamHandler.interrupt(); + inputStreamHandler.join(); + errorStreamHandler.join(); + + if (process.exitValue() != 0) { + return process.exitValue(); + } else { + process.destroy(); + } + + if (deleteInput) { + new File(cmd).delete(); + } + + return 0; + } catch (Exception e) { + e.printStackTrace(); + return -1; + } + } + + public boolean isSilent() { + return silent; + } + + public void setSilent(boolean silent) { + this.silent = silent; + } + + public boolean isDeleteInput() { + return deleteInput; + } + + public void setDeleteInput(boolean deleteInput) { + this.deleteInput = deleteInput; + } + + public void setDirectory(String directory) { + this.directory = directory; + } + + public String getDirectory() { + return directory; + } + + @Override + public String toString() { + String result = cmd; + if (params != null) { + for (String param : params) { + result += " " + param; + } + } + return result; + } + + public String getStdOut() { + return stout.toString(); + } + + public void addInput(String input) { + inputs.add(input); + } + + public void addOutput(String output) { + outputs.add(output); + } + + public List getInputs() { + return inputs; + } + + public List getOutputs() { + return outputs; + } + + private boolean isNoFile(String param) { + return !inputs.contains(param) && !outputs.contains(param); + } + + public String getSignature() { + + String fullCommand = cmd; + for (String param : params) { + if (isNoFile(param)) { + fullCommand += param; + } + } + + return DigestUtils.md5Hex(fullCommand); + + } + + public String getName() { + return cmd; + } + + public String getExecutedCommand() { + String executedCommand = cmd; + if (params != null) { + for (String param : params) { + executedCommand += " " + param; + } + } + return executedCommand; + } + +} \ No newline at end of file diff --git a/src/main/java/cloudgene/mapred/util/command/CommandStreamHandler.java b/src/main/java/cloudgene/mapred/util/command/CommandStreamHandler.java new file mode 100644 index 00000000..35e94811 --- /dev/null +++ b/src/main/java/cloudgene/mapred/util/command/CommandStreamHandler.java @@ -0,0 +1,73 @@ +package cloudgene.mapred.util.command; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; + +public class CommandStreamHandler implements Runnable { + + private InputStream is; + + //private StringBuffer stdout = new StringBuffer();; + + private boolean silent = false; + + private String filename = null; + + public CommandStreamHandler(InputStream is) { + this.is = is; + } + + public CommandStreamHandler(InputStream is, String filename) { + this.is = is; + this.filename = filename; + } + + public void setSilent(boolean silent) { + this.silent = silent; + } + + public void setFilename(String filename) { + this.filename = filename; + } + + @Override + public void run() { + + try { + + boolean save = (filename != null && !filename.isEmpty()); + FileOutputStream writer = null; + + byte[] buffer = new byte[200]; + + if (save) { + writer = new FileOutputStream(filename); + } + + int size = 0; + + while ((size = is.read(buffer)) > 0) { + //stdout.append(line); + if (!silent) { + String line = new String(buffer, 0, size); + System.out.println(line); + } + if (save) { + writer.write(buffer, 0, size); + } + } + + if (save) { + writer.close(); + } + + is.close(); + + } catch (IOException e) { + e.printStackTrace(); + } + + } + +} \ No newline at end of file diff --git a/src/test/java/cloudgene/mapred/TestApplication.java b/src/test/java/cloudgene/mapred/TestApplication.java index 5fceeab9..a303534c 100644 --- a/src/test/java/cloudgene/mapred/TestApplication.java +++ b/src/test/java/cloudgene/mapred/TestApplication.java @@ -1,7 +1,6 @@ package cloudgene.mapred; import java.io.FileNotFoundException; -import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Vector; @@ -15,7 +14,6 @@ import cloudgene.mapred.util.Config; import cloudgene.mapred.util.HashUtil; import cloudgene.mapred.util.Settings; -import cloudgene.mapred.util.TestCluster; import cloudgene.mapred.util.TestMailServer; import genepi.io.FileUtil; import io.micronaut.context.annotation.Context; @@ -38,15 +36,6 @@ public TestApplication() throws Exception { protected static Settings loadSettings(Config config) throws FileNotFoundException, YamlException { - System.out.println("Starting test Hadoop cluster..."); - try { - TestCluster.getInstance().start(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - System.out.println("Test Hadoop cluster started."); - Settings settings = new Settings(new Config()); HashMap mail = new HashMap(); @@ -155,32 +144,8 @@ protected static List registerApplications(Settings settings) { app14.setPermission("public"); applications.add(app14); - // hdfs - - Application app10 = new Application(); - app10.setId("all-possible-inputs-hdfs"); - app10.setFilename("test-data/all-possible-inputs-hdfs.yaml"); - app10.setPermission("public"); - applications.add(app10); - - Application app11 = new Application(); - app11.setId("write-files-to-hdfs-folder"); - app11.setFilename("test-data/write-files-to-hdfs-folder.yaml"); - app11.setPermission("public"); - applications.add(app11); - - Application app12 = new Application(); - app12.setId("write-text-to-hdfs-file"); - app12.setFilename("test-data/write-text-to-hdfs-file.yaml"); - app12.setPermission("public"); - applications.add(app12); - - Application app16 = new Application(); - app16.setId("sftp-import"); - app16.setFilename("test-data/sftp-import.yaml"); - app16.setPermission("public"); - applications.add(app16); - + //app links + Application app17 = new Application(); app17.setId("app-links"); app17.setFilename("test-data/app-links.yaml"); @@ -205,18 +170,6 @@ protected static List registerApplications(Settings settings) { app19.setPermission("protected"); applications.add(app19); - Application app20 = new Application(); - app20.setId("app-installation2"); - app20.setFilename("test-data/app-installation2.yaml"); - app20.setPermission("public"); - applications.add(app20); - - Application app21 = new Application(); - app21.setId("app-installation-child"); - app21.setFilename("test-data/app-installation-child.yaml"); - app21.setPermission("public"); - applications.add(app21); - Application app22 = new Application(); app22.setId("print-hidden-inputs"); app22.setFilename("test-data/print-hidden-inputs.yaml"); diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java index 57ed7b4a..b15db9f6 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java @@ -94,40 +94,6 @@ public void testDownloadSingleFolder() throws InterruptedException { } - @Test - public void testDownloadSingleHdfsFolder() throws InterruptedException { - - Header accessToken = client.loginAsPublicUser(); - - // submit job - String id = RestAssured.given().header(accessToken).and().multiPart("inputtext", "lukas_text").when() - .post("/api/v2/jobs/submit/write-files-to-hdfs-folder").then().statusCode(200).and().extract() - .jsonPath().getString("id"); - - // wait until submitted job is complete - client.waitForJob(id, accessToken); - - // TODO: check why file is not available without this sleep - Thread.sleep(5000); - - // get details - Response response = RestAssured.given().header(accessToken).when().get("/api/v2/jobs/" + id).thenReturn(); - response.then().statusCode(200).and().body("state", equalTo(AbstractJob.STATE_SUCCESS)).and() - .body("outputParams[0].name", equalTo("output")).and().body("outputParams[0].files.size()", equalTo(5)); - - // get path and download all 5 files - for (int i = 0; i < 5; i++) { - String path = response.body().jsonPath().getString("outputParams[0].files[" + i + "].path"); - String name = response.jsonPath().getString("outputParams[0].files[" + i + "].name"); - String hash = response.jsonPath().getString("outputParams[0].files[" + i + "].hash"); - - assertEquals(id + "/output/file" + (i + 1) + ".txt", path); - RestAssured.given().header(accessToken).when().get("/downloads/" + id + "/" + hash + "/" + name).then() - .statusCode(200).and().body(equalTo("lukas_text")); - } - - } - @Test public void testDownloadCounter() throws InterruptedException { diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java index 0d651433..17ce60b9 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java @@ -91,32 +91,6 @@ public void testSubmitAllPossibleInputs() { } - @Test - public void testSubmitAllPossibleInputsHdfs() { - - Header accessToken = client.loginAsPublicUser(); - - // local-file - FileUtil.writeStringBufferToFile("test.txt", new StringBuffer("content-of-my-file")); - FileUtil.writeStringBufferToFile("test1.txt", new StringBuffer("content-of-my-file-in-folder1")); - FileUtil.writeStringBufferToFile("test2.txt", new StringBuffer("content-of-my-file-in-folder2")); - - // submit job with different inputs and file uploads - String id = RestAssured.given().header(accessToken).and().multiPart("job-name", "my-job-name").and() - .multiPart("input-file", new File("test.txt")).and().multiPart("input-folder", new File("test1.txt")) - .and().multiPart("input-folder", new File("test2.txt")).when() - .post("/api/v2/jobs/submit/all-possible-inputs-hdfs").then().statusCode(200).and().extract().jsonPath() - .getString("id"); - - // wait until job is complete - client.waitForJob(id, accessToken); - - // get details and check state - RestAssured.given().header(accessToken).when().get("/api/v2/jobs/" + id).then().statusCode(200).and() - .body("state", equalTo(AbstractJob.STATE_SUCCESS)).and().body("name", equalTo("my-job-name")); - - } - @Test public void testSubmitReturnTrueStepPublic() { @@ -203,35 +177,6 @@ public void testSubmitWriteTextToFilePublic() throws InterruptedException { } - @Test - public void testSubmitWriteTextToHdfsFilePublic() throws InterruptedException { - - Header accessToken = client.loginAsPublicUser(); - - // submit jobs - String id = RestAssured.given().header(accessToken).and().multiPart("input-inputtext", "lukas_text").when() - .post("/api/v2/jobs/submit/write-text-to-hdfs-file").then().statusCode(200).and().extract().jsonPath() - .getString("id"); - - // wait until job is complete - client.waitForJob(id, accessToken); - - // TODO: change! - Thread.sleep(5000); - - Response response = RestAssured.given().header(accessToken).when().get("/api/v2/jobs/" + id).thenReturn(); - response.then().statusCode(200).and().body("state", equalTo(AbstractJob.STATE_SUCCESS)); - - // get file details - String name = response.jsonPath().getString("outputParams[0].files[0].name"); - String hash = response.jsonPath().getString("outputParams[0].files[0].hash"); - - // download file and check content - RestAssured.given().header(accessToken).when().get("/downloads/" + id + "/" + hash + "/" + name).then() - .statusCode(200).and().body(equalTo("lukas_text")); - - } - @Test public void testSubmitThreeTasksStepPublic() { diff --git a/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java b/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java index 910639d1..0dd2a525 100644 --- a/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java +++ b/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java @@ -12,10 +12,10 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; @@ -24,9 +24,13 @@ public class PriorityThreadPoolExecutorTest { private static final int WAIT_FOR_CANCEL = 8000; + @Inject TestApplication application; + @Inject + ExternalWorkspaceFactory workspaceFactory; + @Test public void testCancelRunningJob() throws Exception { @@ -469,17 +473,15 @@ public CloudgeneJob createJobFromWdl(WdlApp app, String id, Map Settings settings = application.getSettings(); - String hdfsWorkspace = HdfsUtil.path(settings.getHdfsWorkspace(), id); String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), id); FileUtil.createDirectory(localWorkspace); CloudgeneJob job = new CloudgeneJob(user, id,app, inputs); job.setId(id); job.setName(id); + job.setExternalWorkspace(workspaceFactory.getDefault()); job.setLocalWorkspace(localWorkspace); - job.setHdfsWorkspace(hdfsWorkspace); job.setSettings(settings); - job.setRemoveHdfsWorkspace(true); job.setApplication(app.getName() + " " + app.getVersion()); job.setApplicationId(app.getId()); diff --git a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java index 3ed2b636..df5420d1 100644 --- a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java +++ b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java @@ -13,11 +13,11 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; import cloudgene.sdk.internal.WorkflowContext; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; @@ -28,6 +28,10 @@ public class WorkflowEngineTest { @Inject TestApplication application; + @Inject + ExternalWorkspaceFactory workspaceFactory; + + @Test public void testReturnTrueStep() throws Exception { @@ -719,133 +723,6 @@ public void testApplicationLinksWrongPermissions() throws Exception { } - @Test - public void testApplicationInstallation() throws Exception { - - WorkflowEngine engine = application.getWorkflowEngine(); - - WdlApp app = WdlReader.loadAppFromFile("test-data/app-installation.yaml"); - - Map params = new HashMap(); - - AbstractJob job = createJobFromWdl(app, params); - job.forceInstallation(true); - engine.submit(job); - while (!job.isComplete()) { - Thread.sleep(500); - } - - assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getStartTime() > 0); - assertTrue(job.getEndTime() > 0); - assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); - - // single file - Message message = job.getSteps().get(0).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of metafile.txt")); - - // folder file1 - message = job.getSteps().get(1).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of file1.txt")); - - // folder file2 - message = job.getSteps().get(2).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of file2.txt")); - - // zip file1 - message = job.getSteps().get(3).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of file1.txt")); - - // zip file2 - message = job.getSteps().get(4).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of file2.txt")); - - // gz file1 - message = job.getSteps().get(5).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of file1.txt")); - - // gz file2 - message = job.getSteps().get(6).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of file2.txt")); - - // http single file - message = job.getSteps().get(7).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().contains("name: hello-cloudgene")); - } - - @Test - public void testApplicationInstallationAndLinks() throws Exception { - - WorkflowEngine engine = application.getWorkflowEngine(); - - WdlApp app = WdlReader.loadAppFromFile("test-data/app-installation2.yaml"); - - Map params = new HashMap(); - params.put("dataset", "apps@app-installation-child"); - - AbstractJob job = createJobFromWdl(app, params); - engine.submit(job); - while (!job.isComplete()) { - Thread.sleep(500); - } - - assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getStartTime() > 0); - assertTrue(job.getEndTime() > 0); - assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); - - Message message = job.getSteps().get(0).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of metafile2.txt")); - } - - @Test - public void testApplicationLinksAndPropertyList() throws Exception { - - WorkflowEngine engine = application.getWorkflowEngine(); - - WdlApp app = WdlReader.loadAppFromFile("test-data/app-installation3.yaml"); - - Map params = new HashMap(); - params.put("dataset", "apps@app-installation-child"); - - AbstractJob job = createJobFromWdl(app, params); - engine.submit(job); - while (!job.isComplete()) { - Thread.sleep(500); - } - - assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getStartTime() > 0); - assertTrue(job.getEndTime() > 0); - assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); - - } - - - // TODO: merge and zip export. - - // TODO: write to hdfs temp and local temp (temp output params)! - - // TODO: check if removehdfsworkspace works! - // TODO: check cloudgene counters (successful and failed) public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs) throws Exception { @@ -870,17 +747,15 @@ public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs, Use String id = "test_" + System.currentTimeMillis(); - String hdfsWorkspace = HdfsUtil.path(settings.getHdfsWorkspace(), id); String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), id); FileUtil.createDirectory(localWorkspace); CloudgeneJob job = new CloudgeneJob(user, id, app, inputs); job.setId(id); job.setName(id); + job.setExternalWorkspace(workspaceFactory.getDefault()); job.setLocalWorkspace(localWorkspace); - job.setHdfsWorkspace(hdfsWorkspace); job.setSettings(settings); - job.setRemoveHdfsWorkspace(true); job.setApplication(app.getName() + " " + app.getVersion()); job.setApplicationId(app.getId()); diff --git a/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java b/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java index 77ad4a69..73dfe0b1 100644 --- a/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java +++ b/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java @@ -11,10 +11,10 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.JobDao; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; @@ -25,6 +25,9 @@ public class WrongWorkspaceTest { @Inject TestApplication application; + @Inject + ExternalWorkspaceFactory workspaceFactory; + @Test public void testReturnTrueStep() throws Exception { @@ -60,17 +63,15 @@ public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs) thr String id = "test_" + System.currentTimeMillis(); - String hdfsWorkspace = HdfsUtil.path("/gsfgdfgdf/vdadsadwa", id); String localWorkspace = FileUtil.path("/gsfgdfgdf/vdadsadwa", id); FileUtil.createDirectory(localWorkspace); CloudgeneJob job = new CloudgeneJob(user, id, app, inputs); job.setId(id); job.setName(id); + job.setExternalWorkspace(workspaceFactory.getDefault()); job.setLocalWorkspace(localWorkspace); - job.setHdfsWorkspace(hdfsWorkspace); job.setSettings(settings); - job.setRemoveHdfsWorkspace(true); job.setApplication(app.getName() + " " + app.getVersion()); job.setApplicationId(app.getId()); diff --git a/src/test/java/cloudgene/mapred/jobs/steps/CheckHdfsInputs.java b/src/test/java/cloudgene/mapred/jobs/steps/CheckHdfsInputs.java deleted file mode 100644 index 4aa43042..00000000 --- a/src/test/java/cloudgene/mapred/jobs/steps/CheckHdfsInputs.java +++ /dev/null @@ -1,50 +0,0 @@ -package cloudgene.mapred.jobs.steps; - -import java.io.IOException; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.util.LineReader; - -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; -import genepi.hadoop.HdfsUtil; - -public class CheckHdfsInputs extends WorkflowStep { - - @Override - public boolean run(WorkflowContext context) { - - try { - - String fileContent = readFrom(context.get("file")); - - String fileContentInFolder1 = readFrom(context.get("folder") + "/" - + "test1.txt"); - String fileContentInFolder2 = readFrom(context.get("folder") + "/" - + "test2.txt"); - - boolean result = fileContent.equals("content-of-my-file") - && fileContentInFolder1 - .equals("content-of-my-file-in-folder1") - && fileContentInFolder2 - .equals("content-of-my-file-in-folder2"); - - return result; - } catch (Exception e) { - e.printStackTrace(); - return false; - } - } - - public String readFrom(String hdfs) throws IOException { - FileSystem fs = HdfsUtil.getFileSystem(); - LineReader reader = new LineReader(fs.open(new Path(hdfs))); - Text text = new Text(); - reader.readLine(text); - reader.close(); - return text.toString(); - } - -} diff --git a/src/test/java/cloudgene/mapred/jobs/steps/PrintDataset.java b/src/test/java/cloudgene/mapred/jobs/steps/PrintDataset.java index a752a32d..148dfaa9 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/PrintDataset.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/PrintDataset.java @@ -2,8 +2,8 @@ import java.util.Map; -import genepi.hadoop.common.WorkflowContext; -import genepi.hadoop.common.WorkflowStep; +import cloudgene.sdk.internal.WorkflowContext; +import cloudgene.sdk.internal.WorkflowStep; public class PrintDataset extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToHdfsFolderStep.java b/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToHdfsFolderStep.java deleted file mode 100644 index e7a7790c..00000000 --- a/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToHdfsFolderStep.java +++ /dev/null @@ -1,51 +0,0 @@ -package cloudgene.mapred.jobs.steps; - -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; - -public class WriteFilesToHdfsFolderStep extends WorkflowStep { - - @Override - public boolean run(WorkflowContext context) { - - String output = context.getOutput("output"); - String temp = context.getLocalTemp(); - - String text = context.getInput("inputtext"); - - try { - FileUtil.writeStringBufferToFile(FileUtil.path(temp, "file1.txt"), - new StringBuffer(text)); - HdfsUtil.put(FileUtil.path(temp, "file1.txt"), - HdfsUtil.path(output, "file1.txt")); - - FileUtil.writeStringBufferToFile(FileUtil.path(temp, "file2.txt"), - new StringBuffer(text)); - HdfsUtil.put(FileUtil.path(temp, "file2.txt"), - HdfsUtil.path(output, "file2.txt")); - - FileUtil.writeStringBufferToFile(FileUtil.path(temp, "file3.txt"), - new StringBuffer(text)); - HdfsUtil.put(FileUtil.path(temp, "file3.txt"), - HdfsUtil.path(output, "file3.txt")); - - FileUtil.writeStringBufferToFile(FileUtil.path(temp, "file4.txt"), - new StringBuffer(text)); - HdfsUtil.put(FileUtil.path(temp, "file4.txt"), - HdfsUtil.path(output, "file4.txt")); - - FileUtil.writeStringBufferToFile(FileUtil.path(temp, "file5.txt"), - new StringBuffer(text)); - HdfsUtil.put(FileUtil.path(temp, "file5.txt"), - HdfsUtil.path(output, "file5.txt")); - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - return true; - } - -} diff --git a/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToHdfsFileStep.java b/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToHdfsFileStep.java deleted file mode 100644 index 9952d890..00000000 --- a/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToHdfsFileStep.java +++ /dev/null @@ -1,28 +0,0 @@ -package cloudgene.mapred.jobs.steps; - -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; - -public class WriteTextToHdfsFileStep extends WorkflowStep { - - @Override - public boolean run(WorkflowContext context) { - - String temp = context.getLocalTemp(); - String hdfsFilename = context.getOutput("output"); - String text = context.getInput("inputtext"); - - try { - FileUtil.writeStringBufferToFile(FileUtil.path(temp, "temp.txt"), - new StringBuffer(text)); - HdfsUtil.put(FileUtil.path(temp, "temp.txt"), hdfsFilename); - } catch (Exception e) { - e.printStackTrace(); - return false; - } - return true; - } - -} diff --git a/src/test/java/cloudgene/mapred/steps/TestCommand.java b/src/test/java/cloudgene/mapred/steps/TestCommand.java index 0045c546..93714ea7 100644 --- a/src/test/java/cloudgene/mapred/steps/TestCommand.java +++ b/src/test/java/cloudgene/mapred/steps/TestCommand.java @@ -16,11 +16,11 @@ import cloudgene.mapred.jobs.CloudgeneJob; import cloudgene.mapred.jobs.Message; import cloudgene.mapred.jobs.WorkflowEngine; +import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; import cloudgene.sdk.internal.WorkflowContext; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; @@ -32,6 +32,9 @@ public class TestCommand { @Inject TestApplication application; + @Inject + ExternalWorkspaceFactory workspaceFactory; + @Test public void testValidCommand() throws Exception { @@ -137,17 +140,15 @@ public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs) thr String id = "test_" + System.currentTimeMillis(); - String hdfsWorkspace = HdfsUtil.path(settings.getHdfsWorkspace(), id); String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), id); FileUtil.createDirectory(localWorkspace); CloudgeneJob job = new CloudgeneJob(user, id, app, inputs); job.setId(id); + job.setExternalWorkspace(workspaceFactory.getDefault()); job.setName(id); job.setLocalWorkspace(localWorkspace); - job.setHdfsWorkspace(hdfsWorkspace); job.setSettings(settings); - job.setRemoveHdfsWorkspace(true); job.setApplication(app.getName() + " " + app.getVersion()); job.setApplicationId(app.getId()); diff --git a/src/test/java/cloudgene/mapred/util/TestCluster.java b/src/test/java/cloudgene/mapred/util/TestCluster.java deleted file mode 100644 index 070eda3d..00000000 --- a/src/test/java/cloudgene/mapred/util/TestCluster.java +++ /dev/null @@ -1,70 +0,0 @@ -package cloudgene.mapred.util; - -import genepi.hadoop.HdfsUtil; - -import java.io.File; -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapred.ClusterStatus; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MiniMRCluster; - -public class TestCluster { - - private static TestCluster instance; - - private static String WORKING_DIRECTORY = "test-cluster"; - - private MiniDFSCluster cluster; - - private FileSystem fs; - - private Configuration conf; - - private TestCluster() { - - } - - public static TestCluster getInstance() { - if (instance == null) { - instance = new TestCluster(); - } - return instance; - } - - public void start() throws IOException { - - if (cluster == null) { - - File testDataCluster1 = new File(WORKING_DIRECTORY); - if (testDataCluster1.exists()) { - testDataCluster1.delete(); - } - conf = new HdfsConfiguration(); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, - testDataCluster1.getAbsolutePath()); - cluster = new MiniDFSCluster.Builder(conf).build(); - fs = cluster.getFileSystem(); - - // set mincluster as default config - HdfsUtil.setDefaultConfiguration(conf); - System.setProperty("hadoop.log.dir", "test-log-dir"); - MiniMRCluster mrCluster = new MiniMRCluster(1, fs.getUri() - .toString(), 1, null, null, new JobConf(conf)); - JobConf mrClusterConf = mrCluster.createJobConf(); - HdfsUtil.setDefaultConfiguration(new Configuration(mrClusterConf)); - - System.out.println("------"); - - JobClient client = new JobClient(mrClusterConf); - ClusterStatus status = client.getClusterStatus(true); - System.out.println(status.getActiveTrackerNames()); - } - } - -} diff --git a/src/test/java/cloudgene/mapred/util/TestMailServer.java b/src/test/java/cloudgene/mapred/util/TestMailServer.java index f9c7c8b5..353d2580 100644 --- a/src/test/java/cloudgene/mapred/util/TestMailServer.java +++ b/src/test/java/cloudgene/mapred/util/TestMailServer.java @@ -1,11 +1,11 @@ package cloudgene.mapred.util; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import com.dumbster.smtp.SimpleSmtpServer; import com.dumbster.smtp.SmtpMessage; -import com.google.common.collect.Lists; /** * A JUnit Rule which runs a mock SMTP server @@ -47,7 +47,12 @@ public synchronized Iterator getReceivedEmail() { } public List getReceivedEmailAsList() { - return Lists.newArrayList((Iterator) getReceivedEmail()); + Iterator iterator = (Iterator) getReceivedEmail(); + List list = new ArrayList(); + while (iterator.hasNext()) { + list.add(iterator.next()); + } + return list; } } \ No newline at end of file diff --git a/test-data/all-possible-inputs-hdfs.yaml b/test-data/all-possible-inputs-hdfs.yaml deleted file mode 100644 index 2f8fc8b7..00000000 --- a/test-data/all-possible-inputs-hdfs.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: WriteTextToFileStep -description:

This job tests your configuration in order to ensure that Cloudgene is able to communicate with your Hadoop Cluster.
If the job fails, please follow the error message and adapt your configuration until the job runs successfully.
Useful informations about the configuration can be found on our website http://cloudgene.uibk.ac.at. -version: 1.0.1 -website: http://cloudgene.uibk.ac.at -category: cloudgene - -mapred: - - steps: - - name: WriteTextToFileStep - classname: cloudgene.mapred.jobs.steps.CheckHdfsInputs - - inputs: - - - id: file - description: Input-file - type: hdfs-file - - - id: folder - description: Input-folder - type: hdfs-folder - - outputs: - - - id: outputFile - description: OutputFile - type: hdfs-file - download: true - temp: false - zip: false - removeHeader: false - mergeOutput: false - - - id: outputFolder - description: outputFolder - type: hdfs-folder - download: true - temp: false - zip: false - removeHeader: true - mergeOutput: true diff --git a/test-data/app-installation-child.yaml b/test-data/app-installation-child.yaml deleted file mode 100644 index 3c475f8e..00000000 --- a/test-data/app-installation-child.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: app-installation-child -category: app-installation-datasets -version: 1.0.0 -properties: - metafile: ${hdfs_app_folder}/metafile2.txt - populations: - eur: EUR - afr: AFR - asn: ASN -installation: - - import: - source: ${local_app_folder}/metafile2.txt - target: ${hdfs_app_folder}/metafile2.txt diff --git a/test-data/app-installation.yaml b/test-data/app-installation.yaml deleted file mode 100644 index cd07114c..00000000 --- a/test-data/app-installation.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: app-installation -version: 1.0.0 -installation: - - import: - source: ${local_app_folder}/metafile.txt - target: ${hdfs_app_folder}/metafile.txt - - import: - source: ${local_app_folder}/folder - target: ${hdfs_app_folder}/folder - - import: - source: ${local_app_folder}/folder.zip - target: ${hdfs_app_folder}/zip - - import: - source: ${local_app_folder}/folder.tar.gz - target: ${hdfs_app_folder}/gz - - import: - source: https://raw.githubusercontent.com/genepi/cloudgene-website/master/hello-cloudgene.yaml - target: ${hdfs_app_folder}/http.yaml -workflow: - steps: - - name: tests single file - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/metafile.txt - - - name: tests folder / file1 - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/folder/file1.txt - - name: tests folder / file2 - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/folder/subfolder/file2.txt - - - name: tests zip archive / file1 - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/zip/file1.txt - - name: tests zip archive / file2 - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/zip/subfolder/file2.txt - - - name: tests gz archive / file1 - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/gz/file1.txt - - name: tests gz archive / file2 - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/gz/subfolder/file2.txt - - - name: tests http file - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/http.yaml diff --git a/test-data/app-installation2.yaml b/test-data/app-installation2.yaml deleted file mode 100644 index 642fafdc..00000000 --- a/test-data/app-installation2.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: app-installation2 -version: 1.0.0 -workflow: - steps: - - name: print content of hfs file - type: groovy - script: print-hdfs-file-dataset.groovy - inputs: - - id: dataset - description: Dataset - type: app_list - category: app-installation-datasets diff --git a/test-data/app-installation3.yaml b/test-data/app-installation3.yaml deleted file mode 100644 index 377b9280..00000000 --- a/test-data/app-installation3.yaml +++ /dev/null @@ -1,11 +0,0 @@ -name: app-installation2 -version: 1.0.0 -workflow: - steps: - - name: print content of hfs file - classname: cloudgene.mapred.jobs.steps.PrintDataset - inputs: - - id: dataset - description: Dataset - type: app_list - category: app-installation-datasets diff --git a/test-data/sftp-import.yaml b/test-data/sftp-import.yaml deleted file mode 100644 index 79eae817..00000000 --- a/test-data/sftp-import.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: sftp-import -description:

This job tests your configuration in order to ensure that Cloudgene is able to communicate with your Hadoop Cluster.
If the job fails, please follow the error message and adapt your configuration until the job runs successfully.
Useful informations about the configuration can be found on our website http://cloudgene.uibk.ac.at. -version: 1.0.1 -website: http://cloudgene.uibk.ac.at -category: cloudgene - - -mapred: - - setups: - - name: Import Files - classname: cloudgene.mapred.steps.SftpStep - - inputs: - - id: input - description: Dummy Input - type: local-file - - outputs: - - id: output - description: output - type: local-file - download: true - temp: false - zip: false - removeHeader: flase - mergeOutput: false diff --git a/test-data/write-files-to-hdfs-folder.yaml b/test-data/write-files-to-hdfs-folder.yaml deleted file mode 100644 index f96af95d..00000000 --- a/test-data/write-files-to-hdfs-folder.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: WriteFilesToHdfsFolderStep -description:

This job tests your configuration in order to ensure that Cloudgene is able to communicate with your Hadoop Cluster.
If the job fails, please follow the error message and adapt your configuration until the job runs successfully.
Useful informations about the configuration can be found on our website http://cloudgene.uibk.ac.at. -version: 1.0.1 -website: http://cloudgene.uibk.ac.at -category: cloudgene - -mapred: - - steps: - - name: WriteFilesToHdfsFolderStep - classname: cloudgene.mapred.jobs.steps.WriteFilesToHdfsFolderStep - - inputs: - - - id: inputtext - description: Input-Text - type: text - - outputs: - - - id: output - description: Output Folder - type: hdfs-folder - download: true - temp: false - zip: false - removeHeader: false - mergeOutput: false diff --git a/test-data/write-text-to-hdfs-file.yaml b/test-data/write-text-to-hdfs-file.yaml deleted file mode 100644 index 8d958633..00000000 --- a/test-data/write-text-to-hdfs-file.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: WriteTextToHdfsFileStep -description:

This job tests your configuration in order to ensure that Cloudgene is able to communicate with your Hadoop Cluster.
If the job fails, please follow the error message and adapt your configuration until the job runs successfully.
Useful informations about the configuration can be found on our website http://cloudgene.uibk.ac.at. -version: 1.0.1 -website: http://cloudgene.uibk.ac.at -category: cloudgene - -mapred: - - steps: - - name: WriteTextToHdfsFileStep - classname: cloudgene.mapred.jobs.steps.WriteTextToHdfsFileStep - - inputs: - - - id: inputtext - description: Input-Text - type: text - - outputs: - - - id: output - description: OutputFile - type: hdfs-file - download: true - temp: false - zip: false - removeHeader: false - mergeOutput: false From 178813585171315f36742f9d7c4af803aed38f7a Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Wed, 23 Aug 2023 12:41:53 +0200 Subject: [PATCH 02/35] Refactor workspace classes --- .../cloudgene/mapred/jobs/AbstractJob.java | 13 ++++---- .../cloudgene/mapred/jobs/CloudgeneJob.java | 19 +++++------- ...ExternalWorkspace.java => IWorkspace.java} | 2 +- .../mapred/jobs/workspace/LocalWorkspace.java | 2 +- .../mapred/jobs/workspace/S3Workspace.java | 2 +- ...paceFactory.java => WorkspaceFactory.java} | 8 ++--- .../jobs/workspace/WorkspaceWrapper.java | 4 +-- .../mapred/plugins/nextflow/NextflowStep.java | 8 ++--- .../server/controller/DownloadController.java | 3 +- .../server/services/DownloadService.java | 12 +++---- .../server/services/JobCleanUpService.java | 8 ++--- .../mapred/server/services/JobService.java | 31 +++++++++---------- .../jobs/PriorityThreadPoolExecutorTest.java | 6 ++-- .../mapred/jobs/WorkflowEngineTest.java | 6 ++-- .../mapred/jobs/WrongWorkspaceTest.java | 6 ++-- .../cloudgene/mapred/steps/TestCommand.java | 6 ++-- 16 files changed, 64 insertions(+), 72 deletions(-) rename src/main/java/cloudgene/mapred/jobs/workspace/{IExternalWorkspace.java => IWorkspace.java} (95%) rename src/main/java/cloudgene/mapred/jobs/workspace/{ExternalWorkspaceFactory.java => WorkspaceFactory.java} (85%) diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java index 88eceba1..e5279087 100644 --- a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java +++ b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java @@ -21,9 +21,8 @@ import cloudgene.mapred.apps.ApplicationRepository; import cloudgene.mapred.core.User; import cloudgene.mapred.jobs.queue.PriorityRunnable; -import cloudgene.mapred.jobs.workspace.IExternalWorkspace; +import cloudgene.mapred.jobs.workspace.IWorkspace; import cloudgene.mapred.util.Settings; -import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlParameterInputType; import genepi.io.FileUtil; @@ -119,7 +118,7 @@ abstract public class AbstractJob extends PriorityRunnable { private boolean canceld = false; - protected IExternalWorkspace externalWorkspace; + protected IWorkspace workspace; private String workspaceSize = null; @@ -743,12 +742,12 @@ public void setLocalWorkspace(String localWorkspace) { this.localWorkspace = localWorkspace; } - public void setExternalWorkspace(IExternalWorkspace externalWorkspace) { - this.externalWorkspace = externalWorkspace; + public void setWorkspace(IWorkspace workspace) { + this.workspace = workspace; } - public IExternalWorkspace getExternalWorkspace() { - return externalWorkspace; + public IWorkspace getWorkspace() { + return workspace; } public void setApplication(String application) { diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java index 8cc61ed5..1b7404b4 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java @@ -14,7 +14,6 @@ import cloudgene.mapred.jobs.engine.Planner; import cloudgene.mapred.jobs.engine.graph.Graph; import cloudgene.mapred.jobs.engine.graph.GraphNode; -import cloudgene.mapred.jobs.workspace.IExternalWorkspace; import cloudgene.mapred.jobs.workspace.WorkspaceWrapper; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlParameterInput; @@ -104,9 +103,9 @@ public boolean setup() { FileUtil.createDirectory(context.getLocalTemp()); try { - context.log("Setup External Workspace on " + externalWorkspace.getName()); - externalWorkspace.setup(this.getId()); - context.setExternalWorkspace(new WorkspaceWrapper(externalWorkspace)); + context.log("Setup External Workspace on " + workspace.getName()); + workspace.setup(this.getId()); + context.setExternalWorkspace(new WorkspaceWrapper(workspace)); } catch (Exception e) { writeLog(e.toString()); log.info("Error setup external workspace", e); @@ -124,12 +123,12 @@ public boolean setup() { throw new RuntimeException("HDFS support was removed in Cloudgene 3"); case LOCAL_FILE: - String filename = externalWorkspace.createFile(param.getName(), param.getName()); + String filename = workspace.createFile(param.getName(), param.getName()); param.setValue(filename); break; case LOCAL_FOLDER: - String folder = externalWorkspace.createFolder(param.getName()); + String folder = workspace.createFolder(param.getName()); param.setValue(folder); break; } @@ -293,7 +292,7 @@ public boolean executeFailureStep(WdlStep failedStep) { public boolean cleanUp() { try { - externalWorkspace.cleanup(getId()); + workspace.cleanup(getId()); } catch (IOException e) { writeLog("Cleanup failed."); writeLog(e.getMessage()); @@ -333,7 +332,7 @@ public boolean exportParameter(CloudgeneParameterOutput out) { out.setJobId(getId()); - List downloads = externalWorkspace.getDownloads(out.getValue()); + List downloads = workspace.getDownloads(out.getValue()); for (Download download : downloads) { download.setParameter(out); download.setCount(MAX_DOWNLOAD); @@ -384,8 +383,4 @@ public void updateProgress() { } - public IExternalWorkspace getExternalWorkspace() { - return externalWorkspace; - } - } diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/IExternalWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java similarity index 95% rename from src/main/java/cloudgene/mapred/jobs/workspace/IExternalWorkspace.java rename to src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java index b7cf0cd6..51745501 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/IExternalWorkspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java @@ -7,7 +7,7 @@ import cloudgene.mapred.jobs.Download; -public interface IExternalWorkspace { +public interface IWorkspace { public void setup(String job) throws IOException; diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java index 31de2b81..18575aed 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java @@ -17,7 +17,7 @@ import cloudgene.mapred.util.HashUtil; import genepi.io.FileUtil; -public class LocalWorkspace implements IExternalWorkspace { +public class LocalWorkspace implements IWorkspace { private static final String INPUT_DIRECTORY = "input"; diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java index 9059a988..8c48e345 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java @@ -24,7 +24,7 @@ import cloudgene.mapred.util.S3Util; import genepi.io.FileUtil; -public class S3Workspace implements IExternalWorkspace { +public class S3Workspace implements IWorkspace { private static final String OUTPUT_DIRECTORY = "outputs"; diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/ExternalWorkspaceFactory.java b/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceFactory.java similarity index 85% rename from src/main/java/cloudgene/mapred/jobs/workspace/ExternalWorkspaceFactory.java rename to src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceFactory.java index 58342468..561d7a58 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/ExternalWorkspaceFactory.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceFactory.java @@ -7,12 +7,12 @@ import jakarta.inject.Singleton; @Singleton -public class ExternalWorkspaceFactory { +public class WorkspaceFactory { @Inject protected Application application; - public IExternalWorkspace getDefault() { + public IWorkspace getDefault() { Settings settings = application.getSettings(); @@ -31,7 +31,7 @@ public IExternalWorkspace getDefault() { } - public IExternalWorkspace getByUrl(String url) { + public IWorkspace getByUrl(String url) { Settings settings = application.getSettings(); @@ -48,7 +48,7 @@ public IExternalWorkspace getByUrl(String url) { } - public IExternalWorkspace getByJob(AbstractJob job) { + public IWorkspace getByJob(AbstractJob job) { return getDefault(); } diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceWrapper.java b/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceWrapper.java index 1d5f84c8..12826cd3 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceWrapper.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceWrapper.java @@ -8,9 +8,9 @@ public class WorkspaceWrapper implements IExternalWorkspace { - private cloudgene.mapred.jobs.workspace.IExternalWorkspace newWorkspace; + private cloudgene.mapred.jobs.workspace.IWorkspace newWorkspace; - public WorkspaceWrapper(cloudgene.mapred.jobs.workspace.IExternalWorkspace newWorkspace) { + public WorkspaceWrapper(cloudgene.mapred.jobs.workspace.IWorkspace newWorkspace) { this.newWorkspace = newWorkspace; } diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java index 70ea0ea7..0a6b0935 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java @@ -14,8 +14,8 @@ import cloudgene.mapred.jobs.CloudgeneJob; import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; -import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; -import cloudgene.mapred.jobs.workspace.IExternalWorkspace; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; +import cloudgene.mapred.jobs.workspace.IWorkspace; import cloudgene.mapred.util.HashUtil; import cloudgene.mapred.wdl.WdlStep; import genepi.io.FileUtil; @@ -88,8 +88,8 @@ public boolean run(WdlStep step, CloudgeneContext context) { nextflowCommand.add("-w"); nextflowCommand.add(work); } else { - IExternalWorkspace externalWorkspace = job.getExternalWorkspace(); - String workDir = externalWorkspace.createTempFolder("nextflow"); + IWorkspace workspace = job.getWorkspace(); + String workDir = workspace.createTempFolder("nextflow"); FileUtil.createDirectory(workDir); nextflowCommand.add("-w"); nextflowCommand.add(workDir); diff --git a/src/main/java/cloudgene/mapred/server/controller/DownloadController.java b/src/main/java/cloudgene/mapred/server/controller/DownloadController.java index 9f76d9dd..5c10147d 100644 --- a/src/main/java/cloudgene/mapred/server/controller/DownloadController.java +++ b/src/main/java/cloudgene/mapred/server/controller/DownloadController.java @@ -22,7 +22,6 @@ import cloudgene.mapred.server.services.DownloadService; import cloudgene.mapred.server.services.JobService; import genepi.io.FileUtil; -import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import io.micronaut.http.MutableHttpResponse; import io.micronaut.http.annotation.Controller; @@ -81,7 +80,7 @@ public MutableHttpResponse downloadExternalResults(String jobId, St @Get("/share/results/{hash}/{filename:.+}") @Secured(SecurityRule.IS_ANONYMOUS) public MutableHttpResponse downloadPublicLink(String hash, String filename) throws URISyntaxException, IOException { -System.out.println("------> " + filename); + DownloadDao dao = new DownloadDao(application.getDatabase()); Download download = dao.findByHash(hash); diff --git a/src/main/java/cloudgene/mapred/server/services/DownloadService.java b/src/main/java/cloudgene/mapred/server/services/DownloadService.java index 60a6d1b9..ae3f877c 100644 --- a/src/main/java/cloudgene/mapred/server/services/DownloadService.java +++ b/src/main/java/cloudgene/mapred/server/services/DownloadService.java @@ -7,8 +7,8 @@ import cloudgene.mapred.database.DownloadDao; import cloudgene.mapred.jobs.Download; -import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; -import cloudgene.mapred.jobs.workspace.IExternalWorkspace; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; +import cloudgene.mapred.jobs.workspace.IWorkspace; import cloudgene.mapred.server.Application; import cloudgene.mapred.server.exceptions.JsonHttpStatusException; import io.micronaut.http.HttpResponse; @@ -22,7 +22,7 @@ public class DownloadService { protected Application application; @Inject - protected ExternalWorkspaceFactory workspaceFactory; + protected WorkspaceFactory workspaceFactory; public MutableHttpResponse download(Download download) throws URISyntaxException, IOException { @@ -42,15 +42,15 @@ public MutableHttpResponse download(Download download) throws URISy dao.update(download); } - IExternalWorkspace externalWorkspace = workspaceFactory.getByUrl(download.getPath()); + IWorkspace workspace = workspaceFactory.getByUrl(download.getPath()); // external workspace found, use link method and create redirect response - String publicUrl = externalWorkspace.createPublicLink(download.getPath()); + String publicUrl = workspace.createPublicLink(download.getPath()); if (publicUrl != null) { URI location = new URI(publicUrl); return HttpResponse.redirect(location); } else { - return HttpResponse.ok(externalWorkspace.download(download.getPath())); + return HttpResponse.ok(workspace.download(download.getPath())); } } diff --git a/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java b/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java index 9443b7ae..6f0a322a 100644 --- a/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java +++ b/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java @@ -9,8 +9,8 @@ import cloudgene.mapred.database.JobDao; import cloudgene.mapred.database.util.Database; import cloudgene.mapred.jobs.AbstractJob; -import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; -import cloudgene.mapred.jobs.workspace.IExternalWorkspace; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; +import cloudgene.mapred.jobs.workspace.IWorkspace; import cloudgene.mapred.server.Application; import cloudgene.mapred.util.MailUtil; import cloudgene.mapred.util.Settings; @@ -27,7 +27,7 @@ public class JobCleanUpService { protected Application application; @Inject - protected ExternalWorkspaceFactory workspaceFactory; + protected WorkspaceFactory workspaceFactory; public int executeRetire() { @@ -54,7 +54,7 @@ public int executeRetire() { log.info("Job " + job.getId() + " retired."); deleted++; - IExternalWorkspace externalWorkspace = workspaceFactory.getByJob(job); + IWorkspace externalWorkspace = workspaceFactory.getByJob(job); try { externalWorkspace.delete(job.getId()); diff --git a/src/main/java/cloudgene/mapred/server/services/JobService.java b/src/main/java/cloudgene/mapred/server/services/JobService.java index 2a3d3af2..3c138cf0 100644 --- a/src/main/java/cloudgene/mapred/server/services/JobService.java +++ b/src/main/java/cloudgene/mapred/server/services/JobService.java @@ -21,9 +21,8 @@ import cloudgene.mapred.jobs.CloudgeneParameterOutput; import cloudgene.mapred.jobs.Download; import cloudgene.mapred.jobs.WorkflowEngine; -import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; -import cloudgene.mapred.jobs.workspace.IExternalWorkspace; -import cloudgene.mapred.jobs.workspace.LocalWorkspace; +import cloudgene.mapred.jobs.workspace.IWorkspace; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.server.Application; import cloudgene.mapred.server.exceptions.JsonHttpStatusException; import cloudgene.mapred.util.FormUtil.Parameter; @@ -46,7 +45,7 @@ public class JobService { protected Application application; @Inject - protected ExternalWorkspaceFactory workspaceFactory; + protected WorkspaceFactory workspaceFactory; private static final String PARAM_JOB_NAME = "job-name"; @@ -121,15 +120,15 @@ public AbstractJob submitJob(String appId, List form, User user) { Map inputParams = null; - IExternalWorkspace externalWorkspace = workspaceFactory.getDefault(); + IWorkspace workspace = workspaceFactory.getDefault(); try { // setup workspace - externalWorkspace.setup(id); + workspace.setup(id); // parse input params - inputParams = parseAndUpdateInputParams(form, app, externalWorkspace); + inputParams = parseAndUpdateInputParams(form, app, workspace); } catch (Exception e) { throw new JsonHttpStatusException(HttpStatus.BAD_REQUEST, e.getMessage()); @@ -148,7 +147,7 @@ public AbstractJob submitJob(String appId, List form, User user) { job.setId(id); job.setName(name); job.setLocalWorkspace(localWorkspace); - job.setExternalWorkspace(externalWorkspace); + job.setWorkspace(workspace); job.setSettings(settings); job.setApplication(app.getName() + " " + app.getVersion()); job.setApplicationId(appId); @@ -229,9 +228,9 @@ public AbstractJob delete(AbstractJob job) { // delete all results that are stored on external workspaces - IExternalWorkspace externalWorkspace = workspaceFactory.getByJob(job); + IWorkspace workspace = workspaceFactory.getByJob(job); try { - externalWorkspace.delete(job.getId()); + workspace.delete(job.getId()); } catch (Exception e) { log.error("Deleteting " + job.getId() + " form workspace failed.", e); } @@ -319,10 +318,10 @@ public String archive(AbstractJob job) { job.setState(AbstractJob.STATE_RETIRED); dao.update(job); - IExternalWorkspace externalWorkspace = workspaceFactory.getByJob(job); + IWorkspace workspace = workspaceFactory.getByJob(job); try { - externalWorkspace.delete(job.getId()); + workspace.delete(job.getId()); } catch (Exception e) { log.error("Deleteting " + job.getId() + " from workspace failed.", e); } @@ -367,8 +366,8 @@ public String createId() { // TODO: refactore and combine this method with CommandLineUtil.parseArgs... - private Map parseAndUpdateInputParams(List form, WdlApp app, - IExternalWorkspace externalWorkspace) throws Exception { + private Map parseAndUpdateInputParams(List form, WdlApp app, IWorkspace workspace) + throws Exception { Map props = new HashMap(); Map params = new HashMap(); @@ -402,10 +401,10 @@ private Map parseAndUpdateInputParams(List form, WdlA } // copy to workspace in input directory - String target = externalWorkspace.uploadInput(fieldName, inputFile); + String target = workspace.uploadInput(fieldName, inputFile); if (inputParam.isFolder()) { - props.put(fieldName, externalWorkspace.getParent(target)); + props.put(fieldName, workspace.getParent(target)); } else { // file props.put(fieldName, target); diff --git a/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java b/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java index 0dd2a525..101f23ef 100644 --- a/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java +++ b/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java @@ -12,7 +12,7 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; -import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; @@ -29,7 +29,7 @@ public class PriorityThreadPoolExecutorTest { TestApplication application; @Inject - ExternalWorkspaceFactory workspaceFactory; + WorkspaceFactory workspaceFactory; @Test public void testCancelRunningJob() throws Exception { @@ -479,7 +479,7 @@ public CloudgeneJob createJobFromWdl(WdlApp app, String id, Map CloudgeneJob job = new CloudgeneJob(user, id,app, inputs); job.setId(id); job.setName(id); - job.setExternalWorkspace(workspaceFactory.getDefault()); + job.setWorkspace(workspaceFactory.getDefault()); job.setLocalWorkspace(localWorkspace); job.setSettings(settings); job.setApplication(app.getName() + " " + app.getVersion()); diff --git a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java index df5420d1..ef366c4e 100644 --- a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java +++ b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java @@ -13,7 +13,7 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; -import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; @@ -29,7 +29,7 @@ public class WorkflowEngineTest { TestApplication application; @Inject - ExternalWorkspaceFactory workspaceFactory; + WorkspaceFactory workspaceFactory; @Test @@ -753,7 +753,7 @@ public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs, Use CloudgeneJob job = new CloudgeneJob(user, id, app, inputs); job.setId(id); job.setName(id); - job.setExternalWorkspace(workspaceFactory.getDefault()); + job.setWorkspace(workspaceFactory.getDefault()); job.setLocalWorkspace(localWorkspace); job.setSettings(settings); job.setApplication(app.getName() + " " + app.getVersion()); diff --git a/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java b/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java index 73dfe0b1..4f73f6e7 100644 --- a/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java +++ b/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java @@ -11,7 +11,7 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.JobDao; import cloudgene.mapred.database.UserDao; -import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; @@ -26,7 +26,7 @@ public class WrongWorkspaceTest { TestApplication application; @Inject - ExternalWorkspaceFactory workspaceFactory; + WorkspaceFactory workspaceFactory; @Test public void testReturnTrueStep() throws Exception { @@ -69,7 +69,7 @@ public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs) thr CloudgeneJob job = new CloudgeneJob(user, id, app, inputs); job.setId(id); job.setName(id); - job.setExternalWorkspace(workspaceFactory.getDefault()); + job.setWorkspace(workspaceFactory.getDefault()); job.setLocalWorkspace(localWorkspace); job.setSettings(settings); job.setApplication(app.getName() + " " + app.getVersion()); diff --git a/src/test/java/cloudgene/mapred/steps/TestCommand.java b/src/test/java/cloudgene/mapred/steps/TestCommand.java index 93714ea7..d426e82f 100644 --- a/src/test/java/cloudgene/mapred/steps/TestCommand.java +++ b/src/test/java/cloudgene/mapred/steps/TestCommand.java @@ -16,7 +16,7 @@ import cloudgene.mapred.jobs.CloudgeneJob; import cloudgene.mapred.jobs.Message; import cloudgene.mapred.jobs.WorkflowEngine; -import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; @@ -33,7 +33,7 @@ public class TestCommand { TestApplication application; @Inject - ExternalWorkspaceFactory workspaceFactory; + WorkspaceFactory workspaceFactory; @Test public void testValidCommand() throws Exception { @@ -145,7 +145,7 @@ public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs) thr CloudgeneJob job = new CloudgeneJob(user, id, app, inputs); job.setId(id); - job.setExternalWorkspace(workspaceFactory.getDefault()); + job.setWorkspace(workspaceFactory.getDefault()); job.setName(id); job.setLocalWorkspace(localWorkspace); job.setSettings(settings); From 8e6bdc400721671d1ad0ebb5924250036d861e63 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Mon, 4 Sep 2023 15:08:43 +0200 Subject: [PATCH 03/35] Support s3 in Nextflow tasks --- .../mapred/plugins/nextflow/NextflowTask.java | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java index 8bc6ba79..eae8049c 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java @@ -1,15 +1,20 @@ package cloudgene.mapred.plugins.nextflow; +import java.io.DataInputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.Map; import cloudgene.mapred.jobs.CloudgeneContext; +import cloudgene.sdk.internal.IExternalWorkspace; import genepi.io.FileUtil; import genepi.io.text.LineReader; public class NextflowTask { + private static final String CLOUDGENE_LOG = "cloudgene.log"; + private int id; private Map trace; @@ -17,8 +22,8 @@ public class NextflowTask { private String log = null; private CloudgeneContext context; - - public NextflowTask( CloudgeneContext context, Map trace) { + + public NextflowTask(CloudgeneContext context, Map trace) { id = (Integer) trace.get("task_id"); this.trace = trace; this.context = context; @@ -33,21 +38,19 @@ public void update(Map trace) throws IOException { String status = (String) trace.get("status"); if (status.equals("COMPLETED") || status.equals("FAILED")) { String workDir = (String) trace.get("workdir"); - String logFilename = FileUtil.path(workDir, "cloudgene.log"); - // TODO: implement s3 support. How to handle other cloud providers? - - if (new File(logFilename).exists()) { - log = FileUtil.readFileAsString(logFilename); - parseFile(logFilename); - } - + String logFilename = FileUtil.path(workDir, CLOUDGENE_LOG); + IExternalWorkspace workspace = context.getExternalWorkspace(); + InputStream stream = workspace.download(logFilename); + log = FileUtil.readFileAsString(stream); + parseFile(logFilename); } } private void parseFile(String logFilename) throws IOException { - LineReader reader = new LineReader(logFilename); - while(reader.next()) { + + LineReader reader = new LineReader(new DataInputStream(context.getExternalWorkspace().download(logFilename))); + while (reader.next()) { String line = reader.get(); if (line.startsWith("[INC]")) { String[] tiles = line.split(" ", 3); @@ -59,9 +62,9 @@ private void parseFile(String logFilename) throws IOException { String[] tiles = line.split(" ", 2); String name = tiles[1]; context.submitCounter(name); - } + } } - reader.close(); + reader.close(); } public int getId() { From f432128a48ec6ea3ae979b09d3a7e36d946b30e2 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Mon, 4 Sep 2023 15:23:13 +0200 Subject: [PATCH 04/35] Fix settings and cleanup old values --- .../cloudgene/mapred/cli/StartServer.java | 4 - .../java/cloudgene/mapred/util/Settings.java | 153 +----------------- 2 files changed, 5 insertions(+), 152 deletions(-) diff --git a/src/main/java/cloudgene/mapred/cli/StartServer.java b/src/main/java/cloudgene/mapred/cli/StartServer.java index d3c493b8..5a7f03e0 100644 --- a/src/main/java/cloudgene/mapred/cli/StartServer.java +++ b/src/main/java/cloudgene/mapred/cli/StartServer.java @@ -130,10 +130,6 @@ protected Settings loadSettings(Config config) throws FileNotFoundException, Yam settings = new Settings(config); } - if (!settings.testPaths()) { - System.exit(1); - } - if (settings.getServerUrl() == null || settings.getServerUrl().trim().isEmpty()) { System.out.println("serverUrl not set. Please set serverUrl in file '" + settingsFilename + "'"); System.exit(1); diff --git a/src/main/java/cloudgene/mapred/util/Settings.java b/src/main/java/cloudgene/mapred/util/Settings.java index 093e9aee..2ee34182 100644 --- a/src/main/java/cloudgene/mapred/util/Settings.java +++ b/src/main/java/cloudgene/mapred/util/Settings.java @@ -23,24 +23,14 @@ public class Settings { - private String serverUrl = "http://localhost:8082"; - - private String hadoopPath = "/usr/"; - - private String pigPath = "/usr/"; + private static final Logger log = LoggerFactory.getLogger(Settings.class); - private String sparkPath = "/usr/bin/spark-submit"; + private String serverUrl = "http://localhost:8082"; private String tempPath = "tmp"; private String localWorkspace = "workspace"; - private String hdfsWorkspace = "cloudgene/data"; - - private String hdfsAppWorkspace = "cloudgene/apps"; - - private String streamingJar = ""; - private String version; private String name = "Cloudgene"; @@ -53,8 +43,6 @@ public class Settings { private Map database; - private Map cluster; - private Map> plugins; private int autoRetireInterval = 5; @@ -73,30 +61,18 @@ public class Settings { private boolean streaming = true; - private boolean removeHdfsWorkspace = true; - - private static final Logger log = LoggerFactory.getLogger(Settings.class); - private boolean writeStatistics = true; private boolean https = false; - private String httpsKeystore = ""; - - private String httpsPassword = ""; - private boolean maintenance = false; private String adminMail = null; - private String slack = null; - private String urlPrefix = ""; private List navigation = new Vector(); - private boolean secureCookie = false; - private Map externalWorkspace = null; private int uploadLimit = 500; @@ -116,7 +92,7 @@ public class Settings { private ApplicationRepository repository; - public Settings() { + private Settings() { repository = new ApplicationRepository(); @@ -197,6 +173,8 @@ public static Settings load(Config config) throws FileNotFoundException, YamlExc log.info("Notify user after " + settings.notificationAfter + " days."); log.info("Write statistics: " + settings.writeStatistics); + settings.config = config; + // workspace in config has higher priority if (config.getWorkspace() != null) { settings.setLocalWorkspace(config.getWorkspace()); @@ -291,22 +269,6 @@ public void save(String filename) { } - public String getHadoopPath() { - return hadoopPath; - } - - public void setHadoopPath(String hadoopPath) { - this.hadoopPath = hadoopPath; - } - - public void setPigPath(String pigPath) { - this.pigPath = pigPath; - } - - public String getPigPath() { - return pigPath; - } - public String getTempPath() { return tempPath; } @@ -323,30 +285,6 @@ public void setLocalWorkspace(String localWorkspace) { this.localWorkspace = localWorkspace; } - public String getHdfsWorkspace() { - return hdfsWorkspace; - } - - public void setHdfsWorkspace(String hdfsWorkspace) { - this.hdfsWorkspace = hdfsWorkspace; - } - - public String getHdfsAppWorkspace() { - return hdfsAppWorkspace; - } - - public void setHdfsAppWorkspace(String hdfsAppWorkspace) { - this.hdfsAppWorkspace = hdfsAppWorkspace; - } - - public String getStreamingJar() { - return streamingJar; - } - - public void setStreamingJar(String streamingJar) { - this.streamingJar = streamingJar; - } - public boolean isStreaming() { return streaming; } @@ -355,39 +293,6 @@ public void setStreaming(boolean streaming) { this.streaming = streaming; } - public boolean isRemoveHdfsWorkspace() { - return removeHdfsWorkspace; - } - - public void setRemoveHdfsWorkspace(boolean removeHdfsWorkspace) { - this.removeHdfsWorkspace = removeHdfsWorkspace; - } - - public boolean testPaths() { - - String hadoop = FileUtil.path(hadoopPath, "bin", "hadoop"); - - if (!new File(hadoop).exists()) { - - log.warn("hadoop '" + hadoop + "' does not exist. please change it."); - - // return false; - - } - /* - * if (!new File(streamingJar).exists()) { - * - * log.error("streamingJar '" + streamingJar + "' does not exist."); - * - * return false; - * - * } - */ - - return true; - - } - public String getVersion() { return version; } @@ -404,22 +309,6 @@ public void setMail(Map mail) { this.mail = mail; } - public Map getCluster() { - return cluster; - } - - public void setCluster(Map cluster) { - this.cluster = cluster; - } - - public String getSlack() { - return slack; - } - - public void setSlack(String slack) { - this.slack = slack; - } - public void setName(String name) { this.name = name; } @@ -482,22 +371,6 @@ public boolean isHttps() { return https; } - public void setHttpsKeystore(String httpsKeystore) { - this.httpsKeystore = httpsKeystore; - } - - public String getHttpsKeystore() { - return httpsKeystore; - } - - public void setHttpsPassword(String httpsPassword) { - this.httpsPassword = httpsPassword; - } - - public String getHttpsPassword() { - return httpsPassword; - } - public void setMaintenance(boolean maintenance) { this.maintenance = maintenance; } @@ -514,14 +387,6 @@ public String getAdminMail() { return adminMail; } - public String getSparkPath() { - return sparkPath; - } - - public void setSparkPath(String sparkPath) { - this.sparkPath = sparkPath; - } - public void setThreadsQueue(int threadsQueue) { this.threadsQueue = threadsQueue; } @@ -570,14 +435,6 @@ public List getNavigation() { return navigation; } - public boolean isSecureCookie() { - return secureCookie; - } - - public void setSecureCookie(boolean secureCookie) { - this.secureCookie = secureCookie; - } - public int getUploadLimit() { return uploadLimit; } From 7ec922f6861a2c05534f7a896241eb3109c14678 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Mon, 4 Sep 2023 16:04:16 +0200 Subject: [PATCH 05/35] Fix uploadLimit --- src/main/java/cloudgene/mapred/cli/StartServer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/cloudgene/mapred/cli/StartServer.java b/src/main/java/cloudgene/mapred/cli/StartServer.java index 5a7f03e0..a2feed07 100644 --- a/src/main/java/cloudgene/mapred/cli/StartServer.java +++ b/src/main/java/cloudgene/mapred/cli/StartServer.java @@ -63,6 +63,7 @@ public int run() { Map properties = new HashMap(); properties.put("micronaut.server.port", port); if (Application.settings.getUploadLimit() != -1) { + properties.put("micronaut.server.maxRequestSize", Application.settings.getUploadLimit() + "MB"); properties.put("micronaut.server.multipart.maxFileSize", Application.settings.getUploadLimit() + "MB"); } From 11cd4fa94cc1f16e0f869e8fbeae1fe445765986 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Tue, 5 Sep 2023 11:02:01 +0200 Subject: [PATCH 06/35] Fix log files of NextflowTasks --- .../java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java index 18575aed..af0f59c3 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java @@ -62,7 +62,10 @@ public String uploadInput(String id, File file) throws IOException { @Override public InputStream download(String path) throws IOException { - String absolutePath = FileUtil.path(location, path); + String absolutePath = path; + if (!absolutePath.startsWith("/")) { + absolutePath = FileUtil.path(location, path); + } File file = new File(absolutePath); if (file.exists()) { return new FileInputStream(file); From 8e26679ab88067fd4e28a1cb9f7fc26d0ea21bde Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Tue, 5 Sep 2023 13:55:35 +0200 Subject: [PATCH 07/35] Add reporting and improve Nextflow support --- .../mapred/jobs/workspace/IWorkspace.java | 4 ++ .../mapred/jobs/workspace/LocalWorkspace.java | 37 +++++++++++++++---- .../mapred/jobs/workspace/S3Workspace.java | 18 ++++++++- .../mapred/plugins/nextflow/NextflowStep.java | 21 +++++++++-- .../mapred/plugins/nextflow/NextflowTask.java | 14 ++++--- .../java/cloudgene/mapred/util/Settings.java | 2 +- 6 files changed, 75 insertions(+), 21 deletions(-) diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java index 51745501..2667ab82 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java @@ -29,10 +29,14 @@ public interface IWorkspace { public String createFile(String name, String name2); + public String createLogFile(String name); + public String createTempFolder(String string); public List getDownloads(String url); public void cleanup(String job) throws IOException; + public boolean exists(String path) throws IOException; + } diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java index af0f59c3..458f27b6 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java @@ -19,10 +19,14 @@ public class LocalWorkspace implements IWorkspace { + private static final String OUTPUT_DIRECTORY = "outputs"; + private static final String INPUT_DIRECTORY = "input"; private static final String TEMP_DIRECTORY = "temp"; + private static final String LOGS_DIRECTORY = "logs"; + private static final Logger log = LoggerFactory.getLogger(LocalWorkspace.class); private String location; @@ -54,7 +58,7 @@ public String upload(String id, File file) throws IOException { FileUtil.copy(file.getAbsolutePath(), target); return target; } - + @Override public String uploadInput(String id, File file) throws IOException { return upload(FileUtil.path(INPUT_DIRECTORY, id), file); @@ -64,7 +68,7 @@ public String uploadInput(String id, File file) throws IOException { public InputStream download(String path) throws IOException { String absolutePath = path; if (!absolutePath.startsWith("/")) { - absolutePath = FileUtil.path(location, path); + absolutePath = FileUtil.path(location, path); } File file = new File(absolutePath); if (file.exists()) { @@ -74,6 +78,16 @@ public InputStream download(String path) throws IOException { } } + @Override + public boolean exists(String path) throws IOException { + String absolutePath = path; + if (!absolutePath.startsWith("/")) { + absolutePath = FileUtil.path(location, path); + } + File file = new File(absolutePath); + return file.exists(); + } + @Override public void delete(String job) throws IOException { @@ -90,12 +104,12 @@ public void delete(String job) throws IOException { } } - + @Override public void cleanup(String job) throws IOException { - - //TODO: add flag to disable cleanup (e.g. debugging) - + + // TODO: add flag to disable cleanup (e.g. debugging) + try { log.debug("Cleanup " + job + " on local workspace..."); String temp = FileUtil.path(location, job, TEMP_DIRECTORY); @@ -125,14 +139,21 @@ public String getParent(String url) { @Override public String createFolder(String id) { - String folder = FileUtil.path(workspace, id); + String folder = FileUtil.path(workspace, OUTPUT_DIRECTORY, id); FileUtil.createDirectory(folder); return folder; } @Override public String createFile(String parent, String id) { - String folder = FileUtil.path(workspace, parent); + String folder = FileUtil.path(workspace, OUTPUT_DIRECTORY, parent); + FileUtil.createDirectory(folder); + return FileUtil.path(folder, id); + } + + @Override + public String createLogFile(String id) { + String folder = FileUtil.path(workspace, LOGS_DIRECTORY); FileUtil.createDirectory(folder); return FileUtil.path(folder, id); } diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java index 8c48e345..c9b3e091 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java @@ -28,12 +28,14 @@ public class S3Workspace implements IWorkspace { private static final String OUTPUT_DIRECTORY = "outputs"; - public static long EXPIRATION_MS = 1000 * 60 * 60; - private static final String INPUT_DIRECTORY = "input"; + private static final String LOGS_DIRECTORY = "logs"; + private static final String TEMP_DIRECTORY = "temp"; + public static long EXPIRATION_MS = 1000 * 60 * 60; + private static final Logger log = LoggerFactory.getLogger(S3Workspace.class); private String location; @@ -98,6 +100,13 @@ public InputStream download(String url) throws IOException { return s3is; } + public boolean exists(String url) { + String bucket = S3Util.getBucket(url); + String key = S3Util.getKey(url); + AmazonS3 s3 = S3Util.getAmazonS3(); + return s3.doesObjectExist(bucket, key); + } + @Override public void delete(String job) throws IOException { @@ -192,6 +201,11 @@ public String createFile(String folder, String id) { return location + "/" + job + "/" + OUTPUT_DIRECTORY + "/" + folder + "/" + id; } + @Override + public String createLogFile(String id) { + return location + "/" + job + "/" + LOGS_DIRECTORY + "/" + id; + } + @Override public String createTempFolder(String id) { return location + "/" + job + "/" + TEMP_DIRECTORY + "/" + id; diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java index 0a6b0935..e861b110 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java @@ -90,7 +90,6 @@ public boolean run(WdlStep step, CloudgeneContext context) { } else { IWorkspace workspace = job.getWorkspace(); String workDir = workspace.createTempFolder("nextflow"); - FileUtil.createDirectory(workDir); nextflowCommand.add("-w"); nextflowCommand.add(workDir); } @@ -135,16 +134,30 @@ public boolean run(WdlStep step, CloudgeneContext context) { return false; } + IWorkspace workspace = job.getWorkspace(); + nextflowCommand.add("-params-file"); nextflowCommand.add(paramsFile.getAbsolutePath()); nextflowCommand.add("-ansi-log"); nextflowCommand.add("false"); - + nextflowCommand.add("-with-weblog"); nextflowCommand.add(context.getSettings().getServerUrl() + context.getSettings().getUrlPrefix() + "/api/v2/collect/" + makeSecretJobId(context.getJobId())); - + + //nextflowCommand.add("-log"); + //nextflowCommand.add(workspace.createLogFile("nextflow.log")); + + nextflowCommand.add("-with-trace"); + nextflowCommand.add(workspace.createLogFile("trace.csv")); + + nextflowCommand.add("-with-report"); + nextflowCommand.add(workspace.createLogFile("report.html")); + + nextflowCommand.add("-with-timeline"); + nextflowCommand.add(workspace.createLogFile("timeline.html")); + StringBuilder output = new StringBuilder(); List command = new Vector(); @@ -260,7 +273,7 @@ private String join(List array) { String result = ""; for (int i = 0; i < array.size(); i++) { if (i > 0) { - result += " \\\n"; + result += " "; } result += array.get(i); } diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java index eae8049c..8c3ffc73 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java @@ -1,13 +1,12 @@ package cloudgene.mapred.plugins.nextflow; import java.io.DataInputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.Map; import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.sdk.internal.IExternalWorkspace; +import cloudgene.mapred.jobs.workspace.IWorkspace; import genepi.io.FileUtil; import genepi.io.text.LineReader; @@ -39,10 +38,13 @@ public void update(Map trace) throws IOException { if (status.equals("COMPLETED") || status.equals("FAILED")) { String workDir = (String) trace.get("workdir"); String logFilename = FileUtil.path(workDir, CLOUDGENE_LOG); - IExternalWorkspace workspace = context.getExternalWorkspace(); - InputStream stream = workspace.download(logFilename); - log = FileUtil.readFileAsString(stream); - parseFile(logFilename); + IWorkspace workspace = context.getJob().getWorkspace(); + if (workspace.exists(logFilename)) { + context.log("Load log file from '" + logFilename + "'"); + InputStream stream = workspace.download(logFilename); + log = FileUtil.readFileAsString(stream); + parseFile(logFilename); + } } } diff --git a/src/main/java/cloudgene/mapred/util/Settings.java b/src/main/java/cloudgene/mapred/util/Settings.java index 2ee34182..2b19eb4f 100644 --- a/src/main/java/cloudgene/mapred/util/Settings.java +++ b/src/main/java/cloudgene/mapred/util/Settings.java @@ -75,7 +75,7 @@ public class Settings { private Map externalWorkspace = null; - private int uploadLimit = 500; + private int uploadLimit = 5000; private String googleAnalytics = ""; From 4658d04fc023f92161fdbe68ad91aba849f670c3 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Tue, 5 Sep 2023 14:26:06 +0200 Subject: [PATCH 08/35] Fix local workspace and downloads --- .../java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java index 458f27b6..057690d4 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java @@ -139,14 +139,14 @@ public String getParent(String url) { @Override public String createFolder(String id) { - String folder = FileUtil.path(workspace, OUTPUT_DIRECTORY, id); + String folder = FileUtil.path(workspace, id); FileUtil.createDirectory(folder); return folder; } @Override public String createFile(String parent, String id) { - String folder = FileUtil.path(workspace, OUTPUT_DIRECTORY, parent); + String folder = FileUtil.path(workspace, parent); FileUtil.createDirectory(folder); return FileUtil.path(folder, id); } From 85c0c99d57f00bf08f8a5b4d3519a99d8a6f27de Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Wed, 6 Sep 2023 13:18:12 +0200 Subject: [PATCH 09/35] Cleanup engine and remove seconda queue and DAG --- .../mapred/CommandLineInterface.java | 2 - .../cloudgene/mapred/cli/RunApplication.java | 372 ------------------ .../cloudgene/mapred/jobs/AbstractJob.java | 275 ++++--------- .../cloudgene/mapred/jobs/CloudgeneJob.java | 159 ++------ .../java/cloudgene/mapred/jobs/Download.java | 6 + .../cloudgene/mapred/jobs/WorkflowEngine.java | 110 +----- .../GraphNode.java => ExecutableStep.java} | 51 +-- .../mapred/jobs/engine/Executor.java | 56 +-- .../cloudgene/mapred/jobs/engine/Planner.java | 102 ----- .../mapred/jobs/engine/graph/Graph.java | 127 ------ .../mapred/jobs/engine/graph/GraphEdge.java | 43 -- .../cloudgene/mapred/jobs/queue/Queue.java | 27 +- .../mapred/server/services/JobService.java | 3 +- .../mapred/api/v2/jobs/SubmitJobTest.java | 2 +- .../jobs/PriorityThreadPoolExecutorTest.java | 20 +- .../mapred/jobs/WorkflowEngineTest.java | 99 +++-- 16 files changed, 221 insertions(+), 1233 deletions(-) delete mode 100644 src/main/java/cloudgene/mapred/cli/RunApplication.java rename src/main/java/cloudgene/mapred/jobs/engine/{graph/GraphNode.java => ExecutableStep.java} (82%) delete mode 100644 src/main/java/cloudgene/mapred/jobs/engine/graph/Graph.java delete mode 100644 src/main/java/cloudgene/mapred/jobs/engine/graph/GraphEdge.java diff --git a/src/main/java/cloudgene/mapred/CommandLineInterface.java b/src/main/java/cloudgene/mapred/CommandLineInterface.java index 4f183faf..993da818 100644 --- a/src/main/java/cloudgene/mapred/CommandLineInterface.java +++ b/src/main/java/cloudgene/mapred/CommandLineInterface.java @@ -5,7 +5,6 @@ import cloudgene.mapred.cli.InstallGitHubApplication; import cloudgene.mapred.cli.ListApplications; import cloudgene.mapred.cli.RemoveApplication; -import cloudgene.mapred.cli.RunApplication; import cloudgene.mapred.cli.ShowPlugins; import cloudgene.mapred.cli.ShowVersion; import cloudgene.mapred.cli.StartServer; @@ -32,7 +31,6 @@ private void printHeader() { public static void main(String[] args) throws Exception { CommandLineInterface toolbox = new CommandLineInterface("cloudgene", args); - toolbox.addTool("run", RunApplication.class); toolbox.addTool("install", InstallApplication.class); toolbox.addTool("gh", InstallGitHubApplication.class); toolbox.addTool("github-install", InstallGitHubApplication.class); diff --git a/src/main/java/cloudgene/mapred/cli/RunApplication.java b/src/main/java/cloudgene/mapred/cli/RunApplication.java deleted file mode 100644 index 738a17a3..00000000 --- a/src/main/java/cloudgene/mapred/cli/RunApplication.java +++ /dev/null @@ -1,372 +0,0 @@ -package cloudgene.mapred.cli; - -import java.io.File; -import java.io.FileNotFoundException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; - -import com.esotericsoftware.yamlbeans.YamlException; - -import cloudgene.mapred.apps.Application; -import cloudgene.mapred.core.User; -import cloudgene.mapred.jobs.CloudgeneJob; -import cloudgene.mapred.jobs.CloudgeneStep; -import cloudgene.mapred.jobs.Message; -import cloudgene.mapred.jobs.WorkflowEngine; -import cloudgene.mapred.plugins.PluginManager; -import cloudgene.mapred.wdl.WdlApp; -import cloudgene.mapred.wdl.WdlParameterInput; -import cloudgene.mapred.wdl.WdlParameterInputType; -import cloudgene.mapred.wdl.WdlParameterOutput; -import cloudgene.mapred.wdl.WdlReader; -import genepi.io.FileUtil; - -public class RunApplication extends BaseTool { - - public static final String DEFAULT_HADOOP_USER = "cloudgene"; - - private WdlApp app = null; - - private boolean output = false; - - private boolean logging = false; - - public RunApplication(String[] args) { - super(args); - } - - @Override - public int run() { - return 0; - } - - // we override start instead of run, because we use our own cli parser based - // on inputs defined in the yaml file - @Override - public int start() { - - // call init manualy - init(); - - if (args.length < 1) { - printError("No filename of a cloudgene.yaml file or id of an application found."); - System.out.println("cloudgene run "); - System.out.println(); - return 1; - } - - String filename = args[0]; - - // check if file exists - if (new File(filename).exists()) { - // load wdl app from yaml file - try { - app = WdlReader.loadAppFromFile(filename); - - } catch (FileNotFoundException e1) { - printError("File '" + filename + "' not found."); - return 1; - } catch (YamlException e) { - printError("Syntax error in file '" + filename + "':"); - printError(e.getMessage()); - return 1; - } catch (Exception e2) { - printError("Error loading file '" + filename + "':"); - printError(e2.getMessage()); - return 1; - } - } else { - // check if application name is installed - Application application = repository.getById(filename); - if (application == null) { - printError("Application or file " + filename + " not found."); - return 1; - } - - if (application.hasSyntaxError()) { - printError("Syntax error in file '" + application.getFilename() + "':"); - printError(application.getErrorMessage()); - return 1; - } - - app = application.getWdlApp(); - - } - - // print application details - System.out.println(); - System.out.println(app.getName() + " " + app.getVersion()); - if (app.getWebsite() != null && !app.getWebsite().isEmpty()) { - System.out.println(app.getWebsite()); - } - if (app.getAuthor() != null && !app.getAuthor().isEmpty()) { - System.out.println(app.getAuthor()); - } - System.out.println(); - - if (app.getWorkflow() == null || app.getWorkflow().getSteps() == null - || app.getWorkflow().getSteps().size() == 0) { - printError("Application has no workflow. It seems that " + app.getName() + " is a data package."); - return 1; - } - - // create the command line parser - CommandLineParser parser = new DefaultParser(); - - // create options for each input param in yaml file - Options options = CommandLineUtil.createOptionsFromApp(app); - - // add general options: run on docker - Option loggingOption = new Option(null, "show-log", false, "Stream logging messages to stdout"); - loggingOption.setRequired(false); - options.addOption(loggingOption); - - // add general options: run on docker - Option noOutputOption = new Option(null, "show-output", false, "Stream output to stdout"); - noOutputOption.setRequired(false); - options.addOption(noOutputOption); - - // add general options: output folder - Option outputFolderOption = new Option(null, "output", true, "Output folder"); - outputFolderOption.setRequired(false); - options.addOption(outputFolderOption); - - // parse the command line arguments - CommandLine line = null; - try { - - line = parser.parse(options, args); - - } catch (Exception e) { - printError(e.getMessage()); - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("input parameters:", options); - System.out.println(); - return 1; - } - - if (line.hasOption("show-log")) { - logging = true; - } - - if (line.hasOption("show-output")) { - output = true; - } - - // show supported plugins - PluginManager manager = PluginManager.getInstance(); - manager.initPlugins(settings); - - /* - * for (IPlugin plugin : manager.getPlugins()) { if (manager.isEnabled(plugin)) - * { printText(0, spaces("[INFO]", 8) + plugin.getStatus()); } else { - * printText(0, spaces("[WARN]", 8) + plugin.getStatus()); } } - */ - - // create directories - FileUtil.createDirectory(settings.getTempPath()); - - // start workflow engine - WorkflowEngine engine = new WorkflowEngine(1, 1); - new Thread(engine).start(); - - // init job - - SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd-HHmmss"); - String id = "job-" + sdf.format(new Date()); - String local = FileUtil.path(id); - - if (line.hasOption("output")) { - local = line.getOptionValue("output"); - } - - FileUtil.createDirectory(local); - - // file params with values from cmdline - try { - Map params = CommandLineUtil.createParams(app, line, local, ""); - - // dummy user - User user = new User(); - user.setUsername("local"); - user.setPassword("local"); - user.makeAdmin(); - - CloudgeneJob job = new CloudgeneJob(user, id, app, params) { - - private int count = 0; - - @Override - public boolean afterSubmission() { - boolean result = super.afterSubmission(); - if (result) { - printParameters(this, app); - } - return result; - } - - @Override - public void onStepStarted(CloudgeneStep step) { - super.onStepStarted(step); - System.out.println(); - printText(0, spaces("[INFO]", 8) + step.getName() + "..."); - } - - @Override - public void onStepFinished(CloudgeneStep step) { - super.onStepFinished(step); - int c = 0; - for (Message message : step.getLogMessages()) { - String text = message.getMessage().replaceAll("
", "\n").replaceAll("\n", - "\n" + spaces(4 + 8)); - String type = getDescription(message.getType()); - type = spaces("[" + type + "]", 8); - if (message.getType() == Message.OK) { - type = makeGreen(type); - } else if (message.getType() == Message.ERROR) { - type = makeRed(type); - } - - printText(0, spaces(type, 8) + text); - c++; - if (c < step.getLogMessages().size()) { - printSingleLine(4); - } - } - } - - @Override - public void writeOutputln(String line) { - super.writeOutputln(line); - if (output) { - printText(0, spaces("[OUT]", 8) + line); - } - } - - @Override - public void writeLog(String line) { - super.writeLog(line); - if (logging) { - printText(0, spaces("[LOG]", 8) + line); - } - } - - }; - job.setId(id); - job.setName(id); - job.setLocalWorkspace(local); - //job.setHdfsWorkspace(hdfs); - job.setSettings(settings); - job.setApplication(app.getName() + " " + app.getVersion()); - job.setApplicationId(app.getId()); - - - printText(0, spaces("[INFO]", 8) + "Submit job " + id + "..."); - - // submit job - engine.submit(job); - - // wait until job is complete. - while (!job.isComplete()) { - Thread.sleep(1000); - } - long wallTime = (job.getFinishedOn() - job.getSubmittedOn()) / 1000; - long setupTime = (job.getSetupEndTime() - job.getSetupStartTime()) / 1000; - long pipelineTime = (job.getFinishedOn() - job.getSubmittedOn()) / 1000; - // print steps and feedback - System.out.println(); - - if (job.getState() == CloudgeneJob.STATE_SUCCESS) { - - printlnInGreen("Done! Executed without errors."); - System.out.println("Results can be found in file://" + (new File(local)).getAbsolutePath()); - System.out.println(); - if (logging) { - printText(0, spaces("[LOG]", 8) + "Wall-Time: " + wallTime + " sec [Setup: " + setupTime - + " sec, Pipeline: " + pipelineTime + " sec]"); - } - System.out.println(); - return 0; - - } else { - printlnInRed("Error: Execution failed."); - System.out.println(); - System.out.println(); - if (logging) { - printText(0, spaces("[LOG]", 8) + "Wall-Time: " + wallTime + " [Setup: " + setupTime - + ", Pipeline: " + pipelineTime); - System.out.println(); - } - - System.out.println(); - - return 1; - } - } catch (FileNotFoundException e) { - printlnInRed("Error: " + e.getMessage()); - System.out.println(); - System.out.println(); - return 1; - - } catch (Exception e) { - printlnInRed("Error: Execution failed."); - System.out.println("Details:"); - e.printStackTrace(); - System.out.println(); - System.out.println(); - return 1; - } - - } - - public void printParameters(CloudgeneJob job, WdlApp app) { - if (app.getWorkflow().getInputs().size() > 0) { - printText(2 + 6, "Input values: "); - for (WdlParameterInput input : app.getWorkflow().getInputs()) { - if (input.getTypeAsEnum() != WdlParameterInputType.AGBCHECKBOX - && input.getTypeAsEnum() != WdlParameterInputType.TERMS_CHECKBOX && !input.isAdminOnly() - && input.isVisible()) { - printText(4 + 6, input.getId() + ": " + job.getContext().get(input.getId())); - } - } - } - - if (app.getWorkflow().getOutputs().size() > 0) { - printText(2 + 6, "Results:"); - for (WdlParameterOutput output : app.getWorkflow().getOutputs()) { - if (output.isDownload() && !output.isAdminOnly()) { - printText(4 + 6, output.getDescription() + ": " + job.getContext().get(output.getId())); - } - } - } - }; - - public String getDescription(int type) { - switch (type) { - case Message.ERROR: - return "ERROR"; - case Message.OK: - return "OK"; - case Message.WARNING: - return "WARN"; - case Message.RUNNING: - return "RUN"; - default: - return "??"; - } - } - - @Override - public void createParameters() { - - } - -} diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java index e5279087..580563e5 100644 --- a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java +++ b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java @@ -289,7 +289,7 @@ public boolean afterSubmission() { } } - public void runSetupSteps() { + public void runInstallationAndResolveAppLinks() { Settings settings = getSettings(); ApplicationRepository repository = settings.getApplicationRepository(); @@ -303,163 +303,67 @@ public void runSetupSteps() { linkedAppId = value.replaceAll("apps@", ""); } - if (!value.isEmpty()) { - Application linkedApp = repository.getByIdAndUser(linkedAppId, getUser()); - if (linkedApp != null) { - // update environment variables - Map envApp = Environment.getApplicationVariables(linkedApp.getWdlApp(), - settings); - Map envJob = Environment.getJobVariables(context); - Map properties = linkedApp.getWdlApp().getProperties(); - for (String property : properties.keySet()) { - Object propertyValue = properties.get(property); - if (propertyValue instanceof String) { - propertyValue = Environment.env(propertyValue.toString(), envApp); - propertyValue = Environment.env(propertyValue.toString(), envJob); - } - properties.put(property, propertyValue); - } - - getContext().setData(input.getName(), properties); - } else { - String error = "Application " + linkedAppId + " is not installed or wrong permissions."; - log.info(error); - writeOutput(error); - setError(error); - setSetupComplete(false); - setState(AbstractJob.STATE_FAILED); - return; - } + if (value.isEmpty()) { + continue; } - } - } - - // execute setup steps - - try { - - log.info("Job " + getId() + ": executing setups..."); - writeLog("Executing Job setups...."); - - boolean succesfull = executeSetupSteps(); - - if (succesfull && hasSteps()) { - // all okey - - log.info("Job " + getId() + ": executed successful. job has steps."); - setSetupComplete(true); - return; - - } else if (succesfull && !hasSteps()) { - // all okey and no more steps - - log.info("Job " + getId() + ": executed successful. job has no more steps."); - - writeLog("Job execution successful."); - writeLog("Exporting Data..."); - - setState(AbstractJob.STATE_EXPORTING); - - try { - - boolean successfulAfter = after(); - - if (successfulAfter) { - - setState(AbstractJob.STATE_SUCCESS); - setSetupComplete(true); - log.info("Job " + getId() + ": data export successful."); - writeLog("Data export successful."); - - } else { - - setSetupComplete(false); - setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": data export failed."); - writeLog("Data export failed."); - - } - - } catch (Error | Exception e) { - - Writer writer = new StringWriter(); - PrintWriter printWriter = new PrintWriter(writer); - e.printStackTrace(printWriter); - String s = writer.toString(); - - setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": data export failed.", e); - writeLog("Data export failed: " + e.getLocalizedMessage() + "\n" + s); - + Application linkedApp = repository.getByIdAndUser(linkedAppId, getUser()); + if (linkedApp == null) { + String error = "Application " + linkedAppId + " is not installed or wrong permissions."; + log.info(error); + writeOutput(error); + setError(error); setSetupComplete(false); - + setState(AbstractJob.STATE_FAILED); + return; } - - writeLog("Cleaning up..."); - cleanUp(); - log.info("Job " + getId() + ": cleanup successful."); - writeLog("Cleanup successful."); - - closeStdOutFiles(); - - } else if (!succesfull) { - - setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": execution failed. " + getError()); - writeLog("Job execution failed: " + getError()); - writeLog("Cleaning up..."); - onFailure(); - log.info("Job " + getId() + ": cleanup successful."); - writeLog("Cleanup successful."); - - if (canceld) { - setState(AbstractJob.STATE_CANCELED); + // update environment variables + Map envApp = Environment.getApplicationVariables(linkedApp.getWdlApp(), settings); + Map envJob = Environment.getJobVariables(context); + Map properties = linkedApp.getWdlApp().getProperties(); + for (String property : properties.keySet()) { + Object propertyValue = properties.get(property); + if (propertyValue instanceof String) { + propertyValue = Environment.env(propertyValue.toString(), envApp); + propertyValue = Environment.env(propertyValue.toString(), envJob); + } + properties.put(property, propertyValue); } - setSetupComplete(false); - closeStdOutFiles(); + getContext().setData(input.getName(), properties); } + } - } catch (Exception | Error e) { - - setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": initialization failed.", e); - - Writer writer = new StringWriter(); - PrintWriter printWriter = new PrintWriter(writer); - e.printStackTrace(printWriter); - String s = writer.toString(); - - writeLog("Initialization failed: " + e.getLocalizedMessage() + "\n" + s); - - writeLog("Cleaning up..."); - onFailure(); - log.info("Job " + getId() + ": cleanup successful."); - writeLog("Cleanup successful."); - setSetupComplete(false); - - closeStdOutFiles(); - - setSetupRunning(false); + setSetupComplete(true); - } } @Override public void run() { - - if (state == AbstractJob.STATE_CANCELED || state == AbstractJob.STATE_FAILED) { - onFailure(); - setError("Job Execution failed."); + + if (isCanceld()) { return; } - - log.info("Job " + getId() + ": running."); + + log.info("Setup job " + getId() + "..."); + setState(AbstractJob.STATE_RUNNING); + + setSetupStartTime(System.currentTimeMillis()); + setSetupRunning(true); + runInstallationAndResolveAppLinks(); + setSetupEndTime(System.currentTimeMillis()); + setSetupRunning(false); + if (!isSetupComplete()) { + log.info("Setup failed for job " + getId() + ". Not added to Long Time Queue."); + setFinishedOn(System.currentTimeMillis()); + setComplete(true); + return; + } + + log.info("Run job " + getId() + "..."); setStartTime(System.currentTimeMillis()); try { - setState(AbstractJob.STATE_RUNNING); writeLog("Details:"); writeLog(" Name: " + getName()); writeLog(" Job-Id: " + getId()); @@ -478,69 +382,56 @@ public void run() { writeLog(" " + parameter.getDescription() + ": " + context.get(parameter.getName())); } - writeLog("Preparing Job...."); - boolean successfulBefore = before(); - - if (!successfulBefore) { - - setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": job preparation failed."); - writeLog("Job Preparation failed."); - - } else { - - log.info("Job " + getId() + ": executing."); - writeLog("Executing Job...."); - - boolean succesfull = execute(); - - if (succesfull) { - - log.info("Job " + getId() + ": executed successful."); + writeLog("Executing Job...."); - writeLog("Job Execution successful."); - writeLog("Exporting Data..."); + boolean succesfull = execute(); - setState(AbstractJob.STATE_EXPORTING); + if (succesfull) { - try { + log.info("Job " + getId() + ": executed successful."); - boolean successfulAfter = after(); - - if (successfulAfter) { + writeLog("Job Execution successful."); + writeLog("Exporting Data..."); - setState(AbstractJob.STATE_SUCCESS); - log.info("Job " + getId() + ": data export successful."); - writeLog("Data Export successful."); + setState(AbstractJob.STATE_EXPORTING); - } else { + try { - setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": data export failed."); - writeLog("Data Export failed."); + boolean successfulAfter = after(); - } + if (successfulAfter) { - } catch (Error | Exception e) { + setState(AbstractJob.STATE_SUCCESS); + log.info("Job " + getId() + ": data export successful."); + writeLog("Data Export successful."); - Writer writer = new StringWriter(); - PrintWriter printWriter = new PrintWriter(writer); - e.printStackTrace(printWriter); - String s = writer.toString(); + } else { setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": data export failed.", e); - writeLog("Data Export failed: " + e.getLocalizedMessage() + "\n" + s); + log.error("Job " + getId() + ": data export failed."); + writeLog("Data Export failed."); } - } else { + } catch (Error | Exception e) { + + Writer writer = new StringWriter(); + PrintWriter printWriter = new PrintWriter(writer); + e.printStackTrace(printWriter); + String s = writer.toString(); setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": execution failed. " + getError()); - writeLog("Job Execution failed: " + getError()); + log.error("Job " + getId() + ": data export failed.", e); + writeLog("Data Export failed: " + e.getLocalizedMessage() + "\n" + s); } + + } else { + + setState(AbstractJob.STATE_FAILED); + log.error("Job " + getId() + ": execution failed. " + getError()); + writeLog("Job Execution failed: " + getError()); + } if (getState() == AbstractJob.STATE_FAILED || getState() == AbstractJob.STATE_CANCELED) { @@ -699,10 +590,6 @@ public boolean isSetupRunning() { return setupRunning; } - public boolean hasSteps() { - return true; - } - public CloudgeneContext getContext() { return context; } @@ -784,26 +671,14 @@ public boolean isRunning() { abstract public boolean execute(); - abstract public boolean executeSetupSteps(); - abstract public boolean setup(); - abstract public boolean before(); - abstract public boolean after(); abstract public boolean onFailure(); abstract public boolean cleanUp(); - public void onStepFinished(CloudgeneStep step) { - - } - - public void onStepStarted(CloudgeneStep step) { - - } - public void kill() { } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java index 1b7404b4..cf06403c 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java @@ -11,9 +11,8 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.jobs.engine.Executor; +import cloudgene.mapred.jobs.engine.ExecutableStep; import cloudgene.mapred.jobs.engine.Planner; -import cloudgene.mapred.jobs.engine.graph.Graph; -import cloudgene.mapred.jobs.engine.graph.GraphNode; import cloudgene.mapred.jobs.workspace.WorkspaceWrapper; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlParameterInput; @@ -139,42 +138,27 @@ public boolean setup() { } - @Override - public boolean hasSteps() { - try { - // evaluate WDL derictives - Planner planner = new Planner(); - WdlApp app = planner.evaluateWDL(this.app, context, getSettings()); - return app.getWorkflow().getSteps().size() > 0; - } catch (Exception e) { - // e.printStackTrace(); - writeOutput(e.getMessage()); - setError(e.getMessage()); - return false; - } - } - @Override public boolean execute() { - + try { - log.info("Job " + getId() + " submitted..."); - - // evaluate WDL derictives + // evaluate WDL and replace all variables (e.g. ${job_id}) Planner planner = new Planner(); WdlApp app = planner.evaluateWDL(this.app, context, getSettings()); - // create dag from wdl document - Graph graph = planner.buildDAG(app.getWorkflow().getSteps(), app.getWorkflow(), context); + // merge setup steps and normal steps + List steps = new Vector(app.getWorkflow().getSetups()); + steps.addAll(app.getWorkflow().getSteps()); + log.info("Job " + getId() + " execute " + steps.size() + " steps"); - // execute optimzed dag + // execute steps executor = new Executor(); - boolean successful = executor.execute(graph); + boolean successful = executor.execute(steps, context); if (!successful) { setError("Job Execution failed."); - GraphNode failedNode = executor.getCurrentNode(); + ExecutableStep failedNode = executor.getCurrentNode(); executeFailureStep(failedNode.getStep()); return false; } @@ -191,54 +175,11 @@ public boolean execute() { } - @Override - public boolean executeSetupSteps() { - - try { - // evaluate WDL - Planner planner = new Planner(); - WdlApp app = planner.evaluateWDL(this.app, context, getSettings()); - - // if a single setup step is set, add it to setups - WdlStep setup = app.getWorkflow().getSetup(); - if (setup != null) { - app.getWorkflow().getSetups().add(0, setup); - } - - Graph graph = planner.buildDAG(app.getWorkflow().getSetups(), app.getWorkflow(), context); - - // execute optimized DAG - executor = new Executor(); - boolean successful = executor.execute(graph); - - if (!successful) { - setError("Job Execution failed."); - GraphNode failedNode = executor.getCurrentNode(); - executeFailureStep(failedNode.getStep()); - return false; - } - - setError(null); - return true; - } catch (Exception e) { - // e.printStackTrace(); - writeOutput(e.getMessage()); - setError(e.getMessage()); - return false; - } - - } - - @Override - public boolean before() { - - return true; - - } - @Override public boolean onFailure() { + after(); + cleanUp(); return true; @@ -247,45 +188,34 @@ public boolean onFailure() { public boolean executeFailureStep(WdlStep failedStep) { WdlStep step = app.getWorkflow().getOnFailure(); + cleanUp(); - if (step != null) { - try { - writeLog("Executing onFailure... "); - GraphNode node = new GraphNode(step, context); - context.setData("cloudgene.failedStep", failedStep); - context.setData("cloudgene.failedStep.classname", failedStep.getClassname()); - node.run(); - boolean result = node.isSuccessful(); - if (result) { - - // export parameters generated by onFailure step - for (CloudgeneParameterOutput out : getOutputParams()) { - if (out.isAutoExport() && step.getGenerates().contains(out.getName())) { - log.info("Export parameter '" + out.getName() + "'..."); - context.println("Export parameter '" + out.getName() + "'..."); - exportParameter(out); - } - } - - writeLog("onFailure execution successful."); - return true; - } else { - writeLog("onFailure execution failed."); - return false; - } + if (step == null) { + return true; + } - } catch (Exception e) { + try { + writeLog("Executing onFailure... "); + ExecutableStep node = new ExecutableStep(step, context); + context.setData("cloudgene.failedStep", failedStep); + context.setData("cloudgene.failedStep.classname", failedStep.getClassname()); + node.run(); + boolean result = node.isSuccessful(); + if (result) { + writeLog("onFailure execution successful."); + return true; + } else { writeLog("onFailure execution failed."); - writeLog(e.getMessage()); - setError(e.getMessage()); return false; } + } catch (Exception e) { + writeLog("onFailure execution failed."); + writeLog(e.getMessage()); + setError(e.getMessage()); + return false; } - cleanUp(); - - return true; } @Override @@ -306,15 +236,12 @@ public boolean cleanUp() { @Override public boolean after() { - // create output zip file for hdfs folders - for (CloudgeneParameterOutput out : getOutputParams()) { + log.info("Execute after and export params..."); - if (out.isDownload() && !out.isAutoExport()) { - // export to local folder for faster download + for (CloudgeneParameterOutput out : getOutputParams()) { + if (out.isDownload()) { exportParameter(out); - } - } return true; @@ -336,19 +263,13 @@ public boolean exportParameter(CloudgeneParameterOutput out) { for (Download download : downloads) { download.setParameter(out); download.setCount(MAX_DOWNLOAD); - } - writeLog(" Added " + downloads.size() + " downloads."); - - List customDownloads = context.getDownloads(out.getName()); - if (customDownloads != null) { - for (Download download : customDownloads) { - download.setParameter(out); - download.setCount(MAX_DOWNLOAD); + if (!out.getFiles().contains(download)) { + out.getFiles().add(download); + writeLog(" Added new download " + download.getName() + "."); + } else { + writeLog(" Download " + download.getName() + " already added."); } - writeLog(" Added " + customDownloads.size() + " custom downloads."); - downloads.addAll(customDownloads); } - Collections.sort(downloads); out.setFiles(downloads); diff --git a/src/main/java/cloudgene/mapred/jobs/Download.java b/src/main/java/cloudgene/mapred/jobs/Download.java index 4b121ff2..da74ac08 100644 --- a/src/main/java/cloudgene/mapred/jobs/Download.java +++ b/src/main/java/cloudgene/mapred/jobs/Download.java @@ -85,5 +85,11 @@ public int compareTo(Download o) { return name.compareTo(o.getName()); } + + @Override + public boolean equals(Object object) { + Download download = (Download) object; + return name.equals(download.getName()); + } } diff --git a/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java b/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java index 022eb3d5..42733522 100644 --- a/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java +++ b/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java @@ -16,12 +16,8 @@ public class WorkflowEngine implements Runnable { private Thread threadLongTimeQueue; - private Thread threadShortTimeQueue; - private Queue longTimeQueue; - private Queue shortTimeQueue; - private boolean running = false; private AtomicLong priorityCounter = new AtomicLong(); @@ -30,38 +26,6 @@ public class WorkflowEngine implements Runnable { public WorkflowEngine(int ltqThreads, int stqThreads) { - shortTimeQueue = new Queue("ShortTimeQueue", stqThreads, false, false) { - - @Override - public PriorityRunnable createRunnable(AbstractJob job) { - return new SetupThread(job); - } - - @Override - public void onComplete(AbstractJob job) { - - if (job.isSetupComplete()) { - - if (job.hasSteps()) { - statusUpdated(job); - longTimeQueue.submit(job); - } else { - job.setFinishedOn(System.currentTimeMillis()); - jobCompleted(job); - job.setComplete(true); - } - - } else { - log.info("Setup failed for Job " + job.getId() + ". Not added to Long Time Queue."); - job.setFinishedOn(System.currentTimeMillis()); - jobCompleted(job); - job.setComplete(true); - } - - } - - }; - longTimeQueue = new Queue("LongTimeQueue", ltqThreads, true, true) { @Override @@ -92,11 +56,11 @@ public void submit(AbstractJob job, long priority) { boolean okey = job.afterSubmission(); if (okey) { - shortTimeQueue.submit(job); + longTimeQueue.submit(job); } else { job.setFinishedOn(System.currentTimeMillis()); statusUpdated(job); - job.setComplete(true); + job.setComplete(true); } } @@ -119,7 +83,7 @@ public void restart(AbstractJob job, long priority) { boolean okey = job.afterSubmission(); if (okey) { - shortTimeQueue.submit(job); + longTimeQueue.submit(job); } else { job.setFinishedOn(System.currentTimeMillis()); statusUpdated(job); @@ -134,21 +98,13 @@ public void updatePriority(AbstractJob job, long priority) { } public void cancel(AbstractJob job) { - - if (shortTimeQueue.isInQueue(job)) { - shortTimeQueue.cancel(job); - } - if (longTimeQueue.isInQueue(job)) { longTimeQueue.cancel(job); } - } @Override public void run() { - threadShortTimeQueue = new Thread(shortTimeQueue); - threadShortTimeQueue.start(); threadLongTimeQueue = new Thread(longTimeQueue); threadLongTimeQueue.start(); running = true; @@ -156,36 +112,29 @@ public void run() { } public void stop() { - threadShortTimeQueue.stop(); threadLongTimeQueue.stop(); } public void block() { - shortTimeQueue.pause(); longTimeQueue.pause(); running = false; } public void resume() { - shortTimeQueue.resume(); longTimeQueue.resume(); running = true; } public boolean isRunning() { - return running && shortTimeQueue.isRunning() && longTimeQueue.isRunning(); + return running && longTimeQueue.isRunning(); } public int getActiveCount() { - return shortTimeQueue.getActiveCount() + longTimeQueue.getActiveCount(); + return longTimeQueue.getActiveCount(); } public AbstractJob getJobById(String id) { - AbstractJob job = longTimeQueue.getJobById(id); - if (job == null) { - job = shortTimeQueue.getJobById(id); - } - return job; + return longTimeQueue.getJobById(id); } public Map getCounters(int state) { @@ -211,8 +160,7 @@ public Map getCounters(int state) { public List getJobsByUser(User user) { - List jobs = shortTimeQueue.getJobsByUser(user); - jobs.addAll(longTimeQueue.getJobsByUser(user)); + List jobs = longTimeQueue.getJobsByUser(user); for (AbstractJob job : jobs) { @@ -227,23 +175,6 @@ public List getJobsByUser(User user) { return jobs; } - public List getAllJobsInShortTimeQueue() { - - List jobs = shortTimeQueue.getAllJobs(); - - for (AbstractJob job : jobs) { - - if (job instanceof CloudgeneJob) { - - ((CloudgeneJob) job).updateProgress(); - - } - - } - - return shortTimeQueue.getAllJobs(); - } - public List getAllJobsInLongTimeQueue() { List jobs = longTimeQueue.getAllJobs(); @@ -261,33 +192,8 @@ public List getAllJobsInLongTimeQueue() { return jobs; } - class SetupThread extends PriorityRunnable { - - private AbstractJob job; - - public SetupThread(AbstractJob job) { - this.job = job; - } - - @Override - public void run() { - log.info("Start input validation for job " + job.getId() + "..."); - job.setSetupStartTime(System.currentTimeMillis()); - job.setSetupRunning(true); - job.runSetupSteps(); - job.setSetupEndTime(System.currentTimeMillis()); - job.setSetupRunning(false); - log.info("Input Validation for job " + job.getId() + " finished. Result: " + job.isSetupComplete()); - } - - } - public boolean isInQueue(AbstractJob job) { - if (!shortTimeQueue.isInQueue(job)) { - return longTimeQueue.isInQueue(job); - } else { - return true; - } + return longTimeQueue.isInQueue(job); } protected void statusUpdated(AbstractJob job) { diff --git a/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphNode.java b/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java similarity index 82% rename from src/main/java/cloudgene/mapred/jobs/engine/graph/GraphNode.java rename to src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java index 7d0fceb1..1964181a 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphNode.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java @@ -1,4 +1,4 @@ -package cloudgene.mapred.jobs.engine.graph; +package cloudgene.mapred.jobs.engine; import java.io.File; import java.io.PrintWriter; @@ -7,8 +7,6 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; -import java.util.List; -import java.util.Vector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +23,8 @@ import cloudgene.sdk.internal.WorkflowStep; import genepi.io.FileUtil; -public class GraphNode implements Runnable { +public class ExecutableStep implements Runnable { + private WdlStep step; private CloudgeneContext context; @@ -38,23 +37,14 @@ public class GraphNode implements Runnable { private boolean successful = false; - private boolean finish = false; - - private List inputs; - - private List outputs; - private long time; private String id = ""; - public GraphNode(WdlStep step, CloudgeneContext context) - throws MalformedURLException, ClassNotFoundException, InstantiationException, IllegalAccessException { + public ExecutableStep(WdlStep step, CloudgeneContext context) throws Exception { this.step = step; this.context = context; this.job = context.getJob(); - inputs = new Vector(); - outputs = new Vector(); instance(); } @@ -98,7 +88,7 @@ private void instance() if (object instanceof CloudgeneStep) { instance = (CloudgeneStep) object; } else if (object instanceof WorkflowStep) { - instance = new JavaInternalStep((WorkflowStep) object); + instance = new JavaInternalStep((WorkflowStep) object); } else { instance = new ErrorStep("Error during initialization: class " + step.getClassname() + " ( " + object.getClass().getSuperclass().getCanonicalName() + ") " @@ -108,7 +98,7 @@ private void instance() // check requirements PluginManager pluginManager = PluginManager.getInstance(); - for (String plugin : instance.getRequirements()) { + for (String plugin : instance.getRequirements()) { if (!pluginManager.isEnabled(plugin)) { instance = new ErrorStep( "Requirements not fullfilled. This steps needs plugin '" + plugin + "'"); @@ -140,8 +130,6 @@ public void run() { context.setCurrentStep(instance); job.getSteps().add(instance); - job.onStepStarted(instance); - job.writeLog("------------------------------------------------------"); job.writeLog(step.getName()); job.writeLog("------------------------------------------------------"); @@ -156,11 +144,9 @@ public void run() { if (!successful) { job.writeLog(" " + step.getName() + " [ERROR]"); successful = false; - finish = true; context.incCounter("steps.failure." + id, 1); context.submitCounter("steps.failure." + id); - job.onStepFinished(instance); return; } else { @@ -184,17 +170,13 @@ public void run() { context.submitCounter("steps.failure." + id); successful = false; - finish = true; - job.onStepFinished(instance); return; } context.incCounter("steps.success." + id, 1); context.submitCounter("steps.success." + id); - finish = true; successful = true; - job.onStepFinished(instance); } @@ -204,6 +186,7 @@ public boolean isSuccessful() { public void kill() { if (instance != null) { + log.info("Get kill signal for job " + job.getId()); instance.kill(); } } @@ -222,26 +205,6 @@ public int getProgress() { } } - public boolean isFinish() { - return finish; - } - - public void addInput(String input) { - inputs.add(input); - } - - public void addOutput(String output) { - outputs.add(output); - } - - public List getInputs() { - return inputs; - } - - public List getOutputs() { - return outputs; - } - public void setTime(long time) { this.time = time; } diff --git a/src/main/java/cloudgene/mapred/jobs/engine/Executor.java b/src/main/java/cloudgene/mapred/jobs/engine/Executor.java index 659bcaf4..5d02c648 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/Executor.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/Executor.java @@ -2,48 +2,22 @@ import java.util.List; -import cloudgene.mapred.jobs.CloudgeneJob; -import cloudgene.mapred.jobs.CloudgeneParameterOutput; -import cloudgene.mapred.jobs.engine.graph.Graph; -import cloudgene.mapred.jobs.engine.graph.GraphNode; +import cloudgene.mapred.jobs.CloudgeneContext; +import cloudgene.mapred.wdl.WdlStep; public class Executor { - private GraphNode executableNode; + private ExecutableStep executableNode; - public boolean execute(Graph graph) { - - graph.getContext().log("Executor: execute DAG..."); - - while (graph.getSize() > 0) { - List nodes = graph.getSources(); - boolean successful = false; - successful = executeNodesSequential(graph, nodes); - if (!successful) { - return false; - } - - } - - return true; - } - - private boolean executeNodesSequential(Graph graph, List nodes) { - - for (GraphNode node : nodes) { - - executableNode = node; + public boolean execute(List steps, CloudgeneContext context) throws Exception { + context.log("Executor: execute " +steps.size() + " steps..."); + for (WdlStep step : steps) { + executableNode = new ExecutableStep(step, context); executableNode.run(); - // export results - exportResults(graph, node); - if (!executableNode.isSuccessful()) { return false; } - // TODO: cache.addToCache(node, graph.getContext()); - - graph.remove(node); } return true; @@ -67,21 +41,7 @@ public int getProgress() { } } - private void exportResults(Graph graph, GraphNode node) { - - CloudgeneJob job = (CloudgeneJob) graph.getContext().getJob(); - - for (CloudgeneParameterOutput out : job.getOutputParams()) { - if (out.isAutoExport() && node.getOutputs().contains(out.getName())) { - graph.getContext().println( - "Export parameter '" + out.getName() + "'..."); - job.exportParameter(out); - } - } - - } - - public GraphNode getCurrentNode() { + public ExecutableStep getCurrentNode() { return executableNode; } diff --git a/src/main/java/cloudgene/mapred/jobs/engine/Planner.java b/src/main/java/cloudgene/mapred/jobs/engine/Planner.java index 0da6d568..3eb83630 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/Planner.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/Planner.java @@ -2,8 +2,6 @@ import java.io.File; import java.io.StringWriter; -import java.net.MalformedURLException; -import java.util.List; import java.util.Map; import org.apache.velocity.Template; @@ -12,19 +10,13 @@ import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.Environment; -import cloudgene.mapred.jobs.engine.graph.Graph; -import cloudgene.mapred.jobs.engine.graph.GraphEdge; -import cloudgene.mapred.jobs.engine.graph.GraphNode; import cloudgene.mapred.jobs.engine.plugins.ParameterValueInput; import cloudgene.mapred.jobs.engine.plugins.ParameterValueOutput; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; -import cloudgene.mapred.wdl.WdlParameter; import cloudgene.mapred.wdl.WdlParameterInput; import cloudgene.mapred.wdl.WdlParameterOutput; import cloudgene.mapred.wdl.WdlReader; -import cloudgene.mapred.wdl.WdlStep; -import cloudgene.mapred.wdl.WdlWorkflow; public class Planner { @@ -78,98 +70,4 @@ public WdlApp evaluateWDL(WdlApp app, CloudgeneContext context, Settings setting return app2; } - public Graph buildDAG(List steps, WdlWorkflow config, CloudgeneContext context) - throws MalformedURLException, ClassNotFoundException, InstantiationException, IllegalAccessException { - - Graph graph = new Graph(context); - - // build nodes - GraphNode lastNode = null; - for (WdlStep step : steps) { - GraphNode node = new GraphNode(step, context); - graph.addNode(node); - if (lastNode != null) { - graph.connect(lastNode, node, null); - } - lastNode = node; - } - - // add output parameters - for (GraphNode node : graph.getNodes()) { - - for (WdlParameter param : config.getOutputs()) { - if (stepConsumesParameter(node.getStep(), param, context) - && !stepProducesParameter(node.getStep(), param, context)) { - node.addInput(param.getId()); - } - - if (stepProducesParameter(node.getStep(), param, context)) { - node.addOutput(param.getId()); - } - } - - for (WdlParameter param : config.getInputs()) { - if (stepConsumesParameter(node.getStep(), param, context)) { - node.addInput(param.getId()); - } - } - } - - context.log("Planner: DAG created."); - context.log(" Nodes: " + graph.getSize()); - for (GraphNode node : graph.getNodes()) { - context.log(" " + node.getStep().getName()); - String inputs = ""; - for (String input : node.getInputs()) { - inputs += input + " "; - } - context.log(" Inputs: " + inputs); - String outputs = ""; - for (String output : node.getOutputs()) { - outputs += output + " "; - } - context.log(" Outputs: " + outputs); - } - - context.log(" Dependencies: " + graph.getEdges().size()); - for (GraphEdge edge : graph.getEdges()) { - context.log(" " + edge.getSource().getStep().getName() + "->" + edge.getTarget().getStep().getName()); - } - - return graph; - - } - - private boolean stepConsumesParameter(WdlStep step, WdlParameter param, CloudgeneContext context) { - - if (step.get("params") != null) { - if (step.get("params").contains(context.get(param.getId()))) { - return true; - } - } - - if (step.get("mapper") != null) { - if (step.get("mapper").contains(context.get(param.getId()))) { - return true; - } - } - - if (step.get("reducer") != null) { - if (step.get("reducer").contains(context.get(param.getId()))) { - return true; - } - } - - return false; - - } - - private boolean stepProducesParameter(WdlStep step, WdlParameter param, CloudgeneContext context) { - if (step.getGenerates() != null) { - return (step.getGenerates().contains(context.get(param.getId()))); - } else { - return false; - } - } - } diff --git a/src/main/java/cloudgene/mapred/jobs/engine/graph/Graph.java b/src/main/java/cloudgene/mapred/jobs/engine/graph/Graph.java deleted file mode 100644 index b08feb8e..00000000 --- a/src/main/java/cloudgene/mapred/jobs/engine/graph/Graph.java +++ /dev/null @@ -1,127 +0,0 @@ -package cloudgene.mapred.jobs.engine.graph; - -import java.util.List; -import java.util.Vector; - -import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.wdl.WdlParameter; - -public class Graph { - - private List nodes; - - private List edges; - - private CloudgeneContext context; - - public Graph(CloudgeneContext context) { - - this.context = context; - - nodes = new Vector(); - edges = new Vector(); - - } - - public CloudgeneContext getContext() { - return context; - } - - public List getNodes() { - return nodes; - } - - public List getEdges() { - return edges; - } - - public void connect(GraphNode source, GraphNode target, WdlParameter param) { - GraphEdge edge = new GraphEdge(source, target, param); - edges.add(edge); - } - - public boolean areConnected(GraphNode source, GraphNode target) { - for (GraphEdge edge : edges) { - if (edge.getSource() == source && edge.getTarget() == target) { - return true; - } - } - return false; - } - - public boolean remove(GraphNode node) { - - List remove = new Vector(); - - nodes.remove(node); - for (GraphEdge edge : edges) { - if (edge.getSource() == node || edge.getTarget() == node) { - remove.add(edge); - } - } - edges.removeAll(remove); - - return true; - } - - public int getInDegree(GraphNode node) { - int degree = 0; - - for (GraphEdge edge : edges) { - if (edge.getTarget() == node) { - degree++; - } - } - - return degree; - - } - - public int getOutDegree(GraphNode node) { - int degree = 0; - - for (GraphEdge edge : edges) { - if (edge.getSource() == node) { - degree++; - } - } - - return degree; - } - - public List getSources() { - - List sources = new Vector(); - - for (GraphNode node : nodes) { - if (getInDegree(node) == 0) { - sources.add(node); - } - } - - return sources; - - } - - public List getTargets() { - - List targets = new Vector(); - - for (GraphNode node : nodes) { - if (getOutDegree(node) == 0) { - targets.add(node); - } - } - - return targets; - } - - public int getSize() { - return nodes.size(); - } - - public void addNode(GraphNode node) { - nodes.add(node); - } - -} diff --git a/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphEdge.java b/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphEdge.java deleted file mode 100644 index 1129e43f..00000000 --- a/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphEdge.java +++ /dev/null @@ -1,43 +0,0 @@ -package cloudgene.mapred.jobs.engine.graph; - -import cloudgene.mapred.wdl.WdlParameter; - -public class GraphEdge { - - private WdlParameter parameter; - - private GraphNode source; - - private GraphNode target; - - public GraphEdge(GraphNode source, GraphNode target, WdlParameter parameter) { - this.source = source; - this.target = target; - this.parameter = parameter; - } - - public WdlParameter getParameter() { - return parameter; - } - - public void setParameter(WdlParameter parameter) { - this.parameter = parameter; - } - - public GraphNode getSource() { - return source; - } - - public void setSource(GraphNode source) { - this.source = source; - } - - public GraphNode getTarget() { - return target; - } - - public void setTarget(GraphNode target) { - this.target = target; - } - -} diff --git a/src/main/java/cloudgene/mapred/jobs/queue/Queue.java b/src/main/java/cloudgene/mapred/jobs/queue/Queue.java index 4ff2a6dc..29d1d8a3 100644 --- a/src/main/java/cloudgene/mapred/jobs/queue/Queue.java +++ b/src/main/java/cloudgene/mapred/jobs/queue/Queue.java @@ -15,6 +15,8 @@ public abstract class Queue implements Runnable { + private static final int POLL_FRQUENCY_MS = 100; + private List queue; private HashMap> futures; @@ -56,7 +58,7 @@ public void submit(AbstractJob job) { log.info(name + ": Submit job" + (priority ? " (P: " + job.getPriority() + ")" : "") + "..."); if (priority) { - // sorty by state and by priority + // sort by state and by priority Collections.sort(queue, new PriorityComparator()); } @@ -71,11 +73,12 @@ public void submit(AbstractJob job) { } public void cancel(AbstractJob job) { - + if (job.getState() == AbstractJob.STATE_RUNNING || job.getState() == AbstractJob.STATE_EXPORTING) { - log.info(name + ": Cancel Job " + job.getId() + "..."); + log.info(name + ": Cancel running job " + job.getId() + "..."); + if (job.getSetupStartTime() > 0 && job.getSetupEndTime() == 0){ job.setSetupEndTime(System.currentTimeMillis()); } @@ -83,20 +86,25 @@ public void cancel(AbstractJob job) { job.kill(); job.cancel(); + log.info(name + ": Job " + job.getId() + " canceled."); + + if (updatePositions) { updatePositionInQueue(); } - } - - if (job.getState() == AbstractJob.STATE_WAITING) { + } else if (job.getState() == AbstractJob.STATE_WAITING) { + log.info(name + ": Cancel waiting job " + job.getId() + "..."); + synchronized (futures) { synchronized (queue) { PriorityRunnable runnable = runnables.get(job); if (runnable != null) { + System.out.println("Kill runnable"); scheduler.kill(runnable); + runnables.remove(job); } if (job.getSetupStartTime() > 0 && job.getSetupEndTime() == 0){ job.setSetupEndTime(System.currentTimeMillis()); @@ -106,7 +114,8 @@ public void cancel(AbstractJob job) { queue.remove(job); futures.remove(job); onComplete(job); - log.info(name + ": Cancel Job..."); + + log.info(name + ": Job " + job.getId() + " canceled."); if (updatePositions) { updatePositionInQueue(); @@ -115,6 +124,8 @@ public void cancel(AbstractJob job) { } } + } else { + log.info(name + ": Cancel job " + job.getId() + ". Unkown state: " + job.getState()); } } @@ -163,7 +174,7 @@ public void run() { } try { - Thread.sleep(5000); + Thread.sleep(POLL_FRQUENCY_MS); } catch (InterruptedException e) { e.printStackTrace(); } diff --git a/src/main/java/cloudgene/mapred/server/services/JobService.java b/src/main/java/cloudgene/mapred/server/services/JobService.java index 3c138cf0..eea2afb7 100644 --- a/src/main/java/cloudgene/mapred/server/services/JobService.java +++ b/src/main/java/cloudgene/mapred/server/services/JobService.java @@ -510,7 +510,8 @@ public List getJobs(String state) { case "running-stq": - jobs = engine.getAllJobsInShortTimeQueue(); + //TODO: remove! + jobs = new Vector(); break; case "current": diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java index 17ce60b9..45e392e1 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java @@ -184,7 +184,7 @@ public void testSubmitThreeTasksStepPublic() { // submit jobs String id = RestAssured.given().header(accessToken).and().multiPart("input-input", "input-file").when() - .post("/api/v2/jobs/submit/three-tasks").then().statusCode(200).and().extract().jsonPath() + .post("/api/v2/jobs/submit/three-tasks").then().log().all().statusCode(200).and().extract().jsonPath() .getString("id"); // wait until job is complete diff --git a/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java b/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java index 101f23ef..e06fe7bb 100644 --- a/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java +++ b/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java @@ -36,7 +36,7 @@ public void testCancelRunningJob() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } @@ -67,7 +67,7 @@ public void testCancelRunningJob() throws Exception { List jobsAfterCancel = engine.getAllJobsInLongTimeQueue(); assertEquals(jobsBeforeSubmit.size(), jobsAfterCancel.size()); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } @@ -79,7 +79,7 @@ public void testCancelWaitingJob() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } @@ -119,7 +119,7 @@ public void testCancelWaitingJob() throws Exception { // clear queue engine.cancel(job1); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } } @@ -136,7 +136,7 @@ public void testMultipleJobs() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } @@ -224,7 +224,7 @@ public void testMultipleJobs() throws Exception { assertEquals(AbstractJob.STATE_CANCELED, job3.getState()); assertEquals(AbstractJob.STATE_CANCELED, job4.getState()); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } } @@ -241,7 +241,7 @@ public void testMultipleJobsWithPriority() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } @@ -329,7 +329,7 @@ public void testMultipleJobsWithPriority() throws Exception { assertEquals(AbstractJob.STATE_CANCELED, job3.getState()); assertEquals(AbstractJob.STATE_CANCELED, job4.getState()); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } } @@ -346,7 +346,7 @@ public void testMultipleJobsAndUpdatePriority() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } @@ -460,7 +460,7 @@ public void testMultipleJobsAndUpdatePriority() throws Exception { assertEquals(AbstractJob.STATE_CANCELED, job3.getState()); assertEquals(AbstractJob.STATE_CANCELED, job4.getState()); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } } diff --git a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java index ef366c4e..064b66af 100644 --- a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java +++ b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java @@ -27,16 +27,15 @@ public class WorkflowEngineTest { @Inject TestApplication application; - + @Inject WorkspaceFactory workspaceFactory; - - + @Test public void testReturnTrueStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/return-true.yaml"); Map inputs = new HashMap(); @@ -61,7 +60,6 @@ public void testReturnFalseStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - WdlApp app = WdlReader.loadAppFromFile("test-data/return-false.yaml"); Map params = new HashMap(); @@ -107,7 +105,7 @@ public void testReturnExceptionStep() throws Exception { @Test public void testReturnTrueInSetupStep() throws Exception { - + WorkflowEngine engine = application.getWorkflowEngine(); WdlApp app = WdlReader.loadAppFromFile("test-data/return-true-in-setup.yaml"); @@ -126,8 +124,8 @@ public void testReturnTrueInSetupStep() throws Exception { assertTrue(job.getSetupStartTime() > 0); assertTrue(job.getSetupEndTime() > 0); // no steps - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); } @@ -136,7 +134,6 @@ public void testReturnFalseInSetupStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - WdlApp app = WdlReader.loadAppFromFile("test-data/return-false-in-setup.yaml"); Map params = new HashMap(); @@ -152,8 +149,8 @@ public void testReturnFalseInSetupStep() throws Exception { assertTrue(job.getSetupStartTime() > 0); assertTrue(job.getSetupEndTime() > 0); // no steps - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_FAILED, job.getState()); } @@ -162,7 +159,6 @@ public void testReturnTrueInSecondSetupStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - WdlApp app = WdlReader.loadAppFromFile("test-data/return-true-in-setup2.yaml"); Map params = new HashMap(); @@ -179,8 +175,8 @@ public void testReturnTrueInSecondSetupStep() throws Exception { assertTrue(job.getSetupStartTime() > 0); assertTrue(job.getSetupEndTime() > 0); // no steps - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); } @@ -189,7 +185,6 @@ public void testReturnTrueInSecondSetupStepAndNormalStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - WdlApp app = WdlReader.loadAppFromFile("test-data/return-true-in-setup3.yaml"); Map params = new HashMap(); @@ -232,24 +227,22 @@ public void testHiddenInputsAndDefaultValues() throws Exception { assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); - - //check step ouputs - assertEquals("text1: my-value\n", job.getSteps().get(0).getLogMessages().get(0).getMessage()); - assertEquals("checkbox1: true\n", job.getSteps().get(1).getLogMessages().get(0).getMessage()); - assertEquals("list1: value1\n", job.getSteps().get(2).getLogMessages().get(0).getMessage()); - assertEquals("text2: my-value\n", job.getSteps().get(3).getLogMessages().get(0).getMessage()); - assertEquals("checkbox2: true\n", job.getSteps().get(4).getLogMessages().get(0).getMessage()); - assertEquals("list2: value1\n", job.getSteps().get(5).getLogMessages().get(0).getMessage()); - - + + // check step ouputs + assertEquals("text1: my-value\n", job.getSteps().get(0).getLogMessages().get(0).getMessage()); + assertEquals("checkbox1: true\n", job.getSteps().get(1).getLogMessages().get(0).getMessage()); + assertEquals("list1: value1\n", job.getSteps().get(2).getLogMessages().get(0).getMessage()); + assertEquals("text2: my-value\n", job.getSteps().get(3).getLogMessages().get(0).getMessage()); + assertEquals("checkbox2: true\n", job.getSteps().get(4).getLogMessages().get(0).getMessage()); + assertEquals("list2: value1\n", job.getSteps().get(5).getLogMessages().get(0).getMessage()); + } - + @Test public void testReturnWriteFileInSecondSetupStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - String myContent = "test-test-test-test-text"; WdlApp app = WdlReader.loadAppFromFile("test-data/write-file-in-setup.yaml"); @@ -303,7 +296,7 @@ public void testReturnWriteFileInSecondSetupStep() throws Exception { public void testEmptyStepList() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/no-steps.yaml"); Map params = new HashMap(); @@ -318,8 +311,8 @@ public void testEmptyStepList() throws Exception { assertTrue(job.getFinishedOn() > 0); assertTrue(job.getSetupStartTime() > 0); assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); } @@ -327,7 +320,7 @@ public void testEmptyStepList() throws Exception { public void testReturnFalseInSecondSetupStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/return-false-in-setup2.yaml"); Map params = new HashMap(); @@ -343,8 +336,8 @@ public void testReturnFalseInSecondSetupStep() throws Exception { assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getSetupStartTime() > 0); assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_FAILED, job.getState()); } @@ -352,7 +345,7 @@ public void testReturnFalseInSecondSetupStep() throws Exception { public void testWriteTextToFileJob() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/write-text-to-file.yaml"); Map params = new HashMap(); @@ -383,7 +376,7 @@ public void testWriteTextToFileJob() throws Exception { public void testWriteTextToFileOnFailureInStepJob() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/write-text-to-file-on-failure.yaml"); Map params = new HashMap(); @@ -435,9 +428,8 @@ public void testWriteTextToFileOnFailureInSetupJob() throws Exception { assertTrue(job.getFinishedOn() > 0); assertTrue(job.getSetupStartTime() > 0); assertTrue(job.getSetupEndTime() > 0); - // steps not executed - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals("lukas_text", content); assertEquals(job.getState(), AbstractJob.STATE_FAILED); } @@ -469,8 +461,8 @@ public void testWriteTextToFileOnFailureInSecondSetupJob() throws Exception { // no steps executed assertTrue(job.getSubmittedOn() > 0); assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals("lukas_text", content); assertEquals(job.getState(), AbstractJob.STATE_FAILED); } @@ -479,7 +471,7 @@ public void testWriteTextToFileOnFailureInSecondSetupJob() throws Exception { public void testThreeTasksStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/three-tasks.yaml"); Map params = new HashMap(); @@ -528,8 +520,7 @@ public void testWriteTextToStdOutStep() throws Exception { assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); - String stdout = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), - "std.out"); + String stdout = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), "std.out"); String contentStdOut = FileUtil.readFileAsString(stdout); String log = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), "job.txt"); @@ -580,7 +571,7 @@ public void testApplicationLinks() throws Exception { assertTrue(message.getMessage().contains("property3:hey3!")); } - + @Test public void testApplicationLinksWithoutAppsPrefix() throws Exception { @@ -611,12 +602,12 @@ public void testApplicationLinksWithoutAppsPrefix() throws Exception { assertTrue(message.getMessage().contains("property3:hey3!")); } - + @Test public void testApplicationLinksWithoutVersion() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/app-links.yaml"); Map params = new HashMap(); @@ -642,12 +633,12 @@ public void testApplicationLinksWithoutVersion() throws Exception { assertTrue(message.getMessage().contains("property3:hey3!")); } - + @Test public void testOptionalApplicationLinks() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/app-links-optional.yaml"); Map params = new HashMap(); @@ -672,12 +663,12 @@ public void testOptionalApplicationLinks() throws Exception { assertFalse(message.getMessage().contains("property3:hey3!")); } - + @Test public void testApplicationLinksWrongApplication() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/app-links.yaml"); Map params = new HashMap(); @@ -702,7 +693,7 @@ public void testApplicationLinksWrongApplication() throws Exception { public void testApplicationLinksWrongPermissions() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/app-links.yaml"); Map params = new HashMap(); @@ -727,7 +718,7 @@ public void testApplicationLinksWrongPermissions() throws Exception { public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs) throws Exception { - UserDao userDao = new UserDao(application.getDatabase()); + UserDao userDao = new UserDao(application.getDatabase()); User user = userDao.findByUsername("admin"); return createJobFromWdl(app, inputs, user); @@ -735,9 +726,9 @@ public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs) thr public CloudgeneJob createJobFromWdlAsUser(WdlApp app, Map inputs) throws Exception { - UserDao userDao = new UserDao(application.getDatabase()); + UserDao userDao = new UserDao(application.getDatabase()); User user = userDao.findByUsername("user"); - + return createJobFromWdl(app, inputs, user); } From 9c69209fcff5e9c5624fc4b523322c7f5e4d716a Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 7 Sep 2023 11:39:56 +0200 Subject: [PATCH 10/35] Remove cloudgene-sdk, improve logging and implement reporting --- pom.xml | 6 - .../cloudgene/mapred/jobs/AbstractJob.java | 26 ++-- .../mapred/jobs/CloudgeneContext.java | 97 ++---------- .../cloudgene/mapred/jobs/CloudgeneJob.java | 21 ++- .../cloudgene/mapred/jobs/CloudgeneStep.java | 2 +- .../mapred/jobs/CloudgeneStepFactory.java | 14 +- .../mapred/jobs/engine/ExecutableStep.java | 2 +- .../mapred/jobs/engine/Executor.java | 8 +- .../cloudgene/mapred/jobs/engine/Planner.java | 33 ++-- .../mapred/jobs/sdk/WorkflowContext.java | 105 +++++++++++++ .../mapred/jobs/sdk/WorkflowStep.java | 35 +++++ .../cloudgene/mapred/jobs/steps/Sleep.java | 4 +- .../jobs/workspace/WorkspaceWrapper.java | 47 ------ .../mapred/plugins/docker/DockerStep.java | 6 +- .../nextflow/NextflowProcessConfig.java | 17 +++ .../nextflow/NextflowProcessRenderer.java | 122 +++++++++++++++ .../mapred/plugins/nextflow/NextflowStep.java | 116 +++++++------- .../mapred/plugins/nextflow/NextflowTask.java | 57 ++++--- .../plugins/nextflow/report/Report.java | 110 ++++++++++++++ .../plugins/nextflow/report/ReportEvent.java | 141 ++++++++++++++++++ .../nextflow/report/ReportEventExecutor.java | 57 +++++++ .../mapred/server/services/JobService.java | 1 + .../mapred/steps/BashCommandStep.java | 8 +- .../cloudgene/mapred/steps/GroovyStep.java | 12 +- .../mapred/steps/HtmlWidgetStep.java | 16 +- .../mapred/steps/JavaExternalStep.java | 4 +- .../mapred/steps/JavaInternalStep.java | 2 +- .../mapred/steps/RMarkdownDockerStep.java | 10 +- .../mapred/steps/RMarkdownLocalStep.java | 10 +- .../mapred/steps/ReturnTrueStep.java | 4 +- .../java/cloudgene/mapred/wdl/WdlStep.java | 24 +-- .../mapred/api/v2/jobs/SubmitJobTest.java | 2 +- .../mapred/jobs/WorkflowEngineTest.java | 2 +- .../mapred/jobs/steps/CheckInputs.java | 4 +- .../mapred/jobs/steps/LongSleepStep.java | 4 +- .../mapred/jobs/steps/PrintDataset.java | 4 +- .../jobs/steps/ReturnExceptionStep.java | 4 +- .../mapred/jobs/steps/ReturnFalseStep.java | 4 +- .../mapred/jobs/steps/ReturnTrueStep.java | 4 +- .../mapred/jobs/steps/ThreeTasksStep.java | 4 +- .../jobs/steps/WriteFilesToFolderStep.java | 4 +- .../jobs/steps/WriteTextToFileStep.java | 4 +- .../jobs/steps/WriteTextToStdOutStep.java | 4 +- .../cloudgene/mapred/steps/TestCommand.java | 2 +- test-data/print-hdfs-file-dataset.groovy | 17 --- test-data/print-hdfs-file.groovy | 16 -- test-data/print-properties.groovy | 2 +- 47 files changed, 807 insertions(+), 391 deletions(-) create mode 100644 src/main/java/cloudgene/mapred/jobs/sdk/WorkflowContext.java create mode 100644 src/main/java/cloudgene/mapred/jobs/sdk/WorkflowStep.java delete mode 100644 src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceWrapper.java create mode 100644 src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessConfig.java create mode 100644 src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java create mode 100644 src/main/java/cloudgene/mapred/plugins/nextflow/report/Report.java create mode 100644 src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEvent.java create mode 100644 src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEventExecutor.java delete mode 100644 test-data/print-hdfs-file-dataset.groovy delete mode 100644 test-data/print-hdfs-file.groovy diff --git a/pom.xml b/pom.xml index a842ceca..a8d73db4 100644 --- a/pom.xml +++ b/pom.xml @@ -78,12 +78,6 @@ - - cloudgene - cloudgene-java-sdk - 1.0.2 - - genepi genepi-io diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java index 580563e5..25b931ca 100644 --- a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java +++ b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java @@ -345,7 +345,7 @@ public void run() { return; } - log.info("Setup job " + getId() + "..."); + log.info("[Job {}] Setup job...", getId()); setState(AbstractJob.STATE_RUNNING); setSetupStartTime(System.currentTimeMillis()); @@ -354,13 +354,13 @@ public void run() { setSetupEndTime(System.currentTimeMillis()); setSetupRunning(false); if (!isSetupComplete()) { - log.info("Setup failed for job " + getId() + ". Not added to Long Time Queue."); + log.info("[Job {}] Setup failed." , getId()); setFinishedOn(System.currentTimeMillis()); setComplete(true); return; } - log.info("Run job " + getId() + "..."); + log.info("[Job {}] Running job...", getId()); setStartTime(System.currentTimeMillis()); try { @@ -388,7 +388,7 @@ public void run() { if (succesfull) { - log.info("Job " + getId() + ": executed successful."); + log.info("[Job {}] Execution successful.", getId()); writeLog("Job Execution successful."); writeLog("Exporting Data..."); @@ -402,13 +402,13 @@ public void run() { if (successfulAfter) { setState(AbstractJob.STATE_SUCCESS); - log.info("Job " + getId() + ": data export successful."); + log.info("[Job {}] data export successful.", getId()); writeLog("Data Export successful."); } else { setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": data export failed."); + log.error("[Job {}] data export failed.", getId()); writeLog("Data Export failed."); } @@ -421,7 +421,7 @@ public void run() { String s = writer.toString(); setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": data export failed.", e); + log.error("[Job {}] data export failed.", getId(), e); writeLog("Data Export failed: " + e.getLocalizedMessage() + "\n" + s); } @@ -429,7 +429,7 @@ public void run() { } else { setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": execution failed. " + getError()); + log.error("[Job {}] Execution failed. {}", getId(), getError()); writeLog("Job Execution failed: " + getError()); } @@ -438,13 +438,13 @@ public void run() { writeLog("Cleaning up..."); onFailure(); - log.info("Job " + getId() + ": cleanup successful."); + log.info("[Job {}]cleanup successful.", getId()); writeLog("Cleanup successful."); } else { writeLog("Cleaning up..."); cleanUp(); - log.info("Job " + getId() + ": cleanup successful."); + log.info("[Job {}] cleanup successful.", getId()); writeLog("Cleanup successful."); } @@ -460,7 +460,7 @@ public void run() { } catch (Exception | Error e) { setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": initialization failed.", e); + log.error("[Job {}]: initialization failed.", getId(), e); Writer writer = new StringWriter(); PrintWriter printWriter = new PrintWriter(writer); @@ -471,7 +471,7 @@ public void run() { writeLog("Cleaning up..."); onFailure(); - log.info("Job " + getId() + ": cleanup successful."); + log.info("[Job {}]: cleanup successful.", getId()); writeLog("Cleanup successful."); closeStdOutFiles(); @@ -484,7 +484,7 @@ public void run() { public void cancel() { writeLog("Canceled by user."); - log.info("Job " + getId() + ": canceld by user."); + log.info("[Job {}]: canceld by user.", getId()); /* * if (state == STATE_RUNNING) { closeStdOutFiles(); } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java index dc4eb8d7..fc27fc2e 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java @@ -10,10 +10,9 @@ import java.util.Vector; import cloudgene.mapred.core.User; -import cloudgene.mapred.util.HashUtil; +import cloudgene.mapred.jobs.sdk.WorkflowContext; import cloudgene.mapred.util.MailUtil; import cloudgene.mapred.util.Settings; -import cloudgene.sdk.internal.WorkflowContext; import genepi.io.FileUtil; public class CloudgeneContext extends WorkflowContext { @@ -42,12 +41,10 @@ public class CloudgeneContext extends WorkflowContext { private Map data = new HashMap(); - private Map config; + private Map config; private int chunks = 0; - private Map> customDownloads = new HashMap>(); - public CloudgeneContext(CloudgeneJob job) { this.workingDirectory = job.getWorkingDirectory(); @@ -67,10 +64,8 @@ public CloudgeneContext(CloudgeneJob job) { } outputParameters = new HashMap(); - customDownloads = new HashMap>(); for (CloudgeneParameterOutput param : job.getOutputParams()) { outputParameters.put(param.getName(), param); - customDownloads.put(param.getName(), new Vector()); } settings = job.getSettings(); @@ -85,54 +80,6 @@ public CloudgeneStep getCurrentStep() { return step; } - /*public void setupOutputParameters() throws Exception { - - FileUtil.deleteDirectory(getLocalTemp()); - - // create output directories - FileUtil.createDirectory(getLocalOutput()); - FileUtil.createDirectory(getLocalTemp()); - - // create output directories - for (CloudgeneParameterOutput param : outputParameters.values()) { - - switch (param.getType()) { - case HDFS_FILE: - case HDFS_FOLDER: - - throw new Exception("HDFS support was removed in Cloudgene 3"); - - case LOCAL_FILE: - String parent = getLocalOutput(); - if (!param.isDownload()) { - parent = getLocalTemp(); - } - String folder = FileUtil.path(parent, param.getName()); - String filename = FileUtil.path(folder, param.getName()); - // delete and create (needed for restart) - FileUtil.deleteDirectory(folder); - FileUtil.createDirectory(folder); - param.setValue(filename); - break; - - case LOCAL_FOLDER: - String parent2 = getLocalOutput(); - if (!param.isDownload()) { - parent2 = getLocalTemp(); - } - - String folder2 = FileUtil.path(parent2, param.getName()); - // delete and create (needed for restart) - FileUtil.deleteDirectory(folder2); - FileUtil.createDirectory(folder2); - param.setValue(folder2); - break; - } - - } - - }*/ - public String getInput(String param) { if (inputParameters.get(param) != null) { @@ -170,28 +117,6 @@ public String get(String param) { return result; } } - - @Override - public void addDownload(String param, String name, String size, String path) { - List downloads = customDownloads.get(param); - if (downloads == null) { - new RuntimeException("Parameter " + param + " is unknown."); - } - - String hash = HashUtil.getSha256(name + size + path + (Math.random() * 100000)); - Download download = new Download(); - download.setName(name); - download.setSize(size); - download.setPath(path); - download.setHash(hash); - download.setCount(CloudgeneJob.MAX_DOWNLOAD); - - downloads.add(download); - } - - public List getDownloads(String param){ - return customDownloads.get(param); - } public Settings getSettings() { return settings; @@ -257,10 +182,10 @@ public boolean sendMail(String to, String subject, String body) throws Exception Settings settings = getSettings(); if (settings.getMail() != null) { - - MailUtil.send(settings.getMail().get("smtp"), settings.getMail().get("port"), settings.getMail().get("user"), - settings.getMail().get("password"), settings.getMail().get("name"), to, - "[" + settings.getName() + "] " + subject, body); + + MailUtil.send(settings.getMail().get("smtp"), settings.getMail().get("port"), + settings.getMail().get("user"), settings.getMail().get("password"), settings.getMail().get("name"), + to, "[" + settings.getName() + "] " + subject, body); } return true; @@ -286,7 +211,7 @@ public void setInput(String input, String value) { parameter.setValue(value); } - public void setOutput(String input, String value) { + public void setOutput2(String input, String value) { CloudgeneParameterOutput parameter = outputParameters.get(input); parameter.setValue(value); @@ -374,7 +299,6 @@ public void beginTask(String name) { logs.add(status); } - public Message createTask(String name) { Message status = new Message(step, Message.RUNNING, name); @@ -387,8 +311,6 @@ public Message createTask(String name) { return status; } - - public void beginTask(String name, int totalWork) { beginTask(name); } @@ -424,14 +346,15 @@ public void setData(String key, Object object) { } @Override - public void setConfig(Map config) { + public void setConfig(Map config) { this.config = config; } @Override public String getConfig(String param) { if (config != null) { - return config.get(param); + Object value = config.get(param); + return value != null ? value.toString() : null; } else { return null; } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java index cf06403c..71f514d4 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java @@ -10,10 +10,9 @@ import org.slf4j.LoggerFactory; import cloudgene.mapred.core.User; -import cloudgene.mapred.jobs.engine.Executor; import cloudgene.mapred.jobs.engine.ExecutableStep; +import cloudgene.mapred.jobs.engine.Executor; import cloudgene.mapred.jobs.engine.Planner; -import cloudgene.mapred.jobs.workspace.WorkspaceWrapper; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlParameterInput; import cloudgene.mapred.wdl.WdlParameterOutput; @@ -102,12 +101,13 @@ public boolean setup() { FileUtil.createDirectory(context.getLocalTemp()); try { + log.info("[Job {}] Setup workspace {}'", getId(), workspace.getName()); context.log("Setup External Workspace on " + workspace.getName()); workspace.setup(this.getId()); - context.setExternalWorkspace(new WorkspaceWrapper(workspace)); + context.setWorkspace(workspace); } catch (Exception e) { writeLog(e.toString()); - log.info("Error setup external workspace", e); + log.error("[Job {}] Error setup external workspace failed.", getId(), e); setError(e.toString()); return false; } @@ -118,16 +118,18 @@ public boolean setup() { switch (param.getType()) { case HDFS_FILE: case HDFS_FOLDER: - + log.info("[Job {}] Setting output param '{}' failed. HDFS support was removed in Cloudgene 3'", getId(), param.getName()); throw new RuntimeException("HDFS support was removed in Cloudgene 3"); - case LOCAL_FILE: + case LOCAL_FILE: String filename = workspace.createFile(param.getName(), param.getName()); param.setValue(filename); + log.info("[Job {}] Set output file '{}' to '{}'", getId(), param.getName(), param.getValue()); break; case LOCAL_FOLDER: String folder = workspace.createFolder(param.getName()); + log.info("[Job {}] Set output folder '{}' to '{}'", getId(), param.getName(), param.getValue()); param.setValue(folder); break; } @@ -150,7 +152,7 @@ public boolean execute() { // merge setup steps and normal steps List steps = new Vector(app.getWorkflow().getSetups()); steps.addAll(app.getWorkflow().getSteps()); - log.info("Job " + getId() + " execute " + steps.size() + " steps"); + log.info("[Job {}] execute {} steps", getId(), steps.size()); // execute steps executor = new Executor(); @@ -221,12 +223,15 @@ public boolean executeFailureStep(WdlStep failedStep) { @Override public boolean cleanUp() { + log.info("[Job {}] Cleaning up...", getId()); + try { workspace.cleanup(getId()); } catch (IOException e) { writeLog("Cleanup failed."); writeLog(e.getMessage()); setError(e.getMessage()); + log.error("[Job {}] Clean up failed.", getId(), e); return false; } @@ -236,7 +241,7 @@ public boolean cleanUp() { @Override public boolean after() { - log.info("Execute after and export params..."); + log.info("[Job {}] Export parameters...", getId()); for (CloudgeneParameterOutput out : getOutputParams()) { if (out.isDownload()) { diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java index 0580bacf..8a0094ff 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java @@ -7,8 +7,8 @@ import java.io.InputStreamReader; import java.util.List; +import cloudgene.mapred.jobs.sdk.WorkflowContext; import cloudgene.mapred.wdl.WdlStep; -import cloudgene.sdk.internal.WorkflowContext; public abstract class CloudgeneStep { diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneStepFactory.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneStepFactory.java index cc2e6aa2..781cfc5f 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneStepFactory.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneStepFactory.java @@ -34,7 +34,7 @@ public void register(String type, Class clazz) { public String getClassname(WdlStep step) { - String type = step.get("type"); + String type = step.getString("type"); if (type != null) { @@ -55,17 +55,17 @@ public String getClassname(WdlStep step) { } } - if (step.get("pig") != null) { + if (step.getString("pig") != null) { throw new RuntimeException("Hadoop support was removed in Cloudgene 3"); } - if (step.get("spark") != null) { + if (step.getString("spark") != null) { throw new RuntimeException("Hadoop support was removed in Cloudgene 3"); - } else if (step.get("rmd") != null) { + } else if (step.getString("rmd") != null) { // rscript return RMarkdownStep.class.getName(); - } else if (step.get("rmd2") != null) { + } else if (step.getString("rmd2") != null) { // rscript return RMarkdownStep.class.getName(); @@ -75,13 +75,13 @@ public String getClassname(WdlStep step) { // custom class return step.getClassname(); - } else if (step.get("exec") != null || step.get("cmd") != null) { + } else if (step.getString("exec") != null || step.getString("cmd") != null) { // command return BashCommandStep.class.getName(); } else { - String runtime = step.get("runtime"); + String runtime = step.getString("runtime"); if (runtime == null || runtime.isEmpty() || runtime.toLowerCase().equals("hadoop")) { throw new RuntimeException("Hadoop support was removed in Cloudgene 3"); } else if (runtime != null && runtime.toLowerCase().equals("java")) { diff --git a/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java b/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java index 1964181a..9e8f8ee6 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java @@ -16,11 +16,11 @@ import cloudgene.mapred.jobs.CloudgeneJob; import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.CloudgeneStepFactory; +import cloudgene.mapred.jobs.sdk.WorkflowStep; import cloudgene.mapred.plugins.PluginManager; import cloudgene.mapred.steps.ErrorStep; import cloudgene.mapred.steps.JavaInternalStep; import cloudgene.mapred.wdl.WdlStep; -import cloudgene.sdk.internal.WorkflowStep; import genepi.io.FileUtil; public class ExecutableStep implements Runnable { diff --git a/src/main/java/cloudgene/mapred/jobs/engine/Executor.java b/src/main/java/cloudgene/mapred/jobs/engine/Executor.java index 5d02c648..3fa6e44f 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/Executor.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/Executor.java @@ -2,6 +2,9 @@ import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.wdl.WdlStep; @@ -9,11 +12,14 @@ public class Executor { private ExecutableStep executableNode; + private static Logger log = LoggerFactory.getLogger(Executor.class); + public boolean execute(List steps, CloudgeneContext context) throws Exception { - context.log("Executor: execute " +steps.size() + " steps..."); + context.log("Execute " + steps.size() + " steps..."); for (WdlStep step : steps) { executableNode = new ExecutableStep(step, context); + log.info("[Job {}] Executor: execute step '{}'...", context.getJobId(), step.getName()); executableNode.run(); if (!executableNode.isSuccessful()) { return false; diff --git a/src/main/java/cloudgene/mapred/jobs/engine/Planner.java b/src/main/java/cloudgene/mapred/jobs/engine/Planner.java index 3eb83630..89db8fe9 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/Planner.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/Planner.java @@ -1,13 +1,9 @@ package cloudgene.mapred.jobs.engine; import java.io.File; -import java.io.StringWriter; +import java.util.HashMap; import java.util.Map; -import org.apache.velocity.Template; -import org.apache.velocity.VelocityContext; -import org.apache.velocity.app.Velocity; - import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.Environment; import cloudgene.mapred.jobs.engine.plugins.ParameterValueInput; @@ -17,22 +13,22 @@ import cloudgene.mapred.wdl.WdlParameterInput; import cloudgene.mapred.wdl.WdlParameterOutput; import cloudgene.mapred.wdl.WdlReader; +import groovy.text.SimpleTemplateEngine; public class Planner { public WdlApp evaluateWDL(WdlApp app, CloudgeneContext context, Settings settings) throws Exception { - Velocity.setProperty("file.resource.loader.path", "/"); - VelocityContext context2 = new VelocityContext(); + Map context2 = new HashMap(); // add input values to context for (WdlParameterInput param : app.getWorkflow().getInputs()) { - context2.put(param.getId(), new ParameterValueInput(param, context.getInput(param.getId()))); + context2.put(param.getId(), context.getInput(param.getId())); } // add output values to context for (WdlParameterOutput param : app.getWorkflow().getOutputs()) { - context2.put(param.getId(), new ParameterValueOutput(param, context.getOutput(param.getId()))); + context2.put(param.getId(), context.getOutput(param.getId())); } // add job variables @@ -47,20 +43,17 @@ public WdlApp evaluateWDL(WdlApp app, CloudgeneContext context, Settings setting context2.put(key, envApp.get(key)); } - File manifest = new File(app.getManifestFile()); - - StringWriter sw = null; - try { + // add user specific variables + context2.put("user_email", context.getUser().getMail()); + context2.put("user_name", context.getUser().getUsername()); + context2.put("user_full_name", context.getUser().getFullName()); - Template template = Velocity.getTemplate(manifest.getAbsolutePath()); - sw = new StringWriter(); - template.merge(context2, sw); + File manifest = new File(app.getManifestFile()); - } catch (Exception e) { - throw e; - } + SimpleTemplateEngine engine = new SimpleTemplateEngine(); + String content = engine.createTemplate(manifest).make(context2).toString(); - WdlApp app2 = WdlReader.loadAppFromString(manifest.getAbsolutePath(), sw.toString()); + WdlApp app2 = WdlReader.loadAppFromString(manifest.getAbsolutePath(), content); app2.getWorkflow().setInputs(app.getWorkflow().getInputs()); app2.getWorkflow().setOutputs(app.getWorkflow().getOutputs()); diff --git a/src/main/java/cloudgene/mapred/jobs/sdk/WorkflowContext.java b/src/main/java/cloudgene/mapred/jobs/sdk/WorkflowContext.java new file mode 100644 index 00000000..78068b73 --- /dev/null +++ b/src/main/java/cloudgene/mapred/jobs/sdk/WorkflowContext.java @@ -0,0 +1,105 @@ +package cloudgene.mapred.jobs.sdk; + +import java.util.Map; +import java.util.Set; + +import cloudgene.mapred.jobs.workspace.IWorkspace; + +public abstract class WorkflowContext { + + public static final int OK = 0; + + public static final int ERROR = 1; + + public static final int WARNING = 2; + + public static final int RUNNING = 3; + + public abstract String getInput(String param); + + public abstract String getJobId(); + + public abstract String getOutput(String param); + + public abstract String get(String param); + + public abstract void println(String line); + + public abstract void log(String line); + + public abstract String getWorkingDirectory(); + + public abstract boolean sendNotification(String body) throws Exception; + + public abstract boolean sendMail(String subject, String body) throws Exception; + + public abstract boolean sendMail(String to, String subject, String body) throws Exception; + + public abstract Set getInputs(); + + public abstract void setInput(String input, String value); + + //public abstract void setOutput(String input, String value); + + public abstract void incCounter(String name, int value); + + public abstract void submitCounter(String name); + + public abstract Map getCounters(); + + public abstract Object getData(String key); + + public abstract String createLinkToFile(String id); + + public String createLinkToFile(String id, String filename) { + return "[NOT AVAILABLE]"; + + } + + public abstract String getJobName(); + + public abstract String getHdfsTemp(); + + public abstract String getLocalTemp(); + + public abstract void setConfig(Map config); + + public abstract String getConfig(String param); + + public abstract void message(String message, int type); + + private IWorkspace workspace; + + public void setWorkspace(IWorkspace workspace) { + this.workspace = workspace; + } + + public IWorkspace getWorkspace() { + return workspace; + } + + public void ok(String message) { + message(message, OK); + } + + public void error(String message) { + message(message, ERROR); + } + + public void warning(String message) { + message(message, WARNING); + } + + public abstract void beginTask(String name); + + public abstract void updateTask(String name, int type); + + public abstract void endTask(String message, int type); + + public void endTask(String message, Exception e) { + // TODO: add trace + String messageWithTrace = message + ". " + e.toString(); + endTask(messageWithTrace, ERROR); + } + +} diff --git a/src/main/java/cloudgene/mapred/jobs/sdk/WorkflowStep.java b/src/main/java/cloudgene/mapred/jobs/sdk/WorkflowStep.java new file mode 100644 index 00000000..a2641e7e --- /dev/null +++ b/src/main/java/cloudgene/mapred/jobs/sdk/WorkflowStep.java @@ -0,0 +1,35 @@ +package cloudgene.mapred.jobs.sdk; + +import java.io.File; + +public abstract class WorkflowStep { + + public String getFolder(Class clazz) { + return new File(clazz.getProtectionDomain().getCodeSource() + .getLocation().getPath()).getParent(); + } + + public void setup(WorkflowContext context) { + + } + + abstract public boolean run(WorkflowContext context); + + public int getMapProgress() { + return 0; + } + + public int getReduceProgress() { + return 0; + } + + public void updateProgress() { + + } + + public void kill() { + + } + + +} diff --git a/src/main/java/cloudgene/mapred/jobs/steps/Sleep.java b/src/main/java/cloudgene/mapred/jobs/steps/Sleep.java index 366f5102..d4b4dfa7 100644 --- a/src/main/java/cloudgene/mapred/jobs/steps/Sleep.java +++ b/src/main/java/cloudgene/mapred/jobs/steps/Sleep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class Sleep extends WorkflowStep { diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceWrapper.java b/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceWrapper.java deleted file mode 100644 index 12826cd3..00000000 --- a/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceWrapper.java +++ /dev/null @@ -1,47 +0,0 @@ -package cloudgene.mapred.jobs.workspace; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; - -import cloudgene.sdk.internal.IExternalWorkspace; - -public class WorkspaceWrapper implements IExternalWorkspace { - - private cloudgene.mapred.jobs.workspace.IWorkspace newWorkspace; - - public WorkspaceWrapper(cloudgene.mapred.jobs.workspace.IWorkspace newWorkspace) { - this.newWorkspace = newWorkspace; - } - - @Override - public String createPublicLink(String arg0) { - return newWorkspace.createPublicLink(arg0); - } - - @Override - public void delete(String arg0) throws IOException { - newWorkspace.delete(arg0); - } - - @Override - public InputStream download(String arg0) throws IOException { - return newWorkspace.download(arg0); - } - - @Override - public String getName() { - return newWorkspace.getName(); - } - - @Override - public void setup(String arg0) throws IOException { - newWorkspace.setup(arg0); - } - - @Override - public String upload(String arg0, File arg1) throws IOException { - return newWorkspace.upload(arg0, arg1); - } - -} diff --git a/src/main/java/cloudgene/mapred/plugins/docker/DockerStep.java b/src/main/java/cloudgene/mapred/plugins/docker/DockerStep.java index 6ae40ad3..15930889 100644 --- a/src/main/java/cloudgene/mapred/plugins/docker/DockerStep.java +++ b/src/main/java/cloudgene/mapred/plugins/docker/DockerStep.java @@ -18,7 +18,7 @@ public class DockerStep extends CloudgeneStep { @Override public boolean run(WdlStep step, CloudgeneContext context) { - String cmd = step.get("cmd"); + String cmd = step.getString("cmd"); if (cmd == null) { context.error("No 'exec' or 'cmd' parameter found."); @@ -28,12 +28,12 @@ public boolean run(WdlStep step, CloudgeneContext context) { context.error("'exec' or 'cmd' parameter cannot be an empty string."); } - String stdout = step.get("stdout", "false"); + String stdout = step.getString("stdout", "false"); boolean streamStdout = stdout.equals("true"); String[] params = cmd.split(" "); - String image = step.get("image"); + String image = step.getString("image"); if (image == null) { context.error("No 'image' parameter found."); diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessConfig.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessConfig.java new file mode 100644 index 00000000..515996f5 --- /dev/null +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessConfig.java @@ -0,0 +1,17 @@ +package cloudgene.mapred.plugins.nextflow; + +public class NextflowProcessConfig { + + private static final String DEFAULT_VIEW = "list"; + + private String view = DEFAULT_VIEW; + + public String getView() { + return view; + } + + public void setView(String view) { + this.view = view; + } + +} diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java new file mode 100644 index 00000000..c8f1c540 --- /dev/null +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java @@ -0,0 +1,122 @@ +package cloudgene.mapred.plugins.nextflow; + +import cloudgene.mapred.jobs.Message; + +public class NextflowProcessRenderer { + + public static void render(NextflowProcessConfig config, NextflowProcess process, Message message) { + switch (config.getView()) { + case "progressbar": + NextflowProcessRenderer.renderProgressbar(process, message); + break; + default: + NextflowProcessRenderer.renderList(process, message); + } + } + + public static void renderList(NextflowProcess process, Message message) { + + String text = "" + process.getName() + ""; + boolean running = false; + boolean ok = true; + for (NextflowTask task : process.getTasks()) { + + // TODO: use templates. + + String status = (String) task.getTrace().get("status"); + + if (status.equals("RUNNING") || status.equals("SUBMITTED")) { + running = true; + } + if (!status.equals("COMPLETED")) { + ok = false; + + } + text += "
"; + + text += (String) task.getTrace().get("name"); + if (status.equals("RUNNING")) { + text += "..."; + } + if (status.equals("COMPLETED")) { + text += " "; + } + if (status.equals("KILLED") || status.equals("FAILED")) { + text += " "; + } + + if (task.getLog() != null) { + text += "
" + task.getLog(); + } + + text += ""; + } + + message.setMessage(text); + + if (running) { + message.setType(Message.RUNNING); + } else { + if (ok) { + message.setType(Message.OK); + } else { + message.setType(Message.ERROR); + } + } + + } + + public static void renderProgressbar(NextflowProcess process, Message message) { + + String text = "" + process.getName() + "

"; + int running = 0; + boolean ok = true; + for (NextflowTask task : process.getTasks()) { + + String status = (String) task.getTrace().get("status"); + + if (status.equals("RUNNING") || status.equals("SUBMITTED")) { + running++; + } + if (!status.equals("COMPLETED")) { + ok = false; + } + text += " "; + + String style = ""; + String name = " "; + + if (status.equals("COMPLETED")) { + style = "badge-success"; + name = "OK"; + } + + if (status.equals("RUNNING")) { + style = "badge-info"; + name = "..."; + } + + if (status.equals("KILLED") || status.equals("FAILED")) { + style = "badge-danger"; + name = "X"; + } + + text += "" + name + ""; + + } + + message.setMessage(text); + + if (running > 0) { + message.setType(Message.RUNNING); + } else { + if (ok) { + message.setType(Message.OK); + } else { + message.setType(Message.ERROR); + } + } + + } + +} diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java index e861b110..22f9ebdf 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java @@ -11,29 +11,30 @@ import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.jobs.CloudgeneJob; import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; -import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.jobs.workspace.IWorkspace; import cloudgene.mapred.util.HashUtil; import cloudgene.mapred.wdl.WdlStep; import genepi.io.FileUtil; import groovy.json.JsonOutput; -import jakarta.inject.Inject; public class NextflowStep extends CloudgeneStep { + private static final String PROPERTY_PROCESS_CONFIG = "processes"; + private CloudgeneContext context; private Map messages = new HashMap(); - + + private Map configs = new HashMap(); + @Override public boolean run(WdlStep step, CloudgeneContext context) { this.context = context; - String script = step.get("script"); + String script = step.getString("script"); if (script == null) { // no script set. try to find a main.nf. IS the default convention of nf-core. @@ -46,8 +47,11 @@ public boolean run(WdlStep step, CloudgeneContext context) { "Nextflow script '" + scriptPath + "' not found. Please use 'script' to define your nf file."); } - NextflowBinary nextflow = NextflowBinary.build(context.getSettings()); + // load process styling + loadProcessConfigs(step.get(PROPERTY_PROCESS_CONFIG)); + NextflowBinary nextflow = NextflowBinary.build(context.getSettings()); + // TODO: move to bextflow binary. see nftest implementation List nextflowCommand = new Vector(); nextflowCommand.add("PATH=$PATH:/usr/local/bin"); nextflowCommand.add(nextflow.getBinary()); @@ -100,10 +104,14 @@ public boolean run(WdlStep step, CloudgeneContext context) { for (String key : step.keySet()) { if (key.startsWith("params.")) { String param = key.replace("params.", ""); - String value = step.get(key); + Object value = step.get(key); params.put(param, value); } } + if (step.get("params") != null) { + Map paramsMap = (Map) step.get("params"); + params.putAll(paramsMap); + } // add all inputs for (String param : context.getInputs()) { @@ -135,29 +143,29 @@ public boolean run(WdlStep step, CloudgeneContext context) { } IWorkspace workspace = job.getWorkspace(); - + nextflowCommand.add("-params-file"); nextflowCommand.add(paramsFile.getAbsolutePath()); nextflowCommand.add("-ansi-log"); nextflowCommand.add("false"); - + nextflowCommand.add("-with-weblog"); nextflowCommand.add(context.getSettings().getServerUrl() + context.getSettings().getUrlPrefix() + "/api/v2/collect/" + makeSecretJobId(context.getJobId())); - - //nextflowCommand.add("-log"); - //nextflowCommand.add(workspace.createLogFile("nextflow.log")); - + + // nextflowCommand.add("-log"); + // nextflowCommand.add(workspace.createLogFile("nextflow.log")); + nextflowCommand.add("-with-trace"); nextflowCommand.add(workspace.createLogFile("trace.csv")); - + nextflowCommand.add("-with-report"); nextflowCommand.add(workspace.createLogFile("report.html")); - + nextflowCommand.add("-with-timeline"); nextflowCommand.add(workspace.createLogFile("timeline.html")); - + StringBuilder output = new StringBuilder(); List command = new Vector(); @@ -168,8 +176,9 @@ public boolean run(WdlStep step, CloudgeneContext context) { NextflowCollector.getInstance().addContext(makeSecretJobId(context.getJobId()), context); try { - // context.beginTask("Running Nextflow pipeline..."); + boolean successful = executeCommand(command, context, output); + if (successful) { updateProgress(); return true; @@ -189,15 +198,10 @@ public boolean run(WdlStep step, CloudgeneContext context) { } updateProgress(); - // Write nextflow output into step - /* - * context.beginTask("Running Nextflow pipeline..."); String text = ""; - * - * if (killed) { text = output + "\n\n\n" + - * makeRed("Pipeline execution canceled."); } else { text = output + "\n\n\n" + - * makeRed("Pipeline execution failed."); } context.endTask(text, - * Message.ERROR_ANSI); - */ + if (killed) { + context.error("Pipeline execution canceled."); + } + return false; } } catch (Exception e) { @@ -209,66 +213,46 @@ public boolean run(WdlStep step, CloudgeneContext context) { @Override public void updateProgress() { + String job = makeSecretJobId(context.getJobId()); List processes = NextflowCollector.getInstance().getProcesses(job); for (NextflowProcess process : processes) { + NextflowProcessConfig config = getNextflowProcessConfig(process); + Message message = messages.get(process.getName()); if (message == null) { message = context.createTask("" + process.getName() + ""); messages.put(process.getName(), message); } - String text = "" + process.getName() + ""; - boolean running = false; - boolean ok = true; - for (NextflowTask task : process.getTasks()) { - - String status = (String) task.getTrace().get("status"); - - if (status.equals("RUNNING") || status.equals("SUBMITTED")) { - running = true; - } - if (!status.equals("COMPLETED")) { - ok = false; - - } - text += "
"; - - text += (String) task.getTrace().get("name"); - if (status.equals("RUNNING")) { - text += "..."; - } - if (status.equals("COMPLETED")) { - text += " "; - } - if (status.equals("KILLED") || status.equals("FAILED")) { - text += " "; - } + NextflowProcessRenderer.render(config, process, message); - if (task.getLog() != null) { - text += "
" + task.getLog(); - } + } - text += ""; - } - message.setMessage(text); + } - if (running) { - message.setType(Message.RUNNING); - } else { - if (ok) { - message.setType(Message.OK); - } else { - message.setType(Message.ERROR); + private void loadProcessConfigs(Object map) { + if (map != null) { + List> processConfigs = (List>) map; + for (Map processConfig : processConfigs) { + String process = processConfig.get("process").toString(); + NextflowProcessConfig config = new NextflowProcessConfig(); + if (processConfig.get("view") != null) { + config.setView(processConfig.get("view").toString()); } + configs.put(process, config); } - } } + private NextflowProcessConfig getNextflowProcessConfig(NextflowProcess process) { + NextflowProcessConfig config = configs.get(process.getName()); + return config != null ? config : new NextflowProcessConfig(); + } + private String join(List array) { String result = ""; for (int i = 0; i < array.size(); i++) { diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java index 8c3ffc73..3b6308a9 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java @@ -1,19 +1,18 @@ package cloudgene.mapred.plugins.nextflow; -import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Map; import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.workspace.IWorkspace; +import cloudgene.mapred.plugins.nextflow.report.Report; +import cloudgene.mapred.plugins.nextflow.report.ReportEvent; +import cloudgene.mapred.plugins.nextflow.report.ReportEventExecutor; import genepi.io.FileUtil; -import genepi.io.text.LineReader; public class NextflowTask { - private static final String CLOUDGENE_LOG = "cloudgene.log"; - private int id; private Map trace; @@ -34,39 +33,35 @@ public void update(Map trace) throws IOException { // if task is completed or failed check if a cloudgene.log is in workdir and // load its content + // TODO: check if CHACHED os also needed! String status = (String) trace.get("status"); - if (status.equals("COMPLETED") || status.equals("FAILED")) { - String workDir = (String) trace.get("workdir"); - String logFilename = FileUtil.path(workDir, CLOUDGENE_LOG); - IWorkspace workspace = context.getJob().getWorkspace(); - if (workspace.exists(logFilename)) { - context.log("Load log file from '" + logFilename + "'"); - InputStream stream = workspace.download(logFilename); - log = FileUtil.readFileAsString(stream); - parseFile(logFilename); - } + if (!status.equals("COMPLETED") && !status.equals("FAILED")) { + return; + } + + String workDir = (String) trace.get("workdir"); + String reportFilename = FileUtil.path(workDir, Report.DEFAULT_FILENAME); + IWorkspace workspace = context.getJob().getWorkspace(); + if (!workspace.exists(reportFilename)) { + return; + } + + context.log("Load report file from '" + reportFilename + "'"); + InputStream stream = workspace.download(reportFilename); + try { + parseReport(reportFilename); + } catch (Exception e) { + log = "Invalid report file: \n" + FileUtil.readFileAsString(stream); } } - private void parseFile(String logFilename) throws IOException { - - LineReader reader = new LineReader(new DataInputStream(context.getExternalWorkspace().download(logFilename))); - while (reader.next()) { - String line = reader.get(); - if (line.startsWith("[INC]")) { - String[] tiles = line.split(" ", 3); - String name = tiles[1]; - int value = Integer.parseInt(tiles[2]); - context.incCounter(name, value); - } - if (line.startsWith("[SUBMIT]")) { - String[] tiles = line.split(" ", 2); - String name = tiles[1]; - context.submitCounter(name); - } + private void parseReport(String reportFilename) throws IOException { + InputStream stream = context.getWorkspace().download(reportFilename); + Report report = new Report(stream); + for (ReportEvent event : report.getEvents()) { + ReportEventExecutor.execute(event, context); } - reader.close(); } public int getId() { diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/report/Report.java b/src/main/java/cloudgene/mapred/plugins/nextflow/report/Report.java new file mode 100644 index 00000000..68e87ac2 --- /dev/null +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/report/Report.java @@ -0,0 +1,110 @@ +package cloudgene.mapred.plugins.nextflow.report; + +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Type; +import java.util.List; +import java.util.Vector; + +import com.nimbusds.jose.shaded.gson.Gson; +import com.nimbusds.jose.shaded.gson.GsonBuilder; +import com.nimbusds.jose.shaded.gson.JsonIOException; +import com.nimbusds.jose.shaded.gson.JsonSyntaxException; +import com.nimbusds.jose.shaded.gson.reflect.TypeToken; + +import cloudgene.mapred.plugins.nextflow.report.ReportEvent.WebCommand; + +public class Report { + + public static final String DEFAULT_FILENAME = "cloudgene.report.json"; + + private List events = new Vector(); + + public Report() { + + } + + public Report(String filename) throws JsonIOException, JsonSyntaxException, FileNotFoundException { + loadFromFile(filename); + } + + public Report(InputStream in) throws JsonIOException, JsonSyntaxException, IOException { + loadFromInputStream(in); + } + + public void loadFromFile(String filename) throws JsonIOException, JsonSyntaxException, FileNotFoundException { + Type collectionType = new TypeToken>() { + }.getType(); + Gson gson = (new GsonBuilder()).create(); + events = gson.fromJson(new FileReader(filename), collectionType); + } + + public void loadFromInputStream(InputStream in) throws JsonIOException, JsonSyntaxException, IOException { + Type collectionType = new TypeToken>() { + }.getType(); + Gson gson = (new GsonBuilder()).create(); + events = gson.fromJson(new InputStreamReader(in), collectionType); + in.close(); + } + + public boolean hasInMemory(String content) { + for (ReportEvent event : events) { + if (event.toString().contains(content)) { + return true; + } + } + return false; + } + + public void saveToFile(String filename) throws JsonIOException, IOException { + Gson gson = (new GsonBuilder()).create(); + gson.toJson(events, new FileWriter(filename)); + } + + public List getEvents() { + return events; + } + + public void println(String line) { + addEvent(WebCommand.PRINTLN, line); + } + + public void log(String line) { + addEvent(WebCommand.LOG, line); + } + + public void incCounter(String name, int value) { + addEvent(WebCommand.INC_COUNTER, name, value); + } + + public void submitCounter(String name) { + addEvent(WebCommand.SUBMIT_COUNTER, name); + } + + public void message(String message, int type) { + addEvent(WebCommand.MESSAGE, message, type); + } + + public void beginTask(String name) { + addEvent(WebCommand.BEGIN_TASK, name); + } + + public void updateTask(String name, int type) { + addEvent(WebCommand.UPDATE_TASK, name, type); + } + + public void endTask(String message, int type) { + addEvent(WebCommand.END_TASK, message, type); + } + + public void addEvent(WebCommand command, Object... params) { + ReportEvent event = new ReportEvent(command, params); + events.add(event); + // TODO: autosave? + } + +} diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEvent.java b/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEvent.java new file mode 100644 index 00000000..6fd9ff67 --- /dev/null +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEvent.java @@ -0,0 +1,141 @@ +package cloudgene.mapred.plugins.nextflow.report; + +import cloudgene.mapred.jobs.sdk.WorkflowContext; + +public class ReportEvent { + + public enum WebCommand { + + MESSAGE, + + BEGIN_TASK, + + UPDATE_TASK, + + END_TASK, + + INC_COUNTER, + + SUBMIT_COUNTER, + + PRINTLN, + + LOG + + } + + private WebCommand command; + + private Object[] params; + + public ReportEvent(WebCommand command, Object[] params) { + super(); + this.command = command; + this.params = params; + } + + public WebCommand getCommand() { + return command; + } + + public void setCommand(WebCommand command) { + this.command = command; + } + + public Object[] getParams() { + return params; + } + + public void setParams(Object[] params) { + this.params = params; + } + + public String toString() { + return Formatter.format(this); + } + + static private class Formatter { + + public static String format(ReportEvent event) { + + switch (event.getCommand()) { + case SUBMIT_COUNTER: { + String counter = (String) event.getParams()[0]; + return submitCounter(counter); + } + case MESSAGE: { + String message = (String) event.getParams()[0]; + int type = ((Integer) event.getParams()[1]); + return message(message, type); + } + case BEGIN_TASK: { + String name = (String) event.getParams()[0]; + return message(name, WorkflowContext.RUNNING); + } + case UPDATE_TASK: { + // String name = (String) event.getParams()[0]; + // int type = ((Integer) event.getParams()[1]).intValue(); + // message(name, type); + break; + } + case LOG: { + String line = (String) event.getParams()[0]; + return log(line); + } + case PRINTLN: { + String line = (String) event.getParams()[0]; + return info(line); + } + case END_TASK: { + String name = (String) event.getParams()[0]; + int type = ((Integer) event.getParams()[1]).intValue(); + return message(name, type); + } + case INC_COUNTER: { + String counter = (String) event.getParams()[0]; + int value = ((Integer) event.getParams()[1]); + return incCounter(counter, value); + } + default: + + } + + return ""; + + } + + public static String info(String message) { + return ("[INFO] " + message); + } + + public static String log(String message) { + return ("[LOG] " + message); + } + + public static String incCounter(String counter, int value) { + return ("[INC] " + counter + " " + value); + } + + public static String submitCounter(String counter) { + return ("[SUBMIT] " + counter); + } + + public static String message(String message, int type) { + + switch (type) { + case WorkflowContext.OK: + return ("[OK] " + message); + case WorkflowContext.ERROR: + return ("[ERROR] " + message); + case WorkflowContext.WARNING: + return ("[WARN] " + message); + case WorkflowContext.RUNNING: + return ("[RUN] " + message); + default: + return ("[INFO] " + message); + } + + } + } + +} diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEventExecutor.java b/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEventExecutor.java new file mode 100644 index 00000000..07060496 --- /dev/null +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEventExecutor.java @@ -0,0 +1,57 @@ +package cloudgene.mapred.plugins.nextflow.report; + +import java.io.IOException; + +import cloudgene.mapred.jobs.sdk.WorkflowContext; + +public class ReportEventExecutor { + + public static void execute(ReportEvent event, WorkflowContext context) throws IOException { + + switch (event.getCommand()) { + case SUBMIT_COUNTER: { + String name = (String) event.getParams()[0]; + context.submitCounter(name); + break; + } + case MESSAGE: { + String message = (String) event.getParams()[0]; + int type = ((Double) event.getParams()[1]).intValue(); + context.message(message, type); + break; + } + case BEGIN_TASK: { + String name = (String) event.getParams()[0]; + context.beginTask(name); + break; + } + case UPDATE_TASK: { + String name = (String) event.getParams()[0]; + int type = ((Double) event.getParams()[1]).intValue(); + context.updateTask(name, type); + break; + } + case LOG: { + String line = (String) event.getParams()[0]; + context.log(line); + break; + } + case END_TASK: { + String name = (String) event.getParams()[0]; + int type = ((Double) event.getParams()[1]).intValue(); + context.endTask(name, type); + break; + } + case INC_COUNTER: { + String name = (String) event.getParams()[0]; + int count = ((Double) event.getParams()[1]).intValue(); + context.incCounter(name, count); + break; + } + default: + throw new IOException("Command " + event.getCommand() + " not yet supported."); + } + + } + +} diff --git a/src/main/java/cloudgene/mapred/server/services/JobService.java b/src/main/java/cloudgene/mapred/server/services/JobService.java index eea2afb7..5ad8a0bc 100644 --- a/src/main/java/cloudgene/mapred/server/services/JobService.java +++ b/src/main/java/cloudgene/mapred/server/services/JobService.java @@ -140,6 +140,7 @@ public AbstractJob submitJob(String appId, List form, User user) { name = jobName; } + //TODO: remove and solve via workspace! String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), id); FileUtil.createDirectory(localWorkspace); diff --git a/src/main/java/cloudgene/mapred/steps/BashCommandStep.java b/src/main/java/cloudgene/mapred/steps/BashCommandStep.java index 13e95973..8b13e47d 100644 --- a/src/main/java/cloudgene/mapred/steps/BashCommandStep.java +++ b/src/main/java/cloudgene/mapred/steps/BashCommandStep.java @@ -14,9 +14,9 @@ public class BashCommandStep extends CloudgeneStep { @Override public boolean run(WdlStep step, CloudgeneContext context) { - String cmd = step.get("exec"); + String cmd = step.getString("exec"); if (cmd == null) { - cmd = step.get("cmd"); + cmd = step.getString("cmd"); } if (cmd == null) { @@ -27,8 +27,8 @@ public boolean run(WdlStep step, CloudgeneContext context) { context.error("'exec' or 'cmd' parameter cannot be an empty string."); } - String bash = step.get("bash", "false"); - String stdout = step.get("stdout", "false"); + String bash = step.getString("bash", "false"); + String stdout = step.getString("stdout", "false"); boolean useBash = bash.equals("true"); boolean streamStdout = stdout.equals("true"); diff --git a/src/main/java/cloudgene/mapred/steps/GroovyStep.java b/src/main/java/cloudgene/mapred/steps/GroovyStep.java index ac4c691e..6574c84f 100644 --- a/src/main/java/cloudgene/mapred/steps/GroovyStep.java +++ b/src/main/java/cloudgene/mapred/steps/GroovyStep.java @@ -4,20 +4,26 @@ import java.io.StringWriter; import java.lang.reflect.Method; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.CloudgeneStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; import cloudgene.mapred.wdl.WdlStep; -import cloudgene.sdk.internal.WorkflowContext; import genepi.io.FileUtil; import groovy.util.GroovyScriptEngine; public class GroovyStep extends CloudgeneStep { + private static final Logger log = LoggerFactory.getLogger(GroovyStep.class); + + @Override public boolean run(WdlStep step, CloudgeneContext context) { context.setConfig(step); - String script = step.get("script"); + String script = step.getString("script"); String workingDirectory = context.getWorkingDirectory(); String filename = FileUtil.path(workingDirectory, script); @@ -37,8 +43,10 @@ public boolean run(WdlStep step, CloudgeneContext context) { } catch (Exception e) { if (e.getCause() != null) { + log.error("[Job {}] Step '{}': Error in script '{}'", context.getJobId(), step.getName(), script, e.getCause()); context.error("Error in script " + script + ":\n" + getStackTraceAsString(e.getCause())); } else { + log.error("[Job {}] Step '{}': Error in script '{}'", context.getJobId(), step.getName(), script, e); context.error("Error in script " + script + ":\n" + getStackTraceAsString(e)); } return false; diff --git a/src/main/java/cloudgene/mapred/steps/HtmlWidgetStep.java b/src/main/java/cloudgene/mapred/steps/HtmlWidgetStep.java index 332cc567..c06fb481 100644 --- a/src/main/java/cloudgene/mapred/steps/HtmlWidgetStep.java +++ b/src/main/java/cloudgene/mapred/steps/HtmlWidgetStep.java @@ -23,7 +23,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { String workingDirectory = context.getWorkingDirectory(); - String template = step.get("template"); + String template = step.getString("template"); if (template == null || template.isEmpty()) { context.endTask("Execution failed. Please set the 'template' parameter.", Message.ERROR); return false; @@ -51,7 +51,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { // add step values to context for (String variable : step.keySet()) { - String value = step.get(variable); + String value = step.getString(variable); if (value.endsWith(".json")) { // replace json files with content. String jsonFilename = ""; @@ -88,8 +88,8 @@ public boolean run(WdlStep step, CloudgeneContext context) { writer.write(""); writer.write(""); writer.write(""); - if (step.get("stylesheet") != null) { - for (String css : step.get("stylesheet").split(",")) { + if (step.getString("stylesheet") != null) { + for (String css : step.getString("stylesheet").split(",")) { css = css.trim(); String data = ""; if (!css.startsWith("http://") && !css.startsWith("https://")) { @@ -101,8 +101,8 @@ public boolean run(WdlStep step, CloudgeneContext context) { writer.write(""); } } - if (step.get("scripts") != null) { - for (String script : step.get("scripts").split(",")) { + if (step.getString("scripts") != null) { + for (String script : step.getString("scripts").split(",")) { script = script.trim(); String data = ""; if (!script.startsWith("http://") && !script.startsWith("https://")) { @@ -122,7 +122,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { writer.write(""); writer.close(); - String output = step.get("output"); + String output = step.getString("output"); if (output == null || output.isEmpty()) { context.addFile(htmlFile); @@ -131,7 +131,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { new File(htmlFile).renameTo(file); //copy assets folder if set - String assets = step.get("assets"); + String assets = step.getString("assets"); if (assets != null && !assets.isEmpty()) { File parent = file.getParentFile(); String assetsSource = FileUtil.path(workingDirectory, assets); diff --git a/src/main/java/cloudgene/mapred/steps/JavaExternalStep.java b/src/main/java/cloudgene/mapred/steps/JavaExternalStep.java index 68f663a0..6c27b508 100644 --- a/src/main/java/cloudgene/mapred/steps/JavaExternalStep.java +++ b/src/main/java/cloudgene/mapred/steps/JavaExternalStep.java @@ -49,7 +49,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { return false; } - String stdout = step.get("stdout", "false"); + String stdout = step.getString("stdout", "false"); boolean streamStdout = stdout.equals("true"); StringBuilder output = null; @@ -59,7 +59,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { String jar = step.getJar(); // params - String paramsString = step.get("params"); + String paramsString = step.getString("params"); String[] params = new String[] {}; if (paramsString != null) { params = paramsString.split(" "); diff --git a/src/main/java/cloudgene/mapred/steps/JavaInternalStep.java b/src/main/java/cloudgene/mapred/steps/JavaInternalStep.java index 168c48ae..70fbb3f3 100644 --- a/src/main/java/cloudgene/mapred/steps/JavaInternalStep.java +++ b/src/main/java/cloudgene/mapred/steps/JavaInternalStep.java @@ -2,8 +2,8 @@ import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.CloudgeneStep; +import cloudgene.mapred.jobs.sdk.WorkflowStep; import cloudgene.mapred.wdl.WdlStep; -import cloudgene.sdk.internal.WorkflowStep; public class JavaInternalStep extends CloudgeneStep { diff --git a/src/main/java/cloudgene/mapred/steps/RMarkdownDockerStep.java b/src/main/java/cloudgene/mapred/steps/RMarkdownDockerStep.java index 0ca92322..388afc8a 100644 --- a/src/main/java/cloudgene/mapred/steps/RMarkdownDockerStep.java +++ b/src/main/java/cloudgene/mapred/steps/RMarkdownDockerStep.java @@ -18,26 +18,26 @@ public boolean run(WdlStep step, CloudgeneContext context) { String workingDirectory = context.getWorkingDirectory(); - String rmd = step.get("rmd"); + String rmd = step.getString("rmd"); if (rmd == null || rmd.isEmpty()) { - rmd = step.get("rmd2"); + rmd = step.getString("rmd2"); } if (rmd == null || rmd.isEmpty()) { context.error("Execution failed. Please set the 'rmd' parameter."); return false; } - String output = step.get("output"); + String output = step.getString("output"); if (output == null || output.isEmpty()) { context.error("Execution failed. Please set the 'output' parameter."); return false; } - String image = step.get("image"); + String image = step.getString("image"); if (image == null || image.isEmpty()) { image = DOCKER_R_BASE_IMAGE; } - String paramsString = step.get("params"); + String paramsString = step.getString("params"); String[] params = new String[] {}; if (paramsString != null) { params = paramsString.split(" "); diff --git a/src/main/java/cloudgene/mapred/steps/RMarkdownLocalStep.java b/src/main/java/cloudgene/mapred/steps/RMarkdownLocalStep.java index 1380eac6..9e2e08a9 100644 --- a/src/main/java/cloudgene/mapred/steps/RMarkdownLocalStep.java +++ b/src/main/java/cloudgene/mapred/steps/RMarkdownLocalStep.java @@ -22,21 +22,21 @@ public boolean run(WdlStep step, CloudgeneContext context) { String workingDirectory = context.getWorkingDirectory(); - String rmd = step.get("rmd"); + String rmd = step.getString("rmd"); if (rmd == null || rmd.isEmpty()) { - rmd = step.get("rmd2"); + rmd = step.getString("rmd2"); } if (rmd == null || rmd.isEmpty()) { context.endTask("Execution failed. Please set the 'rmd' parameter.", Message.ERROR); return false; } - String output = step.get("output"); + String output = step.getString("output"); if (output == null || output.isEmpty()) { context.endTask("Execution failed. Please set the 'output' parameter.", Message.ERROR); return false; } - String paramsString = step.get("params"); + String paramsString = step.getString("params"); String[] params = new String[] {}; if (paramsString != null) { params = paramsString.split(" "); @@ -55,7 +55,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { if (result == 0) { context.endTask("Execution successful.", Message.OK); - String include = step.get("include"); + String include = step.getString("include"); if (include != null && include.equals("true")) { context.addFile(output); } diff --git a/src/main/java/cloudgene/mapred/steps/ReturnTrueStep.java b/src/main/java/cloudgene/mapred/steps/ReturnTrueStep.java index c529aa90..4de63c8d 100644 --- a/src/main/java/cloudgene/mapred/steps/ReturnTrueStep.java +++ b/src/main/java/cloudgene/mapred/steps/ReturnTrueStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class ReturnTrueStep extends WorkflowStep { diff --git a/src/main/java/cloudgene/mapred/wdl/WdlStep.java b/src/main/java/cloudgene/mapred/wdl/WdlStep.java index 9af7d15e..9bde046f 100644 --- a/src/main/java/cloudgene/mapred/wdl/WdlStep.java +++ b/src/main/java/cloudgene/mapred/wdl/WdlStep.java @@ -2,14 +2,14 @@ import java.util.HashMap; -public class WdlStep extends HashMap{ +public class WdlStep extends HashMap{ public String getName() { - return get("name"); + return getString("name"); } public String getClassname() { - return get("classname"); + return getString("classname"); } public void setClassname(String classname){ @@ -18,20 +18,20 @@ public void setClassname(String classname){ public String getJar() { - return get("jar"); + return getString("jar"); } public String getGenerates() { - return get("generates"); + return getString("generates"); } - public String get(String key, String defaultValue){ - String value = get(key); - if (value == null){ - return defaultValue; - }else{ - return value; - } + public String getString(String key){ + return getString(key, null); + } + + public String getString(String key, String defaultValue){ + Object value = get(key); + return value != null ? value.toString() : defaultValue; } } diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java index 45e392e1..6f94b56a 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java @@ -10,8 +10,8 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.jobs.AbstractJob; +import cloudgene.mapred.jobs.sdk.WorkflowContext; import cloudgene.mapred.util.CloudgeneClientRestAssured; -import cloudgene.sdk.internal.WorkflowContext; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.restassured.RestAssured; diff --git a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java index 064b66af..0233a554 100644 --- a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java +++ b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java @@ -13,11 +13,11 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.jobs.sdk.WorkflowContext; import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; -import cloudgene.sdk.internal.WorkflowContext; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; diff --git a/src/test/java/cloudgene/mapred/jobs/steps/CheckInputs.java b/src/test/java/cloudgene/mapred/jobs/steps/CheckInputs.java index 87f71130..0252466a 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/CheckInputs.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/CheckInputs.java @@ -2,8 +2,8 @@ import java.io.File; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; import genepi.io.FileUtil; public class CheckInputs extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/LongSleepStep.java b/src/test/java/cloudgene/mapred/jobs/steps/LongSleepStep.java index 06c46a86..a22170bc 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/LongSleepStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/LongSleepStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class LongSleepStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/PrintDataset.java b/src/test/java/cloudgene/mapred/jobs/steps/PrintDataset.java index 148dfaa9..5231e8de 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/PrintDataset.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/PrintDataset.java @@ -2,8 +2,8 @@ import java.util.Map; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class PrintDataset extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/ReturnExceptionStep.java b/src/test/java/cloudgene/mapred/jobs/steps/ReturnExceptionStep.java index 516d5e54..9aafa0f0 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/ReturnExceptionStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/ReturnExceptionStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class ReturnExceptionStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/ReturnFalseStep.java b/src/test/java/cloudgene/mapred/jobs/steps/ReturnFalseStep.java index 43721111..007f08b5 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/ReturnFalseStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/ReturnFalseStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class ReturnFalseStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/ReturnTrueStep.java b/src/test/java/cloudgene/mapred/jobs/steps/ReturnTrueStep.java index 1012c777..39107831 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/ReturnTrueStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/ReturnTrueStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class ReturnTrueStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/ThreeTasksStep.java b/src/test/java/cloudgene/mapred/jobs/steps/ThreeTasksStep.java index 0149fe72..c7537f39 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/ThreeTasksStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/ThreeTasksStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class ThreeTasksStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToFolderStep.java b/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToFolderStep.java index 373db16a..6ca81bf1 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToFolderStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToFolderStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; import genepi.io.FileUtil; public class WriteFilesToFolderStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToFileStep.java b/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToFileStep.java index 2553312b..4f5dd1dd 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToFileStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToFileStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; import genepi.io.FileUtil; public class WriteTextToFileStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToStdOutStep.java b/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToStdOutStep.java index 4726ad7f..32f4e4bd 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToStdOutStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToStdOutStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class WriteTextToStdOutStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/steps/TestCommand.java b/src/test/java/cloudgene/mapred/steps/TestCommand.java index d426e82f..50cdcfe0 100644 --- a/src/test/java/cloudgene/mapred/steps/TestCommand.java +++ b/src/test/java/cloudgene/mapred/steps/TestCommand.java @@ -16,11 +16,11 @@ import cloudgene.mapred.jobs.CloudgeneJob; import cloudgene.mapred.jobs.Message; import cloudgene.mapred.jobs.WorkflowEngine; +import cloudgene.mapred.jobs.sdk.WorkflowContext; import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; -import cloudgene.sdk.internal.WorkflowContext; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; diff --git a/test-data/print-hdfs-file-dataset.groovy b/test-data/print-hdfs-file-dataset.groovy deleted file mode 100644 index 28236d33..00000000 --- a/test-data/print-hdfs-file-dataset.groovy +++ /dev/null @@ -1,17 +0,0 @@ -import cloudgene.sdk.internal.WorkflowContext -import genepi.hadoop.HdfsUtil - -def run(WorkflowContext context) { - - def dataset = context.getData("dataset"); - def hdfs = dataset.get("metafile"); - def tempFile = context.getLocalTemp()+"/file.txt"; - - //export - HdfsUtil.get(hdfs, tempFile); - - def content = new File(tempFile).text; - context.ok(content); - - return true; -} diff --git a/test-data/print-hdfs-file.groovy b/test-data/print-hdfs-file.groovy deleted file mode 100644 index 62d454de..00000000 --- a/test-data/print-hdfs-file.groovy +++ /dev/null @@ -1,16 +0,0 @@ -import cloudgene.sdk.internal.WorkflowContext -import genepi.hadoop.HdfsUtil - -def run(WorkflowContext context) { - - def hdfs = context.getConfig("file"); - def tempFile = context.getLocalTemp()+"/file.txt"; - - //export - HdfsUtil.get(hdfs, tempFile); - - def content = new File(tempFile).text; - context.ok(content); - - return true; -} diff --git a/test-data/print-properties.groovy b/test-data/print-properties.groovy index 6bbbc198..7c495447 100644 --- a/test-data/print-properties.groovy +++ b/test-data/print-properties.groovy @@ -1,4 +1,4 @@ -import cloudgene.sdk.internal.WorkflowContext +import cloudgene.mapred.jobs.sdk.WorkflowContext def run(WorkflowContext context) { From a2e9a8ce43bb0046399833e5a7fb9ad16e923575 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 7 Sep 2023 12:37:55 +0200 Subject: [PATCH 11/35] Refactore NextflowCommand and NextflowCollector --- .../cloudgene/mapred/jobs/AbstractJob.java | 17 +- .../mapred/jobs/CloudgeneContext.java | 4 + .../cloudgene/mapred/jobs/engine/Planner.java | 9 +- .../engine/plugins/ParameterValueInput.java | 63 ------ .../engine/plugins/ParameterValueOutput.java | 62 ------ .../plugins/nextflow/NextflowBinary.java | 131 +++++++++++ .../plugins/nextflow/NextflowCollector.java | 42 ++-- .../mapred/plugins/nextflow/NextflowStep.java | 204 +++++++----------- 8 files changed, 264 insertions(+), 268 deletions(-) delete mode 100644 src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueInput.java delete mode 100644 src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueOutput.java diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java index 25b931ca..1b82f107 100644 --- a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java +++ b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java @@ -14,6 +14,7 @@ import java.util.Map; import java.util.Vector; +import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,6 +23,7 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.jobs.queue.PriorityRunnable; import cloudgene.mapred.jobs.workspace.IWorkspace; +import cloudgene.mapred.util.HashUtil; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlParameterInputType; import genepi.io.FileUtil; @@ -122,12 +124,15 @@ abstract public class AbstractJob extends PriorityRunnable { private String workspaceSize = null; + private String publicJobId; + public String getId() { return id; } public void setId(String id) { this.id = id; + this.publicJobId = HashUtil.getSha256(id + RandomStringUtils.random(500)); } public int getState() { @@ -340,11 +345,11 @@ public void runInstallationAndResolveAppLinks() { @Override public void run() { - + if (isCanceld()) { return; } - + log.info("[Job {}] Setup job...", getId()); setState(AbstractJob.STATE_RUNNING); @@ -354,12 +359,12 @@ public void run() { setSetupEndTime(System.currentTimeMillis()); setSetupRunning(false); if (!isSetupComplete()) { - log.info("[Job {}] Setup failed." , getId()); + log.info("[Job {}] Setup failed.", getId()); setFinishedOn(System.currentTimeMillis()); setComplete(true); return; } - + log.info("[Job {}] Running job...", getId()); setStartTime(System.currentTimeMillis()); @@ -691,4 +696,8 @@ public void setCurrentTime(long time) { } + public String getPublicJobId() { + return publicJobId; + } + } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java index fc27fc2e..9ce2095d 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java @@ -95,6 +95,10 @@ public String getJobId() { return job.getId(); } + public String getPublicJobId() { + return job.getPublicJobId(); + } + public String getOutput(String param) { if (outputParameters.get(param) != null) { diff --git a/src/main/java/cloudgene/mapred/jobs/engine/Planner.java b/src/main/java/cloudgene/mapred/jobs/engine/Planner.java index 89db8fe9..cade59b4 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/Planner.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/Planner.java @@ -6,8 +6,6 @@ import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.Environment; -import cloudgene.mapred.jobs.engine.plugins.ParameterValueInput; -import cloudgene.mapred.jobs.engine.plugins.ParameterValueOutput; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlParameterInput; @@ -47,7 +45,12 @@ public WdlApp evaluateWDL(WdlApp app, CloudgeneContext context, Settings setting context2.put("user_email", context.getUser().getMail()); context2.put("user_name", context.getUser().getUsername()); context2.put("user_full_name", context.getUser().getFullName()); - + + // add service specific variables + context2.put("service_name", settings.getName()); + context2.put("service_email", settings.getAdminMail()); + context2.put("service_url", settings.getServerUrl()); + File manifest = new File(app.getManifestFile()); SimpleTemplateEngine engine = new SimpleTemplateEngine(); diff --git a/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueInput.java b/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueInput.java deleted file mode 100644 index 203a2bc7..00000000 --- a/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueInput.java +++ /dev/null @@ -1,63 +0,0 @@ -package cloudgene.mapred.jobs.engine.plugins; - -import java.io.File; - -import cloudgene.mapred.wdl.WdlParameterInput; -import cloudgene.mapred.wdl.WdlParameterInputType; -import genepi.io.FileUtil; -import genepi.io.WildCardFileFilter; - -public class ParameterValueInput { - - private WdlParameterInput parameter; - private String value; - - public ParameterValueInput(WdlParameterInput parameter, String value) { - this.parameter = parameter; - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public String[] listFiles(String ext) { - - if (parameter.getTypeAsEnum() == WdlParameterInputType.LOCAL_FOLDER) { - return getFiles(value, ext); - - } - - return new String[] { value }; - - } - - private static String[] getFiles(String path, String ext) { - File dir = new File(path); - File[] files = dir.listFiles(new WildCardFileFilter(ext)); - - if (files != null) { - - String[] names = new String[files.length]; - - for (int i = 0; i < names.length; ++i) { - names[i] = files[i].getName(); - } - return names; - - - } else { - String[] names = new String[1]; - names[0] = FileUtil.getFilename(path); - return names; - - } - } - - public String getName() { - - return FileUtil.getFilename(value); - - } -} diff --git a/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueOutput.java b/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueOutput.java deleted file mode 100644 index 63f5ec77..00000000 --- a/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueOutput.java +++ /dev/null @@ -1,62 +0,0 @@ -package cloudgene.mapred.jobs.engine.plugins; - -import java.io.File; - -import cloudgene.mapred.wdl.WdlParameterOutput; -import cloudgene.mapred.wdl.WdlParameterOutputType; -import genepi.io.FileUtil; -import genepi.io.WildCardFileFilter; - -public class ParameterValueOutput { - - private WdlParameterOutput parameter; - private String value; - - public ParameterValueOutput(WdlParameterOutput parameter, String value) { - this.parameter = parameter; - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public String[] listFiles(String ext) { - - if (parameter.getTypeAsEnum() == WdlParameterOutputType.LOCAL_FOLDER) { - return getFiles(value, ext); - - } - - return new String[] { value }; - - } - - private static String[] getFiles(String path, String ext) { - File dir = new File(path); - File[] files = dir.listFiles(new WildCardFileFilter(ext)); - - if (files != null) { - - String[] names = new String[files.length]; - - for (int i = 0; i < names.length; ++i) { - names[i] = files[i].getName(); - } - return names; - - } else { - String[] names = new String[1]; - names[0] = FileUtil.getFilename(path); - return names; - - } - } - - public String getName() { - - return FileUtil.getFilename(value); - - } -} diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java index 4126d276..4992c577 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java @@ -1,6 +1,8 @@ package cloudgene.mapred.plugins.nextflow; import java.io.File; +import java.util.List; +import java.util.Vector; import cloudgene.mapred.util.BinaryFinder; import cloudgene.mapred.util.Settings; @@ -11,6 +13,26 @@ public class NextflowBinary { private String binary = ""; + private String script; + + private String profile; + + private File nextflowConfigFile; + + private String work; + + private File paramsFile; + + private String weblog; + + private String trace; + + private String report; + + private String timeline; + + private String log; + public static NextflowBinary build(Settings settings) { String binary = new BinaryFinder("nextflow").settings(settings, "nextflow", "home").env("NEXTFLOW_HOME") .envPath().path("/usr/local/bin").find(); @@ -47,4 +69,113 @@ public String getVersion() { return "Nextflow not installed."; } } + + public void setBinary(String binary) { + this.binary = binary; + } + + public void setScript(String script) { + this.script = script; + } + + public void setProfile(String profile) { + this.profile = profile; + } + + public void setNextflowConfigFile(File nextflowConfigFile) { + this.nextflowConfigFile = nextflowConfigFile; + } + + public void setWork(String work) { + this.work = work; + } + + public void setParamsFile(File paramsFile) { + this.paramsFile = paramsFile; + } + + public void setWeblog(String weblog) { + this.weblog = weblog; + } + + public void setTrace(String trace) { + this.trace = trace; + } + + public void setReport(String report) { + this.report = report; + } + + public void setTimeline(String timeline) { + this.timeline = timeline; + } + + public void setLog(String log) { + this.log = log; + } + + public List buildCommand() { + + List nextflow = new Vector(); + nextflow.add("PATH=$PATH:/usr/local/bin"); + nextflow.add(getBinary()); + nextflow.add("run"); + nextflow.add(script); + + // set profile + if (profile != null && !profile.isEmpty()) { + nextflow.add("-profile"); + nextflow.add(profile); + } + + if (nextflowConfigFile.exists()) { + // set custom configuration + nextflow.add("-c"); + nextflow.add(nextflowConfigFile.getAbsolutePath()); + } + + nextflow.add("-w"); + nextflow.add(work); + + nextflow.add("-params-file"); + nextflow.add(paramsFile.getAbsolutePath()); + + nextflow.add("-ansi-log"); + nextflow.add("false"); + + nextflow.add("-with-weblog"); + nextflow.add(weblog); + + // nextflowCommand.add("-log"); + // nextflowCommand.add(log); + + nextflow.add("-with-trace"); + nextflow.add(trace); + + nextflow.add("-with-report"); + nextflow.add(report); + + nextflow.add("-with-timeline"); + nextflow.add(timeline); + + List command = new Vector(); + command.add("/bin/bash"); + command.add("-c"); + command.add(join(nextflow)); + + return command; + + } + + private String join(List array) { + String result = ""; + for (int i = 0; i < array.size(); i++) { + if (i > 0) { + result += " "; + } + result += array.get(i); + } + return result; + } + } diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowCollector.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowCollector.java index db0ec49b..b20bd574 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowCollector.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowCollector.java @@ -6,16 +6,24 @@ import java.util.Map; import java.util.Vector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import cloudgene.mapred.jobs.CloudgeneContext; +import cloudgene.mapred.util.Settings; public class NextflowCollector { - + + private static final String COLLECTOR_ENDPOINT = "/api/v2/collect/"; + private static NextflowCollector instance; private Map> data; private Map contexts; + private static final Logger log = LoggerFactory.getLogger(NextflowCollector.class); + public static NextflowCollector getInstance() { if (instance == null) { instance = new NextflowCollector(); @@ -29,19 +37,22 @@ private NextflowCollector() { data = new HashMap>(); } - public void addContext(String job, CloudgeneContext context) { - contexts.put(job, context); + public String addContext(CloudgeneContext context) { + contexts.put(context.getPublicJobId(), context); + Settings settings = context.getSettings(); + log.info("[Job {}] Register collector for public job id '{}'", context.getJobId(), context.getPublicJobId()); + return settings.getServerUrl() + settings.getUrlPrefix() + COLLECTOR_ENDPOINT + context.getPublicJobId(); } public void addEvent(String job, Map event) throws IOException { - + CloudgeneContext context = contexts.get(job); - + if (context == null) { - System.out.println("Warning! No context found for job " + job); - return; + log.info("Warning! No context found for public job id '{}'", job); + return; } - + List processes = data.get(job); if (processes == null) { processes = new Vector(); @@ -68,14 +79,19 @@ public void addEvent(String job, Map event) throws IOException { processes.add(process); } - public List getProcesses(String job) { - List processes = data.get(job); + public List getProcesses(CloudgeneContext context) { + List processes = data.get(context.getPublicJobId()); if (processes == null) { return new Vector(); } - { - return processes; - } + return processes; + + } + + public void cleanProcesses(CloudgeneContext context) { + data.remove(context.getPublicJobId()); + contexts.remove(context.getPublicJobId()); + log.info("[Job {}] Removed collector for public job id '{}'", context.getJobId(), context.getPublicJobId()); } } diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java index 22f9ebdf..dfb4d17f 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java @@ -7,14 +7,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Vector; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; import cloudgene.mapred.jobs.workspace.IWorkspace; -import cloudgene.mapred.util.HashUtil; import cloudgene.mapred.wdl.WdlStep; import genepi.io.FileUtil; import groovy.json.JsonOutput; @@ -29,6 +30,10 @@ public class NextflowStep extends CloudgeneStep { private Map configs = new HashMap(); + private NextflowCollector collector = NextflowCollector.getInstance(); + + private static final Logger log = LoggerFactory.getLogger(NextflowStep.class); + @Override public boolean run(WdlStep step, CloudgeneContext context) { @@ -51,142 +56,73 @@ public boolean run(WdlStep step, CloudgeneContext context) { loadProcessConfigs(step.get(PROPERTY_PROCESS_CONFIG)); NextflowBinary nextflow = NextflowBinary.build(context.getSettings()); - // TODO: move to bextflow binary. see nftest implementation - List nextflowCommand = new Vector(); - nextflowCommand.add("PATH=$PATH:/usr/local/bin"); - nextflowCommand.add(nextflow.getBinary()); - nextflowCommand.add("run"); - nextflowCommand.add(script); + nextflow.setScript(script); AbstractJob job = context.getJob(); String appFolder = context.getSettings().getApplicationRepository().getConfigDirectory(job.getApplicationId()); + // set profile String profile = ""; String nextflowProfile = FileUtil.path(appFolder, "nextflow.profile"); if (new File(nextflowProfile).exists()) { profile = FileUtil.readFileAsString(nextflowProfile); } + nextflow.setProfile(profile); - // set profile - if (!profile.isEmpty()) { - nextflowCommand.add("-profile"); - nextflowCommand.add(profile); - } - + // set custom configuration String nextflowConfig = FileUtil.path(appFolder, "nextflow.config"); File nextflowConfigFile = new File(nextflowConfig); - if (nextflowConfigFile.exists()) { - // set custom configuration - nextflowCommand.add("-c"); - nextflowCommand.add(nextflowConfigFile.getAbsolutePath()); - } + nextflow.setNextflowConfigFile(nextflowConfigFile); + // set work directory String work = ""; String nextflowWork = FileUtil.path(appFolder, "nextflow.work"); if (new File(nextflowWork).exists()) { work = FileUtil.readFileAsString(nextflowWork); } + IWorkspace workspace = job.getWorkspace(); + // use workdir if set in settings if (!work.trim().isEmpty()) { - nextflowCommand.add("-w"); - nextflowCommand.add(work); + nextflow.setWork(work); } else { - IWorkspace workspace = job.getWorkspace(); String workDir = workspace.createTempFolder("nextflow"); - nextflowCommand.add("-w"); - nextflowCommand.add(workDir); - } - - Map params = new HashMap(); - - // used to defined hard coded params - for (String key : step.keySet()) { - if (key.startsWith("params.")) { - String param = key.replace("params.", ""); - Object value = step.get(key); - params.put(param, value); - } - } - if (step.get("params") != null) { - Map paramsMap = (Map) step.get("params"); - params.putAll(paramsMap); - } - - // add all inputs - for (String param : context.getInputs()) { - String value = context.getInput(param); - // resolve app links: use all properties as input parameters - if (value.startsWith("apps@")) { - Map linkedApp = (Map) context.getData(param); - params.put(param, linkedApp); - } else { - params.put(param, value); - } - - } - - // add all outputs - for (String param : context.getOutputs()) { - String value = context.getOutput(param); - params.put(param, value); + nextflow.setWork(workDir); } + //params json file String paramsJsonFilename = FileUtil.path(context.getLocalOutput(), "params.json"); File paramsFile = new File(paramsJsonFilename); - try { + Map params = createParamsMap(step); + // TODO: workspace? writeParamsJson(params, paramsFile); - } catch (IOException e1) { - e1.printStackTrace(); + } catch (IOException e) { + log.error("[Job {}] Writing params.json file failed.", context.getJobId(), e); return false; } + nextflow.setParamsFile(paramsFile); + + //register job in webcollector and set created url + String collectorUrl = collector.addContext(context); + nextflow.setWeblog(collectorUrl); - IWorkspace workspace = job.getWorkspace(); - - nextflowCommand.add("-params-file"); - nextflowCommand.add(paramsFile.getAbsolutePath()); - - nextflowCommand.add("-ansi-log"); - nextflowCommand.add("false"); - - nextflowCommand.add("-with-weblog"); - nextflowCommand.add(context.getSettings().getServerUrl() + context.getSettings().getUrlPrefix() - + "/api/v2/collect/" + makeSecretJobId(context.getJobId())); - - // nextflowCommand.add("-log"); - // nextflowCommand.add(workspace.createLogFile("nextflow.log")); - - nextflowCommand.add("-with-trace"); - nextflowCommand.add(workspace.createLogFile("trace.csv")); - - nextflowCommand.add("-with-report"); - nextflowCommand.add(workspace.createLogFile("report.html")); - - nextflowCommand.add("-with-timeline"); - nextflowCommand.add(workspace.createLogFile("timeline.html")); - - StringBuilder output = new StringBuilder(); - - List command = new Vector(); - command.add("/bin/bash"); - command.add("-c"); - command.add(join(nextflowCommand)); - - NextflowCollector.getInstance().addContext(makeSecretJobId(context.getJobId()), context); + //log files and reports + nextflow.setTrace(workspace.createLogFile("trace.csv")); + nextflow.setReport(workspace.createLogFile("report.html")); + nextflow.setTimeline(workspace.createLogFile("timeline.html")); + nextflow.setLog(workspace.createLogFile("nextflow.log")); try { - boolean successful = executeCommand(command, context, output); + StringBuilder output = new StringBuilder(); + boolean successful = executeCommand(nextflow.buildCommand(), context, output); - if (successful) { - updateProgress(); - return true; - } else { + if (!successful) { // set all running processes to failed - List processes = NextflowCollector.getInstance() - .getProcesses(makeSecretJobId(context.getJobId())); + List processes = collector.getProcesses(context); for (NextflowProcess process : processes) { for (NextflowTask task : process.getTasks()) { String status = (String) task.getTrace().get("status"); @@ -202,10 +138,16 @@ public boolean run(WdlStep step, CloudgeneContext context) { context.error("Pipeline execution canceled."); } - return false; } + + updateProgress(); + + collector.cleanProcesses(context); + + return successful; + } catch (Exception e) { - e.printStackTrace(); + log.error("[Job {}] Running nextflow script failed.", context.getJobId(), e); return false; } @@ -214,9 +156,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { @Override public void updateProgress() { - String job = makeSecretJobId(context.getJobId()); - - List processes = NextflowCollector.getInstance().getProcesses(job); + List processes = collector.getProcesses(context); for (NextflowProcess process : processes) { @@ -253,36 +193,54 @@ private NextflowProcessConfig getNextflowProcessConfig(NextflowProcess process) return config != null ? config : new NextflowProcessConfig(); } - private String join(List array) { - String result = ""; - for (int i = 0; i < array.size(); i++) { - if (i > 0) { - result += " "; - } - result += array.get(i); - } - return result; - } - @Override public String[] getRequirements() { return new String[] { NextflowPlugin.ID }; } + + private Map createParamsMap(WdlStep step) { + Map params = new HashMap(); - public String makeSecretJobId(String job) { - return HashUtil.getSha256(job); - } + // used to defined hard coded params + for (String key : step.keySet()) { + if (key.startsWith("params.")) { + String param = key.replace("params.", ""); + Object value = step.get(key); + params.put(param, value); + } + } + if (step.get("params") != null) { + Map paramsMap = (Map) step.get("params"); + params.putAll(paramsMap); + } + + // add all inputs + for (String param : context.getInputs()) { + String value = context.getInput(param); + // resolve app links: use all properties as input parameters + if (value.startsWith("apps@")) { + Map linkedApp = (Map) context.getData(param); + params.put(param, linkedApp); + } else { + params.put(param, value); + } + + } + + // add all outputs + for (String param : context.getOutputs()) { + String value = context.getOutput(param); + params.put(param, value); + } + + return params; - private String makeRed(String text) { - return ((char) 27 + "[31m" + text + (char) 27 + "[0m"); } protected void writeParamsJson(Map params, File paramsFile) throws IOException { - BufferedWriter writer = new BufferedWriter(new FileWriter(paramsFile)); writer.write(JsonOutput.toJson(params)); writer.close(); - } } From d48dfe30f8250dbe3d012cfffcc8045e4fddd276 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 7 Sep 2023 13:37:15 +0200 Subject: [PATCH 12/35] Run Nextflow jobs in workspace directory --- .../java/cloudgene/mapred/jobs/CloudgeneStep.java | 12 +++++++++--- .../mapred/plugins/nextflow/NextflowStep.java | 6 ++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java index 8a0094ff..3d2436a1 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java @@ -98,6 +98,12 @@ protected boolean executeCommand(List command, WorkflowContext context) protected boolean executeCommand(List command, WorkflowContext context, StringBuilder output) throws IOException, InterruptedException { + File workDir = new File(context.getWorkingDirectory()); + return executeCommand(command, context, output, workDir); + } + + protected boolean executeCommand(List command, WorkflowContext context, StringBuilder output, File workDir) + throws IOException, InterruptedException { // set global variables for (int j = 0; j < command.size(); j++) { @@ -106,10 +112,10 @@ protected boolean executeCommand(List command, WorkflowContext context, } context.log("Command: " + command); - context.log("Working Directory: " + new File(context.getWorkingDirectory()).getAbsolutePath()); + context.log("Working Directory: " + workDir.getAbsolutePath()); ProcessBuilder builder = new ProcessBuilder(command); - builder.directory(new File(context.getWorkingDirectory())); + builder.directory(workDir); builder.redirectErrorStream(true); builder.redirectOutput(); process = builder.start(); @@ -126,7 +132,7 @@ protected boolean executeCommand(List command, WorkflowContext context, } } } catch (Exception e) { - //e.printStackTrace(); + // e.printStackTrace(); } br.close(); diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java index dfb4d17f..e3d5e62c 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java @@ -56,7 +56,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { loadProcessConfigs(step.get(PROPERTY_PROCESS_CONFIG)); NextflowBinary nextflow = NextflowBinary.build(context.getSettings()); - nextflow.setScript(script); + nextflow.setScript(scriptPath); AbstractJob job = context.getJob(); String appFolder = context.getSettings().getApplicationRepository().getConfigDirectory(job.getApplicationId()); @@ -116,8 +116,10 @@ public boolean run(WdlStep step, CloudgeneContext context) { try { + File executionDir = new File(context.getLocalOutput()); + StringBuilder output = new StringBuilder(); - boolean successful = executeCommand(nextflow.buildCommand(), context, output); + boolean successful = executeCommand(nextflow.buildCommand(), context, output, executionDir); if (!successful) { From abe27888d3a40c8512d9fd577772f4592de32057 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Thu, 7 Sep 2023 17:53:52 +0200 Subject: [PATCH 13/35] Fix missing command in report --- .../plugins/nextflow/NextflowProcessRenderer.java | 4 ++-- .../mapred/plugins/nextflow/NextflowTask.java | 14 ++++++++++---- .../nextflow/report/ReportEventExecutor.java | 5 +++++ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java index c8f1c540..632e6b27 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java @@ -45,8 +45,8 @@ public static void renderList(NextflowProcess process, Message message) { text += " "; } - if (task.getLog() != null) { - text += "
" + task.getLog(); + if (task.getLogText() != null) { + text += "
" + task.getLogText(); } text += ""; diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java index 3b6308a9..24de0616 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java @@ -4,6 +4,9 @@ import java.io.InputStream; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.workspace.IWorkspace; import cloudgene.mapred.plugins.nextflow.report.Report; @@ -17,10 +20,12 @@ public class NextflowTask { private Map trace; - private String log = null; + private String logText = null; private CloudgeneContext context; + private static final Logger log = LoggerFactory.getLogger(NextflowTask.class); + public NextflowTask(CloudgeneContext context, Map trace) { id = (Integer) trace.get("task_id"); this.trace = trace; @@ -51,7 +56,8 @@ public void update(Map trace) throws IOException { try { parseReport(reportFilename); } catch (Exception e) { - log = "Invalid report file: \n" + FileUtil.readFileAsString(stream); + log.error("[Job {}] Invalid report file.", e); + logText = "Invalid report file: \n" + FileUtil.readFileAsString(stream); } } @@ -72,8 +78,8 @@ public Map getTrace() { return trace; } - public String getLog() { - return log; + public String getLogText() { + return logText; } } diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEventExecutor.java b/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEventExecutor.java index 07060496..ecd7d1ac 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEventExecutor.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEventExecutor.java @@ -48,6 +48,11 @@ public static void execute(ReportEvent event, WorkflowContext context) throws IO context.incCounter(name, count); break; } + case PRINTLN: { + String line = (String) event.getParams()[0]; + context.println(line); + break; + } default: throw new IOException("Command " + event.getCommand() + " not yet supported."); } From 772b0e68dedec86f2bae8cf9633588570e839537 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Fri, 8 Sep 2023 17:16:48 +0200 Subject: [PATCH 14/35] Export environment variables when run Nextflow --- .../cloudgene/mapred/jobs/CloudgeneStep.java | 7 ++--- .../cloudgene/mapred/jobs/engine/Planner.java | 17 +++++------- .../mapred/plugins/nextflow/NextflowStep.java | 2 +- .../java/cloudgene/mapred/util/Settings.java | 26 +++++++++++++++++++ 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java index 3d2436a1..fc3c7c42 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java @@ -91,18 +91,18 @@ public CloudgeneContext getup() { return null; } - protected boolean executeCommand(List command, WorkflowContext context) + protected boolean executeCommand(List command, CloudgeneContext context) throws IOException, InterruptedException { return executeCommand(command, context, null); } - protected boolean executeCommand(List command, WorkflowContext context, StringBuilder output) + protected boolean executeCommand(List command, CloudgeneContext context, StringBuilder output) throws IOException, InterruptedException { File workDir = new File(context.getWorkingDirectory()); return executeCommand(command, context, output, workDir); } - protected boolean executeCommand(List command, WorkflowContext context, StringBuilder output, File workDir) + protected boolean executeCommand(List command, CloudgeneContext context, StringBuilder output, File workDir) throws IOException, InterruptedException { // set global variables for (int j = 0; j < command.size(); j++) { @@ -115,6 +115,7 @@ protected boolean executeCommand(List command, WorkflowContext context, context.log("Working Directory: " + workDir.getAbsolutePath()); ProcessBuilder builder = new ProcessBuilder(command); + builder.environment().putAll(context.getSettings().getEnvironment()); builder.directory(workDir); builder.redirectErrorStream(true); builder.redirectOutput(); diff --git a/src/main/java/cloudgene/mapred/jobs/engine/Planner.java b/src/main/java/cloudgene/mapred/jobs/engine/Planner.java index cade59b4..5bc51360 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/Planner.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/Planner.java @@ -41,16 +41,13 @@ public WdlApp evaluateWDL(WdlApp app, CloudgeneContext context, Settings setting context2.put(key, envApp.get(key)); } - // add user specific variables - context2.put("user_email", context.getUser().getMail()); - context2.put("user_name", context.getUser().getUsername()); - context2.put("user_full_name", context.getUser().getFullName()); - - // add service specific variables - context2.put("service_name", settings.getName()); - context2.put("service_email", settings.getAdminMail()); - context2.put("service_url", settings.getServerUrl()); - + context2.putAll(settings.getEnvironment()); + context2.put("CLOUDGENE_JOB_ID", context.getJobId()); + context2.put("CLOUDGENE_USER_NAME", context.getUser().getUsername()); + context2.put("CLOUDGENE_USER_EMAIL", context.getUser().getMail()); + context2.put("CLOUDGENE_USER_FULL_NAME", context.getUser().getFullName()); + + File manifest = new File(app.getManifestFile()); SimpleTemplateEngine engine = new SimpleTemplateEngine(); diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java index e3d5e62c..eb05ad4b 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java @@ -54,7 +54,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { // load process styling loadProcessConfigs(step.get(PROPERTY_PROCESS_CONFIG)); - + NextflowBinary nextflow = NextflowBinary.build(context.getSettings()); nextflow.setScript(scriptPath); diff --git a/src/main/java/cloudgene/mapred/util/Settings.java b/src/main/java/cloudgene/mapred/util/Settings.java index 2b19eb4f..6aaa3cd3 100644 --- a/src/main/java/cloudgene/mapred/util/Settings.java +++ b/src/main/java/cloudgene/mapred/util/Settings.java @@ -69,6 +69,8 @@ public class Settings { private String adminMail = null; + private String adminName = null; + private String urlPrefix = ""; private List navigation = new Vector(); @@ -387,6 +389,14 @@ public String getAdminMail() { return adminMail; } + public void setAdminName(String adminName) { + this.adminName = adminName; + } + + public String getAdminName() { + return adminName; + } + public void setThreadsQueue(int threadsQueue) { this.threadsQueue = threadsQueue; } @@ -557,4 +567,20 @@ public String getServerUrl() { return serverUrl; } + public Map getEnvironment() { + Map map = new HashMap(); + map.put("CLOUDGENE_SERVICE_NAME", name != null ? name : ""); + map.put("CLOUDGENE_SERVICE_URL", serverUrl != null ? serverUrl : ""); + map.put("CLOUDGENE_CONTACT_EMAIL", adminMail != null ? adminMail : ""); + map.put("CLOUDGENE_CONTACT_NAME", adminName != null ? adminName : ""); + if (mail != null) { + map.put("CLOUDGENE_SMTP_HOST", mail.get("smtp") != null ? mail.get("smtp") : ""); + map.put("CLOUDGENE_SMTP_PORT", mail.get("port") != null ? mail.get("port") : ""); + map.put("CLOUDGENE_SMTP_USER", mail.get("user") != null ? mail.get("user") : ""); + map.put("CLOUDGENE_SMTP_PASSWORD", mail.get("password") != null ? mail.get("password") : ""); + map.put("CLOUDGENE_SMTP_NAME", mail.get("name") != null ? mail.get("name") : ""); + } + return map; + } + } \ No newline at end of file From b6c4c4ed7ec16d8da7fa3240396e5a44e9cb9613 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 9 Sep 2023 11:26:59 +0200 Subject: [PATCH 15/35] Add global nextflow config file and add webui --- src/main/html/webapp/admin.js | 13 +- .../components/admin/layout/layout.stache | 1 + .../admin/settings/nextflow/nextflow.js | 39 ++++++ .../admin/settings/nextflow/nextflow.stache | 32 +++++ .../html/webapp/models/nextflow-config.js | 7 + .../cloudgene/mapred/jobs/AbstractJob.java | 7 +- .../cloudgene/mapred/jobs/CloudgeneStep.java | 5 +- .../cloudgene/mapred/jobs/Environment.java | 121 +++++++++++++----- .../cloudgene/mapred/jobs/engine/Planner.java | 20 +-- .../plugins/nextflow/NextflowBinary.java | 27 ++-- .../mapred/plugins/nextflow/NextflowStep.java | 32 +++-- .../controller/ServerAdminController.java | 18 +++ .../server/responses/ApplicationResponse.java | 4 +- .../responses/NextflowConfigResponse.java | 49 +++++++ .../mapred/server/services/ServerService.java | 7 + .../java/cloudgene/mapred/util/Settings.java | 27 ++-- 16 files changed, 312 insertions(+), 97 deletions(-) create mode 100644 src/main/html/webapp/components/admin/settings/nextflow/nextflow.js create mode 100644 src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache create mode 100644 src/main/html/webapp/models/nextflow-config.js create mode 100644 src/main/java/cloudgene/mapred/server/responses/NextflowConfigResponse.java diff --git a/src/main/html/webapp/admin.js b/src/main/html/webapp/admin.js index 573bc600..b21cffe5 100644 --- a/src/main/html/webapp/admin.js +++ b/src/main/html/webapp/admin.js @@ -17,13 +17,14 @@ import JobDetailControl from 'components/core/job/detail/'; import AppListControl from 'components/admin/app/list/'; import AppRepositoryControl from 'components/admin/app/repository/'; import SettingsGeneralControl from 'components/admin/settings/general/'; +import SettingsNextflowControl from 'components/admin/settings/nextflow/'; import SettingsServerControl from 'components/admin/settings/server/'; import SettingsMailControl from 'components/admin/settings/mail/'; import SettingsTemplatesControl from 'components/admin/settings/templates/'; import SettingsLogsControl from 'components/admin/settings/logs/'; -$(document.links).filter(function() { +$(document.links).filter(function () { return this.hostname != window.location.hostname; }).attr('target', '_blank'); @@ -76,6 +77,10 @@ var routes = [{ path: 'pages/admin-settings-general', control: SettingsGeneralControl, guard: adminGuard +}, { + path: 'pages/admin-settings-nextflow', + control: SettingsNextflowControl, + guard: adminGuard }, { path: 'pages/admin-settings-mail', control: SettingsMailControl, @@ -106,9 +111,9 @@ function adminGuard(appState) { } } -$.ajaxPrefilter(function(options, orig, xhr) { +$.ajaxPrefilter(function (options, orig, xhr) { if (!options.beforeSend) { - options.beforeSend = function(xhr) { + options.beforeSend = function (xhr) { if (localStorage.getItem("cloudgene")) { try { // get data @@ -133,7 +138,7 @@ $.ajaxPrefilter(function(options, orig, xhr) { }); -Server.findOne({}, function(server) { +Server.findOne({}, function (server) { new LayoutControl("#main", { appState: server diff --git a/src/main/html/webapp/components/admin/layout/layout.stache b/src/main/html/webapp/components/admin/layout/layout.stache index e86032c7..95d77850 100644 --- a/src/main/html/webapp/components/admin/layout/layout.stache +++ b/src/main/html/webapp/components/admin/layout/layout.stache @@ -28,6 +28,7 @@

General + Nextflow Templates Mail
diff --git a/src/main/html/webapp/components/admin/settings/nextflow/nextflow.js b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.js new file mode 100644 index 00000000..c59be201 --- /dev/null +++ b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.js @@ -0,0 +1,39 @@ +import Control from 'can-control'; +import $ from 'jquery'; +import bootbox from 'bootbox'; + +import NextflowConfig from 'models/nextflow-config'; + +import template from './nextflow.stache'; +import showErrorDialog from 'helpers/error-dialog'; + +export default Control.extend({ + + "init": function (element, options) { + var that = this; + + NextflowConfig.findOne({}, + function (nextflowConfig) { + $(element).html(template({ + nextflowConfig: nextflowConfig + })); + that.nextflowConfig = nextflowConfig; + $(element).fadeIn(); + }); + + }, + + 'submit': function (form, event) { + event.preventDefault(); + + this.nextflowConfig.attr('content', $(form).find("[name='content']").val()); + this.nextflowConfig.save(function (data) { + bootbox.alert("Nextflow configuration updated."); + }, function (response) { + showErrorDialog("Nextflow configuration not updated", response); + }); + + + } + +}); diff --git a/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache new file mode 100644 index 00000000..a93fa7dd --- /dev/null +++ b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache @@ -0,0 +1,32 @@ +

Nextflow Configuration

+ +

To configure your Nextflow settings, you can access the configuration file and fine-tune it according to your specific needs. Additionally, Nextflow offers robust support for environment variables, allowing you to easily integrate Cloudgene variables into your configuration. This nextflow.config file applies to all applications.

+ +
+
+
+ +
+ +
+ +
+
    +{{#nextflowConfig.variables}} +
  • ${{{{name}}}
    Value: {{value}}
    • +{{/nextflowConfig.variables}} +
+
+
+ +
+
+ +
+ + +
+ diff --git a/src/main/html/webapp/models/nextflow-config.js b/src/main/html/webapp/models/nextflow-config.js new file mode 100644 index 00000000..6efb6a02 --- /dev/null +++ b/src/main/html/webapp/models/nextflow-config.js @@ -0,0 +1,7 @@ +import Model from 'can-connect/can/model/model'; + +export default Model.extend({ + findOne: 'GET /api/v2/admin/server/nextflow/config', + create: 'POST /api/v2/admin/server/nextflow/config/update', + update: 'POST /api/v2/admin/server/nextflow/config/update' +}, {}); diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java index 1b82f107..40afc595 100644 --- a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java +++ b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java @@ -322,14 +322,13 @@ public void runInstallationAndResolveAppLinks() { return; } // update environment variables - Map envApp = Environment.getApplicationVariables(linkedApp.getWdlApp(), settings); - Map envJob = Environment.getJobVariables(context); + Environment environment = settings.buildEnvironment().addApplication(linkedApp.getWdlApp()) + .addContext(context); Map properties = linkedApp.getWdlApp().getProperties(); for (String property : properties.keySet()) { Object propertyValue = properties.get(property); if (propertyValue instanceof String) { - propertyValue = Environment.env(propertyValue.toString(), envApp); - propertyValue = Environment.env(propertyValue.toString(), envJob); + propertyValue = environment.resolve(propertyValue.toString()); } properties.put(property, propertyValue); } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java index fc3c7c42..9d660942 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java @@ -6,8 +6,8 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.util.List; +import java.util.Map; -import cloudgene.mapred.jobs.sdk.WorkflowContext; import cloudgene.mapred.wdl.WdlStep; public abstract class CloudgeneStep { @@ -115,7 +115,8 @@ protected boolean executeCommand(List command, CloudgeneContext context, context.log("Working Directory: " + workDir.getAbsolutePath()); ProcessBuilder builder = new ProcessBuilder(command); - builder.environment().putAll(context.getSettings().getEnvironment()); + Map env = context.getSettings().buildEnvironment().toMap(); + builder.environment().putAll(env); builder.directory(workDir); builder.redirectErrorStream(true); builder.redirectOutput(); diff --git a/src/main/java/cloudgene/mapred/jobs/Environment.java b/src/main/java/cloudgene/mapred/jobs/Environment.java index 26f4bf3c..54c79f58 100644 --- a/src/main/java/cloudgene/mapred/jobs/Environment.java +++ b/src/main/java/cloudgene/mapred/jobs/Environment.java @@ -1,8 +1,10 @@ package cloudgene.mapred.jobs; -import java.io.File; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Vector; import cloudgene.mapred.plugins.IPlugin; import cloudgene.mapred.plugins.PluginManager; @@ -11,48 +13,109 @@ public class Environment { - public static Map getApplicationVariables(WdlApp application, Settings settings) { + public static String PREFIX = "CLOUDGENE_"; - String localFolder = application.getPath(); + private Map env = new HashMap(); - HashMap environment = new HashMap(); - environment.put("app_id", application.getId()); - environment.put("app_name", application.getName()); - environment.put("app_version", application.getVersion()); - environment.put("app_local_folder", localFolder); - // Deprecated - environment.put("local_app_folder", localFolder); - environment.put("app_hdfs_folder", localFolder); - // Technologies - PluginManager manager = PluginManager.getInstance(); - for (IPlugin plugin : manager.getPlugins()) { - environment.put(plugin.getId() + "_installed", manager.isEnabled(plugin) ? "true" : "false"); + public Environment(Settings settings) { + add("SERVICE_NAME", settings.getName()); + add("SERVICE_URL", settings.getServerUrl()); + add("CONTACT_EMAIL", settings.getAdminMail()); + add("CONTACT_NAME", settings.getAdminName()); + if (settings.getMail() != null) { + add("SMTP_HOST", settings.getMail().get("smtp")); + add("SMTP_PORT", settings.getMail().get("port")); + add("SMTP_USER", settings.getMail().get("user")); + add("SMTP_PASSWORD", settings.getMail().get("password")); + add("SMTP_NAME", settings.getMail().get("name")); + add("SMTP_SENDER", settings.getMail().get("name")); } - - return environment; } - public static Map getJobVariables(CloudgeneContext context) { - Map environment = new HashMap(); - environment.put("job_id", context.getJobId()); - environment.put("job_local_temp", context.getLocalTemp()); - environment.put("job_local_output", context.getLocalOutput()); - environment.put("user_username", context.getUser().getUsername()); - environment.put("user_mail", context.getUser().getMail()); - // Deprecated - environment.put("workdir", new File(context.getWorkingDirectory()).getAbsolutePath()); - environment.put("jobId", context.getJobId()); + public Environment addContext(CloudgeneContext context) { + add("JOB_ID", context.getJobId()); + add("USER_NAME", context.getUser().getUsername()); + add("USER_EMAIL", context.getUser().getMail()); + add("USER_FULL_NAME", context.getUser().getFullName()); + return this; + } + + public Environment addApplication(WdlApp application) { + String localFolder = application.getPath(); + add("APP_LOCATION", localFolder); + //old variable names without prefix for compatibility + env.put("app_id", application.getId()); + env.put("app_name", application.getName()); + env.put("app_version", application.getVersion()); + env.put("app_local_folder", localFolder); + env.put("local_app_folder", localFolder); + env.put("app_hdfs_folder", localFolder); + return this; + } - return environment; + public Environment add(String name, String value) { + env.put(PREFIX + name, value != null ? value : ""); + return this; } - public static String env(String value, Map variables) { + public Map toMap() { + return env; + } + public List toList() { + List variables = new Vector(); + for (Entry entry : env.entrySet()) { + if (entry.getKey().endsWith("_PASSWORD")) { + variables.add(new Variable(entry.getKey(), "************")); + } else { + variables.add(new Variable(entry.getKey(), entry.getValue())); + } + } + return variables; + } + + + public static String resolve(String value, Map variables) { for (String key : variables.keySet()) { value = value.replaceAll("\\$\\{" + key + "\\}", variables.get(key)); } + return value; + } + public String resolve(String value) { + for (String key : env.keySet()) { + value = value.replaceAll("\\$\\{" + key + "\\}", env.get(key)); + } return value; } + + public static class Variable { + + private String name; + + private String value; + + public Variable(String name, String value) { + this.name = name; + this.value = value; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setValue(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + } } diff --git a/src/main/java/cloudgene/mapred/jobs/engine/Planner.java b/src/main/java/cloudgene/mapred/jobs/engine/Planner.java index 5bc51360..baf3c338 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/Planner.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/Planner.java @@ -5,7 +5,6 @@ import java.util.Map; import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.jobs.Environment; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlParameterInput; @@ -29,24 +28,7 @@ public WdlApp evaluateWDL(WdlApp app, CloudgeneContext context, Settings setting context2.put(param.getId(), context.getOutput(param.getId())); } - // add job variables - Map envJob = Environment.getJobVariables(context); - for (String key : envJob.keySet()) { - context2.put(key, envJob.get(key)); - } - - // add app variables - Map envApp = Environment.getApplicationVariables(app, settings); - for (String key : envApp.keySet()) { - context2.put(key, envApp.get(key)); - } - - context2.putAll(settings.getEnvironment()); - context2.put("CLOUDGENE_JOB_ID", context.getJobId()); - context2.put("CLOUDGENE_USER_NAME", context.getUser().getUsername()); - context2.put("CLOUDGENE_USER_EMAIL", context.getUser().getMail()); - context2.put("CLOUDGENE_USER_FULL_NAME", context.getUser().getFullName()); - + context2.putAll(settings.buildEnvironment().addApplication(app).addContext(context).toMap()); File manifest = new File(app.getManifestFile()); diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java index 4992c577..86e559b0 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java @@ -17,7 +17,7 @@ public class NextflowBinary { private String profile; - private File nextflowConfigFile; + private List configFiles = new Vector(); private String work; @@ -82,10 +82,14 @@ public void setProfile(String profile) { this.profile = profile; } - public void setNextflowConfigFile(File nextflowConfigFile) { - this.nextflowConfigFile = nextflowConfigFile; + public void addConfig(File configFile) { + this.configFiles.add(configFile); } + public void addConfig(String configFilename) { + this.configFiles.add(new File(configFilename)); + } + public void setWork(String work) { this.work = work; } @@ -119,6 +123,11 @@ public List buildCommand() { List nextflow = new Vector(); nextflow.add("PATH=$PATH:/usr/local/bin"); nextflow.add(getBinary()); + + nextflow.add("-log"); + nextflow.add(log); + + nextflow.add("run"); nextflow.add(script); @@ -128,10 +137,11 @@ public List buildCommand() { nextflow.add(profile); } - if (nextflowConfigFile.exists()) { - // set custom configuration - nextflow.add("-c"); - nextflow.add(nextflowConfigFile.getAbsolutePath()); + for (File configFile : configFiles) { + if (configFile.exists()) { + nextflow.add("-c"); + nextflow.add(configFile.getAbsolutePath()); + } } nextflow.add("-w"); @@ -146,9 +156,6 @@ public List buildCommand() { nextflow.add("-with-weblog"); nextflow.add(weblog); - // nextflowCommand.add("-log"); - // nextflowCommand.add(log); - nextflow.add("-with-trace"); nextflow.add(trace); diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java index eb05ad4b..1e37113b 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java @@ -16,6 +16,7 @@ import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; import cloudgene.mapred.jobs.workspace.IWorkspace; +import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlStep; import genepi.io.FileUtil; import groovy.json.JsonOutput; @@ -38,6 +39,8 @@ public class NextflowStep extends CloudgeneStep { public boolean run(WdlStep step, CloudgeneContext context) { this.context = context; + + Settings settings = context.getSettings(); String script = step.getString("script"); @@ -54,12 +57,12 @@ public boolean run(WdlStep step, CloudgeneContext context) { // load process styling loadProcessConfigs(step.get(PROPERTY_PROCESS_CONFIG)); - - NextflowBinary nextflow = NextflowBinary.build(context.getSettings()); + + NextflowBinary nextflow = NextflowBinary.build(settings); nextflow.setScript(scriptPath); AbstractJob job = context.getJob(); - String appFolder = context.getSettings().getApplicationRepository().getConfigDirectory(job.getApplicationId()); + String appFolder = settings.getApplicationRepository().getConfigDirectory(job.getApplicationId()); // set profile String profile = ""; @@ -69,10 +72,13 @@ public boolean run(WdlStep step, CloudgeneContext context) { } nextflow.setProfile(profile); - // set custom configuration - String nextflowConfig = FileUtil.path(appFolder, "nextflow.config"); - File nextflowConfigFile = new File(nextflowConfig); - nextflow.setNextflowConfigFile(nextflowConfigFile); + // set global configuration + String globalConfig = settings.getNextflowConfig(); + nextflow.addConfig(globalConfig); + + // set application specific configuration + String appConfig = FileUtil.path(appFolder, "nextflow.config"); + nextflow.addConfig(appConfig); // set work directory String work = ""; @@ -91,7 +97,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { nextflow.setWork(workDir); } - //params json file + // params json file String paramsJsonFilename = FileUtil.path(context.getLocalOutput(), "params.json"); File paramsFile = new File(paramsJsonFilename); try { @@ -103,12 +109,12 @@ public boolean run(WdlStep step, CloudgeneContext context) { return false; } nextflow.setParamsFile(paramsFile); - - //register job in webcollector and set created url + + // register job in webcollector and set created url String collectorUrl = collector.addContext(context); nextflow.setWeblog(collectorUrl); - //log files and reports + // log files and reports nextflow.setTrace(workspace.createLogFile("trace.csv")); nextflow.setReport(workspace.createLogFile("report.html")); nextflow.setTimeline(workspace.createLogFile("timeline.html")); @@ -117,7 +123,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { try { File executionDir = new File(context.getLocalOutput()); - + StringBuilder output = new StringBuilder(); boolean successful = executeCommand(nextflow.buildCommand(), context, output, executionDir); @@ -199,7 +205,7 @@ private NextflowProcessConfig getNextflowProcessConfig(NextflowProcess process) public String[] getRequirements() { return new String[] { NextflowPlugin.ID }; } - + private Map createParamsMap(WdlStep step) { Map params = new HashMap(); diff --git a/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java b/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java index 4537f226..160575e9 100644 --- a/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java +++ b/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java @@ -10,6 +10,7 @@ import cloudgene.mapred.database.CounterHistoryDao; import cloudgene.mapred.server.Application; import cloudgene.mapred.server.auth.AuthenticationService; +import cloudgene.mapred.server.responses.NextflowConfigResponse; import cloudgene.mapred.server.responses.ServerResponse; import cloudgene.mapred.server.responses.StatisticsResponse; import cloudgene.mapred.server.services.ServerService; @@ -113,6 +114,23 @@ public ServerResponse updateSettings(String name, String backgroundColor, String return ServerResponse.build(application.getSettings()); } + + @Get("/nextflow/config") + public NextflowConfigResponse getNextflowConfig() { + + return NextflowConfigResponse.build(application.getSettings()); + + } + + @Post("/nextflow/config/update") + public NextflowConfigResponse updateNextflowConfig(String content) { + + serverService.updateNextflowConfig(content); + + return NextflowConfigResponse.build(application.getSettings()); + + } + @Get("/cloudgene-apps") public String list() throws IOException { diff --git a/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java b/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java index 13746079..72d70ac1 100644 --- a/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java @@ -65,7 +65,7 @@ public static ApplicationResponse buildWithDetails(Application app, Settings set appResponse.setState(updateState(app, settings)); - Map environment = Environment.getApplicationVariables(app.getWdlApp(), settings); + Map environment = settings.buildEnvironment().addApplication(app.getWdlApp()).toMap(); appResponse.setEnvironment(environment); Map updatedConfig = repository.getConfig(app.getWdlApp()); @@ -85,7 +85,7 @@ public static List buildWithDetails(List appli } private static String updateState(Application app, Settings settings) { - //TODO: remove + // TODO: remove return "completed"; } diff --git a/src/main/java/cloudgene/mapred/server/responses/NextflowConfigResponse.java b/src/main/java/cloudgene/mapred/server/responses/NextflowConfigResponse.java new file mode 100644 index 00000000..85c52a96 --- /dev/null +++ b/src/main/java/cloudgene/mapred/server/responses/NextflowConfigResponse.java @@ -0,0 +1,49 @@ +package cloudgene.mapred.server.responses; + +import java.io.File; +import java.util.List; +import java.util.Vector; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import cloudgene.mapred.jobs.Environment.Variable; +import cloudgene.mapred.util.Settings; +import genepi.io.FileUtil; + +@JsonInclude(JsonInclude.Include.ALWAYS) +public class NextflowConfigResponse { + + private String content = ""; + + private List variables = new Vector(); + + public static NextflowConfigResponse build(Settings settings) { + NextflowConfigResponse response = new NextflowConfigResponse(); + String configFilename = settings.getNextflowConfig(); + File configFile = new File(configFilename); + if (!configFile.exists()) { + return response; + } + + response.setContent(FileUtil.readFileAsString(configFilename)); + response.setVariables(settings.buildEnvironment().toList()); + return response; + } + + public void setContent(String content) { + this.content = content; + } + + public String getContent() { + return content; + } + + public void setVariables(List variables) { + this.variables = variables; + } + + public List getVariables() { + return variables; + } + +} diff --git a/src/main/java/cloudgene/mapred/server/services/ServerService.java b/src/main/java/cloudgene/mapred/server/services/ServerService.java index 0d0c0210..8c01e1d4 100644 --- a/src/main/java/cloudgene/mapred/server/services/ServerService.java +++ b/src/main/java/cloudgene/mapred/server/services/ServerService.java @@ -24,6 +24,7 @@ import cloudgene.mapred.server.controller.ServerAdminController; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; +import genepi.io.FileUtil; import io.micronaut.security.oauth2.configuration.OauthClientConfigurationProperties; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -257,4 +258,10 @@ public String tail(File file, int lines) { } } + public void updateNextflowConfig(String content) { + Settings settings = application.getSettings(); + String filename = settings.getNextflowConfig(); + FileUtil.writeStringBufferToFile(filename, new StringBuffer(content)); + } + } diff --git a/src/main/java/cloudgene/mapred/util/Settings.java b/src/main/java/cloudgene/mapred/util/Settings.java index 6aaa3cd3..8e3f5bec 100644 --- a/src/main/java/cloudgene/mapred/util/Settings.java +++ b/src/main/java/cloudgene/mapred/util/Settings.java @@ -19,6 +19,7 @@ import cloudgene.mapred.apps.Application; import cloudgene.mapred.apps.ApplicationRepository; +import cloudgene.mapred.jobs.Environment; import genepi.io.FileUtil; public class Settings { @@ -87,6 +88,8 @@ public class Settings { private String port = "8082"; + private String nextflowConfig = FileUtil.path("config", "nextflow.config"); + public static final String DEFAULT_SECURITY_KEY = "default-key-change-me-immediately"; // fake! @@ -567,20 +570,16 @@ public String getServerUrl() { return serverUrl; } - public Map getEnvironment() { - Map map = new HashMap(); - map.put("CLOUDGENE_SERVICE_NAME", name != null ? name : ""); - map.put("CLOUDGENE_SERVICE_URL", serverUrl != null ? serverUrl : ""); - map.put("CLOUDGENE_CONTACT_EMAIL", adminMail != null ? adminMail : ""); - map.put("CLOUDGENE_CONTACT_NAME", adminName != null ? adminName : ""); - if (mail != null) { - map.put("CLOUDGENE_SMTP_HOST", mail.get("smtp") != null ? mail.get("smtp") : ""); - map.put("CLOUDGENE_SMTP_PORT", mail.get("port") != null ? mail.get("port") : ""); - map.put("CLOUDGENE_SMTP_USER", mail.get("user") != null ? mail.get("user") : ""); - map.put("CLOUDGENE_SMTP_PASSWORD", mail.get("password") != null ? mail.get("password") : ""); - map.put("CLOUDGENE_SMTP_NAME", mail.get("name") != null ? mail.get("name") : ""); - } - return map; + public void setNextflowConfig(String nextflowConfig) { + this.nextflowConfig = nextflowConfig; + } + + public String getNextflowConfig() { + return nextflowConfig; + } + + public Environment buildEnvironment() { + return new Environment(this); } } \ No newline at end of file From 243d9ffe600e9bba5af16c8c0eb39c9c33caaf2e Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 9 Sep 2023 13:08:31 +0200 Subject: [PATCH 16/35] Remove preprocessing queue --- .../html/webapp/components/admin/job/list/list.js | 7 +++---- .../html/webapp/components/admin/job/list/list.stache | 11 ++--------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/main/html/webapp/components/admin/job/list/list.js b/src/main/html/webapp/components/admin/job/list/list.js index 34e82ad2..e1f4d6bb 100644 --- a/src/main/html/webapp/components/admin/job/list/list.js +++ b/src/main/html/webapp/components/admin/job/list/list.js @@ -11,13 +11,12 @@ import template from './list.stache'; export default Control.extend({ - "init": function(element, options) { + "init": function (element, options) { $(element).html(template()); $(element).fadeIn(); - new JobTable("#job-list-running-stq", {state: "running-stq"}); - new JobTable("#job-list-running-ltq", {state: "running-ltq"}); - new JobTable("#job-list-current", {state: "current"}); + new JobTable("#job-list-running-ltq", { state: "running-ltq" }); + new JobTable("#job-list-current", { state: "current" }); } diff --git a/src/main/html/webapp/components/admin/job/list/list.stache b/src/main/html/webapp/components/admin/job/list/list.stache index 74a29197..99fa5069 100644 --- a/src/main/html/webapp/components/admin/job/list/list.stache +++ b/src/main/html/webapp/components/admin/job/list/list.stache @@ -2,21 +2,14 @@

This page lists all submitted and running jobs.

-

Preprocessing Queue

- -
-
- -
- -

Long Running Queue

+

Running Jobs

-

Active Jobs

+

Completed Jobs

From e4efab2bfa63b89e64a8cee6551f10efcd9521f0 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 9 Sep 2023 13:19:35 +0200 Subject: [PATCH 17/35] Add contact details and server url to admin panel --- .../admin/settings/general/general.js | 13 ++++---- .../admin/settings/general/general.stache | 19 ++++++++++++ .../controller/ServerAdminController.java | 13 ++++---- .../server/responses/ServerResponse.java | 30 +++++++++++++++++++ .../mapred/server/services/ServerService.java | 5 +++- 5 files changed, 67 insertions(+), 13 deletions(-) diff --git a/src/main/html/webapp/components/admin/settings/general/general.js b/src/main/html/webapp/components/admin/settings/general/general.js index bd4a0078..7efd7763 100644 --- a/src/main/html/webapp/components/admin/settings/general/general.js +++ b/src/main/html/webapp/components/admin/settings/general/general.js @@ -9,11 +9,11 @@ import showErrorDialog from 'helpers/error-dialog'; export default Control.extend({ - "init": function(element, options) { + "init": function (element, options) { var that = this; Settings.findOne({}, - function(settings) { + function (settings) { $(element).html(template({ settings: settings })); @@ -23,16 +23,19 @@ export default Control.extend({ }, - 'submit': function(form, event) { + 'submit': function (form, event) { event.preventDefault(); this.settings.attr('name', $(form).find("[name='name']").val()); + this.settings.attr('adminName', $(form).find("[name='adminName']").val()); + this.settings.attr('adminMail', $(form).find("[name='adminMail']").val()); + this.settings.attr('serverUrl', $(form).find("[name='serverUrl']").val()); this.settings.attr('backgroundColor', $(form).find("[name='background-color']").val()); this.settings.attr('foregroundColor', $(form).find("[name='foreground-color']").val()); this.settings.attr('googleAnalytics', $(form).find("[name='google-analytics']").val()); - this.settings.save(function(data) { + this.settings.save(function (data) { bootbox.alert("Settings updated."); - }, function(response) { + }, function (response) { showErrorDialog("Settings not updated", response); }); diff --git a/src/main/html/webapp/components/admin/settings/general/general.stache b/src/main/html/webapp/components/admin/settings/general/general.stache index 6b1c01ed..3682eca7 100644 --- a/src/main/html/webapp/components/admin/settings/general/general.stache +++ b/src/main/html/webapp/components/admin/settings/general/general.stache @@ -11,6 +11,25 @@ +
+ + +
+ + +
+ + +
+ + +
+ + +
+ +
+
diff --git a/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java b/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java index 160575e9..00ac621f 100644 --- a/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java +++ b/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java @@ -104,17 +104,17 @@ public ServerResponse getSettings() { } @Post("/settings/update") - public ServerResponse updateSettings(String name, String backgroundColor, String foregroundColor, - String googleAnalytics, boolean mail, String mailSmtp, String mailUser, String mailPassword, - String mailPort, String mailName) { + public ServerResponse updateSettings(String name, String adminName, String adminMail, String serverUrl, + String backgroundColor, String foregroundColor, String googleAnalytics, boolean mail, String mailSmtp, + String mailUser, String mailPassword, String mailPort, String mailName) { - serverService.updateSettings(name, backgroundColor, foregroundColor, googleAnalytics, String.valueOf(mail), - mailSmtp, mailPort, mailUser, mailPassword, mailName); + serverService.updateSettings(name, adminName, adminMail, serverUrl, backgroundColor, foregroundColor, + googleAnalytics, String.valueOf(mail), mailSmtp, mailPort, mailUser, mailPassword, mailName); return ServerResponse.build(application.getSettings()); } - + @Get("/nextflow/config") public NextflowConfigResponse getNextflowConfig() { @@ -131,7 +131,6 @@ public NextflowConfigResponse updateNextflowConfig(String content) { } - @Get("/cloudgene-apps") public String list() throws IOException { URL url = new URL(CLOUDGENE_APPS_ENDPOINT); diff --git a/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java b/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java index b97f4688..35269166 100644 --- a/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java @@ -19,10 +19,16 @@ public class ServerResponse { private String mailPassword; private String mailName; private boolean mail; + private String adminName; + private String adminMail; + private String serverUrl; public static ServerResponse build(Settings settings) { ServerResponse response = new ServerResponse(); response.setName(settings.getName()); + response.setAdminName(settings.getAdminName()); + response.setAdminMail(settings.getAdminMail()); + response.setServerUrl(settings.getServerUrl()); response.setBackgroundColor(settings.getColors().get("background")); response.setForegroundColor(settings.getColors().get("foreground")); response.setGoogleAnalytics(settings.getGoogleAnalytics()); @@ -59,6 +65,30 @@ public void setName(String name) { this.name = name; } + public String getAdminName() { + return adminName; + } + + public void setAdminName(String adminName) { + this.adminName = adminName; + } + + public String getAdminMail() { + return adminMail; + } + + public void setAdminMail(String adminMail) { + this.adminMail = adminMail; + } + + public String getServerUrl() { + return serverUrl; + } + + public void setServerUrl(String serverUrl) { + this.serverUrl = serverUrl; + } + public String getBackgroundColor() { return backgroundColor; } diff --git a/src/main/java/cloudgene/mapred/server/services/ServerService.java b/src/main/java/cloudgene/mapred/server/services/ServerService.java index 8c01e1d4..e0fb48ab 100644 --- a/src/main/java/cloudgene/mapred/server/services/ServerService.java +++ b/src/main/java/cloudgene/mapred/server/services/ServerService.java @@ -116,11 +116,14 @@ public String getRoot(User user) { return data.toString(); } - public void updateSettings(String name, String background_color, String foreground_color, String google_analytics, + public void updateSettings(String name, String adminName, String adminMail, String serverUrl, String background_color, String foreground_color, String google_analytics, String mail, String mail_smtp, String mail_port, String mail_user, String mail_password, String mail_name) { Settings settings = application.getSettings(); settings.setName(name); + settings.setAdminName(adminName); + settings.setAdminMail(adminMail); + settings.setServerUrl(serverUrl); settings.getColors().put("background", background_color); settings.getColors().put("foreground", foreground_color); settings.setGoogleAnalytics(google_analytics); From efe4b637b9036815cf56ce0ea71ca2bb44ac8a8a Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 9 Sep 2023 13:28:55 +0200 Subject: [PATCH 18/35] Improve application settings and environment variables --- src/main/html/webapp/admin.js | 6 + .../webapp/components/admin/app/list/list.js | 133 +++++------------- .../components/admin/app/list/list.stache | 23 +-- .../components/admin/app/settings/settings.js | 51 +++++++ .../app/{list => }/settings/settings.stache | 29 +++- .../admin/settings/nextflow/nextflow.stache | 2 +- .../webapp/models/application-settings.js | 8 ++ .../cloudgene/mapred/jobs/CloudgeneStep.java | 11 +- .../server/controller/AppController.java | 29 +++- .../server/responses/ApplicationResponse.java | 28 +--- 10 files changed, 175 insertions(+), 145 deletions(-) create mode 100644 src/main/html/webapp/components/admin/app/settings/settings.js rename src/main/html/webapp/components/admin/app/{list => }/settings/settings.stache (56%) create mode 100644 src/main/html/webapp/models/application-settings.js diff --git a/src/main/html/webapp/admin.js b/src/main/html/webapp/admin.js index b21cffe5..a811db12 100644 --- a/src/main/html/webapp/admin.js +++ b/src/main/html/webapp/admin.js @@ -15,6 +15,8 @@ import UserListControl from 'components/admin/user/list/'; import JobListControl from 'components/admin/job/list/'; import JobDetailControl from 'components/core/job/detail/'; import AppListControl from 'components/admin/app/list/'; +import AppSettingsControl from 'components/admin/app/settings/'; + import AppRepositoryControl from 'components/admin/app/repository/'; import SettingsGeneralControl from 'components/admin/settings/general/'; import SettingsNextflowControl from 'components/admin/settings/nextflow/'; @@ -65,6 +67,10 @@ var routes = [{ path: 'pages/admin-apps', control: AppListControl, guard: adminGuard +}, { + path: 'pages/admin-apps/{app}', + control: AppSettingsControl, + guard: adminGuard }, { path: 'pages/admin-apps-repository', control: AppRepositoryControl, diff --git a/src/main/html/webapp/components/admin/app/list/list.js b/src/main/html/webapp/components/admin/app/list/list.js index 35a7f005..10d1de76 100644 --- a/src/main/html/webapp/components/admin/app/list/list.js +++ b/src/main/html/webapp/components/admin/app/list/list.js @@ -15,14 +15,13 @@ import template from './list.stache'; import templateInstallGithub from './install-github/install-github.stache'; import templateInstallUrl from './install-url/install-url.stache'; import templatePermission from './permission/permission.stache'; -import templateSettings from './settings/settings.stache'; export default Control.extend({ - "init": function(element, options) { + "init": function (element, options) { var that = this; - Application.findAll({}, function(applications) { + Application.findAll({}, function (applications) { that.options.installedApplications = applications; $(element).html(template({ applications: applications @@ -33,10 +32,10 @@ export default Control.extend({ }, - '#install-app-url-btn click': function(el, ev) { + '#install-app-url-btn click': function (el, ev) { bootbox.confirm(templateInstallUrl(), - function(result) { + function (result) { if (result) { var url = $('#url').val(); var app = new Application(); @@ -50,15 +49,15 @@ export default Control.extend({ '
', show: false }); - waitingDialog.on('shown.bs.modal', function() { - app.save(function(application) { + waitingDialog.on('shown.bs.modal', function () { + app.save(function (application) { waitingDialog.modal('hide'); - bootbox.alert('

Congratulations

The application installation was successful.

', function() { + bootbox.alert('

Congratulations

The application installation was successful.

', function () { var router = canRoute.router; router.reload(); }); - }, function(response) { + }, function (response) { waitingDialog.modal('hide'); showErrorDialog("Operation failed", response); }); @@ -69,10 +68,10 @@ export default Control.extend({ }); }, - '#install-app-github-btn click': function(el, ev) { + '#install-app-github-btn click': function (el, ev) { bootbox.confirm(templateInstallGithub(), - function(result) { + function (result) { if (result) { var url = 'github://' + $('#url').val(); @@ -88,16 +87,16 @@ export default Control.extend({ show: false }); - waitingDialog.on('shown.bs.modal', function() { + waitingDialog.on('shown.bs.modal', function () { - app.save(function(application) { + app.save(function (application) { waitingDialog.modal('hide'); - bootbox.alert('

Congratulations

The application installation was successful.

', function() { + bootbox.alert('

Congratulations

The application installation was successful.

', function () { var router = canRoute.router; router.reload(); }); - }, function(response) { + }, function (response) { waitingDialog.modal('hide'); showErrorDialog("Operation failed", response); }); @@ -108,12 +107,12 @@ export default Control.extend({ }); }, - '#reload-apps-btn click': function(el, ev) { + '#reload-apps-btn click': function (el, ev) { var element = this.element; Application.findAll({ reload: 'true' - }, function(applications) { + }, function (applications) { $(element).html(template({ applications: applications @@ -123,12 +122,12 @@ export default Control.extend({ }); }, - '.enable-disable-btn click': function(el, ev) { + '.enable-disable-btn click': function (el, ev) { var card = $(el).closest('.card'); var application = domData.get.call(card[0], 'application'); var enabled = !application.attr('enabled') - bootbox.confirm("Are you sure you want to " + (enabled ? "enable" : "disable") + " application " + application.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to " + (enabled ? "enable" : "disable") + " application " + application.attr('id') + "?", function (result) { if (result) { application.attr('enabled', enabled); @@ -140,13 +139,13 @@ export default Control.extend({ '', show: false }); - waitingDialog.on('shown.bs.modal', function() { + waitingDialog.on('shown.bs.modal', function () { - application.save(function(application) { + application.save(function (application) { waitingDialog.modal('hide'); bootbox.alert('

Congratulations

The application has been successfully ' + (enabled ? 'enabled' : 'disabled') + '.

'); - }, function(response) { + }, function (response) { waitingDialog.modal('hide'); showErrorDialog("Operation failed", response); }); @@ -157,12 +156,12 @@ export default Control.extend({ }); }, - '.delete-app-btn click': function(el, ev) { + '.delete-app-btn click': function (el, ev) { var card = $(el).closest('.card'); var application = domData.get.call(card[0], 'application'); - bootbox.confirm("Are you sure you want to delete " + application.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to delete " + application.attr('id') + "?", function (result) { if (result) { var waitingDialog = bootbox.dialog({ @@ -174,13 +173,13 @@ export default Control.extend({ show: false }); - waitingDialog.on('shown.bs.modal', function() { + waitingDialog.on('shown.bs.modal', function () { - application.destroy(function(application) { + application.destroy(function (application) { waitingDialog.modal('hide'); bootbox.alert('

Congratulations

The application has been successfully removed.

'); - }, function(response) { + }, function (response) { waitingDialog.modal('hide'); showErrorDialog("Operation failed", response); }); @@ -193,65 +192,37 @@ export default Control.extend({ }, - '.edit-settings-btn click': function(el, ev) { - - var card = $(el).closest('.card'); - var application = domData.get.call(card[0], 'application'); - - - bootbox.confirm(templateSettings({ - application: application - }), - function(result) { - if (result) { - var nextflowProfile = $('#nextflow-profile').val(); - var nextflowConfig = $('#nextflow-config').val(); - var nextflowWork = $('#nextflow-work').val(); - - application.attr('config').attr('nextflow.profile', nextflowProfile); - application.attr('config').attr('nextflow.config', nextflowConfig); - application.attr('config').attr('nextflow.work', nextflowWork); - application.save(function(data) {}, - function(response) { - showErrorDialog("Operation failed", response); - }); - } - }).find("div.modal-dialog").css({ "width": "80%" }); - - }, - - - '.edit-permission-btn click': function(el, ev) { + '.edit-permission-btn click': function (el, ev) { var card = $(el).closest('.card'); var application = domData.get.call(card[0], 'application'); Group.findAll({}, - function(groups) { + function (groups) { var selection = new canMap(); selection.attr('group', application.attr('permission')); selection.attr('name', ''); bootbox.confirm(templatePermission({ - selection: selection, - application: application, - groups: groups - }), - function(result) { + selection: selection, + application: application, + groups: groups + }), + function (result) { if (result) { var group = selection.attr('group'); if (group !== '') { application.attr('permission', group); - application.save(function(data) {}, - function(response) { + application.save(function (data) { }, + function (response) { showErrorDialog("Operation failed", response); }); } else { var name = selection.attr('name'); if (name !== '') { application.attr('permission', name); - application.save(function(data) {}, - function(response) { + application.save(function (data) { }, + function (response) { showErrorDialog("Operation failed", response); }); @@ -264,35 +235,5 @@ export default Control.extend({ }); }); - }, - - '.reinstall-btn click': function(el, ev) { - - var card = $(el).closest('.card'); - var application = domData.get.call(card[0], 'application'); - bootbox.confirm('

' + application.attr('id') + '

Force reinstallation of application?
All metafiles in HDFS are deleted and reimported on next job run.

', - function(result) { - if (result) { - application.attr('reinstall', 'true'); - application.save(function(data) {}, - function(response) { - showErrorDialog("Operation failed", response); - }); - } - }); - - - }, - - '.view-source-btn click': function(el, ev) { - - var card = $(el).closest('.card'); - var application = domData.get.call(card[0], 'application'); - var env = ''; - for (var property in application.attr('environment').attr()) { - env += property + '=' + application.attr('environment').attr(property) + '\n'; - } - - bootbox.alert('
File

' + application.attr('filename') + '

Status

' + application.attr('state') + '

' + '
Environment Variables

' + env + '

' + '
Source

' + application.attr('source') + '

'); } }); diff --git a/src/main/html/webapp/components/admin/app/list/list.stache b/src/main/html/webapp/components/admin/app/list/list.stache index c073467a..1bf6d6ad 100644 --- a/src/main/html/webapp/components/admin/app/list/list.stache +++ b/src/main/html/webapp/components/admin/app/list/list.stache @@ -25,11 +25,11 @@ {{#if(wdlApp)}} {{#if(wdlApp.workflow)}}
- Application + Application
{{else}}
- {{wdlApp.category}} + {{wdlApp.category}}
{{/if}} {{else}} @@ -53,20 +53,7 @@ {{else}}

{{{truncate(wdlApp.description, 200)}}}
- {{wdlApp.website}} - {{#is(state, 'completed')}} -
- - Installation: {{state}} - - - {{/is}} - {{#is(state, 'on demand')}} -
- - Installation: {{state}} - - {{/is}} + {{wdlApp.website}}

{{/errorMessage}} @@ -76,7 +63,9 @@
- + {{#if(wdlApp.workflow)}} + Settings + {{/if}} {{#enabled}} diff --git a/src/main/html/webapp/components/admin/app/settings/settings.js b/src/main/html/webapp/components/admin/app/settings/settings.js new file mode 100644 index 00000000..853cf669 --- /dev/null +++ b/src/main/html/webapp/components/admin/app/settings/settings.js @@ -0,0 +1,51 @@ +import Control from 'can-control'; +import domData from 'can-util/dom/data/data'; +import canMap from 'can-map'; +import canRoute from 'can-route'; + +import 'helpers/helpers'; +import $ from 'jquery'; +import bootbox from 'bootbox'; +import showErrorDialog from 'helpers/error-dialog'; + +import ApplicationSettings from 'models/application-settings'; +import template from './settings.stache'; + +export default Control.extend({ + + "init": function (element, options) { + var that = this; + + ApplicationSettings.findOne({ id: options.app }, function (application) { + that.application = application; + $(element).html(template({ + application: application + + })); + $(element).fadeIn(); + + }); + + }, + + + 'submit': function (form, event) { + event.preventDefault(); + + var nextflowProfile = $('#nextflow-profile').val(); + var nextflowConfig = $('#nextflow-config').val(); + var nextflowWork = $('#nextflow-work').val(); + + this.application.attr('config').attr('nextflow.profile', nextflowProfile); + this.application.attr('config').attr('nextflow.config', nextflowConfig); + this.application.attr('config').attr('nextflow.work', nextflowWork); + this.application.save(function (data) { + bootbox.alert("Application settings updated."); + }, + function (response) { + showErrorDialog("Operation failed", response); + }); + } + + +}); diff --git a/src/main/html/webapp/components/admin/app/list/settings/settings.stache b/src/main/html/webapp/components/admin/app/settings/settings.stache similarity index 56% rename from src/main/html/webapp/components/admin/app/list/settings/settings.stache rename to src/main/html/webapp/components/admin/app/settings/settings.stache index 32dceef9..1d06110e 100644 --- a/src/main/html/webapp/components/admin/app/list/settings/settings.stache +++ b/src/main/html/webapp/components/admin/app/settings/settings.stache @@ -1,5 +1,6 @@ +

Applications / {{application.id}}

+

{{application.id}}

-

Settings

Change settings for application {{application.wdlApp.name}}.

Nextflow
@@ -14,7 +15,31 @@

Custom Configuration:
All hardware specific parameters can be set here. If you use the AWSBatch profile this is the place to set your credentials

- + +
+ +
+ +
+
+ +
+ + +
+ +
+
    +{{#application.environment}} +
  • ${{{{name}}}
    Value: {{value}}
    • +{{/application.environment}} +
+
+
+ diff --git a/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache index a93fa7dd..3ee1dafd 100644 --- a/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache +++ b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache @@ -1,4 +1,4 @@ -

Nextflow Configuration

+

Nextflow Global Configuration

To configure your Nextflow settings, you can access the configuration file and fine-tune it according to your specific needs. Additionally, Nextflow offers robust support for environment variables, allowing you to easily integrate Cloudgene variables into your configuration. This nextflow.config file applies to all applications.

diff --git a/src/main/html/webapp/models/application-settings.js b/src/main/html/webapp/models/application-settings.js new file mode 100644 index 00000000..0e264413 --- /dev/null +++ b/src/main/html/webapp/models/application-settings.js @@ -0,0 +1,8 @@ +import Model from 'can-connect/can/model/model'; + +export default Model.extend({ + findOne: 'GET /api/v2/server/apps/{id}/settings', + update: 'PUT /api/v2/server/apps/{id}/settings' +}, { + +}); diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java index 9d660942..fd6f0e56 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.Map; +import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlStep; public abstract class CloudgeneStep { @@ -104,10 +105,13 @@ protected boolean executeCommand(List command, CloudgeneContext context, protected boolean executeCommand(List command, CloudgeneContext context, StringBuilder output, File workDir) throws IOException, InterruptedException { + + WdlApp app = context.getSettings().getApplicationRepository().getById(job.getApplicationId()).getWdlApp(); + Environment environment = context.getSettings().buildEnvironment().addContext(context).addApplication(app); + // set global variables for (int j = 0; j < command.size(); j++) { - - String cmd = command.get(j).replaceAll("\\$job_id", context.getJobId()); + String cmd = environment.resolve(command.get(j)); command.set(j, cmd); } @@ -115,8 +119,7 @@ protected boolean executeCommand(List command, CloudgeneContext context, context.log("Working Directory: " + workDir.getAbsolutePath()); ProcessBuilder builder = new ProcessBuilder(command); - Map env = context.getSettings().buildEnvironment().toMap(); - builder.environment().putAll(env); + builder.environment().putAll(environment.toMap()); builder.directory(workDir); builder.redirectErrorStream(true); builder.redirectOutput(); diff --git a/src/main/java/cloudgene/mapred/server/controller/AppController.java b/src/main/java/cloudgene/mapred/server/controller/AppController.java index ea44465b..789d9f26 100644 --- a/src/main/java/cloudgene/mapred/server/controller/AppController.java +++ b/src/main/java/cloudgene/mapred/server/controller/AppController.java @@ -82,7 +82,31 @@ public ApplicationResponse updateApp(String appId, @Nullable Boolean enabled, @N } // update permissions applicationService.updatePermissions(app, permission); - // update config + + app.checkForChanges(); + + ApplicationResponse appResponse = ApplicationResponse.build(app); + + return appResponse; + } + + @Get("/api/v2/server/apps/{appId}/settings") + @Secured(User.ROLE_ADMIN) + public ApplicationResponse getAppSettings(String appId) { + Application app = applicationService.getById(appId); + ApplicationRepository repository = applicationService.getRepository(); + ApplicationResponse appResponse = ApplicationResponse.buildWithDetails(app, this.application.getSettings(), + repository); + + return appResponse; + } + + @Put("/api/v2/server/apps/{appId}/settings") + @Secured(User.ROLE_ADMIN) + public ApplicationResponse updateAppSettings(String appId, @Nullable Boolean enabled, @Nullable String permission, + @Nullable Boolean reinstall, @Nullable Map config) { + + Application app = applicationService.getById(appId); applicationService.updateConfig(app, config); app.checkForChanges(); @@ -99,8 +123,7 @@ public ApplicationResponse updateApp(String appId, @Nullable Boolean enabled, @N @Secured(User.ROLE_ADMIN) public ApplicationResponse install(@Nullable String url) { Application app = applicationService.installApp(url); - ApplicationRepository repository = applicationService.getRepository(); - return ApplicationResponse.buildWithDetails(app, application.getSettings(), repository); + return ApplicationResponse.build(app); } @Get("/api/v2/server/apps") diff --git a/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java b/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java index 72d70ac1..c9cd5ff7 100644 --- a/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java @@ -32,9 +32,7 @@ public class ApplicationResponse { private String source = ""; - private String state = ""; - - private Map environment; + private List environment; private Map config; @@ -48,6 +46,7 @@ public static ApplicationResponse build(Application app) { appResponse.setErrorMessage(app.getErrorMessage()); appResponse.setChanged(app.isChanged()); appResponse.setPermission(app.getPermission()); + // TODO: check if we need wdl app and file? only in details? appResponse.setWdlApp(app.getWdlApp()); if (new File(app.getFilename()).exists()) { @@ -63,9 +62,7 @@ public static ApplicationResponse buildWithDetails(Application app, Settings set ApplicationResponse appResponse = build(app); - appResponse.setState(updateState(app, settings)); - - Map environment = settings.buildEnvironment().addApplication(app.getWdlApp()).toMap(); + List environment = settings.buildEnvironment().addApplication(app.getWdlApp()).toList(); appResponse.setEnvironment(environment); Map updatedConfig = repository.getConfig(app.getWdlApp()); @@ -79,16 +76,11 @@ public static List buildWithDetails(List appli ApplicationRepository repository) { List response = new Vector(); for (Application app : applications) { - response.add(ApplicationResponse.buildWithDetails(app, settings, repository)); + response.add(ApplicationResponse.build(app)); } return response; } - private static String updateState(Application app, Settings settings) { - // TODO: remove - return "completed"; - } - public String getId() { return id; } @@ -153,14 +145,6 @@ public void setSource(String source) { this.source = source; } - public String getState() { - return state; - } - - public void setState(String state) { - this.state = state; - } - public WdlApp getWdlApp() { return wdlApp; } @@ -169,11 +153,11 @@ public void setWdlApp(WdlApp wdlApp) { this.wdlApp = wdlApp; } - public Map getEnvironment() { + public List getEnvironment() { return environment; } - public void setEnvironment(Map environment) { + public void setEnvironment(List environment) { this.environment = environment; } From ffc82ef38e52c1c926461180beb109d45e1413c9 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 9 Sep 2023 14:46:25 +0200 Subject: [PATCH 19/35] Fix issue with job in environment --- .../java/cloudgene/mapred/jobs/CloudgeneContext.java | 4 ++-- .../java/cloudgene/mapred/jobs/CloudgeneStep.java | 12 +++++------- .../cloudgene/mapred/jobs/engine/ExecutableStep.java | 3 +-- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java index 9ce2095d..49c6127a 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java @@ -37,7 +37,7 @@ public class CloudgeneContext extends WorkflowContext { private Map submitCounters = new HashMap(); - private AbstractJob job; + private CloudgeneJob job; private Map data = new HashMap(); @@ -150,7 +150,7 @@ public void log(String line) { job.writeLog(line); } - public AbstractJob getJob() { + public CloudgeneJob getJob() { return job; } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java index fd6f0e56..dbbd0657 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java @@ -6,9 +6,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.util.List; -import java.util.Map; -import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlStep; public abstract class CloudgeneStep { @@ -17,7 +15,7 @@ public abstract class CloudgeneStep { private String name; - private AbstractJob job; + private CloudgeneJob job; private List logMessages; @@ -51,11 +49,11 @@ public String getName() { return name; } - public AbstractJob getJob() { + public CloudgeneJob getJob() { return job; } - public void setJob(AbstractJob job) { + public void setJob(CloudgeneJob job) { this.job = job; } @@ -106,8 +104,8 @@ protected boolean executeCommand(List command, CloudgeneContext context, protected boolean executeCommand(List command, CloudgeneContext context, StringBuilder output, File workDir) throws IOException, InterruptedException { - WdlApp app = context.getSettings().getApplicationRepository().getById(job.getApplicationId()).getWdlApp(); - Environment environment = context.getSettings().buildEnvironment().addContext(context).addApplication(app); + Environment environment = context.getSettings().buildEnvironment().addContext(context) + .addApplication(job.getApp()); // set global variables for (int j = 0; j < command.size(); j++) { diff --git a/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java b/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java index 9e8f8ee6..3c1978b3 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java @@ -11,7 +11,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.CloudgeneJob; import cloudgene.mapred.jobs.CloudgeneStep; @@ -29,7 +28,7 @@ public class ExecutableStep implements Runnable { private CloudgeneContext context; - private AbstractJob job; + private CloudgeneJob job; private CloudgeneStep instance; From c1795bfe9461996e2d9297177b8a301e6c202f46 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 9 Sep 2023 16:37:04 +0200 Subject: [PATCH 20/35] Fix issue with email settings --- .../java/cloudgene/mapred/server/responses/ServerResponse.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java b/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java index 35269166..c58ec470 100644 --- a/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java @@ -40,8 +40,7 @@ public static ServerResponse build(Settings settings) { response.setMailPort(mail.get("port")); response.setMailUser(mail.get("user")); response.setMailPassword(mail.get("password")); - response.setMailUser(""); - response.setMailPassword(""); + response.setMailUser(mail.get("user")); response.setMailName(mail.get("name")); } else { From cba304f9d9ce840c0ccb35451c6093556e72207e Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 9 Sep 2023 16:53:34 +0200 Subject: [PATCH 21/35] Add links to nextflow.log, trace and report --- .../cloudgene/mapred/jobs/AbstractJob.java | 6 +++++ .../cloudgene/mapred/jobs/CloudgeneJob.java | 25 +++++++++++++++---- .../mapred/jobs/PersistentWorkflowEngine.java | 13 ++++++++++ .../mapred/jobs/workspace/IWorkspace.java | 2 ++ .../mapred/jobs/workspace/LocalWorkspace.java | 6 +++++ .../mapred/jobs/workspace/S3Workspace.java | 6 +++++ 6 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java index 40afc595..1a4af4a1 100644 --- a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java +++ b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java @@ -104,6 +104,8 @@ abstract public class AbstractJob extends PriorityRunnable { protected List outputParams = new Vector(); + protected CloudgeneParameterOutput logParam = new CloudgeneParameterOutput(); + protected List steps = new Vector(); protected BufferedOutputStream stdOutStream; @@ -255,6 +257,10 @@ public void setOutputParams(List outputParams) { this.outputParams = outputParams; } + public CloudgeneParameterOutput getLogParam() { + return logParam; + } + public void setPositionInQueue(int positionInQueue) { this.positionInQueue = positionInQueue; } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java index 71f514d4..7123f756 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java @@ -88,6 +88,13 @@ public CloudgeneJob(User user, String id, WdlApp app, Map params outputParams.add(newOutput); } + logParam = new CloudgeneParameterOutput(); + logParam.setAdminOnly(true); + logParam.setDownload(true); + logParam.setDescription("Logs"); + logParam.setType(WdlParameterOutputType.LOCAL_FOLDER); + logParam.setJob(this); + } @Override @@ -118,10 +125,11 @@ public boolean setup() { switch (param.getType()) { case HDFS_FILE: case HDFS_FOLDER: - log.info("[Job {}] Setting output param '{}' failed. HDFS support was removed in Cloudgene 3'", getId(), param.getName()); + log.info("[Job {}] Setting output param '{}' failed. HDFS support was removed in Cloudgene 3'", getId(), + param.getName()); throw new RuntimeException("HDFS support was removed in Cloudgene 3"); - case LOCAL_FILE: + case LOCAL_FILE: String filename = workspace.createFile(param.getName(), param.getName()); param.setValue(filename); log.info("[Job {}] Set output file '{}' to '{}'", getId(), param.getName(), param.getValue()); @@ -129,8 +137,8 @@ public boolean setup() { case LOCAL_FOLDER: String folder = workspace.createFolder(param.getName()); - log.info("[Job {}] Set output folder '{}' to '{}'", getId(), param.getName(), param.getValue()); param.setValue(folder); + log.info("[Job {}] Set output folder '{}' to '{}'", getId(), param.getName(), param.getValue()); break; } @@ -142,7 +150,7 @@ public boolean setup() { @Override public boolean execute() { - + try { // evaluate WDL and replace all variables (e.g. ${job_id}) @@ -224,7 +232,7 @@ public boolean executeFailureStep(WdlStep failedStep) { public boolean cleanUp() { log.info("[Job {}] Cleaning up...", getId()); - + try { workspace.cleanup(getId()); } catch (IOException e) { @@ -249,6 +257,13 @@ public boolean after() { } } + log.info("[Job {}] Export logs...", getId()); + List logs = workspace.getLogs(); + for (Download log: logs){ + log.setCount(-1); + } + logParam.setFiles(logs); + return true; } diff --git a/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java b/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java index 5506598c..fb410915 100644 --- a/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java +++ b/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java @@ -79,6 +79,16 @@ protected void jobCompleted(AbstractJob job) { } + if (job.getLogParam().getFiles() != null) { + + for (Download download : job.getLogParam().getFiles()) { + download.setParameterId(job.getLogParam().getId()); + download.setParameter(job.getLogParam()); + downloadDao.insert(download); + } + + } + if (job.getSteps() != null) { StepDao dao2 = new StepDao(database); for (CloudgeneStep step : job.getSteps()) { @@ -142,6 +152,9 @@ protected void jobSubmitted(AbstractJob job) { parameter.setJobId(job.getId()); dao.insert(parameter); } + + + dao.insert(job.getLogParam()); } @Override diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java index 2667ab82..701a6fc1 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java @@ -35,6 +35,8 @@ public interface IWorkspace { public List getDownloads(String url); + public List getLogs(); + public void cleanup(String job) throws IOException; public boolean exists(String path) throws IOException; diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java index 057690d4..ff1f8301 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java @@ -221,4 +221,10 @@ protected String absolute(String path) { return new File(path).getAbsolutePath(); } + @Override + public List getLogs() { + String location = FileUtil.path(workspace, LOGS_DIRECTORY); + return getDownloads(location); + } + } diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java index c9b3e091..ec8ff6e4 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java @@ -245,5 +245,11 @@ public List getDownloads(String url) { return downloads; } + + @Override + public List getLogs() { + String url = location + "/" + job + "/" + LOGS_DIRECTORY; + return getDownloads(url); + } } From 53458ebec37d2902edfb6a744ead46faca67b07c Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 9 Sep 2023 16:57:42 +0200 Subject: [PATCH 22/35] Fix issue with wrong username in jobs list --- .../html/webapp/components/admin/job/list/table/table.stache | 2 +- .../java/cloudgene/mapred/server/responses/JobResponse.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/html/webapp/components/admin/job/list/table/table.stache b/src/main/html/webapp/components/admin/job/list/table/table.stache index 6dd1cebc..29c62384 100644 --- a/src/main/html/webapp/components/admin/job/list/table/table.stache +++ b/src/main/html/webapp/components/admin/job/list/table/table.stache @@ -59,7 +59,7 @@

- {{user.username}}
+ {{username}}

diff --git a/src/main/java/cloudgene/mapred/server/responses/JobResponse.java b/src/main/java/cloudgene/mapred/server/responses/JobResponse.java index 871eecfb..33d71bf5 100644 --- a/src/main/java/cloudgene/mapred/server/responses/JobResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/JobResponse.java @@ -292,7 +292,9 @@ public static JobResponse build(AbstractJob job, User user) { response.setLogs(""); } - response.setUsername(user.getUsername()); + if (job.getUser() != null) { + response.setUsername(job.getUser().getUsername()); + } return response; } From 096e29b3cae309245bba0af831e00db075ebcc70 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 9 Sep 2023 17:25:07 +0200 Subject: [PATCH 23/35] Add workspace to admin panel --- .../admin/settings/general/general.js | 2 ++ .../admin/settings/general/general.stache | 33 ++++++++++++++++--- .../cloudgene/mapred/jobs/Environment.java | 2 ++ .../controller/ServerAdminController.java | 6 ++-- .../server/responses/ServerResponse.java | 20 +++++++++++ .../mapred/server/services/ServerService.java | 6 ++-- .../java/cloudgene/mapred/util/Settings.java | 9 +++-- 7 files changed, 68 insertions(+), 10 deletions(-) diff --git a/src/main/html/webapp/components/admin/settings/general/general.js b/src/main/html/webapp/components/admin/settings/general/general.js index 7efd7763..a9a4d080 100644 --- a/src/main/html/webapp/components/admin/settings/general/general.js +++ b/src/main/html/webapp/components/admin/settings/general/general.js @@ -33,6 +33,8 @@ export default Control.extend({ this.settings.attr('backgroundColor', $(form).find("[name='background-color']").val()); this.settings.attr('foregroundColor', $(form).find("[name='foreground-color']").val()); this.settings.attr('googleAnalytics', $(form).find("[name='google-analytics']").val()); + this.settings.attr('workspaceType', $(form).find("[name='workspaceType']").val()); + this.settings.attr('workspaceLocation', $(form).find("[name='workspaceLocation']").val()); this.settings.save(function (data) { bootbox.alert("Settings updated."); }, function (response) { diff --git a/src/main/html/webapp/components/admin/settings/general/general.stache b/src/main/html/webapp/components/admin/settings/general/general.stache index 3682eca7..99af7577 100644 --- a/src/main/html/webapp/components/admin/settings/general/general.stache +++ b/src/main/html/webapp/components/admin/settings/general/general.stache @@ -6,30 +6,55 @@

+
Service
+
- +
- +
+
+ +
Contact
- +
- +
+
Workspace
+ +
+ + +
+ + +
+ + +
+ +
+ +
Appearance
+
diff --git a/src/main/java/cloudgene/mapred/jobs/Environment.java b/src/main/java/cloudgene/mapred/jobs/Environment.java index 54c79f58..61a0a399 100644 --- a/src/main/java/cloudgene/mapred/jobs/Environment.java +++ b/src/main/java/cloudgene/mapred/jobs/Environment.java @@ -30,6 +30,8 @@ public Environment(Settings settings) { add("SMTP_NAME", settings.getMail().get("name")); add("SMTP_SENDER", settings.getMail().get("name")); } + add("WORKSPACE_TYPE", settings.getExternalWorkspaceType()); + add("WORKSPACE_HOME", settings.getExternalWorkspaceLocation()); } public Environment addContext(CloudgeneContext context) { diff --git a/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java b/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java index 00ac621f..96620006 100644 --- a/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java +++ b/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java @@ -106,10 +106,12 @@ public ServerResponse getSettings() { @Post("/settings/update") public ServerResponse updateSettings(String name, String adminName, String adminMail, String serverUrl, String backgroundColor, String foregroundColor, String googleAnalytics, boolean mail, String mailSmtp, - String mailUser, String mailPassword, String mailPort, String mailName) { + String mailUser, String mailPassword, String mailPort, String mailName, String workspaceType, + String workspaceLocation) { serverService.updateSettings(name, adminName, adminMail, serverUrl, backgroundColor, foregroundColor, - googleAnalytics, String.valueOf(mail), mailSmtp, mailPort, mailUser, mailPassword, mailName); + googleAnalytics, String.valueOf(mail), mailSmtp, mailPort, mailUser, mailPassword, mailName, + workspaceType, workspaceLocation); return ServerResponse.build(application.getSettings()); diff --git a/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java b/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java index c58ec470..ec4efe00 100644 --- a/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java @@ -22,6 +22,8 @@ public class ServerResponse { private String adminName; private String adminMail; private String serverUrl; + private String workspaceType; + private String workspaceLocation; public static ServerResponse build(Settings settings) { ServerResponse response = new ServerResponse(); @@ -32,6 +34,8 @@ public static ServerResponse build(Settings settings) { response.setBackgroundColor(settings.getColors().get("background")); response.setForegroundColor(settings.getColors().get("foreground")); response.setGoogleAnalytics(settings.getGoogleAnalytics()); + response.setWorkspaceType(settings.getExternalWorkspaceType()); + response.setWorkspaceLocation(settings.getExternalWorkspaceLocation()); Map mail = settings.getMail(); if (mail != null) { @@ -160,4 +164,20 @@ public void setMail(boolean mail) { this.mail = mail; } + public void setWorkspaceLocation(String workspaceLocation) { + this.workspaceLocation = workspaceLocation; + } + + public String getWorkspaceLocation() { + return workspaceLocation; + } + + public void setWorkspaceType(String workspaceType) { + this.workspaceType = workspaceType; + } + + public String getWorkspaceType() { + return workspaceType; + } + } diff --git a/src/main/java/cloudgene/mapred/server/services/ServerService.java b/src/main/java/cloudgene/mapred/server/services/ServerService.java index e0fb48ab..9af643c2 100644 --- a/src/main/java/cloudgene/mapred/server/services/ServerService.java +++ b/src/main/java/cloudgene/mapred/server/services/ServerService.java @@ -117,7 +117,7 @@ public String getRoot(User user) { } public void updateSettings(String name, String adminName, String adminMail, String serverUrl, String background_color, String foreground_color, String google_analytics, - String mail, String mail_smtp, String mail_port, String mail_user, String mail_password, String mail_name) { + String mail, String mail_smtp, String mail_port, String mail_user, String mail_password, String mail_name, String workspaceType, String workspaceLocation) { Settings settings = application.getSettings(); settings.setName(name); @@ -127,7 +127,9 @@ public void updateSettings(String name, String adminName, String adminMail, Stri settings.getColors().put("background", background_color); settings.getColors().put("foreground", foreground_color); settings.setGoogleAnalytics(google_analytics); - + settings.getExternalWorkspace().put("type", workspaceType); + settings.getExternalWorkspace().put("location", workspaceLocation); + if (mail != null && mail.equals("true")) { Map mailConfig = new HashMap(); mailConfig.put("smtp", mail_smtp); diff --git a/src/main/java/cloudgene/mapred/util/Settings.java b/src/main/java/cloudgene/mapred/util/Settings.java index 8e3f5bec..7d33640c 100644 --- a/src/main/java/cloudgene/mapred/util/Settings.java +++ b/src/main/java/cloudgene/mapred/util/Settings.java @@ -20,6 +20,7 @@ import cloudgene.mapred.apps.Application; import cloudgene.mapred.apps.ApplicationRepository; import cloudgene.mapred.jobs.Environment; +import cloudgene.mapred.jobs.workspace.LocalWorkspace; import genepi.io.FileUtil; public class Settings { @@ -538,7 +539,9 @@ public void setExternalWorkspace(Map externalWorkspace) { public String getExternalWorkspaceLocation() { if (externalWorkspace == null) { - return ""; + externalWorkspace = new HashMap<>(); + externalWorkspace.put("type", "local"); + externalWorkspace.put("location", getLocalWorkspace()); } if (externalWorkspace.get("location") == null) { @@ -551,7 +554,9 @@ public String getExternalWorkspaceLocation() { public String getExternalWorkspaceType() { if (externalWorkspace == null) { - return ""; + externalWorkspace = new HashMap<>(); + externalWorkspace.put("type", "local"); + externalWorkspace.put("location", getLocalWorkspace()); } if (externalWorkspace.get("type") == null) { From b3ffd7fafce33f1c2e0619d3b2787bdcca340b61 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sat, 9 Sep 2023 20:26:11 +0200 Subject: [PATCH 24/35] Fix issue with filenames in local workspace --- .../java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java index ff1f8301..c573a7df 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java @@ -188,7 +188,7 @@ private void exportFolder(String prefix, File folder, List downloads) File[] files = folder.listFiles(); for (File file : files) { - if (folder.isFile()) { + if (file.isFile()) { Download download = createDownload(prefix, file); downloads.add(download); } else { From 7fbf8ab681cb7382a94521e02cb5b8e80c4771c2 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sun, 10 Sep 2023 09:49:33 +0200 Subject: [PATCH 25/35] Clean up downloads and fix logs on job restart --- .../mapred/database/DownloadDao.java | 5 +- .../mapred/database/ParameterDao.java | 3 - .../cloudgene/mapred/jobs/AbstractJob.java | 20 +++++- .../cloudgene/mapred/jobs/CloudgeneJob.java | 66 +++++++++---------- .../mapred/jobs/CloudgeneParameterOutput.java | 55 ---------------- .../java/cloudgene/mapred/jobs/Download.java | 18 ----- .../mapred/jobs/PersistentWorkflowEngine.java | 21 +++--- .../plugins/nextflow/NextflowBinary.java | 13 +++- .../server/controller/DownloadController.java | 24 ++----- .../server/responses/DownloadResponse.java | 2 - .../mapred/server/services/JobService.java | 13 +++- 11 files changed, 90 insertions(+), 150 deletions(-) diff --git a/src/main/java/cloudgene/mapred/database/DownloadDao.java b/src/main/java/cloudgene/mapred/database/DownloadDao.java index 3d511ac6..c3256763 100644 --- a/src/main/java/cloudgene/mapred/database/DownloadDao.java +++ b/src/main/java/cloudgene/mapred/database/DownloadDao.java @@ -30,13 +30,13 @@ public boolean insert(Download download) { try { Object[] params = new Object[7]; - params[0] = download.getParameterId(); + params[0] = download.getParameter().getId(); params[1] = download.getName(); params[2] = download.getPath(); params[3] = download.getHash(); params[4] = download.getCount(); params[5] = download.getSize(); - params[6] = download.getParameter().getId(); + params[6] = -1; update(sql.toString(), params); @@ -165,7 +165,6 @@ public Object mapRow(ResultSet rs, int row) throws SQLException { result.setName(rs.getString("name")); result.setPath(rs.getString("path")); result.setSize(rs.getString("size")); - result.setParameterId(rs.getInt("parameter_id")); return result; } diff --git a/src/main/java/cloudgene/mapred/database/ParameterDao.java b/src/main/java/cloudgene/mapred/database/ParameterDao.java index 8422d40f..b33d7718 100644 --- a/src/main/java/cloudgene/mapred/database/ParameterDao.java +++ b/src/main/java/cloudgene/mapred/database/ParameterDao.java @@ -138,9 +138,6 @@ public List findAllOutputByJob(AbstractJob job) { DownloadDao downloadDao = new DownloadDao(database); for (CloudgeneParameterOutput parameter : result) { List downloads = downloadDao.findAllByParameter(parameter); - for (Download download : downloads) { - download.setUsername(job.getUser().getUsername()); - } parameter.setFiles(downloads); } diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java index 1a4af4a1..c0486276 100644 --- a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java +++ b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java @@ -104,7 +104,7 @@ abstract public class AbstractJob extends PriorityRunnable { protected List outputParams = new Vector(); - protected CloudgeneParameterOutput logParam = new CloudgeneParameterOutput(); + protected CloudgeneParameterOutput logOutput = null; protected List steps = new Vector(); @@ -257,8 +257,8 @@ public void setOutputParams(List outputParams) { this.outputParams = outputParams; } - public CloudgeneParameterOutput getLogParam() { - return logParam; + public CloudgeneParameterOutput getLogOutput() { + return logOutput; } public void setPositionInQueue(int positionInQueue) { @@ -679,6 +679,20 @@ public boolean isRunning() { return !complete; } + public Download findDownloadByHash(String hash) { + for (CloudgeneParameterOutput param : getOutputParams()) { + if (param.getFiles() == null) { + continue; + } + for (Download download : param.getFiles()) { + if (download.getHash().equals(hash)) { + return download; + } + } + } + return null; + } + abstract public boolean execute(); abstract public boolean setup(); diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java index 7123f756..85a535ad 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java @@ -22,6 +22,8 @@ public class CloudgeneJob extends AbstractJob { + private static final String CLOUDGENE_LOGS_PARAM = "cloudgene_logs"; + private WdlApp app; private String workingDirectory; @@ -36,31 +38,6 @@ public CloudgeneJob() { super(); } - public void loadConfig(WdlApp app) { - - this.app = app; - workingDirectory = app.getPath(); - - // set parameter properties that are not stored in database. - // needed for restart - for (CloudgeneParameterOutput outputParam : outputParams) { - - for (WdlParameterOutput output : app.getWorkflow().getOutputs()) { - - if (outputParam.getName().equals(output.getId())) { - outputParam.setMakeAbsolute(output.isMakeAbsolute()); - outputParam.setAutoExport(output.isAutoExport()); - outputParam.setZip(output.isZip()); - outputParam.setMergeOutput(output.isMergeOutput()); - outputParam.setRemoveHeader(output.isRemoveHeader()); - } - - } - - } - - } - public CloudgeneJob(User user, String id, WdlApp app, Map params) { setComplete(false); this.app = app; @@ -88,13 +65,36 @@ public CloudgeneJob(User user, String id, WdlApp app, Map params outputParams.add(newOutput); } - logParam = new CloudgeneParameterOutput(); - logParam.setAdminOnly(true); - logParam.setDownload(true); - logParam.setDescription("Logs"); - logParam.setType(WdlParameterOutputType.LOCAL_FOLDER); - logParam.setJob(this); + initLogOutput(); + + } + + public void loadApp(WdlApp app) { + + this.app = app; + workingDirectory = app.getPath(); + + // find logOutput and remove from outputs + for (CloudgeneParameterOutput param : getOutputParams()) { + if (!param.getName().equals(CLOUDGENE_LOGS_PARAM)) { + continue; + } + logOutput = param; + } + if (logOutput != null) { + getOutputParams().remove(logOutput); + } + + } + protected void initLogOutput() { + logOutput = new CloudgeneParameterOutput(); + logOutput.setAdminOnly(true); + logOutput.setDownload(true); + logOutput.setDescription("Logs"); + logOutput.setName(CLOUDGENE_LOGS_PARAM); + logOutput.setType(WdlParameterOutputType.LOCAL_FOLDER); + logOutput.setJob(this); } @Override @@ -259,10 +259,10 @@ public boolean after() { log.info("[Job {}] Export logs...", getId()); List logs = workspace.getLogs(); - for (Download log: logs){ + for (Download log : logs) { log.setCount(-1); } - logParam.setFiles(logs); + logOutput.setFiles(logs); return true; } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneParameterOutput.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneParameterOutput.java index f9da18fd..6f150967 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneParameterOutput.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneParameterOutput.java @@ -29,16 +29,6 @@ public class CloudgeneParameterOutput { private String jobId; - private boolean autoExport = false; - - private boolean makeAbsolute = false; - - private boolean zip = false; - - private boolean mergeOutput = false; - - private boolean removeHeader = true; - private boolean adminOnly = false; private String hash = ""; @@ -52,11 +42,6 @@ public CloudgeneParameterOutput(WdlParameterOutput parameter) { setType(parameter.getTypeAsEnum()); setDownload(parameter.isDownload()); setDescription(parameter.getDescription()); - setMakeAbsolute(parameter.isMakeAbsolute()); - setAutoExport(parameter.isAutoExport()); - setZip(parameter.isZip()); - setMergeOutput(parameter.isMergeOutput()); - setRemoveHeader(parameter.isRemoveHeader()); setAdminOnly(parameter.isAdminOnly()); files = new Vector(); } @@ -133,46 +118,6 @@ public String getJobId() { return jobId; } - public boolean isMakeAbsolute() { - return makeAbsolute; - } - - public void setMakeAbsolute(boolean absolute) { - this.makeAbsolute = absolute; - } - - public void setAutoExport(boolean autoExport) { - this.autoExport = autoExport; - } - - public boolean isAutoExport() { - return autoExport; - } - - public void setZip(boolean zip) { - this.zip = zip; - } - - public void setMergeOutput(boolean mergeOutput) { - this.mergeOutput = mergeOutput; - } - - public void setRemoveHeader(boolean removeHeader) { - this.removeHeader = removeHeader; - } - - public boolean isMergeOutput() { - return mergeOutput; - } - - public boolean isRemoveHeader() { - return removeHeader; - } - - public boolean isZip() { - return zip; - } - public void setAdminOnly(boolean adminOnly) { this.adminOnly = adminOnly; } diff --git a/src/main/java/cloudgene/mapred/jobs/Download.java b/src/main/java/cloudgene/mapred/jobs/Download.java index da74ac08..7588a1fe 100644 --- a/src/main/java/cloudgene/mapred/jobs/Download.java +++ b/src/main/java/cloudgene/mapred/jobs/Download.java @@ -9,8 +9,6 @@ public class Download implements Comparable { private int count = 0; private String size; private CloudgeneParameterOutput parameter; - private int parameterId; - private String user; public String getName() { return name; @@ -60,26 +58,10 @@ public CloudgeneParameterOutput getParameter() { return parameter; } - public void setParameterId(int parameterId) { - this.parameterId = parameterId; - } - - public int getParameterId() { - return parameterId; - } - public void decCount() { count--; } - public void setUsername(String user) { - this.user = user; - } - - public String getUsername() { - return user; - } - @Override public int compareTo(Download o) { return name.compareTo(o.getName()); diff --git a/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java b/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java index fb410915..80789ea0 100644 --- a/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java +++ b/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java @@ -68,7 +68,6 @@ protected void jobCompleted(AbstractJob job) { if (parameter.getFiles() != null) { for (Download download : parameter.getFiles()) { - download.setParameterId(parameter.getId()); download.setParameter(parameter); downloadDao.insert(download); } @@ -79,16 +78,15 @@ protected void jobCompleted(AbstractJob job) { } - if (job.getLogParam().getFiles() != null) { + if (job.getLogOutput().getFiles() != null) { - for (Download download : job.getLogParam().getFiles()) { - download.setParameterId(job.getLogParam().getId()); - download.setParameter(job.getLogParam()); + for (Download download : job.getLogOutput().getFiles()) { + download.setParameter(job.getLogOutput()); downloadDao.insert(download); } } - + if (job.getSteps() != null) { StepDao dao2 = new StepDao(database); for (CloudgeneStep step : job.getSteps()) { @@ -111,7 +109,7 @@ protected void jobCompleted(AbstractJob job) { submittedCounters.put("runs", 1); } } - + // write all submitted counters into database for (String name : submittedCounters.keySet()) { Integer value = submittedCounters.get(name); @@ -130,8 +128,8 @@ protected void jobCompleted(AbstractJob job) { } } - - //update job updates (state, endtime, ....) + + // update job updates (state, endtime, ....) dao.update(job); } @@ -152,9 +150,8 @@ protected void jobSubmitted(AbstractJob job) { parameter.setJobId(job.getId()); dao.insert(parameter); } - - - dao.insert(job.getLogParam()); + + dao.insert(job.getLogOutput()); } @Override diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java index 86e559b0..b1253171 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java @@ -158,13 +158,22 @@ public List buildCommand() { nextflow.add("-with-trace"); nextflow.add(trace); + if (new File(trace).exists()){ + new File(trace).delete(); + } nextflow.add("-with-report"); nextflow.add(report); - + if (new File(report).exists()){ + new File(report).delete(); + } + nextflow.add("-with-timeline"); nextflow.add(timeline); - + if (new File(timeline).exists()){ + new File(timeline).delete(); + } + List command = new Vector(); command.add("/bin/bash"); command.add("-c"); diff --git a/src/main/java/cloudgene/mapred/server/controller/DownloadController.java b/src/main/java/cloudgene/mapred/server/controller/DownloadController.java index 5c10147d..e268af99 100644 --- a/src/main/java/cloudgene/mapred/server/controller/DownloadController.java +++ b/src/main/java/cloudgene/mapred/server/controller/DownloadController.java @@ -58,18 +58,9 @@ public MutableHttpResponse downloadExternalResults(String jobId, St DownloadDao dao = new DownloadDao(application.getDatabase()); Download download = dao.findByHash(hash); - // job is running and not in database --> download possible of - // autoexport params + // job is running and not in database if (download == null) { - for (CloudgeneParameterOutput param : job.getOutputParams()) { - if (param.isAutoExport() && param.getFiles() != null) { - for (Download download2 : param.getFiles()) { - if (download2.getHash().equals(hash)) { - download = download2; - } - } - } - } + download = job.findDownloadByHash(hash); } String message = String.format("Job: Downloading file '%s' for job %s", filename, job.getId()); log.info(message); @@ -79,17 +70,14 @@ public MutableHttpResponse downloadExternalResults(String jobId, St @Get("/share/results/{hash}/{filename:.+}") @Secured(SecurityRule.IS_ANONYMOUS) - public MutableHttpResponse downloadPublicLink(String hash, String filename) throws URISyntaxException, IOException { + public MutableHttpResponse downloadPublicLink(String hash, String filename) + throws URISyntaxException, IOException { DownloadDao dao = new DownloadDao(application.getDatabase()); Download download = dao.findByHash(hash); - if (download != null) { - String message = String.format("Job: Anonymously downloading file '%s' (hash %s)", download.getName(), - hash); - log.info(message); - } - + String message = String.format("Job: Anonymously downloading file '%s' (hash %s)", filename, hash); + log.info(message); return downloadService.download(download); } diff --git a/src/main/java/cloudgene/mapred/server/responses/DownloadResponse.java b/src/main/java/cloudgene/mapred/server/responses/DownloadResponse.java index 3c6f2da6..65086132 100644 --- a/src/main/java/cloudgene/mapred/server/responses/DownloadResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/DownloadResponse.java @@ -73,10 +73,8 @@ public void setUser(String user) { public static DownloadResponse build(Download param) { DownloadResponse response = new DownloadResponse(); - response.setParameterId(param.getParameterId()); response.setName(param.getName()); response.setHash(param.getHash()); - response.setUser(param.getUsername()); response.setSize(param.getSize()); response.setCount(param.getCount()); response.setPath(param.getPath()); diff --git a/src/main/java/cloudgene/mapred/server/services/JobService.java b/src/main/java/cloudgene/mapred/server/services/JobService.java index 5ad8a0bc..5b9f0f50 100644 --- a/src/main/java/cloudgene/mapred/server/services/JobService.java +++ b/src/main/java/cloudgene/mapred/server/services/JobService.java @@ -266,7 +266,18 @@ public AbstractJob restart(AbstractJob job) { } - ((CloudgeneJob) job).loadConfig(application.getWdlApp()); + + IWorkspace workspace = workspaceFactory.getDefault(); + + try { + // setup workspace + workspace.setup(job.getId()); + } catch (Exception e) { + throw new JsonHttpStatusException(HttpStatus.BAD_REQUEST, e.getMessage()); + } + job.setWorkspace(workspace); + + ((CloudgeneJob) job).loadApp(application.getWdlApp()); this.application.getWorkflowEngine().restart(job); From bf4a03073c2b25ef390c28d524e9deac5d7cf8e3 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Mon, 11 Sep 2023 10:17:04 +0200 Subject: [PATCH 26/35] Remove delete temp directory on setup job to enable resume --- src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java index 85a535ad..077b6993 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java @@ -102,11 +102,6 @@ public boolean setup() { context = new CloudgeneContext(this); - FileUtil.deleteDirectory(context.getLocalTemp()); - - // create output directories - FileUtil.createDirectory(context.getLocalTemp()); - try { log.info("[Job {}] Setup workspace {}'", getId(), workspace.getName()); context.log("Setup External Workspace on " + workspace.getName()); From 9bd1ee6a7fed6fbebf4b4a0b5815cd17e479f10a Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Mon, 11 Sep 2023 10:41:06 +0200 Subject: [PATCH 27/35] Add `label` to process config --- .../plugins/nextflow/NextflowProcessConfig.java | 10 ++++++++++ .../plugins/nextflow/NextflowProcessRenderer.java | 15 +++++++++------ .../mapred/plugins/nextflow/NextflowStep.java | 3 +++ 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessConfig.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessConfig.java index 515996f5..14c111c9 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessConfig.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessConfig.java @@ -6,6 +6,8 @@ public class NextflowProcessConfig { private String view = DEFAULT_VIEW; + private String label = null; + public String getView() { return view; } @@ -14,4 +16,12 @@ public void setView(String view) { this.view = view; } + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + } diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java index 632e6b27..04d2d3d5 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java @@ -5,18 +5,21 @@ public class NextflowProcessRenderer { public static void render(NextflowProcessConfig config, NextflowProcess process, Message message) { + + String label = (config.getLabel() != null ? config.getLabel() : process.getName()); + switch (config.getView()) { case "progressbar": - NextflowProcessRenderer.renderProgressbar(process, message); + NextflowProcessRenderer.renderProgressbar(label, process, message); break; default: - NextflowProcessRenderer.renderList(process, message); + NextflowProcessRenderer.renderList(label, process, message); } } - public static void renderList(NextflowProcess process, Message message) { + public static void renderList(String label, NextflowProcess process, Message message) { - String text = "" + process.getName() + ""; + String text = "" + label + ""; boolean running = false; boolean ok = true; for (NextflowTask task : process.getTasks()) { @@ -66,9 +69,9 @@ public static void renderList(NextflowProcess process, Message message) { } - public static void renderProgressbar(NextflowProcess process, Message message) { + public static void renderProgressbar(String label, NextflowProcess process, Message message) { - String text = "" + process.getName() + "

"; + String text = "" + label + "

"; int running = 0; boolean ok = true; for (NextflowTask task : process.getTasks()) { diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java index 1e37113b..04cfc817 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java @@ -191,6 +191,9 @@ private void loadProcessConfigs(Object map) { if (processConfig.get("view") != null) { config.setView(processConfig.get("view").toString()); } + if (processConfig.get("label") != null) { + config.setLabel(processConfig.get("label").toString()); + } configs.put(process, config); } } From da6f87b04ef302e3242d9e429d5b03a7f2d6493d Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Mon, 11 Sep 2023 12:37:02 +0200 Subject: [PATCH 28/35] Improve Nextflow feedback rendering --- .../core/job/detail/steps/steps.stache | 2 +- .../nextflow/NextflowProcessRenderer.java | 177 +++++++++--------- src/main/resources/templates/list.html | 15 ++ src/main/resources/templates/progressbar.html | 6 + 4 files changed, 115 insertions(+), 85 deletions(-) create mode 100644 src/main/resources/templates/list.html create mode 100644 src/main/resources/templates/progressbar.html diff --git a/src/main/html/webapp/components/core/job/detail/steps/steps.stache b/src/main/html/webapp/components/core/job/detail/steps/steps.stache index a0f3e210..1e61f682 100644 --- a/src/main/html/webapp/components/core/job/detail/steps/steps.stache +++ b/src/main/html/webapp/components/core/job/detail/steps/steps.stache @@ -33,7 +33,7 @@ {{#is(type, 3)}}
-

{{{replaceNL(message)}}}

+

{{{replaceNL(message)}}}

diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java index 04d2d3d5..e67866bb 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java @@ -1,125 +1,134 @@ package cloudgene.mapred.plugins.nextflow; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +import org.codehaus.groovy.control.CompilationFailedException; + import cloudgene.mapred.jobs.Message; +import groovy.text.SimpleTemplateEngine; +import groovy.text.Template; public class NextflowProcessRenderer { - public static void render(NextflowProcessConfig config, NextflowProcess process, Message message) { + private static final String TEMPLATES_LIST = "/templates/list.html"; - String label = (config.getLabel() != null ? config.getLabel() : process.getName()); + private static final String TEMPLATES_PROGRESSBAR = "/templates/progressbar.html"; - switch (config.getView()) { - case "progressbar": - NextflowProcessRenderer.renderProgressbar(label, process, message); - break; - default: - NextflowProcessRenderer.renderList(label, process, message); - } - } + private static final String FAILED = "FAILED"; - public static void renderList(String label, NextflowProcess process, Message message) { + private static final String KILLED = "KILLED"; - String text = "" + label + ""; - boolean running = false; - boolean ok = true; - for (NextflowTask task : process.getTasks()) { + private static final String VIEW_PROGRESSBAR = "progressbar"; - // TODO: use templates. + private static final String TRACE_STATUS = "status"; - String status = (String) task.getTrace().get("status"); + private static final String SUBMITTED = "SUBMITTED"; - if (status.equals("RUNNING") || status.equals("SUBMITTED")) { - running = true; - } - if (!status.equals("COMPLETED")) { - ok = false; + private static final String COMPLETED = "COMPLETED"; - } - text += "
"; + private static final String RUNNING = "RUNNING"; - text += (String) task.getTrace().get("name"); - if (status.equals("RUNNING")) { - text += "..."; - } - if (status.equals("COMPLETED")) { - text += " "; - } - if (status.equals("KILLED") || status.equals("FAILED")) { - text += " "; - } - - if (task.getLogText() != null) { - text += "
" + task.getLogText(); - } + public static final Map CACHE = new HashMap(); - text += ""; - } + public static void render(NextflowProcessConfig config, NextflowProcess process, Message message) { - message.setMessage(text); + String label = (config.getLabel() != null ? config.getLabel() : process.getName()); - if (running) { - message.setType(Message.RUNNING); - } else { - if (ok) { - message.setType(Message.OK); - } else { - message.setType(Message.ERROR); - } + switch (config.getView()) { + case VIEW_PROGRESSBAR: + NextflowProcessRenderer.renderAsProgressbar(label, process, message); + break; + default: + NextflowProcessRenderer.renderAsList(label, process, message); } + } + + public static void renderAsList(String label, NextflowProcess process, Message message) { + render(label, TEMPLATES_LIST, process, message); + } + public static void renderAsProgressbar(String label, NextflowProcess process, Message message) { + render(label, TEMPLATES_PROGRESSBAR, process, message); } - public static void renderProgressbar(String label, NextflowProcess process, Message message) { + public static void render(String label, String template, NextflowProcess process, Message message) { - String text = "" + label + "

"; int running = 0; - boolean ok = true; + int completed = 0; + int failed = 0; for (NextflowTask task : process.getTasks()) { - String status = (String) task.getTrace().get("status"); + String status = (String) task.getTrace().get(TRACE_STATUS); - if (status.equals("RUNNING") || status.equals("SUBMITTED")) { + if (status.equals(RUNNING) || status.equals(SUBMITTED)) { running++; } - if (!status.equals("COMPLETED")) { - ok = false; - } - text += " "; - - String style = ""; - String name = " "; - - if (status.equals("COMPLETED")) { - style = "badge-success"; - name = "OK"; - } - - if (status.equals("RUNNING")) { - style = "badge-info"; - name = "..."; + if (status.equals(COMPLETED)) { + completed++; } - - if (status.equals("KILLED") || status.equals("FAILED")) { - style = "badge-danger"; - name = "X"; + if (status.equals(FAILED) || status.equals(KILLED)) { + failed++; } - - text += "" + name + ""; - } - message.setMessage(text); - + int total = running + completed + failed; + Map bindings = new HashMap(); + bindings.put("label", label); + bindings.put("total", total); + bindings.put("running", running); + bindings.put("completed", completed); + bindings.put("failed", failed); + bindings.put("tasks", process.getTasks()); + + try { + String text = renderTemplate(template, bindings); + message.setMessage(text); + } catch (Exception e) { + message.setMessage("Template could not be renderer: " + e.toString()); + } if (running > 0) { message.setType(Message.RUNNING); + } else if (completed > 0) { + message.setType(Message.OK); } else { - if (ok) { - message.setType(Message.OK); - } else { - message.setType(Message.ERROR); - } + message.setType(Message.ERROR); + } + + } + + public static String renderTemplate(String path, Map bindings) + throws CompilationFailedException, ClassNotFoundException, IOException, URISyntaxException { + Template template = getTemplate(path); + String rendered = template.make(bindings).toString(); + return rendered.replaceAll("\n", ""); + } + + public static synchronized Template getTemplate(String path) + throws IOException, URISyntaxException, CompilationFailedException, ClassNotFoundException { + Template template = CACHE.get(path); + + if (template != null) { + return template; } + SimpleTemplateEngine engine = new SimpleTemplateEngine(); + String content = readTemplate(path); + template = engine.createTemplate(content); + CACHE.put(path, template); + return template; + } + + private static String readTemplate(String path) throws IOException, URISyntaxException { + URI uri = NextflowProcess.class.getResource(path).toURI(); + Path test = Paths.get(uri); + return Files.readString(test); } } diff --git a/src/main/resources/templates/list.html b/src/main/resources/templates/list.html new file mode 100644 index 00000000..dc07d363 --- /dev/null +++ b/src/main/resources/templates/list.html @@ -0,0 +1,15 @@ +${label} (${completed}/${total})
+<% for (task in tasks) { %> + + <%= task.getTrace().get("name") %> + <% if (task.getTrace().get("status").equals("COMPLETED")) { %> +   + <% } %> + <% if (task.getTrace().get("status").equals("RUNNING")) { %> + ... + <% } %> + <% if (task.getTrace().get("status").equals("KILLED")) { %> +   + <% } %> +
+<% } %> \ No newline at end of file diff --git a/src/main/resources/templates/progressbar.html b/src/main/resources/templates/progressbar.html new file mode 100644 index 00000000..490459c8 --- /dev/null +++ b/src/main/resources/templates/progressbar.html @@ -0,0 +1,6 @@ +${label} (${completed}/${total})
+
+
+
+
+
\ No newline at end of file From 11f782b8169e2fd694f480f6e6594aab1ca4c6db Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Mon, 11 Sep 2023 16:48:44 +0200 Subject: [PATCH 29/35] Add job name to environment --- src/main/java/cloudgene/mapred/jobs/Environment.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/cloudgene/mapred/jobs/Environment.java b/src/main/java/cloudgene/mapred/jobs/Environment.java index 61a0a399..07038894 100644 --- a/src/main/java/cloudgene/mapred/jobs/Environment.java +++ b/src/main/java/cloudgene/mapred/jobs/Environment.java @@ -36,6 +36,7 @@ public Environment(Settings settings) { public Environment addContext(CloudgeneContext context) { add("JOB_ID", context.getJobId()); + add("JOB_NAME", context.getJobName()); add("USER_NAME", context.getUser().getUsername()); add("USER_EMAIL", context.getUser().getMail()); add("USER_FULL_NAME", context.getUser().getFullName()); From 240462dcedbae493d4451c319ef6ff78888c8843 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Mon, 11 Sep 2023 16:57:59 +0200 Subject: [PATCH 30/35] Set java version to 11 --- pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a8d73db4..4b3eda68 100644 --- a/pom.xml +++ b/pom.xml @@ -45,8 +45,9 @@ UTF-8 UTF-8 jar - 1.8 - 8 + 11 + 11 + 11 3.3.4 cloudgene.mapred.server.Application netty From d8105db2b5dea13ea447e645211bfed855f55991 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Tue, 12 Sep 2023 17:48:58 +0200 Subject: [PATCH 31/35] Remove deprecated properties from job --- .../components/core/job/detail/detail.js | 118 +++++++------ src/main/html/webapp/models/job.js | 44 +++-- .../cloudgene/mapred/database/JobDao.java | 15 +- .../cloudgene/mapred/jobs/AbstractJob.java | 159 +++++------------- .../cloudgene/mapred/jobs/CloudgeneJob.java | 7 - .../cloudgene/mapred/jobs/WorkflowEngine.java | 14 +- .../cloudgene/mapred/jobs/queue/Queue.java | 10 +- .../mapred/server/responses/JobResponse.java | 83 ++------- .../cloudgene/mapred/database/JobDaoTest.java | 18 +- .../mapred/jobs/WorkflowEngineTest.java | 129 ++++---------- .../mapred/jobs/WrongWorkspaceTest.java | 2 +- .../cloudgene/mapred/steps/TestCommand.java | 32 ++-- 12 files changed, 191 insertions(+), 440 deletions(-) diff --git a/src/main/html/webapp/components/core/job/detail/detail.js b/src/main/html/webapp/components/core/job/detail/detail.js index 5ef45138..288c60a1 100644 --- a/src/main/html/webapp/components/core/job/detail/detail.js +++ b/src/main/html/webapp/components/core/job/detail/detail.js @@ -19,7 +19,7 @@ import template from './detail.stache'; export default Control.extend({ - "init": function(element, options) { + "init": function (element, options) { var that = this; this.active = true; @@ -28,55 +28,55 @@ export default Control.extend({ } JobDetails.findOne({ - id: options.job - }, function(job) { - - $(element).html(template({ - job: job, - tab: options.tab, - admin: options.appState.attr('user').attr('admin') - })); - - switch (options.tab) { - case 'results': - new ResultsControl("#tab-results", { - job: job - }); - break; - - case 'steps': - new StepsControl("#tab-steps", { - job: job - }); - break; - - case 'logs': - new LogsControl("#tab-logs", { - job: job - }); - break; - - default: - } - - $('[data-toggle="tooltip"]').tooltip() - - that.job = job; - that.refresh(); - - }, function(response) { - new ErrorPage(that.element, response); + id: options.job + }, function (job) { + + $(element).html(template({ + job: job, + tab: options.tab, + admin: options.appState.attr('user').attr('admin') + })); + + switch (options.tab) { + case 'results': + new ResultsControl("#tab-results", { + job: job + }); + break; + + case 'steps': + new StepsControl("#tab-steps", { + job: job + }); + break; + + case 'logs': + new LogsControl("#tab-logs", { + job: job + }); + break; + + default: } + $('[data-toggle="tooltip"]').tooltip() + + that.job = job; + that.refresh(); + + }, function (response) { + new ErrorPage(that.element, response); + } + ); }, // delete job - '#delete-btn click': function(el, ev) { + '#delete-btn click': function (el, ev) { var that = this; - bootbox.confirm("Are you sure you want to delete " + that.job.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to delete " + that.job.attr('id') + "?", function (result) { if (result) { var okButton = $("button[data-bb-handler='confirm']"); @@ -85,11 +85,11 @@ export default Control.extend({ var cancelButton = $("button[data-bb-handler='cancel']"); cancelButton.hide('hide'); - that.job.destroy(function() { + that.job.destroy(function () { // go to jobs page bootbox.hideAll(); window.location.hash = "!pages/jobs"; - }, function(response) { + }, function (response) { bootbox.hideAll(); showErrorDialog("Job could not be deleted", response); }); @@ -103,10 +103,10 @@ export default Control.extend({ // cancel job - '#cancel-btn click': function(el, ev) { + '#cancel-btn click': function (el, ev) { var that = this; - bootbox.confirm("Are you sure you want to cancel " + that.job.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to cancel " + that.job.attr('id') + "?", function (result) { if (result) { var okButton = $("button[data-bb-handler='confirm']"); @@ -118,10 +118,10 @@ export default Control.extend({ var operation = new JobOperation(); operation.attr('id', that.job.attr('id')); operation.attr('action', 'cancel'); - operation.save(function() { + operation.save(function () { bootbox.hideAll(); that.refresh(); - }, function(response) { + }, function (response) { bootbox.hideAll(); showErrorDialog("Job could not be canceld", response); }); @@ -133,10 +133,10 @@ export default Control.extend({ }, - '#restart-btn click': function(el, ev) { + '#restart-btn click': function (el, ev) { var that = this; - bootbox.confirm("Are you sure you want to restart " + that.job.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to restart " + that.job.attr('id') + "?", function (result) { if (result) { var okButton = $("button[data-bb-handler='confirm']"); @@ -148,10 +148,10 @@ export default Control.extend({ var operation = new JobOperation(); operation.attr('id', that.job.attr('id')); operation.attr('action', 'restart'); - operation.save(function() { + operation.save(function () { bootbox.hideAll(); window.location.hash = "#!pages/jobs"; - }, function(response) { + }, function (response) { bootbox.hideAll(); showErrorDialog("Job could not be restarted", response); }); @@ -165,40 +165,38 @@ export default Control.extend({ // refresh if job is running - refresh: function() { + refresh: function () { var that = this; if (!JobRefresher.needsUpdate(that.job)) { return; } Job.findOne({ id: that.job.id - }, function(currentJob) { + }, function (currentJob) { currentJob.syncTime(); that.job.attr('state', currentJob.attr('state')); that.job.attr('startTime', currentJob.attr('startTime')); that.job.attr('endTime', currentJob.attr('endTime')); - that.job.attr('setupStartTime', currentJob.attr('setupStartTime')); - that.job.attr('setupEndTime', currentJob.attr('setupEndTime')); that.job.attr('steps', currentJob.attr('steps')); that.job.attr('positionInQueue', currentJob.attr('positionInQueue')); // needs refresh if (JobRefresher.needsUpdate(currentJob) && that.active) { - setTimeout(function() { + setTimeout(function () { that.refresh(); }, 5000); } else { // updates details (results, startTime, endTime, ...) JobDetails.findOne({ id: that.job.id - }, function(job) { + }, function (job) { if (that.active) { var router = canRoute.router; router.reload(); } - }, function(response) { + }, function (response) { new ErrorPage(that.element, response); }); @@ -209,7 +207,7 @@ export default Control.extend({ }, - destroy: function() { + destroy: function () { this.active = false; Control.prototype.destroy.call(this); } @@ -218,6 +216,6 @@ export default Control.extend({ var JobRefresher = {}; -JobRefresher.needsUpdate = function(job) { +JobRefresher.needsUpdate = function (job) { return job.attr("state") == 1 || job.attr("state") == 2 || job.attr("state") == 3; }; diff --git a/src/main/html/webapp/models/job.js b/src/main/html/webapp/models/job.js index 5477eba8..d4acfe9e 100644 --- a/src/main/html/webapp/models/job.js +++ b/src/main/html/webapp/models/job.js @@ -9,33 +9,27 @@ export default Model.extend({ destroy: 'DELETE /api/v2/jobs/{id}', }, { - 'syncTime': function() { + 'syncTime': function () { if (this.attr('startTime') > 0 && this.attr('endTime') === 0) { this.attr('endTime', this.attr('currentTime')); } else { this.attr('endTime', this.attr('endTime')); } - if (this.attr('setupStartTime') > 0 && this.attr('setupEndTime') === 0) { - this.attr('setupEndTime', this.attr('currentTime')); - } else { - this.attr('setupEndTime', this.attr('setupEndTime')); - } - }, define: { 'longName': { - get: function() { + get: function () { return this.attr('id') != this.attr('name') ? this.attr('name') : this.attr('id'); } }, 'executionTime': { - get: function() { - var start = this.attr('setupStartTime') + this.attr('startTime'); - var end = this.attr('setupEndTime') + this.attr('endTime'); + get: function () { + var start = this.attr('startTime'); + var end = this.attr('endTime'); var current = this.attr('currentTime'); if (start === 0 && end === 0) { @@ -61,7 +55,7 @@ export default Model.extend({ }, 'stateAsText': { - get: function() { + get: function () { if (this.attr('state') == 1) { return 'Waiting'; } else if (this.attr('state') == 2) { @@ -85,7 +79,7 @@ export default Model.extend({ }, 'stateAsClass': { - get: function() { + get: function () { if (this.attr('state') == '-1') { return 'dark'; @@ -119,7 +113,7 @@ export default Model.extend({ }, 'stateAsImage': { - get: function() { + get: function () { if (this.attr('state') == '-1') { return "fas fa-moon"; @@ -155,67 +149,67 @@ export default Model.extend({ }, 'isInQueue': { - get: function() { + get: function () { return this.attr('state') == 1 && this.attr('positionInQueue') != -1; } }, 'isPending': { - get: function() { + get: function () { return this.attr('state') == '-1'; } }, 'isRetired': { - get: function() { + get: function () { return this.attr('state') == '7'; } }, 'willBeRetired': { - get: function() { + get: function () { return this.attr('state') == 8 || this.attr('state') == 9; } }, 'canResetCounters': { - get: function() { + get: function () { return this.attr('state') > 3; } }, 'canSendRetireNotification': { - get: function() { + get: function () { return this.attr('state') > 3 && this.attr('state') != '8' && this.attr('state') != '9'; } }, 'canIncreaseRetireDate': { - get: function() { + get: function () { return this.attr('state') == '8' || this.attr('state') == '9'; } }, 'canShowLog': { - get: function() { + get: function () { return this.attr('logs') != undefined && this.attr('logs') != ''; } }, 'canCancel': { - get: function() { + get: function () { return this.attr('state') <= '3' && this.attr('state') != '-1'; } }, 'canRetireJob': { - get: function() { + get: function () { return this.attr('state') > '3' && (this.attr('state') == '4' || this.attr('state') != '5' || this.attr('state') != '6'); } }, 'canDelete': { - get: function() { + get: function () { return this.attr('state') > '3' || this.attr('state') == '-1'; } } diff --git a/src/main/java/cloudgene/mapred/database/JobDao.java b/src/main/java/cloudgene/mapred/database/JobDao.java index 39132ec3..d2e8b407 100644 --- a/src/main/java/cloudgene/mapred/database/JobDao.java +++ b/src/main/java/cloudgene/mapred/database/JobDao.java @@ -47,9 +47,9 @@ public boolean insert(AbstractJob job) { params[8] = job.getApplication(); params[9] = job.getApplicationId(); params[10] = job.getSubmittedOn(); - params[11] = job.getFinishedOn(); - params[12] = job.getSetupStartTime(); - params[13] = job.getSetupEndTime(); + params[11] = job.getEndTime(); + params[12] = -1; + params[13] = -1; params[14] = trimToLength(job.getUserAgent(), 350); update(sql.toString(), params); @@ -86,9 +86,9 @@ public boolean update(AbstractJob job) { params[8] = job.getApplication(); params[9] = job.getApplicationId(); params[10] = job.getSubmittedOn(); - params[11] = job.getFinishedOn(); - params[12] = job.getSetupStartTime(); - params[13] = job.getSetupEndTime(); + params[11] = job.getEndTime(); + params[12] = -1; + params[13] = -1; params[14] = job.getId(); update(sql.toString(), params); @@ -485,9 +485,6 @@ public AbstractJob mapRow(ResultSet rs, int row) throws SQLException { job.setApplication(rs.getString("job.application")); job.setApplicationId(rs.getString("job.application_id")); job.setSubmittedOn(rs.getLong("job.submitted_on")); - job.setFinishedOn(rs.getLong("job.finished_on")); - job.setSetupStartTime(rs.getLong("job.setup_start_time")); - job.setSetupEndTime(rs.getLong("job.setup_end_time")); job.setUserAgent(rs.getString("job.user_agent")); return job; diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java index c0486276..868c6c55 100644 --- a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java +++ b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java @@ -68,14 +68,8 @@ abstract public class AbstractJob extends PriorityRunnable { private long endTime = 0; - private long setupStartTime = 0; - - private long setupEndTime = 0; - private long submittedOn = 0; - private long finishedOn = 0; - private String name; private User user; @@ -90,14 +84,6 @@ abstract public class AbstractJob extends PriorityRunnable { private String error = ""; - private int progress = -1; - - private boolean setupComplete = false; - - private boolean setupRunning = false; - - private boolean complete = true; - private int positionInQueue = -1; protected List inputParams = new Vector(); @@ -161,22 +147,6 @@ public void setEndTime(long endTime) { this.endTime = endTime; } - public void setSetupStartTime(long setupStartTime) { - this.setupStartTime = setupStartTime; - } - - public long getSetupStartTime() { - return setupStartTime; - } - - public void setSetupEndTime(long setupEndTime) { - this.setupEndTime = setupEndTime; - } - - public long getSetupEndTime() { - return setupEndTime; - } - public void setSubmittedOn(long submitedOn) { this.submittedOn = submitedOn; } @@ -185,14 +155,6 @@ public long getSubmittedOn() { return submittedOn; } - public void setFinishedOn(long finishedOn) { - this.finishedOn = finishedOn; - } - - public long getFinishedOn() { - return finishedOn; - } - public String getName() { return name; } @@ -217,14 +179,6 @@ public void setError(String error) { this.error = error; } - public void setProgress(int progress) { - this.progress = progress; - } - - public int getProgress() { - return progress; - } - public void setDeletedOn(long deletedOn) { this.deletedOn = deletedOn; } @@ -288,63 +242,58 @@ public boolean afterSubmission() { } catch (Exception e1) { - // setEndTime(System.currentTimeMillis()); - - setState(AbstractJob.STATE_FAILED); log.error("Job " + getId() + ": initialization failed.", e1); writeLog("Initialization failed: " + e1.getLocalizedMessage()); - setSetupComplete(false); - state = AbstractJob.STATE_FAILED; + setState(STATE_FAILED); return false; } } - public void runInstallationAndResolveAppLinks() { + public boolean runInstallationAndResolveAppLinks() { Settings settings = getSettings(); ApplicationRepository repository = settings.getApplicationRepository(); // resolve application links for (CloudgeneParameterInput input : getInputParams()) { - if (input.getType() == WdlParameterInputType.APP_LIST) { - String value = input.getValue(); - String linkedAppId = value; - if (value.startsWith("apps@")) { - linkedAppId = value.replaceAll("apps@", ""); - } + if (input.getType() != WdlParameterInputType.APP_LIST) { + continue; + } + String value = input.getValue(); + String linkedAppId = value; + if (value.startsWith("apps@")) { + linkedAppId = value.replaceAll("apps@", ""); + } - if (value.isEmpty()) { - continue; - } - Application linkedApp = repository.getByIdAndUser(linkedAppId, getUser()); - if (linkedApp == null) { - String error = "Application " + linkedAppId + " is not installed or wrong permissions."; - log.info(error); - writeOutput(error); - setError(error); - setSetupComplete(false); - setState(AbstractJob.STATE_FAILED); - return; - } - // update environment variables - Environment environment = settings.buildEnvironment().addApplication(linkedApp.getWdlApp()) - .addContext(context); - Map properties = linkedApp.getWdlApp().getProperties(); - for (String property : properties.keySet()) { - Object propertyValue = properties.get(property); - if (propertyValue instanceof String) { - propertyValue = environment.resolve(propertyValue.toString()); - } - properties.put(property, propertyValue); + if (value.isEmpty()) { + continue; + } + Application linkedApp = repository.getByIdAndUser(linkedAppId, getUser()); + if (linkedApp == null) { + String error = "Application " + linkedAppId + " is not installed or wrong permissions."; + log.info(error); + writeOutput(error); + setError(error); + return false; + } + // update environment variables + Environment environment = settings.buildEnvironment().addApplication(linkedApp.getWdlApp()) + .addContext(context); + Map properties = linkedApp.getWdlApp().getProperties(); + for (String property : properties.keySet()) { + Object propertyValue = properties.get(property); + if (propertyValue instanceof String) { + propertyValue = environment.resolve(propertyValue.toString()); } + properties.put(property, propertyValue); + } - getContext().setData(input.getName(), properties); + getContext().setData(input.getName(), properties); - } } - setSetupComplete(true); + return true; } @@ -357,16 +306,12 @@ public void run() { log.info("[Job {}] Setup job...", getId()); setState(AbstractJob.STATE_RUNNING); + setStartTime(System.currentTimeMillis()); - setSetupStartTime(System.currentTimeMillis()); - setSetupRunning(true); - runInstallationAndResolveAppLinks(); - setSetupEndTime(System.currentTimeMillis()); - setSetupRunning(false); - if (!isSetupComplete()) { + if (!runInstallationAndResolveAppLinks()) { log.info("[Job {}] Setup failed.", getId()); - setFinishedOn(System.currentTimeMillis()); - setComplete(true); + setEndTime(System.currentTimeMillis()); + setState(AbstractJob.STATE_FAILED); return; } @@ -496,10 +441,8 @@ public void cancel() { writeLog("Canceled by user."); log.info("[Job {}]: canceld by user.", getId()); - /* - * if (state == STATE_RUNNING) { closeStdOutFiles(); } - */ canceld = true; + setEndTime(System.currentTimeMillis()); setState(AbstractJob.STATE_CANCELED); } @@ -584,22 +527,6 @@ public void setSteps(List steps) { this.steps = steps; } - public void setSetupComplete(boolean setupComplete) { - this.setupComplete = setupComplete; - } - - public boolean isSetupComplete() { - return setupComplete; - } - - public void setSetupRunning(boolean setupRunning) { - this.setupRunning = setupRunning; - } - - public boolean isSetupRunning() { - return setupRunning; - } - public CloudgeneContext getContext() { return context; } @@ -663,20 +590,12 @@ public String getApplicationId() { return applicationId; } - public void setComplete(boolean complete) { - this.complete = complete; - } - - public boolean isComplete() { - return this.complete; - } - public boolean isCanceld() { return canceld; } public boolean isRunning() { - return !complete; + return state == STATE_EXPORTING || state == STATE_RUNNING || state == STATE_WAITING; } public Download findDownloadByHash(String hash) { diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java index 077b6993..895bcbb7 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java @@ -18,7 +18,6 @@ import cloudgene.mapred.wdl.WdlParameterOutput; import cloudgene.mapred.wdl.WdlParameterOutputType; import cloudgene.mapred.wdl.WdlStep; -import genepi.io.FileUtil; public class CloudgeneJob extends AbstractJob { @@ -39,7 +38,6 @@ public CloudgeneJob() { } public CloudgeneJob(User user, String id, WdlApp app, Map params) { - setComplete(false); this.app = app; setId(id); setUser(user); @@ -309,12 +307,7 @@ public void kill() { public void updateProgress() { if (executor != null) { - executor.updateProgress(); - setProgress(executor.getProgress()); - - } else { - setProgress(-1); } } diff --git a/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java b/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java index 42733522..c6e4d0f0 100644 --- a/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java +++ b/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java @@ -35,9 +35,8 @@ public PriorityRunnable createRunnable(AbstractJob job) { @Override public void onComplete(AbstractJob job) { - job.setFinishedOn(System.currentTimeMillis()); + job.setEndTime(System.currentTimeMillis()); jobCompleted(job); - job.setComplete(true); } }; @@ -58,10 +57,8 @@ public void submit(AbstractJob job, long priority) { if (okey) { longTimeQueue.submit(job); } else { - job.setFinishedOn(System.currentTimeMillis()); + job.setEndTime(System.currentTimeMillis()); statusUpdated(job); - job.setComplete(true); - } } @@ -75,9 +72,6 @@ public void restart(AbstractJob job, long priority) { job.setSubmittedOn(System.currentTimeMillis()); job.setStartTime(0); job.setEndTime(0); - job.setSetupStartTime(0); - job.setSetupEndTime(0); - job.setFinishedOn(0); job.setState(AbstractJob.STATE_WAITING); statusUpdated(job); @@ -85,10 +79,8 @@ public void restart(AbstractJob job, long priority) { if (okey) { longTimeQueue.submit(job); } else { - job.setFinishedOn(System.currentTimeMillis()); + job.setEndTime(System.currentTimeMillis()); statusUpdated(job); - job.setComplete(true); - } } diff --git a/src/main/java/cloudgene/mapred/jobs/queue/Queue.java b/src/main/java/cloudgene/mapred/jobs/queue/Queue.java index 29d1d8a3..83cdd4bb 100644 --- a/src/main/java/cloudgene/mapred/jobs/queue/Queue.java +++ b/src/main/java/cloudgene/mapred/jobs/queue/Queue.java @@ -77,11 +77,6 @@ public void cancel(AbstractJob job) { if (job.getState() == AbstractJob.STATE_RUNNING || job.getState() == AbstractJob.STATE_EXPORTING) { log.info(name + ": Cancel running job " + job.getId() + "..."); - - - if (job.getSetupStartTime() > 0 && job.getSetupEndTime() == 0){ - job.setSetupEndTime(System.currentTimeMillis()); - } job.kill(); job.cancel(); @@ -106,10 +101,7 @@ public void cancel(AbstractJob job) { scheduler.kill(runnable); runnables.remove(job); } - if (job.getSetupStartTime() > 0 && job.getSetupEndTime() == 0){ - job.setSetupEndTime(System.currentTimeMillis()); - } - + job.cancel(); queue.remove(job); futures.remove(job); diff --git a/src/main/java/cloudgene/mapred/server/responses/JobResponse.java b/src/main/java/cloudgene/mapred/server/responses/JobResponse.java index 33d71bf5..6f550f00 100644 --- a/src/main/java/cloudgene/mapred/server/responses/JobResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/JobResponse.java @@ -17,24 +17,18 @@ public class JobResponse { private String application; private String applicationId; private boolean canceld; - private boolean complete; private long deletedOn; private long endTime; - private long finishedOn; private String id; private String name; private String logs = ""; private int state; private int positionInQueue; private String userAgent; - private int progress; private long startTime; - private long setupStartTime; - private long setupEndTime; private long submittedOn; - private boolean setupComplete; - private boolean setupRunning; private String username; + private long currentTime; @JsonProperty("steps") private List stepResponses; @@ -66,14 +60,6 @@ public void setCanceld(boolean canceld) { this.canceld = canceld; } - public boolean isComplete() { - return complete; - } - - public void setComplete(boolean complete) { - this.complete = complete; - } - public long getDeletedOn() { return deletedOn; } @@ -90,14 +76,6 @@ public void setEndTime(long endTime) { this.endTime = endTime; } - public long getFinishedOn() { - return finishedOn; - } - - public void setFinishedOn(long finishedOn) { - this.finishedOn = finishedOn; - } - public String getId() { return id; } @@ -145,15 +123,7 @@ public String getUserAgent() { public void setUserAgent(String userAgent) { this.userAgent = userAgent; } - - public int getProgress() { - return progress; - } - - public void setProgress(int progress) { - this.progress = progress; - } - + public long getStartTime() { return startTime; } @@ -162,22 +132,6 @@ public void setStartTime(long startTime) { this.startTime = startTime; } - public long getSetupStartTime() { - return setupStartTime; - } - - public void setSetupStartTime(long setupStartTime) { - this.setupStartTime = setupStartTime; - } - - public long getSetupEndTime() { - return setupEndTime; - } - - public void setSetupEndTime(long setupEndTime) { - this.setupEndTime = setupEndTime; - } - public long getSubmittedOn() { return submittedOn; } @@ -186,22 +140,6 @@ public void setSubmittedOn(long submittedOn) { this.submittedOn = submittedOn; } - public boolean isSetupComplete() { - return setupComplete; - } - - public void setSetupComplete(boolean setupComplete) { - this.setupComplete = setupComplete; - } - - public boolean isSetupRunning() { - return setupRunning; - } - - public void setSetupRunning(boolean setupRunning) { - this.setupRunning = setupRunning; - } - public List getParameterOutputResponse() { return parameterOutputResponse; } @@ -225,6 +163,14 @@ public String getLogs() { public void setLogs(String logs) { this.logs = logs; } + + public void setCurrentTime(long currentTime) { + this.currentTime = currentTime; + } + + public long getCurrentTime() { + return currentTime; + } public static JobResponse build(AbstractJob job, User user) { @@ -257,7 +203,6 @@ public static JobResponse build(AbstractJob job, User user) { response.setApplication(job.getApplication()); response.setApplicationId(job.getApplicationId()); response.setCanceld(job.isCanceld()); - response.setComplete(job.isComplete()); response.setName(job.getName()); response.setId(job.getId()); @@ -265,19 +210,13 @@ public static JobResponse build(AbstractJob job, User user) { response.setLogs(job.getLogs()); response.setPositionInQueue(job.getPositionInQueue()); - response.setProgress(job.getProgress()); response.setUserAgent(job.getUserAgent()); - response.setSetupComplete(job.isSetupComplete()); - response.setSetupRunning(job.isRunning()); response.setDeletedOn(job.getDeletedOn()); response.setStartTime(job.getStartTime()); response.setEndTime(job.getEndTime()); - response.setFinishedOn(job.getFinishedOn()); response.setSubmittedOn(job.getSubmittedOn()); - response.setSetupStartTime(job.getSetupStartTime()); - response.setSetupEndTime(job.getSetupEndTime()); List responses = StepResponse.build(job.getSteps()); response.setStepResponses(responses); @@ -295,6 +234,8 @@ public static JobResponse build(AbstractJob job, User user) { if (job.getUser() != null) { response.setUsername(job.getUser().getUsername()); } + + response.setCurrentTime(System.currentTimeMillis()); return response; } diff --git a/src/test/java/cloudgene/mapred/database/JobDaoTest.java b/src/test/java/cloudgene/mapred/database/JobDaoTest.java index 93df5d59..84415a6b 100644 --- a/src/test/java/cloudgene/mapred/database/JobDaoTest.java +++ b/src/test/java/cloudgene/mapred/database/JobDaoTest.java @@ -42,7 +42,7 @@ public void testFindAllOlderThan() throws Exception { job1.setName("old-dummy-job-1" + System.currentTimeMillis()); job1.setState(CloudgeneJob.STATE_SUCCESS); job1.setSubmittedOn(System.currentTimeMillis() - (8 * DAYS_MS)); - job1.setFinishedOn(System.currentTimeMillis() - (7 * DAYS_MS)); + job1.setEndTime(System.currentTimeMillis() - (7 * DAYS_MS)); job1.setUser(user); job1.setApplication("appplication"); job1.setApplicationId("appplication-id"); @@ -53,7 +53,7 @@ public void testFindAllOlderThan() throws Exception { job2.setName("old-dummy-job-2" + System.currentTimeMillis()); job2.setState(CloudgeneJob.STATE_SUCCESS); job2.setSubmittedOn(System.currentTimeMillis() - (10 * DAYS_MS)); - job2.setFinishedOn(System.currentTimeMillis() - (9 * DAYS_MS)); + job2.setEndTime(System.currentTimeMillis() - (9 * DAYS_MS)); job2.setUser(user); job2.setApplication("appplication"); job2.setApplicationId("appplication-id"); @@ -64,7 +64,7 @@ public void testFindAllOlderThan() throws Exception { job3.setName("old-dummy-job-3" + System.currentTimeMillis()); job3.setState(CloudgeneJob.STATE_SUCCESS); job3.setSubmittedOn(System.currentTimeMillis() - (9 * DAYS_MS)); - job3.setFinishedOn(System.currentTimeMillis() - (8 * DAYS_MS)); + job3.setEndTime(System.currentTimeMillis() - (8 * DAYS_MS)); job3.setUser(user); job3.setApplication("appplication"); job3.setApplicationId("appplication-id"); @@ -75,7 +75,7 @@ public void testFindAllOlderThan() throws Exception { job4.setName("old-dummy-job-4" + System.currentTimeMillis()); job4.setState(CloudgeneJob.STATE_SUCCESS); job4.setSubmittedOn(System.currentTimeMillis() - (3 * DAYS_MS)); - job4.setFinishedOn(System.currentTimeMillis() - (2 * DAYS_MS)); + job4.setEndTime(System.currentTimeMillis() - (2 * DAYS_MS)); job4.setUser(user); job4.setApplication("appplication"); job4.setApplicationId("appplication-id"); @@ -111,7 +111,7 @@ public void testFindAllByState() throws Exception { jobr.setName("old-dummy-running-job-1" + System.currentTimeMillis()); jobr.setState(CloudgeneJob.STATE_RUNNING); jobr.setSubmittedOn(System.currentTimeMillis() - (8 * DAYS_MS)); - jobr.setFinishedOn(0); + jobr.setEndTime(0); jobr.setUser(user); jobr.setApplication("appplication"); jobr.setApplicationId("appplication-id"); @@ -122,7 +122,7 @@ public void testFindAllByState() throws Exception { job1.setName("old-dummy-job-1" + System.currentTimeMillis()); job1.setState(CloudgeneJob.STATE_FAILED); job1.setSubmittedOn(System.currentTimeMillis() - (8 * DAYS_MS)); - job1.setFinishedOn(System.currentTimeMillis() - (7 * DAYS_MS)); + job1.setEndTime(System.currentTimeMillis() - (7 * DAYS_MS)); job1.setUser(user); job1.setApplication("appplication"); job1.setApplicationId("appplication-id"); @@ -133,7 +133,7 @@ public void testFindAllByState() throws Exception { job2.setName("old-dummy-job-2" + System.currentTimeMillis()); job2.setState(CloudgeneJob.STATE_FAILED); job2.setSubmittedOn(System.currentTimeMillis() - (10 * DAYS_MS)); - job2.setFinishedOn(System.currentTimeMillis() - (9 * DAYS_MS)); + job2.setEndTime(System.currentTimeMillis() - (9 * DAYS_MS)); job2.setUser(user); job2.setApplication("appplication"); job2.setApplicationId("appplication-id"); @@ -144,7 +144,7 @@ public void testFindAllByState() throws Exception { job3.setName("old-dummy-job-3" + System.currentTimeMillis()); job3.setState(CloudgeneJob.STATE_FAILED); job3.setSubmittedOn(System.currentTimeMillis() - (9 * DAYS_MS)); - job3.setFinishedOn(System.currentTimeMillis() - (8 * DAYS_MS)); + job3.setEndTime(System.currentTimeMillis() - (8 * DAYS_MS)); job3.setUser(user); job3.setApplication("appplication"); job3.setApplicationId("appplication-id"); @@ -155,7 +155,7 @@ public void testFindAllByState() throws Exception { job4.setName("old-dummy-job-4" + System.currentTimeMillis()); job4.setState(CloudgeneJob.STATE_SUCCESS); job4.setSubmittedOn(System.currentTimeMillis() - (3 * DAYS_MS)); - job4.setFinishedOn(System.currentTimeMillis() - (2 * DAYS_MS)); + job4.setEndTime(System.currentTimeMillis() - (2 * DAYS_MS)); job4.setUser(user); job4.setApplication("appplication"); job4.setApplicationId("appplication-id"); diff --git a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java index 0233a554..b54b3b21 100644 --- a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java +++ b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java @@ -43,13 +43,11 @@ public void testReturnTrueStep() throws Exception { AbstractJob job = createJobFromWdl(app, inputs); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); + assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -67,13 +65,10 @@ public void testReturnFalseStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_FAILED, job.getState()); @@ -91,13 +86,10 @@ public void testReturnExceptionStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_FAILED, job.getState()); @@ -115,14 +107,11 @@ public void testReturnTrueInSetupStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); // no steps assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); @@ -141,13 +130,10 @@ public void testReturnFalseInSetupStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); // no steps assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); @@ -166,14 +152,10 @@ public void testReturnTrueInSecondSetupStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); // no steps assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); @@ -192,14 +174,11 @@ public void testReturnTrueInSecondSetupStepAndNormalStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); // one steps assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); @@ -217,13 +196,10 @@ public void testHiddenInputsAndDefaultValues() throws Exception { AbstractJob job = createJobFromWdl(app, inputs); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -252,7 +228,7 @@ public void testReturnWriteFileInSecondSetupStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -270,7 +246,7 @@ public void testReturnWriteFileInSecondSetupStep() throws Exception { job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertEquals(AbstractJob.STATE_FAILED, job.getState()); @@ -282,10 +258,6 @@ public void testReturnWriteFileInSecondSetupStep() throws Exception { content = FileUtil.readFileAsString(filename); assertEquals(myContent, content); assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); // one steps assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); @@ -304,13 +276,10 @@ public void testEmptyStepList() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -328,14 +297,10 @@ public void testReturnFalseInSecondSetupStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_FAILED, job.getState()); @@ -353,7 +318,7 @@ public void testWriteTextToFileJob() throws Exception { CloudgeneJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } @@ -363,9 +328,6 @@ public void testWriteTextToFileJob() throws Exception { String content = FileUtil.readFileAsString(filename); assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals("lukas_text", content); @@ -384,7 +346,7 @@ public void testWriteTextToFileOnFailureInStepJob() throws Exception { CloudgeneJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } Thread.sleep(4000); @@ -393,9 +355,6 @@ public void testWriteTextToFileOnFailureInStepJob() throws Exception { String filename = FileUtil.path(settings.getLocalWorkspace(), path); String content = FileUtil.readFileAsString(filename); assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals("lukas_text", content); @@ -414,7 +373,7 @@ public void testWriteTextToFileOnFailureInSetupJob() throws Exception { CloudgeneJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } @@ -425,9 +384,6 @@ public void testWriteTextToFileOnFailureInSetupJob() throws Exception { String filename = FileUtil.path(settings.getLocalWorkspace(), path); String content = FileUtil.readFileAsString(filename); assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals("lukas_text", content); @@ -446,7 +402,7 @@ public void testWriteTextToFileOnFailureInSecondSetupJob() throws Exception { CloudgeneJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } @@ -456,11 +412,8 @@ public void testWriteTextToFileOnFailureInSecondSetupJob() throws Exception { String path = job.getOutputParams().get(0).getFiles().get(0).getPath(); String filename = FileUtil.path(settings.getLocalWorkspace(), path); String content = FileUtil.readFileAsString(filename); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); // no steps executed assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals("lukas_text", content); @@ -479,7 +432,7 @@ public void testThreeTasksStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } @@ -495,9 +448,6 @@ public void testThreeTasksStep() throws Exception { assertEquals("cloudgene-task3", messages.get(2).getMessage()); assertEquals(WorkflowContext.OK, messages.get(2).getType()); assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); } @@ -514,7 +464,7 @@ public void testWriteTextToStdOutStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } @@ -534,9 +484,6 @@ public void testWriteTextToStdOutStep() throws Exception { assertTrue(contentlog.contains("taks write to log2")); assertTrue(contentlog.contains("taks write to log3")); assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); } @@ -553,13 +500,10 @@ public void testApplicationLinks() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -584,13 +528,10 @@ public void testApplicationLinksWithoutAppsPrefix() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -615,13 +556,10 @@ public void testApplicationLinksWithoutVersion() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -645,13 +583,10 @@ public void testOptionalApplicationLinks() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -676,15 +611,12 @@ public void testApplicationLinksWrongApplication() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_FAILED, job.getState()); } @@ -701,15 +633,12 @@ public void testApplicationLinksWrongPermissions() throws Exception { AbstractJob job = createJobFromWdlAsUser(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_FAILED, job.getState()); } diff --git a/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java b/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java index 4f73f6e7..9db7f4c4 100644 --- a/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java +++ b/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java @@ -40,7 +40,7 @@ public void testReturnTrueStep() throws Exception { AbstractJob job = createJobFromWdl(app, inputs); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(1000); } Thread.sleep(10000); diff --git a/src/test/java/cloudgene/mapred/steps/TestCommand.java b/src/test/java/cloudgene/mapred/steps/TestCommand.java index 50cdcfe0..b4fc62a2 100644 --- a/src/test/java/cloudgene/mapred/steps/TestCommand.java +++ b/src/test/java/cloudgene/mapred/steps/TestCommand.java @@ -28,18 +28,17 @@ @MicronautTest public class TestCommand { - @Inject TestApplication application; @Inject WorkspaceFactory workspaceFactory; - + @Test public void testValidCommand() throws Exception { - + WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/command/valid-command.yaml"); Map params = new HashMap(); @@ -47,7 +46,7 @@ public void testValidCommand() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(1000); } @@ -58,15 +57,13 @@ public void testValidCommand() throws Exception { assertEquals(messages.get(0).getType(), WorkflowContext.OK); assertTrue(messages.get(0).getMessage().contains("Execution successful.")); - String stdout = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), - "std.out"); + String stdout = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), "std.out"); String contentStdOut = FileUtil.readFileAsString(stdout); // simple ls result check assertTrue(contentStdOut.contains("invalid-command.yaml")); - String jobLog = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), - "job.txt"); + String jobLog = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), "job.txt"); String contentjobLog = FileUtil.readFileAsString(jobLog); // simple check if exit code = 0 @@ -76,9 +73,9 @@ public void testValidCommand() throws Exception { @Test public void testInvalidCommand() throws Exception { - + WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/command/invalid-command.yaml"); Map params = new HashMap(); @@ -86,7 +83,7 @@ public void testInvalidCommand() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(1000); } @@ -105,8 +102,8 @@ public void testInvalidCommand() throws Exception { * Map params = new HashMap(); * params.put("input", "input-file"); * - * AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - * while (job.isRunning()) { Thread.sleep(1000); } + * AbstractJob job = createJobFromWdl(app, params); engine.submit(job); while + * (job.isRunning()) { Thread.sleep(1000); } * * assertEquals(AbstractJob.STATE_FAILED, job.getState()); * @@ -116,9 +113,8 @@ public void testInvalidCommand() throws Exception { * assertTrue(messages.get(0).getMessage().contains("Execution failed.")); * * String stdout = FileUtil.path(TestServer.getInstance().getSettings() - * .getLocalWorkspace(), job.getId(), "std.out"); - * System.out.println(stdout); String contentStdOut = - * FileUtil.readFileAsString(stdout); + * .getLocalWorkspace(), job.getId(), "std.out"); System.out.println(stdout); + * String contentStdOut = FileUtil.readFileAsString(stdout); * * //simple check for unrecognized option * assertTrue(contentStdOut.contains("unrecognized option")); @@ -135,7 +131,7 @@ public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs) thr UserDao userDao = new UserDao(application.getDatabase()); User user = userDao.findByUsername("user"); - + Settings settings = application.getSettings(); String id = "test_" + System.currentTimeMillis(); From cb1d646a13604711826caf5453c9674ec478b67b Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Wed, 13 Sep 2023 08:52:21 +0200 Subject: [PATCH 32/35] Remove deprectated property progress --- .../cloudgene/mapred/jobs/AbstractJob.java | 7 ----- .../cloudgene/mapred/jobs/CloudgeneStep.java | 4 --- .../mapred/jobs/engine/ExecutableStep.java | 17 ++--------- .../mapred/jobs/engine/Executor.java | 8 ------ .../mapred/server/responses/StepResponse.java | 28 ++++++------------- .../java/cloudgene/mapred/util/TimeUtil.java | 12 ++++++++ 6 files changed, 23 insertions(+), 53 deletions(-) create mode 100644 src/main/java/cloudgene/mapred/util/TimeUtil.java diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java index 868c6c55..aaed803e 100644 --- a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java +++ b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java @@ -448,15 +448,8 @@ public void cancel() { } private void initStdOutFiles() throws FileNotFoundException { - - // if (stdOutStream == null) { stdOutStream = new BufferedOutputStream(new FileOutputStream(FileUtil.path(localWorkspace, "std.out"))); - - // } - // if (logStream == null) { logStream = new BufferedOutputStream(new FileOutputStream(FileUtil.path(localWorkspace, "job.txt"))); - // } - } private void closeStdOutFiles() { diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java index dbbd0657..a10c5431 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java @@ -65,10 +65,6 @@ public boolean run(WdlStep step, CloudgeneContext context) { return true; } - public int getProgress() { - return -1; - } - public void updateProgress() { } diff --git a/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java b/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java index 3c1978b3..2af2dc9c 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java @@ -19,6 +19,7 @@ import cloudgene.mapred.plugins.PluginManager; import cloudgene.mapred.steps.ErrorStep; import cloudgene.mapred.steps.JavaInternalStep; +import cloudgene.mapred.util.TimeUtil; import cloudgene.mapred.wdl.WdlStep; import genepi.io.FileUtil; @@ -152,13 +153,7 @@ public void run() { long end = System.currentTimeMillis(); long time = end - start; - long h = (long) (Math.floor((time / 1000) / 60 / 60)); - long m = (long) ((Math.floor((time / 1000) / 60)) % 60); - - String t = (h > 0 ? h + " h " : "") + (m > 0 ? m + " min " : "") - + (int) ((Math.floor(time / 1000)) % 60) + " sec"; - - job.writeLog(" " + step.getName() + " [" + t + "]"); + job.writeLog(" " + step.getName() + " [" + TimeUtil.format(time) + "]"); setTime(time); } @@ -196,14 +191,6 @@ public void updateProgress() { } } - public int getProgress() { - if (instance != null) { - return instance.getProgress(); - } else { - return 0; - } - } - public void setTime(long time) { this.time = time; } diff --git a/src/main/java/cloudgene/mapred/jobs/engine/Executor.java b/src/main/java/cloudgene/mapred/jobs/engine/Executor.java index 3fa6e44f..4f8d93e7 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/Executor.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/Executor.java @@ -39,14 +39,6 @@ public void updateProgress() { } } - public int getProgress() { - if (executableNode != null) { - return executableNode.getProgress(); - } else { - return 0; - } - } - public ExecutableStep getCurrentNode() { return executableNode; } diff --git a/src/main/java/cloudgene/mapred/server/responses/StepResponse.java b/src/main/java/cloudgene/mapred/server/responses/StepResponse.java index 980f4869..dff58b80 100644 --- a/src/main/java/cloudgene/mapred/server/responses/StepResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/StepResponse.java @@ -12,11 +12,11 @@ public class StepResponse { private int id; - private String name; - private int progress; - @JsonProperty("logMessages") - private List messageResponse; + private String name; + + @JsonProperty("logMessages") + private List messages; public int getId() { return id; @@ -34,22 +34,12 @@ public void setName(String name) { this.name = name; } - public int getProgress() { - return progress; - } - - public void setProgress(int progress) { - this.progress = progress; - } - - public static StepResponse build(CloudgeneStep step) { StepResponse response = new StepResponse(); response.setId(step.getId()); response.setName(step.getName()); - response.setProgress(step.getProgress()); List responses = MessageResponse.build(step.getLogMessages()); - response.setMessageResponse(responses); + response.setMessages(responses); return response; } @@ -61,12 +51,12 @@ public static List build(List steps) { return response; } - public List getMessageResponse() { - return messageResponse; + public List getMessages() { + return messages; } - public void setMessageResponse(List messageResponse) { - this.messageResponse = messageResponse; + public void setMessages(List messages) { + this.messages = messages; } } diff --git a/src/main/java/cloudgene/mapred/util/TimeUtil.java b/src/main/java/cloudgene/mapred/util/TimeUtil.java new file mode 100644 index 00000000..9f91753d --- /dev/null +++ b/src/main/java/cloudgene/mapred/util/TimeUtil.java @@ -0,0 +1,12 @@ +package cloudgene.mapred.util; + +public class TimeUtil { + + public static String format(long time) { + long h = (long) (Math.floor((time / 1000) / 60 / 60)); + long m = (long) ((Math.floor((time / 1000) / 60)) % 60); + + return (h > 0 ? h + " h " : "") + (m > 0 ? m + " min " : "") + (int) ((Math.floor(time / 1000)) % 60) + " sec"; + } + +} From 44afc36434096c81c08e82f549308ff9633a1c26 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Sun, 17 Sep 2023 11:26:02 +0200 Subject: [PATCH 33/35] Extended binded lists --- .../server/responses/PropertyResponse.java | 24 ++++++++++++++++--- .../responses/WdlParameterInputResponse.java | 24 +++++++++---------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/main/java/cloudgene/mapred/server/responses/PropertyResponse.java b/src/main/java/cloudgene/mapred/server/responses/PropertyResponse.java index fc3282cb..b190077b 100644 --- a/src/main/java/cloudgene/mapred/server/responses/PropertyResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/PropertyResponse.java @@ -69,12 +69,30 @@ public static List buildWithValues(Map values) { return response; } - public static PropertyResponse build(String key, String value, Map values) { + public static List buildWithValues(List values) { + List response = new Vector(); + + for (Map object : values) { + if (object.containsKey("id") && object.containsKey("name")) { + System.out.println(object); + response.add(PropertyResponse.build(object.get("id").toString(), object.get("name").toString())); + } + } + + return response; + } + + public static PropertyResponse build(String key, String value, Object values) { PropertyResponse response = new PropertyResponse(); response.setKey(key); response.setLabel(value); - List responseWithValues = buildWithValues(values); - response.setValues(responseWithValues); + if (values instanceof Map) { + List responseWithValues = buildWithValues((Map) values); + response.setValues(responseWithValues); + } else if (values instanceof List) { + List responseWithValues = buildWithValues((List) values); + response.setValues(responseWithValues); + } return response; } diff --git a/src/main/java/cloudgene/mapred/server/responses/WdlParameterInputResponse.java b/src/main/java/cloudgene/mapred/server/responses/WdlParameterInputResponse.java index a1375320..5402635f 100644 --- a/src/main/java/cloudgene/mapred/server/responses/WdlParameterInputResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/WdlParameterInputResponse.java @@ -136,7 +136,7 @@ public String getEmptySelection() { public void setEmptySelection(String emptySelection) { this.emptySelection = emptySelection; } - + public List getValues() { return values; } @@ -169,7 +169,6 @@ public void setLabel(String label) { this.label = label; } - public static WdlParameterInputResponse build(WdlParameterInput input, List apps) { WdlParameterInputResponse response = new WdlParameterInputResponse(); response.setId(input.getId()); @@ -211,17 +210,16 @@ public static WdlParameterInputResponse build(WdlParameterInput input, List propertyResponses = new ArrayList(); for (WdlApp app : apps) { - if (category != null && !category.isEmpty()) { - if (app.getCategory() != null && app.getCategory().equals(category)) { - Map values = (Map) app.getProperties().get(property); - PropertyResponse propertyResponse = PropertyResponse.build("apps@" + app.getId(), app.getName(), - values); - propertyResponses.add(propertyResponse); - } - - } else { - // TODO:!! + if (category == null || category.isEmpty()) { + continue; } + if (app.getCategory() == null || !app.getCategory().equals(category)) { + continue; + } + Object values = app.getProperties().get(property); + PropertyResponse propertyResponse = PropertyResponse.build("apps@" + app.getId(), app.getName(), + values); + propertyResponses.add(propertyResponse); } response.setValues(propertyResponses); @@ -275,7 +273,7 @@ public static List build(List inpu for (WdlParameterInput input : inputs) { if (input.isVisible()) { - response.add(WdlParameterInputResponse.build(input, apps)); + response.add(WdlParameterInputResponse.build(input, apps)); } } return response; From 3b26d30989195057f8e5c40a548320becc498d99 Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Mon, 18 Sep 2023 09:58:09 +0200 Subject: [PATCH 34/35] Upload job infos to workspace --- .../cloudgene/mapred/database/JobDao.java | 18 ++-- .../cloudgene/mapred/jobs/AbstractJob.java | 87 +++++++------------ .../cloudgene/mapred/jobs/CloudgeneJob.java | 2 +- .../mapred/jobs/workspace/IWorkspace.java | 8 +- .../mapred/jobs/workspace/LocalWorkspace.java | 28 +++++- .../mapred/jobs/workspace/S3Workspace.java | 23 ++++- .../jobs/workspace/WorkspaceFactory.java | 4 +- .../server/controller/LogController.java | 30 +++---- .../mapred/server/responses/JobResponse.java | 11 --- .../mapred/server/services/JobService.java | 24 +++-- 10 files changed, 127 insertions(+), 108 deletions(-) diff --git a/src/main/java/cloudgene/mapred/database/JobDao.java b/src/main/java/cloudgene/mapred/database/JobDao.java index d2e8b407..244ba398 100644 --- a/src/main/java/cloudgene/mapred/database/JobDao.java +++ b/src/main/java/cloudgene/mapred/database/JobDao.java @@ -66,11 +66,12 @@ public boolean insert(AbstractJob job) { public boolean update(AbstractJob job) { StringBuilder sql = new StringBuilder(); - sql.append("update job "); - sql.append(" set name = ?, state = ?, "); + sql.append("update job set "); + sql.append(" name = ?, state = ?, "); sql.append(" start_time = ?, end_time = ?, "); - sql.append( - " user_id = ?, s3_url = ?, type = ?, deleted_on = ?, application = ?, application_id = ?, submitted_on = ?, finished_on = ?, setup_start_time = ?, setup_end_time = ? "); + sql.append(" user_id = ?, s3_url = ?, type = ?, deleted_on = ?, "); + sql.append(" application = ?, application_id = ?, submitted_on = ?, "); + sql.append(" finished_on = ?, setup_start_time = ?, setup_end_time = ? "); sql.append("where id = ? "); try { @@ -105,9 +106,8 @@ public boolean update(AbstractJob job) { public boolean updateUser(User oldUser, User newUser) { StringBuilder sql = new StringBuilder(); - sql.append("update job "); - sql.append("set user_id = ?, name = ? "); - sql.append("where user_id = ?"); + sql.append("update job set user_id = ?, name = ? "); + sql.append("where user_id = ?"); try { Object[] params = new Object[3]; @@ -132,7 +132,7 @@ public boolean updateUser(User oldUser, User newUser) { public boolean delete(AbstractJob job) { StringBuilder sql = new StringBuilder(); sql.append("delete from job "); - sql.append("where id = ? "); + sql.append("where id = ?"); try { Object[] params = new Object[1]; @@ -277,7 +277,7 @@ public List findAllNotRetiredJobs() { params[1] = AbstractJob.STATE_RUNNING; params[2] = AbstractJob.STATE_EXPORTING; params[3] = AbstractJob.STATE_RETIRED; - params[4] = AbstractJob.STATE_DELETED; + params[4] = AbstractJob.STATE_DELETED; params[5] = AbstractJob.STATE_RETIRED; params[6] = AbstractJob.STATE_DELETED; diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java index aaed803e..c1e93dff 100644 --- a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java +++ b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java @@ -1,6 +1,7 @@ package cloudgene.mapred.jobs; import java.io.BufferedOutputStream; +import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -30,6 +31,10 @@ abstract public class AbstractJob extends PriorityRunnable { + public static final String JOB_LOG = "job.txt"; + + public static final String JOB_OUT = "std.out"; + private static final Logger log = LoggerFactory.getLogger(AbstractJob.class); private DateFormat formatter = new SimpleDateFormat("yy/MM/dd HH:mm:ss"); @@ -102,8 +107,6 @@ abstract public class AbstractJob extends PriorityRunnable { private Settings settings; - private String logs; - private String localWorkspace; private boolean canceld = false; @@ -300,7 +303,7 @@ public boolean runInstallationAndResolveAppLinks() { @Override public void run() { - if (isCanceld()) { + if (canceld) { return; } @@ -389,29 +392,19 @@ public void run() { } + writeLog("Cleaning up..."); if (getState() == AbstractJob.STATE_FAILED || getState() == AbstractJob.STATE_CANCELED) { - - writeLog("Cleaning up..."); onFailure(); - log.info("[Job {}]cleanup successful.", getId()); - writeLog("Cleanup successful."); - } else { - writeLog("Cleaning up..."); cleanUp(); - log.info("[Job {}] cleanup successful.", getId()); - writeLog("Cleanup successful."); - } + log.info("[Job {}]cleanup successful.", getId()); + writeLog("Cleanup successful."); if (canceld) { setState(AbstractJob.STATE_CANCELED); } - closeStdOutFiles(); - - setEndTime(System.currentTimeMillis()); - } catch (Exception | Error e) { setState(AbstractJob.STATE_FAILED); @@ -429,11 +422,10 @@ public void run() { log.info("[Job {}]: cleanup successful.", getId()); writeLog("Cleanup successful."); - closeStdOutFiles(); - - setEndTime(System.currentTimeMillis()); - } + + closeStdOutFiles(); + setEndTime(System.currentTimeMillis()); } public void cancel() { @@ -448,8 +440,8 @@ public void cancel() { } private void initStdOutFiles() throws FileNotFoundException { - stdOutStream = new BufferedOutputStream(new FileOutputStream(FileUtil.path(localWorkspace, "std.out"))); - logStream = new BufferedOutputStream(new FileOutputStream(FileUtil.path(localWorkspace, "job.txt"))); + stdOutStream = new BufferedOutputStream(new FileOutputStream(FileUtil.path(localWorkspace, JOB_OUT))); + logStream = new BufferedOutputStream(new FileOutputStream(FileUtil.path(localWorkspace, JOB_LOG))); } private void closeStdOutFiles() { @@ -459,8 +451,15 @@ private void closeStdOutFiles() { stdOutStream.close(); logStream.close(); - } catch (IOException e) { + // stage files to workspace + workspace.uploadLog(new File(FileUtil.path(localWorkspace, JOB_OUT))); + workspace.uploadLog(new File(FileUtil.path(localWorkspace, JOB_LOG))); + + FileUtil.deleteFile(FileUtil.path(localWorkspace, JOB_OUT)); + FileUtil.deleteFile(FileUtil.path(localWorkspace, JOB_LOG)); + } catch (IOException e) { + log.error("[Job {}]: Staging log files failed.", getId(), e); } } @@ -474,25 +473,13 @@ public void writeOutput(String line) { } } catch (IOException e) { - + log.error("[Job {}]: Write output failed.", getId(), e); } } public void writeOutputln(String line) { - - try { - if (stdOutStream == null) { - initStdOutFiles(); - } - - stdOutStream.write(line.getBytes("UTF-8")); - stdOutStream.write("\n".getBytes("UTF-8")); - stdOutStream.flush(); - - } catch (IOException e) { - } - + writeOutput(line + "\n"); } public void writeLog(String line) { @@ -508,6 +495,7 @@ public void writeLog(String line) { logStream.flush(); } catch (IOException e) { + log.error("[Job {}]: Write output failed.", getId(), e); } } @@ -524,14 +512,6 @@ public CloudgeneContext getContext() { return context; } - public void setLogs(String logs) { - this.logs = logs; - } - - public String getLogs() { - return logs; - } - @Override public boolean equals(Object obj) { @@ -583,10 +563,6 @@ public String getApplicationId() { return applicationId; } - public boolean isCanceld() { - return canceld; - } - public boolean isRunning() { return state == STATE_EXPORTING || state == STATE_RUNNING || state == STATE_WAITING; } @@ -619,16 +595,13 @@ public void kill() { } - public long getCurrentTime() { - return System.currentTimeMillis(); - } - - public void setCurrentTime(long time) { - - } - public String getPublicJobId() { return publicJobId; } + public String getLog(String name) { + String logFilename = FileUtil.path(settings.getLocalWorkspace(), getId(), name); + return FileUtil.readFileAsString(logFilename); + } + } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java index 895bcbb7..96f14f4d 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java @@ -103,7 +103,7 @@ public boolean setup() { try { log.info("[Job {}] Setup workspace {}'", getId(), workspace.getName()); context.log("Setup External Workspace on " + workspace.getName()); - workspace.setup(this.getId()); + workspace.setup(); context.setWorkspace(workspace); } catch (Exception e) { writeLog(e.toString()); diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java index 701a6fc1..edff9bb1 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java @@ -9,11 +9,15 @@ public interface IWorkspace { - public void setup(String job) throws IOException; + public void setJob(String job); + + public void setup() throws IOException; public String upload(String id, File file) throws IOException; public String uploadInput(String id, File file) throws IOException; + + public String uploadLog(File file) throws IOException; public InputStream download(String url) throws IOException; @@ -40,5 +44,7 @@ public interface IWorkspace { public void cleanup(String job) throws IOException; public boolean exists(String path) throws IOException; + + public String downloadLog(String string) throws IOException; } diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java index c573a7df..6e35a4bf 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java @@ -43,8 +43,17 @@ public String getName() { } @Override - public void setup(String job) throws IOException { + public void setJob(String job) { workspace = FileUtil.path(location, job); + } + + @Override + public void setup() throws IOException { + + if (workspace == null) { + throw new IOException("No job id provided."); + } + log.info("Init workspace " + workspace); FileUtil.createDirectory(workspace); } @@ -59,6 +68,11 @@ public String upload(String id, File file) throws IOException { return target; } + @Override + public String uploadLog(File file) throws IOException { + return upload(LOGS_DIRECTORY, file); + } + @Override public String uploadInput(String id, File file) throws IOException { return upload(FileUtil.path(INPUT_DIRECTORY, id), file); @@ -74,8 +88,18 @@ public InputStream download(String path) throws IOException { if (file.exists()) { return new FileInputStream(file); } else { - throw new IOException("File '" + path + "' not found in workspace."); + throw new IOException("File '" + absolutePath + "' not found in workspace."); + } + } + + @Override + public String downloadLog(String name) throws IOException { + + if (workspace == null) { + throw new IOException("No job id provided."); } + + return FileUtil.readFileAsString(download(FileUtil.path(workspace, LOGS_DIRECTORY, name))); } @Override diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java index ec8ff6e4..656779b9 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java @@ -52,9 +52,16 @@ public String getName() { } @Override - public void setup(String job) throws IOException { + public void setJob(String job) { + this.job = job; + } + + @Override + public void setup() throws IOException { - this.job = job; + if (job == null) { + throw new IOException("No job id provided."); + } if (location == null) { throw new IOException("No S3 Output Bucket specified."); @@ -86,6 +93,11 @@ public String uploadInput(String id, File file) throws IOException { return upload(FileUtil.path(INPUT_DIRECTORY, id), file); } + @Override + public String uploadLog(File file) throws IOException { + return upload(LOGS_DIRECTORY, file); + } + @Override public InputStream download(String url) throws IOException { @@ -99,6 +111,11 @@ public InputStream download(String url) throws IOException { return s3is; } + + @Override + public String downloadLog(String name) throws IOException { + return FileUtil.readFileAsString(download(FileUtil.path(LOGS_DIRECTORY, name))); + } public boolean exists(String url) { String bucket = S3Util.getBucket(url); @@ -245,7 +262,7 @@ public List getDownloads(String url) { return downloads; } - + @Override public List getLogs() { String url = location + "/" + job + "/" + LOGS_DIRECTORY; diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceFactory.java b/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceFactory.java index 561d7a58..18f1c140 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceFactory.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceFactory.java @@ -49,7 +49,9 @@ public IWorkspace getByUrl(String url) { } public IWorkspace getByJob(AbstractJob job) { - return getDefault(); + IWorkspace workspace = getDefault(); + workspace.setJob(job.getId()); + return workspace; } } diff --git a/src/main/java/cloudgene/mapred/server/controller/LogController.java b/src/main/java/cloudgene/mapred/server/controller/LogController.java index 6b5f20ea..700f3668 100644 --- a/src/main/java/cloudgene/mapred/server/controller/LogController.java +++ b/src/main/java/cloudgene/mapred/server/controller/LogController.java @@ -1,16 +1,17 @@ package cloudgene.mapred.server.controller; +import java.io.IOException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import cloudgene.mapred.core.User; import cloudgene.mapred.jobs.AbstractJob; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.server.Application; import cloudgene.mapred.server.auth.AuthenticationService; import cloudgene.mapred.server.auth.AuthenticationType; import cloudgene.mapred.server.services.JobService; -import cloudgene.mapred.util.Settings; -import genepi.io.FileUtil; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; @@ -24,7 +25,7 @@ public class LogController { private static Logger log = LoggerFactory.getLogger(LogController.class); - + @Inject protected Application application; @@ -34,25 +35,21 @@ public class LogController { @Inject protected JobService jobService; + @Inject + protected WorkspaceFactory workspaceFactory; + @Get("/logs/{id}") @Secured(SecurityRule.IS_AUTHENTICATED) @Produces(MediaType.TEXT_PLAIN) - public String getByJobs(Authentication authentication, String id) { + public String getByJob(Authentication authentication, String id) throws IOException { User user = authenticationService.getUserByAuthentication(authentication, AuthenticationType.ALL_TOKENS); AbstractJob job = jobService.getByIdAndUser(id, user); - Settings settings = application.getSettings(); - // log file - String logFilename = FileUtil.path(settings.getLocalWorkspace(), job.getId(), "job.txt"); - String logContent = FileUtil.readFileAsString(logFilename); - - // std out - String outputFilename = FileUtil.path(settings.getLocalWorkspace(), job.getId(), "std.out"); - String outputContent = FileUtil.readFileAsString(outputFilename); - + String logContent = jobService.getJobLog(job, AbstractJob.JOB_LOG); + String outputContent = jobService.getJobLog(job, AbstractJob.JOB_OUT); + StringBuffer buffer = new StringBuffer(); - if (!logContent.isEmpty()) { buffer.append("job.txt:\n\n"); buffer.append(logContent); @@ -62,14 +59,13 @@ public String getByJobs(Authentication authentication, String id) { buffer.append("\n\nstd.out:\n\n"); buffer.append(outputContent); } - - + String message = String.format("Job: viewing logs for job ID %s", job.getId()); if (user.isAdmin()) { message += String.format(" (by ADMIN user ID %s - email %s)", user.getId(), user.getMail()); } log.info(message); - + return buffer.toString(); } diff --git a/src/main/java/cloudgene/mapred/server/responses/JobResponse.java b/src/main/java/cloudgene/mapred/server/responses/JobResponse.java index 6f550f00..f48bbb1f 100644 --- a/src/main/java/cloudgene/mapred/server/responses/JobResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/JobResponse.java @@ -16,7 +16,6 @@ public class JobResponse { private String application; private String applicationId; - private boolean canceld; private long deletedOn; private long endTime; private String id; @@ -52,14 +51,6 @@ public void setApplicationId(String applicationId) { this.applicationId = applicationId; } - public boolean isCanceld() { - return canceld; - } - - public void setCanceld(boolean canceld) { - this.canceld = canceld; - } - public long getDeletedOn() { return deletedOn; } @@ -202,12 +193,10 @@ public static JobResponse build(AbstractJob job, User user) { JobResponse response = new JobResponse(); response.setApplication(job.getApplication()); response.setApplicationId(job.getApplicationId()); - response.setCanceld(job.isCanceld()); response.setName(job.getName()); response.setId(job.getId()); response.setState(job.getState()); - response.setLogs(job.getLogs()); response.setPositionInQueue(job.getPositionInQueue()); response.setUserAgent(job.getUserAgent()); diff --git a/src/main/java/cloudgene/mapred/server/services/JobService.java b/src/main/java/cloudgene/mapred/server/services/JobService.java index 5b9f0f50..6ecc2fea 100644 --- a/src/main/java/cloudgene/mapred/server/services/JobService.java +++ b/src/main/java/cloudgene/mapred/server/services/JobService.java @@ -1,6 +1,7 @@ package cloudgene.mapred.server.services; import java.io.File; +import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashMap; @@ -125,7 +126,8 @@ public AbstractJob submitJob(String appId, List form, User user) { try { // setup workspace - workspace.setup(id); + workspace.setJob(id); + workspace.setup(); // parse input params inputParams = parseAndUpdateInputParams(form, app, workspace); @@ -140,7 +142,7 @@ public AbstractJob submitJob(String appId, List form, User user) { name = jobName; } - //TODO: remove and solve via workspace! + // TODO: remove and solve via workspace! String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), id); FileUtil.createDirectory(localWorkspace); @@ -266,17 +268,17 @@ public AbstractJob restart(AbstractJob job) { } - IWorkspace workspace = workspaceFactory.getDefault(); try { // setup workspace - workspace.setup(job.getId()); + workspace.setJob(job.getId()); + workspace.setup(); } catch (Exception e) { throw new JsonHttpStatusException(HttpStatus.BAD_REQUEST, e.getMessage()); } job.setWorkspace(workspace); - + ((CloudgeneJob) job).loadApp(application.getWdlApp()); this.application.getWorkflowEngine().restart(job); @@ -522,7 +524,7 @@ public List getJobs(String state) { case "running-stq": - //TODO: remove! + // TODO: remove! jobs = new Vector(); break; @@ -548,6 +550,16 @@ public List getJobs(String state) { return jobs; } + public String getJobLog(AbstractJob job, String name) throws IOException { + if (job.isRunning()) { + // files are locally when job is running + return job.getLog(name); + } else { + IWorkspace workspace = workspaceFactory.getByJob(job); + return workspace.downloadLog(name); + } + } + public boolean needsImport(String url) { return url.startsWith("sftp://") || url.startsWith("http://") || url.startsWith("https://") || url.startsWith("ftp://") || url.startsWith("s3://"); From 45011a497e5ebcf6badc6d21ff059de1d1c6751a Mon Sep 17 00:00:00 2001 From: Lukas Forer Date: Mon, 18 Sep 2023 11:07:10 +0200 Subject: [PATCH 35/35] Support cloudgene.report.json on step level --- .../mapred/plugins/nextflow/NextflowStep.java | 25 +++++++++++++++++++ .../plugins/nextflow/report/ReportEvent.java | 13 +++++++--- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java index 04cfc817..9189a7b0 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -16,6 +17,9 @@ import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; import cloudgene.mapred.jobs.workspace.IWorkspace; +import cloudgene.mapred.plugins.nextflow.report.Report; +import cloudgene.mapred.plugins.nextflow.report.ReportEvent; +import cloudgene.mapred.plugins.nextflow.report.ReportEventExecutor; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlStep; import genepi.io.FileUtil; @@ -152,6 +156,18 @@ public boolean run(WdlStep step, CloudgeneContext context) { collector.cleanProcesses(context); + File report = new File(executionDir, Report.DEFAULT_FILENAME); + if (!report.exists()) { + return successful; + } + + context.log("Load report file from '" + report.getCanonicalPath() + "'"); + try { + parseReport(report); + } catch (Exception e) { + log.error("[Job {}] Invalid report file.", e); + } + return successful; } catch (Exception e) { @@ -161,6 +177,15 @@ public boolean run(WdlStep step, CloudgeneContext context) { } + private void parseReport(File file) throws IOException { + Report report = new Report(file.getAbsolutePath()); + context.log("Execute " + report.getEvents().size() + " events."); + for (ReportEvent event : report.getEvents()) { + context.log("Event: " + event); + ReportEventExecutor.execute(event, context); + } + } + @Override public void updateProgress() { diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEvent.java b/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEvent.java index 6fd9ff67..e8226e3c 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEvent.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEvent.java @@ -65,7 +65,7 @@ public static String format(ReportEvent event) { } case MESSAGE: { String message = (String) event.getParams()[0]; - int type = ((Integer) event.getParams()[1]); + int type = asInteger(event.getParams()[1]); return message(message, type); } case BEGIN_TASK: { @@ -88,12 +88,12 @@ public static String format(ReportEvent event) { } case END_TASK: { String name = (String) event.getParams()[0]; - int type = ((Integer) event.getParams()[1]).intValue(); + int type = asInteger(event.getParams()[1]); return message(name, type); } case INC_COUNTER: { String counter = (String) event.getParams()[0]; - int value = ((Integer) event.getParams()[1]); + int value = asInteger(event.getParams()[1]); return incCounter(counter, value); } default: @@ -137,5 +137,12 @@ public static String message(String message, int type) { } } + + public static int asInteger(Object object) { + //groovy writes integers also as doubles + Double value = Double.parseDouble(object.toString()); + return value.intValue(); + + } }