diff --git a/pom.xml b/pom.xml index 0b0825f9..4b3eda68 100644 --- a/pom.xml +++ b/pom.xml @@ -45,8 +45,9 @@ UTF-8 UTF-8 jar - 1.8 - 8 + 11 + 11 + 11 3.3.4 cloudgene.mapred.server.Application netty @@ -67,15 +68,9 @@ - jfrog-genepi-maven - jfrog-genepi-maven - https://genepi.jfrog.io/artifactory/maven/ - - - - cloudera - Cloudera repository - https://repository.cloudera.com/artifactory/cloudera-repos/ + genepi-maven + genepi-maven + https://genepi.i-med.ac.at/maven @@ -84,12 +79,6 @@ - - cloudgene - cloudgene-java-sdk - 1.0.2 - - genepi genepi-io @@ -106,26 +95,6 @@ - - genepi - genepi-hadoop - mr1-1.4.1 - - - org.apache.hadoop - hadoop-client - - - org.apache.httpcomponents - httpcore - - - org.apache.httpcomponents - httpclient - - - - commons-dbutils commons-dbutils @@ -217,62 +186,6 @@ 5.8.3 - - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - test - test-jar - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop.version} - test - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - test - - - org.apache.httpcomponents - httpcore - - - org.slf4j - slf4j-reload4j - - - - - - org.apache.hadoop - hadoop-common - ${hadoop.version} - test - test-jar - - - org.slf4j - slf4j-reload4j - - - - - - org.apache.hadoop - hadoop-test - 2.6.0-mr1-cdh5.16.1 - test - - org.junit.jupiter junit-jupiter diff --git a/src/main/html/webapp/admin.js b/src/main/html/webapp/admin.js index 573bc600..a811db12 100644 --- a/src/main/html/webapp/admin.js +++ b/src/main/html/webapp/admin.js @@ -15,15 +15,18 @@ import UserListControl from 'components/admin/user/list/'; import JobListControl from 'components/admin/job/list/'; import JobDetailControl from 'components/core/job/detail/'; import AppListControl from 'components/admin/app/list/'; +import AppSettingsControl from 'components/admin/app/settings/'; + import AppRepositoryControl from 'components/admin/app/repository/'; import SettingsGeneralControl from 'components/admin/settings/general/'; +import SettingsNextflowControl from 'components/admin/settings/nextflow/'; import SettingsServerControl from 'components/admin/settings/server/'; import SettingsMailControl from 'components/admin/settings/mail/'; import SettingsTemplatesControl from 'components/admin/settings/templates/'; import SettingsLogsControl from 'components/admin/settings/logs/'; -$(document.links).filter(function() { +$(document.links).filter(function () { return this.hostname != window.location.hostname; }).attr('target', '_blank'); @@ -64,6 +67,10 @@ var routes = [{ path: 'pages/admin-apps', control: AppListControl, guard: adminGuard +}, { + path: 'pages/admin-apps/{app}', + control: AppSettingsControl, + guard: adminGuard }, { path: 'pages/admin-apps-repository', control: AppRepositoryControl, @@ -76,6 +83,10 @@ var routes = [{ path: 'pages/admin-settings-general', control: SettingsGeneralControl, guard: adminGuard +}, { + path: 'pages/admin-settings-nextflow', + control: SettingsNextflowControl, + guard: adminGuard }, { path: 'pages/admin-settings-mail', control: SettingsMailControl, @@ -106,9 +117,9 @@ function adminGuard(appState) { } } -$.ajaxPrefilter(function(options, orig, xhr) { +$.ajaxPrefilter(function (options, orig, xhr) { if (!options.beforeSend) { - options.beforeSend = function(xhr) { + options.beforeSend = function (xhr) { if (localStorage.getItem("cloudgene")) { try { // get data @@ -133,7 +144,7 @@ $.ajaxPrefilter(function(options, orig, xhr) { }); -Server.findOne({}, function(server) { +Server.findOne({}, function (server) { new LayoutControl("#main", { appState: server diff --git a/src/main/html/webapp/components/admin/app/list/list.js b/src/main/html/webapp/components/admin/app/list/list.js index 35a7f005..10d1de76 100644 --- a/src/main/html/webapp/components/admin/app/list/list.js +++ b/src/main/html/webapp/components/admin/app/list/list.js @@ -15,14 +15,13 @@ import template from './list.stache'; import templateInstallGithub from './install-github/install-github.stache'; import templateInstallUrl from './install-url/install-url.stache'; import templatePermission from './permission/permission.stache'; -import templateSettings from './settings/settings.stache'; export default Control.extend({ - "init": function(element, options) { + "init": function (element, options) { var that = this; - Application.findAll({}, function(applications) { + Application.findAll({}, function (applications) { that.options.installedApplications = applications; $(element).html(template({ applications: applications @@ -33,10 +32,10 @@ export default Control.extend({ }, - '#install-app-url-btn click': function(el, ev) { + '#install-app-url-btn click': function (el, ev) { bootbox.confirm(templateInstallUrl(), - function(result) { + function (result) { if (result) { var url = $('#url').val(); var app = new Application(); @@ -50,15 +49,15 @@ export default Control.extend({ '', show: false }); - waitingDialog.on('shown.bs.modal', function() { - app.save(function(application) { + waitingDialog.on('shown.bs.modal', function () { + app.save(function (application) { waitingDialog.modal('hide'); - bootbox.alert('

Congratulations

The application installation was successful.

', function() { + bootbox.alert('

Congratulations

The application installation was successful.

', function () { var router = canRoute.router; router.reload(); }); - }, function(response) { + }, function (response) { waitingDialog.modal('hide'); showErrorDialog("Operation failed", response); }); @@ -69,10 +68,10 @@ export default Control.extend({ }); }, - '#install-app-github-btn click': function(el, ev) { + '#install-app-github-btn click': function (el, ev) { bootbox.confirm(templateInstallGithub(), - function(result) { + function (result) { if (result) { var url = 'github://' + $('#url').val(); @@ -88,16 +87,16 @@ export default Control.extend({ show: false }); - waitingDialog.on('shown.bs.modal', function() { + waitingDialog.on('shown.bs.modal', function () { - app.save(function(application) { + app.save(function (application) { waitingDialog.modal('hide'); - bootbox.alert('

Congratulations

The application installation was successful.

', function() { + bootbox.alert('

Congratulations

The application installation was successful.

', function () { var router = canRoute.router; router.reload(); }); - }, function(response) { + }, function (response) { waitingDialog.modal('hide'); showErrorDialog("Operation failed", response); }); @@ -108,12 +107,12 @@ export default Control.extend({ }); }, - '#reload-apps-btn click': function(el, ev) { + '#reload-apps-btn click': function (el, ev) { var element = this.element; Application.findAll({ reload: 'true' - }, function(applications) { + }, function (applications) { $(element).html(template({ applications: applications @@ -123,12 +122,12 @@ export default Control.extend({ }); }, - '.enable-disable-btn click': function(el, ev) { + '.enable-disable-btn click': function (el, ev) { var card = $(el).closest('.card'); var application = domData.get.call(card[0], 'application'); var enabled = !application.attr('enabled') - bootbox.confirm("Are you sure you want to " + (enabled ? "enable" : "disable") + " application " + application.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to " + (enabled ? "enable" : "disable") + " application " + application.attr('id') + "?", function (result) { if (result) { application.attr('enabled', enabled); @@ -140,13 +139,13 @@ export default Control.extend({ '', show: false }); - waitingDialog.on('shown.bs.modal', function() { + waitingDialog.on('shown.bs.modal', function () { - application.save(function(application) { + application.save(function (application) { waitingDialog.modal('hide'); bootbox.alert('

Congratulations

The application has been successfully ' + (enabled ? 'enabled' : 'disabled') + '.

'); - }, function(response) { + }, function (response) { waitingDialog.modal('hide'); showErrorDialog("Operation failed", response); }); @@ -157,12 +156,12 @@ export default Control.extend({ }); }, - '.delete-app-btn click': function(el, ev) { + '.delete-app-btn click': function (el, ev) { var card = $(el).closest('.card'); var application = domData.get.call(card[0], 'application'); - bootbox.confirm("Are you sure you want to delete " + application.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to delete " + application.attr('id') + "?", function (result) { if (result) { var waitingDialog = bootbox.dialog({ @@ -174,13 +173,13 @@ export default Control.extend({ show: false }); - waitingDialog.on('shown.bs.modal', function() { + waitingDialog.on('shown.bs.modal', function () { - application.destroy(function(application) { + application.destroy(function (application) { waitingDialog.modal('hide'); bootbox.alert('

Congratulations

The application has been successfully removed.

'); - }, function(response) { + }, function (response) { waitingDialog.modal('hide'); showErrorDialog("Operation failed", response); }); @@ -193,65 +192,37 @@ export default Control.extend({ }, - '.edit-settings-btn click': function(el, ev) { - - var card = $(el).closest('.card'); - var application = domData.get.call(card[0], 'application'); - - - bootbox.confirm(templateSettings({ - application: application - }), - function(result) { - if (result) { - var nextflowProfile = $('#nextflow-profile').val(); - var nextflowConfig = $('#nextflow-config').val(); - var nextflowWork = $('#nextflow-work').val(); - - application.attr('config').attr('nextflow.profile', nextflowProfile); - application.attr('config').attr('nextflow.config', nextflowConfig); - application.attr('config').attr('nextflow.work', nextflowWork); - application.save(function(data) {}, - function(response) { - showErrorDialog("Operation failed", response); - }); - } - }).find("div.modal-dialog").css({ "width": "80%" }); - - }, - - - '.edit-permission-btn click': function(el, ev) { + '.edit-permission-btn click': function (el, ev) { var card = $(el).closest('.card'); var application = domData.get.call(card[0], 'application'); Group.findAll({}, - function(groups) { + function (groups) { var selection = new canMap(); selection.attr('group', application.attr('permission')); selection.attr('name', ''); bootbox.confirm(templatePermission({ - selection: selection, - application: application, - groups: groups - }), - function(result) { + selection: selection, + application: application, + groups: groups + }), + function (result) { if (result) { var group = selection.attr('group'); if (group !== '') { application.attr('permission', group); - application.save(function(data) {}, - function(response) { + application.save(function (data) { }, + function (response) { showErrorDialog("Operation failed", response); }); } else { var name = selection.attr('name'); if (name !== '') { application.attr('permission', name); - application.save(function(data) {}, - function(response) { + application.save(function (data) { }, + function (response) { showErrorDialog("Operation failed", response); }); @@ -264,35 +235,5 @@ export default Control.extend({ }); }); - }, - - '.reinstall-btn click': function(el, ev) { - - var card = $(el).closest('.card'); - var application = domData.get.call(card[0], 'application'); - bootbox.confirm('

' + application.attr('id') + '

Force reinstallation of application?
All metafiles in HDFS are deleted and reimported on next job run.

', - function(result) { - if (result) { - application.attr('reinstall', 'true'); - application.save(function(data) {}, - function(response) { - showErrorDialog("Operation failed", response); - }); - } - }); - - - }, - - '.view-source-btn click': function(el, ev) { - - var card = $(el).closest('.card'); - var application = domData.get.call(card[0], 'application'); - var env = ''; - for (var property in application.attr('environment').attr()) { - env += property + '=' + application.attr('environment').attr(property) + '\n'; - } - - bootbox.alert('
File

' + application.attr('filename') + '

Status

' + application.attr('state') + '

' + '
Environment Variables

' + env + '

' + '
Source

' + application.attr('source') + '

'); } }); diff --git a/src/main/html/webapp/components/admin/app/list/list.stache b/src/main/html/webapp/components/admin/app/list/list.stache index c073467a..1bf6d6ad 100644 --- a/src/main/html/webapp/components/admin/app/list/list.stache +++ b/src/main/html/webapp/components/admin/app/list/list.stache @@ -25,11 +25,11 @@ {{#if(wdlApp)}} {{#if(wdlApp.workflow)}}
- Application + Application
{{else}}
- {{wdlApp.category}} + {{wdlApp.category}}
{{/if}} {{else}} @@ -53,20 +53,7 @@ {{else}}

{{{truncate(wdlApp.description, 200)}}}
- {{wdlApp.website}} - {{#is(state, 'completed')}} -
- - Installation: {{state}} - - - {{/is}} - {{#is(state, 'on demand')}} -
- - Installation: {{state}} - - {{/is}} + {{wdlApp.website}}

{{/errorMessage}} @@ -76,7 +63,9 @@
- + {{#if(wdlApp.workflow)}} + Settings + {{/if}} {{#enabled}} diff --git a/src/main/html/webapp/components/admin/app/settings/settings.js b/src/main/html/webapp/components/admin/app/settings/settings.js new file mode 100644 index 00000000..853cf669 --- /dev/null +++ b/src/main/html/webapp/components/admin/app/settings/settings.js @@ -0,0 +1,51 @@ +import Control from 'can-control'; +import domData from 'can-util/dom/data/data'; +import canMap from 'can-map'; +import canRoute from 'can-route'; + +import 'helpers/helpers'; +import $ from 'jquery'; +import bootbox from 'bootbox'; +import showErrorDialog from 'helpers/error-dialog'; + +import ApplicationSettings from 'models/application-settings'; +import template from './settings.stache'; + +export default Control.extend({ + + "init": function (element, options) { + var that = this; + + ApplicationSettings.findOne({ id: options.app }, function (application) { + that.application = application; + $(element).html(template({ + application: application + + })); + $(element).fadeIn(); + + }); + + }, + + + 'submit': function (form, event) { + event.preventDefault(); + + var nextflowProfile = $('#nextflow-profile').val(); + var nextflowConfig = $('#nextflow-config').val(); + var nextflowWork = $('#nextflow-work').val(); + + this.application.attr('config').attr('nextflow.profile', nextflowProfile); + this.application.attr('config').attr('nextflow.config', nextflowConfig); + this.application.attr('config').attr('nextflow.work', nextflowWork); + this.application.save(function (data) { + bootbox.alert("Application settings updated."); + }, + function (response) { + showErrorDialog("Operation failed", response); + }); + } + + +}); diff --git a/src/main/html/webapp/components/admin/app/list/settings/settings.stache b/src/main/html/webapp/components/admin/app/settings/settings.stache similarity index 56% rename from src/main/html/webapp/components/admin/app/list/settings/settings.stache rename to src/main/html/webapp/components/admin/app/settings/settings.stache index 32dceef9..1d06110e 100644 --- a/src/main/html/webapp/components/admin/app/list/settings/settings.stache +++ b/src/main/html/webapp/components/admin/app/settings/settings.stache @@ -1,5 +1,6 @@ +

Applications / {{application.id}}

+

{{application.id}}

-

Settings

Change settings for application {{application.wdlApp.name}}.

Nextflow
@@ -14,7 +15,31 @@

Custom Configuration:
All hardware specific parameters can be set here. If you use the AWSBatch profile this is the place to set your credentials

- + +
+ +
+ +
+
+ +
+ + +
+ +
+
    +{{#application.environment}} +
  • ${{{{name}}}
    Value: {{value}}
  • +{{/application.environment}} +
+
+
+ diff --git a/src/main/html/webapp/components/admin/job/list/list.js b/src/main/html/webapp/components/admin/job/list/list.js index 34e82ad2..e1f4d6bb 100644 --- a/src/main/html/webapp/components/admin/job/list/list.js +++ b/src/main/html/webapp/components/admin/job/list/list.js @@ -11,13 +11,12 @@ import template from './list.stache'; export default Control.extend({ - "init": function(element, options) { + "init": function (element, options) { $(element).html(template()); $(element).fadeIn(); - new JobTable("#job-list-running-stq", {state: "running-stq"}); - new JobTable("#job-list-running-ltq", {state: "running-ltq"}); - new JobTable("#job-list-current", {state: "current"}); + new JobTable("#job-list-running-ltq", { state: "running-ltq" }); + new JobTable("#job-list-current", { state: "current" }); } diff --git a/src/main/html/webapp/components/admin/job/list/list.stache b/src/main/html/webapp/components/admin/job/list/list.stache index 74a29197..99fa5069 100644 --- a/src/main/html/webapp/components/admin/job/list/list.stache +++ b/src/main/html/webapp/components/admin/job/list/list.stache @@ -2,21 +2,14 @@

This page lists all submitted and running jobs.

-

Preprocessing Queue

- -
-
- -
- -

Long Running Queue

+

Running Jobs


-

Active Jobs

+

Completed Jobs

diff --git a/src/main/html/webapp/components/admin/job/list/table/table.stache b/src/main/html/webapp/components/admin/job/list/table/table.stache index 6dd1cebc..29c62384 100644 --- a/src/main/html/webapp/components/admin/job/list/table/table.stache +++ b/src/main/html/webapp/components/admin/job/list/table/table.stache @@ -59,7 +59,7 @@

- {{user.username}}
+ {{username}}

diff --git a/src/main/html/webapp/components/admin/layout/layout.stache b/src/main/html/webapp/components/admin/layout/layout.stache index e86032c7..95d77850 100644 --- a/src/main/html/webapp/components/admin/layout/layout.stache +++ b/src/main/html/webapp/components/admin/layout/layout.stache @@ -28,6 +28,7 @@

diff --git a/src/main/html/webapp/components/admin/settings/general/general.js b/src/main/html/webapp/components/admin/settings/general/general.js index bd4a0078..a9a4d080 100644 --- a/src/main/html/webapp/components/admin/settings/general/general.js +++ b/src/main/html/webapp/components/admin/settings/general/general.js @@ -9,11 +9,11 @@ import showErrorDialog from 'helpers/error-dialog'; export default Control.extend({ - "init": function(element, options) { + "init": function (element, options) { var that = this; Settings.findOne({}, - function(settings) { + function (settings) { $(element).html(template({ settings: settings })); @@ -23,16 +23,21 @@ export default Control.extend({ }, - 'submit': function(form, event) { + 'submit': function (form, event) { event.preventDefault(); this.settings.attr('name', $(form).find("[name='name']").val()); + this.settings.attr('adminName', $(form).find("[name='adminName']").val()); + this.settings.attr('adminMail', $(form).find("[name='adminMail']").val()); + this.settings.attr('serverUrl', $(form).find("[name='serverUrl']").val()); this.settings.attr('backgroundColor', $(form).find("[name='background-color']").val()); this.settings.attr('foregroundColor', $(form).find("[name='foreground-color']").val()); this.settings.attr('googleAnalytics', $(form).find("[name='google-analytics']").val()); - this.settings.save(function(data) { + this.settings.attr('workspaceType', $(form).find("[name='workspaceType']").val()); + this.settings.attr('workspaceLocation', $(form).find("[name='workspaceLocation']").val()); + this.settings.save(function (data) { bootbox.alert("Settings updated."); - }, function(response) { + }, function (response) { showErrorDialog("Settings not updated", response); }); diff --git a/src/main/html/webapp/components/admin/settings/general/general.stache b/src/main/html/webapp/components/admin/settings/general/general.stache index 6b1c01ed..99af7577 100644 --- a/src/main/html/webapp/components/admin/settings/general/general.stache +++ b/src/main/html/webapp/components/admin/settings/general/general.stache @@ -6,11 +6,55 @@
+
Service
+
- +
+
+ + +
+ +
+ +
Contact
+ +
+ + +
+ + +
+ + +
+ +
+ +
Workspace
+ +
+ + +
+ + +
+ + +
+ +
+ +
Appearance
+
diff --git a/src/main/html/webapp/components/admin/settings/nextflow/nextflow.js b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.js new file mode 100644 index 00000000..c59be201 --- /dev/null +++ b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.js @@ -0,0 +1,39 @@ +import Control from 'can-control'; +import $ from 'jquery'; +import bootbox from 'bootbox'; + +import NextflowConfig from 'models/nextflow-config'; + +import template from './nextflow.stache'; +import showErrorDialog from 'helpers/error-dialog'; + +export default Control.extend({ + + "init": function (element, options) { + var that = this; + + NextflowConfig.findOne({}, + function (nextflowConfig) { + $(element).html(template({ + nextflowConfig: nextflowConfig + })); + that.nextflowConfig = nextflowConfig; + $(element).fadeIn(); + }); + + }, + + 'submit': function (form, event) { + event.preventDefault(); + + this.nextflowConfig.attr('content', $(form).find("[name='content']").val()); + this.nextflowConfig.save(function (data) { + bootbox.alert("Nextflow configuration updated."); + }, function (response) { + showErrorDialog("Nextflow configuration not updated", response); + }); + + + } + +}); diff --git a/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache new file mode 100644 index 00000000..3ee1dafd --- /dev/null +++ b/src/main/html/webapp/components/admin/settings/nextflow/nextflow.stache @@ -0,0 +1,32 @@ +

Nextflow Global Configuration

+ +

To configure your Nextflow settings, you can access the configuration file and fine-tune it according to your specific needs. Additionally, Nextflow offers robust support for environment variables, allowing you to easily integrate Cloudgene variables into your configuration. This nextflow.config file applies to all applications.

+ +
+ +
+ +
+ +
+ +
+
    +{{#nextflowConfig.variables}} +
  • ${{{{name}}}
    Value: {{value}}
  • +{{/nextflowConfig.variables}} +
+
+
+ +
+
+ +
+ + + + diff --git a/src/main/html/webapp/components/core/job/detail/detail.js b/src/main/html/webapp/components/core/job/detail/detail.js index 5ef45138..288c60a1 100644 --- a/src/main/html/webapp/components/core/job/detail/detail.js +++ b/src/main/html/webapp/components/core/job/detail/detail.js @@ -19,7 +19,7 @@ import template from './detail.stache'; export default Control.extend({ - "init": function(element, options) { + "init": function (element, options) { var that = this; this.active = true; @@ -28,55 +28,55 @@ export default Control.extend({ } JobDetails.findOne({ - id: options.job - }, function(job) { - - $(element).html(template({ - job: job, - tab: options.tab, - admin: options.appState.attr('user').attr('admin') - })); - - switch (options.tab) { - case 'results': - new ResultsControl("#tab-results", { - job: job - }); - break; - - case 'steps': - new StepsControl("#tab-steps", { - job: job - }); - break; - - case 'logs': - new LogsControl("#tab-logs", { - job: job - }); - break; - - default: - } - - $('[data-toggle="tooltip"]').tooltip() - - that.job = job; - that.refresh(); - - }, function(response) { - new ErrorPage(that.element, response); + id: options.job + }, function (job) { + + $(element).html(template({ + job: job, + tab: options.tab, + admin: options.appState.attr('user').attr('admin') + })); + + switch (options.tab) { + case 'results': + new ResultsControl("#tab-results", { + job: job + }); + break; + + case 'steps': + new StepsControl("#tab-steps", { + job: job + }); + break; + + case 'logs': + new LogsControl("#tab-logs", { + job: job + }); + break; + + default: } + $('[data-toggle="tooltip"]').tooltip() + + that.job = job; + that.refresh(); + + }, function (response) { + new ErrorPage(that.element, response); + } + ); }, // delete job - '#delete-btn click': function(el, ev) { + '#delete-btn click': function (el, ev) { var that = this; - bootbox.confirm("Are you sure you want to delete " + that.job.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to delete " + that.job.attr('id') + "?", function (result) { if (result) { var okButton = $("button[data-bb-handler='confirm']"); @@ -85,11 +85,11 @@ export default Control.extend({ var cancelButton = $("button[data-bb-handler='cancel']"); cancelButton.hide('hide'); - that.job.destroy(function() { + that.job.destroy(function () { // go to jobs page bootbox.hideAll(); window.location.hash = "!pages/jobs"; - }, function(response) { + }, function (response) { bootbox.hideAll(); showErrorDialog("Job could not be deleted", response); }); @@ -103,10 +103,10 @@ export default Control.extend({ // cancel job - '#cancel-btn click': function(el, ev) { + '#cancel-btn click': function (el, ev) { var that = this; - bootbox.confirm("Are you sure you want to cancel " + that.job.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to cancel " + that.job.attr('id') + "?", function (result) { if (result) { var okButton = $("button[data-bb-handler='confirm']"); @@ -118,10 +118,10 @@ export default Control.extend({ var operation = new JobOperation(); operation.attr('id', that.job.attr('id')); operation.attr('action', 'cancel'); - operation.save(function() { + operation.save(function () { bootbox.hideAll(); that.refresh(); - }, function(response) { + }, function (response) { bootbox.hideAll(); showErrorDialog("Job could not be canceld", response); }); @@ -133,10 +133,10 @@ export default Control.extend({ }, - '#restart-btn click': function(el, ev) { + '#restart-btn click': function (el, ev) { var that = this; - bootbox.confirm("Are you sure you want to restart " + that.job.attr('id') + "?", function(result) { + bootbox.confirm("Are you sure you want to restart " + that.job.attr('id') + "?", function (result) { if (result) { var okButton = $("button[data-bb-handler='confirm']"); @@ -148,10 +148,10 @@ export default Control.extend({ var operation = new JobOperation(); operation.attr('id', that.job.attr('id')); operation.attr('action', 'restart'); - operation.save(function() { + operation.save(function () { bootbox.hideAll(); window.location.hash = "#!pages/jobs"; - }, function(response) { + }, function (response) { bootbox.hideAll(); showErrorDialog("Job could not be restarted", response); }); @@ -165,40 +165,38 @@ export default Control.extend({ // refresh if job is running - refresh: function() { + refresh: function () { var that = this; if (!JobRefresher.needsUpdate(that.job)) { return; } Job.findOne({ id: that.job.id - }, function(currentJob) { + }, function (currentJob) { currentJob.syncTime(); that.job.attr('state', currentJob.attr('state')); that.job.attr('startTime', currentJob.attr('startTime')); that.job.attr('endTime', currentJob.attr('endTime')); - that.job.attr('setupStartTime', currentJob.attr('setupStartTime')); - that.job.attr('setupEndTime', currentJob.attr('setupEndTime')); that.job.attr('steps', currentJob.attr('steps')); that.job.attr('positionInQueue', currentJob.attr('positionInQueue')); // needs refresh if (JobRefresher.needsUpdate(currentJob) && that.active) { - setTimeout(function() { + setTimeout(function () { that.refresh(); }, 5000); } else { // updates details (results, startTime, endTime, ...) JobDetails.findOne({ id: that.job.id - }, function(job) { + }, function (job) { if (that.active) { var router = canRoute.router; router.reload(); } - }, function(response) { + }, function (response) { new ErrorPage(that.element, response); }); @@ -209,7 +207,7 @@ export default Control.extend({ }, - destroy: function() { + destroy: function () { this.active = false; Control.prototype.destroy.call(this); } @@ -218,6 +216,6 @@ export default Control.extend({ var JobRefresher = {}; -JobRefresher.needsUpdate = function(job) { +JobRefresher.needsUpdate = function (job) { return job.attr("state") == 1 || job.attr("state") == 2 || job.attr("state") == 3; }; diff --git a/src/main/html/webapp/components/core/job/detail/steps/steps.stache b/src/main/html/webapp/components/core/job/detail/steps/steps.stache index a0f3e210..1e61f682 100644 --- a/src/main/html/webapp/components/core/job/detail/steps/steps.stache +++ b/src/main/html/webapp/components/core/job/detail/steps/steps.stache @@ -33,7 +33,7 @@ {{#is(type, 3)}}
-

{{{replaceNL(message)}}}

+

{{{replaceNL(message)}}}

diff --git a/src/main/html/webapp/models/application-settings.js b/src/main/html/webapp/models/application-settings.js new file mode 100644 index 00000000..0e264413 --- /dev/null +++ b/src/main/html/webapp/models/application-settings.js @@ -0,0 +1,8 @@ +import Model from 'can-connect/can/model/model'; + +export default Model.extend({ + findOne: 'GET /api/v2/server/apps/{id}/settings', + update: 'PUT /api/v2/server/apps/{id}/settings' +}, { + +}); diff --git a/src/main/html/webapp/models/job.js b/src/main/html/webapp/models/job.js index 5477eba8..d4acfe9e 100644 --- a/src/main/html/webapp/models/job.js +++ b/src/main/html/webapp/models/job.js @@ -9,33 +9,27 @@ export default Model.extend({ destroy: 'DELETE /api/v2/jobs/{id}', }, { - 'syncTime': function() { + 'syncTime': function () { if (this.attr('startTime') > 0 && this.attr('endTime') === 0) { this.attr('endTime', this.attr('currentTime')); } else { this.attr('endTime', this.attr('endTime')); } - if (this.attr('setupStartTime') > 0 && this.attr('setupEndTime') === 0) { - this.attr('setupEndTime', this.attr('currentTime')); - } else { - this.attr('setupEndTime', this.attr('setupEndTime')); - } - }, define: { 'longName': { - get: function() { + get: function () { return this.attr('id') != this.attr('name') ? this.attr('name') : this.attr('id'); } }, 'executionTime': { - get: function() { - var start = this.attr('setupStartTime') + this.attr('startTime'); - var end = this.attr('setupEndTime') + this.attr('endTime'); + get: function () { + var start = this.attr('startTime'); + var end = this.attr('endTime'); var current = this.attr('currentTime'); if (start === 0 && end === 0) { @@ -61,7 +55,7 @@ export default Model.extend({ }, 'stateAsText': { - get: function() { + get: function () { if (this.attr('state') == 1) { return 'Waiting'; } else if (this.attr('state') == 2) { @@ -85,7 +79,7 @@ export default Model.extend({ }, 'stateAsClass': { - get: function() { + get: function () { if (this.attr('state') == '-1') { return 'dark'; @@ -119,7 +113,7 @@ export default Model.extend({ }, 'stateAsImage': { - get: function() { + get: function () { if (this.attr('state') == '-1') { return "fas fa-moon"; @@ -155,67 +149,67 @@ export default Model.extend({ }, 'isInQueue': { - get: function() { + get: function () { return this.attr('state') == 1 && this.attr('positionInQueue') != -1; } }, 'isPending': { - get: function() { + get: function () { return this.attr('state') == '-1'; } }, 'isRetired': { - get: function() { + get: function () { return this.attr('state') == '7'; } }, 'willBeRetired': { - get: function() { + get: function () { return this.attr('state') == 8 || this.attr('state') == 9; } }, 'canResetCounters': { - get: function() { + get: function () { return this.attr('state') > 3; } }, 'canSendRetireNotification': { - get: function() { + get: function () { return this.attr('state') > 3 && this.attr('state') != '8' && this.attr('state') != '9'; } }, 'canIncreaseRetireDate': { - get: function() { + get: function () { return this.attr('state') == '8' || this.attr('state') == '9'; } }, 'canShowLog': { - get: function() { + get: function () { return this.attr('logs') != undefined && this.attr('logs') != ''; } }, 'canCancel': { - get: function() { + get: function () { return this.attr('state') <= '3' && this.attr('state') != '-1'; } }, 'canRetireJob': { - get: function() { + get: function () { return this.attr('state') > '3' && (this.attr('state') == '4' || this.attr('state') != '5' || this.attr('state') != '6'); } }, 'canDelete': { - get: function() { + get: function () { return this.attr('state') > '3' || this.attr('state') == '-1'; } } diff --git a/src/main/html/webapp/models/nextflow-config.js b/src/main/html/webapp/models/nextflow-config.js new file mode 100644 index 00000000..6efb6a02 --- /dev/null +++ b/src/main/html/webapp/models/nextflow-config.js @@ -0,0 +1,7 @@ +import Model from 'can-connect/can/model/model'; + +export default Model.extend({ + findOne: 'GET /api/v2/admin/server/nextflow/config', + create: 'POST /api/v2/admin/server/nextflow/config/update', + update: 'POST /api/v2/admin/server/nextflow/config/update' +}, {}); diff --git a/src/main/java/cloudgene/mapred/CommandLineInterface.java b/src/main/java/cloudgene/mapred/CommandLineInterface.java index b25d2fec..993da818 100644 --- a/src/main/java/cloudgene/mapred/CommandLineInterface.java +++ b/src/main/java/cloudgene/mapred/CommandLineInterface.java @@ -5,12 +5,10 @@ import cloudgene.mapred.cli.InstallGitHubApplication; import cloudgene.mapred.cli.ListApplications; import cloudgene.mapred.cli.RemoveApplication; -import cloudgene.mapred.cli.RunApplication; import cloudgene.mapred.cli.ShowPlugins; import cloudgene.mapred.cli.ShowVersion; import cloudgene.mapred.cli.StartServer; import cloudgene.mapred.cli.ValidateApplication; -import cloudgene.mapred.cli.VerifyCluster; import cloudgene.mapred.server.Application; import cloudgene.mapred.util.BuildUtil; import genepi.base.Toolbox; @@ -33,7 +31,6 @@ private void printHeader() { public static void main(String[] args) throws Exception { CommandLineInterface toolbox = new CommandLineInterface("cloudgene", args); - toolbox.addTool("run", RunApplication.class); toolbox.addTool("install", InstallApplication.class); toolbox.addTool("gh", InstallGitHubApplication.class); toolbox.addTool("github-install", InstallGitHubApplication.class); @@ -42,7 +39,6 @@ public static void main(String[] args) throws Exception { toolbox.addTool("remove", RemoveApplication.class); toolbox.addTool("server", StartServer.class); toolbox.addTool("validate", ValidateApplication.class); - toolbox.addTool("verify-cluster", VerifyCluster.class); toolbox.addTool("plugins", ShowPlugins.class); toolbox.addTool("version", ShowVersion.class); toolbox.start(); diff --git a/src/main/java/cloudgene/mapred/apps/ApplicationInstaller.java b/src/main/java/cloudgene/mapred/apps/ApplicationInstaller.java deleted file mode 100644 index 7da435e1..00000000 --- a/src/main/java/cloudgene/mapred/apps/ApplicationInstaller.java +++ /dev/null @@ -1,146 +0,0 @@ -package cloudgene.mapred.apps; - -import java.io.File; -import java.io.IOException; -import java.net.URL; -import java.util.List; -import java.util.Map; - -import org.apache.commons.io.FileUtils; - -import cloudgene.mapred.jobs.Environment; -import cloudgene.mapred.plugins.PluginManager; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; -import cloudgene.mapred.util.Settings; -import cloudgene.mapred.wdl.WdlApp; -import genepi.hadoop.HdfsUtil; -import genepi.hadoop.S3Util; -import genepi.hadoop.io.HdfsLineWriter; -import genepi.io.FileUtil; - -public class ApplicationInstaller { - - public static boolean isInstalled(WdlApp app, Settings settings) { - - PluginManager manager = PluginManager.getInstance(); - //skip hdfs installation when Hadoop plugin is not activated - if (!manager.isEnabled(HadoopPlugin.ID)) { - return true; - } - - Map environment = Environment.getApplicationVariables(app, settings); - String target = environment.get("hdfs_app_folder"); - String installationFile = HdfsUtil.path(target, "installed"); - return HdfsUtil.exists(installationFile); - } - - public static void uninstall(WdlApp app, Settings settings) throws IOException { - - PluginManager manager = PluginManager.getInstance(); - if (!manager.isEnabled(HadoopPlugin.ID)) { - return; - } - - Map environment = Environment.getApplicationVariables(app, settings); - String target = environment.get("hdfs_app_folder"); - HdfsUtil.delete(target); - } - - public static void install(WdlApp app, Settings settings) throws IOException { - - PluginManager manager = PluginManager.getInstance(); - if (!manager.isEnabled(HadoopPlugin.ID)) { - return; - } - - Map environment = Environment.getApplicationVariables(app, settings); - String target = environment.get("hdfs_app_folder"); - String installationFile = HdfsUtil.path(target, "installed"); - HdfsUtil.delete(target); - ApplicationInstaller.runCommands(app.getInstallation(), environment); - HdfsLineWriter lineWriter = new HdfsLineWriter(installationFile); - lineWriter.write(System.currentTimeMillis() + ""); - lineWriter.close(); - } - - private static void runCommands(List> commands, Map environment) - throws IOException { - - for (Map commandObject : commands) { - runCommand(commandObject, environment); - } - } - - private static void runCommand(Map commandObject, Map environment) - throws IOException { - if (commandObject.keySet().size() == 1) { - for (String command : commandObject.keySet()) { - Object parametersObject = commandObject.get(command); - Map parameters = (Map) parametersObject; - runCommand(command, parameters, environment); - } - } else { - throw new IOException("Unknown command structure."); - } - } - - private static void runCommand(String command, Map parameters, Map environment) - throws IOException { - switch (command) { - case "import": - String source = Environment.env((String) parameters.get("source"), environment); - String target = Environment.env((String) parameters.get("target"), environment); - System.out.println("Import data from " + source + " to " + target + "..."); - runImportCommand(source, target); - break; - default: - throw new IOException("Unknown command '" + command + "'"); - } - } - - private static void runImportCommand(String source, String target) throws IOException { - String tempFilename = FileUtil.path("temp", "hdfs_import.tempfile"); - if (source.startsWith("http://") || source.startsWith("https://")) { - // download - FileUtils.copyURLToFile(new URL(source), new File(tempFilename)); - - if (source.endsWith(".zip")) { - HdfsUtil.createDirectory(target); - HdfsUtil.putZip(tempFilename, target); - } else if (source.endsWith(".gz")) { - HdfsUtil.createDirectory(target); - HdfsUtil.putTarGz(tempFilename, target); - } else { - HdfsUtil.put(tempFilename, target); - } - - } else if (source.startsWith("s3://")) { - - S3Util.copyToFile(source, new File(tempFilename)); - - if (source.endsWith(".zip")) { - HdfsUtil.createDirectory(target); - HdfsUtil.putZip(tempFilename, target); - } else if (source.endsWith(".gz")) { - HdfsUtil.createDirectory(target); - HdfsUtil.putTarGz(tempFilename, target); - } else { - HdfsUtil.put(tempFilename, target); - } - - } else { - if (source.endsWith(".zip")) { - HdfsUtil.createDirectory(target); - HdfsUtil.putZip(source, target); - } else if (source.endsWith(".gz")) { - HdfsUtil.createDirectory(target); - HdfsUtil.putTarGz(source, target); - } else { - HdfsUtil.put(source, target); - } - } - - FileUtil.deleteFile(tempFilename); - } - -} diff --git a/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java b/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java index bc2cc641..2f979201 100644 --- a/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java +++ b/src/main/java/cloudgene/mapred/apps/ApplicationRepository.java @@ -22,8 +22,8 @@ import cloudgene.mapred.util.GitHubException; import cloudgene.mapred.util.GitHubUtil; import cloudgene.mapred.util.GitHubUtil.Repository; +import cloudgene.mapred.util.S3Util; import cloudgene.mapred.wdl.WdlApp; -import genepi.hadoop.S3Util; import genepi.io.FileUtil; import net.lingala.zip4j.ZipFile; import net.lingala.zip4j.exception.ZipException; diff --git a/src/main/java/cloudgene/mapred/cli/CloneApplications.java b/src/main/java/cloudgene/mapred/cli/CloneApplications.java index e2cdae85..5250b7c8 100644 --- a/src/main/java/cloudgene/mapred/cli/CloneApplications.java +++ b/src/main/java/cloudgene/mapred/cli/CloneApplications.java @@ -10,7 +10,7 @@ import com.esotericsoftware.yamlbeans.YamlReader; import cloudgene.mapred.apps.Application; -import genepi.hadoop.S3Util; +import cloudgene.mapred.util.S3Util; public class CloneApplications extends BaseTool { diff --git a/src/main/java/cloudgene/mapred/cli/CommandLineUtil.java b/src/main/java/cloudgene/mapred/cli/CommandLineUtil.java index 3cb21a9d..0a530bcf 100644 --- a/src/main/java/cloudgene/mapred/cli/CommandLineUtil.java +++ b/src/main/java/cloudgene/mapred/cli/CommandLineUtil.java @@ -12,8 +12,6 @@ import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlParameterInput; import cloudgene.mapred.wdl.WdlParameterInputType; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; public class CommandLineUtil { @@ -79,29 +77,7 @@ public static Map createParams(WdlApp app, CommandLine line, Str String entryName = tmpFile.getName(); if (input.isHdfs()) { - String targetPath = HdfsUtil.path(hdfs, "input", input.getId()); - if (tmpFile.isDirectory()) { - String[] files = FileUtil.getFiles(value, ""); - for (String sourceFile : files) { - if (!new File(sourceFile).isDirectory()) { - String name = FileUtil.getFilename(sourceFile); - String target = HdfsUtil.path(targetPath, name); - HdfsUtil.put(sourceFile, target); - } - - } - } else { - String target = HdfsUtil.path(targetPath, entryName); - HdfsUtil.put(value, target); - } - - if (input.isFolder()) { - // folder - props.put(input.getId(), HdfsUtil.path(hdfs, "input", input.getId())); - } else { - // file - props.put(input.getId(), HdfsUtil.path(hdfs, "input", input.getId(), entryName)); - } + throw new RuntimeException("HDFS not supported in CG3"); } else { props.put(input.getId(), value); diff --git a/src/main/java/cloudgene/mapred/cli/RunApplication.java b/src/main/java/cloudgene/mapred/cli/RunApplication.java deleted file mode 100644 index cd325e1a..00000000 --- a/src/main/java/cloudgene/mapred/cli/RunApplication.java +++ /dev/null @@ -1,422 +0,0 @@ -package cloudgene.mapred.cli; - -import java.io.File; -import java.io.FileNotFoundException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.Map; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; - -import com.esotericsoftware.yamlbeans.YamlException; - -import cloudgene.mapred.apps.Application; -import cloudgene.mapred.core.User; -import cloudgene.mapred.jobs.CloudgeneJob; -import cloudgene.mapred.jobs.CloudgeneStep; -import cloudgene.mapred.jobs.Message; -import cloudgene.mapred.jobs.WorkflowEngine; -import cloudgene.mapred.plugins.PluginManager; -import cloudgene.mapred.wdl.WdlApp; -import cloudgene.mapred.wdl.WdlParameterInput; -import cloudgene.mapred.wdl.WdlParameterInputType; -import cloudgene.mapred.wdl.WdlParameterOutput; -import cloudgene.mapred.wdl.WdlReader; -import genepi.hadoop.HadoopCluster; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; - -public class RunApplication extends BaseTool { - - public static final String DEFAULT_HADOOP_USER = "cloudgene"; - - private WdlApp app = null; - - private boolean output = false; - - private boolean logging = false; - - private boolean force = false; - - public RunApplication(String[] args) { - super(args); - } - - @Override - public int run() { - return 0; - } - - // we override start instead of run, because we use our own cli parser based - // on inputs defined in the yaml file - @Override - public int start() { - - // call init manualy - init(); - - if (args.length < 1) { - printError("No filename of a cloudgene.yaml file or id of an application found."); - System.out.println("cloudgene run "); - System.out.println(); - return 1; - } - - String filename = args[0]; - - // check if file exists - if (new File(filename).exists()) { - // load wdl app from yaml file - try { - app = WdlReader.loadAppFromFile(filename); - - } catch (FileNotFoundException e1) { - printError("File '" + filename + "' not found."); - return 1; - } catch (YamlException e) { - printError("Syntax error in file '" + filename + "':"); - printError(e.getMessage()); - return 1; - } catch (Exception e2) { - printError("Error loading file '" + filename + "':"); - printError(e2.getMessage()); - return 1; - } - } else { - // check if application name is installed - Application application = repository.getById(filename); - if (application == null) { - printError("Application or file " + filename + " not found."); - return 1; - } - - if (application.hasSyntaxError()) { - printError("Syntax error in file '" + application.getFilename() + "':"); - printError(application.getErrorMessage()); - return 1; - } - - app = application.getWdlApp(); - - } - - // print application details - System.out.println(); - System.out.println(app.getName() + " " + app.getVersion()); - if (app.getWebsite() != null && !app.getWebsite().isEmpty()) { - System.out.println(app.getWebsite()); - } - if (app.getAuthor() != null && !app.getAuthor().isEmpty()) { - System.out.println(app.getAuthor()); - } - System.out.println(); - - if (app.getWorkflow() == null || app.getWorkflow().getSteps() == null - || app.getWorkflow().getSteps().size() == 0) { - printError("Application has no workflow. It seems that " + app.getName() + " is a data package."); - return 1; - } - - // create the command line parser - CommandLineParser parser = new DefaultParser(); - - // create options for each input param in yaml file - Options options = CommandLineUtil.createOptionsFromApp(app); - - // add general options: run on docker - Option loggingOption = new Option(null, "show-log", false, "Stream logging messages to stdout"); - loggingOption.setRequired(false); - options.addOption(loggingOption); - - // add general options: run on docker - Option noOutputOption = new Option(null, "show-output", false, "Stream output to stdout"); - noOutputOption.setRequired(false); - options.addOption(noOutputOption); - - // add general options: run on docker - Option forceOption = new Option(null, "force", false, - "Force Cloudgene to reinstall application in HDFS even if it already installed."); - forceOption.setRequired(false); - options.addOption(forceOption); - - // add general options: hadoop configuration - Option confOption = new Option(null, "conf", true, "Hadoop configuration folder"); - confOption.setRequired(false); - options.addOption(confOption); - - // add general options: output folder - Option outputFolderOption = new Option(null, "output", true, "Output folder"); - outputFolderOption.setRequired(false); - options.addOption(outputFolderOption); - - // add general options: hadoop user - Option usernameOption = new Option(null, "user", true, - "Hadoop username [default: " + DEFAULT_HADOOP_USER + "]"); - usernameOption.setRequired(false); - options.addOption(usernameOption); - - // parse the command line arguments - CommandLine line = null; - try { - - line = parser.parse(options, args); - - } catch (Exception e) { - printError(e.getMessage()); - HelpFormatter formatter = new HelpFormatter(); - formatter.printHelp("input parameters:", options); - System.out.println(); - return 1; - } - - if (line.hasOption("show-log")) { - logging = true; - } - - if (line.hasOption("show-output")) { - output = true; - } - - if (line.hasOption("force")) { - force = true; - } - - if (line.hasOption("conf")) { - - String conf = line.getOptionValue("conf"); - String username = line.getOptionValue("user", null); - printText(0, spaces("[INFO]", 8) + "Use Haddop configuration folder " + conf - + (username != null ? " with username " + username : "")); - HadoopCluster.setConfPath("Unknown", conf, username); - - } else { - if (settings.getCluster() == null) { - // printText(0, spaces("[INFO]", 8) + "No external Haddop cluster set."); - } - } - - // show supported plugins - PluginManager manager = PluginManager.getInstance(); - manager.initPlugins(settings); - - /* - * for (IPlugin plugin : manager.getPlugins()) { if (manager.isEnabled(plugin)) - * { printText(0, spaces("[INFO]", 8) + plugin.getStatus()); } else { - * printText(0, spaces("[WARN]", 8) + plugin.getStatus()); } } - */ - - // create directories - FileUtil.createDirectory(settings.getTempPath()); - - // start workflow engine - WorkflowEngine engine = new WorkflowEngine(1, 1); - new Thread(engine).start(); - - // init job - - SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd-HHmmss"); - String id = "job-" + sdf.format(new Date()); - String hdfs = ""; - try { - hdfs = HdfsUtil.path("cloudgene-cli", id); - } catch (NoClassDefFoundError e) { - - } - String local = FileUtil.path(id); - - if (line.hasOption("output")) { - local = line.getOptionValue("output"); - } - - FileUtil.createDirectory(local); - - // file params with values from cmdline - try { - Map params = CommandLineUtil.createParams(app, line, local, hdfs); - - // dummy user - User user = new User(); - user.setUsername("local"); - user.setPassword("local"); - user.makeAdmin(); - - CloudgeneJob job = new CloudgeneJob(user, id, app, params) { - - private int count = 0; - - @Override - public boolean afterSubmission() { - boolean result = super.afterSubmission(); - if (result) { - printParameters(this, app); - } - return result; - } - - @Override - public void onStepStarted(CloudgeneStep step) { - super.onStepStarted(step); - System.out.println(); - printText(0, spaces("[INFO]", 8) + step.getName() + "..."); - } - - @Override - public void onStepFinished(CloudgeneStep step) { - super.onStepFinished(step); - int c = 0; - for (Message message : step.getLogMessages()) { - String text = message.getMessage().replaceAll("
", "\n").replaceAll("\n", - "\n" + spaces(4 + 8)); - String type = getDescription(message.getType()); - type = spaces("[" + type + "]", 8); - if (message.getType() == Message.OK) { - type = makeGreen(type); - } else if (message.getType() == Message.ERROR) { - type = makeRed(type); - } - - printText(0, spaces(type, 8) + text); - c++; - if (c < step.getLogMessages().size()) { - printSingleLine(4); - } - } - } - - @Override - public void writeOutputln(String line) { - super.writeOutputln(line); - if (output) { - printText(0, spaces("[OUT]", 8) + line); - } - } - - @Override - public void writeLog(String line) { - super.writeLog(line); - if (logging) { - printText(0, spaces("[LOG]", 8) + line); - } - } - - }; - job.setId(id); - job.setName(id); - job.setLocalWorkspace(local); - job.setHdfsWorkspace(hdfs); - job.setSettings(settings); - job.setRemoveHdfsWorkspace(true); - job.setApplication(app.getName() + " " + app.getVersion()); - job.setApplicationId(app.getId()); - job.forceInstallation(force); - - if (force) { - printText(0, spaces("[INFO]", 8) + "Force Cloudgene to reinstall application in HDFS"); - } - - printText(0, spaces("[INFO]", 8) + "Submit job " + id + "..."); - - // submit job - engine.submit(job); - - // wait until job is complete. - while (!job.isComplete()) { - Thread.sleep(1000); - } - long wallTime = (job.getFinishedOn() - job.getSubmittedOn()) / 1000; - long setupTime = (job.getSetupEndTime() - job.getSetupStartTime()) / 1000; - long pipelineTime = (job.getFinishedOn() - job.getSubmittedOn()) / 1000; - // print steps and feedback - System.out.println(); - - if (job.getState() == CloudgeneJob.STATE_SUCCESS) { - - printlnInGreen("Done! Executed without errors."); - System.out.println("Results can be found in file://" + (new File(local)).getAbsolutePath()); - System.out.println(); - if (logging) { - printText(0, spaces("[LOG]", 8) + "Wall-Time: " + wallTime + " sec [Setup: " + setupTime - + " sec, Pipeline: " + pipelineTime + " sec]"); - } - System.out.println(); - return 0; - - } else { - printlnInRed("Error: Execution failed."); - System.out.println(); - System.out.println(); - if (logging) { - printText(0, spaces("[LOG]", 8) + "Wall-Time: " + wallTime + " [Setup: " + setupTime - + ", Pipeline: " + pipelineTime); - System.out.println(); - } - - System.out.println(); - - return 1; - } - } catch (FileNotFoundException e) { - printlnInRed("Error: " + e.getMessage()); - System.out.println(); - System.out.println(); - return 1; - - } catch (Exception e) { - printlnInRed("Error: Execution failed."); - System.out.println("Details:"); - e.printStackTrace(); - System.out.println(); - System.out.println(); - return 1; - } - - } - - public void printParameters(CloudgeneJob job, WdlApp app) { - if (app.getWorkflow().getInputs().size() > 0) { - printText(2 + 6, "Input values: "); - for (WdlParameterInput input : app.getWorkflow().getInputs()) { - if (input.getTypeAsEnum() != WdlParameterInputType.AGBCHECKBOX - && input.getTypeAsEnum() != WdlParameterInputType.TERMS_CHECKBOX && !input.isAdminOnly() - && input.isVisible()) { - printText(4 + 6, input.getId() + ": " + job.getContext().get(input.getId())); - } - } - } - - if (app.getWorkflow().getOutputs().size() > 0) { - printText(2 + 6, "Results:"); - for (WdlParameterOutput output : app.getWorkflow().getOutputs()) { - if (output.isDownload() && !output.isAdminOnly()) { - printText(4 + 6, output.getDescription() + ": " + job.getContext().get(output.getId())); - } - } - } - }; - - public String getDescription(int type) { - switch (type) { - case Message.ERROR: - return "ERROR"; - case Message.OK: - return "OK"; - case Message.WARNING: - return "WARN"; - case Message.RUNNING: - return "RUN"; - default: - return "??"; - } - } - - @Override - public void createParameters() { - - } - -} diff --git a/src/main/java/cloudgene/mapred/cli/StartServer.java b/src/main/java/cloudgene/mapred/cli/StartServer.java index a055d90c..a2feed07 100644 --- a/src/main/java/cloudgene/mapred/cli/StartServer.java +++ b/src/main/java/cloudgene/mapred/cli/StartServer.java @@ -16,7 +16,6 @@ import cloudgene.mapred.util.Config; import cloudgene.mapred.util.Settings; import genepi.base.Tool; -import genepi.hadoop.HadoopCluster; import io.micronaut.context.env.Environment; import io.micronaut.runtime.Micronaut; @@ -35,9 +34,7 @@ public StartServer(String[] args) { @Override public void createParameters() { - addOptionalParameter("user", "Hadoop username [default: " + DEFAULT_HADOOP_USER + "]", Tool.STRING); addOptionalParameter("port", "running webinterface on this port [default: 8082]", Tool.STRING); - addOptionalParameter("conf", "Hadoop configuration folder", Tool.STRING); } @Override @@ -49,24 +46,6 @@ public int run() { System.setProperty(ContextInitializer.CONFIG_FILE_PROPERTY, "logback-dev.xml"); } - - if (getValue("conf") != null) { - - String conf = getValue("conf").toString(); - - String username = null; - if (getValue("user") != null) { - username = getValue("user").toString(); - } - System.out.println( - "Use Haddop configuration folder " + conf + (username != null ? " with username " + username : "")); - HadoopCluster.setConfPath("Unknown", conf, username); - - } else { - // if (settings.getCluster() == null) { - // System.out.println("No external Haddop cluster set."); - // } - } try { @@ -84,6 +63,7 @@ public int run() { Map properties = new HashMap(); properties.put("micronaut.server.port", port); if (Application.settings.getUploadLimit() != -1) { + properties.put("micronaut.server.maxRequestSize", Application.settings.getUploadLimit() + "MB"); properties.put("micronaut.server.multipart.maxFileSize", Application.settings.getUploadLimit() + "MB"); } @@ -151,10 +131,6 @@ protected Settings loadSettings(Config config) throws FileNotFoundException, Yam settings = new Settings(config); } - if (!settings.testPaths()) { - System.exit(1); - } - if (settings.getServerUrl() == null || settings.getServerUrl().trim().isEmpty()) { System.out.println("serverUrl not set. Please set serverUrl in file '" + settingsFilename + "'"); System.exit(1); diff --git a/src/main/java/cloudgene/mapred/cli/VerifyCluster.java b/src/main/java/cloudgene/mapred/cli/VerifyCluster.java deleted file mode 100644 index 84bf5f94..00000000 --- a/src/main/java/cloudgene/mapred/cli/VerifyCluster.java +++ /dev/null @@ -1,52 +0,0 @@ -package cloudgene.mapred.cli; - -import genepi.hadoop.HadoopCluster; -import genepi.hadoop.HadoopUtil; - -public class VerifyCluster extends BaseTool { - - public VerifyCluster(String[] args) { - super(args); - } - - @Override - public void createParameters() { - } - - @Override - public int run() { - - try { - HadoopCluster.verifyCluster(); - - StringBuffer state = new StringBuffer(); - state.append("Mode: " + (HadoopUtil.getInstance().isInSafeMode() ? "Safe Mode" : "Running")); - state.append("JobTracker: " + HadoopCluster.getJobTracker() + "\n"); - state.append("Default FS: " + HadoopCluster.getDefaultFS() + "\n"); - state.append("State: " + HadoopCluster.getJobTrackerStatus().toString() + "\n"); - state.append("MapTask: " + HadoopCluster.getMaxMapTasks() + "\n"); - state.append("ReduceTask: " + HadoopCluster.getMaxReduceTasks() + "\n"); - state.append("Nodes\n"); - for (String tracker : HadoopCluster.getActiveTrackerNames()) { - state.append(" " + tracker + "\n"); - } - state.append("Blacklist:\n"); - for (String tracker : HadoopCluster.getBlacklistedTrackerNames()) { - state.append(" " + tracker + "\n"); - } - System.out.println(state.toString()); - System.out.println(); - printlnInGreen("Hadoop cluster is ready to use."); - System.out.println(); - - return 0; - - } catch (Exception e) { - System.out.println(); - printlnInRed(e.getMessage()); - System.out.println(); - return 1; - } - - } -} \ No newline at end of file diff --git a/src/main/java/cloudgene/mapred/database/DownloadDao.java b/src/main/java/cloudgene/mapred/database/DownloadDao.java index 3d511ac6..c3256763 100644 --- a/src/main/java/cloudgene/mapred/database/DownloadDao.java +++ b/src/main/java/cloudgene/mapred/database/DownloadDao.java @@ -30,13 +30,13 @@ public boolean insert(Download download) { try { Object[] params = new Object[7]; - params[0] = download.getParameterId(); + params[0] = download.getParameter().getId(); params[1] = download.getName(); params[2] = download.getPath(); params[3] = download.getHash(); params[4] = download.getCount(); params[5] = download.getSize(); - params[6] = download.getParameter().getId(); + params[6] = -1; update(sql.toString(), params); @@ -165,7 +165,6 @@ public Object mapRow(ResultSet rs, int row) throws SQLException { result.setName(rs.getString("name")); result.setPath(rs.getString("path")); result.setSize(rs.getString("size")); - result.setParameterId(rs.getInt("parameter_id")); return result; } diff --git a/src/main/java/cloudgene/mapred/database/JobDao.java b/src/main/java/cloudgene/mapred/database/JobDao.java index 39132ec3..244ba398 100644 --- a/src/main/java/cloudgene/mapred/database/JobDao.java +++ b/src/main/java/cloudgene/mapred/database/JobDao.java @@ -47,9 +47,9 @@ public boolean insert(AbstractJob job) { params[8] = job.getApplication(); params[9] = job.getApplicationId(); params[10] = job.getSubmittedOn(); - params[11] = job.getFinishedOn(); - params[12] = job.getSetupStartTime(); - params[13] = job.getSetupEndTime(); + params[11] = job.getEndTime(); + params[12] = -1; + params[13] = -1; params[14] = trimToLength(job.getUserAgent(), 350); update(sql.toString(), params); @@ -66,11 +66,12 @@ public boolean insert(AbstractJob job) { public boolean update(AbstractJob job) { StringBuilder sql = new StringBuilder(); - sql.append("update job "); - sql.append(" set name = ?, state = ?, "); + sql.append("update job set "); + sql.append(" name = ?, state = ?, "); sql.append(" start_time = ?, end_time = ?, "); - sql.append( - " user_id = ?, s3_url = ?, type = ?, deleted_on = ?, application = ?, application_id = ?, submitted_on = ?, finished_on = ?, setup_start_time = ?, setup_end_time = ? "); + sql.append(" user_id = ?, s3_url = ?, type = ?, deleted_on = ?, "); + sql.append(" application = ?, application_id = ?, submitted_on = ?, "); + sql.append(" finished_on = ?, setup_start_time = ?, setup_end_time = ? "); sql.append("where id = ? "); try { @@ -86,9 +87,9 @@ public boolean update(AbstractJob job) { params[8] = job.getApplication(); params[9] = job.getApplicationId(); params[10] = job.getSubmittedOn(); - params[11] = job.getFinishedOn(); - params[12] = job.getSetupStartTime(); - params[13] = job.getSetupEndTime(); + params[11] = job.getEndTime(); + params[12] = -1; + params[13] = -1; params[14] = job.getId(); update(sql.toString(), params); @@ -105,9 +106,8 @@ public boolean update(AbstractJob job) { public boolean updateUser(User oldUser, User newUser) { StringBuilder sql = new StringBuilder(); - sql.append("update job "); - sql.append("set user_id = ?, name = ? "); - sql.append("where user_id = ?"); + sql.append("update job set user_id = ?, name = ? "); + sql.append("where user_id = ?"); try { Object[] params = new Object[3]; @@ -132,7 +132,7 @@ public boolean updateUser(User oldUser, User newUser) { public boolean delete(AbstractJob job) { StringBuilder sql = new StringBuilder(); sql.append("delete from job "); - sql.append("where id = ? "); + sql.append("where id = ?"); try { Object[] params = new Object[1]; @@ -277,7 +277,7 @@ public List findAllNotRetiredJobs() { params[1] = AbstractJob.STATE_RUNNING; params[2] = AbstractJob.STATE_EXPORTING; params[3] = AbstractJob.STATE_RETIRED; - params[4] = AbstractJob.STATE_DELETED; + params[4] = AbstractJob.STATE_DELETED; params[5] = AbstractJob.STATE_RETIRED; params[6] = AbstractJob.STATE_DELETED; @@ -485,9 +485,6 @@ public AbstractJob mapRow(ResultSet rs, int row) throws SQLException { job.setApplication(rs.getString("job.application")); job.setApplicationId(rs.getString("job.application_id")); job.setSubmittedOn(rs.getLong("job.submitted_on")); - job.setFinishedOn(rs.getLong("job.finished_on")); - job.setSetupStartTime(rs.getLong("job.setup_start_time")); - job.setSetupEndTime(rs.getLong("job.setup_end_time")); job.setUserAgent(rs.getString("job.user_agent")); return job; diff --git a/src/main/java/cloudgene/mapred/database/ParameterDao.java b/src/main/java/cloudgene/mapred/database/ParameterDao.java index 8422d40f..b33d7718 100644 --- a/src/main/java/cloudgene/mapred/database/ParameterDao.java +++ b/src/main/java/cloudgene/mapred/database/ParameterDao.java @@ -138,9 +138,6 @@ public List findAllOutputByJob(AbstractJob job) { DownloadDao downloadDao = new DownloadDao(database); for (CloudgeneParameterOutput parameter : result) { List downloads = downloadDao.findAllByParameter(parameter); - for (Download download : downloads) { - download.setUsername(job.getUser().getUsername()); - } parameter.setFiles(downloads); } diff --git a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java index 6fbcff2c..c1e93dff 100644 --- a/src/main/java/cloudgene/mapred/jobs/AbstractJob.java +++ b/src/main/java/cloudgene/mapred/jobs/AbstractJob.java @@ -1,6 +1,7 @@ package cloudgene.mapred.jobs; import java.io.BufferedOutputStream; +import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -11,19 +12,29 @@ import java.text.SimpleDateFormat; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.Vector; +import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import cloudgene.mapred.apps.Application; +import cloudgene.mapred.apps.ApplicationRepository; import cloudgene.mapred.core.User; import cloudgene.mapred.jobs.queue.PriorityRunnable; -import cloudgene.mapred.steps.ErrorStep; +import cloudgene.mapred.jobs.workspace.IWorkspace; +import cloudgene.mapred.util.HashUtil; import cloudgene.mapred.util.Settings; +import cloudgene.mapred.wdl.WdlParameterInputType; import genepi.io.FileUtil; abstract public class AbstractJob extends PriorityRunnable { + public static final String JOB_LOG = "job.txt"; + + public static final String JOB_OUT = "std.out"; + private static final Logger log = LoggerFactory.getLogger(AbstractJob.class); private DateFormat formatter = new SimpleDateFormat("yy/MM/dd HH:mm:ss"); @@ -62,14 +73,8 @@ abstract public class AbstractJob extends PriorityRunnable { private long endTime = 0; - private long setupStartTime = 0; - - private long setupEndTime = 0; - private long submittedOn = 0; - private long finishedOn = 0; - private String name; private User user; @@ -84,20 +89,14 @@ abstract public class AbstractJob extends PriorityRunnable { private String error = ""; - private int progress = -1; - - private boolean setupComplete = false; - - private boolean setupRunning = false; - - private boolean complete = true; - private int positionInQueue = -1; protected List inputParams = new Vector(); protected List outputParams = new Vector(); + protected CloudgeneParameterOutput logOutput = null; + protected List steps = new Vector(); protected BufferedOutputStream stdOutStream; @@ -108,26 +107,23 @@ abstract public class AbstractJob extends PriorityRunnable { private Settings settings; - private String logs; - - private boolean removeHdfsWorkspace; - private String localWorkspace; - private String hdfsWorkspace; - private boolean canceld = false; - private boolean forceInstallation = false; + protected IWorkspace workspace; private String workspaceSize = null; + private String publicJobId; + public String getId() { return id; } public void setId(String id) { this.id = id; + this.publicJobId = HashUtil.getSha256(id + RandomStringUtils.random(500)); } public int getState() { @@ -154,22 +150,6 @@ public void setEndTime(long endTime) { this.endTime = endTime; } - public void setSetupStartTime(long setupStartTime) { - this.setupStartTime = setupStartTime; - } - - public long getSetupStartTime() { - return setupStartTime; - } - - public void setSetupEndTime(long setupEndTime) { - this.setupEndTime = setupEndTime; - } - - public long getSetupEndTime() { - return setupEndTime; - } - public void setSubmittedOn(long submitedOn) { this.submittedOn = submitedOn; } @@ -178,14 +158,6 @@ public long getSubmittedOn() { return submittedOn; } - public void setFinishedOn(long finishedOn) { - this.finishedOn = finishedOn; - } - - public long getFinishedOn() { - return finishedOn; - } - public String getName() { return name; } @@ -210,14 +182,6 @@ public void setError(String error) { this.error = error; } - public void setProgress(int progress) { - this.progress = progress; - } - - public int getProgress() { - return progress; - } - public void setDeletedOn(long deletedOn) { this.deletedOn = deletedOn; } @@ -229,7 +193,7 @@ public long getDeletedOn() { public void setUserAgent(String userAgent) { this.userAgent = userAgent; } - + public String getUserAgent() { return userAgent; } @@ -250,6 +214,10 @@ public void setOutputParams(List outputParams) { this.outputParams = outputParams; } + public CloudgeneParameterOutput getLogOutput() { + return logOutput; + } + public void setPositionInQueue(int positionInQueue) { this.positionInQueue = positionInQueue; } @@ -277,164 +245,83 @@ public boolean afterSubmission() { } catch (Exception e1) { - // setEndTime(System.currentTimeMillis()); - - setState(AbstractJob.STATE_FAILED); log.error("Job " + getId() + ": initialization failed.", e1); writeLog("Initialization failed: " + e1.getLocalizedMessage()); - setSetupComplete(false); - state = AbstractJob.STATE_FAILED; + setState(STATE_FAILED); return false; } } - public void runSetupSteps() { - - log.info("Job " + getId() + ": executing installation..."); - writeLog("Executing Job installation...."); - - // execute installation - - boolean result = executeInstallation(forceInstallation); - - if (result == false || state == AbstractJob.STATE_CANCELED || state == AbstractJob.STATE_FAILED) { - ErrorStep errorStep = new ErrorStep(getError() != null ? getError() : "Error"); - errorStep.setJob(this); - errorStep.setName("Job Setup failed: " + getError()); - getSteps().add(errorStep); - setState(AbstractJob.STATE_FAILED); - onFailure(); - setSetupComplete(false); - return; - } - - // execute setup steps - - try { - - log.info("Job " + getId() + ": executing setups..."); - writeLog("Executing Job setups...."); - - boolean succesfull = executeSetupSteps(); - - if (succesfull && hasSteps()) { - // all okey - - log.info("Job " + getId() + ": executed successful. job has steps."); - setSetupComplete(true); - return; - - } else if (succesfull && !hasSteps()) { - // all okey and no more steps - - log.info("Job " + getId() + ": executed successful. job has no more steps."); - - writeLog("Job execution successful."); - writeLog("Exporting Data..."); - - setState(AbstractJob.STATE_EXPORTING); - - try { - - boolean successfulAfter = after(); - - if (successfulAfter) { - - setState(AbstractJob.STATE_SUCCESS); - setSetupComplete(true); - log.info("Job " + getId() + ": data export successful."); - writeLog("Data export successful."); - - } else { - - setSetupComplete(false); - setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": data export failed."); - writeLog("Data export failed."); - - } - - } catch (Error | Exception e) { - - Writer writer = new StringWriter(); - PrintWriter printWriter = new PrintWriter(writer); - e.printStackTrace(printWriter); - String s = writer.toString(); - - setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": data export failed.", e); - writeLog("Data export failed: " + e.getLocalizedMessage() + "\n" + s); - - setSetupComplete(false); - - } - - writeLog("Cleaning up..."); - cleanUp(); - log.info("Job " + getId() + ": cleanup successful."); - writeLog("Cleanup successful."); - - closeStdOutFiles(); + public boolean runInstallationAndResolveAppLinks() { - } else if (!succesfull) { + Settings settings = getSettings(); + ApplicationRepository repository = settings.getApplicationRepository(); - setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": execution failed. " + getError()); - writeLog("Job execution failed: " + getError()); - writeLog("Cleaning up..."); - onFailure(); - log.info("Job " + getId() + ": cleanup successful."); - writeLog("Cleanup successful."); + // resolve application links + for (CloudgeneParameterInput input : getInputParams()) { + if (input.getType() != WdlParameterInputType.APP_LIST) { + continue; + } + String value = input.getValue(); + String linkedAppId = value; + if (value.startsWith("apps@")) { + linkedAppId = value.replaceAll("apps@", ""); + } - if (canceld) { - setState(AbstractJob.STATE_CANCELED); + if (value.isEmpty()) { + continue; + } + Application linkedApp = repository.getByIdAndUser(linkedAppId, getUser()); + if (linkedApp == null) { + String error = "Application " + linkedAppId + " is not installed or wrong permissions."; + log.info(error); + writeOutput(error); + setError(error); + return false; + } + // update environment variables + Environment environment = settings.buildEnvironment().addApplication(linkedApp.getWdlApp()) + .addContext(context); + Map properties = linkedApp.getWdlApp().getProperties(); + for (String property : properties.keySet()) { + Object propertyValue = properties.get(property); + if (propertyValue instanceof String) { + propertyValue = environment.resolve(propertyValue.toString()); } - setSetupComplete(false); - - closeStdOutFiles(); - + properties.put(property, propertyValue); } - } catch (Exception | Error e) { + getContext().setData(input.getName(), properties); - setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": initialization failed.", e); - - Writer writer = new StringWriter(); - PrintWriter printWriter = new PrintWriter(writer); - e.printStackTrace(printWriter); - String s = writer.toString(); - - writeLog("Initialization failed: " + e.getLocalizedMessage() + "\n" + s); - - writeLog("Cleaning up..."); - onFailure(); - log.info("Job " + getId() + ": cleanup successful."); - writeLog("Cleanup successful."); - setSetupComplete(false); - - closeStdOutFiles(); + } - setSetupRunning(false); + return true; - } } @Override public void run() { - if (state == AbstractJob.STATE_CANCELED || state == AbstractJob.STATE_FAILED) { - onFailure(); - setError("Job Execution failed."); + if (canceld) { + return; + } + + log.info("[Job {}] Setup job...", getId()); + setState(AbstractJob.STATE_RUNNING); + setStartTime(System.currentTimeMillis()); + + if (!runInstallationAndResolveAppLinks()) { + log.info("[Job {}] Setup failed.", getId()); + setEndTime(System.currentTimeMillis()); + setState(AbstractJob.STATE_FAILED); return; } - log.info("Job " + getId() + ": running."); + log.info("[Job {}] Running job...", getId()); setStartTime(System.currentTimeMillis()); try { - setState(AbstractJob.STATE_RUNNING); writeLog("Details:"); writeLog(" Name: " + getName()); writeLog(" Job-Id: " + getId()); @@ -453,98 +340,75 @@ public void run() { writeLog(" " + parameter.getDescription() + ": " + context.get(parameter.getName())); } - writeLog("Preparing Job...."); - boolean successfulBefore = before(); - - if (!successfulBefore) { + writeLog("Executing Job...."); - setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": job preparation failed."); - writeLog("Job Preparation failed."); - - } else { + boolean succesfull = execute(); - log.info("Job " + getId() + ": executing."); - writeLog("Executing Job...."); + if (succesfull) { - boolean succesfull = execute(); + log.info("[Job {}] Execution successful.", getId()); - if (succesfull) { - - log.info("Job " + getId() + ": executed successful."); - - writeLog("Job Execution successful."); - writeLog("Exporting Data..."); - - setState(AbstractJob.STATE_EXPORTING); - - try { - - boolean successfulAfter = after(); - - if (successfulAfter) { + writeLog("Job Execution successful."); + writeLog("Exporting Data..."); - setState(AbstractJob.STATE_SUCCESS); - log.info("Job " + getId() + ": data export successful."); - writeLog("Data Export successful."); + setState(AbstractJob.STATE_EXPORTING); - } else { + try { - setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": data export failed."); - writeLog("Data Export failed."); + boolean successfulAfter = after(); - } + if (successfulAfter) { - } catch (Error | Exception e) { + setState(AbstractJob.STATE_SUCCESS); + log.info("[Job {}] data export successful.", getId()); + writeLog("Data Export successful."); - Writer writer = new StringWriter(); - PrintWriter printWriter = new PrintWriter(writer); - e.printStackTrace(printWriter); - String s = writer.toString(); + } else { setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": data export failed.", e); - writeLog("Data Export failed: " + e.getLocalizedMessage() + "\n" + s); + log.error("[Job {}] data export failed.", getId()); + writeLog("Data Export failed."); } - } else { + } catch (Error | Exception e) { + + Writer writer = new StringWriter(); + PrintWriter printWriter = new PrintWriter(writer); + e.printStackTrace(printWriter); + String s = writer.toString(); setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": execution failed. " + getError()); - writeLog("Job Execution failed: " + getError()); + log.error("[Job {}] data export failed.", getId(), e); + writeLog("Data Export failed: " + e.getLocalizedMessage() + "\n" + s); } + + } else { + + setState(AbstractJob.STATE_FAILED); + log.error("[Job {}] Execution failed. {}", getId(), getError()); + writeLog("Job Execution failed: " + getError()); + } + writeLog("Cleaning up..."); if (getState() == AbstractJob.STATE_FAILED || getState() == AbstractJob.STATE_CANCELED) { - - writeLog("Cleaning up..."); onFailure(); - log.info("Job " + getId() + ": cleanup successful."); - writeLog("Cleanup successful."); - } else { - writeLog("Cleaning up..."); cleanUp(); - log.info("Job " + getId() + ": cleanup successful."); - writeLog("Cleanup successful."); - } + log.info("[Job {}]cleanup successful.", getId()); + writeLog("Cleanup successful."); if (canceld) { setState(AbstractJob.STATE_CANCELED); } - closeStdOutFiles(); - - setEndTime(System.currentTimeMillis()); - } catch (Exception | Error e) { setState(AbstractJob.STATE_FAILED); - log.error("Job " + getId() + ": initialization failed.", e); + log.error("[Job {}]: initialization failed.", getId(), e); Writer writer = new StringWriter(); PrintWriter printWriter = new PrintWriter(writer); @@ -555,39 +419,29 @@ public void run() { writeLog("Cleaning up..."); onFailure(); - log.info("Job " + getId() + ": cleanup successful."); + log.info("[Job {}]: cleanup successful.", getId()); writeLog("Cleanup successful."); - closeStdOutFiles(); - - setEndTime(System.currentTimeMillis()); - } + + closeStdOutFiles(); + setEndTime(System.currentTimeMillis()); } public void cancel() { writeLog("Canceled by user."); - log.info("Job " + getId() + ": canceld by user."); + log.info("[Job {}]: canceld by user.", getId()); - /* - * if (state == STATE_RUNNING) { closeStdOutFiles(); } - */ canceld = true; + setEndTime(System.currentTimeMillis()); setState(AbstractJob.STATE_CANCELED); } private void initStdOutFiles() throws FileNotFoundException { - - // if (stdOutStream == null) { - stdOutStream = new BufferedOutputStream(new FileOutputStream(FileUtil.path(localWorkspace, "std.out"))); - - // } - // if (logStream == null) { - logStream = new BufferedOutputStream(new FileOutputStream(FileUtil.path(localWorkspace, "job.txt"))); - // } - + stdOutStream = new BufferedOutputStream(new FileOutputStream(FileUtil.path(localWorkspace, JOB_OUT))); + logStream = new BufferedOutputStream(new FileOutputStream(FileUtil.path(localWorkspace, JOB_LOG))); } private void closeStdOutFiles() { @@ -597,8 +451,15 @@ private void closeStdOutFiles() { stdOutStream.close(); logStream.close(); - } catch (IOException e) { + // stage files to workspace + workspace.uploadLog(new File(FileUtil.path(localWorkspace, JOB_OUT))); + workspace.uploadLog(new File(FileUtil.path(localWorkspace, JOB_LOG))); + FileUtil.deleteFile(FileUtil.path(localWorkspace, JOB_OUT)); + FileUtil.deleteFile(FileUtil.path(localWorkspace, JOB_LOG)); + + } catch (IOException e) { + log.error("[Job {}]: Staging log files failed.", getId(), e); } } @@ -612,25 +473,13 @@ public void writeOutput(String line) { } } catch (IOException e) { - + log.error("[Job {}]: Write output failed.", getId(), e); } } public void writeOutputln(String line) { - - try { - if (stdOutStream == null) { - initStdOutFiles(); - } - - stdOutStream.write(line.getBytes("UTF-8")); - stdOutStream.write("\n".getBytes("UTF-8")); - stdOutStream.flush(); - - } catch (IOException e) { - } - + writeOutput(line + "\n"); } public void writeLog(String line) { @@ -646,6 +495,7 @@ public void writeLog(String line) { logStream.flush(); } catch (IOException e) { + log.error("[Job {}]: Write output failed.", getId(), e); } } @@ -658,38 +508,10 @@ public void setSteps(List steps) { this.steps = steps; } - public void setSetupComplete(boolean setupComplete) { - this.setupComplete = setupComplete; - } - - public boolean isSetupComplete() { - return setupComplete; - } - - public void setSetupRunning(boolean setupRunning) { - this.setupRunning = setupRunning; - } - - public boolean isSetupRunning() { - return setupRunning; - } - - public boolean hasSteps() { - return true; - } - public CloudgeneContext getContext() { return context; } - public void setLogs(String logs) { - this.logs = logs; - } - - public String getLogs() { - return logs; - } - @Override public boolean equals(Object obj) { @@ -709,14 +531,6 @@ public void setSettings(Settings settings) { this.settings = settings; } - public boolean isRemoveHdfsWorkspace() { - return removeHdfsWorkspace; - } - - public void setRemoveHdfsWorkspace(boolean removeHdfsWorkspace) { - this.removeHdfsWorkspace = removeHdfsWorkspace; - } - public String getLocalWorkspace() { return localWorkspace; } @@ -725,12 +539,12 @@ public void setLocalWorkspace(String localWorkspace) { this.localWorkspace = localWorkspace; } - public String getHdfsWorkspace() { - return hdfsWorkspace; + public void setWorkspace(IWorkspace workspace) { + this.workspace = workspace; } - public void setHdfsWorkspace(String hdfsWorkspace) { - this.hdfsWorkspace = hdfsWorkspace; + public IWorkspace getWorkspace() { + return workspace; } public void setApplication(String application) { @@ -749,60 +563,45 @@ public String getApplicationId() { return applicationId; } - public void setComplete(boolean complete) { - this.complete = complete; - } - - public boolean isComplete() { - return this.complete; - } - - public boolean isCanceld() { - return canceld; + public boolean isRunning() { + return state == STATE_EXPORTING || state == STATE_RUNNING || state == STATE_WAITING; } - public boolean isRunning() { - return !complete; + public Download findDownloadByHash(String hash) { + for (CloudgeneParameterOutput param : getOutputParams()) { + if (param.getFiles() == null) { + continue; + } + for (Download download : param.getFiles()) { + if (download.getHash().equals(hash)) { + return download; + } + } + } + return null; } abstract public boolean execute(); - abstract public boolean executeSetupSteps(); - - abstract public boolean executeInstallation(boolean forceInstallation); - abstract public boolean setup(); - abstract public boolean before(); - abstract public boolean after(); abstract public boolean onFailure(); abstract public boolean cleanUp(); - public void onStepFinished(CloudgeneStep step) { - - } - - public void onStepStarted(CloudgeneStep step) { - - } - public void kill() { } - public void forceInstallation(boolean forceInstallation) { - this.forceInstallation = forceInstallation; + public String getPublicJobId() { + return publicJobId; } - public long getCurrentTime() { - return System.currentTimeMillis(); - } - - public void setCurrentTime(long time) { - + public String getLog(String name) { + String logFilename = FileUtil.path(settings.getLocalWorkspace(), getId(), name); + return FileUtil.readFileAsString(logFilename); } } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java index fb6163fd..49c6127a 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneContext.java @@ -10,31 +10,19 @@ import java.util.Vector; import cloudgene.mapred.core.User; -import cloudgene.mapred.util.HashUtil; +import cloudgene.mapred.jobs.sdk.WorkflowContext; import cloudgene.mapred.util.MailUtil; import cloudgene.mapred.util.Settings; -import cloudgene.sdk.internal.WorkflowContext; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; public class CloudgeneContext extends WorkflowContext { - private String hdfsTemp; - private String localTemp; - private String localInput; - - private String hdfsOutput; - - private String hdfsInput; - private String localOutput; private String workingDirectory; - private String workspace; - private Settings settings; private CloudgeneStep step; @@ -49,16 +37,14 @@ public class CloudgeneContext extends WorkflowContext { private Map submitCounters = new HashMap(); - private AbstractJob job; + private CloudgeneJob job; private Map data = new HashMap(); - private Map config; + private Map config; private int chunks = 0; - private Map> customDownloads = new HashMap>(); - public CloudgeneContext(CloudgeneJob job) { this.workingDirectory = job.getWorkingDirectory(); @@ -69,32 +55,17 @@ public CloudgeneContext(CloudgeneJob job) { setData("cloudgene.user.mail", user.getMail()); setData("cloudgene.user.name", user.getFullName()); - workspace = job.getHdfsWorkspace(); - - try { - hdfsTemp = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace, "temp")); - hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace)); - hdfsInput = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace)); - } catch (Error e) { - log("No hdfs folders created."); - } - localOutput = new File(job.getLocalWorkspace()).getAbsolutePath(); - localTemp = new File(FileUtil.path(job.getLocalWorkspace(), "temp")).getAbsolutePath(); - localInput = new File(FileUtil.path(job.getLocalWorkspace(), "input")).getAbsolutePath(); - inputParameters = new HashMap(); for (CloudgeneParameterInput param : job.getInputParams()) { inputParameters.put(param.getName(), param); } outputParameters = new HashMap(); - customDownloads = new HashMap>(); for (CloudgeneParameterOutput param : job.getOutputParams()) { outputParameters.put(param.getName(), param); - customDownloads.put(param.getName(), new Vector()); } settings = job.getSettings(); @@ -109,83 +80,6 @@ public CloudgeneStep getCurrentStep() { return step; } - public void setupOutputParameters(boolean hasHdfsOutputs) { - - // cleanup temp directories - if (hasHdfsOutputs) { - try { - HdfsUtil.delete(getHdfsTemp()); - } catch (Exception e) { - System.out.println("Warning: problems during hdfs init."); - } catch (Error e) { - System.out.println("Warning: problems during hdfs init."); - } - } - - FileUtil.deleteDirectory(getLocalTemp()); - - // create output directories - FileUtil.createDirectory(getLocalOutput()); - FileUtil.createDirectory(getLocalTemp()); - - // create output directories - for (CloudgeneParameterOutput param : outputParameters.values()) { - - switch (param.getType()) { - case HDFS_FILE: - case HDFS_FOLDER: - - String value = ""; - - if (param.isDownload()) { - value = HdfsUtil.path(getHdfsOutput(), param.getName()); - } else { - value = HdfsUtil.path(getHdfsTemp(), param.getName()); - } - - if (!HdfsUtil.isAbsolute(value)) { - value = HdfsUtil.makeAbsolute(value); - } - // delete (needed for restart) - try { - HdfsUtil.delete(value); - } catch (Exception e) { - System.out.println("Warning: problems during hdfs init."); - } - param.setValue(value); - break; - - case LOCAL_FILE: - String parent = getLocalOutput(); - if (!param.isDownload()) { - parent = getLocalTemp(); - } - String folder = FileUtil.path(parent, param.getName()); - String filename = FileUtil.path(folder, param.getName()); - // delete and create (needed for restart) - FileUtil.deleteDirectory(folder); - FileUtil.createDirectory(folder); - param.setValue(filename); - break; - - case LOCAL_FOLDER: - String parent2 = getLocalOutput(); - if (!param.isDownload()) { - parent2 = getLocalTemp(); - } - - String folder2 = FileUtil.path(parent2, param.getName()); - // delete and create (needed for restart) - FileUtil.deleteDirectory(folder2); - FileUtil.createDirectory(folder2); - param.setValue(folder2); - break; - } - - } - - } - public String getInput(String param) { if (inputParameters.get(param) != null) { @@ -201,6 +95,10 @@ public String getJobId() { return job.getId(); } + public String getPublicJobId() { + return job.getPublicJobId(); + } + public String getOutput(String param) { if (outputParameters.get(param) != null) { @@ -223,53 +121,15 @@ public String get(String param) { return result; } } - - @Override - public void addDownload(String param, String name, String size, String path) { - List downloads = customDownloads.get(param); - if (downloads == null) { - new RuntimeException("Parameter " + param + " is unknown."); - } - - String hash = HashUtil.getSha256(name + size + path + (Math.random() * 100000)); - Download download = new Download(); - download.setName(name); - download.setSize(size); - download.setPath(path); - download.setHash(hash); - download.setCount(CloudgeneJob.MAX_DOWNLOAD); - - downloads.add(download); - } - - public List getDownloads(String param){ - return customDownloads.get(param); - } public Settings getSettings() { return settings; } - public String getHdfsTemp() { - return hdfsTemp; - } - public String getLocalTemp() { return localTemp; } - public String getLocalInput() { - return localInput; - } - - public String getHdfsOutput() { - return hdfsOutput; - } - - public String getHdfsInput() { - return hdfsInput; - } - public String getLocalOutput() { return localOutput; } @@ -290,7 +150,7 @@ public void log(String line) { job.writeLog(line); } - public AbstractJob getJob() { + public CloudgeneJob getJob() { return job; } @@ -326,10 +186,10 @@ public boolean sendMail(String to, String subject, String body) throws Exception Settings settings = getSettings(); if (settings.getMail() != null) { - - MailUtil.send(settings.getMail().get("smtp"), settings.getMail().get("port"), settings.getMail().get("user"), - settings.getMail().get("password"), settings.getMail().get("name"), to, - "[" + settings.getName() + "] " + subject, body); + + MailUtil.send(settings.getMail().get("smtp"), settings.getMail().get("port"), + settings.getMail().get("user"), settings.getMail().get("password"), settings.getMail().get("name"), + to, "[" + settings.getName() + "] " + subject, body); } return true; @@ -355,7 +215,7 @@ public void setInput(String input, String value) { parameter.setValue(value); } - public void setOutput(String input, String value) { + public void setOutput2(String input, String value) { CloudgeneParameterOutput parameter = outputParameters.get(input); parameter.setValue(value); @@ -443,7 +303,6 @@ public void beginTask(String name) { logs.add(status); } - public Message createTask(String name) { Message status = new Message(step, Message.RUNNING, name); @@ -456,8 +315,6 @@ public Message createTask(String name) { return status; } - - public void beginTask(String name, int totalWork) { beginTask(name); } @@ -493,14 +350,15 @@ public void setData(String key, Object object) { } @Override - public void setConfig(Map config) { + public void setConfig(Map config) { this.config = config; } @Override public String getConfig(String param) { if (config != null) { - return config.get(param); + Object value = config.get(param); + return value != null ? value.toString() : null; } else { return null; } @@ -516,4 +374,9 @@ public void addFile(String filename) { message(chunkFilename, 27); } + @Override + public String getHdfsTemp() { + throw new RuntimeException("Not support in cg3"); + } + } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java index 6d688fd4..96f14f4d 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneJob.java @@ -1,39 +1,28 @@ package cloudgene.mapred.jobs; -import java.io.File; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Vector; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import cloudgene.mapred.apps.Application; -import cloudgene.mapred.apps.ApplicationInstaller; -import cloudgene.mapred.apps.ApplicationRepository; import cloudgene.mapred.core.User; +import cloudgene.mapred.jobs.engine.ExecutableStep; import cloudgene.mapred.jobs.engine.Executor; import cloudgene.mapred.jobs.engine.Planner; -import cloudgene.mapred.jobs.engine.graph.Graph; -import cloudgene.mapred.jobs.engine.graph.GraphNode; -import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; -import cloudgene.mapred.util.HashUtil; -import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlParameterInput; -import cloudgene.mapred.wdl.WdlParameterInputType; import cloudgene.mapred.wdl.WdlParameterOutput; import cloudgene.mapred.wdl.WdlParameterOutputType; import cloudgene.mapred.wdl.WdlStep; -import cloudgene.sdk.internal.IExternalWorkspace; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; public class CloudgeneJob extends AbstractJob { + private static final String CLOUDGENE_LOGS_PARAM = "cloudgene_logs"; + private WdlApp app; private String workingDirectory; @@ -44,48 +33,16 @@ public class CloudgeneJob extends AbstractJob { private static Logger log = LoggerFactory.getLogger(CloudgeneJob.class); - private String externalOutput = null; - - private IExternalWorkspace externalWorkspace; - public CloudgeneJob() { super(); } - public void loadConfig(WdlApp app) { - - this.app = app; - workingDirectory = app.getPath(); - - // set parameter properties that are not stored in database. - // needed for restart - for (CloudgeneParameterOutput outputParam : outputParams) { - - for (WdlParameterOutput output : app.getWorkflow().getOutputs()) { - - if (outputParam.getName().equals(output.getId())) { - outputParam.setMakeAbsolute(output.isMakeAbsolute()); - outputParam.setAutoExport(output.isAutoExport()); - outputParam.setZip(output.isZip()); - outputParam.setMergeOutput(output.isMergeOutput()); - outputParam.setRemoveHeader(output.isRemoveHeader()); - } - - } - - } - - } - public CloudgeneJob(User user, String id, WdlApp app, Map params) { - setComplete(false); this.app = app; setId(id); setUser(user); workingDirectory = app.getPath(); - externalOutput = params.get("external-output"); - // init parameters inputParams = new Vector(); for (WdlParameterInput input : app.getWorkflow().getInputs()) { @@ -106,232 +63,126 @@ public CloudgeneJob(User user, String id, WdlApp app, Map params outputParams.add(newOutput); } + initLogOutput(); + } - @Override - public boolean setup() { + public void loadApp(WdlApp app) { - context = new CloudgeneContext(this); - // context.updateInputParameters(); - context.setupOutputParameters(app.getWorkflow().hasHdfsOutputs()); + this.app = app; + workingDirectory = app.getPath(); - return true; + // find logOutput and remove from outputs + for (CloudgeneParameterOutput param : getOutputParams()) { + if (!param.getName().equals(CLOUDGENE_LOGS_PARAM)) { + continue; + } + logOutput = param; + } + if (logOutput != null) { + getOutputParams().remove(logOutput); + } } - @Override - public boolean hasSteps() { - try { - // evaluate WDL derictives - Planner planner = new Planner(); - WdlApp app = planner.evaluateWDL(this.app, context, getSettings()); - return app.getWorkflow().getSteps().size() > 0; - } catch (Exception e) { - // e.printStackTrace(); - writeOutput(e.getMessage()); - setError(e.getMessage()); - return false; - } + protected void initLogOutput() { + logOutput = new CloudgeneParameterOutput(); + logOutput.setAdminOnly(true); + logOutput.setDownload(true); + logOutput.setDescription("Logs"); + logOutput.setName(CLOUDGENE_LOGS_PARAM); + logOutput.setType(WdlParameterOutputType.LOCAL_FOLDER); + logOutput.setJob(this); } @Override - public boolean execute() { - - try { - - log.info("Job " + getId() + " submitted..."); - - // evaluate WDL derictives - Planner planner = new Planner(); - WdlApp app = planner.evaluateWDL(this.app, context, getSettings()); + public boolean setup() { - // create dag from wdl document - Graph graph = planner.buildDAG(app.getWorkflow().getSteps(), app.getWorkflow(), context); + context = new CloudgeneContext(this); - // execute optimzed dag - executor = new Executor(); - boolean successful = executor.execute(graph); + try { + log.info("[Job {}] Setup workspace {}'", getId(), workspace.getName()); + context.log("Setup External Workspace on " + workspace.getName()); + workspace.setup(); + context.setWorkspace(workspace); + } catch (Exception e) { + writeLog(e.toString()); + log.error("[Job {}] Error setup external workspace failed.", getId(), e); + setError(e.toString()); + return false; + } - if (!successful) { - setError("Job Execution failed."); - GraphNode failedNode = executor.getCurrentNode(); - executeFailureStep(failedNode.getStep()); - return false; + // create output directories + for (CloudgeneParameterOutput param : outputParams) { + + switch (param.getType()) { + case HDFS_FILE: + case HDFS_FOLDER: + log.info("[Job {}] Setting output param '{}' failed. HDFS support was removed in Cloudgene 3'", getId(), + param.getName()); + throw new RuntimeException("HDFS support was removed in Cloudgene 3"); + + case LOCAL_FILE: + String filename = workspace.createFile(param.getName(), param.getName()); + param.setValue(filename); + log.info("[Job {}] Set output file '{}' to '{}'", getId(), param.getName(), param.getValue()); + break; + + case LOCAL_FOLDER: + String folder = workspace.createFolder(param.getName()); + param.setValue(folder); + log.info("[Job {}] Set output folder '{}' to '{}'", getId(), param.getName(), param.getValue()); + break; } - setError(null); - return true; - - } catch (Exception e) { - e.printStackTrace(); - writeOutput(e.getMessage()); - setError(e.getMessage()); - return false; } + return true; + } @Override - public boolean executeSetupSteps() { + public boolean execute() { try { - // evaluate WDL + + // evaluate WDL and replace all variables (e.g. ${job_id}) Planner planner = new Planner(); WdlApp app = planner.evaluateWDL(this.app, context, getSettings()); - // if a single setup step is set, add it to setups - WdlStep setup = app.getWorkflow().getSetup(); - if (setup != null) { - app.getWorkflow().getSetups().add(0, setup); - } - - Graph graph = planner.buildDAG(app.getWorkflow().getSetups(), app.getWorkflow(), context); + // merge setup steps and normal steps + List steps = new Vector(app.getWorkflow().getSetups()); + steps.addAll(app.getWorkflow().getSteps()); + log.info("[Job {}] execute {} steps", getId(), steps.size()); - // execute optimized DAG + // execute steps executor = new Executor(); - boolean successful = executor.execute(graph); + boolean successful = executor.execute(steps, context); if (!successful) { setError("Job Execution failed."); - GraphNode failedNode = executor.getCurrentNode(); + ExecutableStep failedNode = executor.getCurrentNode(); executeFailureStep(failedNode.getStep()); return false; } setError(null); return true; - } catch (Exception e) { - // e.printStackTrace(); - writeOutput(e.getMessage()); - setError(e.getMessage()); - return false; - } - - } - - @Override - public boolean executeInstallation(boolean forceInstallation) { - - try { - - Settings settings = getSettings(); - ApplicationRepository repository = settings.getApplicationRepository(); - - // find dependencies - List applications = new Vector<>(); - - // install application - applications.add(app); - - for (CloudgeneParameterInput input : getInputParams()) { - if (input.getType() == WdlParameterInputType.APP_LIST) { - String value = input.getValue(); - String linkedAppId = value; - if (value.startsWith("apps@")) { - linkedAppId = value.replaceAll("apps@", ""); - } - - if (!value.isEmpty()) { - Application linkedApp = repository.getByIdAndUser(linkedAppId, getUser()); - if (linkedApp != null) { - applications.add(linkedApp.getWdlApp()); - // update environment variables - Map envApp = Environment.getApplicationVariables(linkedApp.getWdlApp(), - settings); - Map envJob = Environment.getJobVariables(context); - Map properties = linkedApp.getWdlApp().getProperties(); - for (String property : properties.keySet()) { - Object propertyValue = properties.get(property); - if (propertyValue instanceof String) { - propertyValue = Environment.env(propertyValue.toString(), envApp); - propertyValue = Environment.env(propertyValue.toString(), envJob); - } - properties.put(property, propertyValue); - } - - getContext().setData(input.getName(), properties); - } else { - String error = "Application " + linkedAppId + " is not installed or wrong permissions."; - log.info(error); - writeOutput(error); - setError(error); - return false; - } - } - } - } - - for (WdlApp app : applications) { - - log.info("Job " + getId() + ": executing installation for " + app.getId() + "..."); - - if (app.needsInstallation()) { - - writeLog(" Preparing Application '" + app.getName() + "'..."); - boolean installed = ApplicationInstaller.isInstalled(app, settings); - if (!installed || forceInstallation) { - try { - writeLog(" Installing Application " + app.getId() + "..."); - ApplicationInstaller.install(app, settings); - log.info("Installation of application " + app.getId() + " finished."); - writeLog(" Installation finished."); - } catch (IOException e) { - log.info("Installation of application " + app.getId() + " failed.", e); - writeLog(" Installation failed."); - writeLog(e.getMessage()); - setError(e.getMessage()); - return false; - } - } else { - String info = "Application '" + app.getName() + "'is already installed."; - log.info(info); - writeLog(" " + info); - } - } - - } - - // if default location is set, override user defined - if (!settings.getExternalWorkspaceLocation().isEmpty()) { - externalOutput = settings.getExternalWorkspaceLocation(); - } - - externalWorkspace = ExternalWorkspaceFactory.get(settings.getExternalWorkspaceType(), externalOutput); - - if (externalWorkspace != null) { - - try { - context.log("Setup External Workspace on " + externalWorkspace.getName()); - externalWorkspace.setup(this.getId()); - context.setExternalWorkspace(externalWorkspace); - } catch (Exception e) { - writeLog(e.toString()); - log.info("Error setup external workspace", e); - setError(e.toString()); - return false; - } - } - - return true; } catch (Exception e) { - writeLog("Installation of Application " + getApplicationId() + " failed."); - log.info("Installation of application " + app.getId() + " failed.", e); + e.printStackTrace(); + writeOutput(e.getMessage()); setError(e.getMessage()); return false; } - } - - @Override - public boolean before() { - - return true; } @Override public boolean onFailure() { + after(); + cleanUp(); return true; @@ -340,78 +191,49 @@ public boolean onFailure() { public boolean executeFailureStep(WdlStep failedStep) { WdlStep step = app.getWorkflow().getOnFailure(); + cleanUp(); - if (step != null) { - try { - writeLog("Executing onFailure... "); - GraphNode node = new GraphNode(step, context); - context.setData("cloudgene.failedStep", failedStep); - context.setData("cloudgene.failedStep.classname", failedStep.getClassname()); - node.run(); - boolean result = node.isSuccessful(); - if (result) { - - // export parameters generated by onFailure step - for (CloudgeneParameterOutput out : getOutputParams()) { - if (out.isAutoExport() && step.getGenerates().contains(out.getName())) { - log.info("Export parameter '" + out.getName() + "'..."); - context.println("Export parameter '" + out.getName() + "'..."); - exportParameter(out); - } - } - - writeLog("onFailure execution successful."); - return true; - } else { - writeLog("onFailure execution failed."); - return false; - } - - } catch (Exception e) { + if (step == null) { + return true; + } + + try { + writeLog("Executing onFailure... "); + ExecutableStep node = new ExecutableStep(step, context); + context.setData("cloudgene.failedStep", failedStep); + context.setData("cloudgene.failedStep.classname", failedStep.getClassname()); + node.run(); + boolean result = node.isSuccessful(); + if (result) { + writeLog("onFailure execution successful."); + return true; + } else { writeLog("onFailure execution failed."); - writeLog(e.getMessage()); - setError(e.getMessage()); return false; } + } catch (Exception e) { + writeLog("onFailure execution failed."); + writeLog(e.getMessage()); + setError(e.getMessage()); + return false; } - cleanUp(); - - return true; } @Override public boolean cleanUp() { - // delete local temp folders - - writeLog("Cleaning up uploaded local files..."); - FileUtil.deleteDirectory(context.getLocalInput()); - - writeLog("Cleaning up temporary local files..."); - FileUtil.deleteDirectory(context.getLocalTemp()); - - if (app.getWorkflow().hasHdfsOutputs()) { - - try { - // delete hdfs temp folders - writeLog("Cleaning up temporary hdfs files..."); - HdfsUtil.delete(context.getHdfsTemp()); - - // delete hdfs workspace - if (isRemoveHdfsWorkspace()) { - if (context.getHdfsOutput() != null) { - writeLog("Cleaning up hdfs files..."); - HdfsUtil.delete(context.getHdfsOutput()); - HdfsUtil.delete(context.getHdfsInput()); - } - } - } catch (Exception e) { - log.warn("Warning: problems during hdfs cleanup."); - } catch (Error e) { - log.warn("Warning: problems during hdfs cleanup."); - } + log.info("[Job {}] Cleaning up...", getId()); + + try { + workspace.cleanup(getId()); + } catch (IOException e) { + writeLog("Cleanup failed."); + writeLog(e.getMessage()); + setError(e.getMessage()); + log.error("[Job {}] Clean up failed.", getId(), e); + return false; } return true; @@ -420,16 +242,20 @@ public boolean cleanUp() { @Override public boolean after() { - // create output zip file for hdfs folders - for (CloudgeneParameterOutput out : getOutputParams()) { + log.info("[Job {}] Export parameters...", getId()); - if (out.isDownload() && !out.isAutoExport()) { - // export to local folder for faster download + for (CloudgeneParameterOutput out : getOutputParams()) { + if (out.isDownload()) { exportParameter(out); - } + } + log.info("[Job {}] Export logs...", getId()); + List logs = workspace.getLogs(); + for (Download log : logs) { + log.setCount(-1); } + logOutput.setFiles(logs); return true; } @@ -438,173 +264,31 @@ public boolean exportParameter(CloudgeneParameterOutput out) { writeLog(" Exporting parameter " + out.getName() + "..."); - String localOutput = context.getLocalOutput(); - String workspace = getHdfsWorkspace(); - - if (out.getType() == WdlParameterOutputType.HDFS_FOLDER) { - - String localOutputDirectory = FileUtil.path(localOutput, out.getName()); - - FileUtil.createDirectory(localOutputDirectory); - - String filename = context.getOutput(out.getName()); - String hdfsPath = null; - if (filename.startsWith("hdfs://") || filename.startsWith("file:/")) { - hdfsPath = filename; - - } else { - - hdfsPath = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace, filename)); - } - - if (HdfsUtil.exists(hdfsPath)) { - - if (out.isZip()) { - - String zipName = FileUtil.path(localOutputDirectory, out.getName() + ".zip"); - - if (out.isMergeOutput()) { - - HdfsUtil.compressAndMerge(zipName, hdfsPath, out.isRemoveHeader()); + if (out.getType() == WdlParameterOutputType.HDFS_FOLDER || out.getType() == WdlParameterOutputType.HDFS_FILE) { - } else { - - HdfsUtil.compress(zipName, hdfsPath); - - } - - } else { - - if (out.isMergeOutput()) { - - HdfsUtil.exportDirectoryAndMerge(localOutputDirectory, out.getName(), hdfsPath, - out.isRemoveHeader()); - - } else { - - HdfsUtil.exportDirectory(localOutputDirectory, out.getName(), hdfsPath); - - } - - } - - } - - } - - if (out.getType() == WdlParameterOutputType.HDFS_FILE) { - - String localOutputDirectory = FileUtil.path(localOutput, out.getName()); - - FileUtil.createDirectory(localOutputDirectory); - - String filename = context.getOutput(out.getName()); - String hdfsPath = null; - if (filename.startsWith("hdfs://") || filename.startsWith("file:/")) { - - hdfsPath = filename; - - } else { - - hdfsPath = HdfsUtil.makeAbsolute(HdfsUtil.path(workspace, filename)); - } - - if (!out.isZip()) { - - if (HdfsUtil.exists(hdfsPath)) { - HdfsUtil.exportFile(localOutputDirectory, hdfsPath); - } - - } else { - if (HdfsUtil.exists(hdfsPath)) { - HdfsUtil.compressFile(localOutputDirectory, hdfsPath); - } - } + throw new RuntimeException("HDFS support was removed in Cloudgene 3"); } out.setJobId(getId()); - String name = out.getName(); - String n = FileUtil.path(localOutput, name); - - File f = new File(n); - - List downloads = new Vector(); - - if (f.exists() && f.isDirectory()) { - - try { - exportFolder(out, "", name, f, downloads); - } catch (Exception e) { - - writeLog("Export paramater '" + out.getName() + "' failed:" + e); - log.info("Export paramater '" + out.getName() + "' failed:" + e); - setError("Export paramater '" + out.getName() + "' failed:" + e); - return false; - } - writeLog(" Added " + downloads.size() + " downloads."); - } - - List customDownloads = context.getDownloads(out.getName()); - if (customDownloads != null) { - for (Download download : customDownloads) { - download.setParameter(out); + List downloads = workspace.getDownloads(out.getValue()); + for (Download download : downloads) { + download.setParameter(out); + download.setCount(MAX_DOWNLOAD); + if (!out.getFiles().contains(download)) { + out.getFiles().add(download); + writeLog(" Added new download " + download.getName() + "."); + } else { + writeLog(" Download " + download.getName() + " already added."); } - writeLog(" Added " + customDownloads.size() + " custom downloads."); - downloads.addAll(customDownloads); } - Collections.sort(downloads); out.setFiles(downloads); return true; } - private void exportFolder(CloudgeneParameterOutput out, String prefix, String name, File folder, - List downloads) throws Exception { - if (folder.exists() && folder.isDirectory()) { - - File[] files = folder.listFiles(); - - for (int i = 0; i < files.length; i++) { - if (!files[i].isDirectory()) { - - String filename = prefix.equals("") ? files[i].getName() : prefix + "/" + files[i].getName(); - String id = name + "/" + files[i].getName(); - String size = FileUtils.byteCountToDisplaySize(files[i].length()); - String hash = HashUtil.getSha256(filename + id + size + getId() + (Math.random() * 100000)); - Download download = new Download(); - download.setName(filename); - download.setPath(FileUtil.path(getId(), id)); - download.setSize(size); - download.setHash(hash); - download.setParameter(out); - download.setCount(MAX_DOWNLOAD); - downloads.add(download); - - if (externalWorkspace != null) { - // upload to s3 bucket, update path and delete local file - try { - context.log(" Uploading file " + files[i].getAbsolutePath() + " to external workspace"); - String url = externalWorkspace.upload(name, files[i]); - context.log(" Uploaded file to " + url + "."); - download.setPath(url); - files[i].delete(); - } catch (Exception e) { - throw new Exception("Error uploading output '" + files[i].getAbsolutePath() + "'. " + e); - } - - } - - } else { - exportFolder(out, prefix.equals("") ? files[i].getName() : prefix + "/" + files[i].getName(), - name + "/" + files[i].getName(), files[i], downloads); - } - } - } - } - public String getWorkingDirectory() { return workingDirectory; } @@ -623,12 +307,7 @@ public void kill() { public void updateProgress() { if (executor != null) { - executor.updateProgress(); - setProgress(executor.getProgress()); - - } else { - setProgress(-1); } } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneParameterOutput.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneParameterOutput.java index f9da18fd..6f150967 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneParameterOutput.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneParameterOutput.java @@ -29,16 +29,6 @@ public class CloudgeneParameterOutput { private String jobId; - private boolean autoExport = false; - - private boolean makeAbsolute = false; - - private boolean zip = false; - - private boolean mergeOutput = false; - - private boolean removeHeader = true; - private boolean adminOnly = false; private String hash = ""; @@ -52,11 +42,6 @@ public CloudgeneParameterOutput(WdlParameterOutput parameter) { setType(parameter.getTypeAsEnum()); setDownload(parameter.isDownload()); setDescription(parameter.getDescription()); - setMakeAbsolute(parameter.isMakeAbsolute()); - setAutoExport(parameter.isAutoExport()); - setZip(parameter.isZip()); - setMergeOutput(parameter.isMergeOutput()); - setRemoveHeader(parameter.isRemoveHeader()); setAdminOnly(parameter.isAdminOnly()); files = new Vector(); } @@ -133,46 +118,6 @@ public String getJobId() { return jobId; } - public boolean isMakeAbsolute() { - return makeAbsolute; - } - - public void setMakeAbsolute(boolean absolute) { - this.makeAbsolute = absolute; - } - - public void setAutoExport(boolean autoExport) { - this.autoExport = autoExport; - } - - public boolean isAutoExport() { - return autoExport; - } - - public void setZip(boolean zip) { - this.zip = zip; - } - - public void setMergeOutput(boolean mergeOutput) { - this.mergeOutput = mergeOutput; - } - - public void setRemoveHeader(boolean removeHeader) { - this.removeHeader = removeHeader; - } - - public boolean isMergeOutput() { - return mergeOutput; - } - - public boolean isRemoveHeader() { - return removeHeader; - } - - public boolean isZip() { - return zip; - } - public void setAdminOnly(boolean adminOnly) { this.adminOnly = adminOnly; } diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java index 0580bacf..a10c5431 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneStep.java @@ -8,7 +8,6 @@ import java.util.List; import cloudgene.mapred.wdl.WdlStep; -import cloudgene.sdk.internal.WorkflowContext; public abstract class CloudgeneStep { @@ -16,7 +15,7 @@ public abstract class CloudgeneStep { private String name; - private AbstractJob job; + private CloudgeneJob job; private List logMessages; @@ -50,11 +49,11 @@ public String getName() { return name; } - public AbstractJob getJob() { + public CloudgeneJob getJob() { return job; } - public void setJob(AbstractJob job) { + public void setJob(CloudgeneJob job) { this.job = job; } @@ -66,10 +65,6 @@ public boolean run(WdlStep step, CloudgeneContext context) { return true; } - public int getProgress() { - return -1; - } - public void updateProgress() { } @@ -91,25 +86,35 @@ public CloudgeneContext getup() { return null; } - protected boolean executeCommand(List command, WorkflowContext context) + protected boolean executeCommand(List command, CloudgeneContext context) throws IOException, InterruptedException { return executeCommand(command, context, null); } - protected boolean executeCommand(List command, WorkflowContext context, StringBuilder output) + protected boolean executeCommand(List command, CloudgeneContext context, StringBuilder output) + throws IOException, InterruptedException { + File workDir = new File(context.getWorkingDirectory()); + return executeCommand(command, context, output, workDir); + } + + protected boolean executeCommand(List command, CloudgeneContext context, StringBuilder output, File workDir) throws IOException, InterruptedException { + + Environment environment = context.getSettings().buildEnvironment().addContext(context) + .addApplication(job.getApp()); + // set global variables for (int j = 0; j < command.size(); j++) { - - String cmd = command.get(j).replaceAll("\\$job_id", context.getJobId()); + String cmd = environment.resolve(command.get(j)); command.set(j, cmd); } context.log("Command: " + command); - context.log("Working Directory: " + new File(context.getWorkingDirectory()).getAbsolutePath()); + context.log("Working Directory: " + workDir.getAbsolutePath()); ProcessBuilder builder = new ProcessBuilder(command); - builder.directory(new File(context.getWorkingDirectory())); + builder.environment().putAll(environment.toMap()); + builder.directory(workDir); builder.redirectErrorStream(true); builder.redirectOutput(); process = builder.start(); @@ -126,7 +131,7 @@ protected boolean executeCommand(List command, WorkflowContext context, } } } catch (Exception e) { - //e.printStackTrace(); + // e.printStackTrace(); } br.close(); diff --git a/src/main/java/cloudgene/mapred/jobs/CloudgeneStepFactory.java b/src/main/java/cloudgene/mapred/jobs/CloudgeneStepFactory.java index 017e391e..781cfc5f 100644 --- a/src/main/java/cloudgene/mapred/jobs/CloudgeneStepFactory.java +++ b/src/main/java/cloudgene/mapred/jobs/CloudgeneStepFactory.java @@ -5,13 +5,9 @@ import cloudgene.mapred.steps.BashCommandStep; import cloudgene.mapred.steps.GroovyStep; -import cloudgene.mapred.steps.HadoopMapReduceStep; -import cloudgene.mapred.steps.HadoopPigStep; -import cloudgene.mapred.steps.HadoopSparkStep; import cloudgene.mapred.steps.HtmlWidgetStep; import cloudgene.mapred.steps.JavaExternalStep; import cloudgene.mapred.steps.RMarkdownDockerStep; -import cloudgene.mapred.steps.RMarkdownLocalStep; import cloudgene.mapred.steps.RMarkdownStep; import cloudgene.mapred.wdl.WdlStep; @@ -38,7 +34,7 @@ public void register(String type, Class clazz) { public String getClassname(WdlStep step) { - String type = step.get("type"); + String type = step.getString("type"); if (type != null) { @@ -59,23 +55,17 @@ public String getClassname(WdlStep step) { } } - if (step.get("pig") != null) { - - // pig script - return HadoopPigStep.class.getName(); - + if (step.getString("pig") != null) { + throw new RuntimeException("Hadoop support was removed in Cloudgene 3"); } - if (step.get("spark") != null) { - - // spark - return HadoopSparkStep.class.getName(); - - } else if (step.get("rmd") != null) { + if (step.getString("spark") != null) { + throw new RuntimeException("Hadoop support was removed in Cloudgene 3"); + } else if (step.getString("rmd") != null) { // rscript return RMarkdownStep.class.getName(); - } else if (step.get("rmd2") != null) { + } else if (step.getString("rmd2") != null) { // rscript return RMarkdownStep.class.getName(); @@ -85,16 +75,15 @@ public String getClassname(WdlStep step) { // custom class return step.getClassname(); - } else if (step.get("exec") != null || step.get("cmd") != null) { + } else if (step.getString("exec") != null || step.getString("cmd") != null) { // command return BashCommandStep.class.getName(); } else { - String runtime = step.get("runtime"); + String runtime = step.getString("runtime"); if (runtime == null || runtime.isEmpty() || runtime.toLowerCase().equals("hadoop")) { - // mapreduce - return HadoopMapReduceStep.class.getName(); + throw new RuntimeException("Hadoop support was removed in Cloudgene 3"); } else if (runtime != null && runtime.toLowerCase().equals("java")) { // normal java when no Hadoop suppport return JavaExternalStep.class.getName(); diff --git a/src/main/java/cloudgene/mapred/jobs/Download.java b/src/main/java/cloudgene/mapred/jobs/Download.java index 4b121ff2..7588a1fe 100644 --- a/src/main/java/cloudgene/mapred/jobs/Download.java +++ b/src/main/java/cloudgene/mapred/jobs/Download.java @@ -9,8 +9,6 @@ public class Download implements Comparable { private int count = 0; private String size; private CloudgeneParameterOutput parameter; - private int parameterId; - private String user; public String getName() { return name; @@ -60,30 +58,20 @@ public CloudgeneParameterOutput getParameter() { return parameter; } - public void setParameterId(int parameterId) { - this.parameterId = parameterId; - } - - public int getParameterId() { - return parameterId; - } - public void decCount() { count--; } - public void setUsername(String user) { - this.user = user; - } - - public String getUsername() { - return user; - } - @Override public int compareTo(Download o) { return name.compareTo(o.getName()); } + + @Override + public boolean equals(Object object) { + Download download = (Download) object; + return name.equals(download.getName()); + } } diff --git a/src/main/java/cloudgene/mapred/jobs/Environment.java b/src/main/java/cloudgene/mapred/jobs/Environment.java index 61f9ccd3..07038894 100644 --- a/src/main/java/cloudgene/mapred/jobs/Environment.java +++ b/src/main/java/cloudgene/mapred/jobs/Environment.java @@ -1,73 +1,124 @@ package cloudgene.mapred.jobs; -import java.io.File; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Map.Entry; +import java.util.Vector; import cloudgene.mapred.plugins.IPlugin; import cloudgene.mapred.plugins.PluginManager; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; -import genepi.io.FileUtil; public class Environment { - public static Map getApplicationVariables(WdlApp application, Settings settings) { + public static String PREFIX = "CLOUDGENE_"; - String localFolder = application.getPath(); - String hdfsFolder = ""; - - // set hdfsFolder to localFolder when hadoop plugin is not activated - PluginManager manager = PluginManager.getInstance(); - if (manager.isEnabled(HadoopPlugin.ID)) { - String hdfsAppFolder = settings.getHdfsAppWorkspace(); - hdfsFolder = FileUtil.path(hdfsAppFolder, application.getId().split(":")[0], application.getVersion()); - } else { - hdfsFolder = localFolder; - } + private Map env = new HashMap(); - HashMap environment = new HashMap(); - environment.put("app_id", application.getId()); - environment.put("app_name", application.getName()); - environment.put("app_version", application.getVersion()); - environment.put("app_hdfs_folder", hdfsFolder); - environment.put("app_local_folder", localFolder); - // Deprecated - environment.put("hdfs_app_folder", hdfsFolder); - environment.put("local_app_folder", localFolder); - // Technologies - - for (IPlugin plugin : manager.getPlugins()) { - environment.put(plugin.getId() + "_installed", manager.isEnabled(plugin) ? "true" : "false"); + public Environment(Settings settings) { + add("SERVICE_NAME", settings.getName()); + add("SERVICE_URL", settings.getServerUrl()); + add("CONTACT_EMAIL", settings.getAdminMail()); + add("CONTACT_NAME", settings.getAdminName()); + if (settings.getMail() != null) { + add("SMTP_HOST", settings.getMail().get("smtp")); + add("SMTP_PORT", settings.getMail().get("port")); + add("SMTP_USER", settings.getMail().get("user")); + add("SMTP_PASSWORD", settings.getMail().get("password")); + add("SMTP_NAME", settings.getMail().get("name")); + add("SMTP_SENDER", settings.getMail().get("name")); } + add("WORKSPACE_TYPE", settings.getExternalWorkspaceType()); + add("WORKSPACE_HOME", settings.getExternalWorkspaceLocation()); + } - return environment; + public Environment addContext(CloudgeneContext context) { + add("JOB_ID", context.getJobId()); + add("JOB_NAME", context.getJobName()); + add("USER_NAME", context.getUser().getUsername()); + add("USER_EMAIL", context.getUser().getMail()); + add("USER_FULL_NAME", context.getUser().getFullName()); + return this; + } + + public Environment addApplication(WdlApp application) { + String localFolder = application.getPath(); + add("APP_LOCATION", localFolder); + //old variable names without prefix for compatibility + env.put("app_id", application.getId()); + env.put("app_name", application.getName()); + env.put("app_version", application.getVersion()); + env.put("app_local_folder", localFolder); + env.put("local_app_folder", localFolder); + env.put("app_hdfs_folder", localFolder); + return this; } - public static Map getJobVariables(CloudgeneContext context) { - Map environment = new HashMap(); - environment.put("job_id", context.getJobId()); - environment.put("job_local_temp", context.getLocalTemp()); - environment.put("job_hdfs_temp", context.getHdfsTemp()); - environment.put("job_local_output", context.getLocalOutput()); - environment.put("job_hdfs_output", context.getHdfsOutput()); - environment.put("user_username", context.getUser().getUsername()); - environment.put("user_mail", context.getUser().getMail()); - // Deprecated - environment.put("workdir", new File(context.getWorkingDirectory()).getAbsolutePath()); - environment.put("jobId", context.getJobId()); - - return environment; + public Environment add(String name, String value) { + env.put(PREFIX + name, value != null ? value : ""); + return this; } - public static String env(String value, Map variables) { + public Map toMap() { + return env; + } + public List toList() { + List variables = new Vector(); + for (Entry entry : env.entrySet()) { + if (entry.getKey().endsWith("_PASSWORD")) { + variables.add(new Variable(entry.getKey(), "************")); + } else { + variables.add(new Variable(entry.getKey(), entry.getValue())); + } + } + return variables; + } + + + public static String resolve(String value, Map variables) { for (String key : variables.keySet()) { value = value.replaceAll("\\$\\{" + key + "\\}", variables.get(key)); } + return value; + } + public String resolve(String value) { + for (String key : env.keySet()) { + value = value.replaceAll("\\$\\{" + key + "\\}", env.get(key)); + } return value; } + + public static class Variable { + + private String name; + + private String value; + + public Variable(String name, String value) { + this.name = name; + this.value = value; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public void setValue(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + } } diff --git a/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java b/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java index 5506598c..80789ea0 100644 --- a/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java +++ b/src/main/java/cloudgene/mapred/jobs/PersistentWorkflowEngine.java @@ -68,7 +68,6 @@ protected void jobCompleted(AbstractJob job) { if (parameter.getFiles() != null) { for (Download download : parameter.getFiles()) { - download.setParameterId(parameter.getId()); download.setParameter(parameter); downloadDao.insert(download); } @@ -79,6 +78,15 @@ protected void jobCompleted(AbstractJob job) { } + if (job.getLogOutput().getFiles() != null) { + + for (Download download : job.getLogOutput().getFiles()) { + download.setParameter(job.getLogOutput()); + downloadDao.insert(download); + } + + } + if (job.getSteps() != null) { StepDao dao2 = new StepDao(database); for (CloudgeneStep step : job.getSteps()) { @@ -101,7 +109,7 @@ protected void jobCompleted(AbstractJob job) { submittedCounters.put("runs", 1); } } - + // write all submitted counters into database for (String name : submittedCounters.keySet()) { Integer value = submittedCounters.get(name); @@ -120,8 +128,8 @@ protected void jobCompleted(AbstractJob job) { } } - - //update job updates (state, endtime, ....) + + // update job updates (state, endtime, ....) dao.update(job); } @@ -142,6 +150,8 @@ protected void jobSubmitted(AbstractJob job) { parameter.setJobId(job.getId()); dao.insert(parameter); } + + dao.insert(job.getLogOutput()); } @Override diff --git a/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java b/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java index 022eb3d5..c6e4d0f0 100644 --- a/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java +++ b/src/main/java/cloudgene/mapred/jobs/WorkflowEngine.java @@ -16,12 +16,8 @@ public class WorkflowEngine implements Runnable { private Thread threadLongTimeQueue; - private Thread threadShortTimeQueue; - private Queue longTimeQueue; - private Queue shortTimeQueue; - private boolean running = false; private AtomicLong priorityCounter = new AtomicLong(); @@ -30,38 +26,6 @@ public class WorkflowEngine implements Runnable { public WorkflowEngine(int ltqThreads, int stqThreads) { - shortTimeQueue = new Queue("ShortTimeQueue", stqThreads, false, false) { - - @Override - public PriorityRunnable createRunnable(AbstractJob job) { - return new SetupThread(job); - } - - @Override - public void onComplete(AbstractJob job) { - - if (job.isSetupComplete()) { - - if (job.hasSteps()) { - statusUpdated(job); - longTimeQueue.submit(job); - } else { - job.setFinishedOn(System.currentTimeMillis()); - jobCompleted(job); - job.setComplete(true); - } - - } else { - log.info("Setup failed for Job " + job.getId() + ". Not added to Long Time Queue."); - job.setFinishedOn(System.currentTimeMillis()); - jobCompleted(job); - job.setComplete(true); - } - - } - - }; - longTimeQueue = new Queue("LongTimeQueue", ltqThreads, true, true) { @Override @@ -71,9 +35,8 @@ public PriorityRunnable createRunnable(AbstractJob job) { @Override public void onComplete(AbstractJob job) { - job.setFinishedOn(System.currentTimeMillis()); + job.setEndTime(System.currentTimeMillis()); jobCompleted(job); - job.setComplete(true); } }; @@ -92,12 +55,10 @@ public void submit(AbstractJob job, long priority) { boolean okey = job.afterSubmission(); if (okey) { - shortTimeQueue.submit(job); + longTimeQueue.submit(job); } else { - job.setFinishedOn(System.currentTimeMillis()); + job.setEndTime(System.currentTimeMillis()); statusUpdated(job); - job.setComplete(true); - } } @@ -111,20 +72,15 @@ public void restart(AbstractJob job, long priority) { job.setSubmittedOn(System.currentTimeMillis()); job.setStartTime(0); job.setEndTime(0); - job.setSetupStartTime(0); - job.setSetupEndTime(0); - job.setFinishedOn(0); job.setState(AbstractJob.STATE_WAITING); statusUpdated(job); boolean okey = job.afterSubmission(); if (okey) { - shortTimeQueue.submit(job); + longTimeQueue.submit(job); } else { - job.setFinishedOn(System.currentTimeMillis()); + job.setEndTime(System.currentTimeMillis()); statusUpdated(job); - job.setComplete(true); - } } @@ -134,21 +90,13 @@ public void updatePriority(AbstractJob job, long priority) { } public void cancel(AbstractJob job) { - - if (shortTimeQueue.isInQueue(job)) { - shortTimeQueue.cancel(job); - } - if (longTimeQueue.isInQueue(job)) { longTimeQueue.cancel(job); } - } @Override public void run() { - threadShortTimeQueue = new Thread(shortTimeQueue); - threadShortTimeQueue.start(); threadLongTimeQueue = new Thread(longTimeQueue); threadLongTimeQueue.start(); running = true; @@ -156,36 +104,29 @@ public void run() { } public void stop() { - threadShortTimeQueue.stop(); threadLongTimeQueue.stop(); } public void block() { - shortTimeQueue.pause(); longTimeQueue.pause(); running = false; } public void resume() { - shortTimeQueue.resume(); longTimeQueue.resume(); running = true; } public boolean isRunning() { - return running && shortTimeQueue.isRunning() && longTimeQueue.isRunning(); + return running && longTimeQueue.isRunning(); } public int getActiveCount() { - return shortTimeQueue.getActiveCount() + longTimeQueue.getActiveCount(); + return longTimeQueue.getActiveCount(); } public AbstractJob getJobById(String id) { - AbstractJob job = longTimeQueue.getJobById(id); - if (job == null) { - job = shortTimeQueue.getJobById(id); - } - return job; + return longTimeQueue.getJobById(id); } public Map getCounters(int state) { @@ -211,8 +152,7 @@ public Map getCounters(int state) { public List getJobsByUser(User user) { - List jobs = shortTimeQueue.getJobsByUser(user); - jobs.addAll(longTimeQueue.getJobsByUser(user)); + List jobs = longTimeQueue.getJobsByUser(user); for (AbstractJob job : jobs) { @@ -227,23 +167,6 @@ public List getJobsByUser(User user) { return jobs; } - public List getAllJobsInShortTimeQueue() { - - List jobs = shortTimeQueue.getAllJobs(); - - for (AbstractJob job : jobs) { - - if (job instanceof CloudgeneJob) { - - ((CloudgeneJob) job).updateProgress(); - - } - - } - - return shortTimeQueue.getAllJobs(); - } - public List getAllJobsInLongTimeQueue() { List jobs = longTimeQueue.getAllJobs(); @@ -261,33 +184,8 @@ public List getAllJobsInLongTimeQueue() { return jobs; } - class SetupThread extends PriorityRunnable { - - private AbstractJob job; - - public SetupThread(AbstractJob job) { - this.job = job; - } - - @Override - public void run() { - log.info("Start input validation for job " + job.getId() + "..."); - job.setSetupStartTime(System.currentTimeMillis()); - job.setSetupRunning(true); - job.runSetupSteps(); - job.setSetupEndTime(System.currentTimeMillis()); - job.setSetupRunning(false); - log.info("Input Validation for job " + job.getId() + " finished. Result: " + job.isSetupComplete()); - } - - } - public boolean isInQueue(AbstractJob job) { - if (!shortTimeQueue.isInQueue(job)) { - return longTimeQueue.isInQueue(job); - } else { - return true; - } + return longTimeQueue.isInQueue(job); } protected void statusUpdated(AbstractJob job) { diff --git a/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphNode.java b/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java similarity index 71% rename from src/main/java/cloudgene/mapred/jobs/engine/graph/GraphNode.java rename to src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java index 540cd0d5..2af2dc9c 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphNode.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/ExecutableStep.java @@ -1,4 +1,4 @@ -package cloudgene.mapred.jobs.engine.graph; +package cloudgene.mapred.jobs.engine; import java.io.File; import java.io.PrintWriter; @@ -7,31 +7,29 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; -import java.util.List; -import java.util.Vector; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.CloudgeneJob; import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.CloudgeneStepFactory; +import cloudgene.mapred.jobs.sdk.WorkflowStep; import cloudgene.mapred.plugins.PluginManager; import cloudgene.mapred.steps.ErrorStep; import cloudgene.mapred.steps.JavaInternalStep; -import cloudgene.mapred.steps.JavaInternalStepDeprecrated; +import cloudgene.mapred.util.TimeUtil; import cloudgene.mapred.wdl.WdlStep; -import cloudgene.sdk.internal.WorkflowStep; import genepi.io.FileUtil; -public class GraphNode implements Runnable { +public class ExecutableStep implements Runnable { + private WdlStep step; private CloudgeneContext context; - private AbstractJob job; + private CloudgeneJob job; private CloudgeneStep instance; @@ -39,23 +37,14 @@ public class GraphNode implements Runnable { private boolean successful = false; - private boolean finish = false; - - private List inputs; - - private List outputs; - private long time; private String id = ""; - public GraphNode(WdlStep step, CloudgeneContext context) - throws MalformedURLException, ClassNotFoundException, InstantiationException, IllegalAccessException { + public ExecutableStep(WdlStep step, CloudgeneContext context) throws Exception { this.step = step; this.context = context; this.job = context.getJob(); - inputs = new Vector(); - outputs = new Vector(); instance(); } @@ -100,9 +89,6 @@ private void instance() instance = (CloudgeneStep) object; } else if (object instanceof WorkflowStep) { instance = new JavaInternalStep((WorkflowStep) object); - //old genepi-hadoop support! this is deprecreated! - } else if (object instanceof genepi.hadoop.common.WorkflowStep) { - instance = new JavaInternalStepDeprecrated((genepi.hadoop.common.WorkflowStep) object); } else { instance = new ErrorStep("Error during initialization: class " + step.getClassname() + " ( " + object.getClass().getSuperclass().getCanonicalName() + ") " @@ -112,7 +98,7 @@ private void instance() // check requirements PluginManager pluginManager = PluginManager.getInstance(); - for (String plugin : instance.getRequirements()) { + for (String plugin : instance.getRequirements()) { if (!pluginManager.isEnabled(plugin)) { instance = new ErrorStep( "Requirements not fullfilled. This steps needs plugin '" + plugin + "'"); @@ -144,8 +130,6 @@ public void run() { context.setCurrentStep(instance); job.getSteps().add(instance); - job.onStepStarted(instance); - job.writeLog("------------------------------------------------------"); job.writeLog(step.getName()); job.writeLog("------------------------------------------------------"); @@ -160,24 +144,16 @@ public void run() { if (!successful) { job.writeLog(" " + step.getName() + " [ERROR]"); successful = false; - finish = true; context.incCounter("steps.failure." + id, 1); context.submitCounter("steps.failure." + id); - job.onStepFinished(instance); return; } else { long end = System.currentTimeMillis(); long time = end - start; - long h = (long) (Math.floor((time / 1000) / 60 / 60)); - long m = (long) ((Math.floor((time / 1000) / 60)) % 60); - - String t = (h > 0 ? h + " h " : "") + (m > 0 ? m + " min " : "") - + (int) ((Math.floor(time / 1000)) % 60) + " sec"; - - job.writeLog(" " + step.getName() + " [" + t + "]"); + job.writeLog(" " + step.getName() + " [" + TimeUtil.format(time) + "]"); setTime(time); } @@ -188,17 +164,13 @@ public void run() { context.submitCounter("steps.failure." + id); successful = false; - finish = true; - job.onStepFinished(instance); return; } context.incCounter("steps.success." + id, 1); context.submitCounter("steps.success." + id); - finish = true; successful = true; - job.onStepFinished(instance); } @@ -208,6 +180,7 @@ public boolean isSuccessful() { public void kill() { if (instance != null) { + log.info("Get kill signal for job " + job.getId()); instance.kill(); } } @@ -218,34 +191,6 @@ public void updateProgress() { } } - public int getProgress() { - if (instance != null) { - return instance.getProgress(); - } else { - return 0; - } - } - - public boolean isFinish() { - return finish; - } - - public void addInput(String input) { - inputs.add(input); - } - - public void addOutput(String output) { - outputs.add(output); - } - - public List getInputs() { - return inputs; - } - - public List getOutputs() { - return outputs; - } - public void setTime(long time) { this.time = time; } diff --git a/src/main/java/cloudgene/mapred/jobs/engine/Executor.java b/src/main/java/cloudgene/mapred/jobs/engine/Executor.java index 659bcaf4..4f8d93e7 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/Executor.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/Executor.java @@ -2,48 +2,28 @@ import java.util.List; -import cloudgene.mapred.jobs.CloudgeneJob; -import cloudgene.mapred.jobs.CloudgeneParameterOutput; -import cloudgene.mapred.jobs.engine.graph.Graph; -import cloudgene.mapred.jobs.engine.graph.GraphNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class Executor { - - private GraphNode executableNode; - - public boolean execute(Graph graph) { - - graph.getContext().log("Executor: execute DAG..."); - - while (graph.getSize() > 0) { - List nodes = graph.getSources(); - boolean successful = false; - successful = executeNodesSequential(graph, nodes); - if (!successful) { - return false; - } +import cloudgene.mapred.jobs.CloudgeneContext; +import cloudgene.mapred.wdl.WdlStep; - } - - return true; - } +public class Executor { - private boolean executeNodesSequential(Graph graph, List nodes) { + private ExecutableStep executableNode; - for (GraphNode node : nodes) { + private static Logger log = LoggerFactory.getLogger(Executor.class); - executableNode = node; + public boolean execute(List steps, CloudgeneContext context) throws Exception { + context.log("Execute " + steps.size() + " steps..."); + for (WdlStep step : steps) { + executableNode = new ExecutableStep(step, context); + log.info("[Job {}] Executor: execute step '{}'...", context.getJobId(), step.getName()); executableNode.run(); - // export results - exportResults(graph, node); - if (!executableNode.isSuccessful()) { return false; } - // TODO: cache.addToCache(node, graph.getContext()); - - graph.remove(node); } return true; @@ -59,29 +39,7 @@ public void updateProgress() { } } - public int getProgress() { - if (executableNode != null) { - return executableNode.getProgress(); - } else { - return 0; - } - } - - private void exportResults(Graph graph, GraphNode node) { - - CloudgeneJob job = (CloudgeneJob) graph.getContext().getJob(); - - for (CloudgeneParameterOutput out : job.getOutputParams()) { - if (out.isAutoExport() && node.getOutputs().contains(out.getName())) { - graph.getContext().println( - "Export parameter '" + out.getName() + "'..."); - job.exportParameter(out); - } - } - - } - - public GraphNode getCurrentNode() { + public ExecutableStep getCurrentNode() { return executableNode; } diff --git a/src/main/java/cloudgene/mapred/jobs/engine/Planner.java b/src/main/java/cloudgene/mapred/jobs/engine/Planner.java index 0da6d568..baf3c338 100644 --- a/src/main/java/cloudgene/mapred/jobs/engine/Planner.java +++ b/src/main/java/cloudgene/mapred/jobs/engine/Planner.java @@ -1,74 +1,41 @@ package cloudgene.mapred.jobs.engine; import java.io.File; -import java.io.StringWriter; -import java.net.MalformedURLException; -import java.util.List; +import java.util.HashMap; import java.util.Map; -import org.apache.velocity.Template; -import org.apache.velocity.VelocityContext; -import org.apache.velocity.app.Velocity; - import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.jobs.Environment; -import cloudgene.mapred.jobs.engine.graph.Graph; -import cloudgene.mapred.jobs.engine.graph.GraphEdge; -import cloudgene.mapred.jobs.engine.graph.GraphNode; -import cloudgene.mapred.jobs.engine.plugins.ParameterValueInput; -import cloudgene.mapred.jobs.engine.plugins.ParameterValueOutput; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; -import cloudgene.mapred.wdl.WdlParameter; import cloudgene.mapred.wdl.WdlParameterInput; import cloudgene.mapred.wdl.WdlParameterOutput; import cloudgene.mapred.wdl.WdlReader; -import cloudgene.mapred.wdl.WdlStep; -import cloudgene.mapred.wdl.WdlWorkflow; +import groovy.text.SimpleTemplateEngine; public class Planner { public WdlApp evaluateWDL(WdlApp app, CloudgeneContext context, Settings settings) throws Exception { - Velocity.setProperty("file.resource.loader.path", "/"); - VelocityContext context2 = new VelocityContext(); + Map context2 = new HashMap(); // add input values to context for (WdlParameterInput param : app.getWorkflow().getInputs()) { - context2.put(param.getId(), new ParameterValueInput(param, context.getInput(param.getId()))); + context2.put(param.getId(), context.getInput(param.getId())); } // add output values to context for (WdlParameterOutput param : app.getWorkflow().getOutputs()) { - context2.put(param.getId(), new ParameterValueOutput(param, context.getOutput(param.getId()))); + context2.put(param.getId(), context.getOutput(param.getId())); } - // add job variables - Map envJob = Environment.getJobVariables(context); - for (String key : envJob.keySet()) { - context2.put(key, envJob.get(key)); - } - - // add app variables - Map envApp = Environment.getApplicationVariables(app, settings); - for (String key : envApp.keySet()) { - context2.put(key, envApp.get(key)); - } + context2.putAll(settings.buildEnvironment().addApplication(app).addContext(context).toMap()); File manifest = new File(app.getManifestFile()); - StringWriter sw = null; - try { + SimpleTemplateEngine engine = new SimpleTemplateEngine(); + String content = engine.createTemplate(manifest).make(context2).toString(); - Template template = Velocity.getTemplate(manifest.getAbsolutePath()); - sw = new StringWriter(); - template.merge(context2, sw); - - } catch (Exception e) { - throw e; - } - - WdlApp app2 = WdlReader.loadAppFromString(manifest.getAbsolutePath(), sw.toString()); + WdlApp app2 = WdlReader.loadAppFromString(manifest.getAbsolutePath(), content); app2.getWorkflow().setInputs(app.getWorkflow().getInputs()); app2.getWorkflow().setOutputs(app.getWorkflow().getOutputs()); @@ -78,98 +45,4 @@ public WdlApp evaluateWDL(WdlApp app, CloudgeneContext context, Settings setting return app2; } - public Graph buildDAG(List steps, WdlWorkflow config, CloudgeneContext context) - throws MalformedURLException, ClassNotFoundException, InstantiationException, IllegalAccessException { - - Graph graph = new Graph(context); - - // build nodes - GraphNode lastNode = null; - for (WdlStep step : steps) { - GraphNode node = new GraphNode(step, context); - graph.addNode(node); - if (lastNode != null) { - graph.connect(lastNode, node, null); - } - lastNode = node; - } - - // add output parameters - for (GraphNode node : graph.getNodes()) { - - for (WdlParameter param : config.getOutputs()) { - if (stepConsumesParameter(node.getStep(), param, context) - && !stepProducesParameter(node.getStep(), param, context)) { - node.addInput(param.getId()); - } - - if (stepProducesParameter(node.getStep(), param, context)) { - node.addOutput(param.getId()); - } - } - - for (WdlParameter param : config.getInputs()) { - if (stepConsumesParameter(node.getStep(), param, context)) { - node.addInput(param.getId()); - } - } - } - - context.log("Planner: DAG created."); - context.log(" Nodes: " + graph.getSize()); - for (GraphNode node : graph.getNodes()) { - context.log(" " + node.getStep().getName()); - String inputs = ""; - for (String input : node.getInputs()) { - inputs += input + " "; - } - context.log(" Inputs: " + inputs); - String outputs = ""; - for (String output : node.getOutputs()) { - outputs += output + " "; - } - context.log(" Outputs: " + outputs); - } - - context.log(" Dependencies: " + graph.getEdges().size()); - for (GraphEdge edge : graph.getEdges()) { - context.log(" " + edge.getSource().getStep().getName() + "->" + edge.getTarget().getStep().getName()); - } - - return graph; - - } - - private boolean stepConsumesParameter(WdlStep step, WdlParameter param, CloudgeneContext context) { - - if (step.get("params") != null) { - if (step.get("params").contains(context.get(param.getId()))) { - return true; - } - } - - if (step.get("mapper") != null) { - if (step.get("mapper").contains(context.get(param.getId()))) { - return true; - } - } - - if (step.get("reducer") != null) { - if (step.get("reducer").contains(context.get(param.getId()))) { - return true; - } - } - - return false; - - } - - private boolean stepProducesParameter(WdlStep step, WdlParameter param, CloudgeneContext context) { - if (step.getGenerates() != null) { - return (step.getGenerates().contains(context.get(param.getId()))); - } else { - return false; - } - } - } diff --git a/src/main/java/cloudgene/mapred/jobs/engine/graph/Graph.java b/src/main/java/cloudgene/mapred/jobs/engine/graph/Graph.java deleted file mode 100644 index b08feb8e..00000000 --- a/src/main/java/cloudgene/mapred/jobs/engine/graph/Graph.java +++ /dev/null @@ -1,127 +0,0 @@ -package cloudgene.mapred.jobs.engine.graph; - -import java.util.List; -import java.util.Vector; - -import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.wdl.WdlParameter; - -public class Graph { - - private List nodes; - - private List edges; - - private CloudgeneContext context; - - public Graph(CloudgeneContext context) { - - this.context = context; - - nodes = new Vector(); - edges = new Vector(); - - } - - public CloudgeneContext getContext() { - return context; - } - - public List getNodes() { - return nodes; - } - - public List getEdges() { - return edges; - } - - public void connect(GraphNode source, GraphNode target, WdlParameter param) { - GraphEdge edge = new GraphEdge(source, target, param); - edges.add(edge); - } - - public boolean areConnected(GraphNode source, GraphNode target) { - for (GraphEdge edge : edges) { - if (edge.getSource() == source && edge.getTarget() == target) { - return true; - } - } - return false; - } - - public boolean remove(GraphNode node) { - - List remove = new Vector(); - - nodes.remove(node); - for (GraphEdge edge : edges) { - if (edge.getSource() == node || edge.getTarget() == node) { - remove.add(edge); - } - } - edges.removeAll(remove); - - return true; - } - - public int getInDegree(GraphNode node) { - int degree = 0; - - for (GraphEdge edge : edges) { - if (edge.getTarget() == node) { - degree++; - } - } - - return degree; - - } - - public int getOutDegree(GraphNode node) { - int degree = 0; - - for (GraphEdge edge : edges) { - if (edge.getSource() == node) { - degree++; - } - } - - return degree; - } - - public List getSources() { - - List sources = new Vector(); - - for (GraphNode node : nodes) { - if (getInDegree(node) == 0) { - sources.add(node); - } - } - - return sources; - - } - - public List getTargets() { - - List targets = new Vector(); - - for (GraphNode node : nodes) { - if (getOutDegree(node) == 0) { - targets.add(node); - } - } - - return targets; - } - - public int getSize() { - return nodes.size(); - } - - public void addNode(GraphNode node) { - nodes.add(node); - } - -} diff --git a/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphEdge.java b/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphEdge.java deleted file mode 100644 index 1129e43f..00000000 --- a/src/main/java/cloudgene/mapred/jobs/engine/graph/GraphEdge.java +++ /dev/null @@ -1,43 +0,0 @@ -package cloudgene.mapred.jobs.engine.graph; - -import cloudgene.mapred.wdl.WdlParameter; - -public class GraphEdge { - - private WdlParameter parameter; - - private GraphNode source; - - private GraphNode target; - - public GraphEdge(GraphNode source, GraphNode target, WdlParameter parameter) { - this.source = source; - this.target = target; - this.parameter = parameter; - } - - public WdlParameter getParameter() { - return parameter; - } - - public void setParameter(WdlParameter parameter) { - this.parameter = parameter; - } - - public GraphNode getSource() { - return source; - } - - public void setSource(GraphNode source) { - this.source = source; - } - - public GraphNode getTarget() { - return target; - } - - public void setTarget(GraphNode target) { - this.target = target; - } - -} diff --git a/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueInput.java b/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueInput.java deleted file mode 100644 index 64a316dc..00000000 --- a/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueInput.java +++ /dev/null @@ -1,85 +0,0 @@ -package cloudgene.mapred.jobs.engine.plugins; - -import java.io.File; -import java.io.IOException; -import java.util.List; - -import cloudgene.mapred.wdl.WdlParameterInput; -import cloudgene.mapred.wdl.WdlParameterInputType; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; -import genepi.io.WildCardFileFilter; - -public class ParameterValueInput { - - private WdlParameterInput parameter; - private String value; - - public ParameterValueInput(WdlParameterInput parameter, String value) { - this.parameter = parameter; - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public String[] listFiles(String ext) { - - if (parameter.getTypeAsEnum() == WdlParameterInputType.HDFS_FOLDER) { - List files = null; - try { - - files = HdfsUtil.getFiles(value, ext); - String[] result = new String[files.size()]; - for (int i = 0; i < files.size(); i++) { - result[i] = FileUtil.getFilename(files.get(i)); - } - return result; - } catch (IOException e) { - e.printStackTrace(); - String[] result = new String[1]; - result[0] = FileUtil.getFilename(value); - ; - return result; - } - } - - if (parameter.getTypeAsEnum() == WdlParameterInputType.LOCAL_FOLDER) { - return getFiles(value, ext); - - } - - return new String[] { value }; - - } - - private static String[] getFiles(String path, String ext) { - File dir = new File(path); - File[] files = dir.listFiles(new WildCardFileFilter(ext)); - - if (files != null) { - - String[] names = new String[files.length]; - - for (int i = 0; i < names.length; ++i) { - names[i] = files[i].getName(); - } - return names; - - - } else { - String[] names = new String[1]; - names[0] = FileUtil.getFilename(path); - return names; - - } - } - - public String getName() { - - return FileUtil.getFilename(value); - - } -} diff --git a/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueOutput.java b/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueOutput.java deleted file mode 100644 index fbf55356..00000000 --- a/src/main/java/cloudgene/mapred/jobs/engine/plugins/ParameterValueOutput.java +++ /dev/null @@ -1,84 +0,0 @@ -package cloudgene.mapred.jobs.engine.plugins; - -import java.io.File; -import java.io.IOException; -import java.util.List; - -import cloudgene.mapred.wdl.WdlParameterOutput; -import cloudgene.mapred.wdl.WdlParameterOutputType; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; -import genepi.io.WildCardFileFilter; - -public class ParameterValueOutput { - - private WdlParameterOutput parameter; - private String value; - - public ParameterValueOutput(WdlParameterOutput parameter, String value) { - this.parameter = parameter; - this.value = value; - } - - @Override - public String toString() { - return value; - } - - public String[] listFiles(String ext) { - - if (parameter.getTypeAsEnum() == WdlParameterOutputType.HDFS_FOLDER) { - List files = null; - try { - - files = HdfsUtil.getFiles(value, ext); - String[] result = new String[files.size()]; - for (int i = 0; i < files.size(); i++) { - result[i] = FileUtil.getFilename(files.get(i)); - } - return result; - } catch (IOException e) { - e.printStackTrace(); - String[] result = new String[1]; - result[0] = FileUtil.getFilename(value); - ; - return result; - } - } - - if (parameter.getTypeAsEnum() == WdlParameterOutputType.LOCAL_FOLDER) { - return getFiles(value, ext); - - } - - return new String[] { value }; - - } - - private static String[] getFiles(String path, String ext) { - File dir = new File(path); - File[] files = dir.listFiles(new WildCardFileFilter(ext)); - - if (files != null) { - - String[] names = new String[files.length]; - - for (int i = 0; i < names.length; ++i) { - names[i] = files[i].getName(); - } - return names; - - } else { - String[] names = new String[1]; - names[0] = FileUtil.getFilename(path); - return names; - - } - } - - public String getName() { - - return FileUtil.getFilename(value); - - } -} diff --git a/src/main/java/cloudgene/mapred/jobs/queue/Queue.java b/src/main/java/cloudgene/mapred/jobs/queue/Queue.java index 4ff2a6dc..83cdd4bb 100644 --- a/src/main/java/cloudgene/mapred/jobs/queue/Queue.java +++ b/src/main/java/cloudgene/mapred/jobs/queue/Queue.java @@ -15,6 +15,8 @@ public abstract class Queue implements Runnable { + private static final int POLL_FRQUENCY_MS = 100; + private List queue; private HashMap> futures; @@ -56,7 +58,7 @@ public void submit(AbstractJob job) { log.info(name + ": Submit job" + (priority ? " (P: " + job.getPriority() + ")" : "") + "..."); if (priority) { - // sorty by state and by priority + // sort by state and by priority Collections.sort(queue, new PriorityComparator()); } @@ -71,42 +73,41 @@ public void submit(AbstractJob job) { } public void cancel(AbstractJob job) { - + if (job.getState() == AbstractJob.STATE_RUNNING || job.getState() == AbstractJob.STATE_EXPORTING) { - log.info(name + ": Cancel Job " + job.getId() + "..."); - - if (job.getSetupStartTime() > 0 && job.getSetupEndTime() == 0){ - job.setSetupEndTime(System.currentTimeMillis()); - } + log.info(name + ": Cancel running job " + job.getId() + "..."); job.kill(); job.cancel(); + log.info(name + ": Job " + job.getId() + " canceled."); + + if (updatePositions) { updatePositionInQueue(); } - } - - if (job.getState() == AbstractJob.STATE_WAITING) { + } else if (job.getState() == AbstractJob.STATE_WAITING) { + log.info(name + ": Cancel waiting job " + job.getId() + "..."); + synchronized (futures) { synchronized (queue) { PriorityRunnable runnable = runnables.get(job); if (runnable != null) { + System.out.println("Kill runnable"); scheduler.kill(runnable); + runnables.remove(job); } - if (job.getSetupStartTime() > 0 && job.getSetupEndTime() == 0){ - job.setSetupEndTime(System.currentTimeMillis()); - } - + job.cancel(); queue.remove(job); futures.remove(job); onComplete(job); - log.info(name + ": Cancel Job..."); + + log.info(name + ": Job " + job.getId() + " canceled."); if (updatePositions) { updatePositionInQueue(); @@ -115,6 +116,8 @@ public void cancel(AbstractJob job) { } } + } else { + log.info(name + ": Cancel job " + job.getId() + ". Unkown state: " + job.getState()); } } @@ -163,7 +166,7 @@ public void run() { } try { - Thread.sleep(5000); + Thread.sleep(POLL_FRQUENCY_MS); } catch (InterruptedException e) { e.printStackTrace(); } diff --git a/src/main/java/cloudgene/mapred/jobs/sdk/WorkflowContext.java b/src/main/java/cloudgene/mapred/jobs/sdk/WorkflowContext.java new file mode 100644 index 00000000..78068b73 --- /dev/null +++ b/src/main/java/cloudgene/mapred/jobs/sdk/WorkflowContext.java @@ -0,0 +1,105 @@ +package cloudgene.mapred.jobs.sdk; + +import java.util.Map; +import java.util.Set; + +import cloudgene.mapred.jobs.workspace.IWorkspace; + +public abstract class WorkflowContext { + + public static final int OK = 0; + + public static final int ERROR = 1; + + public static final int WARNING = 2; + + public static final int RUNNING = 3; + + public abstract String getInput(String param); + + public abstract String getJobId(); + + public abstract String getOutput(String param); + + public abstract String get(String param); + + public abstract void println(String line); + + public abstract void log(String line); + + public abstract String getWorkingDirectory(); + + public abstract boolean sendNotification(String body) throws Exception; + + public abstract boolean sendMail(String subject, String body) throws Exception; + + public abstract boolean sendMail(String to, String subject, String body) throws Exception; + + public abstract Set getInputs(); + + public abstract void setInput(String input, String value); + + //public abstract void setOutput(String input, String value); + + public abstract void incCounter(String name, int value); + + public abstract void submitCounter(String name); + + public abstract Map getCounters(); + + public abstract Object getData(String key); + + public abstract String createLinkToFile(String id); + + public String createLinkToFile(String id, String filename) { + return "[NOT AVAILABLE]"; + + } + + public abstract String getJobName(); + + public abstract String getHdfsTemp(); + + public abstract String getLocalTemp(); + + public abstract void setConfig(Map config); + + public abstract String getConfig(String param); + + public abstract void message(String message, int type); + + private IWorkspace workspace; + + public void setWorkspace(IWorkspace workspace) { + this.workspace = workspace; + } + + public IWorkspace getWorkspace() { + return workspace; + } + + public void ok(String message) { + message(message, OK); + } + + public void error(String message) { + message(message, ERROR); + } + + public void warning(String message) { + message(message, WARNING); + } + + public abstract void beginTask(String name); + + public abstract void updateTask(String name, int type); + + public abstract void endTask(String message, int type); + + public void endTask(String message, Exception e) { + // TODO: add trace + String messageWithTrace = message + ". " + e.toString(); + endTask(messageWithTrace, ERROR); + } + +} diff --git a/src/main/java/cloudgene/mapred/jobs/sdk/WorkflowStep.java b/src/main/java/cloudgene/mapred/jobs/sdk/WorkflowStep.java new file mode 100644 index 00000000..a2641e7e --- /dev/null +++ b/src/main/java/cloudgene/mapred/jobs/sdk/WorkflowStep.java @@ -0,0 +1,35 @@ +package cloudgene.mapred.jobs.sdk; + +import java.io.File; + +public abstract class WorkflowStep { + + public String getFolder(Class clazz) { + return new File(clazz.getProtectionDomain().getCodeSource() + .getLocation().getPath()).getParent(); + } + + public void setup(WorkflowContext context) { + + } + + abstract public boolean run(WorkflowContext context); + + public int getMapProgress() { + return 0; + } + + public int getReduceProgress() { + return 0; + } + + public void updateProgress() { + + } + + public void kill() { + + } + + +} diff --git a/src/main/java/cloudgene/mapred/jobs/steps/Sleep.java b/src/main/java/cloudgene/mapred/jobs/steps/Sleep.java index 366f5102..d4b4dfa7 100644 --- a/src/main/java/cloudgene/mapred/jobs/steps/Sleep.java +++ b/src/main/java/cloudgene/mapred/jobs/steps/Sleep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class Sleep extends WorkflowStep { diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/ExternalWorkspaceFactory.java b/src/main/java/cloudgene/mapred/jobs/workspace/ExternalWorkspaceFactory.java deleted file mode 100644 index b9f743b8..00000000 --- a/src/main/java/cloudgene/mapred/jobs/workspace/ExternalWorkspaceFactory.java +++ /dev/null @@ -1,33 +0,0 @@ -package cloudgene.mapred.jobs.workspace; - -import cloudgene.sdk.internal.IExternalWorkspace; - -public class ExternalWorkspaceFactory { - - public static IExternalWorkspace get(String type, String location) { - - if (type == null) { - return null; - } - - if (type.equalsIgnoreCase("S3")) { - return new S3Workspace(location); - } - - return null; - } - - public static IExternalWorkspace get(String url) { - - if (url == null) { - return null; - } - - if (url.startsWith("s3://")) { - return new S3Workspace(""); - } - - return null; - } - -} diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java new file mode 100644 index 00000000..edff9bb1 --- /dev/null +++ b/src/main/java/cloudgene/mapred/jobs/workspace/IWorkspace.java @@ -0,0 +1,50 @@ +package cloudgene.mapred.jobs.workspace; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +import cloudgene.mapred.jobs.Download; + +public interface IWorkspace { + + public void setJob(String job); + + public void setup() throws IOException; + + public String upload(String id, File file) throws IOException; + + public String uploadInput(String id, File file) throws IOException; + + public String uploadLog(File file) throws IOException; + + public InputStream download(String url) throws IOException; + + public void delete(String job) throws IOException; + + public String getName(); + + public String createPublicLink(String url); + + public String getParent(String url); + + public String createFolder(String id); + + public String createFile(String name, String name2); + + public String createLogFile(String name); + + public String createTempFolder(String string); + + public List getDownloads(String url); + + public List getLogs(); + + public void cleanup(String job) throws IOException; + + public boolean exists(String path) throws IOException; + + public String downloadLog(String string) throws IOException; + +} diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java new file mode 100644 index 00000000..6e35a4bf --- /dev/null +++ b/src/main/java/cloudgene/mapred/jobs/workspace/LocalWorkspace.java @@ -0,0 +1,254 @@ +package cloudgene.mapred.jobs.workspace; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Vector; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import cloudgene.mapred.jobs.Download; +import cloudgene.mapred.util.HashUtil; +import genepi.io.FileUtil; + +public class LocalWorkspace implements IWorkspace { + + private static final String OUTPUT_DIRECTORY = "outputs"; + + private static final String INPUT_DIRECTORY = "input"; + + private static final String TEMP_DIRECTORY = "temp"; + + private static final String LOGS_DIRECTORY = "logs"; + + private static final Logger log = LoggerFactory.getLogger(LocalWorkspace.class); + + private String location; + + private String workspace; + + public LocalWorkspace(String location) { + this.location = absolute(location); + } + + @Override + public String getName() { + return "Local Workspace"; + } + + @Override + public void setJob(String job) { + workspace = FileUtil.path(location, job); + } + + @Override + public void setup() throws IOException { + + if (workspace == null) { + throw new IOException("No job id provided."); + } + + log.info("Init workspace " + workspace); + FileUtil.createDirectory(workspace); + } + + @Override + public String upload(String id, File file) throws IOException { + String folder = FileUtil.path(workspace, id); + FileUtil.createDirectory(folder); + String target = FileUtil.path(folder, file.getName()); + log.info("Copy file " + file.getAbsolutePath() + " to " + target); + FileUtil.copy(file.getAbsolutePath(), target); + return target; + } + + @Override + public String uploadLog(File file) throws IOException { + return upload(LOGS_DIRECTORY, file); + } + + @Override + public String uploadInput(String id, File file) throws IOException { + return upload(FileUtil.path(INPUT_DIRECTORY, id), file); + } + + @Override + public InputStream download(String path) throws IOException { + String absolutePath = path; + if (!absolutePath.startsWith("/")) { + absolutePath = FileUtil.path(location, path); + } + File file = new File(absolutePath); + if (file.exists()) { + return new FileInputStream(file); + } else { + throw new IOException("File '" + absolutePath + "' not found in workspace."); + } + } + + @Override + public String downloadLog(String name) throws IOException { + + if (workspace == null) { + throw new IOException("No job id provided."); + } + + return FileUtil.readFileAsString(download(FileUtil.path(workspace, LOGS_DIRECTORY, name))); + } + + @Override + public boolean exists(String path) throws IOException { + String absolutePath = path; + if (!absolutePath.startsWith("/")) { + absolutePath = FileUtil.path(location, path); + } + File file = new File(absolutePath); + return file.exists(); + } + + @Override + public void delete(String job) throws IOException { + + try { + log.debug("Deleting " + job + " on local workspace..."); + String workspace = FileUtil.path(location, job); + FileUtil.deleteDirectory(workspace); + + log.debug("Deleted all files on local workspace for job " + job + "."); + + } catch (Exception e) { + log.error("Deleting " + job + " failed.", e); + throw new IOException("Deleting " + job + " failed.", e); + } + + } + + @Override + public void cleanup(String job) throws IOException { + + // TODO: add flag to disable cleanup (e.g. debugging) + + try { + log.debug("Cleanup " + job + " on local workspace..."); + String temp = FileUtil.path(location, job, TEMP_DIRECTORY); + FileUtil.deleteDirectory(temp); + + String inputs = FileUtil.path(location, job, INPUT_DIRECTORY); + FileUtil.deleteDirectory(inputs); + + log.debug("Deleted all files on local workspace for job " + job + "."); + + } catch (Exception e) { + log.error("Deleting " + job + " failed.", e); + throw new IOException("Deleting " + job + " failed.", e); + } + + } + + @Override + public String createPublicLink(String url) { + return null; + } + + @Override + public String getParent(String url) { + return new File(url).getParent(); + } + + @Override + public String createFolder(String id) { + String folder = FileUtil.path(workspace, id); + FileUtil.createDirectory(folder); + return folder; + } + + @Override + public String createFile(String parent, String id) { + String folder = FileUtil.path(workspace, parent); + FileUtil.createDirectory(folder); + return FileUtil.path(folder, id); + } + + @Override + public String createLogFile(String id) { + String folder = FileUtil.path(workspace, LOGS_DIRECTORY); + FileUtil.createDirectory(folder); + return FileUtil.path(folder, id); + } + + @Override + public String createTempFolder(String id) { + String folder = FileUtil.path(workspace, TEMP_DIRECTORY, id); + FileUtil.createDirectory(folder); + return folder; + } + + @Override + public List getDownloads(String url) { + File folder = new File(url); + List downloads = new Vector(); + exportFolder("", folder, downloads); + return downloads; + } + + private void exportFolder(String prefix, File folder, List downloads) { + + if (!folder.exists()) { + return; + } + + if (folder.isFile()) { + Download download = createDownload(prefix, folder); + downloads.add(download); + return; + } + + File[] files = folder.listFiles(); + + for (File file : files) { + if (file.isFile()) { + Download download = createDownload(prefix, file); + downloads.add(download); + } else { + exportFolder(prefix.equals("") ? file.getName() : prefix + "/" + file.getName(), file, downloads); + } + } + + } + + protected Download createDownload(String prefix, File file) { + String filename = prefix.equals("") ? file.getName() : prefix + "/" + file.getName(); + String size = FileUtils.byteCountToDisplaySize(file.length()); + String hash = HashUtil.getSha256(filename + size + (Math.random() * 100000)); + Download download = new Download(); + download.setName(filename); + download.setPath(relative(file.getAbsolutePath())); + download.setSize(size); + download.setHash(hash); + return download; + } + + protected String relative(String absolute) { + Path pathAbsolute = Paths.get(absolute); + Path pathRoot = Paths.get(location); + Path pathRelative = pathRoot.relativize(pathAbsolute); + return pathRelative.toString(); + } + + protected String absolute(String path) { + return new File(path).getAbsolutePath(); + } + + @Override + public List getLogs() { + String location = FileUtil.path(workspace, LOGS_DIRECTORY); + return getDownloads(location); + } + +} diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java index 265a6a12..656779b9 100644 --- a/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java +++ b/src/main/java/cloudgene/mapred/jobs/workspace/S3Workspace.java @@ -4,23 +4,35 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.util.List; +import java.util.Vector; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.amazonaws.HttpMethod; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.amazonaws.services.s3.model.S3ObjectSummary; -import cloudgene.sdk.internal.IExternalWorkspace; -import genepi.hadoop.S3Util; +import cloudgene.mapred.jobs.Download; +import cloudgene.mapred.util.HashUtil; +import cloudgene.mapred.util.S3Util; +import genepi.io.FileUtil; -public class S3Workspace implements IExternalWorkspace { +public class S3Workspace implements IWorkspace { + + private static final String OUTPUT_DIRECTORY = "outputs"; + + private static final String INPUT_DIRECTORY = "input"; + + private static final String LOGS_DIRECTORY = "logs"; + + private static final String TEMP_DIRECTORY = "temp"; public static long EXPIRATION_MS = 1000 * 60 * 60; @@ -40,9 +52,16 @@ public String getName() { } @Override - public void setup(String job) throws IOException { + public void setJob(String job) { + this.job = job; + } + + @Override + public void setup() throws IOException { - this.job = job; + if (job == null) { + throw new IOException("No job id provided."); + } if (location == null) { throw new IOException("No S3 Output Bucket specified."); @@ -55,7 +74,8 @@ public void setup(String job) throws IOException { try { S3Util.copyToS3(job, location + "/" + job + "/version.txt"); } catch (Exception e) { - throw new IOException("Output Url '" + location + "' is not writable."); + log.error("Copy file to '" + location + "/" + job + "/version.txt' failed.", e); + throw new IOException("Output Url '" + location + "' is not writable.", e); } } @@ -63,10 +83,21 @@ public void setup(String job) throws IOException { @Override public String upload(String id, File file) throws IOException { String target = location + "/" + job + "/" + id + "/" + file.getName(); + log.info("Copy file " + file.getAbsolutePath() + " to " + target); S3Util.copyToS3(file, target); return target; } + @Override + public String uploadInput(String id, File file) throws IOException { + return upload(FileUtil.path(INPUT_DIRECTORY, id), file); + } + + @Override + public String uploadLog(File file) throws IOException { + return upload(LOGS_DIRECTORY, file); + } + @Override public InputStream download(String url) throws IOException { @@ -80,6 +111,18 @@ public InputStream download(String url) throws IOException { return s3is; } + + @Override + public String downloadLog(String name) throws IOException { + return FileUtil.readFileAsString(download(FileUtil.path(LOGS_DIRECTORY, name))); + } + + public boolean exists(String url) { + String bucket = S3Util.getBucket(url); + String key = S3Util.getKey(url); + AmazonS3 s3 = S3Util.getAmazonS3(); + return s3.doesObjectExist(bucket, key); + } @Override public void delete(String job) throws IOException { @@ -88,36 +131,44 @@ public void delete(String job) throws IOException { throw new IOException("Output Url '" + location + "' is not a valid S3 bucket."); } + String url = location + "/" + job; + try { - String url = location + "/" + job; - String bucket = S3Util.getBucket(url); - String key = S3Util.getKey(url); + log.info("Deleting " + job + " on S3 workspace: '" + url + "'..."); - AmazonS3 s3 = S3Util.getAmazonS3(); + S3Util.deleteFolder(url); - log.debug("Deleting " + job + " on S3 workspace..."); + log.info("Deleted all files on S3 for job " + job + "."); - ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(key); + } catch (Exception e) { + throw new IOException("Folder '" + url + "' could not be deleted.", e); + } - ObjectListing objectListing = s3.listObjects(listObjectsRequest); + } - while (true) { - for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { - log.debug(" Deleting file " + bucket + " " + objectSummary.getKey() + " ..."); - s3.deleteObject(bucket, objectSummary.getKey()); - } - if (objectListing.isTruncated()) { - objectListing = s3.listNextBatchOfObjects(objectListing); - } else { - break; - } - } + @Override + public void cleanup(String job) throws IOException { + if (!S3Util.isValidS3Url(location)) { + throw new IOException("Output Url '" + location + "' is not a valid S3 bucket."); + } - log.debug("Deleted all files on S3 for job " + job + "."); + String temp = location + "/" + job + "/" + TEMP_DIRECTORY; + try { + log.info("Deleting temp directory for " + job + " on S3 workspace: '" + temp + "'..."); + S3Util.deleteFolder(temp); + log.info("Deleted all files on S3 for job " + job + "."); + } catch (Exception e) { + throw new IOException("Folder '" + temp + "' could not be deleted.", e); + } + String input = location + "/" + job + "/" + INPUT_DIRECTORY; + try { + log.info("Deleting input directory for " + input + " on S3 workspace: '" + input + "'..."); + S3Util.deleteFolder(input); + log.info("Deleted all files on S3 for job " + job + "."); } catch (Exception e) { - throw new IOException("Output Url '" + location + "' is not writable."); + throw new IOException("Folder '" + input + "' could not be deleted.", e); } } @@ -144,4 +195,78 @@ public String createPublicLink(String url) { return publicUrl.toString(); } + @Override + public String getParent(String url) { + if (url.startsWith("s3://")) { + int index = url.lastIndexOf('/'); + if (index > 0) { + return url.substring(0, index); + } + return null; + } else { + return null; + } + } + + @Override + public String createFolder(String id) { + return location + "/" + job + "/" + OUTPUT_DIRECTORY + "/" + id; + } + + @Override + public String createFile(String folder, String id) { + return location + "/" + job + "/" + OUTPUT_DIRECTORY + "/" + folder + "/" + id; + } + + @Override + public String createLogFile(String id) { + return location + "/" + job + "/" + LOGS_DIRECTORY + "/" + id; + } + + @Override + public String createTempFolder(String id) { + return location + "/" + job + "/" + TEMP_DIRECTORY + "/" + id; + } + + @Override + public List getDownloads(String url) { + List downloads = new Vector(); + ObjectListing listing; + try { + listing = S3Util.listObjects(url); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + return downloads; + } + + String baseKey = S3Util.getKey(url); + + for (S3ObjectSummary summary : listing.getObjectSummaries()) { + + if (summary.getKey().endsWith("/")) { + continue; + } + + String filename = summary.getKey().replaceAll(baseKey + "/", ""); + String size = FileUtils.byteCountToDisplaySize(summary.getSize()); + String hash = HashUtil.getSha256(filename + size + (Math.random() * 100000)); + Download download = new Download(); + download.setName(filename); + download.setPath("s3://" + summary.getBucketName() + "/" + summary.getKey()); + download.setSize(size); + download.setHash(hash); + downloads.add(download); + + } + + return downloads; + } + + @Override + public List getLogs() { + String url = location + "/" + job + "/" + LOGS_DIRECTORY; + return getDownloads(url); + } + } diff --git a/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceFactory.java b/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceFactory.java new file mode 100644 index 00000000..18f1c140 --- /dev/null +++ b/src/main/java/cloudgene/mapred/jobs/workspace/WorkspaceFactory.java @@ -0,0 +1,57 @@ +package cloudgene.mapred.jobs.workspace; + +import cloudgene.mapred.jobs.AbstractJob; +import cloudgene.mapred.server.Application; +import cloudgene.mapred.util.Settings; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; + +@Singleton +public class WorkspaceFactory { + + @Inject + protected Application application; + + public IWorkspace getDefault() { + + Settings settings = application.getSettings(); + + String type = settings.getExternalWorkspaceType(); + + if (type == null) { + return new LocalWorkspace(settings.getLocalWorkspace()); + } + + if (type.equalsIgnoreCase("S3")) { + String bucket = settings.getExternalWorkspaceLocation(); + return new S3Workspace(bucket); + } + + return new LocalWorkspace(settings.getLocalWorkspace()); + + } + + public IWorkspace getByUrl(String url) { + + Settings settings = application.getSettings(); + + if (url == null || url.isEmpty()) { + throw new RuntimeException("Workspace type could not determined for empty url."); + } + + if (url.startsWith("s3://")) { + String bucket = settings.getExternalWorkspaceLocation(); + return new S3Workspace(bucket); + } + + return new LocalWorkspace(settings.getLocalWorkspace()); + + } + + public IWorkspace getByJob(AbstractJob job) { + IWorkspace workspace = getDefault(); + workspace.setJob(job.getId()); + return workspace; + } + +} diff --git a/src/main/java/cloudgene/mapred/plugins/PluginManager.java b/src/main/java/cloudgene/mapred/plugins/PluginManager.java index 34dd7148..1be23a36 100644 --- a/src/main/java/cloudgene/mapred/plugins/PluginManager.java +++ b/src/main/java/cloudgene/mapred/plugins/PluginManager.java @@ -9,7 +9,6 @@ import org.slf4j.LoggerFactory; import cloudgene.mapred.plugins.docker.DockerPlugin; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; import cloudgene.mapred.plugins.nextflow.NextflowPlugin; import cloudgene.mapred.plugins.rscript.RMarkdownPlugin; import cloudgene.mapred.plugins.rscript.RScriptPlugin; @@ -34,7 +33,6 @@ public static PluginManager getInstance() { private PluginManager() { plugins = new Vector(); - plugins.add(new HadoopPlugin()); plugins.add(new DockerPlugin()); plugins.add(new RScriptPlugin()); plugins.add(new RMarkdownPlugin()); diff --git a/src/main/java/cloudgene/mapred/plugins/docker/DockerBinary.java b/src/main/java/cloudgene/mapred/plugins/docker/DockerBinary.java index 55e561d0..4f0f469f 100644 --- a/src/main/java/cloudgene/mapred/plugins/docker/DockerBinary.java +++ b/src/main/java/cloudgene/mapred/plugins/docker/DockerBinary.java @@ -4,7 +4,7 @@ import cloudgene.mapred.util.BinaryFinder; import cloudgene.mapred.util.Settings; -import genepi.hadoop.command.Command; +import cloudgene.mapred.util.command.Command; import genepi.io.FileUtil; public class DockerBinary { diff --git a/src/main/java/cloudgene/mapred/plugins/docker/DockerStep.java b/src/main/java/cloudgene/mapred/plugins/docker/DockerStep.java index c382a517..15930889 100644 --- a/src/main/java/cloudgene/mapred/plugins/docker/DockerStep.java +++ b/src/main/java/cloudgene/mapred/plugins/docker/DockerStep.java @@ -1,7 +1,6 @@ package cloudgene.mapred.plugins.docker; import java.io.File; -import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -9,8 +8,6 @@ import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; import cloudgene.mapred.wdl.WdlStep; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; public class DockerStep extends CloudgeneStep { @@ -21,7 +18,7 @@ public class DockerStep extends CloudgeneStep { @Override public boolean run(WdlStep step, CloudgeneContext context) { - String cmd = step.get("cmd"); + String cmd = step.getString("cmd"); if (cmd == null) { context.error("No 'exec' or 'cmd' parameter found."); @@ -31,12 +28,12 @@ public boolean run(WdlStep step, CloudgeneContext context) { context.error("'exec' or 'cmd' parameter cannot be an empty string."); } - String stdout = step.get("stdout", "false"); + String stdout = step.getString("stdout", "false"); boolean streamStdout = stdout.equals("true"); String[] params = cmd.split(" "); - String image = step.get("image"); + String image = step.getString("image"); if (image == null) { context.error("No 'image' parameter found."); @@ -64,23 +61,7 @@ protected boolean runInDockerContainer(CloudgeneContext context, String image, S String[] newParams = new String[cmd.length]; for (int i = 0; i < newParams.length; i++) { String param = cmd[i]; - - // checkout hdfs file - if (param.startsWith("hdfs://")) { - String name = FileUtil.getFilename(param); - String localFile = FileUtil.path(((CloudgeneContext) context).getLocalTemp(), "local_" + name); - try { - HdfsUtil.checkOut(param, localFile); - String localFilename = new File(localFile).getAbsolutePath(); - newParams[i] = localFilename; - } catch (IOException e) { - context.log(e.getMessage()); - newParams[i] = param.replaceAll(localWorkspace, DOCKER_WORKSPACE); - } - - } else { - newParams[i] = param.replaceAll(localWorkspace, DOCKER_WORKSPACE); - } + newParams[i] = param.replaceAll(localWorkspace, DOCKER_WORKSPACE); } if (!image.contains(":")) { diff --git a/src/main/java/cloudgene/mapred/plugins/hadoop/HadoopPlugin.java b/src/main/java/cloudgene/mapred/plugins/hadoop/HadoopPlugin.java deleted file mode 100644 index 41b1507a..00000000 --- a/src/main/java/cloudgene/mapred/plugins/hadoop/HadoopPlugin.java +++ /dev/null @@ -1,76 +0,0 @@ -package cloudgene.mapred.plugins.hadoop; - -import cloudgene.mapred.plugins.IPlugin; -import cloudgene.mapred.util.Settings; -import genepi.hadoop.HadoopCluster; - -public class HadoopPlugin implements IPlugin { - - public static final String ID = "hadoop"; - - @Override - public String getId() { - return ID; - } - - @Override - public String getName() { - return "Hadoop Cluster"; - } - - @Override - public boolean isInstalled() { - try { - if (HadoopCluster.verifyCluster()) { - return true; - } else { - return false; - } - } catch (Exception e) { - return false; - } - - } - - @Override - public String getDetails() { - StringBuffer state = new StringBuffer(); - state.append("JobTracker: " + HadoopCluster.getJobTracker() + "\n"); - state.append("Default FS: " + HadoopCluster.getDefaultFS() + "\n"); - state.append("State: " + HadoopCluster.getJobTrackerStatus().toString() + "\n"); - state.append("MapTask: " + HadoopCluster.getMaxMapTasks() + "\n"); - state.append("ReduceTask: " + HadoopCluster.getMaxReduceTasks() + "\n"); - state.append("Nodes\n"); - for (String tracker : HadoopCluster.getActiveTrackerNames()) { - state.append(" " + tracker + "\n"); - } - state.append("Blacklist:\n"); - for (String tracker : HadoopCluster.getBlacklistedTrackerNames()) { - state.append(" " + tracker + "\n"); - } - return state.toString(); - } - - @Override - public void configure(Settings settings) { - // TODO Auto-generated method stub - } - - @Override - public String getStatus() { - if (isInstalled()) { - int nodes = HadoopCluster.getActiveTrackerNames().size(); - int mappTasks = HadoopCluster.getMaxMapTasks(); - int reduceTasks = HadoopCluster.getMaxReduceTasks(); - return "Cluster has " + nodes + " nodes, " + mappTasks + " map tasks and " + reduceTasks + " reduce tasks"; - } else { - try { - HadoopCluster.verifyCluster(); - return "Hadoop support disabled."; - } catch (Exception e) { - return "Hadoop support disabled. " + e.getMessage(); - } - } - } - -} diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java index 352c22e7..b1253171 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowBinary.java @@ -1,16 +1,38 @@ package cloudgene.mapred.plugins.nextflow; import java.io.File; +import java.util.List; +import java.util.Vector; import cloudgene.mapred.util.BinaryFinder; import cloudgene.mapred.util.Settings; -import genepi.hadoop.command.Command; +import cloudgene.mapred.util.command.Command; import genepi.io.FileUtil; public class NextflowBinary { private String binary = ""; + private String script; + + private String profile; + + private List configFiles = new Vector(); + + private String work; + + private File paramsFile; + + private String weblog; + + private String trace; + + private String report; + + private String timeline; + + private String log; + public static NextflowBinary build(Settings settings) { String binary = new BinaryFinder("nextflow").settings(settings, "nextflow", "home").env("NEXTFLOW_HOME") .envPath().path("/usr/local/bin").find(); @@ -47,4 +69,129 @@ public String getVersion() { return "Nextflow not installed."; } } + + public void setBinary(String binary) { + this.binary = binary; + } + + public void setScript(String script) { + this.script = script; + } + + public void setProfile(String profile) { + this.profile = profile; + } + + public void addConfig(File configFile) { + this.configFiles.add(configFile); + } + + public void addConfig(String configFilename) { + this.configFiles.add(new File(configFilename)); + } + + public void setWork(String work) { + this.work = work; + } + + public void setParamsFile(File paramsFile) { + this.paramsFile = paramsFile; + } + + public void setWeblog(String weblog) { + this.weblog = weblog; + } + + public void setTrace(String trace) { + this.trace = trace; + } + + public void setReport(String report) { + this.report = report; + } + + public void setTimeline(String timeline) { + this.timeline = timeline; + } + + public void setLog(String log) { + this.log = log; + } + + public List buildCommand() { + + List nextflow = new Vector(); + nextflow.add("PATH=$PATH:/usr/local/bin"); + nextflow.add(getBinary()); + + nextflow.add("-log"); + nextflow.add(log); + + + nextflow.add("run"); + nextflow.add(script); + + // set profile + if (profile != null && !profile.isEmpty()) { + nextflow.add("-profile"); + nextflow.add(profile); + } + + for (File configFile : configFiles) { + if (configFile.exists()) { + nextflow.add("-c"); + nextflow.add(configFile.getAbsolutePath()); + } + } + + nextflow.add("-w"); + nextflow.add(work); + + nextflow.add("-params-file"); + nextflow.add(paramsFile.getAbsolutePath()); + + nextflow.add("-ansi-log"); + nextflow.add("false"); + + nextflow.add("-with-weblog"); + nextflow.add(weblog); + + nextflow.add("-with-trace"); + nextflow.add(trace); + if (new File(trace).exists()){ + new File(trace).delete(); + } + + nextflow.add("-with-report"); + nextflow.add(report); + if (new File(report).exists()){ + new File(report).delete(); + } + + nextflow.add("-with-timeline"); + nextflow.add(timeline); + if (new File(timeline).exists()){ + new File(timeline).delete(); + } + + List command = new Vector(); + command.add("/bin/bash"); + command.add("-c"); + command.add(join(nextflow)); + + return command; + + } + + private String join(List array) { + String result = ""; + for (int i = 0; i < array.size(); i++) { + if (i > 0) { + result += " "; + } + result += array.get(i); + } + return result; + } + } diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowCollector.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowCollector.java index db0ec49b..b20bd574 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowCollector.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowCollector.java @@ -6,16 +6,24 @@ import java.util.Map; import java.util.Vector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import cloudgene.mapred.jobs.CloudgeneContext; +import cloudgene.mapred.util.Settings; public class NextflowCollector { - + + private static final String COLLECTOR_ENDPOINT = "/api/v2/collect/"; + private static NextflowCollector instance; private Map> data; private Map contexts; + private static final Logger log = LoggerFactory.getLogger(NextflowCollector.class); + public static NextflowCollector getInstance() { if (instance == null) { instance = new NextflowCollector(); @@ -29,19 +37,22 @@ private NextflowCollector() { data = new HashMap>(); } - public void addContext(String job, CloudgeneContext context) { - contexts.put(job, context); + public String addContext(CloudgeneContext context) { + contexts.put(context.getPublicJobId(), context); + Settings settings = context.getSettings(); + log.info("[Job {}] Register collector for public job id '{}'", context.getJobId(), context.getPublicJobId()); + return settings.getServerUrl() + settings.getUrlPrefix() + COLLECTOR_ENDPOINT + context.getPublicJobId(); } public void addEvent(String job, Map event) throws IOException { - + CloudgeneContext context = contexts.get(job); - + if (context == null) { - System.out.println("Warning! No context found for job " + job); - return; + log.info("Warning! No context found for public job id '{}'", job); + return; } - + List processes = data.get(job); if (processes == null) { processes = new Vector(); @@ -68,14 +79,19 @@ public void addEvent(String job, Map event) throws IOException { processes.add(process); } - public List getProcesses(String job) { - List processes = data.get(job); + public List getProcesses(CloudgeneContext context) { + List processes = data.get(context.getPublicJobId()); if (processes == null) { return new Vector(); } - { - return processes; - } + return processes; + + } + + public void cleanProcesses(CloudgeneContext context) { + data.remove(context.getPublicJobId()); + contexts.remove(context.getPublicJobId()); + log.info("[Job {}] Removed collector for public job id '{}'", context.getJobId(), context.getPublicJobId()); } } diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessConfig.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessConfig.java new file mode 100644 index 00000000..14c111c9 --- /dev/null +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessConfig.java @@ -0,0 +1,27 @@ +package cloudgene.mapred.plugins.nextflow; + +public class NextflowProcessConfig { + + private static final String DEFAULT_VIEW = "list"; + + private String view = DEFAULT_VIEW; + + private String label = null; + + public String getView() { + return view; + } + + public void setView(String view) { + this.view = view; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + +} diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java new file mode 100644 index 00000000..e67866bb --- /dev/null +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowProcessRenderer.java @@ -0,0 +1,134 @@ +package cloudgene.mapred.plugins.nextflow; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.Map; + +import org.codehaus.groovy.control.CompilationFailedException; + +import cloudgene.mapred.jobs.Message; +import groovy.text.SimpleTemplateEngine; +import groovy.text.Template; + +public class NextflowProcessRenderer { + + private static final String TEMPLATES_LIST = "/templates/list.html"; + + private static final String TEMPLATES_PROGRESSBAR = "/templates/progressbar.html"; + + private static final String FAILED = "FAILED"; + + private static final String KILLED = "KILLED"; + + private static final String VIEW_PROGRESSBAR = "progressbar"; + + private static final String TRACE_STATUS = "status"; + + private static final String SUBMITTED = "SUBMITTED"; + + private static final String COMPLETED = "COMPLETED"; + + private static final String RUNNING = "RUNNING"; + + public static final Map CACHE = new HashMap(); + + public static void render(NextflowProcessConfig config, NextflowProcess process, Message message) { + + String label = (config.getLabel() != null ? config.getLabel() : process.getName()); + + switch (config.getView()) { + case VIEW_PROGRESSBAR: + NextflowProcessRenderer.renderAsProgressbar(label, process, message); + break; + default: + NextflowProcessRenderer.renderAsList(label, process, message); + } + } + + public static void renderAsList(String label, NextflowProcess process, Message message) { + render(label, TEMPLATES_LIST, process, message); + } + + public static void renderAsProgressbar(String label, NextflowProcess process, Message message) { + render(label, TEMPLATES_PROGRESSBAR, process, message); + } + + public static void render(String label, String template, NextflowProcess process, Message message) { + + int running = 0; + int completed = 0; + int failed = 0; + for (NextflowTask task : process.getTasks()) { + + String status = (String) task.getTrace().get(TRACE_STATUS); + + if (status.equals(RUNNING) || status.equals(SUBMITTED)) { + running++; + } + if (status.equals(COMPLETED)) { + completed++; + } + if (status.equals(FAILED) || status.equals(KILLED)) { + failed++; + } + } + + int total = running + completed + failed; + Map bindings = new HashMap(); + bindings.put("label", label); + bindings.put("total", total); + bindings.put("running", running); + bindings.put("completed", completed); + bindings.put("failed", failed); + bindings.put("tasks", process.getTasks()); + + try { + String text = renderTemplate(template, bindings); + message.setMessage(text); + } catch (Exception e) { + message.setMessage("Template could not be renderer: " + e.toString()); + } + if (running > 0) { + message.setType(Message.RUNNING); + } else if (completed > 0) { + message.setType(Message.OK); + } else { + message.setType(Message.ERROR); + } + + } + + public static String renderTemplate(String path, Map bindings) + throws CompilationFailedException, ClassNotFoundException, IOException, URISyntaxException { + Template template = getTemplate(path); + String rendered = template.make(bindings).toString(); + return rendered.replaceAll("\n", ""); + } + + public static synchronized Template getTemplate(String path) + throws IOException, URISyntaxException, CompilationFailedException, ClassNotFoundException { + Template template = CACHE.get(path); + + if (template != null) { + return template; + } + + SimpleTemplateEngine engine = new SimpleTemplateEngine(); + String content = readTemplate(path); + template = engine.createTemplate(content); + CACHE.put(path, template); + return template; + } + + private static String readTemplate(String path) throws IOException, URISyntaxException { + URI uri = NextflowProcess.class.getResource(path).toURI(); + Path test = Paths.get(uri); + return Files.readString(test); + } + +} diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java index 4ea90fe3..9189a7b0 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowStep.java @@ -4,32 +4,49 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Vector; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; -import cloudgene.mapred.util.HashUtil; +import cloudgene.mapred.jobs.workspace.IWorkspace; +import cloudgene.mapred.plugins.nextflow.report.Report; +import cloudgene.mapred.plugins.nextflow.report.ReportEvent; +import cloudgene.mapred.plugins.nextflow.report.ReportEventExecutor; +import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlStep; import genepi.io.FileUtil; import groovy.json.JsonOutput; public class NextflowStep extends CloudgeneStep { + private static final String PROPERTY_PROCESS_CONFIG = "processes"; + private CloudgeneContext context; private Map messages = new HashMap(); + private Map configs = new HashMap(); + + private NextflowCollector collector = NextflowCollector.getInstance(); + + private static final Logger log = LoggerFactory.getLogger(NextflowStep.class); + @Override public boolean run(WdlStep step, CloudgeneContext context) { this.context = context; + + Settings settings = context.getSettings(); - String script = step.get("script"); + String script = step.getString("script"); if (script == null) { // no script set. try to find a main.nf. IS the default convention of nf-core. @@ -42,124 +59,82 @@ public boolean run(WdlStep step, CloudgeneContext context) { "Nextflow script '" + scriptPath + "' not found. Please use 'script' to define your nf file."); } - NextflowBinary nextflow = NextflowBinary.build(context.getSettings()); + // load process styling + loadProcessConfigs(step.get(PROPERTY_PROCESS_CONFIG)); - List nextflowCommand = new Vector(); - nextflowCommand.add("PATH=$PATH:/usr/local/bin"); - nextflowCommand.add(nextflow.getBinary()); - nextflowCommand.add("run"); - nextflowCommand.add(script); + NextflowBinary nextflow = NextflowBinary.build(settings); + nextflow.setScript(scriptPath); AbstractJob job = context.getJob(); - String appFolder = context.getSettings().getApplicationRepository().getConfigDirectory(job.getApplicationId()); + String appFolder = settings.getApplicationRepository().getConfigDirectory(job.getApplicationId()); + // set profile String profile = ""; String nextflowProfile = FileUtil.path(appFolder, "nextflow.profile"); if (new File(nextflowProfile).exists()) { profile = FileUtil.readFileAsString(nextflowProfile); } + nextflow.setProfile(profile); - // set profile - if (!profile.isEmpty()) { - nextflowCommand.add("-profile"); - nextflowCommand.add(profile); - } + // set global configuration + String globalConfig = settings.getNextflowConfig(); + nextflow.addConfig(globalConfig); - String nextflowConfig = FileUtil.path(appFolder, "nextflow.config"); - File nextflowConfigFile = new File(nextflowConfig); - if (nextflowConfigFile.exists()) { - // set custom configuration - nextflowCommand.add("-c"); - nextflowCommand.add(nextflowConfigFile.getAbsolutePath()); - } + // set application specific configuration + String appConfig = FileUtil.path(appFolder, "nextflow.config"); + nextflow.addConfig(appConfig); + // set work directory String work = ""; String nextflowWork = FileUtil.path(appFolder, "nextflow.work"); if (new File(nextflowWork).exists()) { work = FileUtil.readFileAsString(nextflowWork); } + IWorkspace workspace = job.getWorkspace(); + // use workdir if set in settings if (!work.trim().isEmpty()) { - nextflowCommand.add("-w"); - nextflowCommand.add(work); + nextflow.setWork(work); } else { - String workDir = FileUtil.path(context.getLocalOutput(), "nextflow"); - FileUtil.createDirectory(workDir); - nextflowCommand.add("-w"); - nextflowCommand.add(workDir); - } - - Map params = new HashMap(); - - // used to defined hard coded params - for (String key : step.keySet()) { - if (key.startsWith("params.")) { - String param = key.replace("params.", ""); - String value = step.get(key); - params.put(param, value); - } - } - - // add all inputs - for (String param : context.getInputs()) { - String value = context.getInput(param); - // resolve app links: use all properties as input parameters - if (value.startsWith("apps@")) { - Map linkedApp = (Map) context.getData(param); - params.put(param, linkedApp); - } else { - params.put(param, value); - } - - } - - // add all outputs - for (String param : context.getOutputs()) { - String value = context.getOutput(param); - params.put(param, value); + String workDir = workspace.createTempFolder("nextflow"); + nextflow.setWork(workDir); } + // params json file String paramsJsonFilename = FileUtil.path(context.getLocalOutput(), "params.json"); File paramsFile = new File(paramsJsonFilename); - try { + Map params = createParamsMap(step); + // TODO: workspace? writeParamsJson(params, paramsFile); - } catch (IOException e1) { - e1.printStackTrace(); + } catch (IOException e) { + log.error("[Job {}] Writing params.json file failed.", context.getJobId(), e); return false; } + nextflow.setParamsFile(paramsFile); - nextflowCommand.add("-params-file"); - nextflowCommand.add(paramsFile.getAbsolutePath()); + // register job in webcollector and set created url + String collectorUrl = collector.addContext(context); + nextflow.setWeblog(collectorUrl); - nextflowCommand.add("-ansi-log"); - nextflowCommand.add("false"); + // log files and reports + nextflow.setTrace(workspace.createLogFile("trace.csv")); + nextflow.setReport(workspace.createLogFile("report.html")); + nextflow.setTimeline(workspace.createLogFile("timeline.html")); + nextflow.setLog(workspace.createLogFile("nextflow.log")); - nextflowCommand.add("-with-weblog"); - nextflowCommand.add(context.getSettings().getServerUrl() + context.getSettings().getUrlPrefix() - + "/api/v2/collect/" + makeSecretJobId(context.getJobId())); + try { - StringBuilder output = new StringBuilder(); + File executionDir = new File(context.getLocalOutput()); - List command = new Vector(); - command.add("/bin/bash"); - command.add("-c"); - command.add(join(nextflowCommand)); + StringBuilder output = new StringBuilder(); + boolean successful = executeCommand(nextflow.buildCommand(), context, output, executionDir); - NextflowCollector.getInstance().addContext(makeSecretJobId(context.getJobId()), context); - - try { - // context.beginTask("Running Nextflow pipeline..."); - boolean successful = executeCommand(command, context, output); - if (successful) { - updateProgress(); - return true; - } else { + if (!successful) { // set all running processes to failed - List processes = NextflowCollector.getInstance() - .getProcesses(makeSecretJobId(context.getJobId())); + List processes = collector.getProcesses(context); for (NextflowProcess process : processes) { for (NextflowTask task : process.getTasks()) { String status = (String) task.getTrace().get("status"); @@ -171,95 +146,87 @@ public boolean run(WdlStep step, CloudgeneContext context) { } updateProgress(); - // Write nextflow output into step - /* - * context.beginTask("Running Nextflow pipeline..."); String text = ""; - * - * if (killed) { text = output + "\n\n\n" + - * makeRed("Pipeline execution canceled."); } else { text = output + "\n\n\n" + - * makeRed("Pipeline execution failed."); } context.endTask(text, - * Message.ERROR_ANSI); - */ - return false; + if (killed) { + context.error("Pipeline execution canceled."); + } + } + + updateProgress(); + + collector.cleanProcesses(context); + + File report = new File(executionDir, Report.DEFAULT_FILENAME); + if (!report.exists()) { + return successful; + } + + context.log("Load report file from '" + report.getCanonicalPath() + "'"); + try { + parseReport(report); + } catch (Exception e) { + log.error("[Job {}] Invalid report file.", e); + } + + return successful; + } catch (Exception e) { - e.printStackTrace(); + log.error("[Job {}] Running nextflow script failed.", context.getJobId(), e); return false; } } + private void parseReport(File file) throws IOException { + Report report = new Report(file.getAbsolutePath()); + context.log("Execute " + report.getEvents().size() + " events."); + for (ReportEvent event : report.getEvents()) { + context.log("Event: " + event); + ReportEventExecutor.execute(event, context); + } + } + @Override public void updateProgress() { - String job = makeSecretJobId(context.getJobId()); - List processes = NextflowCollector.getInstance().getProcesses(job); + List processes = collector.getProcesses(context); for (NextflowProcess process : processes) { + NextflowProcessConfig config = getNextflowProcessConfig(process); + Message message = messages.get(process.getName()); if (message == null) { message = context.createTask("" + process.getName() + ""); messages.put(process.getName(), message); } - String text = "" + process.getName() + ""; - boolean running = false; - boolean ok = true; - for (NextflowTask task : process.getTasks()) { - - String status = (String) task.getTrace().get("status"); + NextflowProcessRenderer.render(config, process, message); - if (status.equals("RUNNING") || status.equals("SUBMITTED")) { - running = true; - } - if (!status.equals("COMPLETED")) { - ok = false; - - } - text += "
"; + } - text += (String) task.getTrace().get("name"); - if (status.equals("RUNNING")) { - text += "..."; - } - if (status.equals("COMPLETED")) { - text += " "; - } - if (status.equals("KILLED") || status.equals("FAILED")) { - text += " "; - } + } - if (task.getLog() != null) { - text += "
" + task.getLog(); + private void loadProcessConfigs(Object map) { + if (map != null) { + List> processConfigs = (List>) map; + for (Map processConfig : processConfigs) { + String process = processConfig.get("process").toString(); + NextflowProcessConfig config = new NextflowProcessConfig(); + if (processConfig.get("view") != null) { + config.setView(processConfig.get("view").toString()); } - - text += "
"; - } - message.setMessage(text); - - if (running) { - message.setType(Message.RUNNING); - } else { - if (ok) { - message.setType(Message.OK); - } else { - message.setType(Message.ERROR); + if (processConfig.get("label") != null) { + config.setLabel(processConfig.get("label").toString()); } + configs.put(process, config); } - } } - private String join(List array) { - String result = ""; - for (int i = 0; i < array.size(); i++) { - if (i > 0) { - result += " \\\n"; - } - result += array.get(i); - } - return result; + private NextflowProcessConfig getNextflowProcessConfig(NextflowProcess process) { + NextflowProcessConfig config = configs.get(process.getName()); + return config != null ? config : new NextflowProcessConfig(); } @Override @@ -267,20 +234,49 @@ public String[] getRequirements() { return new String[] { NextflowPlugin.ID }; } - public String makeSecretJobId(String job) { - return HashUtil.getSha256(job); - } + private Map createParamsMap(WdlStep step) { + Map params = new HashMap(); + + // used to defined hard coded params + for (String key : step.keySet()) { + if (key.startsWith("params.")) { + String param = key.replace("params.", ""); + Object value = step.get(key); + params.put(param, value); + } + } + if (step.get("params") != null) { + Map paramsMap = (Map) step.get("params"); + params.putAll(paramsMap); + } + + // add all inputs + for (String param : context.getInputs()) { + String value = context.getInput(param); + // resolve app links: use all properties as input parameters + if (value.startsWith("apps@")) { + Map linkedApp = (Map) context.getData(param); + params.put(param, linkedApp); + } else { + params.put(param, value); + } + + } + + // add all outputs + for (String param : context.getOutputs()) { + String value = context.getOutput(param); + params.put(param, value); + } + + return params; - private String makeRed(String text) { - return ((char) 27 + "[31m" + text + (char) 27 + "[0m"); } protected void writeParamsJson(Map params, File paramsFile) throws IOException { - BufferedWriter writer = new BufferedWriter(new FileWriter(paramsFile)); writer.write(JsonOutput.toJson(params)); writer.close(); - } } diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java index d3646053..24de0616 100644 --- a/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/NextflowTask.java @@ -1,12 +1,18 @@ package cloudgene.mapred.plugins.nextflow; -import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import cloudgene.mapred.jobs.CloudgeneContext; +import cloudgene.mapred.jobs.workspace.IWorkspace; +import cloudgene.mapred.plugins.nextflow.report.Report; +import cloudgene.mapred.plugins.nextflow.report.ReportEvent; +import cloudgene.mapred.plugins.nextflow.report.ReportEventExecutor; import genepi.io.FileUtil; -import genepi.io.text.LineReader; public class NextflowTask { @@ -14,11 +20,13 @@ public class NextflowTask { private Map trace; - private String log = null; + private String logText = null; private CloudgeneContext context; + + private static final Logger log = LoggerFactory.getLogger(NextflowTask.class); - public NextflowTask( CloudgeneContext context, Map trace) { + public NextflowTask(CloudgeneContext context, Map trace) { id = (Integer) trace.get("task_id"); this.trace = trace; this.context = context; @@ -30,39 +38,36 @@ public void update(Map trace) throws IOException { // if task is completed or failed check if a cloudgene.log is in workdir and // load its content + // TODO: check if CHACHED os also needed! String status = (String) trace.get("status"); - if (status.equals("COMPLETED") || status.equals("FAILED")) { - String workDir = (String) trace.get("workdir"); - String logFilename = FileUtil.path(workDir, "cloudgene.log"); + if (!status.equals("COMPLETED") && !status.equals("FAILED")) { + return; + } - // TODO: implement s3 support. How to handle other cloud providers? + String workDir = (String) trace.get("workdir"); + String reportFilename = FileUtil.path(workDir, Report.DEFAULT_FILENAME); + IWorkspace workspace = context.getJob().getWorkspace(); + if (!workspace.exists(reportFilename)) { + return; + } - if (new File(logFilename).exists()) { - log = FileUtil.readFileAsString(logFilename); - parseFile(logFilename); - } - + context.log("Load report file from '" + reportFilename + "'"); + InputStream stream = workspace.download(reportFilename); + try { + parseReport(reportFilename); + } catch (Exception e) { + log.error("[Job {}] Invalid report file.", e); + logText = "Invalid report file: \n" + FileUtil.readFileAsString(stream); } } - private void parseFile(String logFilename) throws IOException { - LineReader reader = new LineReader(logFilename); - while(reader.next()) { - String line = reader.get(); - if (line.startsWith("[INC]")) { - String[] tiles = line.split(" ", 3); - String name = tiles[1]; - int value = Integer.parseInt(tiles[2]); - context.incCounter(name, value); - } - if (line.startsWith("[SUBMIT]")) { - String[] tiles = line.split(" ", 2); - String name = tiles[1]; - context.submitCounter(name); - } + private void parseReport(String reportFilename) throws IOException { + InputStream stream = context.getWorkspace().download(reportFilename); + Report report = new Report(stream); + for (ReportEvent event : report.getEvents()) { + ReportEventExecutor.execute(event, context); } - reader.close(); } public int getId() { @@ -73,8 +78,8 @@ public Map getTrace() { return trace; } - public String getLog() { - return log; + public String getLogText() { + return logText; } } diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/report/Report.java b/src/main/java/cloudgene/mapred/plugins/nextflow/report/Report.java new file mode 100644 index 00000000..68e87ac2 --- /dev/null +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/report/Report.java @@ -0,0 +1,110 @@ +package cloudgene.mapred.plugins.nextflow.report; + +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Type; +import java.util.List; +import java.util.Vector; + +import com.nimbusds.jose.shaded.gson.Gson; +import com.nimbusds.jose.shaded.gson.GsonBuilder; +import com.nimbusds.jose.shaded.gson.JsonIOException; +import com.nimbusds.jose.shaded.gson.JsonSyntaxException; +import com.nimbusds.jose.shaded.gson.reflect.TypeToken; + +import cloudgene.mapred.plugins.nextflow.report.ReportEvent.WebCommand; + +public class Report { + + public static final String DEFAULT_FILENAME = "cloudgene.report.json"; + + private List events = new Vector(); + + public Report() { + + } + + public Report(String filename) throws JsonIOException, JsonSyntaxException, FileNotFoundException { + loadFromFile(filename); + } + + public Report(InputStream in) throws JsonIOException, JsonSyntaxException, IOException { + loadFromInputStream(in); + } + + public void loadFromFile(String filename) throws JsonIOException, JsonSyntaxException, FileNotFoundException { + Type collectionType = new TypeToken>() { + }.getType(); + Gson gson = (new GsonBuilder()).create(); + events = gson.fromJson(new FileReader(filename), collectionType); + } + + public void loadFromInputStream(InputStream in) throws JsonIOException, JsonSyntaxException, IOException { + Type collectionType = new TypeToken>() { + }.getType(); + Gson gson = (new GsonBuilder()).create(); + events = gson.fromJson(new InputStreamReader(in), collectionType); + in.close(); + } + + public boolean hasInMemory(String content) { + for (ReportEvent event : events) { + if (event.toString().contains(content)) { + return true; + } + } + return false; + } + + public void saveToFile(String filename) throws JsonIOException, IOException { + Gson gson = (new GsonBuilder()).create(); + gson.toJson(events, new FileWriter(filename)); + } + + public List getEvents() { + return events; + } + + public void println(String line) { + addEvent(WebCommand.PRINTLN, line); + } + + public void log(String line) { + addEvent(WebCommand.LOG, line); + } + + public void incCounter(String name, int value) { + addEvent(WebCommand.INC_COUNTER, name, value); + } + + public void submitCounter(String name) { + addEvent(WebCommand.SUBMIT_COUNTER, name); + } + + public void message(String message, int type) { + addEvent(WebCommand.MESSAGE, message, type); + } + + public void beginTask(String name) { + addEvent(WebCommand.BEGIN_TASK, name); + } + + public void updateTask(String name, int type) { + addEvent(WebCommand.UPDATE_TASK, name, type); + } + + public void endTask(String message, int type) { + addEvent(WebCommand.END_TASK, message, type); + } + + public void addEvent(WebCommand command, Object... params) { + ReportEvent event = new ReportEvent(command, params); + events.add(event); + // TODO: autosave? + } + +} diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEvent.java b/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEvent.java new file mode 100644 index 00000000..e8226e3c --- /dev/null +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEvent.java @@ -0,0 +1,148 @@ +package cloudgene.mapred.plugins.nextflow.report; + +import cloudgene.mapred.jobs.sdk.WorkflowContext; + +public class ReportEvent { + + public enum WebCommand { + + MESSAGE, + + BEGIN_TASK, + + UPDATE_TASK, + + END_TASK, + + INC_COUNTER, + + SUBMIT_COUNTER, + + PRINTLN, + + LOG + + } + + private WebCommand command; + + private Object[] params; + + public ReportEvent(WebCommand command, Object[] params) { + super(); + this.command = command; + this.params = params; + } + + public WebCommand getCommand() { + return command; + } + + public void setCommand(WebCommand command) { + this.command = command; + } + + public Object[] getParams() { + return params; + } + + public void setParams(Object[] params) { + this.params = params; + } + + public String toString() { + return Formatter.format(this); + } + + static private class Formatter { + + public static String format(ReportEvent event) { + + switch (event.getCommand()) { + case SUBMIT_COUNTER: { + String counter = (String) event.getParams()[0]; + return submitCounter(counter); + } + case MESSAGE: { + String message = (String) event.getParams()[0]; + int type = asInteger(event.getParams()[1]); + return message(message, type); + } + case BEGIN_TASK: { + String name = (String) event.getParams()[0]; + return message(name, WorkflowContext.RUNNING); + } + case UPDATE_TASK: { + // String name = (String) event.getParams()[0]; + // int type = ((Integer) event.getParams()[1]).intValue(); + // message(name, type); + break; + } + case LOG: { + String line = (String) event.getParams()[0]; + return log(line); + } + case PRINTLN: { + String line = (String) event.getParams()[0]; + return info(line); + } + case END_TASK: { + String name = (String) event.getParams()[0]; + int type = asInteger(event.getParams()[1]); + return message(name, type); + } + case INC_COUNTER: { + String counter = (String) event.getParams()[0]; + int value = asInteger(event.getParams()[1]); + return incCounter(counter, value); + } + default: + + } + + return ""; + + } + + public static String info(String message) { + return ("[INFO] " + message); + } + + public static String log(String message) { + return ("[LOG] " + message); + } + + public static String incCounter(String counter, int value) { + return ("[INC] " + counter + " " + value); + } + + public static String submitCounter(String counter) { + return ("[SUBMIT] " + counter); + } + + public static String message(String message, int type) { + + switch (type) { + case WorkflowContext.OK: + return ("[OK] " + message); + case WorkflowContext.ERROR: + return ("[ERROR] " + message); + case WorkflowContext.WARNING: + return ("[WARN] " + message); + case WorkflowContext.RUNNING: + return ("[RUN] " + message); + default: + return ("[INFO] " + message); + } + + } + } + + public static int asInteger(Object object) { + //groovy writes integers also as doubles + Double value = Double.parseDouble(object.toString()); + return value.intValue(); + + } + +} diff --git a/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEventExecutor.java b/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEventExecutor.java new file mode 100644 index 00000000..ecd7d1ac --- /dev/null +++ b/src/main/java/cloudgene/mapred/plugins/nextflow/report/ReportEventExecutor.java @@ -0,0 +1,62 @@ +package cloudgene.mapred.plugins.nextflow.report; + +import java.io.IOException; + +import cloudgene.mapred.jobs.sdk.WorkflowContext; + +public class ReportEventExecutor { + + public static void execute(ReportEvent event, WorkflowContext context) throws IOException { + + switch (event.getCommand()) { + case SUBMIT_COUNTER: { + String name = (String) event.getParams()[0]; + context.submitCounter(name); + break; + } + case MESSAGE: { + String message = (String) event.getParams()[0]; + int type = ((Double) event.getParams()[1]).intValue(); + context.message(message, type); + break; + } + case BEGIN_TASK: { + String name = (String) event.getParams()[0]; + context.beginTask(name); + break; + } + case UPDATE_TASK: { + String name = (String) event.getParams()[0]; + int type = ((Double) event.getParams()[1]).intValue(); + context.updateTask(name, type); + break; + } + case LOG: { + String line = (String) event.getParams()[0]; + context.log(line); + break; + } + case END_TASK: { + String name = (String) event.getParams()[0]; + int type = ((Double) event.getParams()[1]).intValue(); + context.endTask(name, type); + break; + } + case INC_COUNTER: { + String name = (String) event.getParams()[0]; + int count = ((Double) event.getParams()[1]).intValue(); + context.incCounter(name, count); + break; + } + case PRINTLN: { + String line = (String) event.getParams()[0]; + context.println(line); + break; + } + default: + throw new IOException("Command " + event.getCommand() + " not yet supported."); + } + + } + +} diff --git a/src/main/java/cloudgene/mapred/plugins/rscript/RScriptBinary.java b/src/main/java/cloudgene/mapred/plugins/rscript/RScriptBinary.java index c1d31198..9dbda6de 100644 --- a/src/main/java/cloudgene/mapred/plugins/rscript/RScriptBinary.java +++ b/src/main/java/cloudgene/mapred/plugins/rscript/RScriptBinary.java @@ -4,7 +4,7 @@ import cloudgene.mapred.util.BinaryFinder; import cloudgene.mapred.util.Settings; -import genepi.hadoop.command.Command; +import cloudgene.mapred.util.command.Command; import genepi.io.FileUtil; public class RScriptBinary { diff --git a/src/main/java/cloudgene/mapred/server/controller/AppController.java b/src/main/java/cloudgene/mapred/server/controller/AppController.java index 01bea502..789d9f26 100644 --- a/src/main/java/cloudgene/mapred/server/controller/AppController.java +++ b/src/main/java/cloudgene/mapred/server/controller/AppController.java @@ -82,12 +82,32 @@ public ApplicationResponse updateApp(String appId, @Nullable Boolean enabled, @N } // update permissions applicationService.updatePermissions(app, permission); - // update config + + app.checkForChanges(); + + ApplicationResponse appResponse = ApplicationResponse.build(app); + + return appResponse; + } + + @Get("/api/v2/server/apps/{appId}/settings") + @Secured(User.ROLE_ADMIN) + public ApplicationResponse getAppSettings(String appId) { + Application app = applicationService.getById(appId); + ApplicationRepository repository = applicationService.getRepository(); + ApplicationResponse appResponse = ApplicationResponse.buildWithDetails(app, this.application.getSettings(), + repository); + + return appResponse; + } + + @Put("/api/v2/server/apps/{appId}/settings") + @Secured(User.ROLE_ADMIN) + public ApplicationResponse updateAppSettings(String appId, @Nullable Boolean enabled, @Nullable String permission, + @Nullable Boolean reinstall, @Nullable Map config) { + + Application app = applicationService.getById(appId); applicationService.updateConfig(app, config); - // reinstall application - if (reinstall != null) { - applicationService.reinstallApp(app, reinstall); - } app.checkForChanges(); @@ -103,8 +123,7 @@ public ApplicationResponse updateApp(String appId, @Nullable Boolean enabled, @N @Secured(User.ROLE_ADMIN) public ApplicationResponse install(@Nullable String url) { Application app = applicationService.installApp(url); - ApplicationRepository repository = applicationService.getRepository(); - return ApplicationResponse.buildWithDetails(app, application.getSettings(), repository); + return ApplicationResponse.build(app); } @Get("/api/v2/server/apps") diff --git a/src/main/java/cloudgene/mapred/server/controller/DownloadController.java b/src/main/java/cloudgene/mapred/server/controller/DownloadController.java index 14f7d0c6..e268af99 100644 --- a/src/main/java/cloudgene/mapred/server/controller/DownloadController.java +++ b/src/main/java/cloudgene/mapred/server/controller/DownloadController.java @@ -1,6 +1,8 @@ package cloudgene.mapred.server.controller; import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.net.URISyntaxException; import java.util.List; @@ -20,8 +22,8 @@ import cloudgene.mapred.server.services.DownloadService; import cloudgene.mapred.server.services.JobService; import genepi.io.FileUtil; -import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; +import io.micronaut.http.MutableHttpResponse; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; import io.micronaut.security.annotation.Secured; @@ -46,28 +48,19 @@ public class DownloadController { @Inject protected JobService jobService; - @Get("/downloads/{jobId}/{hash}/{filename}") + @Get("/downloads/{jobId}/{hash}/{filename:.+}") @Secured(SecurityRule.IS_ANONYMOUS) - public HttpResponse downloadExternalResults(String jobId, String hash, String filename) - throws URISyntaxException { + public MutableHttpResponse downloadExternalResults(String jobId, String hash, String filename) + throws URISyntaxException, IOException { AbstractJob job = jobService.getById(jobId); DownloadDao dao = new DownloadDao(application.getDatabase()); Download download = dao.findByHash(hash); - // job is running and not in database --> download possible of - // autoexport params + // job is running and not in database if (download == null) { - for (CloudgeneParameterOutput param : job.getOutputParams()) { - if (param.isAutoExport() && param.getFiles() != null) { - for (Download download2 : param.getFiles()) { - if (download2.getHash().equals(hash)) { - download = download2; - } - } - } - } + download = job.findDownloadByHash(hash); } String message = String.format("Job: Downloading file '%s' for job %s", filename, job.getId()); log.info(message); @@ -75,19 +68,16 @@ public HttpResponse downloadExternalResults(String jobId, String hash, Str } - @Get("/share/results/{hash}/{filename}") + @Get("/share/results/{hash}/{filename:.+}") @Secured(SecurityRule.IS_ANONYMOUS) - public HttpResponse downloadPublicLink(String hash, String filename) throws URISyntaxException { + public MutableHttpResponse downloadPublicLink(String hash, String filename) + throws URISyntaxException, IOException { DownloadDao dao = new DownloadDao(application.getDatabase()); Download download = dao.findByHash(hash); - if (download != null) { - String message = String.format("Job: Anonymously downloading file '%s' (hash %s)", download.getName(), - hash); - log.info(message); - } - + String message = String.format("Job: Anonymously downloading file '%s' (hash %s)", filename, hash); + log.info(message); return downloadService.download(download); } diff --git a/src/main/java/cloudgene/mapred/server/controller/LogController.java b/src/main/java/cloudgene/mapred/server/controller/LogController.java index 6b5f20ea..700f3668 100644 --- a/src/main/java/cloudgene/mapred/server/controller/LogController.java +++ b/src/main/java/cloudgene/mapred/server/controller/LogController.java @@ -1,16 +1,17 @@ package cloudgene.mapred.server.controller; +import java.io.IOException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import cloudgene.mapred.core.User; import cloudgene.mapred.jobs.AbstractJob; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.server.Application; import cloudgene.mapred.server.auth.AuthenticationService; import cloudgene.mapred.server.auth.AuthenticationType; import cloudgene.mapred.server.services.JobService; -import cloudgene.mapred.util.Settings; -import genepi.io.FileUtil; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.Controller; import io.micronaut.http.annotation.Get; @@ -24,7 +25,7 @@ public class LogController { private static Logger log = LoggerFactory.getLogger(LogController.class); - + @Inject protected Application application; @@ -34,25 +35,21 @@ public class LogController { @Inject protected JobService jobService; + @Inject + protected WorkspaceFactory workspaceFactory; + @Get("/logs/{id}") @Secured(SecurityRule.IS_AUTHENTICATED) @Produces(MediaType.TEXT_PLAIN) - public String getByJobs(Authentication authentication, String id) { + public String getByJob(Authentication authentication, String id) throws IOException { User user = authenticationService.getUserByAuthentication(authentication, AuthenticationType.ALL_TOKENS); AbstractJob job = jobService.getByIdAndUser(id, user); - Settings settings = application.getSettings(); - // log file - String logFilename = FileUtil.path(settings.getLocalWorkspace(), job.getId(), "job.txt"); - String logContent = FileUtil.readFileAsString(logFilename); - - // std out - String outputFilename = FileUtil.path(settings.getLocalWorkspace(), job.getId(), "std.out"); - String outputContent = FileUtil.readFileAsString(outputFilename); - + String logContent = jobService.getJobLog(job, AbstractJob.JOB_LOG); + String outputContent = jobService.getJobLog(job, AbstractJob.JOB_OUT); + StringBuffer buffer = new StringBuffer(); - if (!logContent.isEmpty()) { buffer.append("job.txt:\n\n"); buffer.append(logContent); @@ -62,14 +59,13 @@ public String getByJobs(Authentication authentication, String id) { buffer.append("\n\nstd.out:\n\n"); buffer.append(outputContent); } - - + String message = String.format("Job: viewing logs for job ID %s", job.getId()); if (user.isAdmin()) { message += String.format(" (by ADMIN user ID %s - email %s)", user.getId(), user.getMail()); } log.info(message); - + return buffer.toString(); } diff --git a/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java b/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java index 4537f226..96620006 100644 --- a/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java +++ b/src/main/java/cloudgene/mapred/server/controller/ServerAdminController.java @@ -10,6 +10,7 @@ import cloudgene.mapred.database.CounterHistoryDao; import cloudgene.mapred.server.Application; import cloudgene.mapred.server.auth.AuthenticationService; +import cloudgene.mapred.server.responses.NextflowConfigResponse; import cloudgene.mapred.server.responses.ServerResponse; import cloudgene.mapred.server.responses.StatisticsResponse; import cloudgene.mapred.server.services.ServerService; @@ -103,17 +104,35 @@ public ServerResponse getSettings() { } @Post("/settings/update") - public ServerResponse updateSettings(String name, String backgroundColor, String foregroundColor, - String googleAnalytics, boolean mail, String mailSmtp, String mailUser, String mailPassword, - String mailPort, String mailName) { + public ServerResponse updateSettings(String name, String adminName, String adminMail, String serverUrl, + String backgroundColor, String foregroundColor, String googleAnalytics, boolean mail, String mailSmtp, + String mailUser, String mailPassword, String mailPort, String mailName, String workspaceType, + String workspaceLocation) { - serverService.updateSettings(name, backgroundColor, foregroundColor, googleAnalytics, String.valueOf(mail), - mailSmtp, mailPort, mailUser, mailPassword, mailName); + serverService.updateSettings(name, adminName, adminMail, serverUrl, backgroundColor, foregroundColor, + googleAnalytics, String.valueOf(mail), mailSmtp, mailPort, mailUser, mailPassword, mailName, + workspaceType, workspaceLocation); return ServerResponse.build(application.getSettings()); } + @Get("/nextflow/config") + public NextflowConfigResponse getNextflowConfig() { + + return NextflowConfigResponse.build(application.getSettings()); + + } + + @Post("/nextflow/config/update") + public NextflowConfigResponse updateNextflowConfig(String content) { + + serverService.updateNextflowConfig(content); + + return NextflowConfigResponse.build(application.getSettings()); + + } + @Get("/cloudgene-apps") public String list() throws IOException { URL url = new URL(CLOUDGENE_APPS_ENDPOINT); diff --git a/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java b/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java index 790d68ef..c9cd5ff7 100644 --- a/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/ApplicationResponse.java @@ -6,7 +6,6 @@ import java.util.Vector; import cloudgene.mapred.apps.Application; -import cloudgene.mapred.apps.ApplicationInstaller; import cloudgene.mapred.apps.ApplicationRepository; import cloudgene.mapred.jobs.Environment; import cloudgene.mapred.util.Settings; @@ -33,9 +32,7 @@ public class ApplicationResponse { private String source = ""; - private String state = ""; - - private Map environment; + private List environment; private Map config; @@ -49,6 +46,7 @@ public static ApplicationResponse build(Application app) { appResponse.setErrorMessage(app.getErrorMessage()); appResponse.setChanged(app.isChanged()); appResponse.setPermission(app.getPermission()); + // TODO: check if we need wdl app and file? only in details? appResponse.setWdlApp(app.getWdlApp()); if (new File(app.getFilename()).exists()) { @@ -64,9 +62,7 @@ public static ApplicationResponse buildWithDetails(Application app, Settings set ApplicationResponse appResponse = build(app); - appResponse.setState(updateState(app, settings)); - - Map environment = Environment.getApplicationVariables(app.getWdlApp(), settings); + List environment = settings.buildEnvironment().addApplication(app.getWdlApp()).toList(); appResponse.setEnvironment(environment); Map updatedConfig = repository.getConfig(app.getWdlApp()); @@ -80,32 +76,11 @@ public static List buildWithDetails(List appli ApplicationRepository repository) { List response = new Vector(); for (Application app : applications) { - response.add(ApplicationResponse.buildWithDetails(app, settings, repository)); + response.add(ApplicationResponse.build(app)); } return response; } - private static String updateState(Application app, Settings settings) { - WdlApp wdlApp = app.getWdlApp(); - if (wdlApp != null) { - if (wdlApp.needsInstallation()) { - try { - boolean installed = ApplicationInstaller.isInstalled(wdlApp, settings); - if (installed) { - return "completed"; - } else { - return "on demand"; - } - } catch (NoClassDefFoundError e) { - // TODO: handle exception - } - } else { - return "n/a"; - } - } - return ""; - } - public String getId() { return id; } @@ -170,14 +145,6 @@ public void setSource(String source) { this.source = source; } - public String getState() { - return state; - } - - public void setState(String state) { - this.state = state; - } - public WdlApp getWdlApp() { return wdlApp; } @@ -186,11 +153,11 @@ public void setWdlApp(WdlApp wdlApp) { this.wdlApp = wdlApp; } - public Map getEnvironment() { + public List getEnvironment() { return environment; } - public void setEnvironment(Map environment) { + public void setEnvironment(List environment) { this.environment = environment; } diff --git a/src/main/java/cloudgene/mapred/server/responses/DownloadResponse.java b/src/main/java/cloudgene/mapred/server/responses/DownloadResponse.java index 3c6f2da6..65086132 100644 --- a/src/main/java/cloudgene/mapred/server/responses/DownloadResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/DownloadResponse.java @@ -73,10 +73,8 @@ public void setUser(String user) { public static DownloadResponse build(Download param) { DownloadResponse response = new DownloadResponse(); - response.setParameterId(param.getParameterId()); response.setName(param.getName()); response.setHash(param.getHash()); - response.setUser(param.getUsername()); response.setSize(param.getSize()); response.setCount(param.getCount()); response.setPath(param.getPath()); diff --git a/src/main/java/cloudgene/mapred/server/responses/JobResponse.java b/src/main/java/cloudgene/mapred/server/responses/JobResponse.java index 871eecfb..f48bbb1f 100644 --- a/src/main/java/cloudgene/mapred/server/responses/JobResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/JobResponse.java @@ -16,25 +16,18 @@ public class JobResponse { private String application; private String applicationId; - private boolean canceld; - private boolean complete; private long deletedOn; private long endTime; - private long finishedOn; private String id; private String name; private String logs = ""; private int state; private int positionInQueue; private String userAgent; - private int progress; private long startTime; - private long setupStartTime; - private long setupEndTime; private long submittedOn; - private boolean setupComplete; - private boolean setupRunning; private String username; + private long currentTime; @JsonProperty("steps") private List stepResponses; @@ -58,22 +51,6 @@ public void setApplicationId(String applicationId) { this.applicationId = applicationId; } - public boolean isCanceld() { - return canceld; - } - - public void setCanceld(boolean canceld) { - this.canceld = canceld; - } - - public boolean isComplete() { - return complete; - } - - public void setComplete(boolean complete) { - this.complete = complete; - } - public long getDeletedOn() { return deletedOn; } @@ -90,14 +67,6 @@ public void setEndTime(long endTime) { this.endTime = endTime; } - public long getFinishedOn() { - return finishedOn; - } - - public void setFinishedOn(long finishedOn) { - this.finishedOn = finishedOn; - } - public String getId() { return id; } @@ -145,15 +114,7 @@ public String getUserAgent() { public void setUserAgent(String userAgent) { this.userAgent = userAgent; } - - public int getProgress() { - return progress; - } - - public void setProgress(int progress) { - this.progress = progress; - } - + public long getStartTime() { return startTime; } @@ -162,22 +123,6 @@ public void setStartTime(long startTime) { this.startTime = startTime; } - public long getSetupStartTime() { - return setupStartTime; - } - - public void setSetupStartTime(long setupStartTime) { - this.setupStartTime = setupStartTime; - } - - public long getSetupEndTime() { - return setupEndTime; - } - - public void setSetupEndTime(long setupEndTime) { - this.setupEndTime = setupEndTime; - } - public long getSubmittedOn() { return submittedOn; } @@ -186,22 +131,6 @@ public void setSubmittedOn(long submittedOn) { this.submittedOn = submittedOn; } - public boolean isSetupComplete() { - return setupComplete; - } - - public void setSetupComplete(boolean setupComplete) { - this.setupComplete = setupComplete; - } - - public boolean isSetupRunning() { - return setupRunning; - } - - public void setSetupRunning(boolean setupRunning) { - this.setupRunning = setupRunning; - } - public List getParameterOutputResponse() { return parameterOutputResponse; } @@ -225,6 +154,14 @@ public String getLogs() { public void setLogs(String logs) { this.logs = logs; } + + public void setCurrentTime(long currentTime) { + this.currentTime = currentTime; + } + + public long getCurrentTime() { + return currentTime; + } public static JobResponse build(AbstractJob job, User user) { @@ -256,28 +193,19 @@ public static JobResponse build(AbstractJob job, User user) { JobResponse response = new JobResponse(); response.setApplication(job.getApplication()); response.setApplicationId(job.getApplicationId()); - response.setCanceld(job.isCanceld()); - response.setComplete(job.isComplete()); response.setName(job.getName()); response.setId(job.getId()); response.setState(job.getState()); - response.setLogs(job.getLogs()); response.setPositionInQueue(job.getPositionInQueue()); - response.setProgress(job.getProgress()); response.setUserAgent(job.getUserAgent()); - response.setSetupComplete(job.isSetupComplete()); - response.setSetupRunning(job.isRunning()); response.setDeletedOn(job.getDeletedOn()); response.setStartTime(job.getStartTime()); response.setEndTime(job.getEndTime()); - response.setFinishedOn(job.getFinishedOn()); response.setSubmittedOn(job.getSubmittedOn()); - response.setSetupStartTime(job.getSetupStartTime()); - response.setSetupEndTime(job.getSetupEndTime()); List responses = StepResponse.build(job.getSteps()); response.setStepResponses(responses); @@ -292,7 +220,11 @@ public static JobResponse build(AbstractJob job, User user) { response.setLogs(""); } - response.setUsername(user.getUsername()); + if (job.getUser() != null) { + response.setUsername(job.getUser().getUsername()); + } + + response.setCurrentTime(System.currentTimeMillis()); return response; } diff --git a/src/main/java/cloudgene/mapred/server/responses/NextflowConfigResponse.java b/src/main/java/cloudgene/mapred/server/responses/NextflowConfigResponse.java new file mode 100644 index 00000000..85c52a96 --- /dev/null +++ b/src/main/java/cloudgene/mapred/server/responses/NextflowConfigResponse.java @@ -0,0 +1,49 @@ +package cloudgene.mapred.server.responses; + +import java.io.File; +import java.util.List; +import java.util.Vector; + +import com.fasterxml.jackson.annotation.JsonInclude; + +import cloudgene.mapred.jobs.Environment.Variable; +import cloudgene.mapred.util.Settings; +import genepi.io.FileUtil; + +@JsonInclude(JsonInclude.Include.ALWAYS) +public class NextflowConfigResponse { + + private String content = ""; + + private List variables = new Vector(); + + public static NextflowConfigResponse build(Settings settings) { + NextflowConfigResponse response = new NextflowConfigResponse(); + String configFilename = settings.getNextflowConfig(); + File configFile = new File(configFilename); + if (!configFile.exists()) { + return response; + } + + response.setContent(FileUtil.readFileAsString(configFilename)); + response.setVariables(settings.buildEnvironment().toList()); + return response; + } + + public void setContent(String content) { + this.content = content; + } + + public String getContent() { + return content; + } + + public void setVariables(List variables) { + this.variables = variables; + } + + public List getVariables() { + return variables; + } + +} diff --git a/src/main/java/cloudgene/mapred/server/responses/PropertyResponse.java b/src/main/java/cloudgene/mapred/server/responses/PropertyResponse.java index fc3282cb..b190077b 100644 --- a/src/main/java/cloudgene/mapred/server/responses/PropertyResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/PropertyResponse.java @@ -69,12 +69,30 @@ public static List buildWithValues(Map values) { return response; } - public static PropertyResponse build(String key, String value, Map values) { + public static List buildWithValues(List values) { + List response = new Vector(); + + for (Map object : values) { + if (object.containsKey("id") && object.containsKey("name")) { + System.out.println(object); + response.add(PropertyResponse.build(object.get("id").toString(), object.get("name").toString())); + } + } + + return response; + } + + public static PropertyResponse build(String key, String value, Object values) { PropertyResponse response = new PropertyResponse(); response.setKey(key); response.setLabel(value); - List responseWithValues = buildWithValues(values); - response.setValues(responseWithValues); + if (values instanceof Map) { + List responseWithValues = buildWithValues((Map) values); + response.setValues(responseWithValues); + } else if (values instanceof List) { + List responseWithValues = buildWithValues((List) values); + response.setValues(responseWithValues); + } return response; } diff --git a/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java b/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java index b97f4688..ec4efe00 100644 --- a/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/ServerResponse.java @@ -19,13 +19,23 @@ public class ServerResponse { private String mailPassword; private String mailName; private boolean mail; + private String adminName; + private String adminMail; + private String serverUrl; + private String workspaceType; + private String workspaceLocation; public static ServerResponse build(Settings settings) { ServerResponse response = new ServerResponse(); response.setName(settings.getName()); + response.setAdminName(settings.getAdminName()); + response.setAdminMail(settings.getAdminMail()); + response.setServerUrl(settings.getServerUrl()); response.setBackgroundColor(settings.getColors().get("background")); response.setForegroundColor(settings.getColors().get("foreground")); response.setGoogleAnalytics(settings.getGoogleAnalytics()); + response.setWorkspaceType(settings.getExternalWorkspaceType()); + response.setWorkspaceLocation(settings.getExternalWorkspaceLocation()); Map mail = settings.getMail(); if (mail != null) { @@ -34,8 +44,7 @@ public static ServerResponse build(Settings settings) { response.setMailPort(mail.get("port")); response.setMailUser(mail.get("user")); response.setMailPassword(mail.get("password")); - response.setMailUser(""); - response.setMailPassword(""); + response.setMailUser(mail.get("user")); response.setMailName(mail.get("name")); } else { @@ -59,6 +68,30 @@ public void setName(String name) { this.name = name; } + public String getAdminName() { + return adminName; + } + + public void setAdminName(String adminName) { + this.adminName = adminName; + } + + public String getAdminMail() { + return adminMail; + } + + public void setAdminMail(String adminMail) { + this.adminMail = adminMail; + } + + public String getServerUrl() { + return serverUrl; + } + + public void setServerUrl(String serverUrl) { + this.serverUrl = serverUrl; + } + public String getBackgroundColor() { return backgroundColor; } @@ -131,4 +164,20 @@ public void setMail(boolean mail) { this.mail = mail; } + public void setWorkspaceLocation(String workspaceLocation) { + this.workspaceLocation = workspaceLocation; + } + + public String getWorkspaceLocation() { + return workspaceLocation; + } + + public void setWorkspaceType(String workspaceType) { + this.workspaceType = workspaceType; + } + + public String getWorkspaceType() { + return workspaceType; + } + } diff --git a/src/main/java/cloudgene/mapred/server/responses/StepResponse.java b/src/main/java/cloudgene/mapred/server/responses/StepResponse.java index 980f4869..dff58b80 100644 --- a/src/main/java/cloudgene/mapred/server/responses/StepResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/StepResponse.java @@ -12,11 +12,11 @@ public class StepResponse { private int id; - private String name; - private int progress; - @JsonProperty("logMessages") - private List messageResponse; + private String name; + + @JsonProperty("logMessages") + private List messages; public int getId() { return id; @@ -34,22 +34,12 @@ public void setName(String name) { this.name = name; } - public int getProgress() { - return progress; - } - - public void setProgress(int progress) { - this.progress = progress; - } - - public static StepResponse build(CloudgeneStep step) { StepResponse response = new StepResponse(); response.setId(step.getId()); response.setName(step.getName()); - response.setProgress(step.getProgress()); List responses = MessageResponse.build(step.getLogMessages()); - response.setMessageResponse(responses); + response.setMessages(responses); return response; } @@ -61,12 +51,12 @@ public static List build(List steps) { return response; } - public List getMessageResponse() { - return messageResponse; + public List getMessages() { + return messages; } - public void setMessageResponse(List messageResponse) { - this.messageResponse = messageResponse; + public void setMessages(List messages) { + this.messages = messages; } } diff --git a/src/main/java/cloudgene/mapred/server/responses/WdlParameterInputResponse.java b/src/main/java/cloudgene/mapred/server/responses/WdlParameterInputResponse.java index a1375320..5402635f 100644 --- a/src/main/java/cloudgene/mapred/server/responses/WdlParameterInputResponse.java +++ b/src/main/java/cloudgene/mapred/server/responses/WdlParameterInputResponse.java @@ -136,7 +136,7 @@ public String getEmptySelection() { public void setEmptySelection(String emptySelection) { this.emptySelection = emptySelection; } - + public List getValues() { return values; } @@ -169,7 +169,6 @@ public void setLabel(String label) { this.label = label; } - public static WdlParameterInputResponse build(WdlParameterInput input, List apps) { WdlParameterInputResponse response = new WdlParameterInputResponse(); response.setId(input.getId()); @@ -211,17 +210,16 @@ public static WdlParameterInputResponse build(WdlParameterInput input, List propertyResponses = new ArrayList(); for (WdlApp app : apps) { - if (category != null && !category.isEmpty()) { - if (app.getCategory() != null && app.getCategory().equals(category)) { - Map values = (Map) app.getProperties().get(property); - PropertyResponse propertyResponse = PropertyResponse.build("apps@" + app.getId(), app.getName(), - values); - propertyResponses.add(propertyResponse); - } - - } else { - // TODO:!! + if (category == null || category.isEmpty()) { + continue; } + if (app.getCategory() == null || !app.getCategory().equals(category)) { + continue; + } + Object values = app.getProperties().get(property); + PropertyResponse propertyResponse = PropertyResponse.build("apps@" + app.getId(), app.getName(), + values); + propertyResponses.add(propertyResponse); } response.setValues(propertyResponses); @@ -275,7 +273,7 @@ public static List build(List inpu for (WdlParameterInput input : inputs) { if (input.isVisible()) { - response.add(WdlParameterInputResponse.build(input, apps)); + response.add(WdlParameterInputResponse.build(input, apps)); } } return response; diff --git a/src/main/java/cloudgene/mapred/server/services/ApplicationService.java b/src/main/java/cloudgene/mapred/server/services/ApplicationService.java index aacba423..5235ae24 100644 --- a/src/main/java/cloudgene/mapred/server/services/ApplicationService.java +++ b/src/main/java/cloudgene/mapred/server/services/ApplicationService.java @@ -1,6 +1,5 @@ package cloudgene.mapred.server.services; -import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; @@ -10,11 +9,8 @@ import org.slf4j.LoggerFactory; import cloudgene.mapred.apps.Application; -import cloudgene.mapred.apps.ApplicationInstaller; import cloudgene.mapred.apps.ApplicationRepository; import cloudgene.mapred.core.User; -import cloudgene.mapred.plugins.PluginManager; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; import cloudgene.mapred.server.exceptions.JsonHttpStatusException; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; @@ -79,14 +75,6 @@ public void checkRequirements(Application app) { String.format(APPLICATION_IS_DATA_PACKAGE, app.getId())); } - if (wdlApp.getWorkflow().hasHdfsInputs()) { - - PluginManager manager = PluginManager.getInstance(); - if (!manager.isEnabled(HadoopPlugin.ID)) { - throw new JsonHttpStatusException(HttpStatus.SERVICE_UNAVAILABLE, HADOOP_CLUSTER_UNREACHABLE); - } - } - } public Application removeApp(String appId) { @@ -151,29 +139,6 @@ public void updateConfig(Application app, Map config) { } - public void reinstallApp(Application app, boolean reinstall) { - - WdlApp wdlApp = app.getWdlApp(); - - if (!reinstall) { - return; - } - - boolean installed = ApplicationInstaller.isInstalled(wdlApp, this.application.getSettings()); - if (!installed) { - return; - } - - try { - ApplicationInstaller.uninstall(wdlApp, this.application.getSettings()); - } catch (IOException e) { - e.printStackTrace(); - throw new JsonHttpStatusException(HttpStatus.BAD_REQUEST, - String.format(APPLICATION_NOT_UPDATED, e.getMessage())); - } - - } - public List listApps(boolean reload) { ApplicationRepository repository = application.getSettings().getApplicationRepository(); diff --git a/src/main/java/cloudgene/mapred/server/services/DownloadService.java b/src/main/java/cloudgene/mapred/server/services/DownloadService.java index fd4dc4b2..ae3f877c 100644 --- a/src/main/java/cloudgene/mapred/server/services/DownloadService.java +++ b/src/main/java/cloudgene/mapred/server/services/DownloadService.java @@ -1,18 +1,19 @@ package cloudgene.mapred.server.services; -import java.io.File; +import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import cloudgene.mapred.database.DownloadDao; import cloudgene.mapred.jobs.Download; -import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; +import cloudgene.mapred.jobs.workspace.IWorkspace; import cloudgene.mapred.server.Application; import cloudgene.mapred.server.exceptions.JsonHttpStatusException; -import cloudgene.sdk.internal.IExternalWorkspace; -import genepi.io.FileUtil; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; +import io.micronaut.http.MutableHttpResponse; import jakarta.inject.Inject; public class DownloadService { @@ -20,7 +21,10 @@ public class DownloadService { @Inject protected Application application; - public HttpResponse download(Download download) throws URISyntaxException { + @Inject + protected WorkspaceFactory workspaceFactory; + + public MutableHttpResponse download(Download download) throws URISyntaxException, IOException { DownloadDao dao = new DownloadDao(application.getDatabase()); @@ -37,20 +41,17 @@ public HttpResponse download(Download download) throws URISyntaxException download.decCount(); dao.update(download); } - - IExternalWorkspace externalWorkspace = ExternalWorkspaceFactory.get(download.getPath()); - if (externalWorkspace != null) { - // external workspace found, use link method and create redirect response - String publicUrl = externalWorkspace.createPublicLink(download.getPath()); + + IWorkspace workspace = workspaceFactory.getByUrl(download.getPath()); + + // external workspace found, use link method and create redirect response + String publicUrl = workspace.createPublicLink(download.getPath()); + if (publicUrl != null) { URI location = new URI(publicUrl); return HttpResponse.redirect(location); } else { - // no external workspace found, use local workspace - String localWorkspace = application.getSettings().getLocalWorkspace(); - String resultFile = FileUtil.path(localWorkspace, download.getPath()); - return HttpResponse.ok(new File(resultFile)); + return HttpResponse.ok(workspace.download(download.getPath())); } - } } diff --git a/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java b/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java index 0158ef98..6f0a322a 100644 --- a/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java +++ b/src/main/java/cloudgene/mapred/server/services/JobCleanUpService.java @@ -9,12 +9,11 @@ import cloudgene.mapred.database.JobDao; import cloudgene.mapred.database.util.Database; import cloudgene.mapred.jobs.AbstractJob; -import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; +import cloudgene.mapred.jobs.workspace.IWorkspace; import cloudgene.mapred.server.Application; import cloudgene.mapred.util.MailUtil; import cloudgene.mapred.util.Settings; -import cloudgene.sdk.internal.IExternalWorkspace; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -27,6 +26,9 @@ public class JobCleanUpService { @Inject protected Application application; + @Inject + protected WorkspaceFactory workspaceFactory; + public int executeRetire() { Database database = application.getDatabase(); @@ -38,39 +40,26 @@ public int executeRetire() { int deleted = 0; - IExternalWorkspace externalWorkspace = null; - if (!settings.getExternalWorkspaceLocation().isEmpty()) { - String externalOutput = settings.getExternalWorkspaceLocation(); - externalWorkspace = ExternalWorkspaceFactory.get(settings.getExternalWorkspaceType(), externalOutput); - } - for (AbstractJob job : oldJobs) { if (job.getDeletedOn() < System.currentTimeMillis()) { - // delete local directory and hdfs directory + // delete local directory String localOutput = FileUtil.path(settings.getLocalWorkspace(), job.getId()); FileUtil.deleteDirectory(localOutput); - try { - String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(settings.getHdfsWorkspace(), job.getId())); - HdfsUtil.delete(hdfsOutput); - } catch (NoClassDefFoundError e) { - // TODO: handle exception - } - job.setState(AbstractJob.STATE_RETIRED); dao.update(job); log.info("Job " + job.getId() + " retired."); deleted++; - if (externalWorkspace != null) { - try { - externalWorkspace.delete(job.getId()); - } catch (Exception e) { - log.error("Retire " + job.getId() + " failed.", e); - } + IWorkspace externalWorkspace = workspaceFactory.getByJob(job); + + try { + externalWorkspace.delete(job.getId()); + } catch (Exception e) { + log.error("Retire " + job.getId() + " failed.", e); } } diff --git a/src/main/java/cloudgene/mapred/server/services/JobService.java b/src/main/java/cloudgene/mapred/server/services/JobService.java index b8c4266a..6ecc2fea 100644 --- a/src/main/java/cloudgene/mapred/server/services/JobService.java +++ b/src/main/java/cloudgene/mapred/server/services/JobService.java @@ -1,6 +1,7 @@ package cloudgene.mapred.server.services; import java.io.File; +import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashMap; @@ -9,6 +10,8 @@ import java.util.Vector; import org.apache.commons.lang.StringEscapeUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import cloudgene.mapred.apps.ApplicationRepository; import cloudgene.mapred.core.User; @@ -19,7 +22,8 @@ import cloudgene.mapred.jobs.CloudgeneParameterOutput; import cloudgene.mapred.jobs.Download; import cloudgene.mapred.jobs.WorkflowEngine; -import cloudgene.mapred.jobs.workspace.ExternalWorkspaceFactory; +import cloudgene.mapred.jobs.workspace.IWorkspace; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.server.Application; import cloudgene.mapred.server.exceptions.JsonHttpStatusException; import cloudgene.mapred.util.FormUtil.Parameter; @@ -28,9 +32,6 @@ import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlParameterInput; import cloudgene.mapred.wdl.WdlParameterInputType; -import cloudgene.sdk.internal.IExternalWorkspace; -import genepi.hadoop.HdfsUtil; -import genepi.hadoop.importer.ImporterFactory; import genepi.io.FileUtil; import io.micronaut.http.HttpStatus; import jakarta.inject.Inject; @@ -39,9 +40,14 @@ @Singleton public class JobService { + private static final Logger log = LoggerFactory.getLogger(JobService.class); + @Inject protected Application application; + @Inject + protected WorkspaceFactory workspaceFactory; + private static final String PARAM_JOB_NAME = "job-name"; public AbstractJob getById(String id) { @@ -113,20 +119,19 @@ public AbstractJob submitJob(String appId, List form, User user) { String id = createId(); - String hdfsWorkspace = ""; + Map inputParams = null; + + IWorkspace workspace = workspaceFactory.getDefault(); + try { - hdfsWorkspace = HdfsUtil.path(settings.getHdfsWorkspace(), id); - } catch (NoClassDefFoundError e) { - // Ignore HDFS exceptions to work also without Hadoop; - } - String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), id); - FileUtil.createDirectory(localWorkspace); + // setup workspace + workspace.setJob(id); + workspace.setup(); - Map inputParams = null; + // parse input params + inputParams = parseAndUpdateInputParams(form, app, workspace); - try { - inputParams = parseAndUpdateInputParams(form, app, hdfsWorkspace, localWorkspace); } catch (Exception e) { throw new JsonHttpStatusException(HttpStatus.BAD_REQUEST, e.getMessage()); } @@ -137,13 +142,16 @@ public AbstractJob submitJob(String appId, List form, User user) { name = jobName; } + // TODO: remove and solve via workspace! + String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), id); + FileUtil.createDirectory(localWorkspace); + CloudgeneJob job = new CloudgeneJob(user, id, app, inputParams); job.setId(id); job.setName(name); job.setLocalWorkspace(localWorkspace); - job.setHdfsWorkspace(hdfsWorkspace); + job.setWorkspace(workspace); job.setSettings(settings); - job.setRemoveHdfsWorkspace(settings.isRemoveHdfsWorkspace()); job.setApplication(app.getName() + " " + app.getVersion()); job.setApplicationId(appId); @@ -211,17 +219,10 @@ public Page getAllByUserAndPage(User user, Integer page, int pageSi public AbstractJob delete(AbstractJob job) { Settings settings = application.getSettings(); - // delete local directory and hdfs directory + // delete local directory String localOutput = FileUtil.path(settings.getLocalWorkspace(), job.getId()); FileUtil.deleteDirectory(localOutput); - try { - String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(settings.getHdfsWorkspace(), job.getId())); - HdfsUtil.delete(hdfsOutput); - } catch (NoClassDefFoundError e) { - // ignore HDFS exception to work also without hadoop. - } - // delete job from database job.setState(AbstractJob.STATE_DELETED); @@ -229,16 +230,14 @@ public AbstractJob delete(AbstractJob job) { dao.update(job); // delete all results that are stored on external workspaces - IExternalWorkspace externalWorkspace = null; - if (!settings.getExternalWorkspaceLocation().isEmpty()) { - String externalOutput = settings.getExternalWorkspaceLocation(); - externalWorkspace = ExternalWorkspaceFactory.get(settings.getExternalWorkspaceType(), externalOutput); - try { - externalWorkspace.delete(job.getId()); - } catch (Exception e) { - e.printStackTrace(); - } + + IWorkspace workspace = workspaceFactory.getByJob(job); + try { + workspace.delete(job.getId()); + } catch (Exception e) { + log.error("Deleteting " + job.getId() + " form workspace failed.", e); } + return job; } @@ -255,18 +254,10 @@ public AbstractJob restart(AbstractJob job) { throw new JsonHttpStatusException(HttpStatus.BAD_REQUEST, "Job " + job.getId() + " is not pending."); } - String hdfsWorkspace = ""; - try { - hdfsWorkspace = HdfsUtil.path(settings.getHdfsWorkspace(), job.getId()); - } catch (NoClassDefFoundError e) { - // ignore HDFS exception to work also without hadoop. - } String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), job.getId()); job.setLocalWorkspace(localWorkspace); - job.setHdfsWorkspace(hdfsWorkspace); job.setSettings(settings); - job.setRemoveHdfsWorkspace(settings.isRemoveHdfsWorkspace()); String appId = job.getApplicationId(); @@ -277,7 +268,18 @@ public AbstractJob restart(AbstractJob job) { } - ((CloudgeneJob) job).loadConfig(application.getWdlApp()); + IWorkspace workspace = workspaceFactory.getDefault(); + + try { + // setup workspace + workspace.setJob(job.getId()); + workspace.setup(); + } catch (Exception e) { + throw new JsonHttpStatusException(HttpStatus.BAD_REQUEST, e.getMessage()); + } + job.setWorkspace(workspace); + + ((CloudgeneJob) job).loadApp(application.getWdlApp()); this.application.getWorkflowEngine().restart(job); @@ -323,29 +325,19 @@ public String archive(AbstractJob job) { try { - IExternalWorkspace externalWorkspace = null; - if (!settings.getExternalWorkspaceLocation().isEmpty()) { - String externalOutput = settings.getExternalWorkspaceLocation(); - externalWorkspace = ExternalWorkspaceFactory.get(settings.getExternalWorkspaceType(), externalOutput); - } - // delete local directory and hdfs directory String localOutput = FileUtil.path(settings.getLocalWorkspace(), job.getId()); FileUtil.deleteDirectory(localOutput); - try { - String hdfsOutput = HdfsUtil.makeAbsolute(HdfsUtil.path(settings.getHdfsWorkspace(), job.getId())); - HdfsUtil.delete(hdfsOutput); - } catch (NoClassDefFoundError e) { - } + job.setState(AbstractJob.STATE_RETIRED); dao.update(job); - if (externalWorkspace != null) { - try { - externalWorkspace.delete(job.getId()); - } catch (Exception e) { - e.printStackTrace(); - } + IWorkspace workspace = workspaceFactory.getByJob(job); + + try { + workspace.delete(job.getId()); + } catch (Exception e) { + log.error("Deleteting " + job.getId() + " from workspace failed.", e); } return "Retired job " + job.getId(); @@ -388,8 +380,8 @@ public String createId() { // TODO: refactore and combine this method with CommandLineUtil.parseArgs... - private Map parseAndUpdateInputParams(List form, WdlApp app, String hdfsWorkspace, - String localWorkspace) throws Exception { + private Map parseAndUpdateInputParams(List form, WdlApp app, IWorkspace workspace) + throws Exception { Map props = new HashMap(); Map params = new HashMap(); @@ -406,8 +398,6 @@ private Map parseAndUpdateInputParams(List form, WdlA try { - String entryName = inputFile.getName(); - // remove upload indentification! String fieldName = name.replace("-upload", "").replace("input-", ""); @@ -420,43 +410,18 @@ private Map parseAndUpdateInputParams(List form, WdlA if (inputParam.isHdfs()) { - String targetPath = HdfsUtil.path(hdfsWorkspace, fieldName); - - String target = HdfsUtil.path(targetPath, entryName); + throw new Exception("HDFS support was removed in Cloudgene 3"); - HdfsUtil.put(inputFile.getAbsolutePath(), target); + } - if (inputParam.isFolder()) { - // folder - props.put(fieldName, HdfsUtil.makeAbsolute(HdfsUtil.path(hdfsWorkspace, fieldName))); - } else { - // file - props.put(fieldName, - HdfsUtil.makeAbsolute(HdfsUtil.path(hdfsWorkspace, fieldName, entryName))); - } + // copy to workspace in input directory + String target = workspace.uploadInput(fieldName, inputFile); + if (inputParam.isFolder()) { + props.put(fieldName, workspace.getParent(target)); } else { - - // copy to workspace in input directory - String targetPath = FileUtil.path(localWorkspace, "input", fieldName); - FileUtil.createDirectory(targetPath); - - String target = FileUtil.path(targetPath, entryName); - - FileUtil.copy(inputFile.getAbsolutePath(), target); - - if (inputParam.isFolder()) { - // folder - if (inputParam.getPattern() != null && !inputParam.getPattern().isEmpty()) { - props.put(fieldName, new File(targetPath).getAbsolutePath()); - } else { - props.put(fieldName, new File(targetPath).getAbsolutePath()); - } - } else { - // file - props.put(fieldName, new File(target).getAbsolutePath()); - } - + // file + props.put(fieldName, target); } // deletes temporary file @@ -482,7 +447,7 @@ private Map parseAndUpdateInputParams(List form, WdlA String cleanedValue = StringEscapeUtils.escapeHtml(value.toString()); - if (input != null && input.isFileOrFolder() && ImporterFactory.needsImport(cleanedValue)) { + if (input != null && input.isFileOrFolder() && needsImport(cleanedValue)) { throw new Exception("Parameter '" + input.getId() + "': URL-based uploads are no longer supported. Please use direct file uploads instead."); } @@ -559,7 +524,8 @@ public List getJobs(String state) { case "running-stq": - jobs = engine.getAllJobsInShortTimeQueue(); + // TODO: remove! + jobs = new Vector(); break; case "current": @@ -584,4 +550,19 @@ public List getJobs(String state) { return jobs; } + public String getJobLog(AbstractJob job, String name) throws IOException { + if (job.isRunning()) { + // files are locally when job is running + return job.getLog(name); + } else { + IWorkspace workspace = workspaceFactory.getByJob(job); + return workspace.downloadLog(name); + } + } + + public boolean needsImport(String url) { + return url.startsWith("sftp://") || url.startsWith("http://") || url.startsWith("https://") + || url.startsWith("ftp://") || url.startsWith("s3://"); + } + } diff --git a/src/main/java/cloudgene/mapred/server/services/ServerService.java b/src/main/java/cloudgene/mapred/server/services/ServerService.java index 0d0c0210..9af643c2 100644 --- a/src/main/java/cloudgene/mapred/server/services/ServerService.java +++ b/src/main/java/cloudgene/mapred/server/services/ServerService.java @@ -24,6 +24,7 @@ import cloudgene.mapred.server.controller.ServerAdminController; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; +import genepi.io.FileUtil; import io.micronaut.security.oauth2.configuration.OauthClientConfigurationProperties; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -115,15 +116,20 @@ public String getRoot(User user) { return data.toString(); } - public void updateSettings(String name, String background_color, String foreground_color, String google_analytics, - String mail, String mail_smtp, String mail_port, String mail_user, String mail_password, String mail_name) { + public void updateSettings(String name, String adminName, String adminMail, String serverUrl, String background_color, String foreground_color, String google_analytics, + String mail, String mail_smtp, String mail_port, String mail_user, String mail_password, String mail_name, String workspaceType, String workspaceLocation) { Settings settings = application.getSettings(); settings.setName(name); + settings.setAdminName(adminName); + settings.setAdminMail(adminMail); + settings.setServerUrl(serverUrl); settings.getColors().put("background", background_color); settings.getColors().put("foreground", foreground_color); settings.setGoogleAnalytics(google_analytics); - + settings.getExternalWorkspace().put("type", workspaceType); + settings.getExternalWorkspace().put("location", workspaceLocation); + if (mail != null && mail.equals("true")) { Map mailConfig = new HashMap(); mailConfig.put("smtp", mail_smtp); @@ -257,4 +263,10 @@ public String tail(File file, int lines) { } } + public void updateNextflowConfig(String content) { + Settings settings = application.getSettings(); + String filename = settings.getNextflowConfig(); + FileUtil.writeStringBufferToFile(filename, new StringBuffer(content)); + } + } diff --git a/src/main/java/cloudgene/mapred/server/tasks/ServerTasks.java b/src/main/java/cloudgene/mapred/server/tasks/ServerTasks.java index 073175a0..e2f6c76a 100644 --- a/src/main/java/cloudgene/mapred/server/tasks/ServerTasks.java +++ b/src/main/java/cloudgene/mapred/server/tasks/ServerTasks.java @@ -10,8 +10,6 @@ import cloudgene.mapred.jobs.AbstractJob; import cloudgene.mapred.jobs.WorkflowEngine; import cloudgene.mapred.server.Application; -import cloudgene.mapred.util.MailUtil; -import genepi.hadoop.HadoopUtil; import io.micronaut.scheduling.annotation.Scheduled; import jakarta.inject.Inject; import jakarta.inject.Singleton; @@ -22,47 +20,6 @@ public class ServerTasks { @Inject protected Application application; - @Scheduled(fixedDelay = "1m") - public void checkHadoopCluster() { - - // check namenode state - - try { - boolean safemode = HadoopUtil.getInstance().isInSafeMode(); - - if (safemode) { - - if (application.getWorkflowEngine().isRunning()) { - - try { - - MailUtil.notifyAdmin(application.getSettings(), - "[" + application.getSettings().getName() + "] Problems with your Hadoop cluster", - "Hi,\n\n" + "This is a notification sent by Cloudgene.\n\n" - + "Your Hadoop cluster is in Safemode. " - + "Don't worry, we blocked the queue for you!"); - } catch (Exception e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - application.getWorkflowEngine().block(); - - } - - } - } catch (NoClassDefFoundError e) { - // TODO: handle exception - } - - // if (cluster.getJobTrackerStatus() == ) - - // block queue if tasktracker is in safemode - - // send notification to admins - - } - @Scheduled(fixedDelay = "5m") public void writeStatistics() { diff --git a/src/main/java/cloudgene/mapred/steps/BashCommandStep.java b/src/main/java/cloudgene/mapred/steps/BashCommandStep.java index c725516a..8b13e47d 100644 --- a/src/main/java/cloudgene/mapred/steps/BashCommandStep.java +++ b/src/main/java/cloudgene/mapred/steps/BashCommandStep.java @@ -1,7 +1,6 @@ package cloudgene.mapred.steps; import java.io.File; -import java.io.IOException; import java.util.List; import java.util.Vector; @@ -9,17 +8,15 @@ import cloudgene.mapred.jobs.CloudgeneStep; import cloudgene.mapred.jobs.Message; import cloudgene.mapred.wdl.WdlStep; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; public class BashCommandStep extends CloudgeneStep { @Override public boolean run(WdlStep step, CloudgeneContext context) { - String cmd = step.get("exec"); + String cmd = step.getString("exec"); if (cmd == null) { - cmd = step.get("cmd"); + cmd = step.getString("cmd"); } if (cmd == null) { @@ -30,8 +27,8 @@ public boolean run(WdlStep step, CloudgeneContext context) { context.error("'exec' or 'cmd' parameter cannot be an empty string."); } - String bash = step.get("bash", "false"); - String stdout = step.get("stdout", "false"); + String bash = step.getString("bash", "false"); + String stdout = step.getString("stdout", "false"); boolean useBash = bash.equals("true"); boolean streamStdout = stdout.equals("true"); @@ -62,35 +59,10 @@ public boolean run(WdlStep step, CloudgeneContext context) { String bashCommand = ""; for (String param : params) { - // checkout hdfs file - if (param.startsWith("hdfs://")) { - String name = FileUtil.getFilename(param); - String localFile = FileUtil.path(((CloudgeneContext) context).getLocalTemp(), "local_" + name); - try { - HdfsUtil.checkOut(param, localFile); - String localFilename = new File(localFile).getAbsolutePath(); - if (useBash) { - bashCommand += localFilename + " "; - } else { - command.add(localFilename); - } - - } catch (IOException e) { - context.log(e.getMessage()); - if (useBash) { - bashCommand += param + " "; - } else { - command.add(param); - } - } - + if (useBash) { + bashCommand += param + " "; } else { - if (useBash) { - bashCommand += param + " "; - } else { - command.add(param); - } - + command.add(param); } } @@ -117,7 +89,9 @@ public boolean run(WdlStep step, CloudgeneContext context) { if (streamStdout) { context.endTask(output.toString(), Message.ERROR); } else { - context.endTask("Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", Message.ERROR); + context.endTask( + "Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", + Message.ERROR); } return false; } diff --git a/src/main/java/cloudgene/mapred/steps/GroovyStep.java b/src/main/java/cloudgene/mapred/steps/GroovyStep.java index 9e0ec43a..6574c84f 100644 --- a/src/main/java/cloudgene/mapred/steps/GroovyStep.java +++ b/src/main/java/cloudgene/mapred/steps/GroovyStep.java @@ -4,19 +4,26 @@ import java.io.StringWriter; import java.lang.reflect.Method; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.CloudgeneStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; import cloudgene.mapred.wdl.WdlStep; -import cloudgene.sdk.internal.WorkflowContext; import genepi.io.FileUtil; import groovy.util.GroovyScriptEngine; + public class GroovyStep extends CloudgeneStep { + private static final Logger log = LoggerFactory.getLogger(GroovyStep.class); + + @Override public boolean run(WdlStep step, CloudgeneContext context) { context.setConfig(step); - String script = step.get("script"); + String script = step.getString("script"); String workingDirectory = context.getWorkingDirectory(); String filename = FileUtil.path(workingDirectory, script); @@ -26,33 +33,20 @@ public boolean run(WdlStep step, CloudgeneContext context) { Class scriptClass = new GroovyScriptEngine(".", getClass().getClassLoader()).loadScriptByName(filename); Object scriptInstance = scriptClass.newInstance(); - try { - Method method = scriptClass.getDeclaredMethod("run", new Class[] { WorkflowContext.class }); - Object result = method.invoke(scriptInstance, new Object[] { context }); - if (result instanceof Boolean) { - return (Boolean) result; - } else { - return true; - } - } catch (NoSuchMethodException e) { - - // old dependency to genepi.hadoop - Method method = scriptClass.getDeclaredMethod("run", - new Class[] { genepi.hadoop.common.WorkflowContext.class }); - Object result = method.invoke(scriptInstance, - new Object[] { JavaInternalStepDeprecrated.adapter(context) }); - if (result instanceof Boolean) { - return (Boolean) result; - } else { - return true; - } - + Method method = scriptClass.getDeclaredMethod("run", new Class[] { WorkflowContext.class }); + Object result = method.invoke(scriptInstance, new Object[] { context }); + if (result instanceof Boolean) { + return (Boolean) result; + } else { + return true; } } catch (Exception e) { if (e.getCause() != null) { + log.error("[Job {}] Step '{}': Error in script '{}'", context.getJobId(), step.getName(), script, e.getCause()); context.error("Error in script " + script + ":\n" + getStackTraceAsString(e.getCause())); } else { + log.error("[Job {}] Step '{}': Error in script '{}'", context.getJobId(), step.getName(), script, e); context.error("Error in script " + script + ":\n" + getStackTraceAsString(e)); } return false; @@ -61,9 +55,9 @@ public boolean run(WdlStep step, CloudgeneContext context) { } public static String getStackTraceAsString(Throwable throwable) { - StringWriter stringWriter = new StringWriter(); - throwable.printStackTrace(new PrintWriter(stringWriter)); - return stringWriter.toString(); - } - + StringWriter stringWriter = new StringWriter(); + throwable.printStackTrace(new PrintWriter(stringWriter)); + return stringWriter.toString(); + } + } diff --git a/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java b/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java deleted file mode 100644 index d2e45e0b..00000000 --- a/src/main/java/cloudgene/mapred/steps/HadoopMapReduceStep.java +++ /dev/null @@ -1,281 +0,0 @@ -package cloudgene.mapred.steps; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.List; -import java.util.Vector; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.jobs.CloudgeneStep; -import cloudgene.mapred.jobs.Message; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; -import cloudgene.mapred.wdl.WdlStep; -import cloudgene.sdk.internal.WorkflowContext; -import genepi.hadoop.HadoopUtil; -import genepi.io.FileUtil; - -public class HadoopMapReduceStep extends CloudgeneStep { - - private String jobId; - - private int map = 0; - - private int reduce = 0; - - @Override - public boolean run(WdlStep step, CloudgeneContext context) { - - String hadoopPath = context.getSettings().getHadoopPath(); - - File path = new File(hadoopPath); - - if (!path.exists()) { - context.error("Hadoop Binary was not found. Please set the correct path in the admin panel."); - return false; - } - - String hadoop = ""; - - if (path.isDirectory()) { - hadoop = FileUtil.path(hadoopPath, "bin", "hadoop"); - } else { - hadoop = hadoopPath; - } - - File file = new File(hadoop); - - if (!file.exists()) { - context.error("Hadoop Binary was not found. Please set the correct path in the admin panel."); - return false; - } - - if (!file.canExecute()) { - context.error( - "Hadoop Binary was found (" + hadoop + ") but can not be executed. Please check the permissions."); - return false; - } - - String streamingJar = context.getSettings().getStreamingJar(); - - // params - String paramsString = step.get("params"); - String[] params = new String[] {}; - if (paramsString != null) { - params = paramsString.split(" "); - } - - // hadoop jar or streaming - List command = new Vector(); - - command.add(hadoop); - - // -fs and -jt and -config are supported by generic tool runner. - - command.add("jar"); - - if (step.getJar() != null) { - - // classical - command.add(step.getJar()); - - } else { - - // streaming - - if (context.getSettings().isStreaming()) { - - command.add(streamingJar); - - } else { - - context.error( - "Streaming mode is disabled.\nPlease specify the streaming-jar file in config/settings.yaml to run this job.."); - return false; - - } - - } - - for (String tile : params) { - command.add(tile.trim()); - } - - // mapper and reducer - - if (step.getJar() == null) { - - if (step.get("mapper") != null) { - - String tiles[] = step.get("mapper").split(" ", 2); - String filename = tiles[0]; - - command.add("-mapper"); - - if (tiles.length > 1) { - String params2 = tiles[1]; - command.add(filename + " " + params2); - } else { - command.add(filename); - } - - } - - if (step.get("reducer") != null) { - - String tiles[] = step.get("reducer").split(" ", 2); - String filename = tiles[0]; - - command.add("-reducer"); - - if (tiles.length > 1) { - String params2 = tiles[1]; - command.add(filename + " " + params2); - } else { - command.add(filename); - } - - } - - } - - try { - context.beginTask("Running Hadoop Job..."); - boolean successful = executeCommand(command, context); - if (successful) { - context.endTask("Execution successful.", Message.OK); - return true; - } else { - context.endTask("Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", Message.ERROR); - return false; - } - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - } - - @Override - protected boolean executeCommand(List command, WorkflowContext context) - throws IOException, InterruptedException { - // set global variables - for (int j = 0; j < command.size(); j++) { - - String cmd = command.get(j).replaceAll("\\$job_id", context.getJobId()); - command.set(j, cmd); - } - - context.log("Command: " + command); - context.log("Working Directory: " + new File(context.getWorkingDirectory()).getAbsolutePath()); - - ProcessBuilder builder = new ProcessBuilder(command); - builder.directory(new File(context.getWorkingDirectory())); - builder.redirectErrorStream(true); - Process process = builder.start(); - InputStream is = process.getInputStream(); - InputStreamReader isr = new InputStreamReader(is); - BufferedReader br = new BufferedReader(isr); - String line = null; - - // Find job id and write output into file - Pattern pattern = Pattern.compile("Running job: (.*)"); - Pattern pattern2 = Pattern.compile("HadoopJobId: (.*)"); - - while ((line = br.readLine()) != null) { - - Matcher matcher = pattern.matcher(line); - if (matcher.find()) { - // write statistics from old job - jobId = matcher.group(1).trim(); - context.log("Job " + context.getJobId() + " -> HadoopJob " + jobId); - } else { - Matcher matcher2 = pattern2.matcher(line); - if (matcher2.find()) { - jobId = matcher2.group(1).trim(); - context.log("Job " + context.getJobId() + " -> HadoopJob " + jobId); - } - } - - context.println(line); - } - - br.close(); - isr.close(); - is.close(); - - process.waitFor(); - context.log("Exit Code: " + process.exitValue()); - - if (process.exitValue() != 0) { - return false; - } else { - process.destroy(); - } - return true; - } - - @Override - public void updateProgress() { - - /*RunningJob job = HadoopUtil.getInstance().getJob(jobId); - if (job != null) { - - try { - - if (job.setupProgress() >= 1) { - map = ((int) (job.mapProgress() * 100)); - reduce = ((int) (job.reduceProgress() * 100)); - } else { - map = 0; - reduce = 0; - } - - } catch (Exception e) { - map = 0; - reduce = 0; - } - - } else { - map = 0; - reduce = 0; - }*/ - - } - - @Override - public int getProgress() { - return map / 2 + reduce / 2; - } - - @Override - public void kill() { - - try { - - if (jobId != null) { - - context.log(" Cancel Job " + jobId); - - HadoopUtil.getInstance().kill(jobId); - - } - - } catch (IOException e) { - - context.log(" Cancel Job failed: " + e); - - } - - } - - @Override - public String[] getRequirements() { - return new String[] { HadoopPlugin.ID }; - } - -} diff --git a/src/main/java/cloudgene/mapred/steps/HadoopPigStep.java b/src/main/java/cloudgene/mapred/steps/HadoopPigStep.java deleted file mode 100644 index da78fe12..00000000 --- a/src/main/java/cloudgene/mapred/steps/HadoopPigStep.java +++ /dev/null @@ -1,63 +0,0 @@ -package cloudgene.mapred.steps; - -import java.util.List; -import java.util.Vector; - -import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.jobs.CloudgeneStep; -import cloudgene.mapred.jobs.Message; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; -import cloudgene.mapred.wdl.WdlStep; -import genepi.io.FileUtil; - -public class HadoopPigStep extends CloudgeneStep { - - @Override - public boolean run(WdlStep step, CloudgeneContext context) { - - String pigPath = context.getSettings().getPigPath(); - String pig = FileUtil.path(pigPath, "bin", "pig"); - - // params - String paramsString = step.get("params"); - String[] params = new String[] {}; - if (paramsString != null) { - params = paramsString.split(" "); - } - - // pig script - List command = new Vector(); - - command.add(pig); - command.add("-f"); - command.add(step.get("pig")); - - // params - for (String tile : params) { - command.add(tile.trim()); - } - - try { - context.beginTask("Running Pig Script..."); - boolean successful = executeCommand(command, context); - if (successful) { - context.endTask("Execution successful.", Message.OK); - return true; - } else { - context.endTask("Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", - Message.ERROR); - return false; - } - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - } - - @Override - public String[] getRequirements() { - return new String[] { HadoopPlugin.ID }; - } - -} diff --git a/src/main/java/cloudgene/mapred/steps/HadoopSparkStep.java b/src/main/java/cloudgene/mapred/steps/HadoopSparkStep.java deleted file mode 100644 index 65362518..00000000 --- a/src/main/java/cloudgene/mapred/steps/HadoopSparkStep.java +++ /dev/null @@ -1,62 +0,0 @@ -package cloudgene.mapred.steps; - -import java.util.List; -import java.util.Vector; - -import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.jobs.CloudgeneStep; -import cloudgene.mapred.jobs.Message; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; -import cloudgene.mapred.wdl.WdlStep; - -public class HadoopSparkStep extends CloudgeneStep { - - @Override - public boolean run(WdlStep step, CloudgeneContext context) { - - String pigPath = context.getSettings().getSparkPath(); - - // params - String paramsString = step.get("params"); - String[] params = new String[] {}; - if (paramsString != null) { - params = paramsString.split(" "); - } - - // spark script - List command = new Vector(); - - command.add(pigPath); - command.add("--class"); - command.add(step.get("main_class")); - command.add("--master"); - command.add("yarn"); - command.add(step.get("spark")); - - // params - for (String tile : params) { - command.add(tile.trim()); - } - - try { - context.beginTask("Running Spark Script..."); - boolean successful = executeCommand(command, context); - if (successful) { - context.endTask("Execution successful.", Message.OK); - return true; - } else { - context.endTask("Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", Message.ERROR); - return false; - } - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - } - - @Override - public String[] getRequirements() { - return new String[] { HadoopPlugin.ID }; - } -} diff --git a/src/main/java/cloudgene/mapred/steps/HdfsImporterStep.java b/src/main/java/cloudgene/mapred/steps/HdfsImporterStep.java deleted file mode 100644 index 4b589529..00000000 --- a/src/main/java/cloudgene/mapred/steps/HdfsImporterStep.java +++ /dev/null @@ -1,97 +0,0 @@ -package cloudgene.mapred.steps; - -import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.jobs.CloudgeneStep; -import cloudgene.mapred.jobs.Message; -import cloudgene.mapred.plugins.hadoop.HadoopPlugin; -import cloudgene.mapred.wdl.WdlStep; -import genepi.hadoop.HdfsUtil; -import genepi.hadoop.importer.IImporter; -import genepi.hadoop.importer.ImporterFactory; - -public class HdfsImporterStep extends CloudgeneStep { - - @Override - public boolean run(WdlStep step, CloudgeneContext context) { - - try { - - for (String input : context.getInputs()) { - - if (ImporterFactory.needsImport(context.get(input))) { - - String[] urlList = context.get(input).split(";")[0].split("\\s+"); - - String username = ""; - if (context.get(input).split(";").length > 1) { - username = context.get(input).split(";")[1]; - } - - String password = ""; - if (context.get(input).split(";").length > 2) { - password = context.get(input).split(";")[2]; - } - - for (String url2 : urlList) { - - String url = url2 + ";" + username + ";" + password; - - String target = HdfsUtil.path(((CloudgeneContext) context).getHdfsTemp(), "importer", input); - - try { - - context.beginTask("Import File(s) " + url2 + "..."); - - IImporter importer = ImporterFactory.createImporter(url, target); - - if (importer != null) { - - boolean successful = importer.importFiles(); - - if (successful) { - - context.setInput(input, target); - - context.endTask("Import File(s) " + url2 + " successful.", Message.OK); - - } else { - - context.endTask("Import File(s) " + url2 + " failed: " + importer.getErrorMessage(), - Message.ERROR); - - return false; - - } - - } else { - - context.endTask("Import File(s) " + url2 + " failed: Protocol not supported", - Message.ERROR); - - return false; - - } - - } catch (Exception e) { - context.endTask("Import File(s) " + url2 + " failed: " + e.toString(), Message.ERROR); - return false; - } - - } - - } - } - return true; - - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - } - - @Override - public String[] getRequirements() { - return new String[] { HadoopPlugin.ID }; - } -} diff --git a/src/main/java/cloudgene/mapred/steps/HtmlWidgetStep.java b/src/main/java/cloudgene/mapred/steps/HtmlWidgetStep.java index 332cc567..c06fb481 100644 --- a/src/main/java/cloudgene/mapred/steps/HtmlWidgetStep.java +++ b/src/main/java/cloudgene/mapred/steps/HtmlWidgetStep.java @@ -23,7 +23,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { String workingDirectory = context.getWorkingDirectory(); - String template = step.get("template"); + String template = step.getString("template"); if (template == null || template.isEmpty()) { context.endTask("Execution failed. Please set the 'template' parameter.", Message.ERROR); return false; @@ -51,7 +51,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { // add step values to context for (String variable : step.keySet()) { - String value = step.get(variable); + String value = step.getString(variable); if (value.endsWith(".json")) { // replace json files with content. String jsonFilename = ""; @@ -88,8 +88,8 @@ public boolean run(WdlStep step, CloudgeneContext context) { writer.write(""); writer.write(""); writer.write(""); - if (step.get("stylesheet") != null) { - for (String css : step.get("stylesheet").split(",")) { + if (step.getString("stylesheet") != null) { + for (String css : step.getString("stylesheet").split(",")) { css = css.trim(); String data = ""; if (!css.startsWith("http://") && !css.startsWith("https://")) { @@ -101,8 +101,8 @@ public boolean run(WdlStep step, CloudgeneContext context) { writer.write(""); } } - if (step.get("scripts") != null) { - for (String script : step.get("scripts").split(",")) { + if (step.getString("scripts") != null) { + for (String script : step.getString("scripts").split(",")) { script = script.trim(); String data = ""; if (!script.startsWith("http://") && !script.startsWith("https://")) { @@ -122,7 +122,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { writer.write(""); writer.close(); - String output = step.get("output"); + String output = step.getString("output"); if (output == null || output.isEmpty()) { context.addFile(htmlFile); @@ -131,7 +131,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { new File(htmlFile).renameTo(file); //copy assets folder if set - String assets = step.get("assets"); + String assets = step.getString("assets"); if (assets != null && !assets.isEmpty()) { File parent = file.getParentFile(); String assetsSource = FileUtil.path(workingDirectory, assets); diff --git a/src/main/java/cloudgene/mapred/steps/JavaExternalStep.java b/src/main/java/cloudgene/mapred/steps/JavaExternalStep.java index 68f663a0..6c27b508 100644 --- a/src/main/java/cloudgene/mapred/steps/JavaExternalStep.java +++ b/src/main/java/cloudgene/mapred/steps/JavaExternalStep.java @@ -49,7 +49,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { return false; } - String stdout = step.get("stdout", "false"); + String stdout = step.getString("stdout", "false"); boolean streamStdout = stdout.equals("true"); StringBuilder output = null; @@ -59,7 +59,7 @@ public boolean run(WdlStep step, CloudgeneContext context) { String jar = step.getJar(); // params - String paramsString = step.get("params"); + String paramsString = step.getString("params"); String[] params = new String[] {}; if (paramsString != null) { params = paramsString.split(" "); diff --git a/src/main/java/cloudgene/mapred/steps/JavaInternalStep.java b/src/main/java/cloudgene/mapred/steps/JavaInternalStep.java index 168c48ae..70fbb3f3 100644 --- a/src/main/java/cloudgene/mapred/steps/JavaInternalStep.java +++ b/src/main/java/cloudgene/mapred/steps/JavaInternalStep.java @@ -2,8 +2,8 @@ import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.CloudgeneStep; +import cloudgene.mapred.jobs.sdk.WorkflowStep; import cloudgene.mapred.wdl.WdlStep; -import cloudgene.sdk.internal.WorkflowStep; public class JavaInternalStep extends CloudgeneStep { diff --git a/src/main/java/cloudgene/mapred/steps/JavaInternalStepDeprecrated.java b/src/main/java/cloudgene/mapred/steps/JavaInternalStepDeprecrated.java deleted file mode 100644 index 58bb32aa..00000000 --- a/src/main/java/cloudgene/mapred/steps/JavaInternalStepDeprecrated.java +++ /dev/null @@ -1,187 +0,0 @@ -package cloudgene.mapred.steps; - -import java.util.Map; -import java.util.Set; - -import cloudgene.mapred.jobs.CloudgeneContext; -import cloudgene.mapred.jobs.CloudgeneStep; -import cloudgene.mapred.wdl.WdlStep; -import genepi.hadoop.common.WorkflowContext; -import genepi.hadoop.common.WorkflowStep; - -public class JavaInternalStepDeprecrated extends CloudgeneStep { - - private WorkflowStep workflowStep; - - public JavaInternalStepDeprecrated(WorkflowStep step) { - this.workflowStep = step; - } - - @Override - public void setup(CloudgeneContext context) { - super.setup(context); - workflowStep.setup(adapter(context)); - } - - @Override - public void kill() { - workflowStep.kill(); - } - - @Override - public void updateProgress() { - workflowStep.updateProgress(); - } - - @Override - public boolean run(WdlStep step, CloudgeneContext context) { - context.setConfig(step); - return workflowStep.run(adapter(context)); - } - - public static genepi.hadoop.common.WorkflowContext adapter(CloudgeneContext context) { - return new WorkflowContext() { - - @Override - public void updateTask(String name, int type) { - context.updateTask(name, type); - } - - @Override - public void submitCounter(String name) { - context.submitCounter(name); - } - - @Override - public void setOutput(String input, String value) { - context.setOutput(input, value); - } - - @Override - public void setInput(String input, String value) { - context.setInput(input, value); - } - - @Override - public void setConfig(Map config) { - context.setConfig(config); - } - - @Override - public boolean sendNotification(String body) throws Exception { - return context.sendNotification(body); - } - - @Override - public boolean sendMail(String to, String subject, String body) throws Exception { - return context.sendMail(to, subject, body); - } - - @Override - public boolean sendMail(String subject, String body) throws Exception { - return context.sendMail(subject, body); - } - - @Override - public void println(String line) { - context.println(line); - } - - @Override - public void message(String message, int type) { - context.message(message, type); - } - - @Override - public void log(String line) { - context.log(line); - } - - @Override - public void incCounter(String name, int value) { - context.incCounter(name, value); - } - - @Override - public String getWorkingDirectory() { - return context.getWorkingDirectory(); - } - - @Override - public String getOutput(String param) { - return context.getOutput(param); - } - - @Override - public String getLocalTemp() { - return context.getLocalTemp(); - } - - @Override - public String getJobName() { - return context.getJobName(); - } - - @Override - public String getJobId() { - return context.getJobId(); - } - - @Override - public Set getInputs() { - return context.getInputs(); - } - - @Override - public String getInput(String param) { - return context.getInput(param); - } - - @Override - public String getHdfsTemp() { - return context.getHdfsTemp(); - } - - @Override - public Object getData(String key) { - return context.getData(key); - } - - @Override - public Map getCounters() { - return context.getCounters(); - } - - @Override - public String getConfig(String param) { - return context.getConfig(param); - } - - @Override - public String get(String param) { - return context.get(param); - } - - @Override - public void endTask(String message, int type) { - context.endTask(message, type); - } - - @Override - public String createLinkToFile(String id) { - return context.createLinkToFile(id); - } - - @Override - public String createLinkToFile(String id, String filename) { - return context.createLinkToFile(id, filename); - } - - @Override - public void beginTask(String name) { - context.beginTask(name); - } - }; - } - -} diff --git a/src/main/java/cloudgene/mapred/steps/RMarkdownDockerStep.java b/src/main/java/cloudgene/mapred/steps/RMarkdownDockerStep.java index 0ca92322..388afc8a 100644 --- a/src/main/java/cloudgene/mapred/steps/RMarkdownDockerStep.java +++ b/src/main/java/cloudgene/mapred/steps/RMarkdownDockerStep.java @@ -18,26 +18,26 @@ public boolean run(WdlStep step, CloudgeneContext context) { String workingDirectory = context.getWorkingDirectory(); - String rmd = step.get("rmd"); + String rmd = step.getString("rmd"); if (rmd == null || rmd.isEmpty()) { - rmd = step.get("rmd2"); + rmd = step.getString("rmd2"); } if (rmd == null || rmd.isEmpty()) { context.error("Execution failed. Please set the 'rmd' parameter."); return false; } - String output = step.get("output"); + String output = step.getString("output"); if (output == null || output.isEmpty()) { context.error("Execution failed. Please set the 'output' parameter."); return false; } - String image = step.get("image"); + String image = step.getString("image"); if (image == null || image.isEmpty()) { image = DOCKER_R_BASE_IMAGE; } - String paramsString = step.get("params"); + String paramsString = step.getString("params"); String[] params = new String[] {}; if (paramsString != null) { params = paramsString.split(" "); diff --git a/src/main/java/cloudgene/mapred/steps/RMarkdownLocalStep.java b/src/main/java/cloudgene/mapred/steps/RMarkdownLocalStep.java index e1780304..9e2e08a9 100644 --- a/src/main/java/cloudgene/mapred/steps/RMarkdownLocalStep.java +++ b/src/main/java/cloudgene/mapred/steps/RMarkdownLocalStep.java @@ -1,7 +1,6 @@ package cloudgene.mapred.steps; import java.io.File; -import java.io.IOException; import cloudgene.mapred.jobs.CloudgeneContext; import cloudgene.mapred.jobs.CloudgeneStep; @@ -10,9 +9,8 @@ import cloudgene.mapred.plugins.rscript.RScriptBinary; import cloudgene.mapred.plugins.rscript.RScriptFile; import cloudgene.mapred.plugins.rscript.RScriptPlugin; +import cloudgene.mapred.util.command.Command; import cloudgene.mapred.wdl.WdlStep; -import genepi.hadoop.HdfsUtil; -import genepi.hadoop.command.Command; import genepi.io.FileUtil; public class RMarkdownLocalStep extends CloudgeneStep { @@ -24,21 +22,21 @@ public boolean run(WdlStep step, CloudgeneContext context) { String workingDirectory = context.getWorkingDirectory(); - String rmd = step.get("rmd"); + String rmd = step.getString("rmd"); if (rmd == null || rmd.isEmpty()) { - rmd = step.get("rmd2"); + rmd = step.getString("rmd2"); } if (rmd == null || rmd.isEmpty()) { context.endTask("Execution failed. Please set the 'rmd' parameter.", Message.ERROR); return false; } - String output = step.get("output"); + String output = step.getString("output"); if (output == null || output.isEmpty()) { context.endTask("Execution failed. Please set the 'output' parameter.", Message.ERROR); return false; } - String paramsString = step.get("params"); + String paramsString = step.getString("params"); String[] params = new String[] {}; if (paramsString != null) { params = paramsString.split(" "); @@ -57,13 +55,15 @@ public boolean run(WdlStep step, CloudgeneContext context) { if (result == 0) { context.endTask("Execution successful.", Message.OK); - String include = step.get("include"); + String include = step.getString("include"); if (include != null && include.equals("true")) { context.addFile(output); } return true; } else { - context.endTask("Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", Message.ERROR); + context.endTask( + "Execution failed. Please contact the server administrators for help if you believe this job should have completed successfully.", + Message.ERROR); return false; } @@ -96,32 +96,7 @@ public int convert(String rmdScript, String outputHtml, String[] args, Cloudgene argsForScript[0] = scriptFilename; // argsForScript[1] = "--args"; for (int i = 0; i < args.length; i++) { - - // checkout hdfs file - if (args[i].startsWith("hdfs://")) { - - String localFile = FileUtil.path(folder, "local_file_" + i); - context.log("Check out file " + args[i] + "..."); - try { - HdfsUtil.checkOut(args[i], localFile); - argsForScript[i + 1] = localFile; - } catch (IOException e) { - context.log(e.getMessage()); - argsForScript[i + 1] = args[i]; - } - - try { - context.log("Number of lines: " + FileUtil.getLineCount(localFile)); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - } else { - - argsForScript[i + 1] = args[i]; - - } + argsForScript[i + 1] = args[i]; } rScript.setParams(argsForScript); diff --git a/src/main/java/cloudgene/mapred/steps/ReturnTrueStep.java b/src/main/java/cloudgene/mapred/steps/ReturnTrueStep.java index c529aa90..4de63c8d 100644 --- a/src/main/java/cloudgene/mapred/steps/ReturnTrueStep.java +++ b/src/main/java/cloudgene/mapred/steps/ReturnTrueStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class ReturnTrueStep extends WorkflowStep { diff --git a/src/main/java/cloudgene/mapred/steps/SftpStep.java b/src/main/java/cloudgene/mapred/steps/SftpStep.java deleted file mode 100644 index 21438be4..00000000 --- a/src/main/java/cloudgene/mapred/steps/SftpStep.java +++ /dev/null @@ -1,111 +0,0 @@ -package cloudgene.mapred.steps; - -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; -import genepi.hadoop.importer.IImporter; -import genepi.hadoop.importer.ImporterFactory; -import genepi.io.FileUtil; - -public class SftpStep extends WorkflowStep { - - @Override - public boolean run(WorkflowContext context) { - - try { - Thread.sleep(5000); - return importVcfFiles(context); - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - } - - private boolean importVcfFiles(WorkflowContext context) { - try { - for (String input : context.getInputs()) { - - if (ImporterFactory.needsImport(context.get(input))) { - - context.beginTask("Importing files..."); - - String[] urlList = context.get(input).split(";")[0].split("\\s+"); - - String username = ""; - if (context.get(input).split(";").length > 1) { - username = context.get(input).split(";")[1]; - } - - String password = ""; - if (context.get(input).split(";").length > 2) { - password = context.get(input).split(";")[2]; - } - - for (String url2 : urlList) { - - String url = url2 + ";" + username + ";" + password; - String target = FileUtil.path(context.getLocalTemp(), "importer", input); - FileUtil.createDirectory(target); - context.log("Import to local workspace " + target + "..."); - - try { - - context.updateTask("Import " + url2 + "...", WorkflowContext.RUNNING); - - IImporter importer = ImporterFactory.createImporter(url, target); - - if (importer != null) { - - boolean successful = importer.importFiles("vcf.gz"); - - if (successful) { - - context.setInput(input, target); - - } else { - - context.updateTask("Import " + url2 + " failed: " + importer.getErrorMessage(), - WorkflowContext.ERROR); - - return false; - - } - - } else { - - context.updateTask("Import " + url2 + " failed: Protocol not supported", - WorkflowContext.ERROR); - - return false; - - } - - } catch (Exception e) { - context.updateTask("Import File(s) " + url2 + " failed: " + e.toString(), - WorkflowContext.ERROR); - e.printStackTrace(); - - return false; - } - - } - - context.updateTask("File Import successful. ", WorkflowContext.OK); - - } - - } - - return true; - } catch (Exception e) { - // context.updateTask("Import File(s) " + url2 + " failed: " + - // e.toString(), - // WorkflowContext.ERROR); - e.printStackTrace(); - - return false; - } - - } - -} diff --git a/src/main/java/cloudgene/mapred/util/S3Util.java b/src/main/java/cloudgene/mapred/util/S3Util.java new file mode 100644 index 00000000..1116d577 --- /dev/null +++ b/src/main/java/cloudgene/mapred/util/S3Util.java @@ -0,0 +1,184 @@ +package cloudgene.mapred.util; + +import java.io.File; +import java.io.IOException; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.services.s3.transfer.Download; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; +import com.amazonaws.services.s3.transfer.Upload; + +public class S3Util { + + private static AmazonS3 s3; + + private static TransferManager tm; + + public static AmazonS3 getAmazonS3() { + if (s3 == null) { + s3 = AmazonS3ClientBuilder.defaultClient(); + } + return s3; + } + + public static TransferManager getTransferManager() { + if (tm == null) { + s3 = getAmazonS3(); + tm = TransferManagerBuilder.standard().withS3Client(s3).build(); + } + return tm; + } + + public static boolean isValidS3Url(String url) { + if (url.startsWith("s3://")) { + String temp = url.replaceAll("s3://", ""); + String[] tiles = temp.split("/", 2); + if (tiles.length == 2) { + return true; + } else { + return false; + } + } else { + return false; + } + + } + + public static String getBucket(String url) { + if (url.startsWith("s3://")) { + String temp = url.replaceAll("s3://", ""); + String[] tiles = temp.split("/", 2); + if (tiles.length == 2) { + return tiles[0]; + } else { + return null; + } + } else { + return null; + } + } + + public static String getKey(String url) { + if (url.startsWith("s3://")) { + String temp = url.replaceAll("s3://", ""); + String[] tiles = temp.split("/", 2); + if (tiles.length == 2) { + return tiles[1]; + } else { + return null; + } + } else { + return null; + } + + } + + public static void copyToFile(String url, File file) throws IOException { + if (isValidS3Url(url)) { + String bucket = getBucket(url); + String key = getKey(url); + copyToFile(bucket, key, file); + } else { + throw new IOException("Url '" + url + "' is not a valid S3 bucket."); + } + } + + public static void copyToFile(String bucket, String key, File file) throws IOException { + + TransferManager tm = getTransferManager(); + Download download = tm.download(bucket, key, file); + try { + download.waitForCompletion(); + } catch (InterruptedException e) { + throw new IOException(e); + } + + } + + public static void copyToS3(File file, String url) throws IOException { + if (isValidS3Url(url)) { + String bucket = getBucket(url); + String key = getKey(url); + copyToS3(file, bucket, key); + } else { + throw new IOException("Url '" + url + "' is not a valid S3 bucket."); + } + } + + public static void copyToS3(String content, String url) throws IOException { + if (isValidS3Url(url)) { + String bucket = getBucket(url); + String key = getKey(url); + copyToS3(content, bucket, key); + } else { + throw new IOException("Url '" + url + "' is not a valid S3 bucket."); + } + } + + public static void copyToS3(File file, String bucket, String key) throws IOException { + + TransferManager tm = getTransferManager(); + Upload upload = tm.upload(bucket, key, file); + try { + upload.waitForCompletion(); + } catch (InterruptedException e) { + throw new IOException(e); + } + + } + + public static void copyToS3(String content, String bucket, String key) throws IOException { + + AmazonS3 s3 = getAmazonS3(); + s3.putObject(bucket, key, content); + + } + + public static ObjectListing listObjects(String url) throws IOException { + + AmazonS3 s3 = getAmazonS3(); + + if (isValidS3Url(url)) { + + String bucket = getBucket(url); + String key = getKey(url); + + ObjectListing objects = s3.listObjects(bucket, key); + + return objects; + } else { + throw new IOException("Url '" + url + "' is not a valid S3 bucket."); + } + + } + + public static void deleteFolder(String url) { + + String bucket = S3Util.getBucket(url); + String key = S3Util.getKey(url); + + AmazonS3 s3 = S3Util.getAmazonS3(); + + ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(key); + + ObjectListing objectListing = s3.listObjects(listObjectsRequest); + + while (true) { + for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { + s3.deleteObject(bucket, objectSummary.getKey()); + } + if (objectListing.isTruncated()) { + objectListing = s3.listNextBatchOfObjects(objectListing); + } else { + break; + } + } + + } + +} \ No newline at end of file diff --git a/src/main/java/cloudgene/mapred/util/Settings.java b/src/main/java/cloudgene/mapred/util/Settings.java index e675aad8..7d33640c 100644 --- a/src/main/java/cloudgene/mapred/util/Settings.java +++ b/src/main/java/cloudgene/mapred/util/Settings.java @@ -19,29 +19,20 @@ import cloudgene.mapred.apps.Application; import cloudgene.mapred.apps.ApplicationRepository; -import genepi.hadoop.HadoopCluster; +import cloudgene.mapred.jobs.Environment; +import cloudgene.mapred.jobs.workspace.LocalWorkspace; import genepi.io.FileUtil; public class Settings { - private String serverUrl = "http://localhost:8082"; - - private String hadoopPath = "/usr/"; - - private String pigPath = "/usr/"; + private static final Logger log = LoggerFactory.getLogger(Settings.class); - private String sparkPath = "/usr/bin/spark-submit"; + private String serverUrl = "http://localhost:8082"; private String tempPath = "tmp"; private String localWorkspace = "workspace"; - private String hdfsWorkspace = "cloudgene/data"; - - private String hdfsAppWorkspace = "cloudgene/apps"; - - private String streamingJar = ""; - private String version; private String name = "Cloudgene"; @@ -54,8 +45,6 @@ public class Settings { private Map database; - private Map cluster; - private Map> plugins; private int autoRetireInterval = 5; @@ -74,33 +63,23 @@ public class Settings { private boolean streaming = true; - private boolean removeHdfsWorkspace = true; - - private static final Logger log = LoggerFactory.getLogger(Settings.class); - private boolean writeStatistics = true; private boolean https = false; - private String httpsKeystore = ""; - - private String httpsPassword = ""; - private boolean maintenance = false; private String adminMail = null; - private String slack = null; + private String adminName = null; private String urlPrefix = ""; private List navigation = new Vector(); - private boolean secureCookie = false; - private Map externalWorkspace = null; - private int uploadLimit = 500; + private int uploadLimit = 5000; private String googleAnalytics = ""; @@ -110,6 +89,8 @@ public class Settings { private String port = "8082"; + private String nextflowConfig = FileUtil.path("config", "nextflow.config"); + public static final String DEFAULT_SECURITY_KEY = "default-key-change-me-immediately"; // fake! @@ -117,7 +98,7 @@ public class Settings { private ApplicationRepository repository; - public Settings() { + private Settings() { repository = new ApplicationRepository(); @@ -198,20 +179,6 @@ public static Settings load(Config config) throws FileNotFoundException, YamlExc log.info("Notify user after " + settings.notificationAfter + " days."); log.info("Write statistics: " + settings.writeStatistics); - if (settings.cluster != null) { - String conf = settings.cluster.get("conf"); - String username = settings.cluster.get("user"); - String name = settings.cluster.get("name"); - if (conf != null) { - log.info("Use Haddop configuration folder '" + conf + "'" - + (username != null ? " with username " + username : "")); - try { - HadoopCluster.setConfPath(name, conf, username); - } catch (NoClassDefFoundError e) { - } - } - } - settings.config = config; // workspace in config has higher priority @@ -308,22 +275,6 @@ public void save(String filename) { } - public String getHadoopPath() { - return hadoopPath; - } - - public void setHadoopPath(String hadoopPath) { - this.hadoopPath = hadoopPath; - } - - public void setPigPath(String pigPath) { - this.pigPath = pigPath; - } - - public String getPigPath() { - return pigPath; - } - public String getTempPath() { return tempPath; } @@ -340,30 +291,6 @@ public void setLocalWorkspace(String localWorkspace) { this.localWorkspace = localWorkspace; } - public String getHdfsWorkspace() { - return hdfsWorkspace; - } - - public void setHdfsWorkspace(String hdfsWorkspace) { - this.hdfsWorkspace = hdfsWorkspace; - } - - public String getHdfsAppWorkspace() { - return hdfsAppWorkspace; - } - - public void setHdfsAppWorkspace(String hdfsAppWorkspace) { - this.hdfsAppWorkspace = hdfsAppWorkspace; - } - - public String getStreamingJar() { - return streamingJar; - } - - public void setStreamingJar(String streamingJar) { - this.streamingJar = streamingJar; - } - public boolean isStreaming() { return streaming; } @@ -372,39 +299,6 @@ public void setStreaming(boolean streaming) { this.streaming = streaming; } - public boolean isRemoveHdfsWorkspace() { - return removeHdfsWorkspace; - } - - public void setRemoveHdfsWorkspace(boolean removeHdfsWorkspace) { - this.removeHdfsWorkspace = removeHdfsWorkspace; - } - - public boolean testPaths() { - - String hadoop = FileUtil.path(hadoopPath, "bin", "hadoop"); - - if (!new File(hadoop).exists()) { - - log.warn("hadoop '" + hadoop + "' does not exist. please change it."); - - // return false; - - } - /* - * if (!new File(streamingJar).exists()) { - * - * log.error("streamingJar '" + streamingJar + "' does not exist."); - * - * return false; - * - * } - */ - - return true; - - } - public String getVersion() { return version; } @@ -421,22 +315,6 @@ public void setMail(Map mail) { this.mail = mail; } - public Map getCluster() { - return cluster; - } - - public void setCluster(Map cluster) { - this.cluster = cluster; - } - - public String getSlack() { - return slack; - } - - public void setSlack(String slack) { - this.slack = slack; - } - public void setName(String name) { this.name = name; } @@ -499,22 +377,6 @@ public boolean isHttps() { return https; } - public void setHttpsKeystore(String httpsKeystore) { - this.httpsKeystore = httpsKeystore; - } - - public String getHttpsKeystore() { - return httpsKeystore; - } - - public void setHttpsPassword(String httpsPassword) { - this.httpsPassword = httpsPassword; - } - - public String getHttpsPassword() { - return httpsPassword; - } - public void setMaintenance(boolean maintenance) { this.maintenance = maintenance; } @@ -531,12 +393,12 @@ public String getAdminMail() { return adminMail; } - public String getSparkPath() { - return sparkPath; + public void setAdminName(String adminName) { + this.adminName = adminName; } - public void setSparkPath(String sparkPath) { - this.sparkPath = sparkPath; + public String getAdminName() { + return adminName; } public void setThreadsQueue(int threadsQueue) { @@ -587,14 +449,6 @@ public List getNavigation() { return navigation; } - public boolean isSecureCookie() { - return secureCookie; - } - - public void setSecureCookie(boolean secureCookie) { - this.secureCookie = secureCookie; - } - public int getUploadLimit() { return uploadLimit; } @@ -685,7 +539,9 @@ public void setExternalWorkspace(Map externalWorkspace) { public String getExternalWorkspaceLocation() { if (externalWorkspace == null) { - return ""; + externalWorkspace = new HashMap<>(); + externalWorkspace.put("type", "local"); + externalWorkspace.put("location", getLocalWorkspace()); } if (externalWorkspace.get("location") == null) { @@ -698,7 +554,9 @@ public String getExternalWorkspaceLocation() { public String getExternalWorkspaceType() { if (externalWorkspace == null) { - return ""; + externalWorkspace = new HashMap<>(); + externalWorkspace.put("type", "local"); + externalWorkspace.put("location", getLocalWorkspace()); } if (externalWorkspace.get("type") == null) { @@ -717,4 +575,16 @@ public String getServerUrl() { return serverUrl; } + public void setNextflowConfig(String nextflowConfig) { + this.nextflowConfig = nextflowConfig; + } + + public String getNextflowConfig() { + return nextflowConfig; + } + + public Environment buildEnvironment() { + return new Environment(this); + } + } \ No newline at end of file diff --git a/src/main/java/cloudgene/mapred/util/TimeUtil.java b/src/main/java/cloudgene/mapred/util/TimeUtil.java new file mode 100644 index 00000000..9f91753d --- /dev/null +++ b/src/main/java/cloudgene/mapred/util/TimeUtil.java @@ -0,0 +1,12 @@ +package cloudgene.mapred.util; + +public class TimeUtil { + + public static String format(long time) { + long h = (long) (Math.floor((time / 1000) / 60 / 60)); + long m = (long) ((Math.floor((time / 1000) / 60)) % 60); + + return (h > 0 ? h + " h " : "") + (m > 0 ? m + " min " : "") + (int) ((Math.floor(time / 1000)) % 60) + " sec"; + } + +} diff --git a/src/main/java/cloudgene/mapred/util/command/Command.java b/src/main/java/cloudgene/mapred/util/command/Command.java new file mode 100644 index 00000000..5d6df9e8 --- /dev/null +++ b/src/main/java/cloudgene/mapred/util/command/Command.java @@ -0,0 +1,204 @@ +package cloudgene.mapred.util.command; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Vector; + +import org.apache.commons.codec.digest.DigestUtils; + +public class Command { + + protected String cmd; + + private String[] params; + + private boolean silent = false; + + private boolean deleteInput = false; + + private String directory = null; + + private StringBuffer stout = new StringBuffer(); + + private String stdoutFileName = null; + + private String stderrFileName = null; + + private List inputs = new Vector(); + + private List outputs = new Vector(); + + public Command(String cmd, String... params) { + this.cmd = cmd; + this.params = params; + } + + public Command(String cmd) { + this.cmd = cmd; + } + + public void setParams(String... params) { + this.params = params; + } + + public void setParams(List params) { + this.params = new String[params.size()]; + for (int i = 0; i < params.size(); i++) { + this.params[i] = params.get(i); + } + } + + public void saveStdOut(String filename) { + this.stdoutFileName = filename; + } + + public void saveStdErr(String filename) { + this.stderrFileName = filename; + } + + public int execute() { + + List command = new ArrayList(); + + command.add(cmd); + + if (params != null) { + for (String param : params) { + command.add(param); + } + } + + try { + + ProcessBuilder builder = new ProcessBuilder(command); + // builder.redirectErrorStream(true); + if (directory != null) { + builder.directory(new File(directory)); + } + + Process process = builder.start(); + CommandStreamHandler handler = new CommandStreamHandler( + process.getInputStream(), stdoutFileName); + handler.setSilent(silent); + Thread inputStreamHandler = new Thread(handler); + + CommandStreamHandler handler2 = new CommandStreamHandler( + process.getErrorStream(), stderrFileName); + handler2.setSilent(silent); + Thread errorStreamHandler = new Thread(handler2); + + inputStreamHandler.start(); + errorStreamHandler.start(); + + process.waitFor(); + + inputStreamHandler.interrupt(); + errorStreamHandler.interrupt(); + inputStreamHandler.join(); + errorStreamHandler.join(); + + if (process.exitValue() != 0) { + return process.exitValue(); + } else { + process.destroy(); + } + + if (deleteInput) { + new File(cmd).delete(); + } + + return 0; + } catch (Exception e) { + e.printStackTrace(); + return -1; + } + } + + public boolean isSilent() { + return silent; + } + + public void setSilent(boolean silent) { + this.silent = silent; + } + + public boolean isDeleteInput() { + return deleteInput; + } + + public void setDeleteInput(boolean deleteInput) { + this.deleteInput = deleteInput; + } + + public void setDirectory(String directory) { + this.directory = directory; + } + + public String getDirectory() { + return directory; + } + + @Override + public String toString() { + String result = cmd; + if (params != null) { + for (String param : params) { + result += " " + param; + } + } + return result; + } + + public String getStdOut() { + return stout.toString(); + } + + public void addInput(String input) { + inputs.add(input); + } + + public void addOutput(String output) { + outputs.add(output); + } + + public List getInputs() { + return inputs; + } + + public List getOutputs() { + return outputs; + } + + private boolean isNoFile(String param) { + return !inputs.contains(param) && !outputs.contains(param); + } + + public String getSignature() { + + String fullCommand = cmd; + for (String param : params) { + if (isNoFile(param)) { + fullCommand += param; + } + } + + return DigestUtils.md5Hex(fullCommand); + + } + + public String getName() { + return cmd; + } + + public String getExecutedCommand() { + String executedCommand = cmd; + if (params != null) { + for (String param : params) { + executedCommand += " " + param; + } + } + return executedCommand; + } + +} \ No newline at end of file diff --git a/src/main/java/cloudgene/mapred/util/command/CommandStreamHandler.java b/src/main/java/cloudgene/mapred/util/command/CommandStreamHandler.java new file mode 100644 index 00000000..35e94811 --- /dev/null +++ b/src/main/java/cloudgene/mapred/util/command/CommandStreamHandler.java @@ -0,0 +1,73 @@ +package cloudgene.mapred.util.command; + +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; + +public class CommandStreamHandler implements Runnable { + + private InputStream is; + + //private StringBuffer stdout = new StringBuffer();; + + private boolean silent = false; + + private String filename = null; + + public CommandStreamHandler(InputStream is) { + this.is = is; + } + + public CommandStreamHandler(InputStream is, String filename) { + this.is = is; + this.filename = filename; + } + + public void setSilent(boolean silent) { + this.silent = silent; + } + + public void setFilename(String filename) { + this.filename = filename; + } + + @Override + public void run() { + + try { + + boolean save = (filename != null && !filename.isEmpty()); + FileOutputStream writer = null; + + byte[] buffer = new byte[200]; + + if (save) { + writer = new FileOutputStream(filename); + } + + int size = 0; + + while ((size = is.read(buffer)) > 0) { + //stdout.append(line); + if (!silent) { + String line = new String(buffer, 0, size); + System.out.println(line); + } + if (save) { + writer.write(buffer, 0, size); + } + } + + if (save) { + writer.close(); + } + + is.close(); + + } catch (IOException e) { + e.printStackTrace(); + } + + } + +} \ No newline at end of file diff --git a/src/main/java/cloudgene/mapred/wdl/WdlStep.java b/src/main/java/cloudgene/mapred/wdl/WdlStep.java index 9af7d15e..9bde046f 100644 --- a/src/main/java/cloudgene/mapred/wdl/WdlStep.java +++ b/src/main/java/cloudgene/mapred/wdl/WdlStep.java @@ -2,14 +2,14 @@ import java.util.HashMap; -public class WdlStep extends HashMap{ +public class WdlStep extends HashMap{ public String getName() { - return get("name"); + return getString("name"); } public String getClassname() { - return get("classname"); + return getString("classname"); } public void setClassname(String classname){ @@ -18,20 +18,20 @@ public void setClassname(String classname){ public String getJar() { - return get("jar"); + return getString("jar"); } public String getGenerates() { - return get("generates"); + return getString("generates"); } - public String get(String key, String defaultValue){ - String value = get(key); - if (value == null){ - return defaultValue; - }else{ - return value; - } + public String getString(String key){ + return getString(key, null); + } + + public String getString(String key, String defaultValue){ + Object value = get(key); + return value != null ? value.toString() : defaultValue; } } diff --git a/src/main/resources/templates/list.html b/src/main/resources/templates/list.html new file mode 100644 index 00000000..dc07d363 --- /dev/null +++ b/src/main/resources/templates/list.html @@ -0,0 +1,15 @@ +${label} (${completed}/${total})
+<% for (task in tasks) { %> + + <%= task.getTrace().get("name") %> + <% if (task.getTrace().get("status").equals("COMPLETED")) { %> +   + <% } %> + <% if (task.getTrace().get("status").equals("RUNNING")) { %> + ... + <% } %> + <% if (task.getTrace().get("status").equals("KILLED")) { %> +   + <% } %> +
+<% } %> \ No newline at end of file diff --git a/src/main/resources/templates/progressbar.html b/src/main/resources/templates/progressbar.html new file mode 100644 index 00000000..490459c8 --- /dev/null +++ b/src/main/resources/templates/progressbar.html @@ -0,0 +1,6 @@ +${label} (${completed}/${total})
+
+
+
+
+
\ No newline at end of file diff --git a/src/test/java/cloudgene/mapred/TestApplication.java b/src/test/java/cloudgene/mapred/TestApplication.java index 5fceeab9..a303534c 100644 --- a/src/test/java/cloudgene/mapred/TestApplication.java +++ b/src/test/java/cloudgene/mapred/TestApplication.java @@ -1,7 +1,6 @@ package cloudgene.mapred; import java.io.FileNotFoundException; -import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Vector; @@ -15,7 +14,6 @@ import cloudgene.mapred.util.Config; import cloudgene.mapred.util.HashUtil; import cloudgene.mapred.util.Settings; -import cloudgene.mapred.util.TestCluster; import cloudgene.mapred.util.TestMailServer; import genepi.io.FileUtil; import io.micronaut.context.annotation.Context; @@ -38,15 +36,6 @@ public TestApplication() throws Exception { protected static Settings loadSettings(Config config) throws FileNotFoundException, YamlException { - System.out.println("Starting test Hadoop cluster..."); - try { - TestCluster.getInstance().start(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - System.out.println("Test Hadoop cluster started."); - Settings settings = new Settings(new Config()); HashMap mail = new HashMap(); @@ -155,32 +144,8 @@ protected static List registerApplications(Settings settings) { app14.setPermission("public"); applications.add(app14); - // hdfs - - Application app10 = new Application(); - app10.setId("all-possible-inputs-hdfs"); - app10.setFilename("test-data/all-possible-inputs-hdfs.yaml"); - app10.setPermission("public"); - applications.add(app10); - - Application app11 = new Application(); - app11.setId("write-files-to-hdfs-folder"); - app11.setFilename("test-data/write-files-to-hdfs-folder.yaml"); - app11.setPermission("public"); - applications.add(app11); - - Application app12 = new Application(); - app12.setId("write-text-to-hdfs-file"); - app12.setFilename("test-data/write-text-to-hdfs-file.yaml"); - app12.setPermission("public"); - applications.add(app12); - - Application app16 = new Application(); - app16.setId("sftp-import"); - app16.setFilename("test-data/sftp-import.yaml"); - app16.setPermission("public"); - applications.add(app16); - + //app links + Application app17 = new Application(); app17.setId("app-links"); app17.setFilename("test-data/app-links.yaml"); @@ -205,18 +170,6 @@ protected static List registerApplications(Settings settings) { app19.setPermission("protected"); applications.add(app19); - Application app20 = new Application(); - app20.setId("app-installation2"); - app20.setFilename("test-data/app-installation2.yaml"); - app20.setPermission("public"); - applications.add(app20); - - Application app21 = new Application(); - app21.setId("app-installation-child"); - app21.setFilename("test-data/app-installation-child.yaml"); - app21.setPermission("public"); - applications.add(app21); - Application app22 = new Application(); app22.setId("print-hidden-inputs"); app22.setFilename("test-data/print-hidden-inputs.yaml"); diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java index 57ed7b4a..b15db9f6 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/DownloadResultsTest.java @@ -94,40 +94,6 @@ public void testDownloadSingleFolder() throws InterruptedException { } - @Test - public void testDownloadSingleHdfsFolder() throws InterruptedException { - - Header accessToken = client.loginAsPublicUser(); - - // submit job - String id = RestAssured.given().header(accessToken).and().multiPart("inputtext", "lukas_text").when() - .post("/api/v2/jobs/submit/write-files-to-hdfs-folder").then().statusCode(200).and().extract() - .jsonPath().getString("id"); - - // wait until submitted job is complete - client.waitForJob(id, accessToken); - - // TODO: check why file is not available without this sleep - Thread.sleep(5000); - - // get details - Response response = RestAssured.given().header(accessToken).when().get("/api/v2/jobs/" + id).thenReturn(); - response.then().statusCode(200).and().body("state", equalTo(AbstractJob.STATE_SUCCESS)).and() - .body("outputParams[0].name", equalTo("output")).and().body("outputParams[0].files.size()", equalTo(5)); - - // get path and download all 5 files - for (int i = 0; i < 5; i++) { - String path = response.body().jsonPath().getString("outputParams[0].files[" + i + "].path"); - String name = response.jsonPath().getString("outputParams[0].files[" + i + "].name"); - String hash = response.jsonPath().getString("outputParams[0].files[" + i + "].hash"); - - assertEquals(id + "/output/file" + (i + 1) + ".txt", path); - RestAssured.given().header(accessToken).when().get("/downloads/" + id + "/" + hash + "/" + name).then() - .statusCode(200).and().body(equalTo("lukas_text")); - } - - } - @Test public void testDownloadCounter() throws InterruptedException { diff --git a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java index 0d651433..6f94b56a 100644 --- a/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java +++ b/src/test/java/cloudgene/mapred/api/v2/jobs/SubmitJobTest.java @@ -10,8 +10,8 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.jobs.AbstractJob; +import cloudgene.mapred.jobs.sdk.WorkflowContext; import cloudgene.mapred.util.CloudgeneClientRestAssured; -import cloudgene.sdk.internal.WorkflowContext; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.restassured.RestAssured; @@ -91,32 +91,6 @@ public void testSubmitAllPossibleInputs() { } - @Test - public void testSubmitAllPossibleInputsHdfs() { - - Header accessToken = client.loginAsPublicUser(); - - // local-file - FileUtil.writeStringBufferToFile("test.txt", new StringBuffer("content-of-my-file")); - FileUtil.writeStringBufferToFile("test1.txt", new StringBuffer("content-of-my-file-in-folder1")); - FileUtil.writeStringBufferToFile("test2.txt", new StringBuffer("content-of-my-file-in-folder2")); - - // submit job with different inputs and file uploads - String id = RestAssured.given().header(accessToken).and().multiPart("job-name", "my-job-name").and() - .multiPart("input-file", new File("test.txt")).and().multiPart("input-folder", new File("test1.txt")) - .and().multiPart("input-folder", new File("test2.txt")).when() - .post("/api/v2/jobs/submit/all-possible-inputs-hdfs").then().statusCode(200).and().extract().jsonPath() - .getString("id"); - - // wait until job is complete - client.waitForJob(id, accessToken); - - // get details and check state - RestAssured.given().header(accessToken).when().get("/api/v2/jobs/" + id).then().statusCode(200).and() - .body("state", equalTo(AbstractJob.STATE_SUCCESS)).and().body("name", equalTo("my-job-name")); - - } - @Test public void testSubmitReturnTrueStepPublic() { @@ -203,35 +177,6 @@ public void testSubmitWriteTextToFilePublic() throws InterruptedException { } - @Test - public void testSubmitWriteTextToHdfsFilePublic() throws InterruptedException { - - Header accessToken = client.loginAsPublicUser(); - - // submit jobs - String id = RestAssured.given().header(accessToken).and().multiPart("input-inputtext", "lukas_text").when() - .post("/api/v2/jobs/submit/write-text-to-hdfs-file").then().statusCode(200).and().extract().jsonPath() - .getString("id"); - - // wait until job is complete - client.waitForJob(id, accessToken); - - // TODO: change! - Thread.sleep(5000); - - Response response = RestAssured.given().header(accessToken).when().get("/api/v2/jobs/" + id).thenReturn(); - response.then().statusCode(200).and().body("state", equalTo(AbstractJob.STATE_SUCCESS)); - - // get file details - String name = response.jsonPath().getString("outputParams[0].files[0].name"); - String hash = response.jsonPath().getString("outputParams[0].files[0].hash"); - - // download file and check content - RestAssured.given().header(accessToken).when().get("/downloads/" + id + "/" + hash + "/" + name).then() - .statusCode(200).and().body(equalTo("lukas_text")); - - } - @Test public void testSubmitThreeTasksStepPublic() { @@ -239,7 +184,7 @@ public void testSubmitThreeTasksStepPublic() { // submit jobs String id = RestAssured.given().header(accessToken).and().multiPart("input-input", "input-file").when() - .post("/api/v2/jobs/submit/three-tasks").then().statusCode(200).and().extract().jsonPath() + .post("/api/v2/jobs/submit/three-tasks").then().log().all().statusCode(200).and().extract().jsonPath() .getString("id"); // wait until job is complete diff --git a/src/test/java/cloudgene/mapred/database/JobDaoTest.java b/src/test/java/cloudgene/mapred/database/JobDaoTest.java index 93df5d59..84415a6b 100644 --- a/src/test/java/cloudgene/mapred/database/JobDaoTest.java +++ b/src/test/java/cloudgene/mapred/database/JobDaoTest.java @@ -42,7 +42,7 @@ public void testFindAllOlderThan() throws Exception { job1.setName("old-dummy-job-1" + System.currentTimeMillis()); job1.setState(CloudgeneJob.STATE_SUCCESS); job1.setSubmittedOn(System.currentTimeMillis() - (8 * DAYS_MS)); - job1.setFinishedOn(System.currentTimeMillis() - (7 * DAYS_MS)); + job1.setEndTime(System.currentTimeMillis() - (7 * DAYS_MS)); job1.setUser(user); job1.setApplication("appplication"); job1.setApplicationId("appplication-id"); @@ -53,7 +53,7 @@ public void testFindAllOlderThan() throws Exception { job2.setName("old-dummy-job-2" + System.currentTimeMillis()); job2.setState(CloudgeneJob.STATE_SUCCESS); job2.setSubmittedOn(System.currentTimeMillis() - (10 * DAYS_MS)); - job2.setFinishedOn(System.currentTimeMillis() - (9 * DAYS_MS)); + job2.setEndTime(System.currentTimeMillis() - (9 * DAYS_MS)); job2.setUser(user); job2.setApplication("appplication"); job2.setApplicationId("appplication-id"); @@ -64,7 +64,7 @@ public void testFindAllOlderThan() throws Exception { job3.setName("old-dummy-job-3" + System.currentTimeMillis()); job3.setState(CloudgeneJob.STATE_SUCCESS); job3.setSubmittedOn(System.currentTimeMillis() - (9 * DAYS_MS)); - job3.setFinishedOn(System.currentTimeMillis() - (8 * DAYS_MS)); + job3.setEndTime(System.currentTimeMillis() - (8 * DAYS_MS)); job3.setUser(user); job3.setApplication("appplication"); job3.setApplicationId("appplication-id"); @@ -75,7 +75,7 @@ public void testFindAllOlderThan() throws Exception { job4.setName("old-dummy-job-4" + System.currentTimeMillis()); job4.setState(CloudgeneJob.STATE_SUCCESS); job4.setSubmittedOn(System.currentTimeMillis() - (3 * DAYS_MS)); - job4.setFinishedOn(System.currentTimeMillis() - (2 * DAYS_MS)); + job4.setEndTime(System.currentTimeMillis() - (2 * DAYS_MS)); job4.setUser(user); job4.setApplication("appplication"); job4.setApplicationId("appplication-id"); @@ -111,7 +111,7 @@ public void testFindAllByState() throws Exception { jobr.setName("old-dummy-running-job-1" + System.currentTimeMillis()); jobr.setState(CloudgeneJob.STATE_RUNNING); jobr.setSubmittedOn(System.currentTimeMillis() - (8 * DAYS_MS)); - jobr.setFinishedOn(0); + jobr.setEndTime(0); jobr.setUser(user); jobr.setApplication("appplication"); jobr.setApplicationId("appplication-id"); @@ -122,7 +122,7 @@ public void testFindAllByState() throws Exception { job1.setName("old-dummy-job-1" + System.currentTimeMillis()); job1.setState(CloudgeneJob.STATE_FAILED); job1.setSubmittedOn(System.currentTimeMillis() - (8 * DAYS_MS)); - job1.setFinishedOn(System.currentTimeMillis() - (7 * DAYS_MS)); + job1.setEndTime(System.currentTimeMillis() - (7 * DAYS_MS)); job1.setUser(user); job1.setApplication("appplication"); job1.setApplicationId("appplication-id"); @@ -133,7 +133,7 @@ public void testFindAllByState() throws Exception { job2.setName("old-dummy-job-2" + System.currentTimeMillis()); job2.setState(CloudgeneJob.STATE_FAILED); job2.setSubmittedOn(System.currentTimeMillis() - (10 * DAYS_MS)); - job2.setFinishedOn(System.currentTimeMillis() - (9 * DAYS_MS)); + job2.setEndTime(System.currentTimeMillis() - (9 * DAYS_MS)); job2.setUser(user); job2.setApplication("appplication"); job2.setApplicationId("appplication-id"); @@ -144,7 +144,7 @@ public void testFindAllByState() throws Exception { job3.setName("old-dummy-job-3" + System.currentTimeMillis()); job3.setState(CloudgeneJob.STATE_FAILED); job3.setSubmittedOn(System.currentTimeMillis() - (9 * DAYS_MS)); - job3.setFinishedOn(System.currentTimeMillis() - (8 * DAYS_MS)); + job3.setEndTime(System.currentTimeMillis() - (8 * DAYS_MS)); job3.setUser(user); job3.setApplication("appplication"); job3.setApplicationId("appplication-id"); @@ -155,7 +155,7 @@ public void testFindAllByState() throws Exception { job4.setName("old-dummy-job-4" + System.currentTimeMillis()); job4.setState(CloudgeneJob.STATE_SUCCESS); job4.setSubmittedOn(System.currentTimeMillis() - (3 * DAYS_MS)); - job4.setFinishedOn(System.currentTimeMillis() - (2 * DAYS_MS)); + job4.setEndTime(System.currentTimeMillis() - (2 * DAYS_MS)); job4.setUser(user); job4.setApplication("appplication"); job4.setApplicationId("appplication-id"); diff --git a/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java b/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java index 910639d1..e06fe7bb 100644 --- a/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java +++ b/src/test/java/cloudgene/mapred/jobs/PriorityThreadPoolExecutorTest.java @@ -12,10 +12,10 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; @@ -24,15 +24,19 @@ public class PriorityThreadPoolExecutorTest { private static final int WAIT_FOR_CANCEL = 8000; + @Inject TestApplication application; + @Inject + WorkspaceFactory workspaceFactory; + @Test public void testCancelRunningJob() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } @@ -63,7 +67,7 @@ public void testCancelRunningJob() throws Exception { List jobsAfterCancel = engine.getAllJobsInLongTimeQueue(); assertEquals(jobsBeforeSubmit.size(), jobsAfterCancel.size()); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } @@ -75,7 +79,7 @@ public void testCancelWaitingJob() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } @@ -115,7 +119,7 @@ public void testCancelWaitingJob() throws Exception { // clear queue engine.cancel(job1); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } } @@ -132,7 +136,7 @@ public void testMultipleJobs() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } @@ -220,7 +224,7 @@ public void testMultipleJobs() throws Exception { assertEquals(AbstractJob.STATE_CANCELED, job3.getState()); assertEquals(AbstractJob.STATE_CANCELED, job4.getState()); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } } @@ -237,7 +241,7 @@ public void testMultipleJobsWithPriority() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } @@ -325,7 +329,7 @@ public void testMultipleJobsWithPriority() throws Exception { assertEquals(AbstractJob.STATE_CANCELED, job3.getState()); assertEquals(AbstractJob.STATE_CANCELED, job4.getState()); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } } @@ -342,7 +346,7 @@ public void testMultipleJobsAndUpdatePriority() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } @@ -456,7 +460,7 @@ public void testMultipleJobsAndUpdatePriority() throws Exception { assertEquals(AbstractJob.STATE_CANCELED, job3.getState()); assertEquals(AbstractJob.STATE_CANCELED, job4.getState()); - while (engine.getAllJobsInLongTimeQueue().size() > 0 || engine.getAllJobsInShortTimeQueue().size() > 0) { + while (engine.getAllJobsInLongTimeQueue().size() > 0) { Thread.sleep(6000); } } @@ -469,17 +473,15 @@ public CloudgeneJob createJobFromWdl(WdlApp app, String id, Map Settings settings = application.getSettings(); - String hdfsWorkspace = HdfsUtil.path(settings.getHdfsWorkspace(), id); String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), id); FileUtil.createDirectory(localWorkspace); CloudgeneJob job = new CloudgeneJob(user, id,app, inputs); job.setId(id); job.setName(id); + job.setWorkspace(workspaceFactory.getDefault()); job.setLocalWorkspace(localWorkspace); - job.setHdfsWorkspace(hdfsWorkspace); job.setSettings(settings); - job.setRemoveHdfsWorkspace(true); job.setApplication(app.getName() + " " + app.getVersion()); job.setApplicationId(app.getId()); diff --git a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java index 3ed2b636..b54b3b21 100644 --- a/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java +++ b/src/test/java/cloudgene/mapred/jobs/WorkflowEngineTest.java @@ -13,11 +13,11 @@ import cloudgene.mapred.TestApplication; import cloudgene.mapred.core.User; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; -import cloudgene.sdk.internal.WorkflowContext; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; @@ -27,12 +27,15 @@ public class WorkflowEngineTest { @Inject TestApplication application; - + + @Inject + WorkspaceFactory workspaceFactory; + @Test public void testReturnTrueStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/return-true.yaml"); Map inputs = new HashMap(); @@ -40,13 +43,11 @@ public void testReturnTrueStep() throws Exception { AbstractJob job = createJobFromWdl(app, inputs); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); + assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -57,7 +58,6 @@ public void testReturnFalseStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - WdlApp app = WdlReader.loadAppFromFile("test-data/return-false.yaml"); Map params = new HashMap(); @@ -65,13 +65,10 @@ public void testReturnFalseStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_FAILED, job.getState()); @@ -89,13 +86,10 @@ public void testReturnExceptionStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_FAILED, job.getState()); @@ -103,7 +97,7 @@ public void testReturnExceptionStep() throws Exception { @Test public void testReturnTrueInSetupStep() throws Exception { - + WorkflowEngine engine = application.getWorkflowEngine(); WdlApp app = WdlReader.loadAppFromFile("test-data/return-true-in-setup.yaml"); @@ -113,17 +107,14 @@ public void testReturnTrueInSetupStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); // no steps - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); } @@ -132,7 +123,6 @@ public void testReturnFalseInSetupStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - WdlApp app = WdlReader.loadAppFromFile("test-data/return-false-in-setup.yaml"); Map params = new HashMap(); @@ -140,16 +130,13 @@ public void testReturnFalseInSetupStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); // no steps - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_FAILED, job.getState()); } @@ -158,7 +145,6 @@ public void testReturnTrueInSecondSetupStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - WdlApp app = WdlReader.loadAppFromFile("test-data/return-true-in-setup2.yaml"); Map params = new HashMap(); @@ -166,17 +152,13 @@ public void testReturnTrueInSecondSetupStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); // no steps - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); } @@ -185,7 +167,6 @@ public void testReturnTrueInSecondSetupStepAndNormalStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - WdlApp app = WdlReader.loadAppFromFile("test-data/return-true-in-setup3.yaml"); Map params = new HashMap(); @@ -193,14 +174,11 @@ public void testReturnTrueInSecondSetupStepAndNormalStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); // one steps assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); @@ -218,34 +196,29 @@ public void testHiddenInputsAndDefaultValues() throws Exception { AbstractJob job = createJobFromWdl(app, inputs); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); - - //check step ouputs - assertEquals("text1: my-value\n", job.getSteps().get(0).getLogMessages().get(0).getMessage()); - assertEquals("checkbox1: true\n", job.getSteps().get(1).getLogMessages().get(0).getMessage()); - assertEquals("list1: value1\n", job.getSteps().get(2).getLogMessages().get(0).getMessage()); - assertEquals("text2: my-value\n", job.getSteps().get(3).getLogMessages().get(0).getMessage()); - assertEquals("checkbox2: true\n", job.getSteps().get(4).getLogMessages().get(0).getMessage()); - assertEquals("list2: value1\n", job.getSteps().get(5).getLogMessages().get(0).getMessage()); - - + + // check step ouputs + assertEquals("text1: my-value\n", job.getSteps().get(0).getLogMessages().get(0).getMessage()); + assertEquals("checkbox1: true\n", job.getSteps().get(1).getLogMessages().get(0).getMessage()); + assertEquals("list1: value1\n", job.getSteps().get(2).getLogMessages().get(0).getMessage()); + assertEquals("text2: my-value\n", job.getSteps().get(3).getLogMessages().get(0).getMessage()); + assertEquals("checkbox2: true\n", job.getSteps().get(4).getLogMessages().get(0).getMessage()); + assertEquals("list2: value1\n", job.getSteps().get(5).getLogMessages().get(0).getMessage()); + } - + @Test public void testReturnWriteFileInSecondSetupStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - String myContent = "test-test-test-test-text"; WdlApp app = WdlReader.loadAppFromFile("test-data/write-file-in-setup.yaml"); @@ -255,7 +228,7 @@ public void testReturnWriteFileInSecondSetupStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -273,7 +246,7 @@ public void testReturnWriteFileInSecondSetupStep() throws Exception { job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertEquals(AbstractJob.STATE_FAILED, job.getState()); @@ -285,10 +258,6 @@ public void testReturnWriteFileInSecondSetupStep() throws Exception { content = FileUtil.readFileAsString(filename); assertEquals(myContent, content); assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); // one steps assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); @@ -299,7 +268,7 @@ public void testReturnWriteFileInSecondSetupStep() throws Exception { public void testEmptyStepList() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/no-steps.yaml"); Map params = new HashMap(); @@ -307,15 +276,12 @@ public void testEmptyStepList() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); } @@ -323,7 +289,7 @@ public void testEmptyStepList() throws Exception { public void testReturnFalseInSecondSetupStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/return-false-in-setup2.yaml"); Map params = new HashMap(); @@ -331,16 +297,12 @@ public void testReturnFalseInSecondSetupStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_FAILED, job.getState()); } @@ -348,7 +310,7 @@ public void testReturnFalseInSecondSetupStep() throws Exception { public void testWriteTextToFileJob() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/write-text-to-file.yaml"); Map params = new HashMap(); @@ -356,7 +318,7 @@ public void testWriteTextToFileJob() throws Exception { CloudgeneJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } @@ -366,9 +328,6 @@ public void testWriteTextToFileJob() throws Exception { String content = FileUtil.readFileAsString(filename); assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals("lukas_text", content); @@ -379,7 +338,7 @@ public void testWriteTextToFileJob() throws Exception { public void testWriteTextToFileOnFailureInStepJob() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/write-text-to-file-on-failure.yaml"); Map params = new HashMap(); @@ -387,7 +346,7 @@ public void testWriteTextToFileOnFailureInStepJob() throws Exception { CloudgeneJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } Thread.sleep(4000); @@ -396,9 +355,6 @@ public void testWriteTextToFileOnFailureInStepJob() throws Exception { String filename = FileUtil.path(settings.getLocalWorkspace(), path); String content = FileUtil.readFileAsString(filename); assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals("lukas_text", content); @@ -417,7 +373,7 @@ public void testWriteTextToFileOnFailureInSetupJob() throws Exception { CloudgeneJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } @@ -428,12 +384,8 @@ public void testWriteTextToFileOnFailureInSetupJob() throws Exception { String filename = FileUtil.path(settings.getLocalWorkspace(), path); String content = FileUtil.readFileAsString(filename); assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); - // steps not executed - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals("lukas_text", content); assertEquals(job.getState(), AbstractJob.STATE_FAILED); } @@ -450,7 +402,7 @@ public void testWriteTextToFileOnFailureInSecondSetupJob() throws Exception { CloudgeneJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } @@ -460,13 +412,10 @@ public void testWriteTextToFileOnFailureInSecondSetupJob() throws Exception { String path = job.getOutputParams().get(0).getFiles().get(0).getPath(); String filename = FileUtil.path(settings.getLocalWorkspace(), path); String content = FileUtil.readFileAsString(filename); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); // no steps executed assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals("lukas_text", content); assertEquals(job.getState(), AbstractJob.STATE_FAILED); } @@ -475,7 +424,7 @@ public void testWriteTextToFileOnFailureInSecondSetupJob() throws Exception { public void testThreeTasksStep() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/three-tasks.yaml"); Map params = new HashMap(); @@ -483,7 +432,7 @@ public void testThreeTasksStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } @@ -499,9 +448,6 @@ public void testThreeTasksStep() throws Exception { assertEquals("cloudgene-task3", messages.get(2).getMessage()); assertEquals(WorkflowContext.OK, messages.get(2).getType()); assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); } @@ -518,14 +464,13 @@ public void testWriteTextToStdOutStep() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); - String stdout = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), - "std.out"); + String stdout = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), "std.out"); String contentStdOut = FileUtil.readFileAsString(stdout); String log = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), "job.txt"); @@ -539,9 +484,6 @@ public void testWriteTextToStdOutStep() throws Exception { assertTrue(contentlog.contains("taks write to log2")); assertTrue(contentlog.contains("taks write to log3")); assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); } @@ -558,13 +500,10 @@ public void testApplicationLinks() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -576,7 +515,7 @@ public void testApplicationLinks() throws Exception { assertTrue(message.getMessage().contains("property3:hey3!")); } - + @Test public void testApplicationLinksWithoutAppsPrefix() throws Exception { @@ -589,13 +528,10 @@ public void testApplicationLinksWithoutAppsPrefix() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -607,12 +543,12 @@ public void testApplicationLinksWithoutAppsPrefix() throws Exception { assertTrue(message.getMessage().contains("property3:hey3!")); } - + @Test public void testApplicationLinksWithoutVersion() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/app-links.yaml"); Map params = new HashMap(); @@ -620,13 +556,10 @@ public void testApplicationLinksWithoutVersion() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -638,25 +571,22 @@ public void testApplicationLinksWithoutVersion() throws Exception { assertTrue(message.getMessage().contains("property3:hey3!")); } - + @Test public void testOptionalApplicationLinks() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/app-links-optional.yaml"); Map params = new HashMap(); AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); @@ -668,12 +598,12 @@ public void testOptionalApplicationLinks() throws Exception { assertFalse(message.getMessage().contains("property3:hey3!")); } - + @Test public void testApplicationLinksWrongApplication() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/app-links.yaml"); Map params = new HashMap(); @@ -681,15 +611,12 @@ public void testApplicationLinksWrongApplication() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); + assertTrue(job.getStartTime() > 0); + assertTrue(job.getEndTime() > 0); assertEquals(AbstractJob.STATE_FAILED, job.getState()); } @@ -698,7 +625,7 @@ public void testApplicationLinksWrongApplication() throws Exception { public void testApplicationLinksWrongPermissions() throws Exception { WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/app-links.yaml"); Map params = new HashMap(); @@ -706,151 +633,21 @@ public void testApplicationLinksWrongPermissions() throws Exception { AbstractJob job = createJobFromWdlAsUser(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(500); } assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getStartTime() == 0); - assertTrue(job.getEndTime() == 0); - assertEquals(AbstractJob.STATE_FAILED, job.getState()); - - } - - @Test - public void testApplicationInstallation() throws Exception { - - WorkflowEngine engine = application.getWorkflowEngine(); - - WdlApp app = WdlReader.loadAppFromFile("test-data/app-installation.yaml"); - - Map params = new HashMap(); - - AbstractJob job = createJobFromWdl(app, params); - job.forceInstallation(true); - engine.submit(job); - while (!job.isComplete()) { - Thread.sleep(500); - } - - assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); assertTrue(job.getStartTime() > 0); assertTrue(job.getEndTime() > 0); - assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); - - // single file - Message message = job.getSteps().get(0).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of metafile.txt")); - - // folder file1 - message = job.getSteps().get(1).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of file1.txt")); - - // folder file2 - message = job.getSteps().get(2).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of file2.txt")); - - // zip file1 - message = job.getSteps().get(3).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of file1.txt")); - - // zip file2 - message = job.getSteps().get(4).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of file2.txt")); - - // gz file1 - message = job.getSteps().get(5).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of file1.txt")); - - // gz file2 - message = job.getSteps().get(6).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of file2.txt")); - - // http single file - message = job.getSteps().get(7).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().contains("name: hello-cloudgene")); - } - - @Test - public void testApplicationInstallationAndLinks() throws Exception { - - WorkflowEngine engine = application.getWorkflowEngine(); - - WdlApp app = WdlReader.loadAppFromFile("test-data/app-installation2.yaml"); - - Map params = new HashMap(); - params.put("dataset", "apps@app-installation-child"); - - AbstractJob job = createJobFromWdl(app, params); - engine.submit(job); - while (!job.isComplete()) { - Thread.sleep(500); - } - - assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getStartTime() > 0); - assertTrue(job.getEndTime() > 0); - assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); - - Message message = job.getSteps().get(0).getLogMessages().get(0); - assertEquals(Message.OK, message.getType()); - assertTrue(message.getMessage().equals("content of metafile2.txt")); - } - - @Test - public void testApplicationLinksAndPropertyList() throws Exception { - - WorkflowEngine engine = application.getWorkflowEngine(); - - WdlApp app = WdlReader.loadAppFromFile("test-data/app-installation3.yaml"); - - Map params = new HashMap(); - params.put("dataset", "apps@app-installation-child"); - - AbstractJob job = createJobFromWdl(app, params); - engine.submit(job); - while (!job.isComplete()) { - Thread.sleep(500); - } - - assertTrue(job.getSubmittedOn() > 0); - assertTrue(job.getFinishedOn() > 0); - assertTrue(job.getSetupStartTime() > 0); - assertTrue(job.getSetupEndTime() > 0); - assertTrue(job.getStartTime() > 0); - assertTrue(job.getEndTime() > 0); - assertEquals(AbstractJob.STATE_SUCCESS, job.getState()); + assertEquals(AbstractJob.STATE_FAILED, job.getState()); } - - - // TODO: merge and zip export. - - // TODO: write to hdfs temp and local temp (temp output params)! - - // TODO: check if removehdfsworkspace works! // TODO: check cloudgene counters (successful and failed) public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs) throws Exception { - UserDao userDao = new UserDao(application.getDatabase()); + UserDao userDao = new UserDao(application.getDatabase()); User user = userDao.findByUsername("admin"); return createJobFromWdl(app, inputs, user); @@ -858,9 +655,9 @@ public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs) thr public CloudgeneJob createJobFromWdlAsUser(WdlApp app, Map inputs) throws Exception { - UserDao userDao = new UserDao(application.getDatabase()); + UserDao userDao = new UserDao(application.getDatabase()); User user = userDao.findByUsername("user"); - + return createJobFromWdl(app, inputs, user); } @@ -870,17 +667,15 @@ public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs, Use String id = "test_" + System.currentTimeMillis(); - String hdfsWorkspace = HdfsUtil.path(settings.getHdfsWorkspace(), id); String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), id); FileUtil.createDirectory(localWorkspace); CloudgeneJob job = new CloudgeneJob(user, id, app, inputs); job.setId(id); job.setName(id); + job.setWorkspace(workspaceFactory.getDefault()); job.setLocalWorkspace(localWorkspace); - job.setHdfsWorkspace(hdfsWorkspace); job.setSettings(settings); - job.setRemoveHdfsWorkspace(true); job.setApplication(app.getName() + " " + app.getVersion()); job.setApplicationId(app.getId()); diff --git a/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java b/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java index 77ad4a69..9db7f4c4 100644 --- a/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java +++ b/src/test/java/cloudgene/mapred/jobs/WrongWorkspaceTest.java @@ -11,10 +11,10 @@ import cloudgene.mapred.core.User; import cloudgene.mapred.database.JobDao; import cloudgene.mapred.database.UserDao; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; @@ -25,6 +25,9 @@ public class WrongWorkspaceTest { @Inject TestApplication application; + @Inject + WorkspaceFactory workspaceFactory; + @Test public void testReturnTrueStep() throws Exception { @@ -37,7 +40,7 @@ public void testReturnTrueStep() throws Exception { AbstractJob job = createJobFromWdl(app, inputs); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(1000); } Thread.sleep(10000); @@ -60,17 +63,15 @@ public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs) thr String id = "test_" + System.currentTimeMillis(); - String hdfsWorkspace = HdfsUtil.path("/gsfgdfgdf/vdadsadwa", id); String localWorkspace = FileUtil.path("/gsfgdfgdf/vdadsadwa", id); FileUtil.createDirectory(localWorkspace); CloudgeneJob job = new CloudgeneJob(user, id, app, inputs); job.setId(id); job.setName(id); + job.setWorkspace(workspaceFactory.getDefault()); job.setLocalWorkspace(localWorkspace); - job.setHdfsWorkspace(hdfsWorkspace); job.setSettings(settings); - job.setRemoveHdfsWorkspace(true); job.setApplication(app.getName() + " " + app.getVersion()); job.setApplicationId(app.getId()); diff --git a/src/test/java/cloudgene/mapred/jobs/steps/CheckHdfsInputs.java b/src/test/java/cloudgene/mapred/jobs/steps/CheckHdfsInputs.java deleted file mode 100644 index 4aa43042..00000000 --- a/src/test/java/cloudgene/mapred/jobs/steps/CheckHdfsInputs.java +++ /dev/null @@ -1,50 +0,0 @@ -package cloudgene.mapred.jobs.steps; - -import java.io.IOException; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.util.LineReader; - -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; -import genepi.hadoop.HdfsUtil; - -public class CheckHdfsInputs extends WorkflowStep { - - @Override - public boolean run(WorkflowContext context) { - - try { - - String fileContent = readFrom(context.get("file")); - - String fileContentInFolder1 = readFrom(context.get("folder") + "/" - + "test1.txt"); - String fileContentInFolder2 = readFrom(context.get("folder") + "/" - + "test2.txt"); - - boolean result = fileContent.equals("content-of-my-file") - && fileContentInFolder1 - .equals("content-of-my-file-in-folder1") - && fileContentInFolder2 - .equals("content-of-my-file-in-folder2"); - - return result; - } catch (Exception e) { - e.printStackTrace(); - return false; - } - } - - public String readFrom(String hdfs) throws IOException { - FileSystem fs = HdfsUtil.getFileSystem(); - LineReader reader = new LineReader(fs.open(new Path(hdfs))); - Text text = new Text(); - reader.readLine(text); - reader.close(); - return text.toString(); - } - -} diff --git a/src/test/java/cloudgene/mapred/jobs/steps/CheckInputs.java b/src/test/java/cloudgene/mapred/jobs/steps/CheckInputs.java index 87f71130..0252466a 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/CheckInputs.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/CheckInputs.java @@ -2,8 +2,8 @@ import java.io.File; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; import genepi.io.FileUtil; public class CheckInputs extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/LongSleepStep.java b/src/test/java/cloudgene/mapred/jobs/steps/LongSleepStep.java index 06c46a86..a22170bc 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/LongSleepStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/LongSleepStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class LongSleepStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/PrintDataset.java b/src/test/java/cloudgene/mapred/jobs/steps/PrintDataset.java index a752a32d..5231e8de 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/PrintDataset.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/PrintDataset.java @@ -2,8 +2,8 @@ import java.util.Map; -import genepi.hadoop.common.WorkflowContext; -import genepi.hadoop.common.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class PrintDataset extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/ReturnExceptionStep.java b/src/test/java/cloudgene/mapred/jobs/steps/ReturnExceptionStep.java index 516d5e54..9aafa0f0 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/ReturnExceptionStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/ReturnExceptionStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class ReturnExceptionStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/ReturnFalseStep.java b/src/test/java/cloudgene/mapred/jobs/steps/ReturnFalseStep.java index 43721111..007f08b5 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/ReturnFalseStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/ReturnFalseStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class ReturnFalseStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/ReturnTrueStep.java b/src/test/java/cloudgene/mapred/jobs/steps/ReturnTrueStep.java index 1012c777..39107831 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/ReturnTrueStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/ReturnTrueStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class ReturnTrueStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/ThreeTasksStep.java b/src/test/java/cloudgene/mapred/jobs/steps/ThreeTasksStep.java index 0149fe72..c7537f39 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/ThreeTasksStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/ThreeTasksStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class ThreeTasksStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToFolderStep.java b/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToFolderStep.java index 373db16a..6ca81bf1 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToFolderStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToFolderStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; import genepi.io.FileUtil; public class WriteFilesToFolderStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToHdfsFolderStep.java b/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToHdfsFolderStep.java deleted file mode 100644 index e7a7790c..00000000 --- a/src/test/java/cloudgene/mapred/jobs/steps/WriteFilesToHdfsFolderStep.java +++ /dev/null @@ -1,51 +0,0 @@ -package cloudgene.mapred.jobs.steps; - -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; - -public class WriteFilesToHdfsFolderStep extends WorkflowStep { - - @Override - public boolean run(WorkflowContext context) { - - String output = context.getOutput("output"); - String temp = context.getLocalTemp(); - - String text = context.getInput("inputtext"); - - try { - FileUtil.writeStringBufferToFile(FileUtil.path(temp, "file1.txt"), - new StringBuffer(text)); - HdfsUtil.put(FileUtil.path(temp, "file1.txt"), - HdfsUtil.path(output, "file1.txt")); - - FileUtil.writeStringBufferToFile(FileUtil.path(temp, "file2.txt"), - new StringBuffer(text)); - HdfsUtil.put(FileUtil.path(temp, "file2.txt"), - HdfsUtil.path(output, "file2.txt")); - - FileUtil.writeStringBufferToFile(FileUtil.path(temp, "file3.txt"), - new StringBuffer(text)); - HdfsUtil.put(FileUtil.path(temp, "file3.txt"), - HdfsUtil.path(output, "file3.txt")); - - FileUtil.writeStringBufferToFile(FileUtil.path(temp, "file4.txt"), - new StringBuffer(text)); - HdfsUtil.put(FileUtil.path(temp, "file4.txt"), - HdfsUtil.path(output, "file4.txt")); - - FileUtil.writeStringBufferToFile(FileUtil.path(temp, "file5.txt"), - new StringBuffer(text)); - HdfsUtil.put(FileUtil.path(temp, "file5.txt"), - HdfsUtil.path(output, "file5.txt")); - } catch (Exception e) { - e.printStackTrace(); - return false; - } - - return true; - } - -} diff --git a/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToFileStep.java b/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToFileStep.java index 2553312b..4f5dd1dd 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToFileStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToFileStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; import genepi.io.FileUtil; public class WriteTextToFileStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToHdfsFileStep.java b/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToHdfsFileStep.java deleted file mode 100644 index 9952d890..00000000 --- a/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToHdfsFileStep.java +++ /dev/null @@ -1,28 +0,0 @@ -package cloudgene.mapred.jobs.steps; - -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; -import genepi.hadoop.HdfsUtil; -import genepi.io.FileUtil; - -public class WriteTextToHdfsFileStep extends WorkflowStep { - - @Override - public boolean run(WorkflowContext context) { - - String temp = context.getLocalTemp(); - String hdfsFilename = context.getOutput("output"); - String text = context.getInput("inputtext"); - - try { - FileUtil.writeStringBufferToFile(FileUtil.path(temp, "temp.txt"), - new StringBuffer(text)); - HdfsUtil.put(FileUtil.path(temp, "temp.txt"), hdfsFilename); - } catch (Exception e) { - e.printStackTrace(); - return false; - } - return true; - } - -} diff --git a/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToStdOutStep.java b/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToStdOutStep.java index 4726ad7f..32f4e4bd 100644 --- a/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToStdOutStep.java +++ b/src/test/java/cloudgene/mapred/jobs/steps/WriteTextToStdOutStep.java @@ -1,7 +1,7 @@ package cloudgene.mapred.jobs.steps; -import cloudgene.sdk.internal.WorkflowContext; -import cloudgene.sdk.internal.WorkflowStep; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.sdk.WorkflowStep; public class WriteTextToStdOutStep extends WorkflowStep { diff --git a/src/test/java/cloudgene/mapred/steps/TestCommand.java b/src/test/java/cloudgene/mapred/steps/TestCommand.java index 0045c546..b4fc62a2 100644 --- a/src/test/java/cloudgene/mapred/steps/TestCommand.java +++ b/src/test/java/cloudgene/mapred/steps/TestCommand.java @@ -16,11 +16,11 @@ import cloudgene.mapred.jobs.CloudgeneJob; import cloudgene.mapred.jobs.Message; import cloudgene.mapred.jobs.WorkflowEngine; +import cloudgene.mapred.jobs.sdk.WorkflowContext; +import cloudgene.mapred.jobs.workspace.WorkspaceFactory; import cloudgene.mapred.util.Settings; import cloudgene.mapred.wdl.WdlApp; import cloudgene.mapred.wdl.WdlReader; -import cloudgene.sdk.internal.WorkflowContext; -import genepi.hadoop.HdfsUtil; import genepi.io.FileUtil; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import jakarta.inject.Inject; @@ -28,15 +28,17 @@ @MicronautTest public class TestCommand { - @Inject TestApplication application; + @Inject + WorkspaceFactory workspaceFactory; + @Test public void testValidCommand() throws Exception { - + WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/command/valid-command.yaml"); Map params = new HashMap(); @@ -44,7 +46,7 @@ public void testValidCommand() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(1000); } @@ -55,15 +57,13 @@ public void testValidCommand() throws Exception { assertEquals(messages.get(0).getType(), WorkflowContext.OK); assertTrue(messages.get(0).getMessage().contains("Execution successful.")); - String stdout = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), - "std.out"); + String stdout = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), "std.out"); String contentStdOut = FileUtil.readFileAsString(stdout); // simple ls result check assertTrue(contentStdOut.contains("invalid-command.yaml")); - String jobLog = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), - "job.txt"); + String jobLog = FileUtil.path(application.getSettings().getLocalWorkspace(), job.getId(), "job.txt"); String contentjobLog = FileUtil.readFileAsString(jobLog); // simple check if exit code = 0 @@ -73,9 +73,9 @@ public void testValidCommand() throws Exception { @Test public void testInvalidCommand() throws Exception { - + WorkflowEngine engine = application.getWorkflowEngine(); - + WdlApp app = WdlReader.loadAppFromFile("test-data/command/invalid-command.yaml"); Map params = new HashMap(); @@ -83,7 +83,7 @@ public void testInvalidCommand() throws Exception { AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - while (!job.isComplete()) { + while (job.isRunning()) { Thread.sleep(1000); } @@ -102,8 +102,8 @@ public void testInvalidCommand() throws Exception { * Map params = new HashMap(); * params.put("input", "input-file"); * - * AbstractJob job = createJobFromWdl(app, params); engine.submit(job); - * while (job.isRunning()) { Thread.sleep(1000); } + * AbstractJob job = createJobFromWdl(app, params); engine.submit(job); while + * (job.isRunning()) { Thread.sleep(1000); } * * assertEquals(AbstractJob.STATE_FAILED, job.getState()); * @@ -113,9 +113,8 @@ public void testInvalidCommand() throws Exception { * assertTrue(messages.get(0).getMessage().contains("Execution failed.")); * * String stdout = FileUtil.path(TestServer.getInstance().getSettings() - * .getLocalWorkspace(), job.getId(), "std.out"); - * System.out.println(stdout); String contentStdOut = - * FileUtil.readFileAsString(stdout); + * .getLocalWorkspace(), job.getId(), "std.out"); System.out.println(stdout); + * String contentStdOut = FileUtil.readFileAsString(stdout); * * //simple check for unrecognized option * assertTrue(contentStdOut.contains("unrecognized option")); @@ -132,22 +131,20 @@ public CloudgeneJob createJobFromWdl(WdlApp app, Map inputs) thr UserDao userDao = new UserDao(application.getDatabase()); User user = userDao.findByUsername("user"); - + Settings settings = application.getSettings(); String id = "test_" + System.currentTimeMillis(); - String hdfsWorkspace = HdfsUtil.path(settings.getHdfsWorkspace(), id); String localWorkspace = FileUtil.path(settings.getLocalWorkspace(), id); FileUtil.createDirectory(localWorkspace); CloudgeneJob job = new CloudgeneJob(user, id, app, inputs); job.setId(id); + job.setWorkspace(workspaceFactory.getDefault()); job.setName(id); job.setLocalWorkspace(localWorkspace); - job.setHdfsWorkspace(hdfsWorkspace); job.setSettings(settings); - job.setRemoveHdfsWorkspace(true); job.setApplication(app.getName() + " " + app.getVersion()); job.setApplicationId(app.getId()); diff --git a/src/test/java/cloudgene/mapred/util/TestCluster.java b/src/test/java/cloudgene/mapred/util/TestCluster.java deleted file mode 100644 index 070eda3d..00000000 --- a/src/test/java/cloudgene/mapred/util/TestCluster.java +++ /dev/null @@ -1,70 +0,0 @@ -package cloudgene.mapred.util; - -import genepi.hadoop.HdfsUtil; - -import java.io.File; -import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hdfs.HdfsConfiguration; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapred.ClusterStatus; -import org.apache.hadoop.mapred.JobClient; -import org.apache.hadoop.mapred.JobConf; -import org.apache.hadoop.mapred.MiniMRCluster; - -public class TestCluster { - - private static TestCluster instance; - - private static String WORKING_DIRECTORY = "test-cluster"; - - private MiniDFSCluster cluster; - - private FileSystem fs; - - private Configuration conf; - - private TestCluster() { - - } - - public static TestCluster getInstance() { - if (instance == null) { - instance = new TestCluster(); - } - return instance; - } - - public void start() throws IOException { - - if (cluster == null) { - - File testDataCluster1 = new File(WORKING_DIRECTORY); - if (testDataCluster1.exists()) { - testDataCluster1.delete(); - } - conf = new HdfsConfiguration(); - conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, - testDataCluster1.getAbsolutePath()); - cluster = new MiniDFSCluster.Builder(conf).build(); - fs = cluster.getFileSystem(); - - // set mincluster as default config - HdfsUtil.setDefaultConfiguration(conf); - System.setProperty("hadoop.log.dir", "test-log-dir"); - MiniMRCluster mrCluster = new MiniMRCluster(1, fs.getUri() - .toString(), 1, null, null, new JobConf(conf)); - JobConf mrClusterConf = mrCluster.createJobConf(); - HdfsUtil.setDefaultConfiguration(new Configuration(mrClusterConf)); - - System.out.println("------"); - - JobClient client = new JobClient(mrClusterConf); - ClusterStatus status = client.getClusterStatus(true); - System.out.println(status.getActiveTrackerNames()); - } - } - -} diff --git a/src/test/java/cloudgene/mapred/util/TestMailServer.java b/src/test/java/cloudgene/mapred/util/TestMailServer.java index f9c7c8b5..353d2580 100644 --- a/src/test/java/cloudgene/mapred/util/TestMailServer.java +++ b/src/test/java/cloudgene/mapred/util/TestMailServer.java @@ -1,11 +1,11 @@ package cloudgene.mapred.util; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; import com.dumbster.smtp.SimpleSmtpServer; import com.dumbster.smtp.SmtpMessage; -import com.google.common.collect.Lists; /** * A JUnit Rule which runs a mock SMTP server @@ -47,7 +47,12 @@ public synchronized Iterator getReceivedEmail() { } public List getReceivedEmailAsList() { - return Lists.newArrayList((Iterator) getReceivedEmail()); + Iterator iterator = (Iterator) getReceivedEmail(); + List list = new ArrayList(); + while (iterator.hasNext()) { + list.add(iterator.next()); + } + return list; } } \ No newline at end of file diff --git a/test-data/all-possible-inputs-hdfs.yaml b/test-data/all-possible-inputs-hdfs.yaml deleted file mode 100644 index 2f8fc8b7..00000000 --- a/test-data/all-possible-inputs-hdfs.yaml +++ /dev/null @@ -1,41 +0,0 @@ -name: WriteTextToFileStep -description:

This job tests your configuration in order to ensure that Cloudgene is able to communicate with your Hadoop Cluster.
If the job fails, please follow the error message and adapt your configuration until the job runs successfully.
Useful informations about the configuration can be found on our website http://cloudgene.uibk.ac.at. -version: 1.0.1 -website: http://cloudgene.uibk.ac.at -category: cloudgene - -mapred: - - steps: - - name: WriteTextToFileStep - classname: cloudgene.mapred.jobs.steps.CheckHdfsInputs - - inputs: - - - id: file - description: Input-file - type: hdfs-file - - - id: folder - description: Input-folder - type: hdfs-folder - - outputs: - - - id: outputFile - description: OutputFile - type: hdfs-file - download: true - temp: false - zip: false - removeHeader: false - mergeOutput: false - - - id: outputFolder - description: outputFolder - type: hdfs-folder - download: true - temp: false - zip: false - removeHeader: true - mergeOutput: true diff --git a/test-data/app-installation-child.yaml b/test-data/app-installation-child.yaml deleted file mode 100644 index 3c475f8e..00000000 --- a/test-data/app-installation-child.yaml +++ /dev/null @@ -1,13 +0,0 @@ -name: app-installation-child -category: app-installation-datasets -version: 1.0.0 -properties: - metafile: ${hdfs_app_folder}/metafile2.txt - populations: - eur: EUR - afr: AFR - asn: ASN -installation: - - import: - source: ${local_app_folder}/metafile2.txt - target: ${hdfs_app_folder}/metafile2.txt diff --git a/test-data/app-installation.yaml b/test-data/app-installation.yaml deleted file mode 100644 index cd07114c..00000000 --- a/test-data/app-installation.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: app-installation -version: 1.0.0 -installation: - - import: - source: ${local_app_folder}/metafile.txt - target: ${hdfs_app_folder}/metafile.txt - - import: - source: ${local_app_folder}/folder - target: ${hdfs_app_folder}/folder - - import: - source: ${local_app_folder}/folder.zip - target: ${hdfs_app_folder}/zip - - import: - source: ${local_app_folder}/folder.tar.gz - target: ${hdfs_app_folder}/gz - - import: - source: https://raw.githubusercontent.com/genepi/cloudgene-website/master/hello-cloudgene.yaml - target: ${hdfs_app_folder}/http.yaml -workflow: - steps: - - name: tests single file - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/metafile.txt - - - name: tests folder / file1 - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/folder/file1.txt - - name: tests folder / file2 - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/folder/subfolder/file2.txt - - - name: tests zip archive / file1 - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/zip/file1.txt - - name: tests zip archive / file2 - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/zip/subfolder/file2.txt - - - name: tests gz archive / file1 - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/gz/file1.txt - - name: tests gz archive / file2 - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/gz/subfolder/file2.txt - - - name: tests http file - type: groovy - script: print-hdfs-file.groovy - file: ${hdfs_app_folder}/http.yaml diff --git a/test-data/app-installation2.yaml b/test-data/app-installation2.yaml deleted file mode 100644 index 642fafdc..00000000 --- a/test-data/app-installation2.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: app-installation2 -version: 1.0.0 -workflow: - steps: - - name: print content of hfs file - type: groovy - script: print-hdfs-file-dataset.groovy - inputs: - - id: dataset - description: Dataset - type: app_list - category: app-installation-datasets diff --git a/test-data/app-installation3.yaml b/test-data/app-installation3.yaml deleted file mode 100644 index 377b9280..00000000 --- a/test-data/app-installation3.yaml +++ /dev/null @@ -1,11 +0,0 @@ -name: app-installation2 -version: 1.0.0 -workflow: - steps: - - name: print content of hfs file - classname: cloudgene.mapred.jobs.steps.PrintDataset - inputs: - - id: dataset - description: Dataset - type: app_list - category: app-installation-datasets diff --git a/test-data/print-hdfs-file-dataset.groovy b/test-data/print-hdfs-file-dataset.groovy deleted file mode 100644 index 28236d33..00000000 --- a/test-data/print-hdfs-file-dataset.groovy +++ /dev/null @@ -1,17 +0,0 @@ -import cloudgene.sdk.internal.WorkflowContext -import genepi.hadoop.HdfsUtil - -def run(WorkflowContext context) { - - def dataset = context.getData("dataset"); - def hdfs = dataset.get("metafile"); - def tempFile = context.getLocalTemp()+"/file.txt"; - - //export - HdfsUtil.get(hdfs, tempFile); - - def content = new File(tempFile).text; - context.ok(content); - - return true; -} diff --git a/test-data/print-hdfs-file.groovy b/test-data/print-hdfs-file.groovy deleted file mode 100644 index 62d454de..00000000 --- a/test-data/print-hdfs-file.groovy +++ /dev/null @@ -1,16 +0,0 @@ -import cloudgene.sdk.internal.WorkflowContext -import genepi.hadoop.HdfsUtil - -def run(WorkflowContext context) { - - def hdfs = context.getConfig("file"); - def tempFile = context.getLocalTemp()+"/file.txt"; - - //export - HdfsUtil.get(hdfs, tempFile); - - def content = new File(tempFile).text; - context.ok(content); - - return true; -} diff --git a/test-data/print-properties.groovy b/test-data/print-properties.groovy index 6bbbc198..7c495447 100644 --- a/test-data/print-properties.groovy +++ b/test-data/print-properties.groovy @@ -1,4 +1,4 @@ -import cloudgene.sdk.internal.WorkflowContext +import cloudgene.mapred.jobs.sdk.WorkflowContext def run(WorkflowContext context) { diff --git a/test-data/sftp-import.yaml b/test-data/sftp-import.yaml deleted file mode 100644 index 79eae817..00000000 --- a/test-data/sftp-import.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: sftp-import -description:

This job tests your configuration in order to ensure that Cloudgene is able to communicate with your Hadoop Cluster.
If the job fails, please follow the error message and adapt your configuration until the job runs successfully.
Useful informations about the configuration can be found on our website http://cloudgene.uibk.ac.at. -version: 1.0.1 -website: http://cloudgene.uibk.ac.at -category: cloudgene - - -mapred: - - setups: - - name: Import Files - classname: cloudgene.mapred.steps.SftpStep - - inputs: - - id: input - description: Dummy Input - type: local-file - - outputs: - - id: output - description: output - type: local-file - download: true - temp: false - zip: false - removeHeader: flase - mergeOutput: false diff --git a/test-data/write-files-to-hdfs-folder.yaml b/test-data/write-files-to-hdfs-folder.yaml deleted file mode 100644 index f96af95d..00000000 --- a/test-data/write-files-to-hdfs-folder.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: WriteFilesToHdfsFolderStep -description:

This job tests your configuration in order to ensure that Cloudgene is able to communicate with your Hadoop Cluster.
If the job fails, please follow the error message and adapt your configuration until the job runs successfully.
Useful informations about the configuration can be found on our website http://cloudgene.uibk.ac.at. -version: 1.0.1 -website: http://cloudgene.uibk.ac.at -category: cloudgene - -mapred: - - steps: - - name: WriteFilesToHdfsFolderStep - classname: cloudgene.mapred.jobs.steps.WriteFilesToHdfsFolderStep - - inputs: - - - id: inputtext - description: Input-Text - type: text - - outputs: - - - id: output - description: Output Folder - type: hdfs-folder - download: true - temp: false - zip: false - removeHeader: false - mergeOutput: false diff --git a/test-data/write-text-to-hdfs-file.yaml b/test-data/write-text-to-hdfs-file.yaml deleted file mode 100644 index 8d958633..00000000 --- a/test-data/write-text-to-hdfs-file.yaml +++ /dev/null @@ -1,28 +0,0 @@ -name: WriteTextToHdfsFileStep -description:

This job tests your configuration in order to ensure that Cloudgene is able to communicate with your Hadoop Cluster.
If the job fails, please follow the error message and adapt your configuration until the job runs successfully.
Useful informations about the configuration can be found on our website http://cloudgene.uibk.ac.at. -version: 1.0.1 -website: http://cloudgene.uibk.ac.at -category: cloudgene - -mapred: - - steps: - - name: WriteTextToHdfsFileStep - classname: cloudgene.mapred.jobs.steps.WriteTextToHdfsFileStep - - inputs: - - - id: inputtext - description: Input-Text - type: text - - outputs: - - - id: output - description: OutputFile - type: hdfs-file - download: true - temp: false - zip: false - removeHeader: false - mergeOutput: false