From eeedb8a40ca80dc4d84edfe49609e7c78acba7e2 Mon Sep 17 00:00:00 2001 From: Robert Baertsch Date: Mon, 11 Jul 2016 20:02:21 -0700 Subject: [PATCH 01/11] remove meteor method to send email --- webapp/server/lib/email.js | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/webapp/server/lib/email.js b/webapp/server/lib/email.js index 0e9ceb6..4444740 100644 --- a/webapp/server/lib/email.js +++ b/webapp/server/lib/email.js @@ -1,19 +1,3 @@ -Meteor.methods({ - sendEmail: function (to, subject, text) { - check([to, subject, text], [String]); - - // Let other method calls from the same client start running, - // without waiting for the email sending to complete. - this.unblock(); - console.log('user' , to, 'notified', subject); - Email.send({ - to: to, - from: "info@medbook.ucsc.edu", - subject: subject, - text: text - }); - } -}); Meteor.methods({ get_email : function(user_id) { check(user_id, String); From b0c703ef45517a16fbbdc6d7e2bee048b0e80d4a Mon Sep 17 00:00:00 2001 From: Teo Fleming Date: Mon, 11 Jul 2016 09:54:39 -0700 Subject: [PATCH 02/11] clinical importer works --- webapp/.meteor/versions | 5 +++-- webapp/packages/blobs | 2 +- webapp/packages/primary-collections | 2 +- webapp/packages/wrangler-collections | 2 +- webapp/server/startup.js | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/webapp/.meteor/versions b/webapp/.meteor/versions index 4dd990f..a75d412 100644 --- a/webapp/.meteor/versions +++ b/webapp/.meteor/versions @@ -16,13 +16,13 @@ cfs:collection-filters@0.2.4 cfs:data-man@0.0.6 cfs:file@0.1.17 cfs:gridfs@0.0.33 -cfs:http-methods@0.0.30 +cfs:http-methods@0.0.32 cfs:http-publish@0.0.13 cfs:power-queue@0.9.11 cfs:reactive-list@0.0.9 cfs:reactive-property@0.0.4 cfs:standard-packages@0.5.9 -cfs:storage-adapter@0.2.2 +cfs:storage-adapter@0.2.3 cfs:tempstore@0.1.5 cfs:upload-http@0.0.20 cfs:worker@0.1.4 @@ -44,6 +44,7 @@ less@1.0.14 livedata@1.0.13 localstorage@1.0.3 logging@1.0.7 +matb33:collection-hooks@0.8.1 medbook:adapters@0.0.2 medbook:blobs@0.0.1 medbook:namespace@0.0.2 diff --git a/webapp/packages/blobs b/webapp/packages/blobs index 585a441..f5e05bf 160000 --- a/webapp/packages/blobs +++ b/webapp/packages/blobs @@ -1 +1 @@ -Subproject commit 585a441f80782a8fad4aa15a876b6742d5d6d7b2 +Subproject commit f5e05bf2df6fe1488a7e755c5a91d652bb341afe diff --git a/webapp/packages/primary-collections b/webapp/packages/primary-collections index 896d518..b40b3db 160000 --- a/webapp/packages/primary-collections +++ b/webapp/packages/primary-collections @@ -1 +1 @@ -Subproject commit 896d518dfc83cdb7a527b14825e2275bea551a9b +Subproject commit b40b3db48e956b311d87509823e0ce352a8e509a diff --git a/webapp/packages/wrangler-collections b/webapp/packages/wrangler-collections index 95aa30c..f9f4d5d 160000 --- a/webapp/packages/wrangler-collections +++ b/webapp/packages/wrangler-collections @@ -1 +1 @@ -Subproject commit 95aa30c3b211d854afb34cc038fe21cfe10d031d +Subproject commit f9f4d5daafb40c103743b68c1621c552b0cc8e81 diff --git a/webapp/server/startup.js b/webapp/server/startup.js index ae2f086..8414e1b 100644 --- a/webapp/server/startup.js +++ b/webapp/server/startup.js @@ -186,7 +186,7 @@ Meteor.startup(function () { // make sure the DataSets collection is okay and we can still wrangle things DataSets.update({}, { $set: { - gene_expression_wrangling: false + currently_wrangling: false } }, {multi: true}); From a67d79d9f4248f492bbbce59fe1de0343af22ea6 Mon Sep 17 00:00:00 2001 From: Teo Fleming Date: Wed, 13 Jul 2016 19:12:58 -0700 Subject: [PATCH 03/11] DataSets/Studies, bugfixes --- config/teo/run.sh | 10 +---- config/teo/settings.json | 16 +++---- external-tools | 2 +- webapp/.meteor/.gitignore | 1 + webapp/packages/blobs | 2 +- webapp/packages/primary-collections | 2 +- webapp/packages/referential-integrity | 2 +- webapp/packages/wrangler-collections | 2 +- .../classes/FinishWranglerSubmission.js | 17 ++++++- webapp/server/classes/ParseWranglerFile.js | 44 +++++++------------ webapp/server/classes/RunLimmaGSEA.js | 31 ++++++++++--- webapp/server/lib/email.js | 44 ------------------- 12 files changed, 72 insertions(+), 101 deletions(-) delete mode 100644 webapp/server/lib/email.js diff --git a/config/teo/run.sh b/config/teo/run.sh index bcdb72a..70d171f 100755 --- a/config/teo/run.sh +++ b/config/teo/run.sh @@ -1,10 +1,4 @@ export MONGO_URL="mongodb://localhost:27017/MedBook" +export MEDBOOK_FILESTORE=/tmp/filestore -export settings_file="../config/teo/settings.json" -export port="3003" - -if [ -z "$1" ]; then - meteor --port $port --settings $settings_file -else - meteor $1 --port $port --settings $settings_file -fi +meteor --port 3003 --settings ../config/teo/settings.json diff --git a/config/teo/settings.json b/config/teo/settings.json index 12734a1..7480264 100644 --- a/config/teo/settings.json +++ b/config/teo/settings.json @@ -1,12 +1,12 @@ { "sh": "/bin/sh", "rscript": "Rscript", - "limma_path": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/limma/limma_ng.R", - "outlier_analysis": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/OutlierAnalysis/outlier-analysis.sh", - "calculate_outlier_genes": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/OutlierAnalysis/calculate_outlier_genes.R", - "gene_expression_export": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/exporters/gene_expression_export.py", - "gene_set_collection_export": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/exporters/gene_set_collection_export.py", - "limma_phenotype_export": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/exporters/limma_phenotype_export.py", - "gsea_path": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/gsea/rgGSEA.py", - "gsea_jar_path": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/gsea/gsea2-2.2.2.jar" + "limma_path": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/limma/limma_ng.R", + "outlier_analysis": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/OutlierAnalysis/outlier-analysis.sh", + "calculate_outlier_genes": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/OutlierAnalysis/calculate_outlier_genes.R", + "gene_expression_export": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/exporters/gene_expression_export.py", + "gene_set_collection_export": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/exporters/gene_set_collection_export.py", + "limma_phenotype_export": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/exporters/limma_phenotype_export.py", + "gsea_path": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/gsea/rgGSEA.py", + "gsea_jar_path": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/gsea/gsea2-2.2.2.jar" } diff --git a/external-tools b/external-tools index b8015b6..6474b81 160000 --- a/external-tools +++ b/external-tools @@ -1 +1 @@ -Subproject commit b8015b6d3230dd55552d5546b9e960a6ddbd3444 +Subproject commit 6474b815a1465c00e8b55594c508343ade45c359 diff --git a/webapp/.meteor/.gitignore b/webapp/.meteor/.gitignore index 4083037..2b8ab36 100644 --- a/webapp/.meteor/.gitignore +++ b/webapp/.meteor/.gitignore @@ -1 +1,2 @@ local +dev_bundle diff --git a/webapp/packages/blobs b/webapp/packages/blobs index f5e05bf..59c6a50 160000 --- a/webapp/packages/blobs +++ b/webapp/packages/blobs @@ -1 +1 @@ -Subproject commit f5e05bf2df6fe1488a7e755c5a91d652bb341afe +Subproject commit 59c6a50a7d7a7b426c4cc477192dc02e7e7943fc diff --git a/webapp/packages/primary-collections b/webapp/packages/primary-collections index b40b3db..6ea388e 160000 --- a/webapp/packages/primary-collections +++ b/webapp/packages/primary-collections @@ -1 +1 @@ -Subproject commit b40b3db48e956b311d87509823e0ce352a8e509a +Subproject commit 6ea388e8eab7ade027ff92c6c4a7a1305599178f diff --git a/webapp/packages/referential-integrity b/webapp/packages/referential-integrity index b6e7be3..8763409 160000 --- a/webapp/packages/referential-integrity +++ b/webapp/packages/referential-integrity @@ -1 +1 @@ -Subproject commit b6e7be3e8ac83368bd64ccac6778eb4a5eba13f6 +Subproject commit 8763409b5b787916fd000080153b15f78d8870c2 diff --git a/webapp/packages/wrangler-collections b/webapp/packages/wrangler-collections index f9f4d5d..4301085 160000 --- a/webapp/packages/wrangler-collections +++ b/webapp/packages/wrangler-collections @@ -1 +1 @@ -Subproject commit f9f4d5daafb40c103743b68c1621c552b0cc8e81 +Subproject commit 4301085753dfe1356a0b3ce5fae65c95b6e50aa2 diff --git a/webapp/server/classes/FinishWranglerSubmission.js b/webapp/server/classes/FinishWranglerSubmission.js index 53a2cc5..a9d9ff0 100644 --- a/webapp/server/classes/FinishWranglerSubmission.js +++ b/webapp/server/classes/FinishWranglerSubmission.js @@ -38,8 +38,21 @@ FinishWranglerSubmission.prototype.run = function () { written_to_database: {$ne: true}, }); if (notWrittenCursor.count() > 0) { - this.retry("files not done being written"); - return; + var errorWritingFiles = notWrittenCursor.map(function (wranglerFile) { + var job = Jobs.findOne({ + name: "ParseWranglerFile", + "args.wrangler_file_id": wranglerFile._id + }); + + return !job || job.status === "error"; + }); + + if (errorWritingFiles.indexOf(true) !== -1) { + throw "Internal error writing files to database"; + } else { + this.retry("files not done being written"); + return; + } } // we did it! diff --git a/webapp/server/classes/ParseWranglerFile.js b/webapp/server/classes/ParseWranglerFile.js index 35661d9..2b865a0 100644 --- a/webapp/server/classes/ParseWranglerFile.js +++ b/webapp/server/classes/ParseWranglerFile.js @@ -55,36 +55,25 @@ ParseWranglerFile.prototype.run = function () { // try to guess file_type if (!options.file_type) { - if (extensionEquals(".vcf")) { - setFileOptions({ file_type: "MutationVCF" }); - } else if (blobName.match(/\.rsem\.genes\.[a-z_]*\.(hugo\.|)tab/g)) { + if (blobName.match(/\.rsem\.genes\.[a-z_]*\.(hugo\.|)tab/g)) { // http://regexr.com/3d9i7 - setFileOptions({ file_type: "RectangularGeneExpression" }); - } else if (blobName.match(/\.rsem\.isoform\.[a-z_]*\.tab/g)) { - setFileOptions({ file_type: "RectangularIsoformExpression" }); - } else if (extensionEquals(".xls") || extensionEquals("xlsx")) { - setFileOptions({ file_type: "BasicClinical" }); + setFileOptions({ file_type: "RectGenomicExpression" }); } } - // try to guess normalization - if (!options.normalization) { - // try to guess normalization - if (blobName.match(/raw_counts/g)) { - setFileOptions({ normalization: "raw_counts" }); - } else if (blobName.match(/norm_counts/g)) { - setFileOptions({ normalization: "quantile_counts" }); - } else if (blobName.match(/norm_tpm/g)) { - setFileOptions({ normalization: "tpm" }); - } else if (blobName.match(/norm_fpkm/g)) { - setFileOptions({ normalization: "fpkm" }); - } - } - - // force certain options - if (options.file_type === "TCGAGeneExpression") { - setFileOptions({ normalization: "quantile_counts" }); - } + // // try to guess normalization + // if (!options.normalization) { + // // try to guess normalization + // if (blobName.match(/raw_counts/g)) { + // setFileOptions({ normalization: "raw_counts" }); + // } else if (blobName.match(/norm_counts/g)) { + // setFileOptions({ normalization: "quantile_counts" }); + // } else if (blobName.match(/norm_tpm/g)) { + // setFileOptions({ normalization: "tpm" }); + // } else if (blobName.match(/norm_fpkm/g)) { + // setFileOptions({ normalization: "fpkm" }); + // } + // } if (self.blob.metadata && self.blob.metadata.wrangler_file_options) { setFileOptions(self.blob.metadata.wrangler_file_options); @@ -97,7 +86,8 @@ ParseWranglerFile.prototype.run = function () { // make sure we've got a file_type if (!options.file_type) { - throw "File type could not be inferred. Please manually select a file type"; + throw "File type could not be inferred. " + + "Please manually select a file type"; } var fileHandlerClass = WranglerFileHandlers[options.file_type]; diff --git a/webapp/server/classes/RunLimmaGSEA.js b/webapp/server/classes/RunLimmaGSEA.js index c04db9f..d95da00 100644 --- a/webapp/server/classes/RunLimmaGSEA.js +++ b/webapp/server/classes/RunLimmaGSEA.js @@ -36,26 +36,43 @@ RunLimmaGSEA.prototype.run = function () { // combine samples of same data set into single array var dataSetHash = {}; _.each(groupA.data_sets.concat(groupB.data_sets), function (dataSet) { - var oldSamples = dataSetHash[dataSet.data_set_id]; - if (!oldSamples) { - oldSamples = []; + // check if we've seen this data set already + var seenAlready = dataSetHash[dataSet.data_set_id]; + if (!seenAlready) { + // if we haven't, set it up + seenAlready = { + data_set_name: dataSet.data_set_name, + sample_labels: [], + }; } - dataSetHash[dataSet.data_set_id] = oldSamples.concat(dataSet.sample_labels); + // combine the samples together + seenAlready.sample_labels = + seenAlready.sample_labels.concat(dataSet.sample_labels) + dataSetHash[dataSet.data_set_id] = seenAlready; }); var comboSampleGroupDataSets = _.map(dataSetHash, - function (sample_labels, data_set_id) { + function (samplesAndName, data_set_id) { return { data_set_id: data_set_id, - sample_labels: sample_labels, + data_set_name: samplesAndName.data_set_name, + sample_labels: samplesAndName.sample_labels, + + // I think we can fake this + unfiltered_sample_count: 1, }; }); + console.log("comboSampleGroupDataSets:", comboSampleGroupDataSets); + var comboSampleGroupId = SampleGroups.insert({ name: "temp - created in RunLimmaGSEA to call an adapter", version: 1, - collaborations: [], // invisible data_sets: comboSampleGroupDataSets, + value_type: groupA.value_type, + + // invisible + collaborations: [], }); // star the promise chain: woohoo! diff --git a/webapp/server/lib/email.js b/webapp/server/lib/email.js deleted file mode 100644 index 0e9ceb6..0000000 --- a/webapp/server/lib/email.js +++ /dev/null @@ -1,44 +0,0 @@ -Meteor.methods({ - sendEmail: function (to, subject, text) { - check([to, subject, text], [String]); - - // Let other method calls from the same client start running, - // without waiting for the email sending to complete. - this.unblock(); - console.log('user' , to, 'notified', subject); - Email.send({ - to: to, - from: "info@medbook.ucsc.edu", - subject: subject, - text: text - }); - } -}); -Meteor.methods({ - get_email : function(user_id) { - check(user_id, String); - var user = ""; - if (user_id) { - try { - Users = new Meteor.Collection("users"); - } - catch(err) { - console.log('error creating users collections', err); - } - user = Users.findOne({_id:user_id}); - try { - return user.profile.email; - } - catch(err) { - try { - return user.emails[0].address; - } - catch(err) { - if (user.services) { - return user.services.google.email; - } - } - } - } - } -}); From a1afc9e3279310c0be22fa3653c3c9d26b045e2d Mon Sep 17 00:00:00 2001 From: Teo Fleming Date: Wed, 13 Jul 2016 23:12:53 -0700 Subject: [PATCH 04/11] update packages: bugfixes --- webapp/packages/primary-collections | 2 +- webapp/packages/referential-integrity | 2 +- webapp/packages/wrangler-collections | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/webapp/packages/primary-collections b/webapp/packages/primary-collections index 6ea388e..317bea7 160000 --- a/webapp/packages/primary-collections +++ b/webapp/packages/primary-collections @@ -1 +1 @@ -Subproject commit 6ea388e8eab7ade027ff92c6c4a7a1305599178f +Subproject commit 317bea735ee997b2167bd026f7d29f7018446e39 diff --git a/webapp/packages/referential-integrity b/webapp/packages/referential-integrity index 8763409..9462815 160000 --- a/webapp/packages/referential-integrity +++ b/webapp/packages/referential-integrity @@ -1 +1 @@ -Subproject commit 8763409b5b787916fd000080153b15f78d8870c2 +Subproject commit 946281562ba5c54337a6a8f85b55ab5efc99c651 diff --git a/webapp/packages/wrangler-collections b/webapp/packages/wrangler-collections index 4301085..643cc69 160000 --- a/webapp/packages/wrangler-collections +++ b/webapp/packages/wrangler-collections @@ -1 +1 @@ -Subproject commit 4301085753dfe1356a0b3ce5fae65c95b6e50aa2 +Subproject commit 643cc69032016c56c44faf5be0e8deade30ed874 From 74d727ecb6b12f1d863c9cc73e0ae9f3da546b15 Mon Sep 17 00:00:00 2001 From: Teo Fleming Date: Thu, 14 Jul 2016 11:13:00 -0700 Subject: [PATCH 05/11] bugfix: renamed script --- config/teo/settings.json | 2 +- webapp/server/classes/RunLimmaGSEA.js | 3 ++- webapp/server/classes/UpDownGenes.js | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/config/teo/settings.json b/config/teo/settings.json index 7480264..5bac9ec 100644 --- a/config/teo/settings.json +++ b/config/teo/settings.json @@ -4,7 +4,7 @@ "limma_path": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/limma/limma_ng.R", "outlier_analysis": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/OutlierAnalysis/outlier-analysis.sh", "calculate_outlier_genes": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/OutlierAnalysis/calculate_outlier_genes.R", - "gene_expression_export": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/exporters/gene_expression_export.py", + "genomic_expression_export": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/exporters/genomic_expression_export.py", "gene_set_collection_export": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/exporters/gene_set_collection_export.py", "limma_phenotype_export": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/exporters/limma_phenotype_export.py", "gsea_path": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/gsea/rgGSEA.py", diff --git a/webapp/server/classes/RunLimmaGSEA.js b/webapp/server/classes/RunLimmaGSEA.js index d95da00..adb83af 100644 --- a/webapp/server/classes/RunLimmaGSEA.js +++ b/webapp/server/classes/RunLimmaGSEA.js @@ -94,7 +94,7 @@ RunLimmaGSEA.prototype.run = function () { // write mongo data to files // expression data to a file for use in Limma - spawnCommand(getSetting("gene_expression_export"), [ + spawnCommand(getSetting("genomic_expression_export"), [ "--sample_group_id", comboSampleGroupId, ], workDir), // phenotype file for Limma @@ -117,6 +117,7 @@ RunLimmaGSEA.prototype.run = function () { }); // save the file paths... order maters for spawnResults + // (the order depends on the order of `spawnCommand`s in `Q.all`) var expressionDataPath = spawnResults[0].stdoutPath; var limmaPhenotypePath = spawnResults[1].stdoutPath; diff --git a/webapp/server/classes/UpDownGenes.js b/webapp/server/classes/UpDownGenes.js index 53bae07..7457709 100644 --- a/webapp/server/classes/UpDownGenes.js +++ b/webapp/server/classes/UpDownGenes.js @@ -12,7 +12,7 @@ UpDownGenes.prototype.run = function () { var deferred = Q.defer(); var self = this; - var exportScript = getSetting("gene_expression_export"); + var exportScript = getSetting("genomic_expression_export"); Q.all([ // single sample data spawnCommand(exportScript, [ From 95f1f3da2ec4a778bd942e0fb4a5c64b31d8a797 Mon Sep 17 00:00:00 2001 From: Robert Baertsch Date: Thu, 14 Jul 2016 11:29:01 -0700 Subject: [PATCH 06/11] getting started --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..06cee4f --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +# MedBook-JobRunner +## Runs and monitors Jobs (Current UNIX processes, Galaxy and other Environments coming) +### steps to adding a new tools + +1. create a feature branch in git +2. look at https://github.com/UCSC-MedBook/MedBook-JobRunner/blob/f-gsea-new/webapp/server/classes/RunLimmaGSEA.js +3. create a new class +4. add adapters (importers and exporters) to convert from MedBook objects to files that tools understand and store check them into external-tools +4. add external code to external-tools repo (or mechansim to install it) +5. add pointers to external code in settings.json +6. add gui to appropriate MedBook app, that initiates job by inserting into jobs collection + for example: + Jobs.insert({ + name: "UpDownGenes", + status: "waiting", + user_id: user._id, + collaborations: [ user.personalCollaboration() ], + args + }); +7. read errors from jobs.error_description From ae5c5b7962d15544c6527206020661b54dc744c9 Mon Sep 17 00:00:00 2001 From: Teo Fleming Date: Thu, 14 Jul 2016 14:13:07 -0700 Subject: [PATCH 07/11] patient-care = home page --- webapp/server/classes/UpDownGenes.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/webapp/server/classes/UpDownGenes.js b/webapp/server/classes/UpDownGenes.js index 7457709..c30acd5 100644 --- a/webapp/server/classes/UpDownGenes.js +++ b/webapp/server/classes/UpDownGenes.js @@ -116,7 +116,7 @@ UpDownGenes.prototype.onSuccess = function (result) { check(emailAddress, String); var resultsID = self.job._id; check(resultsID, String); - var resultsURL = "https://medbook.io/patient-care/tools/outlier-analysis/" + resultsID; + var resultsURL = "https://medbook.io/tools/outlier-analysis/" + resultsID; Email.send({ to: emailAddress, @@ -126,8 +126,8 @@ UpDownGenes.prototype.onSuccess = function (result) { "'>" + resultsURL + "" , }); - console.log("Notification email sent for job ", self.job._id); - + console.log("Notification email sent for job ", self.job._id); + }; From 744731d726f76795410aa0e1fd6d0a702a8d55c0 Mon Sep 17 00:00:00 2001 From: Teo Fleming Date: Fri, 15 Jul 2016 10:04:56 -0700 Subject: [PATCH 08/11] removed old run scripts --- config/development/env.sh | 1 - config/development/run.autostart | 13 ------------- config/development/settings.json | 14 -------------- config/starrynight-teo/run.sh | 10 ---------- config/starrynight-teo/settings.json | 4 ---- config/su2c-dev/run.sh | 10 ---------- config/su2c-dev/settings.json | 7 ------- config/teo-medbook/settings.json | 4 ---- 8 files changed, 63 deletions(-) delete mode 100644 config/development/env.sh delete mode 100755 config/development/run.autostart delete mode 100644 config/development/settings.json delete mode 100755 config/starrynight-teo/run.sh delete mode 100644 config/starrynight-teo/settings.json delete mode 100755 config/su2c-dev/run.sh delete mode 100644 config/su2c-dev/settings.json delete mode 100644 config/teo-medbook/settings.json diff --git a/config/development/env.sh b/config/development/env.sh deleted file mode 100644 index 65dd726..0000000 --- a/config/development/env.sh +++ /dev/null @@ -1 +0,0 @@ -export MONGO_URL="mongodb://localhost:27017/MedBook" diff --git a/config/development/run.autostart b/config/development/run.autostart deleted file mode 100755 index 7575ccd..0000000 --- a/config/development/run.autostart +++ /dev/null @@ -1,13 +0,0 @@ -# export NODE_OPTIONS='--debug' -export MEDBOOK_SCRIPTS=`pwd`/scripts/ -export MEDBOOK_WORKSPACE=`pwd`/workspace/ -echo MEDBOOK_WORKSPACE is $MEDBOOK_WORKSPACE -export MAIL_URL="smtp://localhost" - -cd webapp - -export MONGO_URL=mongodb://localhost:27017/MedBook -export ROOT_URL=https://su2c-dev.ucsc.edu/JobRunner/ -export ROOT_URL_PATH_PREFIX=JobRunner -echo $MONGO_URL $ROOT_URL $ROOT_URL_PATH_PREFIX -meteor --port $PORT --settings settings.json >& "/data/MedBook/logs/log.job.runner.$$" & diff --git a/config/development/settings.json b/config/development/settings.json deleted file mode 100644 index 91b3271..0000000 --- a/config/development/settings.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "rscript": "Rscript", - "python": "python", - "limma_path": "/data/MedBook/External-Tools/limma/limma_ng.R", - "apply_signature_path" : "/data/MedBook/External-Tools/signature/apply_signature.py", - "viper_supervised_path": "/data/MedBook/UCSC_VIPER/bin/run-viper-supervised.R", - "viper_unsupervised_path": "/data/MedBook/UCSC_VIPER/bin/run-viper-unsupervised.R", - "viper_path": "/data/MedBook/UCSC_VIPER/bin/run-viper.R", - "marina_path": "/data/MedBook/UCSC_VIPER/bin/run-marina.R, - "gsea_path": "/data/MedBook/External-Tools/gsea/rgGSEA.py" - "boxplot_new_path": "/data/MedBook/Tools/MedBook-Adapters/boxplot//boxplot.new.R - "boxplot_path": "/data/MedBook/Tools/MedBook-Adapters/boxplot//boxplot.R - "violinplot_path": "/data/MedBook/Tools/MedBook-Adapters/boxplot//violinplot.R -} diff --git a/config/starrynight-teo/run.sh b/config/starrynight-teo/run.sh deleted file mode 100755 index 3db363e..0000000 --- a/config/starrynight-teo/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -export MONGO_URL="mongodb://localhost:27017/MedBook" - -export settings_file="../config/starrynight-teo/settings.json" -export port="3003" - -if [ -z "$1" ]; then - meteor --port $port --settings $settings_file -else - meteor $1 --port $port --settings $settings_file -fi diff --git a/config/starrynight-teo/settings.json b/config/starrynight-teo/settings.json deleted file mode 100644 index 66d02c0..0000000 --- a/config/starrynight-teo/settings.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "rscript": "/bin/sh", - "limma_path": "/Users/mokolodi1/work/medbook/job-runner/travis-tools/limma.sh" -} diff --git a/config/su2c-dev/run.sh b/config/su2c-dev/run.sh deleted file mode 100755 index 5ad9f16..0000000 --- a/config/su2c-dev/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -export MONGO_URL="mongodb://localhost:27042/MedBook" - -export settings_file="../config/su2c-dev/settings.json" -export port="3003" - -if [ -z "$1" ]; then - meteor --port $port --settings $settings_file -else - meteor $1 --port $port --settings $settings_file -fi diff --git a/config/su2c-dev/settings.json b/config/su2c-dev/settings.json deleted file mode 100644 index 2fc71d4..0000000 --- a/config/su2c-dev/settings.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "sh": "/bin/sh", - "rscript": "Rscript", - "limma_path": "/data/home/dtflemin/core-genomic/External-Tools/limma/limma_ng.R", - "up_down_genes": "/data/home/dtflemin/core-genomic/External-Tools/UpDownGenes/up_down_genes.sh", - "calculate_outlier_genes": "/data/home/dtflemin/core-genomic/External-Tools/UpDownGenes/calculate_outlier_genes.R" -} diff --git a/config/teo-medbook/settings.json b/config/teo-medbook/settings.json deleted file mode 100644 index 9bf2411..0000000 --- a/config/teo-medbook/settings.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "outlier_analysis": "/home/ubuntu/medbook/MedBook-JobRunner/external-tools/OutlierAnalysis/outlier-analysis.sh", - "calculate_outlier_genes": "/home/ubuntu/medbook/MedBook-JobRunner/external-tools/OutlierAnalysis/calculate_outlier_genes.R", -} From 833c29a0aefc152709aeaaf95044eca08f08513b Mon Sep 17 00:00:00 2001 From: Teo Fleming Date: Sat, 16 Jul 2016 11:41:27 -0700 Subject: [PATCH 09/11] v1.0.2: cache outlier analysis intermediary files --- external-tools | 2 +- webapp/packages/blobs | 2 +- webapp/packages/wrangler-collections | 2 +- webapp/server/classes/RunLimmaGSEA.js | 2 +- webapp/server/classes/UpDownGenes.js | 132 +++++++++++++++++++------- webapp/server/lib/globals.js | 46 ++++----- 6 files changed, 124 insertions(+), 62 deletions(-) diff --git a/external-tools b/external-tools index 6474b81..41a1626 160000 --- a/external-tools +++ b/external-tools @@ -1 +1 @@ -Subproject commit 6474b815a1465c00e8b55594c508343ade45c359 +Subproject commit 41a162665b30e6fa13954d504488b3d1090c6d36 diff --git a/webapp/packages/blobs b/webapp/packages/blobs index 59c6a50..6fc0ad8 160000 --- a/webapp/packages/blobs +++ b/webapp/packages/blobs @@ -1 +1 @@ -Subproject commit 59c6a50a7d7a7b426c4cc477192dc02e7e7943fc +Subproject commit 6fc0ad876fbb990b76288717f9f962482f624c84 diff --git a/webapp/packages/wrangler-collections b/webapp/packages/wrangler-collections index 643cc69..46ae785 160000 --- a/webapp/packages/wrangler-collections +++ b/webapp/packages/wrangler-collections @@ -1 +1 @@ -Subproject commit 643cc69032016c56c44faf5be0e8deade30ed874 +Subproject commit 46ae7856efe0824fc07f600f2350983b938ce310 diff --git a/webapp/server/classes/RunLimmaGSEA.js b/webapp/server/classes/RunLimmaGSEA.js index adb83af..c610eed 100644 --- a/webapp/server/classes/RunLimmaGSEA.js +++ b/webapp/server/classes/RunLimmaGSEA.js @@ -214,7 +214,7 @@ RunLimmaGSEA.prototype.run = function () { Blobs2.create(path.join(gseaOutput, fileName), { collection_name: "Jobs", mongo_id: self.job._id, - }, function (err, out) { + }, {}, function (err, out) { if (err) { console.log("err:", err); def.reject("Error inserting blob: " + fileName); diff --git a/webapp/server/classes/UpDownGenes.js b/webapp/server/classes/UpDownGenes.js index c30acd5..9292ddc 100644 --- a/webapp/server/classes/UpDownGenes.js +++ b/webapp/server/classes/UpDownGenes.js @@ -12,58 +12,119 @@ UpDownGenes.prototype.run = function () { var deferred = Q.defer(); var self = this; + // if the first parts of this command have already been run, + // grab the paths for those output files + var sample_group_id = self.job.args.sample_group_id; + var iqr_multiplier = self.job.args.iqr_multiplier; + var associated_object = { + collection_name: "SampleGroups", + mongo_id: sample_group_id, + }; + + function getStoragePath(file_name) { + var blob = Blobs2.findOne({ + file_name: file_name, + associated_object: associated_object, + "metadata.iqr_multiplier": iqr_multiplier, + }); + + if (blob) { + return blob.getFilePath(); + } + } + var medianPath = getStoragePath("median.tsv"); + var highThresholdPath = getStoragePath("highthreshold.tsv"); + var lowThresholdPath = getStoragePath("lowthreshold.tsv"); + var regenerateFiles = false; + + // medianPath is the one to check if they exist or not so make sure + // it's null if any of them don't exist + if (!medianPath || !highThresholdPath || !lowThresholdPath) { + regenerateFiles = true; + } + var exportScript = getSetting("genomic_expression_export"); - Q.all([ - // single sample data - spawnCommand(exportScript, [ - "--data_set_id", self.job.args.data_set_id, - "--sample_label", self.job.args.sample_label, - ], workDir), - // sample group data - spawnCommand(exportScript, [ - "--sample_group_id", self.job.args.sample_group_id, - ], workDir), - ]) + var exportCommands = [ + // single sample data + spawnCommand(exportScript, [ + "--data_set_id", self.job.args.data_set_id, + "--sample_label", self.job.args.sample_label, + ], workDir), + ]; + + // also write out sample group data if necessary + if (regenerateFiles) { + exportCommands.push(spawnCommand(exportScript, [ + "--sample_group_id", sample_group_id, + ], workDir)); + } + + Q.all(exportCommands) .then(function (spawnResults) { - console.log("spawnResults:", spawnResults); - - // check if there was a problem + // check if there was a problem exporting the data var uniqueExitCodes = _.uniq(_.pluck(spawnResults, "exitCode")); - console.log("uniqueExitCodes:", uniqueExitCodes); if (uniqueExitCodes.length !== 1 || uniqueExitCodes[0] !== 0) { throw new Error("Writing files failed (exit code not 0)"); } // save this result for use in a future chained promise self.testSamplePath = spawnResults[0].stdoutPath; - var sampleGroupPath = spawnResults[1].stdoutPath; - - // // pulled from upDownGenes.sh - // # arg 1: matrix file - // # arg 2: default 1.5 - // /usr/bin/Rscript outlier.R mRNA.NBL.POG.pancan.combat.5.tab 2 - var outlierGenesPath = getSetting("calculate_outlier_genes"); - - return spawnCommand("Rscript", [ - outlierGenesPath, - sampleGroupPath, - self.job.args.iqr_multiplier, - ], workDir); + if (regenerateFiles) { + // // pulled from upDownGenes.sh + // # arg 1: matrix file + // # arg 2: default 1.5 + // /usr/bin/Rscript outlier.R mRNA.NBL.POG.pancan.combat.5.tab 2 + + return spawnCommand("Rscript", [ + getSetting("calculate_outlier_genes"), + spawnResults[1].stdoutPath, + iqr_multiplier, + ], workDir); + } }) .then(function (commandResult) { - if (commandResult.exitCode !== 0) { - throw new Error("Error code running up/down genes Rscript"); + // if we just regenerated the files, use them + if (regenerateFiles) { + if (commandResult.exitCode !== 0) { + throw new Error("Error code running up/down genes Rscript"); + } + + medianPath = path.join(workDir, "median.tsv"); + highThresholdPath = path.join(workDir, "highthreshold.tsv"); + lowThresholdPath = path.join(workDir, "lowthreshold.tsv"); } return spawnCommand("/bin/sh", [ getSetting("outlier_analysis"), - self.testSamplePath + self.testSamplePath, + medianPath, + highThresholdPath, + lowThresholdPath ], workDir); }) .then(Meteor.bindEnvironment(function (commandResult) { + if (commandResult.exitCode !== 0) { + throw new Error("Error code running outlier analysis script"); + } console.log("done with single sample analysis"); - console.log("commandResult:", commandResult); + + // save the intermediary files if necessary + if (regenerateFiles) { + // NOTE: We don't technically need to wait until these are saved + // until the job is done. + // (This is an assumption that might not be true) + + function printError (err) { + if (err) { + console.log("error creating blob:", err); + } + } + var meta = { iqr_multiplier: iqr_multiplier }; + Blobs2.create(medianPath, associated_object, meta, printError); + Blobs2.create(highThresholdPath, associated_object, meta, printError); + Blobs2.create(lowThresholdPath, associated_object, meta, printError); + } // calculate the paths for the output files upPath = path.join(workDir, "up_outlier_genes") @@ -121,9 +182,10 @@ UpDownGenes.prototype.onSuccess = function (result) { Email.send({ to: emailAddress, from: "ucscmedbook@gmail.com", - subject: "Outlier analysis for " + self.job.args["sample_label"] + " complete.", - html: "Your outlier analysis job has completed. Results:\n" + resultsURL + "" , + subject: "Outlier analysis for " + self.job.args["sample_label"] + + " complete.", + html: "Your outlier analysis job has completed. Results:\n" + resultsURL + "" , }); console.log("Notification email sent for job ", self.job._id); diff --git a/webapp/server/lib/globals.js b/webapp/server/lib/globals.js index a382164..dcb8501 100644 --- a/webapp/server/lib/globals.js +++ b/webapp/server/lib/globals.js @@ -149,26 +149,26 @@ setBlobMetadata = function (blob, userId, otherMetadata) { }); }; -// NOTE: pass in the this object of the job function -// (so that we can get to this.job.user_id) -// ex. spawnedCommandFailedResolve.call(self, commandResult, deferred); -spawnedCommandFailedResolve = function (commandResult, deferred) { - var stdout = Blobs.insert(commandResult.stdoutPath); - var stderr = Blobs.insert(commandResult.stderrPath); - setBlobMetadata(stdout, this.job.user_id); - setBlobMetadata(stderr, this.job.user_id); - - deferred.resolve({ - result: "Error code " + commandResult.exitCode, - blobs: [ - { - name: "Command output (stdout)", - blob_id: stdout._id - }, - { - name: "Command error output (stderr)", - blob_id: stderr._id - }, - ], - }); -}; +// // NOTE: pass in the this object of the job function +// // (so that we can get to this.job.user_id) +// // ex. spawnedCommandFailedResolve.call(self, commandResult, deferred); +// spawnedCommandFailedResolve = function (commandResult, deferred) { +// var stdout = Blobs.insert(commandResult.stdoutPath); +// var stderr = Blobs.insert(commandResult.stderrPath); +// setBlobMetadata(stdout, this.job.user_id); +// setBlobMetadata(stderr, this.job.user_id); +// +// deferred.resolve({ +// result: "Error code " + commandResult.exitCode, +// blobs: [ +// { +// name: "Command output (stdout)", +// blob_id: stdout._id +// }, +// { +// name: "Command error output (stderr)", +// blob_id: stderr._id +// }, +// ], +// }); +// }; From fdcbc5a078396f2c1e99da440494743e7e8821b0 Mon Sep 17 00:00:00 2001 From: Teo Fleming Date: Sat, 16 Jul 2016 13:24:27 -0700 Subject: [PATCH 10/11] v1.0.4: onSuccess now fires after job is marked as done, doesn't block --- webapp/server/classes/ParseWranglerFile.js | 17 +++++----- webapp/server/classes/SubmitWranglerFile.js | 23 ++++++++----- webapp/server/classes/UpDownGenes.js | 37 ++++++++++----------- webapp/server/classes/global/topLevel.js | 2 +- webapp/server/startup.js | 16 +++++---- 5 files changed, 52 insertions(+), 43 deletions(-) diff --git a/webapp/server/classes/ParseWranglerFile.js b/webapp/server/classes/ParseWranglerFile.js index 2b865a0..f01b88b 100644 --- a/webapp/server/classes/ParseWranglerFile.js +++ b/webapp/server/classes/ParseWranglerFile.js @@ -106,7 +106,15 @@ ParseWranglerFile.prototype.run = function () { } fileHandler.parse() - .then(deferred.resolve) + .then(Meteor.bindEnvironment(function () { + WranglerFiles.update(self.wranglerFile._id, { + $set: { + status: "done", + } + }); + + deferred.resolve(); + }, deferred.reject)) .catch(deferred.reject); }, deferred.reject)) .catch(deferred.reject); @@ -128,12 +136,5 @@ ParseWranglerFile.prototype.onError = function (error) { } }); }; -ParseWranglerFile.prototype.onSuccess = function (result) { - WranglerFiles.update(this.wranglerFile._id, { - $set: { - status: "done", - } - }); -}; JobClasses.ParseWranglerFile = ParseWranglerFile; diff --git a/webapp/server/classes/SubmitWranglerFile.js b/webapp/server/classes/SubmitWranglerFile.js index 43fcab8..7b93fd3 100644 --- a/webapp/server/classes/SubmitWranglerFile.js +++ b/webapp/server/classes/SubmitWranglerFile.js @@ -15,7 +15,21 @@ SubmitWranglerFile.prototype.run = function () { throw "Invalid options"; } - return fileHandler.parse(); + var self = this; + var deferred = Q.defer(); + + fileHandler.parse() + .then(Meteor.bindEnvironment(function () { + WranglerFiles.update(self.wranglerFile._id, { + $set: { + written_to_database: true, + } + }); + deferred.resolve(); + }, deferred.reject)) + .catch(deferred.reject); + + return deferred.promise; }; SubmitWranglerFile.prototype.onError = function (e) { // TODO: should this be the correct behaviour? @@ -31,12 +45,5 @@ SubmitWranglerFile.prototype.onError = function (e) { } }); }; -SubmitWranglerFile.prototype.onSuccess = function (result) { - WranglerFiles.update(this.wranglerFile._id, { - $set: { - written_to_database: true, - } - }); -}; JobClasses.SubmitWranglerFile = SubmitWranglerFile; diff --git a/webapp/server/classes/UpDownGenes.js b/webapp/server/classes/UpDownGenes.js index 9292ddc..09f8d4e 100644 --- a/webapp/server/classes/UpDownGenes.js +++ b/webapp/server/classes/UpDownGenes.js @@ -169,27 +169,24 @@ UpDownGenes.prototype.run = function () { // Emails the creator with an alert & link to results page UpDownGenes.prototype.onSuccess = function (result) { console.log("UpDownGenes -- Success -- sending notification email."); - var self = this; - var userID = self.job.user_id; - check(userID, String); - var user = Meteor.users.findOne({_id:userID}); - var emailAddress = user.collaborations.email_address; - check(emailAddress, String); - var resultsID = self.job._id; - check(resultsID, String); - var resultsURL = "https://medbook.io/tools/outlier-analysis/" + resultsID; - - Email.send({ - to: emailAddress, - from: "ucscmedbook@gmail.com", - subject: "Outlier analysis for " + self.job.args["sample_label"] + - " complete.", - html: "Your outlier analysis job has completed. Results:\n" + resultsURL + "" , - }); - - console.log("Notification email sent for job ", self.job._id); + var user = Meteor.users.findOne({ _id: this.job.user_id }); + var resultsURL = "https://medbook.io/tools/outlier-analysis/" + this.job._id; + + try { + Email.send({ + to: user.collaborations.email_address, + from: "ucscmedbook@gmail.com", + subject: "Outlier analysis for " + this.job.args["sample_label"] + + " complete.", + html: "Your outlier analysis job has completed. Results:\n" + resultsURL + "" , + }); + + console.log("Notification email sent for job ", this.job._id); + } catch (e) { + console.log("Error: Notification email failed for job ", this.job._id); + } }; diff --git a/webapp/server/classes/global/topLevel.js b/webapp/server/classes/global/topLevel.js index 3690b32..343f498 100644 --- a/webapp/server/classes/global/topLevel.js +++ b/webapp/server/classes/global/topLevel.js @@ -19,7 +19,7 @@ Job.prototype.retry = function(reasonForRetry) { this.reasonForRetry = reasonForRetry; }; Job.prototype.onError = function(e) { - console.log("onSuccess has not been overridden in the job"); + console.log("onError has not been overridden in the job"); }; Job.prototype.onSuccess = function () { console.log("onSuccess has not been overridden in the job"); diff --git a/webapp/server/startup.js b/webapp/server/startup.js index 965ab3d..aa10bd8 100644 --- a/webapp/server/startup.js +++ b/webapp/server/startup.js @@ -68,7 +68,7 @@ function runNextJob () { // that comes to pass console.log(""); console.log("job: running - ", mongoJob.name, - "{ _id: " + mongoJob._id + " }"); + "{ _id: \"" + mongoJob._id + "\" }"); // check to see if something else has to be done first var mustHaveFinished = Jobs.find({ @@ -155,12 +155,10 @@ function runNextJob () { if (job.reasonForRetry) { retryLater(job.reasonForRetry); } else { - try { - job.onSuccess(output); - } catch (e) { - nope(e); - } + // if no output, default to empty object + if (!output) { output = {}; } + // set the job as done in mongo Jobs.update(mongoJob._id, { $set: { output: output, @@ -168,6 +166,12 @@ function runNextJob () { } }); console.log("job: done"); + + try { + job.onSuccess(output); + } catch (e) { + console.log("Error in onSuccess for " + mongoJob._id); + } } }), boundNope) .catch(boundNope); From 72239f73dc9ab90d83caeeae7ec3e922f4f515c6 Mon Sep 17 00:00:00 2001 From: Teo Fleming Date: Mon, 18 Jul 2016 10:27:39 -0700 Subject: [PATCH 11/11] packages on master branch --- webapp/packages/primary-collections | 2 +- webapp/packages/referential-integrity | 2 +- webapp/packages/wrangler-collections | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/webapp/packages/primary-collections b/webapp/packages/primary-collections index 317bea7..ad45829 160000 --- a/webapp/packages/primary-collections +++ b/webapp/packages/primary-collections @@ -1 +1 @@ -Subproject commit 317bea735ee997b2167bd026f7d29f7018446e39 +Subproject commit ad4582993ac773ce1b1ecb7377c4c6fefcae3ca2 diff --git a/webapp/packages/referential-integrity b/webapp/packages/referential-integrity index 9462815..f77a81f 160000 --- a/webapp/packages/referential-integrity +++ b/webapp/packages/referential-integrity @@ -1 +1 @@ -Subproject commit 946281562ba5c54337a6a8f85b55ab5efc99c651 +Subproject commit f77a81f6e0c5b2010162b2142b7e0506926007b4 diff --git a/webapp/packages/wrangler-collections b/webapp/packages/wrangler-collections index 46ae785..b2f87a0 160000 --- a/webapp/packages/wrangler-collections +++ b/webapp/packages/wrangler-collections @@ -1 +1 @@ -Subproject commit 46ae7856efe0824fc07f600f2350983b938ce310 +Subproject commit b2f87a07d95efe9f4774ecc8b18b92d453d97692