From 6ea2331d83625369ba619d7c66455ef1b16b8801 Mon Sep 17 00:00:00 2001 From: ETK Date: Thu, 21 Jul 2016 14:03:45 -0700 Subject: [PATCH 1/6] Saves top5percent genes file, adds to output Top 5% overexpressed genes is generated from outlier analysis as single_sample.head.sort ; capture this file and save it using new Blobs2. Also add its info to output as top5percent_blob_id, top5percent_genes for use in patient care UI. --- webapp/server/classes/UpDownGenes.js | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/webapp/server/classes/UpDownGenes.js b/webapp/server/classes/UpDownGenes.js index 09f8d4e..ebf84e9 100644 --- a/webapp/server/classes/UpDownGenes.js +++ b/webapp/server/classes/UpDownGenes.js @@ -129,17 +129,39 @@ UpDownGenes.prototype.run = function () { // calculate the paths for the output files upPath = path.join(workDir, "up_outlier_genes") downPath = path.join(workDir, "down_outlier_genes") + top5Path = path.join(workDir, "single_sample.head.sort") // insert blobs into mongo + // TODO -- convert these to Blobs2 var output = { up_blob_id: Blobs.insert(upPath)._id, down_blob_id: Blobs.insert(downPath)._id, }; + // Save output files as Blobs2 "synchronously" + + var associated_job_object = { + collection_name: "Jobs", + mongo_id: self.job._id, + }; + var createBlob2Sync = Meteor.wrapAsync(Blobs2.create); + try{ + var top5blob = createBlob2Sync(top5Path, associated_job_object, {}); + console.log("created a blob and got", top5blob); + output["top5percent_blob_id"] = top5blob._id; + }catch(error){ + // Log the error and throw it again to properly fail the outlier analysis job + console.log("Error storing output files for Outlier Analysis:", error); + throw(error); + } + + // parse strings _.each([ + // TODO - convert paths for up_ and down_ genes to use Blobs2 { name: "up_genes", fileString: fs.readFileSync(upPath, "utf8") }, { name: "down_genes", fileString: fs.readFileSync(downPath, "utf8") }, + { name: "top5percent_genes", fileString: fs.readFileSync(top5blob.getFilePath(), "utf8") }, ], function (outlier) { var lineArray = outlier.fileString.split("\n"); var filteredLines = _.filter(lineArray, function (line) { @@ -159,7 +181,7 @@ UpDownGenes.prototype.run = function () { deferred.resolve(output); }, deferred.reject)) - // NOTE: Meteor.bindEnvironment returns immidiately, meaning we can't + // NOTE: Meteor.bindEnvironment returns immediately, meaning we can't // quite use the nice promise syntax of chainging .thens .catch(deferred.reject); return deferred.promise; From a49d64dd8f096cb3493fe834c95a13d10c53fc04 Mon Sep 17 00:00:00 2001 From: ETK Date: Thu, 21 Jul 2016 14:07:45 -0700 Subject: [PATCH 2/6] Add config settings for ekephart --- config/ekephart/run.sh | 4 ++++ config/ekephart/settings.json | 12 ++++++++++++ 2 files changed, 16 insertions(+) create mode 100755 config/ekephart/run.sh create mode 100644 config/ekephart/settings.json diff --git a/config/ekephart/run.sh b/config/ekephart/run.sh new file mode 100755 index 0000000..4f30898 --- /dev/null +++ b/config/ekephart/run.sh @@ -0,0 +1,4 @@ +export MONGO_URL="mongodb://localhost:27017/MedBook" +export MEDBOOK_FILESTORE=/tmp/filestore + +meteor --port 3003 --settings ../config/ekephart/settings.json diff --git a/config/ekephart/settings.json b/config/ekephart/settings.json new file mode 100644 index 0000000..2cde0b8 --- /dev/null +++ b/config/ekephart/settings.json @@ -0,0 +1,12 @@ +{ + "sh": "/bin/sh", + "rscript": "Rscript", + "limma_path": "/Users/ekephart/code/jobrunner/external-tools/limma/limma_ng.R", + "outlier_analysis": "/Users/ekephart/code/jobrunner/external-tools/OutlierAnalysis/outlier-analysis.sh", + "calculate_outlier_genes": "/Users/ekephart/code/jobrunner/external-tools/OutlierAnalysis/calculate_outlier_genes.R", + "genomic_expression_export": "/Users/ekephart/code/jobrunner/external-tools/exporters/genomic_expression_export.py", + "gene_set_collection_export": "/Users/ekephart/code/jobrunner/external-tools/exporters/gene_set_collection_export.py", + "limma_phenotype_export": "/Users/ekephart/code/jobrunner/external-tools/exporters/limma_phenotype_export.py", + "gsea_path": "/Users/ekephart/code/jobrunner/external-tools/gsea/rgGSEA.py", + "gsea_jar_path": "/Users/ekephart/code/jobrunner/external-tools/gsea/gsea2-2.2.2.jar" +} From e12c403f13a270a9de9221bef8e43e9291da542f Mon Sep 17 00:00:00 2001 From: ETK Date: Thu, 21 Jul 2016 16:27:17 -0700 Subject: [PATCH 3/6] Parse top5% file correctly when import to meteor top5% has a different format than other input files. (tab instead of space) --- webapp/server/classes/UpDownGenes.js | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/webapp/server/classes/UpDownGenes.js b/webapp/server/classes/UpDownGenes.js index ebf84e9..bcbbde0 100644 --- a/webapp/server/classes/UpDownGenes.js +++ b/webapp/server/classes/UpDownGenes.js @@ -147,7 +147,6 @@ UpDownGenes.prototype.run = function () { var createBlob2Sync = Meteor.wrapAsync(Blobs2.create); try{ var top5blob = createBlob2Sync(top5Path, associated_job_object, {}); - console.log("created a blob and got", top5blob); output["top5percent_blob_id"] = top5blob._id; }catch(error){ // Log the error and throw it again to properly fail the outlier analysis job @@ -170,19 +169,31 @@ UpDownGenes.prototype.run = function () { // loop for each line output[outlier.name] = _.map(filteredLines, function (line) { - var tabSplit = line.split(" "); - return { - gene_label: tabSplit[0], - background_median: parseFloat(tabSplit[1]), - sample_value: parseFloat(tabSplit[2]), - }; + // Populate the found genes. + // The top5percent overexpressed file has a different format from the + // other files so split its columns separately. + if(outlier.name == "top5percent_genes"){ + var tabSplit = line.split("\t"); + return { + gene_label: tabSplit[0], + sample_value: parseFloat(tabSplit[1]), + // no background_median + } + }else{ + var tabSplit = line.split(" "); + return { + gene_label: tabSplit[0], + background_median: parseFloat(tabSplit[1]), + sample_value: parseFloat(tabSplit[2]), + }; + } }); }); deferred.resolve(output); }, deferred.reject)) // NOTE: Meteor.bindEnvironment returns immediately, meaning we can't - // quite use the nice promise syntax of chainging .thens + // quite use the nice promise syntax of chaining .thens .catch(deferred.reject); return deferred.promise; }; From 719fb9c06cbb47f617bc06b9a2b9e956a70caea9 Mon Sep 17 00:00:00 2001 From: ETK Date: Thu, 21 Jul 2016 16:46:26 -0700 Subject: [PATCH 4/6] Rename single_sample.head.sort to top_5_percent_most_highly_expressed_genes.tsv --- external-tools | 2 +- webapp/server/classes/UpDownGenes.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/external-tools b/external-tools index 41a1626..751130b 160000 --- a/external-tools +++ b/external-tools @@ -1 +1 @@ -Subproject commit 41a162665b30e6fa13954d504488b3d1090c6d36 +Subproject commit 751130b5901931bacc8c1f41a5bc78de92d6778b diff --git a/webapp/server/classes/UpDownGenes.js b/webapp/server/classes/UpDownGenes.js index bcbbde0..1afcd16 100644 --- a/webapp/server/classes/UpDownGenes.js +++ b/webapp/server/classes/UpDownGenes.js @@ -129,7 +129,7 @@ UpDownGenes.prototype.run = function () { // calculate the paths for the output files upPath = path.join(workDir, "up_outlier_genes") downPath = path.join(workDir, "down_outlier_genes") - top5Path = path.join(workDir, "single_sample.head.sort") + top5Path = path.join(workDir, "top_5_percent_most_highly_expressed_genes.tsv") // insert blobs into mongo // TODO -- convert these to Blobs2 From 31b32d86d3c1fd453c4ca9ed5633b0c6b9fce400 Mon Sep 17 00:00:00 2001 From: ETK Date: Thu, 21 Jul 2016 17:05:56 -0700 Subject: [PATCH 5/6] Convert up, down genes to blobs2 from blobs --- webapp/server/classes/UpDownGenes.js | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/webapp/server/classes/UpDownGenes.js b/webapp/server/classes/UpDownGenes.js index 1afcd16..ee69d3a 100644 --- a/webapp/server/classes/UpDownGenes.js +++ b/webapp/server/classes/UpDownGenes.js @@ -131,22 +131,22 @@ UpDownGenes.prototype.run = function () { downPath = path.join(workDir, "down_outlier_genes") top5Path = path.join(workDir, "top_5_percent_most_highly_expressed_genes.tsv") - // insert blobs into mongo - // TODO -- convert these to Blobs2 - var output = { - up_blob_id: Blobs.insert(upPath)._id, - down_blob_id: Blobs.insert(downPath)._id, - }; - // Save output files as Blobs2 "synchronously" + // Save output files as Blobs2 "synchronously" with wrapAsync + var output = {}; var associated_job_object = { collection_name: "Jobs", mongo_id: self.job._id, }; var createBlob2Sync = Meteor.wrapAsync(Blobs2.create); + try{ + var upGenesBlob = createBlob2Sync(upPath, associated_job_object, {}); + var downGenesBlob = createBlob2Sync(downPath, associated_job_object, {}); var top5blob = createBlob2Sync(top5Path, associated_job_object, {}); + output["up_blob_id"] = upGenesBlob._id; + output["down_blob_id"] = downGenesBlob._id; output["top5percent_blob_id"] = top5blob._id; }catch(error){ // Log the error and throw it again to properly fail the outlier analysis job @@ -157,9 +157,8 @@ UpDownGenes.prototype.run = function () { // parse strings _.each([ - // TODO - convert paths for up_ and down_ genes to use Blobs2 - { name: "up_genes", fileString: fs.readFileSync(upPath, "utf8") }, - { name: "down_genes", fileString: fs.readFileSync(downPath, "utf8") }, + { name: "up_genes", fileString: fs.readFileSync(upGenesBlob.getFilePath(), "utf8") }, + { name: "down_genes", fileString: fs.readFileSync(downGenesBlob.getFilePath(), "utf8") }, { name: "top5percent_genes", fileString: fs.readFileSync(top5blob.getFilePath(), "utf8") }, ], function (outlier) { var lineArray = outlier.fileString.split("\n"); From d2de7f20fbb69500dd51f4e20ce93e26def52292 Mon Sep 17 00:00:00 2001 From: ETK Date: Fri, 22 Jul 2016 11:43:02 -0700 Subject: [PATCH 6/6] updating submodule external-tools to newest master --- external-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external-tools b/external-tools index 751130b..13e83e1 160000 --- a/external-tools +++ b/external-tools @@ -1 +1 @@ -Subproject commit 751130b5901931bacc8c1f41a5bc78de92d6778b +Subproject commit 13e83e1e2d7287c5ae399ea227f6901c7e5503ef