v1.0.4.1: DataSets/Studies, cache outlier analysis files

README.md

@@ -0,0 +1,20 @@
+# MedBook-JobRunner
+## Runs and monitors Jobs (Current UNIX processes, Galaxy and other Environments coming)
+### steps to adding a new tools
+
+1. create a feature branch in git
+2. look at https://github.com/UCSC-MedBook/MedBook-JobRunner/blob/f-gsea-new/webapp/server/classes/RunLimmaGSEA.js
+3. create a new class
+4. add adapters (importers and exporters) to convert from MedBook objects to files that tools understand and store check them into external-tools
+4. add external code to external-tools repo (or mechansim to install it)
+5. add pointers to external code in settings.json
+6. add gui to appropriate MedBook app, that initiates job by inserting into jobs collection
+  for example: 
+   Jobs.insert({
+      name: "UpDownGenes",
+      status: "waiting",
+      user_id: user._id,
+      collaborations: [ user.personalCollaboration() ],
+      args
+    });
+7. read errors from jobs.error_description

config/teo/run.sh

@@ -1,10 +1,4 @@
 export MONGO_URL="mongodb://localhost:27017/MedBook"
+export MEDBOOK_FILESTORE=/tmp/filestore
 
-export settings_file="../config/teo/settings.json"
-export port="3003"
-
-if [ -z "$1" ]; then
-    meteor --port $port --settings $settings_file
-else
-    meteor $1 --port $port --settings $settings_file
-fi
+meteor --port 3003 --settings ../config/teo/settings.json

config/teo/settings.json

@@ -1,12 +1,12 @@
 {
   "sh": "/bin/sh",
   "rscript": "Rscript",
-  "limma_path": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/limma/limma_ng.R",
-  "outlier_analysis": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/OutlierAnalysis/outlier-analysis.sh",
-  "calculate_outlier_genes": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/OutlierAnalysis/calculate_outlier_genes.R",
-  "gene_expression_export": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/exporters/gene_expression_export.py",
-  "gene_set_collection_export": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/exporters/gene_set_collection_export.py",
-  "limma_phenotype_export": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/exporters/limma_phenotype_export.py",
-  "gsea_path": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/gsea/rgGSEA.py",
-  "gsea_jar_path": "/Users/mokolodi1/work/medbook/medbook/MedBook-JobRunner/external-tools/gsea/gsea2-2.2.2.jar"
+  "limma_path": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/limma/limma_ng.R",
+  "outlier_analysis": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/OutlierAnalysis/outlier-analysis.sh",
+  "calculate_outlier_genes": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/OutlierAnalysis/calculate_outlier_genes.R",
+  "genomic_expression_export": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/exporters/genomic_expression_export.py",
+  "gene_set_collection_export": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/exporters/gene_set_collection_export.py",
+  "limma_phenotype_export": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/exporters/limma_phenotype_export.py",
+  "gsea_path": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/gsea/rgGSEA.py",
+  "gsea_jar_path": "/Users/mokolodi1/work/medbook/next/MedBook-JobRunner/external-tools/gsea/gsea2-2.2.2.jar"
 }

webapp/.meteor/.gitignore

@@ -1 +1,2 @@
 local
+dev_bundle

webapp/.meteor/versions

@@ -16,13 +16,13 @@ cfs:[email protected]
 cfs:[email protected]
 cfs:[email protected]
 cfs:[email protected]
-cfs:[email protected].30
+cfs:[email protected].32
 cfs:[email protected]
 cfs:[email protected]
 cfs:[email protected]
 cfs:[email protected]
 cfs:[email protected]
-cfs:[email protected].2
+cfs:[email protected].3
 cfs:[email protected]
 cfs:[email protected]
 cfs:[email protected]
@@ -44,6 +44,7 @@ [email protected]
 [email protected]
 [email protected]
 [email protected]
+matb33:[email protected]
 medbook:[email protected]
 medbook:[email protected]
 medbook:[email protected]

webapp/server/classes/FinishWranglerSubmission.js

@@ -38,8 +38,21 @@ FinishWranglerSubmission.prototype.run = function () {
     written_to_database: {$ne: true},
   });
   if (notWrittenCursor.count() > 0) {
-    this.retry("files not done being written");
-    return;
+    var errorWritingFiles = notWrittenCursor.map(function (wranglerFile) {
+      var job = Jobs.findOne({
+        name: "ParseWranglerFile",
+        "args.wrangler_file_id": wranglerFile._id
+      });
+
+      return !job || job.status === "error";
+    });
+
+    if (errorWritingFiles.indexOf(true) !== -1) {
+      throw "Internal error writing files to database";
+    } else {
+      this.retry("files not done being written");
+      return;
+    }
   }
 
   // we did it!

webapp/server/classes/ParseWranglerFile.js

@@ -55,36 +55,25 @@ ParseWranglerFile.prototype.run = function () {
 
       // try to guess file_type
       if (!options.file_type) {
-        if (extensionEquals(".vcf")) {
-          setFileOptions({ file_type: "MutationVCF" });
-        } else if (blobName.match(/\.rsem\.genes\.[a-z_]*\.(hugo\.|)tab/g)) {
+        if (blobName.match(/\.rsem\.genes\.[a-z_]*\.(hugo\.|)tab/g)) {
           // http://regexr.com/3d9i7
-          setFileOptions({ file_type: "RectangularGeneExpression" });
-        } else if (blobName.match(/\.rsem\.isoform\.[a-z_]*\.tab/g)) {
-          setFileOptions({ file_type: "RectangularIsoformExpression" });
-        } else if (extensionEquals(".xls") || extensionEquals("xlsx")) {
-          setFileOptions({ file_type: "BasicClinical" });
+          setFileOptions({ file_type: "RectGenomicExpression" });
         }
       }
 
-      // try to guess normalization
-      if (!options.normalization) {
-        // try to guess normalization
-        if (blobName.match(/raw_counts/g)) {
-          setFileOptions({ normalization: "raw_counts" });
-        } else if (blobName.match(/norm_counts/g)) {
-          setFileOptions({ normalization: "quantile_counts" });
-        } else if (blobName.match(/norm_tpm/g)) {
-          setFileOptions({ normalization: "tpm" });
-        } else if (blobName.match(/norm_fpkm/g)) {
-          setFileOptions({ normalization: "fpkm" });
-        }
-      }
-
-      // force certain options
-      if (options.file_type === "TCGAGeneExpression") {
-        setFileOptions({ normalization: "quantile_counts" });
-      }
+      // // try to guess normalization
+      // if (!options.normalization) {
+      //   // try to guess normalization
+      //   if (blobName.match(/raw_counts/g)) {
+      //     setFileOptions({ normalization: "raw_counts" });
+      //   } else if (blobName.match(/norm_counts/g)) {
+      //     setFileOptions({ normalization: "quantile_counts" });
+      //   } else if (blobName.match(/norm_tpm/g)) {
+      //     setFileOptions({ normalization: "tpm" });
+      //   } else if (blobName.match(/norm_fpkm/g)) {
+      //     setFileOptions({ normalization: "fpkm" });
+      //   }
+      // }
 
       if (self.blob.metadata && self.blob.metadata.wrangler_file_options) {
         setFileOptions(self.blob.metadata.wrangler_file_options);
@@ -97,7 +86,8 @@ ParseWranglerFile.prototype.run = function () {
 
       // make sure we've got a file_type
       if (!options.file_type) {
-        throw "File type could not be inferred. Please manually select a file type";
+        throw "File type could not be inferred. " +
+            "Please manually select a file type";
       }
 
       var fileHandlerClass = WranglerFileHandlers[options.file_type];
@@ -116,7 +106,15 @@ ParseWranglerFile.prototype.run = function () {
       }
 
       fileHandler.parse()
-        .then(deferred.resolve)
+        .then(Meteor.bindEnvironment(function () {
+          WranglerFiles.update(self.wranglerFile._id, {
+            $set: {
+              status: "done",
+            }
+          });
+
+          deferred.resolve();
+        }, deferred.reject))
         .catch(deferred.reject);
     }, deferred.reject))
     .catch(deferred.reject);
@@ -138,12 +136,5 @@ ParseWranglerFile.prototype.onError = function (error) {
     }
   });
 };
-ParseWranglerFile.prototype.onSuccess = function (result) {
-  WranglerFiles.update(this.wranglerFile._id, {
-    $set: {
-      status: "done",
-    }
-  });
-};
 
 JobClasses.ParseWranglerFile = ParseWranglerFile;

webapp/server/classes/RunLimmaGSEA.js

@@ -36,26 +36,43 @@ RunLimmaGSEA.prototype.run = function () {
   // combine samples of same data set into single array
   var dataSetHash = {};
   _.each(groupA.data_sets.concat(groupB.data_sets), function (dataSet) {
-    var oldSamples = dataSetHash[dataSet.data_set_id];
-    if (!oldSamples) {
-      oldSamples = [];
+    // check if we've seen this data set already
+    var seenAlready = dataSetHash[dataSet.data_set_id];
+    if (!seenAlready) {
+      // if we haven't, set it up
+      seenAlready = {
+        data_set_name: dataSet.data_set_name,
+        sample_labels: [],
+      };
     }
 
-    dataSetHash[dataSet.data_set_id] = oldSamples.concat(dataSet.sample_labels);
+    // combine the samples together
+    seenAlready.sample_labels =
+        seenAlready.sample_labels.concat(dataSet.sample_labels)
+    dataSetHash[dataSet.data_set_id] = seenAlready;
   });
   var comboSampleGroupDataSets = _.map(dataSetHash,
-      function (sample_labels, data_set_id) {
+      function (samplesAndName, data_set_id) {
     return {
       data_set_id: data_set_id,
-      sample_labels: sample_labels,
+      data_set_name: samplesAndName.data_set_name,
+      sample_labels: samplesAndName.sample_labels,
+
+      // I think we can fake this
+      unfiltered_sample_count: 1,
     };
   });
 
+  console.log("comboSampleGroupDataSets:", comboSampleGroupDataSets);
+
   var comboSampleGroupId = SampleGroups.insert({
     name: "temp - created in RunLimmaGSEA to call an adapter",
     version: 1,
-    collaborations: [], // invisible
     data_sets: comboSampleGroupDataSets,
+    value_type: groupA.value_type,
+
+    // invisible
+    collaborations: [],
   });
 
   // star the promise chain: woohoo!
@@ -77,7 +94,7 @@ RunLimmaGSEA.prototype.run = function () {
       // write mongo data to files
 
       // expression data to a file for use in Limma
-      spawnCommand(getSetting("gene_expression_export"), [
+      spawnCommand(getSetting("genomic_expression_export"), [
         "--sample_group_id", comboSampleGroupId,
       ], workDir),
       // phenotype file for Limma
@@ -100,6 +117,7 @@ RunLimmaGSEA.prototype.run = function () {
       });
 
       // save the file paths... order maters for spawnResults
+      // (the order depends on the order of `spawnCommand`s in `Q.all`)
       var expressionDataPath = spawnResults[0].stdoutPath;
       var limmaPhenotypePath = spawnResults[1].stdoutPath;
 
@@ -196,7 +214,7 @@ RunLimmaGSEA.prototype.run = function () {
         Blobs2.create(path.join(gseaOutput, fileName), {
           collection_name: "Jobs",
           mongo_id: self.job._id,
-        }, function (err, out) {
+        }, {}, function (err, out) {
           if (err) {
             console.log("err:", err);
             def.reject("Error inserting blob: " + fileName);

webapp/server/classes/SubmitWranglerFile.js

@@ -15,7 +15,21 @@ SubmitWranglerFile.prototype.run = function () {
     throw "Invalid options";
   }
 
-  return fileHandler.parse();
+  var self = this;
+  var deferred = Q.defer();
+
+  fileHandler.parse()
+    .then(Meteor.bindEnvironment(function () {
+      WranglerFiles.update(self.wranglerFile._id, {
+        $set: {
+          written_to_database: true,
+        }
+      });
+      deferred.resolve();
+    }, deferred.reject))
+    .catch(deferred.reject);
+
+  return deferred.promise;
 };
 SubmitWranglerFile.prototype.onError = function (e) {
   // TODO: should this be the correct behaviour?
@@ -31,12 +45,5 @@ SubmitWranglerFile.prototype.onError = function (e) {
     }
   });
 };
-SubmitWranglerFile.prototype.onSuccess = function (result) {
-  WranglerFiles.update(this.wranglerFile._id, {
-    $set: {
-      written_to_database: true,
-    }
-  });
-};
 
 JobClasses.SubmitWranglerFile = SubmitWranglerFile;