Skip to content

Commit

Permalink
Merge branch 'brainlife:master' into show_apps_not_supported_in_search
Browse files Browse the repository at this point in the history
  • Loading branch information
bhatiadheeraj authored Jan 17, 2024
2 parents 32674d9 + 2f70545 commit f3f668e
Showing 127 changed files with 29,732 additions and 30,371 deletions.
8 changes: 8 additions & 0 deletions .codespellrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
[codespell]
skip = .git,*.pdf,*.svg,package-lock.json,*.min.js,vendor,.sass-cache,locales,ui-text.yml
# hsi - server name
# crate - seems some real thing in the API
# deactive - someone could later change to deactivated or inactive
# onTop - variable name used, not picked up by codespell to be ignored
# pres - variable name used
ignore-words-list = keypair,nd,nin,hsi,crate,deactive,ontop,pres,doubleclick
19 changes: 19 additions & 0 deletions .github/workflows/codespell.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
---
name: Codespell

on:
push:
branches: [master]
pull_request:
branches: [master]

jobs:
codespell:
name: Check for spelling errors
runs-on: ubuntu-latest

steps:
- name: Checkout
uses: actions/checkout@v3
- name: Codespell
uses: codespell-project/actions-codespell@v1
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
@@ -6,13 +6,14 @@ ui/node_modules
ui/dist
ui/npm-debug.log
ui/yarn-error.log
ui/config/dev.env.js
**/barn
docker/config
ui/config/index.js
ui/config/dev.env.js
ui/config/dev.env
ui/config/npm-debug.log
api/config
api/config/index.js
docs/_site
docs/Gemfile.lock
**/npm-debug.log
18 changes: 7 additions & 11 deletions api/common.js
Original file line number Diff line number Diff line change
@@ -2,17 +2,13 @@
const request = require('request'); //TODO switch to axios!
const rp = require('request-promise-native');
const keywordEx = require("keyword-extractor");
const tmp = require('tmp');
const path = require('path');
const child_process = require('child_process');
const fs = require('fs');
const async = require('async');
const redis = require('redis');
const xmlescape = require('xml-escape');
const amqp = require("amqp"); //switch to amqplib?
const axios = require('axios');
const jwt = require('express-jwt');
const crypto = require('crypto');

const config = require('./config');
const db = require('./models');
@@ -33,7 +29,7 @@ exports.getprojects = function(user, cb) {
//everyone has read access to public project
let project_query = {access: "public"};

//logged in user may have acess to more projects
//logged in user may have access to more projects
project_query = {
$or: [
project_query,
@@ -208,7 +204,7 @@ exports.issue_archiver_jwt = async function(user_id) {
///add warehouse group that allows user to submit
gids = gids.concat(config.archive.gid);

//issue user token with added gids priviledge
//issue user token with added gids privilege
let {jwt: user_jwt} = await rp.get({
url: config.auth.api+"/jwt/"+user_id,
json: true,
@@ -762,7 +758,7 @@ exports.cacheContact = function(cb) {
console.debug("caching contacts", config.warehouse.jwt);
axios.get(config.auth.api+"/profile/list", {
params: {
limit: 5000, //TODO -- really!?
limit: 6000, //TODO -- really!?
},
headers: { authorization: "Bearer "+config.warehouse.jwt }, //config.auth.jwt is deprecated
}).then(res=>{
@@ -862,7 +858,7 @@ exports.split_product = function(task_product, outputs) {
return products;
}

//TODO - this has to match up between amaretti/bin/metrics and warehouse/api/controller querying for graphite daa
//TODO - this has to match up between amaretti/bin/metrics and warehouse/api/controller querying for graphite data
exports.sensu_name = function(name) {
name = name.toLowerCase();
name = name.replace(/[_.@$#\/]/g, "-");
@@ -899,12 +895,12 @@ exports.disconnectAMQP = function(cb) {
if(!amqpConn) return console.error("AMQP not connected");
console.debug("disconnecting from amqp");
amqpConn.setImplOptions({reconnect: false}); //https://github.com/postwait/node-amqp/issues/462
amqpconn.disconnect();
amqpConn.disconnect();
}

//connect to the amqp exchange and wait for a event to occur
exports.wait_for_event = function(exchange, key, cb) {
if(!amqpConn) return console.error("wait_for_event called before connecting to amqp");
if (!amqpConn) return console.error("wait_for_event called before connecting to amqp");
amqpConn.queue('', q=>{
q.bind(exchange, key, ()=>{
q.subscribe((message, header, deliveryInfo, messageObject)=>{
@@ -1480,7 +1476,7 @@ exports.enumXnatObjects = async (project)=>{
"/subjects/"+oSubject.ID+
"/experiments", {auth});
for(const oExperiment of exres.data.ResultSet.Result) {
console.log(" experiement", oExperiment.ID);
console.log(" experiment", oExperiment.ID);
//console.log(" <experiment>");
/* oExperiment
{
8 changes: 3 additions & 5 deletions api/config/index.js.dev
Original file line number Diff line number Diff line change
@@ -13,8 +13,6 @@ const request = require('request'); //deprecate!
const zlib = require('zlib');
const async = require('async');

const atob = require('atob');

exports.mongodb = "mongodb://brainlife_mongodb/warehouse";

exports.debug = true;
@@ -23,7 +21,7 @@ exports.debug = true;
exports.redis = { url: "redis://brainlife_redis" };

//get it from https://github.com/settings/tokens
//exports.github = { access_token: "..." };
exports.github = { access_token: fs.readFileSync(__dirname+'/github.access_token', 'ascii').trim()};

//admin+client scope oauth2 token used to invite users
//once you create oauth app, you will need to re-authorize it with admin+client (it gave with just admin scope)
@@ -81,7 +79,7 @@ exports.mail = {

//node mailer config
mailer: {
host: 'mail-relay.iu.edu', //max recipents per email: 30
host: 'mail-relay.iu.edu', //max recipients per email: 30
secure: true, //port 465
auth: {
user: 'mememe',
@@ -427,7 +425,7 @@ exports.storage_systems.osiris = {
console.error("stream failed but train has already left the station", err);
});
stream.on('close', code=>{
console.log("stream clsoed.. closing connection");
console.log("stream closed.. closing connection");
conn.end();
});
//I believe listening to stream is required for close event to fire..?
6 changes: 3 additions & 3 deletions api/controllers/app.js
Original file line number Diff line number Diff line change
@@ -52,7 +52,7 @@ router.get('/', common.jwt({credentialsRequired: false}), (req, res, next)=>{
{projects: []}, //if projects is empty array, it's available to everyone

//for backward compatibility
{projects: null}, //if projects is set to null, it's avalable to everyoone
{projects: null}, //if projects is set to null, it's available to everyoone
{projects: {$exists: false}}, //if projects not set, it's availableo to everyone
]});

@@ -103,7 +103,7 @@ router.get('/query', common.jwt({credentialsRequired: false}), (req, res, next)=
{projects: []}, //if projects is empty array, it's available to everyone

//for backward compatibility
{projects: null}, //if projects is set to null, it's avalable to everyoone
{projects: null}, //if projects is set to null, it's available to everyoone
{projects: {$exists: false}}, //if projects not set, it's availableo to everyone
]
};
@@ -371,7 +371,7 @@ router.put('/:id', common.jwt(), (req, res, next)=>{

//let's not validate project - as different maintainer has access to different set of projects
//and if the project is entered by different maintainer, we don't want to error out..
//multiprojectselecter on UI will list project that user doesn't have access using just project ID
//multiprojectselector on UI will list project that user doesn't have access using just project ID
//so user can retain project ID that user don't have access
//common.validate_projects(req.user, req.body.projects, err=>{
// if(err) return next(err);
21 changes: 11 additions & 10 deletions api/controllers/dataset.js
Original file line number Diff line number Diff line change
@@ -353,7 +353,7 @@ router.get('/provscript/:id', async (req, res, next)=>{
replace_path(node._config);
});

//output scrips
//output scripts
while(nodes.length) {
let node = nodes.pop();
if(!node.label) continue; //"This Dataset" doesn't have label, and we don't need it (TODO make output link?)
@@ -533,7 +533,7 @@ This boutique descriptor that can be used to run the workflow used to generate t
app_config[k] = vkey;
if(unwrap) app_config[k] = "__unwrap__"+vkey;

//convert default value to string - boutique invocation value needs to be string accordint to tristan
//convert default value to string - boutique invocation value needs to be string according to tristan
let def = app.config[k].default;
if(def === null) def = "null";
if(def.toString) def = def.toString();
@@ -1210,7 +1210,7 @@ function stream_dataset(dataset, req, res, next) {
res.on('finish', ()=>{
//console.debug("done piping.. meter count:%s dataset.size %d", m.bytes, dataset.size);
if(!dataset.size) {
/* this is not good idea.. as .tar file size might change if versionn of tar get updates
/* this is not good idea.. as .tar file size might change if version of tar get updates
console.debug("updating dataset size based on m.bytes");
dataset.size = m.bytes;
*/
@@ -1350,7 +1350,7 @@ router.get('/download/:id', common.jwt({
});
},

//check aggreements
//check agreements
cb=>{
//load project agreements
get_project_agreements(dataset.project, (err, project_agreements)=>{
@@ -1520,16 +1520,17 @@ set -e
authors = authors.map(id=>common.deref_contact(id)).filter(a=>!!a).map(a=>a.fullname||a.email);

//write README and bids/dataset_description.json
let root = "./proj-"+project_id;
script += "mkdir -p "+root+"/bids\n";
script += "cat << '__ENDREADME__' > "+root+"/README\n";
const uniqueHeredoc = Math.random().toString(36).substring(2);
let root = `./proj-${project_id}`;
script += `mkdir -p ${root}/bids\n`;
script += `cat << '__${uniqueHeredoc}__' > ${root}/README\n`;
script += `${p.name}
${config.warehouse.url}/project/${project_id}
${p.desc}`;

script += "\n__ENDREADME__\n";
script += `\n__${uniqueHeredoc}__\n`;

let dataset_description = {
BIDSVersion: "1.0.1",
@@ -1568,7 +1569,7 @@ ${p.desc}`;
//download info.json
script += "curl ";
if(req.headers.authorization) script += "-H \"$auth\" ";
script += config.warehouse.url+"/api/warehouse/dataset?single=true\\&find='\\{\"_id\":\""+dataset._id+"\"\\}' > "+path+"/_info.json\n";
script += config.warehouse.url+"/api/warehouse/dataset?single=true\\&find='\\{\"_id\":\""+dataset._id+"\"\\}' > '"+path+"/_info.json'\n";

if(dataset.datatype.bids) {
//Create BIDS symlinks
@@ -1637,7 +1638,7 @@ ${p.desc}`;

script+=`
echo
echo "All requested objects succesfully downloaded!!"
echo "All requested objects successfully downloaded!!"
echo
echo "---------------------------------------------------------------------------------"
echo "---------------------------------------------------------------------------------"
24 changes: 15 additions & 9 deletions api/controllers/rule.js
Original file line number Diff line number Diff line change
@@ -77,18 +77,24 @@ function check_access(req, rule, cb) {

//check user has access to the project
common.getprojects(req.user, function(err, canread_project_ids, canwrite_project_ids) {
if(err) return cb(err);
if (err) return cb(err);
let project_id = mongoose.Types.ObjectId(rule.project);
let found = canwrite_project_ids.find(id=>id.equals(rule.project));
if(!found) return cb("can't access rule under this project");
if (!found) return cb("can't access rule under this project");

//check to see if user has read accesses to all input_project_override
if(rule.input_project_override) for(let id in rule.input_project_override) {
if (rule.input_project_override) {
for (let id in rule.input_project_override) {
let project_id = rule.input_project_override[id];
if(!project_id) continue; //ignore null..
let o_project_id = mongoose.Types.ObjectId(project_id); //null gets converted to a valid mongoose id.. new id?)
let found = canread_project_ids.find(id=>id.equals(o_project_id));
if(!found) return cb("can't use project selected in override:"+o_project_id+" for id:"+id);
if (!project_id) continue;

let o_project_id = mongoose.Types.ObjectId(project_id);
let found = canread_project_ids.find(id => id.equals(o_project_id));
if (!found) {
cb("can't use project selected in override:"+o_project_id+" for id:"+id);
return;
}
}
}

cb(); //a-ok
@@ -97,7 +103,7 @@ function check_access(req, rule, cb) {

/**
* @apiGroup Pipeline Rules
* @api {post} /rule/:pubid Register new rule
* @api {post} /rule Register new rule
*
* @apiDescription Register a new pipeline rule.
*
@@ -143,7 +149,7 @@ router.post('/', common.jwt(), (req, res, next)=>{

/**
* @apiGroup Pipeline Rules
* @api {put} /rule/:pubid Update Rule
* @api {put} /rule/:id Update Rule
*
* @apiDescription Update pipeline rule
*
4 changes: 2 additions & 2 deletions api/controllers/secondary.js
Original file line number Diff line number Diff line change
@@ -222,7 +222,7 @@ async function issueGAJwt(instance_id, user, authorization, cb) {
router.post('/launchga', common.jwt(), (req, res, next)=>{
if(!req.body.instance_id) return next("instance_id is not set");
if(!req.body.config) return next("please set config");
if(!req.body.config.container) return next("please set contianer (with tag)");
if(!req.body.config.container) return next("please set container (with tag)");

issueGAJwt(req.body.instance_id, req.user, req.headers.authorization,
async (err, jwt, instance, project)=>{
@@ -231,7 +231,7 @@ router.post('/launchga', common.jwt(), (req, res, next)=>{
let _config = Object.assign({}, req.body.config);
_config.project = {
_id: project._id,
name: project.name, //galauncher juse need _id, but just in case it might become handy..
name: project.name, //galauncher just need _id, but just in case it might become handy..
}
_config.group = instance.group_id;

8 changes: 3 additions & 5 deletions api/lib/provenance.js
Original file line number Diff line number Diff line change
@@ -7,8 +7,6 @@ const mongoose = require('mongoose');
const common = require('../common');
const dbscan = require('@cdxoo/dbscan');

const math = require('mathjs');

exports.traverseProvenance = async (startTaskId) => {
console.debug("traversing from task:", startTaskId);

@@ -202,7 +200,7 @@ exports.traverseProvenance = async (startTaskId) => {

const dirTokens = datasetConfig.dir.split("/");
const sourceTask = dirTokens[1]; //original task that produed the output
const subdir = dirTokens[2]; //original task's subdir that contains this ouput
const subdir = dirTokens[2]; //original task's subdir that contains this output

//figure out input task/subdir (../60874c557f09362173e40866/bold_mask)
let id = datasetConfig.dataset_id;
@@ -258,7 +256,7 @@ exports.traverseProvenance = async (startTaskId) => {
//outdir will be set to indicate the copy dataset id which we want
datasetId = datasetConfig.outdir;

//let's go ahead and register source dataset now in case datset.prov might be missing
//let's go ahead and register source dataset now in case dataset.prov might be missing
//and we can't setup app-stage node for it
const source = await db.Datasets.findById(datasetConfig.id).lean();
registerDataset(source);
@@ -479,7 +477,7 @@ exports.setupShortcuts = (prov)=>{
}

//TODO
//I think this is dangerous.. there could be multpile edges with the same from/to
//I think this is dangerous.. there could be multiple edges with the same from/to
//I think it's safer to just return the edge itself, and let caller specifically use that edge.idx
//for shortcut array
function findEdgeIdx(from, to) {
6 changes: 3 additions & 3 deletions api/models.js
Original file line number Diff line number Diff line change
@@ -94,7 +94,7 @@ var projectSchema = mongoose.Schema({
resources: [{
resource_id: String, //amaretti resource_id

//for quick referncing the resource detail
//for quick referencing the resource detail
name: String,

//becomes too big
@@ -555,7 +555,7 @@ datasetSchema.pre('save', function(next) {
datasetSchema.index({project: 1, 'prov.task.instance_id': 1, removed: 1, 'meta.subject': 1, 'meta.session': 1, create_date: -1}); //is this deprecated by project/remove/subject/session/-create_ate?
//datasetSchema.index({project: 1, removed: 1, "meta.subject": 1, "meta.session": 1, "create_date": -1}); //for dataset search by the archive view
datasetSchema.index({project: 1, datatype: 1, removed: 1, status: 1, "meta.subject": 1, "meta.session": 1, create_date: -1});
datasetSchema.index({project: 1, update_date: 1, removed: 1}); //rule to query the lastest dataset touched
datasetSchema.index({project: 1, update_date: 1, removed: 1}); //rule to query the latest dataset touched
datasetSchema.index({project: 1, update_date: -1, removed: 1}); //rule handler to find the last dataset update date for each project
datasetSchema.index({'prov.task_id': 1, 'prov.output_id': 1, removed: 1, status: 1}); //for event_handler
datasetSchema.index({datatype: 1, removed: 1}); //for searching projects that provides distinct datatypes
@@ -737,7 +737,7 @@ var ruleSchema = mongoose.Schema({
//any tags to look for each input id (object with key(output id)=>array(tags))
input_tags: mongoose.Schema.Types.Mixed,

//count of object that should match for each intput id for the rule to be submitted
//count of object that should match for each input id for the rule to be submitted
input_multicount: mongoose.Schema.Types.Mixed,

//if user wants to override where the input data comes from, specify projects IDs keyed by input id
Loading

0 comments on commit f3f668e

Please sign in to comment.