Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat improve rule logs #352

Closed
wants to merge 4 commits into from
Closed

Feat improve rule logs #352

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
61de18a
feat: add cleaner logging
nicoalee Oct 3, 2024
ef3d2a8
chore: cleanup
nicoalee Oct 3, 2024
bbed5fc
fix: remove unnecessary check
nicoalee Oct 3, 2024
0f68d65
chore: remove spaces
nicoalee Oct 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 62 additions & 22 deletions bin/rule_handler.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,29 +122,43 @@ function createRuleLogger(rule) {
//message: string
return {
info(message, groupkey) {
add('info', groupkey, message);
add("info", groupkey, message);
},
debug(message, groupkey) {
add('debug', groupkey, message);
add("debug", groupkey, message);
},
warning(message, groupkey) {
add('debug', groupkey, message);
add("debug", groupkey, message);
},
error(message, groupkey) {
add('debug', groupkey, message);
add("debug", groupkey, message);
},

set(obj, groupkey) {
const group = getGroup(groupkey);
Object.assign(group, obj);
},

addToOrCreateList(key, list, groupKey) {
const group = getGroup(groupKey);
if (!key || !Array.isArray(list)) return;

if (group[key]) {
if (!Array.isArray(group[key])) return; // lets be safe and not overwrite existing values
const oldArray = group[key];
const newArray = [...oldArray, ...list];
Object.assign(group, { [key]: newArray });
} else {
Object.assign(group, { [key]: list });
}
},

//save logs to disk
close(cb) {
const logpath = config.warehouse.rule_logdir+"/"+rule._id.toString()+".json";
fs.writeFile(logpath, JSON.stringify({root, groups}), cb);
const logpath = `${config.warehouse.rule_logdir}/${rule._id.toString()}.json`;
fs.writeFile(logpath, JSON.stringify({ root, groups }), cb);
},
}
};
}

function handle_rule(rule, cb) {
Expand Down Expand Up @@ -488,14 +502,25 @@ function handle_rule(rule, cb) {
//find all outputs from the app with tags specified in rule.output_tags[output_id]
let output_missing = false;
rule.app.outputs.forEach(output=>{
//I should ignore missing output if user doesn't want to archive it?
if(!rule.archive || !rule.archive[output.id] || !rule.archive[output.id].do) {
log.debug(output.id+" not archived - skip", group_id);
log.addToOrCreateList(
"skippedOutputIds",
[output.id],
group_id
);
// ignore missing output if user doesn't want to archive it
return;
}

if(!~output._groups.indexOf(group_id)) {
log.debug("output dataset not yet created for id:"+output.id, group_id);

log.addToOrCreateList(
"unfulfilledOutputIds",
[output.id],
group_id
);
output_missing = true;
}
});
Expand Down Expand Up @@ -543,22 +568,40 @@ function handle_rule(rule, cb) {
if(!input._datasets[input_group_id]) {
missing = true;
log.info("Found the output data that need to be generated, but we can't identify all required inputs and can't submit the app. missing input for "+input.id, group_id);
log.addToOrCreateList(
"missingDataForInputIds",
[input.id],
group_id
);
} else {
const candidates = input._datasets[input_group_id];

//check count
if(input.multi && rule.input_multicount) {
if(input.multi && rule.input_multicount && candidates.length !== rule.input_multicount[input.id]) {
//make sure we have exactly the expected number of candidates
if(candidates.length != rule.input_multicount[input.id]) {
log.info("We found "+candidates.length +" candidates objects for input:"+input.id+", but the rule is expecting "+rule.input_multicount[input.id]+" objects", group_id);
ambiguous = true;
}
} else {
log.info("We found "+candidates.length +" candidates objects for input:"+input.id+", but the rule is expecting "+rule.input_multicount[input.id]+" objects", group_id);
log.addToOrCreateList(
"ambiguousCases",
[{
inputId: input.id,
expectedMultiCount: rule.input_multicount[input.id],
datasetCandidateIds: candidates.map((d) => d._id),
}],
group_id
);
ambiguous = true;
} else if (candidates.length > 1) {
//let's not submit jobs if there are more than 1 candidates
if(candidates.length > 1) {
log.info("We found "+candidates.length +" candidates objects for input:"+input.id+". Please increase input specificity by adding tags.", group_id);
ambiguous = true;
}
log.info("We found "+candidates.length +" candidates objects for input:"+input.id+". Please increase input specificity by adding tags.", group_id);
log.addToOrCreateList(
"ambiguousCases",
[{
inputId: input.id,
expectedMultiCount: rule.input_multicount[input.id],
datasetCandidateIds: candidates.map((d) => d._id),
}],
group_id
);
ambiguous = true;
}

inputs[input.id] = candidates;
Expand Down Expand Up @@ -705,8 +748,6 @@ function handle_rule(rule, cb) {
const inputSpec = inputs[input_id]._inputSpec;
inputs[input_id].forEach(input=>{
log.debug("looking for source/staged data "+input._id+" for input "+input_id, group_id);

console.log(input);
//although we need to construct _outputs for product_raw, we are reusing most of the info
//for the main app's input. since product-raw doesn't really have id anyway, so let's just use
//app's input id as product_raw's id
Expand Down Expand Up @@ -918,7 +959,6 @@ function handle_rule(rule, cb) {
const datasets = inputs[output.datatype_tags_pass];
if(!datasets) {
console.error("datatype_tags_pass set but can't find the input:"+output.datatype_tags_pass);
console.log("inputs dump");
console.dir(inputs);
} else {
//for multi inputs.. let's just aggregate all meta for now
Expand Down
Loading