Skip to content

Commit

Permalink
removed more old files, added queries for the lookups to the query maker
Browse files Browse the repository at this point in the history
  • Loading branch information
@punkish
punkish committed Apr 20, 2020
1 parent 0f3606c commit 0cbb294
Show file tree
Hide file tree
Showing 12 changed files with 162 additions and 260 deletions.
30 changes: 26 additions & 4 deletions api/v2/lib/dd2queries.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ const calcConstraint = function(ddKeys, queryObject) {
// LIMIT <limit>
// OFFSET <offset>
const calcQuery = function(ddKeys, queryGroup, query, queryObject, matchTables, addedConstraint) {

const columns = query.columns;
const tables = JSON.parse(JSON.stringify(query.tables));
const constraint = JSON.parse(JSON.stringify(query.constraint));
Expand Down Expand Up @@ -242,6 +242,7 @@ const dd2queries = function(queryObject) {

for (let queryName in groupQueries) {
const query = groupQueries[queryName];

const sql = calcQuery(
ddKeys,
queryGroup,
Expand All @@ -265,7 +266,7 @@ const dd2queries = function(queryObject) {

const test = function() {

const queryObject = {
const q1 = {
communities: ['biosyslit', 'belgiumherbarium'],
refreshCache: false,
page: 1,
Expand All @@ -284,8 +285,29 @@ const test = function() {
// treatmentId: '58F12CC7CCAD08F32CF9920D36C9992E'
};

const q = dd2queries(queryObject);
console.log(q);
const lookups = [
{
resource: 'authors',
q: 'ago'
},
{
resource: 'keywords',
q: 'son'
},
{
resource: 'families',
q: 'cho'
},
{
resource: 'taxa',
q: 'tin'
},
];

lookups.forEach(l => {
const q = dd2queries(l);
console.log(q);
});

};

Expand Down
96 changes: 96 additions & 0 deletions api/v2/lib/qparts.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -765,6 +765,102 @@ const queryParts = {
facets: {}
}

},

authors: {
//pk: '',

queries: {

essential: {
//count: {},

data: {
columns: ['author'],
tables: ['authors'],
constraint: ['0 = 0'],
sortBy: {},
group: [],
pagination: false
}
},

// related: {},
// stats: {},
// facets: {}
}
},

keywords: {
//pk: '',

queries: {

essential: {
//count: {},

data: {
columns: ['keyword'],
tables: ['keywords'],
constraint: ['0 = 0'],
sortBy: {},
group: [],
pagination: false
}
},

// related: {},
// stats: {},
// facets: {}
}
},

taxa: {
//pk: '',

queries: {

essential: {
//count: {},

data: {
columns: ['taxon'],
tables: ['taxa'],
constraint: ['0 = 0'],
sortBy: {},
group: [],
pagination: false
}
},

// related: {},
// stats: {},
// facets: {}
}
},

families: {
//pk: '',

queries: {

essential: {
//count: {},

data: {
columns: ['family'],
tables: ['families'],
constraint: ['0 = 0'],
sortBy: {},
group: [],
pagination: false
}
},

// related: {},
// stats: {},
// facets: {}
}
}
};

Expand Down
19 changes: 9 additions & 10 deletions api/v2/lib/zenodeo.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,6 @@ const getManyRecords = async function(queryObject) {
// first find total number of matches
const countSql = q.essential.count.sql;


try {

let t = process.hrtime();
Expand All @@ -221,6 +220,7 @@ const getManyRecords = async function(queryObject) {
.numOfRecords;

t = process.hrtime(t);

const p = {
sql: Utils.strfmt(countSql, queryObject),
took: Utils.timerFormat(t)
Expand Down Expand Up @@ -253,16 +253,19 @@ const getManyRecords = async function(queryObject) {
return Utils.dataForDelivery(timer, data, debug);
}

// get the records
// The count query above returned a number successfully
// which is why we have reached here. So now we get the
// actual records
const dataSql = q.essential.data.sql;

try {

let t = process.hrtime();

data.records = db.prepare(dataSql).all(queryObject) || [];
data.records = db.prepare(dataSql).all(queryObject);

t = process.hrtime(t);

const p = {
sql: Utils.strfmt(dataSql, queryObject),
took: Utils.timerFormat(t)
Expand Down Expand Up @@ -317,7 +320,7 @@ const getManyRecords = async function(queryObject) {

data.previd = id;

data.prevpage = queryObject.page >= 1 ? +queryObject.page - 1 : '';
data.prevpage = queryObject.page >= 2 ? +queryObject.page - 1 : '';
data.nextpage = num < queryObject.size ? '' : +queryObject.page + 1;

data._links.prev = Utils.makeLink({
Expand All @@ -333,9 +336,7 @@ const getManyRecords = async function(queryObject) {
});

// finally, get facets and stats, if requested
const groupedQueries = ['facets', 'stats'];
//console.log(queryObject);
groupedQueries.forEach(g => {
['facets', 'stats'].forEach(g => {
if (g in queryObject && queryObject[g] === 'true') {
data[g] = getStatsFacets(g, q, queryObject, debug);
}
Expand All @@ -357,14 +358,12 @@ const getStatsFacets = function(type, q, queryObject, debug) {

const sql = q[type][query].sql;
const foo = q[type][query].sql;
//console.log(`foo: ${Utils.strfmt(foo, queryObject)}`)

try {
result[query] = db.prepare(sql).all(queryObject);
}
catch (error) {
//plog.error(error, Utils.strfmt(sql, queryObject));
console.log(error)
plog.error(error, Utils.strfmt(sql, queryObject));
}

t = process.hrtime(t);
Expand Down
35 changes: 19 additions & 16 deletions api/v2/utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,16 @@ const config = require('config');
const plog = require(config.get('plog'));
const cacheOn = config.get('v2.cache.on');
const Database = require('better-sqlite3');
const dbFacets = new Database(config.get('data.facets'));
const dbLookups = new Database(config.get('data.lookups'));
const dbTreatments = new Database(config.get('data.treatments'));
const dbQueries = new Database(config.get('data.queries'));

const facets = {
authors: 'author',
families: 'family',
keywords: 'keyword',
taxa: 'taxon'
};
const dbQueryStats = new Database(config.get('data.queryStats'));

// Modify the string prototype to mimic strfmt() so SQL statements
// can be sent back in debug or printed to the console with all the
// parameters visible. See the following SO link for details.
// https://stackoverflow.com/a/18234317/183692
String.prototype.formatUnicorn = String.prototype.formatUnicorn || function () {

"use strict";
var str = this.toString();
if (arguments.length) {
Expand Down Expand Up @@ -56,15 +50,15 @@ module.exports = {
const {queryObject, sqls} = debug;
const inserts = sqls.length;

const s1 = dbQueries.prepare(`INSERT INTO webqueries (qp) VALUES(@qp) ON CONFLICT(qp) DO UPDATE SET count=count+1`);
const s1 = dbQueryStats.prepare(`INSERT INTO webqueries (qp) VALUES(@qp) ON CONFLICT(qp) DO UPDATE SET count=count+1`);

const s2 = dbQueries.prepare('SELECT Max(id) AS id FROM webqueries');
const s2 = dbQueryStats.prepare('SELECT Max(id) AS id FROM webqueries');

const s3 = dbQueries.prepare(`INSERT INTO sqlqueries (sql) VALUES(@sql) ON CONFLICT(sql) DO NOTHING`);
const s3 = dbQueryStats.prepare(`INSERT INTO sqlqueries (sql) VALUES(@sql) ON CONFLICT(sql) DO NOTHING`);

const s4 = dbQueries.prepare('SELECT Max(id) AS id FROM sqlqueries');
const s4 = dbQueryStats.prepare('SELECT Max(id) AS id FROM sqlqueries');

const s5 = dbQueries.prepare('INSERT INTO stats (webqueries_id, sqlqueries_id, timeTaken) VALUES (@webqueries_id, @sqlqueries_id, @timeTaken)');
const s5 = dbQueryStats.prepare('INSERT INTO stats (webqueries_id, sqlqueries_id, timeTaken) VALUES (@webqueries_id, @sqlqueries_id, @timeTaken)');

if (inserts) {

Expand Down Expand Up @@ -94,7 +88,7 @@ module.exports = {
}
}
catch (error) {
console.log(error);
plog.error(error);
}

}
Expand Down Expand Up @@ -135,7 +129,15 @@ module.exports = {
},

find: function(pattern, source) {
return dbFacets.prepare(`SELECT ${facets[source]} FROM ${source} WHERE ${facets[source]} LIKE ?`)

const lookups = {
authors: 'author',
families: 'family',
keywords: 'keyword',
taxa: 'taxon'
};

return dbLookups.prepare(`SELECT ${lookups[source]} FROM ${source} WHERE ${lookups[source]} LIKE ?`)
.raw()
.all(`%${pattern}%`)
.map(r => r[0]);
Expand Down Expand Up @@ -385,6 +387,7 @@ module.exports = {
},

makeLink: function({uri, params, page}) {

const qs = Object.entries(params)
.filter(e => e[0] !== 'path')
.map(e => page ? `${e[0]}=${(e[0] === 'page' ? page : e[1])}`
Expand Down
35 changes: 0 additions & 35 deletions bin/calcTaxonStats.js
View file

This file was deleted.

Loading

0 comments on commit 0cbb294

Please sign in to comment.