diff --git a/ctrack/art/un_logos/41111_old.png b/ctrack/art/un_logos/41111_old.png deleted file mode 100755 index 24fdec15..00000000 Binary files a/ctrack/art/un_logos/41111_old.png and /dev/null differ diff --git a/ctrack/art/un_logos/XM-DAC-41111.png b/ctrack/art/un_logos/XM-DAC-41111.png new file mode 100644 index 00000000..8600ddbb Binary files /dev/null and b/ctrack/art/un_logos/XM-DAC-41111.png differ diff --git a/ctrack/js/view_list_activities.js b/ctrack/js/view_list_activities.js index 93fd69e3..1db67700 100644 --- a/ctrack/js/view_list_activities.js +++ b/ctrack/js/view_list_activities.js @@ -1,146 +1,171 @@ // Copyright (c) 2014 International Aid Transparency Initiative (IATI) // Licensed under the MIT license whose full text can be found at http://opensource.org/licenses/MIT +var view_list_activities = exports; +exports.name = "view_list_activities"; -var view_list_activities=exports; -exports.name="view_list_activities"; +var csvw = require("./csvw.js"); -var csvw=require("./csvw.js") +var ctrack = require("./ctrack.js"); +var plate = require("./plate.js"); +var iati = require("./iati.js"); +var fetch = require("./fetch.js"); +var views = require("./views.js"); -var ctrack=require("./ctrack.js") -var plate=require("./plate.js") -var iati=require("./iati.js") -var fetch=require("./fetch.js") -var views=require("./views.js") +var refry = require("../../dstore/js/refry.js"); +var iati_codes = require("../../dstore/json/iati_codes.json"); +var crs_year = require("../../dstore/json/crs_2013.json"); -var refry=require("../../dstore/js/refry.js") -var iati_codes=require("../../dstore/json/iati_codes.json") -var crs_year=require("../../dstore/json/crs_2013.json") - -var commafy=function(s) { return (""+s).replace(/(^|[^\w.])(\d{4,})/g, function($0, $1, $2) { - return $1 + $2.replace(/\d(?=(?:\d\d\d)+(?!\d))/g, "$&,"); }) }; +var commafy = function(s) { + return ("" + s).replace(/(^|[^\w.])(\d{4,})/g, function($0, $1, $2) { + return $1 + $2.replace(/\d(?=(?:\d\d\d)+(?!\d))/g, "$&,"); + }); +}; // the chunk names this view will fill with new data -view_list_activities.chunks=[ - "list_activities_datas", - "list_activities_count", +view_list_activities.chunks = [ + "list_activities_datas", + "list_activities_count" ]; // // display the view // -view_list_activities.view=function() -{ - view_list_activities.chunks.forEach(function(n){ctrack.chunk(n,"{spinner_in_table_row}");}); - ctrack.setcrumb(2); - ctrack.change_hash(); - view_list_activities.ajax({q:ctrack.hash}); +view_list_activities.view = function() { + view_list_activities.chunks.forEach(function(n) { + ctrack.chunk(n, "{spinner_in_table_row}"); + }); + ctrack.setcrumb(2); + ctrack.change_hash(); + view_list_activities.ajax({ q: ctrack.hash }); }; // // Perform ajax call to get data // -view_list_activities.ajax=function(args) -{ - args=args || {}; - args.zerodata=args.zerodata||"{alert_no_data1}"; - - var dat={ - "from":"act", - "limit":args.limit || -1, - "select":"title,aid,funder_ref,"+ctrack.convert_str("commitment")+","+ctrack.convert_str("spend")+",reporting,reporting_ref,sector_ref,day_start,day_end", - "orderby":"4-", - "groupby":"aid", -// "country_code":(args.country || ctrack.args.country_select), -// "reporting_ref":(args.publisher || ctrack.args.publisher_select), -// "title_like":(args.search || ctrack.args.search), -// "sector_code":(args.sector_code || ctrack.args.sector_code_select), -// "sector_group":(args.sector_group || ctrack.args.sector_group_select), -// "funder_ref":(args.funder_ref || ctrack.args.funder_ref_select), -// "day_start_lteq":(args.date_max || ctrack.args.date_max), -// "day_end_gteq":(args.date_min || ctrack.args.date_min), - }; -// for(var n in ctrack.hash) { dat[n]=ctrack.hash[n]; } -// for(var n in args.q) { dat[n]=args.q[n]; } -// if(dat.sector_code||dat.sector_group) { dat.from+=",sector"; } -// if(dat.country_code) { dat.from+=",country"; } -// if(dat.location_latitude && dat.location_longitude) { dat.from+=",location"; } - - fetch.ajax_dat_fix(dat,args); - if(args.output=="count") // just count please - { - let y = Date.UTC(ctrack.args.selected_year, 1, 1) / (1000 * 60 * 60 * 24); - dat.select="count_aid"; - dat.between = [y,"day_start","day_end"], - delete dat.limit; - delete dat.orderby; - delete dat.groupby; - } - var eightyeight = 88; - fetch.ajax(dat,function(data){ - if(args.output=="count") - { - ctrack.chunk(args.chunk || "list_activities_count",data.rows[0]["count_aid"]); - views.stats.calc(); - } - else - { - - if(args.compare) - { - data.rows.sort(args.compare); - } - - var s=[]; - ctrack.args.chunks["table_header_amount"]=undefined; - if((data.rows.length==0)&&(args.zerodata)) - { - s.push( plate.replace(args.zerodata,{}) ); - ctrack.args.chunks["table_header_amount"]=""; - } - ctrack.chunk("list_activities_count",data.rows.length); - var a=[]; - for(var i=0;i100){d.pct=100;} - } - a.push(d); - s.push( plate.replace(args.plate || "{list_activities_data}",d) ); - } - ctrack.chunk(args.chunk || "list_activities_datas",s.join("")); - ctrack.chunk("total",data.rows.length); - - - var cc=[]; - cc[0]=["aid","title","reporting","commitment","spend","currency"]; - a.forEach(function(v){ - cc[cc.length]=[v.aid,v.title,v.reporting,v.commitment,v.spend,v.currency]; - }); - ctrack.chunk((args.chunk || "list_activities_datas")+"_csv","data:text/csv;charset=UTF-8,"+encodeURIComponent(csvw.arrayToCSV(cc))); - - } - if(args.callback){args.callback(data);} - ctrack.display(); - }); -} +view_list_activities.ajax = function(args) { + args = args || {}; + args.zerodata = args.zerodata || "{alert_no_data1}"; + + var dat = { + from: "act", + limit: args.limit || -1, + select: + "title,aid,funder_ref," + + ctrack.convert_str("commitment") + + "," + + ctrack.convert_str("spend") + + ",reporting,reporting_ref,sector_ref,day_start,day_end", + orderby: "4-", + groupby: "aid" + // "country_code":(args.country || ctrack.args.country_select), + // "reporting_ref":(args.publisher || ctrack.args.publisher_select), + // "title_like":(args.search || ctrack.args.search), + // "sector_code":(args.sector_code || ctrack.args.sector_code_select), + // "sector_group":(args.sector_group || ctrack.args.sector_group_select), + // "funder_ref":(args.funder_ref || ctrack.args.funder_ref_select), + // "day_start_lteq":(args.date_max || ctrack.args.date_max), + // "day_end_gteq":(args.date_min || ctrack.args.date_min), + }; + // for(var n in ctrack.hash) { dat[n]=ctrack.hash[n]; } + // for(var n in args.q) { dat[n]=args.q[n]; } + // if(dat.sector_code||dat.sector_group) { dat.from+=",sector"; } + // if(dat.country_code) { dat.from+=",country"; } + // if(dat.location_latitude && dat.location_longitude) { dat.from+=",location"; } + + fetch.ajax_dat_fix(dat, args); + if (args.output == "count") { + // just count please + let y = ctrack.args.selected_year; //Date.UTC(ctrack.args.selected_year, 1, 1) / (1000 * 60 * 60 * 24); + dat.select = "count_aid"; + (dat.between = [y, "day_start", "day_end"]), delete dat.limit; + delete dat.orderby; + delete dat.groupby; + } + var eightyeight = 88; + fetch.ajax(dat, function(data) { + if (args.output == "count") { + ctrack.chunk( + args.chunk || "list_activities_count", + data.rows[0]["count_aid"] + ); + views.stats.calc(); + } else { + if (args.compare) { + data.rows.sort(args.compare); + } + + var s = []; + ctrack.args.chunks["table_header_amount"] = undefined; + if (data.rows.length == 0 && args.zerodata) { + s.push(plate.replace(args.zerodata, {})); + ctrack.args.chunks["table_header_amount"] = ""; + } + ctrack.chunk("list_activities_count", data.rows.length); + var a = []; + for (var i = 0; i < data.rows.length; i++) { + var v = data.rows[i]; + var d = {}; + d.num = i + 1; + d.funder = v.funder || "N/A"; + d.aid = encodeURIComponent(v.aid || "N/A"); + d.title = v.title || v.aid || "N/A"; + + d.date_start = "N/A"; + d.date_end = "N/A"; + if (v.day_start !== null) { + d.date_start = fetch.get_nday(v.day_start); + } + if (v.day_end !== null) { + d.date_end = fetch.get_nday(v.day_end); + } + + d.reporting = + iati_codes.publisher_names[v.reporting_ref] || + v.reporting || + v.reporting_ref || + "N/A"; + d.commitment = commafy( + "" + Math.floor(ctrack.convert_num("commitment", v)) + ); + d.spend = commafy("" + Math.floor(ctrack.convert_num("spend", v))); + d.currency = ctrack.display_usd; + d.pct = 0; + if (v.commitment && v.commitment != 0) { + d.pct = Math.floor(100 * v.spend / v.commitment); + if (d.pct < 0) { + d.pct = 0; + } + if (d.pct > 100) { + d.pct = 100; + } + } + a.push(d); + s.push(plate.replace(args.plate || "{list_activities_data}", d)); + } + ctrack.chunk(args.chunk || "list_activities_datas", s.join("")); + ctrack.chunk("total", data.rows.length); + + var cc = []; + cc[0] = ["aid", "title", "reporting", "commitment", "spend", "currency"]; + a.forEach(function(v) { + cc[cc.length] = [ + v.aid, + v.title, + v.reporting, + v.commitment, + v.spend, + v.currency + ]; + }); + ctrack.chunk( + (args.chunk || "list_activities_datas") + "_csv", + "data:text/csv;charset=UTF-8," + encodeURIComponent(csvw.arrayToCSV(cc)) + ); + } + if (args.callback) { + args.callback(data); + } + ctrack.display(); + }); +}; diff --git a/ctrack/js/view_stats.js b/ctrack/js/view_stats.js index 655f6d05..d23783fe 100644 --- a/ctrack/js/view_stats.js +++ b/ctrack/js/view_stats.js @@ -1,42 +1,40 @@ // Copyright (c) 2014 International Aid Transparency Initiative (IATI) // Licensed under the MIT license whose full text can be found at http://opensource.org/licenses/MIT - var view_stats = exports; exports.name = "view_stats"; -var ctrack = require("./ctrack.js") -var plate = require("./plate.js") -var iati = require("./iati.js") -var fetch = require("./fetch.js") +var ctrack = require("./ctrack.js"); +var plate = require("./plate.js"); +var iati = require("./iati.js"); +var fetch = require("./fetch.js"); -var views = require("./views.js") +var views = require("./views.js"); // the chunk names this view will fill with new data view_stats.chunks = [ - "total_projects", - "missing_projects", - "active_projects", - "ended_projects", - "planned_projects", - "numof_publishers", - "total_budget", - "total_expenditure", - "percent_of_activities_with_location" + "total_projects", + "missing_projects", + "active_projects", + "ended_projects", + "planned_projects", + "numof_publishers", + "total_budget", + "total_expenditure", + "percent_of_activities_with_location" ]; -view_stats.calc = function () { - var tot = ctrack.chunk("total_projects") || 0; - var num = ctrack.chunk("total_activities_with_location") || 0; - if (num < 1 || tot < 1) { - ctrack.chunk("percent_of_activities_with_location", 0); - } - else { - var pct = Math.ceil(100 * num / tot); - ctrack.chunk("percent_of_activities_with_location", pct); - } +view_stats.calc = function() { + var tot = ctrack.chunk("total_projects") || 0; + var num = ctrack.chunk("total_activities_with_location") || 0; + if (num < 1 || tot < 1) { + ctrack.chunk("percent_of_activities_with_location", 0); + } else { + var pct = Math.ceil(100 * num / tot); + ctrack.chunk("percent_of_activities_with_location", pct); + } - /* + /* var pt=parseInt(ctrack.chunk("total_projects"))||0; var pa=parseInt(ctrack.chunk("active_projects"))||0; @@ -54,103 +52,125 @@ view_stats.calc = function () { } */ -// console.log(pm); -} - + // console.log(pm); +}; // // Perform ajax call to get numof data // -view_stats.ajax = function (args) { - args = args || {}; - let y = Date.UTC(ctrack.args.selected_year, 1, 1) / (1000 * 60 * 60 * 24); - - var dat = { - "select": "sum_budget_value", - "from": "act,budget", - "budget": "budget", - "between":[y,"budget_day_start","budget_day_end"], - - }; - - fetch.ajax_dat_fix(dat, args); - // console.log('==================dat==================',JSON.stringify(dat)) - fetch.ajax(dat, args.callback || function (data) { - // console.log("Fetching Budget"); - // console.log(dat); - // console.log(JSON.stringify(data)); - - if (data.rows[0]) { - ctrack.chunk("total_budget", changeToMillions(data.rows[0]["sum_budget_value"], ctrack.convert_usd)); - } - ctrack.display(); // every fetch.ajax must call display once - }); - - var dat = { - "select": "stats", - "from": "act", - "between":[y,"day_start","day_end"], - }; - fetch.ajax_dat_fix(dat, args); - fetch.ajax(dat, args.callback || function (data) { - if (data.rows[0]) { - ctrack.chunk("total_projects", data.rows[0]["distinct_aid"]); - ctrack.chunk("numof_publishers", data.rows[0]["distinct_reporting_ref"]); - //ctrack.chunk("total_budget", changeToMillions(data.rows[0]["budget_value)"], ctrack.convert_usd)); - ctrack.chunk("total_expenditure", changeToMillions(data.rows[0]["sum_spend"], ctrack.convert_usd)); - } - - view_stats.calc(); - - ctrack.display(); // every fetch.ajax must call display once - }); - - - var dat = { - "select": "stats", - "from": "act,location", - "limit": -1, - "location_longitude_not_null": 1, // must have a location - "location_latitude_not_null": 1, // must have a location - }; - fetch.ajax_dat_fix(dat, args); - if (dat.country_code) { - dat.country_percent = 100; - } - - fetch.ajax(dat, args.callback || function (data) { - - if (data.rows[0]) { - ctrack.chunk("total_activities_with_location", data.rows[0]["distinct_aid"]); - } - view_stats.calc(); - - ctrack.display(); // every fetch.ajax must call display once - }); - - views.planned.ajax({output: "count"}); - views.active.ajax({output: "count"}); - views.ended.ajax({output: "count"}); - views.missing.ajax({output: "count"}); - +view_stats.ajax = function(args) { + args = args || {}; + let y = ctrack.args.selected_year; //Date.UTC(ctrack.args.selected_year, 1, 1) / (1000 * 60 * 60 * 24); + + var dat = { + select: "sum_budget_value", + from: "act,budget", + budget: "budget", + between: [y, "budget_day_start", "budget_day_end"] + }; + + fetch.ajax_dat_fix(dat, args); + // console.log('==================dat==================',JSON.stringify(dat)) + fetch.ajax( + dat, + args.callback || + function(data) { + // console.log("Fetching Budget"); + // console.log(dat); + // console.log(JSON.stringify(data)); + + if (data.rows[0]) { + ctrack.chunk( + "total_budget", + changeToMillions( + data.rows[0]["sum_budget_value"], + ctrack.convert_usd + ) + ); + } + ctrack.display(); // every fetch.ajax must call display once + } + ); + + var dat = { + select: "stats", + from: "act", + between: [y, "day_start", "day_end"] + }; + fetch.ajax_dat_fix(dat, args); + fetch.ajax( + dat, + args.callback || + function(data) { + if (data.rows[0]) { + ctrack.chunk("total_projects", data.rows[0]["distinct_aid"]); + ctrack.chunk( + "numof_publishers", + data.rows[0]["distinct_reporting_ref"] + ); + //ctrack.chunk("total_budget", changeToMillions(data.rows[0]["budget_value)"], ctrack.convert_usd)); + ctrack.chunk( + "total_expenditure", + changeToMillions(data.rows[0]["sum_spend"], ctrack.convert_usd) + ); + } + + view_stats.calc(); + + ctrack.display(); // every fetch.ajax must call display once + } + ); + + var dat = { + select: "stats", + from: "act,location", + limit: -1, + location_longitude_not_null: 1, // must have a location + location_latitude_not_null: 1 // must have a location + }; + fetch.ajax_dat_fix(dat, args); + if (dat.country_code) { + dat.country_percent = 100; + } + + fetch.ajax( + dat, + args.callback || + function(data) { + if (data.rows[0]) { + ctrack.chunk( + "total_activities_with_location", + data.rows[0]["distinct_aid"] + ); + } + view_stats.calc(); + + ctrack.display(); // every fetch.ajax must call display once + } + ); + + views.planned.ajax({ output: "count" }); + views.active.ajax({ output: "count" }); + views.ended.ajax({ output: "count" }); + views.missing.ajax({ output: "count" }); }; -var changeToMillions = function (number, currencyVal) { - function numberWithCommas(x) { - return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); - } - var prettyNumber = "?"; - number = Math.round(number * currencyVal); - - var digits = new Number(number).toString().length; - - if (digits > 6) { - number = number / (1000000); - var roundedNumber = Math.round(number); - prettyNumber = numberWithCommas(roundedNumber) + ' M'; - } else { - prettyNumber = new Number(number).toString(); - } - - return prettyNumber; -} +var changeToMillions = function(number, currencyVal) { + function numberWithCommas(x) { + return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); + } + var prettyNumber = "?"; + number = Math.round(number * currencyVal); + + var digits = new Number(number).toString().length; + + if (digits > 6) { + number = number / 1000000; + var roundedNumber = Math.round(number); + prettyNumber = numberWithCommas(roundedNumber) + " M"; + } else { + prettyNumber = new Number(number).toString(); + } + + return prettyNumber; +}; diff --git a/dportal/js/undg.js b/dportal/js/undg.js index 528a55de..20c0c8a0 100644 --- a/dportal/js/undg.js +++ b/dportal/js/undg.js @@ -1,7 +1,7 @@ //TODO code quality -var fs = require('fs'); +var fs = require("fs"); var sqlite3 = require("sqlite3").verbose(); -var path = '../dstore/db/dstore.sqlite'; +var path = "../dstore/db/dstore.sqlite"; var sipac = new sqlite3.Database(path); var chunkopts = {}; @@ -9,252 +9,311 @@ var json_iati_codes = require("../../dstore/json/iati_codes.json"); var un_agencies_data = require("../../dstore/json/un_agencies_data.json") || {}; var geojson = require("../../dstore/json/countries.geo.json"); -sipac.serialize(function () { - var d = new Date(); - var year = parseInt(d.getFullYear()); - var month = parseInt(d.getMonth()); - var day = parseInt(d.getDate()); - var r = Date.UTC(year, month, day) / (1000 * 60 * 60 * 24); - - un_agencies_data['total_un_agencies'] = Object.keys(json_iati_codes.un_publisher_names).length; - un_agencies_data['un_agencies_in_iati'] = Object.keys(json_iati_codes.iati_un_publishers).length; - un_agencies_data['totalSectors'] = Object.keys(json_iati_codes.sector).length; - - //fetch Total Budget - let budget = {}; - let project = {}; - let expense = {}; - let activeProject = {}; - - for (let y = year; y >= (year - 5); y--) { - let start = Date.UTC(y, 1, 1) / (1000 * 60 * 60 * 24); - let end = Date.UTC(y, 12, 31) / (1000 * 60 * 60 * 24); - sipac.get("SELECT TOTAL(budget_value) FROM act JOIN budget USING (aid) WHERE budget =? AND ? BETWEEN budget_day_start AND budget_day_end", ['budget', start], function (err, row) { - budget[y] = row["TOTAL(budget_value)"]; - un_agencies_data['total_budget'] = budget; - }); - - //fetch Total activities and expenditure - var sql = "SELECT COUNT(DISTINCT aid) , TOTAL(spend) FROM act where reporting_ref NOTNULL AND ? BETWEEN day_start AND day_end"; - sipac.get(sql, [start], function (err, data) { - project[y] = data['COUNT(DISTINCT aid)']; - expense[y] = data['TOTAL(spend)']; - un_agencies_data['total_projects'] = project; - un_agencies_data['total_expenditure'] = expense; - }); - - //fetch Total active activities - sql = "SELECT COUNT(DISTINCT aid) AS count_aid FROM act WHERE ? BETWEEN day_start AND day_end AND day_end >= ? AND day_length IS NOT NULL"; - sipac.get(sql, [start, r], function (err, data) { - activeProject[y] = data['count_aid']; - un_agencies_data["active_projects"] = activeProject; - }); - } - - sipac.get("SELECT TOTAL(budget_value) FROM act JOIN budget USING (aid) WHERE budget =?", ['budget'], function (err, row) { - budget['all'] = row["TOTAL(budget_value)"]; - un_agencies_data['total_budget'] = budget; - }); - - var sql = "SELECT COUNT(DISTINCT aid) , TOTAL(spend) FROM act where reporting_ref NOT NULL"; - sipac.get(sql, function (err, data) { - project['all'] = data['COUNT(DISTINCT aid)']; - expense['all'] = data['TOTAL(spend)']; - un_agencies_data['total_projects'] = project; - un_agencies_data['total_expenditure'] = expense; +sipac.serialize(function() { + var d = new Date(); + var year = parseInt(d.getFullYear()); + var month = parseInt(d.getMonth()); + var day = parseInt(d.getDate()); + var r = Date.UTC(year, month, day) / (1000 * 60 * 60 * 24); + + un_agencies_data["total_un_agencies"] = Object.keys( + json_iati_codes.un_publisher_names + ).length; + un_agencies_data["un_agencies_in_iati"] = Object.keys( + json_iati_codes.iati_un_publishers + ).length; + un_agencies_data["totalSectors"] = Object.keys(json_iati_codes.sector).length; + + //fetch Total Budget + let budget = {}; + let project = {}; + let expense = {}; + let activeProject = {}; + + for (let y = year; y >= year - 5; y--) { + // let start = Date.UTC(y, 1, 1) / (1000 * 60 * 60 * 24); + // let end = Date.UTC(y, 12, 31) / (1000 * 60 * 60 * 24); + + sipac.get( + "SELECT TOTAL(budget_value) FROM act JOIN budget USING (aid) WHERE budget =? AND ? BETWEEN cast(strftime('%Y', datetime(budget_day_start * 60 * 60 * 24, 'unixepoch')) as integer) and cast(strftime('%Y', datetime(budget_day_end * 60 * 60 * 24, 'unixepoch')) as integer)", + ["budget", y], + function(err, row) { + budget[y] = row["TOTAL(budget_value)"]; + un_agencies_data["total_budget"] = budget; + } + ); + + //fetch Total activities and expenditure + var sql = + "SELECT COUNT(DISTINCT aid) , TOTAL(spend) FROM act where reporting_ref NOTNULL AND ? BETWEEN cast(strftime('%Y', datetime(day_start * 60 * 60 * 24, 'unixepoch')) as integer) and cast(strftime('%Y', datetime(day_end * 60 * 60 * 24, 'unixepoch')) as integer)"; + sipac.get(sql, [y], function(err, data) { + project[y] = data["COUNT(DISTINCT aid)"]; + expense[y] = data["TOTAL(spend)"]; + un_agencies_data["total_projects"] = project; + un_agencies_data["total_expenditure"] = expense; }); - sql = "SELECT COUNT(DISTINCT aid) AS count_aid FROM act WHERE day_start <= " + r + " AND ( day_end >= " + r + " OR day_end IS NULL ) AND day_length IS NOT NULL"; - sipac.get(sql, function (err, data) { - activeProject['all'] = data['count_aid']; - un_agencies_data["active_projects"] = activeProject; + //fetch Total active activities + sql = + "SELECT COUNT(DISTINCT aid) AS count_aid FROM act WHERE ? BETWEEN cast(strftime('%Y', datetime(day_start * 60 * 60 * 24, 'unixepoch')) as integer) and cast(strftime('%Y', datetime(day_end * 60 * 60 * 24, 'unixepoch')) as integer) AND day_end >= ? AND day_length IS NOT NULL"; + sipac.get(sql, [y, r], function(err, data) { + activeProject[y] = data["count_aid"]; + un_agencies_data["active_projects"] = activeProject; }); - - //fetch Total countries - sql = "Select Distinct country_code from country"; - var arr = {}; - arr['global-regional'] = {}; - sipac.all(sql, function (err, data) { - for (var code in data) { - if (json_iati_codes['country'].hasOwnProperty(data[code]['country_code'])) { - arr[data[code]['country_code']] = json_iati_codes['country'][data[code]['country_code']]; - } else if (json_iati_codes['region'].hasOwnProperty(data[code]['country_code'])) { - arr[data[code]['country_code']] = json_iati_codes['region'][data[code]['country_code']]; - } else { - arr['global-regional'][data[code]['country_code']] = data[code]['country_code']; - } - } - un_agencies_data['countries'] = arr; - }); - - //fetch projects trends - sql = "Select day_start,day_end from act"; - var currentProjectTrend = {}; - sipac.all(sql, function (err, data) { - for (var i in data) { - var start_year = getYearFromTimeStamp(parseInt(data[i]['day_start']) * (1000 * 60 * 60 * 24)); - var end_year = getYearFromTimeStamp(parseInt(data[i]['day_end']) * (1000 * 60 * 60 * 24)); - currentProjectTrend = getCurrentTrends(currentProjectTrend, [start_year, end_year]); - } - un_agencies_data['un_current_trends'] = currentProjectTrend; - }); - - //fetch projects sector_group - - sql = "Select Distinct sector_code from sector"; - var sectorGroupList = {}; - sipac.all(sql, function (err, data) { - for (var i in data) { - if (typeof json_iati_codes['sector'][data[i].sector_code] != 'undefined') { - sectorGroupList[data[i].sector_code] = json_iati_codes['sector'][data[i].sector_code]; - } - } - un_agencies_data['sectors'] = sectorGroupList; - - }); - + } + + sipac.get( + "SELECT TOTAL(budget_value) FROM act JOIN budget USING (aid) WHERE budget =?", + ["budget"], + function(err, row) { + budget["all"] = row["TOTAL(budget_value)"]; + un_agencies_data["total_budget"] = budget; + } + ); + + var sql = + "SELECT COUNT(DISTINCT aid) , TOTAL(spend) FROM act where reporting_ref NOT NULL"; + sipac.get(sql, function(err, data) { + project["all"] = data["COUNT(DISTINCT aid)"]; + expense["all"] = data["TOTAL(spend)"]; + un_agencies_data["total_projects"] = project; + un_agencies_data["total_expenditure"] = expense; + }); + + sql = + "SELECT COUNT(DISTINCT aid) AS count_aid FROM act WHERE day_start <= " + + r + + " AND ( day_end >= " + + r + + " OR day_end IS NULL ) AND day_length IS NOT NULL"; + sipac.get(sql, function(err, data) { + activeProject["all"] = data["count_aid"]; + un_agencies_data["active_projects"] = activeProject; + }); + + //fetch Total countries + sql = "Select Distinct country_code from country"; + var arr = {}; + arr["global-regional"] = {}; + sipac.all(sql, function(err, data) { + for (var code in data) { + if ( + json_iati_codes["country"].hasOwnProperty(data[code]["country_code"]) + ) { + arr[data[code]["country_code"]] = + json_iati_codes["country"][data[code]["country_code"]]; + } else if ( + json_iati_codes["region"].hasOwnProperty(data[code]["country_code"]) + ) { + arr[data[code]["country_code"]] = + json_iati_codes["region"][data[code]["country_code"]]; + } else { + arr["global-regional"][data[code]["country_code"]] = + data[code]["country_code"]; + } + } + un_agencies_data["countries"] = arr; + }); + + //fetch projects trends + sql = "Select day_start,day_end from act"; + var currentProjectTrend = {}; + sipac.all(sql, function(err, data) { + for (var i in data) { + var start_year = getYearFromTimeStamp( + parseInt(data[i]["day_start"]) * (1000 * 60 * 60 * 24) + ); + var end_year = getYearFromTimeStamp( + parseInt(data[i]["day_end"]) * (1000 * 60 * 60 * 24) + ); + currentProjectTrend = getCurrentTrends(currentProjectTrend, [ + start_year, + end_year + ]); + } + un_agencies_data["un_current_trends"] = currentProjectTrend; + }); + + //fetch projects sector_group + + sql = "Select Distinct sector_code from sector"; + var sectorGroupList = {}; + sipac.all(sql, function(err, data) { + for (var i in data) { + if ( + typeof json_iati_codes["sector"][data[i].sector_code] != "undefined" + ) { + sectorGroupList[data[i].sector_code] = + json_iati_codes["sector"][data[i].sector_code]; + } + } + un_agencies_data["sectors"] = sectorGroupList; + }); }); -var getYearFromTimeStamp = function (timeStamp) { - var d = new Date(timeStamp); - return d.getFullYear(); +var getYearFromTimeStamp = function(timeStamp) { + var d = new Date(timeStamp); + return d.getFullYear(); }; function getCurrentTrends(currentProjectTrend, yearly) { - var year = 0; - if (yearly != 0) { - var diff = yearly[1] - yearly[0]; - for (var i = 0; i <= diff; i++) { - year = parseInt(yearly[0]) + i; - if (year in currentProjectTrend) { - currentProjectTrend[year] = currentProjectTrend[year] + 1; - } else { - currentProjectTrend[year] = 1; - } - } + var year = 0; + if (yearly != 0) { + var diff = yearly[1] - yearly[0]; + for (var i = 0; i <= diff; i++) { + year = parseInt(yearly[0]) + i; + if (year in currentProjectTrend) { + currentProjectTrend[year] = currentProjectTrend[year] + 1; + } else { + currentProjectTrend[year] = 1; + } } - return currentProjectTrend; -}; + } + return currentProjectTrend; +} sipac.close(); setTimeout(function generate() { - fs.writeFileSync(__dirname + "/../../dstore/json/un_agencies_data.json", JSON.stringify(un_agencies_data, null, '\t')); - //update the publisher chunk - var pubs = []; - for (var id in un_agencies_data["sectors"]) { - var name = un_agencies_data["sectors"][id]; - var d = { - name: name - } - pubs.push(d); - } - for (var id in json_iati_codes["un_publisher_names"]) { - var name = json_iati_codes["un_publisher_names"][id]; - var d = { - name: name, - id: id - }; - pubs.push(d); + fs.writeFileSync( + __dirname + "/../../dstore/json/un_agencies_data.json", + JSON.stringify(un_agencies_data, null, "\t") + ); + //update the publisher chunk + var pubs = []; + for (var id in un_agencies_data["sectors"]) { + var name = un_agencies_data["sectors"][id]; + var d = { + name: name + }; + pubs.push(d); + } + for (var id in json_iati_codes["un_publisher_names"]) { + var name = json_iati_codes["un_publisher_names"][id]; + var d = { + name: name, + id: id + }; + pubs.push(d); + } + pubs.sort(function(a, b) { + var ta = a.name.toUpperCase(); + var tb = b.name.toUpperCase(); + return ta < tb ? -1 : ta > tb ? 1 : 0; + }); + chunkopts["publishers"] = pubs; + + // auto update the countries chunk + var ccs = []; + for (var id in un_agencies_data["countries"]) { + var name = un_agencies_data["countries"][id]; + + if (id === "global-regional") { + name = "Regional and global"; } - pubs.sort(function (a, b) { - var ta = a.name.toUpperCase(); - var tb = b.name.toUpperCase(); - return (ta < tb) ? -1 : (ta > tb) ? 1 : 0; - }); - chunkopts["publishers"] = pubs; - - // auto update the countries chunk - var ccs = []; - for (var id in un_agencies_data["countries"]) { - - var name = un_agencies_data["countries"][id]; - - if (id === 'global-regional') { - name = 'Regional and global'; - } - if (name) { - var d = { - name: name, - id: id - }; - ccs.push(d); - } + if (name) { + var d = { + name: name, + id: id + }; + ccs.push(d); } - ccs.sort(function (a, b) { - var ta = a.name.toUpperCase(); - var tb = b.name.toUpperCase(); - return (ta < tb) ? -1 : (ta > tb) ? 1 : 0; - }); - chunkopts["countries"] = ccs; - - var sec = []; - var secId = []; - for (var id in un_agencies_data["sectors"]) { - var cat = id.substring(0, 3); - - if (secId.indexOf(cat) == -1) { - var s = { - name: json_iati_codes["sector_category"][cat], - id: cat - }; - sec.push(s); - secId.push(cat); - } + } + ccs.sort(function(a, b) { + var ta = a.name.toUpperCase(); + var tb = b.name.toUpperCase(); + return ta < tb ? -1 : ta > tb ? 1 : 0; + }); + chunkopts["countries"] = ccs; + + var sec = []; + var secId = []; + for (var id in un_agencies_data["sectors"]) { + var cat = id.substring(0, 3); + + if (secId.indexOf(cat) == -1) { + var s = { + name: json_iati_codes["sector_category"][cat], + id: cat + }; + sec.push(s); + secId.push(cat); } - - sec.sort(function (a, b) { - var ta = a.name.toUpperCase(); - var tb = b.name.toUpperCase(); - - return (ta < tb) ? -1 : (ta > tb) ? 1 : 0; - }); - - chunkopts["sector"] = JSON.stringify(sec); - chunkopts["sector_un_operates"] = JSON.stringify(Object.keys(un_agencies_data["sectors"]).length); - chunkopts["publisher_names_json"] = JSON.stringify(json_iati_codes["un_publisher_names"]); - chunkopts["iati_un_publishers"] = JSON.stringify(json_iati_codes["iati_un_publishers"]); - chunkopts["country_names_json"] = JSON.stringify(un_agencies_data["countries"]); - chunkopts["crs_countries_json"] = JSON.stringify(un_agencies_data["countries"]); - chunkopts["total_projects"] = JSON.stringify(un_agencies_data["total_projects"]); - chunkopts["active_projects"] = JSON.stringify(un_agencies_data["active_projects"]); - chunkopts["total_budget"] = JSON.stringify(un_agencies_data["total_budget"]); - chunkopts["total_expenditure"] = JSON.stringify(un_agencies_data["total_expenditure"]); - chunkopts["total_un_agencies"] = JSON.stringify(un_agencies_data["total_un_agencies"]); - chunkopts["un_agencies_in_iati"] = JSON.stringify(un_agencies_data["un_agencies_in_iati"]); - chunkopts["total_sector"] = JSON.stringify(un_agencies_data["totalSectors"]); - - - var arr = Object.keys(un_agencies_data["un_current_trends"]).map(function (k) { - return k; - }); - var newYear = new Date(); - var currentYear = newYear.getFullYear(); - var currentYearStart = currentYear - 5; - currentYearStart = currentYearStart.toString(); - var currentYearEnd = currentYear + 2; - currentYearEnd = currentYearEnd.toString(); - var arr = arr.slice(arr.indexOf(currentYearStart), arr.indexOf(currentYearEnd)); - var un_trending = {}; - arr.forEach(function (year) { - un_trending[year] = un_agencies_data["un_current_trends"][year]; - }); - chunkopts["un_current_trends"] = JSON.stringify(un_trending); - - chunkopts["geojson"] = JSON.stringify(geojson); - - generateText(chunkopts); - -}, 5000); + } + + sec.sort(function(a, b) { + var ta = a.name.toUpperCase(); + var tb = b.name.toUpperCase(); + + return ta < tb ? -1 : ta > tb ? 1 : 0; + }); + + chunkopts["sector"] = JSON.stringify(sec); + chunkopts["sector_un_operates"] = JSON.stringify( + Object.keys(un_agencies_data["sectors"]).length + ); + chunkopts["publisher_names_json"] = JSON.stringify( + json_iati_codes["un_publisher_names"] + ); + chunkopts["iati_un_publishers"] = JSON.stringify( + json_iati_codes["iati_un_publishers"] + ); + chunkopts["country_names_json"] = JSON.stringify( + un_agencies_data["countries"] + ); + chunkopts["crs_countries_json"] = JSON.stringify( + un_agencies_data["countries"] + ); + chunkopts["total_projects"] = JSON.stringify( + un_agencies_data["total_projects"] + ); + chunkopts["active_projects"] = JSON.stringify( + un_agencies_data["active_projects"] + ); + chunkopts["total_budget"] = JSON.stringify(un_agencies_data["total_budget"]); + chunkopts["total_expenditure"] = JSON.stringify( + un_agencies_data["total_expenditure"] + ); + chunkopts["total_un_agencies"] = JSON.stringify( + un_agencies_data["total_un_agencies"] + ); + chunkopts["un_agencies_in_iati"] = JSON.stringify( + un_agencies_data["un_agencies_in_iati"] + ); + chunkopts["total_sector"] = JSON.stringify(un_agencies_data["totalSectors"]); + + var arr = Object.keys(un_agencies_data["un_current_trends"]).map(function(k) { + return k; + }); + var newYear = new Date(); + var currentYear = newYear.getFullYear(); + var currentYearStart = currentYear - 5; + currentYearStart = currentYearStart.toString(); + var currentYearEnd = currentYear + 2; + currentYearEnd = currentYearEnd.toString(); + var arr = arr.slice( + arr.indexOf(currentYearStart), + arr.indexOf(currentYearEnd) + ); + var un_trending = {}; + arr.forEach(function(year) { + un_trending[year] = un_agencies_data["un_current_trends"][year]; + }); + chunkopts["un_current_trends"] = JSON.stringify(un_trending); + + chunkopts["geojson"] = JSON.stringify(geojson); + + generateText(chunkopts); +}, 8000); function generateText(data) { - - let text = ""; - for (var key in data) { - var value = data[key] instanceof Object ? JSON.stringify(data[key]) : data[key]; - text += "#^" + key + "\n" + value + "\n\n"; - } - - fs.writeFileSync(__dirname + "/../source/\^.undg.txt", text); + let text = ""; + for (var key in data) { + var value = + data[key] instanceof Object ? JSON.stringify(data[key]) : data[key]; + text += "#^" + key + "\n" + value + "\n\n"; + } + + fs.writeFileSync(__dirname + "/../source/^.undg.txt", text); } diff --git a/dportal/source/index.^.html b/dportal/source/index.^.html index 77772a06..2521dd96 100644 --- a/dportal/source/index.^.html +++ b/dportal/source/index.^.html @@ -453,7 +453,7 @@

}); aa.push(""); - console.log("publishers",aa); + $("#publisher_dropmenu").html(aa.join("")); $("#publisher_dropmenu select").change(change); diff --git a/dstore/js/query.js b/dstore/js/query.js index fdb29229..47851670 100644 --- a/dstore/js/query.js +++ b/dstore/js/query.js @@ -1,41 +1,41 @@ // Copyright (c) 2014 International Aid Transparency Initiative (IATI) // Licensed under the MIT license whose full text can be found at http://opensource.org/licenses/MIT -var query=exports; +var query = exports; -var util=require('util'); -var fs=require('fs'); +var util = require("util"); +var fs = require("fs"); -var refry=require('./refry'); -var exs=require('./exs'); -var iati_xml=require('./iati_xml'); -var dstore_db=require("./dstore_db"); +var refry = require("./refry"); +var exs = require("./exs"); +var iati_xml = require("./iati_xml"); +var dstore_db = require("./dstore_db"); -var ls=function(a) { console.log(util.inspect(a,{depth:null})); } - - -query.mustbenumber=function(v) -{ - var n=Number(v); - if("number" == typeof n) - { - if(n==n) //check for nan - { - return n; - } - } - return undefined; -} - -query.maybenumber=function(v,ty) -{ - if(ty=="char") { return ""+v; } // force string - var n=query.mustbenumber(v); - if("number" == typeof n) { return n; } - return v; -} +var ls = function(a) { + console.log(util.inspect(a, { depth: null })); +}; +query.mustbenumber = function(v) { + var n = Number(v); + if ("number" == typeof n) { + if (n == n) { + //check for nan + return n; + } + } + return undefined; +}; +query.maybenumber = function(v, ty) { + if (ty == "char") { + return "" + v; + } // force string + var n = query.mustbenumber(v); + if ("number" == typeof n) { + return n; + } + return v; +}; // // 3 ways of passing query values @@ -48,778 +48,746 @@ query.maybenumber=function(v,ty) // //--REMOVED--- Finally you may perform a post request with json in the body. // -query.get_q = function(req){ - var q={}; - - q.start_time=Date.now(); - - var cp=function(f,unesc){ - for(var n in f) // single depth copy only - { - var v=f[n]; - if(unesc){ v=unesc(v); } // use unescape function? - if(q[n]===undefined) // only set if not exists, so call cp in order of priority from high to low - { - q[n]=v; - } - } - }; - -// use file extension as form - var aa=req.url.split("?"); - if(aa[0]) { - aa=aa[0].split("."); - if(aa[1]) { q.form=aa[1]; } - } - -// start with normal query - cp(req.query); - -// possibly containing an encoded json string? -// if(q._json) // expand json values for ?json=jsondata (also remove the this unexpanded value) -// { -// console.log(q._json); -// var t=JSON.parse(q._json); -// q.json=undefined; -// cp(t); -// } - -// finally the body may contain json so add that as well - if(req.body) - { - cp(req.body); - } - -// defaults - if(!q.from) - { - q.from="act"; // default act - } - if( q.form=="xml" || q.form=="html" ) // xml view so we... - { - if(q.from.indexOf("jml")==-1) // only add once - { - q.from+=",jml"; // ...need a jml join to spit out xml (jml is jsoned xml) - } - } - - -// we now have a json style chunk of data that consists of many possible inputs - return q; +query.get_q = function(req) { + var q = {}; + + q.start_time = Date.now(); + + var cp = function(f, unesc) { + for (var n in f) { + // single depth copy only + var v = f[n]; + if (unesc) { + v = unesc(v); + } // use unescape function? + if (q[n] === undefined) { + // only set if not exists, so call cp in order of priority from high to low + q[n] = v; + } + } + }; + + // use file extension as form + var aa = req.url.split("?"); + if (aa[0]) { + aa = aa[0].split("."); + if (aa[1]) { + q.form = aa[1]; + } + } + + // start with normal query + cp(req.query); + + // possibly containing an encoded json string? + // if(q._json) // expand json values for ?json=jsondata (also remove the this unexpanded value) + // { + // console.log(q._json); + // var t=JSON.parse(q._json); + // q.json=undefined; + // cp(t); + // } + + // finally the body may contain json so add that as well + if (req.body) { + cp(req.body); + } + + // defaults + if (!q.from) { + q.from = "act"; // default act + } + if (q.form == "xml" || q.form == "html") { + // xml view so we... + if (q.from.indexOf("jml") == -1) { + // only add once + q.from += ",jml"; // ...need a jml join to spit out xml (jml is jsoned xml) + } + } + + // we now have a json style chunk of data that consists of many possible inputs + return q; }; -query.getsql_select=function(q,qv){ - var ss=[]; - -// var stats_skip={ // ignore these columns -// "xml":true, -// "jml":true, -// "json":true -// }; - - - var ns=q[0]; - -// extra special calculations - var pcts={"country":true,"sector":true,"location":true}; - var percents=function(ret,name,agg){ - var mults=[]; - var aa=q.from.split(","); - for(var i=0;i", - "_lteq":"<=", - "_gteq":">=", - "_nteq":"!=", - "_eq":"=", - "_glob":"GLOB", - "_like":"LIKE", - "_null":"NULL", - "_not_null":"NOT NULL", - "":"=", - "_bt": "BETWEEN" - }; - - var niq=0; - for(var n in ns) - { - for( var qe in qemap ) - { - var ty=ns[n]; - var v=q[n+qe]; - var eq=qemap[qe]; - - if( v !== undefined ) // got a value - { - if( eq=="NOT NULL") { ss.push( " "+n+" IS NOT NULL " ); } - else - if( eq=="NULL") { ss.push( " "+n+" IS NULL " ); } - else - { - var t=typeof v; - if(t=="string") - { - var sa=v.split("|"); - var sb=/^([0-9]{4})-([0-9]{2})-([0-9]{2})$/.exec(sa[0]); - if( sa[0].length==10 && sb && sb.length==4 && ty=="int") // date string, convert to number if dest is an int - { - v=iati_xml.isodate_to_number(v); - - if(sa.length==2 && (/null/i).test(sa[1]) ) // allow an explicit or |null case - { - ss.push( " ( "+n+" "+eq+" "+dstore_db.text_plate(n+qe)+" OR "+n+" IS NULL ) " ); qv[dstore_db.text_name(n+qe)]=query.maybenumber(v,ty); - } - else - { - ss.push( " "+n+" "+eq+" "+dstore_db.text_plate(n+qe)+" " ); qv[dstore_db.text_name(n+qe)]=query.maybenumber(v,ty); - } - } - else - if(sa[1]) // there was an "|" - { - v=sa; - t="object"; // do object below - } - else - { - ss.push( " "+n+" "+eq+" "+dstore_db.text_plate(n+qe)+" " ); qv[dstore_db.text_name(n+qe)]=query.maybenumber(v,ty); - } - } - else - if(t=="number") - { - ss.push( " "+n+" "+eq+" "+dstore_db.text_plate(n+qe)+" " ); qv[dstore_db.text_name(n+qe)]=v; - } - - if(t=="object") // array, string above may also have been split into array - { - var so=[]; - for(var i=0;i0) { st.push( " OR " ); } - st.push( " "+n+" "+eq+v ); - } - st.push( " ) " ); - - ss.push(st.join("")); - } - } - } - } - } - } - - var v=q["text_search"]; - if( (ns["title"]) && (ns["description"]) && v ) // description and title and text_search available - { -//console.log("text_search "+v) - if(argv.pg) // can use better pg search code - { - ss.push( " to_tsvector('simple',title || ' ' || description) @@ plainto_tsquery('simple',"+dstore_db.text_plate("text_search")+") " ); - qv[dstore_db.text_name("text_search")]=v; - } - else // can only use old sqlite search code that only checks title - { - ss.push( " title LIKE "+dstore_db.text_plate("text_search") ); - qv[dstore_db.text_name("text_search")]="%"+v+"%"; - } - } - - - var ret=""; - if(ss[0]) { ret=" WHERE "+ss.join(" AND "); } - - return ret; +query.getsql_where = function(q, qv) { + var ss = []; + + var ns = q[0]; + + var joins = {}; + + var qemap = { + // possible comparisons + _lt: "<", + _gt: ">", + _lteq: "<=", + _gteq: ">=", + _nteq: "!=", + _eq: "=", + _glob: "GLOB", + _like: "LIKE", + _null: "NULL", + _not_null: "NOT NULL", + "": "=", + _bt: "BETWEEN" + }; + + var niq = 0; + for (var n in ns) { + for (var qe in qemap) { + var ty = ns[n]; + var v = q[n + qe]; + var eq = qemap[qe]; + + if (v !== undefined) { + // got a value + if (eq == "NOT NULL") { + ss.push(" " + n + " IS NOT NULL "); + } else if (eq == "NULL") { + ss.push(" " + n + " IS NULL "); + } else { + var t = typeof v; + if (t == "string") { + var sa = v.split("|"); + var sb = /^([0-9]{4})-([0-9]{2})-([0-9]{2})$/.exec(sa[0]); + if (sa[0].length == 10 && sb && sb.length == 4 && ty == "int") { + // date string, convert to number if dest is an int + v = iati_xml.isodate_to_number(v); + + if (sa.length == 2 && /null/i.test(sa[1])) { + // allow an explicit or |null case + ss.push( + " ( " + + n + + " " + + eq + + " " + + dstore_db.text_plate(n + qe) + + " OR " + + n + + " IS NULL ) " + ); + qv[dstore_db.text_name(n + qe)] = query.maybenumber(v, ty); + } else { + ss.push( + " " + n + " " + eq + " " + dstore_db.text_plate(n + qe) + " " + ); + qv[dstore_db.text_name(n + qe)] = query.maybenumber(v, ty); + } + } else if (sa[1]) { + // there was an "|" + v = sa; + t = "object"; // do object below + } else { + ss.push( + " " + n + " " + eq + " " + dstore_db.text_plate(n + qe) + " " + ); + qv[dstore_db.text_name(n + qe)] = query.maybenumber(v, ty); + } + } else if (t == "number") { + ss.push( + " " + n + " " + eq + " " + dstore_db.text_plate(n + qe) + " " + ); + qv[dstore_db.text_name(n + qe)] = v; + } + + if (t == "object") { + // array, string above may also have been split into array + var so = []; + for (var i = 0; i < v.length; i++) { + so.push(" " + dstore_db.text_plate(n + qe + "_" + i) + " "); + qv[dstore_db.text_name(n + qe + "_" + i)] = query.maybenumber( + v[i], + ty + ); + } + if (v.length == 2 && /null/i.test(v[1])) { + // allow an explicit or null case for base comparisons + ss.push( + " ( " + n + " " + eq + so[0] + " OR " + n + " IS NULL ) " + ); + qv[so[1].trim()] = undefined; // not going to be used + } else if (eq == "=") { + // make an IN statement + ss.push(" " + n + " IN (" + so.join(",") + ") "); + } else { + // explode into a bunch of OR statements + var st = []; + st.push(" ( "); + for (var i = 0; i < so.length; i++) { + var v = so[i]; + if (i > 0) { + st.push(" OR "); + } + st.push(" " + n + " " + eq + v); + } + st.push(" ) "); + + ss.push(st.join("")); + } + } + } + } + } + } + + var v = q["text_search"]; + if (ns["title"] && ns["description"] && v) { + // description and title and text_search available + //console.log("text_search "+v) + if (argv.pg) { + // can use better pg search code + ss.push( + " to_tsvector('simple',title || ' ' || description) @@ plainto_tsquery('simple'," + + dstore_db.text_plate("text_search") + + ") " + ); + qv[dstore_db.text_name("text_search")] = v; + } else { + // can only use old sqlite search code that only checks title + ss.push(" title LIKE " + dstore_db.text_plate("text_search")); + qv[dstore_db.text_name("text_search")] = "%" + v + "%"; + } + } + + var ret = ""; + if (ss[0]) { + ret = " WHERE " + ss.join(" AND "); + } + + return ret; }; -query.getsql_group_by=function(q,qv){ - var ss=[]; - - var ns=q[0]; - - if(q.groupby) - { - var qq; - qq=q.groupby.split(","); - for(var i=0;i0)&&(n<100)&&(n==n)) - { - ss.push(n+xtra); - } - } - } - } - - if(ss[0]) { return " ORDER BY "+ss.join(" , "); } - return ""; +query.getsql_order_by = function(q, qv) { + var ss = []; + + var ns = q[0]; + + if (q.orderby) { + var qq; + qq = q.orderby.split(","); + for (var i = 0; i < qq.length; i++) { + var xtra = ""; + var v = qq[i]; + if (v.slice(-1) == "-") { + xtra = " DESC"; + v = v.slice(0, -1); + } + if (ns[v]) { + // valid member names only + ss.push(v + xtra); + } else { + var n = parseInt(v); // or allow numerical indexes into results row + if (n > 0 && n < 100 && n == n) { + ss.push(n + xtra); + } + } + } + } + + if (ss[0]) { + return " ORDER BY " + ss.join(" , "); + } + return ""; }; -query.getsql_limit=function(q,qv){ - var ss=[]; - var limit=100; - - if( q.limit ) - { - var n=query.mustbenumber(q.limit); - if( "number" == typeof n) - { - limit=n - } - } - - if(limit>=0) - { - ss.push(" LIMIT "+limit+" "); - } - - if( q.page ) - { - var n=query.mustbenumber(q.page); - if( "number" == typeof n) - { - ss.push(" OFFSET "+n*limit+" "); - } - } - else - if( q.offset ) - { - var n=query.mustbenumber(q.offset); - if( "number" == typeof n) - { - ss.push(" OFFSET "+n+" "); - } - } - - if(ss[0]) { return ss.join(""); } - return ""; +query.getsql_limit = function(q, qv) { + var ss = []; + var limit = 100; + + if (q.limit) { + var n = query.mustbenumber(q.limit); + if ("number" == typeof n) { + limit = n; + } + } + + if (limit >= 0) { + ss.push(" LIMIT " + limit + " "); + } + + if (q.page) { + var n = query.mustbenumber(q.page); + if ("number" == typeof n) { + ss.push(" OFFSET " + n * limit + " "); + } + } else if (q.offset) { + var n = query.mustbenumber(q.offset); + if ("number" == typeof n) { + ss.push(" OFFSET " + n + " "); + } + } + + if (ss[0]) { + return ss.join(""); + } + return ""; }; -query.getsql_build_column_names=function(q,qv){ - - var ns={}; - for(var name in dstore_db.tables ) - { - for(var n in dstore_db.tables_active[name]) - { - ns[n]=dstore_db.tables_active[name][n]; - } - } +query.getsql_build_column_names = function(q, qv) { + var ns = {}; + for (var name in dstore_db.tables) { + for (var n in dstore_db.tables_active[name]) { + ns[n] = dstore_db.tables_active[name][n]; + } + } - q[0]=ns; // special array of valid column names + q[0] = ns; // special array of valid column names +}; +query.do_select_response = function(q, res, r) { + var send_json = function(r) { + if (q.callback) { + res.jsonp(r); // seems to only get headers right with a callback + } else { + res.set("charset", "utf8"); + res.set("Content-Type", "application/json"); + res.json(r); + } + }; + + res.set("charset", "utf8"); // This is always the correct answer. + + if (q.form == "html") { + res.set("Content-Type", "text/xml"); + + var xsl = + '\n'; + if (q.xsl == "!") { + xsl = ""; + } // disable pretty view + + res.write( + '\n' + + xsl + + '\n' + ); + + for (var i = 0; i < r.rows.length; i++) { + var v = r.rows[i]; + if (v && v.jml) { + res.write(refry.json(v.jml)); + res.write("\n"); + } + } + + res.end("\n"); + } else if (q.form == "xml") { + res.set("Content-Type", "text/xml"); + + res.write( + '\n' + ); + + for (var i = 0; i < r.rows.length; i++) { + var v = r.rows[i]; + if (v && v.jml) { + var d = JSON.parse(v.jml); + delete d["dstore:slug"]; // remove dstore tags + delete d["dstore:idx"]; + + res.write(refry.json(d)); + res.write("\n"); + } + } + + res.end("\n"); + } else if (q.form == "csv") { + res.set("Content-Type", "text/csv"); + + var head = []; + if (r.rows[0]) { + for (var n in r.rows[0]) { + head.push(n); + } + head.sort(); + res.write(head.join(",") + "\n"); + for (var i = 0; i < r.rows.length; i++) { + var v = r.rows[i]; + var t = []; + head.forEach(function(n) { + var s = "" + v[n]; + if ("string" == typeof s) { + // may need to escape + if (s.split(",")[1] || s.split("\n")[1]) { + // need to escape + s = '"' + s.split('"').join('""') + '"'; // wrap in quotes and double quotes in string + } + } + t.push(s); + }); + res.write(t.join(",") + "\n"); + } + res.end(""); + } else { + res.end(""); + } + } else if (q.form == "jcsv") { + // a jsoned csv (much smaller than json for large table data) + if (r.rows[0]) { + var head = []; + var ta = []; + for (var n in r.rows[0]) { + head.push(n); + } + head.sort(); // result order will be fixed + ta[0] = head; + for (var i = 0; i < r.rows.length; i++) { + var v = r.rows[i]; + var t = []; + ta[i + 1] = t; + head.forEach(function(n) { + t.push(v[n] || null); + }); + } + send_json(ta); + } else { + send_json([]); // nothing to see, but still trigger callback + } + } else { + r.time = (Date.now() - q.start_time) / 1000; + send_json(r); + } }; -query.do_select_response=function(q,res,r){ - - var send_json=function(r) - { - if(q.callback) - { - res.jsonp(r); // seems to only get headers right with a callback - } - else - { - res.set('charset','utf8'); - res.set('Content-Type', 'application/json'); - res.json(r); - } - }; - - res.set('charset','utf8'); // This is always the correct answer. - - if(q.form=="html") - { - res.set('Content-Type', 'text/xml'); - - var xsl='\n'; - if(q.xsl=="!") { xsl=""; } // disable pretty view - - res.write( '\n'+ - xsl+ - '\n'); - - for(var i=0;i\n'); - - } - else - if(q.form=="xml") - { - res.set('Content-Type', 'text/xml'); - - res.write( '\n'); - - for(var i=0;i\n'); - - } - else - if(q.form=="csv") - { - res.set('Content-Type', 'text/csv'); - - var head=[]; - if(r.rows[0]) - { - for(var n in r.rows[0]) { head.push(n); } - head.sort(); - res.write( head.join(",")+"\n" ); - for(var i=0;i