remove slonik, insert postgres.js

lib/bin/run-server.js

@@ -13,15 +13,14 @@
 const { merge } = require('ramda');
 const config = require('config');
 const exit = require('express-graceful-exit');
-
-global.tap = (x) => { console.log(x); return x; }; // eslint-disable-line no-console
+require('../util/tap');
 
 ////////////////////////////////////////////////////////////////////////////////
 // CONTAINER SETUP
 
 // initialize our slonik connection pool.
-const { slonikPool } = require('../external/slonik');
-const db = slonikPool(config.get('default.database'));
+const { postgres } = require('../external/postgres');
+const db = postgres(config.get('default.database'));
 
 // set up our mailer.
 const env = config.get('default.env');

lib/data/attachments.js

@@ -30,9 +30,7 @@ const streamAttachments = (inStream, decryptor) => {
   const writable = new Writable({
     objectMode: true,
     highWaterMark: 5, // the default is 16, we'll be a little more conservative.
-    write(x, _, done) {
-      const att = x.row;
-
+    write(att, _, done) {
       // this sanitization means that two filenames could end up identical.
       // luckily, this is not actually illegal in the zip spec; two files can live at precisely
       // the same location, and the conflict is dealt with interactively by the unzipping client.

lib/data/briefcase.js

@@ -339,7 +339,8 @@ const streamBriefcaseCsvs = (inStream, inFields, xmlFormId, selectValues, decryp
   // to provide the data to do so.
   const archive = zipPart();
   const name = `${sanitize(xmlFormId)}.csv`;
-  archive.append(PartialPipe.of(inStream, rootStream, csv()), { name });
+  archive.append(PartialPipe.of(inStream, rootStream, csv()), { name }, () => { archive.finalize(); });
+
   {
     // two passes; this first pass counts field names (so we know later whether
     // to append a ~1 number).
@@ -364,7 +365,7 @@ const streamBriefcaseCsvs = (inStream, inFields, xmlFormId, selectValues, decryp
       }
     }
   }
-  archive.finalize();
+
   return archive;
 };
 

lib/data/client-audits.js

@@ -82,11 +82,10 @@ const streamClientAudits = (inStream, form, decryptor) => {
   let first = true;
   const csvifier = new Transform({
     objectMode: true,
-    transform(x, _, done) {
+    transform(data, _, done) {
       // data here contains ClientAudit attchement info as well as associated
       // submission instanceId fetched through query in
       // model/query/client-audits.js
-      const data = x.row;
 
       // TODO: we do not currently try/catch this block because it feels low risk.
       // this may not actually be the case..

lib/data/odata-filter.js

@@ -7,8 +7,7 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { sql } = require('slonik');
-const { raw } = require('slonik-sql-tag-raw');
+const { sql } = require('../external/postgres');
 const odataParser = require('odata-v4-parser');
 const Problem = require('../util/problem');
 
@@ -20,24 +19,24 @@ const methodCall = (fn, params) => {
   // n.b. odata-v4-parser appears to already validate function name and arity.
   const lowerName = fn.toLowerCase();
   if (extractFunctions.includes(lowerName))
-    return sql`extract(${raw(lowerName)} from ${op(params[0])})`; // eslint-disable-line no-use-before-define
+    return sql`extract(${sql.unsafe(lowerName)} from ${op(params[0])})`; // eslint-disable-line no-use-before-define
   else if (fn === 'now')
     return sql`now()`;
 };
 const binaryOp = (left, right, operator) =>
   // always use parens to ensure the original AST op precedence.
-  sql`(${op(left)} ${raw(operator)} ${op(right)})`; // eslint-disable-line no-use-before-define
+  sql`(${op(left)} ${sql.unsafe(operator)} ${op(right)})`; // eslint-disable-line no-use-before-define
 
 const op = (node) => {
   if (node.type === 'FirstMemberExpression') {
     if (node.raw === '__system/submissionDate') {
-      return sql.identifier([ 'submissions', 'createdAt' ]); // TODO: HACK HACK
+      return sql('submissions.createdAt'); // TODO: HACK HACK
     } else if (node.raw === '__system/updatedAt') {
-      return sql.identifier([ 'submissions', 'updatedAt' ]); // TODO: HACK HACK
+      return sql('submissions.updatedAt'); // TODO: HACK HACK
     } else if (node.raw === '__system/submitterId') {
-      return sql.identifier([ 'submissions', 'submitterId' ]); // TODO: HACK HACK
+      return sql('submissions.submitterId'); // TODO: HACK HACK
     } else if (node.raw === '__system/reviewState') {
-      return sql.identifier([ 'submissions', 'reviewState' ]); // TODO: HACK HACK
+      return sql('submissions.reviewState'); // TODO: HACK HACK
     } else {
       throw Problem.internal.unsupportedODataField({ at: node.position, text: node.raw });
     }

lib/external/postgres.js

@@ -0,0 +1,42 @@
+// Copyright 2022 ODK Central Developers
+// See the NOTICE file at the top-level directory of this distribution and at
+// https://github.com/getodk/central-backend/blob/master/NOTICE.
+// This file is part of ODK Central. It is subject to the license terms in
+// the LICENSE file found in the top-level directory of this distribution and at
+// https://www.apache.org/licenses/LICENSE-2.0. No part of ODK Central,
+// including this file, may be copied, modified, propagated, or distributed
+// except according to the terms contained in the LICENSE file.
+
+// CRCRCR uhhhh confusing naming maybe idk
+const _postgres = require('postgres');
+const { connectionString } = require('../util/db');
+
+const options = {
+  // when saving to the database we transform all undefined to null rather than
+  // throw. this ought to be safe at time of writing because it's exactly what
+  // we did with slonik. possibly someday with better hygiene this can go away.
+  transform: { undefined: null },
+  types: {
+    // the functions here are how postgres implements them for numerics.
+    // the issue is just that for range safety reasons they do not include
+    // bigint in their default impl, which is a problem we are unlikely to have.
+    // the practical problem is that count() in postgres yields a bigint.
+    bigint: { to: 0, from: [ 20 ], serialize: (x => '' + x), parse: (x => +x) },
+
+    // we don't want to automatically assume all non-true values can be safely
+    // equal to 'f', since we can take user input here.
+    boolean: {
+      to: 16, from: 16,
+      serialize: (x => (x === true ? 't' : x === false ? 'f' : x)),
+      parse: (x => x === 't')
+    }
+  }
+};
+const postgres = (config) => _postgres(connectionString(config), options);
+
+// turns out you can get the templater just by omitting connection info completely.
+// and p/postgres is happy to mix template literals across "connections".
+const sql = _postgres(undefined, options);
+
+module.exports = { postgres, sql, options };
+

lib/model/container.js

@@ -13,7 +13,7 @@
 
 const { merge, head } = require('ramda');
 const { postgresErrorToProblem, queryFuncs } = require('../util/db');
-const { resolve, ignoringResult } = require('../util/promise');
+const { ignoringResult } = require('../util/promise');
 
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -28,7 +28,7 @@ const queryModuleBuilder = (definition, container) => {
     module[methodName] = (...args) => {
       const fn = definition[methodName];
       const result = fn(...args)(container);
-      const wrapped = (result.pipe != null) ? resolve(result) :
+      const wrapped = (result.pipe != null) ? result :
         (result.catch != null) ? result.catch(postgresErrorToProblem) :
         result; // eslint-disable-line indent
 
@@ -59,8 +59,8 @@ const queryModuleBuilder = (definition, container) => {
 class Container {
   constructor(...resources) { Object.assign(this, ...resources); }
   transacting(proc) {
-    if (this.isTransacting === true) return proc(this);
-    return this.db.transaction((trxn) => proc(this.with({ db: trxn, isTransacting: true })));
+    if (this.isTransacting === true) return proc(this); // needed due to testing.
+    return this.db.begin((trxn) => proc(this.with({ db: trxn, isTransacting: true })));
   }
 }
 

lib/model/frame.js

@@ -7,9 +7,9 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { raw } = require('slonik-sql-tag-raw');
 const { pick, without } = require('ramda');
 const uuid = require('uuid/v4');
+const { sql } = require('../external/postgres');
 const { pickAll } = require('../util/util');
 const Option = require('../util/option');
 
@@ -62,9 +62,8 @@ class Frame {
     { /* eslint-disable no-shadow */
       // TODO: precomputing is good but this is sort of dirty :/
       const Frame = class extends this { static get def() { return def; } };
-      Frame.fieldlist = raw(def.fields.map((s) => `"${s}"`).join(','));
+      Frame.fieldlist = sql.unsafe(def.fields.map((s) => `"${s}"`).join(','));
       Frame.insertfields = without([ 'id' ], def.fields);
-      Frame.insertlist = raw(Frame.insertfields.map((s) => `"${s}"`).join(','));
       Frame.hasCreatedAt = def.fields.includes('createdAt');
       Frame.hasUpdatedAt = def.fields.includes('updatedAt');
       return Frame;

lib/model/frames.js

@@ -7,7 +7,7 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { sql } = require('slonik');
+const { sql } = require('../external/postgres');
 const { Frame, table, readable, writable, aux, species, embedded } = require('./frame');
 const { isBlank } = require('../util/util');
 const Option = require('../util/option');

lib/model/query/actees.js

@@ -8,7 +8,7 @@
 // except according to the terms contained in the LICENSE file.
 
 const uuid = require('uuid/v4');
-const { sql } = require('slonik');
+const { sql } = require('../../external/postgres');
 const { Actee } = require('../frames');
 const { construct } = require('../../util/util');
 

lib/model/query/actors.js

@@ -7,7 +7,7 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { sql } = require('slonik');
+const { sql } = require('../../external/postgres');
 const { map } = require('ramda');
 const { insert, markDeleted } = require('../../util/db');
 const { resolve } = require('../../util/promise');

lib/model/query/analytics.js

@@ -8,7 +8,7 @@
 // except according to the terms contained in the LICENSE file.
 
 const config = require('config');
-const { sql } = require('slonik');
+const { sql } = require('../../external/postgres');
 const { clone } = require('ramda');
 const { metricsTemplate } = require('../../data/analytics');
 

lib/model/query/assignments.js

@@ -7,7 +7,7 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { sql } = require('slonik');
+const { sql } = require('../../external/postgres');
 const { Actor, Assignment } = require('../frames');
 const { extender, equals, QueryOptions } = require('../../util/db');
 const { getOrReject } = require('../../util/promise');
@@ -36,9 +36,9 @@ const grantSystem = (actor, systemName, actee) => ({ Assignments, Roles }) =>
     .then(getOrReject(Problem.internal.missingSystemRow('role')))
     .then((role) => Assignments.grant(actor, role, actee));
 
-const _revoke = (actor, roleId, acteeId) => ({ db }) =>
-  db.query(sql`delete from assignments where ${equals({ actorId: actor.id, roleId, acteeId })}`)
-    .then(({ rowCount }) => Number(rowCount) > 0);
+const _revoke = (actor, roleId, acteeId) => ({ q }) =>
+  q(sql`delete from assignments where ${equals({ actorId: actor.id, roleId, acteeId })}`)
+    .then(({ count }) => count > 0);
 
 _revoke.audit = (actor, roleId, acteeId) => (log) => {
   if (actor.type === 'singleUse') return null;

lib/model/query/audits.js

@@ -7,10 +7,10 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { sql } = require('slonik');
+const { sql } = require('../../external/postgres');
 const { map } = require('ramda');
 const { Actee, Actor, Audit, Form, Project } = require('../frames');
-const { extender, equals, page, QueryOptions } = require('../../util/db');
+const { extender, equals, joinSqlStr, page, QueryOptions } = require('../../util/db');
 const Option = require('../../util/option');
 const { construct } = require('../../util/util');
 
@@ -25,7 +25,7 @@ const log = (actor, action, actee, details) => ({ run, context }) => {
 
   return run(sql`
 insert into audits ("actorId", action, "acteeId", details, notes, "loggedAt", processed, failures)
-values (${actorId}, ${action}, ${acteeId}, ${(details == null) ? null : JSON.stringify(details)}, ${notes}, clock_timestamp(), ${processed}, 0)`);
+values (${actorId}, ${action}, ${acteeId}, ${details}, ${notes}, clock_timestamp(), ${processed}, 0)`);
 };
 
 
@@ -63,7 +63,7 @@ const auditFilterer = (options) => {
   options.ifArg('start', (start) => result.push(sql`"loggedAt" >= ${start}`));
   options.ifArg('end', (end) => result.push(sql`"loggedAt" <= ${end}`));
   options.ifArg('action', (action) => result.push(actionCondition(action)));
-  return (result.length === 0) ? sql`true` : sql.join(result, sql` and `);
+  return (result.length === 0) ? sql`true` : joinSqlStr(result, sql` and `);
 };
 
 const _get = extender(Audit)(Option.of(Actor), Option.of(Actor.alias('actee_actor', 'acteeActor')), Option.of(Form), Option.of(Form.Def), Option.of(Project), Option.of(Actee))((fields, extend, options) => sql`

lib/model/query/auth.js

@@ -7,7 +7,7 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { sql } = require('slonik');
+const { sql } = require('../../external/postgres');
 const { compose, uniq, flatten, map } = require('ramda');
 const { Actor, Session } = require('../frames');
 const { resolve, reject } = require('../../util/promise');

lib/model/query/blobs.js

@@ -7,7 +7,7 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { sql } = require('slonik');
+const { sql } = require('../../external/postgres');
 const { map } = require('ramda');
 const { Blob } = require('../frames');
 const { construct } = require('../../util/util');
@@ -22,7 +22,7 @@ const { construct } = require('../../util/util');
 const ensure = (blob) => ({ oneFirst }) => oneFirst(sql`
 with ensured as
 (insert into blobs (sha, md5, content, "contentType") values
-    (${blob.sha}, ${blob.md5}, ${sql.binary(blob.content)}, ${blob.contentType || null})
+    (${blob.sha}, ${blob.md5}, ${blob.content}, ${blob.contentType})
   on conflict (sha) do update set sha = ${blob.sha}
   returning id)
 select id from ensured`);

lib/model/query/client-audits.js

@@ -7,8 +7,8 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { sql } = require('slonik');
-const { insertMany, QueryOptions } = require('../../util/db');
+const { sql } = require('../../external/postgres');
+const { insertMany, QueryOptions, joinSqlStr } = require('../../util/db');
 const { odataFilter } = require('../../data/odata-filter');
 
 
@@ -20,7 +20,7 @@ const existsForBlob = (blobId) => ({ maybeOne }) =>
 
 // TODO: copy-pasted from query/submission-attachments.js
 const keyIdCondition = (keyIds) =>
-  sql.join((((keyIds == null) || (keyIds.length === 0)) ? [ -1 ] : keyIds), sql`,`);
+  joinSqlStr((((keyIds == null) || (keyIds.length === 0)) ? [ -1 ] : keyIds), sql`,`);
 
 const streamForExport = (formId, draft, keyIds, options = QueryOptions.none) => ({ stream }) => stream(sql`
 select client_audits.*, blobs.content, submissions."instanceId", "localKey", "keyId", index, submissions."instanceId" from submission_defs

lib/model/query/comments.js

@@ -7,7 +7,7 @@
 // including this file, may be copied, modified, propagated, or distributed
 // except according to the terms contained in the LICENSE file.
 
-const { sql } = require('slonik');
+const { sql } = require('../../external/postgres');
 const { map } = require('ramda');
 const { Actor, Comment } = require('../frames');
 const { extender, insert, QueryOptions } = require('../../util/db');