From 897a0a8cde2c08e3e55824bb9b243754342afcf2 Mon Sep 17 00:00:00 2001 From: Miroslav Kovar Date: Mon, 14 Feb 2022 10:20:15 +0100 Subject: [PATCH] Periodically prune reviewed and rejected messages (#141) * Prune processed msgs Signed-off-by: Miroslav Kovar Co-authored-by: Patrik Stas --- .../20220210204408-create-events.js | 53 +++++++++++++++++++ .../20220210204408-create-events-down.sql | 1 + .../sqls/20220210204408-create-events-up.sql | 12 +++++ dbutils/src/db-schemas.js | 4 +- dbutils/src/db-testutils.js | 23 ++++---- .../test/unit/messaging/agent-msgs.spec.js | 27 ++++++++-- 6 files changed, 104 insertions(+), 16 deletions(-) create mode 100644 dbutils/migrations_scheme_application/20220210204408-create-events.js create mode 100644 dbutils/migrations_scheme_application/sqls/20220210204408-create-events-down.sql create mode 100644 dbutils/migrations_scheme_application/sqls/20220210204408-create-events-up.sql diff --git a/dbutils/migrations_scheme_application/20220210204408-create-events.js b/dbutils/migrations_scheme_application/20220210204408-create-events.js new file mode 100644 index 00000000..d8121451 --- /dev/null +++ b/dbutils/migrations_scheme_application/20220210204408-create-events.js @@ -0,0 +1,53 @@ +'use strict'; + +var dbm; +var type; +var seed; +var fs = require('fs'); +var path = require('path'); +var Promise; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function(options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; + Promise = options.Promise; +}; + +exports.up = function(db) { + var filePath = path.join(__dirname, 'sqls', '20220210204408-create-events-up.sql'); + return new Promise( function( resolve, reject ) { + fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){ + if (err) return reject(err); + console.log('received data: ' + data); + + resolve(data); + }); + }) + .then(function(data) { + return db.runSql(data); + }); +}; + +exports.down = function(db) { + var filePath = path.join(__dirname, 'sqls', '20220210204408-create-events-down.sql'); + return new Promise( function( resolve, reject ) { + fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){ + if (err) return reject(err); + console.log('received data: ' + data); + + resolve(data); + }); + }) + .then(function(data) { + return db.runSql(data); + }); +}; + +exports._meta = { + "version": 1 +}; diff --git a/dbutils/migrations_scheme_application/sqls/20220210204408-create-events-down.sql b/dbutils/migrations_scheme_application/sqls/20220210204408-create-events-down.sql new file mode 100644 index 00000000..44f074ea --- /dev/null +++ b/dbutils/migrations_scheme_application/sqls/20220210204408-create-events-down.sql @@ -0,0 +1 @@ +/* Replace with your SQL commands */ \ No newline at end of file diff --git a/dbutils/migrations_scheme_application/sqls/20220210204408-create-events-up.sql b/dbutils/migrations_scheme_application/sqls/20220210204408-create-events-up.sql new file mode 100644 index 00000000..c381417a --- /dev/null +++ b/dbutils/migrations_scheme_application/sqls/20220210204408-create-events-up.sql @@ -0,0 +1,12 @@ +CREATE PROCEDURE prune_msgs() +BEGIN + DELETE LOW_PRIORITY FROM messages WHERE status_code = 'MS-105' OR status_code = 'MS-106'; +END; + + +CREATE EVENT msgs_cleanup + ON SCHEDULE EVERY 10 MINUTE + ON COMPLETION PRESERVE + COMMENT 'Cleanup processed messages (in states MS-105 or MS-106)' + DO + CALL prune_msgs(); diff --git a/dbutils/src/db-schemas.js b/dbutils/src/db-schemas.js index d3b322a6..4bf1671a 100644 --- a/dbutils/src/db-schemas.js +++ b/dbutils/src/db-schemas.js @@ -20,12 +20,13 @@ async function _runSingleCmd(dbConfig, cmd) { } } -async function runSingleCmd(user, password, host, port, cmd) { +async function runSingleCmd(user, password, host, port, cmd, database) { const dbConfig = { host, port, user, password, + database, multipleStatements: false } await _runSingleCmd(dbConfig, cmd) @@ -110,4 +111,5 @@ module.exports = { createMysqlDatabase, assureMysqlDatabase, enableConsoleLogging, + runSingleCmd } diff --git a/dbutils/src/db-testutils.js b/dbutils/src/db-testutils.js index 2ac7012b..ab889fe1 100644 --- a/dbutils/src/db-testutils.js +++ b/dbutils/src/db-testutils.js @@ -18,15 +18,16 @@ const uuid = require('uuid') const { migrateSchemaData, migrateSchemaWallet } = require('./migration') -const { dropMysqlDatabase, createMysqlDatabase } = require('./db-schemas') +const { dropMysqlDatabase, createMysqlDatabase, runSingleCmd } = require('./db-schemas') + +const user = 'root' +const password = 'mysecretpassword' +const host = 'localhost' +const port = 3306 async function createDbSchemaApplication (dbNameId) { const dbId = dbNameId || uuid.v4().split('-').join("") const database = `agencytest_data_${dbId}` - const user = 'root' - const password = 'mysecretpassword' - const host = 'localhost' - const port = 3306 await createMysqlDatabase(user, password, host, port, database) await migrateSchemaData(user, password, host, port, database) @@ -45,10 +46,6 @@ async function createDbSchemaApplication (dbNameId) { async function createDbSchemaWallets (dbNameId) { const dbId = dbNameId || uuid.v4().split('-').join("") const database = `agencytest_wallet_${dbId}` - const user = 'root' - const password = 'mysecretpassword' - const host = 'localhost' - const port = 3306 await createMysqlDatabase(user, password, host, port, database) await migrateSchemaWallet(user, password, host, port, database) @@ -64,7 +61,13 @@ async function createDbSchemaWallets (dbNameId) { } } +async function pruneMsgs (dbName) { + const cmd = 'CALL prune_msgs()' + await runSingleCmd(user, password, host, port, cmd, dbName) +} + module.exports = { createDbSchemaApplication, - createDbSchemaWallets + createDbSchemaWallets, + pruneMsgs } diff --git a/vcxagency-node/test/unit/messaging/agent-msgs.spec.js b/vcxagency-node/test/unit/messaging/agent-msgs.spec.js index 2711a750..952bd569 100644 --- a/vcxagency-node/test/unit/messaging/agent-msgs.spec.js +++ b/vcxagency-node/test/unit/messaging/agent-msgs.spec.js @@ -34,7 +34,7 @@ const uuid = require('uuid') const rimraf = require('rimraf') const os = require('os') const { getBaseAppConfig } = require('./common') -const { createDbSchemaApplication, createDbSchemaWallets } = require('dbutils') +const { createDbSchemaApplication, createDbSchemaWallets, pruneMsgs } = require('dbutils') const { setupVcxLogging } = require('../../utils') const { buildApplication, cleanUpApplication } = require('../../../src/setup/app') const { buildAgencyClientVirtual } = require('./common') @@ -369,24 +369,41 @@ describe('onboarding', () => { expect(updated2.uids.length).toBe(1) expect(updated2.uids.includes(msg6Id)).toBeTruthy() - const msgReply = await vcxFlowGetMsgsFromAgent(agencyUserWh, sendToAgency, agent1Did, agent1Verkey, agencyUserVerkey, [aconn1UserPwDid, aconn2UserPwDid], [], []) - const { msgsByConns } = msgReply + let msgReply = await vcxFlowGetMsgsFromAgent(agencyUserWh, sendToAgency, agent1Did, agent1Verkey, agencyUserVerkey, [aconn1UserPwDid, aconn2UserPwDid], [], []) + let { msgsByConns } = msgReply expect(Array.isArray(msgsByConns)).toBeTruthy() expect(msgsByConns.length).toBe(2) - const msgsByConn1 = msgsByConns.find(msgsByConn => msgsByConn.pairwiseDID === aconn1UserPwDid) + let msgsByConn1 = msgsByConns.find(msgsByConn => msgsByConn.pairwiseDID === aconn1UserPwDid) expect(msgsByConn1).toBeDefined() expect(msgsByConn1.msgs.length).toBe(3) assertMsgsByConHasMessageWithStatus(msgsByConn1, msg1Id, 'MS-106') assertMsgsByConHasMessageWithStatus(msgsByConn1, msg2Id, 'MS-106') assertMsgsByConHasMessageWithStatus(msgsByConn1, msg3Id, 'MS-104') - const msgsByConn2 = msgsByConns.find(msgsByConn => msgsByConn.pairwiseDID === aconn2UserPwDid) + let msgsByConn2 = msgsByConns.find(msgsByConn => msgsByConn.pairwiseDID === aconn2UserPwDid) expect(msgsByConn2).toBeDefined() expect(msgsByConn2.msgs.length).toBe(3) assertMsgsByConHasMessageWithStatus(msgsByConn2, msg4Id, 'MS-103') assertMsgsByConHasMessageWithStatus(msgsByConn2, msg5Id, 'MS-103') assertMsgsByConHasMessageWithStatus(msgsByConn2, msg6Id, 'MS-106') + + await pruneMsgs(tmpDbData.info.database) + msgReply = await vcxFlowGetMsgsFromAgent(agencyUserWh, sendToAgency, agent1Did, agent1Verkey, agencyUserVerkey, [aconn1UserPwDid, aconn2UserPwDid], [], []); + ({ msgsByConns } = msgReply) + expect(Array.isArray(msgsByConns)).toBeTruthy() + expect(msgsByConns.length).toBe(2) + + msgsByConn1 = msgsByConns.find(msgsByConn => msgsByConn.pairwiseDID === aconn1UserPwDid) + expect(msgsByConn1).toBeDefined() + expect(msgsByConn1.msgs.length).toBe(1) + assertMsgsByConHasMessageWithStatus(msgsByConn1, msg3Id, 'MS-104') + + msgsByConn2 = msgsByConns.find(msgsByConn => msgsByConn.pairwiseDID === aconn2UserPwDid) + expect(msgsByConn2).toBeDefined() + expect(msgsByConn2.msgs.length).toBe(2) + assertMsgsByConHasMessageWithStatus(msgsByConn2, msg4Id, 'MS-103') + assertMsgsByConHasMessageWithStatus(msgsByConn2, msg5Id, 'MS-103') }) it('should not update any message statusCode of no uids are specified in update request', async () => {