diff --git a/api/src/services/monitoring.js b/api/src/services/monitoring.js index 462b2d9d524..5d69c9d41d8 100644 --- a/api/src/services/monitoring.js +++ b/api/src/services/monitoring.js @@ -27,6 +27,15 @@ const VIEW_INDEXES_TO_MONITOR = { users: ['users'], }; +const NOUVEAU_INDEXES_TO_MONITOR = { + medic: { + 'medic-nouveau': [ + 'contacts_by_freetext', + 'reports_by_freetext', + ], + }, +}; + const MESSAGE_QUEUE_STATUS_KEYS = ['due', 'scheduled', 'muted', 'failed', 'delivered']; const fromEntries = (keys, value) => { // "shim" of Object.fromEntries @@ -115,7 +124,7 @@ const getFragmentation = ({ sizes }, viewIndexInfos) => { return totalFile / totalActive; }; -const mapDbInfo = (dbInfo, viewIndexInfos) => { +const mapDbInfo = (dbInfo, viewIndexInfos, nouveauIndexInfos) => { return { name: dbInfo.db_name || '', update_sequence: getSequenceNumber(dbInfo.update_seq), @@ -131,8 +140,13 @@ const mapDbInfo = (dbInfo, viewIndexInfos) => { sizes: { active: defaultNumber(viewIndexInfo.view_index?.sizes?.active), file: defaultNumber(viewIndexInfo.view_index?.sizes?.file), - } - })) + }, + })), + nouveau_indexes: nouveauIndexInfos?.map(nouveauIndexInfo => ({ + name: nouveauIndexInfo.name || '', + num_docs: defaultNumber(nouveauIndexInfo.search_index.num_docs), + disk_size: defaultNumber(nouveauIndexInfo.search_index.disk_size), + })), }; }; @@ -167,11 +181,37 @@ const fetchViewIndexInfosForDb = (db) => Promise.all( const fetchAllViewIndexInfos = () => Promise.all(Object.keys(VIEW_INDEXES_TO_MONITOR).map(fetchViewIndexInfosForDb)); +const fetchNouveauIndexInfo = (db, designDoc, indexName) => request + .get({ + url: `${environment.serverUrl}/${db}/_design/${designDoc}/_nouveau_info/${indexName}`, + json: true + }) + .catch(err => { + logger.error('Error fetching nouveau index info: %o', err); + return null; + }); + +const fetchNouveauIndexInfosForDdoc = (db, ddoc) => NOUVEAU_INDEXES_TO_MONITOR[db][ddoc].map( + indexName => fetchNouveauIndexInfo(DBS_TO_MONITOR[db], ddoc, indexName), +); + +const fetchNouveauIndexInfosForDb = (db) => Promise.all(Object.keys(NOUVEAU_INDEXES_TO_MONITOR[db]).flatMap( + ddoc => fetchNouveauIndexInfosForDdoc(db, ddoc), +)).then((nouveauIndexInfos) => nouveauIndexInfos.filter(info => info)); + +const fetchAllNouveauIndexInfos = () => Promise.all( + Object.keys(NOUVEAU_INDEXES_TO_MONITOR).map(fetchNouveauIndexInfosForDb), +); + const getDbInfos = async () => { - const [dbInfos, viewIndexInfos] = await Promise.all([fetchDbsInfo(), fetchAllViewIndexInfos()]); + const [dbInfos, viewIndexInfos, nouveauIndexInfos] = await Promise.all([ + fetchDbsInfo(), + fetchAllViewIndexInfos(), + fetchAllNouveauIndexInfos(), + ]); const result = {}; Object.keys(DBS_TO_MONITOR).forEach((dbKey, i) => { - result[dbKey] = mapDbInfo(dbInfos[i], viewIndexInfos[i]); + result[dbKey] = mapDbInfo(dbInfos[i], viewIndexInfos[i], nouveauIndexInfos[i]); }); return result; }; diff --git a/api/tests/mocha/services/monitoring.spec.js b/api/tests/mocha/services/monitoring.spec.js index 9b35e821d79..92fef3f45cd 100644 --- a/api/tests/mocha/services/monitoring.spec.js +++ b/api/tests/mocha/services/monitoring.spec.js @@ -161,6 +161,35 @@ const VIEW_INDEX_INFO_BY_DESIGN = { } }; +const NOUVEAU_DDOCS_BY_DB = { + [environment.db]: ['medic-nouveau'], +}; + +const NOUVEAU_INDEX_INFO_BY_DDOC = { + 'medic-nouveau': { + reports_by_freetext: { + name: '_design/medic-nouveau/reports_by_freetext', + search_index: { + update_seq: 1956891, + purge_seq: 0, + num_docs: 183741, + disk_size: 157258510, + signature: 'cfd67cbb4800308021b6547bcf21cbf99b9476186b5251f317b221225714c5d3', + }, + }, + contacts_by_freetext: { + name: '_design/medic-nouveau/contacts_by_freetext', + search_index: { + update_seq: 1956891, + purge_seq: 0, + num_docs: 207734, + disk_size: 76815351, + signature: '46de1dfc576838494f798264571dc59658db7ea164915dd459a7752c31591ae6', + }, + }, + }, +}; + const setUpMocks = () => { sinon.stub(deployInfo, 'get').resolves({ version: '5.3.2' }); sinon.stub(request, 'get') @@ -173,6 +202,16 @@ const setUpMocks = () => { .resolves(VIEW_INDEX_INFO_BY_DESIGN[designDoc]); }); }); + Object.keys(NOUVEAU_DDOCS_BY_DB).forEach(dbName => { + NOUVEAU_DDOCS_BY_DB[dbName].forEach(designDoc => { + Object.keys(NOUVEAU_INDEX_INFO_BY_DDOC[designDoc]).forEach(indexName => { + request.get + .withArgs( + sinon.match({ url: `${environment.serverUrl}/${dbName}/_design/${designDoc}/_nouveau_info/${indexName}` }), + ).resolves(NOUVEAU_INDEX_INFO_BY_DDOC[designDoc][indexName]); + }); + }); + }); sinon.stub(request, 'post').withArgs(sinon.match({ url: `${environment.serverUrl}/_dbs_info` })) .resolves(dbInfos); sinon.stub(db.sentinel, 'get').withArgs('_local/transitions-seq') @@ -250,6 +289,7 @@ const getExpectedViewIndexes = (dbName) => { const getCurrentDdocNames = (db) => getBundledDdocs(db) .then(ddocs => ddocs + .filter(ddoc => !!ddoc.views) .map(ddoc => ddoc._id) .map(ddocId => ddocId.split('/')[1])); @@ -283,9 +323,21 @@ describe('Monitoring service', () => { update_sequence: 100, sizes: { active: 600, - file: 700 + file: 700, }, - view_indexes: getExpectedViewIndexes(environment.db) + view_indexes: getExpectedViewIndexes(environment.db), + nouveau_indexes: [ + { + disk_size: 76815351, + name: '_design/medic-nouveau/contacts_by_freetext', + num_docs: 207734, + }, + { + disk_size: 157258510, + name: '_design/medic-nouveau/reports_by_freetext', + num_docs: 183741, + }, + ], }, sentinel: { doc_count: 30, @@ -297,7 +349,8 @@ describe('Monitoring service', () => { active: 500, file: 500 }, - view_indexes: getExpectedViewIndexes(`${environment.db}-sentinel`) + view_indexes: getExpectedViewIndexes(`${environment.db}-sentinel`), + nouveau_indexes: undefined, }, users: { doc_count: 50, @@ -309,7 +362,8 @@ describe('Monitoring service', () => { active: 500, file: 501 }, - view_indexes: getExpectedViewIndexes('_users') + view_indexes: getExpectedViewIndexes('_users'), + nouveau_indexes: undefined, }, usersmeta: { doc_count: 40, @@ -321,7 +375,8 @@ describe('Monitoring service', () => { active: 500, file: 5000 }, - view_indexes: getExpectedViewIndexes(`${environment.db}-users-meta`) + view_indexes: getExpectedViewIndexes(`${environment.db}-users-meta`), + nouveau_indexes: undefined, } }); chai.expect(actual.messaging).to.deep.equal({ @@ -347,6 +402,14 @@ describe('Monitoring service', () => { [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-admin/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-client/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-conflicts/_info` }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/contacts_by_freetext`, + }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/reports_by_freetext`, + }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-scripts/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-sms/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}-sentinel/_design/sentinel/_info` }], @@ -394,7 +457,19 @@ describe('Monitoring service', () => { active: 600, file: 700 }, - view_indexes: getExpectedViewIndexes(environment.db) + view_indexes: getExpectedViewIndexes(environment.db), + nouveau_indexes: [ + { + disk_size: 76815351, + name: '_design/medic-nouveau/contacts_by_freetext', + num_docs: 207734, + }, + { + disk_size: 157258510, + name: '_design/medic-nouveau/reports_by_freetext', + num_docs: 183741, + }, + ], }, sentinel: { doc_count: 30, @@ -406,7 +481,8 @@ describe('Monitoring service', () => { active: 500, file: 500 }, - view_indexes: getExpectedViewIndexes(`${environment.db}-sentinel`) + view_indexes: getExpectedViewIndexes(`${environment.db}-sentinel`), + nouveau_indexes: undefined, }, users: { doc_count: 50, @@ -418,7 +494,8 @@ describe('Monitoring service', () => { active: 500, file: 501 }, - view_indexes: getExpectedViewIndexes('_users') + view_indexes: getExpectedViewIndexes('_users'), + nouveau_indexes: undefined, }, usersmeta: { doc_count: 40, @@ -430,7 +507,8 @@ describe('Monitoring service', () => { active: 500, file: 5000 }, - view_indexes: getExpectedViewIndexes(`${environment.db}-users-meta`) + view_indexes: getExpectedViewIndexes(`${environment.db}-users-meta`), + nouveau_indexes: undefined, } }); chai.expect(actual.messaging).to.deep.equal({ @@ -483,6 +561,14 @@ describe('Monitoring service', () => { [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-admin/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-client/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-conflicts/_info` }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/contacts_by_freetext`, + }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/reports_by_freetext`, + }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-scripts/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-sms/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}-sentinel/_design/sentinel/_info` }], @@ -531,7 +617,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: [], }, sentinel: { doc_count: -1, @@ -543,7 +630,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: undefined, }, users: { doc_count: -1, @@ -555,7 +643,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: undefined, }, usersmeta: { doc_count: -1, @@ -567,7 +656,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: undefined, } }); chai.expect(actual.messaging).to.deep.equal({ @@ -591,6 +681,14 @@ describe('Monitoring service', () => { [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-admin/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-client/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-conflicts/_info` }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/contacts_by_freetext`, + }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/reports_by_freetext`, + }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-scripts/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-sms/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}-sentinel/_design/sentinel/_info` }], @@ -627,7 +725,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: [], }, sentinel: { doc_count: -1, @@ -639,7 +738,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: undefined, }, users: { doc_count: -1, @@ -651,7 +751,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: undefined, }, usersmeta: { doc_count: -1, @@ -663,7 +764,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: undefined, } }); chai.expect(actual.messaging).to.deep.equal({ @@ -714,6 +816,14 @@ describe('Monitoring service', () => { [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-admin/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-client/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-conflicts/_info` }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/contacts_by_freetext`, + }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/reports_by_freetext`, + }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-scripts/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-sms/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}-sentinel/_design/sentinel/_info` }], diff --git a/couchdb-nouveau/Dockerfile b/couchdb-nouveau/Dockerfile new file mode 100644 index 00000000000..712463369ea --- /dev/null +++ b/couchdb-nouveau/Dockerfile @@ -0,0 +1,11 @@ +FROM couchdb:3.4.2-nouveau + +COPY --chown=nouveau:nouveau nouveau.yaml /opt/nouveau/etc/nouveau.yaml + +VOLUME /data/nouveau + +# 5987: Nouveau App +# 5989: Nouveau Admin +EXPOSE 5987 5989 + +LABEL Authors="MEDIC SRE TEAM" diff --git a/couchdb-nouveau/nouveau.yaml b/couchdb-nouveau/nouveau.yaml new file mode 100644 index 00000000000..351bec8c0f3 --- /dev/null +++ b/couchdb-nouveau/nouveau.yaml @@ -0,0 +1,27 @@ +maxIndexesOpen: 3000 +commitIntervalSeconds: 30 +idleSeconds: 60 +rootDir: ./data/nouveau + +logging: + level: INFO + +server: + applicationConnectors: + - type: http + bindHost: 0.0.0.0 + port: 5987 + useDateHeader: false + adminConnectors: + - type: http + bindHost: 0.0.0.0 + port: 5989 + useDateHeader: false + gzip: + includedMethods: + - GET + - POST + requestLog: + appenders: + - type: console + target: stderr diff --git a/couchdb/10-docker-default.ini b/couchdb/10-docker-default.ini index 8c924a92d54..2a976ac0b11 100644 --- a/couchdb/10-docker-default.ini +++ b/couchdb/10-docker-default.ini @@ -42,3 +42,7 @@ n=1 [attachments] compressible_types = text/*, application/javascript, application/json, application/xml compression_level = 8 + +[nouveau] +enable = true +url = http://nouveau:5987 diff --git a/couchdb/Dockerfile b/couchdb/Dockerfile index 2afba060d56..97435c8528f 100644 --- a/couchdb/Dockerfile +++ b/couchdb/Dockerfile @@ -1,4 +1,4 @@ -FROM couchdb:3.4.2 as base_couchdb_build +FROM couchdb:3.4.2 AS base_couchdb_build COPY --chown=couchdb:couchdb 10-docker-default.ini /opt/couchdb/etc/default.d/ COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/ diff --git a/ddocs/medic-db/medic-client/views/contacts_by_freetext/map.js b/ddocs/medic-db/medic-client/views/contacts_by_freetext/map.js deleted file mode 100644 index 86e0d9cbe20..00000000000 --- a/ddocs/medic-db/medic-client/views/contacts_by_freetext/map.js +++ /dev/null @@ -1,52 +0,0 @@ -function(doc) { - var skip = [ '_id', '_rev', 'type', 'refid', 'geolocation' ]; - - var usedKeys = []; - var emitMaybe = function(key, value) { - if (usedKeys.indexOf(key) === -1 && // Not already used - key.length > 2 // Not too short - ) { - usedKeys.push(key); - emit([key], value); - } - }; - - var emitField = function(key, value, order) { - if (!key || !value) { - return; - } - key = key.toLowerCase(); - if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { - return; - } - if (typeof value === 'string') { - value = value.toLowerCase(); - value.split(/\s+/).forEach(function(word) { - emitMaybe(word, order); - }); - } - if (typeof value === 'number' || typeof value === 'string') { - emitMaybe(key + ':' + value, order); - } - }; - - var types = [ 'district_hospital', 'health_center', 'clinic', 'person' ]; - var idx; - if (doc.type === 'contact') { - idx = types.indexOf(doc.contact_type); - if (idx === -1) { - idx = doc.contact_type; - } - } else { - idx = types.indexOf(doc.type); - } - - if (idx !== -1) { - var dead = !!doc.date_of_death; - var muted = !!doc.muted; - var order = dead + ' ' + muted + ' ' + idx + ' ' + (doc.name && doc.name.toLowerCase()); - Object.keys(doc).forEach(function(key) { - emitField(key, doc[key], order); - }); - } -} diff --git a/ddocs/medic-db/medic-client/views/contacts_by_type_freetext/map.js b/ddocs/medic-db/medic-client/views/contacts_by_type_freetext/map.js deleted file mode 100644 index 85015e29f9d..00000000000 --- a/ddocs/medic-db/medic-client/views/contacts_by_type_freetext/map.js +++ /dev/null @@ -1,54 +0,0 @@ -function(doc) { - var skip = [ '_id', '_rev', 'type', 'refid', 'geolocation' ]; - - var usedKeys = []; - var emitMaybe = function(type, key, value) { - if (usedKeys.indexOf(key) === -1 && // Not already used - key.length > 2 // Not too short - ) { - usedKeys.push(key); - emit([ type, key ], value); - } - }; - - var emitField = function(type, key, value, order) { - if (!key || !value) { - return; - } - key = key.toLowerCase(); - if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { - return; - } - if (typeof value === 'string') { - value = value.toLowerCase(); - value.split(/\s+/).forEach(function(word) { - emitMaybe(type, word, order); - }); - } - if (typeof value === 'number' || typeof value === 'string') { - emitMaybe(type, key + ':' + value, order); - } - }; - - var types = [ 'district_hospital', 'health_center', 'clinic', 'person' ]; - var idx; - var type; - if (doc.type === 'contact') { - type = doc.contact_type; - idx = types.indexOf(type); - if (idx === -1) { - idx = type; - } - } else { - type = doc.type; - idx = types.indexOf(type); - } - if (idx !== -1) { - var dead = !!doc.date_of_death; - var muted = !!doc.muted; - var order = dead + ' ' + muted + ' ' + idx + ' ' + (doc.name && doc.name.toLowerCase()); - Object.keys(doc).forEach(function(key) { - emitField(type, key, doc[key], order); - }); - } -} diff --git a/ddocs/medic-db/medic-client/views/reports_by_freetext/map.js b/ddocs/medic-db/medic-client/views/reports_by_freetext/map.js deleted file mode 100644 index 41550efee4c..00000000000 --- a/ddocs/medic-db/medic-client/views/reports_by_freetext/map.js +++ /dev/null @@ -1,46 +0,0 @@ -function(doc) { - var skip = [ '_id', '_rev', 'type', 'refid', 'content' ]; - - var usedKeys = []; - var emitMaybe = function(key, value) { - if (usedKeys.indexOf(key) === -1 && // Not already used - key.length > 2 // Not too short - ) { - usedKeys.push(key); - emit([key], value); - } - }; - - var emitField = function(key, value, reportedDate) { - if (!key || !value) { - return; - } - key = key.toLowerCase(); - if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { - return; - } - if (typeof value === 'string') { - value = value.toLowerCase(); - value.split(/\s+/).forEach(function(word) { - emitMaybe(word, reportedDate); - }); - } - if (typeof value === 'number' || typeof value === 'string') { - emitMaybe(key + ':' + value, reportedDate); - } - }; - - if (doc.type === 'data_record' && doc.form) { - Object.keys(doc).forEach(function(key) { - emitField(key, doc[key], doc.reported_date); - }); - if (doc.fields) { - Object.keys(doc.fields).forEach(function(key) { - emitField(key, doc.fields[key], doc.reported_date); - }); - } - if (doc.contact && doc.contact._id) { - emitMaybe('contact:' + doc.contact._id.toLowerCase(), doc.reported_date); - } - } -} diff --git a/ddocs/medic-db/medic-nouveau/_id b/ddocs/medic-db/medic-nouveau/_id new file mode 100644 index 00000000000..3585031d0eb --- /dev/null +++ b/ddocs/medic-db/medic-nouveau/_id @@ -0,0 +1 @@ +_design/medic-nouveau diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js new file mode 100644 index 00000000000..2502f61245b --- /dev/null +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js @@ -0,0 +1,53 @@ +function (doc) { + var skip = ['_id', '_rev', 'type', 'refid', 'geolocation']; + var toIndex = ''; + + var types = ['district_hospital', 'health_center', 'clinic', 'person']; + var idx; + var type; + if (doc.type === 'contact') { + type = doc.contact_type; + idx = types.indexOf(type); + if (idx === -1) { + idx = type; + } + } else { + type = doc.type; + idx = types.indexOf(type); + } + + var isContactDoc = idx !== -1; + if (isContactDoc) { + Object.keys(doc).forEach(function (key) { + var value = doc[key]; + if (!key || !value) { + return; + } + + key = key.toLowerCase().trim(); + if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { + return; + } + + if (typeof value === 'string') { + toIndex += ' ' + value; + index('text', key, value, { store: true }); + } + + if (typeof value === 'number') { + index('double', key, value, { store: true }); + } + }); + + var dead = !!doc.date_of_death; + var muted = !!doc.muted; + var order = dead + ' ' + muted + ' ' + idx + ' ' + (doc.name && doc.name.toLowerCase()); + index('string', 'cht_sort_order', order, { store: false }); + index('text', 'cht_contact_type', type, { store: false }); + + toIndex = toIndex.trim(); + if (toIndex) { + index('text', 'default', toIndex, { store: true }); + } + } +} diff --git a/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js new file mode 100644 index 00000000000..3e901ea1592 --- /dev/null +++ b/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js @@ -0,0 +1,40 @@ +function (doc) { + var skip = ['_id', '_rev', 'type', 'refid', 'content', '_attachments']; + var toIndex = ''; + + var emitField = function (key, value) { + if (!key || !value) { + return; + } + + key = key.toLowerCase().trim(); + if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { + return; + } + + if (typeof value === 'string') { + toIndex += ' ' + value; + index('text', key, value, { store: true }); + } + + if (typeof value === 'number') { + index('double', key, value, { store: true }); + } + }; + + if (doc.type === 'data_record' && doc.form) { + Object.keys(doc).forEach(function (key) { + emitField(key, doc[key]); + }); + if (doc.fields) { + Object.keys(doc.fields).forEach(function (key) { + emitField(key, doc.fields[key]); + }); + } + + toIndex = toIndex.trim(); + if (toIndex) { + index('text', 'default', toIndex, { store: true }); + } + } +} diff --git a/nginx/Dockerfile b/nginx/Dockerfile index 1c55842a50e..289eeb0af22 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -1,5 +1,5 @@ # base build -FROM nginx:1.25.1-alpine as base_nginx +FROM nginx:1.25.1-alpine AS base_nginx RUN apk add --update --no-cache \ curl \ socat \ diff --git a/patches/eslint-plugin-couchdb+0.2.0.patch b/patches/eslint-plugin-couchdb+0.2.0.patch new file mode 100644 index 00000000000..03488adbb4c --- /dev/null +++ b/patches/eslint-plugin-couchdb+0.2.0.patch @@ -0,0 +1,27 @@ +diff --git a/node_modules/eslint-plugin-couchdb/lib/index.js b/node_modules/eslint-plugin-couchdb/lib/index.js +index 3399a8f..418f58b 100644 +--- a/node_modules/eslint-plugin-couchdb/lib/index.js ++++ b/node_modules/eslint-plugin-couchdb/lib/index.js +@@ -18,12 +18,14 @@ var front = function(functions, jsonObject) { + } + + // https://docs.couchdb.org/en/stable/query-server/javascript.html# ++var indexFront = front(['index', 'isArray', 'log', 'require', 'sum', 'toJSON'], true); + var mapFront = front(['emit', 'isArray', 'log', 'require', 'sum', 'toJSON'], true); + var reduceFront = front(['isArray', 'log', 'sum', 'toJSON'], true); + var vdoFront = front(['isArray', 'log', 'require', 'sum', 'toJSON'], true); + var showFront = front(['isArray', 'log', 'provides', 'registerType', 'require', 'sum', 'toJSON'], true); + var listFront = front(['getRow', 'isArray', 'log', 'provides', 'registerType', 'require', 'send', 'start', 'sum', 'toJSON'], true); + ++var indexTest = /nouveau\/[^\/]+\/index\.js$/; + var mapTest = /views\/[^\/]+\/map\.js$/; + var reduceTest = /views\/[^\/]+\/reduce\.js$/; + var vdoTest = /validate_doc_update.js$/; +@@ -31,6 +33,7 @@ var showTest = /shows\/.*.js$/; + var listTest = /lists\/.*.js$/; + + var FN_TYPES = [ ++ {test: indexTest, front: indexFront}, + {test: mapTest, front: mapFront}, + {test: reduceTest, front: reduceFront}, + {test: vdoTest, front: vdoFront}, diff --git a/scripts/build/cht-couchdb-cluster.yml.template b/scripts/build/cht-couchdb-cluster.yml.template index 0cb08e22b68..48af139660a 100644 --- a/scripts/build/cht-couchdb-cluster.yml.template +++ b/scripts/build/cht-couchdb-cluster.yml.template @@ -63,6 +63,23 @@ services: networks: cht-net: + nouveau: + image: {{{ repo }}}/cht-couchdb-nouveau:{{ tag }} + volumes: + - ${COUCHDB_NOUVEAU_DATA:-./srv_nouveau}:/data/nouveau + restart: always + depends_on: + - couchdb-1.local + - couchdb-2.local + - couchdb-3.local + logging: + driver: "local" + options: + max-size: "${LOG_MAX_SIZE:-50m}" + max-file: "${LOG_MAX_FILES:-20}" + networks: + cht-net: + volumes: cht-credentials: diff --git a/scripts/build/cht-couchdb-single-node.yml.template b/scripts/build/cht-couchdb-single-node.yml.template index 458b31f0e8c..921720ba968 100644 --- a/scripts/build/cht-couchdb-single-node.yml.template +++ b/scripts/build/cht-couchdb-single-node.yml.template @@ -20,6 +20,21 @@ services: networks: cht-net: + nouveau: + image: {{{ repo }}}/cht-couchdb-nouveau:{{ tag }} + volumes: + - ${COUCHDB_NOUVEAU_DATA:-./srv_nouveau}:/data/nouveau + restart: always + depends_on: + - couchdb + logging: + driver: "local" + options: + max-size: "${LOG_MAX_SIZE:-50m}" + max-file: "${LOG_MAX_FILES:-20}" + networks: + cht-net: + volumes: cht-credentials: diff --git a/scripts/build/versions.js b/scripts/build/versions.js index a1ed5be17eb..b67b8680edc 100644 --- a/scripts/build/versions.js +++ b/scripts/build/versions.js @@ -61,5 +61,5 @@ module.exports = { getRepo, escapeBranchName, SERVICES: ['api', 'sentinel'], - INFRASTRUCTURE: ['couchdb', 'haproxy', 'haproxy-healthcheck', 'nginx'], + INFRASTRUCTURE: ['couchdb', 'couchdb-nouveau', 'haproxy', 'haproxy-healthcheck', 'nginx'], }; diff --git a/tests/integration/api/controllers/monitoring.spec.js b/tests/integration/api/controllers/monitoring.spec.js index 8fc24483f09..01bfd1a2af8 100644 --- a/tests/integration/api/controllers/monitoring.spec.js +++ b/tests/integration/api/controllers/monitoring.spec.js @@ -5,7 +5,7 @@ const utils = require('@utils'); const sentinelUtils = require('@utils/sentinel'); const VIEW_INDEXES_BY_DB = { - ['medic-test']: [ + 'medic-test': [ 'medic', 'medic-admin', 'medic-client', @@ -13,11 +13,17 @@ const VIEW_INDEXES_BY_DB = { 'medic-scripts', 'medic-sms', ], - ['medic-test-sentinel']: ['sentinel'], - ['medic-test-users-meta']: ['users-meta'], + 'medic-test-sentinel': ['sentinel'], + 'medic-test-users-meta': ['users-meta'], _users: ['users'], }; +const NOUVEAU_INDEXES_BY_DB = { + ['medic-test']: { + 'medic-nouveau': ['contacts_by_freetext', 'reports_by_freetext'], + }, +}; + const getAppVersion = async () => { const deployInfo = await utils.request({ path: '/api/deploy-info' }); return deployInfo.version; @@ -37,7 +43,18 @@ const getExpectedViewIndexes = (db) => { })); }; -const INDETERMINATE_FIELDS = ['current', 'uptime', 'date', 'fragmentation', 'node', 'sizes']; +const getExpectedNouveauIndexes = (db) => { + if (!NOUVEAU_INDEXES_BY_DB[db]) { + return; + } + + const ddocs = Object.entries(NOUVEAU_INDEXES_BY_DB[db]); + return ddocs.flatMap(([ddocName, indexes]) => indexes.map(indexName => ({ + name: `_design/${ddocName}/${indexName}`, + }))); +}; + +const INDETERMINATE_FIELDS = ['current', 'uptime', 'date', 'fragmentation', 'node', 'sizes', 'disk_size', 'num_docs']; const assertCouchDbDataSizeFields = (couchData) => { chai.expect(couchData.fragmentation).to.be.gte(0); @@ -86,6 +103,7 @@ describe('monitoring', () => { doc_count: medicInfo.doc_count, doc_del_count: medicInfo.doc_del_count, view_indexes: getExpectedViewIndexes('medic-test'), + nouveau_indexes: getExpectedNouveauIndexes('medic-test'), }, sentinel: { name: 'medic-test-sentinel', @@ -164,6 +182,7 @@ describe('monitoring', () => { doc_count: medicInfo.doc_count, doc_del_count: medicInfo.doc_del_count, view_indexes: getExpectedViewIndexes('medic-test'), + nouveau_indexes: getExpectedNouveauIndexes('medic-test'), }, sentinel: { name: 'medic-test-sentinel', diff --git a/tests/utils/index.js b/tests/utils/index.js index 89e1c5bda1c..250ae826f0e 100644 --- a/tests/utils/index.js +++ b/tests/utils/index.js @@ -1163,6 +1163,7 @@ const startServices = async () => { env.DB1_DATA = makeTempDir('ci-dbdata'); env.DB2_DATA = makeTempDir('ci-dbdata'); env.DB3_DATA = makeTempDir('ci-dbdata'); + env.COUCHDB_NOUVEAU_DATA = makeTempDir('ci-nouveaudata'); await dockerComposeCmd('up -d'); const services = await dockerComposeCmd('ps -q'); diff --git a/webapp/tests/mocha/unit/views/contacts_by_freetext.spec.js b/webapp/tests/mocha/unit/views/contacts_by_freetext.spec.js index 6d5d5e3f1eb..9aba6f9b506 100644 --- a/webapp/tests/mocha/unit/views/contacts_by_freetext.spec.js +++ b/webapp/tests/mocha/unit/views/contacts_by_freetext.spec.js @@ -36,7 +36,7 @@ const nonAsciiDoc = { reported_date: 1496068842996 }; -describe('contacts_by_freetext view', () => { +describe.skip('contacts_by_freetext view', () => { it('indexes doc name', () => { // given diff --git a/webapp/tests/mocha/unit/views/contacts_by_type_freetext.spec.js b/webapp/tests/mocha/unit/views/contacts_by_type_freetext.spec.js index 4384080ba94..977d933c278 100644 --- a/webapp/tests/mocha/unit/views/contacts_by_type_freetext.spec.js +++ b/webapp/tests/mocha/unit/views/contacts_by_type_freetext.spec.js @@ -53,7 +53,7 @@ const configurableHierarchyDoc = { let map; -describe('contacts_by_type_freetext view', () => { +describe.skip('contacts_by_type_freetext view', () => { beforeEach(() => map = utils.loadView('medic-db', 'medic-client', 'contacts_by_type_freetext')); diff --git a/webapp/tests/mocha/unit/views/reports_by_freetext.spec.js b/webapp/tests/mocha/unit/views/reports_by_freetext.spec.js index 49cdaa1b0fc..8c93f02c26f 100644 --- a/webapp/tests/mocha/unit/views/reports_by_freetext.spec.js +++ b/webapp/tests/mocha/unit/views/reports_by_freetext.spec.js @@ -112,7 +112,7 @@ const doc = { ] }; -describe('reports_by_freetext view', () => { +describe.skip('reports_by_freetext view', () => { it('indexes doc name', () => { // given