diff --git a/api/src/db.js b/api/src/db.js index e9bdab6b64..ee71ce6b01 100644 --- a/api/src/db.js +++ b/api/src/db.js @@ -149,6 +149,17 @@ if (UNIT_TEST_ENV) { }); }; + // TODO: couch is complaining about `Content-Type must be application/json` + module.exports.compactView = (db, ddoc) => request.post({ + url: `${environment.serverUrl}/${db}/_compact/${ddoc}`, + json: true, + }); + + module.exports.compactNouveau = () => request.post({ + url: `${environment.couchUrl}/_nouveau_cleanup`, + json: true, + }); + const saveDocs = async (db, docs) => { try { return await db.bulkDocs(docs); diff --git a/api/src/services/setup/check-install.js b/api/src/services/setup/check-install.js index 825849a2a2..599a7ec2de 100644 --- a/api/src/services/setup/check-install.js +++ b/api/src/services/setup/check-install.js @@ -5,7 +5,7 @@ const ddocsService = require('./ddocs'); const { DATABASES } = require('./databases'); const startupLog = require('./startup-log'); - +// TODO const checkInstallForDb = async (database) => { const check = {}; const allDdocs = await ddocsService.getDdocs(database); @@ -14,8 +14,8 @@ const checkInstallForDb = async (database) => { const liveDdocs = allDdocs.filter(ddoc => !ddocsService.isStaged(ddoc._id)); const stagedDdocs = allDdocs.filter(ddoc => ddocsService.isStaged(ddoc._id)); - const liveDdocsCheck = ddocsService.compareDdocs(bundledDdocs, liveDdocs); + console.log("liveDdocsCheck.different", liveDdocsCheck.different); check.missing = liveDdocsCheck.missing; check.different = liveDdocsCheck.different; @@ -69,7 +69,7 @@ const checkInstall = async () => { ddocValidation.push(await checkInstallForDb(database)); } - const allDbsUpToDate = ddocValidation.every(check => check.upToDate); + const allDbsUpToDate = ddocValidation.every(check => console.log({check}) || check.upToDate); if (allDbsUpToDate) { logger.info('Installation valid.'); await upgradeUtils.interruptPreviousUpgrade(); diff --git a/api/src/services/setup/databases.js b/api/src/services/setup/databases.js index bbeea0299b..918c21db08 100644 --- a/api/src/services/setup/databases.js +++ b/api/src/services/setup/databases.js @@ -5,12 +5,21 @@ const db = require('../../db'); * @property {string} name * @property {PouchDB} db * @property {string} jsonFileName + * @property {string[]?} ddocs */ const MEDIC_DATABASE = { name: environment.db, db: db.medic, jsonFileName: 'medic.json', + ddocs: [ + 'medic', + 'medic-admin', + 'medic-client', + 'medic-conflicts', + 'medic-scripts', + 'medic-sms', + ], }; /** @@ -22,6 +31,7 @@ const DATABASES = [ name: `${environment.db}-sentinel`, db: db.sentinel, jsonFileName: 'sentinel.json', + ddocs: ['sentinel'], }, { name: `${environment.db}-logs`, @@ -32,11 +42,13 @@ const DATABASES = [ name: `${environment.db}-users-meta`, db: db.medicUsersMeta, jsonFileName: 'users-meta.json', + ddocs: ['users-meta'], }, { name: `_users`, db: db.users, jsonFileName: 'users.json', + ddocs: ['users'], }, ]; diff --git a/api/src/services/setup/upgrade-steps.js b/api/src/services/setup/upgrade-steps.js index c0b14359e2..67b572b87a 100644 --- a/api/src/services/setup/upgrade-steps.js +++ b/api/src/services/setup/upgrade-steps.js @@ -102,7 +102,8 @@ const indexStagedViews = async () => { const viewsToIndex = await viewIndexer.getViewsToIndex(); const viewIndexingPromise = viewIndexer.indexViews(viewsToIndex); const stopQueryingIndexers = viewIndexerProgress.log(); - await viewIndexingPromise; + const nouveauIndexingPromise = Promise.resolve(); + await Promise.all([viewIndexingPromise, nouveauIndexingPromise]); stopQueryingIndexers(); }; diff --git a/api/src/services/setup/utils.js b/api/src/services/setup/utils.js index 5c24bba5c3..0197e09d7f 100644 --- a/api/src/services/setup/utils.js +++ b/api/src/services/setup/utils.js @@ -55,15 +55,20 @@ const deleteStagedDdocs = async () => { const cleanup = () => { for (const database of DATABASES) { logger.info(`Running DB compact and view cleanup for ${database.name}`); + const ddocs = database.ddocs ?? []; Promise .all([ database.db.compact(), database.db.viewCleanup(), + ...ddocs.map((ddoc) => db.compactView(database.name, ddoc)), ]) .catch(err => { logger.error('Error while running cleanup: %o', err); }); } + db.compactNouveau().catch(err => { + logger.error('Error while running cleanup: %o', err); + }); }; /** * Updates json ddocs to add the :staged: ddoc name prefix diff --git a/api/src/services/setup/view-indexer-progress.js b/api/src/services/setup/view-indexer-progress.js index e9e9b0129b..68a1f80c85 100644 --- a/api/src/services/setup/view-indexer-progress.js +++ b/api/src/services/setup/view-indexer-progress.js @@ -56,6 +56,7 @@ const updateRunningTasks = (indexers, activeTasks = []) => { indexers.push(indexer); } + console.log(`${task.node}-${task.pid} progress`, task.progress); indexer.tasks[`${task.node}-${task.pid}`] = task.progress; }); }; @@ -89,11 +90,23 @@ const logIndexersProgress = (indexers) => { const getIndexers = async (indexers = []) => { try { const activeTasks = await db.activeTasks(); - const tasks = activeTasks.filter(task => task.type === 'indexer' && DDOC_PREFIX.test(String(task.design_document))); + const tasks = activeTasks.filter(task => { + const isFirstNouveauIndexingTask = task.type === 'search_indexer' && + task.design_document === '_design/medic-nouveau'; + if (isFirstNouveauIndexingTask) { + return true; + } + + return DDOC_PREFIX.test(String(task.design_document)); + }); // We assume all previous tasks have finished. + console.log("indexers 1", indexers); indexers.forEach(setTasksToComplete); + console.log("indexers 2", indexers); updateRunningTasks(indexers, tasks); + console.log("indexers 3", indexers); indexers.forEach(calculateAverageProgress); + console.log("indexers 4", indexers); return indexers; } catch (err) { logger.error('Error while querying active tasks: %o', err); diff --git a/api/src/services/setup/view-indexer.js b/api/src/services/setup/view-indexer.js index 6dac3ff347..fc82ec4e42 100644 --- a/api/src/services/setup/view-indexer.js +++ b/api/src/services/setup/view-indexer.js @@ -12,13 +12,17 @@ let continueIndexing; const indexViews = async (viewsToIndex) => { continueIndexing = true; + console.log("viewsToIndex", viewsToIndex); if (!Array.isArray(viewsToIndex)) { + console.log("not array"); await upgradeLogService.setIndexed(); return; } await upgradeLogService.setIndexing(); const indexResult = await Promise.all(viewsToIndex.map(indexView => indexView())); + console.log("indexResult", indexResult); + console.log("continueIndexing", continueIndexing); if (continueIndexing) { await upgradeLogService.setIndexed(); } @@ -38,14 +42,19 @@ const getViewsToIndex = async () => { for (const database of DATABASES) { const stagedDdocs = await ddocsService.getStagedDdocs(database); stagedDdocs.forEach(ddoc => { - if (!ddoc.views || !_.isObject(ddoc.views)) { - return; + if (ddoc.views && !_.isObject(ddoc.views)) { + const ddocViewIndexPromises = Object + .keys(ddoc.views) + .map(viewName => indexView.bind({}, database.name, ddoc._id, viewName)); + viewsToIndex.push(...ddocViewIndexPromises); } - const ddocViewIndexPromises = Object - .keys(ddoc.views) - .map(viewName => indexView.bind({}, database.name, ddoc._id, viewName)); - viewsToIndex.push(...ddocViewIndexPromises); + if (ddoc.nouveau && !_.isObject(ddoc.nouveau)) { + const ddocNouveauIndexPromises = Object + .keys(ddoc.nouveau) + .map(indexName => indexNouveauIndex.bind({}, database.name, ddoc._id, indexName)); + viewsToIndex.push(...ddocNouveauIndexPromises); + } }); } return viewsToIndex; @@ -79,7 +88,36 @@ const indexView = async (dbName, ddocId, viewName) => { } while (continueIndexing); }; +/** + * Returns a promise that resolves when a nouveau index is indexed. + * Retries querying the index until no error is thrown + * @param {String} dbName + * @param {String} ddocId + * @param {String} indexName + * @return {Promise} + */ +const indexNouveauIndex = async (dbName, ddocId, indexName) => { + do { + try { + return await request.get({ + uri: `${environment.serverUrl}/${dbName}/_design/${ddocId}/_nouveau/${indexName}`, + json: true, + qs: { q: '*:*', limit: 1 }, + }); + } catch (requestError) { + if (!continueIndexing) { + return; + } + + if (!requestError || !requestError.error || !SOCKET_TIMEOUT_ERROR_CODE.includes(requestError.error.code)) { + throw requestError; + } + } + } while (continueIndexing); +}; + const stopIndexing = () => { + console.trace("************ stopIndexing"); continueIndexing = false; }; diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js index 2502f61245..7fb443356c 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js @@ -1,4 +1,5 @@ function (doc) { + 'different 2'; var skip = ['_id', '_rev', 'type', 'refid', 'geolocation']; var toIndex = '';