From 6ab3758956560df97ebdb1c8cfa9446ce8c6a281 Mon Sep 17 00:00:00 2001 From: m5r Date: Mon, 16 Dec 2024 16:49:11 +0100 Subject: [PATCH 1/2] wip --- api/src/db.js | 11 +++++++++++ api/src/services/setup/check-install.js | 7 ++++--- api/src/services/setup/databases.js | 12 ++++++++++++ api/src/services/setup/upgrade-steps.js | 3 ++- api/src/services/setup/utils.js | 5 +++++ .../services/setup/view-indexer-progress.js | 18 +++++++++++++++++- 6 files changed, 51 insertions(+), 5 deletions(-) diff --git a/api/src/db.js b/api/src/db.js index e9bdab6b643..ee71ce6b013 100644 --- a/api/src/db.js +++ b/api/src/db.js @@ -149,6 +149,17 @@ if (UNIT_TEST_ENV) { }); }; + // TODO: couch is complaining about `Content-Type must be application/json` + module.exports.compactView = (db, ddoc) => request.post({ + url: `${environment.serverUrl}/${db}/_compact/${ddoc}`, + json: true, + }); + + module.exports.compactNouveau = () => request.post({ + url: `${environment.couchUrl}/_nouveau_cleanup`, + json: true, + }); + const saveDocs = async (db, docs) => { try { return await db.bulkDocs(docs); diff --git a/api/src/services/setup/check-install.js b/api/src/services/setup/check-install.js index 825849a2a20..e8ceb6beaf0 100644 --- a/api/src/services/setup/check-install.js +++ b/api/src/services/setup/check-install.js @@ -5,16 +5,17 @@ const ddocsService = require('./ddocs'); const { DATABASES } = require('./databases'); const startupLog = require('./startup-log'); - +// TODO const checkInstallForDb = async (database) => { const check = {}; const allDdocs = await ddocsService.getDdocs(database); + // console.log("allDdocs", allDdocs); const bundledDdocs = await upgradeUtils.getBundledDdocs(database); + console.log("bundledDdocs", bundledDdocs); const liveDdocs = allDdocs.filter(ddoc => !ddocsService.isStaged(ddoc._id)); const stagedDdocs = allDdocs.filter(ddoc => ddocsService.isStaged(ddoc._id)); - const liveDdocsCheck = ddocsService.compareDdocs(bundledDdocs, liveDdocs); check.missing = liveDdocsCheck.missing; check.different = liveDdocsCheck.different; @@ -69,7 +70,7 @@ const checkInstall = async () => { ddocValidation.push(await checkInstallForDb(database)); } - const allDbsUpToDate = ddocValidation.every(check => check.upToDate); + const allDbsUpToDate = ddocValidation.every(check => console.log({check}) || check.upToDate); if (allDbsUpToDate) { logger.info('Installation valid.'); await upgradeUtils.interruptPreviousUpgrade(); diff --git a/api/src/services/setup/databases.js b/api/src/services/setup/databases.js index bbeea0299b8..918c21db089 100644 --- a/api/src/services/setup/databases.js +++ b/api/src/services/setup/databases.js @@ -5,12 +5,21 @@ const db = require('../../db'); * @property {string} name * @property {PouchDB} db * @property {string} jsonFileName + * @property {string[]?} ddocs */ const MEDIC_DATABASE = { name: environment.db, db: db.medic, jsonFileName: 'medic.json', + ddocs: [ + 'medic', + 'medic-admin', + 'medic-client', + 'medic-conflicts', + 'medic-scripts', + 'medic-sms', + ], }; /** @@ -22,6 +31,7 @@ const DATABASES = [ name: `${environment.db}-sentinel`, db: db.sentinel, jsonFileName: 'sentinel.json', + ddocs: ['sentinel'], }, { name: `${environment.db}-logs`, @@ -32,11 +42,13 @@ const DATABASES = [ name: `${environment.db}-users-meta`, db: db.medicUsersMeta, jsonFileName: 'users-meta.json', + ddocs: ['users-meta'], }, { name: `_users`, db: db.users, jsonFileName: 'users.json', + ddocs: ['users'], }, ]; diff --git a/api/src/services/setup/upgrade-steps.js b/api/src/services/setup/upgrade-steps.js index c0b14359e23..220d2a96374 100644 --- a/api/src/services/setup/upgrade-steps.js +++ b/api/src/services/setup/upgrade-steps.js @@ -102,7 +102,8 @@ const indexStagedViews = async () => { const viewsToIndex = await viewIndexer.getViewsToIndex(); const viewIndexingPromise = viewIndexer.indexViews(viewsToIndex); const stopQueryingIndexers = viewIndexerProgress.log(); - await viewIndexingPromise; + const nouveauIndexingPromise = Promise.resolve(); // TODO + await Promise.all([viewIndexingPromise, nouveauIndexingPromise]); stopQueryingIndexers(); }; diff --git a/api/src/services/setup/utils.js b/api/src/services/setup/utils.js index 5c24bba5c3c..0197e09d7fc 100644 --- a/api/src/services/setup/utils.js +++ b/api/src/services/setup/utils.js @@ -55,15 +55,20 @@ const deleteStagedDdocs = async () => { const cleanup = () => { for (const database of DATABASES) { logger.info(`Running DB compact and view cleanup for ${database.name}`); + const ddocs = database.ddocs ?? []; Promise .all([ database.db.compact(), database.db.viewCleanup(), + ...ddocs.map((ddoc) => db.compactView(database.name, ddoc)), ]) .catch(err => { logger.error('Error while running cleanup: %o', err); }); } + db.compactNouveau().catch(err => { + logger.error('Error while running cleanup: %o', err); + }); }; /** * Updates json ddocs to add the :staged: ddoc name prefix diff --git a/api/src/services/setup/view-indexer-progress.js b/api/src/services/setup/view-indexer-progress.js index e9e9b0129bf..86e98b7e7e8 100644 --- a/api/src/services/setup/view-indexer-progress.js +++ b/api/src/services/setup/view-indexer-progress.js @@ -89,11 +89,27 @@ const logIndexersProgress = (indexers) => { const getIndexers = async (indexers = []) => { try { const activeTasks = await db.activeTasks(); - const tasks = activeTasks.filter(task => task.type === 'indexer' && DDOC_PREFIX.test(String(task.design_document))); + const tasks = activeTasks.filter(task => { + const isViewIndexingTask = task.type === 'indexer'; + if (isViewIndexingTask) { + return DDOC_PREFIX.test(String(task.design_document)); + } + + const isNouveauIndexingTask = task.type === 'search_indexer'; + if (isNouveauIndexingTask) { + return task.design_document === '_design/medic-nouveau'; + } + + return false; + }); // We assume all previous tasks have finished. + console.log("indexers 1", indexers); indexers.forEach(setTasksToComplete); + console.log("indexers 2", indexers); updateRunningTasks(indexers, tasks); + console.log("indexers 3", indexers); indexers.forEach(calculateAverageProgress); + console.log("indexers 4", indexers); return indexers; } catch (err) { logger.error('Error while querying active tasks: %o', err); From 23d20e82f6a55cdde89b6fd6973db5ab54d4c40a Mon Sep 17 00:00:00 2001 From: m5r Date: Thu, 19 Dec 2024 17:01:17 +0100 Subject: [PATCH 2/2] wip part 2 --- api/src/services/setup/check-install.js | 3 +- api/src/services/setup/upgrade-steps.js | 2 +- .../services/setup/view-indexer-progress.js | 15 +++--- api/src/services/setup/view-indexer.js | 50 ++++++++++++++++--- .../nouveau/contacts_by_freetext/index.js | 1 + 5 files changed, 53 insertions(+), 18 deletions(-) diff --git a/api/src/services/setup/check-install.js b/api/src/services/setup/check-install.js index e8ceb6beaf0..599a7ec2de0 100644 --- a/api/src/services/setup/check-install.js +++ b/api/src/services/setup/check-install.js @@ -9,14 +9,13 @@ const startupLog = require('./startup-log'); const checkInstallForDb = async (database) => { const check = {}; const allDdocs = await ddocsService.getDdocs(database); - // console.log("allDdocs", allDdocs); const bundledDdocs = await upgradeUtils.getBundledDdocs(database); - console.log("bundledDdocs", bundledDdocs); const liveDdocs = allDdocs.filter(ddoc => !ddocsService.isStaged(ddoc._id)); const stagedDdocs = allDdocs.filter(ddoc => ddocsService.isStaged(ddoc._id)); const liveDdocsCheck = ddocsService.compareDdocs(bundledDdocs, liveDdocs); + console.log("liveDdocsCheck.different", liveDdocsCheck.different); check.missing = liveDdocsCheck.missing; check.different = liveDdocsCheck.different; diff --git a/api/src/services/setup/upgrade-steps.js b/api/src/services/setup/upgrade-steps.js index 220d2a96374..67b572b87a5 100644 --- a/api/src/services/setup/upgrade-steps.js +++ b/api/src/services/setup/upgrade-steps.js @@ -102,7 +102,7 @@ const indexStagedViews = async () => { const viewsToIndex = await viewIndexer.getViewsToIndex(); const viewIndexingPromise = viewIndexer.indexViews(viewsToIndex); const stopQueryingIndexers = viewIndexerProgress.log(); - const nouveauIndexingPromise = Promise.resolve(); // TODO + const nouveauIndexingPromise = Promise.resolve(); await Promise.all([viewIndexingPromise, nouveauIndexingPromise]); stopQueryingIndexers(); }; diff --git a/api/src/services/setup/view-indexer-progress.js b/api/src/services/setup/view-indexer-progress.js index 86e98b7e7e8..68a1f80c856 100644 --- a/api/src/services/setup/view-indexer-progress.js +++ b/api/src/services/setup/view-indexer-progress.js @@ -56,6 +56,7 @@ const updateRunningTasks = (indexers, activeTasks = []) => { indexers.push(indexer); } + console.log(`${task.node}-${task.pid} progress`, task.progress); indexer.tasks[`${task.node}-${task.pid}`] = task.progress; }); }; @@ -90,17 +91,13 @@ const getIndexers = async (indexers = []) => { try { const activeTasks = await db.activeTasks(); const tasks = activeTasks.filter(task => { - const isViewIndexingTask = task.type === 'indexer'; - if (isViewIndexingTask) { - return DDOC_PREFIX.test(String(task.design_document)); + const isFirstNouveauIndexingTask = task.type === 'search_indexer' && + task.design_document === '_design/medic-nouveau'; + if (isFirstNouveauIndexingTask) { + return true; } - const isNouveauIndexingTask = task.type === 'search_indexer'; - if (isNouveauIndexingTask) { - return task.design_document === '_design/medic-nouveau'; - } - - return false; + return DDOC_PREFIX.test(String(task.design_document)); }); // We assume all previous tasks have finished. console.log("indexers 1", indexers); diff --git a/api/src/services/setup/view-indexer.js b/api/src/services/setup/view-indexer.js index 6dac3ff3476..fc82ec4e428 100644 --- a/api/src/services/setup/view-indexer.js +++ b/api/src/services/setup/view-indexer.js @@ -12,13 +12,17 @@ let continueIndexing; const indexViews = async (viewsToIndex) => { continueIndexing = true; + console.log("viewsToIndex", viewsToIndex); if (!Array.isArray(viewsToIndex)) { + console.log("not array"); await upgradeLogService.setIndexed(); return; } await upgradeLogService.setIndexing(); const indexResult = await Promise.all(viewsToIndex.map(indexView => indexView())); + console.log("indexResult", indexResult); + console.log("continueIndexing", continueIndexing); if (continueIndexing) { await upgradeLogService.setIndexed(); } @@ -38,14 +42,19 @@ const getViewsToIndex = async () => { for (const database of DATABASES) { const stagedDdocs = await ddocsService.getStagedDdocs(database); stagedDdocs.forEach(ddoc => { - if (!ddoc.views || !_.isObject(ddoc.views)) { - return; + if (ddoc.views && !_.isObject(ddoc.views)) { + const ddocViewIndexPromises = Object + .keys(ddoc.views) + .map(viewName => indexView.bind({}, database.name, ddoc._id, viewName)); + viewsToIndex.push(...ddocViewIndexPromises); } - const ddocViewIndexPromises = Object - .keys(ddoc.views) - .map(viewName => indexView.bind({}, database.name, ddoc._id, viewName)); - viewsToIndex.push(...ddocViewIndexPromises); + if (ddoc.nouveau && !_.isObject(ddoc.nouveau)) { + const ddocNouveauIndexPromises = Object + .keys(ddoc.nouveau) + .map(indexName => indexNouveauIndex.bind({}, database.name, ddoc._id, indexName)); + viewsToIndex.push(...ddocNouveauIndexPromises); + } }); } return viewsToIndex; @@ -79,7 +88,36 @@ const indexView = async (dbName, ddocId, viewName) => { } while (continueIndexing); }; +/** + * Returns a promise that resolves when a nouveau index is indexed. + * Retries querying the index until no error is thrown + * @param {String} dbName + * @param {String} ddocId + * @param {String} indexName + * @return {Promise} + */ +const indexNouveauIndex = async (dbName, ddocId, indexName) => { + do { + try { + return await request.get({ + uri: `${environment.serverUrl}/${dbName}/_design/${ddocId}/_nouveau/${indexName}`, + json: true, + qs: { q: '*:*', limit: 1 }, + }); + } catch (requestError) { + if (!continueIndexing) { + return; + } + + if (!requestError || !requestError.error || !SOCKET_TIMEOUT_ERROR_CODE.includes(requestError.error.code)) { + throw requestError; + } + } + } while (continueIndexing); +}; + const stopIndexing = () => { + console.trace("************ stopIndexing"); continueIndexing = false; }; diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js index 2502f61245b..7fb443356cd 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js @@ -1,4 +1,5 @@ function (doc) { + 'different 2'; var skip = ['_id', '_rev', 'type', 'refid', 'geolocation']; var toIndex = '';