From eeafdccfd7d68f24d4dcbae8d2991801bc625a8d Mon Sep 17 00:00:00 2001 From: Joshua Kuestersteffen Date: Fri, 18 Oct 2024 14:58:19 -0500 Subject: [PATCH] Add offline ddoc logic to webapp along with initial contacts_by_freetext view --- webapp/package.json | 2 +- webapp/src/js/offline-ddocs/.eslintrc | 5 + .../contacts_by_freetext.js | 53 ++++++++ webapp/src/ts/app.component.ts | 3 +- webapp/src/ts/services/db-sync.service.ts | 28 ++++ .../ts/services/offline-ddocs/design-doc.ts | 11 ++ .../medic-offline-freetext.ddoc.ts | 10 ++ webapp/tests/karma/ts/app.component.spec.ts | 3 + .../karma/ts/services/db-sync.service.spec.ts | 59 ++++++++ webapp/tests/mocha/ts/.mocharc.js | 8 ++ .../medic-offline-freetext.spec.ts | 126 ++++++++++++++++++ webapp/tests/mocha/unit/views/utils.js | 10 +- webapp/tsconfig.spec.json | 3 +- 13 files changed, 315 insertions(+), 6 deletions(-) create mode 100644 webapp/src/js/offline-ddocs/.eslintrc create mode 100644 webapp/src/js/offline-ddocs/medic-offline-freetext/contacts_by_freetext.js create mode 100644 webapp/src/ts/services/offline-ddocs/design-doc.ts create mode 100644 webapp/src/ts/services/offline-ddocs/medic-offline-freetext.ddoc.ts create mode 100644 webapp/tests/mocha/ts/.mocharc.js create mode 100644 webapp/tests/mocha/ts/offline-ddocs/medic-offline-freetext.spec.ts diff --git a/webapp/package.json b/webapp/package.json index 7a272c60fe7..b894fe699a3 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -14,7 +14,7 @@ }, "scripts": { "postinstall": "patch-package && ng cache clean", - "unit:mocha": "UNIT_TEST_ENV=1 mocha 'tests/mocha/**/*.spec.js'", + "unit:mocha": "UNIT_TEST_ENV=1 mocha 'tests/mocha/**/*.spec.js' && mocha --config tests/mocha/ts/.mocharc.js", "unit:mocha:tz": "TZ=Canada/Pacific npm run unit:mocha && TZ=Africa/Monrovia npm run unit:mocha && TZ=Pacific/Auckland npm run unit:mocha", "unit:cht-form": "ng test cht-form", "unit": "UNIT_TEST_ENV=1 ng test webapp", diff --git a/webapp/src/js/offline-ddocs/.eslintrc b/webapp/src/js/offline-ddocs/.eslintrc new file mode 100644 index 00000000000..15b79d3fd83 --- /dev/null +++ b/webapp/src/js/offline-ddocs/.eslintrc @@ -0,0 +1,5 @@ +{ + "globals": { + "emit": true + } +} diff --git a/webapp/src/js/offline-ddocs/medic-offline-freetext/contacts_by_freetext.js b/webapp/src/js/offline-ddocs/medic-offline-freetext/contacts_by_freetext.js new file mode 100644 index 00000000000..2f1de80bfab --- /dev/null +++ b/webapp/src/js/offline-ddocs/medic-offline-freetext/contacts_by_freetext.js @@ -0,0 +1,53 @@ +module.exports.map = function(doc) { + const skip = [ '_id', '_rev', 'type', 'contact_type', 'refid', 'geolocation' ]; + + const usedKeys = []; + const emitMaybe = (key, value) => { + if (usedKeys.indexOf(key) === -1 && // Not already used + key.length > 2 // Not too short + ) { + usedKeys.push(key); + emit([key], value); + } + }; + + const emitField = (key, value, order) => { + if (!value) { + return; + } + const lowerKey = key.toLowerCase(); + if (skip.indexOf(lowerKey) !== -1 || /_date$/.test(lowerKey)) { + return; + } + if (typeof value === 'string') { + value + .toLowerCase() + .split(/\s+/) + .forEach((word) => emitMaybe(word, order)); + } + }; + + const getTypeIndex = () => { + const types = [ 'district_hospital', 'health_center', 'clinic', 'person' ]; + if (doc.type !== 'contact') { + return types.indexOf(doc.type); + } + + const contactTypeIdx = types.indexOf(doc.contact_type); + if (contactTypeIdx >= 0) { + return contactTypeIdx; + } + + return doc.contact_type; + }; + + const idx = getTypeIndex(); + if (idx !== -1) { + const dead = !!doc.date_of_death; + const muted = !!doc.muted; + const order = dead + ' ' + muted + ' ' + idx + ' ' + (doc.name && doc.name.toLowerCase()); + Object.keys(doc).forEach(function(key) { + emitField(key, doc[key], order); + }); + } +}; diff --git a/webapp/src/ts/app.component.ts b/webapp/src/ts/app.component.ts index 6716a0e5b39..044bbb75c5b 100644 --- a/webapp/src/ts/app.component.ts +++ b/webapp/src/ts/app.component.ts @@ -281,7 +281,7 @@ export class AppComponent implements OnInit, AfterViewInit { }); } - ngOnInit(): void { + async ngOnInit() { this.recordStartupTelemetry(); this.subscribeToStore(); this.setupRouter(); @@ -294,6 +294,7 @@ export class AppComponent implements OnInit, AfterViewInit { // initialisation tasks that can occur after the UI has been rendered this.setupPromise = Promise.resolve() + .then(() => this.dbSyncService.init()) .then(() => this.chtDatasourceService.isInitialized()) .then(() => this.checkPrivacyPolicy()) .then(() => (this.initialisationComplete = true)) diff --git a/webapp/src/ts/services/db-sync.service.ts b/webapp/src/ts/services/db-sync.service.ts index a8beac99ccd..eabc80af627 100644 --- a/webapp/src/ts/services/db-sync.service.ts +++ b/webapp/src/ts/services/db-sync.service.ts @@ -14,6 +14,8 @@ import { TranslateService } from '@mm-services/translate.service'; import { MigrationsService } from '@mm-services/migrations.service'; import { ReplicationService } from '@mm-services/replication.service'; import { PerformanceService } from '@mm-services/performance.service'; +import medicOfflineDdoc from '@mm-services/offline-ddocs/medic-offline-freetext.ddoc'; +import { DesignDoc } from '@mm-services/offline-ddocs/design-doc'; const READ_ONLY_TYPES = ['form', 'translations']; const READ_ONLY_IDS = ['resources', 'branding', 'service-worker-meta', 'zscore-charts', 'settings', 'partners']; @@ -24,6 +26,9 @@ const SYNC_INTERVAL = 5 * 60 * 1000; // 5 minutes const META_SYNC_INTERVAL = 30 * 60 * 1000; // 30 minutes const BATCH_SIZE = 100; const MAX_SUCCESSIVE_SYNCS = 2; +const OFFLINE_DDOCS = [ + medicOfflineDdoc, +]; const readOnlyFilter = function(doc) { // Never replicate "purged" documents upwards @@ -63,6 +68,21 @@ type SyncState = { type SyncStateListener = Parameters['subscribe']>[0]; +const getRev = async (db, id: string): Promise => db + .get(id) + .then(({ _rev }) => _rev as string) + .catch((e) => { + if (e.status === 404) { + return undefined; + } + throw e; + }); + +const initDdoc = (db) => async (ddoc: DesignDoc) => db.put({ + ...ddoc, + _rev: await getRev(db, ddoc._id), +}); + @Injectable({ providedIn: 'root' }) @@ -100,6 +120,14 @@ export class DBSyncService { return !this.sessionService.isOnlineOnly(); } + init = async () => { + if (!this.isEnabled()) { + return; + } + const medicDb = await this.dbService.get(); + return Promise.all(OFFLINE_DDOCS.map(initDdoc(medicDb))); + }; + private replicateToRetry({ batchSize=BATCH_SIZE }={}) { const telemetryEntry = new DbSyncTelemetry( this.telemetryService, diff --git a/webapp/src/ts/services/offline-ddocs/design-doc.ts b/webapp/src/ts/services/offline-ddocs/design-doc.ts new file mode 100644 index 00000000000..270a77cc6e3 --- /dev/null +++ b/webapp/src/ts/services/offline-ddocs/design-doc.ts @@ -0,0 +1,11 @@ +export interface DesignDoc { + readonly _id: `_design/${string}`; + readonly _rev?: string; + readonly views: { + [key: string]: { + map: string; + }; + }; +} + +export const packageView = ({ map }: { map: Function }) => ({ map: map.toString() }); diff --git a/webapp/src/ts/services/offline-ddocs/medic-offline-freetext.ddoc.ts b/webapp/src/ts/services/offline-ddocs/medic-offline-freetext.ddoc.ts new file mode 100644 index 00000000000..56ae4073533 --- /dev/null +++ b/webapp/src/ts/services/offline-ddocs/medic-offline-freetext.ddoc.ts @@ -0,0 +1,10 @@ +import { DesignDoc, packageView } from './design-doc'; + +import * as contactByFreetext from '../../../js/offline-ddocs/medic-offline-freetext/contacts_by_freetext.js'; + +export default { + _id: '_design/medic-offline-freetext', + views: { + contacts_by_freetext: packageView(contactByFreetext), + } +}; diff --git a/webapp/tests/karma/ts/app.component.spec.ts b/webapp/tests/karma/ts/app.component.spec.ts index 1744063b60c..9ffdfd7c027 100644 --- a/webapp/tests/karma/ts/app.component.spec.ts +++ b/webapp/tests/karma/ts/app.component.spec.ts @@ -145,6 +145,7 @@ describe('AppComponent', () => { isOnlineOnly: sinon.stub() }; dbSyncService = { + init: sinon.stub().resolves(), addUpdateListener: sinon.stub(), isEnabled: sinon.stub().returns(false), sync: sinon.stub(), @@ -479,6 +480,7 @@ describe('AppComponent', () => { }]); expect(globalActions.updateReplicationStatus.getCall(1).args).to.deep.equal([{disabled: true}]); expect(dbSyncService.subscribe.callCount).to.equal(1); + expect(dbSyncService.init.calledOnceWithExactly()).to.be.true; }); it('should sync db if enabled', async () => { @@ -499,6 +501,7 @@ describe('AppComponent', () => { expect(dbSyncService.sync.callCount).to.equal(1); expect(dbSyncService.subscribe.callCount).to.equal(1); + expect(dbSyncService.init.calledOnceWithExactly()).to.be.true; }); it('should set dbSync replication status in subcription callback', async () => { diff --git a/webapp/tests/karma/ts/services/db-sync.service.spec.ts b/webapp/tests/karma/ts/services/db-sync.service.spec.ts index 5feab563cae..2b4cc231517 100644 --- a/webapp/tests/karma/ts/services/db-sync.service.spec.ts +++ b/webapp/tests/karma/ts/services/db-sync.service.spec.ts @@ -15,6 +15,7 @@ import { PerformanceService } from '@mm-services/performance.service'; import { TranslateService } from '@mm-services/translate.service'; import { MigrationsService } from '@mm-services/migrations.service'; import { ReplicationService } from '@mm-services/replication.service'; +import medicOfflineDdoc from '@mm-services/offline-ddocs/medic-offline-freetext.ddoc'; describe('DBSync service', () => { let service:DBSyncService; @@ -102,6 +103,8 @@ describe('DBSync service', () => { replicate: { to: to }, info: sinon.stub().resolves({ update_seq: 99 }), allDocs: sinon.stub(), + get: sinon.stub(), + put: sinon.stub(), }; localMetaDb = { replicate: { to: metaTo, from: metaFrom }, @@ -148,6 +151,62 @@ describe('DBSync service', () => { clock.restore(); }); + describe('init', () => { + it('adds new ddoc to database', async () => { + isOnlineOnly.returns(false); + localMedicDb.get.rejects({ status: 404 }); + + await service.init(); + + expect(isOnlineOnly.calledOnceWithExactly()).to.be.true; + expect(db.calledOnceWithExactly()).to.be.true; + expect(localMedicDb.get.calledOnceWithExactly(medicOfflineDdoc._id)).to.be.true; + expect(localMedicDb.put.calledOnceWithExactly({ + ...medicOfflineDdoc, + _rev: undefined, + })).to.be.true; + }); + + it('updates existing ddoc in database', async () => { + isOnlineOnly.returns(false); + localMedicDb.get.resolves({ _rev: '1-abc' }); + + await service.init(); + + expect(isOnlineOnly.calledOnceWithExactly()).to.be.true; + expect(db.calledOnceWithExactly()).to.be.true; + expect(localMedicDb.get.calledOnceWithExactly(medicOfflineDdoc._id)).to.be.true; + expect(localMedicDb.put.calledOnceWithExactly({ + ...medicOfflineDdoc, + _rev: '1-abc', + })).to.be.true; + }); + + it('rejects when there is an error getting the ddoc', async () => { + isOnlineOnly.returns(false); + const expectedError = new Error('Error getting ddoc'); + localMedicDb.get.rejects(expectedError); + + await expect(service.init()).to.be.rejectedWith(expectedError); + + expect(isOnlineOnly.calledOnceWithExactly()).to.be.true; + expect(db.calledOnceWithExactly()).to.be.true; + expect(localMedicDb.get.calledOnceWithExactly(medicOfflineDdoc._id)).to.be.true; + expect(localMedicDb.put.notCalled).to.be.true; + }); + + it('does nothing when online only', async () => { + isOnlineOnly.returns(true); + + await service.init(); + + expect(isOnlineOnly.calledOnceWithExactly()).to.be.true; + expect(db.notCalled).to.be.true; + expect(localMedicDb.get.notCalled).to.be.true; + expect(localMedicDb.put.notCalled).to.be.true; + }); + }); + describe('sync', () => { it('does nothing for admins', () => { isOnlineOnly.returns(true); diff --git a/webapp/tests/mocha/ts/.mocharc.js b/webapp/tests/mocha/ts/.mocharc.js new file mode 100644 index 00000000000..96b1e599f3e --- /dev/null +++ b/webapp/tests/mocha/ts/.mocharc.js @@ -0,0 +1,8 @@ +const chaiAsPromised = require('chai-as-promised'); +const chai = require('chai'); +chai.use(chaiAsPromised); + +module.exports = { + spec: 'tests/mocha/ts/**/*.spec.ts', + require: 'ts-node/register', +}; diff --git a/webapp/tests/mocha/ts/offline-ddocs/medic-offline-freetext.spec.ts b/webapp/tests/mocha/ts/offline-ddocs/medic-offline-freetext.spec.ts new file mode 100644 index 00000000000..c9931a41b1f --- /dev/null +++ b/webapp/tests/mocha/ts/offline-ddocs/medic-offline-freetext.spec.ts @@ -0,0 +1,126 @@ +import medicOfflineFreetext from '../../../../src/ts/services/offline-ddocs/medic-offline-freetext.ddoc'; +import { expect } from 'chai'; +import { buildViewMapFn } from '../../unit/views/utils.js'; + +const expectedValue = ( + {typeIndex, name, dead = false, muted = false }: Record = {} +) => `${dead} ${muted} ${typeIndex} ${name}`; + +describe('medic-offline-freetext', () => { + it('has the correct _id', () => { + expect(medicOfflineFreetext._id).to.equal('_design/medic-offline-freetext'); + }); + + describe('contacts_by_freetext', () => { + const mapFn = buildViewMapFn(medicOfflineFreetext.views.contacts_by_freetext.map); + + afterEach(() => mapFn.reset()); + + [ + ['district_hospital', 0], + ['health_center', 1], + ['clinic', 2], + ['person', 3], + ['contact', 0, 'district_hospital'], + ['contact', 1, 'health_center'], + ['contact', 2, 'clinic'], + ['contact', 3, 'person'] + ].forEach(([type, typeIndex, contactType]) => it('emits numerical index for default type', () => { + const doc = { type, hello: 'world', contact_type: contactType }; + const emitted = mapFn(doc, true); + expect(emitted).to.deep.equal([{ key: ['world'], value: expectedValue({ typeIndex }) }]); + })); + + it('emits contact_type index for custom type', () => { + const typeIndex = 'my_custom_type'; + const doc = { contact_type: typeIndex, type: 'contact', hello: 'world' }; + const emitted = mapFn(doc, true); + expect(emitted).to.deep.equal([{ key: ['world'], value: expectedValue({ typeIndex }) }]); + }); + + it('emits nothing when type is invalid', () => { + const doc = { type: 'invalid', hello: 'world' }; + const emitted = mapFn(doc, true); + expect(emitted).to.be.empty; + }); + + it('emits death status in value', () => { + const doc = { type: 'district_hospital', date_of_death: '2021-01-01' }; + const emitted = mapFn(doc, true); + expect(emitted).to.deep.equal([{ key: ['2021-01-01'], value: expectedValue({ typeIndex: 0, dead: true }) }]); + }); + + it('emits muted status in value', () => { + const doc = { type: 'district_hospital', muted: true, hello: 'world' }; + const emitted = mapFn(doc, true); + expect(emitted).to.deep.equal([{ key: ['world'], value: expectedValue({ typeIndex: 0, muted: true }) }]); + }); + + [ + 'hello', 'HeLlO' + ].forEach(name => it('emits name in value', () => { + const doc = { type: 'district_hospital', name }; + const emitted = mapFn(doc, true); + expect(emitted).to.deep.equal([ + { key: [name.toLowerCase()], value: expectedValue({ typeIndex: 0, name: name.toLowerCase() }) } + ]); + })); + + [ + null, undefined, { hello: 'world' }, {}, 1, true + ].forEach(hello => it('emits nothing when value is not a string', () => { + const doc = { type: 'district_hospital', hello }; + const emitted = mapFn(doc, true); + expect(emitted).to.be.empty; + })); + + [ + '', 't', 'to' + ].forEach(hello => it('emits nothing when value is too short', () => { + const doc = { type: 'district_hospital', hello }; + const emitted = mapFn(doc, true); + expect(emitted).to.be.empty; + })); + + [ + '_id', '_rev', 'type', 'contact_type', 'refid', 'geolocation' + ].forEach(key => it('emits nothing for a skipped field', () => { + const doc = { type: 'district_hospital', [key]: 'world' }; + const emitted = mapFn(doc, true); + expect(emitted).to.be.empty; + })); + + it('emits nothing for fields that end with "_date"', () => { + const doc = { type: 'district_hospital', reported_date: 'world' }; + const emitted = mapFn(doc, true); + expect(emitted).to.be.empty; + }); + + it('emits value only once', () => { + const doc = { + type: 'district_hospital', + hello: 'world world', + hello1: 'world', + hello3: 'world', + }; + const emitted = mapFn(doc, true); + expect(emitted).to.deep.equal([{ key: ['world'], value: expectedValue({ typeIndex: 0 }) }]); + }); + + it('emits each word in a string', () => { + const doc = { + type: 'district_hospital', + hello: `the quick\nbrown\tfox`, + }; + const emitted = mapFn(doc, true); + + const value = expectedValue({ typeIndex: 0 }); + expect(emitted).to.deep.equal([ + { key: ['the'], value }, + { key: ['quick'], value }, + { key: ['brown'], value }, + { key: ['fox'], value }, + ]); + }); + }); +}); diff --git a/webapp/tests/mocha/unit/views/utils.js b/webapp/tests/mocha/unit/views/utils.js index e827ab349a5..39b0cb1d068 100644 --- a/webapp/tests/mocha/unit/views/utils.js +++ b/webapp/tests/mocha/unit/views/utils.js @@ -5,9 +5,7 @@ const vm = require('vm'); const MAP_ARG_NAME = 'doc'; -module.exports.loadView = (dbName, ddocName, viewName) => { - const mapPath = path.join(__dirname, '../../../../../ddocs', dbName, ddocName, 'views', viewName, '/map.js'); - const mapString = fs.readFileSync(mapPath, 'utf8'); +module.exports.buildViewMapFn = (mapString) => { const mapScript = new vm.Script('(' + mapString + ')(' + MAP_ARG_NAME + ');'); const emitted = []; @@ -35,6 +33,12 @@ module.exports.loadView = (dbName, ddocName, viewName) => { return mapFn; }; +module.exports.loadView = (dbName, ddocName, viewName) => { + const mapPath = path.join(__dirname, '../../../../../ddocs', dbName, ddocName, 'views', viewName, '/map.js'); + const mapString = fs.readFileSync(mapPath, 'utf8'); + return module.exports.buildViewMapFn(mapString); +}; + module.exports.assertIncludesPair = (array, pair) => { assert.ok(array.find((keyArray) => keyArray[0] === pair[0] && keyArray[1] === pair[1])); }; diff --git a/webapp/tsconfig.spec.json b/webapp/tsconfig.spec.json index 6301e1bcc3c..74acd4099e7 100755 --- a/webapp/tsconfig.spec.json +++ b/webapp/tsconfig.spec.json @@ -16,6 +16,7 @@ ], "include": [ "tests/karma/**/*.spec.ts", - "tests/karma/**/*.d.ts" + "tests/karma/**/*.d.ts", + "tests/mocha/ts/**/*.spec.ts", ] }