diff --git a/apps/web/src/lib/server/appConfig/index.ts b/apps/web/src/lib/server/appConfig/index.ts
index f8a2c4c..352c030 100644
--- a/apps/web/src/lib/server/appConfig/index.ts
+++ b/apps/web/src/lib/server/appConfig/index.ts
@@ -1,22 +1,11 @@
-import { PipelinesController, validateConfigs } from '@onaio/symbology-calc-core';
+import { PipelinesController } from '@onaio/symbology-calc-core';
import type { SingleApiSymbolConfig } from 'src/lib/shared/types';
import { geoSymbolLogger } from '../logger/winston';
import { allSymbolConfigsAccessor } from '$lib/server/constants';
import { uniqWith } from 'lodash-es';
-import yup from 'yup';
-import type { Config } from '@onaio/symbology-calc-core';
import { getConfig } from './utils';
import { readMetricOverride, writePripelineMetrics } from '../logger/configMetrics';
-export function webValidateConfigs(config: Config) {
- const extraSchema = yup.object().shape({
- uuid: yup.string().uuid().required('Symbology config does not have a uuid')
- });
-
- validateConfigs(config);
- extraSchema.validateSync(config);
-}
-
export function getAllSymbologyConfigs() {
const rawSymbologyConfigs = getConfig(
allSymbolConfigsAccessor,
@@ -31,7 +20,6 @@ export function getAllSymbologyConfigs() {
readMetric: readMetricOverride
};
});
- allSymbologyConfigs.forEach(webValidateConfigs);
const uniqAllSymbolConfigs = uniqWith(allSymbologyConfigs, (config1, config2) => {
return config1.uuid === config2.uuid;
});
diff --git a/apps/web/src/routes/configs/+server.ts b/apps/web/src/routes/configs/+server.ts
index 4c41714..8778865 100644
--- a/apps/web/src/routes/configs/+server.ts
+++ b/apps/web/src/routes/configs/+server.ts
@@ -40,7 +40,6 @@ export async function PUT({ request }) {
...configsByUuid,
[payload.uuid]: { ...payload, apiToken: configToBeReplaced.apiToken }
};
- console.log({ newConfigs }, configToBeReplaced.apiToken);
const newDataConfigs = {
...data,
allSymbologyConfigs: Object.values(newConfigs)
diff --git a/apps/web/src/routes/workflows/+page.server.ts b/apps/web/src/routes/workflows/+page.server.ts
index 0c29882..77511d7 100644
--- a/apps/web/src/routes/workflows/+page.server.ts
+++ b/apps/web/src/routes/workflows/+page.server.ts
@@ -6,12 +6,15 @@ import type { ConfigRunner } from '@onaio/symbology-calc-core';
export function load() {
const configs = getClientSideSymbologyConfigs();
const ConfigsWithMetrics = configs.map((config) => {
- const metricForThisConfig = getLastPipelineMetricForConfig(config.uuid);
- const isRunning = (pipelineController.getPipelines(config.uuid) as ConfigRunner)?.isRunning();
+ const configId = config.uuid;
+ const metricForThisConfig = getLastPipelineMetricForConfig(configId);
+ const pipeLineRunner = pipelineController.getPipelines(configId) as ConfigRunner;
+ const isRunning = pipeLineRunner?.isRunning();
return {
...config,
metric: metricForThisConfig,
- isRunning
+ isRunning,
+ invalidityErrror: pipeLineRunner.invalidError
};
});
return {
diff --git a/apps/web/src/routes/workflows/+page.svelte b/apps/web/src/routes/workflows/+page.svelte
index 069eb89..f74a916 100644
--- a/apps/web/src/routes/workflows/+page.svelte
+++ b/apps/web/src/routes/workflows/+page.svelte
@@ -90,6 +90,13 @@
+ {#if config.invalidityErrror !== null}
+
+
+ {config.invalidityErrror}
+
+
+ {/if}
- API Base url
- {config.baseUrl}
diff --git a/packages/core/README.md b/packages/core/README.md
index fb9dea8..90c389f 100644
--- a/packages/core/README.md
+++ b/packages/core/README.md
@@ -76,6 +76,10 @@ export interface Config {
// cron-like syntax that represents when the pipeline represented by this config runs.
// this can be more easily generated using an online tool like https://crontab.cronhub.io/
schedule: CronTabString;
+ // how many registration form submissions to process at a time.
+ regFormSubmissionChunks?: number;
+ // store metric; progress information regarding a running pipeline or the last run of an pipeline
+ writeMetric: WriteMetric;
}
```
diff --git a/packages/core/src/evaluator/configRunner.ts b/packages/core/src/evaluator/configRunner.ts
index f5d2780..3de27ad 100644
--- a/packages/core/src/evaluator/configRunner.ts
+++ b/packages/core/src/evaluator/configRunner.ts
@@ -5,7 +5,8 @@ import {
createInfoLog,
createErrorLog,
defaultWriteMetric,
- colorDeciderFactory
+ colorDeciderFactory,
+ validateConfigs
} from '../helpers/utils';
import cron from 'node-cron';
import { createMetricFactory } from './helpers/utils';
@@ -22,9 +23,19 @@ export class ConfigRunner {
public config: Config;
/** Whether pipeline/runner is currently evaluating */
private running = false;
+ /** request abort controller */
+ private abortController;
+ /** stores validity of config */
+ public invalidError: string | null = null;
constructor(config: Config) {
this.config = config;
+ this.abortController = new AbortController();
+ try {
+ validateConfigs(config);
+ } catch (err: unknown) {
+ this.invalidError = (err as Error).message;
+ }
}
/** Runs the pipeline, generator that yields metric information regarding the current run */
@@ -37,14 +48,11 @@ export class ConfigRunner {
logger,
baseUrl,
apiToken,
- requestController,
uuid: configId,
regFormSubmissionChunks: facilityProcessingChunks
} = config;
const regFormSubmissionChunks = facilityProcessingChunks ?? 1000;
- this.running = true;
-
const startTime = Date.now();
const createMetric = createMetricFactory(startTime, configId);
let evaluatedSubmissions = 0;
@@ -62,13 +70,13 @@ export class ConfigRunner {
totalRegFormSubmissions
);
- const service = new OnaApiService(baseUrl, apiToken, logger, requestController);
+ const service = new OnaApiService(baseUrl, apiToken, logger, this.abortController);
const colorDecider = colorDeciderFactory(symbolConfig, logger);
abortableBlock: {
const regForm = await service.fetchSingleForm(regFormId);
if (regForm.isFailure) {
- return createMetric(
+ yield createMetric(
evaluatedSubmissions,
notModifiedWithoutError,
notModifiedWithError,
@@ -76,6 +84,7 @@ export class ConfigRunner {
totalRegFormSubmissions,
true
);
+ return;
}
const regFormSubmissionsNum = regForm.getValue()[numOfSubmissionsAccessor];
totalRegFormSubmissions = regFormSubmissionsNum;
@@ -140,24 +149,28 @@ export class ConfigRunner {
totalRegFormSubmissions,
true
);
- this.running = false;
}
/** Wrapper around the transform generator, collates the metrics and calls a callback that
* inverts the control of writing the metric information to the configs writeMetric method.
*/
async transform() {
- // create a function that parses the config and supplies default values.
const config = this.config;
+ if (this.invalidError) {
+ return Result.fail(`Configuration is not valid, ${this.invalidError}`);
+ }
+ // create a function that parses the config and supplies default values.
const WriteMetric = config.writeMetric ?? defaultWriteMetric;
if (this.running) {
return Result.fail('Pipeline is already running.');
} else {
+ this.running = true;
let finalMetric;
for await (const metric of this.transformGenerator()) {
WriteMetric(metric);
finalMetric = metric;
}
+ this.running = false;
return Result.ok(finalMetric);
}
}
@@ -176,11 +189,7 @@ export class ConfigRunner {
/** Cancel evaluation using a configured abortController */
cancel() {
- const config = this.config;
- const abortController = config.requestController;
- if (!abortController) {
- return Result.fail(`Abort controller not set for config : ${config.uuid}`);
- }
+ const abortController = this.abortController;
if (!this.running) {
return;
}
@@ -198,4 +207,9 @@ export class ConfigRunner {
isRunning() {
return this.running;
}
+
+ /** Whether config for this runner is valid */
+ isValid() {
+ return this.invalidError === null;
+ }
}
diff --git a/packages/core/src/evaluator/pipelinesController.ts b/packages/core/src/evaluator/pipelinesController.ts
index 22cb1d3..6f9dd90 100644
--- a/packages/core/src/evaluator/pipelinesController.ts
+++ b/packages/core/src/evaluator/pipelinesController.ts
@@ -4,8 +4,6 @@ import { ConfigRunner } from './configRunner';
import { isEqual } from 'lodash-es';
import { Result } from '../helpers/Result';
-console.log({ cron });
-
interface GetConfigs {
(): Config[];
}
diff --git a/packages/core/src/evaluator/tests/fixtures/fixtures.ts b/packages/core/src/evaluator/tests/fixtures/fixtures.ts
index 8107583..84def04 100644
--- a/packages/core/src/evaluator/tests/fixtures/fixtures.ts
+++ b/packages/core/src/evaluator/tests/fixtures/fixtures.ts
@@ -7,10 +7,7 @@ import { defaultWriteMetric } from '../../../helpers/utils';
export { form3623, form3623Submissions, form3624Submissions };
export const apiToken = 'apiToken';
-export const createConfigs = (
- loggerMock: jest.Mock,
- controller = new AbortController()
-): Config => ({
+export const createConfigs = (loggerMock: jest.Mock): Config => ({
uuid: 'uuid',
regFormId: '3623',
visitFormId: '3624',
@@ -80,6 +77,5 @@ export const createConfigs = (
apiToken,
baseUrl: 'https://test-api.ona.io',
schedule: '* * * * *',
- writeMetric: defaultWriteMetric,
- requestController: controller
+ writeMetric: defaultWriteMetric
});
diff --git a/packages/core/src/evaluator/tests/helpers.test.ts b/packages/core/src/evaluator/tests/helpers.test.ts
index 1a1e0e7..4b93191 100644
--- a/packages/core/src/evaluator/tests/helpers.test.ts
+++ b/packages/core/src/evaluator/tests/helpers.test.ts
@@ -167,7 +167,6 @@ describe('transform facility tests', () => {
);
expect(response.modified).toBeFalsy();
- console.log(typeof response.error);
expect(response.error).toEqual('400: {"message":"error"}: Network request failed.');
});
diff --git a/packages/core/src/evaluator/tests/pipelinesController.test.ts b/packages/core/src/evaluator/tests/pipelinesController.test.ts
index 445e21d..34dab1b 100644
--- a/packages/core/src/evaluator/tests/pipelinesController.test.ts
+++ b/packages/core/src/evaluator/tests/pipelinesController.test.ts
@@ -24,7 +24,8 @@ jest.mock('node-cron', () => {
// eslint-disable-next-line @typescript-eslint/no-unused-vars
schedule: (cronString: string, _callback: () => unknown) => {
return `task-${cronString}`;
- }
+ },
+ validate: () => true
};
});
@@ -112,10 +113,11 @@ it('works correctly nominal case', async () => {
const configRunner = pipelinesController.getPipelines(configs.uuid);
const runner = configRunner as ConfigRunner;
expect(runner.isRunning()).toBeFalsy();
+ expect(runner.isValid()).toBeTruthy();
const response = runner.transform();
expect(runner.isRunning()).toBeTruthy();
const metric = await response;
- console.log({ configRunner });
+
expect(runner.isRunning()).toBeFalsy();
expect(loggerMock.mock.calls).toEqual(logCalls);
expect(metric.getValue()).toEqual({
@@ -140,12 +142,12 @@ it('error when fetching the registration form', async () => {
nock(configs.baseUrl).get(`/${formEndpoint}/3623`).replyWithError('Could not find form with id');
const pipelinesController = new PipelinesController(() => [configs]);
- const configRunner = pipelinesController.getPipelines(configs.uuid);
+ const configRunner = pipelinesController.getPipelines(configs.uuid) as ConfigRunner;
const runner = configRunner as ConfigRunner;
await runner.transform().catch((err) => {
throw err;
});
-
+ expect(configRunner.isRunning()).toBeFalsy();
expect(loggerMock.mock.calls).toEqual([
[
{
diff --git a/packages/core/src/helpers/types.ts b/packages/core/src/helpers/types.ts
index ad68012..c789d29 100644
--- a/packages/core/src/helpers/types.ts
+++ b/packages/core/src/helpers/types.ts
@@ -38,10 +38,8 @@ export interface Config {
schedule: CronTabString;
// how many registration form submissions to process at a time.
regFormSubmissionChunks?: number;
- // store metric; progress information regarding a running or the last run of an pipeline
+ // store metric; progress information regarding a running pipeline or the last run of an pipeline
writeMetric: WriteMetric;
- // Abort controller to abort evaluation of this pipeline
- requestController: AbortController;
}
export enum PriorityLevel {
diff --git a/packages/core/src/services/onaApi/services.ts b/packages/core/src/services/onaApi/services.ts
index 14835eb..7183fd1 100644
--- a/packages/core/src/services/onaApi/services.ts
+++ b/packages/core/src/services/onaApi/services.ts
@@ -37,7 +37,6 @@ export const customFetch = async (input: RequestInfo, init?: RequestInit) => {
}
};
const response = await persistentFetch(input, requestOptionsWithRetry).catch((err) => {
- console.log('===>', err);
throw Error(`${err.type}: ${err.name}: ${err.message}.`);
});
if (response?.ok) {
@@ -156,11 +155,8 @@ export class OnaApiService {
const paginatedSubmissionsUrl = `${fullSubmissionsUrl}?${sParams.toString()}`;
page = page + 1;
- // const stop1StartFetchPage = performance.now(); //
- console.log('We also got here');
yield await customFetch(paginatedSubmissionsUrl, { ...this.getCommonFetchOptions() })
.then((res) => {
- console.log({ res });
return (res.json() as Promise).then((res) => {
this.logger?.(
createInfoLog(
@@ -178,18 +174,6 @@ export class OnaApiService {
);
return Result.fail(err.message);
});
- // .finally(() => {
- // const stop2EndFetchPage = performance.now();
- // if (pageSize > 100) {
- // this.logger?.(
- // createInfoLog(
- // `Fetched page: ${page}: from ${stop1StartFetchPage} to ${stop2EndFetchPage} i.e in: ${
- // stop2EndFetchPage - stop1StartFetchPage
- // }`
- // )
- // );
- // }
- // });
} while (page * pageSize <= totalSubmissions);
}
@@ -217,7 +201,6 @@ export class OnaApiService {
};
const fullEditSubmissionUrl = `${this.baseUrl}/${editSubmissionPath}`;
- // const stop1StartEdit = performance.now();
return await customFetch(fullEditSubmissionUrl, {
...this.getCommonFetchOptions(),
method: 'POST',
@@ -234,7 +217,6 @@ export class OnaApiService {
});
})
.catch((err) => {
- console.log('==>', { err });
this.logger?.(
createErrorLog(
`Failed to edit sumbission with _id: ${submissionPayload._id} for form with id: ${formId} with err: ${err.message}`
@@ -242,16 +224,6 @@ export class OnaApiService {
);
return Result.fail(err);
});
- // .finally(() => {
- // const stop1StopEdit = performance.now();
- // this.logger?.(
- // createInfoLog(
- // `Editing submission with _id: ${submissionPayload._id} took ${
- // stop1StopEdit - stop1StartEdit
- // }`
- // )
- // );
- // });
}
}