diff --git a/prisma/seed.ts b/prisma/seed.ts index 436745e0a..97008ced2 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -169,5 +169,6 @@ const userPassword = 'taskany'; prisma.appConfig.create({ data: {} }), createSheep(), createCronJob('goalPing', '0 0 0 1 * *'), + createCronJob('externalTaskCheck', '1/10 * * * *'), ]); })(); diff --git a/src/utils/recalculateCriteriaScore.ts b/src/utils/recalculateCriteriaScore.ts index 12cde48e9..8cfcc27e2 100644 --- a/src/utils/recalculateCriteriaScore.ts +++ b/src/utils/recalculateCriteriaScore.ts @@ -97,10 +97,24 @@ type GoalCalculateScore = Goal & { * then for all goals needs recalculate score and parents project's average score */ +interface CriteriaScoreUpdateBaseApi { + recalcCurrentGoalScore: () => CriteriaScoreUpdateApi & CriteriaScoreUpdateBaseApi; + recalcAverageProjectScore: () => CriteriaScoreUpdateApi & CriteriaScoreUpdateBaseApi; + recalcLinkedGoalsScores: () => CriteriaScoreUpdateApi & CriteriaScoreUpdateBaseApi; +} + +interface CriteriaScoreUpdateApi { + run: () => Promise; + makeChain: ( + ...names: Array + ) => CriteriaScoreUpdateApi & CriteriaScoreUpdateBaseApi; +} + export const recalculateCriteriaScore = (goalId: string) => { let currentGoal: GoalCalculateScore; let countsToUpdate: number; let count = 0; + const getCurrentGoal = async () => { if (!currentGoal || countsToUpdate > count++) { currentGoal = await prisma.goal.findUniqueOrThrow({ @@ -118,7 +132,7 @@ export const recalculateCriteriaScore = (goalId: string) => { let prismaCtx: Omit; // eslint-disable-next-line @typescript-eslint/no-explicit-any - const promisesChain: (() => Promise)[] = []; + const promisesChain: (() => Promise)[] = []; const updateGoalScore = ({ id, score }: { id: string; score: number | null }) => { return prismaCtx.goal.update({ @@ -127,7 +141,16 @@ export const recalculateCriteriaScore = (goalId: string) => { }); }; - const methods = { + const run = () => { + return prisma.$transaction((ctx) => { + prismaCtx = ctx; + countsToUpdate = promisesChain.length; + + return promisesChain.reduce((promise, getter) => promise.then(getter), Promise.resolve()); + }); + }; + + const methods: CriteriaScoreUpdateApi & CriteriaScoreUpdateBaseApi = { recalcCurrentGoalScore: () => { promisesChain.push(async () => { const goal = await getCurrentGoal(); @@ -181,7 +204,7 @@ export const recalculateCriteriaScore = (goalId: string) => { group by 1 `; - return prismaCtx.$executeRaw` + await prismaCtx.$executeRaw` update "Project" as project set "averageScore" = scoreByProject.score from (${countsRequests}) as scoreByProject(projectId, score) @@ -252,13 +275,13 @@ export const recalculateCriteriaScore = (goalId: string) => { return methods; }, - async run() { - return prisma.$transaction((ctx) => { - prismaCtx = ctx; - countsToUpdate = promisesChain.length; - - return promisesChain.reduce((promise, getter) => promise.then(getter), Promise.resolve()); + run, + makeChain(...names) { + names.forEach((name) => { + methods[name](); }); + + return methods; }, }; return methods; diff --git a/src/utils/worker/create.ts b/src/utils/worker/create.ts index 74c30dd2a..3b71a3d15 100644 --- a/src/utils/worker/create.ts +++ b/src/utils/worker/create.ts @@ -1,6 +1,7 @@ import { prisma } from '../prisma'; import * as templates from './mail/templates'; +import { log } from './utils'; export const defaultJobDelay = process.env.WORKER_JOBS_DELAY ? parseInt(process.env.WORKER_JOBS_DELAY, 10) : 1000; @@ -14,6 +15,7 @@ export enum jobKind { email = 'email', cron = 'cron', comment = 'comment', + criteriaToUpdate = 'criteriaToUpdate', } type Templates = typeof templates; @@ -25,15 +27,22 @@ export interface JobDataMap { data: any; }; cron: { - template: 'goalPing'; + template: 'goalPing' | 'externalTaskCheck'; }; comment: { goalId: string; activityId: string; description: string; }; + criteriaToUpdate: { + id: string; + }; } +export const castJobData = (kind: Kind, data: unknown): data is JobDataMap[Kind] => { + return data != null; +}; + export type JobKind = keyof JobDataMap; interface CreateJobProps { @@ -43,10 +52,27 @@ interface CreateJobProps { cron?: string; } +export const pickScheduledLastJob = async (kind: JobKind) => { + const res = await prisma.job.findMany({ + where: { kind, state: jobState.scheduled }, + orderBy: [{ createdAt: 'desc' }, { updatedAt: 'desc' }], + take: 1, + skip: 0, + }); + + if (res.length) { + return res[0]; + } + + return null; +}; + export function createJob( kind: K, { data, priority, delay = defaultJobDelay, cron }: CreateJobProps, ) { + log(`create new ${kind} job`, JSON.stringify(data)); + return prisma.job.create({ data: { state: jobState.scheduled, @@ -86,3 +112,7 @@ export function createCommentJob(data: JobDataMap['comment'], delay?: number) { delay, }); } + +export const createCriteriaToUpdate = (data: JobDataMap['criteriaToUpdate'], delay?: number) => { + return createJob('criteriaToUpdate', { data, delay }); +}; diff --git a/src/utils/worker/externalTasksJob.ts b/src/utils/worker/externalTasksJob.ts new file mode 100644 index 000000000..5684ef251 --- /dev/null +++ b/src/utils/worker/externalTasksJob.ts @@ -0,0 +1,242 @@ +import { ExternalTask, Prisma } from '@prisma/client'; +import assert from 'assert'; + +import type { JiraIssue } from '../integration/jira'; +import { jiraService } from '../integration/jira'; +import { prisma } from '../prisma'; +import { recalculateCriteriaScore } from '../recalculateCriteriaScore'; + +import { getSheep } from './sheep'; +import { castJobData, createCriteriaToUpdate, jobKind, jobState } from './create'; + +export const dayDuration = 24 * 60 * 60 * 1000; + +const createSQLValues = (sources: JiraIssue[]) => + Prisma.join( + sources.map(({ key, id, status, summary, issuetype, reporter, project, resolution }) => + Prisma.join( + [ + id, + key, + summary, + status.name, + status.statusCategory.id, + status?.statusCategory.name, + status.statusCategory?.id ? status.statusCategory?.id : Number(status.id), + status.statusCategory?.colorName ?? null, + status.iconUrl, + project.name, + project.key, + issuetype.name, + issuetype.iconUrl, + issuetype.id, + reporter.emailAddress, + reporter.key, + reporter.displayName || reporter.name || null, + resolution?.name || null, + resolution?.id || null, + ], + ',', + '(', + ')', + ), + ), + ); + +const getResolvedJiraTasks = async (ids: string[]) => { + if (ids.length) { + const results = await jiraService.instance.searchJira( + `key in (${ids.map((id) => `"${id}"`).join(',')}) and resolution is not EMPTY`, + ); + + return results.issues.map((issue: { fields: { [key: string]: unknown } }) => ({ + ...issue, + ...issue.fields, + })) as Array; + } + + return null; +}; + +export const updateAssociatedGoalsByCriteriaIds = (_criteriaIds: string[]) => {}; + +const getSheepOrThrow = async () => { + const { activityId } = (await getSheep()) || {}; + + assert(activityId, 'No avaliable sheeps'); + + return activityId; +}; + +// get all incompleted criteria with associated jira tasks +const getCriteriaWithTasks = async (from: Date) => + prisma.goalAchieveCriteria.findMany({ + where: { + externalTaskId: { not: null }, + isDone: false, + updatedAt: { lte: from }, + AND: [ + { + OR: [{ deleted: false }, { deleted: null }], + }, + ], + }, + select: { + id: true, + goalId: true, + externalTask: true, + }, + }); + +const updateExternalTasks = (tasks: JiraIssue[]) => { + // update external tasks + const values = createSQLValues(tasks); // (val1-1, val1-2, ...), (val2-1, val2-2, ...), ... + + const valuesToUpdate = Prisma.sql`( + VALUES${values} + ) AS task( + "externalId", "key", "title", + "state", "stateId", "stateCategoryName", "stateCategoryId", "stateColor", "stateIcon", + "project", "projectKey", + "type", "typeIcon", "typeId", + "ownerEmail", "ownerKey", "ownerName", + "resolution", "resolutionId" + )`; + + const rawSql = Prisma.sql` + UPDATE "ExternalTask" + SET + "title" = task."title", + "externalId" = task."externalId", + "externalKey" = task."key", + "type" = task."type", + "typeIconUrl" = task."typeIcon", + "typeId" = task."typeId", + "state" = task."state", + "stateId" = task."stateId", + "stateColor" = task."stateColor", + "stateIconUrl" = task."stateIcon", + "stateCategoryId" = cast(task."stateCategoryId" as int), + "stateCategoryName" = task."stateCategoryName", + "project" = task."project", + "projectId" = task."projectKey", + "ownerName" = task."ownerName", + "ownerEmail" = task."ownerEmail", + "ownerId" = task."ownerKey", + "resolution" = task."resolution", + "resolutionId" = task."resolutionId" + FROM ${valuesToUpdate} + WHERE "ExternalTask"."externalKey" = task."key" + RETURNING * + `; + + return prisma.$queryRaw`${rawSql}` as unknown as Promise; +}; + +export const externalTasksJob = async (criteriaId: string) => { + const sheepActivityId = await getSheepOrThrow(); + + const actualCriteria = await prisma.goalAchieveCriteria.findUniqueOrThrow({ + where: { id: criteriaId }, + select: { + id: true, + goalId: true, + }, + }); + + const recalcScore = recalculateCriteriaScore(actualCriteria.goalId).makeChain( + 'recalcCurrentGoalScore', + 'recalcLinkedGoalsScores', + 'recalcAverageProjectScore', + ); + + await prisma + .$transaction(async (ctx) => { + await Promise.all([ + ctx.goalAchieveCriteria.update({ + where: { id: criteriaId }, + data: { + isDone: true, + }, + }), + ctx.goalHistory.create({ + data: { + goalId: actualCriteria.goalId, + subject: 'criteria', + action: 'complete', + previousValue: null, + nextValue: criteriaId, + activityId: sheepActivityId, + }, + }), + ]); + }) + .then(() => recalcScore.run()); +}; + +const updateMinInterval = 300; +const tasksPeriod = 1000 * 60; // every minute + +export const externalTaskCheckJob = async () => { + const atYesterday = new Date(); + atYesterday.setDate(atYesterday.getDate() - 1); + + // getting all criteria which updated at last week or earlier + const criteriaListPromise = getCriteriaWithTasks(atYesterday); + const notCompetedJobsPromise = prisma.job.findMany({ + where: { + kind: jobKind.criteriaToUpdate, + state: jobState.scheduled, + }, + }); + + const [criteriaList, jobs] = await Promise.all([criteriaListPromise, notCompetedJobsPromise]); + + if (!criteriaList.length) { + return; + } + + const externalTaskKeys = criteriaList.reduce((acc, c) => { + if (c.externalTask != null) { + acc.add(c.externalTask.externalKey); + } + + return acc; + }, new Set()); + + const externalTasks = await getResolvedJiraTasks(Array.from(externalTaskKeys)); + + if (!externalTasks?.length) { + return; + } + + const updatedTasks = await updateExternalTasks(externalTasks); + + const updatedExternalTaskKeys = updatedTasks.reduce>((acc, { id }) => { + acc[id] = true; + return acc; + }, {}); + + const criteriaIdsToUpdate = criteriaList.filter(({ externalTask }) => { + if (externalTask) { + return externalTask.id in updatedExternalTaskKeys; + } + + return false; + }); + + const interval = Math.max(Math.floor(tasksPeriod / criteriaList.length), updateMinInterval); + + const plannedCriteriaIds = jobs.reduce((acc, { data }) => { + if (castJobData(jobKind.criteriaToUpdate, data)) { + acc.add(data.id); + } + return acc; + }, new Set()); + + criteriaIdsToUpdate.forEach(async ({ id }, index) => { + if (!plannedCriteriaIds.has(id)) { + await createCriteriaToUpdate({ id }, index * interval); + } + }); +}; diff --git a/src/utils/worker/index.ts b/src/utils/worker/index.ts index 43fc097a2..0f0de9afa 100644 --- a/src/utils/worker/index.ts +++ b/src/utils/worker/index.ts @@ -4,14 +4,12 @@ import parser from 'cron-parser'; import { jobKind, jobState, defaultJobDelay } from './create'; import * as resolve from './resolve'; +import { log } from './utils'; const prisma = new PrismaClient(); const queueInterval = process.env.WORKER_JOBS_INTERVAL ? parseInt(process.env.WORKER_JOBS_INTERVAL, 10) : 3000; const retryLimit = process.env.WORKER_JOBS_RETRY ? parseInt(process.env.WORKER_JOBS_RETRY, 10) : 3; -// eslint-disable-next-line no-console -const log = (...rest: unknown[]) => console.log('[WORKER]:', ...rest); - log('Worker started successfully'); const getNextJob = async (state: jobState, exclude: string[]) => { @@ -44,6 +42,7 @@ const getNextJob = async (state: jobState, exclude: string[]) => { const iterateJobQueue = async (state: jobState, cb: (job: Job) => Promise): Promise => { const watchedIds: string[] = []; + // eslint-disable-next-line no-constant-condition while (true) { // eslint-disable-next-line no-await-in-loop const job = await getNextJob(state, watchedIds); @@ -63,6 +62,7 @@ const iterateJobQueue = async (state: jobState, cb: (job: Job) => Promise) const worker = async () => { try { const completedCount = await iterateJobQueue(jobState.completed, async (job) => { + log(`completed: ${job.id} - ${job.kind}`); setTimeout(async () => { if (job.cron) { log(`plan cron ${job.id}`); @@ -93,14 +93,21 @@ const worker = async () => { currentDate: new Date(job.updatedAt), }); - if (Number(interval.next().toDate()) > Date.now() && !job.force) { + const nextCronIntervalInMinutes = Math.floor(Number(interval.next().toDate()) / 1000 / 60); + const nowToMinutes = Math.floor(Date.now() / 1000 / 60); + + if (nextCronIntervalInMinutes > nowToMinutes && !job.force) { await planJob(); return; } } - if (job.delay && Date.now() - new Date(job.createdAt).valueOf() < job.delay) { + if ( + job.delay && + (Date.now() - new Date(job.createdAt).valueOf() < job.delay || + Date.now() - new Date(job.updatedAt).valueOf() < job.delay) + ) { await planJob(); return; diff --git a/src/utils/worker/resolve.ts b/src/utils/worker/resolve.ts index 32d4fa684..7a66ec767 100644 --- a/src/utils/worker/resolve.ts +++ b/src/utils/worker/resolve.ts @@ -4,6 +4,7 @@ import * as emailTemplates from './mail/templates'; import { sendMail } from './mail'; import { JobDataMap } from './create'; import { goalPingJob } from './goalPingJob'; +import { externalTaskCheckJob, externalTasksJob } from './externalTasksJob'; export const email = async ({ template, data }: JobDataMap['email']) => { const renderedTemplate = await emailTemplates[template](data); @@ -11,10 +12,15 @@ export const email = async ({ template, data }: JobDataMap['email']) => { }; export const cron = async ({ template }: JobDataMap['cron']) => { - if (template === 'goalPing') { - goalPingJob(); - } else { - throw new Error('No supported cron jobs'); + switch (template) { + case 'externalTaskCheck': + externalTaskCheckJob(); + break; + case 'goalPing': + goalPingJob(); + break; + default: + throw new Error('No supported cron jobs'); } }; @@ -27,3 +33,7 @@ export const comment = async ({ activityId, description, goalId }: JobDataMap['c shouldUpdateGoal: false, }); }; + +export const criteriaToUpdate = async ({ id }: JobDataMap['criteriaToUpdate']) => { + await externalTasksJob(id); +}; diff --git a/src/utils/worker/utils.ts b/src/utils/worker/utils.ts new file mode 100644 index 000000000..2fac2cbb3 --- /dev/null +++ b/src/utils/worker/utils.ts @@ -0,0 +1,2 @@ +// eslint-disable-next-line no-console +export const log = (...rest: unknown[]) => console.log('[WORKER]:', ...rest.concat(`at ${new Date().toUTCString()}`)); diff --git a/trpc/router/goal.ts b/trpc/router/goal.ts index c85e39c5b..5ebb2486d 100644 --- a/trpc/router/goal.ts +++ b/trpc/router/goal.ts @@ -1518,18 +1518,33 @@ export const goal = router({ const isPositiveStatus = jiraService.positiveStatuses?.includes(updatedTask.status.name) || false; if (isPositiveStatus) { - await prisma.goalAchieveCriteria.update({ - where: { id: currentCriteria.id }, + // needs update all associated criteria + await prisma.goalAchieveCriteria.updateMany({ + where: { externalTaskId: currentCriteria.externalTask.id, deleted: { not: true } }, data: { isDone: true, }, }); - await recalculateCriteriaScore(currentCriteria.goalId) - .recalcCurrentGoalScore() - .recalcLinkedGoalsScores() - .recalcAverageProjectScore() - .run(); + const goalIdsToUpdate = await prisma.goalAchieveCriteria.findMany({ + where: { + externalTaskId: currentCriteria.externalTask.id, + deleted: { not: true }, + }, + select: { + goalId: true, + }, + }); + + await goalIdsToUpdate.reduce((promiseAcc, { goalId }) => { + return promiseAcc.then( + recalculateCriteriaScore(goalId).makeChain( + 'recalcCurrentGoalScore', + 'recalcLinkedGoalsScores', + 'recalcAverageProjectScore', + ).run, + ); + }, Promise.resolve()); returnMessage = tr('Jira task is completed'); } else {