From ab3e3b6fe566cf7e764af3a0dcf106f0849edf8a Mon Sep 17 00:00:00 2001 From: Safwan Shaheer Date: Fri, 13 Oct 2023 15:07:35 +0600 Subject: [PATCH] Sync page content with sidebar script (#2763) --- .../updatePageContentForSync.spec.ts | 352 ++++++++++++++++++ .../conversions/updatePageContentForSync.ts | 217 +++++++++++ .../documentEvents/documentEvents.ts | 2 +- next.config.js | 9 +- .../2023_10_11_updatePageContentForSync.ts | 6 + scripts/updatePageContentForSync.ts | 71 ---- testing/prosemirror/builders.ts | 1 + 7 files changed, 579 insertions(+), 79 deletions(-) create mode 100644 lib/prosemirror/conversions/__tests__/updatePageContentForSync.spec.ts create mode 100644 lib/prosemirror/conversions/updatePageContentForSync.ts create mode 100644 scripts/migrations/2023_10_11_updatePageContentForSync.ts delete mode 100644 scripts/updatePageContentForSync.ts diff --git a/lib/prosemirror/conversions/__tests__/updatePageContentForSync.spec.ts b/lib/prosemirror/conversions/__tests__/updatePageContentForSync.spec.ts new file mode 100644 index 0000000000..a0ed796426 --- /dev/null +++ b/lib/prosemirror/conversions/__tests__/updatePageContentForSync.spec.ts @@ -0,0 +1,352 @@ +import { prisma } from '@charmverse/core/prisma-client'; +import { testUtilsPages } from '@charmverse/core/test'; +import { v4 } from 'uuid'; + +import { emptyDocument } from 'lib/prosemirror/constants'; +import { builders as _ } from 'testing/prosemirror/builders'; +import { generateUserAndSpaceWithApiToken } from 'testing/setupDatabase'; + +import { updatePageContentForSync } from '../updatePageContentForSync'; + +describe('updatePageContentForSync', () => { + it(`Should update page content and diffs by converting linked page nodes and adding missing nested pages`, async () => { + const { user, space } = await generateUserAndSpaceWithApiToken(undefined, false); + const childPage1Id = v4(); + const childPage1Path = `page-${v4()}`; + const linkedPage1 = await testUtilsPages.generatePage({ + spaceId: space.id, + createdBy: user.id + }); + + const linkedPage2 = await testUtilsPages.generatePage({ + spaceId: space.id, + createdBy: user.id + }); + + const pageContent = _.doc( + _.paragraph('Paragraph 1'), + _.page({ + id: childPage1Id, + type: 'page', + path: childPage1Path + }), + _.page({ + id: childPage1Id, + type: 'page', + path: childPage1Path + }), + _.paragraph('Paragraph 2'), + _.page({ + id: linkedPage1.id, + type: 'page', + path: linkedPage1.path + }), + _.paragraph('Paragraph 3'), + _.page({ + id: linkedPage2.id, + type: 'page', + path: linkedPage2.path + }), + _.linkedPage({ + id: linkedPage2.id, + type: 'page', + path: linkedPage2.path + }), + _.paragraph('Paragraph 4') + ).toJSON(); + + const parentPage = await testUtilsPages.generatePage({ + spaceId: space.id, + createdBy: user.id, + content: pageContent + }); + + const childPage1 = await testUtilsPages.generatePage({ + createdBy: user.id, + spaceId: space.id, + parentId: parentPage.id, + id: childPage1Id, + path: childPage1Path + }); + + const childPage2 = await testUtilsPages.generatePage({ + createdBy: user.id, + spaceId: space.id, + parentId: parentPage.id + }); + + const childPage3 = await testUtilsPages.generatePage({ + createdBy: user.id, + spaceId: space.id, + parentId: parentPage.id + }); + + await updatePageContentForSync({ spaceId: space.id }); + + const updatedParentPage = await prisma.page.findUniqueOrThrow({ + where: { + id: parentPage.id + }, + select: { + content: true, + diffs: { + select: { + data: true, + pageId: true, + version: true + } + }, + version: true + } + }); + + const pageDiffs = updatedParentPage.diffs; + + expect(updatedParentPage.content).toEqual( + _.doc( + _.p('Paragraph 1'), + _.page({ + id: childPage1.id, + path: childPage1.path, + type: childPage1.type + }), + _.linkedPage({ + id: childPage1.id, + type: childPage1.type, + path: childPage1.path + }), + _.paragraph('Paragraph 2'), + _.linkedPage({ + id: linkedPage1.id, + path: linkedPage1.path, + type: linkedPage1.type + }), + _.paragraph('Paragraph 3'), + _.linkedPage({ + id: linkedPage2.id, + path: linkedPage2.path, + type: linkedPage2.type + }), + _.linkedPage({ + id: linkedPage2.id, + path: linkedPage2.path, + type: linkedPage2.type + }), + _.paragraph('Paragraph 4'), + _.page({ + id: childPage2.id, + path: childPage2.path, + type: childPage2.type + }), + _.page({ + id: childPage3.id, + path: childPage3.path, + type: childPage3.type + }) + ).toJSON() + ); + + expect(updatedParentPage.version).toEqual(3); + + expect(pageDiffs).toStrictEqual([ + { + data: { + v: 1, + ds: [ + { + to: 15, + from: 14, + slice: { + content: [ + { + type: 'linkedPage', + attrs: { + id: childPage1.id, + path: childPage1.path, + type: childPage1.type, + track: [] + } + } + ] + }, + stepType: 'replace' + }, + { + from: 28, + to: 29, + slice: { + content: [ + { + type: 'linkedPage', + attrs: { + id: linkedPage1.id, + path: linkedPage1.path, + type: 'page', + track: [] + } + } + ] + }, + stepType: 'replace' + }, + { + from: 42, + to: 43, + slice: { + content: [ + { + type: 'linkedPage', + attrs: { + id: linkedPage2.id, + path: linkedPage2.path, + type: 'page', + track: [] + } + } + ] + }, + stepType: 'replace' + } + ], + cid: 0, + rid: 0, + type: 'diff' + }, + pageId: parentPage.id, + version: 1 + }, + { + data: { + v: 2, + ds: [ + { + to: 57, + from: 57, + slice: { + content: [childPage2, childPage3].map((childPage) => ({ + type: 'page', + attrs: { + id: childPage.id, + path: childPage.path, + type: 'page', + track: [] + } + })) + }, + stepType: 'replace' + } + ], + cid: 0, + rid: 0, + type: 'diff' + }, + pageId: parentPage.id, + version: 2 + } + ]); + }); + + it(`Should not update page content or add diffs if all the linked page nodes are correct and there are no missing nested pages`, async () => { + const { user, space } = await generateUserAndSpaceWithApiToken(undefined, false); + const parentPage = await testUtilsPages.generatePage({ + spaceId: space.id, + createdBy: user.id, + content: emptyDocument + }); + + const childPage1 = await testUtilsPages.generatePage({ + createdBy: user.id, + spaceId: space.id, + parentId: parentPage.id + }); + + const childPage2 = await testUtilsPages.generatePage({ + createdBy: user.id, + spaceId: space.id, + parentId: parentPage.id + }); + + const childPage3 = await testUtilsPages.generatePage({ + createdBy: user.id, + spaceId: space.id, + parentId: parentPage.id + }); + + const linkedPage1 = await testUtilsPages.generatePage({ + spaceId: space.id, + createdBy: user.id + }); + + const linkedPage2 = await testUtilsPages.generatePage({ + spaceId: space.id, + createdBy: user.id + }); + + const pageContent = _.doc( + _.paragraph('Paragraph 1'), + _.page({ + id: childPage1.id, + type: 'page', + path: childPage1.path + }), + _.paragraph('Paragraph 2'), + _.linkedPage({ + id: linkedPage1.id, + type: 'page', + path: linkedPage1.path + }), + _.paragraph('Paragraph 3'), + _.linkedPage({ + id: linkedPage2.id, + type: 'page', + path: linkedPage2.path + }), + _.linkedPage({ + id: linkedPage2.id, + type: 'page', + path: linkedPage2.path + }), + _.paragraph('Paragraph 4'), + _.page({ + id: childPage2.id, + type: 'page', + path: childPage2.path + }), + _.page({ + id: childPage3.id, + type: 'page', + path: childPage3.path + }) + ).toJSON(); + + await prisma.page.update({ + where: { + id: parentPage.id + }, + data: { + content: pageContent + } + }); + + await updatePageContentForSync({ pagesRetrievedPerQuery: 2, spaceId: space.id }); + + const updatedParentPage = await prisma.page.findUniqueOrThrow({ + where: { + id: parentPage.id + }, + select: { + content: true, + diffs: true, + version: true + } + }); + + const pageDiffs = updatedParentPage.diffs; + + expect(pageDiffs.length).toEqual(0); + + expect(updatedParentPage.content).toEqual(pageContent); + + expect(updatedParentPage.version).toEqual(1); + }); +}); diff --git a/lib/prosemirror/conversions/updatePageContentForSync.ts b/lib/prosemirror/conversions/updatePageContentForSync.ts new file mode 100644 index 0000000000..c101c51751 --- /dev/null +++ b/lib/prosemirror/conversions/updatePageContentForSync.ts @@ -0,0 +1,217 @@ +import { log } from '@charmverse/core/log'; +import type { PageMeta } from '@charmverse/core/pages'; +import { pageTree } from '@charmverse/core/pages/utilities'; +import { PageDiff, Prisma, prisma } from '@charmverse/core/prisma-client'; +import { Fragment, Slice } from 'prosemirror-model'; +import { replaceStep } from 'prosemirror-transform'; + +import { applyStepsToNode } from 'lib/prosemirror/applyStepsToNode'; +import { getNodeFromJson } from 'lib/prosemirror/getNodeFromJson'; +import type { PageContent } from 'lib/prosemirror/interfaces'; +import type { ProsemirrorJSONStep } from 'lib/websockets/documentEvents/interfaces'; + +export function recurseDocument(content: PageContent, cb: (node: PageContent) => void) { + function recurse(node: PageContent) { + cb(node); + + if (node.content) { + node.content.forEach((childNode) => { + recurse(childNode); + }); + } + } + + recurse(content); +} + +export async function updatePageContentForSync( + config: { pagesRetrievedPerQuery?: number; spaceId?: string } = { + pagesRetrievedPerQuery: 100 + } +) { + const { pagesRetrievedPerQuery = 100, spaceId } = config; + const pageWhereInput: Prisma.PageWhereInput = { + content: { + not: Prisma.DbNull + }, + spaceId, + type: { + notIn: ['board', 'board_template', 'inline_board', 'inline_linked_board', 'linked_board'] + } + }; + + let completedPages = 0; + const totalPages = await prisma.page.count({ + where: pageWhereInput + }); + let skip = 0; + + while (skip < totalPages) { + const pages = await prisma.page.findMany({ + where: pageWhereInput, + select: { + id: true, + createdBy: true, + content: true + }, + orderBy: { + createdAt: 'asc' + }, + skip, + take: pagesRetrievedPerQuery + }); + + for (const page of pages) { + const { createdBy, id } = page; + try { + const childPages = await prisma.page.findMany({ + where: { + parentId: page.id + }, + select: { + id: true, + index: true, + createdAt: true, + path: true, + type: true + }, + orderBy: { + createdAt: 'asc' + } + }); + + const childPageIds = pageTree.sortNodes(childPages as PageMeta[]).map((childPage) => childPage.id); + const pageContent = page.content as PageContent; + const nestedPageIds: Set = new Set(); + let doc = getNodeFromJson(pageContent); + const linkedPageConversionSteps: ProsemirrorJSONStep[] = []; + doc.nodesBetween(0, doc.content.size, (node, pos) => { + switch (node.type.name) { + case 'page': { + const pageId = node.attrs.id; + if (!pageId) { + return false; + } + if ((pageId && !childPageIds.includes(pageId)) || nestedPageIds.has(pageId)) { + const jsonNode = node.toJSON(); + jsonNode.type = 'linkedPage'; + const newNode = getNodeFromJson(jsonNode); + const linkedPageConversionStep = replaceStep( + doc, + pos, + pos + node.nodeSize, + new Slice(Fragment.from(newNode), 0, 0) + ); + if (linkedPageConversionStep) { + linkedPageConversionSteps.push(linkedPageConversionStep.toJSON()); + } + } else if (pageId) { + nestedPageIds.add(pageId); + } + + return false; + } + default: + return true; + } + }); + + if (linkedPageConversionSteps.length) { + doc = applyStepsToNode(linkedPageConversionSteps, doc); + } + + const childPagesNotInDocument = childPages.filter((childPage) => !nestedPageIds.has(childPage.id)); + const nestedPageAppendStep: ProsemirrorJSONStep | null = + childPagesNotInDocument.length === 0 + ? null + : { + from: doc.content.size, + stepType: 'replace', + slice: { + content: childPagesNotInDocument.map((childPage) => ({ + type: 'page', + attrs: { + id: childPage.id, + path: childPage.path, + type: childPage.type, + track: [] + } + })) + }, + to: doc.content.size + }; + + if (nestedPageAppendStep) { + doc = applyStepsToNode([nestedPageAppendStep], doc); + } + + const newContent = doc.toJSON(); + const parentPage = await prisma.page.findUniqueOrThrow({ + where: { + id: page.id + }, + select: { + version: true + } + }); + let version = parentPage.version; + const pageDiffs: Prisma.PageDiffCreateManyInput[] = []; + if (linkedPageConversionSteps.length) { + pageDiffs.push({ + createdBy, + data: { + rid: 0, + type: 'diff', + v: version, + cid: 0, + ds: linkedPageConversionSteps + }, + pageId: id, + version + }); + version += 1; + } + + if (nestedPageAppendStep) { + pageDiffs.push({ + createdBy, + data: { + rid: 0, + type: 'diff', + v: version, + cid: 0, + ds: [nestedPageAppendStep] + }, + pageId: id, + version + }); + version += 1; + } + + if (pageDiffs.length) { + await prisma.$transaction([ + prisma.pageDiff.createMany({ + data: pageDiffs + }), + prisma.page.update({ + where: { + id + }, + data: { + content: newContent, + version + } + }) + ]); + } + completedPages += 1; + log.info(`Complete updating page [${completedPages}/${totalPages}]: ${page.id}`); + } catch (error) { + log.error(`Failed to update page ${page.id}`, { error, skip }); + throw new Error(); + } + } + + skip += pagesRetrievedPerQuery; + } +} diff --git a/lib/websockets/documentEvents/documentEvents.ts b/lib/websockets/documentEvents/documentEvents.ts index b9162b8f2b..c8add954f7 100644 --- a/lib/websockets/documentEvents/documentEvents.ts +++ b/lib/websockets/documentEvents/documentEvents.ts @@ -14,7 +14,7 @@ import { extractPreviewImage } from 'lib/prosemirror/extractPreviewImage'; import { getNodeFromJson } from 'lib/prosemirror/getNodeFromJson'; import type { PageContent } from 'lib/prosemirror/interfaces'; import { WebhookEventNames } from 'lib/webhookPublisher/interfaces'; -import { publishBountyEvent, publishDocumentEvent, publishProposalEvent } from 'lib/webhookPublisher/publishEvent'; +import { publishDocumentEvent } from 'lib/webhookPublisher/publishEvent'; import type { AuthenticatedSocketData } from '../authentication'; import type { AbstractWebsocketBroadcaster } from '../interfaces'; diff --git a/next.config.js b/next.config.js index 6b8420b68d..b6e37876ea 100644 --- a/next.config.js +++ b/next.config.js @@ -1,11 +1,6 @@ /* eslint-disable @typescript-eslint/no-var-requires */ -const fs = require('fs'); -const path = require('node:path'); - const BundleAnalyzer = require('@next/bundle-analyzer'); const next = require('next/dist/lib/is-serializable-props'); -const uuid = require('uuid'); -const webpack = require('webpack'); const esmModules = require('./next.base').esmModules; @@ -183,10 +178,10 @@ const config = { return { ..._entry, cron: './background/cron.ts', - websockets: './background/initWebsockets.ts' + websockets: './background/initWebsockets.ts', + updatePageContentForSync: './scripts/migrations/2023_10_11_updatePageContentForSync.ts' // countSpaceData: './scripts/countSpaceData.ts', // importFromDiscourse: './scripts/importFromDiscourse.ts', - // updatePageContentForSync: './scripts/updatePageContentForSync.ts' }; }); }; diff --git a/scripts/migrations/2023_10_11_updatePageContentForSync.ts b/scripts/migrations/2023_10_11_updatePageContentForSync.ts new file mode 100644 index 0000000000..616bc42f2b --- /dev/null +++ b/scripts/migrations/2023_10_11_updatePageContentForSync.ts @@ -0,0 +1,6 @@ +import { updatePageContentForSync } from 'lib/prosemirror/conversions/updatePageContentForSync'; + +updatePageContentForSync({ + spaceId: "cce0ff04-54c3-4f35-bc67-b5548b6ebbc4", + pagesRetrievedPerQuery: 5 +}); \ No newline at end of file diff --git a/scripts/updatePageContentForSync.ts b/scripts/updatePageContentForSync.ts deleted file mode 100644 index dcb6e39231..0000000000 --- a/scripts/updatePageContentForSync.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { prisma } from '@charmverse/core/prisma-client'; -import { emptyDocument } from 'lib/prosemirror/constants'; -import { getNodeFromJson } from 'lib/prosemirror/getNodeFromJson'; -import { PageContent } from 'lib/prosemirror/interfaces'; - -export async function updatePageContentForSync() { - const pages = await prisma.page.findMany({ - select: { - id: true, - content: true, - } - }) - - for (const page of pages) { - const pageContent = (page.content ?? emptyDocument) as PageContent; - const nestedPageIds: string[] = []; - const pageContentNode = getNodeFromJson(pageContent); - pageContentNode.forEach((node) => { - if (node.type.name === 'page' && node.attrs) { - nestedPageIds.push(node.attrs.id); - } - }); - const childPages = await prisma.page.findMany({ - where: { - parentId: page.id, - }, - select: { - id: true, - } - }); - const childPageIds = childPages.map((childPage) => childPage.id); - - // Loop through all child pages and check if they are added to the page content - childPageIds.forEach(childPageId => { - if (pageContent.content && !nestedPageIds.includes(childPageId)) { - pageContent.content.push({ - type: "page", - attrs: { - id: childPageId, - } - }) - - nestedPageIds.push(childPageId); - } - }) - - await prisma.page.update({ - where: { - id: page.id, - }, - data: { - content: pageContent, - } - }) - - for (const nestedPageId of nestedPageIds) { - if (!childPageIds.includes(nestedPageId)) { - await prisma.page.update({ - where: { - id: nestedPageId, - }, - data: { - parentId: page.id, - } - }) - } - } - } -} - -updatePageContentForSync(); diff --git a/testing/prosemirror/builders.ts b/testing/prosemirror/builders.ts index 975cd71314..ff883d2e56 100644 --- a/testing/prosemirror/builders.ts +++ b/testing/prosemirror/builders.ts @@ -44,6 +44,7 @@ export type NodeType = | 'italic' | 'label' | 'link' + | 'linkedPage' | 'list_item' | 'listItem' | 'mention'