Skip to content

Commit

Permalink
Rewrite links after google upload
Browse files Browse the repository at this point in the history
  • Loading branch information
@ggodlewski
ggodlewski committed Sep 19, 2023
1 parent b30dda3 commit 8753121
Show file tree
Hide file tree
Showing 6 changed files with 161 additions and 43 deletions.
3 changes: 3 additions & 0 deletions src/LinkTranslator.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ export function convertToRelativeSvgPath(localPath, basePath) {
}

export function convertToAbsolutePath(fullPath: string, relativePath: string) {
if (relativePath.indexOf('://') > -1) {
return '';
}
if (fullPath.startsWith('/')) {
fullPath = '/' + fullPath;
}
Expand Down
152 changes: 119 additions & 33 deletions src/containers/google_folder/UploadContainer.ts
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,27 @@
import winston from 'winston';
import {fileURLToPath} from 'url';
import htmlparser2 from 'htmlparser2';
import domutils from 'domutils';
import {Element} from 'domhandler';
import render from 'dom-serializer';

import {FileId} from '../../model/model';
import {MimeTypes} from '../../model/GoogleFile';
import {LocalFile} from '../../model/LocalFile';
import {Container, ContainerConfig, ContainerConfigArr, ContainerEngine} from '../../ContainerEngine';
import {GoogleDriveService} from '../../google/GoogleDriveService';
import {UserConfigService} from './UserConfigService';
import {FileContentService} from '../../utils/FileContentService';
import {getContentFileService} from '../transform/utils';
import {DirectoryScanner} from '../transform/DirectoryScanner';
import {getDesiredPath} from '../transform/LocalFilesGenerator';
import {FileId} from '../../model/model';
import {LocalFile} from '../../model/LocalFile';
import {markdownToHtml} from '../../google/markdownToHtml';
import {convertToAbsolutePath} from '../../LinkTranslator';

const __filename = fileURLToPath(import.meta.url);

interface FileToUpload {
performRewrite?: string;
path: string;
file: LocalFile;
parent: FileId;
Expand All @@ -27,6 +35,8 @@ export class UploadContainer extends Container {
private userConfigService: UserConfigService;
private generatedFileService: FileContentService;

private pathToIdMap = {};

constructor(public readonly params: ContainerConfig, public readonly paramsArr: ContainerConfigArr = {}) {
super(params, paramsArr);
}
Expand Down Expand Up @@ -55,18 +65,50 @@ export class UploadContainer extends Container {
let cnt = 0;
for (const entry of files) {
const file = entry.file;
if (file.id === 'TO_FILL') {
cnt++;
switch (file.mimeType) {
case MimeTypes.IMAGE_SVG:
case MimeTypes.MARKDOWN:
if (file.id === 'TO_FILL') {
cnt++;
}
break;
}
}

if (cnt > 0) {
const ids = await this.googleDriveService.generateIds(access_token, cnt);
for (const entry of files) {
const file = entry.file;
if (file.id === 'TO_FILL') {
file.id = ids.splice(0, 1)[0];
switch (file.mimeType) {
case MimeTypes.IMAGE_SVG:
case MimeTypes.MARKDOWN:
if (file.id === 'TO_FILL') {
file.id = ids.splice(0, 1)[0];
}
break;
}
}
}
}

async updateLinks(driveIdId: FileId, dirFileService: FileContentService, files: FileToUpload[]) {
// Second pass because of error: Generated IDs are not supported for Docs Editors formats
const access_token = this.params.access_token;

for (const entry of files) {
const file = entry.file;

try {
switch (file.mimeType) {
case MimeTypes.MARKDOWN:
if (file.id && file.id !== 'TO_FILL' && entry.performRewrite) {
const rewrittenHtml = await this.rewriteLinks(entry.performRewrite, entry.path);
const response = await this.googleDriveService.update(access_token, entry.parent, file.title, MimeTypes.HTML, rewrittenHtml, file.id);

Check notice

Code scanning / CodeQL

Unused variable, import, function or class Note

Unused variable response.
}
break;
}
} catch (err) {
this.logger.error('[' + entry.path + ']: ' + err.message);
}
}
}
Expand All @@ -77,21 +119,34 @@ export class UploadContainer extends Container {
for (const entry of files) {
const file = entry.file;

switch (file.mimeType) {
case MimeTypes.FOLDER_MIME:
break;
case MimeTypes.MARKDOWN:
if (file.id && file.id !== 'TO_FILL') {
const content = await dirFileService.readBuffer(entry.path);
const response = await this.googleDriveService.upload(access_token, entry.parent, file.title, file.mimeType, content, file.id);
}
break;
case MimeTypes.IMAGE_SVG:
if (file.id && file.id !== 'TO_FILL') {
const content = await dirFileService.readBuffer(entry.path);
const response = await this.googleDriveService.upload(access_token, entry.parent, file.title, file.mimeType, content, file.id);
}
break;
try {
switch (file.mimeType) {
case MimeTypes.FOLDER_MIME:
break;
case MimeTypes.MARKDOWN:
if (file.id && file.id !== 'TO_FILL') {
const content = await dirFileService.readBuffer(entry.path);
const html = await markdownToHtml(content);
entry.performRewrite = html;
const buffer = Buffer.from(new TextEncoder().encode(html));
const response = await this.googleDriveService.upload(access_token, entry.parent, file.title, MimeTypes.HTML, buffer);
file.id = response.id;
}
break;
case MimeTypes.IMAGE_SVG:
if (file.id && file.id !== 'TO_FILL') {
const content = await dirFileService.readBuffer(entry.path);
const response = await this.googleDriveService.upload(access_token, entry.parent, file.title, file.mimeType, content, file.id);
file.id = response.id;
}
break;
}
} catch (err) {
this.logger.error('[' + entry.path + ']: ' + err.message);
}

if (file.id && file.id !== 'TO_FILL') {
this.pathToIdMap[entry.path] = file.id;
}
}
}
Expand Down Expand Up @@ -119,6 +174,7 @@ export class UploadContainer extends Container {
}

for (const file of Object.values(files)) {
const fullPath = parentPath + '/' + file.fileName;
switch (file.mimeType) {
case MimeTypes.FOLDER_MIME:
if (file.id === 'TO_FILL') {
Expand All @@ -129,17 +185,17 @@ export class UploadContainer extends Container {
file.id = map[file.title].id;
}
}
await this.uploadDir(file.id, await dirFileService.getSubFileService(file.fileName), parentPath + '/' + file.fileName);
await this.uploadDir(file.id, await dirFileService.getSubFileService(file.fileName), fullPath);
break;
case MimeTypes.MARKDOWN:
if (map[getDesiredPath(file.title)]) {
file.id = map[getDesiredPath(file.title)].id;
}
{
const content = await dirFileService.readBuffer(file.fileName);
const response = await this.googleDriveService.upload(access_token, folderId, file.title, file.mimeType, content);
file.id = response.id;
}
retVal.push({
path: parentPath + '/' + file.fileName,
file,
parent: folderId
});
break;
case MimeTypes.IMAGE_SVG:
if (map[getDesiredPath(file.title)]) {
Expand All @@ -151,14 +207,11 @@ export class UploadContainer extends Container {
file,
parent: folderId
});

{
const content = await dirFileService.readBuffer(file.fileName);
const response = await this.googleDriveService.upload(access_token, folderId, file.title, file.mimeType, content);
file.id = response.id;
}
break;
}

if (file.id && file.id !== 'TO_FILL') {
this.pathToIdMap[fullPath] = file.id;
}
}

Expand All @@ -171,13 +224,46 @@ export class UploadContainer extends Container {
throw new Error('Content subdirectory must be set and start with /');
}

this.pathToIdMap = {};

const contentFileService = await getContentFileService(this.generatedFileService, this.userConfigService);
const files = await this.uploadDir(this.params.folderId, contentFileService, '');
await this.addIds(this.params.folderId, contentFileService, files);
await this.uploadFiles(this.params.folderId, contentFileService, files);
await this.updateLinks(this.params.folderId, contentFileService, files);
}

onProgressNotify(callback: ({total, completed, warnings}: { total?: number; completed?: number; warnings?: number }) => void) {
this.progressNotifyCallback = callback;
}

private async rewriteLinks(html: string, entryPath: string): Promise<Buffer> {
const dom = htmlparser2.parseDocument(html);

const links = domutils.findAll((elem: Element) => {
return ['a'].includes(elem.tagName) && !!elem.attribs?.href;
}, dom.childNodes);
const images = domutils.findAll((elem: Element) => {
return ['img'].includes(elem.tagName) && !!elem.attribs?.src;
}, dom.childNodes);

for (const elem of links) {
const targetPath = convertToAbsolutePath(entryPath, elem.attribs.href);
if (!targetPath) {
continue;
}
if (this.pathToIdMap[targetPath]) {
elem.attribs.href = 'https://drive.google.com/open?id=' + this.pathToIdMap[targetPath];
}
}
for (const elem of images) {
const targetPath = convertToAbsolutePath(entryPath, elem.attribs.src);
if (this.pathToIdMap[targetPath]) {
elem.attribs.src = 'https://drive.google.com/open?id=' + this.pathToIdMap[targetPath];
}
}

const rewrittenHtml = render(dom);
return Buffer.from(new TextEncoder().encode(rewrittenHtml));
}
}
1 change: 0 additions & 1 deletion src/containers/server/routes/GoogleDriveController.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ export class GoogleDriveController extends Controller {
async getUpload(@RouteParamUser() user, @RouteParamPath('driveId') driveId: string) {
const serverUrl = process.env.DOMAIN;

console.log('driveId', driveId);
const state = new URLSearchParams(filterParams({
uploadDrive: 1,
driveId: driveId !== 'none' ? (driveId || '') : ''
Expand Down
35 changes: 32 additions & 3 deletions src/google/GoogleDriveService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,10 +332,8 @@ export class GoogleDriveService {
// 'mimeType': MimeTypes.DRAWING_MIME, // Error: Bad Request
googleMimeType = MimeTypes.IMAGE_SVG;
break;
case MimeTypes.MARKDOWN:
case MimeTypes.HTML:
googleMimeType = MimeTypes.DOCUMENT_MIME;
buffer = Buffer.from(new TextEncoder().encode(await markdownToHtml(buffer)));
mimeType = 'text/html';
break;
}

Expand All @@ -356,4 +354,35 @@ export class GoogleDriveService {
supportsAllDrives: true
}, formData);
}

async update(accessToken: string, folderId: FileId, name: string, mimeType: string, buffer: Buffer, fileId: FileId) {
const url = `https://www.googleapis.com/upload/drive/v3/files/${fileId}`;

let googleMimeType = 'application/octet-stream';
switch (mimeType) {
case MimeTypes.IMAGE_SVG:
// 'mimeType': MimeTypes.DRAWING_MIME, // Error: Bad Request
googleMimeType = MimeTypes.IMAGE_SVG;
break;
case MimeTypes.HTML:
googleMimeType = MimeTypes.DOCUMENT_MIME;
break;
}

const metadata = {
name,
mimeType: googleMimeType,
// parents: [folderId],
fields: '*'
};

const formData = new FormData();
formData.append('Metadata', new Blob([JSON.stringify(metadata)], { type: 'application/json; charset=UTF-8' }) );
formData.append('Media', new Blob([buffer], { type: mimeType }), name);

return await driveFetchMultipart(this.quotaLimiter, accessToken, 'PATCH', url, {
uploadType: 'multipart',
supportsAllDrives: true
}, formData);
}
}
10 changes: 5 additions & 5 deletions src/google/driveFetch.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,15 +223,15 @@ export async function driveFetchMultipart(quotaLimiter: QuotaLimiter, accessToke
}
}

const after = `\r\n--${boundary}--`;
const after = `\n--${boundary}--`;
function generateMultipart(image: ArrayBuffer, mimetype) {
const source = new Uint8Array(image); // Wrap in view to get data

const before = [
`\r\n--${boundary}\r\n`,
`Content-Type: ${mimetype}\r\n`,
`Content-Length: ${source.byteLength}\r\n`,
'\r\n'
`\n--${boundary}\n`,
`Content-Type: ${mimetype}\n`,
`Content-Length: ${source.byteLength}\n`,
'\n'
].join('');

const size = before.length + source.byteLength;
Expand Down
3 changes: 2 additions & 1 deletion src/model/GoogleFile.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ export const MimeTypes = {
APPS_SCRIPT: 'application/vnd.google-apps.script',
MARKDOWN: 'text/x-markdown',
SHORTCUT: 'application/vnd.google-apps.shortcut',
IMAGE_SVG: 'image/svg+xml'
IMAGE_SVG: 'image/svg+xml',
HTML: 'text/html'
};
export const MimeToExt = {
'image/svg+xml': '.svg',
Expand Down

0 comments on commit 8753121

Please sign in to comment.