From e57378e65a050f973b48d0f3c036826ac216fa44 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Thu, 10 Sep 2020 17:35:30 +0200 Subject: [PATCH 1/3] Fix watch of root folder Resolves: #83 --- src/MainService.js | 3 +-- src/google/QuotaLimiter.js | 3 +++ src/plugins/WatchChangesPlugin.js | 6 +++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/MainService.js b/src/MainService.js index f33ca6a3..c0a987b0 100644 --- a/src/MainService.js +++ b/src/MainService.js @@ -111,11 +111,10 @@ export class MainService { await this.emitThanAwait('main:init', this.params, [ 'google_api:initialized', 'files_structure:initialized' ]); await this.emitThanAwait('main:fetch_watch_token', {}, [ 'watch:token_ready' ]); - this.eventBus.emit('main:run_list_root'); - this.eventBus.on('list_root:done', () => { this.eventBus.emit('main:run_watch'); }); + this.eventBus.emit('main:run_list_root'); await new Promise(() => {}); break; diff --git a/src/google/QuotaLimiter.js b/src/google/QuotaLimiter.js index d6bc3312..a57ba18b 100644 --- a/src/google/QuotaLimiter.js +++ b/src/google/QuotaLimiter.js @@ -39,6 +39,9 @@ export class QuotaLimiter { if (this.oldCounter === this.counter) { return; } + if (this.jobs.length < 8) { // Don't speed up trivial queue + return; + } const newLimits = {}; newLimits.queries = this.currentLimit.queries + 1; diff --git a/src/plugins/WatchChangesPlugin.js b/src/plugins/WatchChangesPlugin.js index 57b1bd73..e5fc4cad 100644 --- a/src/plugins/WatchChangesPlugin.js +++ b/src/plugins/WatchChangesPlugin.js @@ -42,6 +42,7 @@ export class WatchChangesPlugin extends BasePlugin { async watch() { console.log('Watching changes'); + const rootFolderId = this.googleDriveService.urlToFolderId(this.drive_config['drive']); await new Promise(() => setInterval(async () => { try { @@ -50,6 +51,9 @@ export class WatchChangesPlugin extends BasePlugin { const changedFiles = result.files.filter(file => { let retVal = false; file.parents.forEach((parentId) => { + if (parentId === rootFolderId) { + retVal = true; + } if (this.filesStructure.containsFile(parentId)) { retVal = true; } @@ -63,7 +67,7 @@ export class WatchChangesPlugin extends BasePlugin { this.startTrackToken = result.token; // eslint-disable-line require-atomic-updates console.log('Pulled latest changes'); } else { - console.log('No changes detected. Sleeping for 10 seconds.' + this.startTrackToken); + console.log('No changes detected. Sleeping for 10 seconds.'); } } catch (e) { From 9fddb60049094712efb978efaaf289ac68c1c189 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Thu, 10 Sep 2020 22:29:27 +0200 Subject: [PATCH 2/3] Improve watch --- src/MainService.ts | 8 +- src/google/GoogleDriveService.ts | 113 ++++++++++-------- src/index.js | 3 +- src/plugins/DownloadPlugin.ts | 11 +- ...ChangesPlugin.js => WatchChangesPlugin.ts} | 29 ++++- ...atchMTimePlugin.js => WatchMTimePlugin.ts} | 28 ++++- 6 files changed, 127 insertions(+), 65 deletions(-) rename src/plugins/{WatchChangesPlugin.js => WatchChangesPlugin.ts} (69%) rename src/plugins/{WatchMTimePlugin.js => WatchMTimePlugin.ts} (62%) diff --git a/src/MainService.ts b/src/MainService.ts index 24980752..abf1f309 100644 --- a/src/MainService.ts +++ b/src/MainService.ts @@ -28,7 +28,8 @@ export interface CliParams { drive_id: string; drive: string; command: string; - debug: boolean; + watch_mode: string; + debug: string[]; } export class MainService { @@ -40,7 +41,7 @@ export class MainService { this.command = this.params.command; this.eventBus = new EventEmitter(); this.eventBus.setMaxListeners(0); - if (params.debug) { + if (params.debug.indexOf('main') > -1) { this.attachDebug(); } } @@ -136,6 +137,9 @@ export class MainService { await this.emitThanAwait('main:init', this.params, [ 'google_api:initialized', 'files_structure:initialized' ]); await this.emitThanAwait('main:fetch_watch_token', {}, [ 'watch:token_ready' ]); + this.eventBus.on('transform:dirty', () => { + this.eventBus.emit('download:retry'); + }); this.eventBus.on('list_root:done', () => { this.eventBus.emit('main:run_watch'); }); diff --git a/src/google/GoogleDriveService.ts b/src/google/GoogleDriveService.ts index 8d8c1e39..c1eb52de 100644 --- a/src/google/GoogleDriveService.ts +++ b/src/google/GoogleDriveService.ts @@ -3,6 +3,7 @@ import slugify from 'slugify'; import {google} from 'googleapis'; import {File, FilesStructure} from '../storage/FilesStructure'; +import {GoogleDocsServiceError} from "./GoogleDocsService"; const MAX_FILENAME_LENGTH = 100; @@ -14,6 +15,15 @@ export function getDesiredPath(name) { return name; } +export interface ApiFile extends File { + parents: string[]; +} + +export interface Changes { + token: string; + files: ApiFile[]; +} + function removeDuplicates(files) { const retVal = []; @@ -175,63 +185,60 @@ export class GoogleDriveService { return res.data.startPageToken; } - watchChanges(auth, pageToken, driveId) { - return new Promise(async (resolve, reject) => { /* eslint-disable-line no-async-promise-executor */ - const drive = google.drive({ version: 'v3', auth }); - - const params = { - pageToken: pageToken, - supportsAllDrives: true, - includeItemsFromAllDrives: true, - includeCorpusRemovals: true, - fields: 'newStartPageToken, changes( file(id, name, mimeType, modifiedTime, size, md5Checksum, lastModifyingUser, parents, version))', - drives: undefined - }; + async watchChanges(auth, pageToken, driveId): Promise { + const drive = google.drive({ version: 'v3', auth }); - if (driveId) { - params.drives = [ driveId ]; - } + const params = { + pageToken: pageToken, + supportsAllDrives: true, + includeItemsFromAllDrives: true, + includeCorpusRemovals: true, + fields: 'newStartPageToken, changes( file(id, name, mimeType, modifiedTime, size, md5Checksum, lastModifyingUser, parents, version))', + drives: undefined + }; - try { - const res = await drive.changes.list(params); - - const files = res.data.changes - .map(change => change.file) - .map((apiFile) => { - const file = apiFile; - - file.desiredLocalPath = getDesiredPath(file.name); - if (apiFile.lastModifyingUser) { - file.lastAuthor = apiFile.lastModifyingUser.displayName; - delete apiFile.lastModifyingUser; - } - - if (file.desiredLocalPath.length > MAX_FILENAME_LENGTH) { - file.desiredLocalPath = file.desiredLocalPath.substr(0, MAX_FILENAME_LENGTH); - } - - switch (file.mimeType) { - case 'application/vnd.google-apps.drawing': - file.desiredLocalPath += '.svg'; - break; - case 'application/vnd.google-apps.document': - file.desiredLocalPath += '.md'; - break; - } - - return file; - }); + if (driveId) { + params.drives = [ driveId ]; + } + + try { + const res = await drive.changes.list(params); + + const files = res.data.changes + .map(change => change.file) + .map(apiFile => { + const file = apiFile; + file.desiredLocalPath = getDesiredPath(file.name); + if (apiFile.lastModifyingUser) { + file.lastAuthor = apiFile.lastModifyingUser.displayName; + delete apiFile.lastModifyingUser; + } - resolve({ - token: res.data.newStartPageToken, - files: files + if (file.desiredLocalPath.length > MAX_FILENAME_LENGTH) { + file.desiredLocalPath = file.desiredLocalPath.substr(0, MAX_FILENAME_LENGTH); + } + + switch (file.mimeType) { + case 'application/vnd.google-apps.drawing': + file.desiredLocalPath += '.svg'; + break; + case 'application/vnd.google-apps.document': + file.desiredLocalPath += '.md'; + break; + } + + return file; }); - } catch (err) { - reject(new GoogleDriveServiceError('Error watching changes', { - isQuotaError: err.isQuotaError - })); - } - }); + + return { + token: res.data.newStartPageToken, + files: files + }; + } catch (err) { + throw new GoogleDriveServiceError('Error watching changes', { + isQuotaError: err.isQuotaError + }); + } } async _listFiles(auth, context, modifiedTime, nextPageToken?) { diff --git a/src/index.js b/src/index.js index 7bb02395..89c2f4f2 100644 --- a/src/index.js +++ b/src/index.js @@ -60,12 +60,13 @@ async function index() { params['link_mode'] = argv['link_mode'] || 'mdURLs'; params['flat_folder_structure'] = !!argv['without-folder-structure']; - params['debug'] = !!argv['debug']; + params['debug'] = (argv['debug'] || '').split(',').map(str => str.toLocaleString().trim()); params['drive_id'] = argv['drive_id'] || ''; params['service_account'] = argv['service_account'] || null; params['git_update_delay'] = argv['git_update_delay'] || 60; + process.on('unhandledRejection', function (err) { console.error('process.on:unhandledRejection', err); process.exit(1); diff --git a/src/plugins/DownloadPlugin.ts b/src/plugins/DownloadPlugin.ts index a336a103..2cb5371a 100644 --- a/src/plugins/DownloadPlugin.ts +++ b/src/plugins/DownloadPlugin.ts @@ -10,6 +10,7 @@ import {StringWritable} from '../utils/StringWritable'; import {GoogleDocsService} from '../google/GoogleDocsService'; import {GoogleDriveService} from '../google/GoogleDriveService'; import {ExternalFiles} from "../storage/ExternalFiles"; +import {CliParams} from "../MainService"; export class DownloadPlugin extends BasePlugin { private googleDocsService: GoogleDocsService; @@ -18,14 +19,16 @@ export class DownloadPlugin extends BasePlugin { private auth: any; private googleDriveService: GoogleDriveService; private externalFiles: ExternalFiles; + private debug: string[]; constructor(eventBus) { super(eventBus); this.googleDocsService = new GoogleDocsService(); - eventBus.on('main:init', async (params) => { + eventBus.on('main:init', async (params: CliParams) => { this.config_dir = params.config_dir; + this.debug = params.debug; }); eventBus.on('files_structure:initialized', ({ filesStructure }) => { this.filesStructure = filesStructure; @@ -144,6 +147,9 @@ export class DownloadPlugin extends BasePlugin { for (const file of dirtyFiles) { const targetPath = path.join(this.config_dir, 'files', file.id + '.gdoc'); + if (file.mimeType === FilesStructure.REDIRECT_MIME) { + promises.push(this.filesStructure.markClean([ file ])); + } else if (file.mimeType === FilesStructure.DRAWING_MIME) { promises.push(this.downloadDiagram(file, targetPath)); } else @@ -162,6 +168,9 @@ export class DownloadPlugin extends BasePlugin { const dirtyFilesAfter = this.filesStructure.findFiles(item => !!item.dirty); if (dirtyFilesAfter.length > 0) { + if (this.debug.indexOf('download') > -1) { + console.log('dirtyFilesAfter', dirtyFilesAfter); + } console.log('Download retry required'); process.nextTick(() => { this.eventBus.emit('download:retry'); diff --git a/src/plugins/WatchChangesPlugin.js b/src/plugins/WatchChangesPlugin.ts similarity index 69% rename from src/plugins/WatchChangesPlugin.js rename to src/plugins/WatchChangesPlugin.ts index e5fc4cad..1b41ee20 100644 --- a/src/plugins/WatchChangesPlugin.js +++ b/src/plugins/WatchChangesPlugin.ts @@ -1,17 +1,34 @@ 'use strict'; import {BasePlugin} from './BasePlugin'; +import {CliParams} from "../MainService"; +import {DriveConfig} from "./ConfigDirPlugin"; +import {FilesStructure} from "../storage/FilesStructure"; +import {GoogleDriveService} from "../google/GoogleDriveService"; export class WatchChangesPlugin extends BasePlugin { + private command: string; + private drive_id: string; + private watch_mode: string; + private debug: string[]; + private drive_config: DriveConfig; + private filesStructure: FilesStructure; + private auth: any; + private googleDriveService: GoogleDriveService; + private context: any; + private lastMTime: string; + private startTrackToken: string; + constructor(eventBus) { super(eventBus); - eventBus.on('main:init', async (params) => { + eventBus.on('main:init', async (params: CliParams) => { this.command = params.command; this.drive_id = params.drive_id; this.watch_mode = params.watch_mode; + this.debug = params.debug; }); - eventBus.on('drive_config:loaded', (drive_config) => { + eventBus.on('drive_config:loaded', (drive_config: DriveConfig) => { this.drive_config = drive_config; }); eventBus.on('files_structure:initialized', ({ filesStructure }) => { @@ -62,11 +79,17 @@ export class WatchChangesPlugin extends BasePlugin { }); if (changedFiles.length > 0) { - console.log(changedFiles.length + ' files modified'); + console.log(changedFiles.length + ' files changed'); await this.filesStructure.merge(changedFiles); this.startTrackToken = result.token; // eslint-disable-line require-atomic-updates console.log('Pulled latest changes'); + this.eventBus.emit('files_structure:dirty'); } else { + if (this.debug.indexOf('watch') > -1) { + if (result.files.length > 0) { + console.log('Files outside folder:', result.files); + } + } console.log('No changes detected. Sleeping for 10 seconds.'); } diff --git a/src/plugins/WatchMTimePlugin.js b/src/plugins/WatchMTimePlugin.ts similarity index 62% rename from src/plugins/WatchMTimePlugin.js rename to src/plugins/WatchMTimePlugin.ts index 578e5cf9..03cd1c75 100644 --- a/src/plugins/WatchMTimePlugin.js +++ b/src/plugins/WatchMTimePlugin.ts @@ -1,17 +1,31 @@ 'use strict'; import {BasePlugin} from './BasePlugin'; +import {CliParams} from "../MainService"; +import {DriveConfig} from "./ConfigDirPlugin"; +import {FilesStructure} from "../storage/FilesStructure"; +import {GoogleDriveService} from "../google/GoogleDriveService"; export class WatchMTimePlugin extends BasePlugin { + private command: string; + private drive_id: string; + private watch_mode: string; + private drive_config: DriveConfig; + private filesStructure: FilesStructure; + private auth: any; + private googleDriveService: GoogleDriveService; + private context: any; + private lastMTime: string; + constructor(eventBus) { super(eventBus); - eventBus.on('main:init', async (params) => { + eventBus.on('main:init', async (params: CliParams) => { this.command = params.command; this.drive_id = params.drive_id; this.watch_mode = params.watch_mode; }); - eventBus.on('drive_config:loaded', (drive_config) => { + eventBus.on('drive_config:loaded', (drive_config: DriveConfig) => { this.drive_config = drive_config; }); eventBus.on('files_structure:initialized', ({ filesStructure }) => { @@ -46,9 +60,13 @@ export class WatchMTimePlugin extends BasePlugin { try { lastMTime = this.filesStructure.getMaxModifiedTime(); const changedFiles = await this.googleDriveService.listRootRecursive(this.auth, context, lastMTime); - await this.filesStructure.merge(changedFiles); - - console.log('Sleeping for 10 seconds.'); + if (changedFiles.length > 0) { + console.log(changedFiles.length + ' files modified'); + await this.filesStructure.merge(changedFiles); + this.eventBus.emit('files_structure:dirty'); + } else { + console.log('No files modified. Sleeping for 10 seconds.'); + } await new Promise((resolve) => setTimeout(resolve, 10 * 1000)); } catch (e) { From 441cdd6fcefe5898258b34dd2d688958666e1eae Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Fri, 11 Sep 2020 13:04:00 +0200 Subject: [PATCH 3/3] Fix watching changes with driveId --- package-lock.json | 109 +++++++++++++++++------------- package.json | 2 +- src/google/GoogleDriveService.ts | 22 ++++-- src/plugins/WatchChangesPlugin.ts | 4 +- src/storage/FilesStructure.ts | 5 +- 5 files changed, 85 insertions(+), 57 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9c7575ae..b8632b42 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2438,9 +2438,9 @@ "dev": true }, "agent-base": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.0.tgz", - "integrity": "sha512-j1Q7cSCqN+AwrmDd+pzgqc0/NpC655x2bUf5ZjRIO77DcNBFmh+OgRNzF6OKdCC9RSCb19fGd99+bhXFdkRNqw==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", + "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", "requires": { "debug": "4" }, @@ -2850,9 +2850,9 @@ "dev": true }, "bignumber.js": { - "version": "7.2.1", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-7.2.1.tgz", - "integrity": "sha512-S4XzBk5sMB+Rcb/LNcpzXr57VRTxgAvaAEDAl1AwRx27j00hT84O6OkteE7u8UB3NuaaygCRrEpqox4uDOrbdQ==" + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", + "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==" }, "binary-extensions": { "version": "1.13.1", @@ -4736,9 +4736,9 @@ "dev": true }, "fast-text-encoding": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.2.tgz", - "integrity": "sha512-5rQdinSsycpzvAoHga2EDn+LRX1d5xLFsuNG0Kg61JrAT/tASXcLL0nf/33v+sAxlQcfYmWbTURa1mmAf55jGw==" + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.3.tgz", + "integrity": "sha512-dtm4QZH9nZtcDt8qJiOH9fcQd1NAgi+K1O2DbE6GG1PPCK/BWfOH3idCTRQ4ImXRUOyopDEgDEnVEE7Y/2Wrig==" }, "fd-slicer": { "version": "1.1.0", @@ -5513,9 +5513,9 @@ "dev": true }, "gaxios": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-3.0.3.tgz", - "integrity": "sha512-PkzQludeIFhd535/yucALT/Wxyj/y2zLyrMwPcJmnLHDugmV49NvAi/vb+VUq/eWztATZCNcb8ue+ywPG+oLuw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-3.1.0.tgz", + "integrity": "sha512-DDTn3KXVJJigtz+g0J3vhcfbDbKtAroSTxauWsdnP57sM5KZ3d2c/3D9RKFJ86s43hfw6WULg6TXYw/AYiBlpA==", "requires": { "abort-controller": "^3.0.0", "extend": "^3.0.2", @@ -5532,12 +5532,12 @@ } }, "gcp-metadata": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.1.0.tgz", - "integrity": "sha512-r57SV28+olVsflPlKyVig3Muo/VDlcsObMtvDGOEtEJXj+DDE8bEl0coIkXh//hbkSDTvo+f5lbihZOndYXQQQ==", + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-4.1.4.tgz", + "integrity": "sha512-5J/GIH0yWt/56R3dNaNWPGQ/zXsZOddYECfJaqxFWgrZ9HC2Kvc5vl9upOgUUHKzURjAVf2N+f6tEJiojqXUuA==", "requires": { "gaxios": "^3.0.0", - "json-bigint": "^0.3.0" + "json-bigint": "^1.0.0" } }, "gensync": { @@ -5684,42 +5684,57 @@ } }, "google-auth-library": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.0.0.tgz", - "integrity": "sha512-uLydy1t6SHN/EvYUJrtN3GCHFrnJ0c8HJjOxXiGjoTuYHIoCUT3jVxnzmjHwVnSdkfE9Akasm2rM6qG1COTXfQ==", + "version": "6.0.6", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-6.0.6.tgz", + "integrity": "sha512-fWYdRdg55HSJoRq9k568jJA1lrhg9i2xgfhVIMJbskUmbDpJGHsbv9l41DGhCDXM21F9Kn4kUwdysgxSYBYJUw==", "requires": { "arrify": "^2.0.0", "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "fast-text-encoding": "^1.0.0", "gaxios": "^3.0.0", - "gcp-metadata": "^4.0.0", + "gcp-metadata": "^4.1.0", "gtoken": "^5.0.0", "jws": "^4.0.0", - "lru-cache": "^5.0.0" + "lru-cache": "^6.0.0" + }, + "dependencies": { + "lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "requires": { + "yallist": "^4.0.0" + } + }, + "yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + } } }, "google-p12-pem": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.0.1.tgz", - "integrity": "sha512-VlQgtozgNVVVcYTXS36eQz4PXPt9gIPqLOhHN0QiV6W6h4qSCNVKPtKC5INtJsaHHF2r7+nOIa26MJeJMTaZEQ==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-3.0.3.tgz", + "integrity": "sha512-wS0ek4ZtFx/ACKYF3JhyGe5kzH7pgiQ7J5otlumqR9psmWMYc+U9cErKlCYVYHoUaidXHdZ2xbo34kB+S+24hA==", "requires": { - "node-forge": "^0.9.0" + "node-forge": "^0.10.0" } }, "googleapis": { - "version": "51.0.0", - "resolved": "https://registry.npmjs.org/googleapis/-/googleapis-51.0.0.tgz", - "integrity": "sha512-ZicnODSX5apolF05lNXc3AHW/zazeaWL8zSiQ7a/Bd4pfHU+TJzX7GP38phjyOVPzIG3qnrJ0SMASJgljlFZlQ==", + "version": "59.0.0", + "resolved": "https://registry.npmjs.org/googleapis/-/googleapis-59.0.0.tgz", + "integrity": "sha512-GV/E4KRN89a4GxSk7D7cwUfRYgcJHR05sOgm/WGdwc/u8dxNXG5lWmz9gF5ZwFGk2yKtVxL4VZNn4zBuZ6rmGg==", "requires": { "google-auth-library": "^6.0.0", - "googleapis-common": "^4.1.0" + "googleapis-common": "^4.4.0" } }, "googleapis-common": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-4.2.0.tgz", - "integrity": "sha512-Z6MTCEvvfPEGz+qfYOA8trZ6if3EdGXwJwRa6mCTjJv+LKdUHxjdsak6dOETDqVnIhytWQAUpURFsLCaxDI8rg==", + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-4.4.0.tgz", + "integrity": "sha512-Bgrs8/1OZQFFIfVuX38L9t48rPAkVUXttZy6NzhhXxFOEMSHgfFIjxou7RIXOkBHxmx2pVwct9WjKkbnqMYImQ==", "requires": { "extend": "^3.0.2", "gaxios": "^3.0.0", @@ -5751,9 +5766,9 @@ "dev": true }, "gtoken": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.0.1.tgz", - "integrity": "sha512-33w4FNDkUcyIOq/TqyC+drnKdI4PdXmWp9lZzssyEQKuvu9ZFN3KttaSnDKo52U3E51oujVGop93mKxmqO8HHg==", + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-5.0.3.tgz", + "integrity": "sha512-Nyd1wZCMRc2dj/mAD0LlfQLcAO06uKdpKJXvK85SGrF5+5+Bpfil9u/2aw35ltvEHjvl0h5FMKN5knEU+9JrOg==", "requires": { "gaxios": "^3.0.0", "google-p12-pem": "^3.0.0", @@ -6521,11 +6536,11 @@ } }, "json-bigint": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.3.0.tgz", - "integrity": "sha1-DM2RLEuCcNBfBW+9E4FLU9OCWx4=", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", "requires": { - "bignumber.js": "^7.0.0" + "bignumber.js": "^9.0.0" } }, "json-parse-better-errors": { @@ -6693,6 +6708,7 @@ "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, "requires": { "yallist": "^3.0.2" } @@ -7470,9 +7486,9 @@ "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" }, "node-forge": { - "version": "0.9.1", - "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.9.1.tgz", - "integrity": "sha512-G6RlQt5Sb4GMBzXvhfkeFmbqR6MzhtnT7VTHuLadjkii3rdYHNdw0m8zA4BTxVIh68FicCQ2NSUANpsqkr9jvQ==" + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz", + "integrity": "sha512-PPmu8eEeG9saEUvI97fm4OYxXVB6bFvyNTyiUOBichBpFG8A1Ljw3bY62+5oOjDEMHRnd0Y7HQ+x7uzxOzC6JA==" }, "node-libs-browser": { "version": "2.2.1", @@ -9541,9 +9557,9 @@ "dev": true }, "uuid": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.1.0.tgz", - "integrity": "sha512-CI18flHDznR0lq54xBycOVmphdCYnQLKn8abKn7PXUiKUGdEd+/l9LWNJmugXel4hXq7S+RMNl34ecyC9TntWg==" + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.0.tgz", + "integrity": "sha512-fX6Z5o4m6XsXBdli9g7DtWgAx+osMsRRZFKma1mIUsLCz6vRvv+pz5VNbyu9UEDzpMWulZfvpgb/cmDXVulYFQ==" }, "v8-compile-cache": { "version": "2.1.0", @@ -9977,7 +9993,8 @@ "yallist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", - "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==", + "dev": true }, "yaml": { "version": "1.8.2", diff --git a/package.json b/package.json index a42419f8..c998f4f8 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "buffer": "^5.6.0", "cli-progress": "^3.7.0", "dotenv": "^8.2.0", - "googleapis": "^51.0.0", + "googleapis": "^59.0.0", "minimist": "^1.2.5", "node-fetch": "^2.6.0", "relateurl": "^0.2.7", diff --git a/src/google/GoogleDriveService.ts b/src/google/GoogleDriveService.ts index c1eb52de..6f2ada53 100644 --- a/src/google/GoogleDriveService.ts +++ b/src/google/GoogleDriveService.ts @@ -3,7 +3,6 @@ import slugify from 'slugify'; import {google} from 'googleapis'; import {File, FilesStructure} from '../storage/FilesStructure'; -import {GoogleDocsServiceError} from "./GoogleDocsService"; const MAX_FILENAME_LENGTH = 100; @@ -179,9 +178,19 @@ export class GoogleDriveService { return removeDuplicates(retVal); } - async getStartTrackToken(auth) { + async getStartTrackToken(auth, driveId) { const drive = google.drive({ version: 'v3', auth }); - const res = await drive.changes.getStartPageToken({}); + + const params = { + supportsAllDrives: true, + driveId: undefined + } + + if (driveId) { + params.driveId = driveId; + } + + const res = await drive.changes.getStartPageToken(params); return res.data.startPageToken; } @@ -192,13 +201,12 @@ export class GoogleDriveService { pageToken: pageToken, supportsAllDrives: true, includeItemsFromAllDrives: true, - includeCorpusRemovals: true, fields: 'newStartPageToken, changes( file(id, name, mimeType, modifiedTime, size, md5Checksum, lastModifyingUser, parents, version))', - drives: undefined + driveId: undefined }; if (driveId) { - params.drives = [ driveId ]; + params.driveId = driveId; } try { @@ -250,7 +258,7 @@ export class GoogleDriveService { } const listParams = { - corpora: 'allDrives', + corpora: context.driveId ? 'drive' : 'allDrives', q: query, pageToken: nextPageToken, pageSize: 1000, diff --git a/src/plugins/WatchChangesPlugin.ts b/src/plugins/WatchChangesPlugin.ts index 1b41ee20..9cbd98a3 100644 --- a/src/plugins/WatchChangesPlugin.ts +++ b/src/plugins/WatchChangesPlugin.ts @@ -24,12 +24,12 @@ export class WatchChangesPlugin extends BasePlugin { eventBus.on('main:init', async (params: CliParams) => { this.command = params.command; - this.drive_id = params.drive_id; this.watch_mode = params.watch_mode; this.debug = params.debug; }); eventBus.on('drive_config:loaded', (drive_config: DriveConfig) => { this.drive_config = drive_config; + this.drive_id = drive_config.drive_id; }); eventBus.on('files_structure:initialized', ({ filesStructure }) => { this.filesStructure = filesStructure; @@ -46,7 +46,7 @@ export class WatchChangesPlugin extends BasePlugin { if (this.watch_mode !== 'changes') { return; } - this.startTrackToken = await this.googleDriveService.getStartTrackToken(this.auth); + this.startTrackToken = await this.googleDriveService.getStartTrackToken(this.auth, this.drive_id); eventBus.emit('watch:token_ready'); }); eventBus.on('main:run_watch', async () => { diff --git a/src/storage/FilesStructure.ts b/src/storage/FilesStructure.ts index 820540d7..c9fa98da 100644 --- a/src/storage/FilesStructure.ts +++ b/src/storage/FilesStructure.ts @@ -111,9 +111,12 @@ class FilesStructure { } } - async markClean(files) { + async markClean(files: File[]) { for (const file of files) { this.fileMap[file.id].dirty = false; + if (file.modifiedTime) { + this.fileMap[file.id].modifiedTime = file.modifiedTime; + } await this._putFile(file.id, Object.assign({}, this.fileMap[file.id], { dirty: false }));