From 4d9a28f72cf73d8819bf45540acc2ddb66319da7 Mon Sep 17 00:00:00 2001 From: Henry Thornley Date: Tue, 14 May 2024 08:59:35 -0700 Subject: [PATCH 01/11] Add Excel Export Capability --- packages/export/package.json | 3 +- packages/export/src/csv.js | 1 + packages/export/src/excel.js | 93 ++++++++++++++ packages/export/src/index.js | 3 +- yarn.lock | 227 ++++++++++++++++++++++++++++++++++- 5 files changed, 319 insertions(+), 8 deletions(-) create mode 100644 packages/export/src/excel.js diff --git a/packages/export/package.json b/packages/export/package.json index d02f7590f..a1d3c64b5 100644 --- a/packages/export/package.json +++ b/packages/export/package.json @@ -34,7 +34,8 @@ "contexture-client": "^2.53.7", "futil": "^1.76.4", "lodash": "^4.17.21", - "minimal-csv-formatter": "^1.0.15" + "minimal-csv-formatter": "^1.0.15", + "write-excel-file": "^2.0.1" }, "packageManager": "yarn@3.3.1" } diff --git a/packages/export/src/csv.js b/packages/export/src/csv.js index 64b95bd2e..683eb5440 100644 --- a/packages/export/src/csv.js +++ b/packages/export/src/csv.js @@ -39,6 +39,7 @@ export default ({ recordsWritten = recordsWritten + _.getOr(1, 'recordCount', r) await onWrite({ recordsWritten, record: r }) } + await onWrite({ recordsWritten: recordsWritten, isStreamDone: true }) await stream.end() })(), cancel() { diff --git a/packages/export/src/excel.js b/packages/export/src/excel.js new file mode 100644 index 000000000..87dcddf68 --- /dev/null +++ b/packages/export/src/excel.js @@ -0,0 +1,93 @@ +import _ from 'lodash/fp.js' +import writeXlsxFile from 'write-excel-file/node' +import stream from 'stream' +import { promisify } from 'util' +import { pipeline } from 'stream/promises' + +const isISODate = (dateString) => { + // Convert the date to an ISO string + try { + const date = new Date(dateString) + const isoString = date.toISOString() + return date.toString() === isoString + } catch (e) { + return false + } +} + +const isNumber = (numberString) => { + return !isNaN(Number(numberString)) +} + +const convertToExcelCell = (value) => { + return { + type: String, + typeof: 'string', + wrap: true, + value: `${value}`, + } +} + +let transformLabels = _.map(_.get('label')) + +//const pipeline = promisify(stream.pipeline) + +export default ({ + stream, // writable stream target stream + iterableData, // iterator for each page of an array of objects + // order list of which indicates the header label, + // display function for the field, + // and key of the record. + // [{ key: string(supports lodash dot notation), label: string, display: function(value, {key, record, transform})}...] + transform, + headers = null, // array of strings to use as headers, array or arrays for multi-line headers + onWrite = _.noop, // function to intercept writing a page of records +}) => { + const excelData = [ + _.map( + (value) => ({ value, fontWeight: 'bold' }), + headers || transformLabels(transform) + ), + ] + + let cancel = false + let recordsWritten = 0 + + return { + promise: (async () => { + for await (let r of iterableData) { + if (cancel) break + excelData.push( + _.map( + (t) => + convertToExcelCell( + t.display(_.get(t.key, r), { + key: t.key, + record: r, + transform, + }) + ), + transform + ) + ) + recordsWritten = recordsWritten + _.getOr(1, 'recordCount', r) + await onWrite({ recordsWritten }) + } + let columns = _.map( + () => ({ + width: 50, + }), + excelData[1] + ) + const readStream = await writeXlsxFile(excelData, { columns }) + for await (const chunk of readStream) { + stream.write(chunk) + } + await onWrite({ recordsWritten: recordsWritten, isStreamDone: true }) + await stream.end() + })(), + cancel() { + cancel = true + }, + } +} diff --git a/packages/export/src/index.js b/packages/export/src/index.js index 4b4a71dcf..d09a03a1c 100644 --- a/packages/export/src/index.js +++ b/packages/export/src/index.js @@ -1,5 +1,6 @@ import csv from './csv.js' +import excel from './excel.js' import * as nodes from './nodes/index.js' -export { nodes, csv } +export { nodes, csv, excel } export * from './utils.js' diff --git a/yarn.lock b/yarn.lock index 784925786..e2b79e458 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1940,6 +1940,15 @@ __metadata: languageName: node linkType: hard +"@babel/runtime@npm:^7.17.9": + version: 7.24.5 + resolution: "@babel/runtime@npm:7.24.5" + dependencies: + regenerator-runtime: ^0.14.0 + checksum: 755383192f3ac32ba4c62bd4f1ae92aed5b82d2c6665f39eb28fa94546777cf5c63493ea92dd03f1c2e621b17e860f190c056684b7f234270fdc91e29beda063 + languageName: node + linkType: hard + "@babel/runtime@npm:^7.9.2": version: 7.22.5 resolution: "@babel/runtime@npm:7.22.5" @@ -6276,6 +6285,15 @@ __metadata: languageName: node linkType: hard +"abort-controller@npm:^3.0.0": + version: 3.0.0 + resolution: "abort-controller@npm:3.0.0" + dependencies: + event-target-shim: ^5.0.0 + checksum: 170bdba9b47b7e65906a28c8ce4f38a7a369d78e2271706f020849c1bfe0ee2067d4261df8bbb66eb84f79208fd5b710df759d64191db58cfba7ce8ef9c54b75 + languageName: node + linkType: hard + "accepts@npm:~1.3.5, accepts@npm:~1.3.8": version: 1.3.8 resolution: "accepts@npm:1.3.8" @@ -6551,6 +6569,36 @@ __metadata: languageName: node linkType: hard +"archiver-utils@npm:^5.0.0, archiver-utils@npm:^5.0.2": + version: 5.0.2 + resolution: "archiver-utils@npm:5.0.2" + dependencies: + glob: ^10.0.0 + graceful-fs: ^4.2.0 + is-stream: ^2.0.1 + lazystream: ^1.0.0 + lodash: ^4.17.15 + normalize-path: ^3.0.0 + readable-stream: ^4.0.0 + checksum: 7dc4f3001dc373bd0fa7671ebf08edf6f815cbc539c78b5478a2eaa67e52e3fc0e92f562cdef2ba016c4dcb5468d3d069eb89535c6844da4a5bb0baf08ad5720 + languageName: node + linkType: hard + +"archiver@npm:^7.0.1": + version: 7.0.1 + resolution: "archiver@npm:7.0.1" + dependencies: + archiver-utils: ^5.0.2 + async: ^3.2.4 + buffer-crc32: ^1.0.0 + readable-stream: ^4.0.0 + readdir-glob: ^1.1.2 + tar-stream: ^3.0.0 + zip-stream: ^6.0.1 + checksum: f93bcc00f919e0bbb6bf38fddf111d6e4d1ed34721b73cc073edd37278303a7a9f67aa4abd6fd2beb80f6c88af77f2eb4f60276343f67605e3aea404e5ad93ea + languageName: node + linkType: hard + "are-we-there-yet@npm:^3.0.0": version: 3.0.1 resolution: "are-we-there-yet@npm:3.0.1" @@ -6728,6 +6776,13 @@ __metadata: languageName: node linkType: hard +"async@npm:^3.2.4": + version: 3.2.5 + resolution: "async@npm:3.2.5" + checksum: 5ec77f1312301dee02d62140a6b1f7ee0edd2a0f983b6fd2b0849b969f245225b990b47b8243e7b9ad16451a53e7f68e753700385b706198ced888beedba3af4 + languageName: node + linkType: hard + "asynckit@npm:^0.4.0": version: 0.4.0 resolution: "asynckit@npm:0.4.0" @@ -7192,6 +7247,13 @@ __metadata: languageName: node linkType: hard +"buffer-crc32@npm:^1.0.0": + version: 1.0.0 + resolution: "buffer-crc32@npm:1.0.0" + checksum: bc114c0e02fe621249e0b5093c70e6f12d4c2b1d8ddaf3b1b7bbe3333466700100e6b1ebdc12c050d0db845bc582c4fce8c293da487cc483f97eea027c480b23 + languageName: node + linkType: hard + "buffer-crc32@npm:~0.2.3": version: 0.2.13 resolution: "buffer-crc32@npm:0.2.13" @@ -7223,6 +7285,16 @@ __metadata: languageName: node linkType: hard +"buffer@npm:^6.0.3": + version: 6.0.3 + resolution: "buffer@npm:6.0.3" + dependencies: + base64-js: ^1.3.1 + ieee754: ^1.2.1 + checksum: 5ad23293d9a731e4318e420025800b42bf0d264004c0286c8cc010af7a270c7a0f6522e84f54b9ad65cbd6db20b8badbfd8d2ebf4f80fa03dab093b89e68c3f9 + languageName: node + linkType: hard + "bytes@npm:3.0.0": version: 3.0.0 resolution: "bytes@npm:3.0.0" @@ -7726,6 +7798,19 @@ __metadata: languageName: node linkType: hard +"compress-commons@npm:^6.0.2": + version: 6.0.2 + resolution: "compress-commons@npm:6.0.2" + dependencies: + crc-32: ^1.2.0 + crc32-stream: ^6.0.0 + is-stream: ^2.0.1 + normalize-path: ^3.0.0 + readable-stream: ^4.0.0 + checksum: 37d79a54f91344ecde352588e0a128f28ce619b085acd4f887defd76978a0640e3454a42c7dcadb0191bb3f971724ae4b1f9d6ef9620034aa0427382099ac946 + languageName: node + linkType: hard + "compressible@npm:~2.0.16": version: 2.0.18 resolution: "compressible@npm:2.0.18" @@ -7839,6 +7924,7 @@ __metadata: futil: ^1.76.4 lodash: ^4.17.21 minimal-csv-formatter: ^1.0.15 + write-excel-file: ^2.0.1 languageName: unknown linkType: soft @@ -8060,6 +8146,25 @@ __metadata: languageName: node linkType: hard +"crc-32@npm:^1.2.0": + version: 1.2.2 + resolution: "crc-32@npm:1.2.2" + bin: + crc32: bin/crc32.njs + checksum: ad2d0ad0cbd465b75dcaeeff0600f8195b686816ab5f3ba4c6e052a07f728c3e70df2e3ca9fd3d4484dc4ba70586e161ca5a2334ec8bf5a41bf022a6103ff243 + languageName: node + linkType: hard + +"crc32-stream@npm:^6.0.0": + version: 6.0.0 + resolution: "crc32-stream@npm:6.0.0" + dependencies: + crc-32: ^1.2.0 + readable-stream: ^4.0.0 + checksum: e6edc2f81bc387daef6d18b2ac18c2ffcb01b554d3b5c7d8d29b177505aafffba574658fdd23922767e8dab1183d1962026c98c17e17fb272794c33293ef607c + languageName: node + linkType: hard + "create-emotion@npm:^9.2.12": version: 9.2.12 resolution: "create-emotion@npm:9.2.12" @@ -9497,7 +9602,14 @@ __metadata: languageName: node linkType: hard -"events@npm:^3.2.0": +"event-target-shim@npm:^5.0.0": + version: 5.0.1 + resolution: "event-target-shim@npm:5.0.1" + checksum: 1ffe3bb22a6d51bdeb6bf6f7cf97d2ff4a74b017ad12284cc9e6a279e727dc30a5de6bb613e5596ff4dc3e517841339ad09a7eec44266eccb1aa201a30448166 + languageName: node + linkType: hard + +"events@npm:^3.2.0, events@npm:^3.3.0": version: 3.3.0 resolution: "events@npm:3.3.0" checksum: f6f487ad2198aa41d878fa31452f1a3c00958f46e9019286ff4787c84aac329332ab45c9cdc8c445928fc6d7ded294b9e005a7fce9426488518017831b272780 @@ -9735,6 +9847,13 @@ __metadata: languageName: node linkType: hard +"file-saver@npm:^2.0.5": + version: 2.0.5 + resolution: "file-saver@npm:2.0.5" + checksum: c62d96e5cebc58b4bdf3ae8a60d5cf9607ad82f75f798c33a4ee63435ac2203002584d5256a2a780eda7feb5e19dc3b6351c2212e58b3f529e63d265a7cc79f7 + languageName: node + linkType: hard + "file-system-cache@npm:2.3.0": version: 2.3.0 resolution: "file-system-cache@npm:2.3.0" @@ -10867,7 +10986,7 @@ __metadata: languageName: node linkType: hard -"ieee754@npm:^1.1.13": +"ieee754@npm:^1.1.13, ieee754@npm:^1.2.1": version: 1.2.1 resolution: "ieee754@npm:1.2.1" checksum: 5144c0c9815e54ada181d80a0b810221a253562422e7c6c3a60b1901154184f49326ec239d618c416c1c5945a2e197107aee8d986a3dd836b53dffefd99b5e7e @@ -10881,6 +11000,13 @@ __metadata: languageName: node linkType: hard +"immediate@npm:~3.0.5": + version: 3.0.6 + resolution: "immediate@npm:3.0.6" + checksum: f9b3486477555997657f70318cc8d3416159f208bec4cca3ff3442fd266bc23f50f0c9bd8547e1371a6b5e82b821ec9a7044a4f7b944798b25aa3cc6d5e63e62 + languageName: node + linkType: hard + "import-fresh@npm:^3.0.0, import-fresh@npm:^3.1.0, import-fresh@npm:^3.2.1": version: 3.3.0 resolution: "import-fresh@npm:3.3.0" @@ -11283,7 +11409,7 @@ __metadata: languageName: node linkType: hard -"is-stream@npm:^2.0.0": +"is-stream@npm:^2.0.0, is-stream@npm:^2.0.1": version: 2.0.1 resolution: "is-stream@npm:2.0.1" checksum: b8e05ccdf96ac330ea83c12450304d4a591f9958c11fd17bed240af8d5ffe08aedafa4c0f4cfccd4d28dc9d4d129daca1023633d5c11601a6cbc77521f6fae66 @@ -12276,6 +12402,18 @@ __metadata: languageName: node linkType: hard +"jszip@npm:^3.9.1": + version: 3.10.1 + resolution: "jszip@npm:3.10.1" + dependencies: + lie: ~3.3.0 + pako: ~1.0.2 + readable-stream: ~2.3.6 + setimmediate: ^1.0.5 + checksum: abc77bfbe33e691d4d1ac9c74c8851b5761fba6a6986630864f98d876f3fcc2d36817dfc183779f32c00157b5d53a016796677298272a714ae096dfe6b1c8b60 + languageName: node + linkType: hard + "jwa@npm:^1.4.1": version: 1.4.1 resolution: "jwa@npm:1.4.1" @@ -12338,6 +12476,15 @@ __metadata: languageName: node linkType: hard +"lazystream@npm:^1.0.0": + version: 1.0.1 + resolution: "lazystream@npm:1.0.1" + dependencies: + readable-stream: ^2.0.5 + checksum: 822c54c6b87701a6491c70d4fabc4cafcf0f87d6b656af168ee7bb3c45de9128a801cb612e6eeeefc64d298a7524a698dd49b13b0121ae50c2ae305f0dcc5310 + languageName: node + linkType: hard + "leven@npm:^3.1.0": version: 3.1.0 resolution: "leven@npm:3.1.0" @@ -12362,6 +12509,15 @@ __metadata: languageName: node linkType: hard +"lie@npm:~3.3.0": + version: 3.3.0 + resolution: "lie@npm:3.3.0" + dependencies: + immediate: ~3.0.5 + checksum: 33102302cf19766f97919a6a98d481e01393288b17a6aa1f030a3542031df42736edde8dab29ffdbf90bebeffc48c761eb1d064dc77592ca3ba3556f9fe6d2a8 + languageName: node + linkType: hard + "lines-and-columns@npm:^1.1.6": version: 1.2.4 resolution: "lines-and-columns@npm:1.2.4" @@ -12911,7 +13067,7 @@ __metadata: languageName: node linkType: hard -"minimatch@npm:^5.0.1": +"minimatch@npm:^5.0.1, minimatch@npm:^5.1.0": version: 5.1.6 resolution: "minimatch@npm:5.1.6" dependencies: @@ -13804,6 +13960,13 @@ __metadata: languageName: node linkType: hard +"pako@npm:~1.0.2": + version: 1.0.11 + resolution: "pako@npm:1.0.11" + checksum: 1be2bfa1f807608c7538afa15d6f25baa523c30ec870a3228a89579e474a4d992f4293859524e46d5d87fd30fa17c5edf34dbef0671251d9749820b488660b16 + languageName: node + linkType: hard + "param-case@npm:^3.0.4": version: 3.0.4 resolution: "param-case@npm:3.0.4" @@ -14956,7 +15119,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^2.0.0, readable-stream@npm:^2.2.2, readable-stream@npm:~2.3.6": +"readable-stream@npm:^2.0.0, readable-stream@npm:^2.0.5, readable-stream@npm:^2.2.2, readable-stream@npm:~2.3.6": version: 2.3.8 resolution: "readable-stream@npm:2.3.8" dependencies: @@ -14982,6 +15145,28 @@ __metadata: languageName: node linkType: hard +"readable-stream@npm:^4.0.0": + version: 4.5.2 + resolution: "readable-stream@npm:4.5.2" + dependencies: + abort-controller: ^3.0.0 + buffer: ^6.0.3 + events: ^3.3.0 + process: ^0.11.10 + string_decoder: ^1.3.0 + checksum: c4030ccff010b83e4f33289c535f7830190773e274b3fcb6e2541475070bdfd69c98001c3b0cb78763fc00c8b62f514d96c2b10a8bd35d5ce45203a25fa1d33a + languageName: node + linkType: hard + +"readdir-glob@npm:^1.1.2": + version: 1.1.3 + resolution: "readdir-glob@npm:1.1.3" + dependencies: + minimatch: ^5.1.0 + checksum: 1dc0f7440ff5d9378b593abe9d42f34ebaf387516615e98ab410cf3a68f840abbf9ff1032d15e0a0dbffa78f9e2c46d4fafdbaac1ca435af2efe3264e3f21874 + languageName: node + linkType: hard + "readdirp@npm:~3.6.0": version: 3.6.0 resolution: "readdirp@npm:3.6.0" @@ -15598,6 +15783,13 @@ __metadata: languageName: node linkType: hard +"setimmediate@npm:^1.0.5": + version: 1.0.5 + resolution: "setimmediate@npm:1.0.5" + checksum: c9a6f2c5b51a2dabdc0247db9c46460152ffc62ee139f3157440bd48e7c59425093f42719ac1d7931f054f153e2d26cf37dfeb8da17a794a58198a2705e527fd + languageName: node + linkType: hard + "setprototypeof@npm:1.2.0": version: 1.2.0 resolution: "setprototypeof@npm:1.2.0" @@ -16050,7 +16242,7 @@ __metadata: languageName: node linkType: hard -"string_decoder@npm:^1.1.1": +"string_decoder@npm:^1.1.1, string_decoder@npm:^1.3.0": version: 1.3.0 resolution: "string_decoder@npm:1.3.0" dependencies: @@ -17294,6 +17486,18 @@ __metadata: languageName: node linkType: hard +"write-excel-file@npm:^2.0.1": + version: 2.0.1 + resolution: "write-excel-file@npm:2.0.1" + dependencies: + "@babel/runtime": ^7.17.9 + archiver: ^7.0.1 + file-saver: ^2.0.5 + jszip: ^3.9.1 + checksum: 97f72c5725bba78e39ea1a517c6abad93f1280f508e610b4d2497feb8526dc9f0be7f9155fc3988e6ff9e3994c4fec22f4c98ba1a42f1dc7f2dfe602bb84e1bf + languageName: node + linkType: hard + "write-file-atomic@npm:^2.3.0, write-file-atomic@npm:^2.4.2": version: 2.4.3 resolution: "write-file-atomic@npm:2.4.3" @@ -17519,3 +17723,14 @@ __metadata: checksum: f77b3d8d00310def622123df93d4ee654fc6a0096182af8bd60679ddcdfb3474c56c6c7190817c84a2785648cdee9d721c0154eb45698c62176c322fb46fc700 languageName: node linkType: hard + +"zip-stream@npm:^6.0.1": + version: 6.0.1 + resolution: "zip-stream@npm:6.0.1" + dependencies: + archiver-utils: ^5.0.0 + compress-commons: ^6.0.2 + readable-stream: ^4.0.0 + checksum: aa5abd6a89590eadeba040afbc375f53337f12637e5e98330012a12d9886cde7a3ccc28bd91aafab50576035bbb1de39a9a316eecf2411c8b9009c9f94f0db27 + languageName: node + linkType: hard From f0e5c73aebc73433be5c291ba994a0a77ba3ee55 Mon Sep 17 00:00:00 2001 From: Henry Thornley Date: Tue, 14 May 2024 09:02:06 -0700 Subject: [PATCH 02/11] Bump --- .changeset/silly-months-unite.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/silly-months-unite.md diff --git a/.changeset/silly-months-unite.md b/.changeset/silly-months-unite.md new file mode 100644 index 000000000..c87deecd3 --- /dev/null +++ b/.changeset/silly-months-unite.md @@ -0,0 +1,5 @@ +--- +'contexture-export': minor +--- + +Enable Excel Export Option From f09107327cc8ae068116a8152e6802b71044b344 Mon Sep 17 00:00:00 2001 From: Henry Thornley Date: Tue, 14 May 2024 09:04:55 -0700 Subject: [PATCH 03/11] Cleanup --- packages/export/src/excel.js | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/packages/export/src/excel.js b/packages/export/src/excel.js index 87dcddf68..624f11bdd 100644 --- a/packages/export/src/excel.js +++ b/packages/export/src/excel.js @@ -1,23 +1,5 @@ import _ from 'lodash/fp.js' import writeXlsxFile from 'write-excel-file/node' -import stream from 'stream' -import { promisify } from 'util' -import { pipeline } from 'stream/promises' - -const isISODate = (dateString) => { - // Convert the date to an ISO string - try { - const date = new Date(dateString) - const isoString = date.toISOString() - return date.toString() === isoString - } catch (e) { - return false - } -} - -const isNumber = (numberString) => { - return !isNaN(Number(numberString)) -} const convertToExcelCell = (value) => { return { From ce54bf95c5f14f8446e9e96acb28a6567fc7b203 Mon Sep 17 00:00:00 2001 From: Henry Thornley Date: Tue, 14 May 2024 09:05:20 -0700 Subject: [PATCH 04/11] Cleanup --- packages/export/src/excel.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/export/src/excel.js b/packages/export/src/excel.js index 624f11bdd..13f57cff3 100644 --- a/packages/export/src/excel.js +++ b/packages/export/src/excel.js @@ -12,8 +12,6 @@ const convertToExcelCell = (value) => { let transformLabels = _.map(_.get('label')) -//const pipeline = promisify(stream.pipeline) - export default ({ stream, // writable stream target stream iterableData, // iterator for each page of an array of objects From b596032a504eb4f6585db968100f4998b9e61fb6 Mon Sep 17 00:00:00 2001 From: Henry Thornley Date: Fri, 17 May 2024 06:41:31 -0700 Subject: [PATCH 05/11] Addressed PR Feedback --- packages/export/package.json | 1 + packages/export/src/csv.js | 3 +- packages/export/src/excel.js | 13 ++- packages/export/src/excel.test.js | 66 ++++++++++++++ yarn.lock | 143 ++++++++++++++++++++++++++++-- 5 files changed, 216 insertions(+), 10 deletions(-) create mode 100644 packages/export/src/excel.test.js diff --git a/packages/export/package.json b/packages/export/package.json index a1d3c64b5..e82c75c1b 100644 --- a/packages/export/package.json +++ b/packages/export/package.json @@ -35,6 +35,7 @@ "futil": "^1.76.4", "lodash": "^4.17.21", "minimal-csv-formatter": "^1.0.15", + "read-excel-file": "^5.8.1", "write-excel-file": "^2.0.1" }, "packageManager": "yarn@3.3.1" diff --git a/packages/export/src/csv.js b/packages/export/src/csv.js index 683eb5440..66ac9d8e0 100644 --- a/packages/export/src/csv.js +++ b/packages/export/src/csv.js @@ -11,7 +11,8 @@ export default ({ // and key of the record. // [{ key: string(supports lodash dot notation), label: string, display: function(value, {key, record, transform})}...] transform, - headers = null, // array of strings to use as headers, array or arrays for multi-line headers + //TODO: support multi-line headers in excel and csv when implemented + headers = null, // array of strings to use as headers onWrite = _.noop, // function to intercept writing a page of records }) => { stream.write(csv(headers || transformLabels(transform))) diff --git a/packages/export/src/excel.js b/packages/export/src/excel.js index 13f57cff3..ea2b81edf 100644 --- a/packages/export/src/excel.js +++ b/packages/export/src/excel.js @@ -10,6 +10,12 @@ const convertToExcelCell = (value) => { } } +export const writeStreamData = async (stream, readStream) => { + for await (const chunk of readStream) { + stream.write(chunk) + } +} + let transformLabels = _.map(_.get('label')) export default ({ @@ -20,7 +26,8 @@ export default ({ // and key of the record. // [{ key: string(supports lodash dot notation), label: string, display: function(value, {key, record, transform})}...] transform, - headers = null, // array of strings to use as headers, array or arrays for multi-line headers + //TODO: support multi-line headers in excel and csv when implemented + headers = null, // array of strings to use as headers onWrite = _.noop, // function to intercept writing a page of records }) => { const excelData = [ @@ -60,9 +67,7 @@ export default ({ excelData[1] ) const readStream = await writeXlsxFile(excelData, { columns }) - for await (const chunk of readStream) { - stream.write(chunk) - } + await writeStreamData(stream, readStream) await onWrite({ recordsWritten: recordsWritten, isStreamDone: true }) await stream.end() })(), diff --git a/packages/export/src/excel.test.js b/packages/export/src/excel.test.js new file mode 100644 index 000000000..02686e528 --- /dev/null +++ b/packages/export/src/excel.test.js @@ -0,0 +1,66 @@ +import _ from 'lodash/fp.js' +import { PassThrough } from 'stream' +import writeXlsxFile from 'write-excel-file/node' +import readXlsxFile from 'read-excel-file/node' +import { writeStreamData } from './excel.js' + +const testDate = new Date() + +const HEADER_ROW = [ + { + value: 'Name', + fontWeight: 'bold', + }, + { + value: 'Date of Birth', + fontWeight: 'bold', + }, + { + value: 'Cost', + fontWeight: 'bold', + }, + { + value: 'Paid', + fontWeight: 'bold', + }, +] + +const DATA_ROW_1 = [ + { + type: String, + value: 'John Smith', + }, + { + type: Date, + value: testDate, + format: 'mm/dd/yyyy', + }, + { + type: Number, + value: 1800, + }, + { + type: Boolean, + value: true, + }, +] + +const expectedFileContents = [ + ['Name', 'Date of Birth', 'Cost', 'Paid'], + ['John Smith', testDate, 1800, true], +] + +const excelData = [HEADER_ROW, DATA_ROW_1] + +// These are skipped on purpose as they actual write CSVs +describe('Excel Stream Test', () => { + it('Export to Excel and Validate', async () => { + const writeStream = new PassThrough() + const readStream = await writeXlsxFile(excelData) + await writeStreamData(writeStream, readStream) + const rows = await readXlsxFile(writeStream) + console.log(rows) + console.log(expectedFileContents) + expect(rows).toStrictEqual(expectedFileContents) + }) +}) diff --git a/yarn.lock b/yarn.lock index e2b79e458..51128df9d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6233,6 +6233,13 @@ __metadata: languageName: node linkType: hard +"@xmldom/xmldom@npm:^0.8.2": + version: 0.8.10 + resolution: "@xmldom/xmldom@npm:0.8.10" + checksum: 4c136aec31fb3b49aaa53b6fcbfe524d02a1dc0d8e17ee35bd3bf35e9ce1344560481cd1efd086ad1a4821541482528672306d5e37cdbd187f33d7fadd3e2cf0 + languageName: node + linkType: hard + "@xtuc/ieee754@npm:^1.2.0": version: 1.2.0 resolution: "@xtuc/ieee754@npm:1.2.0" @@ -7061,6 +7068,13 @@ __metadata: languageName: node linkType: hard +"big-integer@npm:^1.6.17": + version: 1.6.52 + resolution: "big-integer@npm:1.6.52" + checksum: 6e86885787a20fed96521958ae9086960e4e4b5e74d04f3ef7513d4d0ad631a9f3bde2730fc8aaa4b00419fc865f6ec573e5320234531ef37505da7da192c40b + languageName: node + linkType: hard + "big-integer@npm:^1.6.44": version: 1.6.51 resolution: "big-integer@npm:1.6.51" @@ -7082,6 +7096,16 @@ __metadata: languageName: node linkType: hard +"binary@npm:~0.3.0": + version: 0.3.0 + resolution: "binary@npm:0.3.0" + dependencies: + buffers: ~0.1.1 + chainsaw: ~0.1.0 + checksum: b4699fda9e2c2981e74a46b0115cf0d472eda9b68c0e9d229ef494e92f29ce81acf0a834415094cffcc340dfee7c4ef8ce5d048c65c18067a7ed850323f777af + languageName: node + linkType: hard + "bl@npm:^4.0.3, bl@npm:^4.1.0": version: 4.1.0 resolution: "bl@npm:4.1.0" @@ -7100,6 +7124,13 @@ __metadata: languageName: node linkType: hard +"bluebird@npm:~3.4.1": + version: 3.4.7 + resolution: "bluebird@npm:3.4.7" + checksum: bffa9dee7d3a41ab15c4f3f24687b49959b4e64e55c058a062176feb8ccefc2163414fb4e1a0f3053bf187600936509660c3ebd168fd9f0e48c7eba23b019466 + languageName: node + linkType: hard + "body-parser@npm:1.20.1": version: 1.20.1 resolution: "body-parser@npm:1.20.1" @@ -7275,6 +7306,13 @@ __metadata: languageName: node linkType: hard +"buffer-indexof-polyfill@npm:~1.0.0": + version: 1.0.2 + resolution: "buffer-indexof-polyfill@npm:1.0.2" + checksum: fbfb2d69c6bb2df235683126f9dc140150c08ac3630da149913a9971947b667df816a913b6993bc48f4d611999cb99a1589914d34c02dccd2234afda5cb75bbc + languageName: node + linkType: hard + "buffer@npm:^5.5.0": version: 5.7.1 resolution: "buffer@npm:5.7.1" @@ -7295,6 +7333,13 @@ __metadata: languageName: node linkType: hard +"buffers@npm:~0.1.1": + version: 0.1.1 + resolution: "buffers@npm:0.1.1" + checksum: ad6f8e483efab39cefd92bdc04edbff6805e4211b002f4d1cfb70c6c472a61cc89fb18c37bcdfdd4ee416ca096e9ff606286698a7d41a18b539bac12fd76d4d5 + languageName: node + linkType: hard + "bytes@npm:3.0.0": version: 3.0.0 resolution: "bytes@npm:3.0.0" @@ -7459,6 +7504,15 @@ __metadata: languageName: node linkType: hard +"chainsaw@npm:~0.1.0": + version: 0.1.0 + resolution: "chainsaw@npm:0.1.0" + dependencies: + traverse: ">=0.3.0 <0.4" + checksum: 22a96b9fb0cd9fb20813607c0869e61817d1acc81b5d455cc6456b5e460ea1dd52630e0f76b291cf8294bfb6c1fc42e299afb52104af9096242699d6d3aa6d3e + languageName: node + linkType: hard + "chalk@npm:^2.0.0, chalk@npm:^2.1.0, chalk@npm:^2.3.0, chalk@npm:^2.4.2": version: 2.4.2 resolution: "chalk@npm:2.4.2" @@ -7924,6 +7978,7 @@ __metadata: futil: ^1.76.4 lodash: ^4.17.21 minimal-csv-formatter: ^1.0.15 + read-excel-file: ^5.8.1 write-excel-file: ^2.0.1 languageName: unknown linkType: soft @@ -8766,6 +8821,15 @@ __metadata: languageName: node linkType: hard +"duplexer2@npm:~0.1.4": + version: 0.1.4 + resolution: "duplexer2@npm:0.1.4" + dependencies: + readable-stream: ^2.0.2 + checksum: 744961f03c7f54313f90555ac20284a3fb7bf22fdff6538f041a86c22499560eb6eac9d30ab5768054137cb40e6b18b40f621094e0261d7d8c35a37b7a5ad241 + languageName: node + linkType: hard + "duplexify@npm:^3.5.0, duplexify@npm:^3.6.0": version: 3.7.1 resolution: "duplexify@npm:3.7.1" @@ -9838,6 +9902,13 @@ __metadata: languageName: node linkType: hard +"fflate@npm:^0.7.3": + version: 0.7.4 + resolution: "fflate@npm:0.7.4" + checksum: b812ab26047432db70ff4c73eb45ad53bd0774575b4818b9c61c2921e89ec65d1259f06ec1618f2ac55e6a2f2e29b6dc09173d213b46580bc69efae5344bf8f1 + languageName: node + linkType: hard + "file-entry-cache@npm:^6.0.1": version: 6.0.1 resolution: "file-entry-cache@npm:6.0.1" @@ -10221,6 +10292,18 @@ __metadata: languageName: node linkType: hard +"fstream@npm:^1.0.12": + version: 1.0.12 + resolution: "fstream@npm:1.0.12" + dependencies: + graceful-fs: ^4.1.2 + inherits: ~2.0.0 + mkdirp: ">=0.5 0" + rimraf: 2 + checksum: e6998651aeb85fd0f0a8a68cec4d05a3ada685ecc4e3f56e0d063d0564a4fc39ad11a856f9020f926daf869fc67f7a90e891def5d48e4cadab875dc313094536 + languageName: node + linkType: hard + "function-bind@npm:^1.1.1": version: 1.1.1 resolution: "function-bind@npm:1.1.1" @@ -10614,6 +10697,13 @@ __metadata: languageName: node linkType: hard +"graceful-fs@npm:^4.2.2": + version: 4.2.11 + resolution: "graceful-fs@npm:4.2.11" + checksum: ac85f94da92d8eb6b7f5a8b20ce65e43d66761c55ce85ac96df6865308390da45a8d3f0296dd3a663de65d30ba497bd46c696cc1e248c72b13d6d567138a4fc7 + languageName: node + linkType: hard + "grapheme-splitter@npm:^1.0.4": version: 1.0.4 resolution: "grapheme-splitter@npm:1.0.4" @@ -11060,7 +11150,7 @@ __metadata: languageName: node linkType: hard -"inherits@npm:2, inherits@npm:2.0.4, inherits@npm:^2.0.1, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.3": +"inherits@npm:2, inherits@npm:2.0.4, inherits@npm:^2.0.1, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.0, inherits@npm:~2.0.3": version: 2.0.4 resolution: "inherits@npm:2.0.4" checksum: 4a48a733847879d6cf6691860a6b1e3f0f4754176e4d71494c41f3475553768b10f84b5ce1d40fbd0e34e6bfbb864ee35858ad4dd2cf31e02fc4a154b724d7f1 @@ -12525,6 +12615,13 @@ __metadata: languageName: node linkType: hard +"listenercount@npm:~1.0.1": + version: 1.0.1 + resolution: "listenercount@npm:1.0.1" + checksum: 0f1c9077cdaf2ebc16473c7d72eb7de6d983898ca42500f03da63c3914b6b312dd5f7a90d2657691ea25adf3fe0ac5a43226e8b2c673fd73415ed038041f4757 + languageName: node + linkType: hard + "load-yaml-file@npm:^0.2.0": version: 0.2.0 resolution: "load-yaml-file@npm:0.2.0" @@ -13210,7 +13307,7 @@ __metadata: languageName: node linkType: hard -"mkdirp@npm:^0.5.1, mkdirp@npm:^0.5.4": +"mkdirp@npm:>=0.5 0, mkdirp@npm:^0.5.1, mkdirp@npm:^0.5.4": version: 0.5.6 resolution: "mkdirp@npm:0.5.6" dependencies: @@ -15084,6 +15181,17 @@ __metadata: languageName: node linkType: hard +"read-excel-file@npm:^5.8.1": + version: 5.8.1 + resolution: "read-excel-file@npm:5.8.1" + dependencies: + "@xmldom/xmldom": ^0.8.2 + fflate: ^0.7.3 + unzipper: ^0.10.11 + checksum: ef0e8aa69112908e1a2aca4353e0dd6093407e2254a88f71273ad31352f6ff57b862b7587e21d7669399e715363c400cc499eba1221dd11357072ab86dbd8362 + languageName: node + linkType: hard + "read-pkg-up@npm:^7.0.1": version: 7.0.1 resolution: "read-pkg-up@npm:7.0.1" @@ -15119,7 +15227,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^2.0.0, readable-stream@npm:^2.0.5, readable-stream@npm:^2.2.2, readable-stream@npm:~2.3.6": +"readable-stream@npm:^2.0.0, readable-stream@npm:^2.0.2, readable-stream@npm:^2.0.5, readable-stream@npm:^2.2.2, readable-stream@npm:~2.3.6": version: 2.3.8 resolution: "readable-stream@npm:2.3.8" dependencies: @@ -15551,7 +15659,7 @@ __metadata: languageName: node linkType: hard -"rimraf@npm:^2.6.1": +"rimraf@npm:2, rimraf@npm:^2.6.1": version: 2.7.1 resolution: "rimraf@npm:2.7.1" dependencies: @@ -15783,7 +15891,7 @@ __metadata: languageName: node linkType: hard -"setimmediate@npm:^1.0.5": +"setimmediate@npm:^1.0.5, setimmediate@npm:~1.0.4": version: 1.0.5 resolution: "setimmediate@npm:1.0.5" checksum: c9a6f2c5b51a2dabdc0247db9c46460152ffc62ee139f3157440bd48e7c59425093f42719ac1d7931f054f153e2d26cf37dfeb8da17a794a58198a2705e527fd @@ -16690,6 +16798,13 @@ __metadata: languageName: node linkType: hard +"traverse@npm:>=0.3.0 <0.4": + version: 0.3.9 + resolution: "traverse@npm:0.3.9" + checksum: 982982e4e249e9bbf063732a41fe5595939892758524bbef5d547c67cdf371b13af72b5434c6a61d88d4bb4351d6dabc6e22d832e0d16bc1bc684ef97a1cc59e + languageName: node + linkType: hard + "trim-newlines@npm:^3.0.0": version: 3.0.1 resolution: "trim-newlines@npm:3.0.1" @@ -17036,6 +17151,24 @@ __metadata: languageName: node linkType: hard +"unzipper@npm:^0.10.11": + version: 0.10.14 + resolution: "unzipper@npm:0.10.14" + dependencies: + big-integer: ^1.6.17 + binary: ~0.3.0 + bluebird: ~3.4.1 + buffer-indexof-polyfill: ~1.0.0 + duplexer2: ~0.1.4 + fstream: ^1.0.12 + graceful-fs: ^4.2.2 + listenercount: ~1.0.1 + readable-stream: ~2.3.6 + setimmediate: ~1.0.4 + checksum: b46ae9a72e4b4c224be6a8f46447dd7cb3761a59450827e869747c4564a8f555f877fc19c7e3b5d146127a7dd3e2ffea186116682f6646e64479f99dd23565bc + languageName: node + linkType: hard + "update-browserslist-db@npm:^1.0.10, update-browserslist-db@npm:^1.0.9": version: 1.0.10 resolution: "update-browserslist-db@npm:1.0.10" From be79412a993e031a9f5100917bc6bb970ebcaf76 Mon Sep 17 00:00:00 2001 From: Henry Thornley Date: Fri, 17 May 2024 11:32:15 -0700 Subject: [PATCH 06/11] Update Test Per Feedback --- packages/export/src/excel.js | 13 +++--- packages/export/src/excel.test.js | 75 +++++++------------------------ packages/export/src/utils.js | 7 +++ 3 files changed, 29 insertions(+), 66 deletions(-) diff --git a/packages/export/src/excel.js b/packages/export/src/excel.js index ea2b81edf..7a6bf06c2 100644 --- a/packages/export/src/excel.js +++ b/packages/export/src/excel.js @@ -1,5 +1,6 @@ import _ from 'lodash/fp.js' import writeXlsxFile from 'write-excel-file/node' +import { writeStreamData } from './utils.js' const convertToExcelCell = (value) => { return { @@ -10,12 +11,6 @@ const convertToExcelCell = (value) => { } } -export const writeStreamData = async (stream, readStream) => { - for await (const chunk of readStream) { - stream.write(chunk) - } -} - let transformLabels = _.map(_.get('label')) export default ({ @@ -66,8 +61,10 @@ export default ({ }), excelData[1] ) - const readStream = await writeXlsxFile(excelData, { columns }) - await writeStreamData(stream, readStream) + await writeStreamData( + stream, + async () => await writeXlsxFile(excelData, { columns }) + ) await onWrite({ recordsWritten: recordsWritten, isStreamDone: true }) await stream.end() })(), diff --git a/packages/export/src/excel.test.js b/packages/export/src/excel.test.js index 02686e528..bcede7efe 100644 --- a/packages/export/src/excel.test.js +++ b/packages/export/src/excel.test.js @@ -1,66 +1,25 @@ -import _ from 'lodash/fp.js' import { PassThrough } from 'stream' -import writeXlsxFile from 'write-excel-file/node' -import readXlsxFile from 'read-excel-file/node' -import { writeStreamData } from './excel.js' +import { writeStreamData } from './utils.js' -const testDate = new Date() +const testData = `Lorem ipsum dolor sit amet, consectetur adipiscing elit, + sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.` +const readStream = new PassThrough() +readStream.end(testData) -const HEADER_ROW = [ - { - value: 'Name', - fontWeight: 'bold', - }, - { - value: 'Date of Birth', - fontWeight: 'bold', - }, - { - value: 'Cost', - fontWeight: 'bold', - }, - { - value: 'Paid', - fontWeight: 'bold', - }, -] +const destinationStream = new PassThrough() -const DATA_ROW_1 = [ - { - type: String, - value: 'John Smith', - }, - { - type: Date, - value: testDate, - format: 'mm/dd/yyyy', - }, - { - type: Number, - value: 1800, - }, - { - type: Boolean, - value: true, - }, -] +describe('Stream write test', () => { + it('writeStreamData()', async () => { + let outputData = '' + // Capture the data written as string + destinationStream.on('data', (chunk) => { + outputData += chunk.toString() + }) -const expectedFileContents = [ - ['Name', 'Date of Birth', 'Cost', 'Paid'], - ['John Smith', testDate, 1800, true], -] + const mockWriteStreamData = async () => readStream + await writeStreamData(destinationStream, mockWriteStreamData) + destinationStream.end() -const excelData = [HEADER_ROW, DATA_ROW_1] - -// These are skipped on purpose as they actual write CSVs -describe('Excel Stream Test', () => { - it('Export to Excel and Validate', async () => { - const writeStream = new PassThrough() - const readStream = await writeXlsxFile(excelData) - await writeStreamData(writeStream, readStream) - const rows = await readXlsxFile(writeStream) - console.log(rows) - console.log(expectedFileContents) - expect(rows).toStrictEqual(expectedFileContents) + expect(outputData).toStrictEqual(testData) }) }) diff --git a/packages/export/src/utils.js b/packages/export/src/utils.js index 5b18f075f..48915361e 100644 --- a/packages/export/src/utils.js +++ b/packages/export/src/utils.js @@ -1,6 +1,13 @@ import F from 'futil' import _ from 'lodash/fp.js' +export const writeStreamData = async (stream, writeStreamData) => { + const readStream = await writeStreamData() + for await (const chunk of readStream) { + stream.write(chunk) + } +} + let Tree = F.tree((x) => x.children) export let setFilterOnly = Tree.transform((x) => { x.filterOnly = true From 1f57edba2aa26d5108df41dbb7d2078d59e8c458 Mon Sep 17 00:00:00 2001 From: Henry Thornley Date: Mon, 20 May 2024 11:11:04 -0700 Subject: [PATCH 07/11] PR Feedback --- packages/export/package.json | 1 - packages/export/src/csv.js | 2 +- packages/export/src/excel.js | 15 ++-- packages/export/src/excel.test.js | 2 +- packages/export/src/utils.js | 7 -- yarn.lock | 143 ++---------------------------- 6 files changed, 17 insertions(+), 153 deletions(-) diff --git a/packages/export/package.json b/packages/export/package.json index e82c75c1b..a1d3c64b5 100644 --- a/packages/export/package.json +++ b/packages/export/package.json @@ -35,7 +35,6 @@ "futil": "^1.76.4", "lodash": "^4.17.21", "minimal-csv-formatter": "^1.0.15", - "read-excel-file": "^5.8.1", "write-excel-file": "^2.0.1" }, "packageManager": "yarn@3.3.1" diff --git a/packages/export/src/csv.js b/packages/export/src/csv.js index 66ac9d8e0..dece870e1 100644 --- a/packages/export/src/csv.js +++ b/packages/export/src/csv.js @@ -40,7 +40,7 @@ export default ({ recordsWritten = recordsWritten + _.getOr(1, 'recordCount', r) await onWrite({ recordsWritten, record: r }) } - await onWrite({ recordsWritten: recordsWritten, isStreamDone: true }) + await onWrite({ recordsWritten, isStreamDone: true }) await stream.end() })(), cancel() { diff --git a/packages/export/src/excel.js b/packages/export/src/excel.js index 7a6bf06c2..0e7abcac7 100644 --- a/packages/export/src/excel.js +++ b/packages/export/src/excel.js @@ -1,16 +1,21 @@ import _ from 'lodash/fp.js' import writeXlsxFile from 'write-excel-file/node' -import { writeStreamData } from './utils.js' const convertToExcelCell = (value) => { return { type: String, - typeof: 'string', wrap: true, value: `${value}`, } } +export const writeStreamData = async (stream, writeStreamData) => { + const readStream = await writeStreamData() + for await (const chunk of readStream) { + stream.write(chunk) + } +} + let transformLabels = _.map(_.get('label')) export default ({ @@ -56,8 +61,8 @@ export default ({ await onWrite({ recordsWritten }) } let columns = _.map( - () => ({ - width: 50, + (column) => ({ + width: column.value.length || 50, }), excelData[1] ) @@ -65,7 +70,7 @@ export default ({ stream, async () => await writeXlsxFile(excelData, { columns }) ) - await onWrite({ recordsWritten: recordsWritten, isStreamDone: true }) + await onWrite({ recordsWritten, isStreamDone: true }) await stream.end() })(), cancel() { diff --git a/packages/export/src/excel.test.js b/packages/export/src/excel.test.js index bcede7efe..f6acd2578 100644 --- a/packages/export/src/excel.test.js +++ b/packages/export/src/excel.test.js @@ -1,5 +1,5 @@ import { PassThrough } from 'stream' -import { writeStreamData } from './utils.js' +import { writeStreamData } from './excel.js' const testData = `Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.` diff --git a/packages/export/src/utils.js b/packages/export/src/utils.js index 48915361e..5b18f075f 100644 --- a/packages/export/src/utils.js +++ b/packages/export/src/utils.js @@ -1,13 +1,6 @@ import F from 'futil' import _ from 'lodash/fp.js' -export const writeStreamData = async (stream, writeStreamData) => { - const readStream = await writeStreamData() - for await (const chunk of readStream) { - stream.write(chunk) - } -} - let Tree = F.tree((x) => x.children) export let setFilterOnly = Tree.transform((x) => { x.filterOnly = true diff --git a/yarn.lock b/yarn.lock index 51128df9d..e2b79e458 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6233,13 +6233,6 @@ __metadata: languageName: node linkType: hard -"@xmldom/xmldom@npm:^0.8.2": - version: 0.8.10 - resolution: "@xmldom/xmldom@npm:0.8.10" - checksum: 4c136aec31fb3b49aaa53b6fcbfe524d02a1dc0d8e17ee35bd3bf35e9ce1344560481cd1efd086ad1a4821541482528672306d5e37cdbd187f33d7fadd3e2cf0 - languageName: node - linkType: hard - "@xtuc/ieee754@npm:^1.2.0": version: 1.2.0 resolution: "@xtuc/ieee754@npm:1.2.0" @@ -7068,13 +7061,6 @@ __metadata: languageName: node linkType: hard -"big-integer@npm:^1.6.17": - version: 1.6.52 - resolution: "big-integer@npm:1.6.52" - checksum: 6e86885787a20fed96521958ae9086960e4e4b5e74d04f3ef7513d4d0ad631a9f3bde2730fc8aaa4b00419fc865f6ec573e5320234531ef37505da7da192c40b - languageName: node - linkType: hard - "big-integer@npm:^1.6.44": version: 1.6.51 resolution: "big-integer@npm:1.6.51" @@ -7096,16 +7082,6 @@ __metadata: languageName: node linkType: hard -"binary@npm:~0.3.0": - version: 0.3.0 - resolution: "binary@npm:0.3.0" - dependencies: - buffers: ~0.1.1 - chainsaw: ~0.1.0 - checksum: b4699fda9e2c2981e74a46b0115cf0d472eda9b68c0e9d229ef494e92f29ce81acf0a834415094cffcc340dfee7c4ef8ce5d048c65c18067a7ed850323f777af - languageName: node - linkType: hard - "bl@npm:^4.0.3, bl@npm:^4.1.0": version: 4.1.0 resolution: "bl@npm:4.1.0" @@ -7124,13 +7100,6 @@ __metadata: languageName: node linkType: hard -"bluebird@npm:~3.4.1": - version: 3.4.7 - resolution: "bluebird@npm:3.4.7" - checksum: bffa9dee7d3a41ab15c4f3f24687b49959b4e64e55c058a062176feb8ccefc2163414fb4e1a0f3053bf187600936509660c3ebd168fd9f0e48c7eba23b019466 - languageName: node - linkType: hard - "body-parser@npm:1.20.1": version: 1.20.1 resolution: "body-parser@npm:1.20.1" @@ -7306,13 +7275,6 @@ __metadata: languageName: node linkType: hard -"buffer-indexof-polyfill@npm:~1.0.0": - version: 1.0.2 - resolution: "buffer-indexof-polyfill@npm:1.0.2" - checksum: fbfb2d69c6bb2df235683126f9dc140150c08ac3630da149913a9971947b667df816a913b6993bc48f4d611999cb99a1589914d34c02dccd2234afda5cb75bbc - languageName: node - linkType: hard - "buffer@npm:^5.5.0": version: 5.7.1 resolution: "buffer@npm:5.7.1" @@ -7333,13 +7295,6 @@ __metadata: languageName: node linkType: hard -"buffers@npm:~0.1.1": - version: 0.1.1 - resolution: "buffers@npm:0.1.1" - checksum: ad6f8e483efab39cefd92bdc04edbff6805e4211b002f4d1cfb70c6c472a61cc89fb18c37bcdfdd4ee416ca096e9ff606286698a7d41a18b539bac12fd76d4d5 - languageName: node - linkType: hard - "bytes@npm:3.0.0": version: 3.0.0 resolution: "bytes@npm:3.0.0" @@ -7504,15 +7459,6 @@ __metadata: languageName: node linkType: hard -"chainsaw@npm:~0.1.0": - version: 0.1.0 - resolution: "chainsaw@npm:0.1.0" - dependencies: - traverse: ">=0.3.0 <0.4" - checksum: 22a96b9fb0cd9fb20813607c0869e61817d1acc81b5d455cc6456b5e460ea1dd52630e0f76b291cf8294bfb6c1fc42e299afb52104af9096242699d6d3aa6d3e - languageName: node - linkType: hard - "chalk@npm:^2.0.0, chalk@npm:^2.1.0, chalk@npm:^2.3.0, chalk@npm:^2.4.2": version: 2.4.2 resolution: "chalk@npm:2.4.2" @@ -7978,7 +7924,6 @@ __metadata: futil: ^1.76.4 lodash: ^4.17.21 minimal-csv-formatter: ^1.0.15 - read-excel-file: ^5.8.1 write-excel-file: ^2.0.1 languageName: unknown linkType: soft @@ -8821,15 +8766,6 @@ __metadata: languageName: node linkType: hard -"duplexer2@npm:~0.1.4": - version: 0.1.4 - resolution: "duplexer2@npm:0.1.4" - dependencies: - readable-stream: ^2.0.2 - checksum: 744961f03c7f54313f90555ac20284a3fb7bf22fdff6538f041a86c22499560eb6eac9d30ab5768054137cb40e6b18b40f621094e0261d7d8c35a37b7a5ad241 - languageName: node - linkType: hard - "duplexify@npm:^3.5.0, duplexify@npm:^3.6.0": version: 3.7.1 resolution: "duplexify@npm:3.7.1" @@ -9902,13 +9838,6 @@ __metadata: languageName: node linkType: hard -"fflate@npm:^0.7.3": - version: 0.7.4 - resolution: "fflate@npm:0.7.4" - checksum: b812ab26047432db70ff4c73eb45ad53bd0774575b4818b9c61c2921e89ec65d1259f06ec1618f2ac55e6a2f2e29b6dc09173d213b46580bc69efae5344bf8f1 - languageName: node - linkType: hard - "file-entry-cache@npm:^6.0.1": version: 6.0.1 resolution: "file-entry-cache@npm:6.0.1" @@ -10292,18 +10221,6 @@ __metadata: languageName: node linkType: hard -"fstream@npm:^1.0.12": - version: 1.0.12 - resolution: "fstream@npm:1.0.12" - dependencies: - graceful-fs: ^4.1.2 - inherits: ~2.0.0 - mkdirp: ">=0.5 0" - rimraf: 2 - checksum: e6998651aeb85fd0f0a8a68cec4d05a3ada685ecc4e3f56e0d063d0564a4fc39ad11a856f9020f926daf869fc67f7a90e891def5d48e4cadab875dc313094536 - languageName: node - linkType: hard - "function-bind@npm:^1.1.1": version: 1.1.1 resolution: "function-bind@npm:1.1.1" @@ -10697,13 +10614,6 @@ __metadata: languageName: node linkType: hard -"graceful-fs@npm:^4.2.2": - version: 4.2.11 - resolution: "graceful-fs@npm:4.2.11" - checksum: ac85f94da92d8eb6b7f5a8b20ce65e43d66761c55ce85ac96df6865308390da45a8d3f0296dd3a663de65d30ba497bd46c696cc1e248c72b13d6d567138a4fc7 - languageName: node - linkType: hard - "grapheme-splitter@npm:^1.0.4": version: 1.0.4 resolution: "grapheme-splitter@npm:1.0.4" @@ -11150,7 +11060,7 @@ __metadata: languageName: node linkType: hard -"inherits@npm:2, inherits@npm:2.0.4, inherits@npm:^2.0.1, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.0, inherits@npm:~2.0.3": +"inherits@npm:2, inherits@npm:2.0.4, inherits@npm:^2.0.1, inherits@npm:^2.0.3, inherits@npm:^2.0.4, inherits@npm:~2.0.3": version: 2.0.4 resolution: "inherits@npm:2.0.4" checksum: 4a48a733847879d6cf6691860a6b1e3f0f4754176e4d71494c41f3475553768b10f84b5ce1d40fbd0e34e6bfbb864ee35858ad4dd2cf31e02fc4a154b724d7f1 @@ -12615,13 +12525,6 @@ __metadata: languageName: node linkType: hard -"listenercount@npm:~1.0.1": - version: 1.0.1 - resolution: "listenercount@npm:1.0.1" - checksum: 0f1c9077cdaf2ebc16473c7d72eb7de6d983898ca42500f03da63c3914b6b312dd5f7a90d2657691ea25adf3fe0ac5a43226e8b2c673fd73415ed038041f4757 - languageName: node - linkType: hard - "load-yaml-file@npm:^0.2.0": version: 0.2.0 resolution: "load-yaml-file@npm:0.2.0" @@ -13307,7 +13210,7 @@ __metadata: languageName: node linkType: hard -"mkdirp@npm:>=0.5 0, mkdirp@npm:^0.5.1, mkdirp@npm:^0.5.4": +"mkdirp@npm:^0.5.1, mkdirp@npm:^0.5.4": version: 0.5.6 resolution: "mkdirp@npm:0.5.6" dependencies: @@ -15181,17 +15084,6 @@ __metadata: languageName: node linkType: hard -"read-excel-file@npm:^5.8.1": - version: 5.8.1 - resolution: "read-excel-file@npm:5.8.1" - dependencies: - "@xmldom/xmldom": ^0.8.2 - fflate: ^0.7.3 - unzipper: ^0.10.11 - checksum: ef0e8aa69112908e1a2aca4353e0dd6093407e2254a88f71273ad31352f6ff57b862b7587e21d7669399e715363c400cc499eba1221dd11357072ab86dbd8362 - languageName: node - linkType: hard - "read-pkg-up@npm:^7.0.1": version: 7.0.1 resolution: "read-pkg-up@npm:7.0.1" @@ -15227,7 +15119,7 @@ __metadata: languageName: node linkType: hard -"readable-stream@npm:^2.0.0, readable-stream@npm:^2.0.2, readable-stream@npm:^2.0.5, readable-stream@npm:^2.2.2, readable-stream@npm:~2.3.6": +"readable-stream@npm:^2.0.0, readable-stream@npm:^2.0.5, readable-stream@npm:^2.2.2, readable-stream@npm:~2.3.6": version: 2.3.8 resolution: "readable-stream@npm:2.3.8" dependencies: @@ -15659,7 +15551,7 @@ __metadata: languageName: node linkType: hard -"rimraf@npm:2, rimraf@npm:^2.6.1": +"rimraf@npm:^2.6.1": version: 2.7.1 resolution: "rimraf@npm:2.7.1" dependencies: @@ -15891,7 +15783,7 @@ __metadata: languageName: node linkType: hard -"setimmediate@npm:^1.0.5, setimmediate@npm:~1.0.4": +"setimmediate@npm:^1.0.5": version: 1.0.5 resolution: "setimmediate@npm:1.0.5" checksum: c9a6f2c5b51a2dabdc0247db9c46460152ffc62ee139f3157440bd48e7c59425093f42719ac1d7931f054f153e2d26cf37dfeb8da17a794a58198a2705e527fd @@ -16798,13 +16690,6 @@ __metadata: languageName: node linkType: hard -"traverse@npm:>=0.3.0 <0.4": - version: 0.3.9 - resolution: "traverse@npm:0.3.9" - checksum: 982982e4e249e9bbf063732a41fe5595939892758524bbef5d547c67cdf371b13af72b5434c6a61d88d4bb4351d6dabc6e22d832e0d16bc1bc684ef97a1cc59e - languageName: node - linkType: hard - "trim-newlines@npm:^3.0.0": version: 3.0.1 resolution: "trim-newlines@npm:3.0.1" @@ -17151,24 +17036,6 @@ __metadata: languageName: node linkType: hard -"unzipper@npm:^0.10.11": - version: 0.10.14 - resolution: "unzipper@npm:0.10.14" - dependencies: - big-integer: ^1.6.17 - binary: ~0.3.0 - bluebird: ~3.4.1 - buffer-indexof-polyfill: ~1.0.0 - duplexer2: ~0.1.4 - fstream: ^1.0.12 - graceful-fs: ^4.2.2 - listenercount: ~1.0.1 - readable-stream: ~2.3.6 - setimmediate: ~1.0.4 - checksum: b46ae9a72e4b4c224be6a8f46447dd7cb3761a59450827e869747c4564a8f555f877fc19c7e3b5d146127a7dd3e2ffea186116682f6646e64479f99dd23565bc - languageName: node - linkType: hard - "update-browserslist-db@npm:^1.0.10, update-browserslist-db@npm:^1.0.9": version: 1.0.10 resolution: "update-browserslist-db@npm:1.0.10" From 9f106e586681c81cd09dc49417c983df1b0d5c51 Mon Sep 17 00:00:00 2001 From: Henry Thornley Date: Mon, 20 May 2024 11:29:03 -0700 Subject: [PATCH 08/11] Fixes --- packages/export/src/excel.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/export/src/excel.js b/packages/export/src/excel.js index 0e7abcac7..841aa19bd 100644 --- a/packages/export/src/excel.js +++ b/packages/export/src/excel.js @@ -3,9 +3,8 @@ import writeXlsxFile from 'write-excel-file/node' const convertToExcelCell = (value) => { return { - type: String, wrap: true, - value: `${value}`, + value: value ? `${value}` : ``, } } @@ -64,7 +63,7 @@ export default ({ (column) => ({ width: column.value.length || 50, }), - excelData[1] + excelData[0] ) await writeStreamData( stream, From c66b8e16461adccb2eebda3e6e278e99acc4d871 Mon Sep 17 00:00:00 2001 From: Henry Thornley Date: Mon, 20 May 2024 11:53:19 -0700 Subject: [PATCH 09/11] Smart Column Width - Added smart column width - Added Sticky Header - Added Background Color like CSV for Header --- packages/export/src/excel.js | 52 ++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/packages/export/src/excel.js b/packages/export/src/excel.js index 841aa19bd..3f83f5b38 100644 --- a/packages/export/src/excel.js +++ b/packages/export/src/excel.js @@ -1,4 +1,5 @@ import _ from 'lodash/fp.js' +import F from 'futil' import writeXlsxFile from 'write-excel-file/node' const convertToExcelCell = (value) => { @@ -16,6 +17,7 @@ export const writeStreamData = async (stream, writeStreamData) => { } let transformLabels = _.map(_.get('label')) +let maxColumnWidth = 200 export default ({ stream, // writable stream target stream @@ -31,43 +33,53 @@ export default ({ }) => { const excelData = [ _.map( - (value) => ({ value, fontWeight: 'bold' }), + (value) => ({ value, fontWeight: 'bold', backgroundColor: '#999999' }), headers || transformLabels(transform) ), ] let cancel = false let recordsWritten = 0 + let columnWidths = _.map( + (column) => ({ width: column.value.length }), + excelData[0] + ) return { promise: (async () => { for await (let r of iterableData) { if (cancel) break - excelData.push( - _.map( - (t) => - convertToExcelCell( - t.display(_.get(t.key, r), { - key: t.key, - record: r, - transform, - }) - ), - transform - ) + let row = _.map( + (t) => + convertToExcelCell( + t.display(_.get(t.key, r), { + key: t.key, + record: r, + transform, + }) + ), + transform ) + columnWidths = F.mapIndexed( + (value, index) => ({ + width: Math.min( + Math.max(value.width, row[index].value.length), + maxColumnWidth + ), + }), + columnWidths + ) + excelData.push(row) recordsWritten = recordsWritten + _.getOr(1, 'recordCount', r) await onWrite({ recordsWritten }) } - let columns = _.map( - (column) => ({ - width: column.value.length || 50, - }), - excelData[0] - ) await writeStreamData( stream, - async () => await writeXlsxFile(excelData, { columns }) + async () => + await writeXlsxFile(excelData, { + columns: columnWidths, + stickyRowsCount: 1, + }) ) await onWrite({ recordsWritten, isStreamDone: true }) await stream.end() From 9c6fa9b6c414fa1330477daf63314b05a3b69c3b Mon Sep 17 00:00:00 2001 From: Henry Thornley Date: Mon, 20 May 2024 12:07:36 -0700 Subject: [PATCH 10/11] Add Index Column BG Color --- packages/export/src/excel.js | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/packages/export/src/excel.js b/packages/export/src/excel.js index 3f83f5b38..4d4f8428d 100644 --- a/packages/export/src/excel.js +++ b/packages/export/src/excel.js @@ -2,10 +2,16 @@ import _ from 'lodash/fp.js' import F from 'futil' import writeXlsxFile from 'write-excel-file/node' -const convertToExcelCell = (value) => { +let transformLabels = _.map(_.get('label')) +let maxColumnWidth = 200 +let headerBackgroundColor = '#999999' +let indexColumnBackgroundColor = '#bbbbbb' + +const convertToExcelCell = (value, index) => { return { wrap: true, value: value ? `${value}` : ``, + ...((index === 0 && { backgroundColor: indexColumnBackgroundColor }) || {}), } } @@ -16,9 +22,6 @@ export const writeStreamData = async (stream, writeStreamData) => { } } -let transformLabels = _.map(_.get('label')) -let maxColumnWidth = 200 - export default ({ stream, // writable stream target stream iterableData, // iterator for each page of an array of objects @@ -33,14 +36,18 @@ export default ({ }) => { const excelData = [ _.map( - (value) => ({ value, fontWeight: 'bold', backgroundColor: '#999999' }), + (value) => ({ + value, + fontWeight: 'bold', + backgroundColor: headerBackgroundColor, + }), headers || transformLabels(transform) ), ] let cancel = false let recordsWritten = 0 - let columnWidths = _.map( + let columns = _.map( (column) => ({ width: column.value.length }), excelData[0] ) @@ -49,25 +56,26 @@ export default ({ promise: (async () => { for await (let r of iterableData) { if (cancel) break - let row = _.map( - (t) => + let row = F.mapIndexed( + (data, index) => convertToExcelCell( - t.display(_.get(t.key, r), { - key: t.key, + data.display(_.get(data.key, r), { + key: data.key, record: r, transform, - }) + }), + index ), transform ) - columnWidths = F.mapIndexed( + columns = F.mapIndexed( (value, index) => ({ width: Math.min( Math.max(value.width, row[index].value.length), maxColumnWidth ), }), - columnWidths + columns ) excelData.push(row) recordsWritten = recordsWritten + _.getOr(1, 'recordCount', r) @@ -77,7 +85,7 @@ export default ({ stream, async () => await writeXlsxFile(excelData, { - columns: columnWidths, + columns, stickyRowsCount: 1, }) ) From 842095b2e5e206d45ff2a98ad989cc915484c96f Mon Sep 17 00:00:00 2001 From: Henry Thornley Date: Mon, 20 May 2024 15:01:35 -0700 Subject: [PATCH 11/11] Update Tests --- packages/export/src/excel.js | 25 +++---- packages/export/src/excel.test.js | 114 +++++++++++++++++++++++++----- 2 files changed, 108 insertions(+), 31 deletions(-) diff --git a/packages/export/src/excel.js b/packages/export/src/excel.js index 4d4f8428d..6e4edb2d7 100644 --- a/packages/export/src/excel.js +++ b/packages/export/src/excel.js @@ -15,15 +15,9 @@ const convertToExcelCell = (value, index) => { } } -export const writeStreamData = async (stream, writeStreamData) => { - const readStream = await writeStreamData() - for await (const chunk of readStream) { - stream.write(chunk) - } -} - export default ({ stream, // writable stream target stream + readStreamData = async (data, options) => await writeXlsxFile(data, options), iterableData, // iterator for each page of an array of objects // order list of which indicates the header label, // display function for the field, @@ -81,14 +75,15 @@ export default ({ recordsWritten = recordsWritten + _.getOr(1, 'recordCount', r) await onWrite({ recordsWritten }) } - await writeStreamData( - stream, - async () => - await writeXlsxFile(excelData, { - columns, - stickyRowsCount: 1, - }) - ) + + const readStream = await readStreamData(excelData, { + columns, + stickyRowsCount: 1, + }) + for await (const chunk of readStream) { + stream.write(chunk) + } + await onWrite({ recordsWritten, isStreamDone: true }) await stream.end() })(), diff --git a/packages/export/src/excel.test.js b/packages/export/src/excel.test.js index f6acd2578..ba1726623 100644 --- a/packages/export/src/excel.test.js +++ b/packages/export/src/excel.test.js @@ -1,25 +1,107 @@ import { PassThrough } from 'stream' -import { writeStreamData } from './excel.js' +import _ from 'lodash/fp.js' +import excel from './excel.js' -const testData = `Lorem ipsum dolor sit amet, consectetur adipiscing elit, - sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.` -const readStream = new PassThrough() -readStream.end(testData) +let iterableData = [ + { name: 'record1', value: 1, nestedValue: { value: 'a' } }, + { name: 'record2', value: 2, nestedValue: { value: 'b' } }, + { name: 'record3', value: 3, nestedValue: { value: 'c' } }, +] -const destinationStream = new PassThrough() +let headers = ['NAME', 'Value', 'Nested Value'] -describe('Stream write test', () => { - it('writeStreamData()', async () => { - let outputData = '' - // Capture the data written as string - destinationStream.on('data', (chunk) => { - outputData += chunk.toString() +let transform = [ + { key: 'name', label: 'THE,NAME', display: _.capitalize }, + { key: 'value', label: 'Value', display: _.identity }, + { + key: 'nestedValue.value', + label: 'Value RecordName Key TransformLength', + display: (value, { key, record, transform }) => + `${value} ${record.name} ${key} ${transform.length}`, + }, +] + +let expectExcelData = [ + [ + { backgroundColor: '#999999', fontWeight: 'bold', value: 'NAME' }, + { backgroundColor: '#999999', fontWeight: 'bold', value: 'Value' }, + { + backgroundColor: '#999999', + fontWeight: 'bold', + value: 'Nested Value', + }, + ], + [ + { backgroundColor: '#bbbbbb', value: 'Record1', wrap: true }, + { value: '1', wrap: true }, + { value: 'a record1 nestedValue.value 3', wrap: true }, + ], + [ + { backgroundColor: '#bbbbbb', value: 'Record2', wrap: true }, + { value: '2', wrap: true }, + { value: 'b record2 nestedValue.value 3', wrap: true }, + ], + [ + { backgroundColor: '#bbbbbb', value: 'Record3', wrap: true }, + { value: '3', wrap: true }, + { value: 'c record3 nestedValue.value 3', wrap: true }, + ], +] + +describe('Excel export tests', () => { + it('Excel data transformation test', async () => { + const readStream = new PassThrough() + const destinationStream = new PassThrough() + readStream.end('fake data') + + let finalizeData = new Promise((res, rej) => { + destinationStream.on('data', () => {}) + destinationStream.on('end', () => res()) + destinationStream.on('error', rej) }) - const mockWriteStreamData = async () => readStream - await writeStreamData(destinationStream, mockWriteStreamData) - destinationStream.end() + let resultData + + let writeData = async (data) => { + resultData = data + return readStream + } + + await excel({ + stream: destinationStream, + readStreamData: writeData, + iterableData, + transform, + headers, + }) + await finalizeData + expect(resultData).toStrictEqual(expectExcelData) + }) + it('Write stream data test', async () => { + const readStream = new PassThrough() + const destinationStream = new PassThrough() + readStream.end(`Lorem ipsum dolor sit amet, consectetur adipiscing elit, + sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.`) + + let destinationData = new Promise((res, rej) => { + let data = '' + destinationStream.on('data', (chunk) => { + data += chunk.toString() + }) + destinationStream.on('end', () => res(data)) + destinationStream.on('error', rej) + }) + + await excel({ + stream: destinationStream, + readStreamData: () => readStream, + iterableData, + transform, + headers, + }) - expect(outputData).toStrictEqual(testData) + expect(await destinationData) + .toStrictEqual(`Lorem ipsum dolor sit amet, consectetur adipiscing elit, + sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.`) }) })