Enable Export with Excel Format

.changeset/silly-months-unite.md

@@ -0,0 +1,5 @@
+---
+'contexture-export': minor
+---
+
+Enable Excel Export Option

packages/export/package.json

@@ -34,7 +34,8 @@
     "contexture-client": "^2.53.7",
     "futil": "^1.76.4",
     "lodash": "^4.17.21",
-    "minimal-csv-formatter": "^1.0.15"
+    "minimal-csv-formatter": "^1.0.15",
+    "write-excel-file": "^2.0.1"
   },
   "packageManager": "yarn@3.3.1"
 }

packages/export/src/csv.js

@@ -11,7 +11,8 @@ export default ({
   // and key of the record.
   // [{ key: string(supports lodash dot notation), label: string, display: function(value, {key, record, transform})}...]
   transform,
-  headers = null, // array of strings to use as headers, array or arrays for multi-line headers
+  //TODO: support multi-line headers in excel and csv when implemented
+  headers = null, // array of strings to use as headers
   onWrite = _.noop, // function to intercept writing a page of records
 }) => {
   stream.write(csv(headers || transformLabels(transform)))
@@ -39,6 +40,7 @@ export default ({
         recordsWritten = recordsWritten + _.getOr(1, 'recordCount', r)
         await onWrite({ recordsWritten, record: r })
       }
+      await onWrite({ recordsWritten, isStreamDone: true })
       await stream.end()
     })(),
     cancel() {

packages/export/src/excel.js

@@ -0,0 +1,94 @@
+import _ from 'lodash/fp.js'
+import F from 'futil'
+import writeXlsxFile from 'write-excel-file/node'
+
+let transformLabels = _.map(_.get('label'))
+let maxColumnWidth = 200
+let headerBackgroundColor = '#999999'
+let indexColumnBackgroundColor = '#bbbbbb'
+
+const convertToExcelCell = (value, index) => {
+  return {
+    wrap: true,
+    value: value ? `${value}` : ``,
+    ...((index === 0 && { backgroundColor: indexColumnBackgroundColor }) || {}),
+  }
+}
+
+export default ({
+  stream, // writable stream target stream
+  readStreamData = async (data, options) => await writeXlsxFile(data, options),
+  iterableData, // iterator for each page of an array of objects
+  // order list of which indicates the header label,
+  // display function for the field,
+  // and key of the record.
+  // [{ key: string(supports lodash dot notation), label: string, display: function(value, {key, record, transform})}...]
+  transform,
+  //TODO: support multi-line headers in excel and csv when implemented
+  headers = null, // array of strings to use as headers
+  onWrite = _.noop, // function to intercept writing a page of records
+}) => {
+  const excelData = [
+    _.map(
+      (value) => ({
+        value,
+        fontWeight: 'bold',
+        backgroundColor: headerBackgroundColor,
+      }),
+      headers || transformLabels(transform)
+    ),
+  ]
+
+  let cancel = false
+  let recordsWritten = 0
+  let columns = _.map(
+    (column) => ({ width: column.value.length }),
+    excelData[0]
+  )
+
+  return {
+    promise: (async () => {
+      for await (let r of iterableData) {
+        if (cancel) break
+        let row = F.mapIndexed(
+          (data, index) =>
+            convertToExcelCell(
+              data.display(_.get(data.key, r), {
+                key: data.key,
+                record: r,
+                transform,
+              }),
+              index
+            ),
+          transform
+        )
+        columns = F.mapIndexed(
+          (value, index) => ({
+            width: Math.min(
+              Math.max(value.width, row[index].value.length),
+              maxColumnWidth
+            ),
+          }),
+          columns
+        )
+        excelData.push(row)
+        recordsWritten = recordsWritten + _.getOr(1, 'recordCount', r)
+        await onWrite({ recordsWritten })
+      }
+
+      const readStream = await readStreamData(excelData, {
+        columns,
+        stickyRowsCount: 1,
+      })
+      for await (const chunk of readStream) {
+        stream.write(chunk)
+      }
+
+      await onWrite({ recordsWritten, isStreamDone: true })
+      await stream.end()
+    })(),
+    cancel() {
+      cancel = true
+    },
+  }
+}

packages/export/src/excel.test.js

@@ -0,0 +1,107 @@
+import { PassThrough } from 'stream'
+import _ from 'lodash/fp.js'
+import excel from './excel.js'
+
+let iterableData = [
+  { name: 'record1', value: 1, nestedValue: { value: 'a' } },
+  { name: 'record2', value: 2, nestedValue: { value: 'b' } },
+  { name: 'record3', value: 3, nestedValue: { value: 'c' } },
+]
+
+let headers = ['NAME', 'Value', 'Nested Value']
+
+let transform = [
+  { key: 'name', label: 'THE,NAME', display: _.capitalize },
+  { key: 'value', label: 'Value', display: _.identity },
+  {
+    key: 'nestedValue.value',
+    label: 'Value RecordName Key TransformLength',
+    display: (value, { key, record, transform }) =>
+      `${value} ${record.name} ${key} ${transform.length}`,
+  },
+]
+
+let expectExcelData = [
+  [
+    { backgroundColor: '#999999', fontWeight: 'bold', value: 'NAME' },
+    { backgroundColor: '#999999', fontWeight: 'bold', value: 'Value' },
+    {
+      backgroundColor: '#999999',
+      fontWeight: 'bold',
+      value: 'Nested Value',
+    },
+  ],
+  [
+    { backgroundColor: '#bbbbbb', value: 'Record1', wrap: true },
+    { value: '1', wrap: true },
+    { value: 'a record1 nestedValue.value 3', wrap: true },
+  ],
+  [
+    { backgroundColor: '#bbbbbb', value: 'Record2', wrap: true },
+    { value: '2', wrap: true },
+    { value: 'b record2 nestedValue.value 3', wrap: true },
+  ],
+  [
+    { backgroundColor: '#bbbbbb', value: 'Record3', wrap: true },
+    { value: '3', wrap: true },
+    { value: 'c record3 nestedValue.value 3', wrap: true },
+  ],
+]
+
+describe('Excel export tests', () => {
+  it('Excel data transformation test', async () => {
+    const readStream = new PassThrough()
+    const destinationStream = new PassThrough()
+    readStream.end('fake data')
+
+    let finalizeData = new Promise((res, rej) => {
+      destinationStream.on('data', () => {})
+      destinationStream.on('end', () => res())
+      destinationStream.on('error', rej)
+    })
+
+    let resultData
+
+    let writeData = async (data) => {
+      resultData = data
+      return readStream
+    }
+
+    await excel({
+      stream: destinationStream,
+      readStreamData: writeData,
+      iterableData,
+      transform,
+      headers,
+    })
+    await finalizeData
+    expect(resultData).toStrictEqual(expectExcelData)
+  })
+  it('Write stream data test', async () => {
+    const readStream = new PassThrough()
+    const destinationStream = new PassThrough()
+    readStream.end(`Lorem ipsum dolor sit amet, consectetur adipiscing elit, 
+      sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.`)
+
+    let destinationData = new Promise((res, rej) => {
+      let data = ''
+      destinationStream.on('data', (chunk) => {
+        data += chunk.toString()
+      })
+      destinationStream.on('end', () => res(data))
+      destinationStream.on('error', rej)
+    })
+
+    await excel({
+      stream: destinationStream,
+      readStreamData: () => readStream,
+      iterableData,
+      transform,
+      headers,
+    })
+
+    expect(await destinationData)
+      .toStrictEqual(`Lorem ipsum dolor sit amet, consectetur adipiscing elit, 
+      sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.`)
+  })
+})

packages/export/src/index.js

@@ -1,5 +1,6 @@
 import csv from './csv.js'
+import excel from './excel.js'
 import * as nodes from './nodes/index.js'
 
-export { nodes, csv }
+export { nodes, csv, excel }
 export * from './utils.js'