diff --git a/apps/api/package.json b/apps/api/package.json index cff01fd70..30524aba5 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -19,9 +19,10 @@ "test": "cross-env TZ=UTC NODE_ENV=test E2E_RUNNER=true mocha --timeout 10000 --require ts-node/register --exit src/**/**/*.spec.ts" }, "dependencies": { - "@impler/dal": "^0.24.1", - "@impler/services": "^0.24.1", - "@impler/shared": "^0.24.1", + "@eyeseetea/xlsx-populate": "^4.3.0", + "@impler/dal": "workspace:^", + "@impler/services": "workspace:^", + "@impler/shared": "workspace:^", "@nestjs/common": "^9.1.2", "@nestjs/core": "^9.1.2", "@nestjs/jwt": "^10.0.1", diff --git a/apps/api/src/app/shared/services/file/file.service.ts b/apps/api/src/app/shared/services/file/file.service.ts index ad5cbb220..31dad2972 100644 --- a/apps/api/src/app/shared/services/file/file.service.ts +++ b/apps/api/src/app/shared/services/file/file.service.ts @@ -1,6 +1,6 @@ import * as XLSX from 'xlsx'; import { cwd } from 'node:process'; -import * as xlsxPopulate from 'xlsx-populate'; +import * as xlsxPopulate from '@eyeseetea/xlsx-populate'; import { CONSTANTS } from '@shared/constants'; import { ParseConfig, parse } from 'papaparse'; import { ColumnDelimiterEnum, ColumnTypesEnum, Defaults, FileEncodingsEnum } from '@impler/shared'; @@ -90,6 +90,14 @@ export class ExcelFileService { multiSelectHeadings[heading.key] = heading.delimiter || ColumnDelimiterEnum.COMMA; } else worksheet.cell(columnHeadingCellName).value(heading.key); worksheet.column(columnName).style('numberFormat', '@'); + if (heading.description) + worksheet.cell(columnHeadingCellName).comment({ + text: heading.description, + width: '200px', + height: '100px', + textAlign: 'left', + horizontalAlignment: 'Left', + }); }); const frozenColumns = headings.filter((heading) => heading.isFrozen).length; @@ -139,7 +147,7 @@ export class ExcelFileService { } const buffer = await workbook.outputAsync(); - return buffer as Promise; + return buffer; } getExcelSheets(file: Express.Multer.File): Promise { return new Promise(async (resolve, reject) => { diff --git a/apps/api/src/app/shared/services/file/old-file.service.ts b/apps/api/src/app/shared/services/file/old-file.service.ts new file mode 100644 index 000000000..ad5cbb220 --- /dev/null +++ b/apps/api/src/app/shared/services/file/old-file.service.ts @@ -0,0 +1,261 @@ +import * as XLSX from 'xlsx'; +import { cwd } from 'node:process'; +import * as xlsxPopulate from 'xlsx-populate'; +import { CONSTANTS } from '@shared/constants'; +import { ParseConfig, parse } from 'papaparse'; +import { ColumnDelimiterEnum, ColumnTypesEnum, Defaults, FileEncodingsEnum } from '@impler/shared'; +import { EmptyFileException } from '@shared/exceptions/empty-file.exception'; +import { InvalidFileException } from '@shared/exceptions/invalid-file.exception'; +import { IExcelFileHeading } from '@shared/types/file.types'; + +export class ExcelFileService { + async convertToCsv(file: Express.Multer.File, sheetName?: string): Promise { + return new Promise(async (resolve, reject) => { + try { + const wb = XLSX.read(file.buffer); + const ws = sheetName && wb.SheetNames.includes(sheetName) ? wb.Sheets[sheetName] : wb.Sheets[wb.SheetNames[0]]; + resolve( + XLSX.utils.sheet_to_csv(ws, { + blankrows: false, + skipHidden: true, + forceQuotes: true, + // rawNumbers: true, // was converting 12:12:12 to 1.3945645673 + }) + ); + } catch (error) { + reject(error); + } + }); + } + formatName(name: string): string { + return ( + CONSTANTS.EXCEL_DATA_SHEET_STARTER + + name + .replace(/[^a-zA-Z0-9]/g, '') + .toLowerCase() + .slice(0, 25) // exceljs don't allow heading more than 30 characters + ); + } + addSelectSheet(wb: any, heading: IExcelFileHeading): string { + const name = this.formatName(heading.key); + const addedSheet = wb.addSheet(name); + addedSheet.cell('A1').value(heading.key); + heading.selectValues.forEach((value, index) => addedSheet.cell(`A${index + 2}`).value(value)); + + return name; + } + + getExcelColumnNameFromIndex(columnNumber: number) { + // To store result (Excel column name) + const columnName = []; + + while (columnNumber > 0) { + // Find remainder + const rem = columnNumber % 26; + + /* + * If remainder is 0, then a + * 'Z' must be there in output + */ + if (rem == 0) { + columnName.push('Z'); + columnNumber = Math.floor(columnNumber / 26) - 1; + } // If remainder is non-zero + else { + columnName.push(String.fromCharCode(rem - 1 + 'A'.charCodeAt(0))); + columnNumber = Math.floor(columnNumber / 26); + } + } + + return columnName.reverse().join(''); + } + async getExcelFileForHeadings(headings: IExcelFileHeading[], data?: string): Promise { + const currentDir = cwd(); + const isMultiSelect = headings.some( + (heading) => heading.type === ColumnTypesEnum.SELECT && heading.allowMultiSelect + ); + const workbook = await xlsxPopulate.fromFileAsync( + `${currentDir}/src/config/${isMultiSelect ? 'Excel Multi Select Template.xlsm' : 'Excel Template.xlsx'}` + ); + const worksheet = workbook.sheet('Data'); + const multiSelectHeadings = {}; + + headings.forEach((heading, index) => { + const columnName = this.getExcelColumnNameFromIndex(index + 1); + const columnHeadingCellName = columnName + '1'; + if (heading.type === ColumnTypesEnum.SELECT && heading.allowMultiSelect) { + worksheet + .cell(columnHeadingCellName) + .value(heading.key + '#MULTI' + '#' + (heading.delimiter || ColumnDelimiterEnum.COMMA)); + multiSelectHeadings[heading.key] = heading.delimiter || ColumnDelimiterEnum.COMMA; + } else worksheet.cell(columnHeadingCellName).value(heading.key); + worksheet.column(columnName).style('numberFormat', '@'); + }); + + const frozenColumns = headings.filter((heading) => heading.isFrozen).length; + if (frozenColumns) worksheet.freezePanes(frozenColumns, 1); // freeze panes (freeze first n column and first row) + else worksheet.freezePanes(0, 1); // freeze 0 column and first row + + headings.forEach((heading, index) => { + if (heading.type === ColumnTypesEnum.SELECT) { + const keyName = this.addSelectSheet(workbook, heading); + const columnName = this.getExcelColumnNameFromIndex(index + 1); + worksheet.range(`${columnName}2:${columnName}9999`).dataValidation({ + type: 'list', + allowBlank: !heading.isRequired, + formula1: `${keyName}!$A$2:$A$9999`, + ...(!heading.allowMultiSelect + ? { + showErrorMessage: true, + error: 'Please select from the list', + errorTitle: 'Invalid Value', + } + : {}), + }); + } + }); + const headingNames = headings.map((heading) => heading.key); + const endColumnPosition = this.getExcelColumnNameFromIndex(headings.length + 1); + + let parsedData = []; + try { + if (data) parsedData = JSON.parse(data); + } catch (error) {} + if (Array.isArray(parsedData) && parsedData.length > 0) { + const rows: string[][] = parsedData.reduce((acc: string[][], rowItem: Record) => { + acc.push( + headingNames.map((headingKey) => + multiSelectHeadings[headingKey] && Array.isArray(rowItem[headingKey]) + ? rowItem[headingKey].join(multiSelectHeadings[headingKey]) + : rowItem[headingKey] + ) + ); + + return acc; + }, []); + const rangeKey = `A2:${endColumnPosition}${rows.length + 1}`; + const range = workbook.sheet(0).range(rangeKey); + range.value(rows); + } + const buffer = await workbook.outputAsync(); + + return buffer as Promise; + } + getExcelSheets(file: Express.Multer.File): Promise { + return new Promise(async (resolve, reject) => { + try { + const wb = XLSX.read(file.buffer); + resolve(wb.SheetNames); + } catch (error) { + reject(error); + } + }); + } + getExcelRowsColumnsCount(file: Express.Multer.File, sheetName?: string): Promise<{ rows: number; columns: number }> { + return new Promise(async (resolve, reject) => { + try { + const wb = XLSX.read(file.buffer); + const ws = sheetName && wb.SheetNames.includes(sheetName) ? wb.Sheets[sheetName] : wb.Sheets[wb.SheetNames[0]]; + const range = ws['!ref']; + const regex = /([A-Z]+)(\d+):([A-Z]+)(\d+)/; + const match = range.match(regex); + + if (!match) reject(new InvalidFileException()); + + const [, startCol, startRow, endCol, endRow] = match; + + function columnToNumber(col: string) { + let num = 0; + for (let i = 0; i < col.length; i++) { + num = num * 26 + (col.charCodeAt(i) - 64); + } + + return num; + } + + const columns = columnToNumber(endCol) - columnToNumber(startCol) + 1; + const rows = parseInt(endRow) - parseInt(startRow) + 1; + + resolve({ + columns, + rows, + }); + } catch (error) { + reject(error); + } + }); + } +} + +export class CSVFileService2 { + getCSVMetaInfo(file: string | Express.Multer.File, options?: ParseConfig) { + return new Promise<{ rows: number; columns: number }>((resolve, reject) => { + let fileContent = ''; + if (typeof file === 'string') { + fileContent = file; + } else { + fileContent = file.buffer.toString(FileEncodingsEnum.CSV); + } + let rows = 0; + let columns = 0; + + parse(fileContent, { + ...(options || {}), + dynamicTyping: false, + skipEmptyLines: true, + step: function (results) { + rows++; + if (Array.isArray(results.data)) { + columns = results.data.length; + } + }, + complete: function () { + resolve({ rows, columns }); + }, + error: (error) => { + if (error.message.includes('Parse Error')) { + reject(new InvalidFileException()); + } else { + reject(error); + } + }, + }); + }); + } + + getFileHeaders(file: string | Express.Multer.File, options?: ParseConfig): Promise { + return new Promise((resolve, reject) => { + let fileContent = ''; + if (typeof file === 'string') { + fileContent = file; + } else { + fileContent = file.buffer.toString(FileEncodingsEnum.CSV); + } + let headings: string[]; + let recordIndex = -1; + parse(fileContent, { + ...(options || {}), + preview: 2, + step: (results) => { + recordIndex++; + if (recordIndex === Defaults.ZERO) { + if (Array.isArray(results.data) && results.data.length > Defaults.ZERO) headings = results.data as string[]; + else reject(new EmptyFileException()); + } else resolve(headings); + }, + error: (error) => { + if (error.message.includes('Parse Error')) { + reject(new InvalidFileException()); + } else { + reject(error); + } + }, + complete: () => { + if (recordIndex !== Defaults.ONE) { + reject(new EmptyFileException()); + } + }, + }); + }); + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 40501c5c0..2afb829a3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -90,14 +90,17 @@ importers: apps/api: dependencies: + '@eyeseetea/xlsx-populate': + specifier: ^4.3.0 + version: 4.3.0 '@impler/dal': - specifier: ^0.24.1 + specifier: workspace:^ version: link:../../libs/dal '@impler/services': - specifier: ^0.24.1 + specifier: workspace:^ version: link:../../libs/services '@impler/shared': - specifier: ^0.24.1 + specifier: workspace:^ version: link:../../libs/shared '@nestjs/common': specifier: ^9.1.2 @@ -4182,6 +4185,15 @@ packages: resolution: {integrity: sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + /@eyeseetea/xlsx-populate@4.3.0: + resolution: {integrity: sha512-pUwHEobDsl/MlUP2MWpVBAnccKKpTiJW6dNpli49LeKoNbnVPFVI9AHlUWHzMMQruwmrujAx6Grxa7QLqyJyaA==} + dependencies: + cfb: 1.2.2 + jszip: 3.10.1 + lodash: 4.17.21 + sax: 1.4.1 + dev: false + /@floating-ui/core@1.6.4: resolution: {integrity: sha512-a4IowK4QkXl4SCWTGUR0INAfEOX3wtsYw3rKK5InQEHMGObkR8Xk44qYQD9P4r6HHw0iIfK6GUKECmY8sTkqRA==} dependencies: @@ -14661,7 +14673,7 @@ packages: resolution: {integrity: sha512-oWb1Z6mkHIskLzEJ/XWX0srkpkTQ7vaopMQkyaEIoq0fmtFVxOthb8cCxeT+p3ynTdkk/RZwbgG4brR5BeWECw==} engines: {node: '>= 4.0'} os: [darwin] - deprecated: The v1 package contains DANGEROUS / INSECURE binaries. Upgrade to safe fsevents v2 + deprecated: Upgrade to fsevents v2 to mitigate potential security issues requiresBuild: true dependencies: bindings: 1.5.0