From 77693e886e815dc506f0ded3f2a3115dbdcede80 Mon Sep 17 00:00:00 2001 From: naman-contentstack Date: Thu, 27 Aug 2026 15:04:05 +0530 Subject: [PATCH] feat(bulk-assets): retry publishing assets whose scan was pending MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assets still being scanned for malware were skipped on every publish run with only a console warning, leaving no way to publish them once the scan finished. Record those skips to pending-scan.json in the bulk operation folder, and add --retry-pending to re-check their status and publish the ones that came back clean. The log is self-pruning: a retry drops the entries it published along with any that turned quarantined, keeping only those still scanning. It is also excluded from clearLogs(), since a scan can outlast the gap between two unrelated runs. Quarantined assets are never retried, and the flag is rejected for unpublish, delete and move — scan status only gates publish. Co-Authored-By: Claude Opus 5 (1M context) --- .talismanrc | 8 +- .../contentstack-bulk-operations/README.md | 10 +- .../src/base-bulk-command.ts | 18 ++ .../src/commands/cm/stacks/bulk-assets.ts | 187 +++++++++++++++++- .../src/interfaces/index.ts | 21 ++ .../src/messages/index.ts | 6 + .../src/utils/bulk-operation-log-handler.ts | 63 +++++- .../src/utils/operation-flag-matrix.ts | 7 + .../unit/commands/bulk-assets-init.test.ts | 48 +++++ .../test/unit/commands/bulk-assets.test.ts | 128 ++++++++++++ .../utils/bulk-operation-log-handler.test.ts | 171 +++++++++++++++- .../unit/utils/operation-flag-matrix.test.ts | 21 ++ 12 files changed, 677 insertions(+), 11 deletions(-) diff --git a/.talismanrc b/.talismanrc index 7efa0e61f..e773c014a 100644 --- a/.talismanrc +++ b/.talismanrc @@ -32,11 +32,17 @@ fileignoreconfig: - filename: packages/contentstack-audit/test/unit/mock/am-contents/stack/stack.json checksum: c4ac3ff80eb38b1c58eb0db21a232bd84f856d4647093542296199036945083c - filename: packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts - checksum: 60f33657cc6e2664c17397a2f6b4b54a3f7bf71618931d63cd2811822bd4837e + checksum: 0b07ebebd0aaf3c1cb2dedd74801c93ed4a083864c96b8457d7e612c5736d0da - filename: packages/contentstack-bulk-operations/src/utils/data-dir-asset-fetcher.ts checksum: ddcf4601ac47be300eba0eb901e4d45d748c2c4eab676c56677cb9a802fe3db0 - filename: packages/contentstack-migration/src/commands/cm/stacks/migration.ts checksum: a64a4d396eddd936a63b799eff58c5c6660b5dcaa3a310fd8b09a027932f1789 - filename: packages/contentstack-migration/README.md checksum: e96006c1a948f766c88ae972b29582fa58eaf8184606bf011eebddc5a06cd7b6 +- filename: packages/contentstack-bulk-operations/src/utils/bulk-operation-log-handler.ts + checksum: e67a7a16a395be315c7464c8a24b142cbd259c59d31b17e1cdd32f613060c2b4 +- filename: packages/contentstack-bulk-operations/test/unit/commands/bulk-assets.test.ts + checksum: 92e0ba427283573dd629ca96e5d2c4286bfd90cf6f36af65b404c09bb32d856f +- filename: packages/contentstack-bulk-operations/test/unit/utils/bulk-operation-log-handler.test.ts + checksum: a660f8caac1418746ef6ec7ef9b69d04065ce0a8c0e7568bcbbd6e5287841304 version: "" diff --git a/packages/contentstack-bulk-operations/README.md b/packages/contentstack-bulk-operations/README.md index d428fbd29..9ae706f73 100644 --- a/packages/contentstack-bulk-operations/README.md +++ b/packages/contentstack-bulk-operations/README.md @@ -44,8 +44,8 @@ USAGE $ csdx cm:stacks:bulk-assets [-a ] [-k ] [--operation publish|unpublish|delete|move] [--environments ...] [--locales ...] [--source-env ] [--source-alias ] [--publish-mode bulk|single] [--branch ] [-c ] [-y] [--retry-failed ] [--revert ] [--bulk-operation-file ] - [--folder-uid ] [-d ] [--dry-run] [--space-uid ] [--org-uid ] [--workspace ] - [--asset-uids-file ] [--locale ] [--target-folder-uid ] + [--folder-uid ] [-d ] [--dry-run] [--retry-pending ] [--space-uid ] [--org-uid ] + [--workspace ] [--asset-uids-file ] [--locale ] [--target-folder-uid ] FLAGS -a, --alias= Uses the name of a saved Management Token to authenticate the command. The command @@ -85,6 +85,10 @@ FLAGS --retry-failed= (optional) Use this option to retry publishing the failed entries/assets from the logfile. Specify the name of the logfile that lists failed publish calls. If this option is used, it will override all other flags. + --retry-pending= (optional) Re-checks the malware scan status of assets a previous run skipped + because scanning was still in progress, and publishes the ones that are now clean. + Specify the bulk operation folder used by that run. Quarantined assets are never + retried. --revert= (optional) Revert publish operations from a log folder. Specify the folder path containing success logs. Works similar to retry-failed. --source-alias= Alias name for source environment delivery token (required for cross-publish). Add @@ -112,6 +116,8 @@ EXAMPLES $ csdx cm:stacks:bulk-assets --retry-failed ./bulk-operation -a myAlias + $ csdx cm:stacks:bulk-assets --retry-pending ./bulk-operation -a myAlias + $ csdx cm:stacks:bulk-assets --revert ./bulk-operation -a myAlias $ csdx cm:stacks:bulk-assets --data-dir ./content --operation publish -k blt123 diff --git a/packages/contentstack-bulk-operations/src/base-bulk-command.ts b/packages/contentstack-bulk-operations/src/base-bulk-command.ts index 3251e19ab..132510c17 100644 --- a/packages/contentstack-bulk-operations/src/base-bulk-command.ts +++ b/packages/contentstack-bulk-operations/src/base-bulk-command.ts @@ -156,6 +156,18 @@ export abstract class BaseBulkCommand extends Command { return false; } + /** + * Hook for a resource-specific retry flow that loads its config from a log file + * rather than the usual flags — like --retry-failed/--revert, but with its own + * pre-publish checks (see BulkAssets and --retry-pending). + * + * Return true when the flow handled the run: init() then returns immediately, + * skipping interactive prompts, the normal pipeline, and clearLogs(). + */ + protected async handleResourceSpecificRetryFlow(_flags: any): Promise { + return false; + } + /** * Initialize common components */ @@ -182,6 +194,12 @@ export abstract class BaseBulkCommand extends Command { this.logger = log; this.loggerContext = { module: this.id }; + // Resource-specific retry flows (e.g. asset scan-status retry) run their own + // init and must bypass clearLogs() for the same reason revert/retry does. + if (await this.handleResourceSpecificRetryFlow(flags)) { + return; + } + // Check for revert/retry EARLY - all config comes from log file const isRevertOrRetry = flags.revert || flags['retry-failed']; diff --git a/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts b/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts index 868f324c2..5d4af89ef 100644 --- a/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts +++ b/packages/contentstack-bulk-operations/src/commands/cm/stacks/bulk-assets.ts @@ -3,7 +3,14 @@ import * as path from 'path'; import { flags, handleAndLogError, log, FlagInput } from '@contentstack/cli-utilities'; -import { AssetPublishData, BulkOperationResult, ResourceType, OperationType, CsAssetsFlags } from '../../../interfaces'; +import { + AssetPublishData, + BulkOperationResult, + ResourceType, + OperationType, + CsAssetsFlags, + PendingScanLogEntry, +} from '../../../interfaces'; import { BaseBulkCommand } from '../../../base-bulk-command'; import { $t, @@ -21,6 +28,12 @@ import { RETRY_REVERT_CONTEXT, } from '../../../utils'; import type { DataDirScanStats } from '../../../utils'; +import { + appendPendingScanLog, + getLogPaths, + readPendingScanLog, + writePendingScanLog, +} from '../../../utils/bulk-operation-log-handler'; import { AssetService } from '../../../services'; type RegionWithOptionalCsAssetsUrl = { csAssetsUrl?: string }; @@ -65,6 +78,9 @@ export default class BulkAssets extends BaseBulkCommand { // Retry failed assets from a log file '<%= config.bin %> <%= command.id %> --retry-failed ./bulk-operation -a myAlias', + // Re-check scan status and publish assets a previous run skipped as still scanning + '<%= config.bin %> <%= command.id %> --retry-pending ./bulk-operation -a myAlias', + // Revert (unpublish) previously published assets using success log '<%= config.bin %> <%= command.id %> --revert ./bulk-operation -a myAlias', @@ -96,6 +112,9 @@ export default class BulkAssets extends BaseBulkCommand { description: messages.DRY_RUN_FLAG_DESC, default: false, }), + 'retry-pending': flags.string({ + description: messages.RETRY_PENDING, + }), // CS Assets delete/move flags 'space-uid': flags.string({ @@ -140,7 +159,9 @@ export default class BulkAssets extends BaseBulkCommand { token === '--retry-failed' || token.startsWith('--retry-failed=') || token === '--revert' || - token.startsWith('--revert=') + token.startsWith('--revert=') || + token === '--retry-pending' || + token.startsWith('--retry-pending=') ); if (!operation && !isRevertOrRetry) { @@ -217,6 +238,20 @@ export default class BulkAssets extends BaseBulkCommand { pending.forEach((a) => this.logger.warn($t(messages.SCAN_STATUS_SKIPPED_PENDING, { uid: a.uid }))); quarantined.forEach((a) => this.logger.warn($t(messages.SCAN_STATUS_SKIPPED_QUARANTINED, { uid: a.uid }))); + // Persist the pending ones so --retry-pending can publish them once the scan clears. + // fetchAssets() builds one row per uid x locale, so each carries its own locale/version. + appendPendingScanLog( + pending.map((a) => + this.buildPendingScanEntry({ + uid: a.uid, + locale: a.locale, + version: a.version ?? a._version, + environments: (a.publish_details || []).map((pd: any) => pd.environment), + }) + ), + this.bulkOperationConfig.bulkOperationFolder + ); + this.printScanningDashboard({ total: assets.length, clean: clean.length, @@ -361,6 +396,9 @@ export default class BulkAssets extends BaseBulkCommand { let workingBatch: AssetPublishData[] = []; let batchNumber = 0; let totalSubmitted = 0; + // Collected across the whole stream and written once at the end — a per-asset + // write would defeat the one-chunk-at-a-time design of this pass. + const pendingScanEntries: PendingScanLogEntry[] = []; this.batchResults.clear(); @@ -394,15 +432,21 @@ export default class BulkAssets extends BaseBulkCommand { this.logger.warn($t(messages.SCAN_STATUS_SKIPPED_QUARANTINED, { uid: targetUid })); continue; } + const assetLocales: string[] = useOverrideLocales + ? cliLocales + : [...new Set(asset.publish_details.map((pd: any) => pd.locale as string))]; + if (scanStatus === 'pending') { this.logger.warn($t(messages.SCAN_STATUS_SKIPPED_PENDING, { uid: targetUid })); + // Record per locale so --retry-pending republishes the same rows this run would have. + for (const locale of assetLocales) { + pendingScanEntries.push( + this.buildPendingScanEntry({ uid: targetUid, locale, version: asset._version, environments }) + ); + } continue; } - const assetLocales: string[] = useOverrideLocales - ? cliLocales - : [...new Set(asset.publish_details.map((pd: any) => pd.locale as string))]; - for (const locale of assetLocales) { workingBatch.push({ type: 'asset', uid: targetUid, locale, version: asset._version }); if (workingBatch.length >= BATCH_SIZE) { @@ -415,6 +459,9 @@ export default class BulkAssets extends BaseBulkCommand { flushBatch(); + // One write for the whole streamed run. + appendPendingScanLog(pendingScanEntries, this.bulkOperationConfig.bulkOperationFolder); + // Invariant: pass 1 and pass 2 use identical counting logic (excluding scan status filtering). // If batchNumber < totalBatches, scan status filtering reduced the published count — expected. if (batchNumber !== totalBatches) { @@ -464,4 +511,132 @@ export default class BulkAssets extends BaseBulkCommand { protected async fetchItems(): Promise { return await fetchAssets(this.bulkOperationConfig, this.managementStack, this.deliveryStack, this.logger); } + + protected async handleResourceSpecificRetryFlow(flags: any): Promise { + if (!flags['retry-pending']) { + return false; + } + await this.initForRetryPendingScan(flags); + return true; + } + + /** + * Initialize the --retry-pending run. Mirrors initForRevertOrRetry: everything + * needed comes from the pending-scan log, with any CLI flag overriding it. + */ + private async initForRetryPendingScan(flags: any): Promise { + const logPath = flags['retry-pending']; + const pendingEntries = readPendingScanLog(logPath); + + if (pendingEntries.length === 0) { + log.warn($t(messages.NO_PENDING_SCAN_ITEMS_IN_LOG, { path: getLogPaths(logPath).pendingScan })); + this.finalizeProgressSummary(); + process.exit(0); + } + + const [first] = pendingEntries; + const mergedFlags = { + ...flags, + 'stack-api-key': flags['stack-api-key'] || first.apiKey, + environments: + flags.environments?.length > 0 + ? flags.environments + : [...new Set(pendingEntries.flatMap((entry) => entry.environments))], + locales: flags.locales?.length > 0 ? flags.locales : [...new Set(pendingEntries.map((entry) => entry.locale))], + branch: flags.branch !== 'main' ? flags.branch : first.branch || 'main', + // Scan status only ever gates publish. + operation: OperationType.PUBLISH, + 'publish-mode': flags['publish-mode'] || 'bulk', + }; + + this.parsedFlags = mergedFlags; + await this.buildConfiguration(mergedFlags); + await this.setupStack(); + await this.initializeComponents(); + + await this.retryPendingScan(logPath, pendingEntries); + + // Mirrors initForRevertOrRetry: this early exit bypasses finally(). + this.finalizeProgressSummary(); + process.exit(0); + } + + /** + * Stamp a pending-scan skip with the run metadata --retry-pending needs to + * rebuild the publish call later without re-fetching the asset. + */ + private buildPendingScanEntry(item: { + uid: string; + locale: string; + version?: number; + environments: string[]; + }): PendingScanLogEntry { + return { + ...item, + operation: 'publish', + timestamp: new Date().toISOString(), + apiKey: this.bulkOperationConfig.apiKey || this.bulkOperationConfig.stackApiKey || '', + branch: this.bulkOperationConfig.branch, + }; + } + + /** + * Re-check the scan status of assets a previous run skipped as still scanning, + * publish the ones now clean, and prune the log down to those still pending. + */ + private async retryPendingScan(logPath: string, pendingEntries: PendingScanLogEntry[]): Promise { + const uids = [...new Set(pendingEntries.map((entry) => entry.uid))]; + const assetService = new AssetService(this.managementStack, this.deliveryStack, this.logger); + const scanStatusMap = await assetService.fetchScanStatusByUIDs(uids); + + const nowClean: PendingScanLogEntry[] = []; + const stillPending: PendingScanLogEntry[] = []; + const nowQuarantined: PendingScanLogEntry[] = []; + + for (const entry of pendingEntries) { + const status = scanStatusMap.get(entry.uid); + if (status === 'pending') stillPending.push(entry); + else if (status === 'quarantined') nowQuarantined.push(entry); + else nowClean.push(entry); // clean, or undefined when scanning is disabled + } + + log.info( + $t(messages.SCAN_RECHECK_SUMMARY, { + total: pendingEntries.length, + clean: nowClean.length, + pending: stillPending.length, + quarantined: nowQuarantined.length, + }) + ); + nowQuarantined.forEach((entry) => + this.logger.warn($t(messages.SCAN_STATUS_SKIPPED_QUARANTINED, { uid: entry.uid })) + ); + + if (nowClean.length === 0) { + this.logger.warn($t(messages.NO_PUBLISHABLE_ASSETS)); + // Still prune: the quarantined ones will never become publishable. + writePendingScanLog(stillPending, logPath); + return; + } + + const items: AssetPublishData[] = nowClean.map((entry) => ({ + type: 'asset', + uid: entry.uid, + locale: entry.locale, + version: entry.version, + publish_details: entry.environments.map((environment) => ({ environment, locale: entry.locale })), + })); + + const confirmed = await this.confirmOperation(items); + if (!confirmed) { + this.logger.warn($t(messages.OPERATION_CANCELLED)); + // Leave the log untouched so the next run sees the same set. + return; + } + + const result = await this.executeBulkOperation(items); + this.printOperationSummary(result); + + writePendingScanLog(stillPending, logPath); + } } diff --git a/packages/contentstack-bulk-operations/src/interfaces/index.ts b/packages/contentstack-bulk-operations/src/interfaces/index.ts index 1474f3fa8..8a9e9ed43 100644 --- a/packages/contentstack-bulk-operations/src/interfaces/index.ts +++ b/packages/contentstack-bulk-operations/src/interfaces/index.ts @@ -400,6 +400,25 @@ export interface SingleModeLogEntry { */ export type LogEntry = BulkModeLogEntry | SingleModeLogEntry; +/** + * Log entry for an asset skipped because its malware scan was still pending. + * + * Flat (no bulk/single split) — the skip happens before the publish mode matters. + * Self-sufficient by design: a later --retry-pending run rebuilds the publish + * payload straight from this record without re-fetching the asset. + */ +export interface PendingScanLogEntry { + uid: string; + locale: string; + version?: number; + environments: string[]; + operation: 'publish'; // scan status only gates publish + timestamp: string; + // Metadata + apiKey: string; + branch?: string; +} + export interface LogPaths { folder: string; // Bulk mode logs @@ -408,6 +427,8 @@ export interface LogPaths { // Single mode logs singleSuccess: string; singleFailed: string; + // Assets skipped for a pending scan, retried via --retry-pending + pendingScan: string; } /** diff --git a/packages/contentstack-bulk-operations/src/messages/index.ts b/packages/contentstack-bulk-operations/src/messages/index.ts index 00b72a124..5603fc62b 100644 --- a/packages/contentstack-bulk-operations/src/messages/index.ts +++ b/packages/contentstack-bulk-operations/src/messages/index.ts @@ -231,6 +231,10 @@ const bulkAssetsMsg = { SCAN_STATUS_SKIPPED_QUARANTINED: 'Skipped (quarantined): {uid}', SCAN_STATUS_FETCHING: 'Checking asset scan status for {count} assets...', NO_PUBLISHABLE_ASSETS: 'No publishable assets — all assets are either still scanning or quarantined.', + SCAN_RECHECK_SUMMARY: + 'Re-checked {total} pending asset(s): {clean} now clean (will publish), {pending} still scanning, {quarantined} now quarantined (skipped permanently).', + NO_PENDING_SCAN_ITEMS_IN_LOG: + 'No pending-scan assets found in {path} — nothing to retry. Assets are recorded there when a publish run skips them for a scan still in progress.', }; /** @@ -423,6 +427,8 @@ const flagDescriptions = { YES: 'Skips interactive confirmation prompts and runs the command immediately using the provided options. Useful for automation and scripts.', RETRY_FAILED: '(optional) Use this option to retry publishing the failed entries/assets from the logfile. Specify the name of the logfile that lists failed publish calls. If this option is used, it will override all other flags.', + RETRY_PENDING: + '(optional) Re-checks the malware scan status of assets a previous run skipped because scanning was still in progress, and publishes the ones that are now clean. Specify the bulk operation folder used by that run. Quarantined assets are never retried.', // Entry-specific flags CONTENT_TYPES: 'Content type UIDs to perform operation on. If not provided, operates on all content types.', diff --git a/packages/contentstack-bulk-operations/src/utils/bulk-operation-log-handler.ts b/packages/contentstack-bulk-operations/src/utils/bulk-operation-log-handler.ts index bd5f9f6b5..964de2724 100644 --- a/packages/contentstack-bulk-operations/src/utils/bulk-operation-log-handler.ts +++ b/packages/contentstack-bulk-operations/src/utils/bulk-operation-log-handler.ts @@ -1,6 +1,6 @@ import * as fs from 'fs'; import * as path from 'path'; -import { LogPaths, LogEntry, BulkModeLogEntry, SingleModeLogEntry } from '../interfaces'; +import { LogPaths, LogEntry, BulkModeLogEntry, SingleModeLogEntry, PendingScanLogEntry } from '../interfaces'; import { $t, messages } from './index'; const DEFAULT_LOG_FOLDER = 'bulk-operation'; @@ -31,6 +31,8 @@ export function getLogPaths(folderPath?: string): LogPaths { // Single mode logs (individual items) singleSuccess: path.join(folder, 'single-success.json'), singleFailed: path.join(folder, 'single-failed.json'), + // Assets skipped for a pending malware scan (retried via --retry-pending) + pendingScan: path.join(folder, 'pending-scan.json'), }; } @@ -261,6 +263,65 @@ export function writeSingleFailedLog(entry: SingleModeLogEntry, folderPath?: str } } +/** + * Read the pending-scan log + * @param folderPath - The base folder path for logs (optional) + * @returns Array of assets skipped for a pending malware scan + */ +export function readPendingScanLog(folderPath?: string): PendingScanLogEntry[] { + const paths = getLogPaths(folderPath); + + if (!fs.existsSync(paths.pendingScan)) { + return []; + } + + try { + const content = fs.readFileSync(paths.pendingScan, 'utf-8'); + return JSON.parse(content) as PendingScanLogEntry[]; + } catch (error) { + console.error($t(messages.ERROR_READING_LOG, { logType: 'pending scan', path: paths.pendingScan }), error); + return []; + } +} + +/** + * Overwrite the pending-scan log with the given entries. + * Used by the --retry-pending flow to prune entries that are no longer pending. + * @param entries - The full set of entries to persist + * @param folderPath - The base folder path for logs (optional) + */ +export function writePendingScanLog(entries: PendingScanLogEntry[], folderPath?: string): void { + ensureLogFolder(folderPath); + const paths = getLogPaths(folderPath); + + try { + fs.writeFileSync(paths.pendingScan, JSON.stringify(entries, null, 2), 'utf-8'); + } catch (error) { + console.error($t(messages.ERROR_WRITING_LOG, { logType: 'pending scan', path: paths.pendingScan }), error); + } +} + +/** + * Merge new entries into the pending-scan log, deduped by uid + locale so an asset + * skipped again on a later run replaces its stale entry instead of piling up. + * + * One read + one write per call — callers batch a whole run's skips into a single + * invocation rather than writing per asset. + * @param newEntries - Entries to merge in (no-op when empty) + * @param folderPath - The base folder path for logs (optional) + */ +export function appendPendingScanLog(newEntries: PendingScanLogEntry[], folderPath?: string): void { + if (newEntries.length === 0) return; + + const key = (entry: PendingScanLogEntry) => `${entry.uid}::${entry.locale}`; + const merged = new Map(readPendingScanLog(folderPath).map((entry) => [key(entry), entry])); + for (const entry of newEntries) { + merged.set(key(entry), entry); + } + + writePendingScanLog([...merged.values()], folderPath); +} + /** * Clear all log files in the folder (overwrite with empty arrays) * This is called when starting a NEW operation to ensure only the latest operation's data can be reverted diff --git a/packages/contentstack-bulk-operations/src/utils/operation-flag-matrix.ts b/packages/contentstack-bulk-operations/src/utils/operation-flag-matrix.ts index 5d3ad02f1..b0056d59d 100644 --- a/packages/contentstack-bulk-operations/src/utils/operation-flag-matrix.ts +++ b/packages/contentstack-bulk-operations/src/utils/operation-flag-matrix.ts @@ -32,6 +32,7 @@ const CMS_ONLY_FLAGS: FlagSpec[] = [ { name: 'branch' }, { name: 'config', char: 'c' }, { name: 'retry-failed' }, + { name: 'retry-pending' }, { name: 'revert' }, { name: 'bulk-operation-file' }, { name: 'folder-uid' }, @@ -119,6 +120,12 @@ export function validateOperationFlagMatrix(operation: string, argv: string[]): violations.push(messages.CS_ASSETS_LOCALE_NOT_ALLOWED_FOR_MOVE); } + // Scan status only gates publish, so --retry-pending is meaningless for unpublish. + // CMS_ONLY_FLAGS cannot express this: publish and unpublish share one rejected list. + if (operation === OperationType.UNPUBLISH && isFlagInArgv(argv, { name: 'retry-pending' })) { + violations.push($t(messages.FLAG_NOT_ALLOWED_FOR_OPERATION, { flag: '--retry-pending', operation, hint: '' })); + } + return violations; } diff --git a/packages/contentstack-bulk-operations/test/unit/commands/bulk-assets-init.test.ts b/packages/contentstack-bulk-operations/test/unit/commands/bulk-assets-init.test.ts index 381347841..5403dd159 100644 --- a/packages/contentstack-bulk-operations/test/unit/commands/bulk-assets-init.test.ts +++ b/packages/contentstack-bulk-operations/test/unit/commands/bulk-assets-init.test.ts @@ -240,5 +240,53 @@ describe('BulkAssets command — init() dispatch', () => { expect(thrown).to.be.instanceOf(OperationFlagMatrixError); expect(logStub.error.firstCall.args[0]).to.include('--retry-failed/--revert'); }); + + it('does not prompt for an operation on the --retry-pending path', async () => { + command = makeCommand(['--retry-pending', './bulk-operation']); + const promptStub = sandbox.stub(interactiveModule, 'promptForOperation'); + sandbox.stub(command as any, 'parse').resolves({ flags: { 'retry-pending': './bulk-operation' } }); + // Stop before the flow itself runs — this asserts only that init() skipped the prompt. + sandbox.stub(command as any, 'initForRetryPendingScan').resolves(undefined); + + await (command as any).init(); + + expect(promptStub.called).to.be.false; + }); + }); + + describe('--retry-pending short circuit', () => { + it('runs the pending-scan flow and skips the normal pipeline and clearLogs', async () => { + command = makeCommand(['--retry-pending', './bulk-operation']); + sandbox.stub(command as any, 'parse').resolves({ flags: { 'retry-pending': './bulk-operation' } }); + + const clearLogsStub = sandbox.stub(logHandlerModule, 'clearLogs').returns(undefined); + const retryFlowStub = sandbox.stub(command as any, 'initForRetryPendingScan').resolves(undefined); + const setupStackStub = sandbox.stub(command as any, 'setupStack').resolves(undefined); + + await (command as any).init(); + + expect(retryFlowStub.calledOnce).to.be.true; + // A retry run must not wipe the sibling logs of the run being retried. + expect(clearLogsStub.called).to.be.false; + expect(setupStackStub.called).to.be.false; + }); + + it('leaves the normal pipeline alone when --retry-pending is absent', async () => { + command = makeCommand(['--operation', 'publish']); + sandbox.stub(command as any, 'parse').resolves({ flags: { operation: 'publish' } }); + sandbox.stub(logHandlerModule, 'clearLogs').returns(undefined); + + const retryFlowStub = sandbox.stub(command as any, 'initForRetryPendingScan').resolves(undefined); + sandbox.stub(command as any, 'resolveFlagsInteractively').resolvesArg(0); + sandbox.stub(command as any, 'buildConfiguration').callsFake(async () => { + (command as any).bulkOperationConfig = { bulkOperationFolder: './bulk-operation' }; + }); + sandbox.stub(command as any, 'setupStack').resolves(undefined); + sandbox.stub(command as any, 'initializeComponents').resolves(undefined); + + await (command as any).init(); + + expect(retryFlowStub.called).to.be.false; + }); }); }); diff --git a/packages/contentstack-bulk-operations/test/unit/commands/bulk-assets.test.ts b/packages/contentstack-bulk-operations/test/unit/commands/bulk-assets.test.ts index c9678f0c6..bcb3d58db 100644 --- a/packages/contentstack-bulk-operations/test/unit/commands/bulk-assets.test.ts +++ b/packages/contentstack-bulk-operations/test/unit/commands/bulk-assets.test.ts @@ -543,6 +543,130 @@ describe('BulkAssets Command', () => { }); }); + describe('retryPendingScan', () => { + let logHandler: any; + let writePendingScanLogStub: sinon.SinonStub; + let fetchScanStatusStub: sinon.SinonStub; + let executeBulkOperationStub: sinon.SinonStub; + let confirmOperationStub: sinon.SinonStub; + + const entry = (uid: string, locale = 'en-us') => ({ + uid, + locale, + version: 1, + environments: ['dev'], + operation: 'publish' as const, + timestamp: '2026-01-09T10:00:00Z', + apiKey: 'test-api-key', + branch: 'main', + }); + + /** Drives retryPendingScan with a stubbed status response for the given entries. */ + const runRetry = async (entries: any[], statuses: Record) => { + fetchScanStatusStub.resolves(new Map(Object.entries(statuses))); + await (command as any).retryPendingScan('/mock/bulk-operation', entries); + }; + + beforeEach(() => { + logHandler = require('../../../src/utils/bulk-operation-log-handler'); + writePendingScanLogStub = sandbox.stub(logHandler, 'writePendingScanLog').returns(undefined); + + const { AssetService } = require('../../../src/services'); + fetchScanStatusStub = sandbox.stub(AssetService.prototype, 'fetchScanStatusByUIDs'); + + (command as any).managementStack = {}; + (command as any).deliveryStack = null; + confirmOperationStub = sandbox.stub(command as any, 'confirmOperation').resolves(true); + executeBulkOperationStub = sandbox + .stub(command as any, 'executeBulkOperation') + .resolves({ success: 1, failed: 0, total: 1 } as BulkOperationResult); + sandbox.stub(command as any, 'printOperationSummary').returns(undefined); + }); + + it('should publish only the assets whose scan is now clean', async () => { + await runRetry([entry('clean1'), entry('pending1'), entry('quarantined1')], { + clean1: 'clean', + pending1: 'pending', + quarantined1: 'quarantined', + }); + + expect(executeBulkOperationStub.calledOnce).to.be.true; + const published = executeBulkOperationStub.firstCall.args[0]; + expect(published.map((i: any) => i.uid)).to.deep.equal(['clean1']); + }); + + it('should treat an unknown status as publishable (scanning disabled on the stack)', async () => { + await runRetry([entry('noStatus1')], {}); + + expect(executeBulkOperationStub.firstCall.args[0].map((i: any) => i.uid)).to.deep.equal(['noStatus1']); + }); + + it('should rebuild publish details from the logged environments', async () => { + await runRetry([{ ...entry('clean1'), environments: ['dev', 'prod'] }], { clean1: 'clean' }); + + const [item] = executeBulkOperationStub.firstCall.args[0]; + expect(item).to.deep.include({ type: 'asset', uid: 'clean1', locale: 'en-us', version: 1 }); + expect(item.publish_details).to.deep.equal([ + { environment: 'dev', locale: 'en-us' }, + { environment: 'prod', locale: 'en-us' }, + ]); + }); + + it('should prune published and quarantined entries, keeping only those still pending', async () => { + await runRetry([entry('clean1'), entry('pending1'), entry('quarantined1')], { + clean1: 'clean', + pending1: 'pending', + quarantined1: 'quarantined', + }); + + expect(writePendingScanLogStub.calledOnce).to.be.true; + const remaining = writePendingScanLogStub.firstCall.args[0]; + expect(remaining.map((e: any) => e.uid)).to.deep.equal(['pending1']); + }); + + it('should skip publishing but still drop quarantined entries when nothing is clean', async () => { + await runRetry([entry('pending1'), entry('quarantined1')], { + pending1: 'pending', + quarantined1: 'quarantined', + }); + + expect(executeBulkOperationStub.called).to.be.false; + expect(logStub.warn.calledWith($t(messages.NO_PUBLISHABLE_ASSETS))).to.be.true; + expect(writePendingScanLogStub.firstCall.args[0].map((e: any) => e.uid)).to.deep.equal(['pending1']); + }); + + it('should leave the log untouched when the user declines confirmation', async () => { + confirmOperationStub.resolves(false); + + await runRetry([entry('clean1'), entry('pending1')], { clean1: 'clean', pending1: 'pending' }); + + expect(executeBulkOperationStub.called).to.be.false; + expect(writePendingScanLogStub.called).to.be.false; + }); + + it('should report the recheck counts', async () => { + await runRetry([entry('clean1'), entry('pending1'), entry('quarantined1')], { + clean1: 'clean', + pending1: 'pending', + quarantined1: 'quarantined', + }); + + expect( + logStub.info.calledWith( + $t(messages.SCAN_RECHECK_SUMMARY, { total: 3, clean: 1, pending: 1, quarantined: 1 }) + ) + ).to.be.true; + }); + + it('should query scan status once per unique uid', async () => { + await runRetry([entry('asset1', 'en-us'), entry('asset1', 'fr-fr')], { asset1: 'clean' }); + + expect(fetchScanStatusStub.firstCall.args[0]).to.deep.equal(['asset1']); + // Both locale rows still get published. + expect(executeBulkOperationStub.firstCall.args[0]).to.have.lengthOf(2); + }); + }); + describe('flag configurations', () => { it('should have folder-uid flag', () => { const flags = BulkAssets.flags; @@ -551,6 +675,10 @@ describe('BulkAssets Command', () => { expect(flags['folder-uid'].description).to.include('folder'); }); + it('should have retry-pending flag', () => { + expect(BulkAssets.flags['retry-pending']).to.exist; + }); + it('should inherit base flags', () => { const flags = BulkAssets.flags; diff --git a/packages/contentstack-bulk-operations/test/unit/utils/bulk-operation-log-handler.test.ts b/packages/contentstack-bulk-operations/test/unit/utils/bulk-operation-log-handler.test.ts index 7f05ccaaa..f084ce6fb 100644 --- a/packages/contentstack-bulk-operations/test/unit/utils/bulk-operation-log-handler.test.ts +++ b/packages/contentstack-bulk-operations/test/unit/utils/bulk-operation-log-handler.test.ts @@ -15,9 +15,25 @@ import { writeBulkFailedLog, writeSingleSuccessLog, writeSingleFailedLog, + readPendingScanLog, + writePendingScanLog, + appendPendingScanLog, clearLogs, } from '../../../src/utils/bulk-operation-log-handler'; -import { BulkModeLogEntry, SingleModeLogEntry } from '../../../src/interfaces'; +import { BulkModeLogEntry, SingleModeLogEntry, PendingScanLogEntry } from '../../../src/interfaces'; + +const pendingEntry = (overrides: Partial = {}): PendingScanLogEntry => ({ + uid: 'asset1', + locale: 'en-us', + version: 1, + environments: ['prod'], + operation: 'publish', + timestamp: '2026-01-09T10:00:00Z', + // deepcode ignore HardcodedNonCryptoSecret: test fixture value, not a real secret + apiKey: 'test-key', + branch: 'main', + ...overrides, +}); describe('Bulk Operation Log Handler', () => { let sandbox: sinon.SinonSandbox; @@ -71,6 +87,7 @@ describe('Bulk Operation Log Handler', () => { bulkFailed: '/home/user/project/bulk-operation/bulk-failed.json', singleSuccess: '/home/user/project/bulk-operation/single-success.json', singleFailed: '/home/user/project/bulk-operation/single-failed.json', + pendingScan: '/home/user/project/bulk-operation/pending-scan.json', }); }); @@ -84,6 +101,7 @@ describe('Bulk Operation Log Handler', () => { bulkFailed: '/home/user/project/custom-logs/bulk-failed.json', singleSuccess: '/home/user/project/custom-logs/single-success.json', singleFailed: '/home/user/project/custom-logs/single-failed.json', + pendingScan: '/home/user/project/custom-logs/pending-scan.json', }); }); @@ -97,6 +115,7 @@ describe('Bulk Operation Log Handler', () => { bulkFailed: path.join(absoluteFolder, 'bulk-failed.json'), singleSuccess: path.join(absoluteFolder, 'single-success.json'), singleFailed: path.join(absoluteFolder, 'single-failed.json'), + pendingScan: path.join(absoluteFolder, 'pending-scan.json'), }); }); @@ -666,6 +685,146 @@ describe('Bulk Operation Log Handler', () => { }); }); + describe('readPendingScanLog', () => { + let existsSyncStub: sinon.SinonStub; + let readFileSyncStub: sinon.SinonStub; + let consoleErrorStub: sinon.SinonStub; + + beforeEach(() => { + const fs = require('fs'); + existsSyncStub = sandbox.stub(fs, 'existsSync'); + readFileSyncStub = sandbox.stub(fs, 'readFileSync'); + consoleErrorStub = sandbox.stub(console, 'error'); + }); + + it('should return empty array if pending scan log file does not exist', () => { + existsSyncStub.returns(false); + + expect(readPendingScanLog()).to.deep.equal([]); + }); + + it('should return empty array and log error if JSON parsing fails', () => { + existsSyncStub.returns(true); + readFileSyncStub.returns('invalid json'); + + expect(readPendingScanLog()).to.deep.equal([]); + expect(consoleErrorStub.called).to.be.true; + }); + + it('should return parsed entries when valid JSON exists', () => { + const entries = [pendingEntry()]; + existsSyncStub.returns(true); + readFileSyncStub.returns(JSON.stringify(entries)); + + expect(readPendingScanLog()).to.deep.equal(entries); + }); + }); + + describe('writePendingScanLog', () => { + let existsSyncStub: sinon.SinonStub; + let writeFileSyncStub: sinon.SinonStub; + let consoleErrorStub: sinon.SinonStub; + + beforeEach(() => { + const fs = require('fs'); + existsSyncStub = sandbox.stub(fs, 'existsSync'); + writeFileSyncStub = sandbox.stub(fs, 'writeFileSync'); + sandbox.stub(fs, 'mkdirSync'); + consoleErrorStub = sandbox.stub(console, 'error'); + }); + + it('should overwrite the log with exactly the given entries', () => { + existsSyncStub.returns(true); + const entries = [pendingEntry(), pendingEntry({ uid: 'asset2' })]; + + writePendingScanLog(entries, './test-logs'); + + expect(writeFileSyncStub.calledOnce).to.be.true; + expect(writeFileSyncStub.firstCall.args[0]).to.equal(path.join('/home/user/project/test-logs', 'pending-scan.json')); + expect(JSON.parse(writeFileSyncStub.firstCall.args[1])).to.deep.equal(entries); + }); + + it('should write an empty array when pruning every entry', () => { + existsSyncStub.returns(true); + + writePendingScanLog([], './test-logs'); + + expect(JSON.parse(writeFileSyncStub.firstCall.args[1])).to.deep.equal([]); + }); + + it('should handle write errors gracefully', () => { + existsSyncStub.returns(true); + writeFileSyncStub.throws(new Error('Write error')); + + writePendingScanLog([pendingEntry()], './test-logs'); + expect(consoleErrorStub.called).to.be.true; + }); + }); + + describe('appendPendingScanLog', () => { + let existsSyncStub: sinon.SinonStub; + let readFileSyncStub: sinon.SinonStub; + let writeFileSyncStub: sinon.SinonStub; + + beforeEach(() => { + const fs = require('fs'); + existsSyncStub = sandbox.stub(fs, 'existsSync'); + readFileSyncStub = sandbox.stub(fs, 'readFileSync'); + writeFileSyncStub = sandbox.stub(fs, 'writeFileSync'); + sandbox.stub(fs, 'mkdirSync'); + sandbox.stub(console, 'error'); + }); + + it('should not touch the log when there is nothing to append', () => { + appendPendingScanLog([], './test-logs'); + + expect(writeFileSyncStub.called).to.be.false; + expect(readFileSyncStub.called).to.be.false; + }); + + it('should merge new entries with existing ones', () => { + existsSyncStub.returns(true); + readFileSyncStub.returns(JSON.stringify([pendingEntry({ uid: 'asset1' })])); + + appendPendingScanLog([pendingEntry({ uid: 'asset2' })], './test-logs'); + + const written = JSON.parse(writeFileSyncStub.firstCall.args[1]); + expect(written.map((e: PendingScanLogEntry) => e.uid)).to.deep.equal(['asset1', 'asset2']); + }); + + it('should dedupe by uid and locale, keeping the newest entry', () => { + existsSyncStub.returns(true); + readFileSyncStub.returns(JSON.stringify([pendingEntry({ version: 1, timestamp: 'old' })])); + + appendPendingScanLog([pendingEntry({ version: 2, timestamp: 'new' })], './test-logs'); + + const written = JSON.parse(writeFileSyncStub.firstCall.args[1]); + expect(written).to.have.lengthOf(1); + expect(written[0].version).to.equal(2); + expect(written[0].timestamp).to.equal('new'); + }); + + it('should treat the same uid in different locales as separate entries', () => { + existsSyncStub.returns(true); + readFileSyncStub.returns(JSON.stringify([pendingEntry({ locale: 'en-us' })])); + + appendPendingScanLog([pendingEntry({ locale: 'fr-fr' })], './test-logs'); + + expect(JSON.parse(writeFileSyncStub.firstCall.args[1])).to.have.lengthOf(2); + }); + + it('should write a single file for a whole batch of entries', () => { + existsSyncStub.returns(false); + + appendPendingScanLog( + [pendingEntry({ uid: 'a' }), pendingEntry({ uid: 'b' }), pendingEntry({ uid: 'c' })], + './test-logs' + ); + + expect(writeFileSyncStub.callCount).to.equal(1); + }); + }); + describe('clearLogs', () => { let existsSyncStub: sinon.SinonStub; let writeFileSyncStub: sinon.SinonStub; @@ -694,6 +853,16 @@ describe('Bulk Operation Log Handler', () => { }); }); + it('should NOT clear the pending scan log', () => { + existsSyncStub.returns(false); + + clearLogs('./test-logs'); + + // Pending assets outlive a single run — only --retry-pending prunes them. + const clearedPaths = writeFileSyncStub.getCalls().map((call) => call.args[0]); + expect(clearedPaths.some((p: string) => p.includes('pending-scan.json'))).to.be.false; + }); + it('should use default folder path when none provided', () => { existsSyncStub.returns(false); diff --git a/packages/contentstack-bulk-operations/test/unit/utils/operation-flag-matrix.test.ts b/packages/contentstack-bulk-operations/test/unit/utils/operation-flag-matrix.test.ts index 84a01cf8b..c33a4fff5 100644 --- a/packages/contentstack-bulk-operations/test/unit/utils/operation-flag-matrix.test.ts +++ b/packages/contentstack-bulk-operations/test/unit/utils/operation-flag-matrix.test.ts @@ -77,6 +77,11 @@ describe('operation-flag-matrix', () => { expect(violations).to.have.lengthOf(1); expect(violations[0]).to.include('--locale'); }); + + it('rejects --retry-pending with delete and move', () => { + expect(validateOperationFlagMatrix('delete', ['--retry-pending', './log'])).to.have.lengthOf(1); + expect(validateOperationFlagMatrix('move', ['--retry-pending', './log'])).to.have.lengthOf(1); + }); }); describe('publish/unpublish reject CS Assets flags', () => { @@ -106,6 +111,22 @@ describe('operation-flag-matrix', () => { expect(violations).to.be.empty; }); + it('rejects --retry-pending with unpublish (scan status only gates publish)', () => { + const violations = validateOperationFlagMatrix('unpublish', ['--retry-pending', './log']); + expect(violations).to.have.lengthOf(1); + expect(violations[0]).to.include('--retry-pending'); + }); + + it('accepts --retry-pending with publish', () => { + const violations = validateOperationFlagMatrix('publish', ['--retry-pending', './log']); + expect(violations).to.be.empty; + }); + + it('accepts --retry-pending on the retry/revert path (no operation given)', () => { + const violations = validateOperationFlagMatrix(RETRY_REVERT_CONTEXT, ['--retry-pending', './log']); + expect(violations).to.be.empty; + }); + it('rejects CS Assets flags on the retry/revert path (no operation given)', () => { const violations = validateOperationFlagMatrix(RETRY_REVERT_CONTEXT, [ '--retry-failed',