From ec55a4896e38a4462707db14c190e44705c42a50 Mon Sep 17 00:00:00 2001 From: elrrrrrrr Date: Thu, 24 Oct 2024 13:40:52 +0800 Subject: [PATCH] feat: strict validate deps --- app/core/service/PackageManagerService.ts | 28 ++++++++++- app/core/service/PackageSyncerService.ts | 9 ++++ app/port/config.ts | 5 ++ config/config.default.ts | 1 + .../PackageManagerService/publish.test.ts | 24 +++++++++ .../PackageSyncerService/executeTask.test.ts | 40 +++++++++++++++ .../registry.npmjs.org/invalid-deps.json | 49 +++++++++++++++++++ 7 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 test/fixtures/registry.npmjs.org/invalid-deps.json diff --git a/app/core/service/PackageManagerService.ts b/app/core/service/PackageManagerService.ts index 767b0d51..e7edd28e 100644 --- a/app/core/service/PackageManagerService.ts +++ b/app/core/service/PackageManagerService.ts @@ -1,11 +1,12 @@ import { stat, readFile } from 'node:fs/promises'; +import { strict as assert } from 'node:assert'; import { AccessLevel, SingletonProto, EventBus, Inject, } from '@eggjs/tegg'; -import { ForbiddenError, NotFoundError } from 'egg-errors'; +import { BadRequestError, ForbiddenError, NotFoundError } from 'egg-errors'; import { RequireAtLeastOne } from 'type-fest'; import npa from 'npm-package-arg'; import semver from 'semver'; @@ -46,6 +47,7 @@ import { BugVersion } from '../entity/BugVersion'; import { RegistryManagerService } from './RegistryManagerService'; import { Registry } from '../entity/Registry'; import { PackageVersionService } from './PackageVersionService'; +import pMap from 'p-map'; export interface PublishPackageCmd { // maintainer: Maintainer; @@ -101,6 +103,9 @@ export class PackageManagerService extends AbstractService { // support user publish private package and sync worker publish public package async publish(cmd: PublishPackageCmd, publisher: User) { + if (this.config.cnpmcore.strictValidatePackageDeps) { + await this._checkPackageDepsVersion(cmd.packageJson); + } let pkg = await this.packageRepository.findPackage(cmd.scope, cmd.name); if (!pkg) { pkg = Package.create({ @@ -978,4 +983,25 @@ export class PackageManagerService extends AbstractService { } return data; } + + private async _checkPackageDepsVersion(pkgJSON: PackageJSONType) { + // 只校验 dependencies + // devDependencies、optionalDependencies、peerDependencies 不会影响依赖安装 不在这里进行校验 + const { dependencies } = pkgJSON; + await pMap(Object.entries(dependencies || {}), async ([ fullname, spec ]) => { + try { + const specResult = npa(`${fullname}@${spec}`); + // 对于 git、alias、file 等类型的依赖，不进行版本校验 + if (!['range', 'tag', 'version'].includes(specResult.type)) { + return; + } + const pkgVersion = await this.packageVersionService.getVersion(npa(`${fullname}@${spec}`)); + assert(pkgVersion); + } catch (e) { + throw new BadRequestError(`deps ${fullname}@${spec} not found`); + } + }, { + concurrency: 12, + }); + } } diff --git a/app/core/service/PackageSyncerService.ts b/app/core/service/PackageSyncerService.ts index 768af7d2..42c374b6 100644 --- a/app/core/service/PackageSyncerService.ts +++ b/app/core/service/PackageSyncerService.ts @@ -360,6 +360,7 @@ export class PackageSyncerService extends AbstractService { if (tips) { logs.push(`[${isoNow()}] 👉👉👉👉👉 Tips: ${tips} 👈👈👈👈👈`); } + const taskQueueLength = await this.taskService.getTaskQueueLength(task.type); const taskQueueHighWaterSize = this.config.cnpmcore.taskQueueHighWaterSize; const taskQueueInHighWaterState = taskQueueLength >= taskQueueHighWaterSize; @@ -730,6 +731,14 @@ export class PackageSyncerService extends AbstractService { this.logger.error(err); lastErrorMessage = `publish error: ${err}`; logs.push(`[${isoNow()}] ❌ [${syncIndex}] Synced version ${version} error, ${lastErrorMessage}`); + if (err.name === 'BadRequestError') { + // 由于当前版本的依赖不满足，尝试重试 + // 默认会在当前队列最后重试 + this.logger.info('[PackageSyncerService.executeTask:fail-validate-deps] taskId: %s, targetName: %s, %s', + task.taskId, task.targetName, task.error); + await this.taskService.retryTask(task, logs.join('\n')); + return; + } } } await this.taskService.appendTaskLog(task, logs.join('\n')); diff --git a/app/port/config.ts b/app/port/config.ts index 2fbd80e1..e87ba150 100644 --- a/app/port/config.ts +++ b/app/port/config.ts @@ -170,4 +170,9 @@ export type CnpmcoreConfig = { * strictly enforces/validates manifest and tgz when publish, https://github.com/cnpm/cnpmcore/issues/542 */ strictValidateTarballPkg?: boolean, + + /** + * strictly enforces/validates dependencies version when publish or sync + */ + strictValidatePackageDeps?: boolean, }; diff --git a/config/config.default.ts b/config/config.default.ts index efe42803..3b1adeff 100644 --- a/config/config.default.ts +++ b/config/config.default.ts @@ -59,6 +59,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = { enableElasticsearch: !!process.env.CNPMCORE_CONFIG_ENABLE_ES, elasticsearchIndex: 'cnpmcore_packages', strictValidateTarballPkg: false, + strictValidatePackageDeps: false, }; export default (appInfo: EggAppConfig) => { diff --git a/test/core/service/PackageManagerService/publish.test.ts b/test/core/service/PackageManagerService/publish.test.ts index 0c85824e..e2915ecd 100644 --- a/test/core/service/PackageManagerService/publish.test.ts +++ b/test/core/service/PackageManagerService/publish.test.ts @@ -112,5 +112,29 @@ describe('test/core/service/PackageManagerService/publish.test.ts', () => { assert.equal(pkgVersion.version, '1.1.0'); assert.equal(pkgVersion.tarDist.size, 2672); }); + + it('should strict validate deps', async () => { + let checked = false; + mock(app.config.cnpmcore, 'strictValidatePackageDeps', true); + + await assert.rejects(async () => { + checked = true; + await packageManagerService.publish({ + dist: { + localFile: TestUtil.getFixtures('registry.npmjs.org/pedding/-/pedding-1.1.0.tgz'), + }, + tags: [ '' ], + scope: '', + name: 'pedding', + description: 'pedding description', + packageJson: { name: 'pedding', test: 'test', version: '1.1.0', dependencies: { 'invalid-pkg': 'some-semver-not-exits' } }, + readme: '', + version: '1.1.0', + isPrivate: false, + }, publisher); + }, /Package invalid-pkg@some-semver-not-exits not found/); + + assert(checked); + }); }); }); diff --git a/test/core/service/PackageSyncerService/executeTask.test.ts b/test/core/service/PackageSyncerService/executeTask.test.ts index 48031d72..79c2391a 100644 --- a/test/core/service/PackageSyncerService/executeTask.test.ts +++ b/test/core/service/PackageSyncerService/executeTask.test.ts @@ -2552,5 +2552,45 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => { assert(/different meta: {"_npmUser":{"name":"banana","email":"banana@cnpmjs.org"}}/.test(log2)); }); }); + + describe('strictValidatePackageDeps = true', async () => { + + // already synced pkg + beforeEach(async () => { + app.mockHttpclient(/^https:\/\/registry\.npmjs\.org\/invalid\-deps/, 'GET', { + data: await TestUtil.readFixturesFile('registry.npmjs.org/invalid-deps.json'), + persist: false, + }); + + app.mockHttpclient('https://registry.npmjs.org/invalid-deps/-/invalid-deps-1.0.0.tgz', 'GET', { + data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'), + persist: false, + }); + }); + + it('should not create pkg when invalid deps', async () => { + // removed in remote + mock(app.config.cnpmcore, 'strictValidatePackageDeps', true); + await packageSyncerService.createTask('invalid-deps', { skipDependencies: true }); + const task = await packageSyncerService.findExecuteTask(); + assert(task); + await packageSyncerService.executeTask(task); + // assert(!await TaskModel.findOne({ taskId: task.taskId })); + // assert(await HistoryTaskModel.findOne({ taskId: task.taskId })); + const stream = await packageSyncerService.findTaskLog(task); + assert(stream); + const log = await TestUtil.readStreamToLog(stream); + assert(log); + // console.log(log); + const model = await PackageModel.findOne({ scope: '', name: 'invalid-pkgs' }); + assert(!model); + + // shoud requeue + const reTask = await packageSyncerService.findExecuteTask(); + assert(reTask.attempts === 2); + + }); + + }); }); }); diff --git a/test/fixtures/registry.npmjs.org/invalid-deps.json b/test/fixtures/registry.npmjs.org/invalid-deps.json new file mode 100644 index 00000000..7a3961af --- /dev/null +++ b/test/fixtures/registry.npmjs.org/invalid-deps.json @@ -0,0 +1,49 @@ +{ + "_id": "invalid-deps", + "name": "invalid-deps", + "dist-tags": { "latest": "1.0.0" }, + "versions": { + "1.0.0": { + "name": "invalid-deps", + "version": "1.0.0", + "main": "index.js", + "dependencies": { "invalid-semver": "@*(5895jk!", "invalid-tag": "gutu" }, + "scripts": { "test": "echo \"Error: no test specified\" && exit 1" }, + "author": "", + "license": "ISC", + "_id": "invalid-deps@1.0.0", + "_nodeVersion": "20.15.0", + "_npmVersion": "9.9.3", + "dist": { + "integrity": "sha512-wSjs5WIvZr78mkV8FHCKR59VVDNTupV8wylWHj05RFdB/apXroctfRe44opTO3PTGLdAFPrIUU1q7oJSebkr6w==", + "shasum": "588f41db968dbff87fc554d43f4af78b0eb35156", + "tarball": "https://registry.npmjs.org/invalid-deps/-/invalid-deps-1.0.0.tgz", + "fileCount": 2, + "unpackedSize": 295, + "signatures": [ + { + "keyid": "SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA", + "sig": "MEUCIQCxErdGOYMudvChG6AOp/bNv7fHKFhAYGJ2F62Cry48twIgFpnYMFxw6qXinx2mL3hSEsyV07n2TkKPy20gA3RQxU0=" + } + ] + }, + "_npmUser": { "name": "elrtmp", "email": "elrrrrrrr+npm@gmail.com" }, + "directories": {}, + "maintainers": [{ "name": "elrtmp", "email": "elrrrrrrr+npm@gmail.com" }], + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/invalid-deps_1.0.0_1729738458514_0.507393390991969" + }, + "_hasShrinkwrap": false + } + }, + "time": { + "created": "2024-10-24T02:54:18.513Z", + "1.0.0": "2024-10-24T02:54:18.686Z", + "modified": "2024-10-24T02:54:18.896Z" + }, + "maintainers": [{ "name": "elrtmp", "email": "elrrrrrrr+npm@gmail.com" }], + "license": "ISC", + "readme": "ERROR: No README data found!", + "readmeFilename": "" +}