Skip to content

Commit

Permalink
feat: 在proxy模式下，合并私有manifest缓存，可以返回私有包
Browse files Browse the repository at this point in the history
  • Loading branch information
@fangzhengjin
fangzhengjin committed Nov 1, 2024
1 parent 4facf90 commit 3aea4c8
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 9 deletions.
58 changes: 50 additions & 8 deletions app/core/service/ProxyCacheService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import { calculateIntegrity } from '../../common/PackageUtil';
import { ABBREVIATED_META_TYPE, PROXY_CACHE_DIR_NAME } from '../../common/constants';
import { DIST_NAMES } from '../entity/Package';
import type { AbbreviatedPackageManifestType, AbbreviatedPackageJSONType, PackageManifestType, PackageJSONType } from '../../repository/PackageRepository';
import { PackageManagerService } from './PackageManagerService';
import { getScopeAndName } from '../../common/PackageUtil';

function isoNow() {
return new Date().toISOString();
Expand Down Expand Up @@ -50,6 +52,8 @@ export class ProxyCacheService extends AbstractService {
private readonly cacheService: CacheService;
@Inject()
private readonly backgroundTaskHelper:BackgroundTaskHelper;
@Inject()
private readonly packageManagerService: PackageManagerService;

async getPackageVersionTarResponse(fullname: string, ctx: EggContext): Promise<HttpClientResponse> {
if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) {
Expand All @@ -59,12 +63,17 @@ export class ProxyCacheService extends AbstractService {
}

async getPackageManifest(fullname: string, fileType: DIST_NAMES.FULL_MANIFESTS| DIST_NAMES.ABBREVIATED_MANIFESTS): Promise<AbbreviatedPackageManifestType|PackageManifestType> {
const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath;
if (cachedStoreKey) {
const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey);
const nfsString = Buffer.from(nfsBytes!).toString();
const nfsPkgManifgest = JSON.parse(nfsString);
return nfsPkgManifgest;
try {
const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath;
if (cachedStoreKey) {
const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey);
const nfsString = Buffer.from(nfsBytes!).toString();
const nfsPkgManifest = JSON.parse(nfsString);
return nfsPkgManifest;
}
} catch (e) {
this.logger.error(e);
this.logger.error('[ProxyCacheService.getPackageManifest:error] get cache error, ignore');
}

const manifest = await this.getRewrittenManifest<typeof fileType>(fullname, fileType);
Expand Down Expand Up @@ -152,6 +161,7 @@ export class ProxyCacheService extends AbstractService {
}

async getRewrittenManifest<T extends DIST_NAMES>(fullname:string, fileType: T, versionOrTag?:string): Promise<GetSourceManifestAndCacheReturnType<T>> {
const [ scope, name ] = getScopeAndName(fullname);
let responseResult;
switch (fileType) {
case DIST_NAMES.FULL_MANIFESTS:
Expand All @@ -171,9 +181,24 @@ export class ProxyCacheService extends AbstractService {
}

// replace tarball url
const manifest = responseResult.data;
const { status, data: manifest } = responseResult;
// sourceRegistry not found, check private package
if (status === 404) {
const { etag, data: manifest, blockReason } = fileType === DIST_NAMES.FULL_MANIFESTS ?
await this.packageManagerService.listPackageFullManifests(scope, name, false) :
await this.packageManagerService.listPackageAbbreviatedManifests(scope, name, false);
// found in private package
if (etag && !blockReason) {
return manifest as any;
}
}
const { sourceRegistry, registry } = this.config.cnpmcore;
if (isPkgManifest(fileType)) {
const { etag, data, blockReason } = fileType === DIST_NAMES.FULL_MANIFESTS ?
await this.packageManagerService.listPackageFullManifests(scope, name, false) :
await this.packageManagerService.listPackageAbbreviatedManifests(scope, name, false);
const hasPrivatePackage = etag && !blockReason;

// pkg manifest
const versionMap = manifest.versions || {};
for (const key in versionMap) {
Expand All @@ -182,9 +207,26 @@ export class ProxyCacheService extends AbstractService {
versionItem.dist.tarball = versionItem.dist.tarball.replace(sourceRegistry, registry);
}
}
// private manifest
if (hasPrivatePackage) {
const privateVersionMap = data?.versions || {};
for (const key in privateVersionMap) {
if (!versionMap[key]) {
versionMap[key] = privateVersionMap[key];
}
}
if (manifest.time) {
const privateTimeMap = data?.time || {};
for (const key in privateTimeMap) {
if (!manifest.time[key]) {
manifest.time[key] = privateTimeMap[key];
}
}
}
}
} else {
// pkg version manifest
const distItem = manifest.dist || {};
const distItem = manifest?.dist || {};
if (distItem.tarball) {
distItem.tarball = distItem.tarball.replace(sourceRegistry, registry);
}
Expand Down
28 changes: 27 additions & 1 deletion app/port/controller/package/UpdatePackageController.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ import { AbstractController } from '../AbstractController';
import { FULLNAME_REG_STRING } from '../../../common/PackageUtil';
import { User as UserEntity } from '../../../core/entity/User';
import { PackageManagerService } from '../../../core/service/PackageManagerService';
import { SyncMode } from '../../../common/constants';
import { ProxyCacheRepository } from '../../../repository/ProxyCacheRepository';
import { isPkgManifest, ProxyCacheService } from '../../../core/service/ProxyCacheService';

const MaintainerDataRule = Type.Object({
maintainers: Type.Array(Type.Object({
Expand All @@ -30,7 +33,10 @@ type Maintainer = Static<typeof MaintainerDataRule>;
export class UpdatePackageController extends AbstractController {
@Inject()
private packageManagerService: PackageManagerService;

@Inject()
private readonly proxyCacheRepository: ProxyCacheRepository;
@Inject()
private readonly proxyCacheService: ProxyCacheService;
// https://github.com/npm/cli/blob/latest/lib/commands/owner.js#L191
@HTTPMethod({
// PUT /:fullname/-rev/:rev
Expand Down Expand Up @@ -65,6 +71,26 @@ export class UpdatePackageController extends AbstractController {
}

await this.packageManagerService.replacePackageMaintainersAndDist(pkg, users);
// 代理模式下，更新代理缓存
if (this.config.cnpmcore.syncMode === SyncMode.proxy) {
const refreshList = await this.proxyCacheRepository.findProxyCaches(fullname);
if (refreshList.length !== 0) {
const taskList = refreshList
// 仅manifests需要更新，指定版本的package.json文件发布后不会改变
.filter(i => isPkgManifest(i.fileType))
.map(async item => {
const task = await this.proxyCacheService.createTask(
`${item.fullname}/${item.fileType}`,
{
fullname: item.fullname,
fileType: item.fileType,
},
);
return task;
});
await Promise.all(taskList);
}
}
return { ok: true };
}

Expand Down

0 comments on commit 3aea4c8

Please sign in to comment.