Skip to content

Commit

Permalink
✨ persist cache (#10)
Browse files Browse the repository at this point in the history
* ✨ persit cache

* 🔖 release note

* 💡 docs options
  • Loading branch information
JiangWeixian authored Jul 15, 2023
1 parent c4ee502 commit 6cd8d0a
Show file tree
Hide file tree
Showing 17 changed files with 547 additions and 89 deletions.
5 changes: 5 additions & 0 deletions .changeset/famous-pumpkins-develop.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"vsit": patch
---

support persist cache in node side
6 changes: 3 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
},
"scripts": {
"test": "pnpm --filter=./packages/** run test",
"dev:cli": "pnpm --filter=./packages/vsit dev",
"dev": "pnpm --filter=./packages/client dev",
"dev": "pnpm --filter=./packages/vsit dev",
"play": "pnpm --filter=./packages/client dev",
"build:cli": "pnpm --filter=./packages/vsit build",
"build:client": "pnpm --filter=./packages/client build",
"build:copy": "esno ./scripts/client.ts",
Expand All @@ -48,6 +48,6 @@
"husky": "^8.0.1",
"lint-staged": "^13.0.3",
"npm-run-all": "^4.1.5",
"typescript": "4.4.4"
"typescript": "5.1.6"
}
}
8 changes: 7 additions & 1 deletion packages/vsit/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
"@rollup/plugin-node-resolve": "15.0.1",
"@rollup/plugin-replace": "^5.0.2",
"@types/debug": "^4.1.8",
"@types/fs-extra": "^11.0.1",
"@types/inquirer": "^8.1.3",
"@types/node": "20.3.1",
"@types/rimraf": "^3.0.2",
Expand All @@ -86,24 +87,29 @@
"debug": "^4.3.4",
"esbuild": "^0.18.6",
"execa": "^6.0.0",
"fs-extra": "^11.1.1",
"husky": "^8.0.3",
"inquirer": "8.2.0",
"npm-run-all": "^4.1.5",
"ofetch": "^1.0.1",
"ora": "6.0.1",
"picocolors": "1.0.0",
"publish-police": "^0.1.0",
"read-yaml-file": "2.1.0",
"rimraf": "^3.0.2",
"rollup": "3.19.1",
"rollup-plugin-condition-exports": "2.0.0-next.3",
"rollup-plugin-esbuild": "^5.0.0",
"rollup-plugin-node-externals": "5.1.2",
"rollup-plugin-size": "^0.3.1",
"source-map-support": "^0.5.21",
"tempy": "^3.1.0",
"ttypescript": "^1.5.15",
"type-fest": "^3.12.0",
"typescript": "^4.6.4",
"typescript-transform-paths": "^3.4.6",
"ufo": "^1.1.2",
"vitest": "^0.22.1"
"vitest": "^0.22.1",
"write-yaml-file": "5.0.0"
}
}
6 changes: 6 additions & 0 deletions packages/vsit/src/common/resolver/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,9 @@ export const VIRTUAL_RE = /^(?:virtual:)|\0/

export const VIRUTAL_NODE_ID = 'fake-node-file.ts'
export const VIRUTAL_WEB_ID = 'fake-web-file.ts'
/**
* Borrowed from vite
*/
export const VALID_ID_PREFIX = '/@id/'
export const NULL_BYTE_PLACEHOLDER = '__x00__'
export const NULL_BYTE = '\x00'
7 changes: 7 additions & 0 deletions packages/vsit/src/common/store/constants.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
import os from 'node:os'
import path from 'node:path'

const STORE_DIR = '.vsit-store'
export const STORE_PATH = path.join(os.homedir(), STORE_DIR)
export const LOCK_FILE = 'vist-lock.yaml'
export const STORE_PACKAGES_DIR = 'packages'
11 changes: 6 additions & 5 deletions packages/vsit/src/common/store/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,20 @@ import { createHash } from 'node:crypto'

import { fetch } from 'ofetch'

import { PersistCache } from './persist-cache'
import { debug } from '@/common/log'

export const createStore = () => {
export const createStore = async () => {
const pool = new Map<string, Promise<string>>()
const globalCache = new Map<string, string>()
const cacheManager = await PersistCache.create()
const createInstance = (id: string, url: string, options?: RequestInit) => {
const promise = (async () => {
try {
debug.store('start fetch %s', url)
return fetch(url, options)
.then(async (res) => {
const content = await res.text()
globalCache.set(id, content)
cacheManager.saveCache(url, content)
pool.delete(id)
return content
})
Expand All @@ -27,14 +28,14 @@ export const createStore = () => {
return promise
}
return {
cache: cacheManager,
async clear(url: string) {
const hash = createHash('sha256').update(url).digest('hex')
globalCache.delete(hash)
pool.delete(hash)
},
async fetch(url: string, options?: RequestInit) {
const hash = createHash('sha256').update(url).digest('hex')
const cache = globalCache.get(hash)
const cache = await cacheManager.getCache(url)
if (cache) {
debug.store('load cache %s', url)
return Promise.resolve(cache)
Expand Down
169 changes: 169 additions & 0 deletions packages/vsit/src/common/store/persist-cache.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
import { resolve } from 'node:path'

import {
existsSync,
outputFile,
readFile,
} from 'fs-extra'
import readYaml from 'read-yaml-file'
import writeYaml from 'write-yaml-file'

import { version } from '../../../package.json'
import {
LOCK_FILE,
STORE_PACKAGES_DIR,
STORE_PATH,
} from './constants'
import { computeCacheKey } from './utils'
import { debug } from '@/common/log'

interface Options {
storePath?: string
}

interface ResolvedLockFileOptions {
/**
* @description Virtual store path
* @default <homedir>/<.vsit-store>
*/
storePath: string
/**
* @description Virtual store lock file path
* @default <homedir>/<.vsit-store>/vsit-lock.yaml
*/
lockFilePath: string
}

export interface Package {
/**
* @description Encoded url
*/
id: string
/**
* @description Remote package url
*/
url: string
/**
* @description Dependent packages
* @todo not used currently, in the future, we will outdated the persist cache, and concurrent download the packages based on deps
*/
deps?: string[]
}

interface LockFileYaml {
version: string
packages?: Record<string, Package>
}

export class LockFile {
options: ResolvedLockFileOptions
lockFile: LockFileYaml = { version }
constructor(options: Options) {
this.options = this.resolveOptions(options)
debug.store('resolved lock-file options %o', this.options)
}

resolveOptions(options: Options): ResolvedLockFileOptions {
const resolvedStorePath = options.storePath ?? STORE_PATH
return {
storePath: resolvedStorePath,
lockFilePath: resolve(resolvedStorePath, LOCK_FILE),
}
}

async init() {
if (!existsSync(this.options.lockFilePath)) {
await writeYaml(this.options.lockFilePath, { version })
}
await this.read()
}

async read(): Promise<LockFileYaml> {
this.lockFile = await readYaml(this.options.lockFilePath)
return this.lockFile as LockFileYaml
}

async write(data: LockFileYaml): Promise<void> {
await writeYaml(this.options.lockFilePath, data)
}

async save() {
await writeYaml(this.options.lockFilePath, this.lockFile)
}

async writePackage(url: string, deps: string[] = []) {
const id = computeCacheKey(url)
this.lockFile.packages = {
...this.lockFile.packages,
[id]: {
id,
url,
deps,
},
}
await this.save()
}

async writePackages(packages: Record<string, Package> = {}) {
this.lockFile.packages = {
...this.lockFile.packages,
...packages,
}
await this.save()
}

static async create(options: Options) {
const instance = new LockFile(options)
await instance.init()
return instance
}
}

interface ResolvedCacheOptions {
storePath: string
packagesPath: string
}

export class PersistCache {
options: ResolvedCacheOptions
lockFile?: LockFile
constructor(options: Options = { storePath: STORE_PATH }) {
this.options = this.resolveOptions(options)
this.lockFile = undefined
}

resolveOptions(options: Options): ResolvedCacheOptions {
const resolvedStorePath = options.storePath ?? STORE_PATH
return {
storePath: resolvedStorePath,
packagesPath: resolve(resolvedStorePath, STORE_PACKAGES_DIR),
}
}

static async create(options: Options = { storePath: STORE_PATH }) {
const instance = new PersistCache(options)
instance.lockFile = await LockFile.create(options)
return instance
}

async writePackages(packages: Record<string, Package> = {}) {
this.lockFile?.writePackages(packages)
}

async getCache(url: string) {
const id = computeCacheKey(url)
const path = resolve(this.options.packagesPath, id)
if (existsSync(path)) {
const content = (await readFile(path)).toString('utf-8')
return content
}
return ''
}

async saveCache(url: string, content: string) {
const id = computeCacheKey(url)
const path = resolve(this.options.packagesPath, id)
await this.lockFile?.writePackage(url)
await outputFile(path, content)
}
}
6 changes: 6 additions & 0 deletions packages/vsit/src/common/store/utils.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import { createHash } from 'node:crypto'

export const computeCacheKey = (url: string) => {
const hash = createHash('sha256').update(url).digest('hex')
return hash
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,23 @@ import { join } from 'node:path'

import { withoutLeadingSlash, withoutTrailingSlash } from 'ufo'

import { pkgRoot } from '../path'
import { pkgRoot } from './path'
import {
ESM_HOST,
ESMSH_HTTP_RE,
ESMSH_HTTP_SUB_RE,
ESMSH_PROTOCOL,
ESMSH_PROTOCOL_RE,
NULL_BYTE,
NULL_BYTE_PLACEHOLDER,
VALID_ID_PREFIX,
VIRTUAL_RE,
} from './constants'
} from './resolver/constants'
import { isEsmSh } from './resolver/is'
import { computeCacheKey } from './store/utils'

import type { ModuleNode } from 'vite'
import type { Package } from './store/persist-cache'

// '\0' tell vite to not resolve this id via internal node resolver algorithm
export const wrapId = (id: string) => {
Expand All @@ -25,12 +33,15 @@ export const wrapId = (id: string) => {
return id
}

export const unWrapId = (id: string) => {
export const unwrapId = (id: string) => {
let stripId = id.replace(VIRTUAL_RE, '')
// unwrap
// https:/esm.sh -> https://esm.sh
// esm.sh: -> https://esm.sh
stripId = stripId
.replace(VALID_ID_PREFIX, '')
.replace(NULL_BYTE_PLACEHOLDER, '')
.replace(NULL_BYTE, '')
.replace(ESMSH_HTTP_RE, withoutTrailingSlash(ESM_HOST))
.replace(ESMSH_PROTOCOL_RE, ESM_HOST)
return stripId
Expand All @@ -56,3 +67,46 @@ globalThis.__hook(consolehook, (log) => {
${content}
`
}

export const parseDeps = (deps: string[] = []) => {
return deps
.map((dep) => {
return unwrapId(dep)
})
.filter((dep) => {
return isEsmSh(dep)
})
}

const createPackage = (url: string, deps: string[]): Record<string, Package> => {
if (!url) {
return {}
}
const id = computeCacheKey(url)
return {
[id]: {
id,
url,
deps,
},
}
}

export const parseModulesDeps = (m?: ModuleNode): Record<string, Package> => {
if (!m || !m.id) {
return {}
}
const id = unwrapId(m.id)
const deps = parseDeps(m.ssrTransformResult?.deps)
let records = id && isEsmSh(id) && deps.length
? createPackage(id, deps)
: {}
m.importedModules.forEach((importedModule) => {
const result = parseModulesDeps(importedModule)
records = {
...records,
...result,
}
})
return records
}
Loading

0 comments on commit 6cd8d0a

Please sign in to comment.