Skip to content

Commit

Permalink
feat: checkout external worktrees
Browse files Browse the repository at this point in the history
  • Loading branch information
@alandefreitas
alandefreitas committed Sep 10, 2024
1 parent 7cbc63b commit bd6e1ca
Show file tree
Hide file tree
Showing 3 changed files with 161 additions and 1,320 deletions.
198 changes: 158 additions & 40 deletions lib/extension.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ const expandPath = require('@antora/expand-path-helper')
const fs = require('node:fs')
const {promises: fsp} = fs
const getUserCacheDir = require('cache-directory')
const git = require('isomorphic-git')
const {globStream} = require('fast-glob')
const ospath = require('node:path')
const {posix: path} = ospath
Expand Down Expand Up @@ -75,6 +76,7 @@ class CppReferenceExtension {
this.logger.debug('Registering cpp-reference-extension')

this.config = config
this.createWorktrees = config.createWorktrees || 'auto'
// playbook = playbook
}

Expand Down Expand Up @@ -349,6 +351,7 @@ class CppReferenceExtension {
// Determine the cache directory and create it if it doesn't exist
const cacheDir = ospath.join(CppReferenceExtension.getBaseCacheDir(playbook), 'reference-collector')
await fsp.mkdir(cacheDir, {recursive: true})
this.logger.debug(`Cache directory: ${cacheDir}`)

// Initialize a cache for git repositories
const gitCache = {}
Expand Down Expand Up @@ -409,6 +412,7 @@ class CppReferenceExtension {
const {name, version} = componentVersionBucket
const {url, gitdir, refname, reftype, remote, worktree, startPath, descriptor} = origin
this.logger.debug(`Processing origin ${url || gitdir} (${reftype}: ${refname}) at path ${startPath}`)
this.logger.debug(CppReferenceExtension.objectSummary(origin), 'Origin')

// Get the reference collector configuration from the descriptor
// The reference collector configuration is an array of objects
Expand All @@ -428,9 +432,16 @@ class CppReferenceExtension {
// branches of a repository checked out at once.
// Each worktree has its own directory.
let worktreeDir = worktree
let checkoutWorktree
if (!worktree) {
worktreeDir = await this.prepareWorktree(collectorConfigs, origin, cacheDir, managedWorktrees);
let worktreeConfig = collectorConfigs[0].worktree || {}
if (!worktreeConfig) {
worktreeConfig = worktreeConfig === false ? {create: 'always'} : {}
}
const createWorktree =
!worktree || ('create' in worktreeConfig ? worktreeConfig.create : this.createWorktrees) === 'always'
const checkoutWorktree = worktreeConfig.checkout !== false
if (createWorktree) {
this.logger.debug(`Worktree directory not provided for ${name} version ${version}`)
worktreeDir = await this.setupManagedWorktree(worktreeConfig, checkoutWorktree, origin, cacheDir, managedWorktrees);
}

// Store the worktree directory in the origin
Expand All @@ -448,19 +459,50 @@ class CppReferenceExtension {
}
this.logger.debug(expandPathContext, 'Expand path context')

// If the worktree doesn't exist, either checkout the worktree or create the directory
if (createWorktree) {
if (checkoutWorktree) {
this.logger.debug(`Checking out worktree: ${worktreeDir}`)
const cache = gitCache[gitdir] || (gitCache[gitdir] = {})
const ref = `refs/${reftype === 'branch' ? 'head' : reftype}s/${refname}`
this.logger.debug(cache, 'Cache')
this.logger.debug(`Ref: ${ref}.`)
await this.prepareWorktree({
fs,
cache,
dir: worktreeDir,
gitdir,
ref,
remote,
bare: worktree === undefined
})
this.logger.debug(`Checked out worktree: ${worktreeDir}`)
} else {
this.logger.debug(`Creating worktree directory: ${worktreeDir}`)
await fsp.mkdir(worktreeDir, {recursive: true})
}
} else {
this.logger.debug(`Using existing worktree directory: ${worktreeDir}`)
}

// Create a list of normalized collectors from the collector configuration
const collectors = (collectorConfigs).map((collectorConfig) => {
let collectors = []
for (const collectorConfig of collectorConfigs) {
// If a config file is specified in the collectorConfig configuration, check if it exists
let mrdocsConfigFile
const defaultMrDocsConfigLocations = ['mrdocs.yml', 'docs/mrdocs.yml', 'doc/mrdocs.yml'];
const mrdocsConfigCandidates = [collectorConfig.config].concat(defaultMrDocsConfigLocations)
this.logger.debug(`Looking for mrdocs.yml file in ${startPath} at locations ${mrdocsConfigCandidates.join(', ')} for component ${name} version ${version}`)
for (const candidate of mrdocsConfigCandidates) {
if (candidate) {
const candidateBasePaths = [expandPathContext.base, expandPathContext.dot]
this.logger.debug(`Base paths: ${candidateBasePaths.join(', ')}`)
for (const basePath of candidateBasePaths) {
const candidatePath = path.join(basePath, candidate)
this.logger.debug(`Checking candidate path: ${candidatePath}`)
if (fs.existsSync(candidatePath)) {
mrdocsConfigFile = candidatePath
this.logger.debug(`Found mrdocs.yml file: ${mrdocsConfigFile}`)
break
}
}
Expand All @@ -471,37 +513,15 @@ class CppReferenceExtension {
}
if (!mrdocsConfigFile) {
this.logger.warn(`No mrdocs.yml file found in ${startPath} at locations ${mrdocsConfigCandidates.join(', ')} for component ${name} version ${version}`)
return undefined
continue
}
this.logger.debug(`Using mrdocs.yml file: ${mrdocsConfigFile}`)

return {
collectors.push({
config: mrdocsConfigFile
}
}).filter(Boolean)

// If the worktree doesn't exist, either checkout the worktree or create the directory
if (!worktree) {
if (checkoutWorktree) {
this.logger.debug(`Checking out worktree: ${worktreeDir}`)
const cache = gitCache[gitdir] || (gitCache[gitdir] = {})
const ref = `refs/${reftype === 'branch' ? 'head' : reftype}s/${refname}`
await this.prepareWorktree({
fs,
cache,
dir: worktreeDir,
gitdir,
ref,
remote,
bare: worktree === undefined
})
} else {
this.logger.debug(`Creating worktree directory: ${worktreeDir}`)
await fsp.mkdir(worktreeDir, {recursive: true})
}
} else {
this.logger.debug(`Using existing worktree directory: ${worktreeDir}`)
})
}
this.logger.debug(collectors, 'Collectors')

// For each collector, perform clean, run, and scan operations
for (const collector of collectors) {
Expand All @@ -526,40 +546,48 @@ class CppReferenceExtension {
* - If the worktree directory is not being managed, it adds it to the managed worktrees map.
* - If the worktree is not being checked out or it's not being kept, it removes the directory.
*
* @param {Array} collectorConfigs - An array of collector configuration objects.
* @param {Object} worktreeConfig - The worktree configuration object.
* @param {boolean} checkoutWorktree - Whether to checkout the worktree.
* @param {Object} origin - The origin object.
* @param {string} cacheDir - The cache directory.
* @param {Map} managedWorktrees - The map to manage worktrees.
* @returns {Promise<string>} A promise that resolves with the worktree directory.
*/
async prepareWorktree(collectorConfigs, origin, cacheDir, managedWorktrees) {
// If no worktree is provided, we create a configuration for it.
const worktreeConfig = collectorConfigs[0].worktree || {}
// Determine whether we should checkout the worktree.
// By default, we checkout unless explicitly set to false.
const checkoutWorktree = worktreeConfig.checkout !== false
async setupManagedWorktree(worktreeConfig, checkoutWorktree, origin, cacheDir, managedWorktrees) {
// Determine whether we should keep the worktree after use.
// By default, we don't keep it unless explicitly set to true.
const keepWorktree = 'keep' in worktreeConfig ? worktreeConfig.keep : keepWorktrees
const keepWorktree =
'keep' in worktreeConfig ?
worktreeConfig.keep :
'keepWorktrees' in this.config ?
this.config.keepWorktrees === true :
false
this.logger.debug(`Creating worktree for ${origin.url} with keepWorktree=${keepWorktree}`)

// Generate a name for the worktree directory and join it with
// the cache directory path.
const worktreeFolderName = CppReferenceExtension.generateWorktreeFolderName(origin, keepWorktree);
let worktreeDir = ospath.join(cacheDir, worktreeFolderName)
this.logger.debug(`Worktree directory: ${worktreeDir}`)

// Check if the worktree directory is already being managed.
// If it is, we add the origin to it.
// Otherwise, we create a new entry in the managed worktrees map.
if (managedWorktrees.has(worktreeDir)) {
this.logger.debug(`Worktree directory ${worktreeDir} is already being managed`)
managedWorktrees.get(worktreeDir).origins.add(origin)
// If we're not checking out the worktree, we remove the directory.
if (!checkoutWorktree) {
this.logger.debug(`Removing worktree directory ${worktreeDir} as we're not checking it out`)
await fsp.rm(worktreeDir, {force: true, recursive: true})
}
} else {
this.logger.debug(`Worktree directory ${worktreeDir} is not being managed`)
// If the worktree directory is not being managed, we add it to the managed worktrees map.
managedWorktrees.set(worktreeDir, {origins: new Set([origin]), keep: keepWorktree})
// If we're not checking out the worktree or we're not keeping it, we remove the directory.
// If we're not checking out the worktree, or we're not keeping it, we remove the directory.
if (!checkoutWorktree || keepWorktree !== true) {
this.logger.debug(`Removing worktree directory ${worktreeDir} as we're not checking it out or keeping it`)
await fsp.rm(worktreeDir, {
force: true,
recursive: true
Expand All @@ -569,6 +597,96 @@ class CppReferenceExtension {
return worktreeDir;
}

/**
* Prepares a git worktree from the specified gitdir, making use of the existing clone.
*
* If the worktree already exists from a previous iteration, the worktree is reset.
*
* A valid worktree is one that contains a .git/index file.
* Otherwise, a fresh worktree is created.
*
* If the gitdir contains an index file, that index file is temporarily overwritten to
* prepare the worktree and later restored before the function returns.
*
* @param {Object} repo - The repository object.
*/
async prepareWorktree(repo) {
const {dir: worktreeDir, gitdir, ref, remote = 'origin', bare, cache} = repo
this.logger.debug(`Preparing worktree for ${worktreeDir} from ${gitdir} at ${ref}`)
delete repo.remote
const currentIndexPath = ospath.join(gitdir, 'index')
this.logger.debug(`Current index: ${currentIndexPath}`)
const currentIndexPathBak = currentIndexPath + '~'
this.logger.debug(`Current index backup: ${currentIndexPathBak}`)
const restoreIndex = (await fsp.rename(currentIndexPath, currentIndexPathBak).catch(() => false)) === undefined
this.logger.debug(`Restore index: ${restoreIndex} because it was not possible to rename ${currentIndexPath} to ${currentIndexPathBak}`)
const worktreeGitdir = ospath.join(worktreeDir, '.git')
this.logger.debug(`Worktree gitdir: ${worktreeGitdir}`)
const worktreeIndexPath = ospath.join(worktreeGitdir, 'index')
this.logger.debug(`Worktree index: ${worktreeIndexPath}`)
try {
let force = true
try {
await CppReferenceExtension.mv(worktreeIndexPath, currentIndexPath)
this.logger.debug(`Moved ${worktreeIndexPath} to ${currentIndexPath}`)
await CppReferenceExtension.removeUntrackedFiles(repo)
this.logger.debug(`Removed untracked files from ${worktreeDir}`)
} catch {
this.logger.debug(`Could not move ${worktreeIndexPath} to ${currentIndexPath}`)
force = false
// index file not needed in this case
await fsp.unlink(currentIndexPath).catch(() => undefined)
await fsp.rm(worktreeDir, {recursive: true, force: true})
await fsp.mkdir(worktreeGitdir, {recursive: true})
this.logger.debug(`Created worktree directory ${worktreeDir}`)
Reflect.ownKeys(cache).forEach((it) => it.toString() === 'Symbol(PackfileCache)' || delete cache[it])
this.logger.debug(`Removed cache for ${worktreeDir}`)
}
let head
if (ref.startsWith('refs/heads/')) {
head = `ref: ${ref}`
const branchName = ref.slice(11)
if (bare || !(await git.listBranches(repo)).includes(branchName)) {
await git.branch({
...repo,
ref: branchName,
object: `refs/remotes/${remote}/${branchName}`,
force: true
})
}
} else {
head = await git.resolveRef(repo)
}
this.logger.debug(`Checking out HEAD: ${head}`)
await git.checkout({...repo, force, noUpdateHead: true, track: false})
this.logger.debug(`Checked out HEAD: ${head} at ${worktreeDir}`)
await fsp.writeFile(ospath.join(worktreeGitdir, 'commondir'), `${gitdir}\n`, 'utf8')
this.logger.debug(`Wrote commondir: ${gitdir}`)
const headPath = ospath.join(worktreeGitdir, 'HEAD');
await fsp.writeFile(headPath, `${head}\n`, 'utf8')
this.logger.debug(`Wrote HEAD path: ${headPath}`)
await CppReferenceExtension.mv(currentIndexPath, worktreeIndexPath)
this.logger.debug(`Moved ${currentIndexPath} to ${worktreeIndexPath}`)
} finally {
if (restoreIndex) await fsp.rename(currentIndexPathBak, currentIndexPath)
}
}

static mv(from, to) {
return fsp.cp(from, to).then(() => fsp.rm(from))
}

static removeUntrackedFiles(repo) {
const trees = [git.STAGE({}), git.WORKDIR()]
const map = (relpath, [sEntry]) => {
if (relpath === '.') return
if (relpath === '.git') return null
if (sEntry == null) return fsp.rm(ospath.join(repo.dir, relpath), {recursive: true}).then(invariably.null)
return sEntry.mode().then((mode) => (mode === 0o120000 ? null : undefined))
}
return git.walk({...repo, trees, map})
}

/**
* Processes a collector of a component version bucket.
*
Expand Down Expand Up @@ -1360,4 +1478,4 @@ class CppReferenceExtension {
}
}

module.exports = CppReferenceExtension
module.exports = CppReferenceExtension
Loading

0 comments on commit bd6e1ca

Please sign in to comment.