Skip to content

Commit

Permalink
feat: Use work trees instead of copying the whole directory
Browse files Browse the repository at this point in the history 
This should speed up backports by not copying the repository but create a clean work tree with the required branches set.

Signed-off-by: Ferdinand Thiessen <[email protected]>
  • Loading branch information
@susnux
susnux committed Sep 3, 2024
1 parent 6c1ba1b commit 4bc27eb
Show file tree
Hide file tree
Showing 5 changed files with 182 additions and 191 deletions.
283 changes: 139 additions & 144 deletions src/backport.ts
View file
Original file line number Diff line number Diff line change
@@ -1,175 +1,170 @@
import { existsSync, rmSync } from 'node:fs'
import { Octokit } from '@octokit/rest'

import { cherryPickCommits, cloneAndCacheRepo, hasDiff, hasEmptyCommits, hasSkipCiCommits, pushBranch } from './gitUtils'
import { CherryPickResult, Task } from './constants'
import { debug, error, info, warn } from './logUtils'
import { Reaction, addReaction, getAuthToken, getAvailableLabels, getLabelsFromPR, getAvailableMilestones, requestReviewers, getReviewers, createBackportPullRequest, setPRLabels, setPRMilestone, getChangesFromPR, updatePRBody, commentOnPR, assignToPR } from './githubUtils'
import { getBackportBody, getFailureCommentBody, getLabelsForPR, getMilestoneFromBase } from './nextcloudUtils'

export const backport = (task: Task) => new Promise<void>((resolve, reject) => {
getAuthToken(task.installationId).then(async token => {
const octokit = new Octokit({ auth: token })

let tmpDir: string = ''
let prNumber: number = 0
let conflicts: CherryPickResult|null = null
const backportBranch = `backport/${task.prNumber}/${task.branch}`

info(task, `Starting backport request`)

// Add a reaction to the comment to indicate that we're processing it
import { cherryPickCommits, cloneAndCacheRepo, hasDiff, hasEmptyCommits, hasSkipCiCommits, pushBranch } from './gitUtils.js'
import { CherryPickResult, Task } from './constants.js'
import { debug, error, info, warn } from './logUtils.js'
import { Reaction, addReaction, getAuthToken, getAvailableLabels, getLabelsFromPR, getAvailableMilestones, requestReviewers, getReviewers, createBackportPullRequest, setPRLabels, setPRMilestone, getChangesFromPR, updatePRBody, commentOnPR, assignToPR } from './githubUtils.js'
import { getBackportBody, getFailureCommentBody, getLabelsForPR, getMilestoneFromBase } from './nextcloudUtils.js'

export async function backport(task: Task): Promise<void> {
const token = await getAuthToken(task.installationId)
const octokit = new Octokit({ auth: token })

let tmpDir: string = ''
let prNumber: number = 0
let conflicts: CherryPickResult|null = null
const backportBranch = `backport/${task.prNumber}/${task.branch}`

info(task, `Starting backport request`)

// Add a reaction to the comment to indicate that we're processing it
try {
await addReaction(octokit, task, Reaction.THUMBS_UP)
} catch (e) {
error(task, `Failed to add reaction to PR: ${e.message}`)
// continue, this is not a fatal error
}

try {
// Clone and cache the repo
try {
await addReaction(octokit, task, Reaction.THUMBS_UP)
tmpDir = await cloneAndCacheRepo(task, backportBranch)
info(task, `Cloned to ${tmpDir}`)
} catch (e) {
error(task, `Failed to add reaction to PR: ${e.message}`)
// continue, this is not a fatal error
throw new Error(`Failed to clone repository: ${e.message}`)
}

// Cherry pick the commits
try {
// Clone and cache the repo
try {
tmpDir = await cloneAndCacheRepo(task, backportBranch)
info(task, `Cloned to ${tmpDir}`)
} catch (e) {
throw new Error(`Failed to clone repository: ${e.message}`)
conflicts = await cherryPickCommits(task, tmpDir)
if (conflicts === CherryPickResult.CONFLICTS) {
warn(task, `Cherry picking commits resulted in conflicts`)
} else {
info(task, `Cherry picking commits successful`)
}
} catch (e) {
throw new Error(`Failed to cherry pick commits: ${e.message}`)
}

// Cherry pick the commits
try {
conflicts = await cherryPickCommits(task, tmpDir)
if (conflicts === CherryPickResult.CONFLICTS) {
warn(task, `Cherry picking commits resulted in conflicts`)
} else {
info(task, `Cherry picking commits successful`)
}
} catch (e) {
throw new Error(`Failed to cherry pick commits: ${e.message}`)
}
// Check if there are any changes to backport
const hasChanges = await hasDiff(tmpDir, task.branch, backportBranch, task)
if (!hasChanges) {
throw new Error(`No changes found in backport branch`)
}

// Check if there are any changes to backport
const hasChanges = await hasDiff(tmpDir, task.branch, backportBranch, task)
if (!hasChanges) {
throw new Error(`No changes found in backport branch`)
}
// Push the branch
try {
await pushBranch(task, tmpDir, token, backportBranch)
info(task, `Pushed branch ${backportBranch}`)
} catch (e) {
throw new Error(`Failed to push branch ${backportBranch}: ${e.message}`)
}

// Push the branch
try {
await pushBranch(task, tmpDir, token, backportBranch)
info(task, `Pushed branch ${backportBranch}`)
} catch (e) {
throw new Error(`Failed to push branch ${backportBranch}: ${e.message}`)
}
// Create the pull request
try {
const reviewers = await getReviewers(octokit, task)
const prCreationResult = await createBackportPullRequest(octokit, task, backportBranch, conflicts === CherryPickResult.CONFLICTS)
prNumber = prCreationResult.data.number
info(task, `Opened Pull Request #${prNumber} on ${prCreationResult.data.html_url}`)

// Create the pull request
try {
const reviewers = await getReviewers(octokit, task)
const prCreationResult = await createBackportPullRequest(octokit, task, backportBranch, conflicts === CherryPickResult.CONFLICTS)
prNumber = prCreationResult.data.number
info(task, `Opened Pull Request #${prNumber} on ${prCreationResult.data.html_url}`)

try {
// Ask for reviews from all reviewers of the original PR
if (reviewers.length !== 0) {
await requestReviewers(octokit, task, prNumber, reviewers)
}

// Also ask the author of the original PR for a review
await requestReviewers(octokit, task, prNumber, [task.author])
info(task, `Requested reviews from ${[...reviewers, task.author].join(', ')}`)
} catch (e) {
error(task, `Failed to request reviews: ${e.message}`)
// Ask for reviews from all reviewers of the original PR
if (reviewers.length !== 0) {
await requestReviewers(octokit, task, prNumber, reviewers)
}
} catch (e) {
throw new Error(`Failed to create pull request: ${e.message}`)
}

// Get labels from original PR and set them on the new PR
try {
const availableLabels = await getAvailableLabels(octokit, task)
const prLabels = await getLabelsFromPR(octokit, task)
const labels = getLabelsForPR(prLabels, availableLabels)
await setPRLabels(octokit, task, prNumber, labels)
info(task, `Set labels: ${labels.join(', ')}`)
// Also ask the author of the original PR for a review
await requestReviewers(octokit, task, prNumber, [task.author])
info(task, `Requested reviews from ${[...reviewers, task.author].join(', ')}`)
} catch (e) {
error(task, `Failed to get and set labels: ${e.message}`)
// continue, this is not a fatal error
error(task, `Failed to request reviews: ${e.message}`)
}
} catch (e) {
throw new Error(`Failed to create pull request: ${e.message}`)
}

// Find new appropriate Milestone and set it on the new PR
try {
const availableMilestone = await getAvailableMilestones(octokit, task)
const milestone = await getMilestoneFromBase(task.branch, availableMilestone)
await setPRMilestone(octokit, task, prNumber, milestone)
info(task, `Set milestone: ${milestone.title}`)
} catch (e) {
error(task, `Failed to find appropriate milestone: ${e.message}`)
// continue, this is not a fatal error
}
// Get labels from original PR and set them on the new PR
try {
const availableLabels = await getAvailableLabels(octokit, task)
const prLabels = await getLabelsFromPR(octokit, task)
const labels = getLabelsForPR(prLabels, availableLabels)
await setPRLabels(octokit, task, prNumber, labels)
info(task, `Set labels: ${labels.join(', ')}`)
} catch (e) {
error(task, `Failed to get and set labels: ${e.message}`)
// continue, this is not a fatal error
}

// Assign the PR to the author of the original PR
try {
await assignToPR(octokit, task, prNumber, [task.author])
info(task, `Assigned original author: ${task.author}`)
} catch (e) {
error(task, `Failed to assign PR: ${e.message}`)
// continue, this is not a fatal error
}
// Find new appropriate Milestone and set it on the new PR
try {
const availableMilestone = await getAvailableMilestones(octokit, task)
const milestone = await getMilestoneFromBase(task.branch, availableMilestone)
await setPRMilestone(octokit, task, prNumber, milestone)
info(task, `Set milestone: ${milestone.title}`)
} catch (e) {
error(task, `Failed to find appropriate milestone: ${e.message}`)
// continue, this is not a fatal error
}

// Assign the PR to the author of the original PR
try {
await assignToPR(octokit, task, prNumber, [task.author])
info(task, `Assigned original author: ${task.author}`)
} catch (e) {
error(task, `Failed to assign PR: ${e.message}`)
// continue, this is not a fatal error
}

// Compare the original PR with the new PR
// Compare the original PR with the new PR
try {
const oldChanges = await getChangesFromPR(octokit, task, task.prNumber)
const newChanges = await getChangesFromPR(octokit, task, prNumber)
const diffChanges = oldChanges.additions !== newChanges.additions
|| oldChanges.deletions !== newChanges.deletions
|| oldChanges.changedFiles !== newChanges.changedFiles
const skipCi = await hasSkipCiCommits(tmpDir, task.commits.length)
const emptyCommits = await hasEmptyCommits(tmpDir, task.commits.length, task)
const hasConflicts = conflicts === CherryPickResult.CONFLICTS

debug(task, `hasConflicts: ${hasConflicts}, diffChanges: ${diffChanges}, emptyCommits: ${emptyCommits}, skipCi: ${skipCi}`)
try {
const oldChanges = await getChangesFromPR(octokit, task, task.prNumber)
const newChanges = await getChangesFromPR(octokit, task, prNumber)
const diffChanges = oldChanges.additions !== newChanges.additions
|| oldChanges.deletions !== newChanges.deletions
|| oldChanges.changedFiles !== newChanges.changedFiles
const skipCi = await hasSkipCiCommits(tmpDir, task.commits.length)
const emptyCommits = await hasEmptyCommits(tmpDir, task.commits.length, task)
const hasConflicts = conflicts === CherryPickResult.CONFLICTS

debug(task, `hasConflicts: ${hasConflicts}, diffChanges: ${diffChanges}, emptyCommits: ${emptyCommits}, skipCi: ${skipCi}`)
try {
if (hasConflicts || diffChanges || emptyCommits || skipCi) {
const newBody = await getBackportBody(task.prNumber, hasConflicts, diffChanges, emptyCommits, skipCi)
await updatePRBody(octokit, task, prNumber, newBody)
}
} catch (e) {
error(task, `Failed to update PR body: ${e.message}`)
// continue, this is not a fatal error
if (hasConflicts || diffChanges || emptyCommits || skipCi) {
const newBody = await getBackportBody(task.prNumber, hasConflicts, diffChanges, emptyCommits, skipCi)
await updatePRBody(octokit, task, prNumber, newBody)
}
} catch (e) {
error(task, `Failed to compare changes: ${e.message}`)
error(task, `Failed to update PR body: ${e.message}`)
// continue, this is not a fatal error
}

// Success! We're done here
addReaction(octokit, task, Reaction.HOORAY)
} catch (e) {
// Add a thumbs down reaction to the comment to indicate that we failed
try {
addReaction(octokit, task, Reaction.THUMBS_DOWN)
const failureComment = getFailureCommentBody(task, backportBranch, e?.message)
await commentOnPR(octokit, task, failureComment)
} catch (e) {
error(task, `Failed to comment failure on PR: ${e.message}`)
// continue, this is not a fatal error
}
error(task, `Failed to compare changes: ${e.message}`)
// continue, this is not a fatal error
}

reject(`Failed to backport: ${e.message}`)
// Success! We're done here
addReaction(octokit, task, Reaction.HOORAY)
} catch (e) {
// Add a thumbs down reaction to the comment to indicate that we failed
try {
addReaction(octokit, task, Reaction.THUMBS_DOWN)
const failureComment = getFailureCommentBody(task, backportBranch, e?.message)
await commentOnPR(octokit, task, failureComment)
} catch (e) {
error(task, `Failed to comment failure on PR: ${e.message}`)
// continue, this is not a fatal error
}

// Remove the temp dir if it exists
if (tmpDir !== '' && existsSync(tmpDir)) {
try {
rmSync(tmpDir, { recursive: true })
info(task, `Removed ${tmpDir}`)
resolve()
} catch (e) {
reject(`Failed to remove ${tmpDir}: ${e.message}`)
}
throw new Error(`Failed to backport: ${e.message}`)
}

// Remove the temp dir if it exists
if (tmpDir !== '' && existsSync(tmpDir)) {
try {
rmSync(tmpDir, { recursive: true })
info(task, `Removed ${tmpDir}`)
} catch (e) {
throw new Error(`Failed to remove ${tmpDir}: ${e.message}`)
}
})
}).catch(e => {
error(task, e)
throw e
})
}
}
51 changes: 21 additions & 30 deletions src/gitUtils.ts
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import { cpSync, existsSync, mkdirSync} from 'node:fs'
import { existsSync, mkdirSync} from 'node:fs'
import { join } from 'node:path'
import { simpleGit } from 'simple-git'

import { CACHE_DIRNAME, CherryPickResult, ROOT_DIR, Task, WORK_DIRNAME } from './constants'
import { debug, error } from './logUtils'
import { CACHE_DIRNAME, CherryPickResult, ROOT_DIR, Task, WORK_DIRNAME } from './constants.js'
import { debug, error } from './logUtils.js'
import { randomBytes } from 'node:crypto'

export const setGlobalGitConfig = async (user: string): Promise<void> => {
const git = simpleGit()
Expand All @@ -27,11 +28,18 @@ export const cloneAndCacheRepo = async (task: Task, backportBranch: string): Pro
// Clone the repo into the cache dir or make sure it already exists
const cachedRepoRoot = join(ROOT_DIR, CACHE_DIRNAME, owner, repo)
try {
if (!existsSync(cachedRepoRoot + '/.git')) {
// Create repo path if needed
if (!existsSync(cachedRepoRoot)) {
mkdirSync(cachedRepoRoot, { recursive: true })
const git = simpleGit(cachedRepoRoot)
}

const git = simpleGit(cachedRepoRoot)
if (!existsSync(cachedRepoRoot + '/.git')) {
// Is not a repository, so clone
await git.clone(`https://github.com/${owner}/${repo}`, '.')
} else {
// Is already a repository so make sure it is clean and follows the default branch
await git.clean(['-X', '-d', '-f'])
debug(task, `Repo already cached at ${cachedRepoRoot}`)
}
} catch (e) {
Expand All @@ -52,36 +60,19 @@ export const cloneAndCacheRepo = async (task: Task, backportBranch: string): Pro
// }

// Init a new temp repo in the work dir
const tmpDirName = Math.random().toString(36).substring(7)
const tmpDirName = randomBytes(7).toString('hex')
const tmpRepoRoot = join(ROOT_DIR, WORK_DIRNAME, tmpDirName)
try {
// Copy the cached repo to the temp repo
mkdirSync(join(ROOT_DIR, WORK_DIRNAME), { recursive: true })
cpSync(cachedRepoRoot, tmpRepoRoot, { recursive: true })
} catch (e) {
throw new Error(`Failed to copy cached repo: ${e.message}`)
}

try {
// Checkout all the branches
const git = simpleGit(tmpRepoRoot)
// TODO: We could do that to the cached repo, but
// this seem to create some concurrency issues.
await git.raw(['fetch', '--all'])
await git.raw(['pull', '--prune'])

// reset and clean the repo
await git.raw(['reset', '--hard', `origin/${branch}`])
await git.raw(['clean', '--force', '-dfx'])

// Checkout the branch we want to backport from
await git.checkout(branch)
await git.checkoutBranch(
backportBranch,
branch
)
// create worktree
const git = simpleGit(cachedRepoRoot)
// fetch upstream version of the branch - well we need to fetch all because we do not know where the commits are located we need to cherry-pick
await git.fetch(['-p', '--all'])
// create work tree with up-to-date content of that branch
await git.raw(['worktree', 'add', '-b', backportBranch, tmpRepoRoot, `origin/${branch}`])
} catch (e) {
throw new Error(`Failed to checkout branches: ${e.message}`)
throw new Error(`Failed to create working tree: ${e.message}`)
}

return tmpRepoRoot
Expand Down
Loading

0 comments on commit 4bc27eb

Please sign in to comment.