From 7fe382b30a40891a770974d2d52181337f60f43c Mon Sep 17 00:00:00 2001 From: Steve Boyd Date: Mon, 14 Aug 2023 14:59:16 +1200 Subject: [PATCH] NEW Add merge-ups --- .github/workflows/ci.yml | 10 +- README.md | 1 + composer.json | 7 +- funcs_scripts.php | 38 ++++++- funcs_utils.php | 77 ++++++++++++- run.php | 14 ++- scripts/cms-any/editorconfig.php | 4 +- scripts/cms5/keepalive.php | 34 ++++++ scripts/cms5/merge-ups.php | 59 ++++++++++ update_command.php | 180 +++++++++++++++++++++---------- 10 files changed, 351 insertions(+), 73 deletions(-) create mode 100644 scripts/cms5/keepalive.php create mode 100644 scripts/cms5/merge-ups.php diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46090ea..830067f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,10 +17,10 @@ jobs: - name: Install PHP uses: shivammathur/setup-php@1a18b2267f80291a81ca1d33e7c851fe09e7dfc4 # v2.22.0 with: - php-version: 7.4 - - - name: Install PHPUnit - run: wget https://phar.phpunit.de/phpunit-9.5.phar + php-version: 8.1 + + - name: Composer install + run: composer install --prefer-dist --no-progress --no-suggest --ansi --no-interaction --no-scripts --no-plugins --optimize-autoloader - name: PHPUnit - run: php phpunit-9.5.phar --verbose --colors=always + run: vendor/bin/phpunit --verbose --colors=always diff --git a/README.md b/README.md index df29409..b47ed95 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ MS_GITHUB_TOKEN=abc123 php run.php update --cms-major=5 --branch=next-minor --dr | --dry-run | Do not push to github or create pull-requests | | --account | GitHub account to use for creating pull-requests (default: creative-commoners) | | --no-delete | Do not delete _data and _modules directories before running | +| --update-prs | Update existing open PRs instead of creating new PRs | ## GitHub API secondary rate limit diff --git a/composer.json b/composer.json index aee3979..88b0f89 100644 --- a/composer.json +++ b/composer.json @@ -1,10 +1,11 @@ { "require": { - "php": ">=7.4", + "php": ">=8.1", "symfony/console": "^6.3", - "symfony/process": "^6.3" + "symfony/process": "^6.3", + "panlatent/cron-expression-descriptor": "^1" }, "require-dev": { - "phpunit/phpunit": "^9.5" + "phpunit/phpunit": "^9.6" } } diff --git a/funcs_scripts.php b/funcs_scripts.php index 7735e1c..e50225e 100644 --- a/funcs_scripts.php +++ b/funcs_scripts.php @@ -82,7 +82,7 @@ function delete_file_if_exists($filename) } /** - * Rename a file to the root of the module being processed if it exists + * Rename a file relative to the root of the module being processed if it exists * * Example usage: * rename_file_if_exists('oldfilename.md', 'newfilename.md') @@ -108,14 +108,30 @@ function rename_file_if_exists($oldFilename, $newFilename) function module_is_recipe() { global $MODULE_DIR; - if (strpos('/recipe-', $MODULE_DIR) !== false - || strpos('/silverstripe-installer', $MODULE_DIR) !== false + if (strpos($MODULE_DIR, '/recipe-') !== false + || strpos($MODULE_DIR, '/silverstripe-installer') !== false ) { return true; } return false; } +/** + * Determine if the module being processed is something installed on a website e.g. gha-* + * + * Example usage: + * module_is_not_for_cms() + */ +function module_is_not_for_cms() +{ + global $MODULE_DIR; + return strpos($MODULE_DIR, '/gha-') !== false + || strpos($MODULE_DIR, '/developer-docs') !== false + || strpos($MODULE_DIR, '/vendor-plugin') !== false + || strpos($MODULE_DIR, '/eslint-config') !== false + || strpos($MODULE_DIR, '/webpack-config') !== false; +} + /** * Determine if the module being processed is one of the modules in a list * @@ -132,7 +148,7 @@ function module_is_one_of($repos) if (!is_string($repo)) { error('repo is not a string'); } - if (strpos("/$repo", $MODULE_DIR) !== false) { + if (strpos($MODULE_DIR, "/$repo") !== false) { return true; } } @@ -192,3 +208,17 @@ function human_cron(string $cron): string $str = preg_replace('# (AM|PM),#', ' $1 UTC,', $str); return $str; } + +/** + * Creates a predicatable random int between 0 and $max based on the module name to be used with the % mod operator. + * $offset variable will offset both the min (0) and $max. e.g. $offset of 1 with a max of 27 will return an int + * between 1 and 28 + * Note that this will return the exact same value every time it is called for a given module. + */ +function predictable_random_int($max, $offset = 0): int +{ + global $MODULE_DIR; + $chars = str_split($MODULE_DIR); + $codes = array_map(fn($c) => ord($c), $chars); + return ((10000 + array_sum($codes)) % $max) + $offset; +} diff --git a/funcs_utils.php b/funcs_utils.php index 1306dfd..3cb907f 100644 --- a/funcs_utils.php +++ b/funcs_utils.php @@ -34,7 +34,7 @@ function write_file($path, $contents) } $dirname = dirname($path); if (!file_exists($dirname)) { - error("Directory $dirname does not exist"); + mkdir($dirname, 0775, true); } $contents = trim($contents) . "\n"; file_put_contents($path, $contents); @@ -67,12 +67,43 @@ function supported_modules($cmsMajor) 'account' => explode('/', $ghrepo)[0], 'repo' => explode('/', $ghrepo)[1], 'cloneUrl' => "git@github.com:$ghrepo.git", - 'branch' => max($module['branches'] ?: [-1]) ]; } return $modules; } +/** + * Hardcoded list of additional repositories to standardise (e.g. silverstripe/gha-*) + * + * Repositories in this list should only have a single supported major version + * This will only be included if the $cmsMajor is the CURRENT_CMS_MAJOR + */ +function extra_repositories() +{ + $modules = []; + // iterating to page 7 should be enough to get all the repos well into the future + for ($i = 0; $i < 7; $i++) { + $json = github_api("https://api.github.com/orgs/silverstripe/repos?per_page=100&page=$i"); + foreach ($json as $repo) { + if ($repo['archived']) { + continue; + } + $ghrepo = $repo['full_name']; + // exclude non gha-* repos + if (strpos($ghrepo, '/gha-') === false) { + continue; + } + $modules[] = [ + 'ghrepo' => $ghrepo, + 'account' => explode('/', $ghrepo)[0], + 'repo' => explode('/', $ghrepo)[1], + 'cloneUrl' => "git@github.com:$ghrepo.git", + ]; + } + } + return $modules; +} + /** * Returns a list of all scripts files to run against a particular cms major version */ @@ -169,6 +200,8 @@ function github_token() */ function github_api($url, $data = []) { + // silverstripe-themes has a kind of weird redirect only for api requests + $url = str_replace('/silverstripe-themes/silverstripe-simple', '/silverstripe/silverstripe-simple', $url); $token = github_token(); $jsonStr = empty($data) ? '' : json_encode($data, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); $ch = curl_init($url); @@ -265,12 +298,38 @@ function branch_to_checkout($branches, $currentBranch, $currentBranchCmsMajor, $ return (string) $branchToCheckout; } +/** + * Uses composer.json to workout the current branch cms major version + * + * If composer.json does not exist then it's assumed to be CURRENT_CMS_MAJOR + */ function current_branch_cms_major( // this param is only used for unit testing string $composerJson = '' ) { - // read __composer.json of the current branch - $contents = $composerJson ?: read_file('composer.json'); + global $MODULE_DIR; + + if ($composerJson) { + $contents = $composerJson; + } elseif (check_file_exists('composer.json')) { + $contents = read_file('composer.json'); + } else { + return CURRENT_CMS_MAJOR; + } + + // special logic for developer-docs + if (strpos($MODULE_DIR, '/developer-docs') !== false) { + $currentBranch = cmd('git rev-parse --abbrev-ref HEAD', $MODULE_DIR); + if (!preg_match('#^(pulls/)?([0-9]+)(\.[0-9]+)?(/|$)#', $currentBranch, $matches)) { + error("Could work out current major for developer-docs from branch $currentBranch"); + } + return $matches[2]; + } + + // special logic for silverstripe-themes/silverstripe-simple + if (strpos($MODULE_DIR, '/silverstripe-simple') !== false) { + return CURRENT_CMS_MAJOR; + } $json = json_decode($contents); if (is_null($json)) { @@ -287,7 +346,15 @@ function current_branch_cms_major( } if (!$version) { $version = preg_replace('#[^0-9\.]#', '', $json->require->{'silverstripe/assets'} ?? ''); - $matchedOnBranchThreeLess = true; + if ($version) { + $matchedOnBranchThreeLess = true; + } + } + if (!$version) { + $version = preg_replace('#[^0-9\.]#', '', $json->require->{'cwp/starter-theme'} ?? ''); + if ($version) { + $version += 1; + } } $cmsMajor = ''; if (preg_match('#^([0-9]+)+\.?[0-9]*$#', $version, $matches)) { diff --git a/run.php b/run.php index e6e10a1..32f4a6a 100644 --- a/run.php +++ b/run.php @@ -72,5 +72,17 @@ InputOption::VALUE_NONE, 'Do not delete _data and _modules directories before running' ) + ->addOption( + 'update-prs', + null, + InputOption::VALUE_NONE, + 'Checkout out and update the latest open PR instead of creating a new one' + ) ->setCode($updateCommand); -$app->run(); + +try { + $app->run(); +} catch (Error|Exception $e) { + // Make sure we output and information about PRs which were raised before killing the process. + error("file: {$e->getFile()}\nline: {$e->getLine()}\nmessage: {$e->getMessage()}"); +} diff --git a/scripts/cms-any/editorconfig.php b/scripts/cms-any/editorconfig.php index d3ee8fa..ef52ea3 100644 --- a/scripts/cms-any/editorconfig.php +++ b/scripts/cms-any/editorconfig.php @@ -34,4 +34,6 @@ insert_final_newline = false EOT; -write_file_if_not_exist('.editorconfig', $contents); +if (!module_is_not_for_cms()) { + write_file_if_not_exist('.editorconfig', $contents); +} diff --git a/scripts/cms5/keepalive.php b/scripts/cms5/keepalive.php new file mode 100644 index 0000000..55265ac --- /dev/null +++ b/scripts/cms5/keepalive.php @@ -0,0 +1,34 @@ +getOption('only')) { $only = explode(',', $input->getOption('only')); $modules = array_filter($modules, function ($module) use ($only) { @@ -62,39 +67,99 @@ $repo = $module['repo']; $cloneUrl = $module['cloneUrl']; $MODULE_DIR = MODULES_DIR . "/$repo"; + // clone repo + // always clone the actual remote even when doing update-prs even though this is slower + // reason is because we read origin in .git/config to workout the actual $account in + // module_account() which is very important when setting up github-action crons if (!file_exists($MODULE_DIR)) { cmd("git clone $cloneUrl", MODULES_DIR); } + // set git remote + $prAccount = $input->getOption('account') ?? DEFAULT_ACCOUNT; + $origin = cmd('git remote get-url origin', $MODULE_DIR); + $prOrigin = str_replace("git@github.com:$account", "git@github.com:$prAccount", $origin); + // remove any existing pr-remote - need to do this in case we change the account option + $remotes = explode("\n", cmd('git remote', $MODULE_DIR)); + if (in_array('pr-remote', $remotes)) { + cmd('git remote remove pr-remote', $MODULE_DIR); + } + cmd("git remote add pr-remote $prOrigin", $MODULE_DIR); - // get all branches - $allBranches = explode("\n", cmd('git branch -r', $MODULE_DIR)); - $allBranches = array_map(fn($branch) => trim(str_replace('origin/', '', $branch)), $allBranches); - - // reset to the default branch so that we can then calculate the correct branch to checkout - // this is needed for scenarios where we may be on something unparsable like pulls/5/lorem-ipsum - $cmd = "git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@'"; - $defaultBranch = cmd($cmd, $MODULE_DIR); - cmd("git checkout $defaultBranch", $MODULE_DIR); - - // checkout the branch to run scripts over - $currentBranch = cmd('git rev-parse --abbrev-ref HEAD', $MODULE_DIR); - $currentBranchCmsMajor = current_branch_cms_major(); - $branchToCheckout = branch_to_checkout( - $allBranches, - $currentBranch, - $currentBranchCmsMajor, - $cmsMajor, - $branchOption - ); - if (!in_array($branchToCheckout, $allBranches)) { - error("Could not find branch to checkout for $repo using --branch=$branchOption"); + if ($input->getOption('update-prs')) { + // checkout latest existing pr branch + cmd('git fetch pr-remote', $MODULE_DIR); + $allBranches = explode("\n", cmd('git branch -r', $MODULE_DIR)); + // example branch name: pulls/5/module-standardiser-1691550112 + $allBranches = array_map('trim', $allBranches); + $allBranches = array_filter($allBranches, function($branch) { + return preg_match('#^pr\-remote/pulls/[0-9\.]+/module\-standardiser\-[0-9]{10}$#', $branch); + }); + if (empty($allBranches)) { + warning("Could not find an existing PR branch for $repo - skipping"); + continue; + } + // sort so that the branch with the highest timestamp goes to position 0 in the array + usort($allBranches, function($a, $b) { + return (substr($a, -10) <=> substr($b, -10)) * -1; + }); + $branchToCheckout = $allBranches[0]; + $branchToCheckout = preg_replace('#^pr\-remote/#', '', $branchToCheckout); + $prBranch = $branchToCheckout; + $allPRs = github_api("https://api.github.com/repos/$account/$repo/pulls?per_page=100"); + $allPRs = array_filter($allPRs, function($pr) use($prBranch) { + return $pr['title'] === PR_TITLE && $pr['head']['ref'] === $prBranch && $pr['state'] === 'open'; + }); + if (count($allPRs) < 1) { + warning("Could not find an existing open PR for $repo for branch $prBranch - skipping"); + continue; + } + } else { + // get all branches + $allBranches = explode("\n", cmd('git branch -r', $MODULE_DIR)); + $allBranches = array_map(fn($branch) => trim(str_replace('origin/', '', $branch)), $allBranches); + + // reset to the default branch so that we can then calculate the correct branch to checkout + // this is needed for scenarios where we may be on something unparsable like pulls/5/lorem-ipsum + $cmd = "git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@'"; + $defaultBranch = cmd($cmd, $MODULE_DIR); + cmd("git checkout $defaultBranch", $MODULE_DIR); + + // checkout the branch to run scripts over + $currentBranch = cmd('git rev-parse --abbrev-ref HEAD', $MODULE_DIR); + // ensure that we're on a standard next-minor style branch + if (!ctype_digit($currentBranch)) { + $tmp = array_filter($allBranches, fn($branch) => ctype_digit($branch)); + if (empty($tmp)) { + error('Could not find a next-minor style branch'); + } + $currentBranch = max($tmp); + cmd("git checkout $currentBranch", $MODULE_DIR); + } + $currentBranchCmsMajor = current_branch_cms_major(); + $branchToCheckout = branch_to_checkout( + $allBranches, + $currentBranch, + $currentBranchCmsMajor, + $cmsMajor, + $branchOption + ); + if (!in_array($branchToCheckout, $allBranches)) { + error("Could not find branch to checkout for $repo using --branch=$branchOption"); + } } cmd("git checkout $branchToCheckout", $MODULE_DIR); + // ensure that this branch actually supports the cmsMajor we're targetting + if (current_branch_cms_major() !== $cmsMajor) { + error("Branch $branchToCheckout does not support CMS major version $cmsMajor"); + } + // create a new branch used for the pull-request - $timestamp = time(); - $prBranch = "pulls/$branchToCheckout/module-standardiser-$timestamp"; - cmd("git checkout -b $prBranch", $MODULE_DIR); + if (!$input->getOption('update-prs')) { + $timestamp = time(); + $prBranch = "pulls/$branchToCheckout/module-standardiser-$timestamp"; + cmd("git checkout -b $prBranch", $MODULE_DIR); + } // run scripts foreach ($scriptFiles as $scriptFile) { @@ -105,42 +170,49 @@ eval($contents); } - // set git remote - $prAccount = $input->getOption('account') ?? DEFAULT_ACCOUNT; - $origin = cmd('git remote get-url origin', $MODULE_DIR); - $prOrigin = str_replace("git@github.com:$account", "git@github.com:$prAccount", $origin); - // remove any existing pr-remote - need to do this in case we change the account option - $remotes = explode("\n", cmd('git remote', $MODULE_DIR)); - if (in_array('pr-remote', $remotes)) { - cmd('git remote remove pr-remote', $MODULE_DIR); - } - cmd("git remote add pr-remote $prOrigin", $MODULE_DIR); - // commit changes, push changes and create pull-request $status = cmd('git status', $MODULE_DIR); if (strpos($status, 'nothing to commit') !== false) { info("No changes to commit for $repo"); + continue; + } + cmd('git add .', $MODULE_DIR); + if ($input->getOption('update-prs')) { + // squash on to existing commit + $lastCommitMessage = cmd('git log -1 --pretty=%B', $MODULE_DIR); + if ($lastCommitMessage !== PR_TITLE) { + error("Last commit message \"$lastCommitMessage\" does not match PR_TITLE \"" . PR_TITLE . "\""); + } + cmd("git commit --amend --no-edit", $MODULE_DIR); } else { - cmd('git add .', $MODULE_DIR); + // create new commit cmd("git commit -m '" . PR_TITLE . "'", $MODULE_DIR); - if ($input->getOption('dry-run')) { - info('Not pushing changes or creating pull-request because --dry-run option is set'); - } else { - // push changes to pr-remote - cmd("git push -u pr-remote $prBranch", $MODULE_DIR); - // create pull-request using github api - // https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#create-a-pull-request - $responseJson = github_api("https://api.github.com/repos/$account/$repo/pulls", [ - 'title' => PR_TITLE, - 'body' => PR_DESCRIPTION, - 'head' => "$prAccount:$prBranch", - 'base' => $branchToCheckout, - ]); - $PRS_CREATED[] = $responseJson['html_url']; - $REPOS_WITH_PRS_CREATED[] = $repo; - info("Created pull-request for $repo"); - } } + if ($input->getOption('dry-run')) { + info('Not pushing changes or creating pull-request because --dry-run option is set'); + continue; + } + // push changes to pr-remote + // force pushing for cases when doing update-prs + // double make check we're on a branch that we are willing to force push + $currentBranch = cmd('git rev-parse --abbrev-ref HEAD', $MODULE_DIR); + if (!preg_match('#^pulls/[0-9\.]+/module\-standardiser\-[0-9]{10}$#', $currentBranch)) { + error("Branch $currentBranch is not a pull-request branch"); + } + cmd("git push -f -u pr-remote $prBranch", $MODULE_DIR); + // create pull-request using github api + if (!$input->getOption('update-prs')) { + // https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#create-a-pull-request + $responseJson = github_api("https://api.github.com/repos/$account/$repo/pulls", [ + 'title' => PR_TITLE, + 'body' => PR_DESCRIPTION, + 'head' => "$prAccount:$prBranch", + 'base' => $branchToCheckout, + ]); + $PRS_CREATED[] = $responseJson['html_url']; + info("Created pull-request for $repo"); + } + $REPOS_WITH_PRS_CREATED[] = $repo; } output_repos_with_prs_created(); output_prs_created();