diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 587a4a0bd..000000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,103 +0,0 @@ -steps: - - group: "Test" - steps: - - label: "test {{ matrix }}" - command: "./builds/run_sbt_tests.sh {{ matrix }}" - matrix: - - "fixtures" - - "http" - - "json" - - "typesafe_app" - - "monitoring" - - "monitoring_typesafe" - - "messaging" - - "messaging_typesafe" - - "storage" - - "storage_typesafe" - - "elasticsearch" - - "elasticsearch_typesafe" - - "sierra" - - agents: - queue: "scala" - - - group: "Report evictions" - steps: - - label: "evictions {{ matrix }}" - command: "./builds/report_sbt_evictions.sh {{ matrix }}" - soft_fail: - - exit_status: 2 - matrix: - - "fixtures" - - "http" - - "json" - - "typesafe_app" - - "monitoring" - - "monitoring_typesafe" - - "messaging" - - "messaging_typesafe" - - "storage" - - "storage_typesafe" - - "elasticsearch" - - "elasticsearch_typesafe" - - "sierra" - agents: - queue: "scala" - artifact_paths: - ".reports/evicted*" - - - wait - - - label: "Collate evictions" - commands: - - "mkdir -p .reports" - - "buildkite-agent artifact download '.reports/evicted_*' .reports/" - - "builds/report_unique_evictions.sh | buildkite-agent annotate --context=evictions" - agents: - queue: nano - - wait - - - - label: "cut release" - if: build.branch == "main" - commands: - - "python3 .buildkite/scripts/release.py" - agents: - queue: nano - - - wait - - - group: "Publish" - steps: - - label: "publish {{ matrix }}" - if: build.branch == "main" - command: ".buildkite/scripts/publish.py {{ matrix }}" - matrix: - - "fixtures" - - "http" - - "http_typesafe" - - "json" - - "typesafe_app" - - "monitoring" - - "monitoring_typesafe" - - "messaging" - - "messaging_typesafe" - - "storage" - - "storage_typesafe" - - "elasticsearch" - - "elasticsearch_typesafe" - - "sierra" - - "sierra_typesafe" - - agents: - queue: "scala" - - - wait - - - label: "open downstream PRs" - if: build.branch == "main" - commands: - - "pip3 install --user boto3 httpx" - - "python3 .buildkite/scripts/open_downstream_prs.py" - agents: - queue: nano diff --git a/.buildkite/scripts/commands.py b/.buildkite/scripts/commands.py deleted file mode 100644 index 7dc7d2826..000000000 --- a/.buildkite/scripts/commands.py +++ /dev/null @@ -1,37 +0,0 @@ -import subprocess -import sys - - -def _subprocess_run(cmd, exit_on_error=True): - print("*** Running %r" % " ".join(cmd)) - - output = [] - pipe = subprocess.Popen( - cmd, encoding="utf8", stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - - # Await command completion and print lines as they come in - for stdout_line in iter(pipe.stdout.readline, ""): - print(stdout_line, end="") - output.append(stdout_line) - - # Extract results - pipe.communicate() - return_code = pipe.returncode - - if return_code != 0 and exit_on_error: - sys.exit(return_code) - - return "".join(output).strip() - - -def git(*args, exit_on_error=True): - """Run a Git command and return its output.""" - cmd = ["git"] + list(args) - - return _subprocess_run(cmd, exit_on_error=exit_on_error) - - -def run_build_script(name, *args): - """Run a build script, and check it completes successfully.""" - _subprocess_run([f"./builds/{name}"] + list(args)) diff --git a/.buildkite/scripts/git_utils.py b/.buildkite/scripts/git_utils.py deleted file mode 100755 index 0da06a2cb..000000000 --- a/.buildkite/scripts/git_utils.py +++ /dev/null @@ -1,60 +0,0 @@ -from commands import git - - -def get_changed_paths(*args, globs=None): - """ - Returns a set of changed paths in a given commit range. - - :param args: Arguments to pass to ``git diff``. - :param globs: List of file globs to include in changed paths. - """ - if globs: - args = list(args) + ["--", *globs] - diff_output = git("diff", "--name-only", *args) - - return {line.strip() for line in diff_output.splitlines()} - - -def get_all_tags(): - """ - Returns a list of all tags in the repo. - """ - git('fetch', '--tags') - result = git('tag') - all_tags = result.split('\n') - - assert len(set(all_tags)) == len(all_tags) - - return set(all_tags) - - -def remote_default_branch(): - """Inspect refs to discover default branch @ remote origin.""" - return git("symbolic-ref", "refs/remotes/origin/HEAD").split("/")[-1] - - -def remote_default_head(): - """Inspect refs to discover default branch HEAD @ remote origin.""" - return git("show-ref", f"refs/remotes/origin/{remote_default_branch()}", "-s") - - -def local_current_head(): - """Use rev-parse to discover hash for current commit AKA HEAD (from .git/HEAD).""" - return git("rev-parse", "HEAD") - - -def get_sha1_for_tag(tag): - """Use show-ref to discover the hash for a given tag (fetch first so we have all remote tags).""" - git("fetch") - return git("show-ref", "--hash", tag) - - -def has_source_changes(commit_range): - """ - Returns True if there are source changes since the previous release, - False if not. - """ - changed_files = [ - f for f in get_changed_paths(commit_range) if f.strip().endswith(('.sbt', '.scala')) - ] - return len(changed_files) != 0 \ No newline at end of file diff --git a/.buildkite/scripts/open_downstream_prs.py b/.buildkite/scripts/open_downstream_prs.py deleted file mode 100755 index ad5d0bb1b..000000000 --- a/.buildkite/scripts/open_downstream_prs.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python -""" -Whenever CI releases a new version of scala-libs, this script will create -a pull request in downstream repos that bumps to the new version. - -This ensures that our repos don't fall behind, and any breaking changes can -be resolved promptly. - -This script is quite light on error handling -- because it runs almost as -soon as the release becomes available, it seems unlikely that it would -conflict with a manually created PR/branch. -""" - -import contextlib -import os -import re -import shutil -import tempfile - -import boto3 -import httpx - -from commands import git -from release import latest_version - - -DOWNSTREAM_REPOS = ("catalogue-api", "catalogue-pipeline", "storage-service") - - -@contextlib.contextmanager -def working_directory(path): - """ - Changes the working directory to the given path, then returns to the - original directory when done. - """ - prev_cwd = os.getcwd() - os.chdir(path) - try: - yield - finally: - os.chdir(prev_cwd) - - -@contextlib.contextmanager -def cloned_repo(git_url): - """ - Clones the repository and changes the working directory to the cloned - repo. Cleans up the clone when it's done. - """ - repo_dir = tempfile.mkdtemp() - - git("clone", git_url, repo_dir) - - try: - with working_directory(repo_dir): - yield - finally: - shutil.rmtree(repo_dir) - - -class AlreadyAtLatestVersionException(Exception): - pass - - -def update_scala_libs_version(new_version): - old_lines = list(open("project/Dependencies.scala")) - - with open("project/Dependencies.scala", "w") as out_file: - for line in old_lines: - if line.startswith(" val defaultVersion"): - version_string = new_version.strip('v') - new_line = f' val defaultVersion = "{version_string}" // This is automatically bumped by the scala-libs release process, do not edit this line manually\n' - - if new_line == line: - raise AlreadyAtLatestVersionException() - - out_file.write(new_line) - else: - out_file.write(line) - - -def get_github_api_key(): - session = boto3.Session() - secrets_client = session.client("secretsmanager") - - secret_value = secrets_client.get_secret_value(SecretId="builds/github_wecobot/scala_libs_pr_bumps") - - return secret_value["SecretString"] - - -def get_changelog_entry(): - # BuildKite checks out the commit before it's done the changelog bump - # and pushed to GitHub. The RELEASE file still exists, so we can use - # that to get the release note. - try: - return open("RELEASE.md").read() - except FileNotFoundError: - pass - - with open("CHANGELOG.md") as f: - changelog = f.read() - - # This gets us something like: - # - # '## v26.18.0 - 2021-06-16\n\nAdd an HTTP typesafe builder for the SierraOauthHttpClient.\n\n' - # - last_entry = changelog.split("## ")[1] - - # Then remove that first header - lines = last_entry.splitlines()[1:] - - return "\n".join(lines).strip() - - -def get_last_merged_pr_number(): - for line in git("log", "--oneline").splitlines(): - m = re.match(r"^[0-9a-f]{8} Merge pull request #(?P\d+)", line) - - if m is not None: - return m.group("pr_number") - - -def create_downstream_pull_requests(new_version): - api_key = get_github_api_key() - - client = httpx.Client(auth=("weco-bot", api_key)) - - changelog = get_changelog_entry() - - pr_number = get_last_merged_pr_number() - - pr_body = "\n".join([ - "Changelog entry:\n" - ] + [f"> {line}" for line in changelog.splitlines()] + [ - f"\nSee wellcomecollection/scala-libs#{pr_number}" - ]) - - for repo in DOWNSTREAM_REPOS: - with cloned_repo(f"git@github.com:wellcomecollection/{repo}.git"): - try: - update_scala_libs_version(new_version) - except AlreadyAtLatestVersionException: - continue - - branch_name = f"bump-scala-libs-to-{new_version}" - - git("config", "--local", "user.email", "wellcomedigitalplatform@wellcome.ac.uk") - git("config", "--local", "user.name", "BuildKite on behalf of Wellcome Collection") - - git("checkout", "-b", branch_name) - git("add", "project/Dependencies.scala") - git("commit", "-m", f"Bump scala-libs to {new_version}") - git("push", "origin", branch_name) - - r = client.post( - f"https://api.github.com/repos/wellcomecollection/{repo}/pulls", - headers={"Accept": "application/vnd.github.v3+json"}, - json={ - "head": branch_name, - "base": "main", - "title": f"Bump scala-libs to {new_version}", - "maintainer_can_modify": True, - "body": pr_body, - } - ) - - try: - r.raise_for_status() - new_pr_number = r.json()["number"] - except Exception: - print(r.json()) - raise - - r = client.post( - f"https://api.github.com/repos/wellcomecollection/{repo}/pulls/{new_pr_number}/requested_reviewers", - headers={"Accept": "application/vnd.github.v3+json"}, - json={"team_reviewers": ["scala-reviewers"]} - ) - - print(r.json()) - - try: - r.raise_for_status() - except Exception: - raise - - -if __name__ == '__main__': - create_downstream_pull_requests( - new_version=latest_version() - ) diff --git a/.buildkite/scripts/provider.py b/.buildkite/scripts/provider.py deleted file mode 100644 index db2aa2477..000000000 --- a/.buildkite/scripts/provider.py +++ /dev/null @@ -1,18 +0,0 @@ -import os - -from git_utils import remote_default_branch - - -def current_branch(): - return os.environ["BUILDKITE_BRANCH"] - - -def is_default_branch(): - current_branch_name = current_branch() - default_branch_name = remote_default_branch() - - return current_branch_name == default_branch_name - - -def repo(): - return os.environ["BUILDKITE_REPO"] diff --git a/.buildkite/scripts/publish.py b/.buildkite/scripts/publish.py deleted file mode 100755 index dba1df690..000000000 --- a/.buildkite/scripts/publish.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 - -import sys - -from commands import git, run_build_script -from git_utils import remote_default_branch -from release import has_release - -def publish(project_name): - if has_release(): - print(f"Release detected, publishing {project_name}.") - git("pull", "origin", remote_default_branch()) - run_build_script("run_sbt_task_in_docker.sh", f"project {project_name}", "publish") - else: - print("No release detected, exit gracefully.") - sys.exit(0) - - -# This script takes environment variables as the "command" step -# when used with the buildkite docker plugin incorrectly parses -# spaces as newlines preventing passing args to this script! -if __name__ == '__main__': - project = sys.argv[1] - - publish(project) diff --git a/.buildkite/scripts/release.py b/.buildkite/scripts/release.py deleted file mode 100755 index c2028681f..000000000 --- a/.buildkite/scripts/release.py +++ /dev/null @@ -1,239 +0,0 @@ -#!/usr/bin/env python - -import datetime as dt -import os -import re -import sys - -from commands import git -from git_utils import ( - get_changed_paths, - get_all_tags, - remote_default_branch, - local_current_head, - get_sha1_for_tag, - remote_default_head, - has_source_changes -) -from provider import current_branch, is_default_branch, repo - - -ROOT = git('rev-parse', '--show-toplevel') - -BUILD_SBT = os.path.join(ROOT, 'build.sbt') - -RELEASE_FILE = os.path.join(ROOT, 'RELEASE.md') -RELEASE_TYPE = re.compile(r"^RELEASE_TYPE: +(major|minor|patch)") - -MAJOR = 'major' -MINOR = 'minor' -PATCH = 'patch' - -VALID_RELEASE_TYPES = (MAJOR, MINOR, PATCH) - -CHANGELOG_HEADER = re.compile(r"^## v\d+\.\d+\.\d+ - \d\d\d\d-\d\d-\d\d$") -CHANGELOG_FILE = os.path.join(ROOT, 'CHANGELOG.md') - - -def changelog(): - with open(CHANGELOG_FILE) as i: - return i.read() - - -def new_version(release_type): - version = latest_version() - version_info = [int(i) for i in version.lstrip('v').split('.')] - - new_version = list(version_info) - bump = VALID_RELEASE_TYPES.index(release_type) - new_version[bump] += 1 - for i in range(bump + 1, len(new_version)): - new_version[i] = 0 - new_version = tuple(new_version) - return 'v' + '.'.join(map(str, new_version)) - - -def update_changelog_and_version(): - contents = changelog() - assert '\r' not in contents - lines = contents.split('\n') - assert contents == '\n'.join(lines) - for i, l in enumerate(lines): - if CHANGELOG_HEADER.match(l): - beginning = '\n'.join(lines[:i]) - rest = '\n'.join(lines[i:]) - assert '\n'.join((beginning, rest)) == contents - break - - release_type, release_contents = parse_release_file() - - new_version_string = new_version(release_type) - - print('New version: %s' % new_version_string) - - now = dt.datetime.utcnow() - - date = max([ - d.strftime('%Y-%m-%d') for d in (now, now + dt.timedelta(hours=1)) - ]) - - heading_for_new_version = '## ' + ' - '.join((new_version_string, date)) - - new_changelog_parts = [ - beginning.strip(), - '', - heading_for_new_version, - '', - release_contents, - '', - rest - ] - - with open(CHANGELOG_FILE, 'w') as o: - o.write('\n'.join(new_changelog_parts)) - - # Update the version specified in build.sbt. We're looking to replace - # a line of the form: - # - # version := "x.y.z" - # - lines = list(open(BUILD_SBT)) - for idx, l in enumerate(lines): - if l.startswith('val projectVersion = '): - lines[idx] = 'val projectVersion = "%s"\n' % new_version_string.strip('v') - break - else: # no break - raise RuntimeError('Never updated version in build.sbt?') - - with open(BUILD_SBT, 'w') as f: - f.write(''.join(lines)) - - return release_type - - -def update_for_pending_release(): - release_type = update_changelog_and_version() - - git('rm', RELEASE_FILE) - git('add', CHANGELOG_FILE) - git('add', BUILD_SBT) - - git( - 'commit', - '-m', 'Bump version to %s and update changelog\n\n[skip ci]' % ( - new_version(release_type)) - ) - git('tag', new_version(release_type)) - - -def has_release(): - """ - Returns True if there is a release file, False if not. - """ - return os.path.exists(RELEASE_FILE) - - -def latest_version(): - """ - Returns the latest version, as specified by the Git tags. - """ - versions = [] - - for t in get_all_tags(): - assert t == t.strip() - parts = t.split('.') - assert len(parts) == 3, t - parts[0] = parts[0].lstrip('v') - v = tuple(map(int, parts)) - - versions.append((v, t)) - - _, latest = max(versions) - - assert latest in get_all_tags() - return latest - - -def parse_release_file(): - """ - Parses the release file, returning a tuple (release_type, release_contents) - """ - with open(RELEASE_FILE) as i: - release_contents = i.read() - - release_lines = release_contents.split('\n') - - m = RELEASE_TYPE.match(release_lines[0]) - if m is not None: - release_type = m.group(1) - if release_type not in VALID_RELEASE_TYPES: - print('Unrecognised release type %r' % (release_type,)) - sys.exit(1) - del release_lines[0] - release_contents = '\n'.join(release_lines).strip() - else: - print( - 'RELEASE.md does not start by specifying release type. The first ' - 'line of the file should be RELEASE_TYPE: followed by one of ' - 'major, minor, or patch, to specify the type of release that ' - 'this is (i.e. which version number to increment). Instead the ' - 'first line was %r' % (release_lines[0],) - ) - sys.exit(1) - - return release_type, release_contents - - -def check_release_file(commit_range): - if has_source_changes(commit_range): - if not has_release(): - print( - 'There are source changes but no RELEASE.md. Please create ' - 'one to describe your changes.' - ) - sys.exit(1) - - print('Source changes detected (RELEASE.md is present).') - parse_release_file() - else: - print('No source changes detected (RELEASE.md not required).') - - -def release(): - local_head = local_current_head() - - if is_default_branch(): - latest_sha = get_sha1_for_tag(latest_version()) - commit_range = f"{latest_sha}..{local_head}" - else: - remote_head = remote_default_head() - commit_range = f"{remote_head}..{local_head}" - - print(f"Working in branch: {current_branch()}") - print(f"On default branch: {is_default_branch()}") - print(f"Commit range: {commit_range}") - - if not is_default_branch(): - print('Trying to release while not on main?') - sys.exit(1) - - if has_release(): - print('Updating changelog and version') - - git("config", "user.name", "Buildkite on behalf of Wellcome Collection") - git("config", "user.email", "wellcomedigitalplatform@wellcome.ac.uk") - - print('Attempting a release.') - update_for_pending_release() - - git("remote", "add", "ssh-origin", repo(), exit_on_error=False) - - git('push', 'ssh-origin', 'HEAD:main') - git('push', 'ssh-origin', '--tag') - else: - print("No release detected, exit gracefully.") - sys.exit(0) - - -if __name__ == '__main__': - release() diff --git a/.github/scripts/create_release.py b/.github/scripts/create_release.py new file mode 100644 index 000000000..eac0f6b02 --- /dev/null +++ b/.github/scripts/create_release.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python + +import datetime as dt +import os +import re +import sys + +RELEASE_TYPE = re.compile(r"^RELEASE_TYPE: +(major|minor|patch)") +VALID_RELEASE_TYPES = ('major', 'minor', 'patch') + +CHANGELOG_HEADER = re.compile(r"^## v\d+\.\d+\.\d+ - \d\d\d\d-\d\d-\d\d$") + + +def get_new_version_tag(latest_version: str, release_type: str): + version_info = [int(i) for i in latest_version.lstrip('v').split('.')] + + bump = VALID_RELEASE_TYPES.index(release_type) + version_info[bump] += 1 + for i in range(bump + 1, len(version_info)): + version_info[i] = 0 + return 'v' + '.'.join(map(str, version_info)) + + +def _get_changelog_version_heading(new_version_tag: str): + date = dt.datetime.utcnow().strftime('%Y-%m-%d') + return f'## {new_version_tag} - {date}' + + +def update_changelog(release_contents: str, new_version_tag: str): + with open(CHANGELOG_FILE) as f: + changelog_contents = f.read() + + assert '\r' not in changelog_contents + lines = changelog_contents.split('\n') + assert changelog_contents == '\n'.join(lines) + + for i, line in enumerate(lines): + if CHANGELOG_HEADER.match(line): + beginning = '\n'.join(lines[:i]) + rest = '\n'.join(lines[i:]) + assert '\n'.join((beginning, rest)) == changelog_contents + break + + new_version_heading = _get_changelog_version_heading(new_version_tag) + + new_changelog_parts = [ + beginning.strip(), + new_version_heading, + release_contents, + rest + ] + + with open(CHANGELOG_FILE, 'w') as f: + f.write('\n\n'.join(new_changelog_parts)) + + +def update_sbt_version(new_version_tag: str): + with open(BUILD_SBT, 'r') as f: + lines = f.readlines() + + new_version = new_version_tag.strip('v') + + for idx, line in enumerate(lines): + if line.startswith('val projectVersion = '): + lines[idx] = f'val projectVersion = "{new_version}"\n' + break + else: # no break + raise RuntimeError('Never updated version in build.sbt?') + + with open(BUILD_SBT, 'w') as f: + f.write(''.join(lines)) + + +def parse_release_file(): + """ + Parses the release file, returning a tuple (release_type, release_contents) + """ + with open(RELEASE_FILE) as i: + release_contents = i.read() + + release_lines = release_contents.split('\n') + + m = RELEASE_TYPE.match(release_lines[0]) + if m is not None: + release_type = m.group(1) + if release_type not in VALID_RELEASE_TYPES: + print('Unrecognised release type %r' % (release_type,)) + sys.exit(1) + del release_lines[0] + release_contents = '\n'.join(release_lines).strip() + else: + print( + 'RELEASE.md does not start by specifying release type. The first ' + 'line of the file should be RELEASE_TYPE: followed by one of ' + 'major, minor, or patch, to specify the type of release that ' + 'this is (i.e. which version number to increment). Instead the ' + 'first line was %r' % (release_lines[0],) + ) + sys.exit(1) + + return release_type, release_contents + + +def create_release(): + latest_version_tag = sys.argv[1] + + release_type, release_contents = parse_release_file() + new_version_tag = get_new_version_tag(latest_version_tag, release_type) + + update_changelog(release_contents, new_version_tag) + update_sbt_version(new_version_tag) + + +if __name__ == '__main__': + ROOT = sys.argv[2] + CHANGELOG_FILE = os.path.join(ROOT, 'CHANGELOG.md') + RELEASE_FILE = os.path.join(ROOT, 'RELEASE.md') + BUILD_SBT = os.path.join(ROOT, 'build.sbt') + + create_release() diff --git a/.github/scripts/post-eviction-report-comment.js b/.github/scripts/post-eviction-report-comment.js new file mode 100644 index 000000000..285e63b0d --- /dev/null +++ b/.github/scripts/post-eviction-report-comment.js @@ -0,0 +1,32 @@ +const postEvictionsComment = async (github, context) => { + const fs = require('fs'); + const reportContent = fs.readFileSync('unique_evictions.txt', 'utf8'); + + const comments = await github.rest.issues.listComments({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo + }); + + const commentMarker = 'Suspected binary incompatible evictions across all projects (summary)'; + const existingComment = comments.data.find(comment => comment.body.includes(commentMarker)); + + if (existingComment) { + await github.rest.issues.updateComment({ + comment_id: existingComment.id, + owner: context.repo.owner, + repo: context.repo.repo, + body: reportContent + }); + } + else { + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: reportContent + }); + } +} + +module.exports = postEvictionsComment; diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..e39a53c08 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,110 @@ +name: "Create release and publish packages to Sonatype" + +on: + push: + branches: + - main + +permissions: + id-token: write + contents: write + +jobs: + # First, check if there is a RELEASE.md file in the root of the repository. + # If not, no release will be created and subsequent steps and jobs will be skipped. + check-for-release-file: + runs-on: ubuntu-latest + outputs: + has-release: ${{ steps.check-for-release-file.outputs.has-release }} + steps: + - uses: actions/checkout@v4 + - name: Check for RELEASE.md file + id: check-for-release-file + run: | + if [ ! -f ./RELEASE.md ]; then + echo "has-release=false" >> $GITHUB_OUTPUT + echo "No release detected. Exiting." + exit 0 + fi + echo "has-release=true" >> $GITHUB_OUTPUT + + # Creating a release involves the following two changes: + # - Updating the CHANGELOG.md file with the contents of the RELEASE.md file + # - Bumping the version number in the build.sbt file + # Once these changes are made, they are pushed to the main branch + create-release: + runs-on: ubuntu-latest + needs: check-for-release-file + if: needs.check-for-release-file.outputs.has-release == 'true' + steps: + - uses: actions/checkout@v4 + - name: Update CHANGELOG.md and build.sbt + run: | + git fetch --tags + LATEST_TAG=$(git describe --tags $(git rev-list --tags --max-count=1)) + python3 .github/scripts/create_release.py ${LATEST_TAG} $(pwd) + - name: Commit and push changes + run: | + NEW_TAG=$(cat CHANGELOG.md | grep -m1 -o 'v[0-9]\+\.[0-9]\+\.[0-9]\+') + git config --global user.name "GitHub on behalf of Wellcome Collection" + git config --global user.email "wellcomedigitalplatform@wellcome.ac.uk" + git checkout main + git pull + git add CHANGELOG.md build.sbt + git rm RELEASE.md + git commit -m "$(printf "Release: Bump version to ${NEW_TAG}\n\n[skip ci]")" + git tag ${NEW_TAG} + git push origin main + git push origin --tags + + # All sbt projects are published to Sonatype (https://central.sonatype.com/namespace/org.wellcomecollection). + # Publishing involves several steps: + # - Configuring a GPG key so that the packages can be signed + # - Configuring Sonatype credentials + # - Publishing the packages to a local staging repository using the sbt-sonatype plugin + # - Releasing the published bundle to Sonatype + publish: + runs-on: ubuntu-latest + needs: [create-release, check-for-release-file] + if: needs.check-for-release-file.outputs.has-release == 'true' + strategy: + matrix: + service: + - fixtures + - http + - json + - typesafe_app + - monitoring + - monitoring_typesafe + - messaging + - messaging_typesafe + - storage + - storage_typesafe + - elasticsearch + - elasticsearch_typesafe + - sierra + steps: + - uses: actions/checkout@v4 + with: + # Checkout the latest version, which includes the changes pushed by the previous step! + # If we didn't do this, we would be publishing using the previous version tag. + ref: main + - name: Set up GPG + run: | + echo "${{ secrets.BASE64_GPG_KEY }}" | base64 -d > secret-keys.gpg + echo "${{ secrets.GPG_PASSPHRASE }}" | gpg --batch --yes --passphrase-fd 0 --import secret-keys.gpg + rm secret-keys.gpg + - name: Set up Sonatype credentials + run: | + mkdir ~/.sbt + echo "${{ secrets.SONATYPE_CREDENTIALS }}" > ~/.sbt/sonatype.credentials + - name: Setup JDK + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 17 + cache: sbt + - name: Publish to Sonatype + run: | + PGP_PASSPHRASE=${{ secrets.GPG_PASSPHRASE }} sbt "project ${{ matrix.service }}" publishSigned + sbt "project ${{ matrix.service }}" sonatypeBundleRelease diff --git a/.github/workflows/report-evictions.yml b/.github/workflows/report-evictions.yml new file mode 100644 index 000000000..b553ecfa8 --- /dev/null +++ b/.github/workflows/report-evictions.yml @@ -0,0 +1,74 @@ +name: "Report evictions" + +on: + pull_request: + types: [opened, edited, synchronize] + +permissions: + id-token: write + issues: write + pull-requests: write + +jobs: + report-evictions: + runs-on: ubuntu-latest + strategy: + matrix: + service: + - fixtures + - http + - json + - typesafe_app + - monitoring + - monitoring_typesafe + - messaging + - messaging_typesafe + - storage + - storage_typesafe + - elasticsearch + - elasticsearch_typesafe + - sierra + steps: + - uses: actions/checkout@v4 + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-west-1 + role-to-assume: ${{ secrets.GHA_SCALA_LIBS_ROLE_ARN }} + - uses: aws-actions/amazon-ecr-login@v2 + + - name: Run eviction report for ${{ matrix.service }} + run: ./builds/report_sbt_evictions.sh ${{ matrix.service }} + continue-on-error: true + + # Upload the eviction report as a GitHub artifact + - name: Upload eviction reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: evicted_${{ matrix.service }} + path: .reports/evicted_${{ matrix.service }} + + collate-evictions: + runs-on: ubuntu-latest + needs: report-evictions + steps: + - name: Checkout code + uses: actions/checkout@v4 + + # Download ALL GitHub artifacts uploaded as part of this run + - name: Download eviction reports + uses: actions/download-artifact@v4 + with: + path: .reports/ + + # Post the eviction report as a comment on the corresponding PR + - name: Collate evictions + run: | + mkdir -p .reports + builds/report_unique_evictions.sh | tee unique_evictions.txt + - name: Post eviction report comment + uses: actions/github-script@v6 + with: + script: | + const postComment = require('./.github/scripts/post-eviction-report-comment.js'); + await postComment(github, context); diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 000000000..89ab902fc --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,38 @@ +name: "Run tests" + +on: + pull_request: + types: [opened, edited, synchronize] + +permissions: + id-token: write + +jobs: + run-tests: + runs-on: ubuntu-latest + strategy: + matrix: + service: + - fixtures + - http + - json + - typesafe_app + - monitoring + - monitoring_typesafe + - messaging + - messaging_typesafe + - storage + - storage_typesafe + - elasticsearch + - elasticsearch_typesafe + - sierra + steps: + - uses: actions/checkout@v4 + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-west-1 + role-to-assume: ${{ secrets.GHA_SCALA_LIBS_ROLE_ARN }} + - uses: aws-actions/amazon-ecr-login@v2 + + - name: Run sbt tests + run: ./builds/run_sbt_tests.sh ${{ matrix.service }} diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..5ae5a11f3 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,3 @@ +RELEASE_TYPE: minor + +Migrate release workflow from BuildKite to GitHub and publish new versions to Sonatype instead of S3. diff --git a/builds/report_unique_evictions.sh b/builds/report_unique_evictions.sh index f6cdc882f..9660f2e5e 100755 --- a/builds/report_unique_evictions.sh +++ b/builds/report_unique_evictions.sh @@ -1,5 +1,5 @@ # Intended to be run after producing eviction reports with report_sbt_evictions.sh echo "# Suspected binary incompatible evictions across all projects (summary)" -cat .reports/evicted_* | grep -E '^\[warn\]\s+\*' | sed 's/.*\*/*/' | sort | uniq +cat .reports/evicted_*/evicted_* | grep -E '^\[warn\]\s+\*' | sed 's/.*\*/*/' | sort | uniq echo "" echo "See individual _evictions_ stages for more detail" diff --git a/project/Common.scala b/project/Common.scala index f29642474..0d2bce3f9 100644 --- a/project/Common.scala +++ b/project/Common.scala @@ -1,13 +1,29 @@ -import java.io.File +import com.jsuereth.sbtpgp.PgpKeys.* import com.tapad.docker.DockerComposePlugin -import sbt.Keys._ -import sbt._ +import sbt.Keys.* +import sbt.* +import xerial.sbt.Sonatype.autoImport.{sonatypeCredentialHost, sonatypePublishToBundle, sonatypeRepository} object Common { def createSettings(projectVersion: String): Seq[Def.Setting[_]] = Seq( scalaVersion := "2.12.15", - organization := "weco", + organization := "org.wellcomecollection", + homepage := Some(url("https://github.com/wellcomecollection/scala-libs")), + scmInfo := Some( + ScmInfo( + url("https://github.com/wellcomecollection/scala-libs"), + "scm:git:git@github.com:wellcomecollection/scala-libs.git" + ) + ), + developers ++= List( + Developer( + id = "sbrychta", + name = "Stepan Brychta", + email = "s.brychta@wellcome.org", + url = url("https://github.com/StepanBrychta") + ) + ), scalacOptions ++= Seq( "-deprecation", "-unchecked", @@ -22,9 +38,11 @@ object Common { ), parallelExecution in Test := false, publishMavenStyle := true, - publishTo := Some( - "S3 releases" at "s3://releases.mvn-repo.wellcomecollection.org/" - ), + credentials += Credentials(Path.userHome / ".sbt" / "sonatype.credentials"), + sonatypeCredentialHost := "central.sonatype.com", + sonatypeRepository := "https://central.sonatype.com/service/local", + licenses := Seq("MIT" -> url("https://github.com/wellcomecollection/scala-libs/blob/main/LICENSE")), + publishTo := sonatypePublishToBundle.value, publishArtifact in Test := true, // Don't build scaladocs // https://www.scala-sbt.org/sbt-native-packager/formats/universal.html#skip-packagedoc-task-on-stage diff --git a/project/plugins.sbt b/project/plugins.sbt index cfc37a230..5a7bd9cd3 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -2,4 +2,6 @@ addSbtPlugin("com.tapad" % "sbt-docker-compose" % "1.0.34") addSbtPlugin("com.frugalmechanic" % "fm-sbt-s3-resolver" % "0.19.0") addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "1.15") addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.3.2") +addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.1") +addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.11.2") addDependencyTreePlugin