From aa87c159446b415735170555858e23b1e09f8244 Mon Sep 17 00:00:00 2001 From: John Davis Date: Mon, 19 Feb 2024 14:47:55 -0500 Subject: [PATCH 1/8] Add missing args --- galaxy_release_util/point_release.py | 32 +++++++++++++++++----------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/galaxy_release_util/point_release.py b/galaxy_release_util/point_release.py index 9f8cc01..42b5e2d 100644 --- a/galaxy_release_util/point_release.py +++ b/galaxy_release_util/point_release.py @@ -403,7 +403,8 @@ def merge_and_resolve_branches( "checkout", new_branch, str(galaxy_root / "lib" / "galaxy" / "version.py"), - ] + ], + cwd=galaxy_root, ) # we rewrite the packages changelog for new_package in packages_to_rewrite: @@ -431,14 +432,16 @@ def merge_and_resolve_branches( dev_version = get_root_version(galaxy_root) previous_package.package_history.insert(0, ChangelogItem(version=dev_version, changes=[], date=None)) previous_package.write_history() - subprocess.run(["git", "add", str(previous_package.history_rst)]) + subprocess.run(["git", "add", str(previous_package.history_rst)], cwd=galaxy_root) # restore setup.cfg - subprocess.run(["git", "checkout", new_branch, str(previous_package.setup_cfg)]).check_returncode() + subprocess.run( + ["git", "checkout", new_branch, str(previous_package.setup_cfg)], cwd=galaxy_root + ).check_returncode() # Commit changes if merge_conflict: - subprocess.run(["git", "commit", "--no-edit"]).check_returncode() + subprocess.run(["git", "commit", "--no-edit"], cwd=galaxy_root).check_returncode() else: - subprocess.run(["git", "commit", "--amend", "--no-edit"]).check_returncode() + subprocess.run(["git", "commit", "--amend", "--no-edit"], cwd=galaxy_root).check_returncode() def get_next_devN_version(galaxy_root) -> Version: @@ -473,12 +476,13 @@ def ensure_branches_up_to_date(branches: List[str], base_branch: str, upstream: # Check that the head commit matches the commit for the same branch at the specified remote repo url result = subprocess.run( ["git", "ls-remote", upstream, f"refs/heads/{branch}"], + cwd=galaxy_root, capture_output=True, text=True, ) result.check_returncode() remote_commit_hash = result.stdout.split("\t")[0] - result = subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True, text=True) + result = subprocess.run(["git", "rev-parse", "HEAD"], cwd=galaxy_root, capture_output=True, text=True) result.check_returncode() local_commit_hash = result.stdout.strip() if remote_commit_hash != local_commit_hash: @@ -488,9 +492,11 @@ def ensure_branches_up_to_date(branches: List[str], base_branch: str, upstream: subprocess.run(["git", "checkout", base_branch], cwd=galaxy_root).check_returncode() -def push_references(references: List[str], upstream: str = "https://github.com/galaxyproject/galaxy.git"): +def push_references( + references: List[str], galaxy_root: pathlib.Path, upstream: str = "https://github.com/galaxyproject/galaxy.git" +): for reference in references: - subprocess.run(["git", "push", upstream, reference]).check_returncode() + subprocess.run(["git", "push", upstream, reference], cwd=galaxy_root).check_returncode() @click.group(help="Subcommands of this script can create new releases and build and upload package artifacts") @@ -621,11 +627,11 @@ def create_point_release( cmd.extend(changed_paths) release_tag = f"v{new_version}" subprocess.run(cmd, cwd=galaxy_root) - subprocess.run(["git", "commit", "-m" f"Create version {new_version}"]) + subprocess.run(["git", "commit", "-m" f"Create version {new_version}"], cwd=galaxy_root) if not no_confirm: click.confirm(f"Create git tag '{release_tag}'?", abort=True) - subprocess.run(["git", "tag", release_tag]) + subprocess.run(["git", "tag", release_tag], cwd=galaxy_root) dev_version = get_next_devN_version(galaxy_root) version_py = set_root_version(galaxy_root, dev_version) modified_paths = [version_py] @@ -636,8 +642,8 @@ def create_point_release( modified_paths.extend(package.modified_paths) cmd = ["git", "add"] cmd.extend([str(p) for p in modified_paths]) - subprocess.run(cmd) - subprocess.run(["git", "commit", "-m", f"Start work on {dev_version}"]) + subprocess.run(cmd, cwd=galaxy_root) + subprocess.run(["git", "commit", "-m", f"Start work on {dev_version}"], cwd=galaxy_root) # merge changes into newer branches # special care needs to be taken for changelog files if not no_confirm and newer_branches: @@ -652,7 +658,7 @@ def create_point_release( current_branch = new_branch references = [release_tag] + all_branches if no_confirm or click.confirm(f"Push {','.join(references)} to upstream '{upstream}' ?", abort=True): - push_references(references=references, upstream=upstream) + push_references(references=references, galaxy_root=galaxy_root, upstream=upstream) if __name__ == "__main__": From 7f65015e43e8ca924a6c671b38cee58f238d19ba Mon Sep 17 00:00:00 2001 From: John Davis Date: Mon, 19 Feb 2024 15:26:44 -0500 Subject: [PATCH 2/8] Abort no-commit merge only if not up-to-date --- galaxy_release_util/point_release.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/galaxy_release_util/point_release.py b/galaxy_release_util/point_release.py index 42b5e2d..34af2fa 100644 --- a/galaxy_release_util/point_release.py +++ b/galaxy_release_util/point_release.py @@ -462,9 +462,10 @@ def is_merge_required(base_branch: str, new_branch: str, galaxy_root: pathlib.Pa process = subprocess.run( ["git", "merge", "--no-commit", "--no-ff", base_branch], cwd=galaxy_root, - capture_output=False, + capture_output=True, ) - subprocess.run(["git", "merge", "--abort"], cwd=galaxy_root) + if not process.stdout == b"Already up to date.\n": + subprocess.run(["git", "merge", "--abort"], cwd=galaxy_root) if process.returncode == 0: return False return True From f9cae9dceb6ef9f536aab7011541b0d67d6cde18 Mon Sep 17 00:00:00 2001 From: John Davis Date: Mon, 19 Feb 2024 15:44:53 -0500 Subject: [PATCH 3/8] Add base branch to confirm message, reformat --- galaxy_release_util/point_release.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/galaxy_release_util/point_release.py b/galaxy_release_util/point_release.py index 34af2fa..2eac35d 100644 --- a/galaxy_release_util/point_release.py +++ b/galaxy_release_util/point_release.py @@ -334,7 +334,7 @@ def set_root_version(galaxy_root: pathlib.Path, new_version: Version) -> pathlib def is_git_clean(galaxy_root: pathlib.Path): - click.echo(f"Making sure galaxy clone at '{galaxy_root}' is clean") + click.echo(f"Making sure galaxy clone at '{galaxy_root}' is clean:") command = ["git", "diff-index", "--quiet", "HEAD"] result = subprocess.run(command, capture_output=True, text=True, cwd=galaxy_root) if result.returncode == 0: @@ -571,10 +571,12 @@ def create_point_release( abort=True, ) root_version = get_root_version(galaxy_root) - click.echo(f"Current Galaxy version is {root_version}, will create new version {new_version}") + base_branch = current_branch = get_current_branch(galaxy_root) + click.echo( + f"- Current Galaxy version: {root_version}\n- New Galaxy version: {new_version}\n- Base branch: {base_branch}" + ) if not no_confirm: click.confirm("Does this look correct?", abort=True) - base_branch = current_branch = get_current_branch(galaxy_root) newer_branches = get_branches(galaxy_root, new_version, base_branch) all_branches = newer_branches + [base_branch] click.echo("Making sure that all branches are up to date") From 6e4ce65ce1c38e6347f82c0c82251661ed0a3de9 Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 20 Feb 2024 15:26:32 -0500 Subject: [PATCH 4/8] Edit CI: remove 3.7, mypy; add 3.12/lint --- .github/workflows/tox.yml | 2 +- tox.ini | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tox.yml b/.github/workflows/tox.yml index 499e570..f7a9dc9 100644 --- a/.github/workflows/tox.yml +++ b/.github/workflows/tox.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v3 diff --git a/tox.ini b/tox.ini index 7eeb539..1128a0d 100644 --- a/tox.ini +++ b/tox.ini @@ -6,12 +6,11 @@ envlist = py37, py38, py39, py310, py311 [gh] python = - 3.12 = unit + 3.12 = unit,lint 3.11 = unit 3.10 = unit 3.9 = unit 3.8 = unit - 3.7 = unit,mypy,lint [testenv] commands = From 5854634356f3d9de6c95ade3a46a363b543da989 Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 20 Feb 2024 15:37:28 -0500 Subject: [PATCH 5/8] Fix .gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b22ac28..1e99419 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ # Python build artifacts build dist -lib/galaxy.egg-info +galaxy_release_util.egg-info .eggs # Python bytecode From ade05900d2e33052718f043c40c81b26c7359976 Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 20 Feb 2024 18:20:16 -0500 Subject: [PATCH 6/8] Reduce calls to github API; add counter to view progress --- galaxy_release_util/point_release.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/galaxy_release_util/point_release.py b/galaxy_release_util/point_release.py index 2eac35d..1d46b62 100644 --- a/galaxy_release_util/point_release.py +++ b/galaxy_release_util/point_release.py @@ -242,9 +242,12 @@ def commits_to_prs(packages: List[Package]): commits = set.union(*(p.commits for p in packages)) pr_cache = {} commit_to_pr = {} - for commit in commits: + repo = g.get_repo(REPO) + total_commits = len(commits) + for i, commit in enumerate(commits): + click.echo(f"Processing commit {i} of {total_commits}") # Get the list of pull requests associated with the commit - commit_obj = g.get_repo(REPO).get_commit(commit) + commit_obj = repo.get_commit(commit) prs = commit_obj.get_pulls() if not prs: raise Exception(f"commit {commit} has no associated PRs") From 05590bc7eac27bbf0926b757d59708db2e2a17ca Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 20 Feb 2024 09:23:47 -0500 Subject: [PATCH 7/8] Handle unpublished releases --- galaxy_release_util/point_release.py | 41 ++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/galaxy_release_util/point_release.py b/galaxy_release_util/point_release.py index 1d46b62..571182b 100644 --- a/galaxy_release_util/point_release.py +++ b/galaxy_release_util/point_release.py @@ -47,6 +47,13 @@ """ RELEASE_BRANCH_REGEX = re.compile(r"^release_(\d{2}\.\d{1,2})$") +FIRST_RELEASE_CHANGELOG_TEXT = "First release" + + +@dataclass +class ReleaseItem: + version: Version + date: Optional[str] = None @dataclass @@ -87,6 +94,7 @@ class Package: prs: Set[PullRequest.PullRequest] = field(default_factory=set) modified_paths: List[pathlib.Path] = field(default_factory=list) package_history: List[ChangelogItem] = field(default_factory=list) + release_items: List[ReleaseItem] = field(default_factory=list) @property def name(self) -> str: @@ -109,6 +117,22 @@ def write_history(self): self.history_rst.write_text(self.changelog) self.modified_paths.append(self.history_rst) + def add_release(self, item: ReleaseItem) -> None: + self.release_items.append(item) + + @property + def is_new(self) -> bool: + """Package has not been released: + - has only one release item, + - which is a devrelease, + - and has no date. + """ + return ( + len(self.release_items) == 1 + and self.release_items[0].version.is_devrelease + and not self.release_items[0].date + ) + def __repr__(self) -> str: pretty_string = f"[Package: {self.name}, Current Version: {self.current_version}" return pretty_string @@ -137,6 +161,9 @@ def read_package(package_path: pathlib.Path) -> Package: def parse_changelog(package: Package) -> List[ChangelogItem]: + def add_changelog_item(changes, child): + changes.append(f"* {child.rawsource.strip()}") + settings = frontend.get_default_settings(Parser) # type: ignore[attr-defined] ## upstream type stubs not updated? document = utils.new_document(str(package.history_rst), settings) Parser().parse(package.history_rst.read_text(), document) @@ -159,12 +186,13 @@ def parse_changelog(package: Package) -> List[ChangelogItem]: # we will just omit this later version_str = release_version current_version = Version(version_str) + package.add_release(ReleaseItem(version=current_version, date=current_date)) changes = [] for changelog_item in release[1:]: # could be bullet list or a nested section with bugfix, docs, etc if changelog_item.tagname == "bullet_list": for child in changelog_item.children: - changes.append(f"* {child.rawsource.strip()}") + add_changelog_item(changes, child) elif changelog_item.tagname == "paragraph": changes = changelog_item.rawsource.splitlines() elif changelog_item.tagname == "section": @@ -173,17 +201,17 @@ def parse_changelog(package: Package) -> List[ChangelogItem]: changes.append(f"\n{section_delimiter}\n{kind}\n{section_delimiter}\n") for section_changelog_item in changelog_item[1:]: for child in section_changelog_item: - changes.append(f"* {child.rawsource.strip()}") + add_changelog_item(changes, child) changelog_items.append(ChangelogItem(version=current_version, date=current_date, changes=changes)) # Filter out dev release versions without changelog, # we're going to add these back after committing the release version clean_changelog_items: List[ChangelogItem] = [] for item in changelog_items: - if not item.is_empty_devrelease: + if not (item.is_empty_devrelease or package.is_new): if item.date is None: raise Exception( - f"Error in '{package.history_rst}'. Changelog entry for version '{item.version}' has no date but contains changes. You have to fix this manually." + f"Error in '{package.history_rst}'. Changelog entry for non-dev version '{item.version}' has no date but contains changes. You have to fix this manually." ) clean_changelog_items.append(item) return sorted( @@ -267,7 +295,10 @@ def update_package_history(package: Package, new_version: Version): "Enhancements": [], "Other changes": [], } - if not package.prs: + if package.is_new: + # For new packages, replace any current text; do not list PRs. + sorted_and_formatted_changes.append(FIRST_RELEASE_CHANGELOG_TEXT) + elif not package.prs: # Skip publishing packages if no change ? sorted_and_formatted_changes.append("No recorded changes since last release") else: From 4b6b0cef488209fa02618ce47e19aa85c416be80 Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 21 Feb 2024 18:33:59 -0500 Subject: [PATCH 8/8] Fix bug: no more empty bullet points --- galaxy_release_util/point_release.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/galaxy_release_util/point_release.py b/galaxy_release_util/point_release.py index 571182b..5306a37 100644 --- a/galaxy_release_util/point_release.py +++ b/galaxy_release_util/point_release.py @@ -162,7 +162,9 @@ def read_package(package_path: pathlib.Path) -> Package: def parse_changelog(package: Package) -> List[ChangelogItem]: def add_changelog_item(changes, child): - changes.append(f"* {child.rawsource.strip()}") + rawsource = child.rawsource.strip() + if rawsource: + changes.append(f"* {rawsource}") settings = frontend.get_default_settings(Parser) # type: ignore[attr-defined] ## upstream type stubs not updated? document = utils.new_document(str(package.history_rst), settings)