Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PM-11376] Improve automated commit message #29

Merged
merged 1 commit into from
Sep 11, 2024
Merged

[PM-11376] Improve automated commit message #29

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions .github/workflows/fdroid.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ jobs:
apps:
name: "Generate repo from apps listing"
runs-on: ubuntu-22.04

env:
COMMIT_MSG_FILE: "${{ github.workspace }}/commit_message.tmp"

steps:
- name: Checkout repo
Expand Down Expand Up @@ -100,10 +101,16 @@ jobs:
with:
go-version: '^1.17.0'

- name: Run update script
- name: Run metascoop
id: run-metascoop
env:
GH_ACCESS_TOKEN: ${{ steps.retrieve-secrets.outputs.github-pat-bitwarden-devops-bot-repo-scope }}
run: |
bash run_metascoop.sh ${{ env.COMMIT_MSG_FILE }}

- name: Update repo
env:
GH_TOKEN: ${{ steps.retrieve-secrets.outputs.github-pat-bitwarden-devops-bot-repo-scope }}
DRY_RUN: ${{ !contains(inputs.dry-run, 'true') }}
if: ${{ (github.event_name == 'schedule' || inputs.dry-run == 'false') }}
run: |
bash update.sh --dry-run ${{ inputs.dry-run }}
bash update_repo.sh ${{ env.COMMIT_MSG_FILE }}
263 changes: 174 additions & 89 deletions metascoop/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"os/exec"
"path/filepath"
"strings"
"sync"
"time"

"metascoop/apps"
Expand All @@ -26,11 +27,11 @@ import (

func main() {
var (
reposFilePath = flag.String("rp", "repos.yaml", "Path to repos.yaml file")
repoDir = flag.String("rd", "fdroid/repo", "Path to fdroid \"repo\" directory")
accessToken = flag.String("pat", "", "GitHub personal access token")

debugMode = flag.Bool("debug", false, "Debug mode won't run the fdroid command")
reposFilePath = flag.String("rp", "repos.yaml", "Path to repos.yaml file")
repoDir = flag.String("rd", "fdroid/repo", "Path to fdroid \"repo\" directory")
accessToken = flag.String("pat", "", "GitHub personal access token")
commitMsgFile = flag.String("cm", "commit_message.tmp", "Path to the commit message file")
debugMode = flag.Bool("debug", false, "Debug mode won't run the fdroid command")
)
flag.Parse()

Expand Down Expand Up @@ -69,112 +70,196 @@ func main() {
haveError bool
apkInfoMap = make(map[string]apps.Application)
toRemovePaths []string
changedRepos = make(map[string]map[string]*github.RepositoryRelease)
mu sync.Mutex
wg sync.WaitGroup
)

// Track if changes are detected that will require re-generating metadata
regenerateMetadata := false
for _, repo := range reposList {
fmt.Printf("::group::Repo: %s/%s\n", repo.Owner, repo.Name)

err, releases := getRepositoryReleases(githubClient, repo)
if err != nil {
log.Printf("Error while listing repo releases for %q: %s\n", repo.GitURL, err.Error())
haveError = true
return
}

log.Printf("Received %d releases", len(releases))

for _, app := range repo.Applications {
fmt.Printf("::group::App %s\n", app.Name)

foundArtifact := false

for _, release := range releases {
fmt.Printf("::group::Release %s\n", release.GetTagName())

if release.GetDraft() {
log.Printf("Skipping draft %q\n", release.GetTagName())
continue
}
if release.GetTagName() == "" {
log.Printf("Skipping release with empty tag name")
continue
}

log.Printf("Working on release with tag name %q", release.GetTagName())
var apk *github.ReleaseAsset = apps.FindAPK(release, app.Filename)

if apk == nil {
log.Printf("Couldn't find any F-Droid assets for application %s in %s with file name %s", app.Filename, release.GetName(), app.Filename)
continue
}

appName := apps.GenerateReleaseFilename(app.Id, release.GetTagName())

log.Printf("Target APK name: %s\n", appName)

appClone := app
for _, repo := range reposList {
wg.Add(1)
go func(repo apps.Repo) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

n🧊

defer wg.Done()

appClone.ReleaseDescription = release.GetBody()
if appClone.ReleaseDescription != "" {
log.Printf("Release notes: \n%s\n", appClone.ReleaseDescription)
}
fmt.Printf("::group::Repo: %s/%s\n", repo.Owner, repo.Name)

apkInfoMap[appName] = appClone
err, releases := getRepositoryReleases(githubClient, repo)
if err != nil {
log.Printf("Error while listing repo releases for %q: %s\n", repo.GitURL, err.Error())
mu.Lock()
haveError = true
mu.Unlock()
return
}

appTargetPath := filepath.Join(*repoDir, appName)
log.Printf("Received %d releases", len(releases))

var appWg sync.WaitGroup
repoChanged := false
for _, app := range repo.Applications {
appWg.Add(1)
go func(app apps.Application) {
defer appWg.Done()

fmt.Printf("::group::App %s\n", app.Name)

foundArtifact := false

for _, release := range releases {
fmt.Printf("::group::Release %s\n", release.GetTagName())

if release.GetDraft() {
log.Printf("Skipping draft %q\n", release.GetTagName())
continue
}
if release.GetTagName() == "" {
log.Printf("Skipping release with empty tag name")
continue
}

log.Printf("Working on release with tag name %q", release.GetTagName())
var apk *github.ReleaseAsset = apps.FindAPK(release, app.Filename)

if apk == nil {
log.Printf("Couldn't find any F-Droid assets for application %s in %s with file name %s", app.Filename, release.GetName(), app.Filename)
continue
}

appName := apps.GenerateReleaseFilename(app.Id, release.GetTagName())

log.Printf("Target APK name: %s\n", appName)

appClone := app

appClone.ReleaseDescription = release.GetBody()
if appClone.ReleaseDescription != "" {
log.Printf("Release notes: \n%s\n", appClone.ReleaseDescription)
}

mu.Lock()
apkInfoMap[appName] = appClone
mu.Unlock()

appTargetPath := filepath.Join(*repoDir, appName)

// If the app file already exists for this version, we stop processing this app and move to the next
if _, err := os.Stat(appTargetPath); !errors.Is(err, os.ErrNotExist) {
log.Printf("Already have APK for version %q at %q\n", release.GetTagName(), appTargetPath)
foundArtifact = true
break
}

log.Printf("Downloading APK %q from release %q to %q", apk.GetName(), release.GetTagName(), appTargetPath)

downloadContext, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()

appStream, _, err := githubClient.Repositories.DownloadReleaseAsset(downloadContext, repo.Owner, repo.Name, apk.GetID(), http.DefaultClient)
if err != nil {
log.Printf("Error while downloading app %q (artifact id %d) from from release %q: %s", repo.GitURL, apk.GetID(), release.GetTagName(), err.Error())
mu.Lock()
haveError = true
mu.Unlock()
break
}

err = downloadStream(appTargetPath, appStream)
if err != nil {
log.Printf("Error while downloading app %q (artifact id %d) from from release %q to %q: %s", repo.GitURL, *apk.ID, *release.TagName, appTargetPath, err.Error())
mu.Lock()
haveError = true
mu.Unlock()
break
}

log.Printf("Successfully downloaded app for version %q", release.GetTagName())
fmt.Printf("::endgroup:App %s\n", app.Name)
mu.Lock()
regenerateMetadata = true
repoChanged = true
if changedRepos[repo.GitURL] == nil {
changedRepos[repo.GitURL] = make(map[string]*github.RepositoryRelease)
}
changedRepos[repo.GitURL][app.Filename] = release
mu.Unlock()
break
}
if foundArtifact || haveError {
// Stop after the first [release] of this [app] is downloaded to prevent back-filling legacy releases.
return
}
}(app)
}

// If the app file already exists for this version, we stop processing this app and move to the next
if _, err := os.Stat(appTargetPath); !errors.Is(err, os.ErrNotExist) {
log.Printf("Already have APK for version %q at %q\n", release.GetTagName(), appTargetPath)
foundArtifact = true
break
}
appWg.Wait()

log.Printf("Downloading APK %q from release %q to %q", apk.GetName(), release.GetTagName(), appTargetPath)
if repoChanged {
log.Printf("Changes detected for repo: %s", repo.GitURL)
}

downloadContext, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
fmt.Printf("::endgroup::Repo: %s/%s\n", repo.Owner, repo.Name)
}(repo)
}

appStream, _, err := githubClient.Repositories.DownloadReleaseAsset(downloadContext, repo.Owner, repo.Name, apk.GetID(), http.DefaultClient)
if err != nil {
log.Printf("Error while downloading app %q (artifact id %d) from from release %q: %s", repo.GitURL, apk.GetID(), release.GetTagName(), err.Error())
haveError = true
break
wg.Wait()

// Write changes to temporary commit message file
if regenerateMetadata {
var commitMsg strings.Builder

// Create the first line with repo names
repoNames := make([]string, 0, len(changedRepos))
for repoURL := range changedRepos {
repoName := strings.TrimPrefix(repoURL, "https://github.com/")
repoNames = append(repoNames, repoName)
}
commitMsg.WriteString(fmt.Sprintf("Updated apps from %s\n\n", strings.Join(repoNames, ", ")))

// Add details for each repo
for repoURL, apps := range changedRepos {
repoFullName := strings.TrimPrefix(repoURL, "https://github.com/")

commitMsg.WriteString(fmt.Sprintf("<details>\n<summary>%s</summary>\n\n", repoFullName))

// Group apps by release
releaseApps := make(map[*github.RepositoryRelease][]string)
for appFilename, release := range apps {
releaseApps[release] = append(releaseApps[release], appFilename)
}

for release, appList := range releaseApps {
releaseName := release.GetName()
if releaseName == "" {
releaseName = release.GetTagName()
}

err = downloadStream(appTargetPath, appStream)
if err != nil {
log.Printf("Error while downloading app %q (artifact id %d) from from release %q to %q: %s", repo.GitURL, *apk.ID, *release.TagName, appTargetPath, err.Error())
haveError = true
break
releaseTagURL := release.GetHTMLURL()
commitMsg.WriteString(fmt.Sprintf("### [%s](%s)\n\n", releaseName, releaseTagURL))

for _, appFilename := range appList {
commitMsg.WriteString(fmt.Sprintf("- %s\n", appFilename))
}

log.Printf("Successfully downloaded app for version %q", release.GetTagName())
fmt.Printf("::endgroup:App %s\n", app.Name)
regenerateMetadata = true
break

}
if foundArtifact || haveError {
// Stop after the first [release] of this [app] is downloaded to prevent back-filling legacy releases.
break
commitMsg.WriteString("\n")
}

commitMsg.WriteString("</details>\n\n")
}

fmt.Printf("::endgroup::Repo: %s/%s\n", repo.Owner, repo.Name)
err := os.WriteFile(*commitMsgFile, []byte(commitMsg.String()), 0644)
if err != nil {
log.Printf("Error writing commit message file: %s", err)
} else {
log.Printf("Commit message written to %s", *commitMsgFile)
}
} else {
log.Printf("No changes detected.")
os.Exit(2)
}

if haveError {
os.Exit(1)
}

if !regenerateMetadata {
log.Printf("No changes detected.")
os.Exit(2)
}

if !*debugMode {
fmt.Println("::group::F-Droid: Creating metadata stubs")
// Now, we run the fdroid update command
Expand Down
33 changes: 33 additions & 0 deletions run_metascoop.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/bin/bash

# Check if the commit message file argument is provided
if [ "$#" -ne 1 ]; then
echo "Usage: $0 <commit_message_file_path>"
exit 1
fi

COMMIT_MSG_FILE="$1"

echo "::group::Building metascoop executable"
cd metascoop
go build -o metascoop
echo "::endgroup::"

echo "::group::Running metascoop"
./metascoop -rp=../repos.yaml -rd=../fdroid/repo -pat="$GH_ACCESS_TOKEN" -cm="$COMMIT_MSG_FILE"
EXIT_CODE=$?
cd ..
echo "::endgroup::"

echo "Metascoop had an exit code of $EXIT_CODE"

if [ $EXIT_CODE -eq 2 ]; then
echo "There were no significant changes"
exit 0
elif [ $EXIT_CODE -eq 0 ]; then
echo "Changes detected"
exit 0
else
echo "This is an unexpected error"
exit 1
fi
Loading
Loading