diff --git a/.github/workflows/fdroid.yml b/.github/workflows/fdroid.yml index 7016ab2..6864899 100644 --- a/.github/workflows/fdroid.yml +++ b/.github/workflows/fdroid.yml @@ -19,7 +19,8 @@ jobs: apps: name: "Generate repo from apps listing" runs-on: ubuntu-22.04 - + env: + COMMIT_MSG_FILE: "${{ github.workspace }}/commit_message.tmp" steps: - name: Checkout repo @@ -100,10 +101,16 @@ jobs: with: go-version: '^1.17.0' - - name: Run update script + - name: Run metascoop + id: run-metascoop env: GH_ACCESS_TOKEN: ${{ steps.retrieve-secrets.outputs.github-pat-bitwarden-devops-bot-repo-scope }} + run: | + bash run_metascoop.sh ${{ env.COMMIT_MSG_FILE }} + + - name: Update repo + env: GH_TOKEN: ${{ steps.retrieve-secrets.outputs.github-pat-bitwarden-devops-bot-repo-scope }} - DRY_RUN: ${{ !contains(inputs.dry-run, 'true') }} + if: ${{ (github.event_name == 'schedule' || inputs.dry-run == 'false') }} run: | - bash update.sh --dry-run ${{ inputs.dry-run }} + bash update_repo.sh ${{ env.COMMIT_MSG_FILE }} diff --git a/metascoop/main.go b/metascoop/main.go index 5805ef4..85422d7 100644 --- a/metascoop/main.go +++ b/metascoop/main.go @@ -13,6 +13,7 @@ import ( "os/exec" "path/filepath" "strings" + "sync" "time" "metascoop/apps" @@ -26,11 +27,11 @@ import ( func main() { var ( - reposFilePath = flag.String("rp", "repos.yaml", "Path to repos.yaml file") - repoDir = flag.String("rd", "fdroid/repo", "Path to fdroid \"repo\" directory") - accessToken = flag.String("pat", "", "GitHub personal access token") - - debugMode = flag.Bool("debug", false, "Debug mode won't run the fdroid command") + reposFilePath = flag.String("rp", "repos.yaml", "Path to repos.yaml file") + repoDir = flag.String("rd", "fdroid/repo", "Path to fdroid \"repo\" directory") + accessToken = flag.String("pat", "", "GitHub personal access token") + commitMsgFile = flag.String("cm", "commit_message.tmp", "Path to the commit message file") + debugMode = flag.Bool("debug", false, "Debug mode won't run the fdroid command") ) flag.Parse() @@ -69,112 +70,196 @@ func main() { haveError bool apkInfoMap = make(map[string]apps.Application) toRemovePaths []string + changedRepos = make(map[string]map[string]*github.RepositoryRelease) + mu sync.Mutex + wg sync.WaitGroup ) - // Track if changes are detected that will require re-generating metadata regenerateMetadata := false - for _, repo := range reposList { - fmt.Printf("::group::Repo: %s/%s\n", repo.Owner, repo.Name) - - err, releases := getRepositoryReleases(githubClient, repo) - if err != nil { - log.Printf("Error while listing repo releases for %q: %s\n", repo.GitURL, err.Error()) - haveError = true - return - } - - log.Printf("Received %d releases", len(releases)) - - for _, app := range repo.Applications { - fmt.Printf("::group::App %s\n", app.Name) - foundArtifact := false - - for _, release := range releases { - fmt.Printf("::group::Release %s\n", release.GetTagName()) - - if release.GetDraft() { - log.Printf("Skipping draft %q\n", release.GetTagName()) - continue - } - if release.GetTagName() == "" { - log.Printf("Skipping release with empty tag name") - continue - } - - log.Printf("Working on release with tag name %q", release.GetTagName()) - var apk *github.ReleaseAsset = apps.FindAPK(release, app.Filename) - - if apk == nil { - log.Printf("Couldn't find any F-Droid assets for application %s in %s with file name %s", app.Filename, release.GetName(), app.Filename) - continue - } - - appName := apps.GenerateReleaseFilename(app.Id, release.GetTagName()) - - log.Printf("Target APK name: %s\n", appName) - - appClone := app + for _, repo := range reposList { + wg.Add(1) + go func(repo apps.Repo) { + defer wg.Done() - appClone.ReleaseDescription = release.GetBody() - if appClone.ReleaseDescription != "" { - log.Printf("Release notes: \n%s\n", appClone.ReleaseDescription) - } + fmt.Printf("::group::Repo: %s/%s\n", repo.Owner, repo.Name) - apkInfoMap[appName] = appClone + err, releases := getRepositoryReleases(githubClient, repo) + if err != nil { + log.Printf("Error while listing repo releases for %q: %s\n", repo.GitURL, err.Error()) + mu.Lock() + haveError = true + mu.Unlock() + return + } - appTargetPath := filepath.Join(*repoDir, appName) + log.Printf("Received %d releases", len(releases)) + + var appWg sync.WaitGroup + repoChanged := false + for _, app := range repo.Applications { + appWg.Add(1) + go func(app apps.Application) { + defer appWg.Done() + + fmt.Printf("::group::App %s\n", app.Name) + + foundArtifact := false + + for _, release := range releases { + fmt.Printf("::group::Release %s\n", release.GetTagName()) + + if release.GetDraft() { + log.Printf("Skipping draft %q\n", release.GetTagName()) + continue + } + if release.GetTagName() == "" { + log.Printf("Skipping release with empty tag name") + continue + } + + log.Printf("Working on release with tag name %q", release.GetTagName()) + var apk *github.ReleaseAsset = apps.FindAPK(release, app.Filename) + + if apk == nil { + log.Printf("Couldn't find any F-Droid assets for application %s in %s with file name %s", app.Filename, release.GetName(), app.Filename) + continue + } + + appName := apps.GenerateReleaseFilename(app.Id, release.GetTagName()) + + log.Printf("Target APK name: %s\n", appName) + + appClone := app + + appClone.ReleaseDescription = release.GetBody() + if appClone.ReleaseDescription != "" { + log.Printf("Release notes: \n%s\n", appClone.ReleaseDescription) + } + + mu.Lock() + apkInfoMap[appName] = appClone + mu.Unlock() + + appTargetPath := filepath.Join(*repoDir, appName) + + // If the app file already exists for this version, we stop processing this app and move to the next + if _, err := os.Stat(appTargetPath); !errors.Is(err, os.ErrNotExist) { + log.Printf("Already have APK for version %q at %q\n", release.GetTagName(), appTargetPath) + foundArtifact = true + break + } + + log.Printf("Downloading APK %q from release %q to %q", apk.GetName(), release.GetTagName(), appTargetPath) + + downloadContext, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + appStream, _, err := githubClient.Repositories.DownloadReleaseAsset(downloadContext, repo.Owner, repo.Name, apk.GetID(), http.DefaultClient) + if err != nil { + log.Printf("Error while downloading app %q (artifact id %d) from from release %q: %s", repo.GitURL, apk.GetID(), release.GetTagName(), err.Error()) + mu.Lock() + haveError = true + mu.Unlock() + break + } + + err = downloadStream(appTargetPath, appStream) + if err != nil { + log.Printf("Error while downloading app %q (artifact id %d) from from release %q to %q: %s", repo.GitURL, *apk.ID, *release.TagName, appTargetPath, err.Error()) + mu.Lock() + haveError = true + mu.Unlock() + break + } + + log.Printf("Successfully downloaded app for version %q", release.GetTagName()) + fmt.Printf("::endgroup:App %s\n", app.Name) + mu.Lock() + regenerateMetadata = true + repoChanged = true + if changedRepos[repo.GitURL] == nil { + changedRepos[repo.GitURL] = make(map[string]*github.RepositoryRelease) + } + changedRepos[repo.GitURL][app.Filename] = release + mu.Unlock() + break + } + if foundArtifact || haveError { + // Stop after the first [release] of this [app] is downloaded to prevent back-filling legacy releases. + return + } + }(app) + } - // If the app file already exists for this version, we stop processing this app and move to the next - if _, err := os.Stat(appTargetPath); !errors.Is(err, os.ErrNotExist) { - log.Printf("Already have APK for version %q at %q\n", release.GetTagName(), appTargetPath) - foundArtifact = true - break - } + appWg.Wait() - log.Printf("Downloading APK %q from release %q to %q", apk.GetName(), release.GetTagName(), appTargetPath) + if repoChanged { + log.Printf("Changes detected for repo: %s", repo.GitURL) + } - downloadContext, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() + fmt.Printf("::endgroup::Repo: %s/%s\n", repo.Owner, repo.Name) + }(repo) + } - appStream, _, err := githubClient.Repositories.DownloadReleaseAsset(downloadContext, repo.Owner, repo.Name, apk.GetID(), http.DefaultClient) - if err != nil { - log.Printf("Error while downloading app %q (artifact id %d) from from release %q: %s", repo.GitURL, apk.GetID(), release.GetTagName(), err.Error()) - haveError = true - break + wg.Wait() + + // Write changes to temporary commit message file + if regenerateMetadata { + var commitMsg strings.Builder + + // Create the first line with repo names + repoNames := make([]string, 0, len(changedRepos)) + for repoURL := range changedRepos { + repoName := strings.TrimPrefix(repoURL, "https://github.com/") + repoNames = append(repoNames, repoName) + } + commitMsg.WriteString(fmt.Sprintf("Updated apps from %s\n\n", strings.Join(repoNames, ", "))) + + // Add details for each repo + for repoURL, apps := range changedRepos { + repoFullName := strings.TrimPrefix(repoURL, "https://github.com/") + + commitMsg.WriteString(fmt.Sprintf("
\n%s\n\n", repoFullName)) + + // Group apps by release + releaseApps := make(map[*github.RepositoryRelease][]string) + for appFilename, release := range apps { + releaseApps[release] = append(releaseApps[release], appFilename) + } + + for release, appList := range releaseApps { + releaseName := release.GetName() + if releaseName == "" { + releaseName = release.GetTagName() } - - err = downloadStream(appTargetPath, appStream) - if err != nil { - log.Printf("Error while downloading app %q (artifact id %d) from from release %q to %q: %s", repo.GitURL, *apk.ID, *release.TagName, appTargetPath, err.Error()) - haveError = true - break + releaseTagURL := release.GetHTMLURL() + commitMsg.WriteString(fmt.Sprintf("### [%s](%s)\n\n", releaseName, releaseTagURL)) + + for _, appFilename := range appList { + commitMsg.WriteString(fmt.Sprintf("- %s\n", appFilename)) } - - log.Printf("Successfully downloaded app for version %q", release.GetTagName()) - fmt.Printf("::endgroup:App %s\n", app.Name) - regenerateMetadata = true - break - - } - if foundArtifact || haveError { - // Stop after the first [release] of this [app] is downloaded to prevent back-filling legacy releases. - break + commitMsg.WriteString("\n") } + + commitMsg.WriteString("
\n\n") } - fmt.Printf("::endgroup::Repo: %s/%s\n", repo.Owner, repo.Name) + err := os.WriteFile(*commitMsgFile, []byte(commitMsg.String()), 0644) + if err != nil { + log.Printf("Error writing commit message file: %s", err) + } else { + log.Printf("Commit message written to %s", *commitMsgFile) + } + } else { + log.Printf("No changes detected.") + os.Exit(2) } if haveError { os.Exit(1) } - if !regenerateMetadata { - log.Printf("No changes detected.") - os.Exit(2) - } - if !*debugMode { fmt.Println("::group::F-Droid: Creating metadata stubs") // Now, we run the fdroid update command diff --git a/run_metascoop.sh b/run_metascoop.sh new file mode 100755 index 0000000..2e0174b --- /dev/null +++ b/run_metascoop.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Check if the commit message file argument is provided +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +COMMIT_MSG_FILE="$1" + +echo "::group::Building metascoop executable" +cd metascoop +go build -o metascoop +echo "::endgroup::" + +echo "::group::Running metascoop" +./metascoop -rp=../repos.yaml -rd=../fdroid/repo -pat="$GH_ACCESS_TOKEN" -cm="$COMMIT_MSG_FILE" +EXIT_CODE=$? +cd .. +echo "::endgroup::" + +echo "Metascoop had an exit code of $EXIT_CODE" + +if [ $EXIT_CODE -eq 2 ]; then + echo "There were no significant changes" + exit 0 +elif [ $EXIT_CODE -eq 0 ]; then + echo "Changes detected" + exit 0 +else + echo "This is an unexpected error" + exit 1 +fi diff --git a/update.sh b/update.sh deleted file mode 100755 index 752133c..0000000 --- a/update.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -dry_run=false - -#region Argument validation -while [[ $# -gt 0 ]]; do - case "$1" in - --dry-run) - if [[ $# -lt 2 ]]; then - echo "Error: --dry-run requires an argument (true or false)" - exit 1 - fi - dry_run=$2 - shift 2 - ;; - *) - echo "Unknown option: $1" - exit 1 - ;; - esac -done - -if [ -z "$dry_run" ]; then - echo "Error: --dry-run flag is required with a value (true or false)." - exit 1 -fi - -if [ "$dry_run" = true ]; then - echo "Performing a dry run. No changes will be pushed." -elif [ "$dry_run" = false ]; then - echo "Changes will be pushed." -else - echo "Error: Invalid value for --dry-run. Use 'true' or 'false'." - exit 1 -fi -#endregion Argument validation - -cd metascoop -echo "::group::Building metascoop executable" -go build -o metascoop -echo "::endgroup::" -./metascoop -rp=../repos.yaml -rd=../fdroid/repo -pat="$GH_ACCESS_TOKEN" -EXIT_CODE=$? -cd .. - -echo "Scoop had an exit code of $EXIT_CODE" - -set -e - -if [ $EXIT_CODE -eq 2 ]; then - # Exit code 2 means that there were no significant changes - echo "There were no significant changes" - exit 0 -elif [ $EXIT_CODE -eq 0 ]; then - # Exit code 0 means that we can commit everything & push - - echo "We have changes to push" - - if [ "$dry_run" = true ]; then - echo "Performing a dry run (no actual push)" - else - echo "Pushing changes..." - git add . - git checkout -b update_fdroid_apps - git commit -m "Automated Bitwarden F-droid repo update" - git push -f -u origin update_fdroid_apps - echo "Creating PR..." - PR_URL=$(gh pr create --title "Automated Bitwarden F-droid repo update" \ - --base main \ - --label "automated pr" \ - --body " - ## Objective - Automated update of Bitwarden F-droid applications to the latest version.") - echo "pr_number=${PR_URL##*/}" - gh pr merge $PR_URL --squash --admin --delete-branch - fi -else - echo "This is an unexpected error" - - exit $EXIT_CODE -fi diff --git a/update_repo.sh b/update_repo.sh new file mode 100755 index 0000000..83c6b16 --- /dev/null +++ b/update_repo.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -e + +# Check if the commit message file argument is provided +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +COMMIT_MSG_FILE="$1" + +if [ -f "$COMMIT_MSG_FILE" ]; then + echo "Changes detected. Proceeding with git operations." + + # Read the first line as the PR title + PR_TITLE=$(head -n 1 "$COMMIT_MSG_FILE") + + # Read the remaining lines as the PR body + PR_BODY=$(tail -n +2 "$COMMIT_MSG_FILE") + + echo "Pushing changes..." + git add . + git checkout -b update_fdroid_apps + git commit -F "$COMMIT_MSG_FILE" + git push -f -u origin update_fdroid_apps + + echo "Creating PR..." + PR_URL=$(gh pr create --title "$PR_TITLE" \ + --base main \ + --label "automated pr" \ + --body "$PR_BODY") + echo "pr_number=${PR_URL##*/}" + + gh pr merge $PR_URL --squash --admin --delete-branch + + # Clean up the temporary commit message file + rm "$COMMIT_MSG_FILE" + +else + echo "Error: Commit message file does not exist or could not be found: $COMMIT_MSG_FILE" >&2 + exit 1 +fi