diff --git a/.github/workflows/fdroid.yml b/.github/workflows/fdroid.yml
index 7016ab2..6864899 100644
--- a/.github/workflows/fdroid.yml
+++ b/.github/workflows/fdroid.yml
@@ -19,7 +19,8 @@ jobs:
apps:
name: "Generate repo from apps listing"
runs-on: ubuntu-22.04
-
+ env:
+ COMMIT_MSG_FILE: "${{ github.workspace }}/commit_message.tmp"
steps:
- name: Checkout repo
@@ -100,10 +101,16 @@ jobs:
with:
go-version: '^1.17.0'
- - name: Run update script
+ - name: Run metascoop
+ id: run-metascoop
env:
GH_ACCESS_TOKEN: ${{ steps.retrieve-secrets.outputs.github-pat-bitwarden-devops-bot-repo-scope }}
+ run: |
+ bash run_metascoop.sh ${{ env.COMMIT_MSG_FILE }}
+
+ - name: Update repo
+ env:
GH_TOKEN: ${{ steps.retrieve-secrets.outputs.github-pat-bitwarden-devops-bot-repo-scope }}
- DRY_RUN: ${{ !contains(inputs.dry-run, 'true') }}
+ if: ${{ (github.event_name == 'schedule' || inputs.dry-run == 'false') }}
run: |
- bash update.sh --dry-run ${{ inputs.dry-run }}
+ bash update_repo.sh ${{ env.COMMIT_MSG_FILE }}
diff --git a/metascoop/main.go b/metascoop/main.go
index 5805ef4..85422d7 100644
--- a/metascoop/main.go
+++ b/metascoop/main.go
@@ -13,6 +13,7 @@ import (
"os/exec"
"path/filepath"
"strings"
+ "sync"
"time"
"metascoop/apps"
@@ -26,11 +27,11 @@ import (
func main() {
var (
- reposFilePath = flag.String("rp", "repos.yaml", "Path to repos.yaml file")
- repoDir = flag.String("rd", "fdroid/repo", "Path to fdroid \"repo\" directory")
- accessToken = flag.String("pat", "", "GitHub personal access token")
-
- debugMode = flag.Bool("debug", false, "Debug mode won't run the fdroid command")
+ reposFilePath = flag.String("rp", "repos.yaml", "Path to repos.yaml file")
+ repoDir = flag.String("rd", "fdroid/repo", "Path to fdroid \"repo\" directory")
+ accessToken = flag.String("pat", "", "GitHub personal access token")
+ commitMsgFile = flag.String("cm", "commit_message.tmp", "Path to the commit message file")
+ debugMode = flag.Bool("debug", false, "Debug mode won't run the fdroid command")
)
flag.Parse()
@@ -69,112 +70,196 @@ func main() {
haveError bool
apkInfoMap = make(map[string]apps.Application)
toRemovePaths []string
+ changedRepos = make(map[string]map[string]*github.RepositoryRelease)
+ mu sync.Mutex
+ wg sync.WaitGroup
)
- // Track if changes are detected that will require re-generating metadata
regenerateMetadata := false
- for _, repo := range reposList {
- fmt.Printf("::group::Repo: %s/%s\n", repo.Owner, repo.Name)
-
- err, releases := getRepositoryReleases(githubClient, repo)
- if err != nil {
- log.Printf("Error while listing repo releases for %q: %s\n", repo.GitURL, err.Error())
- haveError = true
- return
- }
-
- log.Printf("Received %d releases", len(releases))
-
- for _, app := range repo.Applications {
- fmt.Printf("::group::App %s\n", app.Name)
- foundArtifact := false
-
- for _, release := range releases {
- fmt.Printf("::group::Release %s\n", release.GetTagName())
-
- if release.GetDraft() {
- log.Printf("Skipping draft %q\n", release.GetTagName())
- continue
- }
- if release.GetTagName() == "" {
- log.Printf("Skipping release with empty tag name")
- continue
- }
-
- log.Printf("Working on release with tag name %q", release.GetTagName())
- var apk *github.ReleaseAsset = apps.FindAPK(release, app.Filename)
-
- if apk == nil {
- log.Printf("Couldn't find any F-Droid assets for application %s in %s with file name %s", app.Filename, release.GetName(), app.Filename)
- continue
- }
-
- appName := apps.GenerateReleaseFilename(app.Id, release.GetTagName())
-
- log.Printf("Target APK name: %s\n", appName)
-
- appClone := app
+ for _, repo := range reposList {
+ wg.Add(1)
+ go func(repo apps.Repo) {
+ defer wg.Done()
- appClone.ReleaseDescription = release.GetBody()
- if appClone.ReleaseDescription != "" {
- log.Printf("Release notes: \n%s\n", appClone.ReleaseDescription)
- }
+ fmt.Printf("::group::Repo: %s/%s\n", repo.Owner, repo.Name)
- apkInfoMap[appName] = appClone
+ err, releases := getRepositoryReleases(githubClient, repo)
+ if err != nil {
+ log.Printf("Error while listing repo releases for %q: %s\n", repo.GitURL, err.Error())
+ mu.Lock()
+ haveError = true
+ mu.Unlock()
+ return
+ }
- appTargetPath := filepath.Join(*repoDir, appName)
+ log.Printf("Received %d releases", len(releases))
+
+ var appWg sync.WaitGroup
+ repoChanged := false
+ for _, app := range repo.Applications {
+ appWg.Add(1)
+ go func(app apps.Application) {
+ defer appWg.Done()
+
+ fmt.Printf("::group::App %s\n", app.Name)
+
+ foundArtifact := false
+
+ for _, release := range releases {
+ fmt.Printf("::group::Release %s\n", release.GetTagName())
+
+ if release.GetDraft() {
+ log.Printf("Skipping draft %q\n", release.GetTagName())
+ continue
+ }
+ if release.GetTagName() == "" {
+ log.Printf("Skipping release with empty tag name")
+ continue
+ }
+
+ log.Printf("Working on release with tag name %q", release.GetTagName())
+ var apk *github.ReleaseAsset = apps.FindAPK(release, app.Filename)
+
+ if apk == nil {
+ log.Printf("Couldn't find any F-Droid assets for application %s in %s with file name %s", app.Filename, release.GetName(), app.Filename)
+ continue
+ }
+
+ appName := apps.GenerateReleaseFilename(app.Id, release.GetTagName())
+
+ log.Printf("Target APK name: %s\n", appName)
+
+ appClone := app
+
+ appClone.ReleaseDescription = release.GetBody()
+ if appClone.ReleaseDescription != "" {
+ log.Printf("Release notes: \n%s\n", appClone.ReleaseDescription)
+ }
+
+ mu.Lock()
+ apkInfoMap[appName] = appClone
+ mu.Unlock()
+
+ appTargetPath := filepath.Join(*repoDir, appName)
+
+ // If the app file already exists for this version, we stop processing this app and move to the next
+ if _, err := os.Stat(appTargetPath); !errors.Is(err, os.ErrNotExist) {
+ log.Printf("Already have APK for version %q at %q\n", release.GetTagName(), appTargetPath)
+ foundArtifact = true
+ break
+ }
+
+ log.Printf("Downloading APK %q from release %q to %q", apk.GetName(), release.GetTagName(), appTargetPath)
+
+ downloadContext, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+ defer cancel()
+
+ appStream, _, err := githubClient.Repositories.DownloadReleaseAsset(downloadContext, repo.Owner, repo.Name, apk.GetID(), http.DefaultClient)
+ if err != nil {
+ log.Printf("Error while downloading app %q (artifact id %d) from from release %q: %s", repo.GitURL, apk.GetID(), release.GetTagName(), err.Error())
+ mu.Lock()
+ haveError = true
+ mu.Unlock()
+ break
+ }
+
+ err = downloadStream(appTargetPath, appStream)
+ if err != nil {
+ log.Printf("Error while downloading app %q (artifact id %d) from from release %q to %q: %s", repo.GitURL, *apk.ID, *release.TagName, appTargetPath, err.Error())
+ mu.Lock()
+ haveError = true
+ mu.Unlock()
+ break
+ }
+
+ log.Printf("Successfully downloaded app for version %q", release.GetTagName())
+ fmt.Printf("::endgroup:App %s\n", app.Name)
+ mu.Lock()
+ regenerateMetadata = true
+ repoChanged = true
+ if changedRepos[repo.GitURL] == nil {
+ changedRepos[repo.GitURL] = make(map[string]*github.RepositoryRelease)
+ }
+ changedRepos[repo.GitURL][app.Filename] = release
+ mu.Unlock()
+ break
+ }
+ if foundArtifact || haveError {
+ // Stop after the first [release] of this [app] is downloaded to prevent back-filling legacy releases.
+ return
+ }
+ }(app)
+ }
- // If the app file already exists for this version, we stop processing this app and move to the next
- if _, err := os.Stat(appTargetPath); !errors.Is(err, os.ErrNotExist) {
- log.Printf("Already have APK for version %q at %q\n", release.GetTagName(), appTargetPath)
- foundArtifact = true
- break
- }
+ appWg.Wait()
- log.Printf("Downloading APK %q from release %q to %q", apk.GetName(), release.GetTagName(), appTargetPath)
+ if repoChanged {
+ log.Printf("Changes detected for repo: %s", repo.GitURL)
+ }
- downloadContext, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
- defer cancel()
+ fmt.Printf("::endgroup::Repo: %s/%s\n", repo.Owner, repo.Name)
+ }(repo)
+ }
- appStream, _, err := githubClient.Repositories.DownloadReleaseAsset(downloadContext, repo.Owner, repo.Name, apk.GetID(), http.DefaultClient)
- if err != nil {
- log.Printf("Error while downloading app %q (artifact id %d) from from release %q: %s", repo.GitURL, apk.GetID(), release.GetTagName(), err.Error())
- haveError = true
- break
+ wg.Wait()
+
+ // Write changes to temporary commit message file
+ if regenerateMetadata {
+ var commitMsg strings.Builder
+
+ // Create the first line with repo names
+ repoNames := make([]string, 0, len(changedRepos))
+ for repoURL := range changedRepos {
+ repoName := strings.TrimPrefix(repoURL, "https://github.com/")
+ repoNames = append(repoNames, repoName)
+ }
+ commitMsg.WriteString(fmt.Sprintf("Updated apps from %s\n\n", strings.Join(repoNames, ", ")))
+
+ // Add details for each repo
+ for repoURL, apps := range changedRepos {
+ repoFullName := strings.TrimPrefix(repoURL, "https://github.com/")
+
+ commitMsg.WriteString(fmt.Sprintf("\n%s
\n\n", repoFullName))
+
+ // Group apps by release
+ releaseApps := make(map[*github.RepositoryRelease][]string)
+ for appFilename, release := range apps {
+ releaseApps[release] = append(releaseApps[release], appFilename)
+ }
+
+ for release, appList := range releaseApps {
+ releaseName := release.GetName()
+ if releaseName == "" {
+ releaseName = release.GetTagName()
}
-
- err = downloadStream(appTargetPath, appStream)
- if err != nil {
- log.Printf("Error while downloading app %q (artifact id %d) from from release %q to %q: %s", repo.GitURL, *apk.ID, *release.TagName, appTargetPath, err.Error())
- haveError = true
- break
+ releaseTagURL := release.GetHTMLURL()
+ commitMsg.WriteString(fmt.Sprintf("### [%s](%s)\n\n", releaseName, releaseTagURL))
+
+ for _, appFilename := range appList {
+ commitMsg.WriteString(fmt.Sprintf("- %s\n", appFilename))
}
-
- log.Printf("Successfully downloaded app for version %q", release.GetTagName())
- fmt.Printf("::endgroup:App %s\n", app.Name)
- regenerateMetadata = true
- break
-
- }
- if foundArtifact || haveError {
- // Stop after the first [release] of this [app] is downloaded to prevent back-filling legacy releases.
- break
+ commitMsg.WriteString("\n")
}
+
+ commitMsg.WriteString(" \n\n")
}
- fmt.Printf("::endgroup::Repo: %s/%s\n", repo.Owner, repo.Name)
+ err := os.WriteFile(*commitMsgFile, []byte(commitMsg.String()), 0644)
+ if err != nil {
+ log.Printf("Error writing commit message file: %s", err)
+ } else {
+ log.Printf("Commit message written to %s", *commitMsgFile)
+ }
+ } else {
+ log.Printf("No changes detected.")
+ os.Exit(2)
}
if haveError {
os.Exit(1)
}
- if !regenerateMetadata {
- log.Printf("No changes detected.")
- os.Exit(2)
- }
-
if !*debugMode {
fmt.Println("::group::F-Droid: Creating metadata stubs")
// Now, we run the fdroid update command
diff --git a/run_metascoop.sh b/run_metascoop.sh
new file mode 100755
index 0000000..2e0174b
--- /dev/null
+++ b/run_metascoop.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Check if the commit message file argument is provided
+if [ "$#" -ne 1 ]; then
+ echo "Usage: $0 "
+ exit 1
+fi
+
+COMMIT_MSG_FILE="$1"
+
+echo "::group::Building metascoop executable"
+cd metascoop
+go build -o metascoop
+echo "::endgroup::"
+
+echo "::group::Running metascoop"
+./metascoop -rp=../repos.yaml -rd=../fdroid/repo -pat="$GH_ACCESS_TOKEN" -cm="$COMMIT_MSG_FILE"
+EXIT_CODE=$?
+cd ..
+echo "::endgroup::"
+
+echo "Metascoop had an exit code of $EXIT_CODE"
+
+if [ $EXIT_CODE -eq 2 ]; then
+ echo "There were no significant changes"
+ exit 0
+elif [ $EXIT_CODE -eq 0 ]; then
+ echo "Changes detected"
+ exit 0
+else
+ echo "This is an unexpected error"
+ exit 1
+fi
diff --git a/update.sh b/update.sh
deleted file mode 100755
index 752133c..0000000
--- a/update.sh
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/bin/bash
-dry_run=false
-
-#region Argument validation
-while [[ $# -gt 0 ]]; do
- case "$1" in
- --dry-run)
- if [[ $# -lt 2 ]]; then
- echo "Error: --dry-run requires an argument (true or false)"
- exit 1
- fi
- dry_run=$2
- shift 2
- ;;
- *)
- echo "Unknown option: $1"
- exit 1
- ;;
- esac
-done
-
-if [ -z "$dry_run" ]; then
- echo "Error: --dry-run flag is required with a value (true or false)."
- exit 1
-fi
-
-if [ "$dry_run" = true ]; then
- echo "Performing a dry run. No changes will be pushed."
-elif [ "$dry_run" = false ]; then
- echo "Changes will be pushed."
-else
- echo "Error: Invalid value for --dry-run. Use 'true' or 'false'."
- exit 1
-fi
-#endregion Argument validation
-
-cd metascoop
-echo "::group::Building metascoop executable"
-go build -o metascoop
-echo "::endgroup::"
-./metascoop -rp=../repos.yaml -rd=../fdroid/repo -pat="$GH_ACCESS_TOKEN"
-EXIT_CODE=$?
-cd ..
-
-echo "Scoop had an exit code of $EXIT_CODE"
-
-set -e
-
-if [ $EXIT_CODE -eq 2 ]; then
- # Exit code 2 means that there were no significant changes
- echo "There were no significant changes"
- exit 0
-elif [ $EXIT_CODE -eq 0 ]; then
- # Exit code 0 means that we can commit everything & push
-
- echo "We have changes to push"
-
- if [ "$dry_run" = true ]; then
- echo "Performing a dry run (no actual push)"
- else
- echo "Pushing changes..."
- git add .
- git checkout -b update_fdroid_apps
- git commit -m "Automated Bitwarden F-droid repo update"
- git push -f -u origin update_fdroid_apps
- echo "Creating PR..."
- PR_URL=$(gh pr create --title "Automated Bitwarden F-droid repo update" \
- --base main \
- --label "automated pr" \
- --body "
- ## Objective
- Automated update of Bitwarden F-droid applications to the latest version.")
- echo "pr_number=${PR_URL##*/}"
- gh pr merge $PR_URL --squash --admin --delete-branch
- fi
-else
- echo "This is an unexpected error"
-
- exit $EXIT_CODE
-fi
diff --git a/update_repo.sh b/update_repo.sh
new file mode 100755
index 0000000..83c6b16
--- /dev/null
+++ b/update_repo.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+set -e
+
+# Check if the commit message file argument is provided
+if [ "$#" -ne 1 ]; then
+ echo "Usage: $0 "
+ exit 1
+fi
+
+COMMIT_MSG_FILE="$1"
+
+if [ -f "$COMMIT_MSG_FILE" ]; then
+ echo "Changes detected. Proceeding with git operations."
+
+ # Read the first line as the PR title
+ PR_TITLE=$(head -n 1 "$COMMIT_MSG_FILE")
+
+ # Read the remaining lines as the PR body
+ PR_BODY=$(tail -n +2 "$COMMIT_MSG_FILE")
+
+ echo "Pushing changes..."
+ git add .
+ git checkout -b update_fdroid_apps
+ git commit -F "$COMMIT_MSG_FILE"
+ git push -f -u origin update_fdroid_apps
+
+ echo "Creating PR..."
+ PR_URL=$(gh pr create --title "$PR_TITLE" \
+ --base main \
+ --label "automated pr" \
+ --body "$PR_BODY")
+ echo "pr_number=${PR_URL##*/}"
+
+ gh pr merge $PR_URL --squash --admin --delete-branch
+
+ # Clean up the temporary commit message file
+ rm "$COMMIT_MSG_FILE"
+
+else
+ echo "Error: Commit message file does not exist or could not be found: $COMMIT_MSG_FILE" >&2
+ exit 1
+fi