Optional repo file (#75)

* Optional repo file

Fixes #74

Adds a new flag for all commands `--repos=<filename>`, which gives the
ability to select a different file from the default `repos.txt` one.

* Fixing defaultOptions naming

* Renaming test

* Fix typo on CampaignOptions

* Updates following PR

- Changed error text to include filename last

Signed-off-by: Sebastien Le Digabel <[email protected]>

* Adding repofile to update_pr

* Rebasing after #67

The `pr-status` command was added after and needed the extra danse to
include the new repo file option.

Also renamed the `pr_status` package into `prstatus` as gofumpt told me
packages should not include underscores.
https://go.dev/blog/package-names

* Changed the campaign start message with the repo filename

.. addressing the PR comment.
Also renamed the update_prs package name for the same reason as the
pr_status one.

* Fixing the parsing for foreach

Adding a manual parsing of the foreach parameters and extra testing.
Also added some details in the Readme

* Rephrasing comment in foreach

* Removing extraneous comment

---------

Signed-off-by: Sebastien Le Digabel <[email protected]>
Co-authored-by: Cat from Catalyst <[email protected]>
Co-authored-by: Richard North <[email protected]>

README.md

@@ -105,6 +105,17 @@ e.g.
 $ gh-search --repos-with-matches YOUR_GITHUB_CODE_SEARCH_QUERY > repos.txt
 ```
 
+### Working on multiple repo files
+
+Occasionally you may need to work on different repo files. For instance the repos can be divided in sub categories and the same change don't apply to them the same way. 
+The default repo file is called `repos.txt` but you can override this with the `--repos` flag.
+
+```console
+turbolift foreach --repos repoFile1.txt sed 's/pattern1/replacement1/g'
+turbolift foreach --repos repoFile2.txt sed 's/pattern2/replacement2/g'
+```
+
+
 ### Running a mass `clone`
 
 ```turbolift clone```

cmd/clone/clone.go

@@ -33,7 +33,10 @@ var (
 	g  git.Git       = git.NewRealGit()
 )
 
-var nofork bool
+var (
+	nofork   bool
+	repoFile string
+)
 
 func NewCloneCmd() *cobra.Command {
 	cmd := &cobra.Command{
@@ -43,15 +46,18 @@ func NewCloneCmd() *cobra.Command {
 	}
 
 	cmd.Flags().BoolVar(&nofork, "no-fork", false, "Will not fork, just clone and create a branch.")
+	cmd.Flags().StringVar(&repoFile, "repos", "repos.txt", "A file containing a list of repositories to clone.")
 
 	return cmd
 }
 
 func run(c *cobra.Command, _ []string) {
 	logger := logging.NewLogger(c)
 
-	readCampaignActivity := logger.StartActivity("Reading campaign data")
-	dir, err := campaign.OpenCampaign()
+	readCampaignActivity := logger.StartActivity("Reading campaign data (%s)", repoFile)
+	options := campaign.NewCampaignOptions()
+	options.RepoFilename = repoFile
+	dir, err := campaign.OpenCampaign(options)
 	if err != nil {
 		readCampaignActivity.EndWithFailure(err)
 		return

cmd/commit/commit.go

@@ -16,18 +16,23 @@
 package commit
 
 import (
+	"os"
+	"path"
+
+	"github.com/spf13/cobra"
+
 	"github.com/skyscanner/turbolift/internal/campaign"
 	"github.com/skyscanner/turbolift/internal/colors"
 	"github.com/skyscanner/turbolift/internal/git"
 	"github.com/skyscanner/turbolift/internal/logging"
-	"github.com/spf13/cobra"
-	"os"
-	"path"
 )
 
 var g git.Git = git.NewRealGit()
 
-var message string
+var (
+	message  string
+	repoFile string
+)
 
 func NewCommitCmd() *cobra.Command {
 	cmd := &cobra.Command{
@@ -37,6 +42,8 @@ func NewCommitCmd() *cobra.Command {
 	}
 
 	cmd.Flags().StringVarP(&message, "message", "m", "", "Commit message to apply")
+	cmd.Flags().StringVar(&repoFile, "repos", "repos.txt", "A file containing a list of repositories to clone.")
+
 	err := cmd.MarkFlagRequired("message")
 	if err != nil {
 		panic(err)
@@ -48,8 +55,10 @@ func NewCommitCmd() *cobra.Command {
 func run(c *cobra.Command, _ []string) {
 	logger := logging.NewLogger(c)
 
-	readCampaignActivity := logger.StartActivity("Reading campaign data")
-	dir, err := campaign.OpenCampaign()
+	readCampaignActivity := logger.StartActivity("Reading campaign data (%s)", repoFile)
+	options := campaign.NewCampaignOptions()
+	options.RepoFilename = repoFile
+	dir, err := campaign.OpenCampaign(options)
 	if err != nil {
 		readCampaignActivity.EndWithFailure(err)
 		return

cmd/create_prs/create_prs.go

@@ -20,12 +20,13 @@ import (
 	"path"
 	"time"
 
+	"github.com/spf13/cobra"
+
 	"github.com/skyscanner/turbolift/internal/campaign"
 	"github.com/skyscanner/turbolift/internal/colors"
 	"github.com/skyscanner/turbolift/internal/git"
 	"github.com/skyscanner/turbolift/internal/github"
 	"github.com/skyscanner/turbolift/internal/logging"
-	"github.com/spf13/cobra"
 )
 
 var (
@@ -34,8 +35,9 @@ var (
 )
 
 var (
-	sleep   time.Duration
-	isDraft bool
+	isDraft  bool
+	repoFile string
+	sleep    time.Duration
 )
 
 func NewCreatePRsCmd() *cobra.Command {
@@ -47,15 +49,18 @@ func NewCreatePRsCmd() *cobra.Command {
 
 	cmd.Flags().DurationVar(&sleep, "sleep", 0, "Fixed sleep in between PR creations (to spread load on CI infrastructure)")
 	cmd.Flags().BoolVar(&isDraft, "draft", false, "Creates the Pull Request as Draft PR")
+	cmd.Flags().StringVar(&repoFile, "repos", "repos.txt", "A file containing a list of repositories to clone.")
 
 	return cmd
 }
 
 func run(c *cobra.Command, _ []string) {
 	logger := logging.NewLogger(c)
 
-	readCampaignActivity := logger.StartActivity("Reading campaign data")
-	dir, err := campaign.OpenCampaign()
+	readCampaignActivity := logger.StartActivity("Reading campaign data (%s)", repoFile)
+	options := campaign.NewCampaignOptions()
+	options.RepoFilename = repoFile
+	dir, err := campaign.OpenCampaign(options)
 	if err != nil {
 		readCampaignActivity.EndWithFailure(err)
 		return

cmd/foreach/foreach.go

@@ -20,15 +20,35 @@ import (
 	"path"
 	"strings"
 
+	"github.com/spf13/cobra"
+
 	"github.com/skyscanner/turbolift/internal/campaign"
 	"github.com/skyscanner/turbolift/internal/colors"
 	"github.com/skyscanner/turbolift/internal/executor"
 	"github.com/skyscanner/turbolift/internal/logging"
-	"github.com/spf13/cobra"
 )
 
 var exec executor.Executor = executor.NewRealExecutor()
 
+var repoFile string = "repos.txt"
+
+func parseForeachArgs(args []string) (strippedArgs []string) {
+MAIN:
+	for i := 0; i < len(args); i++ {
+		switch args[i] {
+		case "--repos":
+			repoFile = args[i+1]
+			i = i + 1
+		default:
+			// we've parsed everything that could be parsed; this is now the command
+			strippedArgs = args[i:]
+			break MAIN
+		}
+	}
+
+	return
+}
+
 func NewForeachCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:                "foreach SHELL_COMMAND",
@@ -38,14 +58,27 @@ func NewForeachCmd() *cobra.Command {
 		DisableFlagParsing: true,
 	}
 
+	// this flag will not be parsed (DisabledFlagParsing is on) but is here for the help context and auto complete
+	cmd.Flags().StringVar(&repoFile, "repos", "repos.txt", "A file containing a list of repositories to clone.")
+
 	return cmd
 }
 
 func run(c *cobra.Command, args []string) {
 	logger := logging.NewLogger(c)
 
-	readCampaignActivity := logger.StartActivity("Reading campaign data")
-	dir, err := campaign.OpenCampaign()
+	/*
+		Parsing is disabled for this command to make sure it doesn't capture flags from the subsequent command.
+		E.g.: turbolift foreach ls -l   <- here, the -l would be captured by foreach, not by ls
+		Because of this, we need a manual parsing of the arguments.
+		Assumption is the foreach arguments will be parsed before the command and its arguments.
+	*/
+	args = parseForeachArgs(args)
+
+	readCampaignActivity := logger.StartActivity("Reading campaign data (%s)", repoFile)
+	options := campaign.NewCampaignOptions()
+	options.RepoFilename = repoFile
+	dir, err := campaign.OpenCampaign(options)
 	if err != nil {
 		readCampaignActivity.EndWithFailure(err)
 		return

cmd/foreach/foreach_test.go

@@ -17,13 +17,73 @@ package foreach
 
 import (
 	"bytes"
-	"github.com/skyscanner/turbolift/internal/executor"
-	"github.com/skyscanner/turbolift/internal/testsupport"
-	"github.com/stretchr/testify/assert"
 	"os"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/skyscanner/turbolift/internal/executor"
+	"github.com/skyscanner/turbolift/internal/testsupport"
 )
 
+func TestParseForEachArgs(t *testing.T) {
+	testCases := []struct {
+		Name                 string
+		Args                 []string
+		ExpectedCommand      []string
+		ExpectedRepoFileName string
+	}{
+		{
+			Name:                 "simple command",
+			Args:                 []string{"ls", "-l"},
+			ExpectedCommand:      []string{"ls", "-l"},
+			ExpectedRepoFileName: "repos.txt",
+		},
+		{
+			Name:                 "advanced command",
+			Args:                 []string{"sed", "-e", "'s/foo/bar/'", "-e", "'s/bar/baz/'"},
+			ExpectedCommand:      []string{"sed", "-e", "'s/foo/bar/'", "-e", "'s/bar/baz/'"},
+			ExpectedRepoFileName: "repos.txt",
+		},
+		{
+			Name:                 "simple command with repo flag",
+			Args:                 []string{"--repos", "test.txt", "ls", "-l"},
+			ExpectedCommand:      []string{"ls", "-l"},
+			ExpectedRepoFileName: "test.txt",
+		},
+		{
+			Name:                 "advanced command with repos flag",
+			Args:                 []string{"--repos", "test2.txt", "sed", "-e", "'s/foo/bar/'", "-e", "'s/bar/baz/'"},
+			ExpectedCommand:      []string{"sed", "-e", "'s/foo/bar/'", "-e", "'s/bar/baz/'"},
+			ExpectedRepoFileName: "test2.txt",
+		},
+		{
+			Name:                 "repos flag should only be caught when at the beginning",
+			Args:                 []string{"ls", "-l", "--repos", "random.txt"},
+			ExpectedCommand:      []string{"ls", "-l", "--repos", "random.txt"},
+			ExpectedRepoFileName: "repos.txt",
+		},
+		{
+			Name:                 "random flag is not caught",
+			Args:                 []string{"--random", "arg", "ls", "-l"},
+			ExpectedCommand:      []string{"--random", "arg", "ls", "-l"},
+			ExpectedRepoFileName: "repos.txt",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			actual := parseForeachArgs(tc.Args)
+			t.Log(actual)
+			assert.EqualValues(t, tc.ExpectedCommand, actual)
+			assert.Equal(t, repoFile, tc.ExpectedRepoFileName)
+
+			// Cleanup to default repo file name
+			repoFile = "repos.txt"
+		})
+	}
+}
+
 func TestItRejectsEmptyArgs(t *testing.T) {
 	fakeExecutor := executor.NewAlwaysSucceedsFakeExecutor()
 	exec = fakeExecutor
@@ -102,7 +162,6 @@ func runCommand(args ...string) (string, error) {
 	cmd.SetOut(outBuffer)
 	cmd.SetArgs(args)
 	err := cmd.Execute()
-
 	if err != nil {
 		return outBuffer.String(), err
 	}

cmd/pr_status/pr_status.go → cmd/prstatus/prstatus.go

@@ -13,19 +13,21 @@
  *
  */
 
-package pr_status
+package prstatus
 
 import (
 	"fmt"
+	"os"
+	"path"
+	"strings"
+
 	"github.com/fatih/color"
 	"github.com/rodaine/table"
+	"github.com/spf13/cobra"
+
 	"github.com/skyscanner/turbolift/internal/campaign"
 	"github.com/skyscanner/turbolift/internal/github"
 	"github.com/skyscanner/turbolift/internal/logging"
-	"github.com/spf13/cobra"
-	"os"
-	"path"
-	"strings"
 )
 
 var reactionsOrder = []string{
@@ -52,7 +54,10 @@ var reactionsMapping = map[string]string{
 
 var gh github.GitHub = github.NewRealGitHub()
 
-var list bool
+var (
+	list     bool
+	repoFile string
+)
 
 func NewPrStatusCmd() *cobra.Command {
 	cmd := &cobra.Command{
@@ -61,15 +66,19 @@ func NewPrStatusCmd() *cobra.Command {
 		Run:   run,
 	}
 	cmd.Flags().BoolVar(&list, "list", false, "Displays a listing by PR")
+	cmd.Flags().StringVar(&repoFile, "repos", "repos.txt", "A file containing a list of repositories to clone.")
 
 	return cmd
 }
 
 func run(c *cobra.Command, _ []string) {
 	logger := logging.NewLogger(c)
 
-	readCampaignActivity := logger.StartActivity("Reading campaign data")
-	dir, err := campaign.OpenCampaign()
+	readCampaignActivity := logger.StartActivity("Reading campaign data (%s)", repoFile)
+
+	options := campaign.NewCampaignOptions()
+	options.RepoFilename = repoFile
+	dir, err := campaign.OpenCampaign(options)
 	if err != nil {
 		readCampaignActivity.EndWithFailure(err)
 		return

cmd/pr_status/pr_status_test.go → cmd/prstatus/prstatus_test.go

@@ -13,7 +13,7 @@
  *
  */
 
-package pr_status
+package prstatus
 
 import (
 	"bytes"
@@ -22,9 +22,10 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/skyscanner/turbolift/internal/github"
 	"github.com/skyscanner/turbolift/internal/testsupport"
-	"github.com/stretchr/testify/assert"
 )
 
 func init() {
@@ -109,7 +110,6 @@ func runCommand(showList bool) (string, error) {
 	outBuffer := bytes.NewBufferString("")
 	cmd.SetOut(outBuffer)
 	err := cmd.Execute()
-
 	if err != nil {
 		return outBuffer.String(), err
 	}