From f45003b494dc3706a04652d4b8fa7b10e87294fd Mon Sep 17 00:00:00 2001 From: Jamie Magee Date: Wed, 6 Dec 2023 09:00:02 -0800 Subject: [PATCH] Add more support for Azure DevOps --- .env.example | 5 ++ .gitignore | 1 + cmd/dependabot/internal/cmd/test.go | 2 +- cmd/dependabot/internal/cmd/update.go | 71 +++++++++++++++++++++- cmd/dependabot/internal/cmd/update_test.go | 6 +- go.mod | 3 + internal/infra/run.go | 32 ++++++---- internal/infra/updater.go | 22 +++++-- internal/model/azure.go | 31 ++++++++++ internal/model/azure_test.go | 45 ++++++++++++++ 10 files changed, 193 insertions(+), 25 deletions(-) create mode 100644 .env.example create mode 100644 internal/model/azure.go create mode 100644 internal/model/azure_test.go diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..37dfd72 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +LOCAL_AZURE_ACCESS_TOKEN= +LOCAL_GITHUB_ACCESS_TOKEN= +DEPENDABOT_PROVIDER= +DEPENDABOT_PACKAGE_MANAGER= +DEPENDABOT_REPO= diff --git a/.gitignore b/.gitignore index d1a39a3..8e64f09 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ testdata/caches cache out.yaml ./dependabot +.env diff --git a/cmd/dependabot/internal/cmd/test.go b/cmd/dependabot/internal/cmd/test.go index 68a66a0..b1a2d23 100644 --- a/cmd/dependabot/internal/cmd/test.go +++ b/cmd/dependabot/internal/cmd/test.go @@ -32,7 +32,7 @@ func NewTestCommand() *cobra.Command { return err } - processInput(&scenario.Input) + processInput(&scenario.Input, nil) if err := executeTestJob(infra.RunParams{ CacheDir: flags.cache, diff --git a/cmd/dependabot/internal/cmd/update.go b/cmd/dependabot/internal/cmd/update.go index 1ecc1ab..23969c6 100644 --- a/cmd/dependabot/internal/cmd/update.go +++ b/cmd/dependabot/internal/cmd/update.go @@ -8,6 +8,7 @@ import ( "io" "log" "net" + "net/url" "os" "github.com/MakeNowJust/heredoc" @@ -32,6 +33,7 @@ type UpdateFlags struct { commit string dependencies []string inputServerPort int + apiUrl string } func NewUpdateCommand() *cobra.Command { @@ -60,7 +62,10 @@ func NewUpdateCommand() *cobra.Command { return err } - processInput(input) + err = processInput(input, &flags) + if err != nil { + return err + } var writer io.Writer if !flags.debugging { @@ -86,6 +91,7 @@ func NewUpdateCommand() *cobra.Command { UpdaterImage: updaterImage, Volumes: flags.volumes, Writer: writer, + ApiUrl: flags.apiUrl, }); err != nil { log.Fatalf("failed to run updater: %v", err) } @@ -112,6 +118,7 @@ func NewUpdateCommand() *cobra.Command { cmd.Flags().StringArrayVar(&flags.extraHosts, "extra-hosts", nil, "Docker extra hosts setting on the proxy") cmd.Flags().DurationVarP(&flags.timeout, "timeout", "t", 0, "max time to run an update") cmd.Flags().IntVar(&flags.inputServerPort, "input-port", 0, "port to use for securely passing input to the updater") + cmd.Flags().StringVarP(&flags.apiUrl, "api-url", "a", "", "the api dependabot should connect to.") return cmd } @@ -238,7 +245,7 @@ func readInputFile(file string) (*model.Input, error) { return &input, nil } -func processInput(input *model.Input) { +func processInput(input *model.Input, flags *UpdateFlags) error { job := &input.Job // a few of the fields need to be initialized instead of null, // it would be nice if the updater didn't care @@ -258,9 +265,13 @@ func processInput(input *model.Input) { job.DependencyGroups = []model.Group{} } + azureRepo := model.NewAzureRepo(input.Job.PackageManager, input.Job.Source.Repo, input.Job.Source.Directory) + // As a convenience, fill in a git_source if credentials are in the environment and a git_source // doesn't already exist. This way the user doesn't run out of calls from being anonymous. hasLocalToken := os.Getenv("LOCAL_GITHUB_ACCESS_TOKEN") != "" + hasLocalAzureToken := os.Getenv("LOCAL_AZURE_ACCESS_TOKEN") != "" + var isGitSourceInCreds bool for _, cred := range input.Credentials { if cred["type"] == "git_source" { @@ -268,6 +279,16 @@ func processInput(input *model.Input) { break } } + if hasLocalAzureToken && flags.apiUrl != "" && azureRepo != nil { + u, _ := url.Parse(flags.apiUrl) + input.Credentials = append(input.Credentials, model.Credential{ + "type": "git_source", + "host": u.Hostname(), + "username": azureRepo.Org, + "password": "$LOCAL_AZURE_ACCESS_TOKEN", + }) + } + if hasLocalToken && !isGitSourceInCreds { log.Println("Inserting $LOCAL_GITHUB_ACCESS_TOKEN into credentials") input.Credentials = append(input.Credentials, model.Credential{ @@ -285,6 +306,50 @@ func processInput(input *model.Input) { } } + if hasLocalAzureToken && !isGitSourceInCreds && azureRepo != nil { + log.Println("Inserting $LOCAL_AZURE_ACCESS_TOKEN into credentials") + log.Printf("Inserting artifacts credentials for %s organization.", azureRepo.Org) + input.Credentials = append(input.Credentials, model.Credential{ + "type": "git_source", + "host": "dev.azure.com", + "username": "x-access-token", + "password": "$LOCAL_AZURE_ACCESS_TOKEN", + }) + if len(input.Job.CredentialsMetadata) > 0 { + // Add the metadata since the next section will be skipped. + input.Job.CredentialsMetadata = append(input.Job.CredentialsMetadata, map[string]any{ + "type": "git_source", + "host": "dev.azure.com", + }) + } + input.Credentials = append(input.Credentials, model.Credential{ + "type": "git_source", + "host": fmt.Sprintf("%s.pkgs.visualstudio.com", azureRepo.Org), + "username": "x-access-token", + "password": "$LOCAL_AZURE_ACCESS_TOKEN", + }) + if len(input.Job.CredentialsMetadata) > 0 { + // Add the metadata since the next section will be skipped. + input.Job.CredentialsMetadata = append(input.Job.CredentialsMetadata, map[string]any{ + "type": "git_source", + "host": fmt.Sprintf("%s.pkgs.visualstudio.com", azureRepo.Org), + }) + } + input.Credentials = append(input.Credentials, model.Credential{ + "type": "git_source", + "host": "pkgs.dev.azure.com", + "username": "x-access-token", + "password": "$LOCAL_AZURE_ACCESS_TOKEN", + }) + if len(input.Job.CredentialsMetadata) > 0 { + // Add the metadata since the next section will be skipped. + input.Job.CredentialsMetadata = append(input.Job.CredentialsMetadata, map[string]any{ + "type": "git_source", + "host": "pkgs.dev.azure.com", + }) + } + } + // As a convenience, fill credentials-metadata if credentials are provided // which is what happens in production. This way the user doesn't have to // specify credentials-metadata in the scenario file unless they want to. @@ -309,6 +374,8 @@ func processInput(input *model.Input) { input.Job.CredentialsMetadata = append(input.Job.CredentialsMetadata, entry) } } + + return nil } func doesStdinHaveData() bool { diff --git a/cmd/dependabot/internal/cmd/update_test.go b/cmd/dependabot/internal/cmd/update_test.go index 1510c9c..202a37e 100644 --- a/cmd/dependabot/internal/cmd/update_test.go +++ b/cmd/dependabot/internal/cmd/update_test.go @@ -16,7 +16,7 @@ func Test_processInput(t *testing.T) { os.Setenv("LOCAL_GITHUB_ACCESS_TOKEN", "") var input model.Input - processInput(&input) + processInput(&input, nil) if input.Job.ExistingPullRequests == nil { t.Error("expected existing pull requests to be initialized") @@ -38,7 +38,7 @@ func Test_processInput(t *testing.T) { // Adding a dummy metadata to test the inner if input.Job.CredentialsMetadata = []model.Credential{{}} - processInput(&input) + processInput(&input, nil) if len(input.Credentials) != 1 { t.Fatal("expected credentials to be added") @@ -72,7 +72,7 @@ func Test_processInput(t *testing.T) { }, } - processInput(&input) + processInput(&input, nil) if len(input.Job.CredentialsMetadata) != 1 { t.Fatal("expected credentials metadata to be added") diff --git a/go.mod b/go.mod index b5a2ff7..f251338 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/moby/moby v24.0.7+incompatible github.com/moby/sys/signal v0.7.0 github.com/spf13/cobra v1.8.0 + github.com/stretchr/testify v1.8.4 gopkg.in/yaml.v3 v3.0.1 ) @@ -18,6 +19,7 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/containerd/containerd v1.7.11 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -33,6 +35,7 @@ require ( github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect github.com/opencontainers/runc v1.1.6 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/mod v0.11.0 // indirect diff --git a/internal/infra/run.go b/internal/infra/run.go index fe5643e..6095f14 100644 --- a/internal/infra/run.go +++ b/internal/infra/run.go @@ -68,6 +68,7 @@ type RunParams struct { Writer io.Writer InputName string InputRaw []byte + ApiUrl string } var gitShaRegex = regexp.MustCompile(`^[0-9a-f]{40}$`) @@ -125,7 +126,10 @@ func Run(params RunParams) error { return err } - if err := runContainers(ctx, params, api); err != nil { + if params.ApiUrl == "" { + params.ApiUrl = fmt.Sprintf("http://host.docker.internal:%v", api.Port()) + } + if err := runContainers(ctx, params); err != nil { return err } @@ -277,16 +281,18 @@ func setImageNames(params *RunParams) error { } func expandEnvironmentVariables(api *server.API, params *RunParams) { - api.Actual.Input.Credentials = params.Creds - - // Make a copy of the credentials, so we don't inject them into the output file. - params.Creds = []model.Credential{} - for _, cred := range api.Actual.Input.Credentials { - newCred := model.Credential{} - for k, v := range cred { - newCred[k] = v + if api != nil { + api.Actual.Input.Credentials = params.Creds + + // Make a copy of the credentials, so we don't inject them into the output file. + params.Creds = []model.Credential{} + for _, cred := range api.Actual.Input.Credentials { + newCred := model.Credential{} + for k, v := range cred { + newCred[k] = v + } + params.Creds = append(params.Creds, newCred) } - params.Creds = append(params.Creds, newCred) } // Add the actual credentials from the environment. @@ -324,7 +330,7 @@ func generateIgnoreConditions(params *RunParams, actual *model.Scenario) error { return nil } -func runContainers(ctx context.Context, params RunParams, api *server.API) error { +func runContainers(ctx context.Context, params RunParams) error { cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { return fmt.Errorf("failed to create Docker client: %w", err) @@ -389,12 +395,12 @@ func runContainers(ctx context.Context, params RunParams, api *server.API) error } if params.Debug { - if err := updater.RunShell(ctx, prox.url, api.Port()); err != nil { + if err := updater.RunShell(ctx, prox.url, params.ApiUrl); err != nil { return err } } else { const cmd = "update-ca-certificates && bin/run fetch_files && bin/run update_files" - if err := updater.RunCmd(ctx, cmd, dependabot, userEnv(prox.url, api.Port())...); err != nil { + if err := updater.RunCmd(ctx, cmd, dependabot, userEnv(prox.url, params.ApiUrl)...); err != nil { return err } } diff --git a/internal/infra/updater.go b/internal/infra/updater.go index 2ce5e57..ca24130 100644 --- a/internal/infra/updater.go +++ b/internal/infra/updater.go @@ -6,7 +6,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/goware/prefixer" "io" "os" "path" @@ -19,6 +18,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" + "github.com/goware/prefixer" "github.com/moby/moby/client" "github.com/moby/moby/pkg/stdcopy" ) @@ -153,19 +153,19 @@ func mountOptions(v string) (local, remote string, readOnly bool, err error) { return local, remote, readOnly, nil } -func userEnv(proxyURL string, apiPort int) []string { +func userEnv(proxyURL string, apiUrl string) []string { return []string{ "GITHUB_ACTIONS=true", // sets exit code when fetch fails fmt.Sprintf("http_proxy=%s", proxyURL), fmt.Sprintf("HTTP_PROXY=%s", proxyURL), fmt.Sprintf("https_proxy=%s", proxyURL), fmt.Sprintf("HTTPS_PROXY=%s", proxyURL), - fmt.Sprintf("DEPENDABOT_JOB_ID=%v", jobID), + fmt.Sprintf("DEPENDABOT_JOB_ID=%v", firstNonEmpty(os.Getenv("DEPENDABOT_JOB_ID"), jobID)), fmt.Sprintf("DEPENDABOT_JOB_TOKEN=%v", ""), fmt.Sprintf("DEPENDABOT_JOB_PATH=%v", guestInputDir), fmt.Sprintf("DEPENDABOT_OUTPUT_PATH=%v", guestOutput), fmt.Sprintf("DEPENDABOT_REPO_CONTENTS_PATH=%v", guestRepoDir), - fmt.Sprintf("DEPENDABOT_API_URL=http://host.docker.internal:%v", apiPort), + fmt.Sprintf("DEPENDABOT_API_URL=%s", apiUrl), fmt.Sprintf("SSL_CERT_FILE=%v/ca-certificates.crt", certsPath), "UPDATER_ONE_CONTAINER=true", "UPDATER_DETERMINISTIC=true", @@ -173,14 +173,14 @@ func userEnv(proxyURL string, apiPort int) []string { } // RunShell executes an interactive shell, blocks until complete. -func (u *Updater) RunShell(ctx context.Context, proxyURL string, apiPort int) error { +func (u *Updater) RunShell(ctx context.Context, proxyURL string, apiUrl string) error { execCreate, err := u.cli.ContainerExecCreate(ctx, u.containerID, types.ExecConfig{ AttachStdin: true, AttachStdout: true, AttachStderr: true, Tty: true, User: dependabot, - Env: append(userEnv(proxyURL, apiPort), "DEBUG=1"), + Env: append(userEnv(proxyURL, apiUrl), "DEBUG=1"), Cmd: []string{"/bin/bash", "-c", "update-ca-certificates && /bin/bash"}, }) if err != nil { @@ -326,3 +326,13 @@ func addFileToArchive(tw *tar.Writer, name string, mode int64, content string) e return nil } + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + + return "" +} diff --git a/internal/model/azure.go b/internal/model/azure.go new file mode 100644 index 0000000..10b3a09 --- /dev/null +++ b/internal/model/azure.go @@ -0,0 +1,31 @@ +package model + +import "strings" + +type AzureRepo struct { + PackageManger string + Org string + Project string + Repo string + Directory string +} + +// NewAzureRepo parses a repo string and returns an AzureRepo struct +// Expects a repo string in the format org/project/repo +func NewAzureRepo(packageManager string, repo string, directory string) *AzureRepo { + repoParts := strings.Split(repo, "/") + for i, part := range repoParts { + println(i, part) + } + if len(repoParts) != 3 { + return nil + } + + return &AzureRepo{ + PackageManger: packageManager, + Org: repoParts[0], + Project: repoParts[1], + Repo: repoParts[2], + Directory: directory, + } +} diff --git a/internal/model/azure_test.go b/internal/model/azure_test.go new file mode 100644 index 0000000..8982ef2 --- /dev/null +++ b/internal/model/azure_test.go @@ -0,0 +1,45 @@ +package model + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_NewAzureRepo(t *testing.T) { + tests := []struct { + name string + packageManager string + repo string + directory string + expected *AzureRepo + }{ + { + name: "valid repo", + packageManager: "npm_and_yarn", + repo: "my-org/my-project/my-repo", + directory: "/", + expected: &AzureRepo{ + PackageManger: "npm_and_yarn", + Org: "my-org", + Project: "my-project", + Repo: "my-repo", + Directory: "/", + }, + }, + { + name: "invalid repo", + packageManager: "npm_and_yarn", + repo: "my-org/my-project", + directory: "/", + expected: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := NewAzureRepo(test.packageManager, test.repo, test.directory) + assert.Equal(t, test.expected, actual) + }) + } +}