diff --git a/.gitignore b/.gitignore index d1a39a3..aad7a04 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ testdata/caches cache out.yaml ./dependabot +dependabot.exe +.env diff --git a/cmd/dependabot/internal/cmd/test.go b/cmd/dependabot/internal/cmd/test.go index 68a66a0..b1a2d23 100644 --- a/cmd/dependabot/internal/cmd/test.go +++ b/cmd/dependabot/internal/cmd/test.go @@ -32,7 +32,7 @@ func NewTestCommand() *cobra.Command { return err } - processInput(&scenario.Input) + processInput(&scenario.Input, nil) if err := executeTestJob(infra.RunParams{ CacheDir: flags.cache, diff --git a/cmd/dependabot/internal/cmd/update.go b/cmd/dependabot/internal/cmd/update.go index 1ecc1ab..9aa9b09 100644 --- a/cmd/dependabot/internal/cmd/update.go +++ b/cmd/dependabot/internal/cmd/update.go @@ -8,6 +8,7 @@ import ( "io" "log" "net" + "net/url" "os" "github.com/MakeNowJust/heredoc" @@ -28,10 +29,12 @@ type UpdateFlags struct { SharedFlags provider string directory string + branch string local string commit string dependencies []string inputServerPort int + apiUrl string } func NewUpdateCommand() *cobra.Command { @@ -60,7 +63,7 @@ func NewUpdateCommand() *cobra.Command { return err } - processInput(input) + processInput(input, &flags) var writer io.Writer if !flags.debugging { @@ -86,6 +89,7 @@ func NewUpdateCommand() *cobra.Command { UpdaterImage: updaterImage, Volumes: flags.volumes, Writer: writer, + ApiUrl: flags.apiUrl, }); err != nil { log.Fatalf("failed to run updater: %v", err) } @@ -97,6 +101,7 @@ func NewUpdateCommand() *cobra.Command { cmd.Flags().StringVarP(&flags.file, "file", "f", "", "path to input file") cmd.Flags().StringVarP(&flags.provider, "provider", "p", "github", "provider of the repository") + cmd.Flags().StringVarP(&flags.branch, "branch", "b", "", "target branch to update") cmd.Flags().StringVarP(&flags.directory, "directory", "d", "/", "directory to update") cmd.Flags().StringVarP(&flags.commit, "commit", "", "", "commit to update") cmd.Flags().StringArrayVarP(&flags.dependencies, "dep", "", nil, "dependencies to update") @@ -112,6 +117,7 @@ func NewUpdateCommand() *cobra.Command { cmd.Flags().StringArrayVar(&flags.extraHosts, "extra-hosts", nil, "Docker extra hosts setting on the proxy") cmd.Flags().DurationVarP(&flags.timeout, "timeout", "t", 0, "max time to run an update") cmd.Flags().IntVar(&flags.inputServerPort, "input-port", 0, "port to use for securely passing input to the updater") + cmd.Flags().StringVarP(&flags.apiUrl, "api-url", "a", "", "the api dependabot should connect to.") return cmd } @@ -194,6 +200,10 @@ func readArguments(cmd *cobra.Command, flags *UpdateFlags) (*model.Input, error) } } + if flags.branch != "" && flags.commit != "" { + return nil, errors.New("cannot specify both branch and commit") + } + input := &model.Input{ Job: model.Job{ PackageManager: packageManager, @@ -211,7 +221,7 @@ func readArguments(cmd *cobra.Command, flags *UpdateFlags) (*model.Input, error) Repo: repo, Directory: flags.directory, Commit: flags.commit, - Branch: nil, + Branch: flags.branch, Hostname: nil, APIEndpoint: nil, }, @@ -238,7 +248,7 @@ func readInputFile(file string) (*model.Input, error) { return &input, nil } -func processInput(input *model.Input) { +func processInput(input *model.Input, flags *UpdateFlags) { job := &input.Job // a few of the fields need to be initialized instead of null, // it would be nice if the updater didn't care @@ -258,9 +268,13 @@ func processInput(input *model.Input) { job.DependencyGroups = []model.Group{} } + azureRepo := model.NewAzureRepo(input.Job.PackageManager, input.Job.Source.Repo, input.Job.Source.Directory) + // As a convenience, fill in a git_source if credentials are in the environment and a git_source // doesn't already exist. This way the user doesn't run out of calls from being anonymous. hasLocalToken := os.Getenv("LOCAL_GITHUB_ACCESS_TOKEN") != "" + hasLocalAzureToken := os.Getenv("LOCAL_AZURE_ACCESS_TOKEN") != "" + var isGitSourceInCreds bool for _, cred := range input.Credentials { if cred["type"] == "git_source" { @@ -268,6 +282,16 @@ func processInput(input *model.Input) { break } } + if hasLocalAzureToken && flags.apiUrl != "" && azureRepo != nil { + u, _ := url.Parse(flags.apiUrl) + input.Credentials = append(input.Credentials, model.Credential{ + "type": "git_source", + "host": u.Hostname(), + "username": azureRepo.Org, + "password": "$LOCAL_AZURE_ACCESS_TOKEN", + }) + } + if hasLocalToken && !isGitSourceInCreds { log.Println("Inserting $LOCAL_GITHUB_ACCESS_TOKEN into credentials") input.Credentials = append(input.Credentials, model.Credential{ @@ -285,6 +309,50 @@ func processInput(input *model.Input) { } } + if hasLocalAzureToken && !isGitSourceInCreds && azureRepo != nil { + log.Println("Inserting $LOCAL_AZURE_ACCESS_TOKEN into credentials") + log.Printf("Inserting artifacts credentials for %s organization.", azureRepo.Org) + input.Credentials = append(input.Credentials, model.Credential{ + "type": "git_source", + "host": "dev.azure.com", + "username": "x-access-token", + "password": "$LOCAL_AZURE_ACCESS_TOKEN", + }) + if len(input.Job.CredentialsMetadata) > 0 { + // Add the metadata since the next section will be skipped. + input.Job.CredentialsMetadata = append(input.Job.CredentialsMetadata, map[string]any{ + "type": "git_source", + "host": "dev.azure.com", + }) + } + input.Credentials = append(input.Credentials, model.Credential{ + "type": "git_source", + "host": fmt.Sprintf("%s.pkgs.visualstudio.com", azureRepo.Org), + "username": "x-access-token", + "password": "$LOCAL_AZURE_ACCESS_TOKEN", + }) + if len(input.Job.CredentialsMetadata) > 0 { + // Add the metadata since the next section will be skipped. + input.Job.CredentialsMetadata = append(input.Job.CredentialsMetadata, map[string]any{ + "type": "git_source", + "host": fmt.Sprintf("%s.pkgs.visualstudio.com", azureRepo.Org), + }) + } + input.Credentials = append(input.Credentials, model.Credential{ + "type": "git_source", + "host": "pkgs.dev.azure.com", + "username": "x-access-token", + "password": "$LOCAL_AZURE_ACCESS_TOKEN", + }) + if len(input.Job.CredentialsMetadata) > 0 { + // Add the metadata since the next section will be skipped. + input.Job.CredentialsMetadata = append(input.Job.CredentialsMetadata, map[string]any{ + "type": "git_source", + "host": "pkgs.dev.azure.com", + }) + } + } + // As a convenience, fill credentials-metadata if credentials are provided // which is what happens in production. This way the user doesn't have to // specify credentials-metadata in the scenario file unless they want to. diff --git a/cmd/dependabot/internal/cmd/update_test.go b/cmd/dependabot/internal/cmd/update_test.go index 1510c9c..202a37e 100644 --- a/cmd/dependabot/internal/cmd/update_test.go +++ b/cmd/dependabot/internal/cmd/update_test.go @@ -16,7 +16,7 @@ func Test_processInput(t *testing.T) { os.Setenv("LOCAL_GITHUB_ACCESS_TOKEN", "") var input model.Input - processInput(&input) + processInput(&input, nil) if input.Job.ExistingPullRequests == nil { t.Error("expected existing pull requests to be initialized") @@ -38,7 +38,7 @@ func Test_processInput(t *testing.T) { // Adding a dummy metadata to test the inner if input.Job.CredentialsMetadata = []model.Credential{{}} - processInput(&input) + processInput(&input, nil) if len(input.Credentials) != 1 { t.Fatal("expected credentials to be added") @@ -72,7 +72,7 @@ func Test_processInput(t *testing.T) { }, } - processInput(&input) + processInput(&input, nil) if len(input.Job.CredentialsMetadata) != 1 { t.Fatal("expected credentials metadata to be added") diff --git a/internal/infra/run.go b/internal/infra/run.go index fe5643e..6095f14 100644 --- a/internal/infra/run.go +++ b/internal/infra/run.go @@ -68,6 +68,7 @@ type RunParams struct { Writer io.Writer InputName string InputRaw []byte + ApiUrl string } var gitShaRegex = regexp.MustCompile(`^[0-9a-f]{40}$`) @@ -125,7 +126,10 @@ func Run(params RunParams) error { return err } - if err := runContainers(ctx, params, api); err != nil { + if params.ApiUrl == "" { + params.ApiUrl = fmt.Sprintf("http://host.docker.internal:%v", api.Port()) + } + if err := runContainers(ctx, params); err != nil { return err } @@ -277,16 +281,18 @@ func setImageNames(params *RunParams) error { } func expandEnvironmentVariables(api *server.API, params *RunParams) { - api.Actual.Input.Credentials = params.Creds - - // Make a copy of the credentials, so we don't inject them into the output file. - params.Creds = []model.Credential{} - for _, cred := range api.Actual.Input.Credentials { - newCred := model.Credential{} - for k, v := range cred { - newCred[k] = v + if api != nil { + api.Actual.Input.Credentials = params.Creds + + // Make a copy of the credentials, so we don't inject them into the output file. + params.Creds = []model.Credential{} + for _, cred := range api.Actual.Input.Credentials { + newCred := model.Credential{} + for k, v := range cred { + newCred[k] = v + } + params.Creds = append(params.Creds, newCred) } - params.Creds = append(params.Creds, newCred) } // Add the actual credentials from the environment. @@ -324,7 +330,7 @@ func generateIgnoreConditions(params *RunParams, actual *model.Scenario) error { return nil } -func runContainers(ctx context.Context, params RunParams, api *server.API) error { +func runContainers(ctx context.Context, params RunParams) error { cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { return fmt.Errorf("failed to create Docker client: %w", err) @@ -389,12 +395,12 @@ func runContainers(ctx context.Context, params RunParams, api *server.API) error } if params.Debug { - if err := updater.RunShell(ctx, prox.url, api.Port()); err != nil { + if err := updater.RunShell(ctx, prox.url, params.ApiUrl); err != nil { return err } } else { const cmd = "update-ca-certificates && bin/run fetch_files && bin/run update_files" - if err := updater.RunCmd(ctx, cmd, dependabot, userEnv(prox.url, api.Port())...); err != nil { + if err := updater.RunCmd(ctx, cmd, dependabot, userEnv(prox.url, params.ApiUrl)...); err != nil { return err } } diff --git a/internal/infra/updater.go b/internal/infra/updater.go index 2ce5e57..ca24130 100644 --- a/internal/infra/updater.go +++ b/internal/infra/updater.go @@ -6,7 +6,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/goware/prefixer" "io" "os" "path" @@ -19,6 +18,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" + "github.com/goware/prefixer" "github.com/moby/moby/client" "github.com/moby/moby/pkg/stdcopy" ) @@ -153,19 +153,19 @@ func mountOptions(v string) (local, remote string, readOnly bool, err error) { return local, remote, readOnly, nil } -func userEnv(proxyURL string, apiPort int) []string { +func userEnv(proxyURL string, apiUrl string) []string { return []string{ "GITHUB_ACTIONS=true", // sets exit code when fetch fails fmt.Sprintf("http_proxy=%s", proxyURL), fmt.Sprintf("HTTP_PROXY=%s", proxyURL), fmt.Sprintf("https_proxy=%s", proxyURL), fmt.Sprintf("HTTPS_PROXY=%s", proxyURL), - fmt.Sprintf("DEPENDABOT_JOB_ID=%v", jobID), + fmt.Sprintf("DEPENDABOT_JOB_ID=%v", firstNonEmpty(os.Getenv("DEPENDABOT_JOB_ID"), jobID)), fmt.Sprintf("DEPENDABOT_JOB_TOKEN=%v", ""), fmt.Sprintf("DEPENDABOT_JOB_PATH=%v", guestInputDir), fmt.Sprintf("DEPENDABOT_OUTPUT_PATH=%v", guestOutput), fmt.Sprintf("DEPENDABOT_REPO_CONTENTS_PATH=%v", guestRepoDir), - fmt.Sprintf("DEPENDABOT_API_URL=http://host.docker.internal:%v", apiPort), + fmt.Sprintf("DEPENDABOT_API_URL=%s", apiUrl), fmt.Sprintf("SSL_CERT_FILE=%v/ca-certificates.crt", certsPath), "UPDATER_ONE_CONTAINER=true", "UPDATER_DETERMINISTIC=true", @@ -173,14 +173,14 @@ func userEnv(proxyURL string, apiPort int) []string { } // RunShell executes an interactive shell, blocks until complete. -func (u *Updater) RunShell(ctx context.Context, proxyURL string, apiPort int) error { +func (u *Updater) RunShell(ctx context.Context, proxyURL string, apiUrl string) error { execCreate, err := u.cli.ContainerExecCreate(ctx, u.containerID, types.ExecConfig{ AttachStdin: true, AttachStdout: true, AttachStderr: true, Tty: true, User: dependabot, - Env: append(userEnv(proxyURL, apiPort), "DEBUG=1"), + Env: append(userEnv(proxyURL, apiUrl), "DEBUG=1"), Cmd: []string{"/bin/bash", "-c", "update-ca-certificates && /bin/bash"}, }) if err != nil { @@ -326,3 +326,13 @@ func addFileToArchive(tw *tar.Writer, name string, mode int64, content string) e return nil } + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + + return "" +} diff --git a/internal/model/azure.go b/internal/model/azure.go new file mode 100644 index 0000000..10b3a09 --- /dev/null +++ b/internal/model/azure.go @@ -0,0 +1,31 @@ +package model + +import "strings" + +type AzureRepo struct { + PackageManger string + Org string + Project string + Repo string + Directory string +} + +// NewAzureRepo parses a repo string and returns an AzureRepo struct +// Expects a repo string in the format org/project/repo +func NewAzureRepo(packageManager string, repo string, directory string) *AzureRepo { + repoParts := strings.Split(repo, "/") + for i, part := range repoParts { + println(i, part) + } + if len(repoParts) != 3 { + return nil + } + + return &AzureRepo{ + PackageManger: packageManager, + Org: repoParts[0], + Project: repoParts[1], + Repo: repoParts[2], + Directory: directory, + } +} diff --git a/internal/model/azure_test.go b/internal/model/azure_test.go new file mode 100644 index 0000000..5151ebf --- /dev/null +++ b/internal/model/azure_test.go @@ -0,0 +1,46 @@ +package model + +import ( + "reflect" + "testing" +) + +func Test_NewAzureRepo(t *testing.T) { + tests := []struct { + name string + packageManager string + repo string + directory string + expected *AzureRepo + }{ + { + name: "valid repo", + packageManager: "npm_and_yarn", + repo: "my-org/my-project/my-repo", + directory: "/", + expected: &AzureRepo{ + PackageManger: "npm_and_yarn", + Org: "my-org", + Project: "my-project", + Repo: "my-repo", + Directory: "/", + }, + }, + { + name: "invalid repo", + packageManager: "npm_and_yarn", + repo: "my-org/my-project", + directory: "/", + expected: nil, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + actual := NewAzureRepo(test.packageManager, test.repo, test.directory) + if !reflect.DeepEqual(actual, test.expected) { + t.Errorf("expected %v, got %v", test.expected, actual) + } + }) + } +} diff --git a/internal/model/job.go b/internal/model/job.go index 5d46200..6bf9d49 100644 --- a/internal/model/job.go +++ b/internal/model/job.go @@ -57,7 +57,7 @@ type Source struct { Repo string `json:"repo" yaml:"repo,omitempty"` Directory string `json:"directory" yaml:"directory,omitempty"` Directories []string `json:"directories,omitempty" yaml:"directories,omitempty"` - Branch *string `json:"branch" yaml:"branch,omitempty"` + Branch string `json:"branch,omitempty" yaml:"branch,omitempty"` Commit string `json:"commit,omitempty" yaml:"commit,omitempty"` Hostname *string `json:"hostname" yaml:"hostname,omitempty"` // Must be provided if APIEndpoint is diff --git a/internal/model/job_test.go b/internal/model/job_test.go index a307761..8d11bd7 100644 --- a/internal/model/job_test.go +++ b/internal/model/job_test.go @@ -92,7 +92,6 @@ job: provider: github repo: dependabot/dependabot-core directory: "/npm_and_yarn/helpers" - branch: api-endpoint: https://api.github.com/ hostname: github.com dependencies: