From f6dd71eae4734d3f917074cb4f0d43f372d7e1a4 Mon Sep 17 00:00:00 2001 From: dave vader <48764154+plyr4@users.noreply.github.com> Date: Thu, 19 Dec 2024 11:05:55 -0600 Subject: [PATCH 1/2] feat: opt-in gh app integration (#1217) --- api/auth/get_token.go | 40 +++ api/build/compile_publish.go | 2 + api/repo/create.go | 14 + api/repo/repair.go | 35 +- api/scm/sync.go | 26 ++ api/types/repo.go | 31 +- api/types/repo_test.go | 4 +- api/webhook/post.go | 17 +- cmd/vela-server/scm.go | 8 +- compiler/engine.go | 8 + compiler/native/compile.go | 13 + compiler/native/compile_test.go | 106 +++--- compiler/native/environment.go | 25 +- compiler/native/environment_test.go | 95 +++-- compiler/native/native.go | 33 +- compiler/native/script_test.go | 4 +- compiler/native/transform_test.go | 16 +- compiler/registry/github/github.go | 14 +- compiler/registry/github/template.go | 2 +- compiler/types/pipeline/git.go | 35 ++ compiler/types/pipeline/git_test.go | 31 ++ compiler/types/yaml/build.go | 3 + compiler/types/yaml/git.go | 28 ++ compiler/types/yaml/git_test.go | 60 ++++ constants/app_install.go | 20 ++ constants/event.go | 6 + database/integration_test.go | 2 + database/repo/create_test.go | 6 +- database/repo/table.go | 2 + database/repo/update_test.go | 6 +- database/testutils/api_resources.go | 1 + database/types/repo.go | 3 + database/types/repo_test.go | 3 + database/types/schedule_test.go | 5 +- internal/webhook.go | 22 +- mock/server/repo.go | 11 +- router/middleware/build/build_test.go | 1 + router/middleware/repo/repo_test.go | 1 + scm/flags.go | 25 ++ scm/github/access.go | 10 +- scm/github/app_install.go | 156 +++++++++ scm/github/app_permissions.go | 104 ++++++ scm/github/app_permissions_test.go | 195 +++++++++++ scm/github/app_transport.go | 331 ++++++++++++++++++ scm/github/app_transport_test.go | 169 +++++++++ scm/github/authentication.go | 2 +- scm/github/changeset.go | 4 +- scm/github/deployment.go | 8 +- scm/github/driver_test.go | 2 + scm/github/github.go | 183 +++++++--- scm/github/github_client.go | 159 +++++++++ scm/github/github_client_test.go | 128 +++++++ scm/github/github_test.go | 6 +- scm/github/opts.go | 53 ++- scm/github/opts_test.go | 76 +++- scm/github/org.go | 2 +- scm/github/repo.go | 175 ++++++++- scm/github/repo_test.go | 287 +++++++++++++++ .../testdata/hooks/installation_created.json | 100 ++++++ .../testdata/hooks/installation_deleted.json | 100 ++++++ .../installation_repositories_added.json | 103 ++++++ .../installation_repositories_removed.json | 103 ++++++ .../testdata/installation_repositories.json | 123 +++++++ scm/github/testdata/installations.json | 52 +++ .../testdata/installations_access_tokens.json | 134 +++++++ scm/github/user.go | 2 +- scm/github/webhook.go | 61 +++- scm/github/webhook_test.go | 199 ++++++++++- scm/scm.go | 7 +- scm/scm_test.go | 11 +- scm/service.go | 17 + scm/setup.go | 24 +- scm/setup_test.go | 29 +- 73 files changed, 3594 insertions(+), 285 deletions(-) create mode 100644 compiler/types/pipeline/git.go create mode 100644 compiler/types/pipeline/git_test.go create mode 100644 compiler/types/yaml/git.go create mode 100644 compiler/types/yaml/git_test.go create mode 100644 constants/app_install.go create mode 100644 scm/github/app_install.go create mode 100644 scm/github/app_permissions.go create mode 100644 scm/github/app_permissions_test.go create mode 100644 scm/github/app_transport.go create mode 100644 scm/github/app_transport_test.go create mode 100644 scm/github/github_client.go create mode 100644 scm/github/github_client_test.go create mode 100644 scm/github/testdata/hooks/installation_created.json create mode 100644 scm/github/testdata/hooks/installation_deleted.json create mode 100644 scm/github/testdata/hooks/installation_repositories_added.json create mode 100644 scm/github/testdata/hooks/installation_repositories_removed.json create mode 100644 scm/github/testdata/installation_repositories.json create mode 100644 scm/github/testdata/installations.json create mode 100644 scm/github/testdata/installations_access_tokens.json diff --git a/api/auth/get_token.go b/api/auth/get_token.go index 1379ccea3..6b4b4f26b 100644 --- a/api/auth/get_token.go +++ b/api/auth/get_token.go @@ -5,11 +5,13 @@ package auth import ( "fmt" "net/http" + "strconv" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" "github.com/go-vela/server/database" "github.com/go-vela/server/internal/token" "github.com/go-vela/server/scm" @@ -36,6 +38,14 @@ import ( // name: redirect_uri // description: The URL where the user will be sent after authorization // type: string +// - in: query +// name: setup_action +// description: The specific setup action callback identifier +// type: string +// - in: query +// name: installation_id +// description: The specific installation identifier for a GitHub App integration +// type: integer // responses: // '200': // description: Successfully authenticated @@ -46,6 +56,10 @@ import ( // "$ref": "#/definitions/Token" // '307': // description: Redirected for authentication +// '400': +// description: Bad Request +// schema: +// "$ref": "#/definitions/Error" // '401': // description: Unauthorized // schema: @@ -69,6 +83,32 @@ func GetAuthToken(c *gin.Context) { // capture the OAuth state if present oAuthState := c.Request.FormValue("state") + // handle scm setup events + // setup_action==install represents the GitHub App installation callback redirect + if c.Request.FormValue("setup_action") == constants.AppInstallSetupActionInstall { + installID, err := strconv.ParseInt(c.Request.FormValue("installation_id"), 10, 0) + if err != nil { + retErr := fmt.Errorf("unable to parse installation_id: %w", err) + + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + r, err := scm.FromContext(c).FinishInstallation(ctx, c.Request, installID) + if err != nil { + retErr := fmt.Errorf("unable to finish installation: %w", err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + c.Redirect(http.StatusTemporaryRedirect, r) + + return + } + // capture the OAuth code if present code := c.Request.FormValue("code") if len(code) == 0 { diff --git a/api/build/compile_publish.go b/api/build/compile_publish.go index d355488d0..7167d1357 100644 --- a/api/build/compile_publish.go +++ b/api/build/compile_publish.go @@ -269,6 +269,8 @@ func CompileAndPublish( WithRepo(repo). WithUser(u). WithLabels(cfg.Labels). + WithSCM(scm). + WithDatabase(database). Compile(ctx, pipelineFile) if err != nil { // format the error message with extra information diff --git a/api/repo/create.go b/api/repo/create.go index 3e35be3f0..ae766c4c5 100644 --- a/api/repo/create.go +++ b/api/repo/create.go @@ -273,6 +273,18 @@ func CreateRepo(c *gin.Context) { } } + // map this repo to an installation if possible + if r.GetInstallID() == 0 { + r, err = scm.FromContext(c).SyncRepoWithInstallation(ctx, r) + if err != nil { + retErr := fmt.Errorf("unable to sync repo %s with installation: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + } + // if the repo exists but is inactive if len(dbRepo.GetOrg()) > 0 && !dbRepo.GetActive() { // update the repo owner @@ -281,6 +293,8 @@ func CreateRepo(c *gin.Context) { dbRepo.SetBranch(r.GetBranch()) // activate the repo dbRepo.SetActive(true) + // update the install_id + dbRepo.SetInstallID(r.GetInstallID()) // send API call to update the repo // NOTE: not logging modification out separately diff --git a/api/repo/repair.go b/api/repo/repair.go index f48ceea2c..bb15e2734 100644 --- a/api/repo/repair.go +++ b/api/repo/repair.go @@ -62,6 +62,8 @@ import ( // RepairRepo represents the API handler to remove // and then create a webhook for a repo. +// +//nolint:funlen // ignore statement count func RepairRepo(c *gin.Context) { // capture middleware values m := c.MustGet("metadata").(*internal.Metadata) @@ -163,21 +165,48 @@ func RepairRepo(c *gin.Context) { } } + dirty := false + // if the repo was previously inactive, mark it as active if !r.GetActive() { r.SetActive(true) - // send API call to update the repo + dirty = true + + l.Tracef("repo %s repaired - set to active", r.GetFullName()) + } + + // map this repo to an installation, if possible + if r.GetInstallID() == 0 { + r, err = scm.FromContext(c).SyncRepoWithInstallation(ctx, r) + if err != nil { + retErr := fmt.Errorf("unable to sync repo %s with installation: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // install_id was synced + if r.GetInstallID() != 0 { + dirty = true + + l.Tracef("repo %s repaired - set install_id to %d", r.GetFullName(), r.GetInstallID()) + } + } + + // update the repo in the database, if necessary + if dirty { _, err := database.FromContext(c).UpdateRepo(ctx, r) if err != nil { - retErr := fmt.Errorf("unable to set repo %s to active: %w", r.GetFullName(), err) + retErr := fmt.Errorf("unable to update repo %s during repair: %w", r.GetFullName(), err) util.HandleError(c, http.StatusInternalServerError, retErr) return } - l.Infof("repo %s updated - set to active", r.GetFullName()) + l.Infof("repo %s repaired - database updated", r.GetFullName()) } c.JSON(http.StatusOK, fmt.Sprintf("repo %s repaired", r.GetFullName())) diff --git a/api/scm/sync.go b/api/scm/sync.go index 66522510a..1f06ab370 100644 --- a/api/scm/sync.go +++ b/api/scm/sync.go @@ -174,5 +174,31 @@ func SyncRepo(c *gin.Context) { } } + // map this repo to an installation, if necessary + installID := r.GetInstallID() + + r, err = scm.FromContext(c).SyncRepoWithInstallation(ctx, r) + if err != nil { + retErr := fmt.Errorf("unable to sync repo %s with installation: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + // install_id was synced + if r.GetInstallID() != installID { + _, err := database.FromContext(c).UpdateRepo(ctx, r) + if err != nil { + retErr := fmt.Errorf("unable to update repo %s during repair: %w", r.GetFullName(), err) + + util.HandleError(c, http.StatusInternalServerError, retErr) + + return + } + + l.Tracef("repo %s install_id synced to %d", r.GetFullName(), r.GetInstallID()) + } + c.Status(http.StatusNoContent) } diff --git a/api/types/repo.go b/api/types/repo.go index bfad93623..b81a061e9 100644 --- a/api/types/repo.go +++ b/api/types/repo.go @@ -32,6 +32,7 @@ type Repo struct { PipelineType *string `json:"pipeline_type,omitempty"` PreviousName *string `json:"previous_name,omitempty"` ApproveBuild *string `json:"approve_build,omitempty"` + InstallID *int64 `json:"install_id,omitempty"` } // Environment returns a list of environment variables @@ -345,6 +346,19 @@ func (r *Repo) GetApproveBuild() string { return *r.ApproveBuild } +// GetInstallID returns the InstallID field. +// +// When the provided Repo type is nil, or the field within +// the type is nil, it returns the zero value for the field. +func (r *Repo) GetInstallID() int64 { + // return zero value if Repo type or InstallID field is nil + if r == nil || r.InstallID == nil { + return 0 + } + + return *r.InstallID +} + // SetID sets the ID field. // // When the provided Repo type is nil, it @@ -618,6 +632,19 @@ func (r *Repo) SetApproveBuild(v string) { r.ApproveBuild = &v } +// SetInstallID sets the InstallID field. +// +// When the provided Repo type is nil, it +// will set nothing and immediately return. +func (r *Repo) SetInstallID(v int64) { + // return if Repo type is nil + if r == nil { + return + } + + r.InstallID = &v +} + // String implements the Stringer interface for the Repo type. func (r *Repo) String() string { return fmt.Sprintf(`{ @@ -640,7 +667,8 @@ func (r *Repo) String() string { Timeout: %d, Topics: %s, Trusted: %t, - Visibility: %s + Visibility: %s, + InstallID: %d }`, r.GetActive(), r.GetAllowEvents().List(), @@ -662,5 +690,6 @@ func (r *Repo) String() string { r.GetTopics(), r.GetTrusted(), r.GetVisibility(), + r.GetInstallID(), ) } diff --git a/api/types/repo_test.go b/api/types/repo_test.go index c7e20407d..fa8fb2970 100644 --- a/api/types/repo_test.go +++ b/api/types/repo_test.go @@ -303,7 +303,8 @@ func TestTypes_Repo_String(t *testing.T) { Timeout: %d, Topics: %s, Trusted: %t, - Visibility: %s + Visibility: %s, + InstallID: %d }`, r.GetActive(), r.GetAllowEvents().List(), @@ -325,6 +326,7 @@ func TestTypes_Repo_String(t *testing.T) { r.GetTopics(), r.GetTrusted(), r.GetVisibility(), + r.GetInstallID(), ) // run test diff --git a/api/webhook/post.go b/api/webhook/post.go index 52f0d90dd..8ff5b4cc2 100644 --- a/api/webhook/post.go +++ b/api/webhook/post.go @@ -84,6 +84,7 @@ func PostWebhook(c *gin.Context) { // capture middleware values m := c.MustGet("metadata").(*internal.Metadata) l := c.MustGet("logger").(*logrus.Entry) + db := database.FromContext(c) ctx := c.Request.Context() l.Debug("webhook received") @@ -133,6 +134,20 @@ func PostWebhook(c *gin.Context) { return } + if webhook.Installation != nil { + err = scm.FromContext(c).ProcessInstallation(ctx, c.Request, webhook, db) + if err != nil { + retErr := fmt.Errorf("unable to process installation: %w", err) + util.HandleError(c, http.StatusBadRequest, retErr) + + return + } + + c.JSON(http.StatusOK, "installation processed successfully") + + return + } + // check if the hook should be skipped if skip, skipReason := webhook.ShouldSkip(); skip { c.JSON(http.StatusOK, fmt.Sprintf("skipping build: %s", skipReason)) @@ -145,8 +160,6 @@ func PostWebhook(c *gin.Context) { l.Debugf("hook generated from SCM: %v", h) l.Debugf("repo generated from SCM: %v", r) - db := database.FromContext(c) - // if event is repository event, handle separately and return if strings.EqualFold(h.GetEvent(), constants.EventRepository) { r, err = handleRepositoryEvent(ctx, l, db, m, h, r) diff --git a/cmd/vela-server/scm.go b/cmd/vela-server/scm.go index 7124761f0..2efa53bb5 100644 --- a/cmd/vela-server/scm.go +++ b/cmd/vela-server/scm.go @@ -20,16 +20,20 @@ func setupSCM(c *cli.Context, tc *tracing.Client) (scm.Service, error) { Address: c.String("scm.addr"), ClientID: c.String("scm.client"), ClientSecret: c.String("scm.secret"), + AppID: c.Int64("scm.app.id"), + AppPrivateKey: c.String("scm.app.private-key"), + AppPrivateKeyPath: c.String("scm.app.private-key.path"), + AppPermissions: c.StringSlice("scm.app.permissions"), ServerAddress: c.String("server-addr"), ServerWebhookAddress: c.String("scm.webhook.addr"), StatusContext: c.String("scm.context"), WebUIAddress: c.String("webui-addr"), - Scopes: c.StringSlice("scm.scopes"), + OAuthScopes: c.StringSlice("scm.scopes"), Tracing: tc, } // setup the scm // // https://pkg.go.dev/github.com/go-vela/server/scm?tab=doc#New - return scm.New(_setup) + return scm.New(c.Context, _setup) } diff --git a/compiler/engine.go b/compiler/engine.go index 31f85852e..ab4555296 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -10,7 +10,9 @@ import ( "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/database" "github.com/go-vela/server/internal" + "github.com/go-vela/server/scm" ) // Engine represents an interface for converting a yaml @@ -146,6 +148,12 @@ type Engine interface { // WithLabel defines a function that sets // the label(s) in the Engine. WithLabels([]string) Engine + // WithSCM defines a function that sets + // the scm in the Engine. + WithSCM(scm.Service) Engine + // WithDatabase defines a function that sets + // the database in the Engine. + WithDatabase(database.Interface) Engine // WithPrivateGitHub defines a function that sets // the private github client in the Engine. WithPrivateGitHub(context.Context, string, string) Engine diff --git a/compiler/native/compile.go b/compiler/native/compile.go index 313e24db3..04740f412 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -44,6 +44,19 @@ func (c *client) Compile(ctx context.Context, v interface{}) (*pipeline.Build, * return nil, nil, err } + // create the netrc using the scm + // this has to occur after Parse because the scm configurations might be set in yaml + // netrc can be provided directly using WithNetrc for situations like local exec + if c.netrc == nil && c.scm != nil { + // get the netrc password from the scm + netrc, err := c.scm.GetNetrcPassword(ctx, c.db, c.repo, c.user, p.Git) + if err != nil { + return nil, nil, err + } + + c.WithNetrc(netrc) + } + // create the API pipeline object from the yaml configuration _pipeline := p.ToPipelineAPI() _pipeline.SetData(data) diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index 4b0214d6f..e887176d7 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -54,21 +54,21 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { }, } - initEnv := environment(nil, m, nil, nil) + initEnv := environment(nil, m, nil, nil, nil) initEnv["HELLO"] = "Hello, Global Environment" - stageEnvInstall := environment(nil, m, nil, nil) + stageEnvInstall := environment(nil, m, nil, nil, nil) stageEnvInstall["HELLO"] = "Hello, Global Environment" stageEnvInstall["GRADLE_USER_HOME"] = ".gradle" - stageEnvTest := environment(nil, m, nil, nil) + stageEnvTest := environment(nil, m, nil, nil, nil) stageEnvTest["HELLO"] = "Hello, Global Environment" stageEnvTest["GRADLE_USER_HOME"] = "willBeOverwrittenInStep" - cloneEnv := environment(nil, m, nil, nil) + cloneEnv := environment(nil, m, nil, nil, nil) cloneEnv["HELLO"] = "Hello, Global Environment" - installEnv := environment(nil, m, nil, nil) + installEnv := environment(nil, m, nil, nil, nil) installEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" installEnv["GRADLE_USER_HOME"] = ".gradle" installEnv["HOME"] = "/root" @@ -76,7 +76,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { installEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew downloadDependencies"}) installEnv["HELLO"] = "Hello, Global Environment" - testEnv := environment(nil, m, nil, nil) + testEnv := environment(nil, m, nil, nil, nil) testEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" testEnv["GRADLE_USER_HOME"] = ".gradle" testEnv["HOME"] = "/root" @@ -84,7 +84,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { testEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew check"}) testEnv["HELLO"] = "Hello, Global Environment" - buildEnv := environment(nil, m, nil, nil) + buildEnv := environment(nil, m, nil, nil, nil) buildEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" buildEnv["GRADLE_USER_HOME"] = ".gradle" buildEnv["HOME"] = "/root" @@ -92,7 +92,7 @@ func TestNative_Compile_StagesPipeline(t *testing.T) { buildEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew build"}) buildEnv["HELLO"] = "Hello, Global Environment" - dockerEnv := environment(nil, m, nil, nil) + dockerEnv := environment(nil, m, nil, nil, nil) dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" @@ -479,13 +479,13 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { }, } - initEnv := environment(nil, m, nil, nil) + initEnv := environment(nil, m, nil, nil, nil) initEnv["HELLO"] = "Hello, Global Environment" - cloneEnv := environment(nil, m, nil, nil) + cloneEnv := environment(nil, m, nil, nil, nil) cloneEnv["HELLO"] = "Hello, Global Environment" - installEnv := environment(nil, m, nil, nil) + installEnv := environment(nil, m, nil, nil, nil) installEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" installEnv["GRADLE_USER_HOME"] = ".gradle" installEnv["HOME"] = "/root" @@ -493,7 +493,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { installEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew downloadDependencies"}) installEnv["HELLO"] = "Hello, Global Environment" - testEnv := environment(nil, m, nil, nil) + testEnv := environment(nil, m, nil, nil, nil) testEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" testEnv["GRADLE_USER_HOME"] = ".gradle" testEnv["HOME"] = "/root" @@ -501,7 +501,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { testEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew check"}) testEnv["HELLO"] = "Hello, Global Environment" - buildEnv := environment(nil, m, nil, nil) + buildEnv := environment(nil, m, nil, nil, nil) buildEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" buildEnv["GRADLE_USER_HOME"] = ".gradle" buildEnv["HOME"] = "/root" @@ -509,7 +509,7 @@ func TestNative_Compile_StepsPipeline(t *testing.T) { buildEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"./gradlew build"}) buildEnv["HELLO"] = "Hello, Global Environment" - dockerEnv := environment(nil, m, nil, nil) + dockerEnv := environment(nil, m, nil, nil, nil) dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" @@ -690,11 +690,11 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { }, } - setupEnv := environment(nil, m, nil, nil) + setupEnv := environment(nil, m, nil, nil, nil) setupEnv["bar"] = "test4" setupEnv["star"] = "test3" - installEnv := environment(nil, m, nil, nil) + installEnv := environment(nil, m, nil, nil, nil) installEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" installEnv["GRADLE_USER_HOME"] = ".gradle" installEnv["HOME"] = "/root" @@ -703,7 +703,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { installEnv["bar"] = "test4" installEnv["star"] = "test3" - testEnv := environment(nil, m, nil, nil) + testEnv := environment(nil, m, nil, nil, nil) testEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" testEnv["GRADLE_USER_HOME"] = ".gradle" testEnv["HOME"] = "/root" @@ -712,7 +712,7 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { testEnv["bar"] = "test4" testEnv["star"] = "test3" - buildEnv := environment(nil, m, nil, nil) + buildEnv := environment(nil, m, nil, nil, nil) buildEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" buildEnv["GRADLE_USER_HOME"] = ".gradle" buildEnv["HOME"] = "/root" @@ -721,14 +721,14 @@ func TestNative_Compile_StagesPipelineTemplate(t *testing.T) { buildEnv["bar"] = "test4" buildEnv["star"] = "test3" - dockerEnv := environment(nil, m, nil, nil) + dockerEnv := environment(nil, m, nil, nil, nil) dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" dockerEnv["bar"] = "test4" dockerEnv["star"] = "test3" - serviceEnv := environment(nil, m, nil, nil) + serviceEnv := environment(nil, m, nil, nil, nil) serviceEnv["bar"] = "test4" serviceEnv["star"] = "test3" @@ -961,11 +961,11 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { }, } - setupEnv := environment(nil, m, nil, nil) + setupEnv := environment(nil, m, nil, nil, nil) setupEnv["bar"] = "test4" setupEnv["star"] = "test3" - installEnv := environment(nil, m, nil, nil) + installEnv := environment(nil, m, nil, nil, nil) installEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" installEnv["GRADLE_USER_HOME"] = ".gradle" installEnv["HOME"] = "/root" @@ -974,7 +974,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { installEnv["bar"] = "test4" installEnv["star"] = "test3" - testEnv := environment(nil, m, nil, nil) + testEnv := environment(nil, m, nil, nil, nil) testEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" testEnv["GRADLE_USER_HOME"] = ".gradle" testEnv["HOME"] = "/root" @@ -983,7 +983,7 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { testEnv["bar"] = "test4" testEnv["star"] = "test3" - buildEnv := environment(nil, m, nil, nil) + buildEnv := environment(nil, m, nil, nil, nil) buildEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" buildEnv["GRADLE_USER_HOME"] = ".gradle" buildEnv["HOME"] = "/root" @@ -992,14 +992,14 @@ func TestNative_Compile_StepsPipelineTemplate(t *testing.T) { buildEnv["bar"] = "test4" buildEnv["star"] = "test3" - dockerEnv := environment(nil, m, nil, nil) + dockerEnv := environment(nil, m, nil, nil, nil) dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" dockerEnv["bar"] = "test4" dockerEnv["star"] = "test3" - serviceEnv := environment(nil, m, nil, nil) + serviceEnv := environment(nil, m, nil, nil, nil) serviceEnv["bar"] = "test4" serviceEnv["star"] = "test3" @@ -1195,9 +1195,9 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName(t *testi }, } - setupEnv := environment(nil, m, nil, nil) + setupEnv := environment(nil, m, nil, nil, nil) - helloEnv := environment(nil, m, nil, nil) + helloEnv := environment(nil, m, nil, nil, nil) helloEnv["HOME"] = "/root" helloEnv["SHELL"] = "/bin/sh" helloEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"echo sample"}) @@ -1316,9 +1316,9 @@ func TestNative_Compile_StepsPipelineTemplate_VelaFunction_TemplateName_Inline(t }, } - setupEnv := environment(nil, m, nil, nil) + setupEnv := environment(nil, m, nil, nil, nil) - helloEnv := environment(nil, m, nil, nil) + helloEnv := environment(nil, m, nil, nil, nil) helloEnv["HOME"] = "/root" helloEnv["SHELL"] = "/bin/sh" helloEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"echo inline_templatename"}) @@ -1436,11 +1436,11 @@ func TestNative_Compile_InvalidType(t *testing.T) { }, } - gradleEnv := environment(nil, m, nil, nil) + gradleEnv := environment(nil, m, nil, nil, nil) gradleEnv["GRADLE_OPTS"] = "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false" gradleEnv["GRADLE_USER_HOME"] = ".gradle" - dockerEnv := environment(nil, m, nil, nil) + dockerEnv := environment(nil, m, nil, nil, nil) dockerEnv["PARAMETER_REGISTRY"] = "index.docker.io" dockerEnv["PARAMETER_REPO"] = "github/octocat" dockerEnv["PARAMETER_TAGS"] = "latest,dev" @@ -1493,10 +1493,10 @@ func TestNative_Compile_Clone(t *testing.T) { }, } - fooEnv := environment(nil, m, nil, nil) + fooEnv := environment(nil, m, nil, nil, nil) fooEnv["PARAMETER_REGISTRY"] = "foo" - cloneEnv := environment(nil, m, nil, nil) + cloneEnv := environment(nil, m, nil, nil, nil) cloneEnv["PARAMETER_DEPTH"] = "5" wantFalse := &pipeline.Build{ @@ -1512,7 +1512,7 @@ func TestNative_Compile_Clone(t *testing.T) { &pipeline.Container{ ID: "step___0_init", Directory: "/vela/src/foo//", - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, nil), Image: "#init", Name: "init", Number: 1, @@ -1543,7 +1543,7 @@ func TestNative_Compile_Clone(t *testing.T) { &pipeline.Container{ ID: "step___0_init", Directory: "/vela/src/foo//", - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, nil), Image: "#init", Name: "init", Number: 1, @@ -1552,7 +1552,7 @@ func TestNative_Compile_Clone(t *testing.T) { &pipeline.Container{ ID: "step___0_clone", Directory: "/vela/src/foo//", - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, nil), Image: defaultCloneImage, Name: "clone", Number: 2, @@ -1583,7 +1583,7 @@ func TestNative_Compile_Clone(t *testing.T) { &pipeline.Container{ ID: "step___0_init", Directory: "/vela/src/foo//", - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, nil), Image: "#init", Name: "init", Number: 1, @@ -1687,10 +1687,10 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { }, } - defaultFooEnv := environment(nil, m, nil, nil) + defaultFooEnv := environment(nil, m, nil, nil, nil) defaultFooEnv["PARAMETER_REGISTRY"] = "foo" - defaultEnv := environment(nil, m, nil, nil) + defaultEnv := environment(nil, m, nil, nil, nil) wantDefault := &pipeline.Build{ Version: "1", ID: "__0", @@ -1733,10 +1733,10 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { goPipelineType := "go" - goFooEnv := environment(nil, m, &api.Repo{PipelineType: &goPipelineType}, nil) + goFooEnv := environment(nil, m, &api.Repo{PipelineType: &goPipelineType}, nil, nil) goFooEnv["PARAMETER_REGISTRY"] = "foo" - defaultGoEnv := environment(nil, m, &api.Repo{PipelineType: &goPipelineType}, nil) + defaultGoEnv := environment(nil, m, &api.Repo{PipelineType: &goPipelineType}, nil, nil) wantGo := &pipeline.Build{ Version: "1", ID: "__0", @@ -1779,10 +1779,10 @@ func TestNative_Compile_Pipeline_Type(t *testing.T) { starPipelineType := "starlark" - starlarkFooEnv := environment(nil, m, &api.Repo{PipelineType: &starPipelineType}, nil) + starlarkFooEnv := environment(nil, m, &api.Repo{PipelineType: &starPipelineType}, nil, nil) starlarkFooEnv["PARAMETER_REGISTRY"] = "foo" - defaultStarlarkEnv := environment(nil, m, &api.Repo{PipelineType: &starPipelineType}, nil) + defaultStarlarkEnv := environment(nil, m, &api.Repo{PipelineType: &starPipelineType}, nil, nil) wantStarlark := &pipeline.Build{ Version: "1", ID: "__0", @@ -2039,13 +2039,13 @@ func Test_client_modifyConfig(t *testing.T) { }, Steps: yaml.StepSlice{ &yaml.Step{ - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, nil), Image: "#init", Name: "init", Pull: "not_present", }, &yaml.Step{ - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, nil), Image: defaultCloneImage, Name: "clone", Pull: "not_present", @@ -2072,13 +2072,13 @@ func Test_client_modifyConfig(t *testing.T) { }, Steps: yaml.StepSlice{ &yaml.Step{ - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, nil), Image: "#init", Name: "init", Pull: "not_present", }, &yaml.Step{ - Environment: environment(nil, m, nil, nil), + Environment: environment(nil, m, nil, nil, nil), Image: defaultCloneImage, Name: "clone", Pull: "not_present", @@ -2255,7 +2255,7 @@ func convertFileToGithubResponse(file string) (github.RepositoryContent, error) } func generateTestEnv(command string, m *internal.Metadata, pipelineType string) map[string]string { - output := environment(nil, m, nil, nil) + output := environment(nil, m, nil, nil, nil) output["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{command}) output["HOME"] = "/root" output["SHELL"] = "/bin/sh" @@ -2312,15 +2312,15 @@ func Test_Compile_Inline(t *testing.T) { }, } - initEnv := environment(nil, m, nil, nil) - testEnv := environment(nil, m, nil, nil) + initEnv := environment(nil, m, nil, nil, nil) + testEnv := environment(nil, m, nil, nil, nil) testEnv["FOO"] = "Hello, foo!" testEnv["HELLO"] = "Hello, Vela!" - stepEnv := environment(nil, m, nil, nil) + stepEnv := environment(nil, m, nil, nil, nil) stepEnv["FOO"] = "Hello, foo!" stepEnv["HELLO"] = "Hello, Vela!" stepEnv["PARAMETER_FIRST"] = "foo" - golangEnv := environment(nil, m, nil, nil) + golangEnv := environment(nil, m, nil, nil, nil) golangEnv["VELA_REPO_PIPELINE_TYPE"] = "go" type args struct { diff --git a/compiler/native/environment.go b/compiler/native/environment.go index fae7557d1..3f3162137 100644 --- a/compiler/native/environment.go +++ b/compiler/native/environment.go @@ -33,8 +33,9 @@ func (c *client) EnvironmentStages(s yaml.StageSlice, globalEnv raw.StringSliceM func (c *client) EnvironmentStage(s *yaml.Stage, globalEnv raw.StringSliceMap) (*yaml.Stage, error) { // make empty map of environment variables env := make(map[string]string) + // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user) + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, c.netrc) // inject the declared global environment // WARNING: local env can override global @@ -87,8 +88,9 @@ func (c *client) EnvironmentSteps(s yaml.StepSlice, stageEnv raw.StringSliceMap) func (c *client) EnvironmentStep(s *yaml.Step, stageEnv raw.StringSliceMap) (*yaml.Step, error) { // make empty map of environment variables env := make(map[string]string) + // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user) + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, c.netrc) // inject the declared stage environment // WARNING: local env can override global + stage @@ -148,8 +150,9 @@ func (c *client) EnvironmentServices(s yaml.ServiceSlice, globalEnv raw.StringSl for _, service := range s { // make empty map of environment variables env := make(map[string]string) + // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user) + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, c.netrc) // inject the declared global environment // WARNING: local env can override global @@ -188,8 +191,9 @@ func (c *client) EnvironmentSecrets(s yaml.SecretSlice, globalEnv raw.StringSlic // make empty map of environment variables env := make(map[string]string) + // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user) + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, c.netrc) // inject the declared global environment // WARNING: local env can override global @@ -243,11 +247,14 @@ func (c *client) EnvironmentSecrets(s yaml.SecretSlice, globalEnv raw.StringSlic return s, nil } +// EnvironmentBuild injects environment variables +// for the build in a yaml configuration. func (c *client) EnvironmentBuild() map[string]string { // make empty map of environment variables env := make(map[string]string) + // gather set of default environment variables - defaultEnv := environment(c.build, c.metadata, c.repo, c.user) + defaultEnv := environment(c.build, c.metadata, c.repo, c.user, c.netrc) // inject the default environment // variables to the build @@ -281,7 +288,7 @@ func appendMap(originalMap, otherMap map[string]string) map[string]string { } // helper function that creates the standard set of environment variables for a pipeline. -func environment(b *api.Build, m *internal.Metadata, r *api.Repo, u *api.User) map[string]string { +func environment(b *api.Build, m *internal.Metadata, r *api.Repo, u *api.User, netrc *string) map[string]string { // set default workspace workspace := constants.WorkspaceDefault notImplemented := "TODO" @@ -297,7 +304,7 @@ func environment(b *api.Build, m *internal.Metadata, r *api.Repo, u *api.User) m env["VELA_DISTRIBUTION"] = notImplemented env["VELA_HOST"] = notImplemented env["VELA_NETRC_MACHINE"] = notImplemented - env["VELA_NETRC_PASSWORD"] = u.GetToken() + env["VELA_NETRC_PASSWORD"] = notImplemented env["VELA_NETRC_USERNAME"] = "x-oauth-basic" env["VELA_QUEUE"] = notImplemented env["VELA_RUNTIME"] = notImplemented @@ -321,6 +328,10 @@ func environment(b *api.Build, m *internal.Metadata, r *api.Repo, u *api.User) m workspace = fmt.Sprintf("%s/%s/%s/%s", workspace, m.Source.Host, r.GetOrg(), r.GetName()) } + if netrc != nil { + env["VELA_NETRC_PASSWORD"] = *netrc + } + env["VELA_WORKSPACE"] = workspace // populate environment variables from repo api diff --git a/compiler/native/environment_test.go b/compiler/native/environment_test.go index b9ddcdec0..17dce3942 100644 --- a/compiler/native/environment_test.go +++ b/compiler/native/environment_test.go @@ -42,7 +42,7 @@ func TestNative_EnvironmentStages(t *testing.T) { }, } - env := environment(nil, nil, nil, nil) + env := environment(nil, nil, nil, nil, nil) env["HELLO"] = "Hello, Global Message" want := yaml.StageSlice{ @@ -174,7 +174,7 @@ func TestNative_EnvironmentSteps(t *testing.T) { "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "TODO", "VELA_NETRC_MACHINE": "TODO", - "VELA_NETRC_PASSWORD": "", + "VELA_NETRC_PASSWORD": "TODO", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "TODO", "VELA_REPO_ACTIVE": "false", @@ -351,7 +351,7 @@ func TestNative_EnvironmentServices(t *testing.T) { "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "TODO", "VELA_NETRC_MACHINE": "TODO", - "VELA_NETRC_PASSWORD": "", + "VELA_NETRC_PASSWORD": "TODO", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "TODO", "VELA_REPO_ACTIVE": "false", @@ -510,7 +510,7 @@ func TestNative_EnvironmentSecrets(t *testing.T) { "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "TODO", "VELA_NETRC_MACHINE": "TODO", - "VELA_NETRC_PASSWORD": "", + "VELA_NETRC_PASSWORD": "TODO", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "TODO", "VELA_REPO_ACTIVE": "false", @@ -580,56 +580,73 @@ func TestNative_environment(t *testing.T) { // deployment deploy := "deployment" target := "production" + // netrc + netrc := "foo" tests := []struct { - w string - b *api.Build - m *internal.Metadata - r *api.Repo - u *api.User - want map[string]string + w string + b *api.Build + m *internal.Metadata + r *api.Repo + u *api.User + netrc *string + want map[string]string }{ // push { - w: workspace, - b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &push, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &str, BaseRef: &str}, - m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, - r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, - u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, - want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "push", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "foo", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "push", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "foo", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, + w: workspace, + b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &push, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &str, BaseRef: &str}, + m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, + r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, + u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, + want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "push", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "foo", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "push", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "foo", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, // tag { - w: workspace, - b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &tag, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &tagref, BaseRef: &str}, - m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, - r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, - u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, - want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "tag", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/tags/1", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TAG": "1", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "tag", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/tags/1", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TAG": "1", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, + w: workspace, + b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &tag, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &tagref, BaseRef: &str}, + m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, + r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, + u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, + want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "tag", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/tags/1", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TAG": "1", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "tag", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/tags/1", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TAG": "1", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, // pull_request { - w: workspace, - b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &pull, EventAction: &pullact, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Fork: &booL, Author: &str, Branch: &str, Ref: &pullref, BaseRef: &str}, - m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, - r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, - u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, - want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "pull_request", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_PULL_REQUEST_NUMBER": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "pull_request", "VELA_BUILD_EVENT_ACTION": "opened", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_PULL_REQUEST": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_PULL_REQUEST": "1", "VELA_PULL_REQUEST_FORK": "false", "VELA_PULL_REQUEST_SOURCE": "", "VELA_PULL_REQUEST_TARGET": "foo", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, + w: workspace, + b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &pull, EventAction: &pullact, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &pullref, BaseRef: &str}, + m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, + r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, + u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, + want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "pull_request", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_PULL_REQUEST_NUMBER": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "pull_request", "VELA_BUILD_EVENT_ACTION": "opened", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_PULL_REQUEST": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_PULL_REQUEST": "1", "VELA_PULL_REQUEST_FORK": "false", "VELA_PULL_REQUEST_SOURCE": "", "VELA_PULL_REQUEST_TARGET": "foo", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, // deployment { - w: workspace, - b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &deploy, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &target, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &pullref, BaseRef: &str}, - m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, - r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, - u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, - want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "deployment", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TARGET": "production", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "deployment", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TARGET": "production", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DEPLOYMENT": "production", "VELA_DEPLOYMENT_NUMBER": "0", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, + w: workspace, + b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &deploy, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &target, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &pullref, BaseRef: &str}, + m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, + r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, + u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, + want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "deployment", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TARGET": "production", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "deployment", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TARGET": "production", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DEPLOYMENT": "production", "VELA_DEPLOYMENT_NUMBER": "0", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, + }, + // netrc + { + w: workspace, + b: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &deploy, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &target, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &pullref, BaseRef: &str}, + m: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, + r: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, + u: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: nil, + want: map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "deployment", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TARGET": "production", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "deployment", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TARGET": "production", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DEPLOYMENT": "production", "VELA_DEPLOYMENT_NUMBER": "0", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "TODO", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_BRANCH": "foo", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_OWNER": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, } // run test for _, test := range tests { - got := environment(test.b, test.m, test.r, test.u) + got := environment(test.b, test.m, test.r, test.u, test.netrc) if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("environment mismatch (-want +got):\n%s", diff) @@ -694,12 +711,15 @@ func Test_client_EnvironmentBuild(t *testing.T) { // deployment deploy := "deployment" target := "production" + // netrc + netrc := "foo" type fields struct { build *api.Build metadata *internal.Metadata repo *api.Repo user *api.User + netrc *string } tests := []struct { @@ -712,12 +732,14 @@ func Test_client_EnvironmentBuild(t *testing.T) { metadata: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, user: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, }, map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "push", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "foo", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "push", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "foo", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_OWNER": "foo", "VELA_REPO_BRANCH": "foo", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}}, {"tag", fields{ build: &api.Build{ID: &num64, Repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, Number: &num, Parent: &num, Event: &tag, Status: &str, Error: &str, Enqueued: &num64, Created: &num64, Started: &num64, Finished: &num64, Deploy: &str, Clone: &str, Source: &str, Title: &str, Message: &str, Commit: &str, Sender: &str, SenderSCMID: &str, Author: &str, Branch: &str, Ref: &tagref, BaseRef: &str}, metadata: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, user: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, }, map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "tag", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/tags/1", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TAG": "1", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "tag", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/tags/1", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TAG": "1", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_OWNER": "foo", "VELA_REPO_BRANCH": "foo", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, {"pull_request", fields{ @@ -725,6 +747,7 @@ func Test_client_EnvironmentBuild(t *testing.T) { metadata: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, user: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, }, map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "pull_request", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_PULL_REQUEST_NUMBER": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "pull_request", "VELA_BUILD_EVENT_ACTION": "opened", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_PULL_REQUEST": "1", "VELA_PULL_REQUEST_FORK": "false", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_PULL_REQUEST": "1", "VELA_PULL_REQUEST_SOURCE": "", "VELA_PULL_REQUEST_TARGET": "foo", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_OWNER": "foo", "VELA_REPO_BRANCH": "foo", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, {"deployment", fields{ @@ -732,6 +755,7 @@ func Test_client_EnvironmentBuild(t *testing.T) { metadata: &internal.Metadata{Database: &internal.Database{Driver: str, Host: str}, Queue: &internal.Queue{Channel: str, Driver: str, Host: str}, Source: &internal.Source{Driver: str, Host: str}, Vela: &internal.Vela{Address: str, WebAddress: str, OpenIDIssuer: str}}, repo: &api.Repo{ID: &num64, Owner: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, Org: &str, Name: &str, FullName: &str, Link: &str, Clone: &str, Branch: &str, Topics: &topics, BuildLimit: &num64, Timeout: &num64, Visibility: &str, Private: &booL, Trusted: &booL, Active: &booL}, user: &api.User{ID: &num64, Name: &str, Token: &str, Active: &booL, Admin: &booL}, + netrc: &netrc, }, map[string]string{"BUILD_AUTHOR": "foo", "BUILD_AUTHOR_EMAIL": "", "BUILD_BASE_REF": "foo", "BUILD_BRANCH": "foo", "BUILD_CHANNEL": "foo", "BUILD_CLONE": "foo", "BUILD_COMMIT": "foo", "BUILD_CREATED": "1", "BUILD_ENQUEUED": "1", "BUILD_EVENT": "deployment", "BUILD_HOST": "", "BUILD_LINK": "", "BUILD_MESSAGE": "foo", "BUILD_NUMBER": "1", "BUILD_PARENT": "1", "BUILD_REF": "refs/pull/1/head", "BUILD_SENDER": "foo", "BUILD_SOURCE": "foo", "BUILD_STARTED": "1", "BUILD_STATUS": "foo", "BUILD_TARGET": "production", "BUILD_TITLE": "foo", "BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "CI": "true", "REPOSITORY_ACTIVE": "false", "REPOSITORY_ALLOW_EVENTS": "", "REPOSITORY_BRANCH": "foo", "REPOSITORY_CLONE": "foo", "REPOSITORY_FULL_NAME": "foo", "REPOSITORY_LINK": "foo", "REPOSITORY_NAME": "foo", "REPOSITORY_ORG": "foo", "REPOSITORY_PRIVATE": "false", "REPOSITORY_TIMEOUT": "1", "REPOSITORY_TRUSTED": "false", "REPOSITORY_VISIBILITY": "foo", "VELA": "true", "VELA_ADDR": "foo", "VELA_SERVER_ADDR": "foo", "VELA_OPEN_ID_ISSUER": "foo", "VELA_BUILD_APPROVED_AT": "0", "VELA_BUILD_APPROVED_BY": "", "VELA_BUILD_AUTHOR": "foo", "VELA_BUILD_AUTHOR_EMAIL": "", "VELA_BUILD_BASE_REF": "foo", "VELA_BUILD_BRANCH": "foo", "VELA_BUILD_CHANNEL": "foo", "VELA_BUILD_CLONE": "foo", "VELA_BUILD_COMMIT": "foo", "VELA_BUILD_CREATED": "1", "VELA_BUILD_DISTRIBUTION": "", "VELA_BUILD_ENQUEUED": "1", "VELA_BUILD_EVENT": "deployment", "VELA_BUILD_EVENT_ACTION": "", "VELA_BUILD_HOST": "", "VELA_BUILD_LINK": "", "VELA_BUILD_MESSAGE": "foo", "VELA_BUILD_NUMBER": "1", "VELA_BUILD_PARENT": "1", "VELA_BUILD_REF": "refs/pull/1/head", "VELA_BUILD_RUNTIME": "", "VELA_BUILD_SENDER": "foo", "VELA_BUILD_SENDER_SCM_ID": "foo", "VELA_BUILD_SOURCE": "foo", "VELA_BUILD_STARTED": "1", "VELA_BUILD_STATUS": "foo", "VELA_BUILD_TARGET": "production", "VELA_BUILD_TITLE": "foo", "VELA_BUILD_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_CHANNEL": "foo", "VELA_DATABASE": "foo", "VELA_DEPLOYMENT": "production", "VELA_DEPLOYMENT_NUMBER": "0", "VELA_DISTRIBUTION": "TODO", "VELA_HOST": "foo", "VELA_NETRC_MACHINE": "foo", "VELA_NETRC_PASSWORD": "foo", "VELA_NETRC_USERNAME": "x-oauth-basic", "VELA_QUEUE": "foo", "VELA_REPO_ACTIVE": "false", "VELA_REPO_ALLOW_EVENTS": "", "VELA_REPO_APPROVE_BUILD": "", "VELA_REPO_OWNER": "foo", "VELA_REPO_BRANCH": "foo", "VELA_REPO_BUILD_LIMIT": "1", "VELA_REPO_CLONE": "foo", "VELA_REPO_FULL_NAME": "foo", "VELA_REPO_LINK": "foo", "VELA_REPO_NAME": "foo", "VELA_REPO_ORG": "foo", "VELA_REPO_PIPELINE_TYPE": "", "VELA_REPO_PRIVATE": "false", "VELA_REPO_TIMEOUT": "1", "VELA_REPO_TOPICS": "cloud,security", "VELA_REPO_TRUSTED": "false", "VELA_REPO_VISIBILITY": "foo", "VELA_RUNTIME": "TODO", "VELA_SOURCE": "foo", "VELA_USER_ACTIVE": "false", "VELA_USER_ADMIN": "false", "VELA_USER_FAVORITES": "[]", "VELA_USER_NAME": "foo", "VELA_VERSION": "TODO", "VELA_WORKSPACE": "/vela/src/foo/foo/foo", "VELA_ID_TOKEN_REQUEST_URL": "foo/api/v1/repos/foo/builds/1/id_token"}, }, } @@ -742,6 +766,7 @@ func Test_client_EnvironmentBuild(t *testing.T) { metadata: tt.fields.metadata, repo: tt.fields.repo, user: tt.fields.user, + netrc: tt.fields.netrc, } got := c.EnvironmentBuild() if diff := cmp.Diff(got, tt.want); diff != "" { diff --git a/compiler/native/native.go b/compiler/native/native.go index 1b36e39fb..4a10aae2f 100644 --- a/compiler/native/native.go +++ b/compiler/native/native.go @@ -15,8 +15,10 @@ import ( "github.com/go-vela/server/compiler" "github.com/go-vela/server/compiler/registry" "github.com/go-vela/server/compiler/registry/github" + "github.com/go-vela/server/database" "github.com/go-vela/server/internal" "github.com/go-vela/server/internal/image" + "github.com/go-vela/server/scm" ) type ModificationConfig struct { @@ -27,9 +29,10 @@ type ModificationConfig struct { } type client struct { - Github registry.Service - PrivateGithub registry.Service - UsePrivateGithub bool + Github registry.Service + PrivateGithub registry.Service + UsePrivateGithub bool + ModificationService ModificationConfig TemplateCache map[string][]byte @@ -45,6 +48,9 @@ type client struct { repo *api.Repo user *api.User labels []string + db database.Interface + scm scm.Service + netrc *string } // FromCLIContext returns a Pipeline implementation that integrates with the supported registries. @@ -235,3 +241,24 @@ func (c *client) WithLabels(labels []string) compiler.Engine { return c } + +// WithNetrc sets the netrc in the Engine. +func (c *client) WithNetrc(n string) compiler.Engine { + c.netrc = &n + + return c +} + +// WithSCM sets the scm in the Engine. +func (c *client) WithSCM(_scm scm.Service) compiler.Engine { + c.scm = _scm + + return c +} + +// WithDatabase sets the database in the Engine. +func (c *client) WithDatabase(db database.Interface) compiler.Engine { + c.db = db + + return c +} diff --git a/compiler/native/script_test.go b/compiler/native/script_test.go index e12e6a4b8..410c297c0 100644 --- a/compiler/native/script_test.go +++ b/compiler/native/script_test.go @@ -19,7 +19,7 @@ func TestNative_ScriptStages(t *testing.T) { set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) - baseEnv := environment(nil, nil, nil, nil) + baseEnv := environment(nil, nil, nil, nil, nil) s := yaml.StageSlice{ &yaml.Stage{ @@ -109,7 +109,7 @@ func TestNative_ScriptSteps(t *testing.T) { set.String("clone-image", defaultCloneImage, "doc") c := cli.NewContext(nil, set, nil) - emptyEnv := environment(nil, nil, nil, nil) + emptyEnv := environment(nil, nil, nil, nil, nil) baseEnv := emptyEnv baseEnv["HOME"] = "/root" diff --git a/compiler/native/transform_test.go b/compiler/native/transform_test.go index 14da2b332..d2906bc8b 100644 --- a/compiler/native/transform_test.go +++ b/compiler/native/transform_test.go @@ -59,7 +59,7 @@ func TestNative_TransformStages(t *testing.T) { Steps: yaml.StepSlice{ &yaml.Step{ Commands: []string{"./gradlew downloadDependencies"}, - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "install", Pull: "always", @@ -72,7 +72,7 @@ func TestNative_TransformStages(t *testing.T) { Steps: yaml.StepSlice{ &yaml.Step{ Commands: []string{"./gradlew check"}, - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "test", Pull: "always", @@ -138,7 +138,7 @@ func TestNative_TransformStages(t *testing.T) { ID: "__0_install deps_install", Commands: []string{"./gradlew downloadDependencies"}, Directory: "/vela/src", - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "install", Number: 1, @@ -194,7 +194,7 @@ func TestNative_TransformStages(t *testing.T) { ID: "localOrg_localRepo_1_install deps_install", Commands: []string{"./gradlew downloadDependencies"}, Directory: "/vela/src", - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "install", Number: 1, @@ -297,14 +297,14 @@ func TestNative_TransformSteps(t *testing.T) { Steps: yaml.StepSlice{ &yaml.Step{ Commands: []string{"./gradlew downloadDependencies"}, - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "install deps", Pull: "always", }, &yaml.Step{ Commands: []string{"./gradlew check"}, - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "test", Pull: "always", @@ -365,7 +365,7 @@ func TestNative_TransformSteps(t *testing.T) { ID: "step___0_install deps", Commands: []string{"./gradlew downloadDependencies"}, Directory: "/vela/src", - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "install deps", Number: 1, @@ -416,7 +416,7 @@ func TestNative_TransformSteps(t *testing.T) { ID: "step_localOrg_localRepo_1_install deps", Commands: []string{"./gradlew downloadDependencies"}, Directory: "/vela/src", - Environment: environment(nil, nil, nil, nil), + Environment: environment(nil, nil, nil, nil, nil), Image: "openjdk:latest", Name: "install deps", Number: 1, diff --git a/compiler/registry/github/github.go b/compiler/registry/github/github.go index 09924393c..f5027a5b5 100644 --- a/compiler/registry/github/github.go +++ b/compiler/registry/github/github.go @@ -50,7 +50,7 @@ func New(ctx context.Context, address, token string) (*client, error) { if len(token) > 0 { // create GitHub OAuth client with user's token - gitClient = c.newClientToken(ctx, token) + gitClient = c.newOAuthTokenClient(ctx, token) } // overwrite the github client @@ -59,8 +59,8 @@ func New(ctx context.Context, address, token string) (*client, error) { return c, nil } -// newClientToken is a helper function to return the GitHub oauth2 client. -func (c *client) newClientToken(ctx context.Context, token string) *github.Client { +// newOAuthTokenClient is a helper function to return the GitHub oauth2 client. +func (c *client) newOAuthTokenClient(ctx context.Context, token string) *github.Client { // create the token object for the client ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, @@ -68,14 +68,6 @@ func (c *client) newClientToken(ctx context.Context, token string) *github.Clien // create the OAuth client tc := oauth2.NewClient(ctx, ts) - // if c.SkipVerify { - // tc.Transport.(*oauth2.Transport).Base = &http.Transport{ - // Proxy: http.ProxyFromEnvironment, - // TLSClientConfig: &tls.Config{ - // InsecureSkipVerify: true, - // }, - // } - // } // create the GitHub client from the OAuth client github := github.NewClient(tc) diff --git a/compiler/registry/github/template.go b/compiler/registry/github/template.go index 44bc0d1e8..5e1cd927b 100644 --- a/compiler/registry/github/template.go +++ b/compiler/registry/github/template.go @@ -19,7 +19,7 @@ func (c *client) Template(ctx context.Context, u *api.User, s *registry.Source) cli := c.Github if u != nil { // create GitHub OAuth client with user's token - cli = c.newClientToken(ctx, u.GetToken()) + cli = c.newOAuthTokenClient(ctx, u.GetToken()) } // create the options to pass diff --git a/compiler/types/pipeline/git.go b/compiler/types/pipeline/git.go new file mode 100644 index 000000000..a7628abf9 --- /dev/null +++ b/compiler/types/pipeline/git.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pipeline + +// Git is the pipeline representation of git configurations for a pipeline. +// +// swagger:model PipelineGit +type Git struct { + Token *Token `json:"token,omitempty" yaml:"token,omitempty"` +} + +// Token is the pipeline representation of git token access configurations for a pipeline. +// +// swagger:model PipelineGitToken +type Token struct { + Repositories []string `json:"repositories,omitempty" yaml:"repositories,omitempty"` + Permissions map[string]string `json:"permissions,omitempty" yaml:"permissions,omitempty"` +} + +// Empty returns true if the provided struct is empty. +func (g *Git) Empty() bool { + // return false if any of the fields are provided + if g.Token != nil { + if g.Token.Repositories != nil { + return false + } + + if g.Token.Permissions != nil { + return false + } + } + + // return true if all fields are empty + return true +} diff --git a/compiler/types/pipeline/git_test.go b/compiler/types/pipeline/git_test.go new file mode 100644 index 000000000..aa7b328e5 --- /dev/null +++ b/compiler/types/pipeline/git_test.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package pipeline + +import "testing" + +func TestPipeline_Git_Empty(t *testing.T) { + // setup tests + tests := []struct { + git *Git + want bool + }{ + { + git: &Git{&Token{Repositories: []string{}}}, + want: false, + }, + { + git: new(Git), + want: true, + }, + } + + // run tests + for _, test := range tests { + got := test.git.Empty() + + if got != test.want { + t.Errorf("Empty is %v, want %t", got, test.want) + } + } +} diff --git a/compiler/types/yaml/build.go b/compiler/types/yaml/build.go index c0b719c9e..8b9a25a44 100644 --- a/compiler/types/yaml/build.go +++ b/compiler/types/yaml/build.go @@ -19,6 +19,7 @@ type Build struct { Steps StepSlice `yaml:"steps,omitempty" json:"steps,omitempty" jsonschema:"oneof_required=steps,description=Provide sequential execution instructions.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/"` Templates TemplateSlice `yaml:"templates,omitempty" json:"templates,omitempty" jsonschema:"description=Provide the name of templates to expand.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/"` Deployment Deployment `yaml:"deployment,omitempty" json:"deployment,omitempty" jsonschema:"description=Provide deployment configuration.\nReference: https://go-vela.github.io/docs/reference/yaml/deployments/"` + Git Git `yaml:"git,omitempty" json:"git,omitempty" jsonschema:"description=Provide the git access specifications.\nReference: https://go-vela.github.io/docs/reference/yaml/git/"` } // ToPipelineAPI converts the Build type to an API Pipeline type. @@ -75,6 +76,7 @@ func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error { Steps StepSlice Templates TemplateSlice Deployment Deployment + Git Git }) // attempt to unmarshal as a build type @@ -89,6 +91,7 @@ func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error { } // override the values + b.Git = build.Git b.Version = build.Version b.Metadata = build.Metadata b.Environment = build.Environment diff --git a/compiler/types/yaml/git.go b/compiler/types/yaml/git.go new file mode 100644 index 000000000..7391714e5 --- /dev/null +++ b/compiler/types/yaml/git.go @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import "github.com/go-vela/server/compiler/types/pipeline" + +// Git is the yaml representation of git configurations for a pipeline. +type Git struct { + Token `yaml:"token,omitempty" json:"token,omitempty" jsonschema:"description=Provide the git token specifications, primarily used for cloning.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#token"` +} + +// Token is the yaml representation of the git token. +// Only applies when using GitHub App installations. +type Token struct { + Repositories []string `yaml:"repositories,omitempty" json:"repositories,omitempty" jsonschema:"description=Provide a list of repositories to clone.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#repositories"` + Permissions map[string]string `yaml:"permissions,omitempty" json:"permissions,omitempty" jsonschema:"description=Provide a list of repository permissions to apply to the git token.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#permissions"` +} + +// ToPipeline converts the Git type +// to a pipeline Git type. +func (g *Git) ToPipeline() *pipeline.Git { + return &pipeline.Git{ + Token: &pipeline.Token{ + Repositories: g.Repositories, + Permissions: g.Permissions, + }, + } +} diff --git a/compiler/types/yaml/git_test.go b/compiler/types/yaml/git_test.go new file mode 100644 index 000000000..d13d6be48 --- /dev/null +++ b/compiler/types/yaml/git_test.go @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "reflect" + "testing" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_Git_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + git *Git + want *pipeline.Git + }{ + { + git: &Git{ + Token: Token{ + Repositories: []string{"foo", "bar"}, + }, + }, + want: &pipeline.Git{ + Token: &pipeline.Token{ + Repositories: []string{"foo", "bar"}, + }, + }, + }, + { + git: &Git{ + Token: Token{ + Permissions: map[string]string{"foo": "bar"}, + }, + }, + want: &pipeline.Git{ + Token: &pipeline.Token{ + Permissions: map[string]string{"foo": "bar"}, + }, + }, + }, + { + git: &Git{ + Token: Token{}, + }, + want: &pipeline.Git{ + Token: &pipeline.Token{}, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.git.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} diff --git a/constants/app_install.go b/constants/app_install.go new file mode 100644 index 000000000..4e97cb067 --- /dev/null +++ b/constants/app_install.go @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 + +// App Install vars. +package constants + +const ( + // GitHub App install repositories selection when "all" repositories are selected. + AppInstallRepositoriesSelectionAll = "all" + // GitHub App install repositories selection when a subset of repositories are selected. + AppInstallRepositoriesSelectionSelected = "selected" +) + +const ( + // GitHub App install setup_action type 'install'. + AppInstallSetupActionInstall = "install" + // GitHub App install event type 'created'. + AppInstallCreated = "created" + // GitHub App install event type 'deleted'. + AppInstallDeleted = "deleted" +) diff --git a/constants/event.go b/constants/event.go index c2ec26aad..a2ab76e3a 100644 --- a/constants/event.go +++ b/constants/event.go @@ -28,6 +28,12 @@ const ( // EventTag defines the event type for build and repo tag events. EventTag = "tag" + // EventInstallation defines the event type for scm installation events. + EventInstallation = "installation" + + // EventInstallationRepositories defines the event type for scm installation_repositories events. + EventInstallationRepositories = "installation_repositories" + // Alternates for common user inputs that do not match our set constants. // EventPullAlternate defines the alternate event type for build and repo pull_request events. diff --git a/database/integration_test.go b/database/integration_test.go index ce9a3ccfe..9d6258fce 100644 --- a/database/integration_test.go +++ b/database/integration_test.go @@ -2491,6 +2491,7 @@ func newResources() *Resources { repoOne.SetPreviousName("") repoOne.SetApproveBuild(constants.ApproveNever) repoOne.SetAllowEvents(api.NewEventsFromMask(1)) + repoOne.SetInstallID(0) repoTwo := new(api.Repo) repoTwo.SetID(2) @@ -2514,6 +2515,7 @@ func newResources() *Resources { repoTwo.SetPreviousName("") repoTwo.SetApproveBuild(constants.ApproveForkAlways) repoTwo.SetAllowEvents(api.NewEventsFromMask(1)) + repoTwo.SetInstallID(0) buildOne := new(api.Build) buildOne.SetID(1) diff --git a/database/repo/create_test.go b/database/repo/create_test.go index 07b8204ab..939fc186b 100644 --- a/database/repo/create_test.go +++ b/database/repo/create_test.go @@ -34,9 +34,9 @@ func TestRepo_Engine_CreateRepo(t *testing.T) { // ensure the mock expects the query _mock.ExpectQuery(`INSERT INTO "repos" -("user_id","hash","org","name","full_name","link","clone","branch","topics","build_limit","timeout","counter","visibility","private","trusted","active","allow_events","pipeline_type","previous_name","approve_build","id") -VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21) RETURNING "id"`). - WithArgs(1, AnyArgument{}, "foo", "bar", "foo/bar", nil, nil, nil, AnyArgument{}, AnyArgument{}, AnyArgument{}, AnyArgument{}, "public", false, false, false, nil, "yaml", "oldName", nil, 1). +("user_id","hash","org","name","full_name","link","clone","branch","topics","build_limit","timeout","counter","visibility","private","trusted","active","allow_events","pipeline_type","previous_name","approve_build","install_id","id") +VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22) RETURNING "id"`). + WithArgs(1, AnyArgument{}, "foo", "bar", "foo/bar", nil, nil, nil, AnyArgument{}, AnyArgument{}, AnyArgument{}, AnyArgument{}, "public", false, false, false, nil, "yaml", "oldName", nil, 0, 1). WillReturnRows(_rows) _sqlite := testSqlite(t) diff --git a/database/repo/table.go b/database/repo/table.go index f49c26933..daba7a367 100644 --- a/database/repo/table.go +++ b/database/repo/table.go @@ -35,6 +35,7 @@ repos ( pipeline_type TEXT, previous_name VARCHAR(100), approve_build VARCHAR(20), + install_id INTEGER, UNIQUE(full_name) ); ` @@ -65,6 +66,7 @@ repos ( pipeline_type TEXT, previous_name TEXT, approve_build TEXT, + install_id INTEGER, UNIQUE(full_name) ); ` diff --git a/database/repo/update_test.go b/database/repo/update_test.go index a447d6973..dab936f9b 100644 --- a/database/repo/update_test.go +++ b/database/repo/update_test.go @@ -35,9 +35,9 @@ func TestRepo_Engine_UpdateRepo(t *testing.T) { // ensure the mock expects the query _mock.ExpectExec(`UPDATE "repos" -SET "user_id"=$1,"hash"=$2,"org"=$3,"name"=$4,"full_name"=$5,"link"=$6,"clone"=$7,"branch"=$8,"topics"=$9,"build_limit"=$10,"timeout"=$11,"counter"=$12,"visibility"=$13,"private"=$14,"trusted"=$15,"active"=$16,"allow_events"=$17,"pipeline_type"=$18,"previous_name"=$19,"approve_build"=$20 -WHERE "id" = $21`). - WithArgs(1, AnyArgument{}, "foo", "bar", "foo/bar", nil, nil, nil, AnyArgument{}, AnyArgument{}, AnyArgument{}, AnyArgument{}, "public", false, false, false, 1, "yaml", "oldName", constants.ApproveForkAlways, 1). +SET "user_id"=$1,"hash"=$2,"org"=$3,"name"=$4,"full_name"=$5,"link"=$6,"clone"=$7,"branch"=$8,"topics"=$9,"build_limit"=$10,"timeout"=$11,"counter"=$12,"visibility"=$13,"private"=$14,"trusted"=$15,"active"=$16,"allow_events"=$17,"pipeline_type"=$18,"previous_name"=$19,"approve_build"=$20,"install_id"=$21 +WHERE "id" = $22`). + WithArgs(1, AnyArgument{}, "foo", "bar", "foo/bar", nil, nil, nil, AnyArgument{}, AnyArgument{}, AnyArgument{}, AnyArgument{}, "public", false, false, false, 1, "yaml", "oldName", constants.ApproveForkAlways, 0, 1). WillReturnResult(sqlmock.NewResult(1, 1)) _sqlite := testSqlite(t) diff --git a/database/testutils/api_resources.go b/database/testutils/api_resources.go index 70cc45d65..738602534 100644 --- a/database/testutils/api_resources.go +++ b/database/testutils/api_resources.go @@ -127,6 +127,7 @@ func APIRepo() *api.Repo { AllowEvents: APIEvents(), Topics: new([]string), ApproveBuild: new(string), + InstallID: new(int64), } } diff --git a/database/types/repo.go b/database/types/repo.go index 96f45caf9..f02949f2e 100644 --- a/database/types/repo.go +++ b/database/types/repo.go @@ -67,6 +67,7 @@ type Repo struct { PipelineType sql.NullString `sql:"pipeline_type"` PreviousName sql.NullString `sql:"previous_name"` ApproveBuild sql.NullString `sql:"approve_build"` + InstallID sql.NullInt64 `sql:"install_id"` Owner User `gorm:"foreignKey:UserID"` } @@ -250,6 +251,7 @@ func (r *Repo) ToAPI() *api.Repo { repo.SetPipelineType(r.PipelineType.String) repo.SetPreviousName(r.PreviousName.String) repo.SetApproveBuild(r.ApproveBuild.String) + repo.SetInstallID(r.InstallID.Int64) return repo } @@ -342,6 +344,7 @@ func RepoFromAPI(r *api.Repo) *Repo { PipelineType: sql.NullString{String: r.GetPipelineType(), Valid: true}, PreviousName: sql.NullString{String: r.GetPreviousName(), Valid: true}, ApproveBuild: sql.NullString{String: r.GetApproveBuild(), Valid: true}, + InstallID: sql.NullInt64{Int64: r.GetInstallID(), Valid: true}, } return repo.Nullify() diff --git a/database/types/repo_test.go b/database/types/repo_test.go index c9a8b6afd..1027e5fc2 100644 --- a/database/types/repo_test.go +++ b/database/types/repo_test.go @@ -193,6 +193,7 @@ func TestTypes_Repo_ToAPI(t *testing.T) { want.SetPipelineType("yaml") want.SetPreviousName("oldName") want.SetApproveBuild(constants.ApproveNever) + want.SetInstallID(0) // run test got := testRepo().ToAPI() @@ -345,6 +346,7 @@ func TestTypes_RepoFromAPI(t *testing.T) { r.SetPipelineType("yaml") r.SetPreviousName("oldName") r.SetApproveBuild(constants.ApproveNever) + r.SetInstallID(0) want := testRepo() want.Owner = User{} @@ -382,6 +384,7 @@ func testRepo() *Repo { PipelineType: sql.NullString{String: "yaml", Valid: true}, PreviousName: sql.NullString{String: "oldName", Valid: true}, ApproveBuild: sql.NullString{String: constants.ApproveNever, Valid: true}, + InstallID: sql.NullInt64{Int64: 0, Valid: true}, Owner: *testUser(), } diff --git a/database/types/schedule_test.go b/database/types/schedule_test.go index 10b4da233..2937c6c0a 100644 --- a/database/types/schedule_test.go +++ b/database/types/schedule_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/adhocore/gronx" + "github.com/google/go-cmp/cmp" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/constants" @@ -116,8 +117,8 @@ func TestTypes_Schedule_ToAPI(t *testing.T) { // run test got := testSchedule().ToAPI() - if !reflect.DeepEqual(got, want) { - t.Errorf("ToAPI is %v, want %v", got, want) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ScheduleToAPI() mismatch (-want +got):\n%s", diff) } } diff --git a/internal/webhook.go b/internal/webhook.go index 6020d2ab1..7baab3701 100644 --- a/internal/webhook.go +++ b/internal/webhook.go @@ -21,15 +21,27 @@ type PullRequest struct { Labels []string } +// Installation defines the data pulled from an installation +// while processing a webhook. +// Only applies to GitHub Apps. +type Installation struct { + Action string + ID int64 + Org string + RepositoriesAdded []string + RepositoriesRemoved []string +} + // Webhook defines a struct that is used to return // the required data when processing webhook event // a for a source provider event. type Webhook struct { - Hook *api.Hook - Repo *api.Repo - Build *api.Build - PullRequest PullRequest - Deployment *api.Deployment + Hook *api.Hook + Repo *api.Repo + Build *api.Build + PullRequest PullRequest + Deployment *api.Deployment + Installation *Installation } // ShouldSkip uses the build information diff --git a/mock/server/repo.go b/mock/server/repo.go index 9b156669e..1db1fc733 100644 --- a/mock/server/repo.go +++ b/mock/server/repo.go @@ -59,8 +59,9 @@ const ( } }, "approve_build": "fork-always", - "previous_name": "" -}` + "previous_name": "", + "install_id": 0 + }` // ReposResp represents a JSON return for one to many repos. ReposResp = `[ @@ -78,7 +79,8 @@ const ( "visibility": "public", "private": false, "trusted": true, - "active": true + "active": true, + "install_id": 0 }, { "id": 2, @@ -94,7 +96,8 @@ const ( "visibility": "public", "private": false, "trusted": true, - "active": true + "active": true, + "install_id": 0 } ]` ) diff --git a/router/middleware/build/build_test.go b/router/middleware/build/build_test.go index 8db7c8000..42288f017 100644 --- a/router/middleware/build/build_test.go +++ b/router/middleware/build/build_test.go @@ -53,6 +53,7 @@ func TestBuild_Establish(t *testing.T) { r.SetName("bar") r.SetFullName("foo/bar") r.SetVisibility("public") + r.SetInstallID(0) want := new(api.Build) want.SetID(1) diff --git a/router/middleware/repo/repo_test.go b/router/middleware/repo/repo_test.go index c20067c62..a331b5ae8 100644 --- a/router/middleware/repo/repo_test.go +++ b/router/middleware/repo/repo_test.go @@ -66,6 +66,7 @@ func TestRepo_Establish(t *testing.T) { want.SetPipelineType("yaml") want.SetPreviousName("") want.SetApproveBuild("") + want.SetInstallID(0) got := new(api.Repo) diff --git a/scm/flags.go b/scm/flags.go index 0d95ebcb5..ff83d24f6 100644 --- a/scm/flags.go +++ b/scm/flags.go @@ -67,4 +67,29 @@ var Flags = []cli.Flag{ "is behind a Firewall or NAT, or when using something like ngrok to forward webhooks. " + "(defaults to VELA_ADDR).", }, + &cli.Int64Flag{ + EnvVars: []string{"VELA_SCM_APP_ID", "SCM_APP_ID"}, + FilePath: "/vela/scm/app_id", + Name: "scm.app.id", + Usage: "set ID for the SCM App integration (GitHub App)", + }, + &cli.StringFlag{ + EnvVars: []string{"VELA_SCM_APP_PRIVATE_KEY", "SCM_APP_PRIVATE_KEY"}, + FilePath: "/vela/scm/app_private_key", + Name: "scm.app.private-key", + Usage: "set value of base64 encoded SCM App integration (GitHub App) private key", + }, + &cli.StringFlag{ + EnvVars: []string{"VELA_SCM_APP_PRIVATE_KEY_PATH", "SCM_APP_PRIVATE_KEY_PATH"}, + FilePath: "/vela/scm/app_private_key_path", + Name: "scm.app.private-key.path", + Usage: "set filepath to the SCM App integration (GitHub App) private key", + }, + &cli.StringSliceFlag{ + EnvVars: []string{"VELA_SCM_APP_PERMISSIONS", "SCM_APP_PERMISSIONS", "VELA_SOURCE_APP_PERMISSIONS", "SOURCE_APP_PERMISSIONS"}, + FilePath: "/vela/scm/app/permissions", + Name: "scm.app.permissions", + Usage: "SCM App integration (GitHub App) permissions to be used as the allowed set of possible installation token permissions", + Value: cli.NewStringSlice("contents:read", "checks:write"), + }, } diff --git a/scm/github/access.go b/scm/github/access.go index a1e7f5d4d..1bd4dd2e3 100644 --- a/scm/github/access.go +++ b/scm/github/access.go @@ -31,7 +31,7 @@ func (c *client) OrgAccess(ctx context.Context, u *api.User, org string) (string } // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // send API call to capture org access level for user membership, _, err := client.Organizations.GetOrgMembership(ctx, *u.Name, org) @@ -67,7 +67,7 @@ func (c *client) RepoAccess(ctx context.Context, name, token, org, repo string) } // create github oauth client with the given token - client := c.newClientToken(ctx, token) + client := c.newOAuthTokenClient(ctx, token) // send API call to capture repo access level for user perm, _, err := client.Repositories.GetPermissionLevel(ctx, org, repo, name) @@ -98,7 +98,7 @@ func (c *client) TeamAccess(ctx context.Context, u *api.User, org, team string) } // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) teams := []*github.Team{} // set the max per page for the options to capture the list of repos @@ -148,7 +148,7 @@ func (c *client) ListUsersTeamsForOrg(ctx context.Context, u *api.User, org stri }).Tracef("capturing %s team membership for org %s", u.GetName(), org) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) teams := []*github.Team{} // set the max per page for the options to capture the list of repos @@ -193,7 +193,7 @@ func (c *client) RepoContributor(ctx context.Context, owner *api.User, sender, o }).Tracef("capturing %s contributor status for repo %s/%s", sender, org, repo) // create GitHub OAuth client with repo owner's token - client := c.newClientToken(ctx, owner.GetToken()) + client := c.newOAuthTokenClient(ctx, owner.GetToken()) // set the max per page for the options to capture the list of repos opts := github.ListContributorsOptions{ diff --git a/scm/github/app_install.go b/scm/github/app_install.go new file mode 100644 index 000000000..16c03924c --- /dev/null +++ b/scm/github/app_install.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/sirupsen/logrus" + "gorm.io/gorm" + + "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" + "github.com/go-vela/server/database" + "github.com/go-vela/server/internal" +) + +// ProcessInstallation takes a GitHub installation and processes the changes. +func (c *client) ProcessInstallation(ctx context.Context, _ *http.Request, webhook *internal.Webhook, db database.Interface) error { + c.Logger.Tracef("processing GitHub App installation") + + errs := []string{} + + // set install_id for repos added to the installation + for _, repo := range webhook.Installation.RepositoriesAdded { + r, err := db.GetRepoForOrg(ctx, webhook.Installation.Org, repo) + if err != nil { + if !errors.Is(err, gorm.ErrRecordNotFound) { + errs = append(errs, fmt.Sprintf("%s:%s", repo, err.Error())) + } + + // skip repos that dont exist in vela + continue + } + + err = updateRepoInstallationID(ctx, webhook, r, db, webhook.Installation.ID) + if err != nil { + errs = append(errs, fmt.Sprintf("%s:%s", repo, err.Error())) + } + } + + // set install_id for repos removed from the installation + for _, repo := range webhook.Installation.RepositoriesRemoved { + r, err := db.GetRepoForOrg(ctx, webhook.Installation.Org, repo) + if err != nil { + if !errors.Is(err, gorm.ErrRecordNotFound) { + errs = append(errs, fmt.Sprintf("%s:%s", repo, err.Error())) + } + + // skip repos that dont exist in vela + continue + } + + err = updateRepoInstallationID(ctx, webhook, r, db, 0) + if err != nil { + errs = append(errs, fmt.Sprintf("%s:%s", repo, err.Error())) + } + } + + // combine all errors + if len(errs) > 0 { + return errors.New(strings.Join(errs, ", ")) + } + + return nil +} + +// updateRepoInstallationID updates the installation ID for a repo. +func updateRepoInstallationID(ctx context.Context, webhook *internal.Webhook, r *types.Repo, db database.Interface, installID int64) error { + r.SetInstallID(installID) + + h := new(types.Hook) + h.SetNumber(webhook.Hook.GetNumber()) + h.SetSourceID(webhook.Hook.GetSourceID()) + h.SetWebhookID(webhook.Hook.GetWebhookID()) + h.SetCreated(webhook.Hook.GetCreated()) + h.SetHost(webhook.Hook.GetHost()) + h.SetEvent(constants.EventInstallation) + h.SetStatus(webhook.Hook.GetStatus()) + + r, err := db.UpdateRepo(ctx, r) + if err != nil { + h.SetStatus(constants.StatusFailure) + h.SetError(err.Error()) + } + + h.Repo = r + + // number of times to retry + retryLimit := 3 + // implement a loop to process asynchronous operations with a retry limit + // + // Some operations taken during the webhook workflow can lead to race conditions + // failing to successfully process the request. This logic ensures we attempt our + // best efforts to handle these cases gracefully. + for i := 0; i < retryLimit; i++ { + // check if we're on the first iteration of the loop + if i > 0 { + // incrementally sleep in between retries + time.Sleep(time.Duration(i) * time.Second) + } + + // send API call to capture the last hook for the repo + lastHook, err := db.LastHookForRepo(ctx, r) + if err != nil { + // log the error for traceability + logrus.Error(err.Error()) + + // check if the retry limit has been exceeded + if i < retryLimit { + // continue to the next iteration of the loop + continue + } + + return err + } + + // set the Number field + if lastHook != nil { + h.SetNumber( + lastHook.GetNumber() + 1, + ) + } + + // send hook update to db + _, err = db.CreateHook(ctx, h) + if err != nil { + return err + } + + break + } + + return nil +} + +// FinishInstallation completes the web flow for a GitHub App installation, returning a redirect to the app installation page. +func (c *client) FinishInstallation(ctx context.Context, _ *http.Request, installID int64) (string, error) { + c.Logger.Tracef("finishing GitHub App installation for ID %d", installID) + + client, err := c.newGithubAppClient() + if err != nil { + return "", err + } + + install, _, err := client.Apps.GetInstallation(ctx, installID) + if err != nil { + return "", err + } + + return install.GetHTMLURL(), nil +} diff --git a/scm/github/app_permissions.go b/scm/github/app_permissions.go new file mode 100644 index 000000000..411ee3d4a --- /dev/null +++ b/scm/github/app_permissions.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "fmt" + "strings" + + "github.com/google/go-github/v65/github" +) + +// see: https://docs.github.com/en/rest/authentication/permissions-required-for-github-apps?apiVersion=2022-11-28 +const ( + // GitHub App install permission 'none'. + AppInstallPermissionNone = "none" + // GitHub App install permission 'read'. + AppInstallPermissionRead = "read" + // GitHub App install permission 'write'. + AppInstallPermissionWrite = "write" +) + +const ( + // GitHub App install contents resource. + AppInstallResourceContents = "contents" + // GitHub App install checks resource. + AppInstallResourceChecks = "checks" + // GitHub App install packages resource. + AppInstallResourcePackages = "packages" + // add more supported resources as needed. +) + +// GetInstallationPermission takes permissions and returns the permission level if valid. +func GetInstallationPermission(resource string, appPermissions *github.InstallationPermissions) (string, error) { + switch resource { + case AppInstallResourceContents: + return appPermissions.GetContents(), nil + case AppInstallResourceChecks: + return appPermissions.GetChecks(), nil + case AppInstallResourcePackages: + return appPermissions.GetPackages(), nil + // add more supported resources as needed. + default: + return "", fmt.Errorf("given permission resource not supported: %s", resource) + } +} + +// ApplyInstallationPermissions takes permissions and applies a new permission if valid. +func ApplyInstallationPermissions(resource, perm string, perms *github.InstallationPermissions) (*github.InstallationPermissions, error) { + // convert permissions from string + switch strings.ToLower(perm) { + case AppInstallPermissionNone: + case AppInstallPermissionRead: + case AppInstallPermissionWrite: + break + default: + return perms, fmt.Errorf("invalid permission level given for : in %s:%s", resource, perm) + } + + // convert resource from string + switch strings.ToLower(resource) { + case AppInstallResourceContents: + perms.Contents = github.String(perm) + case AppInstallResourceChecks: + perms.Checks = github.String(perm) + case AppInstallResourcePackages: + perms.Packages = github.String(perm) + // add more supported resources as needed. + default: + return perms, fmt.Errorf("invalid permission resource given for : in %s:%s", resource, perm) + } + + return perms, nil +} + +// InstallationHasPermission takes a resource:perm pair and checks if the actual permission matches the expected permission or is supersceded by a higher permission. +func InstallationHasPermission(resource, requiredPerm, actualPerm string) error { + if len(actualPerm) == 0 { + return fmt.Errorf("github app missing permission %s:%s", resource, requiredPerm) + } + + permitted := false + + switch requiredPerm { + case AppInstallPermissionNone: + permitted = true + case AppInstallPermissionRead: + if actualPerm == AppInstallPermissionRead || + actualPerm == AppInstallPermissionWrite { + permitted = true + } + case AppInstallPermissionWrite: + if actualPerm == AppInstallPermissionWrite { + permitted = true + } + default: + return fmt.Errorf("invalid required permission type: %s", requiredPerm) + } + + if !permitted { + return fmt.Errorf("github app requires permission %s:%s, found: %s", AppInstallResourceContents, AppInstallPermissionRead, actualPerm) + } + + return nil +} diff --git a/scm/github/app_permissions_test.go b/scm/github/app_permissions_test.go new file mode 100644 index 000000000..74b94a071 --- /dev/null +++ b/scm/github/app_permissions_test.go @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "testing" + + "github.com/google/go-github/v65/github" +) + +func TestGetInstallationPermission(t *testing.T) { + tests := []struct { + name string + resource string + permissions *github.InstallationPermissions + expectedPerm string + expectedError bool + }{ + { + name: "valid contents permission", + resource: AppInstallResourceContents, + permissions: &github.InstallationPermissions{Contents: github.String(AppInstallPermissionRead)}, + expectedPerm: AppInstallPermissionRead, + }, + { + name: "valid checks permission", + resource: AppInstallResourceChecks, + permissions: &github.InstallationPermissions{Checks: github.String(AppInstallPermissionWrite)}, + expectedPerm: AppInstallPermissionWrite, + }, + { + name: "valid packages permission", + resource: AppInstallResourcePackages, + permissions: &github.InstallationPermissions{Packages: github.String(AppInstallPermissionNone)}, + expectedPerm: AppInstallPermissionNone, + }, + { + name: "invalid resource", + resource: "invalid_resource", + permissions: &github.InstallationPermissions{}, + expectedError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + perm, err := GetInstallationPermission(tt.resource, tt.permissions) + if (err != nil) != tt.expectedError { + t.Errorf("GetInstallationPermission() error = %v, expectedError %v", err, tt.expectedError) + return + } + if perm != tt.expectedPerm { + t.Errorf("GetInstallationPermission() = %v, expected %v", perm, tt.expectedPerm) + } + }) + } +} + +func TestApplyInstallationPermissions(t *testing.T) { + tests := []struct { + name string + resource string + perm string + initialPerms *github.InstallationPermissions + expectedPerms *github.InstallationPermissions + expectedError bool + }{ + { + name: "apply read permission to contents", + resource: AppInstallResourceContents, + perm: AppInstallPermissionRead, + initialPerms: &github.InstallationPermissions{}, + expectedPerms: &github.InstallationPermissions{ + Contents: github.String(AppInstallPermissionRead), + }, + }, + { + name: "apply write permission to checks", + resource: AppInstallResourceChecks, + perm: AppInstallPermissionWrite, + initialPerms: &github.InstallationPermissions{}, + expectedPerms: &github.InstallationPermissions{ + Checks: github.String(AppInstallPermissionWrite), + }, + }, + { + name: "apply none permission to packages", + resource: AppInstallResourcePackages, + perm: AppInstallPermissionNone, + initialPerms: &github.InstallationPermissions{}, + expectedPerms: &github.InstallationPermissions{ + Packages: github.String(AppInstallPermissionNone), + }, + }, + { + name: "invalid permission level", + resource: AppInstallResourceContents, + perm: "invalid_perm", + initialPerms: &github.InstallationPermissions{}, + expectedError: true, + }, + { + name: "invalid resource", + resource: "invalid_resource", + perm: AppInstallPermissionRead, + initialPerms: &github.InstallationPermissions{}, + expectedError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + perms, err := ApplyInstallationPermissions(tt.resource, tt.perm, tt.initialPerms) + if (err != nil) != tt.expectedError { + t.Errorf("ApplyInstallationPermissions() error = %v, expectedError %v", err, tt.expectedError) + return + } + if !tt.expectedError && !comparePermissions(perms, tt.expectedPerms) { + t.Errorf("ApplyInstallationPermissions() = %v, expected %v", perms, tt.expectedPerms) + } + }) + } +} + +func TestInstallationHasPermission(t *testing.T) { + tests := []struct { + name string + resource string + requiredPerm string + actualPerm string + expectedError bool + }{ + { + name: "valid read permission", + resource: AppInstallResourceContents, + requiredPerm: AppInstallPermissionRead, + actualPerm: AppInstallPermissionRead, + }, + { + name: "valid write permission", + resource: AppInstallResourceChecks, + requiredPerm: AppInstallPermissionWrite, + actualPerm: AppInstallPermissionWrite, + }, + { + name: "valid none permission", + resource: AppInstallResourcePackages, + requiredPerm: AppInstallPermissionNone, + actualPerm: AppInstallPermissionNone, + }, + { + name: "read permission superseded by write", + resource: AppInstallResourceContents, + requiredPerm: AppInstallPermissionRead, + actualPerm: AppInstallPermissionWrite, + }, + { + name: "missing permission", + resource: AppInstallResourceChecks, + requiredPerm: AppInstallPermissionWrite, + actualPerm: "", + expectedError: true, + }, + { + name: "invalid required permission", + resource: AppInstallResourcePackages, + requiredPerm: "invalid_perm", + actualPerm: AppInstallPermissionRead, + expectedError: true, + }, + { + name: "insufficient permission", + resource: AppInstallResourceContents, + requiredPerm: AppInstallPermissionWrite, + actualPerm: AppInstallPermissionRead, + expectedError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := InstallationHasPermission(tt.resource, tt.requiredPerm, tt.actualPerm) + if (err != nil) != tt.expectedError { + t.Errorf("InstallationHasPermission() error = %v, expectedError %v", err, tt.expectedError) + } + }) + } +} + +func comparePermissions(a, b *github.InstallationPermissions) bool { + if a == nil || b == nil { + return a == b + } + return github.Stringify(a) == github.Stringify(b) +} diff --git a/scm/github/app_transport.go b/scm/github/app_transport.go new file mode 100644 index 000000000..b0db1f5f8 --- /dev/null +++ b/scm/github/app_transport.go @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptrace" + "strconv" + "strings" + "sync" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/google/go-github/v65/github" + "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" +) + +const ( + acceptHeader = "application/vnd.github.v3+json" +) + +// AppsTransport provides a http.RoundTripper by wrapping an existing +// http.RoundTripper and provides GitHub Apps authentication as a GitHub App. +// +// Client can also be overwritten, and is useful to change to one which +// provides retry logic if you do experience retryable errors. +// +// See https://developer.github.com/apps/building-integrations/setting-up-and-registering-github-apps/about-authentication-options-for-github-apps/ +type AppsTransport struct { + BaseURL string // BaseURL is the scheme and host for GitHub API, defaults to https://api.github.com + Client Client // Client to use to refresh tokens, defaults to http.Client with provided transport + tr http.RoundTripper // tr is the underlying roundtripper being wrapped + signer Signer // signer signs JWT tokens. + appID int64 // appID is the GitHub App's ID +} + +// newGitHubAppTransport creates a new GitHub App transport for authenticating as the GitHub App. +func (c *client) newGitHubAppTransport(appID int64, baseURL string, privateKey *rsa.PrivateKey) *AppsTransport { + transport := c.newAppsTransportFromPrivateKey(http.DefaultTransport, appID, privateKey) + transport.BaseURL = baseURL + + // apply tracing to the transport + if c.Tracing.Config.EnableTracing { + transport.tr = otelhttp.NewTransport( + transport.tr, + otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { + return otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans()) + }), + ) + } + + return transport +} + +// newAppsTransportFromPrivateKey returns an AppsTransport using a crypto/rsa.(*PrivateKey). +func (c *client) newAppsTransportFromPrivateKey(tr http.RoundTripper, appID int64, key *rsa.PrivateKey) *AppsTransport { + return &AppsTransport{ + BaseURL: defaultAPI, + Client: &http.Client{Transport: tr}, + tr: tr, + signer: NewRSASigner(jwt.SigningMethodRS256, key), + appID: appID, + } +} + +// RoundTrip implements http.RoundTripper interface. +func (t *AppsTransport) RoundTrip(req *http.Request) (*http.Response, error) { + // GitHub rejects expiry and issue timestamps that are not an integer, + // while the jwt-go library serializes to fractional timestamps + // then truncate them before passing to jwt-go. + iss := time.Now().Add(-30 * time.Second).Truncate(time.Second) + exp := iss.Add(2 * time.Minute) + claims := &jwt.RegisteredClaims{ + IssuedAt: jwt.NewNumericDate(iss), + ExpiresAt: jwt.NewNumericDate(exp), + Issuer: strconv.FormatInt(t.appID, 10), + } + + ss, err := t.signer.Sign(claims) + if err != nil { + return nil, fmt.Errorf("could not sign jwt: %w", err) + } + + req.Header.Set("Authorization", "Bearer "+ss) + req.Header.Add("Accept", acceptHeader) + + return t.tr.RoundTrip(req) +} + +// Transport provides a http.RoundTripper by wrapping an existing +// http.RoundTripper and provides GitHub Apps authentication as an installation. +// +// Client can also be overwritten, and is useful to change to one which +// provides retry logic if you do experience retryable errors. +// +// See https://developer.github.com/apps/building-integrations/setting-up-and-registering-github-apps/about-authentication-options-for-github-apps/ +type Transport struct { + BaseURL string // BaseURL is the scheme and host for GitHub API, defaults to https://api.github.com + Client Client // Client to use to refresh tokens, defaults to http.Client with provided transport + tr http.RoundTripper // tr is the underlying roundtripper being wrapped + installationID int64 // installationID is the GitHub App Installation ID + InstallationTokenOptions *github.InstallationTokenOptions // parameters restrict a token's access + appsTransport *AppsTransport + + mu *sync.Mutex + token *accessToken // the installation's access token +} + +// accessToken is an installation access token response from GitHub. +type accessToken struct { + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` + Permissions github.InstallationPermissions `json:"permissions,omitempty"` + Repositories []github.Repository `json:"repositories,omitempty"` +} + +var _ http.RoundTripper = &Transport{} + +// Client is a HTTP client which sends a http.Request and returns a http.Response +// or an error. +type Client interface { + Do(*http.Request) (*http.Response, error) +} + +// RoundTrip implements http.RoundTripper interface. +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + reqBodyClosed := false + + if req.Body != nil { + defer func() { + if !reqBodyClosed { + req.Body.Close() + } + }() + } + + token, err := t.Token(req.Context()) + if err != nil { + return nil, err + } + + creq := cloneRequest(req) + creq.Header.Set("Authorization", "token "+token) + + if creq.Header.Get("Accept") == "" { + creq.Header.Add("Accept", acceptHeader) + } + + reqBodyClosed = true + + return t.tr.RoundTrip(creq) +} + +// getRefreshTime returns the time when the token should be refreshed. +func (at *accessToken) getRefreshTime() time.Time { + return at.ExpiresAt.Add(-time.Minute) +} + +// isExpired checks if the access token is expired. +func (at *accessToken) isExpired() bool { + return at == nil || at.getRefreshTime().Before(time.Now()) +} + +// Token checks the active token expiration and renews if necessary. Token returns +// a valid access token. If renewal fails an error is returned. +func (t *Transport) Token(ctx context.Context) (string, error) { + t.mu.Lock() + + defer t.mu.Unlock() + + if t.token.isExpired() { + // token is not set or expired/nearly expired, so refresh + if err := t.refreshToken(ctx); err != nil { + return "", fmt.Errorf("could not refresh installation id %v's token: %w", t.installationID, err) + } + } + + return t.token.Token, nil +} + +// Expiry returns a transport token's expiration time and refresh time. There is a small grace period +// built in where a token will be refreshed before it expires. expiresAt is the actual token expiry, +// and refreshAt is when a call to Token() will cause it to be refreshed. +func (t *Transport) Expiry() (expiresAt time.Time, refreshAt time.Time, err error) { + if t.token == nil { + return time.Time{}, time.Time{}, errors.New("Expiry() = unknown, err: nil token") + } + + return t.token.ExpiresAt, t.token.getRefreshTime(), nil +} + +func (t *Transport) refreshToken(ctx context.Context) error { + // convert InstallationTokenOptions into a ReadWriter to pass as an argument to http.NewRequest + body, err := GetReadWriter(t.InstallationTokenOptions) + if err != nil { + return fmt.Errorf("could not convert installation token parameters into json: %w", err) + } + + requestURL := fmt.Sprintf("%s/app/installations/%v/access_tokens", strings.TrimRight(t.BaseURL, "/"), t.installationID) + + req, err := http.NewRequest("POST", requestURL, body) + if err != nil { + return fmt.Errorf("could not create request: %w", err) + } + + // set Content and Accept headers + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + req.Header.Set("Accept", acceptHeader) + + if ctx != nil { + req = req.WithContext(ctx) + } + + t.appsTransport.BaseURL = t.BaseURL + t.appsTransport.Client = t.Client + + resp, err := t.appsTransport.RoundTrip(req) + if err != nil { + return fmt.Errorf("could not get access_tokens from GitHub API for installation ID %v: %w", t.installationID, err) + } + + if resp.StatusCode/100 != 2 { + return fmt.Errorf("received non 2xx response status %q when fetching %v", resp.Status, req.URL) + } + + // closing body late, to provide caller a chance to inspect body in an error / non-200 response status situation + defer resp.Body.Close() + + return json.NewDecoder(resp.Body).Decode(&t.token) +} + +// GetReadWriter converts a body interface into an io.ReadWriter object. +func GetReadWriter(i interface{}) (io.ReadWriter, error) { + var buf io.ReadWriter + + if i != nil { + buf = new(bytes.Buffer) + + enc := json.NewEncoder(buf) + + err := enc.Encode(i) + if err != nil { + return nil, err + } + } + + return buf, nil +} + +// cloneRequest returns a clone of the provided *http.Request. +// The clone is a shallow copy of the struct and its Header map. +func cloneRequest(r *http.Request) *http.Request { + // shallow copy of the struct + _r := new(http.Request) + + *_r = *r + + // deep copy of the Header + _r.Header = make(http.Header, len(r.Header)) + + for k, s := range r.Header { + _r.Header[k] = append([]string(nil), s...) + } + + return _r +} + +// Signer is a JWT token signer. This is a wrapper around [jwt.SigningMethod] with predetermined +// key material. +type Signer interface { + // sign the given claims and returns a JWT token string, as specified + // by [jwt.Token.SignedString] + Sign(claims jwt.Claims) (string, error) +} + +// RSASigner signs JWT tokens using RSA keys. +type RSASigner struct { + method *jwt.SigningMethodRSA + key *rsa.PrivateKey +} + +// NewRSASigner creates a new RSASigner with the given RSA key. +func NewRSASigner(method *jwt.SigningMethodRSA, key *rsa.PrivateKey) *RSASigner { + return &RSASigner{ + method: method, + key: key, + } +} + +// Sign signs the JWT claims with the RSA key. +func (s *RSASigner) Sign(claims jwt.Claims) (string, error) { + return jwt.NewWithClaims(s.method, claims).SignedString(s.key) +} + +// AppsTransportOption is a func option for configuring an AppsTransport. +type AppsTransportOption func(*AppsTransport) + +// WithSigner configures the AppsTransport to use the given Signer for generating JWT tokens. +func WithSigner(signer Signer) AppsTransportOption { + return func(at *AppsTransport) { + at.signer = signer + } +} + +// NewTestAppsTransport creates a new AppsTransport for testing purposes. +func NewTestAppsTransport(baseURL string) *AppsTransport { + pk, _ := rsa.GenerateKey(rand.Reader, 2048) + + return &AppsTransport{ + BaseURL: baseURL, + Client: &http.Client{Transport: http.DefaultTransport}, + tr: http.DefaultTransport, + signer: &RSASigner{ + method: jwt.SigningMethodRS256, + key: pk, + }, + appID: 1, + } +} diff --git a/scm/github/app_transport_test.go b/scm/github/app_transport_test.go new file mode 100644 index 000000000..3bf27443c --- /dev/null +++ b/scm/github/app_transport_test.go @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/google/go-cmp/cmp" +) + +func TestGitHub_cloneRequest(t *testing.T) { + tests := []struct { + name string + request *http.Request + }{ + { + name: "basic request", + request: &http.Request{ + Method: "GET", + URL: &url.URL{ + Scheme: "https", + Path: "/", + }, + Header: http.Header{ + "Accept": []string{"application/json"}, + }, + }, + }, + { + name: "request with body", + request: &http.Request{ + Method: "POST", + URL: &url.URL{ + Scheme: "https", + Path: "/", + }, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + Body: io.NopCloser(strings.NewReader(`{"key":"value"}`)), + }, + }, + { + name: "request with multiple headers", + request: &http.Request{ + Method: "GET", + URL: &url.URL{ + Scheme: "https", + Path: "/", + }, + Header: http.Header{ + "Accept": []string{"application/json"}, + "Authorization": []string{"Bearer token"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clonedReq := cloneRequest(tt.request) + + if clonedReq == tt.request { + t.Errorf("cloneRequest() = %v, want different instance", clonedReq) + } + + if diff := cmp.Diff(clonedReq.Header, tt.request.Header); diff != "" { + t.Errorf("cloneRequest() headers mismatch (-want +got):\n%s", diff) + } + + if clonedReq.Method != tt.request.Method { + t.Errorf("cloneRequest() method = %v, want %v", clonedReq.Method, tt.request.Method) + } + + if clonedReq.URL.String() != tt.request.URL.String() { + t.Errorf("cloneRequest() URL = %v, want %v", clonedReq.URL, tt.request.URL) + } + }) + } +} + +func TestAppsTransport_RoundTrip(t *testing.T) { + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.GET("/", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + }) + + s := httptest.NewServer(engine) + defer s.Close() + + _url, _ := url.Parse(s.URL) + + tests := []struct { + name string + transport *AppsTransport + request *http.Request + wantHeader string + wantErr bool + }{ + { + name: "valid GET request", + transport: NewTestAppsTransport(s.URL), + request: &http.Request{ + Method: "GET", + URL: _url, + Header: http.Header{ + "Accept": []string{"application/json"}, + }, + }, + wantHeader: "Bearer ", + wantErr: false, + }, + { + name: "valid POST request", + transport: NewTestAppsTransport(s.URL), + request: &http.Request{ + Method: "POST", + URL: _url, + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + Body: io.NopCloser(strings.NewReader(`{"key":"value"}`)), + }, + wantHeader: "Bearer ", + wantErr: false, + }, + { + name: "request with invalid URL", + transport: NewTestAppsTransport(s.URL), + request: &http.Request{ + Method: "GET", + URL: &url.URL{Path: "://invalid-url"}, + Header: http.Header{}, + }, + wantHeader: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp, err := tt.transport.RoundTrip(tt.request) + if (err != nil) != tt.wantErr { + t.Errorf("RoundTrip() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr { + if got := tt.request.Header.Get("Authorization"); !strings.HasPrefix(got, tt.wantHeader) { + t.Errorf("RoundTrip() Authorization header = %v, want prefix %v", got, tt.wantHeader) + } + } + if resp != nil { + resp.Body.Close() + } + }) + } +} diff --git a/scm/github/authentication.go b/scm/github/authentication.go index 30376991f..bd1f4d62c 100644 --- a/scm/github/authentication.go +++ b/scm/github/authentication.go @@ -21,7 +21,7 @@ func (c *client) Authorize(ctx context.Context, token string) (string, error) { c.Logger.Trace("authorizing user with token") // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, token) + client := c.newOAuthTokenClient(ctx, token) // send API call to capture the current user making the call u, _, err := client.Users.Get(ctx, "") diff --git a/scm/github/changeset.go b/scm/github/changeset.go index 2aa07a445..7a9732fc4 100644 --- a/scm/github/changeset.go +++ b/scm/github/changeset.go @@ -21,7 +21,7 @@ func (c *client) Changeset(ctx context.Context, r *api.Repo, sha string) ([]stri }).Tracef("capturing commit changeset for %s/commit/%s", r.GetFullName(), sha) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, r.GetOwner().GetToken()) + client := c.newOAuthTokenClient(ctx, r.GetOwner().GetToken()) s := []string{} // set the max per page for the options to capture the commit @@ -50,7 +50,7 @@ func (c *client) ChangesetPR(ctx context.Context, r *api.Repo, number int) ([]st }).Tracef("capturing pull request changeset for %s/pull/%d", r.GetFullName(), number) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, r.GetOwner().GetToken()) + client := c.newOAuthTokenClient(ctx, r.GetOwner().GetToken()) s := []string{} f := []*github.CommitFile{} diff --git a/scm/github/deployment.go b/scm/github/deployment.go index e7f8ed0b4..f1c32db88 100644 --- a/scm/github/deployment.go +++ b/scm/github/deployment.go @@ -22,7 +22,7 @@ func (c *client) GetDeployment(ctx context.Context, u *api.User, r *api.Repo, id }).Tracef("capturing deployment %d for repo %s", id, r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // send API call to capture the deployment deployment, _, err := client.Repositories.GetDeployment(ctx, r.GetOrg(), r.GetName(), id) @@ -63,7 +63,7 @@ func (c *client) GetDeploymentCount(ctx context.Context, u *api.User, r *api.Rep }).Tracef("counting deployments for repo %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // create variable to track the deployments deployments := []*github.Deployment{} @@ -105,7 +105,7 @@ func (c *client) GetDeploymentList(ctx context.Context, u *api.User, r *api.Repo }).Tracef("listing deployments for repo %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // set pagination options for listing deployments opts := &github.DeploymentsListOptions{ @@ -164,7 +164,7 @@ func (c *client) CreateDeployment(ctx context.Context, u *api.User, r *api.Repo, }).Tracef("creating deployment for repo %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) var payload interface{} if d.Payload == nil { diff --git a/scm/github/driver_test.go b/scm/github/driver_test.go index 7ea6656d8..9315cb626 100644 --- a/scm/github/driver_test.go +++ b/scm/github/driver_test.go @@ -3,6 +3,7 @@ package github import ( + "context" "reflect" "testing" @@ -14,6 +15,7 @@ func TestGitHub_Driver(t *testing.T) { want := constants.DriverGithub _service, err := New( + context.Background(), WithAddress("https://github.com/"), WithClientID("foo"), WithClientSecret("bar"), diff --git a/scm/github/github.go b/scm/github/github.go index dfbffe53a..b8a3082df 100644 --- a/scm/github/github.go +++ b/scm/github/github.go @@ -4,14 +4,16 @@ package github import ( "context" + "crypto/x509" + "encoding/base64" + "encoding/pem" + "errors" "fmt" - "net/http/httptrace" - "net/url" + "os" + "strings" "github.com/google/go-github/v65/github" "github.com/sirupsen/logrus" - "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "golang.org/x/oauth2" "github.com/go-vela/server/tracing" @@ -39,6 +41,14 @@ type config struct { ClientID string // specifies the OAuth client secret from GitHub to use for the GitHub client ClientSecret string + // specifies the ID for the Vela GitHub App + AppID int64 + // specifies the App private key to use for the GitHub client when interacting with App resources + AppPrivateKey string + // specifies the App private key to use for the GitHub client when interacting with App resources + AppPrivateKeyPath string + // specifics the App permissions set + AppPermissions []string // specifies the Vela server address to use for the GitHub client ServerAddress string // specifies the Vela server address that the scm provider should use to send Vela webhooks @@ -48,14 +58,15 @@ type config struct { // specifies the Vela web UI address to use for the GitHub client WebUIAddress string // specifies the OAuth scopes to use for the GitHub client - Scopes []string + OAuthScopes []string } type client struct { - config *config - OAuth *oauth2.Config - AuthReq *github.AuthorizationRequest - Tracing *tracing.Client + config *config + OAuth *oauth2.Config + AuthReq *github.AuthorizationRequest + Tracing *tracing.Client + AppsTransport *AppsTransport // https://pkg.go.dev/github.com/sirupsen/logrus#Entry Logger *logrus.Entry } @@ -64,7 +75,7 @@ type client struct { // a GitHub or a GitHub Enterprise instance. // //nolint:revive // ignore returning unexported client -func New(opts ...ClientOpt) (*client, error) { +func New(ctx context.Context, opts ...ClientOpt) (*client, error) { // create new GitHub client c := new(client) @@ -95,28 +106,133 @@ func New(opts ...ClientOpt) (*client, error) { c.OAuth = &oauth2.Config{ ClientID: c.config.ClientID, ClientSecret: c.config.ClientSecret, - Scopes: c.config.Scopes, + Scopes: c.config.OAuthScopes, Endpoint: oauth2.Endpoint{ AuthURL: fmt.Sprintf("%s/login/oauth/authorize", c.config.Address), TokenURL: fmt.Sprintf("%s/login/oauth/access_token", c.config.Address), }, } - var githubScopes []github.Scope - for _, scope := range c.config.Scopes { - githubScopes = append(githubScopes, github.Scope(scope)) + var oauthScopes []github.Scope + for _, scope := range c.config.OAuthScopes { + oauthScopes = append(oauthScopes, github.Scope(scope)) } // create the GitHub authorization object c.AuthReq = &github.AuthorizationRequest{ ClientID: &c.config.ClientID, ClientSecret: &c.config.ClientSecret, - Scopes: githubScopes, + Scopes: oauthScopes, + } + + var err error + + if c.config.AppID != 0 { + c.Logger.Infof("configurating github app integration for app_id %d", c.config.AppID) + + var privateKeyPEM []byte + + if len(c.config.AppPrivateKey) == 0 && len(c.config.AppPrivateKeyPath) == 0 { + return nil, errors.New("GitHub App ID provided but no valid private key was provided in either VELA_SCM_APP_PRIVATE_KEY or VELA_SCM_APP_PRIVATE_KEY_PATH") + } + + if len(c.config.AppPrivateKey) > 0 { + privateKeyPEM, err = base64.StdEncoding.DecodeString(c.config.AppPrivateKey) + if err != nil { + return nil, fmt.Errorf("error decoding base64: %w", err) + } + } else { + // try reading from path if necessary + c.Logger.Infof("no VELA_SCM_APP_PRIVATE_KEY provided, reading github app private key from path %s", c.config.AppPrivateKeyPath) + + privateKeyPEM, err = os.ReadFile(c.config.AppPrivateKeyPath) + if err != nil { + return nil, err + } + } + + if len(privateKeyPEM) == 0 { + return nil, errors.New("GitHub App ID provided but no valid private key was provided in either VELA_SCM_APP_PRIVATE_KEY or VELA_SCM_APP_PRIVATE_KEY_PATH") + } + + block, _ := pem.Decode(privateKeyPEM) + if block == nil { + return nil, fmt.Errorf("failed to parse GitHub App private key PEM block containing the key") + } + + parsedPrivateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse GitHub App RSA private key: %w", err) + } + + c.AppsTransport = c.newGitHubAppTransport(c.config.AppID, c.config.API, parsedPrivateKey) + + err = c.ValidateGitHubApp(ctx) + if err != nil { + return nil, err + } } return c, nil } +// ValidateGitHubApp ensures the GitHub App configuration is valid. +func (c *client) ValidateGitHubApp(ctx context.Context) error { + client, err := c.newGithubAppClient() + if err != nil { + return fmt.Errorf("error creating github app client: %w", err) + } + + app, _, err := client.Apps.Get(ctx, "") + if err != nil { + return fmt.Errorf("error getting github app: %w", err) + } + + appPermissions := app.GetPermissions() + + type perm struct { + resource string + requiredPermission string + actualPermission string + } + + // the GitHub App installation requires the same permissions as provided at runtime + requiredPermissions := []perm{} + + // retrieve the required permissions for checking + for _, permission := range c.config.AppPermissions { + splitPerm := strings.Split(permission, ":") + if len(splitPerm) != 2 { + return fmt.Errorf("invalid app permission format %s, expected resource:permission", permission) + } + + resource := splitPerm[0] + requiredPermission := splitPerm[1] + + actual, err := GetInstallationPermission(resource, appPermissions) + if err != nil { + return err + } + + perm := perm{ + resource: resource, + requiredPermission: requiredPermission, + actualPermission: actual, + } + requiredPermissions = append(requiredPermissions, perm) + } + + // verify the app permissions + for _, p := range requiredPermissions { + err := InstallationHasPermission(p.resource, p.requiredPermission, p.actualPermission) + if err != nil { + return err + } + } + + return nil +} + // NewTest returns a SCM implementation that integrates with the provided // mock server. Only the url from the mock server is required. // @@ -133,6 +249,7 @@ func NewTest(urls ...string) (*client, error) { } return New( + context.Background(), WithAddress(address), WithClientID("foo"), WithClientSecret("bar"), @@ -143,39 +260,3 @@ func NewTest(urls ...string) (*client, error) { WithTracing(&tracing.Client{Config: tracing.Config{EnableTracing: false}}), ) } - -// helper function to return the GitHub OAuth client. -func (c *client) newClientToken(ctx context.Context, token string) *github.Client { - // create the token object for the client - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - - // create the OAuth client - tc := oauth2.NewClient(ctx, ts) - // if c.SkipVerify { - // tc.Transport.(*oauth2.Transport).Base = &http.Transport{ - // Proxy: http.ProxyFromEnvironment, - // TLSClientConfig: &tls.Config{ - // InsecureSkipVerify: true, - // }, - // } - // } - - if c.Tracing.Config.EnableTracing { - tc.Transport = otelhttp.NewTransport( - tc.Transport, - otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { - return otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans()) - }), - ) - } - - // create the GitHub client from the OAuth client - github := github.NewClient(tc) - - // ensure the proper URL is set in the GitHub client - github.BaseURL, _ = url.Parse(c.config.API) - - return github -} diff --git a/scm/github/github_client.go b/scm/github/github_client.go new file mode 100644 index 000000000..ac79436c8 --- /dev/null +++ b/scm/github/github_client.go @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptrace" + "net/url" + "strings" + + "github.com/google/go-github/v65/github" + "go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "golang.org/x/oauth2" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" +) + +// newOAuthTokenClient returns the GitHub OAuth client. +func (c *client) newOAuthTokenClient(ctx context.Context, token string) *github.Client { + // create the token object for the client + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + + // create the OAuth client + tc := oauth2.NewClient(ctx, ts) + + if c.Tracing.Config.EnableTracing { + tc.Transport = otelhttp.NewTransport( + tc.Transport, + otelhttp.WithClientTrace(func(ctx context.Context) *httptrace.ClientTrace { + return otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans()) + }), + ) + } + + // create the GitHub client from the OAuth client + github := github.NewClient(tc) + + // ensure the proper URL is set in the GitHub client + github.BaseURL, _ = url.Parse(c.config.API) + + return github +} + +// newGithubAppClient returns the GitHub App client for authenticating as the GitHub App itself using the RoundTripper. +func (c *client) newGithubAppClient() (*github.Client, error) { + if c.AppsTransport == nil { + return nil, errors.New("unable to create github app client: no AppsTransport configured") + } + + // create a github client based off the existing GitHub App configuration + client, err := github.NewClient( + &http.Client{ + Transport: c.AppsTransport, + }). + WithEnterpriseURLs(c.config.API, c.config.API) + if err != nil { + return nil, err + } + + return client, nil +} + +// newGithubAppInstallationRepoToken returns the GitHub App installation token for a particular repo with granular permissions. +func (c *client) newGithubAppInstallationRepoToken(ctx context.Context, r *api.Repo, repos []string, permissions *github.InstallationPermissions) (*github.InstallationToken, int64, error) { + // create a github client based off the existing GitHub App configuration + client, err := c.newGithubAppClient() + if err != nil { + return nil, 0, err + } + + opts := &github.InstallationTokenOptions{ + Repositories: repos, + Permissions: permissions, + } + + id := r.GetInstallID() + + // if the source scm repo has an install ID but the Vela db record does not + // then use the source repo to create an installation token + if id == 0 { + // list all installations (a.k.a. orgs) where the GitHub App is installed + installations, _, err := client.Apps.ListInstallations(ctx, &github.ListOptions{}) + if err != nil { + return nil, 0, err + } + + // iterate through the list of installations + for _, install := range installations { + // find the installation that matches the org for the repo + if strings.EqualFold(install.GetAccount().GetLogin(), r.GetOrg()) { + if install.GetRepositorySelection() == constants.AppInstallRepositoriesSelectionSelected { + installationCanReadRepo, err := c.installationCanReadRepo(ctx, r, install) + if err != nil { + return nil, 0, fmt.Errorf("installation for org %s exists but unable to check if it can read repo %s: %w", install.GetAccount().GetLogin(), r.GetFullName(), err) + } + + if !installationCanReadRepo { + return nil, 0, fmt.Errorf("installation for org %s exists but does not have access to repo %s", install.GetAccount().GetLogin(), r.GetFullName()) + } + } + + id = install.GetID() + } + } + } + + // failsafe in case the repo does not belong to an org where the GitHub App is installed + if id == 0 { + return nil, 0, errors.New("unable to find installation ID for repo") + } + + // create installation token for the repo + t, _, err := client.Apps.CreateInstallationToken(ctx, id, opts) + if err != nil { + return nil, 0, err + } + + return t, id, nil +} + +// installationCanReadRepo checks if the installation can read the repo. +func (c *client) installationCanReadRepo(ctx context.Context, r *api.Repo, installation *github.Installation) (bool, error) { + installationCanReadRepo := false + + if installation.GetRepositorySelection() == constants.AppInstallRepositoriesSelectionSelected { + client, err := c.newGithubAppClient() + if err != nil { + return false, err + } + + t, _, err := client.Apps.CreateInstallationToken(ctx, installation.GetID(), &github.InstallationTokenOptions{}) + if err != nil { + return false, err + } + + client = c.newOAuthTokenClient(ctx, t.GetToken()) + + repos, _, err := client.Apps.ListRepos(ctx, &github.ListOptions{}) + if err != nil { + return false, err + } + + for _, repo := range repos.Repositories { + if strings.EqualFold(repo.GetFullName(), r.GetFullName()) { + installationCanReadRepo = true + break + } + } + } + + return installationCanReadRepo, nil +} diff --git a/scm/github/github_client_test.go b/scm/github/github_client_test.go new file mode 100644 index 000000000..57de6f7d7 --- /dev/null +++ b/scm/github/github_client_test.go @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: Apache-2.0 + +package github + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/google/go-github/v65/github" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/constants" +) + +func TestClient_installationCanReadRepo(t *testing.T) { + // setup types + accessibleRepo := new(api.Repo) + accessibleRepo.SetOrg("octocat") + accessibleRepo.SetName("Hello-World") + accessibleRepo.SetFullName("octocat/Hello-World") + accessibleRepo.SetInstallID(0) + + inaccessibleRepo := new(api.Repo) + inaccessibleRepo.SetOrg("octocat") + inaccessibleRepo.SetName("Hello-World") + inaccessibleRepo.SetFullName("octocat/Hello-World2") + inaccessibleRepo.SetInstallID(4) + + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.POST("/api/v3/app/installations/:id/access_tokens", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installations_access_tokens.json") + }) + engine.GET("/api/v3/installation/repositories", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installation_repositories.json") + }) + + s := httptest.NewServer(engine) + defer s.Close() + + oauthClient, _ := NewTest(s.URL) + + appsClient, err := NewTest(s.URL) + if err != nil { + t.Errorf("unable to create GitHub App client: %v", err) + } + + appsClient.AppsTransport = NewTestAppsTransport("") + + // setup tests + tests := []struct { + name string + client *client + repo *api.Repo + installation *github.Installation + appsTransport bool + want bool + wantErr bool + }{ + { + name: "installation can read repo", + client: appsClient, + repo: accessibleRepo, + installation: &github.Installation{ + ID: github.Int64(1), + Account: &github.User{ + Login: github.String("github"), + }, + RepositorySelection: github.String(constants.AppInstallRepositoriesSelectionSelected), + }, + want: true, + wantErr: false, + }, + { + name: "installation cannot read repo", + client: appsClient, + repo: inaccessibleRepo, + installation: &github.Installation{ + ID: github.Int64(2), + Account: &github.User{ + Login: github.String("github"), + }, + RepositorySelection: github.String(constants.AppInstallRepositoriesSelectionSelected), + }, + want: false, + wantErr: false, + }, + { + name: "no GitHub App client", + client: oauthClient, + repo: accessibleRepo, + installation: &github.Installation{ + ID: github.Int64(1), + Account: &github.User{ + Login: github.String("github"), + }, + RepositorySelection: github.String(constants.AppInstallRepositoriesSelectionSelected), + }, + want: false, + wantErr: true, + }, + } + + // run tests + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.client.installationCanReadRepo(context.Background(), tt.repo, tt.installation) + if (err != nil) != tt.wantErr { + t.Errorf("installationCanReadRepo() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("installationCanReadRepo() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/scm/github/github_test.go b/scm/github/github_test.go index 963aa8076..3ce90df3a 100644 --- a/scm/github/github_test.go +++ b/scm/github/github_test.go @@ -32,14 +32,14 @@ func TestGithub_New(t *testing.T) { // run tests for _, test := range tests { - _, err := New( + _, err := New(context.Background(), WithAddress("https://github.com/"), WithClientID(test.id), WithClientSecret("bar"), WithServerAddress("https://vela-server.example.com"), WithStatusContext("continuous-integration/vela"), WithWebUIAddress("https://vela.example.com"), - WithScopes([]string{"repo", "repo:status", "user:email", "read:user", "read:org"}), + WithOAuthScopes([]string{"repo", "repo:status", "user:email", "read:user", "read:org"}), ) if test.failure { @@ -72,7 +72,7 @@ func TestGithub_newClientToken(t *testing.T) { client, _ := NewTest(s.URL) // run test - got := client.newClientToken(context.Background(), "foobar") + got := client.newOAuthTokenClient(context.Background(), "foobar") //nolint:staticcheck // ignore false positive if got == nil { diff --git a/scm/github/opts.go b/scm/github/opts.go index bb7385827..ad64cae0b 100644 --- a/scm/github/opts.go +++ b/scm/github/opts.go @@ -135,8 +135,8 @@ func WithWebUIAddress(address string) ClientOpt { } } -// WithScopes sets the OAuth scopes in the scm client for GitHub. -func WithScopes(scopes []string) ClientOpt { +// WithOAuthScopes sets the OAuth scopes in the scm client for GitHub. +func WithOAuthScopes(scopes []string) ClientOpt { return func(c *client) error { c.Logger.Trace("configuring oauth scopes in github scm client") @@ -146,7 +146,7 @@ func WithScopes(scopes []string) ClientOpt { } // set the scopes in the github client - c.config.Scopes = scopes + c.config.OAuthScopes = scopes return nil } @@ -160,3 +160,50 @@ func WithTracing(tracing *tracing.Client) ClientOpt { return nil } } + +// WithGithubAppID sets the ID for the GitHub App in the scm client. +func WithGithubAppID(id int64) ClientOpt { + return func(c *client) error { + c.Logger.Trace("configuring ID for GitHub App in github scm client") + + // set the ID for the GitHub App in the github client + c.config.AppID = id + + return nil + } +} + +// WithGithubPrivateKey sets the private key for the GitHub App in the scm client. +func WithGithubPrivateKey(key string) ClientOpt { + return func(c *client) error { + c.Logger.Trace("configuring private key for GitHub App in github scm client") + + // set the private key for the GitHub App in the github client + c.config.AppPrivateKey = key + + return nil + } +} + +// WithGithubPrivateKeyPath sets the private key path for the GitHub App in the scm client. +func WithGithubPrivateKeyPath(path string) ClientOpt { + return func(c *client) error { + c.Logger.Trace("configuring private key path for GitHub App in github scm client") + + // set the private key for the GitHub App in the github client + c.config.AppPrivateKeyPath = path + + return nil + } +} + +// WithGitHubAppPermissions sets the App permissions in the scm client for GitHub. +func WithGitHubAppPermissions(permissions []string) ClientOpt { + return func(c *client) error { + c.Logger.Trace("configuring app permissions in github scm client") + + c.config.AppPermissions = permissions + + return nil + } +} diff --git a/scm/github/opts_test.go b/scm/github/opts_test.go index 8a6a2617e..a2aac20bc 100644 --- a/scm/github/opts_test.go +++ b/scm/github/opts_test.go @@ -3,9 +3,12 @@ package github import ( + "context" "reflect" "testing" + "github.com/google/go-cmp/cmp" + "github.com/go-vela/server/tracing" ) @@ -33,7 +36,7 @@ func TestGithub_ClientOpt_WithAddress(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithAddress(test.address), ) @@ -72,7 +75,7 @@ func TestGithub_ClientOpt_WithClientID(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithClientID(test.id), ) @@ -115,7 +118,7 @@ func TestGithub_ClientOpt_WithClientSecret(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithClientSecret(test.secret), ) @@ -158,7 +161,7 @@ func TestGithub_ClientOpt_WithServerAddress(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithServerAddress(test.address), ) @@ -210,7 +213,7 @@ func TestGithub_ClientOpt_WithServerWebhookAddress(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithServerAddress(test.address), WithServerWebhookAddress(test.webhookAddress), ) @@ -254,7 +257,7 @@ func TestGithub_ClientOpt_WithStatusContext(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithStatusContext(test.context), ) @@ -294,7 +297,7 @@ func TestGithub_ClientOpt_WithWebUIAddress(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithWebUIAddress(test.address), ) @@ -308,7 +311,7 @@ func TestGithub_ClientOpt_WithWebUIAddress(t *testing.T) { } } -func TestGithub_ClientOpt_WithScopes(t *testing.T) { +func TestGithub_ClientOpt_WithOAuthScopes(t *testing.T) { // setup tests tests := []struct { failure bool @@ -329,24 +332,24 @@ func TestGithub_ClientOpt_WithScopes(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( - WithScopes(test.scopes), + _service, err := New(context.Background(), + WithOAuthScopes(test.scopes), ) if test.failure { if err == nil { - t.Errorf("WithScopes should have returned err") + t.Errorf("WithOAuthScopes should have returned err") } continue } if err != nil { - t.Errorf("WithScopes returned err: %v", err) + t.Errorf("WithOAuthScopes returned err: %v", err) } - if !reflect.DeepEqual(_service.config.Scopes, test.want) { - t.Errorf("WithScopes is %v, want %v", _service.config.Scopes, test.want) + if !reflect.DeepEqual(_service.config.OAuthScopes, test.want) { + t.Errorf("WithOAuthScopes is %v, want %v", _service.config.OAuthScopes, test.want) } } } @@ -367,7 +370,7 @@ func TestGithub_ClientOpt_WithTracing(t *testing.T) { // run tests for _, test := range tests { - _service, err := New( + _service, err := New(context.Background(), WithTracing(test.tracing), ) @@ -388,3 +391,46 @@ func TestGithub_ClientOpt_WithTracing(t *testing.T) { } } } + +func TestGithub_ClientOpt_WithGitHubAppPermissions(t *testing.T) { + // setup tests + tests := []struct { + failure bool + permissions []string + want []string + }{ + { + failure: false, + permissions: []string{"contents:read"}, + want: []string{"contents:read"}, + }, + { + failure: false, + permissions: []string{}, + want: []string{}, + }, + } + + // run tests + for _, test := range tests { + _service, err := New(context.Background(), + WithGitHubAppPermissions(test.permissions), + ) + + if test.failure { + if err == nil { + t.Errorf("WithGitHubAppPermissions should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("WithGitHubAppPermissions returned err: %v", err) + } + + if diff := cmp.Diff(test.want, _service.config.AppPermissions); diff != "" { + t.Errorf("WithGitHubAppPermissions mismatch (-want +got):\n%s", diff) + } + } +} diff --git a/scm/github/org.go b/scm/github/org.go index 8c9e95986..ec1f8e314 100644 --- a/scm/github/org.go +++ b/scm/github/org.go @@ -19,7 +19,7 @@ func (c *client) GetOrgName(ctx context.Context, u *api.User, o string) (string, }).Tracef("retrieving org information for %s", o) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) // send an API call to get the org info orgInfo, resp, err := client.Organizations.Get(ctx, o) diff --git a/scm/github/repo.go b/scm/github/repo.go index 3cf9dc760..d1648caa6 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -14,7 +14,9 @@ import ( "github.com/sirupsen/logrus" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/compiler/types/yaml" "github.com/go-vela/server/constants" + "github.com/go-vela/server/database" ) // ConfigBackoff is a wrapper for Config that will retry five times if the function @@ -55,7 +57,7 @@ func (c *client) Config(ctx context.Context, u *api.User, r *api.Repo, ref strin }).Tracef("capturing configuration file for %s/commit/%s", r.GetFullName(), ref) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // default pipeline file names files := []string{".vela.yml", ".vela.yaml"} @@ -95,6 +97,11 @@ func (c *client) Config(ctx context.Context, u *api.User, r *api.Repo, ref strin // Disable deactivates a repo by deleting the webhook. func (c *client) Disable(ctx context.Context, u *api.User, org, name string) error { + return c.DestroyWebhook(ctx, u, org, name) +} + +// DestroyWebhook deletes a repo's webhook. +func (c *client) DestroyWebhook(ctx context.Context, u *api.User, org, name string) error { c.Logger.WithFields(logrus.Fields{ "org": org, "repo": name, @@ -102,7 +109,7 @@ func (c *client) Disable(ctx context.Context, u *api.User, org, name string) err }).Tracef("deleting repository webhooks for %s/%s", org, name) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // send API call to capture the hooks for the repo hooks, _, err := client.Repositories.ListHooks(ctx, org, name, nil) @@ -150,6 +157,11 @@ func (c *client) Disable(ctx context.Context, u *api.User, org, name string) err // Enable activates a repo by creating the webhook. func (c *client) Enable(ctx context.Context, u *api.User, r *api.Repo, h *api.Hook) (*api.Hook, string, error) { + return c.CreateWebhook(ctx, u, r, h) +} + +// CreateWebhook creates a repo's webhook. +func (c *client) CreateWebhook(ctx context.Context, u *api.User, r *api.Repo, h *api.Hook) (*api.Hook, string, error) { c.Logger.WithFields(logrus.Fields{ "org": r.GetOrg(), "repo": r.GetName(), @@ -157,7 +169,7 @@ func (c *client) Enable(ctx context.Context, u *api.User, r *api.Repo, h *api.Ho }).Tracef("creating repository webhook for %s/%s", r.GetOrg(), r.GetName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // always listen to repository events in case of repo name change events := []string{eventRepository} @@ -231,7 +243,7 @@ func (c *client) Update(ctx context.Context, u *api.User, r *api.Repo, hookID in }).Tracef("updating repository webhook for %s/%s", r.GetOrg(), r.GetName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // always listen to repository events in case of repo name change events := []string{eventRepository} @@ -295,7 +307,7 @@ func (c *client) Status(ctx context.Context, u *api.User, b *api.Build, org, nam } // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) context := fmt.Sprintf("%s/%s", c.config.StatusContext, b.GetEvent()) url := fmt.Sprintf("%s/%s/%s/%d", c.config.WebUIAddress, org, name, b.GetNumber()) @@ -414,7 +426,7 @@ func (c *client) StepStatus(ctx context.Context, u *api.User, b *api.Build, s *a } // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) context := fmt.Sprintf("%s/%s/%s", c.config.StatusContext, b.GetEvent(), s.GetReportAs()) url := fmt.Sprintf("%s/%s/%s/%d#%d", c.config.WebUIAddress, org, name, b.GetNumber(), s.GetNumber()) @@ -477,7 +489,7 @@ func (c *client) GetRepo(ctx context.Context, u *api.User, r *api.Repo) (*api.Re }).Tracef("retrieving repository information for %s", r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) // send an API call to get the repo info repo, resp, err := client.Repositories.Get(ctx, r.GetOrg(), r.GetName()) @@ -497,7 +509,7 @@ func (c *client) GetOrgAndRepoName(ctx context.Context, u *api.User, o string, r }).Tracef("retrieving repository information for %s/%s", o, r) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) // send an API call to get the repo info repo, _, err := client.Repositories.Get(ctx, o, r) @@ -515,7 +527,7 @@ func (c *client) ListUserRepos(ctx context.Context, u *api.User) ([]*api.Repo, e }).Tracef("listing source repositories for %s", u.GetName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) r := []*github.Repository{} f := []*api.Repo{} @@ -595,7 +607,7 @@ func (c *client) GetPullRequest(ctx context.Context, r *api.Repo, number int) (s }).Tracef("retrieving pull request %d for repo %s", number, r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, r.GetOwner().GetToken()) + client := c.newOAuthTokenClient(ctx, r.GetOwner().GetToken()) pull, _, err := client.PullRequests.Get(ctx, r.GetOrg(), r.GetName(), number) if err != nil { @@ -619,7 +631,7 @@ func (c *client) GetHTMLURL(ctx context.Context, u *api.User, org, repo, name, r }).Tracef("capturing html_url for %s/%s/%s@%s", org, repo, name, ref) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, *u.Token) + client := c.newOAuthTokenClient(ctx, *u.Token) // set the reference for the options to capture the repository contents opts := &github.RepositoryContentGetOptions{ @@ -651,7 +663,7 @@ func (c *client) GetBranch(ctx context.Context, r *api.Repo, branch string) (str }).Tracef("retrieving branch %s for repo %s", branch, r.GetFullName()) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, r.GetOwner().GetToken()) + client := c.newOAuthTokenClient(ctx, r.GetOwner().GetToken()) maxRedirects := 3 @@ -662,3 +674,142 @@ func (c *client) GetBranch(ctx context.Context, r *api.Repo, branch string) (str return data.GetName(), data.GetCommit().GetSHA(), nil } + +// GetNetrcPassword returns a clone token using the repo's github app installation if it exists. +// If not, it defaults to the user OAuth token. +func (c *client) GetNetrcPassword(ctx context.Context, db database.Interface, r *api.Repo, u *api.User, g yaml.Git) (string, error) { + l := c.Logger.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + }) + + l.Tracef("getting netrc password for %s/%s", r.GetOrg(), r.GetName()) + + // no GitHub App configured, use legacy oauth token + if c.AppsTransport == nil { + return u.GetToken(), nil + } + + var err error + + // repos that the token has access to + // providing no repos, nil, or empty slice will default the token permissions to the list + // of repos added to the installation + repos := g.Repositories + + // use triggering repo as a restrictive default + if repos == nil { + repos = []string{r.GetName()} + } + + // convert repo fullname org/name to just name for usability + for i, repo := range repos { + split := strings.Split(repo, "/") + if len(split) == 2 { + repos[i] = split[1] + } + } + + // permissions that are applied to the token for every repo provided + // providing no permissions, nil, or empty map will default to the permissions + // of the GitHub App installation + // + // the Vela compiler follows a least-privileged-defaults model where + // the list contains only the triggering repo, unless provided in the git yaml block + // + // the default is contents:read and checks:write + ghPermissions := &github.InstallationPermissions{ + Contents: github.String(AppInstallPermissionRead), + Checks: github.String(AppInstallPermissionWrite), + } + + permissions := g.Permissions + if permissions == nil { + permissions = map[string]string{} + } + + for resource, perm := range permissions { + ghPermissions, err = ApplyInstallationPermissions(resource, perm, ghPermissions) + if err != nil { + return u.GetToken(), err + } + } + + // the app might not be installed therefore we retain backwards compatibility via the user oauth token + // https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/authenticating-as-a-github-app-installation + // the optional list of repos and permissions are driven by yaml + installToken, installID, err := c.newGithubAppInstallationRepoToken(ctx, r, repos, ghPermissions) + if err != nil { + // return the legacy token along with no error for backwards compatibility + // todo: return an error based based on app installation requirements + l.Tracef("unable to create github app installation token for repos %v with permissions %v: %v", repos, permissions, err) + + return u.GetToken(), nil + } + + if installToken != nil && len(installToken.GetToken()) != 0 { + l.Tracef("using github app installation token for %s/%s", r.GetOrg(), r.GetName()) + + // (optional) sync the install ID with the repo + if db != nil && r.GetInstallID() != installID { + r.SetInstallID(installID) + + _, err = db.UpdateRepo(ctx, r) + if err != nil { + c.Logger.Tracef("unable to update repo with install ID %d: %v", installID, err) + } + } + + return installToken.GetToken(), nil + } + + l.Tracef("using user oauth token for %s/%s", r.GetOrg(), r.GetName()) + + return u.GetToken(), nil +} + +// SyncRepoWithInstallation ensures the repo is synchronized with the scm installation, if it exists. +func (c *client) SyncRepoWithInstallation(ctx context.Context, r *api.Repo) (*api.Repo, error) { + c.Logger.WithFields(logrus.Fields{ + "org": r.GetOrg(), + "repo": r.GetName(), + }).Tracef("syncing app installation for repo %s/%s", r.GetOrg(), r.GetName()) + + // no GitHub App configured, skip + if c.AppsTransport == nil { + return r, nil + } + + client, err := c.newGithubAppClient() + if err != nil { + return r, err + } + + installations, _, err := client.Apps.ListInstallations(ctx, &github.ListOptions{}) + if err != nil { + return r, err + } + + var installation *github.Installation + + for _, install := range installations { + if strings.EqualFold(install.GetAccount().GetLogin(), r.GetOrg()) { + installation = install + } + } + + if installation == nil { + return r, nil + } + + installationCanReadRepo, err := c.installationCanReadRepo(ctx, r, installation) + if err != nil { + return r, err + } + + if installationCanReadRepo { + r.SetInstallID(installation.GetID()) + } + + return r, nil +} diff --git a/scm/github/repo_test.go b/scm/github/repo_test.go index 502c1cc7c..036639674 100644 --- a/scm/github/repo_test.go +++ b/scm/github/repo_test.go @@ -13,8 +13,11 @@ import ( "testing" "github.com/gin-gonic/gin" + "github.com/google/go-cmp/cmp" + "github.com/google/go-github/v65/github" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/compiler/types/yaml" "github.com/go-vela/server/constants" ) @@ -1621,3 +1624,287 @@ func TestGithub_GetBranch(t *testing.T) { t.Errorf("Commit is %v, want %v", gotCommit, wantCommit) } } + +func TestGithub_GetNetrcPassword(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.GET("/api/v3/app/installations", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installations.json") + }) + engine.POST("/api/v3/app/installations/:id/access_tokens", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installations_access_tokens.json") + }) + + s := httptest.NewServer(engine) + defer s.Close() + + installedRepo := new(api.Repo) + installedRepo.SetOrg("octocat") + installedRepo.SetName("Hello-World") + installedRepo.SetInstallID(1) + + oauthRepo := new(api.Repo) + oauthRepo.SetOrg("octocat") + oauthRepo.SetName("Hello-World2") + oauthRepo.SetInstallID(0) + + u := new(api.User) + u.SetName("foo") + u.SetToken("bar") + + tests := []struct { + name string + repo *api.Repo + user *api.User + git yaml.Git + appsTransport bool + wantToken string + wantErr bool + }{ + { + name: "installation token", + repo: installedRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"contents": "read"}, + }, + }, + appsTransport: true, + wantToken: "ghs_16C7e42F292c6912E7710c838347Ae178B4a", + wantErr: false, + }, + { + name: "no app configured returns user oauth token", + repo: installedRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"contents": "read"}, + }, + }, + appsTransport: false, + wantToken: "bar", + wantErr: false, + }, + { + name: "repo not installed returns user oauth token", + repo: oauthRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"contents": "read"}, + }, + }, + appsTransport: true, + wantToken: "bar", + wantErr: false, + }, + { + name: "invalid permission resource", + repo: installedRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"invalid": "read"}, + }, + }, + appsTransport: true, + wantToken: "bar", + wantErr: true, + }, + { + name: "invalid permission level", + repo: installedRepo, + user: u, + git: yaml.Git{ + Token: yaml.Token{ + Repositories: []string{"Hello-World"}, + Permissions: map[string]string{"contents": "invalid"}, + }, + }, + appsTransport: true, + wantToken: "bar", + wantErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client, _ := NewTest(s.URL) + if test.appsTransport { + client.AppsTransport = NewTestAppsTransport(s.URL) + } + + got, err := client.GetNetrcPassword(context.TODO(), nil, test.repo, test.user, test.git) + if (err != nil) != test.wantErr { + t.Errorf("GetNetrcPassword() error = %v, wantErr %v", err, test.wantErr) + return + } + if got != test.wantToken { + t.Errorf("GetNetrcPassword() = %v, want %v", got, test.wantToken) + } + }) + } +} + +func TestGithub_SyncRepoWithInstallation(t *testing.T) { + // setup context + gin.SetMode(gin.TestMode) + + resp := httptest.NewRecorder() + _, engine := gin.CreateTestContext(resp) + + // setup mock server + engine.GET("/api/v3/app/installations", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installations.json") + }) + engine.POST("/api/v3/app/installations/:id/access_tokens", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installations_access_tokens.json") + }) + engine.GET("/api/v3/installation/repositories", func(c *gin.Context) { + c.Header("Content-Type", "application/json") + c.Status(http.StatusOK) + c.File("testdata/installation_repositories.json") + }) + + s := httptest.NewServer(engine) + defer s.Close() + + tests := []struct { + name string + org string + repo string + wantInstallID int64 + wantStatusCode int + }{ + { + name: "match", + org: "octocat", + repo: "Hello-World", + wantInstallID: 1, + wantStatusCode: http.StatusOK, + }, + { + name: "no match", + repo: "octocat/Hello-World2", + wantInstallID: 0, + wantStatusCode: http.StatusOK, + }, + } + for _, test := range tests { + // setup types + r := new(api.Repo) + r.SetOrg(test.org) + r.SetName(test.repo) + r.SetFullName(fmt.Sprintf("%s/%s", test.org, test.repo)) + + client, _ := NewTest(s.URL) + client.AppsTransport = NewTestAppsTransport(s.URL) + + // run test + got, err := client.SyncRepoWithInstallation(context.TODO(), r) + + if resp.Code != test.wantStatusCode { + t.Errorf("SyncRepoWithInstallation %s returned %v, want %v", test.name, resp.Code, http.StatusOK) + } + + if err != nil { + t.Errorf("SyncRepoWithInstallation %s returned err: %v", test.name, err) + } + + if got.GetInstallID() != test.wantInstallID { + t.Errorf("SyncRepoWithInstallation %s returned %v, want %v", test.name, got.GetInstallID(), test.wantInstallID) + } + } +} + +func TestGithub_applyGitHubInstallationPermission(t *testing.T) { + tests := []struct { + name string + perms *github.InstallationPermissions + resource string + perm string + wantPerms *github.InstallationPermissions + wantErr bool + }{ + { + name: "valid read permission for contents", + perms: &github.InstallationPermissions{ + Contents: github.String(AppInstallPermissionNone), + }, + resource: AppInstallResourceContents, + perm: AppInstallPermissionRead, + wantPerms: &github.InstallationPermissions{ + Contents: github.String(AppInstallPermissionRead), + }, + wantErr: false, + }, + { + name: "valid write permission for checks", + perms: &github.InstallationPermissions{ + Checks: github.String(AppInstallPermissionNone), + }, + resource: AppInstallResourceChecks, + perm: AppInstallPermissionWrite, + wantPerms: &github.InstallationPermissions{ + Checks: github.String(AppInstallPermissionWrite), + }, + wantErr: false, + }, + { + name: "invalid permission value", + perms: &github.InstallationPermissions{ + Contents: github.String(AppInstallPermissionNone), + }, + resource: AppInstallResourceContents, + perm: "invalid", + wantPerms: &github.InstallationPermissions{ + Contents: github.String(AppInstallPermissionNone), + }, + wantErr: true, + }, + { + name: "invalid permission key", + perms: &github.InstallationPermissions{ + Contents: github.String(AppInstallPermissionNone), + }, + resource: "invalid", + perm: AppInstallPermissionRead, + wantPerms: &github.InstallationPermissions{ + Contents: github.String(AppInstallPermissionNone), + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ApplyInstallationPermissions(tt.resource, tt.perm, tt.perms) + if (err != nil) != tt.wantErr { + t.Errorf("ToGitHubAppInstallationPermissions() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(tt.wantPerms, got); diff != "" { + t.Errorf("ToGitHubAppInstallationPermissions() mismatch (-want +got):\n%s", diff) + } + }) + } +} diff --git a/scm/github/testdata/hooks/installation_created.json b/scm/github/testdata/hooks/installation_created.json new file mode 100644 index 000000000..c3904b286 --- /dev/null +++ b/scm/github/testdata/hooks/installation_created.json @@ -0,0 +1,100 @@ +{ + "action": "created", + "installation": { + "id": 1, + "account": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + }, + "repository_selection": "selected", + "access_tokens_url": "https://octocoders.github.io/api/v3/app/installations/1/access_tokens", + "repositories_url": "https://octocoders.github.io/api/v3/installation/repositories", + "html_url": "https://octocoders.github.io/settings/installations/1", + "app_id": 282, + "app_slug": "vela", + "target_id": 10919, + "target_type": "User", + "permissions": { + "checks": "write", + "contents": "read", + "metadata": "read" + }, + "events": [ + + ], + "created_at": "2024-10-22T08:50:39.000-05:00", + "updated_at": "2024-10-22T08:50:39.000-05:00", + "single_file_name": null, + "has_multiple_single_files": false, + "single_file_paths": [ + + ], + "suspended_by": null, + "suspended_at": null + }, + "repositories": [ + { + "id": 1, + "node_id": "MDEwOlJlcG9zaXRvcnkxMTg=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "private": true + }, + { + "id": 2, + "node_id": "MDEwOlJlcG9zaXRvcnk0MjI0MzE=", + "name": "Hello-World2", + "full_name": "Codertocat/Hello-World2", + "private": false + } + ], + "requester": null, + "enterprise": { + "id": 1, + "slug": "github", + "name": "GitHub", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://octocoders.github.io/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://octocoders.github.io/businesses/github", + "created_at": "2018-10-24T21:19:19Z", + "updated_at": "2023-06-01T21:03:12Z" + }, + "sender": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + } +} \ No newline at end of file diff --git a/scm/github/testdata/hooks/installation_deleted.json b/scm/github/testdata/hooks/installation_deleted.json new file mode 100644 index 000000000..9972e0cf9 --- /dev/null +++ b/scm/github/testdata/hooks/installation_deleted.json @@ -0,0 +1,100 @@ +{ + "action": "deleted", + "installation": { + "id": 1, + "account": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + }, + "repository_selection": "selected", + "access_tokens_url": "https://octocoders.github.io/api/v3/app/installations/1/access_tokens", + "repositories_url": "https://octocoders.github.io/api/v3/installation/repositories", + "html_url": "https://octocoders.github.io/settings/installations/1", + "app_id": 282, + "app_slug": "vela", + "target_id": 10919, + "target_type": "User", + "permissions": { + "checks": "write", + "contents": "read", + "metadata": "read" + }, + "events": [ + + ], + "created_at": "2024-10-22T08:50:39.000-05:00", + "updated_at": "2024-10-22T08:50:39.000-05:00", + "single_file_name": null, + "has_multiple_single_files": false, + "single_file_paths": [ + + ], + "suspended_by": null, + "suspended_at": null + }, + "repositories": [ + { + "id": 1, + "node_id": "MDEwOlJlcG9zaXRvcnkxMTg=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "private": true + }, + { + "id": 2, + "node_id": "MDEwOlJlcG9zaXRvcnk0MjI0MzE=", + "name": "Hello-World2", + "full_name": "Codertocat/Hello-World2", + "private": false + } + ], + "requester": null, + "enterprise": { + "id": 1, + "slug": "github", + "name": "GitHub", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://octocoders.github.io/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://octocoders.github.io/businesses/github", + "created_at": "2018-10-24T21:19:19Z", + "updated_at": "2023-06-01T21:03:12Z" + }, + "sender": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + } +} \ No newline at end of file diff --git a/scm/github/testdata/hooks/installation_repositories_added.json b/scm/github/testdata/hooks/installation_repositories_added.json new file mode 100644 index 000000000..f75fedcd1 --- /dev/null +++ b/scm/github/testdata/hooks/installation_repositories_added.json @@ -0,0 +1,103 @@ +{ + "action": "added", + "installation": { + "id": 1, + "account": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + }, + "repository_selection": "selected", + "access_tokens_url": "https://octocoders.github.io/api/v3/app/installations/1/access_tokens", + "repositories_url": "https://octocoders.github.io/api/v3/installation/repositories", + "html_url": "https://octocoders.github.io/settings/installations/1", + "app_id": 282, + "app_slug": "vela", + "target_id": 10919, + "target_type": "User", + "permissions": { + "checks": "write", + "contents": "read", + "metadata": "read" + }, + "events": [ + + ], + "created_at": "2024-10-22T08:50:39.000-05:00", + "updated_at": "2024-10-22T08:50:39.000-05:00", + "single_file_name": null, + "has_multiple_single_files": false, + "single_file_paths": [ + + ], + "suspended_by": null, + "suspended_at": null + }, + "repositories_added": [ + { + "id": 1, + "node_id": "MDEwOlJlcG9zaXRvcnkxMTg=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "private": true + }, + { + "id": 2, + "node_id": "MDEwOlJlcG9zaXRvcnk0MjI0MzE=", + "name": "Hello-World2", + "full_name": "Codertocat/Hello-World2", + "private": false + } + ], + "repositories_removed": [ + + ], + "requester": null, + "enterprise": { + "id": 1, + "slug": "github", + "name": "GitHub", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://octocoders.github.io/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://octocoders.github.io/businesses/github", + "created_at": "2018-10-24T21:19:19Z", + "updated_at": "2023-06-01T21:03:12Z" + }, + "sender": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + } + } \ No newline at end of file diff --git a/scm/github/testdata/hooks/installation_repositories_removed.json b/scm/github/testdata/hooks/installation_repositories_removed.json new file mode 100644 index 000000000..b7a82ec5b --- /dev/null +++ b/scm/github/testdata/hooks/installation_repositories_removed.json @@ -0,0 +1,103 @@ +{ + "action": "removed", + "installation": { + "id": 1, + "account": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + }, + "repository_selection": "selected", + "access_tokens_url": "https://octocoders.github.io/api/v3/app/installations/1/access_tokens", + "repositories_url": "https://octocoders.github.io/api/v3/installation/repositories", + "html_url": "https://octocoders.github.io/settings/installations/1", + "app_id": 282, + "app_slug": "vela", + "target_id": 10919, + "target_type": "User", + "permissions": { + "checks": "write", + "contents": "read", + "metadata": "read" + }, + "events": [ + + ], + "created_at": "2024-10-22T08:50:39.000-05:00", + "updated_at": "2024-10-22T08:50:39.000-05:00", + "single_file_name": null, + "has_multiple_single_files": false, + "single_file_paths": [ + + ], + "suspended_by": null, + "suspended_at": null + }, + "repositories_added": [ + + ], + "repositories_removed": [ + { + "id": 1, + "node_id": "MDEwOlJlcG9zaXRvcnkxMTg=", + "name": "Hello-World", + "full_name": "Codertocat/Hello-World", + "private": true + }, + { + "id": 2, + "node_id": "MDEwOlJlcG9zaXRvcnk0MjI0MzE=", + "name": "Hello-World2", + "full_name": "Codertocat/Hello-World2", + "private": false + } + ], + "requester": null, + "enterprise": { + "id": 1, + "slug": "github", + "name": "GitHub", + "node_id": "MDEwOkVudGVycHJpc2Ux", + "avatar_url": "https://octocoders.github.io/avatars/b/1?", + "description": null, + "website_url": null, + "html_url": "https://octocoders.github.io/businesses/github", + "created_at": "2018-10-24T21:19:19Z", + "updated_at": "2023-06-01T21:03:12Z" + }, + "sender": { + "login": "Codertocat", + "id": 4, + "node_id": "MDQ6VXNlcjQ=", + "avatar_url": "https://octocoders.github.io/avatars/u/4?", + "gravatar_id": "", + "url": "https://octocoders.github.io/api/v3/users/Codertocat", + "html_url": "https://octocoders.github.io/Codertocat", + "followers_url": "https://octocoders.github.io/api/v3/users/Codertocat/followers", + "following_url": "https://octocoders.github.io/api/v3/users/Codertocat/following{/other_user}", + "gists_url": "https://octocoders.github.io/api/v3/users/Codertocat/gists{/gist_id}", + "starred_url": "https://octocoders.github.io/api/v3/users/Codertocat/starred{/owner}{/repo}", + "subscriptions_url": "https://octocoders.github.io/api/v3/users/Codertocat/subscriptions", + "organizations_url": "https://octocoders.github.io/api/v3/users/Codertocat/orgs", + "repos_url": "https://octocoders.github.io/api/v3/users/Codertocat/repos", + "events_url": "https://octocoders.github.io/api/v3/users/Codertocat/events{/privacy}", + "received_events_url": "https://octocoders.github.io/api/v3/users/Codertocat/received_events", + "type": "User", + "site_admin": false + } + } \ No newline at end of file diff --git a/scm/github/testdata/installation_repositories.json b/scm/github/testdata/installation_repositories.json new file mode 100644 index 000000000..9eb501cb5 --- /dev/null +++ b/scm/github/testdata/installation_repositories.json @@ -0,0 +1,123 @@ +{ + "total_count": 1, + "repositories": [ + { + "id": 1296269, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5", + "name": "Hello-World", + "full_name": "octocat/Hello-World", + "owner": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/octocat/Hello-World", + "description": "This your first repo!", + "fork": false, + "url": "https://api.github.com/repos/octocat/Hello-World", + "archive_url": "https://api.github.com/repos/octocat/Hello-World/{archive_format}{/ref}", + "assignees_url": "https://api.github.com/repos/octocat/Hello-World/assignees{/user}", + "blobs_url": "https://api.github.com/repos/octocat/Hello-World/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/octocat/Hello-World/branches{/branch}", + "collaborators_url": "https://api.github.com/repos/octocat/Hello-World/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/octocat/Hello-World/comments{/number}", + "commits_url": "https://api.github.com/repos/octocat/Hello-World/commits{/sha}", + "compare_url": "https://api.github.com/repos/octocat/Hello-World/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/octocat/Hello-World/contents/{+path}", + "contributors_url": "https://api.github.com/repos/octocat/Hello-World/contributors", + "deployments_url": "https://api.github.com/repos/octocat/Hello-World/deployments", + "downloads_url": "https://api.github.com/repos/octocat/Hello-World/downloads", + "events_url": "https://api.github.com/repos/octocat/Hello-World/events", + "forks_url": "https://api.github.com/repos/octocat/Hello-World/forks", + "git_commits_url": "https://api.github.com/repos/octocat/Hello-World/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/octocat/Hello-World/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/octocat/Hello-World/git/tags{/sha}", + "git_url": "git:github.com/octocat/Hello-World.git", + "issue_comment_url": "https://api.github.com/repos/octocat/Hello-World/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/octocat/Hello-World/issues/events{/number}", + "issues_url": "https://api.github.com/repos/octocat/Hello-World/issues{/number}", + "keys_url": "https://api.github.com/repos/octocat/Hello-World/keys{/key_id}", + "labels_url": "https://api.github.com/repos/octocat/Hello-World/labels{/name}", + "languages_url": "https://api.github.com/repos/octocat/Hello-World/languages", + "merges_url": "https://api.github.com/repos/octocat/Hello-World/merges", + "milestones_url": "https://api.github.com/repos/octocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/octocat/Hello-World/notifications{?since,all,participating}", + "pulls_url": "https://api.github.com/repos/octocat/Hello-World/pulls{/number}", + "releases_url": "https://api.github.com/repos/octocat/Hello-World/releases{/id}", + "ssh_url": "git@github.com:octocat/Hello-World.git", + "stargazers_url": "https://api.github.com/repos/octocat/Hello-World/stargazers", + "statuses_url": "https://api.github.com/repos/octocat/Hello-World/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/octocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/octocat/Hello-World/subscription", + "tags_url": "https://api.github.com/repos/octocat/Hello-World/tags", + "teams_url": "https://api.github.com/repos/octocat/Hello-World/teams", + "trees_url": "https://api.github.com/repos/octocat/Hello-World/git/trees{/sha}", + "clone_url": "https://github.com/octocat/Hello-World.git", + "mirror_url": "git:git.example.com/octocat/Hello-World", + "hooks_url": "https://api.github.com/repos/octocat/Hello-World/hooks", + "svn_url": "https://svn.github.com/octocat/Hello-World", + "homepage": "https://github.com", + "language": null, + "forks_count": 9, + "stargazers_count": 80, + "watchers_count": 80, + "size": 108, + "default_branch": "master", + "open_issues_count": 0, + "is_template": true, + "topics": [ + "octocat", + "atom", + "electron", + "api" + ], + "has_issues": true, + "has_projects": true, + "has_wiki": true, + "has_pages": false, + "has_downloads": true, + "archived": false, + "disabled": false, + "visibility": "public", + "pushed_at": "2011-01-26T19:06:43Z", + "created_at": "2011-01-26T19:01:12Z", + "updated_at": "2011-01-26T19:14:43Z", + "allow_rebase_merge": true, + "template_repository": null, + "temp_clone_token": "ABTLWHOULUVAXGTRYU7OC2876QJ2O", + "allow_squash_merge": true, + "allow_auto_merge": false, + "delete_branch_on_merge": true, + "allow_merge_commit": true, + "subscribers_count": 42, + "network_count": 0, + "license": { + "key": "mit", + "name": "MIT License", + "url": "https://api.github.com/licenses/mit", + "spdx_id": "MIT", + "node_id": "MDc6TGljZW5zZW1pdA==", + "html_url": "https://github.com/licenses/mit" + }, + "forks": 1, + "open_issues": 1, + "watchers": 1 + } + ] + } \ No newline at end of file diff --git a/scm/github/testdata/installations.json b/scm/github/testdata/installations.json new file mode 100644 index 000000000..736849702 --- /dev/null +++ b/scm/github/testdata/installations.json @@ -0,0 +1,52 @@ +[ + { + "id": 1, + "account": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "access_tokens_url": "https://api.github.com/app/installations/1/access_tokens", + "repositories_url": "https://api.github.com/installation/repositories", + "html_url": "https://github.com/organizations/github/settings/installations/1", + "app_id": 1, + "target_id": 1, + "target_type": "Organization", + "permissions": { + "checks": "write", + "metadata": "read", + "contents": "read" + }, + "events": [ + "push", + "pull_request" + ], + "single_file_name": "config.yaml", + "has_multiple_single_files": true, + "single_file_paths": [ + "config.yml", + ".github/issue_TEMPLATE.md" + ], + "repository_selection": "selected", + "created_at": "2017-07-08T16:18:44-04:00", + "updated_at": "2017-07-08T16:18:44-04:00", + "app_slug": "github-actions", + "suspended_at": null, + "suspended_by": null + } +] \ No newline at end of file diff --git a/scm/github/testdata/installations_access_tokens.json b/scm/github/testdata/installations_access_tokens.json new file mode 100644 index 000000000..86b705880 --- /dev/null +++ b/scm/github/testdata/installations_access_tokens.json @@ -0,0 +1,134 @@ +{ + "token": "ghs_16C7e42F292c6912E7710c838347Ae178B4a", + "expires_at": "2016-07-11T22:14:10Z", + "permissions": { + "issues": "write", + "contents": "read" + }, + "repository_selection": "selected", + "repositories": [ + { + "id": 1296269, + "node_id": "MDEwOlJlcG9zaXRvcnkxMjk2MjY5", + "name": "Hello-World", + "full_name": "octocat/Hello-World", + "owner": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "private": false, + "html_url": "https://github.com/octocat/Hello-World", + "description": "This your first repo!", + "fork": false, + "url": "https://api.github.com/repos/octocat/Hello-World", + "archive_url": "https://api.github.com/repos/octocat/Hello-World/{archive_format}{/ref}", + "assignees_url": "https://api.github.com/repos/octocat/Hello-World/assignees{/user}", + "blobs_url": "https://api.github.com/repos/octocat/Hello-World/git/blobs{/sha}", + "branches_url": "https://api.github.com/repos/octocat/Hello-World/branches{/branch}", + "collaborators_url": "https://api.github.com/repos/octocat/Hello-World/collaborators{/collaborator}", + "comments_url": "https://api.github.com/repos/octocat/Hello-World/comments{/number}", + "commits_url": "https://api.github.com/repos/octocat/Hello-World/commits{/sha}", + "compare_url": "https://api.github.com/repos/octocat/Hello-World/compare/{base}...{head}", + "contents_url": "https://api.github.com/repos/octocat/Hello-World/contents/{+path}", + "contributors_url": "https://api.github.com/repos/octocat/Hello-World/contributors", + "deployments_url": "https://api.github.com/repos/octocat/Hello-World/deployments", + "downloads_url": "https://api.github.com/repos/octocat/Hello-World/downloads", + "events_url": "https://api.github.com/repos/octocat/Hello-World/events", + "forks_url": "https://api.github.com/repos/octocat/Hello-World/forks", + "git_commits_url": "https://api.github.com/repos/octocat/Hello-World/git/commits{/sha}", + "git_refs_url": "https://api.github.com/repos/octocat/Hello-World/git/refs{/sha}", + "git_tags_url": "https://api.github.com/repos/octocat/Hello-World/git/tags{/sha}", + "git_url": "git:github.com/octocat/Hello-World.git", + "issue_comment_url": "https://api.github.com/repos/octocat/Hello-World/issues/comments{/number}", + "issue_events_url": "https://api.github.com/repos/octocat/Hello-World/issues/events{/number}", + "issues_url": "https://api.github.com/repos/octocat/Hello-World/issues{/number}", + "keys_url": "https://api.github.com/repos/octocat/Hello-World/keys{/key_id}", + "labels_url": "https://api.github.com/repos/octocat/Hello-World/labels{/name}", + "languages_url": "https://api.github.com/repos/octocat/Hello-World/languages", + "merges_url": "https://api.github.com/repos/octocat/Hello-World/merges", + "milestones_url": "https://api.github.com/repos/octocat/Hello-World/milestones{/number}", + "notifications_url": "https://api.github.com/repos/octocat/Hello-World/notifications{?since,all,participating}", + "pulls_url": "https://api.github.com/repos/octocat/Hello-World/pulls{/number}", + "releases_url": "https://api.github.com/repos/octocat/Hello-World/releases{/id}", + "ssh_url": "git@github.com:octocat/Hello-World.git", + "stargazers_url": "https://api.github.com/repos/octocat/Hello-World/stargazers", + "statuses_url": "https://api.github.com/repos/octocat/Hello-World/statuses/{sha}", + "subscribers_url": "https://api.github.com/repos/octocat/Hello-World/subscribers", + "subscription_url": "https://api.github.com/repos/octocat/Hello-World/subscription", + "tags_url": "https://api.github.com/repos/octocat/Hello-World/tags", + "teams_url": "https://api.github.com/repos/octocat/Hello-World/teams", + "trees_url": "https://api.github.com/repos/octocat/Hello-World/git/trees{/sha}", + "clone_url": "https://github.com/octocat/Hello-World.git", + "mirror_url": "git:git.example.com/octocat/Hello-World", + "hooks_url": "https://api.github.com/repos/octocat/Hello-World/hooks", + "svn_url": "https://svn.github.com/octocat/Hello-World", + "homepage": "https://github.com", + "language": null, + "forks_count": 9, + "stargazers_count": 80, + "watchers_count": 80, + "size": 108, + "default_branch": "master", + "open_issues_count": 0, + "is_template": true, + "topics": [ + "octocat", + "atom", + "electron", + "api" + ], + "has_issues": true, + "has_projects": true, + "has_wiki": true, + "has_pages": false, + "has_downloads": true, + "archived": false, + "disabled": false, + "visibility": "public", + "pushed_at": "2011-01-26T19:06:43Z", + "created_at": "2011-01-26T19:01:12Z", + "updated_at": "2011-01-26T19:14:43Z", + "permissions": { + "admin": false, + "push": false, + "pull": true + }, + "allow_rebase_merge": true, + "template_repository": null, + "temp_clone_token": "ABTLWHOULUVAXGTRYU7OC2876QJ2O", + "allow_squash_merge": true, + "allow_auto_merge": false, + "delete_branch_on_merge": true, + "allow_merge_commit": true, + "subscribers_count": 42, + "network_count": 0, + "license": { + "key": "mit", + "name": "MIT License", + "url": "https://api.github.com/licenses/mit", + "spdx_id": "MIT", + "node_id": "MDc6TGljZW5zZW1pdA==", + "html_url": "https://github.com/licenses/mit" + }, + "forks": 1, + "open_issues": 1, + "watchers": 1 + } + ] + } \ No newline at end of file diff --git a/scm/github/user.go b/scm/github/user.go index d014256e4..3ceaf75ac 100644 --- a/scm/github/user.go +++ b/scm/github/user.go @@ -16,7 +16,7 @@ func (c *client) GetUserID(ctx context.Context, name string, token string) (stri }).Tracef("capturing SCM user id for %s", name) // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, token) + client := c.newOAuthTokenClient(ctx, token) // send API call to capture user user, _, err := client.Users.Get(ctx, name) diff --git a/scm/github/webhook.go b/scm/github/webhook.go index 61bdd2135..a23c6be1b 100644 --- a/scm/github/webhook.go +++ b/scm/github/webhook.go @@ -60,7 +60,6 @@ func (c *client) ProcessWebhook(ctx context.Context, request *http.Request) (*in // parse the payload from the webhook event, err := github.ParseWebHook(github.WebHookType(request), payload) - if err != nil { return &internal.Webhook{Hook: h}, nil } @@ -77,6 +76,10 @@ func (c *client) ProcessWebhook(ctx context.Context, request *http.Request) (*in return c.processIssueCommentEvent(h, event) case *github.RepositoryEvent: return c.processRepositoryEvent(h, event) + case *github.InstallationEvent: + return c.processInstallationEvent(ctx, h, event) + case *github.InstallationRepositoriesEvent: + return c.processInstallationRepositoriesEvent(ctx, h, event) } return &internal.Webhook{Hook: h}, nil @@ -100,7 +103,7 @@ func (c *client) VerifyWebhook(_ context.Context, request *http.Request, r *api. // RedeliverWebhook redelivers webhooks from GitHub. func (c *client) RedeliverWebhook(ctx context.Context, u *api.User, h *api.Hook) error { // create GitHub OAuth client with user's token - client := c.newClientToken(ctx, u.GetToken()) + client := c.newOAuthTokenClient(ctx, u.GetToken()) // capture the delivery ID of the hook using GitHub API deliveryID, err := c.getDeliveryID(ctx, client, h) @@ -512,7 +515,6 @@ func (c *client) processIssueCommentEvent(h *api.Hook, payload *github.IssueComm } // processRepositoryEvent is a helper function to process the repository event. - func (c *client) processRepositoryEvent(h *api.Hook, payload *github.RepositoryEvent) (*internal.Webhook, error) { logrus.Tracef("processing repository event GitHub webhook for %s", payload.GetRepo().GetFullName()) @@ -543,6 +545,59 @@ func (c *client) processRepositoryEvent(h *api.Hook, payload *github.RepositoryE }, nil } +// processInstallationEvent is a helper function to process the installation event. +func (c *client) processInstallationEvent(_ context.Context, h *api.Hook, payload *github.InstallationEvent) (*internal.Webhook, error) { + h.SetEvent(constants.EventInstallation) + h.SetEventAction(payload.GetAction()) + + install := new(internal.Installation) + + install.Action = payload.GetAction() + install.ID = payload.GetInstallation().GetID() + install.Org = payload.GetInstallation().GetAccount().GetLogin() + + switch payload.GetAction() { + case constants.AppInstallCreated: + for _, repo := range payload.Repositories { + install.RepositoriesAdded = append(install.RepositoriesAdded, repo.GetName()) + } + case constants.AppInstallDeleted: + for _, repo := range payload.Repositories { + install.RepositoriesRemoved = append(install.RepositoriesRemoved, repo.GetName()) + } + } + + return &internal.Webhook{ + Hook: h, + Installation: install, + }, nil +} + +// processInstallationRepositoriesEvent is a helper function to process the installation repositories event. +func (c *client) processInstallationRepositoriesEvent(_ context.Context, h *api.Hook, payload *github.InstallationRepositoriesEvent) (*internal.Webhook, error) { + h.SetEvent(constants.EventInstallationRepositories) + h.SetEventAction(payload.GetAction()) + + install := new(internal.Installation) + + install.Action = payload.GetAction() + install.ID = payload.GetInstallation().GetID() + install.Org = payload.GetInstallation().GetAccount().GetLogin() + + for _, repo := range payload.RepositoriesAdded { + install.RepositoriesAdded = append(install.RepositoriesAdded, repo.GetName()) + } + + for _, repo := range payload.RepositoriesRemoved { + install.RepositoriesRemoved = append(install.RepositoriesRemoved, repo.GetName()) + } + + return &internal.Webhook{ + Hook: h, + Installation: install, + }, nil +} + // getDeliveryID gets the last 100 webhook deliveries for a repo and // finds the matching delivery id with the source id in the hook. func (c *client) getDeliveryID(ctx context.Context, ghClient *github.Client, h *api.Hook) (int64, error) { diff --git a/scm/github/webhook_test.go b/scm/github/webhook_test.go index c736a94d2..b9a118baa 100644 --- a/scm/github/webhook_test.go +++ b/scm/github/webhook_test.go @@ -1200,8 +1200,8 @@ func TestGitHub_ProcessWebhook_RepositoryRename(t *testing.T) { t.Errorf("ProcessWebhook returned err: %v", err) } - if !reflect.DeepEqual(got, want) { - t.Errorf("ProcessWebhook is %v, want %v", got, want) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) } } @@ -1263,8 +1263,8 @@ func TestGitHub_ProcessWebhook_RepositoryTransfer(t *testing.T) { t.Errorf("ProcessWebhook returned err: %v", err) } - if !reflect.DeepEqual(got, want) { - t.Errorf("ProcessWebhook is %v, want %v", got, want) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) } } @@ -1326,8 +1326,8 @@ func TestGitHub_ProcessWebhook_RepositoryArchived(t *testing.T) { t.Errorf("ProcessWebhook returned err: %v", err) } - if !reflect.DeepEqual(got, want) { - t.Errorf("ProcessWebhook is %v, want %v", got, want) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) } } @@ -1389,8 +1389,8 @@ func TestGitHub_ProcessWebhook_RepositoryEdited(t *testing.T) { t.Errorf("ProcessWebhook returned err: %v", err) } - if !reflect.DeepEqual(got, want) { - t.Errorf("ProcessWebhook is %v, want %v", got, want) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) } } @@ -1452,8 +1452,8 @@ func TestGitHub_ProcessWebhook_Repository(t *testing.T) { t.Errorf("ProcessWebhook returned err: %v", err) } - if !reflect.DeepEqual(got, want) { - t.Errorf("ProcessWebhook is %v, want %v", got, want) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) } } @@ -1545,7 +1545,7 @@ func TestGithub_GetDeliveryID(t *testing.T) { client, _ := NewTest(s.URL, "https://foo.bar.com") - ghClient := client.newClientToken(context.Background(), *u.Token) + ghClient := client.newOAuthTokenClient(context.Background(), *u.Token) // run test got, err := client.getDeliveryID(context.TODO(), ghClient, _hook) @@ -1558,3 +1558,180 @@ func TestGithub_GetDeliveryID(t *testing.T) { t.Errorf("getDeliveryID returned: %v; want: %v", got, want) } } + +func TestGitHub_ProcessWebhook_Installation(t *testing.T) { + // setup tests + var createdHook api.Hook + createdHook.SetNumber(1) + createdHook.SetSourceID("7bd477e4-4415-11e9-9359-0d41fdf9567e") + createdHook.SetWebhookID(123456) + createdHook.SetCreated(time.Now().UTC().Unix()) + createdHook.SetHost("github.com") + createdHook.SetEvent(constants.EventInstallation) + createdHook.SetEventAction(constants.AppInstallCreated) + createdHook.SetStatus(constants.StatusSuccess) + + deletedHook := createdHook + deletedHook.SetEventAction(constants.AppInstallDeleted) + + tests := []struct { + name string + file string + wantHook *api.Hook + wantInstall *internal.Installation + }{ + { + name: "installation created", + file: "testdata/hooks/installation_created.json", + wantHook: &createdHook, + wantInstall: &internal.Installation{ + Action: constants.AppInstallCreated, + ID: 1, + RepositoriesAdded: []string{"Hello-World", "Hello-World2"}, + Org: "Codertocat", + }, + }, + { + name: "installation deleted", + file: "testdata/hooks/installation_deleted.json", + wantHook: &deletedHook, + wantInstall: &internal.Installation{ + Action: constants.AppInstallDeleted, + ID: 1, + RepositoriesRemoved: []string{"Hello-World", "Hello-World2"}, + Org: "Codertocat", + }, + }, + } + + // setup router + s := httptest.NewServer(http.NotFoundHandler()) + defer s.Close() + + for _, tt := range tests { + // setup request + body, err := os.Open(tt.file) + if err != nil { + t.Errorf("unable to open file: %v", err) + } + + defer body.Close() + + request, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/test", body) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("User-Agent", "GitHub-Hookshot/a22606a") + request.Header.Set("X-GitHub-Delivery", "7bd477e4-4415-11e9-9359-0d41fdf9567e") + request.Header.Set("X-GitHub-Hook-ID", "123456") + request.Header.Set("X-GitHub-Event", "installation") + + // setup client + client, _ := NewTest(s.URL) + + want := &internal.Webhook{ + Hook: tt.wantHook, + Installation: tt.wantInstall, + } + + got, err := client.ProcessWebhook(context.TODO(), request) + + if err != nil { + t.Errorf("ProcessWebhook returned err: %v", err) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) + } + } +} + +const ( + // GitHub App install event type 'added'. + AppInstallRepositoriesAdded = "added" + // GitHub App install event type 'removed'. + AppInstallRepositoriesRemoved = "removed" +) + +func TestGitHub_ProcessWebhook_InstallationRepositories(t *testing.T) { + // setup tests + var reposAddedHook api.Hook + reposAddedHook.SetNumber(1) + reposAddedHook.SetSourceID("7bd477e4-4415-11e9-9359-0d41fdf9567e") + reposAddedHook.SetWebhookID(123456) + reposAddedHook.SetCreated(time.Now().UTC().Unix()) + reposAddedHook.SetHost("github.com") + reposAddedHook.SetEvent(constants.EventInstallationRepositories) + reposAddedHook.SetEventAction(AppInstallRepositoriesAdded) + reposAddedHook.SetStatus(constants.StatusSuccess) + + reposRemovedHook := reposAddedHook + reposRemovedHook.SetEventAction(AppInstallRepositoriesRemoved) + + tests := []struct { + name string + file string + wantHook *api.Hook + wantInstall *internal.Installation + }{ + { + name: "installation_repositories repos added", + file: "testdata/hooks/installation_repositories_added.json", + wantHook: &reposAddedHook, + wantInstall: &internal.Installation{ + Action: AppInstallRepositoriesAdded, + ID: 1, + RepositoriesAdded: []string{"Hello-World", "Hello-World2"}, + Org: "Codertocat", + }, + }, + { + name: "installation_repositories repos removed", + file: "testdata/hooks/installation_repositories_removed.json", + wantHook: &reposRemovedHook, + wantInstall: &internal.Installation{ + Action: AppInstallRepositoriesRemoved, + ID: 1, + RepositoriesRemoved: []string{"Hello-World", "Hello-World2"}, + Org: "Codertocat", + }, + }, + } + + // setup router + s := httptest.NewServer(http.NotFoundHandler()) + defer s.Close() + + for _, tt := range tests { + // setup request + body, err := os.Open(tt.file) + if err != nil { + t.Errorf("unable to open file: %v", err) + } + + defer body.Close() + + request, _ := http.NewRequestWithContext(context.Background(), http.MethodGet, "/test", body) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("User-Agent", "GitHub-Hookshot/a22606a") + request.Header.Set("X-GitHub-Delivery", "7bd477e4-4415-11e9-9359-0d41fdf9567e") + request.Header.Set("X-GitHub-Hook-ID", "123456") + request.Header.Set("X-GitHub-Event", "installation_repositories") + + // setup client + client, _ := NewTest(s.URL) + + want := &internal.Webhook{ + Hook: tt.wantHook, + Installation: tt.wantInstall, + } + + got, err := client.ProcessWebhook(context.TODO(), request) + + if err != nil { + t.Errorf("ProcessWebhook returned err: %v", err) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("ProcessWebhook() mismatch (-want +got):\n%s", diff) + } + } +} diff --git a/scm/scm.go b/scm/scm.go index b93bde906..037a5d835 100644 --- a/scm/scm.go +++ b/scm/scm.go @@ -3,6 +3,7 @@ package scm import ( + "context" "fmt" "github.com/sirupsen/logrus" @@ -17,7 +18,7 @@ import ( // // * Github // . -func New(s *Setup) (Service, error) { +func New(ctx context.Context, s *Setup) (Service, error) { // validate the setup being provided // // https://pkg.go.dev/github.com/go-vela/server/scm?tab=doc#Setup.Validate @@ -33,12 +34,12 @@ func New(s *Setup) (Service, error) { // handle the Github scm driver being provided // // https://pkg.go.dev/github.com/go-vela/server/scm?tab=doc#Setup.Github - return s.Github() + return s.Github(ctx) case constants.DriverGitlab: // handle the Gitlab scm driver being provided // // https://pkg.go.dev/github.com/go-vela/server/scm?tab=doc#Setup.Gitlab - return s.Gitlab() + return s.Gitlab(ctx) default: // handle an invalid scm driver being provided return nil, fmt.Errorf("invalid scm driver provided: %s", s.Driver) diff --git a/scm/scm_test.go b/scm/scm_test.go index 233108758..974c85b45 100644 --- a/scm/scm_test.go +++ b/scm/scm_test.go @@ -3,6 +3,7 @@ package scm import ( + "context" "testing" ) @@ -23,7 +24,7 @@ func TestSCM_New(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, }, }, { @@ -37,7 +38,7 @@ func TestSCM_New(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, }, }, { @@ -51,7 +52,7 @@ func TestSCM_New(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, }, }, { @@ -65,14 +66,14 @@ func TestSCM_New(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, }, }, } // run tests for _, test := range tests { - _, err := New(test.setup) + _, err := New(context.Background(), test.setup) if test.failure { if err == nil { diff --git a/scm/service.go b/scm/service.go index bc917ce9a..7d1d12ec6 100644 --- a/scm/service.go +++ b/scm/service.go @@ -7,6 +7,8 @@ import ( "net/http" api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/database" "github.com/go-vela/server/internal" ) @@ -140,6 +142,12 @@ type Service interface { // GetHTMLURL defines a function that retrieves // a repository file's html_url. GetHTMLURL(context.Context, *api.User, string, string, string, string) (string, error) + // GetNetrcPassword defines a function that returns the netrc + // password injected into build steps. + GetNetrcPassword(context.Context, database.Interface, *api.Repo, *api.User, yaml.Git) (string, error) + // SyncRepoWithInstallation defines a function that syncs + // a repo with the installation, if it exists. + SyncRepoWithInstallation(context.Context, *api.Repo) (*api.Repo, error) // Webhook SCM Interface Functions @@ -153,5 +161,14 @@ type Service interface { // redelivers the webhook from the SCM. RedeliverWebhook(context.Context, *api.User, *api.Hook) error + // App Integration SCM Interface Functions + + // ProcessInstallation defines a function that + // processes an installation event. + ProcessInstallation(context.Context, *http.Request, *internal.Webhook, database.Interface) error + // FinishInstallation defines a function that + // finishes an installation event and returns a web redirect. + FinishInstallation(context.Context, *http.Request, int64) (string, error) + // TODO: Add convert functions to interface? } diff --git a/scm/setup.go b/scm/setup.go index dc3a3e698..44704fffd 100644 --- a/scm/setup.go +++ b/scm/setup.go @@ -3,6 +3,7 @@ package scm import ( + "context" "fmt" "strings" @@ -27,6 +28,14 @@ type Setup struct { ClientID string // specifies the OAuth client secret from the scm system to use for the scm client ClientSecret string + // specifies App integration id + AppID int64 + // specifies App integration private key + AppPrivateKey string + // specifies App integration path to private key + AppPrivateKeyPath string + // specifies App integration permissions set + AppPermissions []string // specifies the Vela server address to use for the scm client ServerAddress string // specifies the Vela server address that the scm provider should use to send Vela webhooks @@ -36,20 +45,21 @@ type Setup struct { // specifies the Vela web UI address to use for the scm client WebUIAddress string // specifies the OAuth scopes to use for the scm client - Scopes []string + OAuthScopes []string // specifies OTel tracing configurations Tracing *tracing.Client } // Github creates and returns a Vela service capable of // integrating with a Github scm system. -func (s *Setup) Github() (Service, error) { +func (s *Setup) Github(ctx context.Context) (Service, error) { logrus.Trace("creating github scm client from setup") // create new Github scm service // // https://pkg.go.dev/github.com/go-vela/server/scm/github?tab=doc#New return github.New( + ctx, github.WithAddress(s.Address), github.WithClientID(s.ClientID), github.WithClientSecret(s.ClientSecret), @@ -57,14 +67,18 @@ func (s *Setup) Github() (Service, error) { github.WithServerWebhookAddress(s.ServerWebhookAddress), github.WithStatusContext(s.StatusContext), github.WithWebUIAddress(s.WebUIAddress), - github.WithScopes(s.Scopes), + github.WithOAuthScopes(s.OAuthScopes), github.WithTracing(s.Tracing), + github.WithGithubAppID(s.AppID), + github.WithGithubPrivateKey(s.AppPrivateKey), + github.WithGithubPrivateKeyPath(s.AppPrivateKeyPath), + github.WithGitHubAppPermissions(s.AppPermissions), ) } // Gitlab creates and returns a Vela service capable of // integrating with a Gitlab scm system. -func (s *Setup) Gitlab() (Service, error) { +func (s *Setup) Gitlab(_ context.Context) (Service, error) { logrus.Trace("creating gitlab scm client from setup") return nil, fmt.Errorf("unsupported scm driver: %s", constants.DriverGitlab) @@ -110,7 +124,7 @@ func (s *Setup) Validate() error { return fmt.Errorf("no scm status context provided") } - if len(s.Scopes) == 0 { + if len(s.OAuthScopes) == 0 { return fmt.Errorf("no scm scopes provided") } diff --git a/scm/setup_test.go b/scm/setup_test.go index f3ad759a4..3cb381598 100644 --- a/scm/setup_test.go +++ b/scm/setup_test.go @@ -3,6 +3,7 @@ package scm import ( + "context" "reflect" "testing" ) @@ -18,10 +19,10 @@ func TestSCM_Setup_Github(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, } - _github, err := _setup.Github() + _github, err := _setup.Github(context.Background()) if err != nil { t.Errorf("unable to setup scm: %v", err) } @@ -46,7 +47,7 @@ func TestSCM_Setup_Github(t *testing.T) { // run tests for _, test := range tests { - got, err := test.setup.Github() + got, err := test.setup.Github(context.Background()) if test.failure { if err == nil { @@ -80,7 +81,7 @@ func TestSCM_Setup_Gitlab(t *testing.T) { } // run test - got, err := _setup.Gitlab() + got, err := _setup.Gitlab(context.Background()) if err == nil { t.Errorf("Gitlab should have returned err") } @@ -107,7 +108,7 @@ func TestSCM_Setup_Validate(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, }, }, { @@ -121,7 +122,7 @@ func TestSCM_Setup_Validate(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, }, }, { @@ -135,7 +136,7 @@ func TestSCM_Setup_Validate(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, }, }, { @@ -149,7 +150,7 @@ func TestSCM_Setup_Validate(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, }, }, { @@ -163,7 +164,7 @@ func TestSCM_Setup_Validate(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, }, }, { @@ -177,7 +178,7 @@ func TestSCM_Setup_Validate(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, }, }, { @@ -191,7 +192,7 @@ func TestSCM_Setup_Validate(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, }, }, { @@ -205,7 +206,7 @@ func TestSCM_Setup_Validate(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, }, }, { @@ -219,7 +220,7 @@ func TestSCM_Setup_Validate(t *testing.T) { ServerWebhookAddress: "", StatusContext: "", WebUIAddress: "https://vela.example.com", - Scopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, + OAuthScopes: []string{"repo", "repo:status", "user:email", "read:user", "read:org"}, }, }, { @@ -233,7 +234,7 @@ func TestSCM_Setup_Validate(t *testing.T) { ServerWebhookAddress: "", StatusContext: "continuous-integration/vela", WebUIAddress: "https://vela.example.com", - Scopes: []string{}, + OAuthScopes: []string{}, }, }, } From fa93fa875ade5508e90dc82032a0a6ac505a8949 Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Mon, 23 Dec 2024 11:38:34 -0600 Subject: [PATCH 2/2] chore(yaml): add go-yaml types package (#1225) * enhance(yaml): allow for users to parse pipelines using old library * testing file for internal yaml * chore(compiler): convert unmarshaled buildkite to go-yaml * remove tests used in later PRs * lintfix * fix schema * gci --- api/pipeline/template.go | 2 +- api/types/string.go | 2 +- compiler/engine.go | 2 +- compiler/native/clone.go | 2 +- compiler/native/clone_test.go | 2 +- compiler/native/compile.go | 4 +- compiler/native/compile_test.go | 8 +- compiler/native/environment.go | 2 +- compiler/native/environment_test.go | 2 +- compiler/native/expand.go | 2 +- compiler/native/expand_test.go | 2 +- compiler/native/initialize.go | 2 +- compiler/native/initialize_test.go | 2 +- compiler/native/parse.go | 25 +- compiler/native/parse_test.go | 2 +- compiler/native/script.go | 2 +- compiler/native/script_test.go | 2 +- compiler/native/substitute.go | 4 +- compiler/native/substitute_test.go | 2 +- compiler/native/transform.go | 15 +- compiler/native/transform_test.go | 2 +- compiler/native/validate.go | 2 +- compiler/native/validate_test.go | 2 +- compiler/template/native/convert.go | 2 +- compiler/template/native/render.go | 11 +- compiler/template/native/render_test.go | 4 +- compiler/template/starlark/render.go | 13 +- compiler/template/starlark/render_test.go | 4 +- compiler/template/template.go | 2 +- compiler/types/raw/map_test.go | 2 +- compiler/types/raw/slice_test.go | 2 +- compiler/types/yaml/buildkite/build.go | 122 ++++ .../types/yaml/{ => buildkite}/build_test.go | 2 +- compiler/types/yaml/buildkite/deployment.go | 112 +++ .../types/yaml/buildkite/deployment_test.go | 77 ++ compiler/types/yaml/buildkite/doc.go | 8 + compiler/types/yaml/buildkite/git.go | 44 ++ compiler/types/yaml/buildkite/git_test.go | 60 ++ compiler/types/yaml/buildkite/metadata.go | 108 +++ .../types/yaml/buildkite/metadata_test.go | 130 ++++ compiler/types/yaml/buildkite/ruleset.go | 223 ++++++ .../yaml/{ => buildkite}/ruleset_test.go | 2 +- compiler/types/yaml/buildkite/secret.go | 328 +++++++++ .../types/yaml/{ => buildkite}/secret_test.go | 2 +- compiler/types/yaml/buildkite/service.go | 164 +++++ .../yaml/{ => buildkite}/service_test.go | 2 +- compiler/types/yaml/{ => buildkite}/stage.go | 42 +- .../types/yaml/{ => buildkite}/stage_test.go | 2 +- compiler/types/yaml/buildkite/step.go | 188 +++++ .../types/yaml/{ => buildkite}/step_test.go | 2 +- compiler/types/yaml/buildkite/template.go | 121 +++ .../yaml/{ => buildkite}/template_test.go | 2 +- .../yaml/{ => buildkite}/testdata/build.yml | 0 .../testdata/build/validate/bad_pipeline0.yml | 0 .../testdata/build/validate/bad_pipeline1.yml | 0 .../testdata/build/validate/bad_version.yml | 0 .../testdata/build/validate/step.yml | 0 .../testdata/build_anchor_stage.yml | 0 .../testdata/build_anchor_step.yml | 0 .../testdata/build_empty_env.yml | 0 .../testdata/build_with_deploy_config.yml | 0 .../testdata/deploy_parameter.yml | 0 .../yaml/{ => buildkite}/testdata/invalid.yml | 0 .../{ => buildkite}/testdata/merge_anchor.yml | 0 .../{ => buildkite}/testdata/metadata.yml | 0 .../{ => buildkite}/testdata/metadata_env.yml | 0 .../testdata/ruleset_advanced.yml | 0 .../testdata/ruleset_regex.yml | 0 .../testdata/ruleset_simple.yml | 0 .../yaml/{ => buildkite}/testdata/secret.yml | 0 .../testdata/secret/validate/no_name.yml | 0 .../testdata/secret/validate/org.yml | 0 .../secret/validate/org_bad_engine.yml | 0 .../testdata/secret/validate/org_bad_key.yml | 0 .../testdata/secret/validate/plugin.yml | 0 .../secret/validate/plugin_bad_image.yml | 0 .../secret/validate/plugin_bad_name.yml | 0 .../testdata/secret/validate/repo.yml | 0 .../secret/validate/repo_bad_engine.yml | 0 .../testdata/secret/validate/repo_bad_key.yml | 0 .../testdata/secret/validate/shared.yml | 0 .../secret/validate/shared_bad_engine.yml | 0 .../secret/validate/shared_bad_key.yml | 0 .../yaml/{ => buildkite}/testdata/service.yml | 0 .../testdata/service/validate/bad_image.yml | 0 .../testdata/service/validate/minimal.yml | 0 .../service/validate/missing_image.yml | 0 .../service/validate/missing_name.yml | 0 .../{ => buildkite}/testdata/service_nil.yml | 0 .../yaml/{ => buildkite}/testdata/stage.yml | 0 .../testdata/stage/validate/bad_image.yml | 0 .../testdata/stage/validate/minimal.yml | 0 .../testdata/stage/validate/missing.yml | 0 .../testdata/stage/validate/missing_image.yml | 0 .../testdata/stage/validate/missing_name.yml | 0 .../yaml/{ => buildkite}/testdata/step.yml | 0 .../testdata/step/validate/bad_image.yml | 0 .../testdata/step/validate/minimal.yml | 0 .../testdata/step/validate/missing.yml | 0 .../testdata/step/validate/missing_image.yml | 0 .../testdata/step/validate/missing_name.yml | 0 .../testdata/step_malformed.yml | 0 .../{ => buildkite}/testdata/step_nil.yml | 0 .../testdata/step_secret_slice.yml | 0 .../step_secret_slice_invalid_no_source.yml | 0 .../step_secret_slice_invalid_no_target.yml | 0 .../testdata/step_secret_string.yml | 0 .../{ => buildkite}/testdata/template.yml | 0 .../testdata/ulimit_colon_error.yml | 0 .../testdata/ulimit_equal_error.yml | 0 .../testdata/ulimit_hardlimit1_error.yml | 0 .../testdata/ulimit_hardlimit2_error.yml | 0 .../{ => buildkite}/testdata/ulimit_slice.yml | 0 .../testdata/ulimit_softlimit_error.yml | 0 .../testdata/ulimit_string.yml | 0 .../{ => buildkite}/testdata/volume_error.yml | 0 .../{ => buildkite}/testdata/volume_slice.yml | 0 .../testdata/volume_string.yml | 0 compiler/types/yaml/buildkite/ulimit.go | 158 ++++ .../types/yaml/{ => buildkite}/ulimit_test.go | 2 +- compiler/types/yaml/buildkite/volume.go | 147 ++++ compiler/types/yaml/buildkite/volume_test.go | 137 ++++ compiler/types/yaml/buildkite/worker.go | 31 + compiler/types/yaml/buildkite/worker_test.go | 38 + compiler/types/yaml/doc.go | 8 - compiler/types/yaml/{ => yaml}/build.go | 0 compiler/types/yaml/yaml/build_test.go | 686 ++++++++++++++++++ compiler/types/yaml/{ => yaml}/deployment.go | 0 .../types/yaml/{ => yaml}/deployment_test.go | 0 compiler/types/yaml/yaml/doc.go | 8 + compiler/types/yaml/{ => yaml}/git.go | 0 compiler/types/yaml/{ => yaml}/git_test.go | 0 compiler/types/yaml/{ => yaml}/metadata.go | 0 .../types/yaml/{ => yaml}/metadata_test.go | 0 compiler/types/yaml/{ => yaml}/ruleset.go | 0 compiler/types/yaml/yaml/ruleset_test.go | 288 ++++++++ compiler/types/yaml/{ => yaml}/secret.go | 0 compiler/types/yaml/yaml/secret_test.go | 460 ++++++++++++ compiler/types/yaml/{ => yaml}/service.go | 0 compiler/types/yaml/yaml/service_test.go | 186 +++++ compiler/types/yaml/yaml/stage.go | 169 +++++ compiler/types/yaml/yaml/stage_test.go | 474 ++++++++++++ compiler/types/yaml/{ => yaml}/step.go | 0 compiler/types/yaml/yaml/step_test.go | 327 +++++++++ compiler/types/yaml/{ => yaml}/template.go | 0 compiler/types/yaml/yaml/template_test.go | 121 +++ compiler/types/yaml/yaml/testdata/build.yml | 144 ++++ .../testdata/build/validate/bad_pipeline0.yml | 1 + .../testdata/build/validate/bad_pipeline1.yml | 3 + .../testdata/build/validate/bad_version.yml | 2 + .../yaml/testdata/build/validate/step.yml | 47 ++ .../yaml/yaml/testdata/build_anchor_stage.yml | 57 ++ .../yaml/yaml/testdata/build_anchor_step.yml | 48 ++ .../yaml/yaml/testdata/build_empty_env.yml | 27 + .../testdata/build_with_deploy_config.yml | 36 + .../yaml/yaml/testdata/deploy_parameter.yml | 11 + compiler/types/yaml/yaml/testdata/invalid.yml | 2 + .../types/yaml/yaml/testdata/merge_anchor.yml | 45 ++ .../types/yaml/yaml/testdata/metadata.yml | 2 + .../types/yaml/yaml/testdata/metadata_env.yml | 3 + .../yaml/yaml/testdata/ruleset_advanced.yml | 15 + .../yaml/yaml/testdata/ruleset_regex.yml | 7 + .../yaml/yaml/testdata/ruleset_simple.yml | 13 + compiler/types/yaml/yaml/testdata/secret.yml | 39 + .../yaml/testdata/secret/validate/no_name.yml | 11 + .../yaml/testdata/secret/validate/org.yml | 5 + .../secret/validate/org_bad_engine.yml | 9 + .../testdata/secret/validate/org_bad_key.yml | 9 + .../yaml/testdata/secret/validate/plugin.yml | 8 + .../secret/validate/plugin_bad_image.yml | 15 + .../secret/validate/plugin_bad_name.yml | 7 + .../yaml/testdata/secret/validate/repo.yml | 13 + .../secret/validate/repo_bad_engine.yml | 5 + .../testdata/secret/validate/repo_bad_key.yml | 14 + .../yaml/testdata/secret/validate/shared.yml | 5 + .../secret/validate/shared_bad_engine.yml | 9 + .../secret/validate/shared_bad_key.yml | 9 + compiler/types/yaml/yaml/testdata/service.yml | 14 + .../testdata/service/validate/bad_image.yml | 3 + .../testdata/service/validate/minimal.yml | 3 + .../service/validate/missing_image.yml | 2 + .../service/validate/missing_name.yml | 2 + .../types/yaml/yaml/testdata/service_nil.yml | 2 + compiler/types/yaml/yaml/testdata/stage.yml | 44 ++ .../testdata/stage/validate/bad_image.yml | 7 + .../yaml/testdata/stage/validate/minimal.yml | 7 + .../yaml/testdata/stage/validate/missing.yml | 5 + .../testdata/stage/validate/missing_image.yml | 6 + .../testdata/stage/validate/missing_name.yml | 6 + compiler/types/yaml/yaml/testdata/step.yml | 46 ++ .../yaml/testdata/step/validate/bad_image.yml | 5 + .../yaml/testdata/step/validate/minimal.yml | 5 + .../yaml/testdata/step/validate/missing.yml | 3 + .../testdata/step/validate/missing_image.yml | 4 + .../testdata/step/validate/missing_name.yml | 4 + .../yaml/yaml/testdata/step_malformed.yml | 4 + .../types/yaml/yaml/testdata/step_nil.yml | 2 + .../yaml/yaml/testdata/step_secret_slice.yml | 5 + .../step_secret_slice_invalid_no_source.yml | 2 + .../step_secret_slice_invalid_no_target.yml | 2 + .../yaml/yaml/testdata/step_secret_string.yml | 2 + .../types/yaml/yaml/testdata/template.yml | 12 + .../yaml/yaml/testdata/ulimit_colon_error.yml | 2 + .../yaml/yaml/testdata/ulimit_equal_error.yml | 2 + .../yaml/testdata/ulimit_hardlimit1_error.yml | 2 + .../yaml/testdata/ulimit_hardlimit2_error.yml | 2 + .../types/yaml/yaml/testdata/ulimit_slice.yml | 6 + .../yaml/testdata/ulimit_softlimit_error.yml | 2 + .../yaml/yaml/testdata/ulimit_string.yml | 2 + .../types/yaml/yaml/testdata/volume_error.yml | 2 + .../types/yaml/yaml/testdata/volume_slice.yml | 7 + .../yaml/yaml/testdata/volume_string.yml | 2 + compiler/types/yaml/{ => yaml}/ulimit.go | 0 compiler/types/yaml/yaml/ulimit_test.go | 147 ++++ compiler/types/yaml/{ => yaml}/volume.go | 0 compiler/types/yaml/{ => yaml}/volume_test.go | 0 compiler/types/yaml/{ => yaml}/worker.go | 0 compiler/types/yaml/{ => yaml}/worker_test.go | 0 go.mod | 2 +- internal/testdata/buildkite.yml | 18 + internal/testdata/go-yaml.yml | 19 + internal/testdata/invalid.yml | 2 + internal/testdata/no_version.yml | 17 + mock/server/pipeline.go | 4 +- schema/pipeline.go | 2 +- scm/github/repo.go | 2 +- scm/github/repo_test.go | 2 +- scm/service.go | 2 +- 228 files changed, 6083 insertions(+), 94 deletions(-) create mode 100644 compiler/types/yaml/buildkite/build.go rename compiler/types/yaml/{ => buildkite}/build_test.go (99%) create mode 100644 compiler/types/yaml/buildkite/deployment.go create mode 100644 compiler/types/yaml/buildkite/deployment_test.go create mode 100644 compiler/types/yaml/buildkite/doc.go create mode 100644 compiler/types/yaml/buildkite/git.go create mode 100644 compiler/types/yaml/buildkite/git_test.go create mode 100644 compiler/types/yaml/buildkite/metadata.go create mode 100644 compiler/types/yaml/buildkite/metadata_test.go create mode 100644 compiler/types/yaml/buildkite/ruleset.go rename compiler/types/yaml/{ => buildkite}/ruleset_test.go (99%) create mode 100644 compiler/types/yaml/buildkite/secret.go rename compiler/types/yaml/{ => buildkite}/secret_test.go (99%) create mode 100644 compiler/types/yaml/buildkite/service.go rename compiler/types/yaml/{ => buildkite}/service_test.go (99%) rename compiler/types/yaml/{ => buildkite}/stage.go (85%) rename compiler/types/yaml/{ => buildkite}/stage_test.go (99%) create mode 100644 compiler/types/yaml/buildkite/step.go rename compiler/types/yaml/{ => buildkite}/step_test.go (99%) create mode 100644 compiler/types/yaml/buildkite/template.go rename compiler/types/yaml/{ => buildkite}/template_test.go (99%) rename compiler/types/yaml/{ => buildkite}/testdata/build.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/build/validate/bad_pipeline0.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/build/validate/bad_pipeline1.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/build/validate/bad_version.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/build/validate/step.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/build_anchor_stage.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/build_anchor_step.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/build_empty_env.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/build_with_deploy_config.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/deploy_parameter.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/invalid.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/merge_anchor.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/metadata.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/metadata_env.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/ruleset_advanced.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/ruleset_regex.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/ruleset_simple.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret/validate/no_name.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret/validate/org.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret/validate/org_bad_engine.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret/validate/org_bad_key.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret/validate/plugin.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret/validate/plugin_bad_image.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret/validate/plugin_bad_name.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret/validate/repo.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret/validate/repo_bad_engine.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret/validate/repo_bad_key.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret/validate/shared.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret/validate/shared_bad_engine.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/secret/validate/shared_bad_key.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/service.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/service/validate/bad_image.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/service/validate/minimal.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/service/validate/missing_image.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/service/validate/missing_name.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/service_nil.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/stage.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/stage/validate/bad_image.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/stage/validate/minimal.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/stage/validate/missing.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/stage/validate/missing_image.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/stage/validate/missing_name.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/step.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/step/validate/bad_image.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/step/validate/minimal.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/step/validate/missing.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/step/validate/missing_image.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/step/validate/missing_name.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/step_malformed.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/step_nil.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/step_secret_slice.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/step_secret_slice_invalid_no_source.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/step_secret_slice_invalid_no_target.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/step_secret_string.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/template.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/ulimit_colon_error.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/ulimit_equal_error.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/ulimit_hardlimit1_error.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/ulimit_hardlimit2_error.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/ulimit_slice.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/ulimit_softlimit_error.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/ulimit_string.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/volume_error.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/volume_slice.yml (100%) rename compiler/types/yaml/{ => buildkite}/testdata/volume_string.yml (100%) create mode 100644 compiler/types/yaml/buildkite/ulimit.go rename compiler/types/yaml/{ => buildkite}/ulimit_test.go (99%) create mode 100644 compiler/types/yaml/buildkite/volume.go create mode 100644 compiler/types/yaml/buildkite/volume_test.go create mode 100644 compiler/types/yaml/buildkite/worker.go create mode 100644 compiler/types/yaml/buildkite/worker_test.go delete mode 100644 compiler/types/yaml/doc.go rename compiler/types/yaml/{ => yaml}/build.go (100%) create mode 100644 compiler/types/yaml/yaml/build_test.go rename compiler/types/yaml/{ => yaml}/deployment.go (100%) rename compiler/types/yaml/{ => yaml}/deployment_test.go (100%) create mode 100644 compiler/types/yaml/yaml/doc.go rename compiler/types/yaml/{ => yaml}/git.go (100%) rename compiler/types/yaml/{ => yaml}/git_test.go (100%) rename compiler/types/yaml/{ => yaml}/metadata.go (100%) rename compiler/types/yaml/{ => yaml}/metadata_test.go (100%) rename compiler/types/yaml/{ => yaml}/ruleset.go (100%) create mode 100644 compiler/types/yaml/yaml/ruleset_test.go rename compiler/types/yaml/{ => yaml}/secret.go (100%) create mode 100644 compiler/types/yaml/yaml/secret_test.go rename compiler/types/yaml/{ => yaml}/service.go (100%) create mode 100644 compiler/types/yaml/yaml/service_test.go create mode 100644 compiler/types/yaml/yaml/stage.go create mode 100644 compiler/types/yaml/yaml/stage_test.go rename compiler/types/yaml/{ => yaml}/step.go (100%) create mode 100644 compiler/types/yaml/yaml/step_test.go rename compiler/types/yaml/{ => yaml}/template.go (100%) create mode 100644 compiler/types/yaml/yaml/template_test.go create mode 100644 compiler/types/yaml/yaml/testdata/build.yml create mode 100644 compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline0.yml create mode 100644 compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline1.yml create mode 100644 compiler/types/yaml/yaml/testdata/build/validate/bad_version.yml create mode 100644 compiler/types/yaml/yaml/testdata/build/validate/step.yml create mode 100644 compiler/types/yaml/yaml/testdata/build_anchor_stage.yml create mode 100644 compiler/types/yaml/yaml/testdata/build_anchor_step.yml create mode 100644 compiler/types/yaml/yaml/testdata/build_empty_env.yml create mode 100644 compiler/types/yaml/yaml/testdata/build_with_deploy_config.yml create mode 100644 compiler/types/yaml/yaml/testdata/deploy_parameter.yml create mode 100644 compiler/types/yaml/yaml/testdata/invalid.yml create mode 100644 compiler/types/yaml/yaml/testdata/merge_anchor.yml create mode 100644 compiler/types/yaml/yaml/testdata/metadata.yml create mode 100644 compiler/types/yaml/yaml/testdata/metadata_env.yml create mode 100644 compiler/types/yaml/yaml/testdata/ruleset_advanced.yml create mode 100644 compiler/types/yaml/yaml/testdata/ruleset_regex.yml create mode 100644 compiler/types/yaml/yaml/testdata/ruleset_simple.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret/validate/no_name.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret/validate/org.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret/validate/org_bad_engine.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret/validate/org_bad_key.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret/validate/plugin.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_image.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_name.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret/validate/repo.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_engine.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_key.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret/validate/shared.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_engine.yml create mode 100644 compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_key.yml create mode 100644 compiler/types/yaml/yaml/testdata/service.yml create mode 100644 compiler/types/yaml/yaml/testdata/service/validate/bad_image.yml create mode 100644 compiler/types/yaml/yaml/testdata/service/validate/minimal.yml create mode 100644 compiler/types/yaml/yaml/testdata/service/validate/missing_image.yml create mode 100644 compiler/types/yaml/yaml/testdata/service/validate/missing_name.yml create mode 100644 compiler/types/yaml/yaml/testdata/service_nil.yml create mode 100644 compiler/types/yaml/yaml/testdata/stage.yml create mode 100644 compiler/types/yaml/yaml/testdata/stage/validate/bad_image.yml create mode 100644 compiler/types/yaml/yaml/testdata/stage/validate/minimal.yml create mode 100644 compiler/types/yaml/yaml/testdata/stage/validate/missing.yml create mode 100644 compiler/types/yaml/yaml/testdata/stage/validate/missing_image.yml create mode 100644 compiler/types/yaml/yaml/testdata/stage/validate/missing_name.yml create mode 100644 compiler/types/yaml/yaml/testdata/step.yml create mode 100644 compiler/types/yaml/yaml/testdata/step/validate/bad_image.yml create mode 100644 compiler/types/yaml/yaml/testdata/step/validate/minimal.yml create mode 100644 compiler/types/yaml/yaml/testdata/step/validate/missing.yml create mode 100644 compiler/types/yaml/yaml/testdata/step/validate/missing_image.yml create mode 100644 compiler/types/yaml/yaml/testdata/step/validate/missing_name.yml create mode 100644 compiler/types/yaml/yaml/testdata/step_malformed.yml create mode 100644 compiler/types/yaml/yaml/testdata/step_nil.yml create mode 100644 compiler/types/yaml/yaml/testdata/step_secret_slice.yml create mode 100644 compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_source.yml create mode 100644 compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_target.yml create mode 100644 compiler/types/yaml/yaml/testdata/step_secret_string.yml create mode 100644 compiler/types/yaml/yaml/testdata/template.yml create mode 100644 compiler/types/yaml/yaml/testdata/ulimit_colon_error.yml create mode 100644 compiler/types/yaml/yaml/testdata/ulimit_equal_error.yml create mode 100644 compiler/types/yaml/yaml/testdata/ulimit_hardlimit1_error.yml create mode 100644 compiler/types/yaml/yaml/testdata/ulimit_hardlimit2_error.yml create mode 100644 compiler/types/yaml/yaml/testdata/ulimit_slice.yml create mode 100644 compiler/types/yaml/yaml/testdata/ulimit_softlimit_error.yml create mode 100644 compiler/types/yaml/yaml/testdata/ulimit_string.yml create mode 100644 compiler/types/yaml/yaml/testdata/volume_error.yml create mode 100644 compiler/types/yaml/yaml/testdata/volume_slice.yml create mode 100644 compiler/types/yaml/yaml/testdata/volume_string.yml rename compiler/types/yaml/{ => yaml}/ulimit.go (100%) create mode 100644 compiler/types/yaml/yaml/ulimit_test.go rename compiler/types/yaml/{ => yaml}/volume.go (100%) rename compiler/types/yaml/{ => yaml}/volume_test.go (100%) rename compiler/types/yaml/{ => yaml}/worker.go (100%) rename compiler/types/yaml/{ => yaml}/worker_test.go (100%) create mode 100644 internal/testdata/buildkite.yml create mode 100644 internal/testdata/go-yaml.yml create mode 100644 internal/testdata/invalid.yml create mode 100644 internal/testdata/no_version.yml diff --git a/api/pipeline/template.go b/api/pipeline/template.go index aef79d165..0ff2a54bc 100644 --- a/api/pipeline/template.go +++ b/api/pipeline/template.go @@ -13,7 +13,7 @@ import ( "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler" "github.com/go-vela/server/compiler/registry/github" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/internal" "github.com/go-vela/server/router/middleware/org" "github.com/go-vela/server/router/middleware/pipeline" diff --git a/api/types/string.go b/api/types/string.go index 33ddacb59..a7637d714 100644 --- a/api/types/string.go +++ b/api/types/string.go @@ -7,8 +7,8 @@ import ( "strconv" "strings" - "github.com/buildkite/yaml" json "github.com/ghodss/yaml" + "gopkg.in/yaml.v3" ) // ToString is a helper function to convert diff --git a/compiler/engine.go b/compiler/engine.go index ab4555296..b3503b77a 100644 --- a/compiler/engine.go +++ b/compiler/engine.go @@ -9,7 +9,7 @@ import ( "github.com/go-vela/server/api/types/settings" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/database" "github.com/go-vela/server/internal" "github.com/go-vela/server/scm" diff --git a/compiler/native/clone.go b/compiler/native/clone.go index f6f4ffdbc..179e80db5 100644 --- a/compiler/native/clone.go +++ b/compiler/native/clone.go @@ -3,7 +3,7 @@ package native import ( - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) diff --git a/compiler/native/clone_test.go b/compiler/native/clone_test.go index 6bd92697c..ffee2ceaf 100644 --- a/compiler/native/clone_test.go +++ b/compiler/native/clone_test.go @@ -9,7 +9,7 @@ import ( "github.com/urfave/cli/v2" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) const defaultCloneImage = "target/vela-git-slim:latest" diff --git a/compiler/native/compile.go b/compiler/native/compile.go index 04740f412..7f0a71f41 100644 --- a/compiler/native/compile.go +++ b/compiler/native/compile.go @@ -12,14 +12,14 @@ import ( "strings" "time" - yml "github.com/buildkite/yaml" "github.com/hashicorp/go-cleanhttp" "github.com/hashicorp/go-retryablehttp" + yml "gopkg.in/yaml.v3" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index e887176d7..e2134c9db 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -13,16 +13,16 @@ import ( "testing" "time" - yml "github.com/buildkite/yaml" "github.com/gin-gonic/gin" "github.com/google/go-cmp/cmp" "github.com/google/go-github/v65/github" "github.com/urfave/cli/v2" + yml "gopkg.in/yaml.v3" api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" "github.com/go-vela/server/internal" ) @@ -2056,7 +2056,7 @@ func Test_client_modifyConfig(t *testing.T) { Name: "docker", Pull: "always", Parameters: map[string]interface{}{ - "init_options": map[interface{}]interface{}{ + "init_options": map[string]interface{}{ "get_plugins": "true", }, }, @@ -2089,7 +2089,7 @@ func Test_client_modifyConfig(t *testing.T) { Name: "docker", Pull: "always", Parameters: map[string]interface{}{ - "init_options": map[interface{}]interface{}{ + "init_options": map[string]interface{}{ "get_plugins": "true", }, }, diff --git a/compiler/native/environment.go b/compiler/native/environment.go index 3f3162137..34ae5b3a7 100644 --- a/compiler/native/environment.go +++ b/compiler/native/environment.go @@ -9,7 +9,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" "github.com/go-vela/server/internal" ) diff --git a/compiler/native/environment_test.go b/compiler/native/environment_test.go index 17dce3942..24f319a08 100644 --- a/compiler/native/environment_test.go +++ b/compiler/native/environment_test.go @@ -13,7 +13,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/internal" ) diff --git a/compiler/native/expand.go b/compiler/native/expand.go index a19abcd98..ac86d09ce 100644 --- a/compiler/native/expand.go +++ b/compiler/native/expand.go @@ -15,7 +15,7 @@ import ( "github.com/go-vela/server/compiler/template/starlark" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index 412877966..0ca596338 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -17,7 +17,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func TestNative_ExpandStages(t *testing.T) { diff --git a/compiler/native/initialize.go b/compiler/native/initialize.go index bbfdb071c..6380278e4 100644 --- a/compiler/native/initialize.go +++ b/compiler/native/initialize.go @@ -3,7 +3,7 @@ package native import ( - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) diff --git a/compiler/native/initialize_test.go b/compiler/native/initialize_test.go index 26c170581..738d388d7 100644 --- a/compiler/native/initialize_test.go +++ b/compiler/native/initialize_test.go @@ -9,7 +9,7 @@ import ( "github.com/urfave/cli/v2" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func TestNative_InitStage(t *testing.T) { diff --git a/compiler/native/parse.go b/compiler/native/parse.go index 7afeefef3..804b1e015 100644 --- a/compiler/native/parse.go +++ b/compiler/native/parse.go @@ -7,12 +7,13 @@ import ( "io" "os" - "github.com/buildkite/yaml" + bkYaml "github.com/buildkite/yaml" "github.com/go-vela/server/compiler/template/native" "github.com/go-vela/server/compiler/template/starlark" typesRaw "github.com/go-vela/server/compiler/types/raw" - types "github.com/go-vela/server/compiler/types/yaml" + bkYamlTypes "github.com/go-vela/server/compiler/types/yaml/buildkite" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) @@ -41,9 +42,9 @@ func (c *client) ParseRaw(v interface{}) (string, error) { } // Parse converts an object to a yaml configuration. -func (c *client) Parse(v interface{}, pipelineType string, template *types.Template) (*types.Build, []byte, error) { +func (c *client) Parse(v interface{}, pipelineType string, template *yaml.Template) (*yaml.Build, []byte, error) { var ( - p *types.Build + p *yaml.Build raw []byte ) @@ -112,11 +113,11 @@ func (c *client) Parse(v interface{}, pipelineType string, template *types.Templ } // ParseBytes converts a byte slice to a yaml configuration. -func ParseBytes(data []byte) (*types.Build, []byte, error) { - config := new(types.Build) +func ParseBytes(data []byte) (*yaml.Build, []byte, error) { + config := new(bkYamlTypes.Build) // unmarshal the bytes into the yaml configuration - err := yaml.Unmarshal(data, config) + err := bkYaml.Unmarshal(data, config) if err != nil { return nil, data, fmt.Errorf("unable to unmarshal yaml: %w", err) } @@ -128,11 +129,11 @@ func ParseBytes(data []byte) (*types.Build, []byte, error) { config.Environment = typesRaw.StringSliceMap{} } - return config, data, nil + return config.ToYAML(), data, nil } // ParseFile converts an os.File into a yaml configuration. -func ParseFile(f *os.File) (*types.Build, []byte, error) { +func ParseFile(f *os.File) (*yaml.Build, []byte, error) { return ParseReader(f) } @@ -142,7 +143,7 @@ func ParseFileRaw(f *os.File) (string, error) { } // ParsePath converts a file path into a yaml configuration. -func ParsePath(p string) (*types.Build, []byte, error) { +func ParsePath(p string) (*yaml.Build, []byte, error) { // open the file for reading f, err := os.Open(p) if err != nil { @@ -168,7 +169,7 @@ func ParsePathRaw(p string) (string, error) { } // ParseReader converts an io.Reader into a yaml configuration. -func ParseReader(r io.Reader) (*types.Build, []byte, error) { +func ParseReader(r io.Reader) (*yaml.Build, []byte, error) { // read all the bytes from the reader data, err := io.ReadAll(r) if err != nil { @@ -190,6 +191,6 @@ func ParseReaderRaw(r io.Reader) (string, error) { } // ParseString converts a string into a yaml configuration. -func ParseString(s string) (*types.Build, []byte, error) { +func ParseString(s string) (*yaml.Build, []byte, error) { return ParseBytes([]byte(s)) } diff --git a/compiler/native/parse_test.go b/compiler/native/parse_test.go index fa2635a48..ae5b263b7 100644 --- a/compiler/native/parse_test.go +++ b/compiler/native/parse_test.go @@ -15,7 +15,7 @@ import ( api "github.com/go-vela/server/api/types" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) diff --git a/compiler/native/script.go b/compiler/native/script.go index 03aa093bb..049b01f91 100644 --- a/compiler/native/script.go +++ b/compiler/native/script.go @@ -8,7 +8,7 @@ import ( "fmt" "strings" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) // ScriptStages injects the script for each step in every stage in a yaml configuration. diff --git a/compiler/native/script_test.go b/compiler/native/script_test.go index 410c297c0..bbcb95a73 100644 --- a/compiler/native/script_test.go +++ b/compiler/native/script_test.go @@ -10,7 +10,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/urfave/cli/v2" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func TestNative_ScriptStages(t *testing.T) { diff --git a/compiler/native/substitute.go b/compiler/native/substitute.go index a675e92d5..8e3f44b53 100644 --- a/compiler/native/substitute.go +++ b/compiler/native/substitute.go @@ -6,10 +6,10 @@ import ( "fmt" "strings" - "github.com/buildkite/yaml" "github.com/drone/envsubst" + "gopkg.in/yaml.v3" - types "github.com/go-vela/server/compiler/types/yaml" + types "github.com/go-vela/server/compiler/types/yaml/yaml" ) // SubstituteStages replaces every declared environment diff --git a/compiler/native/substitute_test.go b/compiler/native/substitute_test.go index e8db1565c..df7a2230c 100644 --- a/compiler/native/substitute_test.go +++ b/compiler/native/substitute_test.go @@ -9,7 +9,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/urfave/cli/v2" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func Test_client_SubstituteStages(t *testing.T) { diff --git a/compiler/native/transform.go b/compiler/native/transform.go index ebdb6777e..3bad97b89 100644 --- a/compiler/native/transform.go +++ b/compiler/native/transform.go @@ -6,7 +6,7 @@ import ( "fmt" "github.com/go-vela/server/compiler/types/pipeline" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) const ( @@ -65,12 +65,13 @@ func (c *client) TransformStages(r *pipeline.RuleData, p *yaml.Build) (*pipeline // create new executable pipeline pipeline := &pipeline.Build{ - Version: p.Version, - Metadata: *p.Metadata.ToPipeline(), - Stages: *p.Stages.ToPipeline(), - Secrets: *p.Secrets.ToPipeline(), - Services: *p.Services.ToPipeline(), - Worker: *p.Worker.ToPipeline(), + Version: p.Version, + Metadata: *p.Metadata.ToPipeline(), + Stages: *p.Stages.ToPipeline(), + Secrets: *p.Secrets.ToPipeline(), + Services: *p.Services.ToPipeline(), + Worker: *p.Worker.ToPipeline(), + Deployment: *p.Deployment.ToPipeline(), } // set the unique ID for the executable pipeline diff --git a/compiler/native/transform_test.go b/compiler/native/transform_test.go index d2906bc8b..317e9e5dd 100644 --- a/compiler/native/transform_test.go +++ b/compiler/native/transform_test.go @@ -10,7 +10,7 @@ import ( "github.com/urfave/cli/v2" "github.com/go-vela/server/compiler/types/pipeline" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/internal" ) diff --git a/compiler/native/validate.go b/compiler/native/validate.go index 0c0775019..77a8578f2 100644 --- a/compiler/native/validate.go +++ b/compiler/native/validate.go @@ -7,7 +7,7 @@ import ( "github.com/hashicorp/go-multierror" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) diff --git a/compiler/native/validate_test.go b/compiler/native/validate_test.go index 4d3876f0a..a5771e953 100644 --- a/compiler/native/validate_test.go +++ b/compiler/native/validate_test.go @@ -10,7 +10,7 @@ import ( "github.com/urfave/cli/v2" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func TestNative_Validate_NoVersion(t *testing.T) { diff --git a/compiler/template/native/convert.go b/compiler/template/native/convert.go index 4a75a5793..5715cecd5 100644 --- a/compiler/template/native/convert.go +++ b/compiler/template/native/convert.go @@ -5,7 +5,7 @@ package native import ( "strings" - "github.com/buildkite/yaml" + "gopkg.in/yaml.v3" "github.com/go-vela/server/compiler/types/raw" ) diff --git a/compiler/template/native/render.go b/compiler/template/native/render.go index afc3bea51..9fec0fd50 100644 --- a/compiler/template/native/render.go +++ b/compiler/template/native/render.go @@ -11,13 +11,14 @@ import ( "github.com/buildkite/yaml" "github.com/go-vela/server/compiler/types/raw" - types "github.com/go-vela/server/compiler/types/yaml" + bkTypes "github.com/go-vela/server/compiler/types/yaml/buildkite" + types "github.com/go-vela/server/compiler/types/yaml/yaml" ) // Render combines the template with the step in the yaml pipeline. func Render(tmpl string, name string, tName string, environment raw.StringSliceMap, variables map[string]interface{}) (*types.Build, error) { buffer := new(bytes.Buffer) - config := new(types.Build) + config := new(bkTypes.Build) velaFuncs := funcHandler{envs: convertPlatformVars(environment, name)} templateFuncMap := map[string]interface{}{ @@ -57,13 +58,13 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM config.Steps[index].Name = fmt.Sprintf("%s_%s", name, newStep.Name) } - return &types.Build{Metadata: config.Metadata, Deployment: config.Deployment, Steps: config.Steps, Secrets: config.Secrets, Services: config.Services, Environment: config.Environment, Templates: config.Templates}, nil + return &types.Build{Metadata: *config.Metadata.ToYAML(), Steps: *config.Steps.ToYAML(), Secrets: *config.Secrets.ToYAML(), Services: *config.Services.ToYAML(), Environment: config.Environment, Templates: *config.Templates.ToYAML(), Deployment: *config.Deployment.ToYAML()}, nil } // RenderBuild renders the templated build. func RenderBuild(tmpl string, b string, envs map[string]string, variables map[string]interface{}) (*types.Build, error) { buffer := new(bytes.Buffer) - config := new(types.Build) + config := new(bkTypes.Build) velaFuncs := funcHandler{envs: convertPlatformVars(envs, tmpl)} templateFuncMap := map[string]interface{}{ @@ -98,5 +99,5 @@ func RenderBuild(tmpl string, b string, envs map[string]string, variables map[st return nil, fmt.Errorf("unable to unmarshal yaml: %w", err) } - return config, nil + return config.ToYAML(), nil } diff --git a/compiler/template/native/render_test.go b/compiler/template/native/render_test.go index 3b4daacb8..d204f7572 100644 --- a/compiler/template/native/render_test.go +++ b/compiler/template/native/render_test.go @@ -6,11 +6,11 @@ import ( "os" "testing" - goyaml "github.com/buildkite/yaml" "github.com/google/go-cmp/cmp" + goyaml "gopkg.in/yaml.v3" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func TestNative_Render(t *testing.T) { diff --git a/compiler/template/starlark/render.go b/compiler/template/starlark/render.go index edd822463..a89d63f99 100644 --- a/compiler/template/starlark/render.go +++ b/compiler/template/starlark/render.go @@ -7,13 +7,14 @@ import ( "errors" "fmt" - yaml "github.com/buildkite/yaml" + "github.com/buildkite/yaml" "go.starlark.net/starlark" "go.starlark.net/starlarkstruct" "go.starlark.net/syntax" "github.com/go-vela/server/compiler/types/raw" - types "github.com/go-vela/server/compiler/types/yaml" + bkTypes "github.com/go-vela/server/compiler/types/yaml/buildkite" + types "github.com/go-vela/server/compiler/types/yaml/yaml" ) var ( @@ -32,7 +33,7 @@ var ( // Render combines the template with the step in the yaml pipeline. func Render(tmpl string, name string, tName string, environment raw.StringSliceMap, variables map[string]interface{}, limit int64) (*types.Build, error) { - config := new(types.Build) + config := new(bkTypes.Build) thread := &starlark.Thread{Name: name} @@ -134,14 +135,14 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM config.Steps[index].Name = fmt.Sprintf("%s_%s", name, newStep.Name) } - return &types.Build{Steps: config.Steps, Secrets: config.Secrets, Services: config.Services, Environment: config.Environment}, nil + return &types.Build{Steps: *config.Steps.ToYAML(), Secrets: *config.Secrets.ToYAML(), Services: *config.Services.ToYAML(), Environment: config.Environment}, nil } // RenderBuild renders the templated build. // //nolint:lll // ignore function length due to input args func RenderBuild(tmpl string, b string, envs map[string]string, variables map[string]interface{}, limit int64) (*types.Build, error) { - config := new(types.Build) + config := new(bkTypes.Build) thread := &starlark.Thread{Name: "templated-base"} @@ -238,5 +239,5 @@ func RenderBuild(tmpl string, b string, envs map[string]string, variables map[st return nil, fmt.Errorf("unable to unmarshal yaml: %w", err) } - return config, nil + return config.ToYAML(), nil } diff --git a/compiler/template/starlark/render_test.go b/compiler/template/starlark/render_test.go index 33db5a7f0..b9d184dae 100644 --- a/compiler/template/starlark/render_test.go +++ b/compiler/template/starlark/render_test.go @@ -6,11 +6,11 @@ import ( "os" "testing" - goyaml "github.com/buildkite/yaml" "github.com/google/go-cmp/cmp" + goyaml "gopkg.in/yaml.v3" "github.com/go-vela/server/compiler/types/raw" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) func TestStarlark_Render(t *testing.T) { diff --git a/compiler/template/template.go b/compiler/template/template.go index a16cc4057..ee503e980 100644 --- a/compiler/template/template.go +++ b/compiler/template/template.go @@ -2,7 +2,7 @@ package template -import "github.com/go-vela/server/compiler/types/yaml" +import "github.com/go-vela/server/compiler/types/yaml/yaml" // Engine represents the interface for Vela integrating // with the different supported template engines. diff --git a/compiler/types/raw/map_test.go b/compiler/types/raw/map_test.go index 0494c8ed4..918664a65 100644 --- a/compiler/types/raw/map_test.go +++ b/compiler/types/raw/map_test.go @@ -8,7 +8,7 @@ import ( "reflect" "testing" - "github.com/buildkite/yaml" + "gopkg.in/yaml.v3" ) func TestRaw_StringSliceMap_UnmarshalJSON(t *testing.T) { diff --git a/compiler/types/raw/slice_test.go b/compiler/types/raw/slice_test.go index 7a32bd3d7..0ae76f9dc 100644 --- a/compiler/types/raw/slice_test.go +++ b/compiler/types/raw/slice_test.go @@ -7,7 +7,7 @@ import ( "reflect" "testing" - "github.com/buildkite/yaml" + "gopkg.in/yaml.v3" ) func TestRaw_StringSlice_UnmarshalJSON(t *testing.T) { diff --git a/compiler/types/yaml/buildkite/build.go b/compiler/types/yaml/buildkite/build.go new file mode 100644 index 000000000..8bc8dcf60 --- /dev/null +++ b/compiler/types/yaml/buildkite/build.go @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +// Build is the yaml representation of a build for a pipeline. +type Build struct { + Version string `yaml:"version,omitempty" json:"version,omitempty" jsonschema:"required,minLength=1,description=Provide syntax version used to evaluate the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/version/"` + Metadata Metadata `yaml:"metadata,omitempty" json:"metadata,omitempty" jsonschema:"description=Pass extra information.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/"` + Environment raw.StringSliceMap `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Provide global environment variables injected into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-environment-key"` + Worker Worker `yaml:"worker,omitempty" json:"worker,omitempty" jsonschema:"description=Limit the pipeline to certain types of workers.\nReference: https://go-vela.github.io/docs/reference/yaml/worker/"` + Secrets SecretSlice `yaml:"secrets,omitempty" json:"secrets,omitempty" jsonschema:"description=Provide sensitive information.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/"` + Services ServiceSlice `yaml:"services,omitempty" json:"services,omitempty" jsonschema:"description=Provide detached (headless) execution instructions.\nReference: https://go-vela.github.io/docs/reference/yaml/services/"` + Stages StageSlice `yaml:"stages,omitempty" json:"stages,omitempty" jsonschema:"oneof_required=stages,description=Provide parallel execution instructions.\nReference: https://go-vela.github.io/docs/reference/yaml/stages/"` + Steps StepSlice `yaml:"steps,omitempty" json:"steps,omitempty" jsonschema:"oneof_required=steps,description=Provide sequential execution instructions.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/"` + Templates TemplateSlice `yaml:"templates,omitempty" json:"templates,omitempty" jsonschema:"description=Provide the name of templates to expand.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/"` + Deployment Deployment `yaml:"deployment,omitempty" json:"deployment,omitempty" jsonschema:"description=Provide deployment configuration.\nReference: https://go-vela.github.io/docs/reference/yaml/deployments/"` + Git Git `yaml:"git,omitempty" json:"git,omitempty" jsonschema:"description=Provide the git access specifications.\nReference: https://go-vela.github.io/docs/reference/yaml/git/"` +} + +// ToPipelineAPI converts the Build type to an API Pipeline type. +func (b *Build) ToPipelineAPI() *api.Pipeline { + pipeline := new(api.Pipeline) + + pipeline.SetFlavor(b.Worker.Flavor) + pipeline.SetPlatform(b.Worker.Platform) + pipeline.SetVersion(b.Version) + pipeline.SetServices(len(b.Services) > 0) + pipeline.SetStages(len(b.Stages) > 0) + pipeline.SetSteps(len(b.Steps) > 0) + pipeline.SetTemplates(len(b.Templates) > 0) + + // set default for external and internal secrets + external := false + internal := false + + // iterate through all secrets in the build + for _, secret := range b.Secrets { + // check if external and internal secrets have been found + if external && internal { + // exit the loop since both secrets have been found + break + } + + // check if the secret origin is empty + if secret.Origin.Empty() { + // origin was empty so an internal secret was found + internal = true + } else { + // origin was not empty so an external secret was found + external = true + } + } + + pipeline.SetExternalSecrets(external) + pipeline.SetInternalSecrets(internal) + + return pipeline +} + +// UnmarshalYAML implements the Unmarshaler interface for the Build type. +func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error { + // build we try unmarshalling to + build := new(struct { + Version string + Metadata Metadata + Environment raw.StringSliceMap + Worker Worker + Secrets SecretSlice + Services ServiceSlice + Stages StageSlice + Steps StepSlice + Templates TemplateSlice + Deployment Deployment + }) + + // attempt to unmarshal as a build type + err := unmarshal(build) + if err != nil { + return err + } + + // give the documented default value to metadata environment + if build.Metadata.Environment == nil { + build.Metadata.Environment = []string{"steps", "services", "secrets"} + } + + // override the values + b.Version = build.Version + b.Metadata = build.Metadata + b.Environment = build.Environment + b.Worker = build.Worker + b.Secrets = build.Secrets + b.Services = build.Services + b.Stages = build.Stages + b.Steps = build.Steps + b.Templates = build.Templates + b.Deployment = build.Deployment + + return nil +} + +func (b *Build) ToYAML() *yaml.Build { + return &yaml.Build{ + Version: b.Version, + Metadata: *b.Metadata.ToYAML(), + Git: *b.Git.ToYAML(), + Environment: b.Environment, + Worker: *b.Worker.ToYAML(), + Secrets: *b.Secrets.ToYAML(), + Services: *b.Services.ToYAML(), + Stages: *b.Stages.ToYAML(), + Steps: *b.Steps.ToYAML(), + Templates: *b.Templates.ToYAML(), + Deployment: *b.Deployment.ToYAML(), + } +} diff --git a/compiler/types/yaml/build_test.go b/compiler/types/yaml/buildkite/build_test.go similarity index 99% rename from compiler/types/yaml/build_test.go rename to compiler/types/yaml/buildkite/build_test.go index bf2d341f6..ebd0fa267 100644 --- a/compiler/types/yaml/build_test.go +++ b/compiler/types/yaml/buildkite/build_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/buildkite/deployment.go b/compiler/types/yaml/buildkite/deployment.go new file mode 100644 index 000000000..305371a5b --- /dev/null +++ b/compiler/types/yaml/buildkite/deployment.go @@ -0,0 +1,112 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +type ( + // Deployment is the yaml representation of a + // deployment block in a pipeline. + Deployment struct { + Targets raw.StringSlice `yaml:"targets,omitempty" json:"targets,omitempty" jsonschema:"description=List of deployment targets for the deployment block.\nReference: https://go-vela.github.io/docs/reference/yaml/deployments/#the-targets-key"` + Parameters ParameterMap `yaml:"parameters,omitempty" json:"parameters,omitempty" jsonschema:"description=List of parameters for the deployment block.\nReference: https://go-vela.github.io/docs/reference/yaml/deployments/#the-parameters-key"` + Template StepTemplate `yaml:"template,omitempty" json:"template,omitempty" jsonschema:"description=Name of template to expand in the deployment block.\nReference: https://go-vela.github.io/docs/reference/yaml/deployments/#the-template-key"` + } + + // ParameterMap is the yaml representation + // of the parameters block for a deployment block of a pipeline. + ParameterMap map[string]*Parameter + + // Parameters is the yaml representation of the deploy parameters + // from a deployment block in a pipeline. + Parameter struct { + Description string `yaml:"description,omitempty" json:"description,omitempty" jsonschema:"description=Description of the parameter.\nReference: https://go-vela.github.io/docs/reference/yaml/deployments/#the-parameters-key"` + Type string `yaml:"type,omitempty" json:"type,omitempty" jsonschema:"description=Type of the parameter.\nReference: https://go-vela.github.io/docs/reference/yaml/deployments/#the-parameters-key"` + Required bool `yaml:"required,omitempty" json:"required,omitempty" jsonschema:"description=Flag indicating if the parameter is required.\nReference: https://go-vela.github.io/docs/reference/yaml/deployments/#the-parameters-key"` + Options raw.StringSlice `yaml:"options,omitempty" json:"options,omitempty" jsonschema:"description=List of options for the parameter.\nReference: https://go-vela.github.io/docs/reference/yaml/deployments/#the-parameters-key"` + Min int `yaml:"min,omitempty" json:"min,omitempty" jsonschema:"description=Minimum value for the parameter.\nReference: https://go-vela.github.io/docs/reference/yaml/deployments/#the-parameters-key"` + Max int `yaml:"max,omitempty" json:"max,omitempty" jsonschema:"description=Maximum value for the parameter.\nReference: https://go-vela.github.io/docs/reference/yaml/deployments/#the-parameters-key"` + } +) + +// ToPipeline converts the Deployment type +// to a pipeline Deployment type. +func (d *Deployment) ToPipeline() *pipeline.Deployment { + return &pipeline.Deployment{ + Targets: d.Targets, + Parameters: d.Parameters.ToPipeline(), + } +} + +// ToPipeline converts the Parameters type +// to a pipeline Parameters type. +func (p *ParameterMap) ToPipeline() pipeline.ParameterMap { + if len(*p) == 0 { + return nil + } + + // parameter map we want to return + parameterMap := make(pipeline.ParameterMap) + + // iterate through each element in the parameter map + for k, v := range *p { + // add the element to the pipeline parameter map + parameterMap[k] = &pipeline.Parameter{ + Description: v.Description, + Type: v.Type, + Required: v.Required, + Options: v.Options, + Min: v.Min, + Max: v.Max, + } + } + + return parameterMap +} + +func (p *Parameter) ToYAML() *yaml.Parameter { + if p == nil { + return nil + } + + return &yaml.Parameter{ + Description: p.Description, + Type: p.Type, + Required: p.Required, + Options: p.Options, + Min: p.Min, + Max: p.Max, + } +} + +func (p *ParameterMap) ToYAML() yaml.ParameterMap { + if len(*p) == 0 { + return nil + } + + parameterMap := make(yaml.ParameterMap) + + // iterate through each element in the volume slice + for param := range *p { + // append the element to the yaml volume slice + parameterMap[param] = (*p)[param].ToYAML() + } + + return parameterMap +} + +func (d *Deployment) ToYAML() *yaml.Deployment { + if d == nil { + return nil + } + + return &yaml.Deployment{ + Targets: d.Targets, + Parameters: d.Parameters.ToYAML(), + Template: d.Template.ToYAML(), + } +} diff --git a/compiler/types/yaml/buildkite/deployment_test.go b/compiler/types/yaml/buildkite/deployment_test.go new file mode 100644 index 000000000..410920428 --- /dev/null +++ b/compiler/types/yaml/buildkite/deployment_test.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_Deployment_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + name string + deployment *Deployment + want *pipeline.Deployment + }{ + { + name: "deployment with template", + deployment: &Deployment{ + Template: StepTemplate{Name: "foo"}, + }, + want: &pipeline.Deployment{}, + }, + { + name: "deployment with targets and parameters", + deployment: &Deployment{ + Targets: []string{"foo"}, + Parameters: ParameterMap{ + "foo": { + Description: "bar", + Type: "string", + Required: true, + Options: []string{"baz"}, + }, + "bar": { + Description: "baz", + Type: "string", + Required: false, + }, + }, + }, + want: &pipeline.Deployment{ + Targets: []string{"foo"}, + Parameters: pipeline.ParameterMap{ + "foo": { + Description: "bar", + Type: "string", + Required: true, + Options: []string{"baz"}, + }, + "bar": { + Description: "baz", + Type: "string", + Required: false, + }, + }, + }, + }, + { + name: "empty deployment config", + deployment: &Deployment{}, + want: &pipeline.Deployment{}, + }, + } + + // run tests + for _, test := range tests { + got := test.deployment.ToPipeline() + + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("ToPipeline for %s does not match: -want +got):\n%s", test.name, diff) + } + } +} diff --git a/compiler/types/yaml/buildkite/doc.go b/compiler/types/yaml/buildkite/doc.go new file mode 100644 index 000000000..07e33c5a4 --- /dev/null +++ b/compiler/types/yaml/buildkite/doc.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 + +// package buildkite provides the defined yaml types for Vela. +// +// Usage: +// +// import "github.com/go-vela/server/compiler/types/yaml/yaml" +package buildkite diff --git a/compiler/types/yaml/buildkite/git.go b/compiler/types/yaml/buildkite/git.go new file mode 100644 index 000000000..53f8e6bdf --- /dev/null +++ b/compiler/types/yaml/buildkite/git.go @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +// Git is the yaml representation of git configurations for a pipeline. +type Git struct { + Token `yaml:"token,omitempty" json:"token,omitempty" jsonschema:"description=Provide the git token specifications, primarily used for cloning.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#token"` +} + +// Token is the yaml representation of the git token. +// Only applies when using GitHub App installations. +type Token struct { + Repositories []string `yaml:"repositories,omitempty" json:"repositories,omitempty" jsonschema:"description=Provide a list of repositories to clone.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#repositories"` + Permissions map[string]string `yaml:"permissions,omitempty" json:"permissions,omitempty" jsonschema:"description=Provide a list of repository permissions to apply to the git token.\nReference: https://go-vela.github.io/docs/reference/yaml/git/#permissions"` +} + +// ToPipeline converts the Git type +// to a pipeline Git type. +func (g *Git) ToPipeline() *pipeline.Git { + return &pipeline.Git{ + Token: &pipeline.Token{ + Repositories: g.Repositories, + Permissions: g.Permissions, + }, + } +} + +func (g *Git) ToYAML() *yaml.Git { + if g == nil { + return nil + } + + return &yaml.Git{ + Token: yaml.Token{ + Repositories: g.Repositories, + Permissions: g.Permissions, + }, + } +} diff --git a/compiler/types/yaml/buildkite/git_test.go b/compiler/types/yaml/buildkite/git_test.go new file mode 100644 index 000000000..6d23f231d --- /dev/null +++ b/compiler/types/yaml/buildkite/git_test.go @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "reflect" + "testing" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_Git_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + git *Git + want *pipeline.Git + }{ + { + git: &Git{ + Token: Token{ + Repositories: []string{"foo", "bar"}, + }, + }, + want: &pipeline.Git{ + Token: &pipeline.Token{ + Repositories: []string{"foo", "bar"}, + }, + }, + }, + { + git: &Git{ + Token: Token{ + Permissions: map[string]string{"foo": "bar"}, + }, + }, + want: &pipeline.Git{ + Token: &pipeline.Token{ + Permissions: map[string]string{"foo": "bar"}, + }, + }, + }, + { + git: &Git{ + Token: Token{}, + }, + want: &pipeline.Git{ + Token: &pipeline.Token{}, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.git.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/buildkite/metadata.go b/compiler/types/yaml/buildkite/metadata.go new file mode 100644 index 000000000..a57d83273 --- /dev/null +++ b/compiler/types/yaml/buildkite/metadata.go @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +type ( + // Metadata is the yaml representation of + // the metadata block for a pipeline. + Metadata struct { + Template bool `yaml:"template,omitempty" json:"template,omitempty" jsonschema:"description=Enables compiling the pipeline as a template.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-template-key"` + RenderInline bool `yaml:"render_inline,omitempty" json:"render_inline,omitempty" jsonschema:"description=Enables inline compiling for the pipeline templates.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-render-inline-key"` + Clone *bool `yaml:"clone,omitempty" json:"clone,omitempty" jsonschema:"default=true,description=Enables injecting the default clone process.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-clone-key"` + Environment []string `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Controls which containers processes can have global env injected.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-environment-key"` + AutoCancel *CancelOptions `yaml:"auto_cancel,omitempty" json:"auto_cancel,omitempty" jsonschema:"description=Enables auto canceling of queued or running pipelines that become stale due to new push.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-auto-cancel-key"` + } + + // CancelOptions is the yaml representation of + // the auto_cancel block for a pipeline. + CancelOptions struct { + Running *bool `yaml:"running,omitempty" json:"running,omitempty" jsonschema:"description=Enables auto canceling of running pipelines that become stale due to new push.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-auto-cancel-key"` + Pending *bool `yaml:"pending,omitempty" json:"pending,omitempty" jsonschema:"description=Enables auto canceling of queued pipelines that become stale due to new push.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-auto-cancel-key"` + DefaultBranch *bool `yaml:"default_branch,omitempty" json:"default_branch,omitempty" jsonschema:"description=Enables auto canceling of queued or running pipelines that become stale due to new push to default branch.\nReference: https://go-vela.github.io/docs/reference/yaml/metadata/#the-auto-cancel-key"` + } +) + +// ToPipeline converts the Metadata type +// to a pipeline Metadata type. +func (m *Metadata) ToPipeline() *pipeline.Metadata { + var clone bool + if m.Clone == nil { + clone = true + } else { + clone = *m.Clone + } + + autoCancel := new(pipeline.CancelOptions) + + // default to false for all fields if block isn't found + if m.AutoCancel == nil { + autoCancel.Pending = false + autoCancel.Running = false + autoCancel.DefaultBranch = false + } else { + // if block is found but pending field isn't, default to true + if m.AutoCancel.Pending != nil { + autoCancel.Pending = *m.AutoCancel.Pending + } else { + autoCancel.Pending = true + } + + if m.AutoCancel.Running != nil { + autoCancel.Running = *m.AutoCancel.Running + } + + if m.AutoCancel.DefaultBranch != nil { + autoCancel.DefaultBranch = *m.AutoCancel.DefaultBranch + } + } + + return &pipeline.Metadata{ + Template: m.Template, + Clone: clone, + Environment: m.Environment, + AutoCancel: autoCancel, + } +} + +// HasEnvironment checks if the container type +// is contained within the environment list. +func (m *Metadata) HasEnvironment(container string) bool { + for _, e := range m.Environment { + if e == container { + return true + } + } + + return false +} + +func (m *Metadata) ToYAML() *yaml.Metadata { + if m == nil { + return nil + } + + return &yaml.Metadata{ + Template: m.Template, + RenderInline: m.RenderInline, + Clone: m.Clone, + Environment: m.Environment, + AutoCancel: m.AutoCancel.ToYAML(), + } +} + +func (ac *CancelOptions) ToYAML() *yaml.CancelOptions { + if ac == nil { + return nil + } + + return &yaml.CancelOptions{ + Pending: ac.Pending, + Running: ac.Running, + DefaultBranch: ac.DefaultBranch, + } +} diff --git a/compiler/types/yaml/buildkite/metadata_test.go b/compiler/types/yaml/buildkite/metadata_test.go new file mode 100644 index 000000000..a3c92bec8 --- /dev/null +++ b/compiler/types/yaml/buildkite/metadata_test.go @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "reflect" + "testing" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_Metadata_ToPipeline(t *testing.T) { + tBool := true + fBool := false + // setup tests + tests := []struct { + metadata *Metadata + want *pipeline.Metadata + }{ + { + metadata: &Metadata{ + Template: false, + Clone: &fBool, + Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &CancelOptions{ + Pending: &tBool, + Running: &tBool, + DefaultBranch: &fBool, + }, + }, + want: &pipeline.Metadata{ + Template: false, + Clone: false, + Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{ + Pending: true, + Running: true, + DefaultBranch: false, + }, + }, + }, + { + metadata: &Metadata{ + Template: false, + Clone: &tBool, + Environment: []string{"steps", "services"}, + }, + want: &pipeline.Metadata{ + Template: false, + Clone: true, + Environment: []string{"steps", "services"}, + AutoCancel: &pipeline.CancelOptions{ + Pending: false, + Running: false, + DefaultBranch: false, + }, + }, + }, + { + metadata: &Metadata{ + Template: false, + Clone: nil, + Environment: []string{"steps"}, + AutoCancel: &CancelOptions{ + Running: &tBool, + DefaultBranch: &tBool, + }, + }, + want: &pipeline.Metadata{ + Template: false, + Clone: true, + Environment: []string{"steps"}, + AutoCancel: &pipeline.CancelOptions{ + Pending: true, + Running: true, + DefaultBranch: true, + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.metadata.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_Metadata_HasEnvironment(t *testing.T) { + // setup tests + tests := []struct { + metadata *Metadata + container string + want bool + }{ + { + metadata: &Metadata{ + Environment: []string{"steps", "services", "secrets"}, + }, + container: "steps", + want: true, + }, + { + metadata: &Metadata{ + Environment: []string{"services", "secrets"}, + }, + container: "services", + want: true, + }, + { + metadata: &Metadata{ + Environment: []string{"steps", "services", "secrets"}, + }, + container: "notacontainer", + want: false, + }, + } + + // run tests + for _, test := range tests { + got := test.metadata.HasEnvironment(test.container) + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/buildkite/ruleset.go b/compiler/types/yaml/buildkite/ruleset.go new file mode 100644 index 000000000..1b56e3062 --- /dev/null +++ b/compiler/types/yaml/buildkite/ruleset.go @@ -0,0 +1,223 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" + "github.com/go-vela/server/constants" +) + +type ( + // Ruleset is the yaml representation of a + // ruleset block for a step in a pipeline. + Ruleset struct { + If Rules `yaml:"if,omitempty" json:"if,omitempty" jsonschema:"description=Limit execution to when all rules match.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Unless Rules `yaml:"unless,omitempty" json:"unless,omitempty" jsonschema:"description=Limit execution to when all rules do not match.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Matcher string `yaml:"matcher,omitempty" json:"matcher,omitempty" jsonschema:"enum=filepath,enum=regexp,default=filepath,description=Use the defined matching method.\nReference: coming soon"` + Operator string `yaml:"operator,omitempty" json:"operator,omitempty" jsonschema:"enum=or,enum=and,default=and,description=Whether all rule conditions must be met or just any one of them.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Continue bool `yaml:"continue,omitempty" json:"continue,omitempty" jsonschema:"default=false,description=Limits the execution of a step to continuing on any failure.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + } + + // Rules is the yaml representation of the ruletypes + // from a ruleset block for a step in a pipeline. + Rules struct { + Branch []string `yaml:"branch,omitempty,flow" json:"branch,omitempty" jsonschema:"description=Limits the execution of a step to matching build branches.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Comment []string `yaml:"comment,omitempty,flow" json:"comment,omitempty" jsonschema:"description=Limits the execution of a step to matching a pull request comment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Event []string `yaml:"event,omitempty,flow" json:"event,omitempty" jsonschema:"description=Limits the execution of a step to matching build events.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Path []string `yaml:"path,omitempty,flow" json:"path,omitempty" jsonschema:"description=Limits the execution of a step to matching files changed in a repository.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Repo []string `yaml:"repo,omitempty,flow" json:"repo,omitempty" jsonschema:"description=Limits the execution of a step to matching repos.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Sender []string `yaml:"sender,omitempty,flow" json:"sender,omitempty" jsonschema:"description=Limits the execution of a step to matching build senders.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Status []string `yaml:"status,omitempty,flow" json:"status,omitempty" jsonschema:"enum=[failure],enum=[success],description=Limits the execution of a step to matching build statuses.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Tag []string `yaml:"tag,omitempty,flow" json:"tag,omitempty" jsonschema:"description=Limits the execution of a step to matching build tag references.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Target []string `yaml:"target,omitempty,flow" json:"target,omitempty" jsonschema:"description=Limits the execution of a step to matching build deployment targets.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Label []string `yaml:"label,omitempty,flow" json:"label,omitempty" jsonschema:"description=Limits step execution to match on pull requests labels.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Instance []string `yaml:"instance,omitempty,flow" json:"instance,omitempty" jsonschema:"description=Limits step execution to match on certain instances.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + } +) + +// ToPipeline converts the Ruleset type +// to a pipeline Ruleset type. +func (r *Ruleset) ToPipeline() *pipeline.Ruleset { + return &pipeline.Ruleset{ + If: *r.If.ToPipeline(), + Unless: *r.Unless.ToPipeline(), + Matcher: r.Matcher, + Operator: r.Operator, + Continue: r.Continue, + } +} + +// UnmarshalYAML implements the Unmarshaler interface for the Ruleset type. +func (r *Ruleset) UnmarshalYAML(unmarshal func(interface{}) error) error { + // simple struct we try unmarshalling to + simple := new(Rules) + + // advanced struct we try unmarshalling to + advanced := new(struct { + If Rules + Unless Rules + Matcher string + Operator string + Continue bool + }) + + // attempt to unmarshal simple ruleset + //nolint:errcheck // intentionally not handling error + unmarshal(simple) + // attempt to unmarshal advanced ruleset + //nolint:errcheck // intentionally not handling error + unmarshal(advanced) + + // set ruleset `unless` to advanced `unless` rules + r.Unless = advanced.Unless + // set ruleset `matcher` to advanced `matcher` + r.Matcher = advanced.Matcher + // set ruleset `operator` to advanced `operator` + r.Operator = advanced.Operator + // set ruleset `continue` to advanced `continue` + r.Continue = advanced.Continue + + // implicitly add simple ruleset to the advanced ruleset for each rule type + advanced.If.Branch = append(advanced.If.Branch, simple.Branch...) + advanced.If.Comment = append(advanced.If.Comment, simple.Comment...) + advanced.If.Event = append(advanced.If.Event, simple.Event...) + advanced.If.Path = append(advanced.If.Path, simple.Path...) + advanced.If.Repo = append(advanced.If.Repo, simple.Repo...) + advanced.If.Sender = append(advanced.If.Sender, simple.Sender...) + advanced.If.Status = append(advanced.If.Status, simple.Status...) + advanced.If.Tag = append(advanced.If.Tag, simple.Tag...) + advanced.If.Target = append(advanced.If.Target, simple.Target...) + advanced.If.Label = append(advanced.If.Label, simple.Label...) + advanced.If.Instance = append(advanced.If.Instance, simple.Instance...) + + // set ruleset `if` to advanced `if` rules + r.If = advanced.If + + // implicitly set `matcher` field if empty for ruleset + if len(r.Matcher) == 0 { + r.Matcher = constants.MatcherFilepath + } + + // implicitly set `operator` field if empty for ruleset + if len(r.Operator) == 0 { + r.Operator = constants.OperatorAnd + } + + return nil +} + +// ToPipeline converts the Rules +// type to a pipeline Rules type. +func (r *Rules) ToPipeline() *pipeline.Rules { + return &pipeline.Rules{ + Branch: r.Branch, + Comment: r.Comment, + Event: r.Event, + Path: r.Path, + Repo: r.Repo, + Sender: r.Sender, + Status: r.Status, + Tag: r.Tag, + Target: r.Target, + Label: r.Label, + Instance: r.Instance, + } +} + +// UnmarshalYAML implements the Unmarshaler interface for the Rules type. +func (r *Rules) UnmarshalYAML(unmarshal func(interface{}) error) error { + // rules struct we try unmarshalling to + rules := new(struct { + Branch raw.StringSlice + Comment raw.StringSlice + Event raw.StringSlice + Path raw.StringSlice + Repo raw.StringSlice + Sender raw.StringSlice + Status raw.StringSlice + Tag raw.StringSlice + Target raw.StringSlice + Label raw.StringSlice + Instance raw.StringSlice + }) + + // attempt to unmarshal rules + err := unmarshal(rules) + if err == nil { + r.Branch = rules.Branch + r.Comment = rules.Comment + r.Path = rules.Path + r.Repo = rules.Repo + r.Sender = rules.Sender + r.Status = rules.Status + r.Tag = rules.Tag + r.Target = rules.Target + r.Label = rules.Label + r.Instance = rules.Instance + + // account for users who use non-scoped pull_request event + events := []string{} + + for _, e := range rules.Event { + switch e { + // backwards compatibility + // pull_request = pull_request:opened + pull_request:synchronize + pull_request:reopened + // comment = comment:created + comment:edited + case constants.EventPull: + events = append(events, + constants.EventPull+":"+constants.ActionOpened, + constants.EventPull+":"+constants.ActionSynchronize, + constants.EventPull+":"+constants.ActionReopened) + case constants.EventDeploy: + events = append(events, + constants.EventDeploy+":"+constants.ActionCreated) + case constants.EventComment: + events = append(events, + constants.EventComment+":"+constants.ActionCreated, + constants.EventComment+":"+constants.ActionEdited) + default: + events = append(events, e) + } + } + + r.Event = events + } + + return err +} + +func (r *Ruleset) ToYAML() *yaml.Ruleset { + if r == nil { + return nil + } + + return &yaml.Ruleset{ + If: *r.If.ToYAML(), + Unless: *r.Unless.ToYAML(), + Matcher: r.Matcher, + Operator: r.Operator, + Continue: r.Continue, + } +} + +func (r *Rules) ToYAML() *yaml.Rules { + if r == nil { + return nil + } + + return &yaml.Rules{ + Branch: r.Branch, + Comment: r.Comment, + Event: r.Event, + Path: r.Path, + Repo: r.Repo, + Sender: r.Sender, + Status: r.Status, + Tag: r.Tag, + Target: r.Target, + Label: r.Label, + Instance: r.Instance, + } +} diff --git a/compiler/types/yaml/ruleset_test.go b/compiler/types/yaml/buildkite/ruleset_test.go similarity index 99% rename from compiler/types/yaml/ruleset_test.go rename to compiler/types/yaml/buildkite/ruleset_test.go index 6275046b0..adc926c9a 100644 --- a/compiler/types/yaml/ruleset_test.go +++ b/compiler/types/yaml/buildkite/ruleset_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/buildkite/secret.go b/compiler/types/yaml/buildkite/secret.go new file mode 100644 index 000000000..b6ea8d71a --- /dev/null +++ b/compiler/types/yaml/buildkite/secret.go @@ -0,0 +1,328 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "errors" + "fmt" + "strings" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" + "github.com/go-vela/server/constants" +) + +type ( + // SecretSlice is the yaml representation + // of the secrets block for a pipeline. + SecretSlice []*Secret + + // Secret is the yaml representation of a secret + // from the secrets block for a pipeline. + Secret struct { + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Name of secret to reference in the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/#the-name-key"` + Key string `yaml:"key,omitempty" json:"key,omitempty" jsonschema:"minLength=1,description=Path to secret to fetch from storage backend.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/#the-key-key"` + Engine string `yaml:"engine,omitempty" json:"engine,omitempty" jsonschema:"enum=native,enum=vault,default=native,description=Name of storage backend to fetch secret from.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/#the-engine-key"` + Type string `yaml:"type,omitempty" json:"type,omitempty" jsonschema:"enum=repo,enum=org,enum=shared,default=repo,description=Type of secret to fetch from storage backend.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/#the-type-key"` + Origin Origin `yaml:"origin,omitempty" json:"origin,omitempty" jsonschema:"description=Declaration to pull secrets from non-internal secret providers.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/#the-origin-key"` + Pull string `yaml:"pull,omitempty" json:"pull,omitempty" jsonschema:"enum=step_start,enum=build_start,default=build_start,description=When to pull in secrets from storage backend.\nReference: https://go-vela.github.io/docs/reference/yaml/secrets/#the-pull-key"` + } + + // Origin is the yaml representation of a method + // for looking up secrets with a secret plugin. + Origin struct { + Environment raw.StringSliceMap `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Variables to inject into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-environment-key"` + Image string `yaml:"image,omitempty" json:"image,omitempty" jsonschema:"required,minLength=1,description=Docker image to use to create the ephemeral container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-image-key"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Unique name for the secret origin.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-name-key"` + Parameters map[string]interface{} `yaml:"parameters,omitempty" json:"parameters,omitempty" jsonschema:"description=Extra configuration variables for the secret plugin.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-parameters-key"` + Secrets StepSecretSlice `yaml:"secrets,omitempty" json:"secrets,omitempty" jsonschema:"description=Secrets to inject that are necessary to retrieve the secrets.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-secrets-key"` + Pull string `yaml:"pull,omitempty" json:"pull,omitempty" jsonschema:"enum=always,enum=not_present,enum=on_start,enum=never,default=not_present,description=Declaration to configure if and when the Docker image is pulled.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-pull-key"` + Ruleset Ruleset `yaml:"ruleset,omitempty" json:"ruleset,omitempty" jsonschema:"description=Conditions to limit the execution of the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + } +) + +// ToPipeline converts the SecretSlice type +// to a pipeline SecretSlice type. +func (s *SecretSlice) ToPipeline() *pipeline.SecretSlice { + // secret slice we want to return + secretSlice := new(pipeline.SecretSlice) + + // iterate through each element in the secret slice + for _, secret := range *s { + // append the element to the pipeline secret slice + *secretSlice = append(*secretSlice, &pipeline.Secret{ + Name: secret.Name, + Key: secret.Key, + Engine: secret.Engine, + Type: secret.Type, + Origin: secret.Origin.ToPipeline(), + Pull: secret.Pull, + }) + } + + return secretSlice +} + +// UnmarshalYAML implements the Unmarshaler interface for the SecretSlice type. +func (s *SecretSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // secret slice we try unmarshalling to + secretSlice := new([]*Secret) + + // attempt to unmarshal as a secret slice type + err := unmarshal(secretSlice) + if err != nil { + return err + } + + tmp := SecretSlice{} + + // iterate through each element in the secret slice + for _, secret := range *secretSlice { + if secret.Origin.Empty() && len(secret.Name) == 0 { + continue + } + + if secret.Origin.Empty() && len(secret.Key) == 0 { + secret.Key = secret.Name + } + + // implicitly set `engine` field if empty + if secret.Origin.Empty() && len(secret.Engine) == 0 { + secret.Engine = constants.DriverNative + } + + // implicitly set `type` field if empty + if secret.Origin.Empty() && len(secret.Type) == 0 { + secret.Type = constants.SecretRepo + } + + // implicitly set `type` field if empty + if secret.Origin.Empty() && len(secret.Pull) == 0 { + secret.Pull = constants.SecretPullBuild + } + + // implicitly set `pull` field if empty + if !secret.Origin.Empty() && len(secret.Origin.Pull) == 0 { + secret.Origin.Pull = constants.PullNotPresent + } + + // TODO: remove this in a future release + // + // handle true deprecated pull policy + // + // a `true` pull policy equates to `always` + if !secret.Origin.Empty() && strings.EqualFold(secret.Origin.Pull, "true") { + secret.Origin.Pull = constants.PullAlways + } + + // TODO: remove this in a future release + // + // handle false deprecated pull policy + // + // a `false` pull policy equates to `not_present` + if !secret.Origin.Empty() && strings.EqualFold(secret.Origin.Pull, "false") { + secret.Origin.Pull = constants.PullNotPresent + } + + tmp = append(tmp, secret) + } + + // overwrite existing SecretSlice + *s = tmp + + return nil +} + +// Empty returns true if the provided origin is empty. +func (o *Origin) Empty() bool { + // return true if the origin is nil + if o == nil { + return true + } + + // return true if every origin field is empty + if o.Environment == nil && + len(o.Image) == 0 && + len(o.Name) == 0 && + o.Parameters == nil && + len(o.Secrets) == 0 && + len(o.Pull) == 0 { + return true + } + + return false +} + +// MergeEnv takes a list of environment variables and attempts +// to set them in the secret environment. If the environment +// variable already exists in the secret, than this will +// overwrite the existing environment variable. +func (o *Origin) MergeEnv(environment map[string]string) error { + // check if the secret container is empty + if o.Empty() { + // TODO: evaluate if we should error here + // + // immediately return and do nothing + // + // treated as a no-op + return nil + } + + // check if the environment provided is empty + if environment == nil { + return fmt.Errorf("empty environment provided for secret %s", o.Name) + } + + // iterate through all environment variables provided + for key, value := range environment { + // set or update the secret environment variable + o.Environment[key] = value + } + + return nil +} + +// ToPipeline converts the Origin type +// to a pipeline Container type. +func (o *Origin) ToPipeline() *pipeline.Container { + return &pipeline.Container{ + Environment: o.Environment, + Image: o.Image, + Name: o.Name, + Pull: o.Pull, + Ruleset: *o.Ruleset.ToPipeline(), + Secrets: *o.Secrets.ToPipeline(), + } +} + +type ( + // StepSecretSlice is the yaml representation of + // the secrets block for a step in a pipeline. + StepSecretSlice []*StepSecret + + // StepSecret is the yaml representation of a secret + // from a secrets block for a step in a pipeline. + StepSecret struct { + Source string `yaml:"source,omitempty"` + Target string `yaml:"target,omitempty"` + } +) + +// ToPipeline converts the StepSecretSlice type +// to a pipeline StepSecretSlice type. +func (s *StepSecretSlice) ToPipeline() *pipeline.StepSecretSlice { + // step secret slice we want to return + secretSlice := new(pipeline.StepSecretSlice) + + // iterate through each element in the step secret slice + for _, secret := range *s { + // append the element to the pipeline step secret slice + *secretSlice = append(*secretSlice, &pipeline.StepSecret{ + Source: secret.Source, + Target: secret.Target, + }) + } + + return secretSlice +} + +// UnmarshalYAML implements the Unmarshaler interface for the StepSecretSlice type. +func (s *StepSecretSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // string slice we try unmarshalling to + stringSlice := new(raw.StringSlice) + + // attempt to unmarshal as a string slice type + err := unmarshal(stringSlice) + if err == nil { + // iterate through each element in the string slice + for _, secret := range *stringSlice { + // append the element to the step secret slice + *s = append(*s, &StepSecret{ + Source: secret, + Target: strings.ToUpper(secret), + }) + } + + return nil + } + + // step secret slice we try unmarshalling to + secrets := new([]*StepSecret) + + // attempt to unmarshal as a step secret slice type + err = unmarshal(secrets) + if err == nil { + // check for secret source and target + for _, secret := range *secrets { + if len(secret.Source) == 0 || len(secret.Target) == 0 { + return fmt.Errorf("no secret source or target found") + } + + secret.Target = strings.ToUpper(secret.Target) + } + + // overwrite existing StepSecretSlice + *s = StepSecretSlice(*secrets) + + return nil + } + + return errors.New("failed to unmarshal StepSecretSlice") +} + +func (s *StepSecretSlice) ToYAML() *yaml.StepSecretSlice { + // step secret slice we want to return + secretSlice := new(yaml.StepSecretSlice) + + // iterate through each element in the step secret slice + for _, secret := range *s { + // append the element to the yaml step secret slice + *secretSlice = append(*secretSlice, &yaml.StepSecret{ + Source: secret.Source, + Target: secret.Target, + }) + } + + return secretSlice +} + +func (o *Origin) ToYAML() yaml.Origin { + return yaml.Origin{ + Environment: o.Environment, + Image: o.Image, + Name: o.Name, + Parameters: o.Parameters, + Secrets: *o.Secrets.ToYAML(), + Pull: o.Pull, + Ruleset: *o.Ruleset.ToYAML(), + } +} + +func (s *Secret) ToYAML() *yaml.Secret { + if s == nil { + return nil + } + + return &yaml.Secret{ + Name: s.Name, + Key: s.Key, + Engine: s.Engine, + Type: s.Type, + Origin: s.Origin.ToYAML(), + Pull: s.Pull, + } +} + +func (s *SecretSlice) ToYAML() *yaml.SecretSlice { + // secret slice we want to return + secretSlice := new(yaml.SecretSlice) + + // iterate through each element in the secret slice + for _, secret := range *s { + // append the element to the yaml secret slice + *secretSlice = append(*secretSlice, secret.ToYAML()) + } + + return secretSlice +} diff --git a/compiler/types/yaml/secret_test.go b/compiler/types/yaml/buildkite/secret_test.go similarity index 99% rename from compiler/types/yaml/secret_test.go rename to compiler/types/yaml/buildkite/secret_test.go index 68a9ed4d6..1d731e477 100644 --- a/compiler/types/yaml/secret_test.go +++ b/compiler/types/yaml/buildkite/secret_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/buildkite/service.go b/compiler/types/yaml/buildkite/service.go new file mode 100644 index 000000000..9fb9a0ed1 --- /dev/null +++ b/compiler/types/yaml/buildkite/service.go @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "fmt" + "strings" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" + "github.com/go-vela/server/constants" +) + +type ( + // ServiceSlice is the yaml representation + // of the Services block for a pipeline. + ServiceSlice []*Service + + // Service is the yaml representation + // of a Service in a pipeline. + Service struct { + Image string `yaml:"image,omitempty" json:"image,omitempty" jsonschema:"required,minLength=1,description=Docker image used to create ephemeral container.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-image-key"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Unique identifier for the container in the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-name-key"` + Entrypoint raw.StringSlice `yaml:"entrypoint,omitempty" json:"entrypoint,omitempty" jsonschema:"description=Commands to execute inside the container.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-entrypoint-key"` + Environment raw.StringSliceMap `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Variables to inject into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-environment-key"` + Ports raw.StringSlice `yaml:"ports,omitempty" json:"ports,omitempty" jsonschema:"description=List of ports to map for the container in the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-ports-key"` + Pull string `yaml:"pull,omitempty" json:"pull,omitempty" jsonschema:"enum=always,enum=not_present,enum=on_start,enum=never,default=not_present,description=Declaration to configure if and when the Docker image is pulled.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-pul-key"` + Ulimits UlimitSlice `yaml:"ulimits,omitempty" json:"ulimits,omitempty" jsonschema:"description=Set the user limits for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/services/#the-ulimits-key"` + User string `yaml:"user,omitempty" json:"user,omitempty" jsonschema:"description=Set the user for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-user-key"` + } +) + +// ToPipeline converts the ServiceSlice type +// to a pipeline ContainerSlice type. +func (s *ServiceSlice) ToPipeline() *pipeline.ContainerSlice { + // service slice we want to return + serviceSlice := new(pipeline.ContainerSlice) + + // iterate through each element in the service slice + for _, service := range *s { + // append the element to the pipeline container slice + *serviceSlice = append(*serviceSlice, &pipeline.Container{ + Detach: true, + Image: service.Image, + Name: service.Name, + Entrypoint: service.Entrypoint, + Environment: service.Environment, + Ports: service.Ports, + Pull: service.Pull, + Ulimits: *service.Ulimits.ToPipeline(), + User: service.User, + }) + } + + return serviceSlice +} + +// UnmarshalYAML implements the Unmarshaler interface for the ServiceSlice type. +func (s *ServiceSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // service slice we try unmarshalling to + serviceSlice := new([]*Service) + + // attempt to unmarshal as a service slice type + err := unmarshal(serviceSlice) + if err != nil { + return err + } + + // iterate through each element in the service slice + for _, service := range *serviceSlice { + // handle nil service to avoid panic + if service == nil { + return fmt.Errorf("invalid service with nil content found") + } + + // implicitly set `pull` field if empty + if len(service.Pull) == 0 { + service.Pull = constants.PullNotPresent + } + + // TODO: remove this in a future release + // + // handle true deprecated pull policy + // + // a `true` pull policy equates to `always` + if strings.EqualFold(service.Pull, "true") { + service.Pull = constants.PullAlways + } + + // TODO: remove this in a future release + // + // handle false deprecated pull policy + // + // a `false` pull policy equates to `not_present` + if strings.EqualFold(service.Pull, "false") { + service.Pull = constants.PullNotPresent + } + } + + // overwrite existing ServiceSlice + *s = ServiceSlice(*serviceSlice) + + return nil +} + +// MergeEnv takes a list of environment variables and attempts +// to set them in the service environment. If the environment +// variable already exists in the service, than this will +// overwrite the existing environment variable. +func (s *Service) MergeEnv(environment map[string]string) error { + // check if the service container is empty + if s == nil || s.Environment == nil { + // TODO: evaluate if we should error here + // + // immediately return and do nothing + // + // treated as a no-op + return nil + } + + // check if the environment provided is empty + if environment == nil { + return fmt.Errorf("empty environment provided for service %s", s.Name) + } + + // iterate through all environment variables provided + for key, value := range environment { + // set or update the service environment variable + s.Environment[key] = value + } + + return nil +} + +func (s *Service) ToYAML() *yaml.Service { + if s == nil { + return nil + } + + return &yaml.Service{ + Image: s.Image, + Name: s.Name, + Entrypoint: s.Entrypoint, + Environment: s.Environment, + Ports: s.Ports, + Pull: s.Pull, + Ulimits: *s.Ulimits.ToYAML(), + User: s.User, + } +} + +func (s *ServiceSlice) ToYAML() *yaml.ServiceSlice { + // service slice we want to return + serviceSlice := new(yaml.ServiceSlice) + + // iterate through each element in the service slice + for _, service := range *s { + // append the element to the yaml service slice + *serviceSlice = append(*serviceSlice, service.ToYAML()) + } + + return serviceSlice +} diff --git a/compiler/types/yaml/service_test.go b/compiler/types/yaml/buildkite/service_test.go similarity index 99% rename from compiler/types/yaml/service_test.go rename to compiler/types/yaml/buildkite/service_test.go index 09f4fbc16..bc97c258e 100644 --- a/compiler/types/yaml/service_test.go +++ b/compiler/types/yaml/buildkite/service_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/stage.go b/compiler/types/yaml/buildkite/stage.go similarity index 85% rename from compiler/types/yaml/stage.go rename to compiler/types/yaml/buildkite/stage.go index 79b635b05..c7909d2f7 100644 --- a/compiler/types/yaml/stage.go +++ b/compiler/types/yaml/buildkite/stage.go @@ -1,15 +1,16 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "fmt" - "github.com/buildkite/yaml" + bkYaml "github.com/buildkite/yaml" "github.com/invopop/jsonschema" "github.com/go-vela/server/compiler/types/pipeline" "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) type ( @@ -53,7 +54,7 @@ func (s *StageSlice) ToPipeline() *pipeline.StageSlice { // UnmarshalYAML implements the Unmarshaler interface for the StageSlice type. func (s *StageSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { // map slice we try unmarshalling to - mapSlice := new(yaml.MapSlice) + mapSlice := new(bkYaml.MapSlice) // attempt to unmarshal as a map slice type err := unmarshal(mapSlice) @@ -67,10 +68,10 @@ func (s *StageSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { stage := new(Stage) // marshal interface value from ordered map - out, _ := yaml.Marshal(v.Value) + out, _ := bkYaml.Marshal(v.Value) // unmarshal interface value as stage - err = yaml.Unmarshal(out, stage) + err = bkYaml.Unmarshal(out, stage) if err != nil { return err } @@ -103,7 +104,7 @@ func (s *StageSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { // MarshalYAML implements the marshaler interface for the StageSlice type. func (s StageSlice) MarshalYAML() (interface{}, error) { // map slice to return as marshaled output - var output yaml.MapSlice + var output bkYaml.MapSlice // loop over the input stages for _, inputStage := range s { @@ -120,7 +121,7 @@ func (s StageSlice) MarshalYAML() (interface{}, error) { outputStage.Steps = inputStage.Steps // append stage to MapSlice - output = append(output, yaml.MapItem{Key: inputStage.Name, Value: outputStage}) + output = append(output, bkYaml.MapItem{Key: inputStage.Name, Value: outputStage}) } return output, nil @@ -170,3 +171,30 @@ func (s *Stage) MergeEnv(environment map[string]string) error { return nil } + +func (s *Stage) ToYAML() *yaml.Stage { + if s == nil { + return nil + } + + return &yaml.Stage{ + Environment: s.Environment, + Name: s.Name, + Needs: s.Needs, + Independent: s.Independent, + Steps: *s.Steps.ToYAML(), + } +} + +func (s *StageSlice) ToYAML() *yaml.StageSlice { + // stage slice we want to return + stageSlice := new(yaml.StageSlice) + + // iterate through each element in the stage slice + for _, stage := range *s { + // append the element to the yaml stage slice + *stageSlice = append(*stageSlice, stage.ToYAML()) + } + + return stageSlice +} diff --git a/compiler/types/yaml/stage_test.go b/compiler/types/yaml/buildkite/stage_test.go similarity index 99% rename from compiler/types/yaml/stage_test.go rename to compiler/types/yaml/buildkite/stage_test.go index 7c63253b8..3c87a3859 100644 --- a/compiler/types/yaml/stage_test.go +++ b/compiler/types/yaml/buildkite/stage_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/buildkite/step.go b/compiler/types/yaml/buildkite/step.go new file mode 100644 index 000000000..4b36b93fd --- /dev/null +++ b/compiler/types/yaml/buildkite/step.go @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "fmt" + "strings" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" + "github.com/go-vela/server/constants" +) + +type ( + // StepSlice is the yaml representation + // of the steps block for a pipeline. + StepSlice []*Step + + // Step is the yaml representation of a step + // from the steps block for a pipeline. + Step struct { + Ruleset Ruleset `yaml:"ruleset,omitempty" json:"ruleset,omitempty" jsonschema:"description=Conditions to limit the execution of the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ruleset-key"` + Commands raw.StringSlice `yaml:"commands,omitempty" json:"commands,omitempty" jsonschema:"description=Execution instructions to run inside the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-commands-key"` + Entrypoint raw.StringSlice `yaml:"entrypoint,omitempty" json:"entrypoint,omitempty" jsonschema:"description=Command to execute inside the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-entrypoint-key"` + Secrets StepSecretSlice `yaml:"secrets,omitempty" json:"secrets,omitempty" jsonschema:"description=Sensitive variables injected into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-secrets-key"` + Template StepTemplate `yaml:"template,omitempty" json:"template,omitempty" jsonschema:"oneof_required=template,description=Name of template to expand in the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-template-key"` + Ulimits UlimitSlice `yaml:"ulimits,omitempty" json:"ulimits,omitempty" jsonschema:"description=Set the user limits for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ulimits-key"` + Volumes VolumeSlice `yaml:"volumes,omitempty" json:"volumes,omitempty" jsonschema:"description=Mount volumes for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-volume-key"` + Image string `yaml:"image,omitempty" json:"image,omitempty" jsonschema:"oneof_required=image,minLength=1,description=Docker image to use to create the ephemeral container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-image-key"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Unique name for the step.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-name-key"` + Pull string `yaml:"pull,omitempty" json:"pull,omitempty" jsonschema:"enum=always,enum=not_present,enum=on_start,enum=never,default=not_present,description=Declaration to configure if and when the Docker image is pulled.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-pull-key"` + Environment raw.StringSliceMap `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Provide environment variables injected into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-environment-key"` + Parameters map[string]interface{} `yaml:"parameters,omitempty" json:"parameters,omitempty" jsonschema:"description=Extra configuration variables for a plugin.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-parameters-key"` + Detach bool `yaml:"detach,omitempty" json:"detach,omitempty" jsonschema:"description=Run the container in a detached (headless) state.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-detach-key"` + Privileged bool `yaml:"privileged,omitempty" json:"privileged,omitempty" jsonschema:"description=Run the container with extra privileges.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-privileged-key"` + User string `yaml:"user,omitempty" json:"user,omitempty" jsonschema:"description=Set the user for the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-user-key"` + ReportAs string `yaml:"report_as,omitempty" json:"report_as,omitempty" jsonschema:"description=Set the name of the step to report as.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-report_as-key"` + IDRequest string `yaml:"id_request,omitempty" json:"id_request,omitempty" jsonschema:"description=Request ID Request Token for the step.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-id_request-key"` + } +) + +// ToPipeline converts the StepSlice type +// to a pipeline ContainerSlice type. +func (s *StepSlice) ToPipeline() *pipeline.ContainerSlice { + // step slice we want to return + stepSlice := new(pipeline.ContainerSlice) + + // iterate through each element in the step slice + for _, step := range *s { + // append the element to the pipeline container slice + *stepSlice = append(*stepSlice, &pipeline.Container{ + Commands: step.Commands, + Detach: step.Detach, + Entrypoint: step.Entrypoint, + Environment: step.Environment, + Image: step.Image, + Name: step.Name, + Privileged: step.Privileged, + Pull: step.Pull, + Ruleset: *step.Ruleset.ToPipeline(), + Secrets: *step.Secrets.ToPipeline(), + Ulimits: *step.Ulimits.ToPipeline(), + Volumes: *step.Volumes.ToPipeline(), + User: step.User, + ReportAs: step.ReportAs, + IDRequest: step.IDRequest, + }) + } + + return stepSlice +} + +// UnmarshalYAML implements the Unmarshaler interface for the StepSlice type. +func (s *StepSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // step slice we try unmarshalling to + stepSlice := new([]*Step) + + // attempt to unmarshal as a step slice type + err := unmarshal(stepSlice) + if err != nil { + return err + } + + // iterate through each element in the step slice + for _, step := range *stepSlice { + // handle nil step to avoid panic + if step == nil { + return fmt.Errorf("invalid step with nil content found") + } + + // implicitly set `pull` field if empty + if len(step.Pull) == 0 { + step.Pull = constants.PullNotPresent + } + + // TODO: remove this in a future release + // + // handle true deprecated pull policy + // + // a `true` pull policy equates to `always` + if strings.EqualFold(step.Pull, "true") { + step.Pull = constants.PullAlways + } + + // TODO: remove this in a future release + // + // handle false deprecated pull policy + // + // a `false` pull policy equates to `not_present` + if strings.EqualFold(step.Pull, "false") { + step.Pull = constants.PullNotPresent + } + } + + // overwrite existing StepSlice + *s = StepSlice(*stepSlice) + + return nil +} + +// MergeEnv takes a list of environment variables and attempts +// to set them in the step environment. If the environment +// variable already exists in the step, than this will +// overwrite the existing environment variable. +func (s *Step) MergeEnv(environment map[string]string) error { + // check if the step container is empty + if s == nil || s.Environment == nil { + // TODO: evaluate if we should error here + // + // immediately return and do nothing + // + // treated as a no-op + return nil + } + + // check if the environment provided is empty + if environment == nil { + return fmt.Errorf("empty environment provided for step %s", s.Name) + } + + // iterate through all environment variables provided + for key, value := range environment { + // set or update the step environment variable + s.Environment[key] = value + } + + return nil +} + +func (s *Step) ToYAML() *yaml.Step { + if s == nil { + return nil + } + + return &yaml.Step{ + Commands: s.Commands, + Detach: s.Detach, + Entrypoint: s.Entrypoint, + Environment: s.Environment, + Image: s.Image, + Name: s.Name, + Privileged: s.Privileged, + Pull: s.Pull, + Ruleset: *s.Ruleset.ToYAML(), + Secrets: *s.Secrets.ToYAML(), + Template: s.Template.ToYAML(), + Ulimits: *s.Ulimits.ToYAML(), + Volumes: *s.Volumes.ToYAML(), + Parameters: s.Parameters, + User: s.User, + ReportAs: s.ReportAs, + IDRequest: s.IDRequest, + } +} + +func (s *StepSlice) ToYAML() *yaml.StepSlice { + // step slice we want to return + stepSlice := new(yaml.StepSlice) + + // iterate through each element in the step slice + for _, step := range *s { + // append the element to the yaml step slice + *stepSlice = append(*stepSlice, step.ToYAML()) + } + + return stepSlice +} diff --git a/compiler/types/yaml/step_test.go b/compiler/types/yaml/buildkite/step_test.go similarity index 99% rename from compiler/types/yaml/step_test.go rename to compiler/types/yaml/buildkite/step_test.go index 2f5336be4..da4df26fe 100644 --- a/compiler/types/yaml/step_test.go +++ b/compiler/types/yaml/buildkite/step_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/buildkite/template.go b/compiler/types/yaml/buildkite/template.go new file mode 100644 index 000000000..f9380dab3 --- /dev/null +++ b/compiler/types/yaml/buildkite/template.go @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +type ( + // TemplateSlice is the yaml representation + // of the templates block for a pipeline. + TemplateSlice []*Template + + // Template is the yaml representation of a template + // from the templates block for a pipeline. + Template struct { + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Unique identifier for the template.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-name-key"` + Source string `yaml:"source,omitempty" json:"source,omitempty" jsonschema:"required,minLength=1,description=Path to template in remote system.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-source-key"` + Format string `yaml:"format,omitempty" json:"format,omitempty" jsonschema:"enum=starlark,enum=golang,enum=go,default=go,minLength=1,description=language used within the template file \nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-format-key"` + Type string `yaml:"type,omitempty" json:"type,omitempty" jsonschema:"minLength=1,example=github,description=Type of template provided from the remote system.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-type-key"` + Variables map[string]interface{} `yaml:"vars,omitempty" json:"vars,omitempty" jsonschema:"description=Variables injected into the template.\nReference: https://go-vela.github.io/docs/reference/yaml/templates/#the-variables-key"` + } + + // StepTemplate is the yaml representation of the + // template block for a step in a pipeline. + StepTemplate struct { + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Unique identifier for the template.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-template-key"` + Variables map[string]interface{} `yaml:"vars,omitempty" json:"vars,omitempty" jsonschema:"description=Variables injected into the template.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-template-key"` + } +) + +// UnmarshalYAML implements the Unmarshaler interface for the TemplateSlice type. +func (t *TemplateSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // template slice we try unmarshalling to + templateSlice := new([]*Template) + + // attempt to unmarshal as a template slice type + err := unmarshal(templateSlice) + if err != nil { + return err + } + + // overwrite existing TemplateSlice + *t = TemplateSlice(*templateSlice) + + return nil +} + +// ToAPI converts the Template type +// to an API Template type. +func (t *Template) ToAPI() *api.Template { + template := new(api.Template) + + template.SetName(t.Name) + template.SetSource(t.Source) + template.SetType(t.Type) + + return template +} + +// TemplateFromAPI converts the API Template type +// to a yaml Template type. +func TemplateFromAPI(t *api.Template) *Template { + template := &Template{ + Name: t.GetName(), + Source: t.GetSource(), + Type: t.GetType(), + } + + return template +} + +// Map helper function that creates a map of templates from a slice of templates. +func (t *TemplateSlice) Map() map[string]*Template { + m := make(map[string]*Template) + + if t == nil { + return m + } + + for _, tmpl := range *t { + m[tmpl.Name] = tmpl + } + + return m +} + +func (t *Template) ToYAML() *yaml.Template { + if t == nil { + return nil + } + + return &yaml.Template{ + Name: t.Name, + Source: t.Source, + Format: t.Format, + Type: t.Type, + Variables: t.Variables, + } +} + +func (t *TemplateSlice) ToYAML() *yaml.TemplateSlice { + // template slice we want to return + templateSlice := new(yaml.TemplateSlice) + + // iterate through each element in the template slice + for _, template := range *t { + // append the element to the yaml template slice + *templateSlice = append(*templateSlice, template.ToYAML()) + } + + return templateSlice +} + +func (t *StepTemplate) ToYAML() yaml.StepTemplate { + return yaml.StepTemplate{ + Name: t.Name, + Variables: t.Variables, + } +} diff --git a/compiler/types/yaml/template_test.go b/compiler/types/yaml/buildkite/template_test.go similarity index 99% rename from compiler/types/yaml/template_test.go rename to compiler/types/yaml/buildkite/template_test.go index b000cd70b..f81481005 100644 --- a/compiler/types/yaml/template_test.go +++ b/compiler/types/yaml/buildkite/template_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/testdata/build.yml b/compiler/types/yaml/buildkite/testdata/build.yml similarity index 100% rename from compiler/types/yaml/testdata/build.yml rename to compiler/types/yaml/buildkite/testdata/build.yml diff --git a/compiler/types/yaml/testdata/build/validate/bad_pipeline0.yml b/compiler/types/yaml/buildkite/testdata/build/validate/bad_pipeline0.yml similarity index 100% rename from compiler/types/yaml/testdata/build/validate/bad_pipeline0.yml rename to compiler/types/yaml/buildkite/testdata/build/validate/bad_pipeline0.yml diff --git a/compiler/types/yaml/testdata/build/validate/bad_pipeline1.yml b/compiler/types/yaml/buildkite/testdata/build/validate/bad_pipeline1.yml similarity index 100% rename from compiler/types/yaml/testdata/build/validate/bad_pipeline1.yml rename to compiler/types/yaml/buildkite/testdata/build/validate/bad_pipeline1.yml diff --git a/compiler/types/yaml/testdata/build/validate/bad_version.yml b/compiler/types/yaml/buildkite/testdata/build/validate/bad_version.yml similarity index 100% rename from compiler/types/yaml/testdata/build/validate/bad_version.yml rename to compiler/types/yaml/buildkite/testdata/build/validate/bad_version.yml diff --git a/compiler/types/yaml/testdata/build/validate/step.yml b/compiler/types/yaml/buildkite/testdata/build/validate/step.yml similarity index 100% rename from compiler/types/yaml/testdata/build/validate/step.yml rename to compiler/types/yaml/buildkite/testdata/build/validate/step.yml diff --git a/compiler/types/yaml/testdata/build_anchor_stage.yml b/compiler/types/yaml/buildkite/testdata/build_anchor_stage.yml similarity index 100% rename from compiler/types/yaml/testdata/build_anchor_stage.yml rename to compiler/types/yaml/buildkite/testdata/build_anchor_stage.yml diff --git a/compiler/types/yaml/testdata/build_anchor_step.yml b/compiler/types/yaml/buildkite/testdata/build_anchor_step.yml similarity index 100% rename from compiler/types/yaml/testdata/build_anchor_step.yml rename to compiler/types/yaml/buildkite/testdata/build_anchor_step.yml diff --git a/compiler/types/yaml/testdata/build_empty_env.yml b/compiler/types/yaml/buildkite/testdata/build_empty_env.yml similarity index 100% rename from compiler/types/yaml/testdata/build_empty_env.yml rename to compiler/types/yaml/buildkite/testdata/build_empty_env.yml diff --git a/compiler/types/yaml/testdata/build_with_deploy_config.yml b/compiler/types/yaml/buildkite/testdata/build_with_deploy_config.yml similarity index 100% rename from compiler/types/yaml/testdata/build_with_deploy_config.yml rename to compiler/types/yaml/buildkite/testdata/build_with_deploy_config.yml diff --git a/compiler/types/yaml/testdata/deploy_parameter.yml b/compiler/types/yaml/buildkite/testdata/deploy_parameter.yml similarity index 100% rename from compiler/types/yaml/testdata/deploy_parameter.yml rename to compiler/types/yaml/buildkite/testdata/deploy_parameter.yml diff --git a/compiler/types/yaml/testdata/invalid.yml b/compiler/types/yaml/buildkite/testdata/invalid.yml similarity index 100% rename from compiler/types/yaml/testdata/invalid.yml rename to compiler/types/yaml/buildkite/testdata/invalid.yml diff --git a/compiler/types/yaml/testdata/merge_anchor.yml b/compiler/types/yaml/buildkite/testdata/merge_anchor.yml similarity index 100% rename from compiler/types/yaml/testdata/merge_anchor.yml rename to compiler/types/yaml/buildkite/testdata/merge_anchor.yml diff --git a/compiler/types/yaml/testdata/metadata.yml b/compiler/types/yaml/buildkite/testdata/metadata.yml similarity index 100% rename from compiler/types/yaml/testdata/metadata.yml rename to compiler/types/yaml/buildkite/testdata/metadata.yml diff --git a/compiler/types/yaml/testdata/metadata_env.yml b/compiler/types/yaml/buildkite/testdata/metadata_env.yml similarity index 100% rename from compiler/types/yaml/testdata/metadata_env.yml rename to compiler/types/yaml/buildkite/testdata/metadata_env.yml diff --git a/compiler/types/yaml/testdata/ruleset_advanced.yml b/compiler/types/yaml/buildkite/testdata/ruleset_advanced.yml similarity index 100% rename from compiler/types/yaml/testdata/ruleset_advanced.yml rename to compiler/types/yaml/buildkite/testdata/ruleset_advanced.yml diff --git a/compiler/types/yaml/testdata/ruleset_regex.yml b/compiler/types/yaml/buildkite/testdata/ruleset_regex.yml similarity index 100% rename from compiler/types/yaml/testdata/ruleset_regex.yml rename to compiler/types/yaml/buildkite/testdata/ruleset_regex.yml diff --git a/compiler/types/yaml/testdata/ruleset_simple.yml b/compiler/types/yaml/buildkite/testdata/ruleset_simple.yml similarity index 100% rename from compiler/types/yaml/testdata/ruleset_simple.yml rename to compiler/types/yaml/buildkite/testdata/ruleset_simple.yml diff --git a/compiler/types/yaml/testdata/secret.yml b/compiler/types/yaml/buildkite/testdata/secret.yml similarity index 100% rename from compiler/types/yaml/testdata/secret.yml rename to compiler/types/yaml/buildkite/testdata/secret.yml diff --git a/compiler/types/yaml/testdata/secret/validate/no_name.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/no_name.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/no_name.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/no_name.yml diff --git a/compiler/types/yaml/testdata/secret/validate/org.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/org.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/org.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/org.yml diff --git a/compiler/types/yaml/testdata/secret/validate/org_bad_engine.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/org_bad_engine.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/org_bad_engine.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/org_bad_engine.yml diff --git a/compiler/types/yaml/testdata/secret/validate/org_bad_key.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/org_bad_key.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/org_bad_key.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/org_bad_key.yml diff --git a/compiler/types/yaml/testdata/secret/validate/plugin.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/plugin.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/plugin.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/plugin.yml diff --git a/compiler/types/yaml/testdata/secret/validate/plugin_bad_image.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/plugin_bad_image.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/plugin_bad_image.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/plugin_bad_image.yml diff --git a/compiler/types/yaml/testdata/secret/validate/plugin_bad_name.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/plugin_bad_name.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/plugin_bad_name.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/plugin_bad_name.yml diff --git a/compiler/types/yaml/testdata/secret/validate/repo.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/repo.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/repo.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/repo.yml diff --git a/compiler/types/yaml/testdata/secret/validate/repo_bad_engine.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/repo_bad_engine.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/repo_bad_engine.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/repo_bad_engine.yml diff --git a/compiler/types/yaml/testdata/secret/validate/repo_bad_key.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/repo_bad_key.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/repo_bad_key.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/repo_bad_key.yml diff --git a/compiler/types/yaml/testdata/secret/validate/shared.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/shared.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/shared.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/shared.yml diff --git a/compiler/types/yaml/testdata/secret/validate/shared_bad_engine.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/shared_bad_engine.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/shared_bad_engine.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/shared_bad_engine.yml diff --git a/compiler/types/yaml/testdata/secret/validate/shared_bad_key.yml b/compiler/types/yaml/buildkite/testdata/secret/validate/shared_bad_key.yml similarity index 100% rename from compiler/types/yaml/testdata/secret/validate/shared_bad_key.yml rename to compiler/types/yaml/buildkite/testdata/secret/validate/shared_bad_key.yml diff --git a/compiler/types/yaml/testdata/service.yml b/compiler/types/yaml/buildkite/testdata/service.yml similarity index 100% rename from compiler/types/yaml/testdata/service.yml rename to compiler/types/yaml/buildkite/testdata/service.yml diff --git a/compiler/types/yaml/testdata/service/validate/bad_image.yml b/compiler/types/yaml/buildkite/testdata/service/validate/bad_image.yml similarity index 100% rename from compiler/types/yaml/testdata/service/validate/bad_image.yml rename to compiler/types/yaml/buildkite/testdata/service/validate/bad_image.yml diff --git a/compiler/types/yaml/testdata/service/validate/minimal.yml b/compiler/types/yaml/buildkite/testdata/service/validate/minimal.yml similarity index 100% rename from compiler/types/yaml/testdata/service/validate/minimal.yml rename to compiler/types/yaml/buildkite/testdata/service/validate/minimal.yml diff --git a/compiler/types/yaml/testdata/service/validate/missing_image.yml b/compiler/types/yaml/buildkite/testdata/service/validate/missing_image.yml similarity index 100% rename from compiler/types/yaml/testdata/service/validate/missing_image.yml rename to compiler/types/yaml/buildkite/testdata/service/validate/missing_image.yml diff --git a/compiler/types/yaml/testdata/service/validate/missing_name.yml b/compiler/types/yaml/buildkite/testdata/service/validate/missing_name.yml similarity index 100% rename from compiler/types/yaml/testdata/service/validate/missing_name.yml rename to compiler/types/yaml/buildkite/testdata/service/validate/missing_name.yml diff --git a/compiler/types/yaml/testdata/service_nil.yml b/compiler/types/yaml/buildkite/testdata/service_nil.yml similarity index 100% rename from compiler/types/yaml/testdata/service_nil.yml rename to compiler/types/yaml/buildkite/testdata/service_nil.yml diff --git a/compiler/types/yaml/testdata/stage.yml b/compiler/types/yaml/buildkite/testdata/stage.yml similarity index 100% rename from compiler/types/yaml/testdata/stage.yml rename to compiler/types/yaml/buildkite/testdata/stage.yml diff --git a/compiler/types/yaml/testdata/stage/validate/bad_image.yml b/compiler/types/yaml/buildkite/testdata/stage/validate/bad_image.yml similarity index 100% rename from compiler/types/yaml/testdata/stage/validate/bad_image.yml rename to compiler/types/yaml/buildkite/testdata/stage/validate/bad_image.yml diff --git a/compiler/types/yaml/testdata/stage/validate/minimal.yml b/compiler/types/yaml/buildkite/testdata/stage/validate/minimal.yml similarity index 100% rename from compiler/types/yaml/testdata/stage/validate/minimal.yml rename to compiler/types/yaml/buildkite/testdata/stage/validate/minimal.yml diff --git a/compiler/types/yaml/testdata/stage/validate/missing.yml b/compiler/types/yaml/buildkite/testdata/stage/validate/missing.yml similarity index 100% rename from compiler/types/yaml/testdata/stage/validate/missing.yml rename to compiler/types/yaml/buildkite/testdata/stage/validate/missing.yml diff --git a/compiler/types/yaml/testdata/stage/validate/missing_image.yml b/compiler/types/yaml/buildkite/testdata/stage/validate/missing_image.yml similarity index 100% rename from compiler/types/yaml/testdata/stage/validate/missing_image.yml rename to compiler/types/yaml/buildkite/testdata/stage/validate/missing_image.yml diff --git a/compiler/types/yaml/testdata/stage/validate/missing_name.yml b/compiler/types/yaml/buildkite/testdata/stage/validate/missing_name.yml similarity index 100% rename from compiler/types/yaml/testdata/stage/validate/missing_name.yml rename to compiler/types/yaml/buildkite/testdata/stage/validate/missing_name.yml diff --git a/compiler/types/yaml/testdata/step.yml b/compiler/types/yaml/buildkite/testdata/step.yml similarity index 100% rename from compiler/types/yaml/testdata/step.yml rename to compiler/types/yaml/buildkite/testdata/step.yml diff --git a/compiler/types/yaml/testdata/step/validate/bad_image.yml b/compiler/types/yaml/buildkite/testdata/step/validate/bad_image.yml similarity index 100% rename from compiler/types/yaml/testdata/step/validate/bad_image.yml rename to compiler/types/yaml/buildkite/testdata/step/validate/bad_image.yml diff --git a/compiler/types/yaml/testdata/step/validate/minimal.yml b/compiler/types/yaml/buildkite/testdata/step/validate/minimal.yml similarity index 100% rename from compiler/types/yaml/testdata/step/validate/minimal.yml rename to compiler/types/yaml/buildkite/testdata/step/validate/minimal.yml diff --git a/compiler/types/yaml/testdata/step/validate/missing.yml b/compiler/types/yaml/buildkite/testdata/step/validate/missing.yml similarity index 100% rename from compiler/types/yaml/testdata/step/validate/missing.yml rename to compiler/types/yaml/buildkite/testdata/step/validate/missing.yml diff --git a/compiler/types/yaml/testdata/step/validate/missing_image.yml b/compiler/types/yaml/buildkite/testdata/step/validate/missing_image.yml similarity index 100% rename from compiler/types/yaml/testdata/step/validate/missing_image.yml rename to compiler/types/yaml/buildkite/testdata/step/validate/missing_image.yml diff --git a/compiler/types/yaml/testdata/step/validate/missing_name.yml b/compiler/types/yaml/buildkite/testdata/step/validate/missing_name.yml similarity index 100% rename from compiler/types/yaml/testdata/step/validate/missing_name.yml rename to compiler/types/yaml/buildkite/testdata/step/validate/missing_name.yml diff --git a/compiler/types/yaml/testdata/step_malformed.yml b/compiler/types/yaml/buildkite/testdata/step_malformed.yml similarity index 100% rename from compiler/types/yaml/testdata/step_malformed.yml rename to compiler/types/yaml/buildkite/testdata/step_malformed.yml diff --git a/compiler/types/yaml/testdata/step_nil.yml b/compiler/types/yaml/buildkite/testdata/step_nil.yml similarity index 100% rename from compiler/types/yaml/testdata/step_nil.yml rename to compiler/types/yaml/buildkite/testdata/step_nil.yml diff --git a/compiler/types/yaml/testdata/step_secret_slice.yml b/compiler/types/yaml/buildkite/testdata/step_secret_slice.yml similarity index 100% rename from compiler/types/yaml/testdata/step_secret_slice.yml rename to compiler/types/yaml/buildkite/testdata/step_secret_slice.yml diff --git a/compiler/types/yaml/testdata/step_secret_slice_invalid_no_source.yml b/compiler/types/yaml/buildkite/testdata/step_secret_slice_invalid_no_source.yml similarity index 100% rename from compiler/types/yaml/testdata/step_secret_slice_invalid_no_source.yml rename to compiler/types/yaml/buildkite/testdata/step_secret_slice_invalid_no_source.yml diff --git a/compiler/types/yaml/testdata/step_secret_slice_invalid_no_target.yml b/compiler/types/yaml/buildkite/testdata/step_secret_slice_invalid_no_target.yml similarity index 100% rename from compiler/types/yaml/testdata/step_secret_slice_invalid_no_target.yml rename to compiler/types/yaml/buildkite/testdata/step_secret_slice_invalid_no_target.yml diff --git a/compiler/types/yaml/testdata/step_secret_string.yml b/compiler/types/yaml/buildkite/testdata/step_secret_string.yml similarity index 100% rename from compiler/types/yaml/testdata/step_secret_string.yml rename to compiler/types/yaml/buildkite/testdata/step_secret_string.yml diff --git a/compiler/types/yaml/testdata/template.yml b/compiler/types/yaml/buildkite/testdata/template.yml similarity index 100% rename from compiler/types/yaml/testdata/template.yml rename to compiler/types/yaml/buildkite/testdata/template.yml diff --git a/compiler/types/yaml/testdata/ulimit_colon_error.yml b/compiler/types/yaml/buildkite/testdata/ulimit_colon_error.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_colon_error.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_colon_error.yml diff --git a/compiler/types/yaml/testdata/ulimit_equal_error.yml b/compiler/types/yaml/buildkite/testdata/ulimit_equal_error.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_equal_error.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_equal_error.yml diff --git a/compiler/types/yaml/testdata/ulimit_hardlimit1_error.yml b/compiler/types/yaml/buildkite/testdata/ulimit_hardlimit1_error.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_hardlimit1_error.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_hardlimit1_error.yml diff --git a/compiler/types/yaml/testdata/ulimit_hardlimit2_error.yml b/compiler/types/yaml/buildkite/testdata/ulimit_hardlimit2_error.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_hardlimit2_error.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_hardlimit2_error.yml diff --git a/compiler/types/yaml/testdata/ulimit_slice.yml b/compiler/types/yaml/buildkite/testdata/ulimit_slice.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_slice.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_slice.yml diff --git a/compiler/types/yaml/testdata/ulimit_softlimit_error.yml b/compiler/types/yaml/buildkite/testdata/ulimit_softlimit_error.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_softlimit_error.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_softlimit_error.yml diff --git a/compiler/types/yaml/testdata/ulimit_string.yml b/compiler/types/yaml/buildkite/testdata/ulimit_string.yml similarity index 100% rename from compiler/types/yaml/testdata/ulimit_string.yml rename to compiler/types/yaml/buildkite/testdata/ulimit_string.yml diff --git a/compiler/types/yaml/testdata/volume_error.yml b/compiler/types/yaml/buildkite/testdata/volume_error.yml similarity index 100% rename from compiler/types/yaml/testdata/volume_error.yml rename to compiler/types/yaml/buildkite/testdata/volume_error.yml diff --git a/compiler/types/yaml/testdata/volume_slice.yml b/compiler/types/yaml/buildkite/testdata/volume_slice.yml similarity index 100% rename from compiler/types/yaml/testdata/volume_slice.yml rename to compiler/types/yaml/buildkite/testdata/volume_slice.yml diff --git a/compiler/types/yaml/testdata/volume_string.yml b/compiler/types/yaml/buildkite/testdata/volume_string.yml similarity index 100% rename from compiler/types/yaml/testdata/volume_string.yml rename to compiler/types/yaml/buildkite/testdata/volume_string.yml diff --git a/compiler/types/yaml/buildkite/ulimit.go b/compiler/types/yaml/buildkite/ulimit.go new file mode 100644 index 000000000..75271f293 --- /dev/null +++ b/compiler/types/yaml/buildkite/ulimit.go @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "fmt" + "strconv" + "strings" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +type ( + // UlimitSlice is the yaml representation of + // the ulimits block for a step in a pipeline. + UlimitSlice []*Ulimit + + // Ulimit is the yaml representation of a ulimit + // from the ulimits block for a step in a pipeline. + Ulimit struct { + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"required,minLength=1,description=Unique name of the user limit.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ulimits-key"` + Soft int64 `yaml:"soft,omitempty" json:"soft,omitempty" jsonschema:"description=Set the soft limit.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ulimits-key"` + Hard int64 `yaml:"hard,omitempty" json:"hard,omitempty" jsonschema:"description=Set the hard limit.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-ulimits-key"` + } +) + +// ToPipeline converts the UlimitSlice type +// to a pipeline UlimitSlice type. +func (u *UlimitSlice) ToPipeline() *pipeline.UlimitSlice { + // ulimit slice we want to return + ulimitSlice := new(pipeline.UlimitSlice) + + // iterate through each element in the ulimit slice + for _, ulimit := range *u { + // append the element to the pipeline ulimit slice + *ulimitSlice = append(*ulimitSlice, &pipeline.Ulimit{ + Name: ulimit.Name, + Soft: ulimit.Soft, + Hard: ulimit.Hard, + }) + } + + return ulimitSlice +} + +// UnmarshalYAML implements the Unmarshaler interface for the UlimitSlice type. +func (u *UlimitSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // string slice we try unmarshalling to + stringSlice := new(raw.StringSlice) + + // attempt to unmarshal as a string slice type + err := unmarshal(stringSlice) + if err == nil { + // iterate through each element in the string slice + for _, ulimit := range *stringSlice { + // split each slice element into key/value pairs + parts := strings.Split(ulimit, "=") + if len(parts) != 2 { + return fmt.Errorf("ulimit %s must contain 1 `=` (equal)", ulimit) + } + + // split each value into soft and hard limits + limitParts := strings.Split(parts[1], ":") + + switch { + case len(limitParts) == 1: + // capture value for soft and hard limit + value, err := strconv.ParseInt(limitParts[0], 10, 64) + if err != nil { + return err + } + + // append the element to the ulimit slice + *u = append(*u, &Ulimit{ + Name: parts[0], + Soft: value, + Hard: value, + }) + + continue + case len(limitParts) == 2: + // capture value for soft limit + firstValue, err := strconv.ParseInt(limitParts[0], 10, 64) + if err != nil { + return err + } + + // capture value for hard limit + secondValue, err := strconv.ParseInt(limitParts[1], 10, 64) + if err != nil { + return err + } + + // append the element to the ulimit slice + *u = append(*u, &Ulimit{ + Name: parts[0], + Soft: firstValue, + Hard: secondValue, + }) + + continue + default: + return fmt.Errorf("ulimit %s can only contain 1 `:` (colon)", ulimit) + } + } + + return nil + } + + // ulimit slice we try unmarshalling to + ulimits := new([]*Ulimit) + + // attempt to unmarshal as a ulimit slice type + err = unmarshal(ulimits) + if err != nil { + return err + } + + // iterate through each element in the volume slice + for _, ulimit := range *ulimits { + // implicitly set `hard` field if empty + if ulimit.Hard == 0 { + ulimit.Hard = ulimit.Soft + } + } + + // overwrite existing UlimitSlice + *u = UlimitSlice(*ulimits) + + return nil +} + +func (u *Ulimit) ToYAML() *yaml.Ulimit { + if u == nil { + return nil + } + + return &yaml.Ulimit{ + Name: u.Name, + Soft: u.Soft, + Hard: u.Hard, + } +} + +func (u *UlimitSlice) ToYAML() *yaml.UlimitSlice { + // ulimit slice we want to return + ulimitSlice := new(yaml.UlimitSlice) + + // iterate through each element in the ulimit slice + for _, ulimit := range *u { + // append the element to the yaml ulimit slice + *ulimitSlice = append(*ulimitSlice, ulimit.ToYAML()) + } + + return ulimitSlice +} diff --git a/compiler/types/yaml/ulimit_test.go b/compiler/types/yaml/buildkite/ulimit_test.go similarity index 99% rename from compiler/types/yaml/ulimit_test.go rename to compiler/types/yaml/buildkite/ulimit_test.go index 3a2d9fcfb..5337007fa 100644 --- a/compiler/types/yaml/ulimit_test.go +++ b/compiler/types/yaml/buildkite/ulimit_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -package yaml +package buildkite import ( "os" diff --git a/compiler/types/yaml/buildkite/volume.go b/compiler/types/yaml/buildkite/volume.go new file mode 100644 index 000000000..ee7328075 --- /dev/null +++ b/compiler/types/yaml/buildkite/volume.go @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "fmt" + "strings" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +type ( + // VolumeSlice is the yaml representation of + // the volumes block for a step in a pipeline. + VolumeSlice []*Volume + + // Volume is the yaml representation of a volume + // from a volumes block for a step in a pipeline. + Volume struct { + Source string `yaml:"source,omitempty" json:"source,omitempty" jsonschema:"required,minLength=1,description=Set the source directory to be mounted.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-volume-key"` + Destination string `yaml:"destination,omitempty" json:"destination,omitempty" jsonschema:"required,minLength=1,description=Set the destination directory for the mount in the container.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-volume-key"` + AccessMode string `yaml:"access_mode,omitempty" json:"access_mode,omitempty" jsonschema:"default=ro,description=Set the access mode for the mounted volume.\nReference: https://go-vela.github.io/docs/reference/yaml/steps/#the-volume-key"` + } +) + +// ToPipeline converts the VolumeSlice type +// to a pipeline VolumeSlice type. +func (v *VolumeSlice) ToPipeline() *pipeline.VolumeSlice { + // volume slice we want to return + volumes := new(pipeline.VolumeSlice) + + // iterate through each element in the volume slice + for _, volume := range *v { + // append the element to the pipeline volume slice + *volumes = append(*volumes, &pipeline.Volume{ + Source: volume.Source, + Destination: volume.Destination, + AccessMode: volume.AccessMode, + }) + } + + return volumes +} + +// UnmarshalYAML implements the Unmarshaler interface for the VolumeSlice type. +func (v *VolumeSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + // string slice we try unmarshalling to + stringSlice := new(raw.StringSlice) + + // attempt to unmarshal as a string slice type + err := unmarshal(stringSlice) + if err == nil { + // iterate through each element in the string slice + for _, volume := range *stringSlice { + // split each slice element into source, destination and access mode + parts := strings.Split(volume, ":") + + switch { + case len(parts) == 1: + // append the element to the volume slice + *v = append(*v, &Volume{ + Source: parts[0], + Destination: parts[0], + AccessMode: "ro", + }) + + continue + case len(parts) == 2: + // append the element to the volume slice + *v = append(*v, &Volume{ + Source: parts[0], + Destination: parts[1], + AccessMode: "ro", + }) + + continue + case len(parts) == 3: + // append the element to the volume slice + *v = append(*v, &Volume{ + Source: parts[0], + Destination: parts[1], + AccessMode: parts[2], + }) + + continue + default: + return fmt.Errorf("volume %s must contain at least 1 but no more than 2 `:`(colons)", volume) + } + } + + return nil + } + + // volume slice we try unmarshalling to + volumes := new([]*Volume) + + // attempt to unmarshal as a volume slice type + err = unmarshal(volumes) + if err != nil { + return err + } + + // iterate through each element in the volume slice + for _, volume := range *volumes { + // implicitly set `destination` field if empty + if len(volume.Destination) == 0 { + volume.Destination = volume.Source + } + + // implicitly set `access_mode` field if empty + if len(volume.AccessMode) == 0 { + volume.AccessMode = "ro" + } + } + + // overwrite existing VolumeSlice + *v = VolumeSlice(*volumes) + + return nil +} + +func (v *Volume) ToYAML() *yaml.Volume { + if v == nil { + return nil + } + + return &yaml.Volume{ + Source: v.Source, + Destination: v.Destination, + AccessMode: v.AccessMode, + } +} + +func (v *VolumeSlice) ToYAML() *yaml.VolumeSlice { + // volume slice we want to return + volumeSlice := new(yaml.VolumeSlice) + + // iterate through each element in the volume slice + for _, volume := range *v { + // append the element to the yaml volume slice + *volumeSlice = append(*volumeSlice, volume.ToYAML()) + } + + return volumeSlice +} diff --git a/compiler/types/yaml/buildkite/volume_test.go b/compiler/types/yaml/buildkite/volume_test.go new file mode 100644 index 000000000..c2192f944 --- /dev/null +++ b/compiler/types/yaml/buildkite/volume_test.go @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "os" + "reflect" + "testing" + + "github.com/buildkite/yaml" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_VolumeSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + volumes *VolumeSlice + want *pipeline.VolumeSlice + }{ + { + volumes: &VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + want: &pipeline.VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.volumes.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_VolumeSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *VolumeSlice + }{ + { + failure: false, + file: "testdata/volume_slice.yml", + want: &VolumeSlice{ + { + Source: "/foo", + Destination: "/foo", + AccessMode: "ro", + }, + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + { + Source: "/foo", + Destination: "/foobar", + AccessMode: "ro", + }, + }, + }, + { + failure: false, + file: "testdata/volume_string.yml", + want: &VolumeSlice{ + { + Source: "/foo", + Destination: "/foo", + AccessMode: "ro", + }, + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + { + Source: "/foo", + Destination: "/foobar", + AccessMode: "ro", + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + { + failure: true, + file: "testdata/volume_error.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(VolumeSlice) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/buildkite/worker.go b/compiler/types/yaml/buildkite/worker.go new file mode 100644 index 000000000..d7fa9e891 --- /dev/null +++ b/compiler/types/yaml/buildkite/worker.go @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +// Worker is the yaml representation of a worker +// from a worker block in a pipeline. +type Worker struct { + Flavor string `yaml:"flavor,omitempty" json:"flavor,omitempty" jsonschema:"minLength=1,description=Flavor identifier for worker.\nReference: https://go-vela.github.io/docs/reference/yaml/worker/#the-flavor-key,example=large"` + Platform string `yaml:"platform,omitempty" json:"platform,omitempty" jsonschema:"minLength=1,description=Platform identifier for the worker.\nReference: https://go-vela.github.io/docs/reference/yaml/worker/#the-platform-key,example=kubernetes"` +} + +// ToPipeline converts the Worker type +// to a pipeline Worker type. +func (w *Worker) ToPipeline() *pipeline.Worker { + return &pipeline.Worker{ + Flavor: w.Flavor, + Platform: w.Platform, + } +} + +func (w *Worker) ToYAML() *yaml.Worker { + return &yaml.Worker{ + Flavor: w.Flavor, + Platform: w.Platform, + } +} diff --git a/compiler/types/yaml/buildkite/worker_test.go b/compiler/types/yaml/buildkite/worker_test.go new file mode 100644 index 000000000..049fd090c --- /dev/null +++ b/compiler/types/yaml/buildkite/worker_test.go @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: Apache-2.0 + +package buildkite + +import ( + "reflect" + "testing" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_Worker_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + worker *Worker + want *pipeline.Worker + }{ + { + worker: &Worker{ + Flavor: "8cpu16gb", + Platform: "gcp", + }, + want: &pipeline.Worker{ + Flavor: "8cpu16gb", + Platform: "gcp", + }, + }, + } + + // run tests + for _, test := range tests { + got := test.worker.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/doc.go b/compiler/types/yaml/doc.go deleted file mode 100644 index 2a2e39cd3..000000000 --- a/compiler/types/yaml/doc.go +++ /dev/null @@ -1,8 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -// Package yaml provides the defined yaml types for Vela. -// -// Usage: -// -// import "github.com/go-vela/server/compiler/types/yaml" -package yaml diff --git a/compiler/types/yaml/build.go b/compiler/types/yaml/yaml/build.go similarity index 100% rename from compiler/types/yaml/build.go rename to compiler/types/yaml/yaml/build.go diff --git a/compiler/types/yaml/yaml/build_test.go b/compiler/types/yaml/yaml/build_test.go new file mode 100644 index 000000000..f0c284f75 --- /dev/null +++ b/compiler/types/yaml/yaml/build_test.go @@ -0,0 +1,686 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + api "github.com/go-vela/server/api/types" + "github.com/go-vela/server/compiler/types/raw" +) + +func TestYaml_Build_ToAPI(t *testing.T) { + build := new(api.Pipeline) + build.SetFlavor("16cpu8gb") + build.SetPlatform("gcp") + build.SetVersion("1") + build.SetExternalSecrets(true) + build.SetInternalSecrets(true) + build.SetServices(true) + build.SetStages(false) + build.SetSteps(true) + build.SetTemplates(true) + + stages := new(api.Pipeline) + stages.SetFlavor("") + stages.SetPlatform("") + stages.SetVersion("1") + stages.SetExternalSecrets(false) + stages.SetInternalSecrets(false) + stages.SetServices(false) + stages.SetStages(true) + stages.SetSteps(false) + stages.SetTemplates(false) + + steps := new(api.Pipeline) + steps.SetFlavor("") + steps.SetPlatform("") + steps.SetVersion("1") + steps.SetExternalSecrets(false) + steps.SetInternalSecrets(false) + steps.SetServices(false) + steps.SetStages(false) + steps.SetSteps(true) + steps.SetTemplates(false) + + // setup tests + tests := []struct { + name string + file string + want *api.Pipeline + }{ + { + name: "build", + file: "testdata/build.yml", + want: build, + }, + { + name: "stages", + file: "testdata/build_anchor_stage.yml", + want: stages, + }, + { + name: "steps", + file: "testdata/build_anchor_step.yml", + want: steps, + }, + } + + // run tests + for _, test := range tests { + b := new(Build) + + data, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file %s for %s: %v", test.file, test.name, err) + } + + err = yaml.Unmarshal(data, b) + if err != nil { + t.Errorf("unable to unmarshal YAML for %s: %v", test.name, err) + } + + got := b.ToPipelineAPI() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipelineAPI for %s is %v, want %v", test.name, got, test.want) + } + } +} + +func TestYaml_Build_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + file string + want *Build + }{ + { + file: "testdata/build.yml", + want: &Build{ + Version: "1", + Metadata: Metadata{ + Template: false, + Clone: nil, + Environment: []string{"steps", "services", "secrets"}, + }, + Environment: raw.StringSliceMap{ + "HELLO": "Hello, Global Message", + }, + Worker: Worker{ + Flavor: "16cpu8gb", + Platform: "gcp", + }, + Services: ServiceSlice{ + { + Ports: []string{"5432:5432"}, + Environment: raw.StringSliceMap{ + "POSTGRES_DB": "foo", + }, + Name: "postgres", + Image: "postgres:latest", + Pull: "not_present", + }, + }, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"./gradlew downloadDependencies"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:edited"}}, + Matcher: "filepath", + Operator: "and", + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + { + Commands: raw.StringSlice{"./gradlew check"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "test", + Image: "openjdk:latest", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + { + Commands: raw.StringSlice{"./gradlew build"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "build", + Image: "openjdk:latest", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + { + Name: "docker_build", + Parameters: map[string]interface{}{ + "dry_run": true, + "registry": "index.docker.io", + "repo": "github/octocat", + "tags": []interface{}{"latest", "dev"}, + }, + Image: "plugins/docker:18.09", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + }, + { + Name: "docker_publish", + Parameters: map[string]interface{}{ + "registry": "index.docker.io", + "repo": "github/octocat", + "tags": []interface{}{"latest", "dev"}, + }, + Image: "plugins/docker:18.09", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Branch: []string{"main"}, Event: []string{"push"}}, + Matcher: "filepath", + Operator: "and", + }, + Secrets: StepSecretSlice{ + { + Source: "docker_username", + Target: "PLUGIN_USERNAME", + }, + { + Source: "docker_password", + Target: "PLUGIN_PASSWORD", + }, + }, + }, + }, + Secrets: SecretSlice{ + { + Name: "docker_username", + Key: "org/repo/docker/username", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "docker_password", + Key: "org/repo/docker/password", + Engine: "vault", + Type: "repo", + Pull: "build_start", + }, + { + Name: "docker_username", + Key: "org/docker/username", + Engine: "native", + Type: "org", + Pull: "build_start", + }, + { + Name: "docker_password", + Key: "org/docker/password", + Engine: "vault", + Type: "org", + Pull: "build_start", + }, + { + Name: "docker_username", + Key: "org/team/docker/username", + Engine: "native", + Type: "shared", + Pull: "build_start", + }, + { + Name: "docker_password", + Key: "org/team/docker/password", + Engine: "vault", + Type: "shared", + Pull: "build_start", + }, + { + Origin: Origin{ + Image: "target/vela-vault:latest", + Parameters: map[string]interface{}{ + "addr": "vault.example.com", + }, + Pull: "always", + Secrets: StepSecretSlice{ + { + Source: "docker_username", + Target: "DOCKER_USERNAME", + }, + { + Source: "docker_password", + Target: "DOCKER_PASSWORD", + }, + }, + }, + }, + }, + Templates: TemplateSlice{ + { + Name: "docker_publish", + Source: "github.com/go-vela/atlas/stable/docker_publish", + Type: "github", + }, + }, + }, + }, + { + file: "testdata/build_anchor_stage.yml", + want: &Build{ + Version: "1", + Metadata: Metadata{ + Template: false, + Clone: nil, + Environment: []string{"steps", "services", "secrets"}, + }, + Stages: StageSlice{ + { + Name: "dependencies", + Needs: []string{"clone"}, + Independent: false, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"./gradlew downloadDependencies"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + }, + }, + { + Name: "test", + Needs: []string{"dependencies", "clone"}, + Independent: false, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"./gradlew check"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "test", + Image: "openjdk:latest", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + }, + }, + { + Name: "build", + Needs: []string{"dependencies", "clone"}, + Independent: true, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"./gradlew build"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "build", + Image: "openjdk:latest", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + }, + }, + }, + }, + }, + { + file: "testdata/build_anchor_step.yml", + want: &Build{ + Version: "1", + Metadata: Metadata{ + Template: false, + Clone: nil, + Environment: []string{"steps", "services", "secrets"}, + }, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"./gradlew downloadDependencies"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + { + Commands: raw.StringSlice{"./gradlew check"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "test", + Image: "openjdk:latest", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + { + Commands: raw.StringSlice{"./gradlew build"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "build", + Image: "openjdk:latest", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + }, + }, + }, + { + file: "testdata/build_empty_env.yml", + want: &Build{ + Version: "1", + Metadata: Metadata{ + Template: false, + Clone: nil, + Environment: []string{}, + }, + Environment: raw.StringSliceMap{ + "HELLO": "Hello, Global Message", + }, + Worker: Worker{ + Flavor: "16cpu8gb", + Platform: "gcp"}, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"./gradlew downloadDependencies"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + Ruleset: Ruleset{ + If: Rules{Event: []string{"push", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened"}}, + Matcher: "filepath", + Operator: "and", + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + }, + }, + }, + { + file: "testdata/merge_anchor.yml", + want: &Build{ + Version: "1", + Metadata: Metadata{ + Template: false, + Clone: nil, + Environment: []string{"steps", "services", "secrets"}, + }, + Services: ServiceSlice{ + { + Name: "service-a", + Ports: []string{"5432:5432"}, + Environment: raw.StringSliceMap{ + "REGION": "dev", + }, + Image: "postgres", + Pull: "not_present", + }, + }, + Steps: StepSlice{ + { + Commands: raw.StringSlice{"echo alpha"}, + Name: "alpha", + Image: "alpine:latest", + Pull: "not_present", + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + { + Commands: raw.StringSlice{"echo beta"}, + Name: "beta", + Image: "alpine:latest", + Pull: "not_present", + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + { + Commands: raw.StringSlice{"echo gamma"}, + Name: "gamma", + Image: "alpine:latest", + Pull: "not_present", + Environment: raw.StringSliceMap{ + "REGION": "dev", + }, + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := new(Build) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("Reading file for UnmarshalYAML returned err: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/deployment.go b/compiler/types/yaml/yaml/deployment.go similarity index 100% rename from compiler/types/yaml/deployment.go rename to compiler/types/yaml/yaml/deployment.go diff --git a/compiler/types/yaml/deployment_test.go b/compiler/types/yaml/yaml/deployment_test.go similarity index 100% rename from compiler/types/yaml/deployment_test.go rename to compiler/types/yaml/yaml/deployment_test.go diff --git a/compiler/types/yaml/yaml/doc.go b/compiler/types/yaml/yaml/doc.go new file mode 100644 index 000000000..30870a01c --- /dev/null +++ b/compiler/types/yaml/yaml/doc.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 + +// package yaml provides the defined yaml types for Vela. +// +// Usage: +// +// import "github.com/go-vela/server/compiler/types/yaml/yaml" +package yaml diff --git a/compiler/types/yaml/git.go b/compiler/types/yaml/yaml/git.go similarity index 100% rename from compiler/types/yaml/git.go rename to compiler/types/yaml/yaml/git.go diff --git a/compiler/types/yaml/git_test.go b/compiler/types/yaml/yaml/git_test.go similarity index 100% rename from compiler/types/yaml/git_test.go rename to compiler/types/yaml/yaml/git_test.go diff --git a/compiler/types/yaml/metadata.go b/compiler/types/yaml/yaml/metadata.go similarity index 100% rename from compiler/types/yaml/metadata.go rename to compiler/types/yaml/yaml/metadata.go diff --git a/compiler/types/yaml/metadata_test.go b/compiler/types/yaml/yaml/metadata_test.go similarity index 100% rename from compiler/types/yaml/metadata_test.go rename to compiler/types/yaml/yaml/metadata_test.go diff --git a/compiler/types/yaml/ruleset.go b/compiler/types/yaml/yaml/ruleset.go similarity index 100% rename from compiler/types/yaml/ruleset.go rename to compiler/types/yaml/yaml/ruleset.go diff --git a/compiler/types/yaml/yaml/ruleset_test.go b/compiler/types/yaml/yaml/ruleset_test.go new file mode 100644 index 000000000..cb4e0466f --- /dev/null +++ b/compiler/types/yaml/yaml/ruleset_test.go @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_Ruleset_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + ruleset *Ruleset + want *pipeline.Ruleset + }{ + { + ruleset: &Ruleset{ + If: Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push", "pull_request:labeled"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + Label: []string{"enhancement"}, + Instance: []string{"http://localhost:8080"}, + }, + Unless: Rules{ + Branch: []string{"main"}, + Comment: []string{"real comment"}, + Event: []string{"pull_request"}, + Path: []string{"bar.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octokitty"}, + Status: []string{"failure"}, + Tag: []string{"v0.2.0"}, + Target: []string{"production"}, + Instance: []string{"http://localhost:8080"}, + }, + Matcher: "filepath", + Operator: "and", + Continue: false, + }, + want: &pipeline.Ruleset{ + If: pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push", "pull_request:labeled"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + Label: []string{"enhancement"}, + Instance: []string{"http://localhost:8080"}, + }, + Unless: pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"real comment"}, + Event: []string{"pull_request"}, + Path: []string{"bar.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octokitty"}, + Status: []string{"failure"}, + Tag: []string{"v0.2.0"}, + Target: []string{"production"}, + Instance: []string{"http://localhost:8080"}, + }, + Matcher: "filepath", + Operator: "and", + Continue: false, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.ruleset.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_Ruleset_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + file string + want *Ruleset + }{ + { + file: "testdata/ruleset_simple.yml", + want: &Ruleset{ + If: Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push"}, + Instance: []string{"vela-server"}, + Label: []string{"bug"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + }, + Matcher: "filepath", + Operator: "and", + Continue: true, + }, + }, + { + file: "testdata/ruleset_advanced.yml", + want: &Ruleset{ + If: Rules{ + Branch: []string{"main"}, + Event: []string{"push"}, + Tag: []string{"^refs/tags/(\\d+\\.)+\\d+$"}, + }, + Unless: Rules{ + Event: []string{"deployment:created", "pull_request:opened", "pull_request:synchronize", "pull_request:reopened", "comment:created", "comment:edited", "schedule"}, + Path: []string{"foo.txt", "/foo/bar.txt"}, + }, + Matcher: "regexp", + Operator: "or", + Continue: true, + }, + }, + { + file: "testdata/ruleset_regex.yml", + want: &Ruleset{ + If: Rules{ + Branch: []string{"main"}, + Event: []string{"tag"}, + Tag: []string{"^refs/tags/(\\d+\\.)+\\d+$"}, + }, + Operator: "and", + Matcher: "regex", + }, + }, + } + + // run tests + for _, test := range tests { + got := new(Ruleset) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} + +func TestYaml_Rules_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + rules *Rules + want *pipeline.Rules + }{ + { + rules: &Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push", "pull_request:labeled"}, + Instance: []string{"vela-server"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + Label: []string{"enhancement"}, + }, + want: &pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push", "pull_request:labeled"}, + Instance: []string{"vela-server"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + Label: []string{"enhancement"}, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.rules.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_Rules_UnmarshalYAML(t *testing.T) { + // setup types + var ( + b []byte + err error + ) + + // setup tests + tests := []struct { + failure bool + file string + want *Rules + }{ + { + failure: false, + file: "testdata/ruleset_simple.yml", + want: &Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push"}, + Instance: []string{"vela-server"}, + Label: []string{"bug"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Sender: []string{"octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + }, + }, + { + failure: true, + file: "", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(Rules) + + if len(test.file) > 0 { + b, err = os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + } else { + b = []byte("``") + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/secret.go b/compiler/types/yaml/yaml/secret.go similarity index 100% rename from compiler/types/yaml/secret.go rename to compiler/types/yaml/yaml/secret.go diff --git a/compiler/types/yaml/yaml/secret_test.go b/compiler/types/yaml/yaml/secret_test.go new file mode 100644 index 000000000..d35b91171 --- /dev/null +++ b/compiler/types/yaml/yaml/secret_test.go @@ -0,0 +1,460 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_Origin_MergeEnv(t *testing.T) { + // setup tests + tests := []struct { + origin *Origin + environment map[string]string + failure bool + }{ + { + origin: &Origin{ + Name: "vault", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-vault:latest", + Parameters: map[string]interface{}{ + "addr": "vault.example.com", + "auth_method": "token", + "items": []interface{}{ + map[string]string{"source": "secret/docker", "path": "docker"}, + }, + }, + Pull: "always", + Secrets: StepSecretSlice{ + { + Source: "vault_token", + Target: "vault_token", + }, + }, + }, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + origin: &Origin{}, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + origin: nil, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + origin: &Origin{ + Name: "vault", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-vault:latest", + Parameters: map[string]interface{}{ + "addr": "vault.example.com", + "auth_method": "token", + "items": []interface{}{ + map[string]string{"source": "secret/docker", "path": "docker"}, + }, + }, + Pull: "always", + Secrets: StepSecretSlice{ + { + Source: "vault_token", + Target: "vault_token", + }, + }, + }, + environment: nil, + failure: true, + }, + } + + // run tests + for _, test := range tests { + err := test.origin.MergeEnv(test.environment) + + if test.failure { + if err == nil { + t.Errorf("MergeEnv should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("MergeEnv returned err: %v", err) + } + } +} + +func TestYaml_SecretSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + secrets *SecretSlice + want *pipeline.SecretSlice + }{ + { + secrets: &SecretSlice{ + { + Name: "docker_username", + Key: "github/octocat/docker/username", + Engine: "native", + Type: "repo", + Origin: Origin{}, + Pull: "build_start", + }, + { + Name: "docker_username", + Key: "", + Engine: "", + Type: "", + Origin: Origin{ + Name: "vault", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-vault:latest", + Parameters: map[string]interface{}{ + "addr": "vault.company.com", + }, + Pull: "always", + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Operator: "and", + }, + Secrets: StepSecretSlice{ + { + Source: "foo", + Target: "foo", + }, + { + Source: "foobar", + Target: "foobar", + }, + }, + }, + Pull: "build_start", + }, + }, + want: &pipeline.SecretSlice{ + { + Name: "docker_username", + Key: "github/octocat/docker/username", + Engine: "native", + Type: "repo", + Origin: &pipeline.Container{}, + Pull: "build_start", + }, + { + Name: "docker_username", + Key: "", + Engine: "", + Type: "", + Origin: &pipeline.Container{ + Name: "vault", + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-vault:latest", + Pull: "always", + Ruleset: pipeline.Ruleset{ + If: pipeline.Rules{ + Event: []string{"push"}, + }, + Operator: "and", + }, + Secrets: pipeline.StepSecretSlice{ + { + Source: "foo", + Target: "foo", + }, + { + Source: "foobar", + Target: "foobar", + }, + }, + }, + Pull: "build_start", + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.secrets.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_SecretSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *SecretSlice + }{ + { + failure: false, + file: "testdata/secret.yml", + want: &SecretSlice{ + { + Name: "foo", + Key: "bar", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "noKey", + Key: "noKey", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "noType", + Key: "bar", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "noEngine", + Key: "bar", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "noKeyEngineAndType", + Key: "noKeyEngineAndType", + Engine: "native", + Type: "repo", + Pull: "build_start", + }, + { + Name: "externalSecret", + Key: "", + Engine: "", + Type: "", + Origin: Origin{ + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-vault:latest", + Parameters: map[string]interface{}{ + "addr": "vault.company.com", + }, + Pull: "always", + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Operator: "and", + Matcher: "filepath", + }, + Secrets: StepSecretSlice{ + { + Source: "foo", + Target: "FOO", + }, + { + Source: "foobar", + Target: "FOOBAR", + }, + }, + }, + Pull: "", + }, + { + Name: "", + Key: "", + Engine: "", + Type: "", + Origin: Origin{ + Environment: map[string]string{"FOO": "bar"}, + Image: "target/vela-vault:latest", + Parameters: map[string]interface{}{ + "addr": "vault.company.com", + }, + Pull: "always", + Ruleset: Ruleset{ + If: Rules{ + Event: []string{"push"}, + }, + Operator: "and", + Matcher: "filepath", + }, + Secrets: StepSecretSlice{ + { + Source: "foo", + Target: "FOO", + }, + { + Source: "foobar", + Target: "FOOBAR", + }, + }, + }, + Pull: "", + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(SecretSlice) + + // run test + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} + +func TestYaml_StepSecretSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + secrets *StepSecretSlice + want *pipeline.StepSecretSlice + }{ + { + secrets: &StepSecretSlice{ + { + Source: "docker_username", + Target: "plugin_username", + }, + }, + want: &pipeline.StepSecretSlice{ + { + Source: "docker_username", + Target: "plugin_username", + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.secrets.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_StepSecretSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *StepSecretSlice + }{ + { + failure: false, + file: "testdata/step_secret_slice.yml", + want: &StepSecretSlice{ + { + Source: "foo", + Target: "BAR", + }, + { + Source: "hello", + Target: "WORLD", + }, + }, + }, + { + failure: false, + file: "testdata/step_secret_string.yml", + want: &StepSecretSlice{ + { + Source: "foo", + Target: "FOO", + }, + { + Source: "hello", + Target: "HELLO", + }, + }, + }, + { + failure: true, + file: "testdata/step_secret_slice_invalid_no_source.yml", + want: nil, + }, + { + failure: true, + file: "testdata/step_secret_slice_invalid_no_target.yml", + want: nil, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(StepSecretSlice) + + // run test + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/service.go b/compiler/types/yaml/yaml/service.go similarity index 100% rename from compiler/types/yaml/service.go rename to compiler/types/yaml/yaml/service.go diff --git a/compiler/types/yaml/yaml/service_test.go b/compiler/types/yaml/yaml/service_test.go new file mode 100644 index 000000000..85c432d8b --- /dev/null +++ b/compiler/types/yaml/yaml/service_test.go @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" +) + +func TestYaml_ServiceSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + services *ServiceSlice + want *pipeline.ContainerSlice + }{ + { + services: &ServiceSlice{ + { + Entrypoint: []string{"/usr/local/bin/docker-entrypoint.sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + want: &pipeline.ContainerSlice{ + { + Detach: true, + Entrypoint: []string{"/usr/local/bin/docker-entrypoint.sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:12-alpine", + Name: "postgres", + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.services.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_ServiceSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *ServiceSlice + }{ + { + failure: false, + file: "testdata/service.yml", + want: &ServiceSlice{ + { + Environment: raw.StringSliceMap{ + "POSTGRES_DB": "foo", + }, + Image: "postgres:latest", + Name: "postgres", + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + { + Environment: raw.StringSliceMap{ + "MYSQL_DATABASE": "foo", + }, + Image: "mysql:latest", + Name: "mysql", + Ports: []string{"3061:3061"}, + Pull: "not_present", + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + { + failure: true, + file: "testdata/service_nil.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(ServiceSlice) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} + +func TestYaml_Service_MergeEnv(t *testing.T) { + // setup tests + tests := []struct { + service *Service + environment map[string]string + failure bool + }{ + { + service: &Service{ + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:latest", + Name: "postgres", + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + service: &Service{}, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + service: nil, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + service: &Service{ + Environment: map[string]string{"FOO": "bar"}, + Image: "postgres:latest", + Name: "postgres", + Ports: []string{"5432:5432"}, + Pull: "not_present", + }, + environment: nil, + failure: true, + }, + } + + // run tests + for _, test := range tests { + err := test.service.MergeEnv(test.environment) + + if test.failure { + if err == nil { + t.Errorf("MergeEnv should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("MergeEnv returned err: %v", err) + } + } +} diff --git a/compiler/types/yaml/yaml/stage.go b/compiler/types/yaml/yaml/stage.go new file mode 100644 index 000000000..0d7a5deef --- /dev/null +++ b/compiler/types/yaml/yaml/stage.go @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "fmt" + + "github.com/invopop/jsonschema" + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" +) + +type ( + // StageSlice is the yaml representation + // of the stages block for a pipeline. + StageSlice []*Stage + + // Stage is the yaml representation + // of a stage in a pipeline. + Stage struct { + Environment raw.StringSliceMap `yaml:"environment,omitempty" json:"environment,omitempty" jsonschema:"description=Provide environment variables injected into the container environment.\nReference: https://go-vela.github.io/docs/reference/yaml/stages/#the-environment-key"` + Name string `yaml:"name,omitempty" json:"name,omitempty" jsonschema:"minLength=1,description=Unique identifier for the stage in the pipeline.\nReference: https://go-vela.github.io/docs/reference/yaml/stages/#the-name-key"` + Needs raw.StringSlice `yaml:"needs,omitempty,flow" json:"needs,omitempty" jsonschema:"description=Stages that must complete before starting the current one.\nReference: https://go-vela.github.io/docs/reference/yaml/stages/#the-needs-key"` + Independent bool `yaml:"independent,omitempty" json:"independent,omitempty" jsonschema:"description=Stage will continue executing if other stage fails"` + Steps StepSlice `yaml:"steps,omitempty" json:"steps,omitempty" jsonschema:"required,description=Sequential execution instructions for the stage.\nReference: https://go-vela.github.io/docs/reference/yaml/stages/#the-steps-key"` + } +) + +// ToPipeline converts the StageSlice type +// to a pipeline StageSlice type. +func (s *StageSlice) ToPipeline() *pipeline.StageSlice { + // stage slice we want to return + stageSlice := new(pipeline.StageSlice) + + // iterate through each element in the stage slice + for _, stage := range *s { + // append the element to the pipeline stage slice + *stageSlice = append(*stageSlice, &pipeline.Stage{ + Done: make(chan error, 1), + Environment: stage.Environment, + Name: stage.Name, + Needs: stage.Needs, + Independent: stage.Independent, + Steps: *stage.Steps.ToPipeline(), + }) + } + + return stageSlice +} + +// UnmarshalYAML implements the Unmarshaler interface for the StageSlice type. +func (s *StageSlice) UnmarshalYAML(v *yaml.Node) error { + if v.Kind != yaml.MappingNode { + return fmt.Errorf("invalid yaml: expected map node for stage") + } + + // iterate through each element in the map slice + for i := 0; i < len(v.Content); i += 2 { + key := v.Content[i] + value := v.Content[i+1] + + stage := new(Stage) + + // unmarshal value into stage + err := value.Decode(stage) + if err != nil { + return err + } + + // implicitly set stage `name` if empty + if len(stage.Name) == 0 { + stage.Name = fmt.Sprintf("%v", key.Value) + } + + // implicitly set the stage `needs` + if stage.Name != "clone" && stage.Name != "init" { + // add clone if not present + stage.Needs = func(needs []string) []string { + for _, s := range needs { + if s == "clone" { + return needs + } + } + + return append(needs, "clone") + }(stage.Needs) + } + // append stage to stage slice + *s = append(*s, stage) + } + + return nil +} + +// MarshalYAML implements the marshaler interface for the StageSlice type. +func (s StageSlice) MarshalYAML() (interface{}, error) { + output := new(yaml.Node) + output.Kind = yaml.MappingNode + + for _, inputStage := range s { + n := new(yaml.Node) + + // create new stage with existing properties + outputStage := &Stage{ + Name: inputStage.Name, + Needs: inputStage.Needs, + Independent: inputStage.Independent, + Steps: inputStage.Steps, + } + + err := n.Encode(outputStage) + if err != nil { + return nil, err + } + + // append stage to map output + output.Content = append(output.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: inputStage.Name}) + output.Content = append(output.Content, n) + } + + return output, nil +} + +// JSONSchemaExtend handles some overrides that need to be in place +// for this type for the jsonschema generation. +// +// Stages are not really a slice of stages to the user. This change +// supports the map they really are. +func (StageSlice) JSONSchemaExtend(schema *jsonschema.Schema) { + schema.AdditionalProperties = jsonschema.FalseSchema + schema.Items = nil + schema.PatternProperties = map[string]*jsonschema.Schema{ + ".*": { + Ref: "#/$defs/Stage", + }, + } + schema.Type = "object" +} + +// MergeEnv takes a list of environment variables and attempts +// to set them in the stage environment. If the environment +// variable already exists in the stage, than this will +// overwrite the existing environment variable. +func (s *Stage) MergeEnv(environment map[string]string) error { + // check if the stage is empty + if s == nil || s.Environment == nil { + // TODO: evaluate if we should error here + // + // immediately return and do nothing + // + // treated as a no-op + return nil + } + + // check if the environment provided is empty + if environment == nil { + return fmt.Errorf("empty environment provided for stage %s", s.Name) + } + + // iterate through all environment variables provided + for key, value := range environment { + // set or update the stage environment variable + s.Environment[key] = value + } + + return nil +} diff --git a/compiler/types/yaml/yaml/stage_test.go b/compiler/types/yaml/yaml/stage_test.go new file mode 100644 index 000000000..86169f373 --- /dev/null +++ b/compiler/types/yaml/yaml/stage_test.go @@ -0,0 +1,474 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_StageSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + stages *StageSlice + want *pipeline.StageSlice + }{ + { + stages: &StageSlice{ + { + Name: "echo", + Needs: []string{"clone"}, + Steps: StepSlice{ + { + Commands: []string{"echo hello"}, + Detach: false, + Entrypoint: []string{"/bin/sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Privileged: false, + Pull: "not_present", + Ruleset: Ruleset{ + If: Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + }, + Unless: Rules{ + Branch: []string{"main"}, + Comment: []string{"real comment"}, + Event: []string{"pull_request"}, + Path: []string{"bar.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"failure"}, + Tag: []string{"v0.2.0"}, + Target: []string{"production"}, + }, + Operator: "and", + Continue: false, + }, + Secrets: StepSecretSlice{ + { + Source: "docker_username", + Target: "plugin_username", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + }, + }, + }, + want: &pipeline.StageSlice{ + { + Name: "echo", + Needs: []string{"clone"}, + Steps: pipeline.ContainerSlice{ + { + Commands: []string{"echo hello"}, + Detach: false, + Entrypoint: []string{"/bin/sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Privileged: false, + Pull: "not_present", + Ruleset: pipeline.Ruleset{ + If: pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + }, + Unless: pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"real comment"}, + Event: []string{"pull_request"}, + Path: []string{"bar.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"failure"}, + Tag: []string{"v0.2.0"}, + Target: []string{"production"}, + }, + Operator: "and", + Continue: false, + }, + Secrets: pipeline.StepSecretSlice{ + { + Source: "docker_username", + Target: "plugin_username", + }, + }, + Ulimits: pipeline.UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + Volumes: pipeline.VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + }, + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.stages.ToPipeline() + + // WARNING: hack to compare stages + // + // Channel values can only be compared for equality. + // Two channel values are considered equal if they + // originated from the same make call meaning they + // refer to the same channel value in memory. + for i, stage := range *got { + tmp := *test.want + + tmp[i].Done = stage.Done + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_StageSlice_UnmarshalYAML(t *testing.T) { + // setup types + var ( + b []byte + err error + ) + + // setup tests + tests := []struct { + failure bool + file string + want *StageSlice + }{ + { + failure: false, + file: "testdata/stage.yml", + want: &StageSlice{ + { + Name: "dependencies", + Needs: []string{"clone"}, + Environment: map[string]string{ + "STAGE_ENV_VAR": "stage", + }, + Independent: true, + Steps: StepSlice{ + { + Commands: []string{"./gradlew downloadDependencies"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + }, + }, + }, + { + Name: "test", + Needs: []string{"dependencies", "clone"}, + Environment: map[string]string{ + "STAGE_ENV_VAR": "stage", + "SECOND_STAGE_ENV": "stage2", + }, + Independent: false, + Steps: StepSlice{ + { + Commands: []string{"./gradlew check"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "test", + Image: "openjdk:latest", + Pull: "always", + }, + }, + }, + { + Name: "build", + Needs: []string{"dependencies", "clone"}, + Environment: map[string]string{ + "STAGE_ENV_VAR": "stage", + }, + Independent: false, + Steps: StepSlice{ + { + Commands: []string{"./gradlew build"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "build", + Image: "openjdk:latest", + Pull: "always", + }, + }, + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + { + failure: true, + file: "", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(StageSlice) + + if len(test.file) > 0 { + b, err = os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + } else { + b = []byte("- foo") + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if diff := cmp.Diff(test.want, got); diff != "" { + t.Errorf("(Unmarshal mismatch: -want +got):\n%s", diff) + } + } +} + +func TestYaml_StageSlice_MarshalYAML(t *testing.T) { + // setup types + var ( + b []byte + err error + ) + + // setup tests + tests := []struct { + failure bool + file string + want *StageSlice + }{ + { + failure: false, + file: "testdata/stage.yml", + want: &StageSlice{ + { + Name: "dependencies", + Needs: []string{"clone"}, + Independent: true, + Steps: StepSlice{ + { + Commands: []string{"./gradlew downloadDependencies"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + }, + }, + }, + { + Name: "test", + Needs: []string{"dependencies", "clone"}, + Independent: false, + Steps: StepSlice{ + { + Commands: []string{"./gradlew check"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "test", + Image: "openjdk:latest", + Pull: "always", + }, + }, + }, + { + Name: "build", + Needs: []string{"dependencies", "clone"}, + Independent: false, + Steps: StepSlice{ + { + Commands: []string{"./gradlew build"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "build", + Image: "openjdk:latest", + Pull: "always", + }, + }, + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + { + failure: true, + file: "", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(StageSlice) + got2 := new(StageSlice) + + if len(test.file) > 0 { + b, err = os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + } else { + b = []byte("- foo") + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + out, err := yaml.Marshal(got) + if err != nil { + t.Errorf("MarshalYAML returned err: %v", err) + } + + err = yaml.Unmarshal(out, got2) + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if diff := cmp.Diff(got2, test.want); diff != "" { + t.Errorf("(Marshal mismatch: -got +want):\n%s", diff) + } + } +} + +func TestYaml_Stage_MergeEnv(t *testing.T) { + // setup tests + tests := []struct { + stage *Stage + environment map[string]string + failure bool + }{ + { + stage: &Stage{ + Environment: map[string]string{"FOO": "bar"}, + Name: "testStage", + }, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + stage: &Stage{}, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + stage: nil, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + stage: &Stage{ + Environment: map[string]string{"FOO": "bar"}, + Name: "testStage", + }, + environment: nil, + failure: true, + }, + } + + // run tests + for _, test := range tests { + err := test.stage.MergeEnv(test.environment) + + if test.failure { + if err == nil { + t.Errorf("MergeEnv should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("MergeEnv returned err: %v", err) + } + } +} diff --git a/compiler/types/yaml/step.go b/compiler/types/yaml/yaml/step.go similarity index 100% rename from compiler/types/yaml/step.go rename to compiler/types/yaml/yaml/step.go diff --git a/compiler/types/yaml/yaml/step_test.go b/compiler/types/yaml/yaml/step_test.go new file mode 100644 index 000000000..343daf230 --- /dev/null +++ b/compiler/types/yaml/yaml/step_test.go @@ -0,0 +1,327 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" + "github.com/go-vela/server/compiler/types/raw" +) + +func TestYaml_StepSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + steps *StepSlice + want *pipeline.ContainerSlice + }{ + { + steps: &StepSlice{ + { + Commands: []string{"echo hello"}, + Detach: false, + Entrypoint: []string{"/bin/sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Privileged: false, + Pull: "not_present", + ReportAs: "my-step", + IDRequest: "yes", + Ruleset: Ruleset{ + If: Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + }, + Unless: Rules{ + Branch: []string{"main"}, + Comment: []string{"real comment"}, + Event: []string{"pull_request"}, + Path: []string{"bar.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"failure"}, + Tag: []string{"v0.2.0"}, + Target: []string{"production"}, + }, + Operator: "and", + Continue: false, + }, + Secrets: StepSecretSlice{ + { + Source: "docker_username", + Target: "plugin_username", + }, + }, + Ulimits: UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + Volumes: VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + }, + want: &pipeline.ContainerSlice{ + { + Commands: []string{"echo hello"}, + Detach: false, + Entrypoint: []string{"/bin/sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Privileged: false, + Pull: "not_present", + ReportAs: "my-step", + IDRequest: "yes", + Ruleset: pipeline.Ruleset{ + If: pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"test comment"}, + Event: []string{"push"}, + Path: []string{"foo.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"success"}, + Tag: []string{"v0.1.0"}, + Target: []string{"production"}, + }, + Unless: pipeline.Rules{ + Branch: []string{"main"}, + Comment: []string{"real comment"}, + Event: []string{"pull_request"}, + Path: []string{"bar.txt"}, + Repo: []string{"github/octocat"}, + Status: []string{"failure"}, + Tag: []string{"v0.2.0"}, + Target: []string{"production"}, + }, + Operator: "and", + Continue: false, + }, + Secrets: pipeline.StepSecretSlice{ + { + Source: "docker_username", + Target: "plugin_username", + }, + }, + Ulimits: pipeline.UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + Volumes: pipeline.VolumeSlice{ + { + Source: "/foo", + Destination: "/bar", + AccessMode: "ro", + }, + }, + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.steps.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_StepSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *StepSlice + }{ + { + failure: false, + file: "testdata/step.yml", + want: &StepSlice{ + { + Commands: raw.StringSlice{"./gradlew downloadDependencies"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "install", + Image: "openjdk:latest", + Pull: "always", + }, + { + Commands: raw.StringSlice{"./gradlew check"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "test", + Image: "openjdk:latest", + Pull: "always", + }, + { + Commands: raw.StringSlice{"./gradlew build"}, + Environment: raw.StringSliceMap{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Name: "build", + Image: "openjdk:latest", + Pull: "always", + }, + { + Name: "docker_build", + Image: "plugins/docker:18.09", + Pull: "always", + ReportAs: "docker", + Parameters: map[string]interface{}{ + "registry": "index.docker.io", + "repo": "github/octocat", + "tags": []interface{}{"latest", "dev"}, + }, + }, + { + Name: "templated_publish", + Pull: "not_present", + Template: StepTemplate{ + Name: "docker_publish", + Variables: map[string]interface{}{ + "registry": "index.docker.io", + "repo": "github/octocat", + "tags": []interface{}{"latest", "dev"}, + }, + }, + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + { + failure: true, + file: "testdata/step_malformed.yml", + want: nil, + }, + { + failure: true, + file: "testdata/step_nil.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(StepSlice) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} + +func TestYaml_Step_MergeEnv(t *testing.T) { + // setup tests + tests := []struct { + step *Step + environment map[string]string + failure bool + }{ + { + step: &Step{ + Commands: []string{"echo hello"}, + Detach: false, + Entrypoint: []string{"/bin/sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Privileged: false, + Pull: "not_present", + }, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + step: &Step{}, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + step: nil, + environment: map[string]string{"BAR": "baz"}, + failure: false, + }, + { + step: &Step{ + Commands: []string{"echo hello"}, + Detach: false, + Entrypoint: []string{"/bin/sh"}, + Environment: map[string]string{"FOO": "bar"}, + Image: "alpine:latest", + Name: "echo", + Privileged: false, + Pull: "not_present", + }, + environment: nil, + failure: true, + }, + } + + // run tests + for _, test := range tests { + err := test.step.MergeEnv(test.environment) + + if test.failure { + if err == nil { + t.Errorf("MergeEnv should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("MergeEnv returned err: %v", err) + } + } +} diff --git a/compiler/types/yaml/template.go b/compiler/types/yaml/yaml/template.go similarity index 100% rename from compiler/types/yaml/template.go rename to compiler/types/yaml/yaml/template.go diff --git a/compiler/types/yaml/yaml/template_test.go b/compiler/types/yaml/yaml/template_test.go new file mode 100644 index 000000000..128c998a0 --- /dev/null +++ b/compiler/types/yaml/yaml/template_test.go @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + api "github.com/go-vela/server/api/types" +) + +func TestBuild_TemplateSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *TemplateSlice + }{ + { + failure: false, + file: "testdata/template.yml", + want: &TemplateSlice{ + { + Name: "docker_build", + Source: "github.com/go-vela/atlas/stable/docker_create", + Type: "github", + }, + { + Name: "docker_build", + Source: "github.com/go-vela/atlas/stable/docker_build", + Format: "go", + Type: "github", + }, + { + Name: "docker_publish", + Source: "github.com/go-vela/atlas/stable/docker_publish", + Format: "starlark", + Type: "github", + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(TemplateSlice) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} + +func TestYAML_Template_ToAPI(t *testing.T) { + // setup types + want := new(api.Template) + want.SetName("docker_build") + want.SetSource("github.com/go-vela/atlas/stable/docker_build") + want.SetType("github") + + tmpl := &Template{ + Name: "docker_build", + Source: "github.com/go-vela/atlas/stable/docker_build", + Type: "github", + } + + // run test + got := tmpl.ToAPI() + + if !reflect.DeepEqual(got, want) { + t.Errorf("ToAPI is %v, want %v", got, want) + } +} + +func TestYAML_TemplateFromAPI(t *testing.T) { + // setup types + want := &Template{ + Name: "docker_build", + Source: "github.com/go-vela/atlas/stable/docker_build", + Type: "github", + } + + tmpl := new(api.Template) + tmpl.SetName("docker_build") + tmpl.SetSource("github.com/go-vela/atlas/stable/docker_build") + tmpl.SetType("github") + + // run test + got := TemplateFromAPI(tmpl) + + if !reflect.DeepEqual(got, want) { + t.Errorf("TemplateFromAPI is %v, want %v", got, want) + } +} diff --git a/compiler/types/yaml/yaml/testdata/build.yml b/compiler/types/yaml/yaml/testdata/build.yml new file mode 100644 index 000000000..e1a7fbc9f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build.yml @@ -0,0 +1,144 @@ +--- +version: "1" + +environment: + HELLO: "Hello, Global Message" + +templates: + - name: docker_publish + source: github.com/go-vela/atlas/stable/docker_publish + type: github + +worker: + flavor: 16cpu8gb + platform: gcp + +services: + - name: postgres + image: postgres:latest + environment: + POSTGRES_DB: foo + ports: + - "5432:5432" + +steps: + - name: install + commands: + - ./gradlew downloadDependencies + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + ruleset: + event: [ push, pull_request:opened, pull_request:synchronize, pull_request:edited ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] + + - name: test + commands: + - ./gradlew check + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + ruleset: + event: [ push, pull_request ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] + + - name: build + commands: + - ./gradlew build + environment: + - GRADLE_OPTS=-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + - GRADLE_USER_HOME=.gradle + image: openjdk:latest + pull: true + ruleset: + event: [ push, pull_request ] + volumes: + - source: /foo + destination: /bar + access_mode: ro + ulimits: + - name: foo + soft: 1024 + hard: 2048 + + - name: docker_build + image: plugins/docker:18.09 + parameters: + dry_run: true + registry: index.docker.io + repo: github/octocat + tags: + - latest + - dev + pull: true + ruleset: + if: + event: [ push, pull_request ] + operator: and + + - name: docker_publish + image: plugins/docker:18.09 + parameters: + registry: index.docker.io + repo: github/octocat + tags: + - latest + - dev + pull: true + ruleset: + if: + branch: main + event: push + operator: and + secrets: + - source: docker_username + target: plugin_username + - source: docker_password + target: plugin_password + +secrets: + # Repo secrets + - name: docker_username + key: org/repo/docker/username + engine: native + type: repo + + - name: docker_password + key: org/repo/docker/password + engine: vault + type: repo + + # Org secrets + - name: docker_username + key: org/docker/username + engine: native + type: org + + - name: docker_password + key: org/docker/password + engine: vault + type: org + + # Shared secrets + - name: docker_username + key: org/team/docker/username + engine: native + type: shared + + - name: docker_password + key: org/team/docker/password + engine: vault + type: shared + + - origin: + image: target/vela-vault:latest + pull: always + parameters: + addr: vault.example.com + secrets: [ docker_username, docker_password ] diff --git a/compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline0.yml b/compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline0.yml new file mode 100644 index 000000000..8cbe12806 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline0.yml @@ -0,0 +1 @@ +version: 1 \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline1.yml b/compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline1.yml new file mode 100644 index 000000000..b1db03665 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build/validate/bad_pipeline1.yml @@ -0,0 +1,3 @@ +version: "1" +steps: +stages: \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/build/validate/bad_version.yml b/compiler/types/yaml/yaml/testdata/build/validate/bad_version.yml new file mode 100644 index 000000000..e2489f424 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build/validate/bad_version.yml @@ -0,0 +1,2 @@ +--- +steps: \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/build/validate/step.yml b/compiler/types/yaml/yaml/testdata/build/validate/step.yml new file mode 100644 index 000000000..a70942591 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build/validate/step.yml @@ -0,0 +1,47 @@ +--- +version: 1 +steps: + - name: install + commands: + - ./gradlew downloadDependencies + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + + - name: test + commands: + - ./gradlew check + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + + - name: build + commands: + - ./gradlew build + environment: + - GRADLE_OPTS=-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + - GRADLE_USER_HOME=.gradle + image: openjdk:latest + pull: true + + - name: docker_build + image: plugins/docker:18.09 + parameters: + registry: index.docker.io + repo: github/octocat + tags: + - latest + - dev + pull: true + + - name: templated_publish + template: + name: docker_publish + vars: + registry: index.docker.io + repo: github/octocat + tags: [ latest, dev ] diff --git a/compiler/types/yaml/yaml/testdata/build_anchor_stage.yml b/compiler/types/yaml/yaml/testdata/build_anchor_stage.yml new file mode 100644 index 000000000..2fc87932b --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build_anchor_stage.yml @@ -0,0 +1,57 @@ +--- +version: "1" + +metadata: + template: false + +stage-anchor: &stage-anchor + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + +stages: + dependencies: + steps: + - name: install + commands: + - ./gradlew downloadDependencies + <<: *stage-anchor + pull: true + ruleset: + event: [ push, pull_request ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] + + test: + needs: [ dependencies ] + steps: + - name: test + commands: + - ./gradlew check + <<: *stage-anchor + pull: true + ruleset: + event: [ push, pull_request ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] + + build: + needs: [ dependencies ] + independent: true + steps: + - name: build + commands: + - ./gradlew build + <<: *stage-anchor + pull: true + ruleset: + event: [ push, pull_request ] + volumes: + - source: /foo + destination: /bar + access_mode: ro + ulimits: + - name: foo + soft: 1024 + hard: 2048 diff --git a/compiler/types/yaml/yaml/testdata/build_anchor_step.yml b/compiler/types/yaml/yaml/testdata/build_anchor_step.yml new file mode 100644 index 000000000..ffc7abf50 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build_anchor_step.yml @@ -0,0 +1,48 @@ +--- +version: "1" + +metadata: + template: false + +step-anchor: &step-anchor + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + +steps: + - name: install + commands: + - ./gradlew downloadDependencies + <<: *step-anchor + pull: true + ruleset: + event: [ push, pull_request ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] + + - name: test + commands: + - ./gradlew check + <<: *step-anchor + pull: true + ruleset: + event: [ push, pull_request ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] + + - name: build + commands: + - ./gradlew build + <<: *step-anchor + pull: true + ruleset: + event: [ push, pull_request ] + volumes: + - source: /foo + destination: /bar + access_mode: ro + ulimits: + - name: foo + soft: 1024 + hard: 2048 diff --git a/compiler/types/yaml/yaml/testdata/build_empty_env.yml b/compiler/types/yaml/yaml/testdata/build_empty_env.yml new file mode 100644 index 000000000..6b4f7d063 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build_empty_env.yml @@ -0,0 +1,27 @@ +--- +version: "1" + +metadata: + template: false + environment: [] + +environment: + HELLO: "Hello, Global Message" + +worker: + flavor: 16cpu8gb + platform: gcp + +steps: + - name: install + commands: + - ./gradlew downloadDependencies + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + ruleset: + event: [ push, pull_request ] + volumes: [ /foo:/bar:ro ] + ulimits: [ foo=1024:2048 ] \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/build_with_deploy_config.yml b/compiler/types/yaml/yaml/testdata/build_with_deploy_config.yml new file mode 100644 index 000000000..61aea8b48 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/build_with_deploy_config.yml @@ -0,0 +1,36 @@ +version: "1" + +deployment: + targets: [ dev, stage, production ] + parameters: + alpha: + description: primary node name + required: true + type: string + options: + - north + - south + + beta: + description: secondary node name + required: false + type: string + options: + - east + - west + + cluster_count: + description: number of clusters to deploy + required: false + type: integer + + canary: + description: deploy with canary strategy + required: true + type: boolean + +steps: + - name: deploy plugin + image: awesome-plugin:latest + commands: + - ./deploy.sh diff --git a/compiler/types/yaml/yaml/testdata/deploy_parameter.yml b/compiler/types/yaml/yaml/testdata/deploy_parameter.yml new file mode 100644 index 000000000..89a9e0ae6 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/deploy_parameter.yml @@ -0,0 +1,11 @@ +--- +foo: + description: bar + required: true + type: string + options: + - baz +hello: + description: baz + required: false + type: string \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/invalid.yml b/compiler/types/yaml/yaml/testdata/invalid.yml new file mode 100644 index 000000000..23809fe06 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/invalid.yml @@ -0,0 +1,2 @@ +--- +foo: bar diff --git a/compiler/types/yaml/yaml/testdata/merge_anchor.yml b/compiler/types/yaml/yaml/testdata/merge_anchor.yml new file mode 100644 index 000000000..626b695f5 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/merge_anchor.yml @@ -0,0 +1,45 @@ +# test file that uses the non-standard multiple anchor keys in one step to test custom step unmarshaler + +version: "1" + +aliases: + images: + alpine: &alpine-image + image: alpine:latest + postgres: &pg-image + image: postgres + + events: + push: &event-push + ruleset: + event: + - push + env: + dev-env: &dev-environment + environment: + REGION: dev + +services: + - name: service-a + <<: [ *pg-image, *dev-environment ] + ports: + - "5432:5432" + +steps: + - name: alpha + <<: [ *alpine-image, *event-push ] + commands: + - echo alpha + + - name: beta + <<: [ *alpine-image, *event-push ] + commands: + - echo beta + + - name: gamma + <<: + - *alpine-image + - *event-push + - *dev-environment + commands: + - echo gamma \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/metadata.yml b/compiler/types/yaml/yaml/testdata/metadata.yml new file mode 100644 index 000000000..6946050b0 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/metadata.yml @@ -0,0 +1,2 @@ +--- +template: false diff --git a/compiler/types/yaml/yaml/testdata/metadata_env.yml b/compiler/types/yaml/yaml/testdata/metadata_env.yml new file mode 100644 index 000000000..0b7932a30 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/metadata_env.yml @@ -0,0 +1,3 @@ +--- +template: false +environment: [ steps ] diff --git a/compiler/types/yaml/yaml/testdata/ruleset_advanced.yml b/compiler/types/yaml/yaml/testdata/ruleset_advanced.yml new file mode 100644 index 000000000..24039e2a6 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ruleset_advanced.yml @@ -0,0 +1,15 @@ +--- +if: + branch: [ main ] + event: push + tag: "^refs/tags/(\\d+\\.)+\\d+$" +unless: + event: + - deployment + - pull_request + - comment + - schedule + path: [ foo.txt, /foo/bar.txt ] +matcher: regexp +operator: or +continue: true \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/ruleset_regex.yml b/compiler/types/yaml/yaml/testdata/ruleset_regex.yml new file mode 100644 index 000000000..eb6b1fd31 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ruleset_regex.yml @@ -0,0 +1,7 @@ +--- +if: + branch: main + event: tag + tag: [ "^refs/tags/(\\d+\\.)+\\d+$" ] + operator: and +matcher: regex diff --git a/compiler/types/yaml/yaml/testdata/ruleset_simple.yml b/compiler/types/yaml/yaml/testdata/ruleset_simple.yml new file mode 100644 index 000000000..acbe4e198 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ruleset_simple.yml @@ -0,0 +1,13 @@ +--- +branch: main +comment: "test comment" +continue: true +event: push +instance: vela-server +label: bug +path: foo.txt +repo: github/octocat +sender: octocat +status: success +tag: v0.1.0 +target: production diff --git a/compiler/types/yaml/yaml/testdata/secret.yml b/compiler/types/yaml/yaml/testdata/secret.yml new file mode 100644 index 000000000..c432eb291 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret.yml @@ -0,0 +1,39 @@ +--- +- source: foo + target: bar +- name: foo + key: bar + engine: native + type: repo + pull: build_start +- name: noKey + engine: native + type: repo +- name: noType + key: bar + engine: native +- name: noEngine + key: bar + type: repo +- name: noKeyEngineAndType +- name: externalSecret + origin: + environment: + FOO: bar + image: target/vela-vault:latest + pull: true + parameters: + addr: vault.company.com + ruleset: + event: [ push ] + secrets: [ foo, foobar ] +- origin: + environment: + FOO: bar + image: target/vela-vault:latest + pull: true + parameters: + addr: vault.company.com + ruleset: + event: [ push ] + secrets: [ foo, foobar ] diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/no_name.yml b/compiler/types/yaml/yaml/testdata/secret/validate/no_name.yml new file mode 100644 index 000000000..d674817c5 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/no_name.yml @@ -0,0 +1,11 @@ +secrets: +# Declarative repository secret definition. + - key: github/ocotocat/foob + engine: native + type: repo + - key: github/ocotocat + engine: native + type: org + - key: github/octokitties/foobar + engine: native + type: org \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/org.yml b/compiler/types/yaml/yaml/testdata/secret/validate/org.yml new file mode 100644 index 000000000..a5aad5e0d --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/org.yml @@ -0,0 +1,5 @@ +secrets: + - name: foobar + key: github/foobar + engine: native + type: org \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/org_bad_engine.yml b/compiler/types/yaml/yaml/testdata/secret/validate/org_bad_engine.yml new file mode 100644 index 000000000..18f9f8c6f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/org_bad_engine.yml @@ -0,0 +1,9 @@ +secrets: + - name: foo + key: github/foobar + type: org + + - name: foobar + key: github/foobar + engine: badengine + type: org \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/org_bad_key.yml b/compiler/types/yaml/yaml/testdata/secret/validate/org_bad_key.yml new file mode 100644 index 000000000..bae2fcc9a --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/org_bad_key.yml @@ -0,0 +1,9 @@ +secrets: + - name: foo + engine: native + type: org + + - name: foobar + key: github + engine: native + type: org \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/plugin.yml b/compiler/types/yaml/yaml/testdata/secret/validate/plugin.yml new file mode 100644 index 000000000..180ab5da4 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/plugin.yml @@ -0,0 +1,8 @@ +secrets: + - origin: + name: vault secrets + image: target/vela/secret-vault:latest + parameters: + items: + - source: secret/vela/dev/docker + path: docker \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_image.yml b/compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_image.yml new file mode 100644 index 000000000..c21be424e --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_image.yml @@ -0,0 +1,15 @@ +secrets: + - origin: + name: vault secrets + parameters: + items: + - source: secret/vela/dev/docker + path: docker + + - origin: + name: vault secrets + image: bazel/:java:3240943c9ea3f72db51bea0a2428e83f3c5fa1312e19af017d026f9bcf70f84b + parameters: + items: + - source: secret/vela/dev/docker + path: docker \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_name.yml b/compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_name.yml new file mode 100644 index 000000000..6ebb1505f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/plugin_bad_name.yml @@ -0,0 +1,7 @@ +secrets: + - origin: + image: target/vela/secret-vault:latest + parameters: + items: + - source: secret/vela/dev/docker + path: docker \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/repo.yml b/compiler/types/yaml/yaml/testdata/secret/validate/repo.yml new file mode 100644 index 000000000..fcad31edb --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/repo.yml @@ -0,0 +1,13 @@ +secrets: + # Implicit native secret definition. + - name: foo + + # Declarative repository secret definition. + - name: foob + key: github/ocotocat/foob + engine: native + type: repo + - name: foo_bar + key: github/ocotocat/foo/bar + engine: native + type: repo \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_engine.yml b/compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_engine.yml new file mode 100644 index 000000000..3eac1d359 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_engine.yml @@ -0,0 +1,5 @@ +secrets: + - name: foobar + key: github/ocotocat/foobar + engine: badengine + type: repo \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_key.yml b/compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_key.yml new file mode 100644 index 000000000..cf031b30c --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/repo_bad_key.yml @@ -0,0 +1,14 @@ +secrets: + - name: foo + engine: native + type: repo + + - name: bar + key: github/ocotocat + engine: native + type: repo + + - name: foobar + key: github + engine: native + type: repo \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/shared.yml b/compiler/types/yaml/yaml/testdata/secret/validate/shared.yml new file mode 100644 index 000000000..4037a97a3 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/shared.yml @@ -0,0 +1,5 @@ +secrets: + - name: foobar + key: github/ocotokitties/foo + engine: native + type: shared \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_engine.yml b/compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_engine.yml new file mode 100644 index 000000000..ca22067dd --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_engine.yml @@ -0,0 +1,9 @@ +secrets: + - name: foo + key: github/ocotokitties/foo + type: shared + + - name: foobar + key: github/ocotokitties/foo + engine: badengine + type: shared \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_key.yml b/compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_key.yml new file mode 100644 index 000000000..b80945618 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/secret/validate/shared_bad_key.yml @@ -0,0 +1,9 @@ +secrets: + - name: foo + engine: native + type: shared + + - name: foobar + key: github/ocotokitties + engine: native + type: shared \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/service.yml b/compiler/types/yaml/yaml/testdata/service.yml new file mode 100644 index 000000000..b6bc0456f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/service.yml @@ -0,0 +1,14 @@ +--- +- name: postgres + image: postgres:latest + environment: + POSTGRES_DB: foo + ports: + - "5432:5432" + +- name: mysql + image: mysql:latest + environment: + MYSQL_DATABASE: foo + ports: + - "3061:3061" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/service/validate/bad_image.yml b/compiler/types/yaml/yaml/testdata/service/validate/bad_image.yml new file mode 100644 index 000000000..47a875e67 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/service/validate/bad_image.yml @@ -0,0 +1,3 @@ +services: + - name: badimage + image: bazel/:java:3240943c9ea3f72db51bea0a2428e83f3c5fa1312e19af017d026f9bcf70f84b \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/service/validate/minimal.yml b/compiler/types/yaml/yaml/testdata/service/validate/minimal.yml new file mode 100644 index 000000000..469d2dddf --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/service/validate/minimal.yml @@ -0,0 +1,3 @@ +services: + - name: postgres + image: postgres:latest diff --git a/compiler/types/yaml/yaml/testdata/service/validate/missing_image.yml b/compiler/types/yaml/yaml/testdata/service/validate/missing_image.yml new file mode 100644 index 000000000..3c2b76f1f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/service/validate/missing_image.yml @@ -0,0 +1,2 @@ +services: + - name: postgres \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/service/validate/missing_name.yml b/compiler/types/yaml/yaml/testdata/service/validate/missing_name.yml new file mode 100644 index 000000000..0c1034e5d --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/service/validate/missing_name.yml @@ -0,0 +1,2 @@ +services: + - image: postgres:latest \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/service_nil.yml b/compiler/types/yaml/yaml/testdata/service_nil.yml new file mode 100644 index 000000000..41cd65e60 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/service_nil.yml @@ -0,0 +1,2 @@ +--- +- diff --git a/compiler/types/yaml/yaml/testdata/stage.yml b/compiler/types/yaml/yaml/testdata/stage.yml new file mode 100644 index 000000000..543ffdf83 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/stage.yml @@ -0,0 +1,44 @@ +--- +dependencies: + environment: + STAGE_ENV_VAR: stage + independent: true + steps: + - name: install + commands: + - ./gradlew downloadDependencies + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + +test: + needs: [ dependencies ] + environment: + STAGE_ENV_VAR: stage + SECOND_STAGE_ENV: stage2 + independent: false + steps: + - name: test + commands: + - ./gradlew check + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + +build: + needs: [ dependencies ] + environment: + STAGE_ENV_VAR: stage + steps: + - name: build + commands: + - ./gradlew build + environment: + - GRADLE_OPTS=-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + - GRADLE_USER_HOME=.gradle + image: openjdk:latest + pull: true diff --git a/compiler/types/yaml/yaml/testdata/stage/validate/bad_image.yml b/compiler/types/yaml/yaml/testdata/stage/validate/bad_image.yml new file mode 100644 index 000000000..3dbed3107 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/stage/validate/bad_image.yml @@ -0,0 +1,7 @@ +stages: + badimage: + steps: + - name: badimage + image: bazel/:java:3240943c9ea3f72db51bea0a2428e83f3c5fa1312e19af017d026f9bcf70f84b + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/stage/validate/minimal.yml b/compiler/types/yaml/yaml/testdata/stage/validate/minimal.yml new file mode 100644 index 000000000..665887c87 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/stage/validate/minimal.yml @@ -0,0 +1,7 @@ +stages: + hello: + steps: + - name: hello + image: alpine:latest + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/stage/validate/missing.yml b/compiler/types/yaml/yaml/testdata/stage/validate/missing.yml new file mode 100644 index 000000000..954fdeabe --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/stage/validate/missing.yml @@ -0,0 +1,5 @@ +stages: + hello: + steps: + - name: hello + image: alpine:latest \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/stage/validate/missing_image.yml b/compiler/types/yaml/yaml/testdata/stage/validate/missing_image.yml new file mode 100644 index 000000000..90b361748 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/stage/validate/missing_image.yml @@ -0,0 +1,6 @@ +stages: + hello: + steps: + - name: hello + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/stage/validate/missing_name.yml b/compiler/types/yaml/yaml/testdata/stage/validate/missing_name.yml new file mode 100644 index 000000000..1394939a3 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/stage/validate/missing_name.yml @@ -0,0 +1,6 @@ +stages: + hello: + steps: + - image: alpine:latest + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step.yml b/compiler/types/yaml/yaml/testdata/step.yml new file mode 100644 index 000000000..1d6d9cc93 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step.yml @@ -0,0 +1,46 @@ +--- +- name: install + commands: + - ./gradlew downloadDependencies + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + +- name: test + commands: + - ./gradlew check + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + +- name: build + commands: + - ./gradlew build + environment: + - GRADLE_OPTS=-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + - GRADLE_USER_HOME=.gradle + image: openjdk:latest + pull: true + +- name: docker_build + image: plugins/docker:18.09 + report_as: docker + parameters: + registry: index.docker.io + repo: github/octocat + tags: + - latest + - dev + pull: true + +- name: templated_publish + template: + name: docker_publish + vars: + registry: index.docker.io + repo: github/octocat + tags: [ latest, dev ] diff --git a/compiler/types/yaml/yaml/testdata/step/validate/bad_image.yml b/compiler/types/yaml/yaml/testdata/step/validate/bad_image.yml new file mode 100644 index 000000000..a97e8f12c --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step/validate/bad_image.yml @@ -0,0 +1,5 @@ +steps: + - name: badimage + image: bazel/:java:3240943c9ea3f72db51bea0a2428e83f3c5fa1312e19af017d026f9bcf70f84b + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step/validate/minimal.yml b/compiler/types/yaml/yaml/testdata/step/validate/minimal.yml new file mode 100644 index 000000000..da2283d4b --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step/validate/minimal.yml @@ -0,0 +1,5 @@ +steps: + - name: hello + image: alpine:latest + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step/validate/missing.yml b/compiler/types/yaml/yaml/testdata/step/validate/missing.yml new file mode 100644 index 000000000..0aa30db52 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step/validate/missing.yml @@ -0,0 +1,3 @@ +steps: + - name: hello + image: alpine:latest \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step/validate/missing_image.yml b/compiler/types/yaml/yaml/testdata/step/validate/missing_image.yml new file mode 100644 index 000000000..b36acf70d --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step/validate/missing_image.yml @@ -0,0 +1,4 @@ +steps: + - name: hello + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step/validate/missing_name.yml b/compiler/types/yaml/yaml/testdata/step/validate/missing_name.yml new file mode 100644 index 000000000..228ac30ec --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step/validate/missing_name.yml @@ -0,0 +1,4 @@ +steps: + - image: alpine:latest + commands: + - echo "hello vela" \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step_malformed.yml b/compiler/types/yaml/yaml/testdata/step_malformed.yml new file mode 100644 index 000000000..5d70e5a4f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step_malformed.yml @@ -0,0 +1,4 @@ +--- +- name: Testing + environment: + - 'This: Shouldnt Panic' diff --git a/compiler/types/yaml/yaml/testdata/step_nil.yml b/compiler/types/yaml/yaml/testdata/step_nil.yml new file mode 100644 index 000000000..41cd65e60 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step_nil.yml @@ -0,0 +1,2 @@ +--- +- diff --git a/compiler/types/yaml/yaml/testdata/step_secret_slice.yml b/compiler/types/yaml/yaml/testdata/step_secret_slice.yml new file mode 100644 index 000000000..64bc68b7f --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step_secret_slice.yml @@ -0,0 +1,5 @@ +--- +- source: foo + target: bar +- source: hello + target: world diff --git a/compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_source.yml b/compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_source.yml new file mode 100644 index 000000000..1f7e6fc3e --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_source.yml @@ -0,0 +1,2 @@ +--- +- target: foo \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_target.yml b/compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_target.yml new file mode 100644 index 000000000..3e0e29b1c --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step_secret_slice_invalid_no_target.yml @@ -0,0 +1,2 @@ +--- +- source: foo \ No newline at end of file diff --git a/compiler/types/yaml/yaml/testdata/step_secret_string.yml b/compiler/types/yaml/yaml/testdata/step_secret_string.yml new file mode 100644 index 000000000..930977980 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/step_secret_string.yml @@ -0,0 +1,2 @@ +--- +[ foo, hello ] diff --git a/compiler/types/yaml/yaml/testdata/template.yml b/compiler/types/yaml/yaml/testdata/template.yml new file mode 100644 index 000000000..6d5615e01 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/template.yml @@ -0,0 +1,12 @@ +--- +- name: docker_build + source: github.com/go-vela/atlas/stable/docker_create + type: github +- name: docker_build + source: github.com/go-vela/atlas/stable/docker_build + format: go + type: github +- name: docker_publish + source: github.com/go-vela/atlas/stable/docker_publish + format: starlark + type: github diff --git a/compiler/types/yaml/yaml/testdata/ulimit_colon_error.yml b/compiler/types/yaml/yaml/testdata/ulimit_colon_error.yml new file mode 100644 index 000000000..3e948cb08 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_colon_error.yml @@ -0,0 +1,2 @@ +--- +[ foo=bar:1024:2048 ] diff --git a/compiler/types/yaml/yaml/testdata/ulimit_equal_error.yml b/compiler/types/yaml/yaml/testdata/ulimit_equal_error.yml new file mode 100644 index 000000000..f72b3b461 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_equal_error.yml @@ -0,0 +1,2 @@ +--- +[ foo=1024=2048 ] diff --git a/compiler/types/yaml/yaml/testdata/ulimit_hardlimit1_error.yml b/compiler/types/yaml/yaml/testdata/ulimit_hardlimit1_error.yml new file mode 100644 index 000000000..1472c22b7 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_hardlimit1_error.yml @@ -0,0 +1,2 @@ +--- +[ foo=bar:1024 ] diff --git a/compiler/types/yaml/yaml/testdata/ulimit_hardlimit2_error.yml b/compiler/types/yaml/yaml/testdata/ulimit_hardlimit2_error.yml new file mode 100644 index 000000000..4569bc3ad --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_hardlimit2_error.yml @@ -0,0 +1,2 @@ +--- +[ foo=1024:bar ] diff --git a/compiler/types/yaml/yaml/testdata/ulimit_slice.yml b/compiler/types/yaml/yaml/testdata/ulimit_slice.yml new file mode 100644 index 000000000..9ee862c06 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_slice.yml @@ -0,0 +1,6 @@ +--- +- name: foo + soft: 1024 +- name: bar + soft: 1024 + hard: 2048 diff --git a/compiler/types/yaml/yaml/testdata/ulimit_softlimit_error.yml b/compiler/types/yaml/yaml/testdata/ulimit_softlimit_error.yml new file mode 100644 index 000000000..63f68f1c4 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_softlimit_error.yml @@ -0,0 +1,2 @@ +--- +[ foo=bar ] diff --git a/compiler/types/yaml/yaml/testdata/ulimit_string.yml b/compiler/types/yaml/yaml/testdata/ulimit_string.yml new file mode 100644 index 000000000..59669af36 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/ulimit_string.yml @@ -0,0 +1,2 @@ +--- +[ foo=1024, bar=1024:2048 ] diff --git a/compiler/types/yaml/yaml/testdata/volume_error.yml b/compiler/types/yaml/yaml/testdata/volume_error.yml new file mode 100644 index 000000000..8c36e5057 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/volume_error.yml @@ -0,0 +1,2 @@ +--- +[ /foo:/bar:/foo:bar ] diff --git a/compiler/types/yaml/yaml/testdata/volume_slice.yml b/compiler/types/yaml/yaml/testdata/volume_slice.yml new file mode 100644 index 000000000..fbad0133b --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/volume_slice.yml @@ -0,0 +1,7 @@ +--- +- source: /foo +- source: /foo + destination: /bar +- source: /foo + destination: /foobar + access_mode: ro diff --git a/compiler/types/yaml/yaml/testdata/volume_string.yml b/compiler/types/yaml/yaml/testdata/volume_string.yml new file mode 100644 index 000000000..a596a9116 --- /dev/null +++ b/compiler/types/yaml/yaml/testdata/volume_string.yml @@ -0,0 +1,2 @@ +--- +[ /foo, /foo:/bar, /foo:/foobar:ro ] diff --git a/compiler/types/yaml/ulimit.go b/compiler/types/yaml/yaml/ulimit.go similarity index 100% rename from compiler/types/yaml/ulimit.go rename to compiler/types/yaml/yaml/ulimit.go diff --git a/compiler/types/yaml/yaml/ulimit_test.go b/compiler/types/yaml/yaml/ulimit_test.go new file mode 100644 index 000000000..a8da414ee --- /dev/null +++ b/compiler/types/yaml/yaml/ulimit_test.go @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: Apache-2.0 + +package yaml + +import ( + "os" + "reflect" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/go-vela/server/compiler/types/pipeline" +) + +func TestYaml_UlimitSlice_ToPipeline(t *testing.T) { + // setup tests + tests := []struct { + ulimits *UlimitSlice + want *pipeline.UlimitSlice + }{ + { + ulimits: &UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + want: &pipeline.UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 2048, + }, + }, + }, + } + + // run tests + for _, test := range tests { + got := test.ulimits.ToPipeline() + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("ToPipeline is %v, want %v", got, test.want) + } + } +} + +func TestYaml_UlimitSlice_UnmarshalYAML(t *testing.T) { + // setup tests + tests := []struct { + failure bool + file string + want *UlimitSlice + }{ + { + failure: false, + file: "testdata/ulimit_slice.yml", + want: &UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 1024, + }, + { + Name: "bar", + Soft: 1024, + Hard: 2048, + }, + }, + }, + { + failure: false, + file: "testdata/ulimit_string.yml", + want: &UlimitSlice{ + { + Name: "foo", + Soft: 1024, + Hard: 1024, + }, + { + Name: "bar", + Soft: 1024, + Hard: 2048, + }, + }, + }, + { + failure: true, + file: "testdata/invalid.yml", + want: nil, + }, + { + failure: true, + file: "testdata/ulimit_equal_error.yml", + want: nil, + }, + { + failure: true, + file: "testdata/ulimit_colon_error.yml", + want: nil, + }, + { + failure: true, + file: "testdata/ulimit_softlimit_error.yml", + want: nil, + }, + { + failure: true, + file: "testdata/ulimit_hardlimit1_error.yml", + want: nil, + }, + { + failure: true, + file: "testdata/ulimit_hardlimit2_error.yml", + want: nil, + }, + } + + // run tests + for _, test := range tests { + got := new(UlimitSlice) + + b, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + err = yaml.Unmarshal(b, got) + + if test.failure { + if err == nil { + t.Errorf("UnmarshalYAML should have returned err") + } + + continue + } + + if err != nil { + t.Errorf("UnmarshalYAML returned err: %v", err) + } + + if !reflect.DeepEqual(got, test.want) { + t.Errorf("UnmarshalYAML is %v, want %v", got, test.want) + } + } +} diff --git a/compiler/types/yaml/volume.go b/compiler/types/yaml/yaml/volume.go similarity index 100% rename from compiler/types/yaml/volume.go rename to compiler/types/yaml/yaml/volume.go diff --git a/compiler/types/yaml/volume_test.go b/compiler/types/yaml/yaml/volume_test.go similarity index 100% rename from compiler/types/yaml/volume_test.go rename to compiler/types/yaml/yaml/volume_test.go diff --git a/compiler/types/yaml/worker.go b/compiler/types/yaml/yaml/worker.go similarity index 100% rename from compiler/types/yaml/worker.go rename to compiler/types/yaml/yaml/worker.go diff --git a/compiler/types/yaml/worker_test.go b/compiler/types/yaml/yaml/worker_test.go similarity index 100% rename from compiler/types/yaml/worker_test.go rename to compiler/types/yaml/yaml/worker_test.go diff --git a/go.mod b/go.mod index 0c68f970b..81d20dba5 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( golang.org/x/oauth2 v0.23.0 golang.org/x/sync v0.8.0 golang.org/x/time v0.6.0 + gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/postgres v1.5.9 gorm.io/driver/sqlite v1.5.6 gorm.io/gorm v1.25.12 @@ -153,7 +154,6 @@ require ( google.golang.org/grpc v1.66.1 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect ) diff --git a/internal/testdata/buildkite.yml b/internal/testdata/buildkite.yml new file mode 100644 index 000000000..1a74a344b --- /dev/null +++ b/internal/testdata/buildkite.yml @@ -0,0 +1,18 @@ +version: "legacy" + +aliases: + images: + alpine: &alpine-image + image: alpine:latest + + env: + dev-env: &dev-environment + environment: + REGION: dev + +steps: + - name: example + <<: *alpine-image + <<: *dev-environment + commands: + - echo $REGION \ No newline at end of file diff --git a/internal/testdata/go-yaml.yml b/internal/testdata/go-yaml.yml new file mode 100644 index 000000000..7a9d284f5 --- /dev/null +++ b/internal/testdata/go-yaml.yml @@ -0,0 +1,19 @@ +version: "1" + +aliases: + images: + alpine: &alpine-image + image: alpine:latest + + env: + dev-env: &dev-environment + environment: + REGION: dev + +steps: + - name: example + <<: + - *alpine-image + - *dev-environment + commands: + - echo $REGION diff --git a/internal/testdata/invalid.yml b/internal/testdata/invalid.yml new file mode 100644 index 000000000..a1cde504a --- /dev/null +++ b/internal/testdata/invalid.yml @@ -0,0 +1,2 @@ +- sliceNodeA +- sliceNodeB \ No newline at end of file diff --git a/internal/testdata/no_version.yml b/internal/testdata/no_version.yml new file mode 100644 index 000000000..5b59dea2a --- /dev/null +++ b/internal/testdata/no_version.yml @@ -0,0 +1,17 @@ +aliases: + images: + alpine: &alpine-image + image: alpine:latest + + env: + dev-env: &dev-environment + environment: + REGION: dev + +steps: + - name: example + <<: + - *alpine-image + - *dev-environment + commands: + - echo $REGION \ No newline at end of file diff --git a/mock/server/pipeline.go b/mock/server/pipeline.go index f7eabde04..f4d65c596 100644 --- a/mock/server/pipeline.go +++ b/mock/server/pipeline.go @@ -8,11 +8,11 @@ import ( "net/http" "strings" - yml "github.com/buildkite/yaml" "github.com/gin-gonic/gin" + yml "gopkg.in/yaml.v3" api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" ) const ( diff --git a/schema/pipeline.go b/schema/pipeline.go index 0daf45115..165d77d14 100644 --- a/schema/pipeline.go +++ b/schema/pipeline.go @@ -16,7 +16,7 @@ import ( "github.com/invopop/jsonschema" - types "github.com/go-vela/server/compiler/types/yaml" + types "github.com/go-vela/server/compiler/types/yaml/yaml" ) // NewPipelineSchema generates the JSON schema object for a Vela pipeline configuration. diff --git a/scm/github/repo.go b/scm/github/repo.go index d1648caa6..060876e48 100644 --- a/scm/github/repo.go +++ b/scm/github/repo.go @@ -14,7 +14,7 @@ import ( "github.com/sirupsen/logrus" api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" "github.com/go-vela/server/database" ) diff --git a/scm/github/repo_test.go b/scm/github/repo_test.go index 036639674..1a86f089a 100644 --- a/scm/github/repo_test.go +++ b/scm/github/repo_test.go @@ -17,7 +17,7 @@ import ( "github.com/google/go-github/v65/github" api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" ) diff --git a/scm/service.go b/scm/service.go index 7d1d12ec6..191b3d880 100644 --- a/scm/service.go +++ b/scm/service.go @@ -7,7 +7,7 @@ import ( "net/http" api "github.com/go-vela/server/api/types" - "github.com/go-vela/server/compiler/types/yaml" + "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/database" "github.com/go-vela/server/internal" )