From c9f2346c2c854775309b40c61fe0d40c6092cd1d Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Sat, 1 Jun 2019 10:41:51 +0200 Subject: [PATCH 01/41] Supporting multiple yamls in procBuilder --- server/build.go | 4 +- server/hook.go | 2 +- server/hook_test.go | 3 +- server/procBuilder.go | 226 ++++++++++++++++++++---------------------- 4 files changed, 112 insertions(+), 123 deletions(-) diff --git a/server/build.go b/server/build.go index 33161c3c64a..280dd628763 100644 --- a/server/build.go +++ b/server/build.go @@ -323,7 +323,7 @@ func PostApproval(c *gin.Context) { Secs: secs, Regs: regs, Link: httputil.GetURL(c.Request), - Yaml: conf.Data, + Yamls: []string{conf.Data}, Envs: envs, } buildItems, err := b.Build() @@ -512,7 +512,7 @@ func PostBuild(c *gin.Context) { Secs: secs, Regs: regs, Link: httputil.GetURL(c.Request), - Yaml: conf.Data, + Yamls: []string{conf.Data}, Envs: buildParams, } buildItems, err := b.Build() diff --git a/server/hook.go b/server/hook.go index 7604f22d050..09ef1ff2474 100644 --- a/server/hook.go +++ b/server/hook.go @@ -235,7 +235,7 @@ func PostHook(c *gin.Context) { Regs: regs, Envs: envs, Link: httputil.GetURL(c.Request), - Yaml: conf.Data, + Yamls: []string{conf.Data}, } buildItems, err := b.Build() if err != nil { diff --git a/server/hook_test.go b/server/hook_test.go index c34b0070df4..9bf9f5f6a2d 100644 --- a/server/hook_test.go +++ b/server/hook_test.go @@ -32,11 +32,12 @@ bbb`, Secs: []*model.Secret{}, Regs: []*model.Registry{}, Link: "", - Yaml: `pipeline: + Yamls: []string{`pipeline: xxx: image: scratch yyy: ${DRONE_COMMIT_MESSAGE} `, + }, } if _, err := b.Build(); err != nil { diff --git a/server/procBuilder.go b/server/procBuilder.go index f5ddee81375..86cd004d606 100644 --- a/server/procBuilder.go +++ b/server/procBuilder.go @@ -39,7 +39,7 @@ type procBuilder struct { Secs []*model.Secret Regs []*model.Registry Link string - Yaml string + Yamls []string Envs map[string]string } @@ -51,139 +51,127 @@ type buildItem struct { } func (b *procBuilder) Build() ([]*buildItem, error) { - - axes, err := matrix.ParseString(b.Yaml) - if err != nil { - return nil, err - } - if len(axes) == 0 { - axes = append(axes, matrix.Axis{}) - } - var items []*buildItem - for i, axis := range axes { - proc := &model.Proc{ - BuildID: b.Curr.ID, - PID: i + 1, - PGID: i + 1, - State: model.StatusPending, - Environ: axis, - } - metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link) - environ := metadata.Environ() - for k, v := range metadata.EnvironDrone() { - environ[k] = v + for _, y := range b.Yamls { + axes, err := matrix.ParseString(y) + if err != nil { + return nil, err } - for k, v := range axis { - environ[k] = v + if len(axes) == 0 { + axes = append(axes, matrix.Axis{}) } - var secrets []compiler.Secret - for _, sec := range b.Secs { - if !sec.Match(b.Curr.Event) { - continue + for i, axis := range axes { + proc := &model.Proc{ + BuildID: b.Curr.ID, + PID: i + 1, + PGID: i + 1, + State: model.StatusPending, + Environ: axis, } - secrets = append(secrets, compiler.Secret{ - Name: sec.Name, - Value: sec.Value, - Match: sec.Images, - }) - } - y := b.Yaml - s, err := envsubst.Eval(y, func(name string) string { - env := environ[name] - if strings.Contains(env, "\n") { - env = fmt.Sprintf("%q", env) + metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link) + environ := metadata.Environ() + for k, v := range metadata.EnvironDrone() { + environ[k] = v + } + for k, v := range axis { + environ[k] = v } - return env - }) - if err != nil { - return nil, err - } - y = s - - parsed, err := yaml.ParseString(y) - if err != nil { - return nil, err - } - metadata.Sys.Arch = parsed.Platform - if metadata.Sys.Arch == "" { - metadata.Sys.Arch = "linux/amd64" - } - lerr := linter.New( - linter.WithTrusted(b.Repo.IsTrusted), - ).Lint(parsed) - if lerr != nil { - return nil, lerr - } + var secrets []compiler.Secret + for _, sec := range b.Secs { + if !sec.Match(b.Curr.Event) { + continue + } + secrets = append(secrets, compiler.Secret{ + Name: sec.Name, + Value: sec.Value, + Match: sec.Images, + }) + } - var registries []compiler.Registry - for _, reg := range b.Regs { - registries = append(registries, compiler.Registry{ - Hostname: reg.Address, - Username: reg.Username, - Password: reg.Password, - Email: reg.Email, + s, err := envsubst.Eval(y, func(name string) string { + env := environ[name] + if strings.Contains(env, "\n") { + env = fmt.Sprintf("%q", env) + } + return env }) - } + if err != nil { + return nil, err + } + y = s - ir := compiler.New( - compiler.WithEnviron(environ), - compiler.WithEnviron(b.Envs), - compiler.WithEscalated(Config.Pipeline.Privileged...), - compiler.WithResourceLimit(Config.Pipeline.Limits.MemSwapLimit, Config.Pipeline.Limits.MemLimit, Config.Pipeline.Limits.ShmSize, Config.Pipeline.Limits.CPUQuota, Config.Pipeline.Limits.CPUShares, Config.Pipeline.Limits.CPUSet), - compiler.WithVolumes(Config.Pipeline.Volumes...), - compiler.WithNetworks(Config.Pipeline.Networks...), - compiler.WithLocal(false), - compiler.WithOption( - compiler.WithNetrc( - b.Netrc.Login, - b.Netrc.Password, - b.Netrc.Machine, - ), - b.Repo.IsPrivate, - ), - compiler.WithRegistry(registries...), - compiler.WithSecret(secrets...), - compiler.WithPrefix( - fmt.Sprintf( - "%d_%d", - proc.ID, - rand.Int(), - ), - ), - compiler.WithEnviron(proc.Environ), - compiler.WithProxy(), - compiler.WithWorkspaceFromURL("/drone", b.Repo.Link), - compiler.WithMetadata(metadata), - ).Compile(parsed) + parsed, err := yaml.ParseString(y) + if err != nil { + return nil, err + } + metadata.Sys.Arch = parsed.Platform + if metadata.Sys.Arch == "" { + metadata.Sys.Arch = "linux/amd64" + } - // for _, sec := range b.Secs { - // if !sec.MatchEvent(b.Curr.Event) { - // continue - // } - // if b.Curr.Verified || sec.SkipVerify { - // ir.Secrets = append(ir.Secrets, &backend.Secret{ - // Mask: sec.Conceal, - // Name: sec.Name, - // Value: sec.Value, - // }) - // } - // } + lerr := linter.New( + linter.WithTrusted(b.Repo.IsTrusted), + ).Lint(parsed) + if lerr != nil { + return nil, lerr + } - item := &buildItem{ - Proc: proc, - Config: ir, - Labels: parsed.Labels, - Platform: metadata.Sys.Arch, - } - if item.Labels == nil { - item.Labels = map[string]string{} + var registries []compiler.Registry + for _, reg := range b.Regs { + registries = append(registries, compiler.Registry{ + Hostname: reg.Address, + Username: reg.Username, + Password: reg.Password, + Email: reg.Email, + }) + } + + ir := compiler.New( + compiler.WithEnviron(environ), + compiler.WithEnviron(b.Envs), + compiler.WithEscalated(Config.Pipeline.Privileged...), + compiler.WithResourceLimit(Config.Pipeline.Limits.MemSwapLimit, Config.Pipeline.Limits.MemLimit, Config.Pipeline.Limits.ShmSize, Config.Pipeline.Limits.CPUQuota, Config.Pipeline.Limits.CPUShares, Config.Pipeline.Limits.CPUSet), + compiler.WithVolumes(Config.Pipeline.Volumes...), + compiler.WithNetworks(Config.Pipeline.Networks...), + compiler.WithLocal(false), + compiler.WithOption( + compiler.WithNetrc( + b.Netrc.Login, + b.Netrc.Password, + b.Netrc.Machine, + ), + b.Repo.IsPrivate, + ), + compiler.WithRegistry(registries...), + compiler.WithSecret(secrets...), + compiler.WithPrefix( + fmt.Sprintf( + "%d_%d", + proc.ID, + rand.Int(), + ), + ), + compiler.WithEnviron(proc.Environ), + compiler.WithProxy(), + compiler.WithWorkspaceFromURL("/drone", b.Repo.Link), + compiler.WithMetadata(metadata), + ).Compile(parsed) + + item := &buildItem{ + Proc: proc, + Config: ir, + Labels: parsed.Labels, + Platform: metadata.Sys.Arch, + } + if item.Labels == nil { + item.Labels = map[string]string{} + } + items = append(items, item) } - items = append(items, item) } return items, nil From df50b7ef401aa6d925aa67f6bf6a8273656e5854 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Sat, 1 Jun 2019 10:56:12 +0200 Subject: [PATCH 02/41] Testing procBuilder --- server/{hook_test.go => procBuilder_test.go} | 47 +++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) rename server/{hook_test.go => procBuilder_test.go} (53%) diff --git a/server/hook_test.go b/server/procBuilder_test.go similarity index 53% rename from server/hook_test.go rename to server/procBuilder_test.go index 9bf9f5f6a2d..a8f1c755162 100644 --- a/server/hook_test.go +++ b/server/procBuilder_test.go @@ -15,6 +15,7 @@ package server import ( + "fmt" "testing" "github.com/laszlocph/drone-oss-08/model" @@ -36,11 +37,55 @@ bbb`, xxx: image: scratch yyy: ${DRONE_COMMIT_MESSAGE} +`, `pipeline: + build: + image: scratch + yyy: ${DRONE_COMMIT_MESSAGE} `, }, } - if _, err := b.Build(); err != nil { + if buildItems, err := b.Build(); err != nil { t.Fatal(err) + } else { + fmt.Println(buildItems) + build := &model.Build{} + setBuildProcs(build, buildItems) + fmt.Println(build) } } + +func TestMultiPipeline(t *testing.T) { + b := procBuilder{ + Repo: &model.Repo{}, + Curr: &model.Build{}, + Last: &model.Build{}, + Netrc: &model.Netrc{}, + Secs: []*model.Secret{}, + Regs: []*model.Registry{}, + Link: "", + Yamls: []string{`pipeline: + lint: + image: scratch + yyy: ${DRONE_COMMIT_MESSAGE} +`, `pipeline: + test: + image: scratch + yyy: ${DRONE_COMMIT_MESSAGE} +`, + }, + } + + buildItems, err := b.Build() + if err != nil { + t.Fatal(err) + } + if len(buildItems) != 2 { + t.Fatal("Should have generated 2 buildItems") + } + + // fmt.Println(buildItems) + // build := &model.Build{} + // setBuildProcs(build, buildItems) + // fmt.Println(build) +} From 67cdbd250994be8b72f8587ca2c3c8efa962846e Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Mon, 3 Jun 2019 08:49:11 +0200 Subject: [PATCH 03/41] Simplified the interface --- remote/bitbucket/bitbucket.go | 7 +------ remote/bitbucketserver/bitbucketserver.go | 6 ------ remote/coding/coding.go | 10 ---------- remote/coding/coding_test.go | 5 ----- remote/gerrit/gerrit.go | 5 ----- remote/gitea/gitea.go | 5 ----- remote/github/github.go | 7 +------ remote/gitlab/gitlab.go | 11 +++-------- remote/gitlab3/gitlab.go | 15 --------------- remote/gogs/gogs.go | 7 +------ remote/mock/remote.go | 23 ----------------------- remote/remote.go | 17 ----------------- 12 files changed, 6 insertions(+), 112 deletions(-) diff --git a/remote/bitbucket/bitbucket.go b/remote/bitbucket/bitbucket.go index ebef2f01151..5a61eab631c 100644 --- a/remote/bitbucket/bitbucket.go +++ b/remote/bitbucket/bitbucket.go @@ -202,12 +202,7 @@ func (c *config) Perm(u *model.User, owner, name string) (*model.Perm, error) { // File fetches the file from the Bitbucket repository and returns its contents. func (c *config) File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) { - return c.FileRef(u, r, b.Commit, f) -} - -// FileRef fetches the file from the Bitbucket repository and returns its contents. -func (c *config) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { - config, err := c.newClient(u).FindSource(r.Owner, r.Name, ref, f) + config, err := c.newClient(u).FindSource(r.Owner, r.Name, b.Commit, f) if err != nil { return nil, err } diff --git a/remote/bitbucketserver/bitbucketserver.go b/remote/bitbucketserver/bitbucketserver.go index 3fe2cd3d239..0b394821f86 100644 --- a/remote/bitbucketserver/bitbucketserver.go +++ b/remote/bitbucketserver/bitbucketserver.go @@ -179,12 +179,6 @@ func (c *Config) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return client.FindFileForRepo(r.Owner, r.Name, f, b.Ref) } -func (c *Config) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { - client := internal.NewClientWithToken(c.URL, c.Consumer, u.Token) - - return client.FindFileForRepo(r.Owner, r.Name, f, ref) -} - // Status is not supported by the bitbucketserver driver. func (c *Config) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { status := internal.BuildStatus{ diff --git a/remote/coding/coding.go b/remote/coding/coding.go index 115f944983c..1d2229b132e 100644 --- a/remote/coding/coding.go +++ b/remote/coding/coding.go @@ -238,16 +238,6 @@ func (c *Coding) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return data, nil } -// FileRef fetches a file from the remote repository for the given ref -// and returns in string format. -func (c *Coding) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { - data, err := c.newClient(u).GetFile(r.Owner, r.Name, ref, f) - if err != nil { - return nil, err - } - return data, nil -} - // Status sends the commit status to the remote system. func (c *Coding) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { // EMPTY: not implemented in Coding OAuth API diff --git a/remote/coding/coding_test.go b/remote/coding/coding_test.go index bf17180cf32..f24f34d3362 100644 --- a/remote/coding/coding_test.go +++ b/remote/coding/coding_test.go @@ -184,11 +184,6 @@ func Test_coding(t *testing.T) { g.Assert(err == nil).IsTrue() g.Assert(string(data)).Equal("pipeline:\n test:\n image: golang:1.6\n commands:\n - go test\n") }) - g.It("Should return file for specified ref", func() { - data, err := c.FileRef(fakeUser, fakeRepo, "master", ".drone.yml") - g.Assert(err == nil).IsTrue() - g.Assert(string(data)).Equal("pipeline:\n test:\n image: golang:1.6\n commands:\n - go test\n") - }) }) g.Describe("When requesting a netrc config", func() { diff --git a/remote/gerrit/gerrit.go b/remote/gerrit/gerrit.go index f0fc91da586..c8f92610861 100644 --- a/remote/gerrit/gerrit.go +++ b/remote/gerrit/gerrit.go @@ -103,11 +103,6 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return nil, nil } -// File is not supported by the Gerrit driver. -func (c *client) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { - return nil, nil -} - // Status is not supported by the Gogs driver. func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { return nil diff --git a/remote/gitea/gitea.go b/remote/gitea/gitea.go index f1cfe4c60ee..39a5105d6e4 100644 --- a/remote/gitea/gitea.go +++ b/remote/gitea/gitea.go @@ -249,11 +249,6 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return cfg, err } -// FileRef fetches the file from the Gitea repository and returns its contents. -func (c *client) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { - return c.newClientToken(u.Token).GetFile(r.Owner, r.Name, ref, f) -} - // Status is supported by the Gitea driver. func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { client := c.newClientToken(u.Token) diff --git a/remote/github/github.go b/remote/github/github.go index 5c142e71b7f..1fd05789287 100644 --- a/remote/github/github.go +++ b/remote/github/github.go @@ -225,15 +225,10 @@ func (c *client) Perm(u *model.User, owner, name string) (*model.Perm, error) { // File fetches the file from the GitHub repository and returns its contents. func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) { - return c.FileRef(u, r, b.Commit, f) -} - -// FileRef fetches the file from the GitHub repository and returns its contents. -func (c *client) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { client := c.newClientToken(u.Token) opts := new(github.RepositoryContentGetOptions) - opts.Ref = ref + opts.Ref = b.Commit data, _, _, err := client.Repositories.GetContents(r.Owner, r.Name, f, opts) if err != nil { return nil, err diff --git a/remote/gitlab/gitlab.go b/remote/gitlab/gitlab.go index 318c83428e6..e61776b7951 100644 --- a/remote/gitlab/gitlab.go +++ b/remote/gitlab/gitlab.go @@ -325,18 +325,13 @@ func (g *Gitlab) Perm(u *model.User, owner, name string) (*model.Perm, error) { // File fetches a file from the remote repository and returns in string format. func (g *Gitlab) File(user *model.User, repo *model.Repo, build *model.Build, f string) ([]byte, error) { - return g.FileRef(user, repo, build.Commit, f) -} - -// FileRef fetches the file from the GitHub repository and returns its contents. -func (g *Gitlab) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { - var client = NewClient(g.URL, u.Token, g.SkipVerify) - id, err := GetProjectId(g, client, r.Owner, r.Name) + var client = NewClient(g.URL, user.Token, g.SkipVerify) + id, err := GetProjectId(g, client, repo.Owner, repo.Name) if err != nil { return nil, err } - out, err := client.RepoRawFileRef(id, ref, f) + out, err := client.RepoRawFileRef(id, build.Commit, f) if err != nil { return nil, err } diff --git a/remote/gitlab3/gitlab.go b/remote/gitlab3/gitlab.go index cd7eb1f2554..7a0dde3d88d 100644 --- a/remote/gitlab3/gitlab.go +++ b/remote/gitlab3/gitlab.go @@ -338,21 +338,6 @@ func (g *Gitlab) File(user *model.User, repo *model.Repo, build *model.Build, f return out, err } -// FileRef fetches the file from the GitHub repository and returns its contents. -func (g *Gitlab) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { - var client = NewClient(g.URL, u.Token, g.SkipVerify) - id, err := GetProjectId(g, client, r.Owner, r.Name) - if err != nil { - return nil, err - } - - out, err := client.RepoRawFileRef(id, ref, f) - if err != nil { - return nil, err - } - return out, err -} - // NOTE Currently gitlab doesn't support status for commits and events, // also if we want get MR status in gitlab we need implement a special plugin for gitlab, // gitlab uses API to fetch build status on client side. But for now we skip this. diff --git a/remote/gogs/gogs.go b/remote/gogs/gogs.go index 285c8079537..13072c0caff 100644 --- a/remote/gogs/gogs.go +++ b/remote/gogs/gogs.go @@ -22,9 +22,9 @@ import ( "net/url" "strings" + "github.com/gogits/go-gogs-client" "github.com/laszlocph/drone-oss-08/model" "github.com/laszlocph/drone-oss-08/remote" - "github.com/gogits/go-gogs-client" ) // Opts defines configuration options. @@ -202,11 +202,6 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return cfg, err } -// FileRef fetches the file from the Gogs repository and returns its contents. -func (c *client) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { - return c.newClientToken(u.Token).GetFile(r.Owner, r.Name, ref, f) -} - // Status is not supported by the Gogs driver. func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { return nil diff --git a/remote/mock/remote.go b/remote/mock/remote.go index 96f5fba1657..97577c73f5e 100644 --- a/remote/mock/remote.go +++ b/remote/mock/remote.go @@ -98,29 +98,6 @@ func (_m *Remote) File(u *model.User, r *model.Repo, b *model.Build, f string) ( return r0, r1 } -// FileRef provides a mock function with given fields: u, r, ref, f -func (_m *Remote) FileRef(u *model.User, r *model.Repo, ref string, f string) ([]byte, error) { - ret := _m.Called(u, r, ref, f) - - var r0 []byte - if rf, ok := ret.Get(0).(func(*model.User, *model.Repo, string, string) []byte); ok { - r0 = rf(u, r, ref, f) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(*model.User, *model.Repo, string, string) error); ok { - r1 = rf(u, r, ref, f) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // Hook provides a mock function with given fields: r func (_m *Remote) Hook(r *http.Request) (*model.Repo, *model.Build, error) { ret := _m.Called(r) diff --git a/remote/remote.go b/remote/remote.go index e6dd43339f3..297a77aad48 100644 --- a/remote/remote.go +++ b/remote/remote.go @@ -51,10 +51,6 @@ type Remote interface { // format. File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) - // FileRef fetches a file from the remote repository for the given ref - // and returns in string format. - FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) - // Status sends the commit status to the remote system. // An example would be the GitHub pull request status. Status(u *model.User, r *model.Repo, b *model.Build, link string) error @@ -115,18 +111,6 @@ func Perm(c context.Context, u *model.User, owner, repo string) (*model.Perm, er return FromContext(c).Perm(u, owner, repo) } -// File fetches a file from the remote repository and returns in string format. -func File(c context.Context, u *model.User, r *model.Repo, b *model.Build, f string) (out []byte, err error) { - for i := 0; i < 12; i++ { - out, err = FromContext(c).File(u, r, b, f) - if err == nil { - return - } - time.Sleep(5 * time.Second) - } - return -} - // Status sends the commit status to the remote system. // An example would be the GitHub pull request status. func Status(c context.Context, u *model.User, r *model.Repo, b *model.Build, link string) error { @@ -170,7 +154,6 @@ func Refresh(c context.Context, u *model.User) (bool, error) { } // FileBackoff fetches the file using an exponential backoff. -// TODO replace this with a proper backoff func FileBackoff(remote Remote, u *model.User, r *model.Repo, b *model.Build, f string) (out []byte, err error) { for i := 0; i < 5; i++ { select { From 75d30dea099236371576224a158fd75c9309eb89 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Mon, 3 Jun 2019 09:16:15 +0200 Subject: [PATCH 04/41] Function to fetch a folder from the remote --- remote/bitbucket/bitbucket.go | 4 ++++ remote/bitbucketserver/bitbucketserver.go | 4 ++++ remote/coding/coding.go | 4 ++++ remote/gerrit/gerrit.go | 4 ++++ remote/gitea/gitea.go | 4 ++++ remote/github/github.go | 25 +++++++++++++++++++++++ remote/gitlab/gitlab.go | 4 ++++ remote/gitlab3/gitlab.go | 4 ++++ remote/gogs/gogs.go | 4 ++++ remote/mock/remote.go | 6 ++++++ remote/remote.go | 23 +++++++++++++++++++++ 11 files changed, 86 insertions(+) diff --git a/remote/bitbucket/bitbucket.go b/remote/bitbucket/bitbucket.go index 5a61eab631c..143d2aff132 100644 --- a/remote/bitbucket/bitbucket.go +++ b/remote/bitbucket/bitbucket.go @@ -209,6 +209,10 @@ func (c *config) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return []byte(config.Data), err } +func (c *config) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]*remote.FileMeta, error) { + return nil, fmt.Errorf("Not implemented") +} + // Status creates a build status for the Bitbucket commit. func (c *config) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { status := internal.BuildStatus{ diff --git a/remote/bitbucketserver/bitbucketserver.go b/remote/bitbucketserver/bitbucketserver.go index 0b394821f86..9e14a377ad8 100644 --- a/remote/bitbucketserver/bitbucketserver.go +++ b/remote/bitbucketserver/bitbucketserver.go @@ -179,6 +179,10 @@ func (c *Config) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return client.FindFileForRepo(r.Owner, r.Name, f, b.Ref) } +func (c *Config) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]*remote.FileMeta, error) { + return nil, fmt.Errorf("Not implemented") +} + // Status is not supported by the bitbucketserver driver. func (c *Config) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { status := internal.BuildStatus{ diff --git a/remote/coding/coding.go b/remote/coding/coding.go index 1d2229b132e..e488bc2c91b 100644 --- a/remote/coding/coding.go +++ b/remote/coding/coding.go @@ -238,6 +238,10 @@ func (c *Coding) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return data, nil } +func (c *Coding) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]*remote.FileMeta, error) { + return nil, fmt.Errorf("Not implemented") +} + // Status sends the commit status to the remote system. func (c *Coding) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { // EMPTY: not implemented in Coding OAuth API diff --git a/remote/gerrit/gerrit.go b/remote/gerrit/gerrit.go index c8f92610861..a78b9d53011 100644 --- a/remote/gerrit/gerrit.go +++ b/remote/gerrit/gerrit.go @@ -103,6 +103,10 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return nil, nil } +func (c *client) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]*remote.FileMeta, error) { + return nil, fmt.Errorf("Not implemented") +} + // Status is not supported by the Gogs driver. func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { return nil diff --git a/remote/gitea/gitea.go b/remote/gitea/gitea.go index 39a5105d6e4..9efaca9e29d 100644 --- a/remote/gitea/gitea.go +++ b/remote/gitea/gitea.go @@ -249,6 +249,10 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return cfg, err } +func (c *client) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]*remote.FileMeta, error) { + return nil, fmt.Errorf("Not implemented") +} + // Status is supported by the Gitea driver. func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { client := c.newClientToken(u.Token) diff --git a/remote/github/github.go b/remote/github/github.go index 1fd05789287..355a8accca4 100644 --- a/remote/github/github.go +++ b/remote/github/github.go @@ -236,6 +236,31 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return data.Decode() } +func (c *client) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]*remote.FileMeta, error) { + client := c.newClientToken(u.Token) + + opts := new(github.RepositoryContentGetOptions) + opts.Ref = b.Commit + _, data, _, err := client.Repositories.GetContents(r.Owner, r.Name, f, opts) + if err != nil { + return nil, err + } + + var files []*remote.FileMeta + for _, file := range data { + data, err := file.Decode() + if err != nil { + return nil, err + } + files = append(files, &remote.FileMeta{ + Name: *file.Name, + Data: data, + }) + } + + return files, nil +} + // Netrc returns a netrc file capable of authenticating GitHub requests and // cloning GitHub repositories. The netrc will use the global machine account // when configured. diff --git a/remote/gitlab/gitlab.go b/remote/gitlab/gitlab.go index e61776b7951..a0e60725bfa 100644 --- a/remote/gitlab/gitlab.go +++ b/remote/gitlab/gitlab.go @@ -338,6 +338,10 @@ func (g *Gitlab) File(user *model.User, repo *model.Repo, build *model.Build, f return out, err } +func (c *Gitlab) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]*remote.FileMeta, error) { + return nil, fmt.Errorf("Not implemented") +} + // NOTE Currently gitlab doesn't support status for commits and events, // also if we want get MR status in gitlab we need implement a special plugin for gitlab, // gitlab uses API to fetch build status on client side. But for now we skip this. diff --git a/remote/gitlab3/gitlab.go b/remote/gitlab3/gitlab.go index 7a0dde3d88d..7009271eeae 100644 --- a/remote/gitlab3/gitlab.go +++ b/remote/gitlab3/gitlab.go @@ -338,6 +338,10 @@ func (g *Gitlab) File(user *model.User, repo *model.Repo, build *model.Build, f return out, err } +func (c *Gitlab) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]*remote.FileMeta, error) { + return nil, fmt.Errorf("Not implemented") +} + // NOTE Currently gitlab doesn't support status for commits and events, // also if we want get MR status in gitlab we need implement a special plugin for gitlab, // gitlab uses API to fetch build status on client side. But for now we skip this. diff --git a/remote/gogs/gogs.go b/remote/gogs/gogs.go index 13072c0caff..f6d1aa7cae6 100644 --- a/remote/gogs/gogs.go +++ b/remote/gogs/gogs.go @@ -202,6 +202,10 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return cfg, err } +func (c *client) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]*remote.FileMeta, error) { + return nil, fmt.Errorf("Not implemented") +} + // Status is not supported by the Gogs driver. func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { return nil diff --git a/remote/mock/remote.go b/remote/mock/remote.go index 97577c73f5e..aaf8a744259 100644 --- a/remote/mock/remote.go +++ b/remote/mock/remote.go @@ -15,9 +15,11 @@ package mock import ( + "fmt" "net/http" "github.com/laszlocph/drone-oss-08/model" + "github.com/laszlocph/drone-oss-08/remote" "github.com/stretchr/testify/mock" ) @@ -98,6 +100,10 @@ func (_m *Remote) File(u *model.User, r *model.Repo, b *model.Build, f string) ( return r0, r1 } +func (c *Remote) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]*remote.FileMeta, error) { + return nil, fmt.Errorf("Not implemented") +} + // Hook provides a mock function with given fields: r func (_m *Remote) Hook(r *http.Request) (*model.Repo, *model.Build, error) { ret := _m.Called(r) diff --git a/remote/remote.go b/remote/remote.go index 297a77aad48..070882d6ef4 100644 --- a/remote/remote.go +++ b/remote/remote.go @@ -51,6 +51,9 @@ type Remote interface { // format. File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) + // Dir fetches a folder from the remote repository + Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]*FileMeta, error) + // Status sends the commit status to the remote system. // An example would be the GitHub pull request status. Status(u *model.User, r *model.Repo, b *model.Build, link string) error @@ -71,6 +74,12 @@ type Remote interface { Hook(r *http.Request) (*model.Repo, *model.Build, error) } +// FileMeta represents a file in version control +type FileMeta struct { + Name string + Data []byte +} + // Refresher refreshes an oauth token and expiration for the given user. It // returns true if the token was refreshed, false if the token was not refreshed, // and error if it failed to refersh. @@ -166,3 +175,17 @@ func FileBackoff(remote Remote, u *model.User, r *model.Repo, b *model.Build, f } return } + +// DirBackoff fetches the folder using an exponential backoff. +func DirBackoff(remote Remote, u *model.User, r *model.Repo, b *model.Build, f string) (out []*FileMeta, err error) { + for i := 0; i < 5; i++ { + select { + case <-time.After(time.Second * time.Duration(i)): + out, err = remote.Dir(u, r, b, f) + if err == nil { + return + } + } + } + return +} From 5cc4dca56fa2e186ae16020033c99f2970fb70df Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Tue, 4 Jun 2019 15:04:18 +0200 Subject: [PATCH 05/41] Fetching multiple pipelines from Github, skipping some logic for now --- model/config.go | 4 +-- remote/github/github.go | 49 ++++++++++++++++++++++++++------ remote/remote.go | 29 ------------------- server/configFetcher.go | 37 ++++++++++++++++++++++++ server/configFetcher_test.go | 22 +++++++++++++++ server/hook.go | 54 ++++++++++++++++++++---------------- 6 files changed, 131 insertions(+), 64 deletions(-) create mode 100644 server/configFetcher.go create mode 100644 server/configFetcher_test.go diff --git a/model/config.go b/model/config.go index 8eae6e34c19..e81a9724220 100644 --- a/model/config.go +++ b/model/config.go @@ -16,8 +16,8 @@ package model // ConfigStore persists pipeline configuration to storage. type ConfigStore interface { - ConfigLoad(int64) (*Config, error) - ConfigFind(*Repo, string) (*Config, error) + ConfigLoad(ID int64) (*Config, error) + ConfigFind(repo *Repo, sha string) (*Config, error) ConfigFindApproved(*Config) (bool, error) ConfigCreate(*Config) error } diff --git a/remote/github/github.go b/remote/github/github.go index 355a8accca4..694cf9535ea 100644 --- a/remote/github/github.go +++ b/remote/github/github.go @@ -23,6 +23,7 @@ import ( "regexp" "strconv" "strings" + "sync" "github.com/laszlocph/drone-oss-08/model" "github.com/laszlocph/drone-oss-08/remote" @@ -233,6 +234,9 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ if err != nil { return nil, err } + if data == nil { + return nil, fmt.Errorf("%s is a folder not a file use Dir(..)", f) + } return data.Decode() } @@ -246,18 +250,45 @@ func (c *client) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([] return nil, err } - var files []*remote.FileMeta + fc := make(chan *remote.FileMeta) + errc := make(chan error) + + wg := &sync.WaitGroup{} + wg.Add(len(data)) + for _, file := range data { - data, err := file.Decode() - if err != nil { - return nil, err - } - files = append(files, &remote.FileMeta{ - Name: *file.Name, - Data: data, - }) + go func(path string) { + content, err := c.File(u, r, b, path) + if err != nil { + errc <- err + } + fc <- &remote.FileMeta{ + Name: path, + Data: content, + } + }(f + "/" + *file.Name) } + var files []*remote.FileMeta + var errors []error + + go func() { + for { + select { + case err := <-errc: + errors = append(errors, err) + wg.Done() + case fileMeta := <-fc: + files = append(files, fileMeta) + wg.Done() + } + } + }() + + wg.Wait() + close(fc) + close(errc) + return files, nil } diff --git a/remote/remote.go b/remote/remote.go index 070882d6ef4..7804cd97bdc 100644 --- a/remote/remote.go +++ b/remote/remote.go @@ -18,7 +18,6 @@ package remote import ( "net/http" - "time" "github.com/laszlocph/drone-oss-08/model" @@ -161,31 +160,3 @@ func Refresh(c context.Context, u *model.User) (bool, error) { } return refresher.Refresh(u) } - -// FileBackoff fetches the file using an exponential backoff. -func FileBackoff(remote Remote, u *model.User, r *model.Repo, b *model.Build, f string) (out []byte, err error) { - for i := 0; i < 5; i++ { - select { - case <-time.After(time.Second * time.Duration(i)): - out, err = remote.File(u, r, b, f) - if err == nil { - return - } - } - } - return -} - -// DirBackoff fetches the folder using an exponential backoff. -func DirBackoff(remote Remote, u *model.User, r *model.Repo, b *model.Build, f string) (out []*FileMeta, err error) { - for i := 0; i < 5; i++ { - select { - case <-time.After(time.Second * time.Duration(i)): - out, err = remote.Dir(u, r, b, f) - if err == nil { - return - } - } - } - return -} diff --git a/server/configFetcher.go b/server/configFetcher.go new file mode 100644 index 00000000000..6ca48a4de31 --- /dev/null +++ b/server/configFetcher.go @@ -0,0 +1,37 @@ +package server + +import ( + "time" + + "github.com/laszlocph/drone-oss-08/model" + "github.com/laszlocph/drone-oss-08/remote" +) + +type configFetcher struct { + remote_ remote.Remote + user *model.User + repo *model.Repo + build *model.Build +} + +func (cf *configFetcher) Fetch() ([]*remote.FileMeta, error) { + for i := 0; i < 5; i++ { + select { + case <-time.After(time.Second * time.Duration(i)): + file, err := cf.remote_.File(cf.user, cf.repo, cf.build, cf.repo.Config) // either a file + if err == nil { + return []*remote.FileMeta{&remote.FileMeta{ + Name: cf.repo.Config, + Data: file, + }}, nil + } + + dir, err := cf.remote_.Dir(cf.user, cf.repo, cf.build, cf.repo.Config) // or a folder + if err != nil { + return nil, err + } + return dir, nil + } + } + return []*remote.FileMeta{}, nil +} diff --git a/server/configFetcher_test.go b/server/configFetcher_test.go new file mode 100644 index 00000000000..1ac3f6825af --- /dev/null +++ b/server/configFetcher_test.go @@ -0,0 +1,22 @@ +package server + +import ( + "testing" + + "github.com/laszlocph/drone-oss-08/model" + "github.com/laszlocph/drone-oss-08/remote/github" +) + +func TestFetchGithub(t *testing.T) { + github, err := github.New(github.Opts{URL: "https://github.com"}) + if err != nil { + t.Fatal(err) + } + configFetcher := &configFetcher{ + remote_: github, + user: &model.User{Token: "xxx"}, + repo: &model.Repo{Owner: "laszlocph", Name: "drone-multipipeline", Config: ".drone"}, + build: &model.Build{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"}, + } + configFetcher.Fetch() +} diff --git a/server/hook.go b/server/hook.go index 09ef1ff2474..5a71ab9d484 100644 --- a/server/hook.go +++ b/server/hook.go @@ -33,7 +33,6 @@ import ( "github.com/laszlocph/drone-oss-08/shared/token" "github.com/laszlocph/drone-oss-08/store" - "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml" "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/rpc" "github.com/laszlocph/drone-oss-08/cncd/pubsub" "github.com/laszlocph/drone-oss-08/cncd/queue" @@ -143,35 +142,37 @@ func PostHook(c *gin.Context) { } // fetch the build file from the remote - remoteYamlConfig, err := remote.FileBackoff(remote_, user, repo, build, repo.Config) + configFetcher := &configFetcher{remote_: remote_, user: user, repo: repo, build: build} + remoteYamlConfigs, err := configFetcher.Fetch() if err != nil { logrus.Errorf("error: %s: cannot find %s in %s: %s", repo.FullName, repo.Config, build.Ref, err) c.AbortWithError(404, err) return } - conf, err := findOrPersistPipelineConfig(repo, remoteYamlConfig) - if err != nil { - logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err) - c.AbortWithError(500, err) - return - } - build.ConfigID = conf.ID + // persist the build config for historical correctness, restarts, etc + // conf, err := findOrPersistPipelineConfig(repo, remoteYamlConfig) + // if err != nil { + // logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err) + // c.AbortWithError(500, err) + // return + // } + // build.ConfigID = conf.ID // verify that pipeline can be built at all - parsedPipelineConfig, err := yaml.ParseString(conf.Data) - if err == nil { - if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy { - c.String(200, "Branch does not match restrictions defined in yaml") - return - } - } - - if repo.IsGated { - allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf) - if !allowed { - build.Status = model.StatusBlocked - } - } + // parsedPipelineConfig, err := yaml.ParseString(conf.Data) + // if err == nil { + // if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy { + // c.String(200, "Branch does not match restrictions defined in yaml") + // return + // } + // } + + // if repo.IsGated { + // allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf) + // if !allowed { + // build.Status = model.StatusBlocked + // } + // } // update some build fields build.RepoID = repo.ID @@ -226,6 +227,11 @@ func PostHook(c *gin.Context) { } }() + var yamls []string + for _, y := range remoteYamlConfigs { + yamls = append(yamls, string(y.Data)) + } + b := procBuilder{ Repo: repo, Curr: build, @@ -235,7 +241,7 @@ func PostHook(c *gin.Context) { Regs: regs, Envs: envs, Link: httputil.GetURL(c.Request), - Yamls: []string{conf.Data}, + Yamls: yamls, } buildItems, err := b.Build() if err != nil { From 435083db9e1201c6294e6c9326f9adfdba82a3a7 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Wed, 5 Jun 2019 10:08:49 +0200 Subject: [PATCH 06/41] Handling channel close --- remote/github/github.go | 25 +++++++++++++++---------- server/configFetcher.go | 10 +++++----- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/remote/github/github.go b/remote/github/github.go index 694cf9535ea..bcce1b48c95 100644 --- a/remote/github/github.go +++ b/remote/github/github.go @@ -261,10 +261,11 @@ func (c *client) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([] content, err := c.File(u, r, b, path) if err != nil { errc <- err - } - fc <- &remote.FileMeta{ - Name: path, - Data: content, + } else { + fc <- &remote.FileMeta{ + Name: path, + Data: content, + } } }(f + "/" + *file.Name) } @@ -275,12 +276,16 @@ func (c *client) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([] go func() { for { select { - case err := <-errc: - errors = append(errors, err) - wg.Done() - case fileMeta := <-fc: - files = append(files, fileMeta) - wg.Done() + case err, open := <-errc: + if open { + errors = append(errors, err) + wg.Done() + } + case fileMeta, open := <-fc: + if open { + files = append(files, fileMeta) + wg.Done() + } } } }() diff --git a/server/configFetcher.go b/server/configFetcher.go index 6ca48a4de31..2df321b9bbf 100644 --- a/server/configFetcher.go +++ b/server/configFetcher.go @@ -18,17 +18,17 @@ func (cf *configFetcher) Fetch() ([]*remote.FileMeta, error) { for i := 0; i < 5; i++ { select { case <-time.After(time.Second * time.Duration(i)): - file, err := cf.remote_.File(cf.user, cf.repo, cf.build, cf.repo.Config) // either a file - if err == nil { + file, fileerr := cf.remote_.File(cf.user, cf.repo, cf.build, cf.repo.Config) // either a file + if fileerr == nil { return []*remote.FileMeta{&remote.FileMeta{ Name: cf.repo.Config, Data: file, }}, nil } - dir, err := cf.remote_.Dir(cf.user, cf.repo, cf.build, cf.repo.Config) // or a folder - if err != nil { - return nil, err + dir, direrr := cf.remote_.Dir(cf.user, cf.repo, cf.build, ".drone") // or a folder + if direrr != nil { + return nil, direrr } return dir, nil } From 4323148a857471ab0fcb85c40cb48a7a5abe5534 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Wed, 5 Jun 2019 10:11:01 +0200 Subject: [PATCH 07/41] Semantics --- cncd/pipeline/pipeline/frontend/yaml/matrix/matrix.go | 3 +-- cncd/pipeline/pipeline/frontend/yaml/matrix/matrix_test.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cncd/pipeline/pipeline/frontend/yaml/matrix/matrix.go b/cncd/pipeline/pipeline/frontend/yaml/matrix/matrix.go index 76a0f6bedcc..7177e1a2f8d 100644 --- a/cncd/pipeline/pipeline/frontend/yaml/matrix/matrix.go +++ b/cncd/pipeline/pipeline/frontend/yaml/matrix/matrix.go @@ -40,9 +40,8 @@ func Parse(data []byte) ([]Axis, error) { return nil, err } - // if not a matrix build return an array with just the single axis. if len(matrix) == 0 { - return nil, nil + return []Axis{}, nil } return calc(matrix), nil diff --git a/cncd/pipeline/pipeline/frontend/yaml/matrix/matrix_test.go b/cncd/pipeline/pipeline/frontend/yaml/matrix/matrix_test.go index 0c4eef50aa4..690d710823c 100644 --- a/cncd/pipeline/pipeline/frontend/yaml/matrix/matrix_test.go +++ b/cncd/pipeline/pipeline/frontend/yaml/matrix/matrix_test.go @@ -25,10 +25,10 @@ func TestMatrix(t *testing.T) { g.Assert(len(set)).Equal(24) }) - g.It("Should return nil if no matrix", func() { + g.It("Should return empty array if no matrix", func() { axis, err := ParseString("") g.Assert(err == nil).IsTrue() - g.Assert(axis == nil).IsTrue() + g.Assert(len(axis) == 0).IsTrue() }) g.It("Should return included axis", func() { From dbfbd1523856589a277e591d753838402195e2ec Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Wed, 5 Jun 2019 10:36:16 +0200 Subject: [PATCH 08/41] Moved proc id changes closer to each other --- server/hook.go | 29 ----------------------------- server/procBuilder.go | 35 ++++++++++++++++++++++++++++++++--- server/procBuilder_test.go | 5 ----- 3 files changed, 32 insertions(+), 37 deletions(-) diff --git a/server/hook.go b/server/hook.go index 5a71ab9d484..a15f4a3c5a6 100644 --- a/server/hook.go +++ b/server/hook.go @@ -253,8 +253,6 @@ func PostHook(c *gin.Context) { return } - setBuildProcs(build, buildItems) - err = store.FromContext(c).ProcCreate(build.Procs) if err != nil { logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err) @@ -286,33 +284,6 @@ func findOrPersistPipelineConfig(repo *model.Repo, remoteYamlConfig []byte) (*mo return conf, nil } -func setBuildProcs(build *model.Build, buildItems []*buildItem) { - pcounter := len(buildItems) - for _, item := range buildItems { - build.Procs = append(build.Procs, item.Proc) - item.Proc.BuildID = build.ID - - for _, stage := range item.Config.Stages { - var gid int - for _, step := range stage.Steps { - pcounter++ - if gid == 0 { - gid = pcounter - } - proc := &model.Proc{ - BuildID: build.ID, - Name: step.Alias, - PID: pcounter, - PPID: item.Proc.PID, - PGID: gid, - State: model.StatusPending, - } - build.Procs = append(build.Procs, proc) - } - } - } -} - func publishToTopic(c *gin.Context, build *model.Build, repo *model.Repo) { message := pubsub.Message{ Labels: map[string]string{ diff --git a/server/procBuilder.go b/server/procBuilder.go index 86cd004d606..e12c05df88f 100644 --- a/server/procBuilder.go +++ b/server/procBuilder.go @@ -53,7 +53,7 @@ type buildItem struct { func (b *procBuilder) Build() ([]*buildItem, error) { var items []*buildItem - for _, y := range b.Yamls { + for j, y := range b.Yamls { axes, err := matrix.ParseString(y) if err != nil { return nil, err @@ -65,8 +65,8 @@ func (b *procBuilder) Build() ([]*buildItem, error) { for i, axis := range axes { proc := &model.Proc{ BuildID: b.Curr.ID, - PID: i + 1, - PGID: i + 1, + PID: j + i + 1, + PGID: j + i + 1, State: model.StatusPending, Environ: axis, } @@ -174,9 +174,38 @@ func (b *procBuilder) Build() ([]*buildItem, error) { } } + setBuildProcs(b.Curr, items) + return items, nil } +func setBuildProcs(build *model.Build, buildItems []*buildItem) { + pcounter := len(buildItems) + for _, item := range buildItems { + build.Procs = append(build.Procs, item.Proc) + item.Proc.BuildID = build.ID + + for _, stage := range item.Config.Stages { + var gid int + for _, step := range stage.Steps { + pcounter++ + if gid == 0 { + gid = pcounter + } + proc := &model.Proc{ + BuildID: build.ID, + Name: step.Alias, + PID: pcounter, + PPID: item.Proc.PID, + PGID: gid, + State: model.StatusPending, + } + build.Procs = append(build.Procs, proc) + } + } + } +} + // return the metadata from the cli context. func metadataFromStruct(repo *model.Repo, build, last *model.Build, proc *model.Proc, link string) frontend.Metadata { host := link diff --git a/server/procBuilder_test.go b/server/procBuilder_test.go index a8f1c755162..9ba858ed9c6 100644 --- a/server/procBuilder_test.go +++ b/server/procBuilder_test.go @@ -83,9 +83,4 @@ func TestMultiPipeline(t *testing.T) { if len(buildItems) != 2 { t.Fatal("Should have generated 2 buildItems") } - - // fmt.Println(buildItems) - // build := &model.Build{} - // setBuildProcs(build, buildItems) - // fmt.Println(build) } From 105a0708fe94c889f9840f45dd90e447163ea21c Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Wed, 5 Jun 2019 15:58:27 +0200 Subject: [PATCH 09/41] Refactor --- cncd/pipeline/pipeline/frontend/metadata.go | 7 + docker-compose.yml | 2 +- server/build.go | 3 - server/procBuilder.go | 161 +++++++++++--------- server/procBuilder_test.go | 3 - 5 files changed, 94 insertions(+), 82 deletions(-) diff --git a/cncd/pipeline/pipeline/frontend/metadata.go b/cncd/pipeline/pipeline/frontend/metadata.go index edd4b0b6243..c46791add3a 100644 --- a/cncd/pipeline/pipeline/frontend/metadata.go +++ b/cncd/pipeline/pipeline/frontend/metadata.go @@ -222,3 +222,10 @@ func (m *Metadata) EnvironDrone() map[string]string { } var pullRegexp = regexp.MustCompile("\\d+") + +func (m *Metadata) SetPlatform(platform string) { + if platform == "" { + platform = "linux/amd64" + } + m.Sys.Arch = platform +} diff --git a/docker-compose.yml b/docker-compose.yml index 4a2594fb6d4..e9044364490 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,4 +28,4 @@ services: environment: - DRONE_SERVER=drone-server:9000 - DRONE_SECRET=${DRONE_SECRET} - - DRONE_MAX_PROCS=1 \ No newline at end of file + - DRONE_MAX_PROCS=2 \ No newline at end of file diff --git a/server/build.go b/server/build.go index 280dd628763..ed024938d4f 100644 --- a/server/build.go +++ b/server/build.go @@ -336,7 +336,6 @@ func PostApproval(c *gin.Context) { return } - setBuildProcs(build, buildItems) err = store.FromContext(c).ProcCreate(build.Procs) if err != nil { logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err) @@ -525,8 +524,6 @@ func PostBuild(c *gin.Context) { return } - setBuildProcs(build, buildItems) - err = store.FromContext(c).ProcCreate(build.Procs) if err != nil { logrus.Errorf("cannot restart %s#%d: %s", repo.FullName, build.Number, err) diff --git a/server/procBuilder.go b/server/procBuilder.go index e12c05df88f..7cd70e13afd 100644 --- a/server/procBuilder.go +++ b/server/procBuilder.go @@ -54,6 +54,7 @@ func (b *procBuilder) Build() ([]*buildItem, error) { var items []*buildItem for j, y := range b.Yamls { + // matrix axes axes, err := matrix.ParseString(y) if err != nil { return nil, err @@ -70,49 +71,24 @@ func (b *procBuilder) Build() ([]*buildItem, error) { State: model.StatusPending, Environ: axis, } + b.Curr.Procs = append(b.Curr.Procs, proc) metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link) - environ := metadata.Environ() - for k, v := range metadata.EnvironDrone() { - environ[k] = v - } - for k, v := range axis { - environ[k] = v - } - - var secrets []compiler.Secret - for _, sec := range b.Secs { - if !sec.Match(b.Curr.Event) { - continue - } - secrets = append(secrets, compiler.Secret{ - Name: sec.Name, - Value: sec.Value, - Match: sec.Images, - }) - } + environ := b.environmentVariables(metadata, axis) - s, err := envsubst.Eval(y, func(name string) string { - env := environ[name] - if strings.Contains(env, "\n") { - env = fmt.Sprintf("%q", env) - } - return env - }) + // substitute vars + y, err := b.envsubst_(y, environ) if err != nil { return nil, err } - y = s + // parse yaml pipeline parsed, err := yaml.ParseString(y) if err != nil { return nil, err } - metadata.Sys.Arch = parsed.Platform - if metadata.Sys.Arch == "" { - metadata.Sys.Arch = "linux/amd64" - } + // lint pipeline lerr := linter.New( linter.WithTrusted(b.Repo.IsTrusted), ).Lint(parsed) @@ -120,46 +96,9 @@ func (b *procBuilder) Build() ([]*buildItem, error) { return nil, lerr } - var registries []compiler.Registry - for _, reg := range b.Regs { - registries = append(registries, compiler.Registry{ - Hostname: reg.Address, - Username: reg.Username, - Password: reg.Password, - Email: reg.Email, - }) - } + metadata.SetPlatform(parsed.Platform) - ir := compiler.New( - compiler.WithEnviron(environ), - compiler.WithEnviron(b.Envs), - compiler.WithEscalated(Config.Pipeline.Privileged...), - compiler.WithResourceLimit(Config.Pipeline.Limits.MemSwapLimit, Config.Pipeline.Limits.MemLimit, Config.Pipeline.Limits.ShmSize, Config.Pipeline.Limits.CPUQuota, Config.Pipeline.Limits.CPUShares, Config.Pipeline.Limits.CPUSet), - compiler.WithVolumes(Config.Pipeline.Volumes...), - compiler.WithNetworks(Config.Pipeline.Networks...), - compiler.WithLocal(false), - compiler.WithOption( - compiler.WithNetrc( - b.Netrc.Login, - b.Netrc.Password, - b.Netrc.Machine, - ), - b.Repo.IsPrivate, - ), - compiler.WithRegistry(registries...), - compiler.WithSecret(secrets...), - compiler.WithPrefix( - fmt.Sprintf( - "%d_%d", - proc.ID, - rand.Int(), - ), - ), - compiler.WithEnviron(proc.Environ), - compiler.WithProxy(), - compiler.WithWorkspaceFromURL("/drone", b.Repo.Link), - compiler.WithMetadata(metadata), - ).Compile(parsed) + ir := b.toInternalRepresentation(parsed, environ, metadata, proc.ID) item := &buildItem{ Proc: proc, @@ -174,17 +113,89 @@ func (b *procBuilder) Build() ([]*buildItem, error) { } } - setBuildProcs(b.Curr, items) + setBuildSteps(b.Curr, items) return items, nil } -func setBuildProcs(build *model.Build, buildItems []*buildItem) { +func (b *procBuilder) envsubst_(y string, environ map[string]string) (string, error) { + return envsubst.Eval(y, func(name string) string { + env := environ[name] + if strings.Contains(env, "\n") { + env = fmt.Sprintf("%q", env) + } + return env + }) +} + +func (b *procBuilder) environmentVariables(metadata frontend.Metadata, axis matrix.Axis) map[string]string { + environ := metadata.Environ() + for k, v := range metadata.EnvironDrone() { + environ[k] = v + } + for k, v := range axis { + environ[k] = v + } + return environ +} + +func (b *procBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[string]string, metadata frontend.Metadata, procID int64) *backend.Config { + var secrets []compiler.Secret + for _, sec := range b.Secs { + if !sec.Match(b.Curr.Event) { + continue + } + secrets = append(secrets, compiler.Secret{ + Name: sec.Name, + Value: sec.Value, + Match: sec.Images, + }) + } + + var registries []compiler.Registry + for _, reg := range b.Regs { + registries = append(registries, compiler.Registry{ + Hostname: reg.Address, + Username: reg.Username, + Password: reg.Password, + Email: reg.Email, + }) + } + + return compiler.New( + compiler.WithEnviron(environ), + compiler.WithEnviron(b.Envs), + compiler.WithEscalated(Config.Pipeline.Privileged...), + compiler.WithResourceLimit(Config.Pipeline.Limits.MemSwapLimit, Config.Pipeline.Limits.MemLimit, Config.Pipeline.Limits.ShmSize, Config.Pipeline.Limits.CPUQuota, Config.Pipeline.Limits.CPUShares, Config.Pipeline.Limits.CPUSet), + compiler.WithVolumes(Config.Pipeline.Volumes...), + compiler.WithNetworks(Config.Pipeline.Networks...), + compiler.WithLocal(false), + compiler.WithOption( + compiler.WithNetrc( + b.Netrc.Login, + b.Netrc.Password, + b.Netrc.Machine, + ), + b.Repo.IsPrivate, + ), + compiler.WithRegistry(registries...), + compiler.WithSecret(secrets...), + compiler.WithPrefix( + fmt.Sprintf( + "%d_%d", + procID, + rand.Int(), + ), + ), + compiler.WithProxy(), + compiler.WithWorkspaceFromURL("/drone", b.Repo.Link), + compiler.WithMetadata(metadata), + ).Compile(parsed) +} + +func setBuildSteps(build *model.Build, buildItems []*buildItem) { pcounter := len(buildItems) for _, item := range buildItems { - build.Procs = append(build.Procs, item.Proc) - item.Proc.BuildID = build.ID - for _, stage := range item.Config.Stages { var gid int for _, step := range stage.Steps { diff --git a/server/procBuilder_test.go b/server/procBuilder_test.go index 9ba858ed9c6..aaf00282145 100644 --- a/server/procBuilder_test.go +++ b/server/procBuilder_test.go @@ -49,9 +49,6 @@ bbb`, t.Fatal(err) } else { fmt.Println(buildItems) - build := &model.Build{} - setBuildProcs(build, buildItems) - fmt.Println(build) } } From 3f9356c6a77541668d9b83a8183231c1fed51f92 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Thu, 6 Jun 2019 09:41:42 +0200 Subject: [PATCH 10/41] Doh, this wasn't a proper tree --- model/proc.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/model/proc.go b/model/proc.go index c547725e62d..8740c07371e 100644 --- a/model/proc.go +++ b/model/proc.go @@ -14,6 +14,8 @@ package model +import "fmt" + // ProcStore persists process information to storage. type ProcStore interface { ProcLoad(int64) (*Proc, error) @@ -57,18 +59,24 @@ func (p *Proc) Failing() bool { // Tree creates a process tree from a flat process list. func Tree(procs []*Proc) []*Proc { - var ( - nodes []*Proc - parent *Proc - ) + var nodes []*Proc for _, proc := range procs { if proc.PPID == 0 { nodes = append(nodes, proc) - parent = proc - continue } else { + parent, _ := findNode(nodes, proc.PPID) parent.Children = append(parent.Children, proc) } } return nodes } + +func findNode(nodes []*Proc, pid int) (*Proc, error) { + for _, node := range nodes { + if node.PID == pid { + return node, nil + } + } + + return nil, fmt.Errorf("Corrupt proc structure") +} From 3fe710bbe4eee6b5bf1d83aea1bf3b3932f23441 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Fri, 7 Jun 2019 10:40:16 +0200 Subject: [PATCH 11/41] 1:n relationship for build-config --- .gitignore | 1 + model/config.go | 12 +-- server/build.go | 18 +++- store/datastore/config.go | 8 +- store/datastore/config_test.go | 90 +++++++++++-------- store/datastore/ddl/sqlite/ddl_gen.go | 54 ++++++++--- .../files/019_add_column_config_build_id.sql | 7 ++ .../files/020_add_column_config_name.sql | 7 ++ store/datastore/sql/sqlite/files/config.sql | 8 +- store/datastore/sql/sqlite/sql_gen.go | 8 +- store/store.go | 2 +- 11 files changed, 148 insertions(+), 67 deletions(-) create mode 100644 store/datastore/ddl/sqlite/files/019_add_column_config_build_id.sql create mode 100644 store/datastore/ddl/sqlite/files/020_add_column_config_name.sql diff --git a/.gitignore b/.gitignore index 22527af4ab7..e2cebad11bb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ release/ cli/release/ server/swagger/files/*.json +server/swagger/swagger_gen.go .idea/ diff --git a/model/config.go b/model/config.go index e81a9724220..5707d605d10 100644 --- a/model/config.go +++ b/model/config.go @@ -16,7 +16,7 @@ package model // ConfigStore persists pipeline configuration to storage. type ConfigStore interface { - ConfigLoad(ID int64) (*Config, error) + ConfigLoad(buildID int64) ([]*Config, error) ConfigFind(repo *Repo, sha string) (*Config, error) ConfigFindApproved(*Config) (bool, error) ConfigCreate(*Config) error @@ -24,8 +24,10 @@ type ConfigStore interface { // Config represents a pipeline configuration. type Config struct { - ID int64 `json:"-" meddler:"config_id,pk"` - RepoID int64 `json:"-" meddler:"config_repo_id"` - Data string `json:"data" meddler:"config_data"` - Hash string `json:"hash" meddler:"config_hash"` + ID int64 `json:"-" meddler:"config_id,pk"` + RepoID int64 `json:"-" meddler:"config_repo_id"` + BuildID int64 `json:"-" meddler:"config_build_id"` + Data string `json:"data" meddler:"config_data"` + Hash string `json:"hash" meddler:"config_hash"` + Name string `json:"name" meddler:"config_name"` } diff --git a/server/build.go b/server/build.go index ed024938d4f..6e5082d1132 100644 --- a/server/build.go +++ b/server/build.go @@ -268,7 +268,7 @@ func PostApproval(c *gin.Context) { build.Reviewer = user.Login // fetch the build file from the database - conf, err := Config.Storage.Config.ConfigLoad(build.ConfigID) + configs, err := Config.Storage.Config.ConfigLoad(build.ID) if err != nil { logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err) c.AbortWithError(404, err) @@ -315,6 +315,11 @@ func PostApproval(c *gin.Context) { } }() + var yamls []string + for _, y := range configs { + yamls = append(yamls, string(y.Data)) + } + b := procBuilder{ Repo: repo, Curr: build, @@ -323,7 +328,7 @@ func PostApproval(c *gin.Context) { Secs: secs, Regs: regs, Link: httputil.GetURL(c.Request), - Yamls: []string{conf.Data}, + Yamls: yamls, Envs: envs, } buildItems, err := b.Build() @@ -435,7 +440,7 @@ func PostBuild(c *gin.Context) { } // fetch the .drone.yml file from the database - conf, err := Config.Storage.Config.ConfigLoad(build.ConfigID) + configs, err := Config.Storage.Config.ConfigLoad(build.ID) if err != nil { logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err) c.AbortWithError(404, err) @@ -503,6 +508,11 @@ func PostBuild(c *gin.Context) { } } + var yamls []string + for _, y := range configs { + yamls = append(yamls, string(y.Data)) + } + b := procBuilder{ Repo: repo, Curr: build, @@ -511,7 +521,7 @@ func PostBuild(c *gin.Context) { Secs: secs, Regs: regs, Link: httputil.GetURL(c.Request), - Yamls: []string{conf.Data}, + Yamls: yamls, Envs: buildParams, } buildItems, err := b.Build() diff --git a/store/datastore/config.go b/store/datastore/config.go index 7ac58534733..aea0a13e7b0 100644 --- a/store/datastore/config.go +++ b/store/datastore/config.go @@ -22,11 +22,11 @@ import ( "github.com/russross/meddler" ) -func (db *datastore) ConfigLoad(id int64) (*model.Config, error) { +func (db *datastore) ConfigLoad(buildID int64) ([]*model.Config, error) { stmt := sql.Lookup(db.driver, "config-find-id") - conf := new(model.Config) - err := meddler.QueryRow(db, conf, stmt, id) - return conf, err + var configs = []*model.Config{} + err := meddler.QueryAll(db, &configs, stmt, buildID) + return configs, err } func (db *datastore) ConfigFind(repo *model.Repo, hash string) (*model.Config, error) { diff --git a/store/datastore/config_test.go b/store/datastore/config_test.go index dedef4466da..bc09a629321 100644 --- a/store/datastore/config_test.go +++ b/store/datastore/config_test.go @@ -28,15 +28,18 @@ func TestConfig(t *testing.T) { }() var ( - data = "pipeline: [ { image: golang, commands: [ go build, go test ] } ]" - hash = "8d8647c9aa90d893bfb79dddbe901f03e258588121e5202632f8ae5738590b26" + data = "pipeline: [ { image: golang, commands: [ go build, go test ] } ]" + hash = "8d8647c9aa90d893bfb79dddbe901f03e258588121e5202632f8ae5738590b26" + buildID = int64(1) ) if err := s.ConfigCreate( &model.Config{ - RepoID: 2, - Data: data, - Hash: hash, + RepoID: 2, + BuildID: 1, + Data: data, + Hash: hash, + Name: "default", }, ); err != nil { t.Errorf("Unexpected error: insert config: %s", err) @@ -54,19 +57,25 @@ func TestConfig(t *testing.T) { if got, want := config.RepoID, int64(2); got != want { t.Errorf("Want config repo id %d, got %d", want, got) } + if got, want := config.BuildID, buildID; got != want { + t.Errorf("Want config build id %d, got %d", want, got) + } if got, want := config.Data, data; got != want { t.Errorf("Want config data %s, got %s", want, got) } if got, want := config.Hash, hash; got != want { t.Errorf("Want config hash %s, got %s", want, got) } + if got, want := config.Name, "default"; got != want { + t.Errorf("Want config name %s, got %s", want, got) + } - loaded, err := s.ConfigLoad(config.ID) + loaded, err := s.ConfigLoad(buildID) if err != nil { t.Errorf("Want config by id, got error %q", err) return } - if got, want := loaded.ID, config.ID; got != want { + if got, want := loaded[0].ID, config.ID; got != want { t.Errorf("Want config by id %d, got %d", want, got) } } @@ -89,46 +98,57 @@ func TestConfigApproved(t *testing.T) { s.CreateRepo(repo) var ( - data = "pipeline: [ { image: golang, commands: [ go build, go test ] } ]" - hash = "8d8647c9aa90d893bfb79dddbe901f03e258588121e5202632f8ae5738590b26" - conf = &model.Config{ + data = "pipeline: [ { image: golang, commands: [ go build, go test ] } ]" + hash = "8d8647c9aa90d893bfb79dddbe901f03e258588121e5202632f8ae5738590b26" + buildBlocked = &model.Build{ RepoID: repo.ID, - Data: data, - Hash: hash, + Status: model.StatusBlocked, + Commit: "85f8c029b902ed9400bc600bac301a0aadb144ac", + } + buildPending = &model.Build{ + RepoID: repo.ID, + Status: model.StatusPending, + Commit: "85f8c029b902ed9400bc600bac301a0aadb144ac", + } + buildRunning = &model.Build{ + RepoID: repo.ID, + Status: model.StatusRunning, + Commit: "85f8c029b902ed9400bc600bac301a0aadb144ac", } ) + s.CreateBuild(buildBlocked) + s.CreateBuild(buildPending) + conf := &model.Config{ + RepoID: repo.ID, + BuildID: buildBlocked.ID, + Data: data, + Hash: hash, + } if err := s.ConfigCreate(conf); err != nil { t.Errorf("Unexpected error: insert config: %s", err) return } - s.CreateBuild(&model.Build{ - RepoID: repo.ID, - ConfigID: conf.ID, - Status: model.StatusBlocked, - Commit: "85f8c029b902ed9400bc600bac301a0aadb144ac", - }) - s.CreateBuild(&model.Build{ - RepoID: repo.ID, - ConfigID: conf.ID, - Status: model.StatusPending, - Commit: "85f8c029b902ed9400bc600bac301a0aadb144ac", - }) - - if ok, _ := s.ConfigFindApproved(conf); ok == true { - t.Errorf("Want config not approved, when blocked or pending") + + if approved, err := s.ConfigFindApproved(conf); approved != false || err != nil { + t.Errorf("Want config not approved, when blocked or pending. %v", err) return } - s.CreateBuild(&model.Build{ - RepoID: repo.ID, - ConfigID: conf.ID, - Status: model.StatusRunning, - Commit: "85f8c029b902ed9400bc600bac301a0aadb144ac", - }) + s.CreateBuild(buildRunning) + conf2 := &model.Config{ + RepoID: repo.ID, + BuildID: buildRunning.ID, + Data: data, + Hash: "xxx", + } + if err := s.ConfigCreate(conf2); err != nil { + t.Errorf("Unexpected error: insert config: %s", err) + return + } - if ok, _ := s.ConfigFindApproved(conf); ok == false { - t.Errorf("Want config approved, when running.") + if approved, err := s.ConfigFindApproved(conf2); approved != true || err != nil { + t.Errorf("Want config approved, when running. %v", err) return } } diff --git a/store/datastore/ddl/sqlite/ddl_gen.go b/store/datastore/ddl/sqlite/ddl_gen.go index 3318a935843..136dd6d62ef 100644 --- a/store/datastore/ddl/sqlite/ddl_gen.go +++ b/store/datastore/ddl/sqlite/ddl_gen.go @@ -1,17 +1,3 @@ -// Copyright 2018 Drone.IO Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package sqlite import ( @@ -174,6 +160,22 @@ var migrations = []struct { name: "alter-table-update-file-meta", stmt: alterTableUpdateFileMeta, }, + { + name: "alter-table-add-config-build-id", + stmt: alterTableAddConfigBuildId, + }, + { + name: "update-table-set-config-config-id", + stmt: updateTableSetConfigConfigId, + }, + { + name: "alter-table-add-config-name", + stmt: alterTableAddConfigName, + }, + { + name: "update-table-set-config-name", + stmt: updateTableSetConfigName, + }, } // Migrate performs the database migration. If the migration fails @@ -637,3 +639,27 @@ UPDATE files SET ,file_meta_failed=0 ,file_meta_skipped=0 ` + +// +// 019_add_column_config_build_id.sql +// + +var alterTableAddConfigBuildId = ` +ALTER TABLE config ADD COLUMN config_build_id INTEGER +` + +var updateTableSetConfigConfigId = ` +UPDATE config SET config_build_id = (SELECT builds.build_id FROM builds WHERE builds.build_config_id = config.config_id) +` + +// +// 020_add_column_config_name.sql +// + +var alterTableAddConfigName = ` +ALTER TABLE config ADD COLUMN config_name TEXT +` + +var updateTableSetConfigName = ` +UPDATE config SET config_name = "default" +` diff --git a/store/datastore/ddl/sqlite/files/019_add_column_config_build_id.sql b/store/datastore/ddl/sqlite/files/019_add_column_config_build_id.sql new file mode 100644 index 00000000000..469ea867738 --- /dev/null +++ b/store/datastore/ddl/sqlite/files/019_add_column_config_build_id.sql @@ -0,0 +1,7 @@ +-- name: alter-table-add-config-build-id + +ALTER TABLE config ADD COLUMN config_build_id INTEGER + +-- name: update-table-set-config-config-id + +UPDATE config SET config_build_id = (SELECT builds.build_id FROM builds WHERE builds.build_config_id = config.config_id) diff --git a/store/datastore/ddl/sqlite/files/020_add_column_config_name.sql b/store/datastore/ddl/sqlite/files/020_add_column_config_name.sql new file mode 100644 index 00000000000..4e364a7d88d --- /dev/null +++ b/store/datastore/ddl/sqlite/files/020_add_column_config_name.sql @@ -0,0 +1,7 @@ +-- name: alter-table-add-config-name + +ALTER TABLE config ADD COLUMN config_name TEXT + +-- name: update-table-set-config-name + +UPDATE config SET config_name = "default" \ No newline at end of file diff --git a/store/datastore/sql/sqlite/files/config.sql b/store/datastore/sql/sqlite/files/config.sql index 8f29bd5d167..8a358300a71 100644 --- a/store/datastore/sql/sqlite/files/config.sql +++ b/store/datastore/sql/sqlite/files/config.sql @@ -3,18 +3,22 @@ SELECT config_id ,config_repo_id +,config_build_id ,config_hash ,config_data +,config_name FROM config -WHERE config_id = ? +WHERE config_build_id = ? -- name: config-find-repo-hash SELECT config_id ,config_repo_id +,config_build_id ,config_hash ,config_data +,config_name FROM config WHERE config_repo_id = ? AND config_hash = ? @@ -23,6 +27,6 @@ WHERE config_repo_id = ? SELECT build_id FROM builds WHERE build_repo_id = ? -AND build_config_id = ? +AND build_id in (SELECT config_build_id FROM config WHERE config.config_id = ?) AND build_status NOT IN ('blocked', 'pending') LIMIT 1 diff --git a/store/datastore/sql/sqlite/sql_gen.go b/store/datastore/sql/sqlite/sql_gen.go index e98d2e36e57..c0418612d37 100644 --- a/store/datastore/sql/sqlite/sql_gen.go +++ b/store/datastore/sql/sqlite/sql_gen.go @@ -57,18 +57,22 @@ var configFindId = ` SELECT config_id ,config_repo_id +,config_build_id ,config_hash ,config_data +,config_name FROM config -WHERE config_id = ? +WHERE config_build_id = ? ` var configFindRepoHash = ` SELECT config_id ,config_repo_id +,config_build_id ,config_hash ,config_data +,config_name FROM config WHERE config_repo_id = ? AND config_hash = ? @@ -77,7 +81,7 @@ WHERE config_repo_id = ? var configFindApproved = ` SELECT build_id FROM builds WHERE build_repo_id = ? -AND build_config_id = ? +AND build_id in (SELECT config_build_id FROM config WHERE config.config_id = ?) AND build_status NOT IN ('blocked', 'pending') LIMIT 1 ` diff --git a/store/store.go b/store/store.go index 0ffd47328d5..e1839cb4dff 100644 --- a/store/store.go +++ b/store/store.go @@ -111,7 +111,7 @@ type Store interface { PermDelete(perm *model.Perm) error PermFlush(user *model.User, before int64) error - ConfigLoad(int64) (*model.Config, error) + ConfigLoad(int64) ([]*model.Config, error) ConfigFind(*model.Repo, string) (*model.Config, error) ConfigFindApproved(*model.Config) (bool, error) ConfigCreate(*model.Config) error From 9504b00a40744763462e62f27673de8f991f363a Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Fri, 7 Jun 2019 10:43:42 +0200 Subject: [PATCH 12/41] How to generate sql files --- BUILDING | 17 ++++++++ remote/mock/remote.go | 90 +++++++++++++++++-------------------------- 2 files changed, 52 insertions(+), 55 deletions(-) diff --git a/BUILDING b/BUILDING index d7ef8996e72..3e66d99f788 100644 --- a/BUILDING +++ b/BUILDING @@ -10,3 +10,20 @@ go install github.com/laszlocph/drone-oss-08/cmd/drone-agent go install github.com/laszlocph/drone-oss-08/cmd/drone-server + +--- + +0. To generate SQL files + +go get github.com/vektra/mockery/.../ + +export download_url=$(curl -s https://api.github.com/repos/go-swagger/go-swagger/releases/latest | \ + jq -r '.assets[] | select(.name | contains("'"$(uname | tr '[:upper:]' '[:lower:]')"'_amd64")) | .browser_download_url') +curl -o swagger -L'#' "$download_url" +chmod +x swagger +sudo mv swagger /usr/local/bin + +go get github.com/laszlocph/togo + +go generate + diff --git a/remote/mock/remote.go b/remote/mock/remote.go index aaf8a744259..096a25fad66 100644 --- a/remote/mock/remote.go +++ b/remote/mock/remote.go @@ -1,27 +1,11 @@ -// Copyright 2018 Drone.IO Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mock - -import ( - "fmt" - "net/http" - - "github.com/laszlocph/drone-oss-08/model" - "github.com/laszlocph/drone-oss-08/remote" - "github.com/stretchr/testify/mock" -) +// Code generated by mockery v1.0.0. DO NOT EDIT. + +package mocks + +import http "net/http" +import mock "github.com/stretchr/testify/mock" +import model "github.com/laszlocph/drone-oss-08/model" +import remote "github.com/laszlocph/drone-oss-08/remote" // Remote is an autogenerated mock type for the Remote type type Remote struct { @@ -77,6 +61,29 @@ func (_m *Remote) Deactivate(u *model.User, r *model.Repo, link string) error { return r0 } +// Dir provides a mock function with given fields: u, r, b, f +func (_m *Remote) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]*remote.FileMeta, error) { + ret := _m.Called(u, r, b, f) + + var r0 []*remote.FileMeta + if rf, ok := ret.Get(0).(func(*model.User, *model.Repo, *model.Build, string) []*remote.FileMeta); ok { + r0 = rf(u, r, b, f) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*remote.FileMeta) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*model.User, *model.Repo, *model.Build, string) error); ok { + r1 = rf(u, r, b, f) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // File provides a mock function with given fields: u, r, b, f func (_m *Remote) File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) { ret := _m.Called(u, r, b, f) @@ -100,10 +107,6 @@ func (_m *Remote) File(u *model.User, r *model.Repo, b *model.Build, f string) ( return r0, r1 } -func (c *Remote) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]*remote.FileMeta, error) { - return nil, fmt.Errorf("Not implemented") -} - // Hook provides a mock function with given fields: r func (_m *Remote) Hook(r *http.Request) (*model.Repo, *model.Build, error) { ret := _m.Called(r) @@ -229,15 +232,15 @@ func (_m *Remote) Repo(u *model.User, owner string, repo string) (*model.Repo, e } // Repos provides a mock function with given fields: u -func (_m *Remote) Repos(u *model.User) ([]*model.RepoLite, error) { +func (_m *Remote) Repos(u *model.User) ([]*model.Repo, error) { ret := _m.Called(u) - var r0 []*model.RepoLite - if rf, ok := ret.Get(0).(func(*model.User) []*model.RepoLite); ok { + var r0 []*model.Repo + if rf, ok := ret.Get(0).(func(*model.User) []*model.Repo); ok { r0 = rf(u) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]*model.RepoLite) + r0 = ret.Get(0).([]*model.Repo) } } @@ -265,29 +268,6 @@ func (_m *Remote) Status(u *model.User, r *model.Repo, b *model.Build, link stri return r0 } -// TeamPerm provides a mock function with given fields: u, org -func (_m *Remote) TeamPerm(u *model.User, org string) (*model.Perm, error) { - ret := _m.Called(u, org) - - var r0 *model.Perm - if rf, ok := ret.Get(0).(func(*model.User, string) *model.Perm); ok { - r0 = rf(u, org) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*model.Perm) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(*model.User, string) error); ok { - r1 = rf(u, org) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // Teams provides a mock function with given fields: u func (_m *Remote) Teams(u *model.User) ([]*model.Team, error) { ret := _m.Called(u) From f9c5fcec0dfc6cb60b60fa896dfbdbf863924d84 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Tue, 11 Jun 2019 10:50:50 +0200 Subject: [PATCH 13/41] N:N relationship between build and config. Restarts work --- model/config.go | 22 ++++--- server/build.go | 4 +- server/hook.go | 36 +++++++---- store/datastore/config.go | 10 +++- store/datastore/config_test.go | 59 +++++++++++-------- store/datastore/ddl/sqlite/ddl_gen.go | 24 ++++---- .../files/019_add_column_config_build_id.sql | 7 --- .../files/019_create_table_build_config.sql | 9 +++ store/datastore/sql/sqlite/files/config.sql | 13 ++-- store/datastore/sql/sqlite/sql_gen.go | 13 ++-- store/store.go | 5 +- 11 files changed, 122 insertions(+), 80 deletions(-) delete mode 100644 store/datastore/ddl/sqlite/files/019_add_column_config_build_id.sql create mode 100644 store/datastore/ddl/sqlite/files/019_create_table_build_config.sql diff --git a/model/config.go b/model/config.go index 5707d605d10..05aa587ac65 100644 --- a/model/config.go +++ b/model/config.go @@ -16,18 +16,24 @@ package model // ConfigStore persists pipeline configuration to storage. type ConfigStore interface { - ConfigLoad(buildID int64) ([]*Config, error) - ConfigFind(repo *Repo, sha string) (*Config, error) + ConfigsForBuild(buildID int64) ([]*Config, error) + ConfigFindIdentical(repoID int64, sha string) (*Config, error) ConfigFindApproved(*Config) (bool, error) ConfigCreate(*Config) error + BuildConfigCreate(*BuildConfig) error } // Config represents a pipeline configuration. type Config struct { - ID int64 `json:"-" meddler:"config_id,pk"` - RepoID int64 `json:"-" meddler:"config_repo_id"` - BuildID int64 `json:"-" meddler:"config_build_id"` - Data string `json:"data" meddler:"config_data"` - Hash string `json:"hash" meddler:"config_hash"` - Name string `json:"name" meddler:"config_name"` + ID int64 `json:"-" meddler:"config_id,pk"` + RepoID int64 `json:"-" meddler:"config_repo_id"` + Data string `json:"data" meddler:"config_data"` + Hash string `json:"hash" meddler:"config_hash"` + Name string `json:"name" meddler:"config_name"` +} + +// BuildConfig is the n:n relation between Build and Config +type BuildConfig struct { + ConfigID int64 `json:"-" meddler:"config_id"` + BuildID int64 `json:"-" meddler:"build_id"` } diff --git a/server/build.go b/server/build.go index 6e5082d1132..f45f41740b2 100644 --- a/server/build.go +++ b/server/build.go @@ -268,7 +268,7 @@ func PostApproval(c *gin.Context) { build.Reviewer = user.Login // fetch the build file from the database - configs, err := Config.Storage.Config.ConfigLoad(build.ID) + configs, err := Config.Storage.Config.ConfigsForBuild(build.ID) if err != nil { logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err) c.AbortWithError(404, err) @@ -440,7 +440,7 @@ func PostBuild(c *gin.Context) { } // fetch the .drone.yml file from the database - configs, err := Config.Storage.Config.ConfigLoad(build.ID) + configs, err := Config.Storage.Config.ConfigsForBuild(build.ID) if err != nil { logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err) c.AbortWithError(404, err) diff --git a/server/hook.go b/server/hook.go index a15f4a3c5a6..5b64ce8449e 100644 --- a/server/hook.go +++ b/server/hook.go @@ -149,14 +149,6 @@ func PostHook(c *gin.Context) { c.AbortWithError(404, err) return } - // persist the build config for historical correctness, restarts, etc - // conf, err := findOrPersistPipelineConfig(repo, remoteYamlConfig) - // if err != nil { - // logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err) - // c.AbortWithError(500, err) - // return - // } - // build.ConfigID = conf.ID // verify that pipeline can be built at all // parsedPipelineConfig, err := yaml.ParseString(conf.Data) @@ -186,6 +178,17 @@ func PostHook(c *gin.Context) { return } + // persist the build config for historical correctness, restarts, etc + for _, remoteYamlConfig := range remoteYamlConfigs { + conf, err := findOrPersistPipelineConfig(build, remoteYamlConfig.Data) + fmt.Println(conf) + if err != nil { + logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err) + c.AbortWithError(500, err) + return + } + } + c.JSON(200, build) if build.Status == model.StatusBlocked { @@ -262,25 +265,34 @@ func PostHook(c *gin.Context) { queueBuild(build, repo, buildItems) } -func findOrPersistPipelineConfig(repo *model.Repo, remoteYamlConfig []byte) (*model.Config, error) { +func findOrPersistPipelineConfig(build *model.Build, remoteYamlConfig []byte) (*model.Config, error) { sha := shasum(remoteYamlConfig) - conf, err := Config.Storage.Config.ConfigFind(repo, sha) + conf, err := Config.Storage.Config.ConfigFindIdentical(build.RepoID, sha) if err != nil { conf = &model.Config{ - RepoID: repo.ID, + RepoID: build.RepoID, Data: string(remoteYamlConfig), Hash: sha, } err = Config.Storage.Config.ConfigCreate(conf) if err != nil { // retry in case we receive two hooks at the same time - conf, err = Config.Storage.Config.ConfigFind(repo, sha) + conf, err = Config.Storage.Config.ConfigFindIdentical(build.RepoID, sha) if err != nil { return nil, err } } } + buildConfig := &model.BuildConfig{ + ConfigID: conf.ID, + BuildID: build.ID, + } + err = Config.Storage.Config.BuildConfigCreate(buildConfig) + if err != nil { + return nil, err + } + return conf, nil } diff --git a/store/datastore/config.go b/store/datastore/config.go index aea0a13e7b0..1be819bff4f 100644 --- a/store/datastore/config.go +++ b/store/datastore/config.go @@ -22,17 +22,17 @@ import ( "github.com/russross/meddler" ) -func (db *datastore) ConfigLoad(buildID int64) ([]*model.Config, error) { +func (db *datastore) ConfigsForBuild(buildID int64) ([]*model.Config, error) { stmt := sql.Lookup(db.driver, "config-find-id") var configs = []*model.Config{} err := meddler.QueryAll(db, &configs, stmt, buildID) return configs, err } -func (db *datastore) ConfigFind(repo *model.Repo, hash string) (*model.Config, error) { +func (db *datastore) ConfigFindIdentical(repoID int64, hash string) (*model.Config, error) { stmt := sql.Lookup(db.driver, "config-find-repo-hash") conf := new(model.Config) - err := meddler.QueryRow(db, conf, stmt, repo.ID, hash) + err := meddler.QueryRow(db, conf, stmt, repoID, hash) return conf, err } @@ -51,3 +51,7 @@ func (db *datastore) ConfigFindApproved(config *model.Config) (bool, error) { func (db *datastore) ConfigCreate(config *model.Config) error { return meddler.Insert(db, "config", config) } + +func (db *datastore) BuildConfigCreate(buildConfig *model.BuildConfig) error { + return meddler.Insert(db, "build_config", buildConfig) +} diff --git a/store/datastore/config_test.go b/store/datastore/config_test.go index bc09a629321..52ec1057061 100644 --- a/store/datastore/config_test.go +++ b/store/datastore/config_test.go @@ -33,20 +33,28 @@ func TestConfig(t *testing.T) { buildID = int64(1) ) - if err := s.ConfigCreate( - &model.Config{ - RepoID: 2, - BuildID: 1, - Data: data, - Hash: hash, - Name: "default", + config := &model.Config{ + RepoID: 2, + Data: data, + Hash: hash, + Name: "default", + } + if err := s.ConfigCreate(config); err != nil { + t.Errorf("Unexpected error: insert config: %s", err) + return + } + + if err := s.BuildConfigCreate( + &model.BuildConfig{ + ConfigID: config.ID, + BuildID: buildID, }, ); err != nil { t.Errorf("Unexpected error: insert config: %s", err) return } - config, err := s.ConfigFind(&model.Repo{ID: 2}, hash) + config, err := s.ConfigFindIdentical(int64(2), hash) if err != nil { t.Error(err) return @@ -57,9 +65,6 @@ func TestConfig(t *testing.T) { if got, want := config.RepoID, int64(2); got != want { t.Errorf("Want config repo id %d, got %d", want, got) } - if got, want := config.BuildID, buildID; got != want { - t.Errorf("Want config build id %d, got %d", want, got) - } if got, want := config.Data, data; got != want { t.Errorf("Want config data %s, got %s", want, got) } @@ -70,7 +75,7 @@ func TestConfig(t *testing.T) { t.Errorf("Want config name %s, got %s", want, got) } - loaded, err := s.ConfigLoad(buildID) + loaded, err := s.ConfigsForBuild(buildID) if err != nil { t.Errorf("Want config by id, got error %q", err) return @@ -120,13 +125,17 @@ func TestConfigApproved(t *testing.T) { s.CreateBuild(buildBlocked) s.CreateBuild(buildPending) conf := &model.Config{ - RepoID: repo.ID, - BuildID: buildBlocked.ID, - Data: data, - Hash: hash, + ID: int64(8), + RepoID: repo.ID, + Data: data, + Hash: hash, } - if err := s.ConfigCreate(conf); err != nil { - t.Errorf("Unexpected error: insert config: %s", err) + buildConfig := &model.BuildConfig{ + ConfigID: int64(8), + BuildID: buildBlocked.ID, + } + if err := s.BuildConfigCreate(buildConfig); err != nil { + t.Errorf("Unexpected error: insert build_config: %s", err) return } @@ -137,12 +146,16 @@ func TestConfigApproved(t *testing.T) { s.CreateBuild(buildRunning) conf2 := &model.Config{ - RepoID: repo.ID, - BuildID: buildRunning.ID, - Data: data, - Hash: "xxx", + ID: int64(9), + RepoID: repo.ID, + Data: data, + Hash: "xxx", + } + buildConfig2 := &model.BuildConfig{ + ConfigID: int64(9), + BuildID: buildRunning.ID, } - if err := s.ConfigCreate(conf2); err != nil { + if err := s.BuildConfigCreate(buildConfig2); err != nil { t.Errorf("Unexpected error: insert config: %s", err) return } diff --git a/store/datastore/ddl/sqlite/ddl_gen.go b/store/datastore/ddl/sqlite/ddl_gen.go index 136dd6d62ef..34cb2a6ac37 100644 --- a/store/datastore/ddl/sqlite/ddl_gen.go +++ b/store/datastore/ddl/sqlite/ddl_gen.go @@ -161,12 +161,8 @@ var migrations = []struct { stmt: alterTableUpdateFileMeta, }, { - name: "alter-table-add-config-build-id", - stmt: alterTableAddConfigBuildId, - }, - { - name: "update-table-set-config-config-id", - stmt: updateTableSetConfigConfigId, + name: "create-table-build-config", + stmt: createTableBuildConfig, }, { name: "alter-table-add-config-name", @@ -641,15 +637,17 @@ UPDATE files SET ` // -// 019_add_column_config_build_id.sql +// 019_create_table_build_config.sql // -var alterTableAddConfigBuildId = ` -ALTER TABLE config ADD COLUMN config_build_id INTEGER -` - -var updateTableSetConfigConfigId = ` -UPDATE config SET config_build_id = (SELECT builds.build_id FROM builds WHERE builds.build_config_id = config.config_id) +var createTableBuildConfig = ` +CREATE TABLE IF NOT EXISTS build_config ( + config_id INTEGER NOT NULL +,build_id INTEGER NOT NULL +,PRIMARY KEY (config_id, build_id) +,FOREIGN KEY (config_id) REFERENCES config (config_id) +,FOREIGN KEY (build_id) REFERENCES builds (build_id) +); ` // diff --git a/store/datastore/ddl/sqlite/files/019_add_column_config_build_id.sql b/store/datastore/ddl/sqlite/files/019_add_column_config_build_id.sql deleted file mode 100644 index 469ea867738..00000000000 --- a/store/datastore/ddl/sqlite/files/019_add_column_config_build_id.sql +++ /dev/null @@ -1,7 +0,0 @@ --- name: alter-table-add-config-build-id - -ALTER TABLE config ADD COLUMN config_build_id INTEGER - --- name: update-table-set-config-config-id - -UPDATE config SET config_build_id = (SELECT builds.build_id FROM builds WHERE builds.build_config_id = config.config_id) diff --git a/store/datastore/ddl/sqlite/files/019_create_table_build_config.sql b/store/datastore/ddl/sqlite/files/019_create_table_build_config.sql new file mode 100644 index 00000000000..be5d5871831 --- /dev/null +++ b/store/datastore/ddl/sqlite/files/019_create_table_build_config.sql @@ -0,0 +1,9 @@ +-- name: create-table-build-config + +CREATE TABLE IF NOT EXISTS build_config ( + config_id INTEGER NOT NULL +,build_id INTEGER NOT NULL +,PRIMARY KEY (config_id, build_id) +,FOREIGN KEY (config_id) REFERENCES config (config_id) +,FOREIGN KEY (build_id) REFERENCES builds (build_id) +); diff --git a/store/datastore/sql/sqlite/files/config.sql b/store/datastore/sql/sqlite/files/config.sql index 8a358300a71..64f63e7a07f 100644 --- a/store/datastore/sql/sqlite/files/config.sql +++ b/store/datastore/sql/sqlite/files/config.sql @@ -1,21 +1,20 @@ -- name: config-find-id SELECT - config_id + config.config_id ,config_repo_id -,config_build_id ,config_hash ,config_data ,config_name FROM config -WHERE config_build_id = ? +LEFT JOIN build_config ON config.config_id = build_config.config_id +WHERE build_config.build_id = ? -- name: config-find-repo-hash SELECT config_id ,config_repo_id -,config_build_id ,config_hash ,config_data ,config_name @@ -27,6 +26,10 @@ WHERE config_repo_id = ? SELECT build_id FROM builds WHERE build_repo_id = ? -AND build_id in (SELECT config_build_id FROM config WHERE config.config_id = ?) +AND build_id in ( + SELECT build_id + FROM build_config + WHERE build_config.config_id = ? + ) AND build_status NOT IN ('blocked', 'pending') LIMIT 1 diff --git a/store/datastore/sql/sqlite/sql_gen.go b/store/datastore/sql/sqlite/sql_gen.go index c0418612d37..4d2798f9dd8 100644 --- a/store/datastore/sql/sqlite/sql_gen.go +++ b/store/datastore/sql/sqlite/sql_gen.go @@ -55,21 +55,20 @@ var index = map[string]string{ var configFindId = ` SELECT - config_id + config.config_id ,config_repo_id -,config_build_id ,config_hash ,config_data ,config_name FROM config -WHERE config_build_id = ? +LEFT JOIN build_config ON config.config_id = build_config.config_id +WHERE build_config.build_id = ? ` var configFindRepoHash = ` SELECT config_id ,config_repo_id -,config_build_id ,config_hash ,config_data ,config_name @@ -81,7 +80,11 @@ WHERE config_repo_id = ? var configFindApproved = ` SELECT build_id FROM builds WHERE build_repo_id = ? -AND build_id in (SELECT config_build_id FROM config WHERE config.config_id = ?) +AND build_id in ( + SELECT build_id + FROM build_config + WHERE build_config.config_id = ? + ) AND build_status NOT IN ('blocked', 'pending') LIMIT 1 ` diff --git a/store/store.go b/store/store.go index e1839cb4dff..f0849fa2c75 100644 --- a/store/store.go +++ b/store/store.go @@ -111,10 +111,11 @@ type Store interface { PermDelete(perm *model.Perm) error PermFlush(user *model.User, before int64) error - ConfigLoad(int64) ([]*model.Config, error) - ConfigFind(*model.Repo, string) (*model.Config, error) + ConfigsForBuild(buildID int64) ([]*model.Config, error) + ConfigFindIdentical(repoID int64, sha string) (*model.Config, error) ConfigFindApproved(*model.Config) (bool, error) ConfigCreate(*model.Config) error + BuildConfigCreate(*model.BuildConfig) error SenderFind(*model.Repo, string) (*model.Sender, error) SenderList(*model.Repo) ([]*model.Sender, error) From 71f2486ae0ec7c0c3e6b9f602f6479c96854b207 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Wed, 12 Jun 2019 11:03:59 +0200 Subject: [PATCH 14/41] depend_on in yaml --- cncd/pipeline/pipeline/frontend/yaml/config.go | 1 + cncd/pipeline/pipeline/frontend/yaml/config_test.go | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cncd/pipeline/pipeline/frontend/yaml/config.go b/cncd/pipeline/pipeline/frontend/yaml/config.go index e5d52f8c2a7..281e99908b6 100644 --- a/cncd/pipeline/pipeline/frontend/yaml/config.go +++ b/cncd/pipeline/pipeline/frontend/yaml/config.go @@ -22,6 +22,7 @@ type ( Networks Networks Volumes Volumes Labels libcompose.SliceorMap + DependsOn []string `yaml:"depends_on,omitempty"` } // Workspace defines a pipeline workspace. diff --git a/cncd/pipeline/pipeline/frontend/yaml/config_test.go b/cncd/pipeline/pipeline/frontend/yaml/config_test.go index 54549ffda08..ce9cadfc6b6 100644 --- a/cncd/pipeline/pipeline/frontend/yaml/config_test.go +++ b/cncd/pipeline/pipeline/frontend/yaml/config_test.go @@ -7,7 +7,7 @@ import ( "github.com/franela/goblin" ) -func xTestParse(t *testing.T) { +func TestParse(t *testing.T) { g := goblin.Goblin(t) g.Describe("Parser", func() { @@ -35,9 +35,11 @@ func xTestParse(t *testing.T) { g.Assert(out.Pipeline.Containers[1].Commands).Equal(yaml.Stringorslice{"go build"}) g.Assert(out.Pipeline.Containers[2].Name).Equal("notify") g.Assert(out.Pipeline.Containers[2].Image).Equal("slack") - g.Assert(out.Pipeline.Containers[2].NetworkMode).Equal("container:name") + // g.Assert(out.Pipeline.Containers[2].NetworkMode).Equal("container:name") g.Assert(out.Labels["com.example.team"]).Equal("frontend") g.Assert(out.Labels["com.example.type"]).Equal("build") + g.Assert(out.DependsOn[0]).Equal("lint") + g.Assert(out.DependsOn[1]).Equal("test") }) // Check to make sure variable expansion works in yaml.MapSlice // g.It("Should unmarshal variables", func() { @@ -94,6 +96,9 @@ volumes: labels: com.example.type: "build" com.example.team: "frontend" +depends_on: + - lint + - test ` var sampleVarYaml = ` From a3ec40d4380ab01edfdd71c3e0ea9e7d7c5fee02 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Thu, 13 Jun 2019 17:38:19 +0200 Subject: [PATCH 15/41] Fifo queue with dependencies --- cncd/queue/fifo.go | 52 ++++++++++++++++++++++++++++--- cncd/queue/fifo_test.go | 27 ++++++++++++++++ cncd/queue/queue.go | 8 ++++- model/queue.go | 20 +++++++++++- server/build.go | 29 +++++++++++++++--- server/hook.go | 35 +++++++++++++-------- server/procBuilder.go | 35 +++++++++++++-------- server/procBuilder_test.go | 63 ++++++++++++++++++++++++++++++++------ 8 files changed, 224 insertions(+), 45 deletions(-) diff --git a/cncd/queue/fifo.go b/cncd/queue/fifo.go index ced613d64ad..275baeae0c8 100644 --- a/cncd/queue/fifo.go +++ b/cncd/queue/fifo.go @@ -7,6 +7,8 @@ import ( "runtime" "sync" "time" + + "github.com/Sirupsen/logrus" ) type entry struct { @@ -50,6 +52,19 @@ func (q *fifo) Push(c context.Context, task *Task) error { return nil } +// Push pushes an item to the tail of this queue. +func (q *fifo) PushAtOnce(c context.Context, tasks []*Task) error { + q.Lock() + for _, task := range tasks { + q.pending.PushBack(task) + } + q.Unlock() + for range tasks { + go q.process() + } + return nil +} + // Poll retrieves and removes the head of this queue. func (q *fifo) Poll(c context.Context, f Filter) (*Task, error) { q.Lock() @@ -187,21 +202,48 @@ func (q *fifo) process() { loop: for e := q.pending.Front(); e != nil; e = next { next = e.Next() - item := e.Value.(*Task) + task := e.Value.(*Task) + logrus.Debugf("queue: trying to assign task: %v with deps %v", task.ID, task.Dependencies) + if q.depsInQueue(task) { + continue + } for w := range q.workers { - if w.filter(item) { + if w.filter(task) { delete(q.workers, w) q.pending.Remove(e) - q.running[item.ID] = &entry{ - item: item, + q.running[task.ID] = &entry{ + item: task, done: make(chan bool), deadline: time.Now().Add(q.extension), } - w.channel <- item + logrus.Debugf("queue: assigned task: %v with deps %v", task.ID, task.Dependencies) + w.channel <- task break loop } } } } + +func (q *fifo) depsInQueue(task *Task) bool { + var next *list.Element + for e := q.pending.Front(); e != nil; e = next { + next = e.Next() + possibleDep, ok := e.Value.(*Task) + logrus.Debugf("queue: in queue right now: %v", possibleDep.ID) + for _, dep := range task.Dependencies { + if ok && possibleDep.ID == dep { + return true + } + } + } + for possibleDepID := range q.running { + for _, dep := range task.Dependencies { + if possibleDepID == dep { + return true + } + } + } + return false +} diff --git a/cncd/queue/fifo_test.go b/cncd/queue/fifo_test.go index 5123671bdfc..8d73a0f5666 100644 --- a/cncd/queue/fifo_test.go +++ b/cncd/queue/fifo_test.go @@ -117,3 +117,30 @@ func TestFifoEvict(t *testing.T) { t.Errorf("expect not found error when evicting item not in queue, got %s", err) } } + +func TestFifoDependencies(t *testing.T) { + task1 := &Task{ + ID: "1", + } + + task2 := &Task{ + ID: "2", + Dependencies: []string{"1"}, + } + + q := New().(*fifo) + q.Push(noContext, task2) + q.Push(noContext, task1) + + got, _ := q.Poll(noContext, func(*Task) bool { return true }) + if got != task1 { + t.Errorf("expect task1 returned from queue as task2 depends on it") + return + } + + got, _ = q.Poll(noContext, func(*Task) bool { return true }) + if got != task2 { + t.Errorf("expect task2 returned from queue") + return + } +} diff --git a/cncd/queue/queue.go b/cncd/queue/queue.go index fa25a798a61..2fa4626bddb 100644 --- a/cncd/queue/queue.go +++ b/cncd/queue/queue.go @@ -23,6 +23,9 @@ type Task struct { // Labels represents the key-value pairs the entry is lebeled with. Labels map[string]string `json:"labels,omitempty"` + + // Task IDs this task depend on + Dependencies []string } // InfoT provides runtime information. @@ -44,9 +47,12 @@ type Filter func(*Task) bool // Queue defines a task queue for scheduling tasks among // a pool of workers. type Queue interface { - // Push pushes an task to the tail of this queue. + // Push pushes a task to the tail of this queue. Push(c context.Context, task *Task) error + // Push pushes a task to the tail of this queue. + PushAtOnce(c context.Context, tasks []*Task) error + // Poll retrieves and removes a task head of this queue. Poll(c context.Context, f Filter) (*Task, error) diff --git a/model/queue.go b/model/queue.go index 040728ae35d..433fdcc007e 100644 --- a/model/queue.go +++ b/model/queue.go @@ -54,7 +54,7 @@ type persistentQueue struct { store TaskStore } -// Push pushes an task to the tail of this queue. +// Push pushes a task to the tail of this queue. func (q *persistentQueue) Push(c context.Context, task *queue.Task) error { q.store.TaskInsert(&Task{ ID: task.ID, @@ -68,6 +68,24 @@ func (q *persistentQueue) Push(c context.Context, task *queue.Task) error { return err } +// Push pushes multiple tasks to the tail of this queue. +func (q *persistentQueue) PushAtOnce(c context.Context, tasks []*queue.Task) error { + for _, task := range tasks { + q.store.TaskInsert(&Task{ + ID: task.ID, + Data: task.Data, + Labels: task.Labels, + }) + } + err := q.Queue.PushAtOnce(c, tasks) + if err != nil { + for _, task := range tasks { + q.store.TaskDelete(task.ID) + } + } + return err +} + // Poll retrieves and removes a task head of this queue. func (q *persistentQueue) Poll(c context.Context, f queue.Filter) (*queue.Task, error) { task, err := q.Queue.Poll(c, f) diff --git a/server/build.go b/server/build.go index f45f41740b2..94353930876 100644 --- a/server/build.go +++ b/server/build.go @@ -315,9 +315,9 @@ func PostApproval(c *gin.Context) { } }() - var yamls []string + var yamls []*remote.FileMeta for _, y := range configs { - yamls = append(yamls, string(y.Data)) + yamls = append(yamls, &remote.FileMeta{Data: []byte(y.Data), Name: y.Name}) } b := procBuilder{ @@ -478,6 +478,13 @@ func PostBuild(c *gin.Context) { return } + err = persistBuildConfigs(configs, build.ID) + if err != nil { + logrus.Errorf("failure to persist build config for %s. %s", repo.FullName, err) + c.AbortWithError(500, err) + return + } + // Read query string parameters into buildParams, exclude reserved params var buildParams = map[string]string{} for key, val := range c.Request.URL.Query() { @@ -508,9 +515,9 @@ func PostBuild(c *gin.Context) { } } - var yamls []string + var yamls []*remote.FileMeta for _, y := range configs { - yamls = append(yamls, string(y.Data)) + yamls = append(yamls, &remote.FileMeta{Data: []byte(y.Data), Name: y.Name}) } b := procBuilder{ @@ -589,6 +596,20 @@ func DeleteBuildLogs(c *gin.Context) { c.String(204, "") } +func persistBuildConfigs(configs []*model.Config, buildID int64) error { + for _, conf := range configs { + buildConfig := &model.BuildConfig{ + ConfigID: conf.ID, + BuildID: buildID, + } + err := Config.Storage.Config.BuildConfigCreate(buildConfig) + if err != nil { + return err + } + } + return nil +} + var deleteStr = `[ { "proc": %q, diff --git a/server/hook.go b/server/hook.go index 5b64ce8449e..833408e9465 100644 --- a/server/hook.go +++ b/server/hook.go @@ -180,8 +180,7 @@ func PostHook(c *gin.Context) { // persist the build config for historical correctness, restarts, etc for _, remoteYamlConfig := range remoteYamlConfigs { - conf, err := findOrPersistPipelineConfig(build, remoteYamlConfig.Data) - fmt.Println(conf) + _, err := findOrPersistPipelineConfig(build, remoteYamlConfig) if err != nil { logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err) c.AbortWithError(500, err) @@ -230,11 +229,6 @@ func PostHook(c *gin.Context) { } }() - var yamls []string - for _, y := range remoteYamlConfigs { - yamls = append(yamls, string(y.Data)) - } - b := procBuilder{ Repo: repo, Curr: build, @@ -244,7 +238,7 @@ func PostHook(c *gin.Context) { Regs: regs, Envs: envs, Link: httputil.GetURL(c.Request), - Yamls: yamls, + Yamls: remoteYamlConfigs, } buildItems, err := b.Build() if err != nil { @@ -265,14 +259,15 @@ func PostHook(c *gin.Context) { queueBuild(build, repo, buildItems) } -func findOrPersistPipelineConfig(build *model.Build, remoteYamlConfig []byte) (*model.Config, error) { - sha := shasum(remoteYamlConfig) +func findOrPersistPipelineConfig(build *model.Build, remoteYamlConfig *remote.FileMeta) (*model.Config, error) { + sha := shasum(remoteYamlConfig.Data) conf, err := Config.Storage.Config.ConfigFindIdentical(build.RepoID, sha) if err != nil { conf = &model.Config{ RepoID: build.RepoID, - Data: string(remoteYamlConfig), + Data: string(remoteYamlConfig.Data), Hash: sha, + Name: sanitizePath(remoteYamlConfig.Name), } err = Config.Storage.Config.ConfigCreate(conf) if err != nil { @@ -296,6 +291,7 @@ func findOrPersistPipelineConfig(build *model.Build, remoteYamlConfig []byte) (* return conf, nil } +// publishes message to UI clients func publishToTopic(c *gin.Context, build *model.Build, repo *model.Repo) { message := pubsub.Message{ Labels: map[string]string{ @@ -314,6 +310,7 @@ func publishToTopic(c *gin.Context, build *model.Build, repo *model.Repo) { } func queueBuild(build *model.Build, repo *model.Repo, buildItems []*buildItem) { + var tasks []*queue.Task for _, item := range buildItems { task := new(queue.Task) task.ID = fmt.Sprint(item.Proc.ID) @@ -323,6 +320,7 @@ func queueBuild(build *model.Build, repo *model.Repo, buildItems []*buildItem) { } task.Labels["platform"] = item.Platform task.Labels["repo"] = repo.FullName + task.Dependencies = taskIds(item.DependsOn, buildItems) task.Data, _ = json.Marshal(rpc.Pipeline{ ID: fmt.Sprint(item.Proc.ID), @@ -331,8 +329,21 @@ func queueBuild(build *model.Build, repo *model.Repo, buildItems []*buildItem) { }) Config.Services.Logs.Open(context.Background(), task.ID) - Config.Services.Queue.Push(context.Background(), task) + tasks = append(tasks, task) + } + Config.Services.Queue.PushAtOnce(context.Background(), tasks) +} + +func taskIds(dependsOn []string, buildItems []*buildItem) []string { + taskIds := []string{} + for _, dep := range dependsOn { + for _, buildItem := range buildItems { + if buildItem.Proc.Name == dep { + taskIds = append(taskIds, fmt.Sprint(buildItem.Proc.ID)) + } + } } + return taskIds } func shasum(raw []byte) string { diff --git a/server/procBuilder.go b/server/procBuilder.go index 7cd70e13afd..230656dd694 100644 --- a/server/procBuilder.go +++ b/server/procBuilder.go @@ -28,6 +28,7 @@ import ( "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/linter" "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/matrix" "github.com/laszlocph/drone-oss-08/model" + "github.com/laszlocph/drone-oss-08/remote" ) // Takes the hook data and the yaml and returns in internal data model @@ -39,15 +40,16 @@ type procBuilder struct { Secs []*model.Secret Regs []*model.Registry Link string - Yamls []string + Yamls []*remote.FileMeta Envs map[string]string } type buildItem struct { - Proc *model.Proc - Platform string - Labels map[string]string - Config *backend.Config + Proc *model.Proc + Platform string + Labels map[string]string + DependsOn []string + Config *backend.Config } func (b *procBuilder) Build() ([]*buildItem, error) { @@ -55,7 +57,7 @@ func (b *procBuilder) Build() ([]*buildItem, error) { for j, y := range b.Yamls { // matrix axes - axes, err := matrix.ParseString(y) + axes, err := matrix.ParseString(string(y.Data)) if err != nil { return nil, err } @@ -70,6 +72,7 @@ func (b *procBuilder) Build() ([]*buildItem, error) { PGID: j + i + 1, State: model.StatusPending, Environ: axis, + Name: sanitizePath(y.Name), } b.Curr.Procs = append(b.Curr.Procs, proc) @@ -77,13 +80,13 @@ func (b *procBuilder) Build() ([]*buildItem, error) { environ := b.environmentVariables(metadata, axis) // substitute vars - y, err := b.envsubst_(y, environ) + substituted, err := b.envsubst_(string(y.Data), environ) if err != nil { return nil, err } // parse yaml pipeline - parsed, err := yaml.ParseString(y) + parsed, err := yaml.ParseString(substituted) if err != nil { return nil, err } @@ -101,10 +104,11 @@ func (b *procBuilder) Build() ([]*buildItem, error) { ir := b.toInternalRepresentation(parsed, environ, metadata, proc.ID) item := &buildItem{ - Proc: proc, - Config: ir, - Labels: parsed.Labels, - Platform: metadata.Sys.Arch, + Proc: proc, + Config: ir, + Labels: parsed.Labels, + DependsOn: parsed.DependsOn, + Platform: metadata.Sys.Arch, } if item.Labels == nil { item.Labels = map[string]string{} @@ -289,3 +293,10 @@ func metadataFromStruct(repo *model.Repo, build, last *model.Build, proc *model. }, } } + +func sanitizePath(path string) string { + path = strings.TrimSuffix(path, ".yml") + path = strings.TrimPrefix(path, ".drone/") + path = strings.TrimPrefix(path, ".") + return path +} diff --git a/server/procBuilder_test.go b/server/procBuilder_test.go index aaf00282145..177a960c5bb 100644 --- a/server/procBuilder_test.go +++ b/server/procBuilder_test.go @@ -19,6 +19,7 @@ import ( "testing" "github.com/laszlocph/drone-oss-08/model" + "github.com/laszlocph/drone-oss-08/remote" ) func TestMultilineEnvsubst(t *testing.T) { @@ -33,17 +34,20 @@ bbb`, Secs: []*model.Secret{}, Regs: []*model.Registry{}, Link: "", - Yamls: []string{`pipeline: + Yamls: []*remote.FileMeta{ + &remote.FileMeta{Data: []byte(` +pipeline: xxx: image: scratch yyy: ${DRONE_COMMIT_MESSAGE} -`, `pipeline: +`)}, + &remote.FileMeta{Data: []byte(` +pipeline: build: image: scratch yyy: ${DRONE_COMMIT_MESSAGE} -`, - }, - } +`)}, + }} if buildItems, err := b.Build(); err != nil { t.Fatal(err) @@ -61,15 +65,19 @@ func TestMultiPipeline(t *testing.T) { Secs: []*model.Secret{}, Regs: []*model.Registry{}, Link: "", - Yamls: []string{`pipeline: - lint: + Yamls: []*remote.FileMeta{ + &remote.FileMeta{Data: []byte(` +pipeline: + xxx: image: scratch yyy: ${DRONE_COMMIT_MESSAGE} -`, `pipeline: - test: +`)}, + &remote.FileMeta{Data: []byte(` +pipeline: + build: image: scratch yyy: ${DRONE_COMMIT_MESSAGE} -`, +`)}, }, } @@ -81,3 +89,38 @@ func TestMultiPipeline(t *testing.T) { t.Fatal("Should have generated 2 buildItems") } } + +func TestDependsOn(t *testing.T) { + b := procBuilder{ + Repo: &model.Repo{}, + Curr: &model.Build{}, + Last: &model.Build{}, + Netrc: &model.Netrc{}, + Secs: []*model.Secret{}, + Regs: []*model.Registry{}, + Link: "", + Yamls: []*remote.FileMeta{ + &remote.FileMeta{Data: []byte(` +pipeline: + deploy: + image: scratch + +depends_on: + - lint + - test + - build +`)}, + }, + } + + buildItems, err := b.Build() + if err != nil { + t.Fatal(err) + } + if len(buildItems[0].DependsOn) != 3 { + t.Fatal("Should have 3 dependencies") + } + if buildItems[0].DependsOn[1] != "test" { + t.Fatal("Should depend on test") + } +} From 001d6454711a9645b4a4080ae995327f1e5005c2 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Thu, 13 Jun 2019 18:09:25 +0200 Subject: [PATCH 16/41] Order files alphabetically --- server/configFetcher.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/server/configFetcher.go b/server/configFetcher.go index 2df321b9bbf..2bd98e15843 100644 --- a/server/configFetcher.go +++ b/server/configFetcher.go @@ -5,6 +5,8 @@ import ( "github.com/laszlocph/drone-oss-08/model" "github.com/laszlocph/drone-oss-08/remote" + + "sort" ) type configFetcher struct { @@ -30,8 +32,16 @@ func (cf *configFetcher) Fetch() ([]*remote.FileMeta, error) { if direrr != nil { return nil, direrr } + + sort.Sort(byName(dir)) return dir, nil } } return []*remote.FileMeta{}, nil } + +type byName []*remote.FileMeta + +func (a byName) Len() int { return len(a) } +func (a byName) Less(i, j int) bool { return a[i].Name < a[j].Name } +func (a byName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } From 3f50cafe941087db21dd157fce9020230bc170ea Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Fri, 14 Jun 2019 09:03:59 +0200 Subject: [PATCH 17/41] Fixing test --- cncd/queue/fifo_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cncd/queue/fifo_test.go b/cncd/queue/fifo_test.go index 8d73a0f5666..eb4f9b17417 100644 --- a/cncd/queue/fifo_test.go +++ b/cncd/queue/fifo_test.go @@ -138,6 +138,8 @@ func TestFifoDependencies(t *testing.T) { return } + q.Done(noContext, got.ID) + got, _ = q.Poll(noContext, func(*Task) bool { return true }) if got != task2 { t.Errorf("expect task2 returned from queue") From a18b7bb46eca4933b49f5b23738b43b53f184b65 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Fri, 14 Jun 2019 09:20:46 +0200 Subject: [PATCH 18/41] SQL changes for Postgres and MySQL --- store/datastore/builds_test.go | 2 +- store/datastore/config_test.go | 78 ++++++++++++++----- store/datastore/ddl/mysql/ddl_gen.go | 52 +++++++++---- .../files/019_create_table_build_config.sql | 9 +++ .../files/020_add_column_config_name.sql | 7 ++ store/datastore/ddl/postgres/ddl_gen.go | 74 ++++++++++++------ .../files/019_create_table_build_config.sql | 9 +++ .../files/020_add_column_config_name.sql | 7 ++ store/datastore/ddl/sqlite/ddl_gen.go | 2 +- .../files/020_add_column_config_name.sql | 2 +- store/datastore/sql/mysql/files/config.sql | 13 +++- store/datastore/sql/mysql/sql_gen.go | 13 +++- store/datastore/sql/postgres/files/config.sql | 13 +++- store/datastore/sql/postgres/sql_gen.go | 15 +++- 14 files changed, 224 insertions(+), 72 deletions(-) create mode 100644 store/datastore/ddl/mysql/files/019_create_table_build_config.sql create mode 100644 store/datastore/ddl/mysql/files/020_add_column_config_name.sql create mode 100644 store/datastore/ddl/postgres/files/019_create_table_build_config.sql create mode 100644 store/datastore/ddl/postgres/files/020_add_column_config_name.sql diff --git a/store/datastore/builds_test.go b/store/datastore/builds_test.go index cc8f8f12e0a..452936e1c2b 100644 --- a/store/datastore/builds_test.go +++ b/store/datastore/builds_test.go @@ -18,8 +18,8 @@ import ( "fmt" "testing" - "github.com/laszlocph/drone-oss-08/model" "github.com/franela/goblin" + "github.com/laszlocph/drone-oss-08/model" ) func TestBuilds(t *testing.T) { diff --git a/store/datastore/config_test.go b/store/datastore/config_test.go index 52ec1057061..35f2d2398d4 100644 --- a/store/datastore/config_test.go +++ b/store/datastore/config_test.go @@ -23,18 +23,31 @@ import ( func TestConfig(t *testing.T) { s := newTest() defer func() { + s.Exec("delete from repos") + s.Exec("delete from builds") + s.Exec("delete from procs") s.Exec("delete from config") s.Close() }() var ( - data = "pipeline: [ { image: golang, commands: [ go build, go test ] } ]" - hash = "8d8647c9aa90d893bfb79dddbe901f03e258588121e5202632f8ae5738590b26" - buildID = int64(1) + data = "pipeline: [ { image: golang, commands: [ go build, go test ] } ]" + hash = "8d8647c9aa90d893bfb79dddbe901f03e258588121e5202632f8ae5738590b26" ) + repo := &model.Repo{ + UserID: 1, + FullName: "bradrydzewski/drone", + Owner: "bradrydzewski", + Name: "drone", + } + if err := s.CreateRepo(repo); err != nil { + t.Errorf("Unexpected error: insert repo: %s", err) + return + } + config := &model.Config{ - RepoID: 2, + RepoID: repo.ID, Data: data, Hash: hash, Name: "default", @@ -44,17 +57,27 @@ func TestConfig(t *testing.T) { return } + build := &model.Build{ + RepoID: repo.ID, + Status: model.StatusRunning, + Commit: "85f8c029b902ed9400bc600bac301a0aadb144ac", + } + if err := s.CreateBuild(build); err != nil { + t.Errorf("Unexpected error: insert build: %s", err) + return + } + if err := s.BuildConfigCreate( &model.BuildConfig{ ConfigID: config.ID, - BuildID: buildID, + BuildID: build.ID, }, ); err != nil { - t.Errorf("Unexpected error: insert config: %s", err) + t.Errorf("Unexpected error: insert build config: %s", err) return } - config, err := s.ConfigFindIdentical(int64(2), hash) + config, err := s.ConfigFindIdentical(repo.ID, hash) if err != nil { t.Error(err) return @@ -62,7 +85,7 @@ func TestConfig(t *testing.T) { if got, want := config.ID, int64(1); got != want { t.Errorf("Want config id %d, got %d", want, got) } - if got, want := config.RepoID, int64(2); got != want { + if got, want := config.RepoID, repo.ID; got != want { t.Errorf("Want config repo id %d, got %d", want, got) } if got, want := config.Data, data; got != want { @@ -75,7 +98,7 @@ func TestConfig(t *testing.T) { t.Errorf("Want config name %s, got %s", want, got) } - loaded, err := s.ConfigsForBuild(buildID) + loaded, err := s.ConfigsForBuild(build.ID) if err != nil { t.Errorf("Want config by id, got error %q", err) return @@ -88,9 +111,10 @@ func TestConfig(t *testing.T) { func TestConfigApproved(t *testing.T) { s := newTest() defer func() { - s.Exec("delete from config") - s.Exec("delete from builds") s.Exec("delete from repos") + s.Exec("delete from builds") + s.Exec("delete from procs") + s.Exec("delete from config") s.Close() }() @@ -100,7 +124,10 @@ func TestConfigApproved(t *testing.T) { Owner: "bradrydzewski", Name: "drone", } - s.CreateRepo(repo) + if err := s.CreateRepo(repo); err != nil { + t.Errorf("Unexpected error: insert repo: %s", err) + return + } var ( data = "pipeline: [ { image: golang, commands: [ go build, go test ] } ]" @@ -122,16 +149,25 @@ func TestConfigApproved(t *testing.T) { } ) - s.CreateBuild(buildBlocked) - s.CreateBuild(buildPending) + if err := s.CreateBuild(buildBlocked); err != nil { + t.Errorf("Unexpected error: insert build: %s", err) + return + } + if err := s.CreateBuild(buildPending); err != nil { + t.Errorf("Unexpected error: insert build: %s", err) + return + } conf := &model.Config{ - ID: int64(8), RepoID: repo.ID, Data: data, Hash: hash, } + if err := s.ConfigCreate(conf); err != nil { + t.Errorf("Unexpected error: insert config: %s", err) + return + } buildConfig := &model.BuildConfig{ - ConfigID: int64(8), + ConfigID: conf.ID, BuildID: buildBlocked.ID, } if err := s.BuildConfigCreate(buildConfig); err != nil { @@ -146,13 +182,16 @@ func TestConfigApproved(t *testing.T) { s.CreateBuild(buildRunning) conf2 := &model.Config{ - ID: int64(9), RepoID: repo.ID, Data: data, Hash: "xxx", } + if err := s.ConfigCreate(conf2); err != nil { + t.Errorf("Unexpected error: insert config: %s", err) + return + } buildConfig2 := &model.BuildConfig{ - ConfigID: int64(9), + ConfigID: conf2.ID, BuildID: buildRunning.ID, } if err := s.BuildConfigCreate(buildConfig2); err != nil { @@ -169,6 +208,9 @@ func TestConfigApproved(t *testing.T) { func TestConfigIndexes(t *testing.T) { s := newTest() defer func() { + s.Exec("delete from repos") + s.Exec("delete from builds") + s.Exec("delete from procs") s.Exec("delete from config") s.Close() }() diff --git a/store/datastore/ddl/mysql/ddl_gen.go b/store/datastore/ddl/mysql/ddl_gen.go index b12efe12635..1522076be33 100644 --- a/store/datastore/ddl/mysql/ddl_gen.go +++ b/store/datastore/ddl/mysql/ddl_gen.go @@ -1,17 +1,3 @@ -// Copyright 2018 Drone.IO Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package mysql import ( @@ -170,6 +156,18 @@ var migrations = []struct { name: "alter-table-update-file-meta", stmt: alterTableUpdateFileMeta, }, + { + name: "create-table-build-config", + stmt: createTableBuildConfig, + }, + { + name: "alter-table-add-config-name", + stmt: alterTableAddConfigName, + }, + { + name: "update-table-set-config-name", + stmt: updateTableSetConfigName, + }, } // Migrate performs the database migration. If the migration fails @@ -636,3 +634,29 @@ UPDATE files SET ,file_meta_failed=0 ,file_meta_skipped=0 ` + +// +// 019_create_table_build_config.sql +// + +var createTableBuildConfig = ` +CREATE TABLE IF NOT EXISTS build_config ( + config_id INTEGER NOT NULL +,build_id INTEGER NOT NULL +,PRIMARY KEY (config_id, build_id) +,FOREIGN KEY (config_id) REFERENCES config (config_id) +,FOREIGN KEY (build_id) REFERENCES builds (build_id) +); +` + +// +// 020_add_column_config_name.sql +// + +var alterTableAddConfigName = ` +ALTER TABLE config ADD COLUMN config_name TEXT +` + +var updateTableSetConfigName = ` +UPDATE config SET config_name = "drone" +` diff --git a/store/datastore/ddl/mysql/files/019_create_table_build_config.sql b/store/datastore/ddl/mysql/files/019_create_table_build_config.sql new file mode 100644 index 00000000000..be5d5871831 --- /dev/null +++ b/store/datastore/ddl/mysql/files/019_create_table_build_config.sql @@ -0,0 +1,9 @@ +-- name: create-table-build-config + +CREATE TABLE IF NOT EXISTS build_config ( + config_id INTEGER NOT NULL +,build_id INTEGER NOT NULL +,PRIMARY KEY (config_id, build_id) +,FOREIGN KEY (config_id) REFERENCES config (config_id) +,FOREIGN KEY (build_id) REFERENCES builds (build_id) +); diff --git a/store/datastore/ddl/mysql/files/020_add_column_config_name.sql b/store/datastore/ddl/mysql/files/020_add_column_config_name.sql new file mode 100644 index 00000000000..0140b403538 --- /dev/null +++ b/store/datastore/ddl/mysql/files/020_add_column_config_name.sql @@ -0,0 +1,7 @@ +-- name: alter-table-add-config-name + +ALTER TABLE config ADD COLUMN config_name TEXT + +-- name: update-table-set-config-name + +UPDATE config SET config_name = "drone" \ No newline at end of file diff --git a/store/datastore/ddl/postgres/ddl_gen.go b/store/datastore/ddl/postgres/ddl_gen.go index 3915069faf8..e98fbe40d7b 100644 --- a/store/datastore/ddl/postgres/ddl_gen.go +++ b/store/datastore/ddl/postgres/ddl_gen.go @@ -1,17 +1,3 @@ -// Copyright 2018 Drone.IO Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package postgres import ( @@ -170,6 +156,18 @@ var migrations = []struct { name: "alter-table-update-file-meta", stmt: alterTableUpdateFileMeta, }, + { + name: "create-table-build-config", + stmt: createTableBuildConfig, + }, + { + name: "alter-table-add-config-name", + stmt: alterTableAddConfigName, + }, + { + name: "update-table-set-config-name", + stmt: updateTableSetConfigName, + }, } // Migrate performs the database migration. If the migration fails @@ -530,7 +528,7 @@ CREATE INDEX IF NOT EXISTS sender_repo_ix ON senders (sender_repo_id); // var alterTableAddRepoVisibility = ` -ALTER TABLE repos ADD COLUMN repo_visibility VARCHAR(50) +ALTER TABLE repos ADD COLUMN repo_visibility VARCHAR(50); ` var updateTableSetRepoVisibility = ` @@ -538,7 +536,7 @@ UPDATE repos SET repo_visibility = (CASE WHEN repo_private = false THEN 'public' ELSE 'private' - END) + END); ` // @@ -554,12 +552,13 @@ UPDATE repos SET repo_counter = ( SELECT max(build_number) FROM builds WHERE builds.build_repo_id = repos.repo_id -) +); ` var updateTableSetRepoSeqDefault = ` UPDATE repos SET repo_counter = 0 WHERE repo_counter IS NULL +; ` // @@ -567,11 +566,11 @@ WHERE repo_counter IS NULL // var alterTableAddRepoActive = ` -ALTER TABLE repos ADD COLUMN repo_active BOOLEAN +ALTER TABLE repos ADD COLUMN repo_active BOOLEAN; ` var updateTableSetRepoActive = ` -UPDATE repos SET repo_active = true +UPDATE repos SET repo_active = true; ` // @@ -583,7 +582,7 @@ ALTER TABLE users ADD COLUMN user_synced INTEGER; ` var updateTableSetUserSynced = ` -UPDATE users SET user_synced = 0 +UPDATE users SET user_synced = 0; ` // @@ -615,19 +614,19 @@ CREATE INDEX IF NOT EXISTS ix_perms_user ON perms (perm_user_id); // var alterTableAddFilePid = ` -ALTER TABLE files ADD COLUMN file_pid INTEGER +ALTER TABLE files ADD COLUMN file_pid INTEGER; ` var alterTableAddFileMetaPassed = ` -ALTER TABLE files ADD COLUMN file_meta_passed INTEGER +ALTER TABLE files ADD COLUMN file_meta_passed INTEGER; ` var alterTableAddFileMetaFailed = ` -ALTER TABLE files ADD COLUMN file_meta_failed INTEGER +ALTER TABLE files ADD COLUMN file_meta_failed INTEGER; ` var alterTableAddFileMetaSkipped = ` -ALTER TABLE files ADD COLUMN file_meta_skipped INTEGER +ALTER TABLE files ADD COLUMN file_meta_skipped INTEGER; ` var alterTableUpdateFileMeta = ` @@ -635,4 +634,31 @@ UPDATE files SET file_meta_passed=0 ,file_meta_failed=0 ,file_meta_skipped=0 +; +` + +// +// 019_create_table_build_config.sql +// + +var createTableBuildConfig = ` +CREATE TABLE IF NOT EXISTS build_config ( + config_id INTEGER NOT NULL +,build_id INTEGER NOT NULL +,PRIMARY KEY (config_id, build_id) +,FOREIGN KEY (config_id) REFERENCES config (config_id) +,FOREIGN KEY (build_id) REFERENCES builds (build_id) +); +` + +// +// 020_add_column_config_name.sql +// + +var alterTableAddConfigName = ` +ALTER TABLE config ADD COLUMN config_name TEXT +` + +var updateTableSetConfigName = ` +UPDATE config SET config_name = 'drone' ` diff --git a/store/datastore/ddl/postgres/files/019_create_table_build_config.sql b/store/datastore/ddl/postgres/files/019_create_table_build_config.sql new file mode 100644 index 00000000000..be5d5871831 --- /dev/null +++ b/store/datastore/ddl/postgres/files/019_create_table_build_config.sql @@ -0,0 +1,9 @@ +-- name: create-table-build-config + +CREATE TABLE IF NOT EXISTS build_config ( + config_id INTEGER NOT NULL +,build_id INTEGER NOT NULL +,PRIMARY KEY (config_id, build_id) +,FOREIGN KEY (config_id) REFERENCES config (config_id) +,FOREIGN KEY (build_id) REFERENCES builds (build_id) +); diff --git a/store/datastore/ddl/postgres/files/020_add_column_config_name.sql b/store/datastore/ddl/postgres/files/020_add_column_config_name.sql new file mode 100644 index 00000000000..b1581d11d5d --- /dev/null +++ b/store/datastore/ddl/postgres/files/020_add_column_config_name.sql @@ -0,0 +1,7 @@ +-- name: alter-table-add-config-name + +ALTER TABLE config ADD COLUMN config_name TEXT + +-- name: update-table-set-config-name + +UPDATE config SET config_name = 'drone' \ No newline at end of file diff --git a/store/datastore/ddl/sqlite/ddl_gen.go b/store/datastore/ddl/sqlite/ddl_gen.go index 34cb2a6ac37..b84546b5cc6 100644 --- a/store/datastore/ddl/sqlite/ddl_gen.go +++ b/store/datastore/ddl/sqlite/ddl_gen.go @@ -659,5 +659,5 @@ ALTER TABLE config ADD COLUMN config_name TEXT ` var updateTableSetConfigName = ` -UPDATE config SET config_name = "default" +UPDATE config SET config_name = "drone" ` diff --git a/store/datastore/ddl/sqlite/files/020_add_column_config_name.sql b/store/datastore/ddl/sqlite/files/020_add_column_config_name.sql index 4e364a7d88d..0140b403538 100644 --- a/store/datastore/ddl/sqlite/files/020_add_column_config_name.sql +++ b/store/datastore/ddl/sqlite/files/020_add_column_config_name.sql @@ -4,4 +4,4 @@ ALTER TABLE config ADD COLUMN config_name TEXT -- name: update-table-set-config-name -UPDATE config SET config_name = "default" \ No newline at end of file +UPDATE config SET config_name = "drone" \ No newline at end of file diff --git a/store/datastore/sql/mysql/files/config.sql b/store/datastore/sql/mysql/files/config.sql index 8f29bd5d167..64f63e7a07f 100644 --- a/store/datastore/sql/mysql/files/config.sql +++ b/store/datastore/sql/mysql/files/config.sql @@ -1,12 +1,14 @@ -- name: config-find-id SELECT - config_id + config.config_id ,config_repo_id ,config_hash ,config_data +,config_name FROM config -WHERE config_id = ? +LEFT JOIN build_config ON config.config_id = build_config.config_id +WHERE build_config.build_id = ? -- name: config-find-repo-hash @@ -15,6 +17,7 @@ SELECT ,config_repo_id ,config_hash ,config_data +,config_name FROM config WHERE config_repo_id = ? AND config_hash = ? @@ -23,6 +26,10 @@ WHERE config_repo_id = ? SELECT build_id FROM builds WHERE build_repo_id = ? -AND build_config_id = ? +AND build_id in ( + SELECT build_id + FROM build_config + WHERE build_config.config_id = ? + ) AND build_status NOT IN ('blocked', 'pending') LIMIT 1 diff --git a/store/datastore/sql/mysql/sql_gen.go b/store/datastore/sql/mysql/sql_gen.go index 313a53eb48c..d0669c3c940 100644 --- a/store/datastore/sql/mysql/sql_gen.go +++ b/store/datastore/sql/mysql/sql_gen.go @@ -55,12 +55,14 @@ var index = map[string]string{ var configFindId = ` SELECT - config_id + config.config_id ,config_repo_id ,config_hash ,config_data +,config_name FROM config -WHERE config_id = ? +LEFT JOIN build_config ON config.config_id = build_config.config_id +WHERE build_config.build_id = ? ` var configFindRepoHash = ` @@ -69,6 +71,7 @@ SELECT ,config_repo_id ,config_hash ,config_data +,config_name FROM config WHERE config_repo_id = ? AND config_hash = ? @@ -77,7 +80,11 @@ WHERE config_repo_id = ? var configFindApproved = ` SELECT build_id FROM builds WHERE build_repo_id = ? -AND build_config_id = ? +AND build_id in ( + SELECT build_id + FROM build_config + WHERE build_config.config_id = ? + ) AND build_status NOT IN ('blocked', 'pending') LIMIT 1 ` diff --git a/store/datastore/sql/postgres/files/config.sql b/store/datastore/sql/postgres/files/config.sql index 2dc95438447..17076138b39 100644 --- a/store/datastore/sql/postgres/files/config.sql +++ b/store/datastore/sql/postgres/files/config.sql @@ -1,12 +1,14 @@ -- name: config-find-id SELECT - config_id + config.config_id ,config_repo_id ,config_hash ,config_data +,config_name FROM config -WHERE config_id = $1 +LEFT JOIN build_config ON config.config_id = build_config.config_id +WHERE build_config.build_id = $1 -- name: config-find-repo-hash @@ -15,6 +17,7 @@ SELECT ,config_repo_id ,config_hash ,config_data +,config_name FROM config WHERE config_repo_id = $1 AND config_hash = $2 @@ -23,6 +26,10 @@ WHERE config_repo_id = $1 SELECT build_id FROM builds WHERE build_repo_id = $1 -AND build_config_id = $2 +AND build_id in ( + SELECT build_id + FROM build_config + WHERE build_config.config_id = $2 + ) AND build_status NOT IN ('blocked', 'pending') LIMIT 1 diff --git a/store/datastore/sql/postgres/sql_gen.go b/store/datastore/sql/postgres/sql_gen.go index 83bdb729b57..e64d661afe2 100644 --- a/store/datastore/sql/postgres/sql_gen.go +++ b/store/datastore/sql/postgres/sql_gen.go @@ -55,12 +55,14 @@ var index = map[string]string{ var configFindId = ` SELECT - config_id + config.config_id ,config_repo_id ,config_hash ,config_data +,config_name FROM config -WHERE config_id = $1 +LEFT JOIN build_config ON config.config_id = build_config.config_id +WHERE build_config.build_id = $1 ` var configFindRepoHash = ` @@ -69,6 +71,7 @@ SELECT ,config_repo_id ,config_hash ,config_data +,config_name FROM config WHERE config_repo_id = $1 AND config_hash = $2 @@ -77,7 +80,11 @@ WHERE config_repo_id = $1 var configFindApproved = ` SELECT build_id FROM builds WHERE build_repo_id = $1 -AND build_config_id = $2 +AND build_id in ( + SELECT build_id + FROM build_config + WHERE build_config.config_id = $2 + ) AND build_status NOT IN ('blocked', 'pending') LIMIT 1 ` @@ -95,7 +102,7 @@ WHERE repo_active = true var countBuilds = ` SELECT count(1) -FROM builds; +FROM builds ` var feedLatestBuild = ` From 782a59b6cce2a0799ef8e8b358f353df2e7fa42b Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Fri, 14 Jun 2019 10:13:13 +0200 Subject: [PATCH 19/41] Github merge head is outdated --- .drone.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.drone.yml b/.drone.yml index 4c2551ebd82..de443395551 100644 --- a/.drone.yml +++ b/.drone.yml @@ -1,3 +1,7 @@ +clone: + git: + image: plugins/git:next + workspace: base: /go path: src/github.com/laszlocph/drone-oss-08 From 6a2c3f129c233e0e87d53c23791e7869652de2f9 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Sun, 16 Jun 2019 10:54:31 +0200 Subject: [PATCH 20/41] Factor into functions --- server/rpc.go | 117 +++++++++++++++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 45 deletions(-) diff --git a/server/rpc.go b/server/rpc.go index 61ff8648a21..8e20fdb17d4 100644 --- a/server/rpc.go +++ b/server/rpc.go @@ -383,6 +383,43 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error { return err } + s.updateProcState(proc, state) + + if err := s.queue.Done(c, id); err != nil { + log.Printf("error: done: cannot ack proc_id %d: %s", procID, err) + } + + procs, _ := s.store.ProcList(build) + s.completeChildrenIfParentCompleted(procs, proc) + + if !isThereRunningStage(procs) { + build.Status = buildStatus(procs) + build.Finished = proc.Stopped + if err := s.store.UpdateBuild(build); err != nil { + log.Printf("error: done: cannot update build_id %d final state: %s", build.ID, err) + } + + s.updateRemoteStatus(repo, build) + } + + if err := s.logger.Close(c, id); err != nil { + log.Printf("error: done: cannot close build_id %d logger: %s", proc.ID, err) + } + + s.notify(c, repo, build, procs) + + return nil +} + +// Log implements the rpc.Log function +func (s *RPC) Log(c context.Context, id string, line *rpc.Line) error { + entry := new(logging.Entry) + entry.Data, _ = json.Marshal(line) + s.logger.Write(c, id, entry) + return nil +} + +func (s *RPC) updateProcState(proc *model.Proc, state rpc.State) { proc.Stopped = state.Finished proc.Error = state.Error proc.ExitCode = state.ExitCode @@ -391,68 +428,68 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error { proc.State = model.StatusFailure } if err := s.store.ProcUpdate(proc); err != nil { - log.Printf("error: done: cannot update proc_id %d state: %s", procID, err) - } - - if err := s.queue.Done(c, id); err != nil { - log.Printf("error: done: cannot ack proc_id %d: %s", procID, err) + log.Printf("error: done: cannot update proc_id %d state: %s", proc.ID, err) } +} - // TODO handle this error - procs, _ := s.store.ProcList(build) +func (s *RPC) completeChildrenIfParentCompleted(procs []*model.Proc, completedProc *model.Proc) { for _, p := range procs { - if p.Running() && p.PPID == proc.PID { + if p.Running() && p.PPID == completedProc.PID { p.State = model.StatusSkipped if p.Started != 0 { p.State = model.StatusSuccess // for deamons that are killed - p.Stopped = proc.Stopped + p.Stopped = completedProc.Stopped } if err := s.store.ProcUpdate(p); err != nil { log.Printf("error: done: cannot update proc_id %d child state: %s", p.ID, err) } } } +} - running := false - status := model.StatusSuccess +func isThereRunningStage(procs []*model.Proc) bool { for _, p := range procs { if p.PPID == 0 { if p.Running() { - running = true + return true } + } + } + return false +} + +func buildStatus(procs []*model.Proc) string { + status := model.StatusSuccess + + for _, p := range procs { + if p.PPID == 0 { if p.Failing() { status = p.State } } } - if !running { - build.Status = status - build.Finished = proc.Stopped - if err := s.store.UpdateBuild(build); err != nil { - log.Printf("error: done: cannot update build_id %d final state: %s", build.ID, err) - } - // update the status - user, err := s.store.GetUser(repo.UserID) - if err == nil { - if refresher, ok := s.remote.(remote.Refresher); ok { - ok, _ := refresher.Refresh(user) - if ok { - s.store.UpdateUser(user) - } - } - uri := fmt.Sprintf("%s/%s/%d", s.host, repo.FullName, build.Number) - err = s.remote.Status(user, repo, build, uri) - if err != nil { - logrus.Errorf("error setting commit status for %s/%d: %v", repo.FullName, build.Number, err) + return status +} + +func (s *RPC) updateRemoteStatus(repo *model.Repo, build *model.Build) { + user, err := s.store.GetUser(repo.UserID) + if err == nil { + if refresher, ok := s.remote.(remote.Refresher); ok { + ok, _ := refresher.Refresh(user) + if ok { + s.store.UpdateUser(user) } } + uri := fmt.Sprintf("%s/%s/%d", s.host, repo.FullName, build.Number) + err = s.remote.Status(user, repo, build, uri) + if err != nil { + logrus.Errorf("error setting commit status for %s/%d: %v", repo.FullName, build.Number, err) + } } +} - if err := s.logger.Close(c, id); err != nil { - log.Printf("error: done: cannot close build_id %d logger: %s", proc.ID, err) - } - +func (s *RPC) notify(c context.Context, repo *model.Repo, build *model.Build, procs []*model.Proc) { build.Procs = model.Tree(procs) message := pubsub.Message{ Labels: map[string]string{ @@ -465,16 +502,6 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error { Build: *build, }) s.pubsub.Publish(c, "topic/events", message) - - return nil -} - -// Log implements the rpc.Log function -func (s *RPC) Log(c context.Context, id string, line *rpc.Line) error { - entry := new(logging.Entry) - entry.Data, _ = json.Marshal(line) - s.logger.Write(c, id, entry) - return nil } func (s *RPC) checkCancelled(pipeline *rpc.Pipeline) (bool, error) { From a37866179f5ccfae02e9b9729ada8f25495d9428 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Sun, 16 Jun 2019 15:26:45 +0200 Subject: [PATCH 21/41] Tasks should not run on error, unless specified --- cncd/queue/fifo.go | 27 +++++++++++++-- cncd/queue/fifo_test.go | 56 ++++++++++++++++++++++++++++++ cncd/queue/queue.go | 75 +++++++++++++++++------------------------ 3 files changed, 111 insertions(+), 47 deletions(-) diff --git a/cncd/queue/fifo.go b/cncd/queue/fifo.go index 275baeae0c8..3f21111e264 100644 --- a/cncd/queue/fifo.go +++ b/cncd/queue/fifo.go @@ -97,10 +97,11 @@ func (q *fifo) Done(c context.Context, id string) error { // Error signals that the item is done executing with error. func (q *fifo) Error(c context.Context, id string, err error) error { q.Lock() - state, ok := q.running[id] + taskEntry, ok := q.running[id] if ok { - state.error = err - close(state.done) + q.updateDepStatusInQueue(id, err == nil) + taskEntry.error = err + close(taskEntry.done) delete(q.running, id) } q.Unlock() @@ -247,3 +248,23 @@ func (q *fifo) depsInQueue(task *Task) bool { } return false } + +func (q *fifo) updateDepStatusInQueue(taskID string, success bool) { + var next *list.Element + for e := q.pending.Front(); e != nil; e = next { + next = e.Next() + pending, ok := e.Value.(*Task) + for _, dep := range pending.Dependencies { + if ok && taskID == dep { + pending.DepStatus[dep] = success + } + } + } + for _, running := range q.running { + for _, dep := range running.item.Dependencies { + if taskID == dep { + running.item.DepStatus[dep] = success + } + } + } +} diff --git a/cncd/queue/fifo_test.go b/cncd/queue/fifo_test.go index eb4f9b17417..6f3e414a5e4 100644 --- a/cncd/queue/fifo_test.go +++ b/cncd/queue/fifo_test.go @@ -2,6 +2,7 @@ package queue import ( "context" + "fmt" "sync" "testing" "time" @@ -126,6 +127,7 @@ func TestFifoDependencies(t *testing.T) { task2 := &Task{ ID: "2", Dependencies: []string{"1"}, + DepStatus: make(map[string]bool), } q := New().(*fifo) @@ -146,3 +148,57 @@ func TestFifoDependencies(t *testing.T) { return } } + +func TestFifoErrors(t *testing.T) { + task1 := &Task{ + ID: "1", + } + + task2 := &Task{ + ID: "2", + Dependencies: []string{"1"}, + DepStatus: make(map[string]bool), + } + + task3 := &Task{ + ID: "3", + Dependencies: []string{"1"}, + DepStatus: make(map[string]bool), + RunOn: []string{"success", "failure"}, + } + + q := New().(*fifo) + q.Push(noContext, task2) + q.Push(noContext, task3) + q.Push(noContext, task1) + + got, _ := q.Poll(noContext, func(*Task) bool { return true }) + if got != task1 { + t.Errorf("expect task1 returned from queue as task2 depends on it") + return + } + + q.Error(noContext, got.ID, fmt.Errorf("exitcode 1, there was an error")) + + got, _ = q.Poll(noContext, func(*Task) bool { return true }) + if got != task2 { + t.Errorf("expect task2 returned from queue") + return + } + + if got.ShouldRun() { + t.Errorf("expect task2 should not run, since task1 failed") + return + } + + got, _ = q.Poll(noContext, func(*Task) bool { return true }) + if got != task3 { + t.Errorf("expect task3 returned from queue") + return + } + + if !got.ShouldRun() { + t.Errorf("expect task3 should run, task1 failed, but task3 runs on failure too") + return + } +} diff --git a/cncd/queue/queue.go b/cncd/queue/queue.go index 2fa4626bddb..cb0ae753051 100644 --- a/cncd/queue/queue.go +++ b/cncd/queue/queue.go @@ -24,8 +24,38 @@ type Task struct { // Labels represents the key-value pairs the entry is lebeled with. Labels map[string]string `json:"labels,omitempty"` - // Task IDs this task depend on + // Task IDs this task depend Dependencies []string + + // If dep finished sucessfully + DepStatus map[string]bool + + // RunOn failure or success + RunOn []string +} + +// ShouldRun tells if a task should be run or skipped, based on dependencies +func (t *Task) ShouldRun() bool { + if runsOnFailure(t.RunOn) { + return true + } + + for _, success := range t.DepStatus { + if !success { + return false + } + } + + return true +} + +func runsOnFailure(runsOn []string) bool { + for _, status := range runsOn { + if status == "failure" { + return true + } + } + return false } // InfoT provides runtime information. @@ -74,46 +104,3 @@ type Queue interface { // Info returns internal queue information. Info(c context.Context) InfoT } - -// // global instance of the queue. -// var global = New() -// -// // Set sets the global queue. -// func Set(queue Queue) { -// global = queue -// } -// -// // Push pushes an task to the tail of the global queue. -// func Push(c context.Context, task *Task) error { -// return global.Push(c, task) -// } -// -// // Poll retrieves and removes a task head of the global queue. -// func Poll(c context.Context, f Filter) (*Task, error) { -// return global.Poll(c, f) -// } -// -// // Extend extends the deadline for a task. -// func Extend(c context.Context, id string) error { -// return global.Extend(c, id) -// } -// -// // Done signals the task is complete. -// func Done(c context.Context, id string) error { -// return global.Done(c, id) -// } -// -// // Error signals the task is complete with errors. -// func Error(c context.Context, id string, err error) { -// global.Error(c, id, err) -// } -// -// // Wait waits until the task is complete. -// func Wait(c context.Context, id string) error { -// return global.Wait(c, id) -// } -// -// // Info returns internal queue information. -// func Info(c context.Context) InfoT { -// return global.Info(c) -// } From a22747f5c360e8bc0b148f7b7565ed197e6dfb15 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Sun, 16 Jun 2019 15:27:40 +0200 Subject: [PATCH 22/41] Sort later, so it is sorted in restarts too --- remote/remote.go | 6 ++++++ server/configFetcher.go | 9 --------- server/hook.go | 1 + server/procBuilder.go | 3 +++ 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/remote/remote.go b/remote/remote.go index 7804cd97bdc..22563cf477a 100644 --- a/remote/remote.go +++ b/remote/remote.go @@ -79,6 +79,12 @@ type FileMeta struct { Data []byte } +type ByName []*FileMeta + +func (a ByName) Len() int { return len(a) } +func (a ByName) Less(i, j int) bool { return a[i].Name < a[j].Name } +func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + // Refresher refreshes an oauth token and expiration for the given user. It // returns true if the token was refreshed, false if the token was not refreshed, // and error if it failed to refersh. diff --git a/server/configFetcher.go b/server/configFetcher.go index 2bd98e15843..08b2150746b 100644 --- a/server/configFetcher.go +++ b/server/configFetcher.go @@ -5,8 +5,6 @@ import ( "github.com/laszlocph/drone-oss-08/model" "github.com/laszlocph/drone-oss-08/remote" - - "sort" ) type configFetcher struct { @@ -33,15 +31,8 @@ func (cf *configFetcher) Fetch() ([]*remote.FileMeta, error) { return nil, direrr } - sort.Sort(byName(dir)) return dir, nil } } return []*remote.FileMeta{}, nil } - -type byName []*remote.FileMeta - -func (a byName) Len() int { return len(a) } -func (a byName) Less(i, j int) bool { return a[i].Name < a[j].Name } -func (a byName) Swap(i, j int) { a[i], a[j] = a[j], a[i] } diff --git a/server/hook.go b/server/hook.go index 833408e9465..2f0de47854b 100644 --- a/server/hook.go +++ b/server/hook.go @@ -321,6 +321,7 @@ func queueBuild(build *model.Build, repo *model.Repo, buildItems []*buildItem) { task.Labels["platform"] = item.Platform task.Labels["repo"] = repo.FullName task.Dependencies = taskIds(item.DependsOn, buildItems) + task.DepStatus = make(map[string]bool) task.Data, _ = json.Marshal(rpc.Pipeline{ ID: fmt.Sprint(item.Proc.ID), diff --git a/server/procBuilder.go b/server/procBuilder.go index 230656dd694..391bf9f7bd7 100644 --- a/server/procBuilder.go +++ b/server/procBuilder.go @@ -18,6 +18,7 @@ import ( "fmt" "math/rand" "net/url" + "sort" "strings" "github.com/drone/envsubst" @@ -55,6 +56,8 @@ type buildItem struct { func (b *procBuilder) Build() ([]*buildItem, error) { var items []*buildItem + sort.Sort(remote.ByName(b.Yamls)) + for j, y := range b.Yamls { // matrix axes axes, err := matrix.ParseString(string(y.Data)) From 2253ca66dbfa1998667c5aefd53a003296a6fcfa Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Sun, 16 Jun 2019 15:56:32 +0200 Subject: [PATCH 23/41] Tasks should not run on error, unless specified --- server/rpc.go | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/server/rpc.go b/server/rpc.go index 8e20fdb17d4..6bd2884590d 100644 --- a/server/rpc.go +++ b/server/rpc.go @@ -113,26 +113,22 @@ func (s *RPC) Next(c context.Context, filter rpc.Filter) (*rpc.Pipeline, error) if err != nil { return nil, err } - task, err := s.queue.Poll(c, fn) - if err != nil { - return nil, err - } else if task == nil { - return nil, nil - } - pipeline := new(rpc.Pipeline) - - // check if the process was previously cancelled - // cancelled, _ := s.checkCancelled(pipeline) - // if cancelled { - // logrus.Debugf("ignore pid %v: cancelled by user", pipeline.ID) - // if derr := s.queue.Done(c, pipeline.ID); derr != nil { - // logrus.Errorf("error: done: cannot ack proc_id %v: %s", pipeline.ID, err) - // } - // return nil, nil - // } + for { + task, err := s.queue.Poll(c, fn) + if err != nil { + return nil, err + } else if task == nil { + return nil, nil + } - err = json.Unmarshal(task.Data, pipeline) - return pipeline, err + if task.ShouldRun() { + pipeline := new(rpc.Pipeline) + err = json.Unmarshal(task.Data, pipeline) + return pipeline, err + } else { + s.Done(c, task.ID, rpc.State{}) + } + } } // Wait implements the rpc.Wait function @@ -385,7 +381,13 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error { s.updateProcState(proc, state) - if err := s.queue.Done(c, id); err != nil { + var queueErr error + if proc.Failing() { + queueErr = s.queue.Error(c, id, fmt.Errorf("Proc finished with exitcode %d, %s", state.ExitCode, state.Error)) + } else { + queueErr = s.queue.Done(c, id) + } + if queueErr != nil { log.Printf("error: done: cannot ack proc_id %d: %s", procID, err) } From a433591afa2280a9fbcc2172dd95f7989e5b82f5 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Mon, 17 Jun 2019 09:06:36 +0200 Subject: [PATCH 24/41] Introducing runs_on to run pipelines on failure --- .../pipeline/pipeline/frontend/yaml/config.go | 1 + .../pipeline/frontend/yaml/config_test.go | 5 ++ cncd/queue/fifo_test.go | 74 ++++++++++++++++++- cncd/queue/queue.go | 35 +++++++-- server/hook.go | 1 + server/procBuilder.go | 2 + server/procBuilder_test.go | 34 +++++++++ 7 files changed, 143 insertions(+), 9 deletions(-) diff --git a/cncd/pipeline/pipeline/frontend/yaml/config.go b/cncd/pipeline/pipeline/frontend/yaml/config.go index 281e99908b6..afa8e05124f 100644 --- a/cncd/pipeline/pipeline/frontend/yaml/config.go +++ b/cncd/pipeline/pipeline/frontend/yaml/config.go @@ -23,6 +23,7 @@ type ( Volumes Volumes Labels libcompose.SliceorMap DependsOn []string `yaml:"depends_on,omitempty"` + RunsOn []string `yaml:"runs_on,omitempty"` } // Workspace defines a pipeline workspace. diff --git a/cncd/pipeline/pipeline/frontend/yaml/config_test.go b/cncd/pipeline/pipeline/frontend/yaml/config_test.go index ce9cadfc6b6..afe9dcb3bdd 100644 --- a/cncd/pipeline/pipeline/frontend/yaml/config_test.go +++ b/cncd/pipeline/pipeline/frontend/yaml/config_test.go @@ -40,6 +40,8 @@ func TestParse(t *testing.T) { g.Assert(out.Labels["com.example.type"]).Equal("build") g.Assert(out.DependsOn[0]).Equal("lint") g.Assert(out.DependsOn[1]).Equal("test") + g.Assert(out.RunsOn[0]).Equal("success") + g.Assert(out.RunsOn[1]).Equal("failure") }) // Check to make sure variable expansion works in yaml.MapSlice // g.It("Should unmarshal variables", func() { @@ -99,6 +101,9 @@ labels: depends_on: - lint - test +runs_on: + - success + - failure ` var sampleVarYaml = ` diff --git a/cncd/queue/fifo_test.go b/cncd/queue/fifo_test.go index 6f3e414a5e4..5aa8327ff2f 100644 --- a/cncd/queue/fifo_test.go +++ b/cncd/queue/fifo_test.go @@ -127,7 +127,7 @@ func TestFifoDependencies(t *testing.T) { task2 := &Task{ ID: "2", Dependencies: []string{"1"}, - DepStatus: make(map[string]bool), + DepStatus: make(map[string]bool), } q := New().(*fifo) @@ -157,14 +157,14 @@ func TestFifoErrors(t *testing.T) { task2 := &Task{ ID: "2", Dependencies: []string{"1"}, - DepStatus: make(map[string]bool), + DepStatus: make(map[string]bool), } task3 := &Task{ ID: "3", Dependencies: []string{"1"}, - DepStatus: make(map[string]bool), - RunOn: []string{"success", "failure"}, + DepStatus: make(map[string]bool), + RunOn: []string{"success", "failure"}, } q := New().(*fifo) @@ -202,3 +202,69 @@ func TestFifoErrors(t *testing.T) { return } } + +func TestShouldRun(t *testing.T) { + task := &Task{ + ID: "2", + Dependencies: []string{"1"}, + DepStatus: map[string]bool{ + "1": true, + }, + RunOn: []string{"failure"}, + } + if task.ShouldRun() { + t.Errorf("expect task to not run, it runs on failure only") + return + } + + task = &Task{ + ID: "2", + Dependencies: []string{"1"}, + DepStatus: map[string]bool{ + "1": true, + }, + RunOn: []string{"failure", "success"}, + } + if !task.ShouldRun() { + t.Errorf("expect task to run") + return + } + + task = &Task{ + ID: "2", + Dependencies: []string{"1"}, + DepStatus: map[string]bool{ + "1": false, + }, + } + if task.ShouldRun() { + t.Errorf("expect task to not run") + return + } + + task = &Task{ + ID: "2", + Dependencies: []string{"1"}, + DepStatus: map[string]bool{ + "1": true, + }, + RunOn: []string{"success"}, + } + if !task.ShouldRun() { + t.Errorf("expect task to run") + return + } + + task = &Task{ + ID: "2", + Dependencies: []string{"1"}, + DepStatus: map[string]bool{ + "1": false, + }, + RunOn: []string{"failure"}, + } + if !task.ShouldRun() { + t.Errorf("expect task to run") + return + } +} diff --git a/cncd/queue/queue.go b/cncd/queue/queue.go index cb0ae753051..2b70d328572 100644 --- a/cncd/queue/queue.go +++ b/cncd/queue/queue.go @@ -36,17 +36,29 @@ type Task struct { // ShouldRun tells if a task should be run or skipped, based on dependencies func (t *Task) ShouldRun() bool { - if runsOnFailure(t.RunOn) { + if runsOnFailure(t.RunOn) && runsOnSuccess(t.RunOn) { return true } - for _, success := range t.DepStatus { - if !success { - return false + if !runsOnFailure(t.RunOn) && runsOnSuccess(t.RunOn) { + for _, success := range t.DepStatus { + if !success { + return false + } } + return true + } + + if runsOnFailure(t.RunOn) && !runsOnSuccess(t.RunOn) { + for _, success := range t.DepStatus { + if success { + return false + } + } + return true } - return true + return false } func runsOnFailure(runsOn []string) bool { @@ -58,6 +70,19 @@ func runsOnFailure(runsOn []string) bool { return false } +func runsOnSuccess(runsOn []string) bool { + if len(runsOn) == 0 { + return true + } + + for _, status := range runsOn { + if status == "success" { + return true + } + } + return false +} + // InfoT provides runtime information. type InfoT struct { Pending []*Task `json:"pending"` diff --git a/server/hook.go b/server/hook.go index 2f0de47854b..8efa1721822 100644 --- a/server/hook.go +++ b/server/hook.go @@ -321,6 +321,7 @@ func queueBuild(build *model.Build, repo *model.Repo, buildItems []*buildItem) { task.Labels["platform"] = item.Platform task.Labels["repo"] = repo.FullName task.Dependencies = taskIds(item.DependsOn, buildItems) + task.RunOn = item.RunsOn task.DepStatus = make(map[string]bool) task.Data, _ = json.Marshal(rpc.Pipeline{ diff --git a/server/procBuilder.go b/server/procBuilder.go index 391bf9f7bd7..5e0656d761b 100644 --- a/server/procBuilder.go +++ b/server/procBuilder.go @@ -50,6 +50,7 @@ type buildItem struct { Platform string Labels map[string]string DependsOn []string + RunsOn []string Config *backend.Config } @@ -111,6 +112,7 @@ func (b *procBuilder) Build() ([]*buildItem, error) { Config: ir, Labels: parsed.Labels, DependsOn: parsed.DependsOn, + RunsOn: parsed.RunsOn, Platform: metadata.Sys.Arch, } if item.Labels == nil { diff --git a/server/procBuilder_test.go b/server/procBuilder_test.go index 177a960c5bb..1a885b6b658 100644 --- a/server/procBuilder_test.go +++ b/server/procBuilder_test.go @@ -124,3 +124,37 @@ depends_on: t.Fatal("Should depend on test") } } + +func TestRunsOn(t *testing.T) { + b := procBuilder{ + Repo: &model.Repo{}, + Curr: &model.Build{}, + Last: &model.Build{}, + Netrc: &model.Netrc{}, + Secs: []*model.Secret{}, + Regs: []*model.Registry{}, + Link: "", + Yamls: []*remote.FileMeta{ + &remote.FileMeta{Data: []byte(` +pipeline: + deploy: + image: scratch + +runs_on: + - success + - failure +`)}, + }, + } + + buildItems, err := b.Build() + if err != nil { + t.Fatal(err) + } + if len(buildItems[0].RunsOn) != 2 { + t.Fatal("Should run on success and failure") + } + if buildItems[0].RunsOn[1] != "failure" { + t.Fatal("Should run on failure") + } +} From 1d47ba8a3217ca32e7f85f8c70c5fd1ebb93049f Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Mon, 17 Jun 2019 10:48:40 +0200 Subject: [PATCH 25/41] Status line for each pipeline on Github --- remote/bitbucket/bitbucket.go | 2 +- remote/bitbucket/bitbucket_test.go | 2 +- remote/bitbucketserver/bitbucketserver.go | 2 +- remote/coding/coding.go | 2 +- remote/gerrit/gerrit.go | 2 +- remote/gitea/gitea.go | 2 +- remote/gitea/gitea_test.go | 6 +++--- remote/github/github.go | 19 +++++++++++++----- remote/gitlab/gitlab.go | 2 +- remote/gitlab3/gitlab.go | 2 +- remote/gogs/gogs.go | 2 +- remote/gogs/gogs_test.go | 2 +- remote/remote.go | 6 +++--- server/build.go | 24 ++++++++++++++--------- server/hook.go | 22 +++++++++++++-------- server/rpc.go | 22 ++++++++++++++++++--- 16 files changed, 78 insertions(+), 41 deletions(-) diff --git a/remote/bitbucket/bitbucket.go b/remote/bitbucket/bitbucket.go index 143d2aff132..6f2ff1841f0 100644 --- a/remote/bitbucket/bitbucket.go +++ b/remote/bitbucket/bitbucket.go @@ -214,7 +214,7 @@ func (c *config) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([] } // Status creates a build status for the Bitbucket commit. -func (c *config) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { +func (c *config) Status(u *model.User, r *model.Repo, b *model.Build, link string, proc *model.Proc) error { status := internal.BuildStatus{ State: convertStatus(b.Status), Desc: convertDesc(b.Status), diff --git a/remote/bitbucket/bitbucket_test.go b/remote/bitbucket/bitbucket_test.go index 3c9af60b223..854007c9ea1 100644 --- a/remote/bitbucket/bitbucket_test.go +++ b/remote/bitbucket/bitbucket_test.go @@ -283,7 +283,7 @@ func Test_bitbucket(t *testing.T) { }) g.It("Should update the status", func() { - err := c.Status(fakeUser, fakeRepo, fakeBuild, "http://127.0.0.1") + err := c.Status(fakeUser, fakeRepo, fakeBuild, "http://127.0.0.1", nil) g.Assert(err == nil).IsTrue() }) diff --git a/remote/bitbucketserver/bitbucketserver.go b/remote/bitbucketserver/bitbucketserver.go index 9e14a377ad8..e0b74732706 100644 --- a/remote/bitbucketserver/bitbucketserver.go +++ b/remote/bitbucketserver/bitbucketserver.go @@ -184,7 +184,7 @@ func (c *Config) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([] } // Status is not supported by the bitbucketserver driver. -func (c *Config) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { +func (c *Config) Status(u *model.User, r *model.Repo, b *model.Build, link string, proc *model.Proc) error { status := internal.BuildStatus{ State: convertStatus(b.Status), Desc: convertDesc(b.Status), diff --git a/remote/coding/coding.go b/remote/coding/coding.go index e488bc2c91b..7255b2448e1 100644 --- a/remote/coding/coding.go +++ b/remote/coding/coding.go @@ -243,7 +243,7 @@ func (c *Coding) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([] } // Status sends the commit status to the remote system. -func (c *Coding) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { +func (c *Coding) Status(u *model.User, r *model.Repo, b *model.Build, link string, proc *model.Proc) error { // EMPTY: not implemented in Coding OAuth API return nil } diff --git a/remote/gerrit/gerrit.go b/remote/gerrit/gerrit.go index a78b9d53011..f5fd526c3c5 100644 --- a/remote/gerrit/gerrit.go +++ b/remote/gerrit/gerrit.go @@ -108,7 +108,7 @@ func (c *client) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([] } // Status is not supported by the Gogs driver. -func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { +func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string, proc *model.Proc) error { return nil } diff --git a/remote/gitea/gitea.go b/remote/gitea/gitea.go index 9efaca9e29d..14b4bdb54f8 100644 --- a/remote/gitea/gitea.go +++ b/remote/gitea/gitea.go @@ -254,7 +254,7 @@ func (c *client) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([] } // Status is supported by the Gitea driver. -func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { +func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string, proc *model.Proc) error { client := c.newClientToken(u.Token) status := getStatus(b.Status) diff --git a/remote/gitea/gitea_test.go b/remote/gitea/gitea_test.go index a6607bdf27b..ab0933c2c06 100644 --- a/remote/gitea/gitea_test.go +++ b/remote/gitea/gitea_test.go @@ -18,10 +18,10 @@ import ( "net/http/httptest" "testing" - "github.com/laszlocph/drone-oss-08/model" - "github.com/laszlocph/drone-oss-08/remote/gitea/fixtures" "github.com/franela/goblin" "github.com/gin-gonic/gin" + "github.com/laszlocph/drone-oss-08/model" + "github.com/laszlocph/drone-oss-08/remote/gitea/fixtures" ) func Test_gitea(t *testing.T) { @@ -149,7 +149,7 @@ func Test_gitea(t *testing.T) { }) g.It("Should return nil from send build status", func() { - err := c.Status(fakeUser, fakeRepo, fakeBuild, "http://gitea.io") + err := c.Status(fakeUser, fakeRepo, fakeBuild, "http://gitea.io", nil) g.Assert(err == nil).IsTrue() }) diff --git a/remote/github/github.go b/remote/github/github.go index bcce1b48c95..a4cdb3dc4c4 100644 --- a/remote/github/github.go +++ b/remote/github/github.go @@ -430,17 +430,17 @@ func matchingHooks(hooks []github.Hook, rawurl string) *github.Hook { // Status sends the commit status to the remote system. // An example would be the GitHub pull request status. -func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { +func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string, proc *model.Proc) error { client := c.newClientToken(u.Token) switch b.Event { case "deployment": return deploymentStatus(client, r, b, link) default: - return repoStatus(client, r, b, link, c.Context) + return repoStatus(client, r, b, link, c.Context, proc) } } -func repoStatus(client *github.Client, r *model.Repo, b *model.Build, link, ctx string) error { +func repoStatus(client *github.Client, r *model.Repo, b *model.Build, link, ctx string, proc *model.Proc) error { context := ctx switch b.Event { case model.EventPull: @@ -451,10 +451,19 @@ func repoStatus(client *github.Client, r *model.Repo, b *model.Build, link, ctx } } + status := github.String(convertStatus(b.Status)) + desc := github.String(convertDesc(b.Status)) + + if proc != nil { + context += "/" + proc.Name + status = github.String(convertStatus(proc.State)) + desc = github.String(convertDesc(proc.State)) + } + data := github.RepoStatus{ Context: github.String(context), - State: github.String(convertStatus(b.Status)), - Description: github.String(convertDesc(b.Status)), + State: status, + Description: desc, TargetURL: github.String(link), } _, _, err := client.Repositories.CreateStatus(r.Owner, r.Name, b.Commit, &data) diff --git a/remote/gitlab/gitlab.go b/remote/gitlab/gitlab.go index a0e60725bfa..910433ea860 100644 --- a/remote/gitlab/gitlab.go +++ b/remote/gitlab/gitlab.go @@ -345,7 +345,7 @@ func (c *Gitlab) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([] // NOTE Currently gitlab doesn't support status for commits and events, // also if we want get MR status in gitlab we need implement a special plugin for gitlab, // gitlab uses API to fetch build status on client side. But for now we skip this. -func (g *Gitlab) Status(u *model.User, repo *model.Repo, b *model.Build, link string) error { +func (g *Gitlab) Status(u *model.User, repo *model.Repo, b *model.Build, link string, proc *model.Proc) error { client := NewClient(g.URL, u.Token, g.SkipVerify) status := getStatus(b.Status) diff --git a/remote/gitlab3/gitlab.go b/remote/gitlab3/gitlab.go index 7009271eeae..73aafdbdf90 100644 --- a/remote/gitlab3/gitlab.go +++ b/remote/gitlab3/gitlab.go @@ -345,7 +345,7 @@ func (c *Gitlab) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([] // NOTE Currently gitlab doesn't support status for commits and events, // also if we want get MR status in gitlab we need implement a special plugin for gitlab, // gitlab uses API to fetch build status on client side. But for now we skip this. -func (g *Gitlab) Status(u *model.User, repo *model.Repo, b *model.Build, link string) error { +func (g *Gitlab) Status(u *model.User, repo *model.Repo, b *model.Build, link string, proc *model.Proc) error { client := NewClient(g.URL, u.Token, g.SkipVerify) status := getStatus(b.Status) diff --git a/remote/gogs/gogs.go b/remote/gogs/gogs.go index f6d1aa7cae6..7b641384ebf 100644 --- a/remote/gogs/gogs.go +++ b/remote/gogs/gogs.go @@ -207,7 +207,7 @@ func (c *client) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([] } // Status is not supported by the Gogs driver. -func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { +func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string, proc *model.Proc) error { return nil } diff --git a/remote/gogs/gogs_test.go b/remote/gogs/gogs_test.go index 961eae4605e..20f847646a5 100644 --- a/remote/gogs/gogs_test.go +++ b/remote/gogs/gogs_test.go @@ -163,7 +163,7 @@ func Test_gogs(t *testing.T) { g.It("Should return no-op for usupporeted features", func() { _, err1 := c.Auth("octocat", "4vyW6b49Z") - err2 := c.Status(nil, nil, nil, "") + err2 := c.Status(nil, nil, nil, "", nil) err3 := c.Deactivate(nil, nil, "") g.Assert(err1 != nil).IsTrue() g.Assert(err2 == nil).IsTrue() diff --git a/remote/remote.go b/remote/remote.go index 22563cf477a..558aae222ad 100644 --- a/remote/remote.go +++ b/remote/remote.go @@ -55,7 +55,7 @@ type Remote interface { // Status sends the commit status to the remote system. // An example would be the GitHub pull request status. - Status(u *model.User, r *model.Repo, b *model.Build, link string) error + Status(u *model.User, r *model.Repo, b *model.Build, link string, proc *model.Proc) error // Netrc returns a .netrc file that can be used to clone // private repositories from a remote system. @@ -127,8 +127,8 @@ func Perm(c context.Context, u *model.User, owner, repo string) (*model.Perm, er // Status sends the commit status to the remote system. // An example would be the GitHub pull request status. -func Status(c context.Context, u *model.User, r *model.Repo, b *model.Build, link string) error { - return FromContext(c).Status(u, r, b, link) +func Status(c context.Context, u *model.User, r *model.Repo, b *model.Build, link string, proc *model.Proc) error { + return FromContext(c).Status(u, r, b, link, proc) } // Netrc returns a .netrc file that can be used to clone diff --git a/server/build.go b/server/build.go index 94353930876..df2efc093c3 100644 --- a/server/build.go +++ b/server/build.go @@ -307,14 +307,6 @@ func PostApproval(c *gin.Context) { } } - defer func() { - uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) - err = remote_.Status(user, repo, build, uri) - if err != nil { - logrus.Errorf("error setting commit status for %s/%d: %v", repo.FullName, build.Number, err) - } - }() - var yamls []*remote.FileMeta for _, y := range configs { yamls = append(yamls, &remote.FileMeta{Data: []byte(y.Data), Name: y.Name}) @@ -346,6 +338,20 @@ func PostApproval(c *gin.Context) { logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err) } + defer func() { + for _, item := range buildItems { + uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) + if len(buildItems) > 1 { + err = remote_.Status(user, repo, build, uri, item.Proc) + } else { + err = remote_.Status(user, repo, build, uri, nil) + } + if err != nil { + logrus.Errorf("error setting commit status for %s/%d: %v", repo.FullName, build.Number, err) + } + } + }() + publishToTopic(c, build, repo) queueBuild(build, repo, buildItems) } @@ -380,7 +386,7 @@ func PostDecline(c *gin.Context) { } uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) - err = remote_.Status(user, repo, build, uri) + err = remote_.Status(user, repo, build, uri, nil) if err != nil { logrus.Errorf("error setting commit status for %s/%d: %v", repo.FullName, build.Number, err) } diff --git a/server/hook.go b/server/hook.go index 8efa1721822..8bbd5b9e3d9 100644 --- a/server/hook.go +++ b/server/hook.go @@ -221,14 +221,6 @@ func PostHook(c *gin.Context) { // get the previous build so that we can send status change notifications last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) - defer func() { - uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) - err = remote_.Status(user, repo, build, uri) - if err != nil { - logrus.Errorf("error setting commit status for %s/%d: %v", repo.FullName, build.Number, err) - } - }() - b := procBuilder{ Repo: repo, Curr: build, @@ -255,6 +247,20 @@ func PostHook(c *gin.Context) { logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err) } + defer func() { + for _, item := range buildItems { + uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) + if len(buildItems) > 1 { + err = remote_.Status(user, repo, build, uri, item.Proc) + } else { + err = remote_.Status(user, repo, build, uri, nil) + } + if err != nil { + logrus.Errorf("error setting commit status for %s/%d: %v", repo.FullName, build.Number, err) + } + } + }() + publishToTopic(c, build, repo) queueBuild(build, repo, buildItems) } diff --git a/server/rpc.go b/server/rpc.go index 6bd2884590d..c552ea195e6 100644 --- a/server/rpc.go +++ b/server/rpc.go @@ -401,7 +401,13 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error { log.Printf("error: done: cannot update build_id %d final state: %s", build.ID, err) } - s.updateRemoteStatus(repo, build) + if !isMultiPipeline(procs) { + s.updateRemoteStatus(repo, build, nil) + } + } + + if isMultiPipeline(procs) { + s.updateRemoteStatus(repo, build, proc) } if err := s.logger.Close(c, id); err != nil { @@ -413,6 +419,16 @@ func (s *RPC) Done(c context.Context, id string, state rpc.State) error { return nil } +func isMultiPipeline(procs []*model.Proc) bool { + countPPIDZero := 0 + for _, proc := range procs { + if proc.PPID == 0 { + countPPIDZero++ + } + } + return countPPIDZero > 1 +} + // Log implements the rpc.Log function func (s *RPC) Log(c context.Context, id string, line *rpc.Line) error { entry := new(logging.Entry) @@ -474,7 +490,7 @@ func buildStatus(procs []*model.Proc) string { return status } -func (s *RPC) updateRemoteStatus(repo *model.Repo, build *model.Build) { +func (s *RPC) updateRemoteStatus(repo *model.Repo, build *model.Build, proc *model.Proc) { user, err := s.store.GetUser(repo.UserID) if err == nil { if refresher, ok := s.remote.(remote.Refresher); ok { @@ -484,7 +500,7 @@ func (s *RPC) updateRemoteStatus(repo *model.Repo, build *model.Build) { } } uri := fmt.Sprintf("%s/%s/%d", s.host, repo.FullName, build.Number) - err = s.remote.Status(user, repo, build, uri) + err = s.remote.Status(user, repo, build, uri, proc) if err != nil { logrus.Errorf("error setting commit status for %s/%d: %v", repo.FullName, build.Number, err) } From 69de8face1624accc50dd821a8e04c0f2bb682ac Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Wed, 19 Jun 2019 08:36:13 +0200 Subject: [PATCH 26/41] Handling canceled, skipped builds --- cncd/queue/fifo.go | 16 +++++++++++ cncd/queue/fifo_test.go | 35 ++++++++++++++++++++++++ remote/github/convert.go | 2 +- server/build.go | 42 ++++++++++++++++++----------- server/rpc.go | 57 ++++------------------------------------ 5 files changed, 84 insertions(+), 68 deletions(-) diff --git a/cncd/queue/fifo.go b/cncd/queue/fifo.go index 3f21111e264..43d0c50970d 100644 --- a/cncd/queue/fifo.go +++ b/cncd/queue/fifo.go @@ -103,6 +103,8 @@ func (q *fifo) Error(c context.Context, id string, err error) error { taskEntry.error = err close(taskEntry.done) delete(q.running, id) + } else { + q.removeFromPending(id) } q.Unlock() return nil @@ -268,3 +270,17 @@ func (q *fifo) updateDepStatusInQueue(taskID string, success bool) { } } } + +func (q *fifo) removeFromPending(taskID string) { + logrus.Debugf("queue: trying to remove %s", taskID) + var next *list.Element + for e := q.pending.Front(); e != nil; e = next { + next = e.Next() + task := e.Value.(*Task) + if task.ID == taskID { + logrus.Debugf("queue: %s is removed from pending", taskID) + q.pending.Remove(e) + return + } + } +} diff --git a/cncd/queue/fifo_test.go b/cncd/queue/fifo_test.go index 5aa8327ff2f..a5642ba38e4 100644 --- a/cncd/queue/fifo_test.go +++ b/cncd/queue/fifo_test.go @@ -203,6 +203,41 @@ func TestFifoErrors(t *testing.T) { } } +func TestFifoCancel(t *testing.T) { + task1 := &Task{ + ID: "1", + } + + task2 := &Task{ + ID: "2", + Dependencies: []string{"1"}, + DepStatus: make(map[string]bool), + } + + task3 := &Task{ + ID: "3", + Dependencies: []string{"1"}, + DepStatus: make(map[string]bool), + RunOn: []string{"success", "failure"}, + } + + q := New().(*fifo) + q.Push(noContext, task2) + q.Push(noContext, task3) + q.Push(noContext, task1) + + _, _ = q.Poll(noContext, func(*Task) bool { return true }) + q.Error(noContext, task1.ID, fmt.Errorf("cancelled")) + q.Error(noContext, task2.ID, fmt.Errorf("cancelled")) + q.Error(noContext, task3.ID, fmt.Errorf("cancelled")) + + info := q.Info(noContext) + if len(info.Pending) != 0 { + t.Errorf("All pipelines should be cancelled") + return + } +} + func TestShouldRun(t *testing.T) { task := &Task{ ID: "2", diff --git a/remote/github/convert.go b/remote/github/convert.go index b8ce64f653c..4fde67e4ca4 100644 --- a/remote/github/convert.go +++ b/remote/github/convert.go @@ -51,7 +51,7 @@ const ( // GitHub commit status. func convertStatus(status string) string { switch status { - case model.StatusPending, model.StatusRunning, model.StatusBlocked: + case model.StatusPending, model.StatusRunning, model.StatusBlocked, model.StatusSkipped: return statusPending case model.StatusFailure, model.StatusDeclined: return statusFailure diff --git a/server/build.go b/server/build.go index df2efc093c3..b94a7e1198f 100644 --- a/server/build.go +++ b/server/build.go @@ -156,13 +156,12 @@ func GetProcLogs(c *gin.Context) { io.Copy(c.Writer, rc) } +// DeleteBuild cancels a build func DeleteBuild(c *gin.Context) { repo := session.Repo(c) - // parse the build number and job sequence number from - // the repquest parameter. + // parse the build number from the request parameter. num, _ := strconv.Atoi(c.Params.ByName("number")) - seq, _ := strconv.Atoi(c.Params.ByName("job")) build, err := store.GetBuildNumber(c, repo, num) if err != nil { @@ -170,27 +169,40 @@ func DeleteBuild(c *gin.Context) { return } - proc, err := store.FromContext(c).ProcFind(build, seq) + procs, err := store.FromContext(c).ProcList(build) if err != nil { c.AbortWithError(404, err) return } - if proc.State != model.StatusRunning { - c.String(400, "Cannot cancel a non-running build") - return + cancelled := false + for _, proc := range procs { + if proc.PPID != 0 { + continue + } + + if proc.State != model.StatusRunning && proc.State != model.StatusPending { + continue + } + + proc.State = model.StatusKilled + proc.Stopped = time.Now().Unix() + if proc.Started == 0 { + proc.Started = proc.Stopped + } + proc.ExitCode = 137 + // TODO cancel child procs + store.FromContext(c).ProcUpdate(proc) + + Config.Services.Queue.Error(context.Background(), fmt.Sprint(proc.ID), queue.ErrCancel) + cancelled = true } - proc.State = model.StatusKilled - proc.Stopped = time.Now().Unix() - if proc.Started == 0 { - proc.Started = proc.Stopped + if !cancelled { + c.String(400, "Cannot cancel a non-running build") + return } - proc.ExitCode = 137 - // TODO cancel child procs - store.FromContext(c).ProcUpdate(proc) - Config.Services.Queue.Error(context.Background(), fmt.Sprint(proc.ID), queue.ErrCancel) c.String(204, "") } diff --git a/server/rpc.go b/server/rpc.go index c552ea195e6..0410ece42b1 100644 --- a/server/rpc.go +++ b/server/rpc.go @@ -441,7 +441,11 @@ func (s *RPC) updateProcState(proc *model.Proc, state rpc.State) { proc.Stopped = state.Finished proc.Error = state.Error proc.ExitCode = state.ExitCode - proc.State = model.StatusSuccess + if state.Started == 0 { + proc.State = model.StatusSkipped + } else { + proc.State = model.StatusSuccess + } if proc.ExitCode != 0 || proc.Error != "" { proc.State = model.StatusFailure } @@ -522,21 +526,6 @@ func (s *RPC) notify(c context.Context, repo *model.Repo, build *model.Build, pr s.pubsub.Publish(c, "topic/events", message) } -func (s *RPC) checkCancelled(pipeline *rpc.Pipeline) (bool, error) { - pid, err := strconv.ParseInt(pipeline.ID, 10, 64) - if err != nil { - return false, err - } - proc, err := s.store.ProcLoad(pid) - if err != nil { - return false, err - } - if proc.State == model.StatusKilled { - return true, nil - } - return false, err -} - func createFilterFunc(filter rpc.Filter) (queue.Filter, error) { var st *expr.Selector var err error @@ -606,42 +595,6 @@ func (s *DroneServer) Next(c oldcontext.Context, req *proto.NextRequest) (*proto res.Pipeline.Payload, _ = json.Marshal(pipeline.Config) return res, err - - // fn := func(task *queue.Task) bool { - // for k, v := range req.GetFilter().Labels { - // if task.Labels[k] != v { - // return false - // } - // } - // return true - // } - // task, err := s.Queue.Poll(c, fn) - // if err != nil { - // return nil, err - // } else if task == nil { - // return nil, nil - // } - // - // pipeline := new(rpc.Pipeline) - // json.Unmarshal(task.Data, pipeline) - // - // res := new(proto.NextReply) - // res.Pipeline = new(proto.Pipeline) - // res.Pipeline.Id = pipeline.ID - // res.Pipeline.Timeout = pipeline.Timeout - // res.Pipeline.Payload, _ = json.Marshal(pipeline.Config) - // - // // check if the process was previously cancelled - // // cancelled, _ := s.checkCancelled(pipeline) - // // if cancelled { - // // logrus.Debugf("ignore pid %v: cancelled by user", pipeline.ID) - // // if derr := s.queue.Done(c, pipeline.ID); derr != nil { - // // logrus.Errorf("error: done: cannot ack proc_id %v: %s", pipeline.ID, err) - // // } - // // return nil, nil - // // } - // - // return res, nil } func (s *DroneServer) Init(c oldcontext.Context, req *proto.InitRequest) (*proto.Empty, error) { From fded15332825ce4d7fb1abd597ef173d46c19352 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Wed, 19 Jun 2019 09:01:17 +0200 Subject: [PATCH 27/41] Gated builds is disabled as I don't understand the feature --- server/hook.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server/hook.go b/server/hook.go index 8bbd5b9e3d9..6ffd51c05fb 100644 --- a/server/hook.go +++ b/server/hook.go @@ -159,12 +159,9 @@ func PostHook(c *gin.Context) { // } // } - // if repo.IsGated { - // allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf) - // if !allowed { - // build.Status = model.StatusBlocked - // } - // } + if repo.IsGated { // This feature is not clear to me. Reenabling once better understood + build.Status = model.StatusBlocked + } // update some build fields build.RepoID = repo.ID From eed92421c940dc18508338fc1858ab5370bea51f Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Wed, 19 Jun 2019 09:36:54 +0200 Subject: [PATCH 28/41] Branch conditions on pipelines --- server/hook.go | 29 +++++++++++++++++------- server/procBuilder.go | 7 ++++++ server/procBuilder_test.go | 46 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/server/hook.go b/server/hook.go index 6ffd51c05fb..c5f0af97367 100644 --- a/server/hook.go +++ b/server/hook.go @@ -33,6 +33,7 @@ import ( "github.com/laszlocph/drone-oss-08/shared/token" "github.com/laszlocph/drone-oss-08/store" + "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml" "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/rpc" "github.com/laszlocph/drone-oss-08/cncd/pubsub" "github.com/laszlocph/drone-oss-08/cncd/queue" @@ -150,14 +151,10 @@ func PostHook(c *gin.Context) { return } - // verify that pipeline can be built at all - // parsedPipelineConfig, err := yaml.ParseString(conf.Data) - // if err == nil { - // if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy { - // c.String(200, "Branch does not match restrictions defined in yaml") - // return - // } - // } + if branchFiltered(build, remoteYamlConfigs) { + c.String(200, "Branch does not match restrictions defined in yaml") + return + } if repo.IsGated { // This feature is not clear to me. Reenabling once better understood build.Status = model.StatusBlocked @@ -262,6 +259,19 @@ func PostHook(c *gin.Context) { queueBuild(build, repo, buildItems) } +func branchFiltered(build *model.Build, remoteYamlConfigs []*remote.FileMeta) bool { + for _, remoteYamlConfig := range remoteYamlConfigs { + parsedPipelineConfig, err := yaml.ParseString(string(remoteYamlConfig.Data)) + if err == nil { + if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy { + } else { + return false + } + } + } + return true +} + func findOrPersistPipelineConfig(build *model.Build, remoteYamlConfig *remote.FileMeta) (*model.Config, error) { sha := shasum(remoteYamlConfig.Data) conf, err := Config.Storage.Config.ConfigFindIdentical(build.RepoID, sha) @@ -315,6 +325,9 @@ func publishToTopic(c *gin.Context, build *model.Build, repo *model.Repo) { func queueBuild(build *model.Build, repo *model.Repo, buildItems []*buildItem) { var tasks []*queue.Task for _, item := range buildItems { + if item.Proc.State == model.StatusSkipped { + continue + } task := new(queue.Task) task.ID = fmt.Sprint(item.Proc.ID) task.Labels = map[string]string{} diff --git a/server/procBuilder.go b/server/procBuilder.go index 5e0656d761b..5eaadeefa65 100644 --- a/server/procBuilder.go +++ b/server/procBuilder.go @@ -103,6 +103,10 @@ func (b *procBuilder) Build() ([]*buildItem, error) { return nil, lerr } + if !parsed.Branches.Match(b.Curr.Branch) { + proc.State = model.StatusSkipped + } + metadata.SetPlatform(parsed.Platform) ir := b.toInternalRepresentation(parsed, environ, metadata, proc.ID) @@ -220,6 +224,9 @@ func setBuildSteps(build *model.Build, buildItems []*buildItem) { PGID: gid, State: model.StatusPending, } + if item.Proc.State == model.StatusSkipped { + proc.State = model.StatusSkipped + } build.Procs = append(build.Procs, proc) } } diff --git a/server/procBuilder_test.go b/server/procBuilder_test.go index 1a885b6b658..4452dfaf4a1 100644 --- a/server/procBuilder_test.go +++ b/server/procBuilder_test.go @@ -158,3 +158,49 @@ runs_on: t.Fatal("Should run on failure") } } + +func TestBranchFilter(t *testing.T) { + b := procBuilder{ + Repo: &model.Repo{}, + Curr: &model.Build{Branch: "dev"}, + Last: &model.Build{}, + Netrc: &model.Netrc{}, + Secs: []*model.Secret{}, + Regs: []*model.Registry{}, + Link: "", + Yamls: []*remote.FileMeta{ + &remote.FileMeta{Data: []byte(` +pipeline: + xxx: + image: scratch + yyy: ${DRONE_COMMIT_MESSAGE} +branches: master +`)}, + &remote.FileMeta{Data: []byte(` +pipeline: + build: + image: scratch + yyy: ${DRONE_COMMIT_MESSAGE} +`)}, + }, + } + + buildItems, err := b.Build() + if err != nil { + t.Fatal(err) + } + if len(buildItems) != 2 { + t.Fatal("Should have generated 2 buildItems") + } + if buildItems[0].Proc.State != model.StatusSkipped { + t.Fatal("Should not run on dev branch") + } + for _, child := range buildItems[0].Proc.Children { + if child.State != model.StatusSkipped { + t.Fatal("Children should skipped status too") + } + } + if buildItems[1].Proc.State != model.StatusPending { + t.Fatal("Should not run on dev branch") + } +} From d7d360b94e6b11341aef640e0ccabe1398d28b28 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Wed, 19 Jun 2019 10:24:34 +0200 Subject: [PATCH 29/41] Latest UI pulled in --- .../laszlocph/drone-ui/dist/dist_gen.go | 121 +++++++++--------- vendor/vendor.json | 6 +- 2 files changed, 66 insertions(+), 61 deletions(-) diff --git a/vendor/github.com/laszlocph/drone-ui/dist/dist_gen.go b/vendor/github.com/laszlocph/drone-ui/dist/dist_gen.go index 19d52727b23..abcbfed6ccc 100644 --- a/vendor/github.com/laszlocph/drone-ui/dist/dist_gen.go +++ b/vendor/github.com/laszlocph/drone-ui/dist/dist_gen.go @@ -136,20 +136,20 @@ func MustLookup(path string) []byte { // Index of all files var files = map[string]file{ - "/static/bundle.1a4416c2ce69f18a1693.js": { + "/static/bundle.eb8b0239cd35b72cf8c6.js": { data: file0, FileInfo: &fileInfo{ - name: "bundle.1a4416c2ce69f18a1693.js", - size: 368620, - modTime: time.Unix(1559709405, 0), + name: "bundle.eb8b0239cd35b72cf8c6.js", + size: 368903, + modTime: time.Unix(1560931888, 0), }, }, - "/static/vendor.f67324c74172d8a7f0ba.js": { + "/static/vendor.81cce2e51d1d8f062da7.js": { data: file1, FileInfo: &fileInfo{ - name: "vendor.f67324c74172d8a7f0ba.js", + name: "vendor.81cce2e51d1d8f062da7.js", size: 272277, - modTime: time.Unix(1559709405, 0), + modTime: time.Unix(1560931888, 0), }, }, "/favicon.png": { @@ -157,7 +157,7 @@ var files = map[string]file{ FileInfo: &fileInfo{ name: "favicon.png", size: 1374, - modTime: time.Unix(1559709405, 0), + modTime: time.Unix(1560931888, 0), }, }, "/index.html": { @@ -165,7 +165,7 @@ var files = map[string]file{ FileInfo: &fileInfo{ name: "index.html", size: 388, - modTime: time.Unix(1559709405, 0), + modTime: time.Unix(1560931888, 0), }, }, } @@ -174,7 +174,7 @@ var files = map[string]file{ // embedded files. // -// /static/bundle.1a4416c2ce69f18a1693.js +// /static/bundle.eb8b0239cd35b72cf8c6.js var file0 = []byte(`webpackJsonp([0],[ /* 0 */, /* 1 */, @@ -1108,7 +1108,7 @@ var hideMessage = exports.hideMessage = function hideMessage(tree) { exports.__esModule = true; -exports.StatusLabel = exports["default"] = undefined; +exports.StatusText = exports.StatusLabel = exports["default"] = undefined; var _react = __webpack_require__(1); @@ -1222,6 +1222,24 @@ var StatusLabel = exports.StatusLabel = function StatusLabel(_ref) { ); }; +var StatusText = exports.StatusText = function StatusText(_ref2) { + var status = _ref2.status, + text = _ref2.text; + + return _react2["default"].createElement( + "div", + { + className: (0, _classnames2["default"])(_status3["default"].label, _status3["default"][status]), + style: "text-transform: capitalize;padding: 5px 10px;" + }, + _react2["default"].createElement( + "div", + null, + text + ) + ); +}; + /***/ }), /* 87 */ /***/ (function(module, exports, __webpack_require__) { @@ -8974,13 +8992,18 @@ var BuildLogs = (_dec = (0, _higherOrder.branch)(binding), (0, _inject.inject)(_ "section", { className: _index2["default"].sticky }, build.procs.map(function (rootProc) { - return _react2["default"].createElement(_components.ProcList, { - key: rootProc.pid, - repo: repo, - build: build, - rootProc: rootProc, - selectedProc: selectedProc - }); + return _react2["default"].createElement( + "div", + { style: "padding-bottom: 50px;", key: rootProc.pid }, + _react2["default"].createElement(_components.ProcList, { + key: rootProc.pid, + repo: repo, + build: build, + rootProc: rootProc, + selectedProc: selectedProc, + renderName: build.procs.length > 1 + }) + ); }) ) ), @@ -9311,12 +9334,12 @@ var _classnames = __webpack_require__(66); var _classnames2 = _interopRequireDefault(_classnames); +var _elapsed = __webpack_require__(549); + var _status = __webpack_require__(86); var _status2 = _interopRequireDefault(_status); -var _elapsed = __webpack_require__(549); - var _procs = __webpack_require__(550); var _procs2 = _interopRequireDefault(_procs); @@ -9341,14 +9364,23 @@ var renderEnviron = function renderEnviron(data) { var ProcListHolder = function ProcListHolder(_ref) { var vars = _ref.vars, + renderName = _ref.renderName, children = _ref.children; return _react2["default"].createElement( "div", { className: _procs2["default"].list }, + renderName && vars.name !== "drone" ? _react2["default"].createElement( + "div", + null, + _react2["default"].createElement(_status.StatusText, { status: vars.state, text: vars.name }) + ) : null, vars.environ ? _react2["default"].createElement( "div", - { className: _procs2["default"].vars }, - Object.entries(vars.environ).map(renderEnviron) + null, + _react2["default"].createElement(_status.StatusText, { + status: vars.state, + text: Object.entries(vars.environ).map(renderEnviron) + }) ) : null, children ); @@ -9368,11 +9400,12 @@ var ProcList = exports.ProcList = function (_Component) { repo = _props.repo, build = _props.build, rootProc = _props.rootProc, - selectedProc = _props.selectedProc; + selectedProc = _props.selectedProc, + renderName = _props.renderName; return _react2["default"].createElement( ProcListHolder, - { vars: rootProc }, + { vars: rootProc, renderName: renderName }, this.props.rootProc.children.map(function (child) { return _react2["default"].createElement( _reactRouterDom.Link, @@ -9423,29 +9456,6 @@ var ProcListItem = exports.ProcListItem = function ProcListItem(_ref2) { ); }; -// function List({ children }) { -// return
{children}
; -// } -// -// function ListItem({ name, start, finish, state, selected }) { -// const classes = classnames(styles.item, selected ? styles.selected : null); -// return ( -//
-//

{name}

-// -// {finish ? ( -// -// ) : ( -// -// )} -// -//
-// -//
-//
-// ); -// } - /***/ }), /* 549 */ /***/ (function(module, exports, __webpack_require__) { @@ -10309,18 +10319,13 @@ var BuildMenu = (_dec = (0, _higherOrder.branch)(binding), (0, _inject.inject)(_ }; BuildMenu.prototype.render = function render() { - var _props3 = this.props, - build = _props3.build, - match = _props3.match; - var proc = match.params.proc; - + var build = this.props.build; - var hideCancel = (0, _build.assertBuildMatrix)(build) && !proc; var rightSide = !build ? undefined : _react2["default"].createElement( "section", null, - build.status === "pending" || build.status === "running" ? !hideCancel ? _react2["default"].createElement( + build.status === "pending" || build.status === "running" ? _react2["default"].createElement( "button", { onClick: this.handleCancel }, _react2["default"].createElement(_icons.CloseIcon, null), @@ -10329,7 +10334,7 @@ var BuildMenu = (_dec = (0, _higherOrder.branch)(binding), (0, _inject.inject)(_ null, "Cancel" ) - ) : null : _react2["default"].createElement( + ) : _react2["default"].createElement( "button", { onClick: this.handleRestart }, _react2["default"].createElement(_icons.RefreshIcon, null), @@ -10781,8 +10786,8 @@ exports.push([module.i, " {\n}\ndiv,\nspan {\n font-family: 'Roboto';\n font-s /***/ }) ],[201]);`) -// /static/vendor.f67324c74172d8a7f0ba.js -var file1 = []byte(`!function(t){function e(n){if(r[n])return r[n].exports;var o=r[n]={i:n,l:!1,exports:{}};return t[n].call(o.exports,o,o.exports,e),o.l=!0,o.exports}var n=window.webpackJsonp;window.webpackJsonp=function(r,i,a){for(var u,c,s,f=0,l=[];r.length>f;f++)c=r[f],o[c]&&l.push(o[c][0]),o[c]=0;for(u in i)Object.prototype.hasOwnProperty.call(i,u)&&(t[u]=i[u]);for(n&&n(r,i,a);l.length;)l.shift()();if(a)for(f=0;a.length>f;f++)s=e(e.s=a[f]);return s};var r={},o={1:0};e.e=function(t){function n(){u.onerror=u.onload=null,clearTimeout(c);var e=o[t];0!==e&&(e&&e[1](Error("Loading chunk "+t+" failed.")),o[t]=void 0)}var r=o[t];if(0===r)return new Promise(function(t){t()});if(r)return r[2];var i=new Promise(function(e,n){r=o[t]=[e,n]});r[2]=i;var a=document.getElementsByTagName("head")[0],u=document.createElement("script");u.type="text/javascript",u.charset="utf-8",u.async=!0,u.timeout=12e4,e.nc&&u.setAttribute("nonce",e.nc),u.src=e.p+""+t+".static/bundle."+{0:"1a4416c2ce69f18a1693"}[t]+".js";var c=setTimeout(n,12e4);return u.onerror=u.onload=n,a.appendChild(u),i},e.m=t,e.c=r,e.d=function(t,n,r){e.o(t,n)||Object.defineProperty(t,n,{configurable:!1,enumerable:!0,get:r})},e.n=function(t){var n=t&&t.__esModule?function(){return t.default}:function(){return t};return e.d(n,"a",n),n},e.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},e.p="/",e.oe=function(t){throw t},e(e.s=584)}([function(t,e,n){var r=n(5),o=n(32),i=n(18),a=n(19),u=n(28),c=function(t,e,n){var s,f,l,p,h=t&c.F,d=t&c.G,v=t&c.S,y=t&c.P,m=t&c.B,g=d?r:v?r[e]||(r[e]={}):(r[e]||{}).prototype,b=d?o:o[e]||(o[e]={}),_=b.prototype||(b.prototype={});d&&(n=e);for(s in n)f=!h&&g&&void 0!==g[s],l=(f?g:n)[s],p=m&&f?u(l,r):y&&"function"==typeof l?u(Function.call,l):l,g&&a(g,s,l,t&c.U),b[s]!=l&&i(b,s,p),y&&_[s]!=l&&(_[s]=l)};r.core=o,c.F=1,c.G=2,c.S=4,c.P=8,c.B=16,c.W=32,c.U=64,c.R=128,t.exports=c},function(t,e,n){(function(e){!function(e,r){t.exports=r(n(12),n(118))}(0,function(t,n){function r(){return null}function o(t){var e=t.nodeName,n=t.attributes;t.attributes={},e.defaultProps&&w(t.attributes,e.defaultProps),n&&w(t.attributes,n)}function i(t,e){var n,r,o;if(e){for(o in e)if(n=B.test(o))break;if(n){r=t.attributes={};for(o in e)e.hasOwnProperty(o)&&(r[B.test(o)?o.replace(/([A-Z0-9])/,"-$1").toLowerCase():o]=e[o])}}}function a(t,e,r){var o=e&&e._preactCompatRendered&&e._preactCompatRendered.base;o&&o.parentNode!==e&&(o=null),!o&&e&&(o=e.firstElementChild);for(var i=e.childNodes.length;i--;)e.childNodes[i]!==o&&e.removeChild(e.childNodes[i]);var a=n.render(t,e,o);return e&&(e._preactCompatRendered=a&&(a._component||{base:a})),"function"==typeof r&&r(),a&&a._component||a}function u(t,e,r,o){var i=n.h(Y,{context:t.context},e),u=a(i,r),c=u._component||u.base;return o&&o.call(c,u),c}function c(t){var e=t._preactCompatRendered&&t._preactCompatRendered.base;return!(!e||e.parentNode!==t)&&(n.render(n.h(r),t,e),!0)}function s(t){return d.bind(null,t)}function f(t,e){for(var n=e||0;t.length>n;n++){var r=t[n];Array.isArray(r)?f(r):r&&"object"==typeof r&&!m(r)&&(r.props&&r.type||r.attributes&&r.nodeName||r.children)&&(t[n]=d(r.type||r.nodeName,r.props||r.attributes,r.children))}}function l(t){return"function"==typeof t&&!(t.prototype&&t.prototype.render)}function p(t){return E({displayName:t.displayName||t.name,render:function(){return t(this.props,this.context)}})}function h(t){var e=t[U];return e?!0===e?t:e:(e=p(t),Object.defineProperty(e,U,{configurable:!0,value:!0}),e.displayName=t.displayName,e.propTypes=t.propTypes,e.defaultProps=t.defaultProps,Object.defineProperty(t,U,{configurable:!0,value:e}),e)}function d(){for(var t=[],e=arguments.length;e--;)t[e]=arguments[e];return f(t,2),v(n.h.apply(void 0,t))}function v(t){t.preactCompatNormalized=!0,_(t),l(t.nodeName)&&(t.nodeName=h(t.nodeName));var e=t.attributes.ref,n=e&&typeof e;return!$||"string"!==n&&"number"!==n||(t.attributes.ref=g(e,$)),b(t),t}function y(t,e){for(var r=[],o=arguments.length-2;o-- >0;)r[o]=arguments[o+2];if(!m(t))return t;var i=t.attributes||t.props,a=n.h(t.nodeName||t.type,i,t.children||i&&i.children),u=[a,e];return r&&r.length?u.push(r):e&&e.children&&u.push(e.children),v(n.cloneElement.apply(void 0,u))}function m(t){return t&&(t instanceof V||t.$$typeof===D)}function g(t,e){return e._refProxies[t]||(e._refProxies[t]=function(n){e&&e.refs&&(e.refs[t]=n,null===n&&(delete e._refProxies[t],e=null))})}function b(t){var e=t.nodeName,n=t.attributes;if(n&&"string"==typeof e){var r={};for(var o in n)r[o.toLowerCase()]=o;if(r.ondoubleclick&&(n.ondblclick=n[r.ondoubleclick],delete n[r.ondoubleclick]),r.onchange&&("textarea"===e||"input"===e.toLowerCase()&&!/^fil|che|rad/i.test(n.type))){var i=r.oninput||"oninput";n[i]||(n[i]=C([n[i],n[r.onchange]]),delete n[r.onchange])}}}function _(t){var e=t.attributes||(t.attributes={});Z.enumerable="className"in e,e.className&&(e.class=e.className),Object.defineProperty(e,"className",Z)}function w(t){for(var e=arguments,n=1,r=void 0;arguments.length>n;n++)if(r=e[n])for(var o in r)r.hasOwnProperty(o)&&(t[o]=r[o]);return t}function O(t,e){for(var n in t)if(!(n in e))return!0;for(var r in e)if(t[r]!==e[r])return!0;return!1}function x(t){return t&&t.base||t}function P(){}function E(t){function e(t,e){S(this),L.call(this,t,e,z),M.call(this,t,e)}return t=w({constructor:e},t),t.mixins&&j(t,k(t.mixins)),t.statics&&w(e,t.statics),t.propTypes&&(e.propTypes=t.propTypes),t.defaultProps&&(e.defaultProps=t.defaultProps),t.getDefaultProps&&(e.defaultProps=t.getDefaultProps()),P.prototype=L.prototype,e.prototype=w(new P,t),e.displayName=t.displayName||"Component",e}function k(t){for(var e={},n=0;t.length>n;n++){var r=t[n];for(var o in r)r.hasOwnProperty(o)&&"function"==typeof r[o]&&(e[o]||(e[o]=[])).push(r[o])}return e}function j(t,e){for(var n in e)e.hasOwnProperty(n)&&(t[n]=C(e[n].concat(t[n]||K),"getDefaultProps"===n||"getInitialState"===n||"getChildContext"===n))}function S(t){for(var e in t){var n=t[e];"function"!=typeof n||n.__bound||W.hasOwnProperty(e)||((t[e]=n.bind(t)).__bound=!0)}}function T(t,e,n){if("string"==typeof e&&(e=t.constructor.prototype[e]),"function"==typeof e)return e.apply(t,n)}function C(t,e){return function(){for(var n,r=arguments,o=this,i=0;t.length>i;i++){var a=T(o,t[i],r);if(e&&null!=a){n||(n={});for(var u in a)a.hasOwnProperty(u)&&(n[u]=a[u])}else void 0!==a&&(n=a)}return n}}function M(t,e){N.call(this,t,e),this.componentWillReceiveProps=C([N,this.componentWillReceiveProps||"componentWillReceiveProps"]),this.render=C([N,R,this.render||"render",A])}function N(e){if(e){var n=e.children;if(n&&Array.isArray(n)&&1===n.length&&("string"==typeof n[0]||"function"==typeof n[0]||n[0]instanceof V)&&(e.children=n[0])&&"object"==typeof e.children&&(e.children.length=1,e.children[0]=e.children),q){var r="function"==typeof this?this:this.constructor,o=this.propTypes||r.propTypes,i=this.displayName||r.name;o&&t.checkPropTypes(o,e,"prop",i)}}}function R(){$=this}function A(){$===this&&($=null)}function L(t,e,r){n.Component.call(this,t,e),this.state=this.getInitialState?this.getInitialState():{},this.refs={},this._refProxies={},r!==z&&M.call(this,t,e)}function I(t,e){L.call(this,t,e)}t="default"in t?t.default:t;var F="a abbr address area article aside audio b base bdi bdo big blockquote body br button canvas caption cite code col colgroup data datalist dd del details dfn dialog div dl dt em embed fieldset figcaption figure footer form h1 h2 h3 h4 h5 h6 head header hgroup hr html i iframe img input ins kbd keygen label legend li link main map mark menu menuitem meta meter nav noscript object ol optgroup option output p param picture pre progress q rp rt ruby s samp script section select small source span strong style sub summary sup table tbody td textarea tfoot th thead time title tr track u ul var video wbr circle clipPath defs ellipse g image line linearGradient mask path pattern polygon polyline radialGradient rect stop svg text tspan".split(" "),D="undefined"!=typeof Symbol&&Symbol.for&&Symbol.for("react.element")||60103,U="undefined"!=typeof Symbol?Symbol.for("__preactCompatWrapper"):"__preactCompatWrapper",W={constructor:1,render:1,shouldComponentUpdate:1,componentWillReceiveProps:1,componentWillUpdate:1,componentDidUpdate:1,componentWillMount:1,componentDidMount:1,componentWillUnmount:1,componentDidUnmount:1},B=/^(?:accent|alignment|arabic|baseline|cap|clip|color|fill|flood|font|glyph|horiz|marker|overline|paint|stop|strikethrough|stroke|text|underline|unicode|units|v|vector|vert|word|writing|x)[A-Z]/,z={},q=void 0===e||!e.env||"production"!==e.env.NODE_ENV,V=n.h("a",null).constructor;V.prototype.$$typeof=D,V.prototype.preactCompatUpgraded=!1,V.prototype.preactCompatNormalized=!1,Object.defineProperty(V.prototype,"type",{get:function(){return this.nodeName},set:function(t){this.nodeName=t},configurable:!0}),Object.defineProperty(V.prototype,"props",{get:function(){return this.attributes},set:function(t){this.attributes=t},configurable:!0});var G=n.options.event;n.options.event=function(t){return G&&(t=G(t)),t.persist=Object,t.nativeEvent=t,t};var H=n.options.vnode;n.options.vnode=function(t){if(!t.preactCompatUpgraded){t.preactCompatUpgraded=!0;var e=t.nodeName,n=t.attributes=w({},t.attributes);"function"==typeof e?(!0===e[U]||e.prototype&&"isReactComponent"in e.prototype)&&(t.children&&t.children+""==""&&(t.children=void 0),t.children&&(n.children=t.children),t.preactCompatNormalized||v(t),o(t)):(t.children&&t.children+""==""&&(t.children=void 0),t.children&&(n.children=t.children),n.defaultValue&&(n.value||0===n.value||(n.value=n.defaultValue),delete n.defaultValue),i(t,n))}H&&H(t)};var Y=function(){};Y.prototype.getChildContext=function(){return this.props.context},Y.prototype.render=function(t){return t.children[0]};for(var $,K=[],J={map:function(t,e,n){return null==t?null:(t=J.toArray(t),n&&n!==t&&(e=e.bind(n)),t.map(e))},forEach:function(t,e,n){if(null==t)return null;t=J.toArray(t),n&&n!==t&&(e=e.bind(n)),t.forEach(e)},count:function(t){return t&&t.length||0},only:function(t){if(t=J.toArray(t),1!==t.length)throw Error("Children.only() expects only one child.");return t[0]},toArray:function(t){return null==t?[]:K.concat(t)}},Q={},X=F.length;X--;)Q[F[X]]=s(F[X]);var Z={configurable:!0,get:function(){return this.class},set:function(t){this.class=t}};return w(L.prototype=new n.Component,{constructor:L,isReactComponent:{},replaceState:function(t,e){var n=this;this.setState(t,e);for(var r in n.state)r in t||delete n.state[r]},getDOMNode:function(){return this.base},isMounted:function(){return!!this.base}}),P.prototype=L.prototype,I.prototype=new P,I.prototype.isPureReactComponent=!0,I.prototype.shouldComponentUpdate=function(t,e){return O(this.props,t)||O(this.state,e)},{version:"15.1.0",DOM:Q,PropTypes:t,Children:J,render:a,createClass:E,createFactory:s,createElement:d,cloneElement:y,isValidElement:m,findDOMNode:x,unmountComponentAtNode:c,Component:L,PureComponent:I,unstable_renderSubtreeIntoContainer:u,__spread:w}})}).call(e,n(21))},function(t,e,n){var r=n(7);t.exports=function(t){if(!r(t))throw TypeError(t+" is not an object!");return t}},,,function(t){var e=t.exports="undefined"!=typeof window&&window.Math==Math?window:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")();"number"==typeof __g&&(__g=e)},function(t){t.exports=function(t){try{return!!t()}catch(t){return!0}}},function(t){t.exports=function(t){return"object"==typeof t?null!==t:"function"==typeof t}},function(t,e,n){var r=n(68)("wks"),o=n(44),i=n(5).Symbol,a="function"==typeof i;(t.exports=function(t){return r[t]||(r[t]=a&&i[t]||(a?i:o)("Symbol."+t))}).store=r},function(t,e,n){t.exports=!n(6)(function(){return 7!=Object.defineProperty({},"a",{get:function(){return 7}}).a})},function(t,e,n){var r=n(2),o=n(133),i=n(33),a=Object.defineProperty;e.f=n(9)?Object.defineProperty:function(t,e,n){if(r(t),e=i(e,!0),r(n),o)try{return a(t,e,n)}catch(t){}if("get"in n||"set"in n)throw TypeError("Accessors not supported!");return"value"in n&&(t[e]=n.value),t}},function(t,e,n){var r=n(35),o=Math.min;t.exports=function(t){return t>0?o(r(t),9007199254740991):0}},function(t,e,n){(function(e){if("production"!==e.env.NODE_ENV){var r="function"==typeof Symbol&&Symbol.for&&Symbol.for("react.element")||60103,o=function(t){return"object"==typeof t&&null!==t&&t.$$typeof===r};t.exports=n(403)(o,!0)}else t.exports=n(405)()}).call(e,n(21))},function(t,e,n){var r=n(34);t.exports=function(t){return Object(r(t))}},function(t,e,n){"use strict";(function(e){var n=function(){};"production"!==e.env.NODE_ENV&&(n=function(t,e,n){var r=arguments.length;n=Array(r>2?r-2:0);for(var o=2;r>o;o++)n[o-2]=arguments[o];if(void 0===e)throw Error("` + "`" + `warning(condition, format, ...args)` + "`" + ` requires a warning message argument");if(10>e.length||/^[s\W]*$/.test(e))throw Error("The warning format should be able to uniquely identify this warning. Please, use a more descriptive format than: "+e);if(!t){var i=0,a="Warning: "+e.replace(/%s/g,function(){return n[i++]});try{throw Error(a)}catch(t){}}}),t.exports=n}).call(e,n(21))},function(t){t.exports=function(t){if("function"!=typeof t)throw TypeError(t+" is not a function!");return t}},function(t,e,n){var r=n(407);e.root=r.root,e.branch=r.branch},function(t){var e={}.hasOwnProperty;t.exports=function(t,n){return e.call(t,n)}},function(t,e,n){var r=n(10),o=n(43);t.exports=n(9)?function(t,e,n){return r.f(t,e,o(1,n))}:function(t,e,n){return t[e]=n,t}},function(t,e,n){var r=n(5),o=n(18),i=n(17),a=n(44)("src"),u=Function.toString,c=(""+u).split("toString");n(32).inspectSource=function(t){return u.call(t)},(t.exports=function(t,e,n,u){var s="function"==typeof n;s&&(i(n,"name")||o(n,"name",e)),t[e]!==n&&(s&&(i(n,a)||o(n,a,t[e]?""+t[e]:c.join(e+""))),t===r?t[e]=n:u?t[e]?t[e]=n:o(t,e,n):(delete t[e],o(t,e,n)))})(Function.prototype,"toString",function(){return"function"==typeof this&&this[a]||u.call(this)})},function(t,e,n){var r=n(0),o=n(6),i=n(34),a=/"/g,u=function(t,e,n,r){var o=i(t)+"",u="<"+e;return""!==n&&(u+=" "+n+'="'+(r+"").replace(a,""")+'"'),u+">"+o+""};t.exports=function(t,e){var n={};n[t]=e(u),r(r.P+r.F*o(function(){var e=""[t]('"');return e!==e.toLowerCase()||e.split('"').length>3}),"String",n)}},function(t){function e(){throw Error("setTimeout has not been defined")}function n(){throw Error("clearTimeout has not been defined")}function r(t){if(s===setTimeout)return setTimeout(t,0);if((s===e||!s)&&setTimeout)return s=setTimeout,setTimeout(t,0);try{return s(t,0)}catch(e){try{return s.call(null,t,0)}catch(e){return s.call(this,t,0)}}}function o(t){if(f===clearTimeout)return clearTimeout(t);if((f===n||!f)&&clearTimeout)return f=clearTimeout,clearTimeout(t);try{return f(t)}catch(e){try{return f.call(null,t)}catch(e){return f.call(this,t)}}}function i(){d&&p&&(d=!1,p.length?h=p.concat(h):v=-1,h.length&&a())}function a(){if(!d){var t=r(i);d=!0;for(var e=h.length;e;){for(p=h,h=[];++v1)for(var n=1;arguments.length>n;n++)e[n-1]=arguments[n];h.push(new u(t,e)),1!==h.length||d||r(a)},u.prototype.run=function(){this.fun.apply(null,this.array)},l.title="browser",l.browser=!0,l.env={},l.argv=[],l.version="",l.versions={},l.on=c,l.addListener=c,l.once=c,l.off=c,l.removeListener=c,l.removeAllListeners=c,l.emit=c,l.prependListener=c,l.prependOnceListener=c,l.listeners=function(){return[]},l.binding=function(){throw Error("process.binding is not supported")},l.cwd=function(){return"/"},l.chdir=function(){throw Error("process.chdir is not supported")},l.umask=function(){return 0}},,,function(t,e,n){var r=n(59),o=n(34);t.exports=function(t){return r(o(t))}},function(t,e,n){var r=n(60),o=n(43),i=n(24),a=n(33),u=n(17),c=n(133),s=Object.getOwnPropertyDescriptor;e.f=n(9)?s:function(t,e){if(t=i(t),e=a(e,!0),c)try{return s(t,e)}catch(t){}if(u(t,e))return o(!r.f.call(t,e),t[e])}},function(t,e,n){var r=n(17),o=n(13),i=n(91)("IE_PROTO"),a=Object.prototype;t.exports=Object.getPrototypeOf||function(t){return t=o(t),r(t,i)?t[i]:"function"==typeof t.constructor&&t instanceof t.constructor?t.constructor.prototype:t instanceof Object?a:null}},function(t,e,n){"use strict";Object.defineProperty(e,"__esModule",{value:!0});var r=n(424);n.d(e,"BrowserRouter",function(){return r.a});var o=n(426);n.d(e,"HashRouter",function(){return o.a});var i=n(176);n.d(e,"Link",function(){return i.a});var a=n(428);n.d(e,"MemoryRouter",function(){return a.a});var u=n(430);n.d(e,"NavLink",function(){return u.a});var c=n(433);n.d(e,"Prompt",function(){return c.a});var s=n(434);n.d(e,"Redirect",function(){return s.a});var f=n(178);n.d(e,"Route",function(){return f.a});var l=n(123);n.d(e,"Router",function(){return l.a});var p=n(439);n.d(e,"StaticRouter",function(){return p.a});var h=n(440);n.d(e,"Switch",function(){return h.a});var d=n(441);n.d(e,"matchPath",function(){return d.a});var v=n(442);n.d(e,"withRouter",function(){return v.a})},function(t,e,n){var r=n(15);t.exports=function(t,e,n){if(r(t),void 0===e)return t;switch(n){case 1:return function(n){return t.call(e,n)};case 2:return function(n,r){return t.call(e,n,r)};case 3:return function(n,r,o){return t.call(e,n,r,o)}}return function(){return t.apply(e,arguments)}}},function(t){var e={}.toString;t.exports=function(t){return e.call(t).slice(8,-1)}},function(t,e,n){"use strict";var r=n(6);t.exports=function(t,e){return!!t&&r(function(){e?t.call(null,function(){},1):t.call(null)})}},function(t,e,n){"use strict";(function(e){t.exports=function(t,n,r,o,i,a,u,c){if("production"!==e.env.NODE_ENV&&void 0===n)throw Error("invariant requires an error message argument");if(!t){var s;if(void 0===n)s=Error("Minified exception occurred; use the non-minified dev environment for the full error message and additional helpful warnings.");else{var f=[r,o,i,a,u,c],l=0;s=Error(n.replace(/%s/g,function(){return f[l++]})),s.name="Invariant Violation"}throw s.framesToPop=1,s}}}).call(e,n(21))},function(t){var e=t.exports={version:"2.5.1"};"number"==typeof __e&&(__e=e)},function(t,e,n){var r=n(7);t.exports=function(t,e){if(!r(t))return t;var n,o;if(e&&"function"==typeof(n=t.toString)&&!r(o=n.call(t)))return o;if("function"==typeof(n=t.valueOf)&&!r(o=n.call(t)))return o;if(!e&&"function"==typeof(n=t.toString)&&!r(o=n.call(t)))return o;throw TypeError("Can't convert object to primitive value")}},function(t){t.exports=function(t){if(void 0==t)throw TypeError("Can't call method on "+t);return t}},function(t){var e=Math.ceil,n=Math.floor;t.exports=function(t){return isNaN(t=+t)?0:(t>0?n:e)(t)}},function(t,e,n){var r=n(0),o=n(32),i=n(6);t.exports=function(t,e){var n=(o.Object||{})[t]||Object[t],a={};a[t]=e(n),r(r.S+r.F*i(function(){n(1)}),"Object",a)}},function(t,e,n){var r=n(28),o=n(59),i=n(13),a=n(11),u=n(108);t.exports=function(t,e){var n=1==t,c=2==t,s=3==t,f=4==t,l=6==t,p=5==t||l,h=e||u;return function(e,u,d){for(var v,y,m=i(e),g=o(m),b=r(u,d,3),_=a(g.length),w=0,O=n?h(e,_):c?h(e,0):void 0;_>w;w++)if((p||w in g)&&(v=g[w],y=b(v,w,m),t))if(n)O[w]=y;else if(y)switch(t){case 3:return!0;case 5:return v;case 6:return w;case 2:O.push(v)}else if(f)return!1;return l?-1:s||f?f:O}}},function(t,e,n){"use strict";if(n(9)){var r=n(45),o=n(5),i=n(6),a=n(0),u=n(78),c=n(114),s=n(28),f=n(51),l=n(43),p=n(18),h=n(53),d=n(35),v=n(11),y=n(159),m=n(47),g=n(33),b=n(17),_=n(61),w=n(7),O=n(13),x=n(105),P=n(48),E=n(26),k=n(49).f,j=n(107),S=n(44),T=n(8),C=n(37),M=n(69),N=n(76),R=n(110),A=n(56),L=n(73),I=n(50),F=n(109),D=n(149),U=n(10),W=n(25),B=U.f,z=W.f,q=o.RangeError,V=o.TypeError,G=o.Uint8Array,H=Array.prototype,Y=c.ArrayBuffer,$=c.DataView,K=C(0),J=C(2),Q=C(3),X=C(4),Z=C(5),tt=C(6),et=M(!0),nt=M(!1),rt=R.values,ot=R.keys,it=R.entries,at=H.lastIndexOf,ut=H.reduce,ct=H.reduceRight,st=H.join,ft=H.sort,lt=H.slice,pt=H.toString,ht=H.toLocaleString,dt=T("iterator"),vt=T("toStringTag"),yt=S("typed_constructor"),mt=S("def_constructor"),gt=u.CONSTR,bt=u.TYPED,_t=u.VIEW,wt=C(1,function(t,e){return kt(N(t,t[mt]),e)}),Ot=i(function(){return 1===new G(new Uint16Array([1]).buffer)[0]}),xt=!!G&&!!G.prototype.set&&i(function(){new G(1).set({})}),Pt=function(t,e){var n=d(t);if(0>n||n%e)throw q("Wrong offset!");return n},Et=function(t){if(w(t)&&bt in t)return t;throw V(t+" is not a typed array!")},kt=function(t,e){if(!(w(t)&&yt in t))throw V("It is not a typed array constructor!");return new t(e)},jt=function(t,e){return St(N(t,t[mt]),e)},St=function(t,e){for(var n=0,r=e.length,o=kt(t,r);r>n;)o[n]=e[n++];return o},Tt=function(t,e,n){B(t,e,{get:function(){return this._d[n]}})},Ct=function(t){var e,n,r,o,i,a,u=O(t),c=arguments.length,f=c>1?arguments[1]:void 0,l=void 0!==f,p=j(u);if(void 0!=p&&!x(p)){for(a=p.call(u),r=[],e=0;!(i=a.next()).done;e++)r.push(i.value);u=r}for(l&&c>2&&(f=s(f,arguments[2],2)),e=0,n=v(u.length),o=kt(this,n);n>e;e++)o[e]=l?f(u[e],e):u[e];return o},Mt=function(){for(var t=0,e=arguments.length,n=kt(this,e);e>t;)n[t]=arguments[t++];return n},Nt=!!G&&i(function(){ht.call(new G(1))}),Rt=function(){return ht.apply(Nt?lt.call(Et(this)):Et(this),arguments)},At={copyWithin:function(t,e){return D.call(Et(this),t,e,arguments.length>2?arguments[2]:void 0)},every:function(t){return X(Et(this),t,arguments.length>1?arguments[1]:void 0)},fill:function(){return F.apply(Et(this),arguments)},filter:function(t){return jt(this,J(Et(this),t,arguments.length>1?arguments[1]:void 0))},find:function(t){return Z(Et(this),t,arguments.length>1?arguments[1]:void 0)},findIndex:function(t){return tt(Et(this),t,arguments.length>1?arguments[1]:void 0)},forEach:function(t){K(Et(this),t,arguments.length>1?arguments[1]:void 0)},indexOf:function(t){return nt(Et(this),t,arguments.length>1?arguments[1]:void 0)},includes:function(t){return et(Et(this),t,arguments.length>1?arguments[1]:void 0)},join:function(){return st.apply(Et(this),arguments)},lastIndexOf:function(){return at.apply(Et(this),arguments)},map:function(t){return wt(Et(this),t,arguments.length>1?arguments[1]:void 0)},reduce:function(){return ut.apply(Et(this),arguments)},reduceRight:function(){return ct.apply(Et(this),arguments)},reverse:function(){for(var t,e=this,n=Et(e).length,r=Math.floor(n/2),o=0;r>o;)t=e[o],e[o++]=e[--n],e[n]=t;return e},some:function(t){return Q(Et(this),t,arguments.length>1?arguments[1]:void 0)},sort:function(t){return ft.call(Et(this),t)},subarray:function(t,e){var n=Et(this),r=n.length,o=m(t,r);return new(N(n,n[mt]))(n.buffer,n.byteOffset+o*n.BYTES_PER_ELEMENT,v((void 0===e?r:m(e,r))-o))}},Lt=function(t,e){return jt(this,lt.call(Et(this),t,e))},It=function(t){Et(this);var e=Pt(arguments[1],1),n=this.length,r=O(t),o=v(r.length),i=0;if(o+e>n)throw q("Wrong length!");for(;o>i;)this[e+i]=r[i++]},Ft={entries:function(){return it.call(Et(this))},keys:function(){return ot.call(Et(this))},values:function(){return rt.call(Et(this))}},Dt=function(t,e){return w(t)&&t[bt]&&"symbol"!=typeof e&&e in t&&+e+""==e+""},Ut=function(t,e){return Dt(t,e=g(e,!0))?l(2,t[e]):z(t,e)},Wt=function(t,e,n){return!(Dt(t,e=g(e,!0))&&w(n)&&b(n,"value"))||b(n,"get")||b(n,"set")||n.configurable||b(n,"writable")&&!n.writable||b(n,"enumerable")&&!n.enumerable?B(t,e,n):(t[e]=n.value,t)};gt||(W.f=Ut,U.f=Wt),a(a.S+a.F*!gt,"Object",{getOwnPropertyDescriptor:Ut,defineProperty:Wt}),i(function(){pt.call({})})&&(pt=ht=function(){return st.call(this)});var Bt=h({},At);h(Bt,Ft),p(Bt,dt,Ft.values),h(Bt,{slice:Lt,set:It,constructor:function(){},toString:pt,toLocaleString:Rt}),Tt(Bt,"buffer","b"),Tt(Bt,"byteOffset","o"),Tt(Bt,"byteLength","l"),Tt(Bt,"length","e"),B(Bt,vt,{get:function(){return this[bt]}}),t.exports=function(t,e,n,c){c=!!c;var s=t+(c?"Clamped":"")+"Array",l="get"+t,h="set"+t,d=o[s],m=d||{},g=d&&E(d),b=!d||!u.ABV,O={},x=d&&d.prototype,j=function(t,n){var r=t._d;return r.v[l](n*e+r.o,Ot)},S=function(t,n,r){var o=t._d;c&&(r=0>(r=Math.round(r))?0:r>255?255:255&r),o.v[h](n*e+o.o,r,Ot)},T=function(t,e){B(t,e,{get:function(){return j(this,e)},set:function(t){return S(this,e,t)},enumerable:!0})};b?(d=n(function(t,n,r,o){f(t,d,s,"_d");var i,a,u,c,l=0,h=0;if(w(n)){if(!(n instanceof Y||"ArrayBuffer"==(c=_(n))||"SharedArrayBuffer"==c))return bt in n?St(d,n):Ct.call(d,n);i=n,h=Pt(r,e);var m=n.byteLength;if(void 0===o){if(m%e)throw q("Wrong length!");if(0>(a=m-h))throw q("Wrong length!")}else if((a=v(o)*e)+h>m)throw q("Wrong length!");u=a/e}else u=y(n),a=u*e,i=new Y(a);for(p(t,"_d",{b:i,o:h,l:a,e:u,v:new $(i)});u>l;)T(t,l++)}),x=d.prototype=P(Bt),p(x,"constructor",d)):i(function(){d(1)})&&i(function(){new d(-1)})&&L(function(t){new d,new d(null),new d(1.5),new d(t)},!0)||(d=n(function(t,n,r,o){f(t,d,s);var i;return w(n)?n instanceof Y||"ArrayBuffer"==(i=_(n))||"SharedArrayBuffer"==i?void 0!==o?new m(n,Pt(r,e),o):void 0!==r?new m(n,Pt(r,e)):new m(n):bt in n?St(d,n):Ct.call(d,n):new m(y(n))}),K(g!==Function.prototype?k(m).concat(k(g)):k(m),function(t){t in d||p(d,t,m[t])}),d.prototype=x,r||(x.constructor=d));var C=x[dt],M=!!C&&("values"==C.name||void 0==C.name),N=Ft.values;p(d,yt,!0),p(x,bt,s),p(x,_t,!0),p(x,mt,d),(c?new d(1)[vt]==s:vt in x)||B(x,vt,{get:function(){return s}}),O[s]=d,a(a.G+a.W+a.F*(d!=m),O),a(a.S,s,{BYTES_PER_ELEMENT:e}),a(a.S+a.F*i(function(){m.of.call(d,1)}),s,{from:Ct,of:Mt}),"BYTES_PER_ELEMENT"in x||p(x,"BYTES_PER_ELEMENT",e),a(a.P,s,At),I(s),a(a.P+a.F*xt,s,{set:It}),a(a.P+a.F*!M,s,Ft),r||x.toString==pt||(x.toString=pt),a(a.P+a.F*i(function(){new d(1).slice()}),s,{slice:Lt}),a(a.P+a.F*(i(function(){return[1,2].toLocaleString()!=new d([1,2]).toLocaleString()})||!i(function(){x.toLocaleString.call([1,2])})),s,{toLocaleString:Rt}),A[s]=M?C:N,r||M||p(x,dt,N)}}else t.exports=function(){}},function(t,e,n){var r=n(154),o=n(0),i=n(68)("metadata"),a=i.store||(i.store=new(n(157))),u=function(t,e,n){var o=a.get(t);if(!o){if(!n)return;a.set(t,o=new r)}var i=o.get(e);if(!i){if(!n)return;o.set(e,i=new r)}return i};t.exports={store:a,map:u,has:function(t,e,n){var r=u(e,n,!1);return void 0!==r&&r.has(t)},get:function(t,e,n){var r=u(e,n,!1);return void 0===r?void 0:r.get(t)},set:function(t,e,n,r){u(n,r,!0).set(t,e)},keys:function(t,e){var n=u(t,e,!1),r=[];return n&&n.forEach(function(t,e){r.push(e)}),r},key:function(t){return void 0===t||"symbol"==typeof t?t:t+""},exp:function(t){o(o.S,"Reflect",t)}}},,function(t,e,n){var r=n(44)("meta"),o=n(7),i=n(17),a=n(10).f,u=0,c=Object.isExtensible||function(){return!0},s=!n(6)(function(){return c(Object.preventExtensions({}))}),f=function(t){a(t,r,{value:{i:"O"+ ++u,w:{}}})},l=function(t,e){if(!o(t))return"symbol"==typeof t?t:("string"==typeof t?"S":"P")+t;if(!i(t,r)){if(!c(t))return"F";if(!e)return"E";f(t)}return t[r].i},p=function(t,e){if(!i(t,r)){if(!c(t))return!0;if(!e)return!1;f(t)}return t[r].w},h=function(t){return s&&d.NEED&&c(t)&&!i(t,r)&&f(t),t},d=t.exports={KEY:r,NEED:!1,fastKey:l,getWeak:p,onFreeze:h}},function(t,e,n){var r=n(8)("unscopables"),o=Array.prototype;void 0==o[r]&&n(18)(o,r,{}),t.exports=function(t){o[r][t]=!0}},function(t){t.exports=function(t,e){return{enumerable:!(1&t),configurable:!(2&t),writable:!(4&t),value:e}}},function(t){var e=0,n=Math.random();t.exports=function(t){return"Symbol(".concat(void 0===t?"":t,")_",(++e+n).toString(36))}},function(t){t.exports=!1},function(t,e,n){var r=n(135),o=n(92);t.exports=Object.keys||function(t){return r(t,o)}},function(t,e,n){var r=n(35),o=Math.max,i=Math.min;t.exports=function(t,e){return t=r(t),0>t?o(t+e,0):i(t,e)}},function(t,e,n){var r=n(2),o=n(136),i=n(92),a=n(91)("IE_PROTO"),u=function(){},c=function(){var t,e=n(89)("iframe"),r=i.length;for(e.style.display="none",n(93).appendChild(e),e.src="javascript:",t=e.contentWindow.document,t.open(),t.write(" + `) diff --git a/vendor/vendor.json b/vendor/vendor.json index a21cd42b5ba..0804e41c82c 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -425,10 +425,10 @@ "revisionTime": "2016-05-04T02:26:26Z" }, { - "checksumSHA1": "hmMnLbqq5stRt+XMnqnVglxk8pg=", + "checksumSHA1": "c9aYXHdgU88JBRO151jAiA3C/qo=", "path": "github.com/laszlocph/drone-ui/dist", - "revision": "0dbf167b5c9efd822bbdf2723bdfa5b8499992ba", - "revisionTime": "2019-06-05T04:37:12Z" + "revision": "b285ace6d5f88a4eb29a01d8a72e211c536e0a31", + "revisionTime": "2019-06-19T08:13:15Z" }, { "path": "github.com/lib/pq", From 7420131019d2cb9fb204bff0f5a2b9898607dafe Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Wed, 19 Jun 2019 10:27:04 +0200 Subject: [PATCH 30/41] Feature release --- .drone.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.drone.yml b/.drone.yml index de443395551..7dc34c12063 100644 --- a/.drone.yml +++ b/.drone.yml @@ -113,7 +113,7 @@ pipeline: repo: laszlocloud/drone-oss-08-server dockerfile: Dockerfile.alpine secrets: [ docker_username, docker_password ] - tag: [ 0.8.95-alpine ] + tag: [ 0.8.95-multi-pipeline-alpine ] when: event: tag @@ -122,7 +122,7 @@ pipeline: repo: laszlocloud/drone-oss-08-agent dockerfile: Dockerfile.agent.alpine secrets: [ docker_username, docker_password ] - tag: [ 0.8.95-alpine ] + tag: [ 0.8.95-multi-pipeline-alpine ] when: event: tag @@ -130,7 +130,7 @@ pipeline: image: plugins/docker repo: laszlocloud/drone-oss-08-server secrets: [ docker_username, docker_password ] - tag: [ 0.8.95 ] + tag: [ 0.8.95-multi-pipeline ] when: event: tag @@ -139,7 +139,7 @@ pipeline: repo: laszlocloud/drone-oss-08-agent dockerfile: Dockerfile.agent secrets: [ docker_username, docker_password ] - tag: [ 0.8.95 ] + tag: [ 0.8.95-multi-pipeline ] when: event: tag From 806249ec1643f548e4332593dc649fcf5bc8c36a Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Wed, 19 Jun 2019 16:13:09 +0200 Subject: [PATCH 31/41] Migrating build configs to the new schema --- store/datastore/ddl/mysql/ddl_gen.go | 13 +++++++++++++ .../ddl/mysql/files/021_populate_build_config.sql | 4 ++++ store/datastore/ddl/postgres/ddl_gen.go | 13 +++++++++++++ .../postgres/files/021_populate_build_config.sql | 4 ++++ store/datastore/ddl/sqlite/ddl_gen.go | 13 +++++++++++++ .../ddl/sqlite/files/021_populate_build_config.sql | 4 ++++ 6 files changed, 51 insertions(+) create mode 100644 store/datastore/ddl/mysql/files/021_populate_build_config.sql create mode 100644 store/datastore/ddl/postgres/files/021_populate_build_config.sql create mode 100644 store/datastore/ddl/sqlite/files/021_populate_build_config.sql diff --git a/store/datastore/ddl/mysql/ddl_gen.go b/store/datastore/ddl/mysql/ddl_gen.go index 1522076be33..79da2b3f973 100644 --- a/store/datastore/ddl/mysql/ddl_gen.go +++ b/store/datastore/ddl/mysql/ddl_gen.go @@ -168,6 +168,10 @@ var migrations = []struct { name: "update-table-set-config-name", stmt: updateTableSetConfigName, }, + { + name: "populate-build-config", + stmt: populateBuildConfig, + }, } // Migrate performs the database migration. If the migration fails @@ -660,3 +664,12 @@ ALTER TABLE config ADD COLUMN config_name TEXT var updateTableSetConfigName = ` UPDATE config SET config_name = "drone" ` + +// +// 021_populate_build_config.sql +// + +var populateBuildConfig = ` +INSERT INTO build_config (config_id, build_id) +SELECT build_config_id, build_id FROM builds +` diff --git a/store/datastore/ddl/mysql/files/021_populate_build_config.sql b/store/datastore/ddl/mysql/files/021_populate_build_config.sql new file mode 100644 index 00000000000..e004b81c92d --- /dev/null +++ b/store/datastore/ddl/mysql/files/021_populate_build_config.sql @@ -0,0 +1,4 @@ +-- name: populate-build-config + +INSERT INTO build_config (config_id, build_id) +SELECT build_config_id, build_id FROM builds diff --git a/store/datastore/ddl/postgres/ddl_gen.go b/store/datastore/ddl/postgres/ddl_gen.go index e98fbe40d7b..1fe2828e9e0 100644 --- a/store/datastore/ddl/postgres/ddl_gen.go +++ b/store/datastore/ddl/postgres/ddl_gen.go @@ -168,6 +168,10 @@ var migrations = []struct { name: "update-table-set-config-name", stmt: updateTableSetConfigName, }, + { + name: "populate-build-config", + stmt: populateBuildConfig, + }, } // Migrate performs the database migration. If the migration fails @@ -662,3 +666,12 @@ ALTER TABLE config ADD COLUMN config_name TEXT var updateTableSetConfigName = ` UPDATE config SET config_name = 'drone' ` + +// +// 021_populate_build_config.sql +// + +var populateBuildConfig = ` +INSERT INTO build_config (config_id, build_id) +SELECT build_config_id, build_id FROM builds +` diff --git a/store/datastore/ddl/postgres/files/021_populate_build_config.sql b/store/datastore/ddl/postgres/files/021_populate_build_config.sql new file mode 100644 index 00000000000..e004b81c92d --- /dev/null +++ b/store/datastore/ddl/postgres/files/021_populate_build_config.sql @@ -0,0 +1,4 @@ +-- name: populate-build-config + +INSERT INTO build_config (config_id, build_id) +SELECT build_config_id, build_id FROM builds diff --git a/store/datastore/ddl/sqlite/ddl_gen.go b/store/datastore/ddl/sqlite/ddl_gen.go index b84546b5cc6..1da2de86b3a 100644 --- a/store/datastore/ddl/sqlite/ddl_gen.go +++ b/store/datastore/ddl/sqlite/ddl_gen.go @@ -172,6 +172,10 @@ var migrations = []struct { name: "update-table-set-config-name", stmt: updateTableSetConfigName, }, + { + name: "populate-build-config", + stmt: populateBuildConfig, + }, } // Migrate performs the database migration. If the migration fails @@ -661,3 +665,12 @@ ALTER TABLE config ADD COLUMN config_name TEXT var updateTableSetConfigName = ` UPDATE config SET config_name = "drone" ` + +// +// 021_populate_build_config.sql +// + +var populateBuildConfig = ` +INSERT INTO build_config (config_id, build_id) +SELECT build_config_id, build_id FROM builds +` diff --git a/store/datastore/ddl/sqlite/files/021_populate_build_config.sql b/store/datastore/ddl/sqlite/files/021_populate_build_config.sql new file mode 100644 index 00000000000..e004b81c92d --- /dev/null +++ b/store/datastore/ddl/sqlite/files/021_populate_build_config.sql @@ -0,0 +1,4 @@ +-- name: populate-build-config + +INSERT INTO build_config (config_id, build_id) +SELECT build_config_id, build_id FROM builds From 20a0714c1b97dd496771a548ca1ce657f942a62b Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Wed, 19 Jun 2019 20:18:00 +0200 Subject: [PATCH 32/41] Latest UI with bugifx --- .../laszlocph/drone-ui/dist/dist_gen.go | 36 +++++++++---------- vendor/vendor.json | 6 ++-- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/vendor/github.com/laszlocph/drone-ui/dist/dist_gen.go b/vendor/github.com/laszlocph/drone-ui/dist/dist_gen.go index abcbfed6ccc..2d4e27bc197 100644 --- a/vendor/github.com/laszlocph/drone-ui/dist/dist_gen.go +++ b/vendor/github.com/laszlocph/drone-ui/dist/dist_gen.go @@ -136,20 +136,20 @@ func MustLookup(path string) []byte { // Index of all files var files = map[string]file{ - "/static/bundle.eb8b0239cd35b72cf8c6.js": { + "/static/vendor.0710d064e53e3aa085b8.js": { data: file0, FileInfo: &fileInfo{ - name: "bundle.eb8b0239cd35b72cf8c6.js", - size: 368903, - modTime: time.Unix(1560931888, 0), + name: "vendor.0710d064e53e3aa085b8.js", + size: 272277, + modTime: time.Unix(1560968186, 0), }, }, - "/static/vendor.81cce2e51d1d8f062da7.js": { + "/static/bundle.40f2870c3156fb064370.js": { data: file1, FileInfo: &fileInfo{ - name: "vendor.81cce2e51d1d8f062da7.js", - size: 272277, - modTime: time.Unix(1560931888, 0), + name: "bundle.40f2870c3156fb064370.js", + size: 368905, + modTime: time.Unix(1560968186, 0), }, }, "/favicon.png": { @@ -157,7 +157,7 @@ var files = map[string]file{ FileInfo: &fileInfo{ name: "favicon.png", size: 1374, - modTime: time.Unix(1560931888, 0), + modTime: time.Unix(1560968186, 0), }, }, "/index.html": { @@ -165,7 +165,7 @@ var files = map[string]file{ FileInfo: &fileInfo{ name: "index.html", size: 388, - modTime: time.Unix(1560931888, 0), + modTime: time.Unix(1560968186, 0), }, }, } @@ -174,8 +174,11 @@ var files = map[string]file{ // embedded files. // -// /static/bundle.eb8b0239cd35b72cf8c6.js -var file0 = []byte(`webpackJsonp([0],[ +// /static/vendor.0710d064e53e3aa085b8.js +var file0 = []byte(`!function(t){function e(n){if(r[n])return r[n].exports;var o=r[n]={i:n,l:!1,exports:{}};return t[n].call(o.exports,o,o.exports,e),o.l=!0,o.exports}var n=window.webpackJsonp;window.webpackJsonp=function(r,i,a){for(var u,c,s,f=0,l=[];r.length>f;f++)c=r[f],o[c]&&l.push(o[c][0]),o[c]=0;for(u in i)Object.prototype.hasOwnProperty.call(i,u)&&(t[u]=i[u]);for(n&&n(r,i,a);l.length;)l.shift()();if(a)for(f=0;a.length>f;f++)s=e(e.s=a[f]);return s};var r={},o={1:0};e.e=function(t){function n(){u.onerror=u.onload=null,clearTimeout(c);var e=o[t];0!==e&&(e&&e[1](Error("Loading chunk "+t+" failed.")),o[t]=void 0)}var r=o[t];if(0===r)return new Promise(function(t){t()});if(r)return r[2];var i=new Promise(function(e,n){r=o[t]=[e,n]});r[2]=i;var a=document.getElementsByTagName("head")[0],u=document.createElement("script");u.type="text/javascript",u.charset="utf-8",u.async=!0,u.timeout=12e4,e.nc&&u.setAttribute("nonce",e.nc),u.src=e.p+""+t+".static/bundle."+{0:"40f2870c3156fb064370"}[t]+".js";var c=setTimeout(n,12e4);return u.onerror=u.onload=n,a.appendChild(u),i},e.m=t,e.c=r,e.d=function(t,n,r){e.o(t,n)||Object.defineProperty(t,n,{configurable:!1,enumerable:!0,get:r})},e.n=function(t){var n=t&&t.__esModule?function(){return t.default}:function(){return t};return e.d(n,"a",n),n},e.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},e.p="/",e.oe=function(t){throw t},e(e.s=584)}([function(t,e,n){var r=n(5),o=n(32),i=n(18),a=n(19),u=n(28),c=function(t,e,n){var s,f,l,p,h=t&c.F,d=t&c.G,v=t&c.S,y=t&c.P,m=t&c.B,g=d?r:v?r[e]||(r[e]={}):(r[e]||{}).prototype,b=d?o:o[e]||(o[e]={}),_=b.prototype||(b.prototype={});d&&(n=e);for(s in n)f=!h&&g&&void 0!==g[s],l=(f?g:n)[s],p=m&&f?u(l,r):y&&"function"==typeof l?u(Function.call,l):l,g&&a(g,s,l,t&c.U),b[s]!=l&&i(b,s,p),y&&_[s]!=l&&(_[s]=l)};r.core=o,c.F=1,c.G=2,c.S=4,c.P=8,c.B=16,c.W=32,c.U=64,c.R=128,t.exports=c},function(t,e,n){(function(e){!function(e,r){t.exports=r(n(12),n(118))}(0,function(t,n){function r(){return null}function o(t){var e=t.nodeName,n=t.attributes;t.attributes={},e.defaultProps&&w(t.attributes,e.defaultProps),n&&w(t.attributes,n)}function i(t,e){var n,r,o;if(e){for(o in e)if(n=B.test(o))break;if(n){r=t.attributes={};for(o in e)e.hasOwnProperty(o)&&(r[B.test(o)?o.replace(/([A-Z0-9])/,"-$1").toLowerCase():o]=e[o])}}}function a(t,e,r){var o=e&&e._preactCompatRendered&&e._preactCompatRendered.base;o&&o.parentNode!==e&&(o=null),!o&&e&&(o=e.firstElementChild);for(var i=e.childNodes.length;i--;)e.childNodes[i]!==o&&e.removeChild(e.childNodes[i]);var a=n.render(t,e,o);return e&&(e._preactCompatRendered=a&&(a._component||{base:a})),"function"==typeof r&&r(),a&&a._component||a}function u(t,e,r,o){var i=n.h(Y,{context:t.context},e),u=a(i,r),c=u._component||u.base;return o&&o.call(c,u),c}function c(t){var e=t._preactCompatRendered&&t._preactCompatRendered.base;return!(!e||e.parentNode!==t)&&(n.render(n.h(r),t,e),!0)}function s(t){return d.bind(null,t)}function f(t,e){for(var n=e||0;t.length>n;n++){var r=t[n];Array.isArray(r)?f(r):r&&"object"==typeof r&&!m(r)&&(r.props&&r.type||r.attributes&&r.nodeName||r.children)&&(t[n]=d(r.type||r.nodeName,r.props||r.attributes,r.children))}}function l(t){return"function"==typeof t&&!(t.prototype&&t.prototype.render)}function p(t){return E({displayName:t.displayName||t.name,render:function(){return t(this.props,this.context)}})}function h(t){var e=t[U];return e?!0===e?t:e:(e=p(t),Object.defineProperty(e,U,{configurable:!0,value:!0}),e.displayName=t.displayName,e.propTypes=t.propTypes,e.defaultProps=t.defaultProps,Object.defineProperty(t,U,{configurable:!0,value:e}),e)}function d(){for(var t=[],e=arguments.length;e--;)t[e]=arguments[e];return f(t,2),v(n.h.apply(void 0,t))}function v(t){t.preactCompatNormalized=!0,_(t),l(t.nodeName)&&(t.nodeName=h(t.nodeName));var e=t.attributes.ref,n=e&&typeof e;return!$||"string"!==n&&"number"!==n||(t.attributes.ref=g(e,$)),b(t),t}function y(t,e){for(var r=[],o=arguments.length-2;o-- >0;)r[o]=arguments[o+2];if(!m(t))return t;var i=t.attributes||t.props,a=n.h(t.nodeName||t.type,i,t.children||i&&i.children),u=[a,e];return r&&r.length?u.push(r):e&&e.children&&u.push(e.children),v(n.cloneElement.apply(void 0,u))}function m(t){return t&&(t instanceof V||t.$$typeof===D)}function g(t,e){return e._refProxies[t]||(e._refProxies[t]=function(n){e&&e.refs&&(e.refs[t]=n,null===n&&(delete e._refProxies[t],e=null))})}function b(t){var e=t.nodeName,n=t.attributes;if(n&&"string"==typeof e){var r={};for(var o in n)r[o.toLowerCase()]=o;if(r.ondoubleclick&&(n.ondblclick=n[r.ondoubleclick],delete n[r.ondoubleclick]),r.onchange&&("textarea"===e||"input"===e.toLowerCase()&&!/^fil|che|rad/i.test(n.type))){var i=r.oninput||"oninput";n[i]||(n[i]=C([n[i],n[r.onchange]]),delete n[r.onchange])}}}function _(t){var e=t.attributes||(t.attributes={});Z.enumerable="className"in e,e.className&&(e.class=e.className),Object.defineProperty(e,"className",Z)}function w(t){for(var e=arguments,n=1,r=void 0;arguments.length>n;n++)if(r=e[n])for(var o in r)r.hasOwnProperty(o)&&(t[o]=r[o]);return t}function O(t,e){for(var n in t)if(!(n in e))return!0;for(var r in e)if(t[r]!==e[r])return!0;return!1}function x(t){return t&&t.base||t}function P(){}function E(t){function e(t,e){S(this),L.call(this,t,e,z),M.call(this,t,e)}return t=w({constructor:e},t),t.mixins&&j(t,k(t.mixins)),t.statics&&w(e,t.statics),t.propTypes&&(e.propTypes=t.propTypes),t.defaultProps&&(e.defaultProps=t.defaultProps),t.getDefaultProps&&(e.defaultProps=t.getDefaultProps()),P.prototype=L.prototype,e.prototype=w(new P,t),e.displayName=t.displayName||"Component",e}function k(t){for(var e={},n=0;t.length>n;n++){var r=t[n];for(var o in r)r.hasOwnProperty(o)&&"function"==typeof r[o]&&(e[o]||(e[o]=[])).push(r[o])}return e}function j(t,e){for(var n in e)e.hasOwnProperty(n)&&(t[n]=C(e[n].concat(t[n]||K),"getDefaultProps"===n||"getInitialState"===n||"getChildContext"===n))}function S(t){for(var e in t){var n=t[e];"function"!=typeof n||n.__bound||W.hasOwnProperty(e)||((t[e]=n.bind(t)).__bound=!0)}}function T(t,e,n){if("string"==typeof e&&(e=t.constructor.prototype[e]),"function"==typeof e)return e.apply(t,n)}function C(t,e){return function(){for(var n,r=arguments,o=this,i=0;t.length>i;i++){var a=T(o,t[i],r);if(e&&null!=a){n||(n={});for(var u in a)a.hasOwnProperty(u)&&(n[u]=a[u])}else void 0!==a&&(n=a)}return n}}function M(t,e){N.call(this,t,e),this.componentWillReceiveProps=C([N,this.componentWillReceiveProps||"componentWillReceiveProps"]),this.render=C([N,R,this.render||"render",A])}function N(e){if(e){var n=e.children;if(n&&Array.isArray(n)&&1===n.length&&("string"==typeof n[0]||"function"==typeof n[0]||n[0]instanceof V)&&(e.children=n[0])&&"object"==typeof e.children&&(e.children.length=1,e.children[0]=e.children),q){var r="function"==typeof this?this:this.constructor,o=this.propTypes||r.propTypes,i=this.displayName||r.name;o&&t.checkPropTypes(o,e,"prop",i)}}}function R(){$=this}function A(){$===this&&($=null)}function L(t,e,r){n.Component.call(this,t,e),this.state=this.getInitialState?this.getInitialState():{},this.refs={},this._refProxies={},r!==z&&M.call(this,t,e)}function I(t,e){L.call(this,t,e)}t="default"in t?t.default:t;var F="a abbr address area article aside audio b base bdi bdo big blockquote body br button canvas caption cite code col colgroup data datalist dd del details dfn dialog div dl dt em embed fieldset figcaption figure footer form h1 h2 h3 h4 h5 h6 head header hgroup hr html i iframe img input ins kbd keygen label legend li link main map mark menu menuitem meta meter nav noscript object ol optgroup option output p param picture pre progress q rp rt ruby s samp script section select small source span strong style sub summary sup table tbody td textarea tfoot th thead time title tr track u ul var video wbr circle clipPath defs ellipse g image line linearGradient mask path pattern polygon polyline radialGradient rect stop svg text tspan".split(" "),D="undefined"!=typeof Symbol&&Symbol.for&&Symbol.for("react.element")||60103,U="undefined"!=typeof Symbol?Symbol.for("__preactCompatWrapper"):"__preactCompatWrapper",W={constructor:1,render:1,shouldComponentUpdate:1,componentWillReceiveProps:1,componentWillUpdate:1,componentDidUpdate:1,componentWillMount:1,componentDidMount:1,componentWillUnmount:1,componentDidUnmount:1},B=/^(?:accent|alignment|arabic|baseline|cap|clip|color|fill|flood|font|glyph|horiz|marker|overline|paint|stop|strikethrough|stroke|text|underline|unicode|units|v|vector|vert|word|writing|x)[A-Z]/,z={},q=void 0===e||!e.env||"production"!==e.env.NODE_ENV,V=n.h("a",null).constructor;V.prototype.$$typeof=D,V.prototype.preactCompatUpgraded=!1,V.prototype.preactCompatNormalized=!1,Object.defineProperty(V.prototype,"type",{get:function(){return this.nodeName},set:function(t){this.nodeName=t},configurable:!0}),Object.defineProperty(V.prototype,"props",{get:function(){return this.attributes},set:function(t){this.attributes=t},configurable:!0});var G=n.options.event;n.options.event=function(t){return G&&(t=G(t)),t.persist=Object,t.nativeEvent=t,t};var H=n.options.vnode;n.options.vnode=function(t){if(!t.preactCompatUpgraded){t.preactCompatUpgraded=!0;var e=t.nodeName,n=t.attributes=w({},t.attributes);"function"==typeof e?(!0===e[U]||e.prototype&&"isReactComponent"in e.prototype)&&(t.children&&t.children+""==""&&(t.children=void 0),t.children&&(n.children=t.children),t.preactCompatNormalized||v(t),o(t)):(t.children&&t.children+""==""&&(t.children=void 0),t.children&&(n.children=t.children),n.defaultValue&&(n.value||0===n.value||(n.value=n.defaultValue),delete n.defaultValue),i(t,n))}H&&H(t)};var Y=function(){};Y.prototype.getChildContext=function(){return this.props.context},Y.prototype.render=function(t){return t.children[0]};for(var $,K=[],J={map:function(t,e,n){return null==t?null:(t=J.toArray(t),n&&n!==t&&(e=e.bind(n)),t.map(e))},forEach:function(t,e,n){if(null==t)return null;t=J.toArray(t),n&&n!==t&&(e=e.bind(n)),t.forEach(e)},count:function(t){return t&&t.length||0},only:function(t){if(t=J.toArray(t),1!==t.length)throw Error("Children.only() expects only one child.");return t[0]},toArray:function(t){return null==t?[]:K.concat(t)}},Q={},X=F.length;X--;)Q[F[X]]=s(F[X]);var Z={configurable:!0,get:function(){return this.class},set:function(t){this.class=t}};return w(L.prototype=new n.Component,{constructor:L,isReactComponent:{},replaceState:function(t,e){var n=this;this.setState(t,e);for(var r in n.state)r in t||delete n.state[r]},getDOMNode:function(){return this.base},isMounted:function(){return!!this.base}}),P.prototype=L.prototype,I.prototype=new P,I.prototype.isPureReactComponent=!0,I.prototype.shouldComponentUpdate=function(t,e){return O(this.props,t)||O(this.state,e)},{version:"15.1.0",DOM:Q,PropTypes:t,Children:J,render:a,createClass:E,createFactory:s,createElement:d,cloneElement:y,isValidElement:m,findDOMNode:x,unmountComponentAtNode:c,Component:L,PureComponent:I,unstable_renderSubtreeIntoContainer:u,__spread:w}})}).call(e,n(21))},function(t,e,n){var r=n(7);t.exports=function(t){if(!r(t))throw TypeError(t+" is not an object!");return t}},,,function(t){var e=t.exports="undefined"!=typeof window&&window.Math==Math?window:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")();"number"==typeof __g&&(__g=e)},function(t){t.exports=function(t){try{return!!t()}catch(t){return!0}}},function(t){t.exports=function(t){return"object"==typeof t?null!==t:"function"==typeof t}},function(t,e,n){var r=n(68)("wks"),o=n(44),i=n(5).Symbol,a="function"==typeof i;(t.exports=function(t){return r[t]||(r[t]=a&&i[t]||(a?i:o)("Symbol."+t))}).store=r},function(t,e,n){t.exports=!n(6)(function(){return 7!=Object.defineProperty({},"a",{get:function(){return 7}}).a})},function(t,e,n){var r=n(2),o=n(133),i=n(33),a=Object.defineProperty;e.f=n(9)?Object.defineProperty:function(t,e,n){if(r(t),e=i(e,!0),r(n),o)try{return a(t,e,n)}catch(t){}if("get"in n||"set"in n)throw TypeError("Accessors not supported!");return"value"in n&&(t[e]=n.value),t}},function(t,e,n){var r=n(35),o=Math.min;t.exports=function(t){return t>0?o(r(t),9007199254740991):0}},function(t,e,n){(function(e){if("production"!==e.env.NODE_ENV){var r="function"==typeof Symbol&&Symbol.for&&Symbol.for("react.element")||60103,o=function(t){return"object"==typeof t&&null!==t&&t.$$typeof===r};t.exports=n(403)(o,!0)}else t.exports=n(405)()}).call(e,n(21))},function(t,e,n){var r=n(34);t.exports=function(t){return Object(r(t))}},function(t,e,n){"use strict";(function(e){var n=function(){};"production"!==e.env.NODE_ENV&&(n=function(t,e,n){var r=arguments.length;n=Array(r>2?r-2:0);for(var o=2;r>o;o++)n[o-2]=arguments[o];if(void 0===e)throw Error("` + "`" + `warning(condition, format, ...args)` + "`" + ` requires a warning message argument");if(10>e.length||/^[s\W]*$/.test(e))throw Error("The warning format should be able to uniquely identify this warning. Please, use a more descriptive format than: "+e);if(!t){var i=0,a="Warning: "+e.replace(/%s/g,function(){return n[i++]});try{throw Error(a)}catch(t){}}}),t.exports=n}).call(e,n(21))},function(t){t.exports=function(t){if("function"!=typeof t)throw TypeError(t+" is not a function!");return t}},function(t,e,n){var r=n(407);e.root=r.root,e.branch=r.branch},function(t){var e={}.hasOwnProperty;t.exports=function(t,n){return e.call(t,n)}},function(t,e,n){var r=n(10),o=n(43);t.exports=n(9)?function(t,e,n){return r.f(t,e,o(1,n))}:function(t,e,n){return t[e]=n,t}},function(t,e,n){var r=n(5),o=n(18),i=n(17),a=n(44)("src"),u=Function.toString,c=(""+u).split("toString");n(32).inspectSource=function(t){return u.call(t)},(t.exports=function(t,e,n,u){var s="function"==typeof n;s&&(i(n,"name")||o(n,"name",e)),t[e]!==n&&(s&&(i(n,a)||o(n,a,t[e]?""+t[e]:c.join(e+""))),t===r?t[e]=n:u?t[e]?t[e]=n:o(t,e,n):(delete t[e],o(t,e,n)))})(Function.prototype,"toString",function(){return"function"==typeof this&&this[a]||u.call(this)})},function(t,e,n){var r=n(0),o=n(6),i=n(34),a=/"/g,u=function(t,e,n,r){var o=i(t)+"",u="<"+e;return""!==n&&(u+=" "+n+'="'+(r+"").replace(a,""")+'"'),u+">"+o+""};t.exports=function(t,e){var n={};n[t]=e(u),r(r.P+r.F*o(function(){var e=""[t]('"');return e!==e.toLowerCase()||e.split('"').length>3}),"String",n)}},function(t){function e(){throw Error("setTimeout has not been defined")}function n(){throw Error("clearTimeout has not been defined")}function r(t){if(s===setTimeout)return setTimeout(t,0);if((s===e||!s)&&setTimeout)return s=setTimeout,setTimeout(t,0);try{return s(t,0)}catch(e){try{return s.call(null,t,0)}catch(e){return s.call(this,t,0)}}}function o(t){if(f===clearTimeout)return clearTimeout(t);if((f===n||!f)&&clearTimeout)return f=clearTimeout,clearTimeout(t);try{return f(t)}catch(e){try{return f.call(null,t)}catch(e){return f.call(this,t)}}}function i(){d&&p&&(d=!1,p.length?h=p.concat(h):v=-1,h.length&&a())}function a(){if(!d){var t=r(i);d=!0;for(var e=h.length;e;){for(p=h,h=[];++v1)for(var n=1;arguments.length>n;n++)e[n-1]=arguments[n];h.push(new u(t,e)),1!==h.length||d||r(a)},u.prototype.run=function(){this.fun.apply(null,this.array)},l.title="browser",l.browser=!0,l.env={},l.argv=[],l.version="",l.versions={},l.on=c,l.addListener=c,l.once=c,l.off=c,l.removeListener=c,l.removeAllListeners=c,l.emit=c,l.prependListener=c,l.prependOnceListener=c,l.listeners=function(){return[]},l.binding=function(){throw Error("process.binding is not supported")},l.cwd=function(){return"/"},l.chdir=function(){throw Error("process.chdir is not supported")},l.umask=function(){return 0}},,,function(t,e,n){var r=n(59),o=n(34);t.exports=function(t){return r(o(t))}},function(t,e,n){var r=n(60),o=n(43),i=n(24),a=n(33),u=n(17),c=n(133),s=Object.getOwnPropertyDescriptor;e.f=n(9)?s:function(t,e){if(t=i(t),e=a(e,!0),c)try{return s(t,e)}catch(t){}if(u(t,e))return o(!r.f.call(t,e),t[e])}},function(t,e,n){var r=n(17),o=n(13),i=n(91)("IE_PROTO"),a=Object.prototype;t.exports=Object.getPrototypeOf||function(t){return t=o(t),r(t,i)?t[i]:"function"==typeof t.constructor&&t instanceof t.constructor?t.constructor.prototype:t instanceof Object?a:null}},function(t,e,n){"use strict";Object.defineProperty(e,"__esModule",{value:!0});var r=n(424);n.d(e,"BrowserRouter",function(){return r.a});var o=n(426);n.d(e,"HashRouter",function(){return o.a});var i=n(176);n.d(e,"Link",function(){return i.a});var a=n(428);n.d(e,"MemoryRouter",function(){return a.a});var u=n(430);n.d(e,"NavLink",function(){return u.a});var c=n(433);n.d(e,"Prompt",function(){return c.a});var s=n(434);n.d(e,"Redirect",function(){return s.a});var f=n(178);n.d(e,"Route",function(){return f.a});var l=n(123);n.d(e,"Router",function(){return l.a});var p=n(439);n.d(e,"StaticRouter",function(){return p.a});var h=n(440);n.d(e,"Switch",function(){return h.a});var d=n(441);n.d(e,"matchPath",function(){return d.a});var v=n(442);n.d(e,"withRouter",function(){return v.a})},function(t,e,n){var r=n(15);t.exports=function(t,e,n){if(r(t),void 0===e)return t;switch(n){case 1:return function(n){return t.call(e,n)};case 2:return function(n,r){return t.call(e,n,r)};case 3:return function(n,r,o){return t.call(e,n,r,o)}}return function(){return t.apply(e,arguments)}}},function(t){var e={}.toString;t.exports=function(t){return e.call(t).slice(8,-1)}},function(t,e,n){"use strict";var r=n(6);t.exports=function(t,e){return!!t&&r(function(){e?t.call(null,function(){},1):t.call(null)})}},function(t,e,n){"use strict";(function(e){t.exports=function(t,n,r,o,i,a,u,c){if("production"!==e.env.NODE_ENV&&void 0===n)throw Error("invariant requires an error message argument");if(!t){var s;if(void 0===n)s=Error("Minified exception occurred; use the non-minified dev environment for the full error message and additional helpful warnings.");else{var f=[r,o,i,a,u,c],l=0;s=Error(n.replace(/%s/g,function(){return f[l++]})),s.name="Invariant Violation"}throw s.framesToPop=1,s}}}).call(e,n(21))},function(t){var e=t.exports={version:"2.5.1"};"number"==typeof __e&&(__e=e)},function(t,e,n){var r=n(7);t.exports=function(t,e){if(!r(t))return t;var n,o;if(e&&"function"==typeof(n=t.toString)&&!r(o=n.call(t)))return o;if("function"==typeof(n=t.valueOf)&&!r(o=n.call(t)))return o;if(!e&&"function"==typeof(n=t.toString)&&!r(o=n.call(t)))return o;throw TypeError("Can't convert object to primitive value")}},function(t){t.exports=function(t){if(void 0==t)throw TypeError("Can't call method on "+t);return t}},function(t){var e=Math.ceil,n=Math.floor;t.exports=function(t){return isNaN(t=+t)?0:(t>0?n:e)(t)}},function(t,e,n){var r=n(0),o=n(32),i=n(6);t.exports=function(t,e){var n=(o.Object||{})[t]||Object[t],a={};a[t]=e(n),r(r.S+r.F*i(function(){n(1)}),"Object",a)}},function(t,e,n){var r=n(28),o=n(59),i=n(13),a=n(11),u=n(108);t.exports=function(t,e){var n=1==t,c=2==t,s=3==t,f=4==t,l=6==t,p=5==t||l,h=e||u;return function(e,u,d){for(var v,y,m=i(e),g=o(m),b=r(u,d,3),_=a(g.length),w=0,O=n?h(e,_):c?h(e,0):void 0;_>w;w++)if((p||w in g)&&(v=g[w],y=b(v,w,m),t))if(n)O[w]=y;else if(y)switch(t){case 3:return!0;case 5:return v;case 6:return w;case 2:O.push(v)}else if(f)return!1;return l?-1:s||f?f:O}}},function(t,e,n){"use strict";if(n(9)){var r=n(45),o=n(5),i=n(6),a=n(0),u=n(78),c=n(114),s=n(28),f=n(51),l=n(43),p=n(18),h=n(53),d=n(35),v=n(11),y=n(159),m=n(47),g=n(33),b=n(17),_=n(61),w=n(7),O=n(13),x=n(105),P=n(48),E=n(26),k=n(49).f,j=n(107),S=n(44),T=n(8),C=n(37),M=n(69),N=n(76),R=n(110),A=n(56),L=n(73),I=n(50),F=n(109),D=n(149),U=n(10),W=n(25),B=U.f,z=W.f,q=o.RangeError,V=o.TypeError,G=o.Uint8Array,H=Array.prototype,Y=c.ArrayBuffer,$=c.DataView,K=C(0),J=C(2),Q=C(3),X=C(4),Z=C(5),tt=C(6),et=M(!0),nt=M(!1),rt=R.values,ot=R.keys,it=R.entries,at=H.lastIndexOf,ut=H.reduce,ct=H.reduceRight,st=H.join,ft=H.sort,lt=H.slice,pt=H.toString,ht=H.toLocaleString,dt=T("iterator"),vt=T("toStringTag"),yt=S("typed_constructor"),mt=S("def_constructor"),gt=u.CONSTR,bt=u.TYPED,_t=u.VIEW,wt=C(1,function(t,e){return kt(N(t,t[mt]),e)}),Ot=i(function(){return 1===new G(new Uint16Array([1]).buffer)[0]}),xt=!!G&&!!G.prototype.set&&i(function(){new G(1).set({})}),Pt=function(t,e){var n=d(t);if(0>n||n%e)throw q("Wrong offset!");return n},Et=function(t){if(w(t)&&bt in t)return t;throw V(t+" is not a typed array!")},kt=function(t,e){if(!(w(t)&&yt in t))throw V("It is not a typed array constructor!");return new t(e)},jt=function(t,e){return St(N(t,t[mt]),e)},St=function(t,e){for(var n=0,r=e.length,o=kt(t,r);r>n;)o[n]=e[n++];return o},Tt=function(t,e,n){B(t,e,{get:function(){return this._d[n]}})},Ct=function(t){var e,n,r,o,i,a,u=O(t),c=arguments.length,f=c>1?arguments[1]:void 0,l=void 0!==f,p=j(u);if(void 0!=p&&!x(p)){for(a=p.call(u),r=[],e=0;!(i=a.next()).done;e++)r.push(i.value);u=r}for(l&&c>2&&(f=s(f,arguments[2],2)),e=0,n=v(u.length),o=kt(this,n);n>e;e++)o[e]=l?f(u[e],e):u[e];return o},Mt=function(){for(var t=0,e=arguments.length,n=kt(this,e);e>t;)n[t]=arguments[t++];return n},Nt=!!G&&i(function(){ht.call(new G(1))}),Rt=function(){return ht.apply(Nt?lt.call(Et(this)):Et(this),arguments)},At={copyWithin:function(t,e){return D.call(Et(this),t,e,arguments.length>2?arguments[2]:void 0)},every:function(t){return X(Et(this),t,arguments.length>1?arguments[1]:void 0)},fill:function(){return F.apply(Et(this),arguments)},filter:function(t){return jt(this,J(Et(this),t,arguments.length>1?arguments[1]:void 0))},find:function(t){return Z(Et(this),t,arguments.length>1?arguments[1]:void 0)},findIndex:function(t){return tt(Et(this),t,arguments.length>1?arguments[1]:void 0)},forEach:function(t){K(Et(this),t,arguments.length>1?arguments[1]:void 0)},indexOf:function(t){return nt(Et(this),t,arguments.length>1?arguments[1]:void 0)},includes:function(t){return et(Et(this),t,arguments.length>1?arguments[1]:void 0)},join:function(){return st.apply(Et(this),arguments)},lastIndexOf:function(){return at.apply(Et(this),arguments)},map:function(t){return wt(Et(this),t,arguments.length>1?arguments[1]:void 0)},reduce:function(){return ut.apply(Et(this),arguments)},reduceRight:function(){return ct.apply(Et(this),arguments)},reverse:function(){for(var t,e=this,n=Et(e).length,r=Math.floor(n/2),o=0;r>o;)t=e[o],e[o++]=e[--n],e[n]=t;return e},some:function(t){return Q(Et(this),t,arguments.length>1?arguments[1]:void 0)},sort:function(t){return ft.call(Et(this),t)},subarray:function(t,e){var n=Et(this),r=n.length,o=m(t,r);return new(N(n,n[mt]))(n.buffer,n.byteOffset+o*n.BYTES_PER_ELEMENT,v((void 0===e?r:m(e,r))-o))}},Lt=function(t,e){return jt(this,lt.call(Et(this),t,e))},It=function(t){Et(this);var e=Pt(arguments[1],1),n=this.length,r=O(t),o=v(r.length),i=0;if(o+e>n)throw q("Wrong length!");for(;o>i;)this[e+i]=r[i++]},Ft={entries:function(){return it.call(Et(this))},keys:function(){return ot.call(Et(this))},values:function(){return rt.call(Et(this))}},Dt=function(t,e){return w(t)&&t[bt]&&"symbol"!=typeof e&&e in t&&+e+""==e+""},Ut=function(t,e){return Dt(t,e=g(e,!0))?l(2,t[e]):z(t,e)},Wt=function(t,e,n){return!(Dt(t,e=g(e,!0))&&w(n)&&b(n,"value"))||b(n,"get")||b(n,"set")||n.configurable||b(n,"writable")&&!n.writable||b(n,"enumerable")&&!n.enumerable?B(t,e,n):(t[e]=n.value,t)};gt||(W.f=Ut,U.f=Wt),a(a.S+a.F*!gt,"Object",{getOwnPropertyDescriptor:Ut,defineProperty:Wt}),i(function(){pt.call({})})&&(pt=ht=function(){return st.call(this)});var Bt=h({},At);h(Bt,Ft),p(Bt,dt,Ft.values),h(Bt,{slice:Lt,set:It,constructor:function(){},toString:pt,toLocaleString:Rt}),Tt(Bt,"buffer","b"),Tt(Bt,"byteOffset","o"),Tt(Bt,"byteLength","l"),Tt(Bt,"length","e"),B(Bt,vt,{get:function(){return this[bt]}}),t.exports=function(t,e,n,c){c=!!c;var s=t+(c?"Clamped":"")+"Array",l="get"+t,h="set"+t,d=o[s],m=d||{},g=d&&E(d),b=!d||!u.ABV,O={},x=d&&d.prototype,j=function(t,n){var r=t._d;return r.v[l](n*e+r.o,Ot)},S=function(t,n,r){var o=t._d;c&&(r=0>(r=Math.round(r))?0:r>255?255:255&r),o.v[h](n*e+o.o,r,Ot)},T=function(t,e){B(t,e,{get:function(){return j(this,e)},set:function(t){return S(this,e,t)},enumerable:!0})};b?(d=n(function(t,n,r,o){f(t,d,s,"_d");var i,a,u,c,l=0,h=0;if(w(n)){if(!(n instanceof Y||"ArrayBuffer"==(c=_(n))||"SharedArrayBuffer"==c))return bt in n?St(d,n):Ct.call(d,n);i=n,h=Pt(r,e);var m=n.byteLength;if(void 0===o){if(m%e)throw q("Wrong length!");if(0>(a=m-h))throw q("Wrong length!")}else if((a=v(o)*e)+h>m)throw q("Wrong length!");u=a/e}else u=y(n),a=u*e,i=new Y(a);for(p(t,"_d",{b:i,o:h,l:a,e:u,v:new $(i)});u>l;)T(t,l++)}),x=d.prototype=P(Bt),p(x,"constructor",d)):i(function(){d(1)})&&i(function(){new d(-1)})&&L(function(t){new d,new d(null),new d(1.5),new d(t)},!0)||(d=n(function(t,n,r,o){f(t,d,s);var i;return w(n)?n instanceof Y||"ArrayBuffer"==(i=_(n))||"SharedArrayBuffer"==i?void 0!==o?new m(n,Pt(r,e),o):void 0!==r?new m(n,Pt(r,e)):new m(n):bt in n?St(d,n):Ct.call(d,n):new m(y(n))}),K(g!==Function.prototype?k(m).concat(k(g)):k(m),function(t){t in d||p(d,t,m[t])}),d.prototype=x,r||(x.constructor=d));var C=x[dt],M=!!C&&("values"==C.name||void 0==C.name),N=Ft.values;p(d,yt,!0),p(x,bt,s),p(x,_t,!0),p(x,mt,d),(c?new d(1)[vt]==s:vt in x)||B(x,vt,{get:function(){return s}}),O[s]=d,a(a.G+a.W+a.F*(d!=m),O),a(a.S,s,{BYTES_PER_ELEMENT:e}),a(a.S+a.F*i(function(){m.of.call(d,1)}),s,{from:Ct,of:Mt}),"BYTES_PER_ELEMENT"in x||p(x,"BYTES_PER_ELEMENT",e),a(a.P,s,At),I(s),a(a.P+a.F*xt,s,{set:It}),a(a.P+a.F*!M,s,Ft),r||x.toString==pt||(x.toString=pt),a(a.P+a.F*i(function(){new d(1).slice()}),s,{slice:Lt}),a(a.P+a.F*(i(function(){return[1,2].toLocaleString()!=new d([1,2]).toLocaleString()})||!i(function(){x.toLocaleString.call([1,2])})),s,{toLocaleString:Rt}),A[s]=M?C:N,r||M||p(x,dt,N)}}else t.exports=function(){}},function(t,e,n){var r=n(154),o=n(0),i=n(68)("metadata"),a=i.store||(i.store=new(n(157))),u=function(t,e,n){var o=a.get(t);if(!o){if(!n)return;a.set(t,o=new r)}var i=o.get(e);if(!i){if(!n)return;o.set(e,i=new r)}return i};t.exports={store:a,map:u,has:function(t,e,n){var r=u(e,n,!1);return void 0!==r&&r.has(t)},get:function(t,e,n){var r=u(e,n,!1);return void 0===r?void 0:r.get(t)},set:function(t,e,n,r){u(n,r,!0).set(t,e)},keys:function(t,e){var n=u(t,e,!1),r=[];return n&&n.forEach(function(t,e){r.push(e)}),r},key:function(t){return void 0===t||"symbol"==typeof t?t:t+""},exp:function(t){o(o.S,"Reflect",t)}}},,function(t,e,n){var r=n(44)("meta"),o=n(7),i=n(17),a=n(10).f,u=0,c=Object.isExtensible||function(){return!0},s=!n(6)(function(){return c(Object.preventExtensions({}))}),f=function(t){a(t,r,{value:{i:"O"+ ++u,w:{}}})},l=function(t,e){if(!o(t))return"symbol"==typeof t?t:("string"==typeof t?"S":"P")+t;if(!i(t,r)){if(!c(t))return"F";if(!e)return"E";f(t)}return t[r].i},p=function(t,e){if(!i(t,r)){if(!c(t))return!0;if(!e)return!1;f(t)}return t[r].w},h=function(t){return s&&d.NEED&&c(t)&&!i(t,r)&&f(t),t},d=t.exports={KEY:r,NEED:!1,fastKey:l,getWeak:p,onFreeze:h}},function(t,e,n){var r=n(8)("unscopables"),o=Array.prototype;void 0==o[r]&&n(18)(o,r,{}),t.exports=function(t){o[r][t]=!0}},function(t){t.exports=function(t,e){return{enumerable:!(1&t),configurable:!(2&t),writable:!(4&t),value:e}}},function(t){var e=0,n=Math.random();t.exports=function(t){return"Symbol(".concat(void 0===t?"":t,")_",(++e+n).toString(36))}},function(t){t.exports=!1},function(t,e,n){var r=n(135),o=n(92);t.exports=Object.keys||function(t){return r(t,o)}},function(t,e,n){var r=n(35),o=Math.max,i=Math.min;t.exports=function(t,e){return t=r(t),0>t?o(t+e,0):i(t,e)}},function(t,e,n){var r=n(2),o=n(136),i=n(92),a=n(91)("IE_PROTO"),u=function(){},c=function(){var t,e=n(89)("iframe"),r=i.length;for(e.style.display="none",n(93).appendChild(e),e.src="javascript:",t=e.contentWindow.document,t.open(),t.write(" + `) diff --git a/vendor/vendor.json b/vendor/vendor.json index 0804e41c82c..43966a49cd1 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -425,10 +425,10 @@ "revisionTime": "2016-05-04T02:26:26Z" }, { - "checksumSHA1": "c9aYXHdgU88JBRO151jAiA3C/qo=", + "checksumSHA1": "GazIWQylHUrdwMKCxLWBEzHHZpM=", "path": "github.com/laszlocph/drone-ui/dist", - "revision": "b285ace6d5f88a4eb29a01d8a72e211c536e0a31", - "revisionTime": "2019-06-19T08:13:15Z" + "revision": "94da649274d2f27bf8b5a4e87d898f5055da24af", + "revisionTime": "2019-06-19T18:17:07Z" }, { "path": "github.com/lib/pq", From 788973d0d1dd45ed5677a742d41591b660b1cfcb Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Fri, 21 Jun 2019 11:52:36 +0200 Subject: [PATCH 33/41] A finished stage can unlock a pending one --- cncd/queue/fifo.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cncd/queue/fifo.go b/cncd/queue/fifo.go index 43d0c50970d..381631b0f96 100644 --- a/cncd/queue/fifo.go +++ b/cncd/queue/fifo.go @@ -107,6 +107,7 @@ func (q *fifo) Error(c context.Context, id string, err error) error { q.removeFromPending(id) } q.Unlock() + go q.process() return nil } From e411893c627e04cd018ad43e8d84dbf36e0b1384 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Fri, 21 Jun 2019 11:55:43 +0200 Subject: [PATCH 34/41] Small refactor --- cncd/queue/fifo.go | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/cncd/queue/fifo.go b/cncd/queue/fifo.go index 381631b0f96..f05c50a259d 100644 --- a/cncd/queue/fifo.go +++ b/cncd/queue/fifo.go @@ -192,15 +192,7 @@ func (q *fifo) process() { q.Lock() defer q.Unlock() - // TODO(bradrydzewski) move this to a helper function - // push items to the front of the queue if the item expires. - for id, state := range q.running { - if time.Now().After(state.deadline) { - q.pending.PushFront(state.item) - delete(q.running, id) - close(state.done) - } - } + q.resubmitExpiredBuilds() var next *list.Element loop: @@ -209,6 +201,7 @@ loop: task := e.Value.(*Task) logrus.Debugf("queue: trying to assign task: %v with deps %v", task.ID, task.Dependencies) if q.depsInQueue(task) { + logrus.Debugf("queue: skipping due to unmet dependencies %v", task.ID) continue } for w := range q.workers { @@ -230,12 +223,22 @@ loop: } } +func (q *fifo) resubmitExpiredBuilds() { + for id, state := range q.running { + if time.Now().After(state.deadline) { + q.pending.PushFront(state.item) + delete(q.running, id) + close(state.done) + } + } +} + func (q *fifo) depsInQueue(task *Task) bool { var next *list.Element for e := q.pending.Front(); e != nil; e = next { next = e.Next() possibleDep, ok := e.Value.(*Task) - logrus.Debugf("queue: in queue right now: %v", possibleDep.ID) + logrus.Debugf("queue: pending right now: %v", possibleDep.ID) for _, dep := range task.Dependencies { if ok && possibleDep.ID == dep { return true @@ -243,6 +246,7 @@ func (q *fifo) depsInQueue(task *Task) bool { } } for possibleDepID := range q.running { + logrus.Debugf("queue: running right now: %v", possibleDepID) for _, dep := range task.Dependencies { if possibleDepID == dep { return true From 580b7b272f2b73c1edfe6ecfce5b811fcd9d2eac Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Mon, 24 Jun 2019 09:04:30 +0200 Subject: [PATCH 35/41] Pipeline path configurable --- server/configFetcher.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/configFetcher.go b/server/configFetcher.go index 08b2150746b..1c064ca1b34 100644 --- a/server/configFetcher.go +++ b/server/configFetcher.go @@ -1,6 +1,7 @@ package server import ( + "strings" "time" "github.com/laszlocph/drone-oss-08/model" @@ -18,7 +19,8 @@ func (cf *configFetcher) Fetch() ([]*remote.FileMeta, error) { for i := 0; i < 5; i++ { select { case <-time.After(time.Second * time.Duration(i)): - file, fileerr := cf.remote_.File(cf.user, cf.repo, cf.build, cf.repo.Config) // either a file + // either a file + file, fileerr := cf.remote_.File(cf.user, cf.repo, cf.build, cf.repo.Config) if fileerr == nil { return []*remote.FileMeta{&remote.FileMeta{ Name: cf.repo.Config, @@ -26,7 +28,8 @@ func (cf *configFetcher) Fetch() ([]*remote.FileMeta, error) { }}, nil } - dir, direrr := cf.remote_.Dir(cf.user, cf.repo, cf.build, ".drone") // or a folder + // or a folder + dir, direrr := cf.remote_.Dir(cf.user, cf.repo, cf.build, strings.TrimSuffix(cf.repo.Config, "/")) if direrr != nil { return nil, direrr } From f386c4fca339085b50d227111170913cc8f45387 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Mon, 24 Jun 2019 09:25:21 +0200 Subject: [PATCH 36/41] Tests are deadlocking, probably sensitive to wait length --- cncd/logging/log_test.go | 4 ++-- cncd/pubsub/pub_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cncd/logging/log_test.go b/cncd/logging/log_test.go index 20d8e3d5526..2c235297ee4 100644 --- a/cncd/logging/log_test.go +++ b/cncd/logging/log_test.go @@ -30,7 +30,7 @@ func TestLogging(t *testing.T) { logger.Tail(ctx, testPath, func(entry ...*Entry) { wg.Done() }) }() - <-time.After(time.Millisecond) + <-time.After(500 * time.Millisecond) wg.Add(4) go func() { @@ -45,7 +45,7 @@ func TestLogging(t *testing.T) { logger.Tail(ctx, testPath, func(entry ...*Entry) { wg.Done() }) }() - <-time.After(time.Millisecond) + <-time.After(500 * time.Millisecond) wg.Wait() cancel() diff --git a/cncd/pubsub/pub_test.go b/cncd/pubsub/pub_test.go index 574c0e673ad..007f3192d39 100644 --- a/cncd/pubsub/pub_test.go +++ b/cncd/pubsub/pub_test.go @@ -30,7 +30,7 @@ func TestPubsub(t *testing.T) { broker.Subscribe(ctx, testTopic, func(message Message) { wg.Done() }) }() - <-time.After(time.Millisecond) + <-time.After(500 * time.Millisecond) if _, ok := broker.(*publisher).topics[testTopic]; !ok { t.Errorf("Expect topic registered with publisher") @@ -86,7 +86,7 @@ func TestSubscriptionClosed(t *testing.T) { wg.Done() }() - <-time.After(time.Millisecond) + <-time.After(500 * time.Millisecond) if _, ok := broker.(*publisher).topics[testTopic]; !ok { t.Errorf("Expect topic registered with publisher") From e239c2fe74de1b88221f43c997846e69e3ebd3b7 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Mon, 24 Jun 2019 11:19:12 +0200 Subject: [PATCH 37/41] Supporting skip_clone --- cncd/pipeline/pipeline/frontend/yaml/compiler/compiler.go | 4 ++-- cncd/pipeline/pipeline/frontend/yaml/config.go | 1 + cncd/pipeline/pipeline/frontend/yaml/config_test.go | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cncd/pipeline/pipeline/frontend/yaml/compiler/compiler.go b/cncd/pipeline/pipeline/frontend/yaml/compiler/compiler.go index d21311d366e..4049a10f958 100644 --- a/cncd/pipeline/pipeline/frontend/yaml/compiler/compiler.go +++ b/cncd/pipeline/pipeline/frontend/yaml/compiler/compiler.go @@ -97,7 +97,7 @@ func (c *Compiler) Compile(conf *yaml.Config) *backend.Config { } // add default clone step - if c.local == false && len(conf.Clone.Containers) == 0 { + if c.local == false && len(conf.Clone.Containers) == 0 && !conf.SkipClone { container := &yaml.Container{ Name: "clone", Image: "plugins/git:latest", @@ -118,7 +118,7 @@ func (c *Compiler) Compile(conf *yaml.Config) *backend.Config { stage.Steps = append(stage.Steps, step) config.Stages = append(config.Stages, stage) - } else if c.local == false { + } else if c.local == false && !conf.SkipClone { for i, container := range conf.Clone.Containers { if !container.Constraints.Match(c.metadata) { continue diff --git a/cncd/pipeline/pipeline/frontend/yaml/config.go b/cncd/pipeline/pipeline/frontend/yaml/config.go index afa8e05124f..9893dbf261c 100644 --- a/cncd/pipeline/pipeline/frontend/yaml/config.go +++ b/cncd/pipeline/pipeline/frontend/yaml/config.go @@ -24,6 +24,7 @@ type ( Labels libcompose.SliceorMap DependsOn []string `yaml:"depends_on,omitempty"` RunsOn []string `yaml:"runs_on,omitempty"` + SkipClone bool `yaml:"skip_clone"` } // Workspace defines a pipeline workspace. diff --git a/cncd/pipeline/pipeline/frontend/yaml/config_test.go b/cncd/pipeline/pipeline/frontend/yaml/config_test.go index afe9dcb3bdd..79c360d19f7 100644 --- a/cncd/pipeline/pipeline/frontend/yaml/config_test.go +++ b/cncd/pipeline/pipeline/frontend/yaml/config_test.go @@ -42,6 +42,7 @@ func TestParse(t *testing.T) { g.Assert(out.DependsOn[1]).Equal("test") g.Assert(out.RunsOn[0]).Equal("success") g.Assert(out.RunsOn[1]).Equal("failure") + g.Assert(out.SkipClone).Equal(false) }) // Check to make sure variable expansion works in yaml.MapSlice // g.It("Should unmarshal variables", func() { From 844c371f0096a7e0890e4ee8f23571dae17a8023 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Mon, 24 Jun 2019 22:35:50 +0200 Subject: [PATCH 38/41] Assign multiple pending tasks in one go --- cncd/queue/fifo.go | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/cncd/queue/fifo.go b/cncd/queue/fifo.go index f05c50a259d..c0428de9a53 100644 --- a/cncd/queue/fifo.go +++ b/cncd/queue/fifo.go @@ -59,9 +59,7 @@ func (q *fifo) PushAtOnce(c context.Context, tasks []*Task) error { q.pending.PushBack(task) } q.Unlock() - for range tasks { - go q.process() - } + go q.process() return nil } @@ -107,7 +105,6 @@ func (q *fifo) Error(c context.Context, id string, err error) error { q.removeFromPending(id) } q.Unlock() - go q.process() return nil } @@ -194,8 +191,21 @@ func (q *fifo) process() { q.resubmitExpiredBuilds() + for pending, worker := q.assignToWorker(); pending != nil && worker != nil; pending, worker = q.assignToWorker() { + task := pending.Value.(*Task) + delete(q.workers, worker) + q.pending.Remove(pending) + q.running[task.ID] = &entry{ + item: task, + done: make(chan bool), + deadline: time.Now().Add(q.extension), + } + worker.channel <- task + } +} + +func (q *fifo) assignToWorker() (*list.Element, *worker) { var next *list.Element -loop: for e := q.pending.Front(); e != nil; e = next { next = e.Next() task := e.Value.(*Task) @@ -204,23 +214,16 @@ loop: logrus.Debugf("queue: skipping due to unmet dependencies %v", task.ID) continue } + for w := range q.workers { if w.filter(task) { - delete(q.workers, w) - q.pending.Remove(e) - - q.running[task.ID] = &entry{ - item: task, - done: make(chan bool), - deadline: time.Now().Add(q.extension), - } - logrus.Debugf("queue: assigned task: %v with deps %v", task.ID, task.Dependencies) - w.channel <- task - break loop + return e, w } } } + + return nil, nil } func (q *fifo) resubmitExpiredBuilds() { From 84564f9ae561512383fc8d1ba062902ea2f8c991 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Tue, 25 Jun 2019 09:46:18 +0200 Subject: [PATCH 39/41] .drone.yml takes precedence over configured path. Allows incremental rollout --- server/configFetcher.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/configFetcher.go b/server/configFetcher.go index 1c064ca1b34..5b1ce7b8f3e 100644 --- a/server/configFetcher.go +++ b/server/configFetcher.go @@ -19,8 +19,17 @@ func (cf *configFetcher) Fetch() ([]*remote.FileMeta, error) { for i := 0; i < 5; i++ { select { case <-time.After(time.Second * time.Duration(i)): + // .drone.yml takes precedence + file, fileerr := cf.remote_.File(cf.user, cf.repo, cf.build, ".drone.yml") + if fileerr == nil { + return []*remote.FileMeta{&remote.FileMeta{ + Name: cf.repo.Config, + Data: file, + }}, nil + } + // either a file - file, fileerr := cf.remote_.File(cf.user, cf.repo, cf.build, cf.repo.Config) + file, fileerr = cf.remote_.File(cf.user, cf.repo, cf.build, cf.repo.Config) if fileerr == nil { return []*remote.FileMeta{&remote.FileMeta{ Name: cf.repo.Config, From be0d1d73fb5b3e687caacf67821424f2bd9d14db Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Tue, 25 Jun 2019 12:07:36 +0200 Subject: [PATCH 40/41] Task dependencies to survive restarts --- model/queue.go | 37 ++++++++++++------- store/datastore/ddl/mysql/ddl_gen.go | 20 ++++++++++ .../ddl/mysql/files/022_add_task_columns.sql | 6 +++ store/datastore/ddl/postgres/ddl_gen.go | 20 ++++++++++ .../postgres/files/022_add_task_columns.sql | 6 +++ store/datastore/ddl/sqlite/ddl_gen.go | 20 ++++++++++ .../ddl/sqlite/files/022_add_task_columns.sql | 6 +++ store/datastore/sql/mysql/files/task.sql | 2 + store/datastore/sql/mysql/sql_gen.go | 2 + store/datastore/sql/postgres/files/tasks.sql | 2 + store/datastore/sql/postgres/sql_gen.go | 2 + store/datastore/sql/sqlite/files/task.sql | 2 + store/datastore/sql/sqlite/sql_gen.go | 2 + 13 files changed, 114 insertions(+), 13 deletions(-) create mode 100644 store/datastore/ddl/mysql/files/022_add_task_columns.sql create mode 100644 store/datastore/ddl/postgres/files/022_add_task_columns.sql create mode 100644 store/datastore/ddl/sqlite/files/022_add_task_columns.sql diff --git a/model/queue.go b/model/queue.go index 433fdcc007e..3b595268745 100644 --- a/model/queue.go +++ b/model/queue.go @@ -23,9 +23,11 @@ import ( // Task defines scheduled pipeline Task. type Task struct { - ID string `meddler:"task_id"` - Data []byte `meddler:"task_data"` - Labels map[string]string `meddler:"task_labels,json"` + ID string `meddler:"task_id"` + Data []byte `meddler:"task_data"` + Labels map[string]string `meddler:"task_labels,json"` + Dependencies []string `meddler:"task_dependencies,json"` + RunOn []string `meddler:"task_run_on,json"` } // TaskStore defines storage for scheduled Tasks. @@ -39,13 +41,18 @@ type TaskStore interface { // ensures the task Queue can be restored when the system starts. func WithTaskStore(q queue.Queue, s TaskStore) queue.Queue { tasks, _ := s.TaskList() + toEnqueue := []*queue.Task{} for _, task := range tasks { - q.Push(context.Background(), &queue.Task{ - ID: task.ID, - Data: task.Data, - Labels: task.Labels, + toEnqueue = append(toEnqueue, &queue.Task{ + ID: task.ID, + Data: task.Data, + Labels: task.Labels, + Dependencies: task.Dependencies, + RunOn: task.RunOn, + DepStatus: make(map[string]bool), }) } + q.PushAtOnce(context.Background(), toEnqueue) return &persistentQueue{q, s} } @@ -57,9 +64,11 @@ type persistentQueue struct { // Push pushes a task to the tail of this queue. func (q *persistentQueue) Push(c context.Context, task *queue.Task) error { q.store.TaskInsert(&Task{ - ID: task.ID, - Data: task.Data, - Labels: task.Labels, + ID: task.ID, + Data: task.Data, + Labels: task.Labels, + Dependencies: task.Dependencies, + RunOn: task.RunOn, }) err := q.Queue.Push(c, task) if err != nil { @@ -72,9 +81,11 @@ func (q *persistentQueue) Push(c context.Context, task *queue.Task) error { func (q *persistentQueue) PushAtOnce(c context.Context, tasks []*queue.Task) error { for _, task := range tasks { q.store.TaskInsert(&Task{ - ID: task.ID, - Data: task.Data, - Labels: task.Labels, + ID: task.ID, + Data: task.Data, + Labels: task.Labels, + Dependencies: task.Dependencies, + RunOn: task.RunOn, }) } err := q.Queue.PushAtOnce(c, tasks) diff --git a/store/datastore/ddl/mysql/ddl_gen.go b/store/datastore/ddl/mysql/ddl_gen.go index 79da2b3f973..23c77f2de3e 100644 --- a/store/datastore/ddl/mysql/ddl_gen.go +++ b/store/datastore/ddl/mysql/ddl_gen.go @@ -172,6 +172,14 @@ var migrations = []struct { name: "populate-build-config", stmt: populateBuildConfig, }, + { + name: "alter-table-add-task-dependencies", + stmt: alterTableAddTaskDependencies, + }, + { + name: "alter-table-add-task-run-on", + stmt: alterTableAddTaskRunOn, + }, } // Migrate performs the database migration. If the migration fails @@ -673,3 +681,15 @@ var populateBuildConfig = ` INSERT INTO build_config (config_id, build_id) SELECT build_config_id, build_id FROM builds ` + +// +// 022_add_task_columns.sql +// + +var alterTableAddTaskDependencies = ` +ALTER TABLE tasks ADD COLUMN task_dependencies MEDIUMBLOB +` + +var alterTableAddTaskRunOn = ` +ALTER TABLE tasks ADD COLUMN task_run_on MEDIUMBLOB +` diff --git a/store/datastore/ddl/mysql/files/022_add_task_columns.sql b/store/datastore/ddl/mysql/files/022_add_task_columns.sql new file mode 100644 index 00000000000..b98a2180a8d --- /dev/null +++ b/store/datastore/ddl/mysql/files/022_add_task_columns.sql @@ -0,0 +1,6 @@ +-- name: alter-table-add-task-dependencies +ALTER TABLE tasks ADD COLUMN task_dependencies MEDIUMBLOB + +-- name: alter-table-add-task-run-on + +ALTER TABLE tasks ADD COLUMN task_run_on MEDIUMBLOB \ No newline at end of file diff --git a/store/datastore/ddl/postgres/ddl_gen.go b/store/datastore/ddl/postgres/ddl_gen.go index 1fe2828e9e0..06dfdd9e490 100644 --- a/store/datastore/ddl/postgres/ddl_gen.go +++ b/store/datastore/ddl/postgres/ddl_gen.go @@ -172,6 +172,14 @@ var migrations = []struct { name: "populate-build-config", stmt: populateBuildConfig, }, + { + name: "alter-table-add-task-dependencies", + stmt: alterTableAddTaskDependencies, + }, + { + name: "alter-table-add-task-run-on", + stmt: alterTableAddTaskRunOn, + }, } // Migrate performs the database migration. If the migration fails @@ -675,3 +683,15 @@ var populateBuildConfig = ` INSERT INTO build_config (config_id, build_id) SELECT build_config_id, build_id FROM builds ` + +// +// 022_add_task_columns.sql +// + +var alterTableAddTaskDependencies = ` +ALTER TABLE tasks ADD COLUMN task_dependencies BYTEA +` + +var alterTableAddTaskRunOn = ` +ALTER TABLE tasks ADD COLUMN task_run_on BYTEA +` diff --git a/store/datastore/ddl/postgres/files/022_add_task_columns.sql b/store/datastore/ddl/postgres/files/022_add_task_columns.sql new file mode 100644 index 00000000000..555e1d882b7 --- /dev/null +++ b/store/datastore/ddl/postgres/files/022_add_task_columns.sql @@ -0,0 +1,6 @@ +-- name: alter-table-add-task-dependencies +ALTER TABLE tasks ADD COLUMN task_dependencies BYTEA + +-- name: alter-table-add-task-run-on + +ALTER TABLE tasks ADD COLUMN task_run_on BYTEA diff --git a/store/datastore/ddl/sqlite/ddl_gen.go b/store/datastore/ddl/sqlite/ddl_gen.go index 1da2de86b3a..b898bbed97a 100644 --- a/store/datastore/ddl/sqlite/ddl_gen.go +++ b/store/datastore/ddl/sqlite/ddl_gen.go @@ -176,6 +176,14 @@ var migrations = []struct { name: "populate-build-config", stmt: populateBuildConfig, }, + { + name: "alter-table-add-task-dependencies", + stmt: alterTableAddTaskDependencies, + }, + { + name: "alter-table-add-task-run-on", + stmt: alterTableAddTaskRunOn, + }, } // Migrate performs the database migration. If the migration fails @@ -674,3 +682,15 @@ var populateBuildConfig = ` INSERT INTO build_config (config_id, build_id) SELECT build_config_id, build_id FROM builds ` + +// +// 022_add_task_columns.sql +// + +var alterTableAddTaskDependencies = ` +ALTER TABLE tasks ADD COLUMN task_dependencies BLOB +` + +var alterTableAddTaskRunOn = ` +ALTER TABLE tasks ADD COLUMN task_run_on BLOB +` diff --git a/store/datastore/ddl/sqlite/files/022_add_task_columns.sql b/store/datastore/ddl/sqlite/files/022_add_task_columns.sql new file mode 100644 index 00000000000..01b16d89a39 --- /dev/null +++ b/store/datastore/ddl/sqlite/files/022_add_task_columns.sql @@ -0,0 +1,6 @@ +-- name: alter-table-add-task-dependencies +ALTER TABLE tasks ADD COLUMN task_dependencies BLOB + +-- name: alter-table-add-task-run-on + +ALTER TABLE tasks ADD COLUMN task_run_on BLOB diff --git a/store/datastore/sql/mysql/files/task.sql b/store/datastore/sql/mysql/files/task.sql index 5b61c7c5e18..61890f721a4 100644 --- a/store/datastore/sql/mysql/files/task.sql +++ b/store/datastore/sql/mysql/files/task.sql @@ -4,6 +4,8 @@ SELECT task_id ,task_data ,task_labels +,task_dependencies +,task_run_on FROM tasks -- name: task-delete diff --git a/store/datastore/sql/mysql/sql_gen.go b/store/datastore/sql/mysql/sql_gen.go index d0669c3c940..73990a46094 100644 --- a/store/datastore/sql/mysql/sql_gen.go +++ b/store/datastore/sql/mysql/sql_gen.go @@ -554,6 +554,8 @@ SELECT task_id ,task_data ,task_labels +,task_dependencies +,task_run_on FROM tasks ` diff --git a/store/datastore/sql/postgres/files/tasks.sql b/store/datastore/sql/postgres/files/tasks.sql index 896c66e51a5..515ec5e8378 100644 --- a/store/datastore/sql/postgres/files/tasks.sql +++ b/store/datastore/sql/postgres/files/tasks.sql @@ -4,6 +4,8 @@ SELECT task_id ,task_data ,task_labels +,task_dependencies +,task_run_on FROM tasks -- name: task-delete diff --git a/store/datastore/sql/postgres/sql_gen.go b/store/datastore/sql/postgres/sql_gen.go index e64d661afe2..84eb1df9ac2 100644 --- a/store/datastore/sql/postgres/sql_gen.go +++ b/store/datastore/sql/postgres/sql_gen.go @@ -559,6 +559,8 @@ SELECT task_id ,task_data ,task_labels +,task_dependencies +,task_run_on FROM tasks ` diff --git a/store/datastore/sql/sqlite/files/task.sql b/store/datastore/sql/sqlite/files/task.sql index 5b61c7c5e18..61890f721a4 100644 --- a/store/datastore/sql/sqlite/files/task.sql +++ b/store/datastore/sql/sqlite/files/task.sql @@ -4,6 +4,8 @@ SELECT task_id ,task_data ,task_labels +,task_dependencies +,task_run_on FROM tasks -- name: task-delete diff --git a/store/datastore/sql/sqlite/sql_gen.go b/store/datastore/sql/sqlite/sql_gen.go index 4d2798f9dd8..08a2c1b432e 100644 --- a/store/datastore/sql/sqlite/sql_gen.go +++ b/store/datastore/sql/sqlite/sql_gen.go @@ -554,6 +554,8 @@ SELECT task_id ,task_data ,task_labels +,task_dependencies +,task_run_on FROM tasks ` From 8d79f8671e4494205948a65a21171c331c10b8d7 Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Tue, 25 Jun 2019 13:45:43 +0200 Subject: [PATCH 41/41] Fallback to default config. Allows incremental rollout of custom-path --- model/repo.go | 2 + server/configFetcher.go | 27 ++++++----- server/repo.go | 3 ++ store/datastore/ddl/mysql/ddl_gen.go | 20 ++++++++ .../files/023_add_repo_fallback_column.sql | 5 ++ store/datastore/ddl/postgres/ddl_gen.go | 20 ++++++++ .../files/023_add_repo_fallback_column.sql | 5 ++ store/datastore/ddl/sqlite/ddl_gen.go | 20 ++++++++ .../files/023_add_repo_fallback_column.sql | 5 ++ .../laszlocph/drone-ui/dist/dist_gen.go | 47 +++++++++++++------ vendor/vendor.json | 8 ++-- 11 files changed, 133 insertions(+), 29 deletions(-) create mode 100644 store/datastore/ddl/mysql/files/023_add_repo_fallback_column.sql create mode 100644 store/datastore/ddl/postgres/files/023_add_repo_fallback_column.sql create mode 100644 store/datastore/ddl/sqlite/files/023_add_repo_fallback_column.sql diff --git a/model/repo.go b/model/repo.go index 40a523931e9..c02e4a4ff58 100644 --- a/model/repo.go +++ b/model/repo.go @@ -55,6 +55,7 @@ type Repo struct { Config string `json:"config_file" meddler:"repo_config_path"` Hash string `json:"-" meddler:"repo_hash"` Perm *Perm `json:"-" meddler:"-"` + Fallback bool `json:"fallback" meddler:"repo_fallback"` } func (r *Repo) ResetVisibility() { @@ -105,4 +106,5 @@ type RepoPatch struct { AllowDeploy *bool `json:"allow_deploy,omitempty"` AllowTag *bool `json:"allow_tag,omitempty"` BuildCounter *int `json:"build_counter,omitempty"` + Fallback *bool `json:"fallback,omitempty"` } diff --git a/server/configFetcher.go b/server/configFetcher.go index 5b1ce7b8f3e..b4977ff061f 100644 --- a/server/configFetcher.go +++ b/server/configFetcher.go @@ -19,17 +19,8 @@ func (cf *configFetcher) Fetch() ([]*remote.FileMeta, error) { for i := 0; i < 5; i++ { select { case <-time.After(time.Second * time.Duration(i)): - // .drone.yml takes precedence - file, fileerr := cf.remote_.File(cf.user, cf.repo, cf.build, ".drone.yml") - if fileerr == nil { - return []*remote.FileMeta{&remote.FileMeta{ - Name: cf.repo.Config, - Data: file, - }}, nil - } - // either a file - file, fileerr = cf.remote_.File(cf.user, cf.repo, cf.build, cf.repo.Config) + file, fileerr := cf.remote_.File(cf.user, cf.repo, cf.build, cf.repo.Config) if fileerr == nil { return []*remote.FileMeta{&remote.FileMeta{ Name: cf.repo.Config, @@ -39,11 +30,23 @@ func (cf *configFetcher) Fetch() ([]*remote.FileMeta, error) { // or a folder dir, direrr := cf.remote_.Dir(cf.user, cf.repo, cf.build, strings.TrimSuffix(cf.repo.Config, "/")) - if direrr != nil { + + if direrr == nil { + return dir, nil + } else if !cf.repo.Fallback { return nil, direrr } - return dir, nil + // or fallback + file, fileerr = cf.remote_.File(cf.user, cf.repo, cf.build, ".drone.yml") + if fileerr != nil { + return nil, fileerr + } + + return []*remote.FileMeta{&remote.FileMeta{ + Name: cf.repo.Config, + Data: file, + }}, nil } } return []*remote.FileMeta{}, nil diff --git a/server/repo.go b/server/repo.go index 70c68269ecf..46c8ff07eb3 100644 --- a/server/repo.go +++ b/server/repo.go @@ -150,6 +150,9 @@ func PatchRepo(c *gin.Context) { if in.BuildCounter != nil { repo.Counter = *in.BuildCounter } + if in.Fallback != nil { + repo.Fallback = *in.Fallback + } err := store.UpdateRepo(c, repo) if err != nil { diff --git a/store/datastore/ddl/mysql/ddl_gen.go b/store/datastore/ddl/mysql/ddl_gen.go index 23c77f2de3e..4d4048f6f45 100644 --- a/store/datastore/ddl/mysql/ddl_gen.go +++ b/store/datastore/ddl/mysql/ddl_gen.go @@ -180,6 +180,14 @@ var migrations = []struct { name: "alter-table-add-task-run-on", stmt: alterTableAddTaskRunOn, }, + { + name: "alter-table-add-repo-fallback", + stmt: alterTableAddRepoFallback, + }, + { + name: "update-table-set-repo-fallback", + stmt: updateTableSetRepoFallback, + }, } // Migrate performs the database migration. If the migration fails @@ -693,3 +701,15 @@ ALTER TABLE tasks ADD COLUMN task_dependencies MEDIUMBLOB var alterTableAddTaskRunOn = ` ALTER TABLE tasks ADD COLUMN task_run_on MEDIUMBLOB ` + +// +// 023_add_repo_fallback_column.sql +// + +var alterTableAddRepoFallback = ` +ALTER TABLE repos ADD COLUMN repo_fallback BOOLEAN +` + +var updateTableSetRepoFallback = ` +UPDATE repos SET repo_fallback='false' +` diff --git a/store/datastore/ddl/mysql/files/023_add_repo_fallback_column.sql b/store/datastore/ddl/mysql/files/023_add_repo_fallback_column.sql new file mode 100644 index 00000000000..3e3384854fd --- /dev/null +++ b/store/datastore/ddl/mysql/files/023_add_repo_fallback_column.sql @@ -0,0 +1,5 @@ +-- name: alter-table-add-repo-fallback +ALTER TABLE repos ADD COLUMN repo_fallback BOOLEAN + +-- name: update-table-set-repo-fallback +UPDATE repos SET repo_fallback='false' diff --git a/store/datastore/ddl/postgres/ddl_gen.go b/store/datastore/ddl/postgres/ddl_gen.go index 06dfdd9e490..c55674bb9c2 100644 --- a/store/datastore/ddl/postgres/ddl_gen.go +++ b/store/datastore/ddl/postgres/ddl_gen.go @@ -180,6 +180,14 @@ var migrations = []struct { name: "alter-table-add-task-run-on", stmt: alterTableAddTaskRunOn, }, + { + name: "alter-table-add-repo-fallback", + stmt: alterTableAddRepoFallback, + }, + { + name: "update-table-set-repo-fallback", + stmt: updateTableSetRepoFallback, + }, } // Migrate performs the database migration. If the migration fails @@ -695,3 +703,15 @@ ALTER TABLE tasks ADD COLUMN task_dependencies BYTEA var alterTableAddTaskRunOn = ` ALTER TABLE tasks ADD COLUMN task_run_on BYTEA ` + +// +// 023_add_repo_fallback_column.sql +// + +var alterTableAddRepoFallback = ` +ALTER TABLE repos ADD COLUMN repo_fallback BOOLEAN +` + +var updateTableSetRepoFallback = ` +UPDATE repos SET repo_fallback='false' +` diff --git a/store/datastore/ddl/postgres/files/023_add_repo_fallback_column.sql b/store/datastore/ddl/postgres/files/023_add_repo_fallback_column.sql new file mode 100644 index 00000000000..3e3384854fd --- /dev/null +++ b/store/datastore/ddl/postgres/files/023_add_repo_fallback_column.sql @@ -0,0 +1,5 @@ +-- name: alter-table-add-repo-fallback +ALTER TABLE repos ADD COLUMN repo_fallback BOOLEAN + +-- name: update-table-set-repo-fallback +UPDATE repos SET repo_fallback='false' diff --git a/store/datastore/ddl/sqlite/ddl_gen.go b/store/datastore/ddl/sqlite/ddl_gen.go index b898bbed97a..d5fe8eb5c70 100644 --- a/store/datastore/ddl/sqlite/ddl_gen.go +++ b/store/datastore/ddl/sqlite/ddl_gen.go @@ -184,6 +184,14 @@ var migrations = []struct { name: "alter-table-add-task-run-on", stmt: alterTableAddTaskRunOn, }, + { + name: "alter-table-add-repo-fallback", + stmt: alterTableAddRepoFallback, + }, + { + name: "update-table-set-repo-fallback", + stmt: updateTableSetRepoFallback, + }, } // Migrate performs the database migration. If the migration fails @@ -694,3 +702,15 @@ ALTER TABLE tasks ADD COLUMN task_dependencies BLOB var alterTableAddTaskRunOn = ` ALTER TABLE tasks ADD COLUMN task_run_on BLOB ` + +// +// 023_add_repo_fallback_column.sql +// + +var alterTableAddRepoFallback = ` +ALTER TABLE repos ADD COLUMN repo_fallback BOOLEAN +` + +var updateTableSetRepoFallback = ` +UPDATE repos SET repo_fallback='false' +` diff --git a/store/datastore/ddl/sqlite/files/023_add_repo_fallback_column.sql b/store/datastore/ddl/sqlite/files/023_add_repo_fallback_column.sql new file mode 100644 index 00000000000..3e3384854fd --- /dev/null +++ b/store/datastore/ddl/sqlite/files/023_add_repo_fallback_column.sql @@ -0,0 +1,5 @@ +-- name: alter-table-add-repo-fallback +ALTER TABLE repos ADD COLUMN repo_fallback BOOLEAN + +-- name: update-table-set-repo-fallback +UPDATE repos SET repo_fallback='false' diff --git a/vendor/github.com/laszlocph/drone-ui/dist/dist_gen.go b/vendor/github.com/laszlocph/drone-ui/dist/dist_gen.go index 2f01ebe6114..cd5992367d9 100644 --- a/vendor/github.com/laszlocph/drone-ui/dist/dist_gen.go +++ b/vendor/github.com/laszlocph/drone-ui/dist/dist_gen.go @@ -136,20 +136,20 @@ func MustLookup(path string) []byte { // Index of all files var files = map[string]file{ - "/static/bundle.6423f8f41f5c95f505c6.js": { + "/static/bundle.ed8654564546820f8c42.js": { data: file0, FileInfo: &fileInfo{ - name: "bundle.6423f8f41f5c95f505c6.js", - size: 369462, - modTime: time.Unix(1561358593, 0), + name: "bundle.ed8654564546820f8c42.js", + size: 370027, + modTime: time.Unix(1561462765, 0), }, }, - "/static/vendor.b12b1d6de7354306faba.js": { + "/static/vendor.eeeab6d65a2d5969abed.js": { data: file1, FileInfo: &fileInfo{ - name: "vendor.b12b1d6de7354306faba.js", + name: "vendor.eeeab6d65a2d5969abed.js", size: 272277, - modTime: time.Unix(1561358593, 0), + modTime: time.Unix(1561462765, 0), }, }, "/favicon.png": { @@ -157,7 +157,7 @@ var files = map[string]file{ FileInfo: &fileInfo{ name: "favicon.png", size: 1374, - modTime: time.Unix(1561358593, 0), + modTime: time.Unix(1561462764, 0), }, }, "/index.html": { @@ -165,7 +165,7 @@ var files = map[string]file{ FileInfo: &fileInfo{ name: "index.html", size: 388, - modTime: time.Unix(1561358593, 0), + modTime: time.Unix(1561462764, 0), }, }, } @@ -174,7 +174,7 @@ var files = map[string]file{ // embedded files. // -// /static/bundle.6423f8f41f5c95f505c6.js +// /static/bundle.ed8654564546820f8c42.js var file0 = []byte(`webpackJsonp([0],[ /* 0 */, /* 1 */, @@ -6780,6 +6780,7 @@ var Settings = (_dec = (0, _higherOrder.branch)(binding), (0, _inject.inject)(_c _this.handleVisibilityChange = _this.handleVisibilityChange.bind(_this); _this.handleTimeoutChange = _this.handleTimeoutChange.bind(_this); _this.handlePathChange = _this.handlePathChange.bind(_this); + _this.handleFallbackChange = _this.handleFallbackChange.bind(_this); _this.handleChange = _this.handleChange.bind(_this); return _this; } @@ -6827,7 +6828,21 @@ var Settings = (_dec = (0, _higherOrder.branch)(binding), (0, _inject.inject)(_c type: "text", value: repo.config_file, onBlur: this.handlePathChange - }) + }), + _react2["default"].createElement( + "label", + null, + _react2["default"].createElement("input", { + type: "checkbox", + checked: repo.fallback, + onChange: this.handleFallbackChange + }), + _react2["default"].createElement( + "span", + null, + "Fallback to .drone.yml if path not exists" + ) + ) ) ), _react2["default"].createElement( @@ -7063,6 +7078,10 @@ var Settings = (_dec = (0, _higherOrder.branch)(binding), (0, _inject.inject)(_c this.handleChange("config_file", e.target.value); }; + Settings.prototype.handleFallbackChange = function handleFallbackChange(e) { + this.handleChange("fallback", e.target.checked); + }; + Settings.prototype.handleChange = function handleChange(prop, value) { var _props2 = this.props, dispatch = _props2.dispatch, @@ -10809,8 +10828,8 @@ exports.push([module.i, " {\n}\ndiv,\nspan {\n font-family: 'Roboto';\n font-s /***/ }) ],[201]);`) -// /static/vendor.b12b1d6de7354306faba.js -var file1 = []byte(`!function(t){function e(n){if(r[n])return r[n].exports;var o=r[n]={i:n,l:!1,exports:{}};return t[n].call(o.exports,o,o.exports,e),o.l=!0,o.exports}var n=window.webpackJsonp;window.webpackJsonp=function(r,i,a){for(var u,c,s,f=0,l=[];r.length>f;f++)c=r[f],o[c]&&l.push(o[c][0]),o[c]=0;for(u in i)Object.prototype.hasOwnProperty.call(i,u)&&(t[u]=i[u]);for(n&&n(r,i,a);l.length;)l.shift()();if(a)for(f=0;a.length>f;f++)s=e(e.s=a[f]);return s};var r={},o={1:0};e.e=function(t){function n(){u.onerror=u.onload=null,clearTimeout(c);var e=o[t];0!==e&&(e&&e[1](Error("Loading chunk "+t+" failed.")),o[t]=void 0)}var r=o[t];if(0===r)return new Promise(function(t){t()});if(r)return r[2];var i=new Promise(function(e,n){r=o[t]=[e,n]});r[2]=i;var a=document.getElementsByTagName("head")[0],u=document.createElement("script");u.type="text/javascript",u.charset="utf-8",u.async=!0,u.timeout=12e4,e.nc&&u.setAttribute("nonce",e.nc),u.src=e.p+""+t+".static/bundle."+{0:"6423f8f41f5c95f505c6"}[t]+".js";var c=setTimeout(n,12e4);return u.onerror=u.onload=n,a.appendChild(u),i},e.m=t,e.c=r,e.d=function(t,n,r){e.o(t,n)||Object.defineProperty(t,n,{configurable:!1,enumerable:!0,get:r})},e.n=function(t){var n=t&&t.__esModule?function(){return t.default}:function(){return t};return e.d(n,"a",n),n},e.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},e.p="/",e.oe=function(t){throw t},e(e.s=584)}([function(t,e,n){var r=n(5),o=n(32),i=n(18),a=n(19),u=n(28),c=function(t,e,n){var s,f,l,p,h=t&c.F,d=t&c.G,v=t&c.S,y=t&c.P,m=t&c.B,g=d?r:v?r[e]||(r[e]={}):(r[e]||{}).prototype,b=d?o:o[e]||(o[e]={}),_=b.prototype||(b.prototype={});d&&(n=e);for(s in n)f=!h&&g&&void 0!==g[s],l=(f?g:n)[s],p=m&&f?u(l,r):y&&"function"==typeof l?u(Function.call,l):l,g&&a(g,s,l,t&c.U),b[s]!=l&&i(b,s,p),y&&_[s]!=l&&(_[s]=l)};r.core=o,c.F=1,c.G=2,c.S=4,c.P=8,c.B=16,c.W=32,c.U=64,c.R=128,t.exports=c},function(t,e,n){(function(e){!function(e,r){t.exports=r(n(12),n(118))}(0,function(t,n){function r(){return null}function o(t){var e=t.nodeName,n=t.attributes;t.attributes={},e.defaultProps&&w(t.attributes,e.defaultProps),n&&w(t.attributes,n)}function i(t,e){var n,r,o;if(e){for(o in e)if(n=B.test(o))break;if(n){r=t.attributes={};for(o in e)e.hasOwnProperty(o)&&(r[B.test(o)?o.replace(/([A-Z0-9])/,"-$1").toLowerCase():o]=e[o])}}}function a(t,e,r){var o=e&&e._preactCompatRendered&&e._preactCompatRendered.base;o&&o.parentNode!==e&&(o=null),!o&&e&&(o=e.firstElementChild);for(var i=e.childNodes.length;i--;)e.childNodes[i]!==o&&e.removeChild(e.childNodes[i]);var a=n.render(t,e,o);return e&&(e._preactCompatRendered=a&&(a._component||{base:a})),"function"==typeof r&&r(),a&&a._component||a}function u(t,e,r,o){var i=n.h(Y,{context:t.context},e),u=a(i,r),c=u._component||u.base;return o&&o.call(c,u),c}function c(t){var e=t._preactCompatRendered&&t._preactCompatRendered.base;return!(!e||e.parentNode!==t)&&(n.render(n.h(r),t,e),!0)}function s(t){return d.bind(null,t)}function f(t,e){for(var n=e||0;t.length>n;n++){var r=t[n];Array.isArray(r)?f(r):r&&"object"==typeof r&&!m(r)&&(r.props&&r.type||r.attributes&&r.nodeName||r.children)&&(t[n]=d(r.type||r.nodeName,r.props||r.attributes,r.children))}}function l(t){return"function"==typeof t&&!(t.prototype&&t.prototype.render)}function p(t){return E({displayName:t.displayName||t.name,render:function(){return t(this.props,this.context)}})}function h(t){var e=t[U];return e?!0===e?t:e:(e=p(t),Object.defineProperty(e,U,{configurable:!0,value:!0}),e.displayName=t.displayName,e.propTypes=t.propTypes,e.defaultProps=t.defaultProps,Object.defineProperty(t,U,{configurable:!0,value:e}),e)}function d(){for(var t=[],e=arguments.length;e--;)t[e]=arguments[e];return f(t,2),v(n.h.apply(void 0,t))}function v(t){t.preactCompatNormalized=!0,_(t),l(t.nodeName)&&(t.nodeName=h(t.nodeName));var e=t.attributes.ref,n=e&&typeof e;return!$||"string"!==n&&"number"!==n||(t.attributes.ref=g(e,$)),b(t),t}function y(t,e){for(var r=[],o=arguments.length-2;o-- >0;)r[o]=arguments[o+2];if(!m(t))return t;var i=t.attributes||t.props,a=n.h(t.nodeName||t.type,i,t.children||i&&i.children),u=[a,e];return r&&r.length?u.push(r):e&&e.children&&u.push(e.children),v(n.cloneElement.apply(void 0,u))}function m(t){return t&&(t instanceof V||t.$$typeof===D)}function g(t,e){return e._refProxies[t]||(e._refProxies[t]=function(n){e&&e.refs&&(e.refs[t]=n,null===n&&(delete e._refProxies[t],e=null))})}function b(t){var e=t.nodeName,n=t.attributes;if(n&&"string"==typeof e){var r={};for(var o in n)r[o.toLowerCase()]=o;if(r.ondoubleclick&&(n.ondblclick=n[r.ondoubleclick],delete n[r.ondoubleclick]),r.onchange&&("textarea"===e||"input"===e.toLowerCase()&&!/^fil|che|rad/i.test(n.type))){var i=r.oninput||"oninput";n[i]||(n[i]=C([n[i],n[r.onchange]]),delete n[r.onchange])}}}function _(t){var e=t.attributes||(t.attributes={});Z.enumerable="className"in e,e.className&&(e.class=e.className),Object.defineProperty(e,"className",Z)}function w(t){for(var e=arguments,n=1,r=void 0;arguments.length>n;n++)if(r=e[n])for(var o in r)r.hasOwnProperty(o)&&(t[o]=r[o]);return t}function O(t,e){for(var n in t)if(!(n in e))return!0;for(var r in e)if(t[r]!==e[r])return!0;return!1}function x(t){return t&&t.base||t}function P(){}function E(t){function e(t,e){S(this),L.call(this,t,e,z),M.call(this,t,e)}return t=w({constructor:e},t),t.mixins&&j(t,k(t.mixins)),t.statics&&w(e,t.statics),t.propTypes&&(e.propTypes=t.propTypes),t.defaultProps&&(e.defaultProps=t.defaultProps),t.getDefaultProps&&(e.defaultProps=t.getDefaultProps()),P.prototype=L.prototype,e.prototype=w(new P,t),e.displayName=t.displayName||"Component",e}function k(t){for(var e={},n=0;t.length>n;n++){var r=t[n];for(var o in r)r.hasOwnProperty(o)&&"function"==typeof r[o]&&(e[o]||(e[o]=[])).push(r[o])}return e}function j(t,e){for(var n in e)e.hasOwnProperty(n)&&(t[n]=C(e[n].concat(t[n]||K),"getDefaultProps"===n||"getInitialState"===n||"getChildContext"===n))}function S(t){for(var e in t){var n=t[e];"function"!=typeof n||n.__bound||W.hasOwnProperty(e)||((t[e]=n.bind(t)).__bound=!0)}}function T(t,e,n){if("string"==typeof e&&(e=t.constructor.prototype[e]),"function"==typeof e)return e.apply(t,n)}function C(t,e){return function(){for(var n,r=arguments,o=this,i=0;t.length>i;i++){var a=T(o,t[i],r);if(e&&null!=a){n||(n={});for(var u in a)a.hasOwnProperty(u)&&(n[u]=a[u])}else void 0!==a&&(n=a)}return n}}function M(t,e){N.call(this,t,e),this.componentWillReceiveProps=C([N,this.componentWillReceiveProps||"componentWillReceiveProps"]),this.render=C([N,R,this.render||"render",A])}function N(e){if(e){var n=e.children;if(n&&Array.isArray(n)&&1===n.length&&("string"==typeof n[0]||"function"==typeof n[0]||n[0]instanceof V)&&(e.children=n[0])&&"object"==typeof e.children&&(e.children.length=1,e.children[0]=e.children),q){var r="function"==typeof this?this:this.constructor,o=this.propTypes||r.propTypes,i=this.displayName||r.name;o&&t.checkPropTypes(o,e,"prop",i)}}}function R(){$=this}function A(){$===this&&($=null)}function L(t,e,r){n.Component.call(this,t,e),this.state=this.getInitialState?this.getInitialState():{},this.refs={},this._refProxies={},r!==z&&M.call(this,t,e)}function I(t,e){L.call(this,t,e)}t="default"in t?t.default:t;var F="a abbr address area article aside audio b base bdi bdo big blockquote body br button canvas caption cite code col colgroup data datalist dd del details dfn dialog div dl dt em embed fieldset figcaption figure footer form h1 h2 h3 h4 h5 h6 head header hgroup hr html i iframe img input ins kbd keygen label legend li link main map mark menu menuitem meta meter nav noscript object ol optgroup option output p param picture pre progress q rp rt ruby s samp script section select small source span strong style sub summary sup table tbody td textarea tfoot th thead time title tr track u ul var video wbr circle clipPath defs ellipse g image line linearGradient mask path pattern polygon polyline radialGradient rect stop svg text tspan".split(" "),D="undefined"!=typeof Symbol&&Symbol.for&&Symbol.for("react.element")||60103,U="undefined"!=typeof Symbol?Symbol.for("__preactCompatWrapper"):"__preactCompatWrapper",W={constructor:1,render:1,shouldComponentUpdate:1,componentWillReceiveProps:1,componentWillUpdate:1,componentDidUpdate:1,componentWillMount:1,componentDidMount:1,componentWillUnmount:1,componentDidUnmount:1},B=/^(?:accent|alignment|arabic|baseline|cap|clip|color|fill|flood|font|glyph|horiz|marker|overline|paint|stop|strikethrough|stroke|text|underline|unicode|units|v|vector|vert|word|writing|x)[A-Z]/,z={},q=void 0===e||!e.env||"production"!==e.env.NODE_ENV,V=n.h("a",null).constructor;V.prototype.$$typeof=D,V.prototype.preactCompatUpgraded=!1,V.prototype.preactCompatNormalized=!1,Object.defineProperty(V.prototype,"type",{get:function(){return this.nodeName},set:function(t){this.nodeName=t},configurable:!0}),Object.defineProperty(V.prototype,"props",{get:function(){return this.attributes},set:function(t){this.attributes=t},configurable:!0});var G=n.options.event;n.options.event=function(t){return G&&(t=G(t)),t.persist=Object,t.nativeEvent=t,t};var H=n.options.vnode;n.options.vnode=function(t){if(!t.preactCompatUpgraded){t.preactCompatUpgraded=!0;var e=t.nodeName,n=t.attributes=w({},t.attributes);"function"==typeof e?(!0===e[U]||e.prototype&&"isReactComponent"in e.prototype)&&(t.children&&t.children+""==""&&(t.children=void 0),t.children&&(n.children=t.children),t.preactCompatNormalized||v(t),o(t)):(t.children&&t.children+""==""&&(t.children=void 0),t.children&&(n.children=t.children),n.defaultValue&&(n.value||0===n.value||(n.value=n.defaultValue),delete n.defaultValue),i(t,n))}H&&H(t)};var Y=function(){};Y.prototype.getChildContext=function(){return this.props.context},Y.prototype.render=function(t){return t.children[0]};for(var $,K=[],J={map:function(t,e,n){return null==t?null:(t=J.toArray(t),n&&n!==t&&(e=e.bind(n)),t.map(e))},forEach:function(t,e,n){if(null==t)return null;t=J.toArray(t),n&&n!==t&&(e=e.bind(n)),t.forEach(e)},count:function(t){return t&&t.length||0},only:function(t){if(t=J.toArray(t),1!==t.length)throw Error("Children.only() expects only one child.");return t[0]},toArray:function(t){return null==t?[]:K.concat(t)}},Q={},X=F.length;X--;)Q[F[X]]=s(F[X]);var Z={configurable:!0,get:function(){return this.class},set:function(t){this.class=t}};return w(L.prototype=new n.Component,{constructor:L,isReactComponent:{},replaceState:function(t,e){var n=this;this.setState(t,e);for(var r in n.state)r in t||delete n.state[r]},getDOMNode:function(){return this.base},isMounted:function(){return!!this.base}}),P.prototype=L.prototype,I.prototype=new P,I.prototype.isPureReactComponent=!0,I.prototype.shouldComponentUpdate=function(t,e){return O(this.props,t)||O(this.state,e)},{version:"15.1.0",DOM:Q,PropTypes:t,Children:J,render:a,createClass:E,createFactory:s,createElement:d,cloneElement:y,isValidElement:m,findDOMNode:x,unmountComponentAtNode:c,Component:L,PureComponent:I,unstable_renderSubtreeIntoContainer:u,__spread:w}})}).call(e,n(21))},function(t,e,n){var r=n(7);t.exports=function(t){if(!r(t))throw TypeError(t+" is not an object!");return t}},,,function(t){var e=t.exports="undefined"!=typeof window&&window.Math==Math?window:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")();"number"==typeof __g&&(__g=e)},function(t){t.exports=function(t){try{return!!t()}catch(t){return!0}}},function(t){t.exports=function(t){return"object"==typeof t?null!==t:"function"==typeof t}},function(t,e,n){var r=n(68)("wks"),o=n(44),i=n(5).Symbol,a="function"==typeof i;(t.exports=function(t){return r[t]||(r[t]=a&&i[t]||(a?i:o)("Symbol."+t))}).store=r},function(t,e,n){t.exports=!n(6)(function(){return 7!=Object.defineProperty({},"a",{get:function(){return 7}}).a})},function(t,e,n){var r=n(2),o=n(133),i=n(33),a=Object.defineProperty;e.f=n(9)?Object.defineProperty:function(t,e,n){if(r(t),e=i(e,!0),r(n),o)try{return a(t,e,n)}catch(t){}if("get"in n||"set"in n)throw TypeError("Accessors not supported!");return"value"in n&&(t[e]=n.value),t}},function(t,e,n){var r=n(35),o=Math.min;t.exports=function(t){return t>0?o(r(t),9007199254740991):0}},function(t,e,n){(function(e){if("production"!==e.env.NODE_ENV){var r="function"==typeof Symbol&&Symbol.for&&Symbol.for("react.element")||60103,o=function(t){return"object"==typeof t&&null!==t&&t.$$typeof===r};t.exports=n(403)(o,!0)}else t.exports=n(405)()}).call(e,n(21))},function(t,e,n){var r=n(34);t.exports=function(t){return Object(r(t))}},function(t,e,n){"use strict";(function(e){var n=function(){};"production"!==e.env.NODE_ENV&&(n=function(t,e,n){var r=arguments.length;n=Array(r>2?r-2:0);for(var o=2;r>o;o++)n[o-2]=arguments[o];if(void 0===e)throw Error("` + "`" + `warning(condition, format, ...args)` + "`" + ` requires a warning message argument");if(10>e.length||/^[s\W]*$/.test(e))throw Error("The warning format should be able to uniquely identify this warning. Please, use a more descriptive format than: "+e);if(!t){var i=0,a="Warning: "+e.replace(/%s/g,function(){return n[i++]});try{throw Error(a)}catch(t){}}}),t.exports=n}).call(e,n(21))},function(t){t.exports=function(t){if("function"!=typeof t)throw TypeError(t+" is not a function!");return t}},function(t,e,n){var r=n(407);e.root=r.root,e.branch=r.branch},function(t){var e={}.hasOwnProperty;t.exports=function(t,n){return e.call(t,n)}},function(t,e,n){var r=n(10),o=n(43);t.exports=n(9)?function(t,e,n){return r.f(t,e,o(1,n))}:function(t,e,n){return t[e]=n,t}},function(t,e,n){var r=n(5),o=n(18),i=n(17),a=n(44)("src"),u=Function.toString,c=(""+u).split("toString");n(32).inspectSource=function(t){return u.call(t)},(t.exports=function(t,e,n,u){var s="function"==typeof n;s&&(i(n,"name")||o(n,"name",e)),t[e]!==n&&(s&&(i(n,a)||o(n,a,t[e]?""+t[e]:c.join(e+""))),t===r?t[e]=n:u?t[e]?t[e]=n:o(t,e,n):(delete t[e],o(t,e,n)))})(Function.prototype,"toString",function(){return"function"==typeof this&&this[a]||u.call(this)})},function(t,e,n){var r=n(0),o=n(6),i=n(34),a=/"/g,u=function(t,e,n,r){var o=i(t)+"",u="<"+e;return""!==n&&(u+=" "+n+'="'+(r+"").replace(a,""")+'"'),u+">"+o+""};t.exports=function(t,e){var n={};n[t]=e(u),r(r.P+r.F*o(function(){var e=""[t]('"');return e!==e.toLowerCase()||e.split('"').length>3}),"String",n)}},function(t){function e(){throw Error("setTimeout has not been defined")}function n(){throw Error("clearTimeout has not been defined")}function r(t){if(s===setTimeout)return setTimeout(t,0);if((s===e||!s)&&setTimeout)return s=setTimeout,setTimeout(t,0);try{return s(t,0)}catch(e){try{return s.call(null,t,0)}catch(e){return s.call(this,t,0)}}}function o(t){if(f===clearTimeout)return clearTimeout(t);if((f===n||!f)&&clearTimeout)return f=clearTimeout,clearTimeout(t);try{return f(t)}catch(e){try{return f.call(null,t)}catch(e){return f.call(this,t)}}}function i(){d&&p&&(d=!1,p.length?h=p.concat(h):v=-1,h.length&&a())}function a(){if(!d){var t=r(i);d=!0;for(var e=h.length;e;){for(p=h,h=[];++v1)for(var n=1;arguments.length>n;n++)e[n-1]=arguments[n];h.push(new u(t,e)),1!==h.length||d||r(a)},u.prototype.run=function(){this.fun.apply(null,this.array)},l.title="browser",l.browser=!0,l.env={},l.argv=[],l.version="",l.versions={},l.on=c,l.addListener=c,l.once=c,l.off=c,l.removeListener=c,l.removeAllListeners=c,l.emit=c,l.prependListener=c,l.prependOnceListener=c,l.listeners=function(){return[]},l.binding=function(){throw Error("process.binding is not supported")},l.cwd=function(){return"/"},l.chdir=function(){throw Error("process.chdir is not supported")},l.umask=function(){return 0}},,,function(t,e,n){var r=n(59),o=n(34);t.exports=function(t){return r(o(t))}},function(t,e,n){var r=n(60),o=n(43),i=n(24),a=n(33),u=n(17),c=n(133),s=Object.getOwnPropertyDescriptor;e.f=n(9)?s:function(t,e){if(t=i(t),e=a(e,!0),c)try{return s(t,e)}catch(t){}if(u(t,e))return o(!r.f.call(t,e),t[e])}},function(t,e,n){var r=n(17),o=n(13),i=n(91)("IE_PROTO"),a=Object.prototype;t.exports=Object.getPrototypeOf||function(t){return t=o(t),r(t,i)?t[i]:"function"==typeof t.constructor&&t instanceof t.constructor?t.constructor.prototype:t instanceof Object?a:null}},function(t,e,n){"use strict";Object.defineProperty(e,"__esModule",{value:!0});var r=n(424);n.d(e,"BrowserRouter",function(){return r.a});var o=n(426);n.d(e,"HashRouter",function(){return o.a});var i=n(176);n.d(e,"Link",function(){return i.a});var a=n(428);n.d(e,"MemoryRouter",function(){return a.a});var u=n(430);n.d(e,"NavLink",function(){return u.a});var c=n(433);n.d(e,"Prompt",function(){return c.a});var s=n(434);n.d(e,"Redirect",function(){return s.a});var f=n(178);n.d(e,"Route",function(){return f.a});var l=n(123);n.d(e,"Router",function(){return l.a});var p=n(439);n.d(e,"StaticRouter",function(){return p.a});var h=n(440);n.d(e,"Switch",function(){return h.a});var d=n(441);n.d(e,"matchPath",function(){return d.a});var v=n(442);n.d(e,"withRouter",function(){return v.a})},function(t,e,n){var r=n(15);t.exports=function(t,e,n){if(r(t),void 0===e)return t;switch(n){case 1:return function(n){return t.call(e,n)};case 2:return function(n,r){return t.call(e,n,r)};case 3:return function(n,r,o){return t.call(e,n,r,o)}}return function(){return t.apply(e,arguments)}}},function(t){var e={}.toString;t.exports=function(t){return e.call(t).slice(8,-1)}},function(t,e,n){"use strict";var r=n(6);t.exports=function(t,e){return!!t&&r(function(){e?t.call(null,function(){},1):t.call(null)})}},function(t,e,n){"use strict";(function(e){t.exports=function(t,n,r,o,i,a,u,c){if("production"!==e.env.NODE_ENV&&void 0===n)throw Error("invariant requires an error message argument");if(!t){var s;if(void 0===n)s=Error("Minified exception occurred; use the non-minified dev environment for the full error message and additional helpful warnings.");else{var f=[r,o,i,a,u,c],l=0;s=Error(n.replace(/%s/g,function(){return f[l++]})),s.name="Invariant Violation"}throw s.framesToPop=1,s}}}).call(e,n(21))},function(t){var e=t.exports={version:"2.5.1"};"number"==typeof __e&&(__e=e)},function(t,e,n){var r=n(7);t.exports=function(t,e){if(!r(t))return t;var n,o;if(e&&"function"==typeof(n=t.toString)&&!r(o=n.call(t)))return o;if("function"==typeof(n=t.valueOf)&&!r(o=n.call(t)))return o;if(!e&&"function"==typeof(n=t.toString)&&!r(o=n.call(t)))return o;throw TypeError("Can't convert object to primitive value")}},function(t){t.exports=function(t){if(void 0==t)throw TypeError("Can't call method on "+t);return t}},function(t){var e=Math.ceil,n=Math.floor;t.exports=function(t){return isNaN(t=+t)?0:(t>0?n:e)(t)}},function(t,e,n){var r=n(0),o=n(32),i=n(6);t.exports=function(t,e){var n=(o.Object||{})[t]||Object[t],a={};a[t]=e(n),r(r.S+r.F*i(function(){n(1)}),"Object",a)}},function(t,e,n){var r=n(28),o=n(59),i=n(13),a=n(11),u=n(108);t.exports=function(t,e){var n=1==t,c=2==t,s=3==t,f=4==t,l=6==t,p=5==t||l,h=e||u;return function(e,u,d){for(var v,y,m=i(e),g=o(m),b=r(u,d,3),_=a(g.length),w=0,O=n?h(e,_):c?h(e,0):void 0;_>w;w++)if((p||w in g)&&(v=g[w],y=b(v,w,m),t))if(n)O[w]=y;else if(y)switch(t){case 3:return!0;case 5:return v;case 6:return w;case 2:O.push(v)}else if(f)return!1;return l?-1:s||f?f:O}}},function(t,e,n){"use strict";if(n(9)){var r=n(45),o=n(5),i=n(6),a=n(0),u=n(78),c=n(114),s=n(28),f=n(51),l=n(43),p=n(18),h=n(53),d=n(35),v=n(11),y=n(159),m=n(47),g=n(33),b=n(17),_=n(61),w=n(7),O=n(13),x=n(105),P=n(48),E=n(26),k=n(49).f,j=n(107),S=n(44),T=n(8),C=n(37),M=n(69),N=n(76),R=n(110),A=n(56),L=n(73),I=n(50),F=n(109),D=n(149),U=n(10),W=n(25),B=U.f,z=W.f,q=o.RangeError,V=o.TypeError,G=o.Uint8Array,H=Array.prototype,Y=c.ArrayBuffer,$=c.DataView,K=C(0),J=C(2),Q=C(3),X=C(4),Z=C(5),tt=C(6),et=M(!0),nt=M(!1),rt=R.values,ot=R.keys,it=R.entries,at=H.lastIndexOf,ut=H.reduce,ct=H.reduceRight,st=H.join,ft=H.sort,lt=H.slice,pt=H.toString,ht=H.toLocaleString,dt=T("iterator"),vt=T("toStringTag"),yt=S("typed_constructor"),mt=S("def_constructor"),gt=u.CONSTR,bt=u.TYPED,_t=u.VIEW,wt=C(1,function(t,e){return kt(N(t,t[mt]),e)}),Ot=i(function(){return 1===new G(new Uint16Array([1]).buffer)[0]}),xt=!!G&&!!G.prototype.set&&i(function(){new G(1).set({})}),Pt=function(t,e){var n=d(t);if(0>n||n%e)throw q("Wrong offset!");return n},Et=function(t){if(w(t)&&bt in t)return t;throw V(t+" is not a typed array!")},kt=function(t,e){if(!(w(t)&&yt in t))throw V("It is not a typed array constructor!");return new t(e)},jt=function(t,e){return St(N(t,t[mt]),e)},St=function(t,e){for(var n=0,r=e.length,o=kt(t,r);r>n;)o[n]=e[n++];return o},Tt=function(t,e,n){B(t,e,{get:function(){return this._d[n]}})},Ct=function(t){var e,n,r,o,i,a,u=O(t),c=arguments.length,f=c>1?arguments[1]:void 0,l=void 0!==f,p=j(u);if(void 0!=p&&!x(p)){for(a=p.call(u),r=[],e=0;!(i=a.next()).done;e++)r.push(i.value);u=r}for(l&&c>2&&(f=s(f,arguments[2],2)),e=0,n=v(u.length),o=kt(this,n);n>e;e++)o[e]=l?f(u[e],e):u[e];return o},Mt=function(){for(var t=0,e=arguments.length,n=kt(this,e);e>t;)n[t]=arguments[t++];return n},Nt=!!G&&i(function(){ht.call(new G(1))}),Rt=function(){return ht.apply(Nt?lt.call(Et(this)):Et(this),arguments)},At={copyWithin:function(t,e){return D.call(Et(this),t,e,arguments.length>2?arguments[2]:void 0)},every:function(t){return X(Et(this),t,arguments.length>1?arguments[1]:void 0)},fill:function(){return F.apply(Et(this),arguments)},filter:function(t){return jt(this,J(Et(this),t,arguments.length>1?arguments[1]:void 0))},find:function(t){return Z(Et(this),t,arguments.length>1?arguments[1]:void 0)},findIndex:function(t){return tt(Et(this),t,arguments.length>1?arguments[1]:void 0)},forEach:function(t){K(Et(this),t,arguments.length>1?arguments[1]:void 0)},indexOf:function(t){return nt(Et(this),t,arguments.length>1?arguments[1]:void 0)},includes:function(t){return et(Et(this),t,arguments.length>1?arguments[1]:void 0)},join:function(){return st.apply(Et(this),arguments)},lastIndexOf:function(){return at.apply(Et(this),arguments)},map:function(t){return wt(Et(this),t,arguments.length>1?arguments[1]:void 0)},reduce:function(){return ut.apply(Et(this),arguments)},reduceRight:function(){return ct.apply(Et(this),arguments)},reverse:function(){for(var t,e=this,n=Et(e).length,r=Math.floor(n/2),o=0;r>o;)t=e[o],e[o++]=e[--n],e[n]=t;return e},some:function(t){return Q(Et(this),t,arguments.length>1?arguments[1]:void 0)},sort:function(t){return ft.call(Et(this),t)},subarray:function(t,e){var n=Et(this),r=n.length,o=m(t,r);return new(N(n,n[mt]))(n.buffer,n.byteOffset+o*n.BYTES_PER_ELEMENT,v((void 0===e?r:m(e,r))-o))}},Lt=function(t,e){return jt(this,lt.call(Et(this),t,e))},It=function(t){Et(this);var e=Pt(arguments[1],1),n=this.length,r=O(t),o=v(r.length),i=0;if(o+e>n)throw q("Wrong length!");for(;o>i;)this[e+i]=r[i++]},Ft={entries:function(){return it.call(Et(this))},keys:function(){return ot.call(Et(this))},values:function(){return rt.call(Et(this))}},Dt=function(t,e){return w(t)&&t[bt]&&"symbol"!=typeof e&&e in t&&+e+""==e+""},Ut=function(t,e){return Dt(t,e=g(e,!0))?l(2,t[e]):z(t,e)},Wt=function(t,e,n){return!(Dt(t,e=g(e,!0))&&w(n)&&b(n,"value"))||b(n,"get")||b(n,"set")||n.configurable||b(n,"writable")&&!n.writable||b(n,"enumerable")&&!n.enumerable?B(t,e,n):(t[e]=n.value,t)};gt||(W.f=Ut,U.f=Wt),a(a.S+a.F*!gt,"Object",{getOwnPropertyDescriptor:Ut,defineProperty:Wt}),i(function(){pt.call({})})&&(pt=ht=function(){return st.call(this)});var Bt=h({},At);h(Bt,Ft),p(Bt,dt,Ft.values),h(Bt,{slice:Lt,set:It,constructor:function(){},toString:pt,toLocaleString:Rt}),Tt(Bt,"buffer","b"),Tt(Bt,"byteOffset","o"),Tt(Bt,"byteLength","l"),Tt(Bt,"length","e"),B(Bt,vt,{get:function(){return this[bt]}}),t.exports=function(t,e,n,c){c=!!c;var s=t+(c?"Clamped":"")+"Array",l="get"+t,h="set"+t,d=o[s],m=d||{},g=d&&E(d),b=!d||!u.ABV,O={},x=d&&d.prototype,j=function(t,n){var r=t._d;return r.v[l](n*e+r.o,Ot)},S=function(t,n,r){var o=t._d;c&&(r=0>(r=Math.round(r))?0:r>255?255:255&r),o.v[h](n*e+o.o,r,Ot)},T=function(t,e){B(t,e,{get:function(){return j(this,e)},set:function(t){return S(this,e,t)},enumerable:!0})};b?(d=n(function(t,n,r,o){f(t,d,s,"_d");var i,a,u,c,l=0,h=0;if(w(n)){if(!(n instanceof Y||"ArrayBuffer"==(c=_(n))||"SharedArrayBuffer"==c))return bt in n?St(d,n):Ct.call(d,n);i=n,h=Pt(r,e);var m=n.byteLength;if(void 0===o){if(m%e)throw q("Wrong length!");if(0>(a=m-h))throw q("Wrong length!")}else if((a=v(o)*e)+h>m)throw q("Wrong length!");u=a/e}else u=y(n),a=u*e,i=new Y(a);for(p(t,"_d",{b:i,o:h,l:a,e:u,v:new $(i)});u>l;)T(t,l++)}),x=d.prototype=P(Bt),p(x,"constructor",d)):i(function(){d(1)})&&i(function(){new d(-1)})&&L(function(t){new d,new d(null),new d(1.5),new d(t)},!0)||(d=n(function(t,n,r,o){f(t,d,s);var i;return w(n)?n instanceof Y||"ArrayBuffer"==(i=_(n))||"SharedArrayBuffer"==i?void 0!==o?new m(n,Pt(r,e),o):void 0!==r?new m(n,Pt(r,e)):new m(n):bt in n?St(d,n):Ct.call(d,n):new m(y(n))}),K(g!==Function.prototype?k(m).concat(k(g)):k(m),function(t){t in d||p(d,t,m[t])}),d.prototype=x,r||(x.constructor=d));var C=x[dt],M=!!C&&("values"==C.name||void 0==C.name),N=Ft.values;p(d,yt,!0),p(x,bt,s),p(x,_t,!0),p(x,mt,d),(c?new d(1)[vt]==s:vt in x)||B(x,vt,{get:function(){return s}}),O[s]=d,a(a.G+a.W+a.F*(d!=m),O),a(a.S,s,{BYTES_PER_ELEMENT:e}),a(a.S+a.F*i(function(){m.of.call(d,1)}),s,{from:Ct,of:Mt}),"BYTES_PER_ELEMENT"in x||p(x,"BYTES_PER_ELEMENT",e),a(a.P,s,At),I(s),a(a.P+a.F*xt,s,{set:It}),a(a.P+a.F*!M,s,Ft),r||x.toString==pt||(x.toString=pt),a(a.P+a.F*i(function(){new d(1).slice()}),s,{slice:Lt}),a(a.P+a.F*(i(function(){return[1,2].toLocaleString()!=new d([1,2]).toLocaleString()})||!i(function(){x.toLocaleString.call([1,2])})),s,{toLocaleString:Rt}),A[s]=M?C:N,r||M||p(x,dt,N)}}else t.exports=function(){}},function(t,e,n){var r=n(154),o=n(0),i=n(68)("metadata"),a=i.store||(i.store=new(n(157))),u=function(t,e,n){var o=a.get(t);if(!o){if(!n)return;a.set(t,o=new r)}var i=o.get(e);if(!i){if(!n)return;o.set(e,i=new r)}return i};t.exports={store:a,map:u,has:function(t,e,n){var r=u(e,n,!1);return void 0!==r&&r.has(t)},get:function(t,e,n){var r=u(e,n,!1);return void 0===r?void 0:r.get(t)},set:function(t,e,n,r){u(n,r,!0).set(t,e)},keys:function(t,e){var n=u(t,e,!1),r=[];return n&&n.forEach(function(t,e){r.push(e)}),r},key:function(t){return void 0===t||"symbol"==typeof t?t:t+""},exp:function(t){o(o.S,"Reflect",t)}}},,function(t,e,n){var r=n(44)("meta"),o=n(7),i=n(17),a=n(10).f,u=0,c=Object.isExtensible||function(){return!0},s=!n(6)(function(){return c(Object.preventExtensions({}))}),f=function(t){a(t,r,{value:{i:"O"+ ++u,w:{}}})},l=function(t,e){if(!o(t))return"symbol"==typeof t?t:("string"==typeof t?"S":"P")+t;if(!i(t,r)){if(!c(t))return"F";if(!e)return"E";f(t)}return t[r].i},p=function(t,e){if(!i(t,r)){if(!c(t))return!0;if(!e)return!1;f(t)}return t[r].w},h=function(t){return s&&d.NEED&&c(t)&&!i(t,r)&&f(t),t},d=t.exports={KEY:r,NEED:!1,fastKey:l,getWeak:p,onFreeze:h}},function(t,e,n){var r=n(8)("unscopables"),o=Array.prototype;void 0==o[r]&&n(18)(o,r,{}),t.exports=function(t){o[r][t]=!0}},function(t){t.exports=function(t,e){return{enumerable:!(1&t),configurable:!(2&t),writable:!(4&t),value:e}}},function(t){var e=0,n=Math.random();t.exports=function(t){return"Symbol(".concat(void 0===t?"":t,")_",(++e+n).toString(36))}},function(t){t.exports=!1},function(t,e,n){var r=n(135),o=n(92);t.exports=Object.keys||function(t){return r(t,o)}},function(t,e,n){var r=n(35),o=Math.max,i=Math.min;t.exports=function(t,e){return t=r(t),0>t?o(t+e,0):i(t,e)}},function(t,e,n){var r=n(2),o=n(136),i=n(92),a=n(91)("IE_PROTO"),u=function(){},c=function(){var t,e=n(89)("iframe"),r=i.length;for(e.style.display="none",n(93).appendChild(e),e.src="javascript:",t=e.contentWindow.document,t.open(),t.write(" + `) diff --git a/vendor/vendor.json b/vendor/vendor.json index af16c1a54aa..118b36cc6a8 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -425,10 +425,12 @@ "revisionTime": "2016-05-04T02:26:26Z" }, { - "checksumSHA1": "i3dVVpc0/It5nJIt/LEOHNuvqQY=", + "checksumSHA1": "R/gRUF6hXEFbDGSIOKt6VdCPwHE=", "path": "github.com/laszlocph/drone-ui/dist", - "revision": "106788432c8e9f19ee7a73fd977ef15d32cca79d", - "revisionTime": "2019-06-24T07:03:37Z" + "revision": "1c55c6bab89440efc658a708ba7bb3424dbd5d0d", + "revisionTime": "2019-06-25T11:41:53Z", + "version": "fallback-config", + "versionExact": "fallback-config" }, { "path": "github.com/lib/pq",