Skip to content

Commit

Permalink
Remove workflow-level volumes and networks (#4636)
Browse files Browse the repository at this point in the history
  • Loading branch information
@qwerty287
qwerty287 authored Dec 30, 2024
1 parent afa6dee commit 428ba68
Show file tree
Hide file tree
Showing 11 changed files with 52 additions and 412 deletions.
41 changes: 15 additions & 26 deletions pipeline/backend/docker/docker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,30 +144,23 @@ func (e *docker) Load(ctx context.Context) (*backend.BackendInfo, error) {
func (e *docker) SetupWorkflow(ctx context.Context, conf *backend.Config, taskUUID string) error {
log.Trace().Str("taskUUID", taskUUID).Msg("create workflow environment")

for _, vol := range conf.Volumes {
_, err := e.client.VolumeCreate(ctx, volume.CreateOptions{
Name: vol.Name,
Driver: volumeDriver,
})
if err != nil {
return err
}
_, err := e.client.VolumeCreate(ctx, volume.CreateOptions{
Name: conf.Volume.Name,
Driver: volumeDriver,
})
if err != nil {
return err
}

networkDriver := networkDriverBridge
if e.info.OSType == "windows" {
networkDriver = networkDriverNAT
}
for _, n := range conf.Networks {
_, err := e.client.NetworkCreate(ctx, n.Name, network.CreateOptions{
Driver: networkDriver,
EnableIPv6: &e.config.enableIPv6,
})
if err != nil {
return err
}
}
return nil
_, err = e.client.NetworkCreate(ctx, conf.Network.Name, network.CreateOptions{
Driver: networkDriver,
EnableIPv6: &e.config.enableIPv6,
})
return err
}

func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID string) error {
Expand Down Expand Up @@ -330,15 +323,11 @@ func (e *docker) DestroyWorkflow(ctx context.Context, conf *backend.Config, task
}
}
}
for _, v := range conf.Volumes {
if err := e.client.VolumeRemove(ctx, v.Name, true); err != nil {
log.Error().Err(err).Msgf("could not remove volume '%s'", v.Name)
}
if err := e.client.VolumeRemove(ctx, conf.Volume.Name, true); err != nil {
log.Error().Err(err).Msgf("could not remove volume '%s'", conf.Volume.Name)
}
for _, n := range conf.Networks {
if err := e.client.NetworkRemove(ctx, n.Name); err != nil {
log.Error().Err(err).Msgf("could not remove network '%s'", n.Name)
}
if err := e.client.NetworkRemove(ctx, conf.Network.Name); err != nil {
log.Error().Err(err).Msgf("could not remove network '%s'", conf.Network.Name)
}
return nil
}
Expand Down
16 changes: 6 additions & 10 deletions pipeline/backend/kubernetes/kubernetes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,11 +191,9 @@ func (e *kube) getConfig() *config {
func (e *kube) SetupWorkflow(ctx context.Context, conf *types.Config, taskUUID string) error {
log.Trace().Str("taskUUID", taskUUID).Msgf("Setting up Kubernetes primitives")

for _, vol := range conf.Volumes {
_, err := startVolume(ctx, e, vol.Name)
if err != nil {
return err
}
_, err := startVolume(ctx, e, conf.Volume.Name)
if err != nil {
return err
}

var extraHosts []types.HostAlias
Expand Down Expand Up @@ -427,11 +425,9 @@ func (e *kube) DestroyWorkflow(ctx context.Context, conf *types.Config, taskUUID
}
}

for _, vol := range conf.Volumes {
err := stopVolume(ctx, e, vol.Name, defaultDeleteOptions)
if err != nil {
return err
}
err := stopVolume(ctx, e, conf.Volume.Name, defaultDeleteOptions)
if err != nil {
return err
}

return nil
Expand Down
8 changes: 4 additions & 4 deletions pipeline/backend/types/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ package types

// Config defines the runtime configuration of a workflow.
type Config struct {
Stages []*Stage `json:"pipeline"` // workflow stages
Networks []*Network `json:"networks"` // network definitions
Volumes []*Volume `json:"volumes"` // volume definitions
Secrets []*Secret `json:"secrets"` // secret definitions
Stages []*Stage `json:"pipeline"` // workflow stages
Network *Network `json:"network"` // network definitions
Volume *Volume `json:"volume"` // volume definition
Secrets []*Secret `json:"secrets"` // secret definitions
}

// CliCommand is the context key to pass cli context to backends if needed.
Expand Down
8 changes: 4 additions & 4 deletions pipeline/frontend/yaml/compiler/compiler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,14 +129,14 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
}

// create a default volume
config.Volumes = append(config.Volumes, &backend_types.Volume{
config.Volume = &backend_types.Volume{
Name: fmt.Sprintf("%s_default", c.prefix),
})
}

// create a default network
config.Networks = append(config.Networks, &backend_types.Network{
config.Network = &backend_types.Network{
Name: fmt.Sprintf("%s_default", c.prefix),
})
}

// create secrets for mask
for _, sec := range c.secrets {
Expand Down
46 changes: 23 additions & 23 deletions pipeline/frontend/yaml/compiler/compiler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,12 @@ func TestCompilerCompile(t *testing.T) {
WithWorkspaceFromURL("/test", repoURL),
)

defaultNetworks := []*backend_types.Network{{
defaultNetwork := &backend_types.Network{
Name: "test_default",
}}
defaultVolumes := []*backend_types.Volume{{
}
defaultVolume := &backend_types.Volume{
Name: "test_default",
}}
}

defaultCloneStage := &backend_types.Stage{
Steps: []*backend_types.Step{{
Expand All @@ -95,7 +95,7 @@ func TestCompilerCompile(t *testing.T) {
Image: constant.DefaultClonePlugin,
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/woodpecker"},
Volumes: []string{defaultVolume.Name + ":/woodpecker"},
WorkingDir: "/woodpecker/src/github.com/octocat/hello-world",
WorkspaceBase: "/woodpecker",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"clone"}}},
Expand All @@ -113,17 +113,17 @@ func TestCompilerCompile(t *testing.T) {
name: "empty workflow, no clone",
fronConf: &yaml_types.Workflow{SkipClone: true},
backConf: &backend_types.Config{
Networks: defaultNetworks,
Volumes: defaultVolumes,
Network: defaultNetwork,
Volume: defaultVolume,
},
},
{
name: "empty workflow, default clone",
fronConf: &yaml_types.Workflow{},
backConf: &backend_types.Config{
Networks: defaultNetworks,
Volumes: defaultVolumes,
Stages: []*backend_types.Stage{defaultCloneStage},
Network: defaultNetwork,
Volume: defaultVolume,
Stages: []*backend_types.Stage{defaultCloneStage},
},
},
{
Expand All @@ -133,16 +133,16 @@ func TestCompilerCompile(t *testing.T) {
Image: "dummy_img",
}}}},
backConf: &backend_types.Config{
Networks: defaultNetworks,
Volumes: defaultVolumes,
Network: defaultNetwork,
Volume: defaultVolume,
Stages: []*backend_types.Stage{defaultCloneStage, {
Steps: []*backend_types.Step{{
Name: "dummy",
Type: backend_types.StepTypePlugin,
Image: "dummy_img",
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/woodpecker"},
Volumes: []string{defaultVolume.Name + ":/woodpecker"},
WorkingDir: "/woodpecker/src/github.com/octocat/hello-world",
WorkspaceBase: "/woodpecker",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"dummy"}}},
Expand All @@ -167,8 +167,8 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"echo 2"},
}}}},
backConf: &backend_types.Config{
Networks: defaultNetworks,
Volumes: defaultVolumes,
Network: defaultNetwork,
Volume: defaultVolume,
Stages: []*backend_types.Stage{
defaultCloneStage, {
Steps: []*backend_types.Step{{
Expand All @@ -178,7 +178,7 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"env"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/test"},
Volumes: []string{defaultVolume.Name + ":/test"},
WorkingDir: "/test/src/github.com/octocat/hello-world",
WorkspaceBase: "/test",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}},
Expand All @@ -192,7 +192,7 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"echo 1"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/test"},
Volumes: []string{defaultVolume.Name + ":/test"},
WorkingDir: "/test/src/github.com/octocat/hello-world",
WorkspaceBase: "/test",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 1"}}},
Expand All @@ -206,7 +206,7 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"echo 2"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/test"},
Volumes: []string{defaultVolume.Name + ":/test"},
WorkingDir: "/test/src/github.com/octocat/hello-world",
WorkspaceBase: "/test",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 2"}}},
Expand All @@ -233,8 +233,8 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"echo 2"},
}}}},
backConf: &backend_types.Config{
Networks: defaultNetworks,
Volumes: defaultVolumes,
Network: defaultNetwork,
Volume: defaultVolume,
Stages: []*backend_types.Stage{defaultCloneStage, {
Steps: []*backend_types.Step{{
Name: "echo env",
Expand All @@ -243,7 +243,7 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"env"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/test"},
Volumes: []string{defaultVolume.Name + ":/test"},
WorkingDir: "/test/src/github.com/octocat/hello-world",
WorkspaceBase: "/test",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}},
Expand All @@ -255,7 +255,7 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"echo 2"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/test"},
Volumes: []string{defaultVolume.Name + ":/test"},
WorkingDir: "/test/src/github.com/octocat/hello-world",
WorkspaceBase: "/test",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo 2"}}},
Expand All @@ -269,7 +269,7 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"echo 1"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/test"},
Volumes: []string{defaultVolume.Name + ":/test"},
WorkingDir: "/test/src/github.com/octocat/hello-world",
WorkspaceBase: "/test",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo 1"}}},
Expand Down
10 changes: 0 additions & 10 deletions pipeline/frontend/yaml/parse_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,6 @@ func TestParse(t *testing.T) {

assert.Equal(t, "/go", out.Workspace.Base)
assert.Equal(t, "src/github.com/octocat/hello-world", out.Workspace.Path)
assert.Equal(t, "custom", out.Volumes.WorkflowVolumes[0].Name)
assert.Equal(t, "blockbridge", out.Volumes.WorkflowVolumes[0].Driver)
assert.Equal(t, "custom", out.Networks.WorkflowNetworks[0].Name)
assert.Equal(t, "overlay", out.Networks.WorkflowNetworks[0].Driver)
assert.Equal(t, "database", out.Services.ContainerList[0].Name)
assert.Equal(t, "mysql", out.Services.ContainerList[0].Image)
assert.Equal(t, "test", out.Steps.ContainerList[0].Name)
Expand Down Expand Up @@ -201,12 +197,6 @@ steps:
services:
database:
image: mysql
networks:
custom:
driver: overlay
volumes:
custom:
driver: blockbridge
labels:
com.example.type: "build"
com.example.team: "frontend"
Expand Down
4 changes: 0 additions & 4 deletions pipeline/frontend/yaml/types/workflow.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,6 @@ type (
DependsOn []string `yaml:"depends_on,omitempty"`
RunsOn []string `yaml:"runs_on,omitempty"`
SkipClone bool `yaml:"skip_clone"`

// Undocumented
Networks WorkflowNetworks `yaml:"networks,omitempty"`
Volumes WorkflowVolumes `yaml:"volumes,omitempty"`
}

// Workspace defines a pipeline workspace.
Expand Down
52 changes: 0 additions & 52 deletions pipeline/frontend/yaml/types/workflow_network.go
View file

This file was deleted.

Loading

0 comments on commit 428ba68

Please sign in to comment.