Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pod termination fault #359

Merged
merged 8 commits into from
Oct 26, 2023
Merged

Pod termination fault #359

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
90b798e
Make PodController stateless
pablochacin Oct 23, 2023
76882eb
Add helpers for deleting pods
pablochacin Oct 24, 2023
4336626
Implement TerminatePod fault
pablochacin Oct 24, 2023
93fce2e
Implement e2e test for TerminatePod
pablochacin Oct 24, 2023
738e229
Add helper for handing percentages
pablochacin Oct 24, 2023
e42eb59
Use int or percentage in pod termination count
pablochacin Oct 24, 2023
e87dd78
Rename Pod termination fault
pablochacin Oct 25, 2023
e38a6db
Expose pod termination fault in JS API
pablochacin Oct 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
93 changes: 80 additions & 13 deletions e2e/disruptors/pod_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import (
"github.com/grafana/xk6-disruptor/pkg/types/intstr"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8sintstr "k8s.io/apimachinery/pkg/util/intstr"

"github.com/grafana/xk6-disruptor/pkg/disruptors"
Expand Down Expand Up @@ -55,22 +57,24 @@ func Test_PodDisruptor(t *testing.T) {
return
}

t.Run("Test fault injection", func(t *testing.T) {
t.Run("Protocol fault injection", func(t *testing.T) {
t.Parallel()

testCases := []struct {
title string
pod corev1.Pod
replicas int
service corev1.Service
port int
injector func(d disruptors.PodDisruptor) error
check checks.Check
}{
{
title: "Inject Http error 500",
pod: fixtures.BuildHttpbinPod(),
service: fixtures.BuildHttpbinService(),
port: 80,
title: "Inject Http error 500",
pod: fixtures.BuildHttpbinPod(),
replicas: 1,
service: fixtures.BuildHttpbinService(),
port: 80,
injector: func(d disruptors.PodDisruptor) error {
fault := disruptors.HTTPFault{
Port: intstr.FromInt32(80),
Expand All @@ -93,10 +97,11 @@ func Test_PodDisruptor(t *testing.T) {
},
},
{
title: "Inject Grpc error",
pod: fixtures.BuildGrpcpbinPod(),
service: fixtures.BuildGrpcbinService(),
port: 9000,
title: "Inject Grpc error",
pod: fixtures.BuildGrpcpbinPod(),
replicas: 1,
service: fixtures.BuildGrpcbinService(),
port: 9000,
injector: func(d disruptors.PodDisruptor) error {
fault := disruptors.GrpcFault{
Port: intstr.FromInt32(9000),
Expand Down Expand Up @@ -136,6 +141,7 @@ func Test_PodDisruptor(t *testing.T) {
k8s,
namespace,
tc.pod,
tc.replicas,
tc.service,
k8sintstr.FromInt(tc.port),
30*time.Second,
Expand Down Expand Up @@ -197,7 +203,8 @@ func Test_PodDisruptor(t *testing.T) {
t.Fatalf("creating kubectl client from kubeconfig: %v", err)
}

port, err := kc.ForwardPodPort(ctx, namespace, tc.pod.Name, uint(tc.port))
// connect via port forwarding to the first pod in the pod set
port, err := kc.ForwardPodPort(ctx, namespace, tc.pod.Name+"-0", uint(tc.port))
if err != nil {
t.Fatalf("forwarding port from %s/%s: %v", namespace, tc.pod.Name, err)
}
Expand All @@ -221,11 +228,11 @@ func Test_PodDisruptor(t *testing.T) {
}

service := fixtures.BuildHttpbinService()

err = deploy.ExposeApp(
k8s,
namespace,
fixtures.BuildHttpbinPod(),
1,
service,
k8sintstr.FromInt(80),
30*time.Second,
Expand Down Expand Up @@ -265,10 +272,70 @@ func Test_PodDisruptor(t *testing.T) {
t.Fatalf("disruptor did not return an error")
}

// It is not possible to use errors.Is here, as ErrNoRequests is returned inside the agent pod. The controller
// only sees the error message printed to stderr.
// It is not possible to use errors.Is here, as ErrNoRequests is returned inside the agent pod.
// The controller only sees the error message printed to stderr.
if !strings.Contains(err.Error(), protocol.ErrNoRequests.Error()) {
t.Fatalf("expected ErrNoRequests, got: %v", err)
}
})

t.Run("Terminate Pod", func(t *testing.T) {
t.Parallel()

namespace, err := namespace.CreateTestNamespace(context.TODO(), t, k8s.Client())
if err != nil {
t.Fatalf("failed to create test namespace: %v", err)
}

err = deploy.RunPodSet(
k8s,
namespace,
fixtures.BuildHttpbinPod(),
3,
30*time.Second,
)
if err != nil {
t.Fatalf("starting pod replicas %v", err)
}

// create pod disruptor that will select all pods
selector := disruptors.PodSelector{
Namespace: namespace,
}
options := disruptors.PodDisruptorOptions{}
disruptor, err := disruptors.NewPodDisruptor(context.TODO(), k8s, selector, options)
if err != nil {
t.Fatalf("creating disruptor: %v", err)
}

targets, _ := disruptor.Targets(context.TODO())
if len(targets) == 0 {
t.Fatalf("No pods matched the selector")
}

fault := disruptors.PodTerminationFault{
Count: intstr.FromInt32(1),
Timeout: 10 * time.Second,
}

terminated, err := disruptor.TerminatePods(context.TODO(), fault)
if err != nil {
t.Fatalf("terminating pods: %v", err)
}

if len(terminated) != int(fault.Count.Int32()) {
t.Fatalf("Invalid number of pods deleted. Expected %d got %d", fault.Count.Int32(), len(terminated))
}

for _, pod := range terminated {
_, err = k8s.Client().CoreV1().Pods(namespace).Get(context.TODO(), pod, metav1.GetOptions{})
if !errors.IsNotFound(err) {
if err == nil {
t.Fatalf("pod '%s/%s' not deleted", namespace, pod)
}

t.Fatalf("failed %v", err)
}
}
})
}
73 changes: 69 additions & 4 deletions e2e/disruptors/service_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"time"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8sintstr "k8s.io/apimachinery/pkg/util/intstr"

"github.com/grafana/xk6-disruptor/pkg/disruptors"
Expand Down Expand Up @@ -54,16 +56,18 @@ func Test_ServiceDisruptor(t *testing.T) {
testCases := []struct {
title string
pod corev1.Pod
replicas int
service corev1.Service
port int
injector func(d disruptors.ServiceDisruptor) error
check checks.Check
}{
{
title: "Inject Http error 500",
pod: fixtures.BuildHttpbinPod(),
service: fixtures.BuildHttpbinService(),
port: 80,
title: "Inject Http error 500",
pod: fixtures.BuildHttpbinPod(),
replicas: 1,
service: fixtures.BuildHttpbinService(),
port: 80,
injector: func(d disruptors.ServiceDisruptor) error {
fault := disruptors.HTTPFault{
Port: intstr.FromInt32(80),
Expand Down Expand Up @@ -100,6 +104,7 @@ func Test_ServiceDisruptor(t *testing.T) {
k8s,
namespace,
tc.pod,
1,
tc.service,
k8sintstr.FromInt(tc.port),
30*time.Second,
Expand Down Expand Up @@ -145,4 +150,64 @@ func Test_ServiceDisruptor(t *testing.T) {
})
}
})

t.Run("Terminate Pod", func(t *testing.T) {
t.Parallel()

namespace, err := namespace.CreateTestNamespace(context.TODO(), t, k8s.Client())
if err != nil {
t.Fatalf("failed to create test namespace: %v", err)
}

service := fixtures.BuildHttpbinService()
err = deploy.ExposeApp(
k8s,
namespace,
fixtures.BuildHttpbinPod(),
3,
service,
k8sintstr.FromInt(80),
30*time.Second,
)
if err != nil {
t.Fatalf("starting pod replicas %v", err)
}

// create pod disruptor that will select all pods
options := disruptors.ServiceDisruptorOptions{}
disruptor, err := disruptors.NewServiceDisruptor(context.TODO(), k8s, service.Name, namespace, options)
if err != nil {
t.Fatalf("creating disruptor: %v", err)
}

targets, _ := disruptor.Targets(context.TODO())
if len(targets) == 0 {
t.Fatalf("No pods matched the selector")
}

fault := disruptors.PodTerminationFault{
Count: intstr.FromInt32(1),
Timeout: 10 * time.Second,
}

terminated, err := disruptor.TerminatePods(context.TODO(), fault)
if err != nil {
t.Fatalf("terminating pods: %v", err)
}

if len(terminated) != int(fault.Count.Int32()) {
t.Fatalf("Invalid number of pods deleted. Expected %d got %d", fault.Count.Int32(), len(terminated))
}

for _, pod := range terminated {
_, err = k8s.Client().CoreV1().Pods(namespace).Get(context.TODO(), pod, metav1.GetOptions{})
if !errors.IsNotFound(err) {
if err == nil {
t.Fatalf("pod '%s/%s' not deleted", namespace, pod)
}

t.Fatalf("failed %v", err)
}
}
})
}
38 changes: 38 additions & 0 deletions pkg/api/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,9 +128,36 @@ func (p *jsProtocolFaultInjector) InjectGrpcFaults(args ...goja.Value) {
}
}

// jsPodFaultInjector implements methods for injecting faults into Pods
type jsPodFaultInjector struct {
ctx context.Context
rt *goja.Runtime
disruptors.PodFaultInjector
}

// TerminatePods is a proxy method. Validates parameters and delegates to the Pod Fault Injector method
func (p *jsPodFaultInjector) TerminatePods(args ...goja.Value) {
if len(args) == 0 {
common.Throw(p.rt, fmt.Errorf("PodTermination fault is required"))
}

fault := disruptors.PodTerminationFault{}
err := convertValue(p.rt, args[0], &fault)
if err != nil {
common.Throw(p.rt, fmt.Errorf("invalid fault argument: %w", err))
}

// TODO: return list of pods terminated
_, err = p.PodFaultInjector.TerminatePods(p.ctx, fault)
if err != nil {
common.Throw(p.rt, fmt.Errorf("error injecting fault: %w", err))
}
}

type jsPodDisruptor struct {
jsDisruptor
jsProtocolFaultInjector
jsPodFaultInjector
}

// buildJsPodDisruptor builds a goja object that implements the PodDisruptor API
Expand All @@ -150,6 +177,11 @@ func buildJsPodDisruptor(
rt: rt,
ProtocolFaultInjector: disruptor,
},
jsPodFaultInjector: jsPodFaultInjector{
ctx: ctx,
rt: rt,
PodFaultInjector: disruptor,
},
}

return buildObject(rt, d)
Expand All @@ -158,6 +190,7 @@ func buildJsPodDisruptor(
type jsServiceDisruptor struct {
jsDisruptor
jsProtocolFaultInjector
jsPodFaultInjector
}

// buildJsServiceDisruptor builds a goja object that implements the ServiceDisruptor API
Expand All @@ -177,6 +210,11 @@ func buildJsServiceDisruptor(
rt: rt,
ProtocolFaultInjector: disruptor,
},
jsPodFaultInjector: jsPodFaultInjector{
ctx: ctx,
rt: rt,
PodFaultInjector: disruptor,
},
}

return buildObject(rt, d)
Expand Down
47 changes: 47 additions & 0 deletions pkg/api/api_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,53 @@ func Test_JsPodDisruptor(t *testing.T) {
`,
expectError: true,
},
{
description: "Terminate Pods (integer count)",
script: `
const fault = {
count: 1
}

d.terminatePods(fault)
`,
expectError: false,
},
{
description: "Terminate Pods (percentage count)",
script: `
const fault = {
count: '100%'
}

d.terminatePods(fault)
`,
expectError: false,
},
{
description: "Terminate Pods (invalid percentage count)",
script: `
const fault = {
count: '100'
}

d.terminatePods(fault)
`,
expectError: true,
},
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd suggest to add a test for an empty fault object as well

Suggested change
},
},
{
description: "Terminate Pods (empty fault)",
script: `
d.terminatePods({})
`,
expectError: true,
},

{
description: "Terminate Pods (missing argument)",
script: `
d.terminatePods()
`,
expectError: true,
},
{
description: "Terminate Pods (empty argument)",
script: `
d.terminatePods({})
`,
expectError: true,
},
}

for _, tc := range testCases {
Expand Down
Loading