[Cherry-pick for 1.9.1] (#191)

* chore(codeowners): Add codeowners of litmus-go (#184) Signed-off-by: Udit Gaurav <[email protected]> * chore(events): Adding unique event name for multiple experiment in engine (#182) Signed-off-by: shubhamchaudhary <[email protected]> * chore(litmus-sdk): Adding litmus-sdk cli for experiment creation (#168) Signed-off-by: shubhamchaudhary <[email protected]> * chore(pod-io-stress): Add support for volume mount path for pod-io-stress experiment (#183) Signed-off-by: Udit Gaurav <[email protected]> * feat(abort): Add abort in pod autoscaler experiment (#185) * chore(abort): Add abort in pod autoscaler experiment Signed-off-by: Udit Gaurav <[email protected]> * chore(infra-chaos): Adding node-aff-perc & sequence in node-level chaos (#186) * chore(node-aff-perc): Adding node-aff-perc in node-level chaos Signed-off-by: shubhamchaudhary <[email protected]> * chore(autoscaler): Adding support for statefulset resource type (#187) Signed-off-by: shubhamchaudhary <[email protected]> * chore(Exp): Add EBS Loss Experiment in GO (#188) * chore(Exp): Add EBS Loss Experiment in GO Signed-off-by: Udit Gaurav <[email protected]> * chore(Exp): Add AWS EC2 terminate experiment (#189) * chore(Exp): Add AWS EC2 terminate experiment Signed-off-by: Udit Gaurav <[email protected]> Co-authored-by: Udit Gaurav <[email protected]> Co-authored-by: Shubham Chaudhary <[email protected]>
litmuschaos · Oct 28, 2020 · 112c707 · 112c707
1 parent 6128618
commit 112c707
Show file tree

Hide file tree

Showing 51 changed files with 2,375 additions and 351 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1,5 @@
+# Lines starting with '#' are comments.
+# Each line is a file pattern followed by one or more owners.
+
+# These owners will be the default owners for everything in the repo.
+*           @ispeakc0de @ksatchit @uditgaurav
diff --git a/bin/go-runner.go b/bin/go-runner.go
@@ -31,6 +31,8 @@ import (
 	podNetworkLatency "github.com/litmuschaos/litmus-go/experiments/generic/pod-network-latency/experiment"
 	podNetworkLoss "github.com/litmuschaos/litmus-go/experiments/generic/pod-network-loss/experiment"
 	kafkaBrokerPodFailure "github.com/litmuschaos/litmus-go/experiments/kafka/kafka-broker-pod-failure/experiment"
+	ec2terminate "github.com/litmuschaos/litmus-go/experiments/kube-aws/ec2-terminate/experiment"
+	ebsLoss "github.com/litmuschaos/litmus-go/experiments/kube-aws/ebs-loss/experiment"
 
 	"github.com/litmuschaos/litmus-go/pkg/clients"
 	"github.com/litmuschaos/litmus-go/pkg/log"
@@ -100,6 +102,10 @@ func main() {
 		podNetworkLoss.PodNetworkLoss(clients)
 	case "cassandra-pod-delete":
 		cassandraPodDelete.CasssandraPodDelete(clients)
+	case "ec2-terminate":
+		ec2terminate.EC2Terminate(clients)
+	case "ebs-loss":
+		ebsLoss.EBSLoss(clients)
 	default:
 		log.Fatalf("Unsupported -name %v, please provide the correct value of -name args", *experimentName)
 	}

diff --git a/chaoslib/litmus/ebs-loss/lib/ebs-loss.go b/chaoslib/litmus/ebs-loss/lib/ebs-loss.go
@@ -0,0 +1,147 @@
+package lib
+
+import (
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	clients "github.com/litmuschaos/litmus-go/pkg/clients"
+	ebs "github.com/litmuschaos/litmus-go/pkg/cloud/aws"
+	experimentTypes "github.com/litmuschaos/litmus-go/pkg/kube-aws/ebs-loss/types"
+	"github.com/litmuschaos/litmus-go/pkg/log"
+	"github.com/litmuschaos/litmus-go/pkg/types"
+	"github.com/litmuschaos/litmus-go/pkg/utils/common"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+//InjectEBSLoss contains the chaos injection steps for ebs loss
+func InjectEBSLoss(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
+
+	var err error
+	//Waiting for the ramp time before chaos injection
+	if experimentsDetails.RampTime != 0 {
+		log.Infof("[Ramp]: Waiting for the %vs ramp time before injecting chaos", experimentsDetails.RampTime)
+		common.WaitForDuration(experimentsDetails.RampTime)
+	}
+
+	//Detaching the ebs volume from the instance
+	log.Info("[Chaos]: Detaching the EBS volume from the instance")
+	err = EBSVolumeDetach(experimentsDetails)
+	if err != nil {
+		return errors.Errorf("ebs detachment failed err: %v", err)
+	}
+
+	//Wait for chaos duration
+	log.Infof("[Wait]: Waiting for the chaos duration of %vs", experimentsDetails.ChaosDuration)
+	time.Sleep(time.Duration(experimentsDetails.ChaosDuration) * time.Second)
+
+	//Getting the Ebs status
+	EBSStatus, err := ebs.GetEBSStatus(experimentsDetails)
+	if err != nil {
+		return errors.Errorf("fail to get the ebs status err: %v", err)
+	}
+
+	if EBSStatus != "attached" {
+		//Attaching the ebs volume from the instance
+		log.Info("[Chaos]: Attaching the EBS volume from the instance")
+		err = EBSVolumeAttach(experimentsDetails)
+		if err != nil {
+			return errors.Errorf("ebs attachment failed err: %v", err)
+		}
+	} else {
+		log.Info("[Skip]: The EBS volume is already attached")
+	}
+
+	//Waiting for the ramp time after chaos injection
+	if experimentsDetails.RampTime != 0 {
+		log.Infof("[Ramp]: Waiting for the %vs ramp time after injecting chaos", experimentsDetails.RampTime)
+		common.WaitForDuration(experimentsDetails.RampTime)
+	}
+	return nil
+}
+
+// EBSVolumeDetach will detach the ebs vol from ec2 node
+func EBSVolumeDetach(experimentsDetails *experimentTypes.ExperimentDetails) error {
+
+	// Load session from shared config
+	sess := session.Must(session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+		Config:            aws.Config{Region: aws.String(experimentsDetails.Region)},
+	}))
+
+	// Create new EC2 client
+	ec2Svc := ec2.New(sess)
+
+	input := &ec2.DetachVolumeInput{
+		VolumeId: aws.String(experimentsDetails.EBSVolumeID),
+	}
+
+	result, err := ec2Svc.DetachVolume(input)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			default:
+				return errors.Errorf(aerr.Error())
+			}
+		} else {
+			return errors.Errorf(err.Error())
+		}
+	}
+
+	log.InfoWithValues("Detaching ebs having:", logrus.Fields{
+		"VolumeId":   *result.VolumeId,
+		"State":      *result.State,
+		"Device":     *result.Device,
+		"InstanceId": *result.InstanceId,
+	})
+
+	return nil
+}
+
+// EBSVolumeAttach will detach the ebs vol from ec2 node
+func EBSVolumeAttach(experimentsDetails *experimentTypes.ExperimentDetails) error {
+
+	// Load session from shared config
+	sess := session.Must(session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+		Config:            aws.Config{Region: aws.String(experimentsDetails.Region)},
+	}))
+
+	// Create new EC2 client
+	ec2Svc := ec2.New(sess)
+
+	//Attaching the ebs volume after chaos
+	input := &ec2.AttachVolumeInput{
+		Device:     aws.String(experimentsDetails.DeviceName),
+		InstanceId: aws.String(experimentsDetails.Ec2InstanceID),
+		VolumeId:   aws.String(experimentsDetails.EBSVolumeID),
+	}
+
+	result, err := ec2Svc.AttachVolume(input)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			default:
+				return errors.Errorf(aerr.Error())
+			}
+		} else {
+			return errors.Errorf(err.Error())
+		}
+	}
+
+	log.InfoWithValues("Attaching ebs having:", logrus.Fields{
+		"VolumeId":   *result.VolumeId,
+		"State":      *result.State,
+		"Device":     *result.Device,
+		"InstanceId": *result.InstanceId,
+	})
+
+	//Wait for instance to get attached
+	log.Info("[Wait]: Wait for ebs vol to reattach")
+	time.Sleep(10 * time.Second)
+
+	return nil
+}
diff --git a/chaoslib/litmus/ec2-terminate/lib/ec2-terminate.go b/chaoslib/litmus/ec2-terminate/lib/ec2-terminate.go
@@ -0,0 +1,196 @@
+package lib
+
+import (
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	clients "github.com/litmuschaos/litmus-go/pkg/clients"
+	awslib "github.com/litmuschaos/litmus-go/pkg/cloud/aws"
+	experimentTypes "github.com/litmuschaos/litmus-go/pkg/kube-aws/ec2-terminate/types"
+	"github.com/litmuschaos/litmus-go/pkg/log"
+	"github.com/litmuschaos/litmus-go/pkg/types"
+	"github.com/litmuschaos/litmus-go/pkg/utils/common"
+	"github.com/litmuschaos/litmus-go/pkg/utils/retry"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+//InjectEC2Terminate contains the chaos injection steps for ec2 terminate chaos
+func InjectEC2Terminate(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
+
+	var err error
+	//Waiting for the ramp time before chaos injection
+	if experimentsDetails.RampTime != 0 {
+		log.Infof("[Ramp]: Waiting for the %vs ramp time before injecting chaos", experimentsDetails.RampTime)
+		common.WaitForDuration(experimentsDetails.RampTime)
+	}
+
+	//Stoping the EC2 instance
+	log.Info("[Chaos]: Stoping the desired EC2 instance")
+	err = EC2Stop(experimentsDetails)
+	if err != nil {
+		return errors.Errorf("ec2 instance failed to stop err: %v", err)
+	}
+
+	//Wait for ec2 instance to completely stop
+	log.Info("[Wait]: Wait for EC2 instance to come in stopped state")
+	if err = WaitForEC2Down(experimentsDetails); err != nil {
+		return errors.Errorf("unable to stop the ec2 instance err: %v", err)
+	}
+
+	//Wait for chaos duration
+	log.Infof("[Wait]: Waiting for chaos duration of %vs before starting the instance", experimentsDetails.ChaosDuration)
+	time.Sleep(time.Duration(experimentsDetails.ChaosDuration) * time.Second)
+
+	//Starting the EC2 instance
+	log.Info("[Chaos]: Starting back the EC2 instance")
+	err = EC2Start(experimentsDetails)
+	if err != nil {
+		return errors.Errorf("ec2 instance failed to start err: %v", err)
+	}
+
+	//Wait for ec2 instance to come in running state
+	log.Info("[Wait]: Wait for EC2 instance to get in running state")
+	if err = WaitForEC2Up(experimentsDetails); err != nil {
+		return errors.Errorf("unable to start the ec2 instance err: %v", err)
+	}
+
+	//Waiting for the ramp time after chaos injection
+	if experimentsDetails.RampTime != 0 {
+		log.Infof("[Ramp]: Waiting for the %vs ramp time after injecting chaos", experimentsDetails.RampTime)
+		common.WaitForDuration(experimentsDetails.RampTime)
+	}
+	return nil
+}
+
+// EC2Stop will stop an aws ec2 instance
+func EC2Stop(experimentsDetails *experimentTypes.ExperimentDetails) error {
+
+	// Load session from shared config
+	sess := session.Must(session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+		Config:            aws.Config{Region: aws.String(experimentsDetails.Region)},
+	}))
+
+	// Create new EC2 client
+	ec2Svc := ec2.New(sess)
+
+	input := &ec2.StopInstancesInput{
+		InstanceIds: []*string{
+			aws.String(experimentsDetails.Ec2InstanceID),
+		},
+	}
+	result, err := ec2Svc.StopInstances(input)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			default:
+				return errors.Errorf(aerr.Error())
+			}
+		} else {
+			return errors.Errorf(err.Error())
+		}
+	}
+
+	log.InfoWithValues("Stopping an ec2 instance:", logrus.Fields{
+		"CurrentState":  *result.StoppingInstances[0].CurrentState.Name,
+		"PreviousState": *result.StoppingInstances[0].PreviousState.Name,
+		"InstanceId":    *result.StoppingInstances[0].InstanceId,
+	})
+
+	return nil
+}
+
+// EC2Start will stop an aws ec2 instance
+func EC2Start(experimentsDetails *experimentTypes.ExperimentDetails) error {
+
+	// Load session from shared config
+	sess := session.Must(session.NewSessionWithOptions(session.Options{
+		SharedConfigState: session.SharedConfigEnable,
+		Config:            aws.Config{Region: aws.String(experimentsDetails.Region)},
+	}))
+
+	// Create new EC2 client
+	ec2Svc := ec2.New(sess)
+
+	input := &ec2.StartInstancesInput{
+		InstanceIds: []*string{
+			aws.String(experimentsDetails.Ec2InstanceID),
+		},
+	}
+
+	result, err := ec2Svc.StartInstances(input)
+	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			switch aerr.Code() {
+			default:
+				return errors.Errorf(aerr.Error())
+			}
+		} else {
+			return errors.Errorf(err.Error())
+		}
+	}
+
+	log.InfoWithValues("Starting ec2 instance:", logrus.Fields{
+		"CurrentState":  *result.StartingInstances[0].CurrentState.Name,
+		"PreviousState": *result.StartingInstances[0].PreviousState.Name,
+		"InstanceId":    *result.StartingInstances[0].InstanceId,
+	})
+
+	return nil
+}
+
+//WaitForEC2Down will wait for the ec2 instance to get in stopped state
+func WaitForEC2Down(experimentsDetails *experimentTypes.ExperimentDetails) error {
+
+	log.Info("[Status]: Checking EC2 instance status")
+	err := retry.
+		Times(uint(experimentsDetails.Timeout / experimentsDetails.Delay)).
+		Wait(time.Duration(experimentsDetails.Delay) * time.Second).
+		Try(func(attempt uint) error {
+
+			instanceState, err := awslib.GetEC2InstanceStatus(experimentsDetails)
+			if err != nil {
+				return errors.Errorf("fail to get the instance status")
+			}
+			if instanceState != "stopped" {
+				log.Infof("The instance state is %v", instanceState)
+				return errors.Errorf("instance is not yet in stopped state")
+			}
+			log.Infof("The instance state is %v", instanceState)
+			return nil
+		})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+//WaitForEC2Up will wait for the ec2 instance to get in running state
+func WaitForEC2Up(experimentsDetails *experimentTypes.ExperimentDetails) error {
+
+	log.Info("[Status]: Checking EC2 instance status")
+	err := retry.
+		Times(uint(experimentsDetails.Timeout / experimentsDetails.Delay)).
+		Wait(time.Duration(experimentsDetails.Delay) * time.Second).
+		Try(func(attempt uint) error {
+
+			instanceState, err := awslib.GetEC2InstanceStatus(experimentsDetails)
+			if err != nil {
+				return errors.Errorf("fail to get the instance status")
+			}
+			if instanceState != "running" {
+				log.Infof("The instance state is %v", instanceState)
+				return errors.Errorf("instance is not yet in running state")
+			}
+			log.Infof("The instance state is %v", instanceState)
+			return nil
+		})
+	if err != nil {
+		return err
+	}
+	return nil
+}