Skip to content

Commit

Permalink
[Cherry-pick for 2.0.0] (#435)
Browse files Browse the repository at this point in the history
* Auxiliary Application Check for node-memory-hog Experiment and Multiple Target Node Functionality Fix for three Node-Level Experiments (#424)

* Fixed multiple node selection functionality in nodes.go

Signed-off-by: neelanjan00 <neelanjanmanna@gmail.com>

* Added Auxiliary Application Status Check to node-memory-hog experiment

Signed-off-by: neelanjan00 <neelanjanmanna@gmail.com>

* Pod experiment fixes (#429)

* Fixed issue with pod affected percentage > 100

Signed-off-by: Akash Shrivastava <akash@chaosnative.com>

* resolve conflict

Signed-off-by: udit <udit@chaosnative.com>

* fixing bugs (#418)

Signed-off-by: Oum Kale <oumkale@chaosnative.com>

Co-authored-by: Shubham Chaudhary <shubham@chaosnative.com>

* feat(helper-status): fix helper status timeout when helper pod failed (#421)

Signed-off-by: shubham chaudhary <shubham@chaosnative.com>

* resolve conflict

Signed-off-by: udit <udit@chaosnative.com>

* feat(exec-chaos): handle the abort signal (#425)

Signed-off-by: shubham chaudhary <shubham@chaosnative.com>

* feat(target-pods: validating target pods env (#430)

Signed-off-by: shubham chaudhary <shubham@chaosnative.com>

* feat(experiment): derive parent name in pod-delete only (#434)

Signed-off-by: shubham chaudhary <shubham@chaosnative.com>

Co-authored-by: Neelanjan Manna <neelanjanmanna@gmail.com>
Co-authored-by: Akash Shrivastava <akash@chaosnative.com>
Co-authored-by: OUM NIVRATHI KALE <oumkale@chaosnative.com>
Co-authored-by: Shubham Chaudhary <shubham@chaosnative.com>
  • Loading branch information
5 people authored Aug 14, 2021
1 parent f67b15d commit 6ad9db0
Show file tree
Hide file tree
Showing 64 changed files with 610 additions and 422 deletions.
12 changes: 12 additions & 0 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,18 @@ jobs:
sudo apt-get update && sudo apt-get install golint
make gotasks
- name: gofmt check
run: |
if [ "$(gofmt -s -l . | wc -l)" -ne 0 ]
then
echo "The following files were found to be not go formatted:"
gofmt -s -l .
exit 1
fi
- name: golangci-lint
uses: reviewdog/action-golangci-lint@v1

build:
needs: pre-checks
runs-on: ubuntu-latest
Expand Down
12 changes: 12 additions & 0 deletions .github/workflows/push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,18 @@ jobs:
sudo apt-get update && sudo apt-get install golint
make gotasks
- name: gofmt check
run: |
if [ "$(gofmt -s -l . | wc -l)" -ne 0 ]
then
echo "The following files were found to be not go formatted:"
gofmt -s -l .
exit 1
fi
- name: golangci-lint
uses: reviewdog/action-golangci-lint@v1

push:
needs: pre-checks
runs-on: ubuntu-latest
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -91,13 +91,6 @@ func PrepareDockerServiceKill(experimentsDetails *experimentTypes.ExperimentDeta
return errors.Errorf("helper pod failed, err: %v", err)
}

// Checking the status of target nodes
log.Info("[Status]: Getting the status of target nodes")
if err = status.CheckNodeStatus(experimentsDetails.TargetNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
common.DeleteHelperPodBasedOnJobCleanupPolicy(experimentsDetails.ExperimentName+"-helper-"+experimentsDetails.RunID, appLabel, chaosDetails, clients)
log.Warnf("Target nodes are not in the ready state, you may need to manually recover the node, err: %v", err)
}

//Deleting the helper pod
log.Info("[Cleanup]: Deleting the helper pod")
if err = common.DeletePod(experimentsDetails.ExperimentName+"-helper-"+experimentsDetails.RunID, appLabel, experimentsDetails.ChaosNamespace, chaosDetails.Timeout, chaosDetails.Delay, clients); err != nil {
Expand Down
29 changes: 25 additions & 4 deletions chaoslib/litmus/kafka-broker-pod-failure/lib/pod-delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/litmuschaos/litmus-go/pkg/probe"
"github.com/litmuschaos/litmus-go/pkg/status"
"github.com/litmuschaos/litmus-go/pkg/types"
"github.com/litmuschaos/litmus-go/pkg/utils/annotation"
"github.com/litmuschaos/litmus-go/pkg/utils/common"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -74,8 +75,18 @@ func injectChaosInSerialMode(experimentsDetails *experimentTypes.ExperimentDetai
return err
}

for _, target := range chaosDetails.ParentsResources {
common.SetTargets(target, "targeted", chaosDetails.AppDetail.Kind, chaosDetails)
// deriving the parent name of the target resources
if chaosDetails.AppDetail.Kind != "" {
for _, pod := range targetPodList.Items {
parentName, err := annotation.GetParentName(clients, pod, chaosDetails)
if err != nil {
return err
}
common.SetParentName(parentName, chaosDetails)
}
for _, target := range chaosDetails.ParentsResources {
common.SetTargets(target, "targeted", chaosDetails.AppDetail.Kind, chaosDetails)
}
}

if experimentsDetails.ChaoslibDetail.EngineName != "" {
Expand Down Expand Up @@ -152,8 +163,18 @@ func injectChaosInParallelMode(experimentsDetails *experimentTypes.ExperimentDet
return err
}

for _, target := range chaosDetails.ParentsResources {
common.SetTargets(target, "targeted", chaosDetails.AppDetail.Kind, chaosDetails)
// deriving the parent name of the target resources
if chaosDetails.AppDetail.Kind != "" {
for _, pod := range targetPodList.Items {
parentName, err := annotation.GetParentName(clients, pod, chaosDetails)
if err != nil {
return err
}
common.SetParentName(parentName, chaosDetails)
}
for _, target := range chaosDetails.ParentsResources {
common.SetTargets(target, "targeted", chaosDetails.AppDetail.Kind, chaosDetails)
}
}

if experimentsDetails.ChaoslibDetail.EngineName != "" {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,13 +93,6 @@ func PrepareKubeletKill(experimentsDetails *experimentTypes.ExperimentDetails, c
return errors.Errorf("helper pod failed, err: %v", err)
}

// Checking the status of target nodes
log.Info("[Status]: Getting the status of target nodes")
if err = status.CheckNodeStatus(experimentsDetails.TargetNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
common.DeleteHelperPodBasedOnJobCleanupPolicy(experimentsDetails.ExperimentName+"-helper-"+experimentsDetails.RunID, appLabel, chaosDetails, clients)
log.Warnf("Target nodes are not in the ready state, you may need to manually recover the node, err: %v", err)
}

//Deleting the helper pod
log.Info("[Cleanup]: Deleting the helper pod")
if err = common.DeletePod(experimentsDetails.ExperimentName+"-helper-"+experimentsDetails.RunID, appLabel, experimentsDetails.ChaosNamespace, chaosDetails.Timeout, chaosDetails.Delay, clients); err != nil {
Expand Down
17 changes: 0 additions & 17 deletions chaoslib/litmus/node-cpu-hog/lib/node-cpu-hog.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,13 +123,6 @@ func injectChaosInSerialMode(experimentsDetails *experimentTypes.ExperimentDetai
return errors.Errorf("helper pod failed due to, err: %v", err)
}

// Checking the status of target nodes
log.Info("[Status]: Getting the status of target nodes")
if err = status.CheckNodeStatus(appNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
common.DeleteHelperPodBasedOnJobCleanupPolicy(experimentsDetails.ExperimentName+"-helper-"+experimentsDetails.RunID, appLabel, chaosDetails, clients)
log.Warnf("Target nodes are not in the ready state, you may need to manually recover the node, err: %v", err)
}

//Deleting the helper pod
log.Info("[Cleanup]: Deleting the helper pod")
if err = common.DeletePod(experimentsDetails.ExperimentName+"-helper-"+experimentsDetails.RunID, appLabel, experimentsDetails.ChaosNamespace, chaosDetails.Timeout, chaosDetails.Delay, clients); err != nil {
Expand Down Expand Up @@ -201,16 +194,6 @@ func injectChaosInParallelMode(experimentsDetails *experimentTypes.ExperimentDet
return errors.Errorf("helper pod failed due to, err: %v", err)
}

for _, appNode := range targetNodeList {

// Checking the status of application node
log.Info("[Status]: Getting the status of application node")
if err = status.CheckNodeStatus(appNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
common.DeleteAllHelperPodBasedOnJobCleanupPolicy(appLabel, chaosDetails, clients)
log.Warn("Application node is not in the ready state, you may need to manually recover the node")
}
}

//Deleting the helper pod
log.Info("[Cleanup]: Deleting the helper pod")
if err = common.DeleteAllPod(appLabel, experimentsDetails.ChaosNamespace, chaosDetails.Timeout, chaosDetails.Delay, clients); err != nil {
Expand Down
6 changes: 0 additions & 6 deletions chaoslib/litmus/node-drain/lib/node-drain.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,12 +100,6 @@ func PrepareNodeDrain(experimentsDetails *experimentTypes.ExperimentDetails, cli
return err
}

// Checking the status of target nodes
log.Info("[Status]: Getting the status of target nodes")
if err = status.CheckNodeStatus(experimentsDetails.TargetNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
log.Warnf("Target nodes are not in the ready state, you may need to manually recover the node, err: %v", err)
}

//Waiting for the ramp time after chaos injection
if experimentsDetails.RampTime != 0 {
log.Infof("[Ramp]: Waiting for the %vs ramp time after injecting chaos", experimentsDetails.RampTime)
Expand Down
17 changes: 0 additions & 17 deletions chaoslib/litmus/node-io-stress/lib/node-io-stress.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,13 +115,6 @@ func injectChaosInSerialMode(experimentsDetails *experimentTypes.ExperimentDetai
return errors.Errorf("helper pod failed due to, err: %v", err)
}

// Checking the status of target nodes
log.Info("[Status]: Getting the status of target nodes")
if err = status.CheckNodeStatus(appNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
common.DeleteHelperPodBasedOnJobCleanupPolicy(experimentsDetails.ExperimentName+"-helper-"+experimentsDetails.RunID, appLabel, chaosDetails, clients)
log.Warnf("Target nodes are not in the ready state, you may need to manually recover the node, err: %v", err)
}

//Deleting the helper pod
log.Info("[Cleanup]: Deleting the helper pod")
if err = common.DeletePod(experimentsDetails.ExperimentName+"-helper-"+experimentsDetails.RunID, appLabel, experimentsDetails.ChaosNamespace, chaosDetails.Timeout, chaosDetails.Delay, clients); err != nil {
Expand Down Expand Up @@ -188,16 +181,6 @@ func injectChaosInParallelMode(experimentsDetails *experimentTypes.ExperimentDet
return errors.Errorf("helper pod failed due to, err: %v", err)
}

for _, appNode := range targetNodeList {

// Checking the status of application node
log.Info("[Status]: Getting the status of application node")
if err = status.CheckNodeStatus(appNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
common.DeleteAllHelperPodBasedOnJobCleanupPolicy(appLabel, chaosDetails, clients)
log.Warn("Application node is not in the ready state, you may need to manually recover the node")
}
}

//Deleting the helper pod
log.Info("[Cleanup]: Deleting the helper pod")
if err = common.DeleteAllPod(appLabel, experimentsDetails.ChaosNamespace, chaosDetails.Timeout, chaosDetails.Delay, clients); err != nil {
Expand Down
17 changes: 0 additions & 17 deletions chaoslib/litmus/node-memory-hog/lib/node-memory-hog.go
Original file line number Diff line number Diff line change
Expand Up @@ -131,13 +131,6 @@ func injectChaosInSerialMode(experimentsDetails *experimentTypes.ExperimentDetai
return errors.Errorf("helper pod status is %v", podStatus)
}

// Checking the status of target nodes
log.Info("[Status]: Getting the status of target nodes")
if err = status.CheckNodeStatus(appNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
common.DeleteHelperPodBasedOnJobCleanupPolicy(experimentsDetails.ExperimentName+"-helper-"+experimentsDetails.RunID, appLabel, chaosDetails, clients)
log.Warnf("Target nodes are not in the ready state, you may need to manually recover the node, err: %v", err)
}

//Deleting the helper pod
log.Info("[Cleanup]: Deleting the helper pod")
if err = common.DeletePod(experimentsDetails.ExperimentName+"-helper-"+experimentsDetails.RunID, appLabel, experimentsDetails.ChaosNamespace, chaosDetails.Timeout, chaosDetails.Delay, clients); err != nil {
Expand Down Expand Up @@ -217,16 +210,6 @@ func injectChaosInParallelMode(experimentsDetails *experimentTypes.ExperimentDet
return errors.Errorf("helper pod status is %v", podStatus)
}

for _, appNode := range targetNodeList {

// Checking the status of application node
log.Info("[Status]: Getting the status of application node")
if err = status.CheckNodeStatus(appNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
common.DeleteAllHelperPodBasedOnJobCleanupPolicy(appLabel, chaosDetails, clients)
log.Warn("Application node is not in the ready state, you may need to manually recover the node")
}
}

//Deleting the helper pod
log.Info("[Cleanup]: Deleting the helper pod")
if err = common.DeleteAllPod(appLabel, experimentsDetails.ChaosNamespace, chaosDetails.Timeout, chaosDetails.Delay, clients); err != nil {
Expand Down
13 changes: 0 additions & 13 deletions chaoslib/litmus/node-restart/lib/node-restart.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,12 +58,6 @@ func PrepareNodeRestart(experimentsDetails *experimentTypes.ExperimentDetails, c
"Target Node IP": experimentsDetails.TargetNodeIP,
})

// Checking the status of target node
log.Info("[Status]: Getting the status of target node")
if err = status.CheckNodeStatus(experimentsDetails.TargetNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
return errors.Errorf("target node is not in ready state, err: %v", err)
}

experimentsDetails.RunID = common.GetRunID()
appLabel := "name=" + experimentsDetails.ExperimentName + "-helper-" + experimentsDetails.RunID

Expand Down Expand Up @@ -113,13 +107,6 @@ func PrepareNodeRestart(experimentsDetails *experimentTypes.ExperimentDetails, c
return errors.Errorf("helper pod failed due to, err: %v", err)
}

// Checking the status of application node
log.Info("[Status]: Getting the status of application node")
if err = status.CheckNodeStatus(experimentsDetails.TargetNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
common.DeleteHelperPodBasedOnJobCleanupPolicy(experimentsDetails.ExperimentName+"-helper-"+experimentsDetails.RunID, appLabel, chaosDetails, clients)
log.Warnf("Application node is not in the ready state, you may need to manually recover the node, err: %v", err)
}

//Deleting the helper pod
log.Info("[Cleanup]: Deleting the helper pod")
if err = common.DeletePod(experimentsDetails.ExperimentName+"-helper-"+experimentsDetails.RunID, appLabel, experimentsDetails.ChaosNamespace, chaosDetails.Timeout, chaosDetails.Delay, clients); err != nil {
Expand Down
6 changes: 0 additions & 6 deletions chaoslib/litmus/node-taint/lib/node-taint.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,12 +98,6 @@ func PrepareNodeTaint(experimentsDetails *experimentTypes.ExperimentDetails, cli
return err
}

// Checking the status of target nodes
log.Info("[Status]: Getting the status of target nodes")
if err = status.CheckNodeStatus(experimentsDetails.TargetNode, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
log.Warnf("Target nodes are not in the ready state, you may need to manually recover the node, err: %v", err)
}

//Waiting for the ramp time after chaos injection
if experimentsDetails.RampTime != 0 {
log.Infof("[Ramp]: Waiting for the %vs ramp time after injecting chaos", experimentsDetails.RampTime)
Expand Down
Loading

0 comments on commit 6ad9db0

Please sign in to comment.