Skip to content

Commit

Permalink
feat: add repeated test in http ping agent plugin (#17)
Browse files Browse the repository at this point in the history 
* feat: add consecutive test in http ping agent plugin

* rename variables

* rename variables

* warning log
  • Loading branch information
@lunzhou2024
lunzhou2024 authored Sep 11, 2024
1 parent fb58e54 commit beba465
Show file tree
Hide file tree
Showing 3 changed files with 129 additions and 48 deletions.
20 changes: 18 additions & 2 deletions agent/plugins/syntests/httpPing/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,17 @@ Pings a HTTP endpoint and checks if its the expected response.
1. `key`: `_log`
- `value`: details of a request

## Configuration Items

| Key | Description | Required | Memo |
|----------------------|-------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------------------|
| `address` | The address of the endpoint to ping | Yes | |
| `expectedCodeRegex` | The expected http response code regex | Yes | |
| `retries` | The number of max retry times when http request failed | No | The final result is successful within the retry attempts. |
| `timeoutRetries` | The number of max retry times only when http request exceeded timeout. Recommended value is small number, like 1. | No | The final result is successful within the retry attempts. |
| `repeatsWithoutFail` | The number of total repeated http ping test times | No | The final result is successful only if all repeated tests pass. It’s mutually exclusive with `retries` and `timeoutRetries`. |
| `waitBetweenRepeats` | The ping interval in the repeated http ping test | No | Default value is 5s. |

## Example Configuration

```yaml
Expand All @@ -17,8 +28,13 @@ Pings a HTTP endpoint and checks if its the expected response.
timeoutRetries: 1
```
`retries` is max retry times when http request failed except exceeded timeout
`timeoutRetries` is optional, default value is 0. Max retry times only when http request exceeded timeout. Recommended value is small number, like 1.
```yaml
config : |
address: "localhost:9200"
expectedCodeRegex : ^(202|472){1}
repeatsWithoutFail: 3
waitBetweenRepeats: 3s
```
For multiple endpoints (performs the tests in parallel):
Expand Down
142 changes: 96 additions & 46 deletions agent/plugins/syntests/httpPing/httpPing.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,15 @@ type HttpPingTest struct {
}

const ParallelWorkers = 5
const DefaultWaitBetweenRepeats = "5s"

type HttpPingTestConfig struct {
Address string `yaml:"address"`
ExpectedCodeRegex string `yaml:"expectedCodeRegex"`
MaxRetries int `yaml:"retries"`
MaxTimeoutRetry int `yaml:"timeoutRetries"`
RepeatWithoutFail int `yaml:"repeatsWithoutFail"`
WaitBetweenRepeat string `yaml:"waitBetweenRepeats"`
timeout time.Duration
}

Expand All @@ -73,6 +76,15 @@ func (t *HttpPingTest) Initialise(synTestConfig proto.SynTestConfig) error {
// timeout for each url - based on number of urls, no. of workers
t.timeout = (testTimeout / time.Duration(math.Max(float64(len(configs)/ParallelWorkers), 1))) - time.Second
t.configs = configs
for i := range t.configs {
if (t.configs[i].MaxRetries > 0 || t.configs[i].MaxTimeoutRetry > 0) && t.configs[i].RepeatWithoutFail > 0 {
log.Println("Error: retries/timeoutRetries and repeatsWithoutFail are mutually exclusive and cannot be used together.")
return errors.New("retries/timeoutRetries and repeatsWithoutFail cannot be larger than 0 at the same time")
}
if len(t.configs[i].WaitBetweenRepeat) == 0 {
t.configs[i].WaitBetweenRepeat = DefaultWaitBetweenRepeats
}
}
return nil
}

Expand Down Expand Up @@ -135,6 +147,12 @@ func httpPingTest(_ context.Context, log *log.Logger, d interface{}) (interface{
maxRetries := d.(HttpPingTestConfig).MaxRetries
timeout := d.(HttpPingTestConfig).timeout
maxTimeoutRetries := d.(HttpPingTestConfig).MaxTimeoutRetry
repeatsWithoutFail := d.(HttpPingTestConfig).RepeatWithoutFail
waitBetweenRepeats := d.(HttpPingTestConfig).WaitBetweenRepeat
repeatWaitTime, parseErr := time.ParseDuration(waitBetweenRepeats)
if parseErr != nil {
return result, errors.Wrap(parseErr, "error parsing repeated success test interval")
}

log.Println("address: " + address)

Expand All @@ -154,66 +172,98 @@ func httpPingTest(_ context.Context, log *log.Logger, d interface{}) (interface{
}
req = req.WithContext(ctx)

marks := 0
result["marks"] = marks
resp := &http.Response{}
err = error(nil)
var elapsed_time time.Duration
for i := 0; i <= maxRetries && (ctx.Err() == nil || ctx.Err().Error() == context.DeadlineExceeded.Error()); i++ {
// Allow retry when context deadline exceeded. Retry times not larger than maxRetries
// maxRetries is max retry times when http request failed EXCEPT exceeded timeout
// maxTimeoutRetries is max retry times ONLY when http request exceeded timeout
// maxTimeoutRetries default value is 0. Recommended value is small number, like 1
// Retry for timeout cases decline performance of synthetic test. Use maxTimeoutRetries for compromise
if i > maxTimeoutRetries && ctx.Err() != nil && ctx.Err().Error() == context.DeadlineExceeded.Error() {
break
}
if i > 0 {
log.Println(fmt.Sprintf("(%d/%d) retrying...", i, maxRetries))
match := false
if repeatsWithoutFail > 0 {
// when repeatsWithoutFail is larger than 0, zero-tolerance for ping failure
for i := 1; i <= repeatsWithoutFail; i++ {
log.Println(fmt.Sprintf("(%d/%d) repeat success testing...", i, repeatsWithoutFail))
match, elapsed_time, err = runHttpPing(c, req, log, expectedCodeRegex)
result["elapsed_time"] = int(elapsed_time.Milliseconds())
if ctx.Err() != nil {
log.Println(fmt.Sprintf("(%d/%d) error in http request context when repeat success ping, %v", i, repeatsWithoutFail, ctx.Err()))
return result, ctx.Err()
}
if err != nil {
log.Println(fmt.Sprintf("(%d/%d) failed http request when repeat success ping, %v", i, repeatsWithoutFail, err))
return result, err
}
if !match {
log.Println(fmt.Sprintf("(%d/%d) failed to match expected code when repeat success ping", i, repeatsWithoutFail))
return result, nil
}
if i < repeatsWithoutFail {
log.Println(fmt.Sprintf("sleeping %s before next repeatable success test", waitBetweenRepeats))
time.Sleep(repeatWaitTime)
}
}
start := time.Now()
resp, err = c.Do(req)
if err == nil {
elapsed_time = time.Since(start)
log.Println("request latency: " + elapsed_time.String())
log.Println(fmt.Sprintf("whole repeatable success test successfully after ping %d times", repeatsWithoutFail))
result["marks"] = 1
} else {
for i := 0; i <= maxRetries && (ctx.Err() == nil || ctx.Err().Error() == context.DeadlineExceeded.Error()); i++ {
// Allow retry when context deadline exceeded. Retry times not larger than maxRetries
// maxRetries is max retry times when http request failed EXCEPT exceeded timeout
// maxTimeoutRetries is max retry times ONLY when http request exceeded timeout
// maxTimeoutRetries default value is 0. Recommended value is small number, like 1
// Retry for timeout cases decline performance of synthetic test. Use maxTimeoutRetries for compromise
if i > maxTimeoutRetries && ctx.Err() != nil && ctx.Err().Error() == context.DeadlineExceeded.Error() {
break
}
if i > 0 {
log.Println(fmt.Sprintf("(%d/%d) retrying...", i, maxRetries))
}
match, elapsed_time, err = runHttpPing(c, req, log, expectedCodeRegex)
result["elapsed_time"] = int(elapsed_time.Milliseconds())
break
if err == nil {
break
}
}
if err != nil {
return result, err
}
if match {
result["marks"] = 1
}
log.Println("err:", err)
}
return result, nil
}

func runHttpPing(c *http.Client, req *http.Request, log *log.Logger, expectedCodeRegex string) (bool, time.Duration, error) {
start := time.Now()
resp, err := c.Do(req)
defer func() {
if resp != nil && resp.Body != nil {
log.Println("closing response body")
resp.Body.Close()
}
}()
if err != nil {
return result, err
}

log.Println("request returned code " + strconv.Itoa(resp.StatusCode))
log.Println("details:")
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return result, err
}
bodyString := string(bodyBytes)
log.Println(bodyString)
match, err := regexp.MatchString(expectedCodeRegex, strconv.Itoa(resp.StatusCode))
if err != nil {
return result, err
}
if err == nil {
elapsedTime := time.Since(start)
log.Println("request latency: " + elapsedTime.String())
log.Println("request returned code " + strconv.Itoa(resp.StatusCode))
log.Println("details:")
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return false, 0, err
}
bodyString := string(bodyBytes)
log.Println(bodyString)
match, err := regexp.MatchString(expectedCodeRegex, strconv.Itoa(resp.StatusCode))
if err != nil {
return false, elapsedTime, err
}

if match {
log.Println("ping successful")
marks = 1
result["marks"] = marks
} else {
log.Println("ping failed")
marks = 0
result["marks"] = marks
if match {
log.Println("ping successful")
return true, elapsedTime, nil
} else {
log.Println("ping failed")
return false, elapsedTime, nil
}
}
return result, nil
log.Println("err:", err)
return false, 0, err
}

func (t *HttpPingTest) Finish() error {
Expand Down
15 changes: 15 additions & 0 deletions testing/configs/syntest-configs/http-ping-consecutive.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
name: http-ping-consecutive
labels:
foo: bar-1
pluginName: httpPing
displayName: HTTP Ping Consecutive Test
description: HTTP Ping Consecutive Test
namespace: test-ns-01
node: "*"
repeat: 1m
timeouts:
run: 1m
config: |
address: https://api64.ipify.org
expectedCodeRegex: ^(200|302)$
repeatsWithoutFail: 2

0 comments on commit beba465

Please sign in to comment.