Skip to content

N
nexpie-grafana-theme
Commit 364154d8 by bergquist
Options

moves timeout tests to an integration test

parent 35013a28
Showing with 154 additions and 127 deletions
pkg/services/alerting/engine.go
......@@ -205,8 +205,14 @@ func (e *AlertingService) processJob(attemptID int, attemptChan chan int, cancel
}
}
// create new context with timeout for notifications
resultHandleCtx, resultHandleCancelFn := context.WithTimeout(context.Background(), resultHandleTimeout)
cancelChan <- resultHandleCancelFn
// override the context used for evaluation with a new context for notifications.
// This makes it possible for notifiers to execute when datasources
// dont respond within the timeout limit. We should rewrite this so notifications
// dont reuse the evalContext and get its own context.
evalContext.Ctx = resultHandleCtx
evalContext.Rule.State = evalContext.GetNewState()
e.resultHandler.Handle(evalContext)
......
pkg/services/alerting/engine_integration_test.go 0 → 100644
// +build integration
package alerting
import (
"context"
"errors"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
. "github.com/smartystreets/goconvey/convey"
)
func TestEngineTimeouts(t *testing.T) {
Convey("Alerting engine timeout tests", t, func() {
engine := NewEngine()
engine.resultHandler = &FakeResultHandler{}
job := &Job{Running: true, Rule: &Rule{}}
Convey("Should trigger as many retries as needed", func() {
Convey("pended alert for datasource -> result handler should be worked", func() {
// reduce alert timeout to test quickly
originAlertTimeout := alertTimeout
alertTimeout = 2 * time.Second
transportTimeoutInterval := 2 * time.Second
serverBusySleepDuration := 1 * time.Second
evalHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration)
resultHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration)
engine.evalHandler = evalHandler
engine.resultHandler = resultHandler
engine.processJobWithRetry(context.TODO(), job)
So(evalHandler.EvalSucceed, ShouldEqual, true)
So(resultHandler.ResultHandleSucceed, ShouldEqual, true)
// initialize for other tests.
alertTimeout = originAlertTimeout
engine.resultHandler = &FakeResultHandler{}
})
})
})
}
type FakeCommonTimeoutHandler struct {
TransportTimeoutDuration time.Duration
ServerBusySleepDuration time.Duration
EvalSucceed bool
ResultHandleSucceed bool
}
func NewFakeCommonTimeoutHandler(transportTimeoutDuration time.Duration, serverBusySleepDuration time.Duration) *FakeCommonTimeoutHandler {
return &FakeCommonTimeoutHandler{
TransportTimeoutDuration: transportTimeoutDuration,
ServerBusySleepDuration: serverBusySleepDuration,
EvalSucceed: false,
ResultHandleSucceed: false,
}
}
func (handler *FakeCommonTimeoutHandler) Eval(evalContext *EvalContext) {
// 1. prepare mock server
path := "/evaltimeout"
srv := runBusyServer(path, handler.ServerBusySleepDuration)
defer srv.Close()
// 2. send requests
url := srv.URL + path
res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration)
if res != nil {
defer res.Body.Close()
}
if err != nil {
evalContext.Error = errors.New("Fake evaluation timeout test failure")
return
}
if res.StatusCode == 200 {
handler.EvalSucceed = true
}
evalContext.Error = errors.New("Fake evaluation timeout test failure; wrong response")
}
func (handler *FakeCommonTimeoutHandler) Handle(evalContext *EvalContext) error {
// 1. prepare mock server
path := "/resulthandle"
srv := runBusyServer(path, handler.ServerBusySleepDuration)
defer srv.Close()
// 2. send requests
url := srv.URL + path
res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration)
if res != nil {
defer res.Body.Close()
}
if err != nil {
evalContext.Error = errors.New("Fake result handle timeout test failure")
return evalContext.Error
}
if res.StatusCode == 200 {
handler.ResultHandleSucceed = true
return nil
}
evalContext.Error = errors.New("Fake result handle timeout test failure; wrong response")
return evalContext.Error
}
func runBusyServer(path string, serverBusySleepDuration time.Duration) *httptest.Server {
mux := http.NewServeMux()
server := httptest.NewServer(mux)
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
time.Sleep(serverBusySleepDuration)
})
return server
}
func sendRequest(context context.Context, url string, transportTimeoutInterval time.Duration) (resp *http.Response, err error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req = req.WithContext(context)
transport := http.Transport{
Dial: (&net.Dialer{
Timeout: transportTimeoutInterval,
KeepAlive: transportTimeoutInterval,
}).Dial,
}
client := http.Client{
Transport: &transport,
}
return client.Do(req)
}
pkg/services/alerting/engine_test.go
......@@ -4,11 +4,7 @@ import (
"context"
"errors"
"math"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
. "github.com/smartystreets/goconvey/convey"
)
......@@ -38,106 +34,6 @@ func (handler *FakeResultHandler) Handle(evalContext *EvalContext) error {
return nil
}
type FakeCommonTimeoutHandler struct {
TransportTimeoutDuration time.Duration
ServerBusySleepDuration time.Duration
EvalSucceed bool
ResultHandleSucceed bool
}
func NewFakeCommonTimeoutHandler(transportTimeoutDuration time.Duration, serverBusySleepDuration time.Duration) *FakeCommonTimeoutHandler {
return &FakeCommonTimeoutHandler{
TransportTimeoutDuration: transportTimeoutDuration,
ServerBusySleepDuration: serverBusySleepDuration,
EvalSucceed: false,
ResultHandleSucceed: false,
}
}
func (handler *FakeCommonTimeoutHandler) Eval(evalContext *EvalContext) {
// 1. prepare mock server
path := "/evaltimeout"
srv := runBusyServer(path, handler.ServerBusySleepDuration)
defer srv.Close()
// 2. send requests
url := srv.URL + path
res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration)
if res != nil {
defer res.Body.Close()
}
if err != nil {
evalContext.Error = errors.New("Fake evaluation timeout test failure")
return
}
if res.StatusCode == 200 {
handler.EvalSucceed = true
}
evalContext.Error = errors.New("Fake evaluation timeout test failure; wrong response")
}
func (handler *FakeCommonTimeoutHandler) Handle(evalContext *EvalContext) error {
// 1. prepare mock server
path := "/resulthandle"
srv := runBusyServer(path, handler.ServerBusySleepDuration)
defer srv.Close()
// 2. send requests
url := srv.URL + path
res, err := sendRequest(evalContext.Ctx, url, handler.TransportTimeoutDuration)
if res != nil {
defer res.Body.Close()
}
if err != nil {
evalContext.Error = errors.New("Fake result handle timeout test failure")
return evalContext.Error
}
if res.StatusCode == 200 {
handler.ResultHandleSucceed = true
return nil
}
evalContext.Error = errors.New("Fake result handle timeout test failure; wrong response")
return evalContext.Error
}
func runBusyServer(path string, serverBusySleepDuration time.Duration) *httptest.Server {
mux := http.NewServeMux()
server := httptest.NewServer(mux)
mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
time.Sleep(serverBusySleepDuration)
})
return server
}
func sendRequest(context context.Context, url string, transportTimeoutInterval time.Duration) (resp *http.Response, err error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req = req.WithContext(context)
transport := http.Transport{
Dial: (&net.Dialer{
Timeout: transportTimeoutInterval,
KeepAlive: transportTimeoutInterval,
}).Dial,
}
client := http.Client{
Transport: &transport,
}
return client.Do(req)
}
func TestEngineProcessJob(t *testing.T) {
Convey("Alerting engine job processing", t, func() {
engine := NewEngine()
......@@ -217,29 +113,6 @@ func TestEngineProcessJob(t *testing.T) {
engine.processJobWithRetry(context.TODO(), job)
So(evalHandler.CallNb, ShouldEqual, expectedAttempts)
})
Convey("pended alert for datasource -> result handler should be worked", func() {
// reduce alert timeout to test quickly
originAlertTimeout := alertTimeout
alertTimeout = 5 * time.Second
transportTimeoutInterval := 5 * time.Second
serverBusySleepDuration := 4 * time.Second
evalHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration)
resultHandler := NewFakeCommonTimeoutHandler(transportTimeoutInterval, serverBusySleepDuration)
engine.evalHandler = evalHandler
engine.resultHandler = resultHandler
engine.processJobWithRetry(context.TODO(), job)
So(evalHandler.EvalSucceed, ShouldEqual, true)
So(resultHandler.ResultHandleSucceed, ShouldEqual, true)
// initialize for other tests.
alertTimeout = originAlertTimeout
engine.resultHandler = &FakeResultHandler{}
})
})
})
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment