Skip to content

N
nexpie-grafana-theme
Commit 32a45077 by Marcus Efraimsson Committed by GitHub
Options

Elasticsearch: Fix pre-v7.0 and alerting error (#16904) 

This fixes a regression introduced in #16646 where using Elasticsearch 
pre-v7.0 and alerting resulted in an error when trying to deserialize the 
response of total number of hits.
Total number of hits is not in use by the backend so we're removing it 
for now to make ES 6 and 7 being able to deserialize search responses 
without errors.

Closes #15622
parent 575fe7ad
Showing with 150 additions and 105 deletions
pkg/tsdb/elasticsearch/client/client_test.go
...@@ -118,60 +118,39 @@ func TestClient(t *testing.T) { ...@@ -118,60 +118,39 @@ func TestClient(t *testing.T) {
}) })
}) })
Convey("Given a fake http client", func() { httpClientScenario(t, "Given a fake http client and a v2.x client with response", &models.DataSource{
var responseBuffer *bytes.Buffer
var req *http.Request
ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
req = r
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
t.Fatalf("Failed to read response body, err=%v", err)
}
responseBuffer = bytes.NewBuffer(buf)
}))
currentNewDatasourceHttpClient := newDatasourceHttpClient
newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) {
return ts.Client(), nil
}
from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC)
to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC)
fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond))
toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond))
timeRange := tsdb.NewTimeRange(fromStr, toStr)
Convey("and a v2.x client", func() {
ds := models.DataSource{
Database: "[metrics-]YYYY.MM.DD", Database: "[metrics-]YYYY.MM.DD",
Url: ts.URL,
JsonData: simplejson.NewFromAny(map[string]interface{}{ JsonData: simplejson.NewFromAny(map[string]interface{}{
"esVersion": 2, "esVersion": 2,
"timeField": "@timestamp", "timeField": "@timestamp",
"interval": "Daily", "interval": "Daily",
}), }),
}, func(sc *scenarioContext) {
sc.responseBody = `{
"responses": [
{
"hits": { "hits": [], "max_score": 0, "total": 4656 },
"status": 200
} }
]
c, err := NewClient(context.Background(), &ds, timeRange) }`
So(err, ShouldBeNil)
So(c, ShouldNotBeNil)
Convey("When executing multi search", func() { Convey("When executing multi search", func() {
ms, err := createMultisearchForTest(c) ms, err := createMultisearchForTest(sc.client)
So(err, ShouldBeNil)
res, err := sc.client.ExecuteMultisearch(ms)
So(err, ShouldBeNil) So(err, ShouldBeNil)
c.ExecuteMultisearch(ms)
Convey("Should send correct request and payload", func() { Convey("Should send correct request and payload", func() {
So(req, ShouldNotBeNil) So(sc.request, ShouldNotBeNil)
So(req.Method, ShouldEqual, http.MethodPost) So(sc.request.Method, ShouldEqual, http.MethodPost)
So(req.URL.Path, ShouldEqual, "/_msearch") So(sc.request.URL.Path, ShouldEqual, "/_msearch")
So(responseBuffer, ShouldNotBeNil) So(sc.requestBody, ShouldNotBeNil)
headerBytes, err := responseBuffer.ReadBytes('\n') headerBytes, err := sc.requestBody.ReadBytes('\n')
So(err, ShouldBeNil) So(err, ShouldBeNil)
bodyBytes := responseBuffer.Bytes() bodyBytes := sc.requestBody.Bytes()
jHeader, err := simplejson.NewJson(headerBytes) jHeader, err := simplejson.NewJson(headerBytes)
So(err, ShouldBeNil) So(err, ShouldBeNil)
...@@ -192,40 +171,48 @@ func TestClient(t *testing.T) { ...@@ -192,40 +171,48 @@ func TestClient(t *testing.T) {
So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s") So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s")
}) })
}) })
Convey("Should parse response", func() {
So(res.Status, ShouldEqual, 200)
So(res.Responses, ShouldHaveLength, 1)
})
}) })
}) })
Convey("and a v5.x client", func() { httpClientScenario(t, "Given a fake http client and a v5.x client with response", &models.DataSource{
ds := models.DataSource{
Database: "[metrics-]YYYY.MM.DD", Database: "[metrics-]YYYY.MM.DD",
Url: ts.URL,
JsonData: simplejson.NewFromAny(map[string]interface{}{ JsonData: simplejson.NewFromAny(map[string]interface{}{
"esVersion": 5, "esVersion": 5,
"maxConcurrentShardRequests": 100, "maxConcurrentShardRequests": 100,
"timeField": "@timestamp", "timeField": "@timestamp",
"interval": "Daily", "interval": "Daily",
}), }),
}, func(sc *scenarioContext) {
sc.responseBody = `{
"responses": [
{
"hits": { "hits": [], "max_score": 0, "total": 4656 },
"status": 200
} }
]
c, err := NewClient(context.Background(), &ds, timeRange) }`
So(err, ShouldBeNil)
So(c, ShouldNotBeNil)
Convey("When executing multi search", func() { Convey("When executing multi search", func() {
ms, err := createMultisearchForTest(c) ms, err := createMultisearchForTest(sc.client)
So(err, ShouldBeNil)
res, err := sc.client.ExecuteMultisearch(ms)
So(err, ShouldBeNil) So(err, ShouldBeNil)
c.ExecuteMultisearch(ms)
Convey("Should send correct request and payload", func() { Convey("Should send correct request and payload", func() {
So(req, ShouldNotBeNil) So(sc.request, ShouldNotBeNil)
So(req.Method, ShouldEqual, http.MethodPost) So(sc.request.Method, ShouldEqual, http.MethodPost)
So(req.URL.Path, ShouldEqual, "/_msearch") So(sc.request.URL.Path, ShouldEqual, "/_msearch")
So(responseBuffer, ShouldNotBeNil) So(sc.requestBody, ShouldNotBeNil)
headerBytes, err := responseBuffer.ReadBytes('\n') headerBytes, err := sc.requestBody.ReadBytes('\n')
So(err, ShouldBeNil) So(err, ShouldBeNil)
bodyBytes := responseBuffer.Bytes() bodyBytes := sc.requestBody.Bytes()
jHeader, err := simplejson.NewJson(headerBytes) jHeader, err := simplejson.NewJson(headerBytes)
So(err, ShouldBeNil) So(err, ShouldBeNil)
...@@ -246,40 +233,48 @@ func TestClient(t *testing.T) { ...@@ -246,40 +233,48 @@ func TestClient(t *testing.T) {
So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s") So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s")
}) })
}) })
Convey("Should parse response", func() {
So(res.Status, ShouldEqual, 200)
So(res.Responses, ShouldHaveLength, 1)
})
}) })
}) })
Convey("and a v5.6 client", func() { httpClientScenario(t, "Given a fake http client and a v5.6 client with response", &models.DataSource{
ds := models.DataSource{
Database: "[metrics-]YYYY.MM.DD", Database: "[metrics-]YYYY.MM.DD",
Url: ts.URL,
JsonData: simplejson.NewFromAny(map[string]interface{}{ JsonData: simplejson.NewFromAny(map[string]interface{}{
"esVersion": 56, "esVersion": 56,
"maxConcurrentShardRequests": 100, "maxConcurrentShardRequests": 100,
"timeField": "@timestamp", "timeField": "@timestamp",
"interval": "Daily", "interval": "Daily",
}), }),
}, func(sc *scenarioContext) {
sc.responseBody = `{
"responses": [
{
"hits": { "hits": [], "max_score": 0, "total": 4656 },
"status": 200
} }
]
c, err := NewClient(context.Background(), &ds, timeRange) }`
So(err, ShouldBeNil)
So(c, ShouldNotBeNil)
Convey("When executing multi search", func() { Convey("When executing multi search", func() {
ms, err := createMultisearchForTest(c) ms, err := createMultisearchForTest(sc.client)
So(err, ShouldBeNil)
res, err := sc.client.ExecuteMultisearch(ms)
So(err, ShouldBeNil) So(err, ShouldBeNil)
c.ExecuteMultisearch(ms)
Convey("Should send correct request and payload", func() { Convey("Should send correct request and payload", func() {
So(req, ShouldNotBeNil) So(sc.request, ShouldNotBeNil)
So(req.Method, ShouldEqual, http.MethodPost) So(sc.request.Method, ShouldEqual, http.MethodPost)
So(req.URL.Path, ShouldEqual, "/_msearch") So(sc.request.URL.Path, ShouldEqual, "/_msearch")
So(responseBuffer, ShouldNotBeNil) So(sc.requestBody, ShouldNotBeNil)
headerBytes, err := responseBuffer.ReadBytes('\n') headerBytes, err := sc.requestBody.ReadBytes('\n')
So(err, ShouldBeNil) So(err, ShouldBeNil)
bodyBytes := responseBuffer.Bytes() bodyBytes := sc.requestBody.Bytes()
jHeader, err := simplejson.NewJson(headerBytes) jHeader, err := simplejson.NewJson(headerBytes)
So(err, ShouldBeNil) So(err, ShouldBeNil)
...@@ -300,41 +295,49 @@ func TestClient(t *testing.T) { ...@@ -300,41 +295,49 @@ func TestClient(t *testing.T) {
So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s") So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s")
}) })
}) })
Convey("Should parse response", func() {
So(res.Status, ShouldEqual, 200)
So(res.Responses, ShouldHaveLength, 1)
})
}) })
}) })
Convey("and a v7.0 client", func() { httpClientScenario(t, "Given a fake http client and a v7.0 client with response", &models.DataSource{
ds := models.DataSource{
Database: "[metrics-]YYYY.MM.DD", Database: "[metrics-]YYYY.MM.DD",
Url: ts.URL,
JsonData: simplejson.NewFromAny(map[string]interface{}{ JsonData: simplejson.NewFromAny(map[string]interface{}{
"esVersion": 70, "esVersion": 70,
"maxConcurrentShardRequests": 6, "maxConcurrentShardRequests": 6,
"timeField": "@timestamp", "timeField": "@timestamp",
"interval": "Daily", "interval": "Daily",
}), }),
}, func(sc *scenarioContext) {
sc.responseBody = `{
"responses": [
{
"hits": { "hits": [], "max_score": 0, "total": { "value": 4656, "relation": "eq"} },
"status": 200
} }
]
c, err := NewClient(context.Background(), &ds, timeRange) }`
So(err, ShouldBeNil)
So(c, ShouldNotBeNil)
Convey("When executing multi search", func() { Convey("When executing multi search", func() {
ms, err := createMultisearchForTest(c) ms, err := createMultisearchForTest(sc.client)
So(err, ShouldBeNil)
res, err := sc.client.ExecuteMultisearch(ms)
So(err, ShouldBeNil) So(err, ShouldBeNil)
c.ExecuteMultisearch(ms)
Convey("Should send correct request and payload", func() { Convey("Should send correct request and payload", func() {
So(req, ShouldNotBeNil) So(sc.request, ShouldNotBeNil)
So(req.Method, ShouldEqual, http.MethodPost) So(sc.request.Method, ShouldEqual, http.MethodPost)
So(req.URL.Path, ShouldEqual, "/_msearch") So(sc.request.URL.Path, ShouldEqual, "/_msearch")
So(req.URL.RawQuery, ShouldEqual, "max_concurrent_shard_requests=6") So(sc.request.URL.RawQuery, ShouldEqual, "max_concurrent_shard_requests=6")
So(responseBuffer, ShouldNotBeNil) So(sc.requestBody, ShouldNotBeNil)
headerBytes, err := responseBuffer.ReadBytes('\n') headerBytes, err := sc.requestBody.ReadBytes('\n')
So(err, ShouldBeNil) So(err, ShouldBeNil)
bodyBytes := responseBuffer.Bytes() bodyBytes := sc.requestBody.Bytes()
jHeader, err := simplejson.NewJson(headerBytes) jHeader, err := simplejson.NewJson(headerBytes)
So(err, ShouldBeNil) So(err, ShouldBeNil)
...@@ -354,11 +357,11 @@ func TestClient(t *testing.T) { ...@@ -354,11 +357,11 @@ func TestClient(t *testing.T) {
So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s") So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s")
}) })
}) })
})
})
Reset(func() { Convey("Should parse response", func() {
newDatasourceHttpClient = currentNewDatasourceHttpClient So(res.Status, ShouldEqual, 200)
So(res.Responses, ShouldHaveLength, 1)
})
}) })
}) })
}) })
...@@ -376,3 +379,61 @@ func createMultisearchForTest(c Client) (*MultiSearchRequest, error) { ...@@ -376,3 +379,61 @@ func createMultisearchForTest(c Client) (*MultiSearchRequest, error) {
}) })
return msb.Build() return msb.Build()
} }
type scenarioContext struct {
client Client
request *http.Request
requestBody *bytes.Buffer
responseStatus int
responseBody string
}
type scenarioFunc func(*scenarioContext)
func httpClientScenario(t *testing.T, desc string, ds *models.DataSource, fn scenarioFunc) {
t.Helper()
Convey(desc, func() {
sc := &scenarioContext{
responseStatus: 200,
responseBody: `{ "responses": [] }`,
}
ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
sc.request = r
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
t.Fatalf("Failed to read request body, err=%v", err)
}
sc.requestBody = bytes.NewBuffer(buf)
rw.Header().Add("Content-Type", "application/json")
rw.Write([]byte(sc.responseBody))
rw.WriteHeader(sc.responseStatus)
}))
ds.Url = ts.URL
from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC)
to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC)
fromStr := fmt.Sprintf("%d", from.UnixNano()/int64(time.Millisecond))
toStr := fmt.Sprintf("%d", to.UnixNano()/int64(time.Millisecond))
timeRange := tsdb.NewTimeRange(fromStr, toStr)
c, err := NewClient(context.Background(), ds, timeRange)
So(err, ShouldBeNil)
So(c, ShouldNotBeNil)
sc.client = c
currentNewDatasourceHttpClient := newDatasourceHttpClient
newDatasourceHttpClient = func(ds *models.DataSource) (*http.Client, error) {
return ts.Client(), nil
}
defer func() {
ts.Close()
newDatasourceHttpClient = currentNewDatasourceHttpClient
}()
fn(sc)
})
}
pkg/tsdb/elasticsearch/client/models.go
...@@ -42,7 +42,6 @@ func (r *SearchRequest) MarshalJSON() ([]byte, error) { ...@@ -42,7 +42,6 @@ func (r *SearchRequest) MarshalJSON() ([]byte, error) {
// SearchResponseHits represents search response hits // SearchResponseHits represents search response hits
type SearchResponseHits struct { type SearchResponseHits struct {
Hits []map[string]interface{} Hits []map[string]interface{}
Total map[string]interface{}
} }
// SearchResponse represents a search response // SearchResponse represents a search response
...@@ -52,21 +51,6 @@ type SearchResponse struct { ...@@ -52,21 +51,6 @@ type SearchResponse struct {
Hits *SearchResponseHits `json:"hits"` Hits *SearchResponseHits `json:"hits"`
} }
// func (r *Response) getErrMsg() string {
// var msg bytes.Buffer
// errJson := simplejson.NewFromAny(r.Err)
// errType, err := errJson.Get("type").String()
// if err == nil {
// msg.WriteString(fmt.Sprintf("type:%s", errType))
// }
// reason, err := errJson.Get("type").String()
// if err == nil {
// msg.WriteString(fmt.Sprintf("reason:%s", reason))
// }
// return msg.String()
// }
// MultiSearchRequest represents a multi search request // MultiSearchRequest represents a multi search request
type MultiSearchRequest struct { type MultiSearchRequest struct {
Requests []*SearchRequest Requests []*SearchRequest
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment