Commit 71782772 by Mitsuhiro Tanda Committed by GitHub

Cloud Monitoring: MQL support (#26551)

* cloud monitoring mql support

* reduce nesting

* remove resource type from deep link since. its removed for two reasons. first of all it is not needed for the link to work. secondly, by adding the resource type, the the link will differ from the query in grafana which I think is misleading

* use frame.meta.executedQueryString instead of legacy meta

Co-authored-by: Erik Sundell <erik.sundell87@gmail.com>
parent 84ee4143
......@@ -222,6 +222,27 @@ The Alias By field allows you to control the format of the legend keys for SLO q
SLO queries use the same [alignment period functionality as metric queries]({{< relref "#metric-queries" >}}).
### MQL (Monitoring Query Language) queries
> **Note:** Only available in Grafana v7.4+.
The MQL query builder in the Google Cloud Monitoring data source allows you to display MQL results in time series format. To get an understanding of the basic concepts in MQL, refer to [Introduction to Monitoring Query Language](https://cloud.google.com/monitoring/mql).
#### Create an MQL query
To create an MQL query, follow these steps:
1. In the **Query Type** list, select **Metrics**.
2. Click **<> Edit MQL** right next to the **Query Type** field. This will toggle the metric query builder mode so that raw MQL queries can be used.
3. Choose a project from the **Project** list.
4. Add the [MQL](https://cloud.google.com/monitoring/mql/query-language) query of your choice in the text area.
#### Alias patterns for MQL queries
MQL queries use the same alias patterns as [metric queries]({{< relref "#metric-queries" >}}).
`{{metric.service}}` is not supported. `{{metric.type}}` and `{{metric.name}}` show the time series key in the response.
## Templating
Instead of hard-coding things like server, application and sensor name in your metric queries you can use variables in their place.
......
......@@ -2,9 +2,7 @@ package cloudmonitoring
import (
"context"
"strconv"
"strings"
"time"
"github.com/grafana/grafana/pkg/tsdb"
)
......@@ -16,12 +14,12 @@ func (e *CloudMonitoringExecutor) executeAnnotationQuery(ctx context.Context, ts
firstQuery := tsdbQuery.Queries[0]
queries, err := e.buildQueries(tsdbQuery)
queries, err := e.buildQueryExecutors(tsdbQuery)
if err != nil {
return nil, err
}
queryRes, resp, err := e.executeQuery(ctx, queries[0], tsdbQuery)
queryRes, resp, _, err := queries[0].run(ctx, tsdbQuery, e)
if err != nil {
return nil, err
}
......@@ -30,36 +28,12 @@ func (e *CloudMonitoringExecutor) executeAnnotationQuery(ctx context.Context, ts
title := metricQuery.Get("title").MustString()
text := metricQuery.Get("text").MustString()
tags := metricQuery.Get("tags").MustString()
err = e.parseToAnnotations(queryRes, resp, queries[0], title, text, tags)
err = queries[0].parseToAnnotations(queryRes, resp, title, text, tags)
result.Results[firstQuery.RefId] = queryRes
return result, err
}
func (e *CloudMonitoringExecutor) parseToAnnotations(queryRes *tsdb.QueryResult, data cloudMonitoringResponse, query *cloudMonitoringQuery, title string, text string, tags string) error {
annotations := make([]map[string]string, 0)
for _, series := range data.TimeSeries {
// reverse the order to be ascending
for i := len(series.Points) - 1; i >= 0; i-- {
point := series.Points[i]
value := strconv.FormatFloat(point.Value.DoubleValue, 'f', 6, 64)
if series.ValueType == "STRING" {
value = point.Value.StringValue
}
annotation := make(map[string]string)
annotation["time"] = point.Interval.EndTime.UTC().Format(time.RFC3339)
annotation["title"] = formatAnnotationText(title, value, series.Metric.Type, series.Metric.Labels, series.Resource.Labels)
annotation["tags"] = tags
annotation["text"] = formatAnnotationText(text, value, series.Metric.Type, series.Metric.Labels, series.Resource.Labels)
annotations = append(annotations, annotation)
}
}
transformAnnotationToTable(annotations, queryRes)
return nil
}
func transformAnnotationToTable(data []map[string]string, result *tsdb.QueryResult) {
table := &tsdb.Table{
Columns: make([]tsdb.TableColumn, 4),
......
......@@ -15,10 +15,9 @@ func TestCloudMonitoringExecutor_parseToAnnotations(t *testing.T) {
require.Len(t, data.TimeSeries, 3)
res := &tsdb.QueryResult{Meta: simplejson.New(), RefId: "annotationQuery"}
query := &cloudMonitoringQuery{}
query := &cloudMonitoringTimeSeriesFilter{}
executor := &CloudMonitoringExecutor{}
err = executor.parseToAnnotations(res, data, query, "atitle {{metric.label.instance_name}} {{metric.value}}", "atext {{resource.label.zone}}", "atag")
err = query.parseToAnnotations(res, data, "atitle {{metric.label.instance_name}} {{metric.value}}", "atext {{resource.label.zone}}", "atag")
require.NoError(t, err)
require.Len(t, res.Tables, 1)
......
{
"timeSeriesDescriptor": {
"labelDescriptors": [
{
"key": "resource.project_id"
},
{
"key": "resource.zone"
},
{
"key": "resource.instance_id"
}
],
"pointDescriptors": [
{
"key": "value.read_bytes_count",
"valueType": "INT64",
"metricKind": "DELTA"
}
]
},
"timeSeriesData": [
{
"labelValues": [
{
"stringValue": "grafana-prod"
},
{
"stringValue": "asia-northeast1-c"
},
{
"stringValue": "6724404429462225363"
}
],
"pointData": [
{
"values": [
{
"int64Value": "0"
}
],
"timeInterval": {
"startTime": "2020-05-18T09:47:00Z",
"endTime": "2020-05-18T09:48:00Z"
}
},
{
"values": [
{
"int64Value": "0"
}
],
"timeInterval": {
"startTime": "2020-05-18T09:46:00Z",
"endTime": "2020-05-18T09:47:00Z"
}
}
]
}
]
}
package cloudmonitoring
import (
"context"
"net/url"
"time"
"github.com/grafana/grafana/pkg/tsdb"
)
type (
cloudMonitoringQuery struct {
cloudMonitoringQueryExecutor interface {
run(ctx context.Context, tsdbQuery *tsdb.TsdbQuery, e *CloudMonitoringExecutor) (*tsdb.QueryResult, cloudMonitoringResponse, string, error)
parseResponse(queryRes *tsdb.QueryResult, data cloudMonitoringResponse, executedQueryString string) error
parseToAnnotations(queryRes *tsdb.QueryResult, data cloudMonitoringResponse, title string, text string, tags string) error
buildDeepLink() string
getRefID() string
getUnit() string
}
// Used to build time series filters
cloudMonitoringTimeSeriesFilter struct {
Target string
Params url.Values
RefID string
......@@ -19,6 +32,17 @@ type (
Unit string
}
// Used to build MQL queries
cloudMonitoringTimeSeriesQuery struct {
RefID string
ProjectName string
Query string
IntervalMS int64
AliasBy string
timeRange *tsdb.TimeRange
Unit string
}
metricQuery struct {
ProjectName string
MetricType string
......@@ -29,6 +53,8 @@ type (
Filters []string
AliasBy string
View string
EditorMode string
Query string
Unit string
}
......@@ -68,9 +94,60 @@ type (
cloudMonitoringResponse struct {
TimeSeries []timeSeries `json:"timeSeries"`
TimeSeriesDescriptor timeSeriesDescriptor `json:"timeSeriesDescriptor"`
TimeSeriesData timeSeriesData `json:"timeSeriesData"`
}
)
type timeSeriesDescriptor struct {
LabelDescriptors []struct {
Key string `json:"key"`
ValueType string `json:"valueType"`
Description string `json:"description"`
} `json:"labelDescriptors"`
PointDescriptors []struct {
Key string `json:"key"`
ValueType string `json:"valueType"`
MetricKind string `json:"metricKind"`
} `json:"pointDescriptors"`
}
type timeSeriesData []struct {
LabelValues []struct {
BoolValue bool `json:"boolValue"`
Int64Value int64 `json:"int64Value"`
StringValue string `json:"stringValue"`
} `json:"labelValues"`
PointData []struct {
Values []struct {
BoolValue bool `json:"boolValue"`
Int64Value string `json:"int64Value"`
DoubleValue float64 `json:"doubleValue"`
StringValue string `json:"stringValue"`
DistributionValue struct {
Count string `json:"count"`
Mean float64 `json:"mean"`
SumOfSquaredDeviation float64 `json:"sumOfSquaredDeviation"`
Range struct {
Min int `json:"min"`
Max int `json:"max"`
} `json:"range"`
BucketOptions cloudMonitoringBucketOptions `json:"bucketOptions"`
BucketCounts []string `json:"bucketCounts"`
Examplars []struct {
Value float64 `json:"value"`
Timestamp string `json:"timestamp"`
// attachments
} `json:"examplars"`
} `json:"distributionValue"`
} `json:"values"`
TimeInterval struct {
EndTime time.Time `json:"endTime"`
StartTime time.Time `json:"startTime"`
} `json:"timeInterval"`
} `json:"pointData"`
}
type timeSeries struct {
Metric struct {
Labels map[string]string `json:"labels"`
......
......@@ -6,7 +6,7 @@ import { SelectableValue } from '@grafana/data';
import CloudMonitoringDatasource from '../datasource';
import { AnnotationsHelp, LabelFilter, Metrics, Project } from './';
import { toOption } from '../functions';
import { AnnotationTarget, MetricDescriptor } from '../types';
import { AnnotationTarget, EditorMode, MetricDescriptor } from '../types';
const { Input } = LegacyForms;
......@@ -25,6 +25,7 @@ interface State extends AnnotationTarget {
}
const DefaultTarget: State = {
editorMode: EditorMode.Visual,
projectName: '',
projects: [],
metricType: '',
......@@ -42,7 +43,7 @@ const DefaultTarget: State = {
export class AnnotationQueryEditor extends React.Component<Props, State> {
state: State = DefaultTarget;
async UNSAFE_UNSAFE_componentWillMount() {
async UNSAFE_componentWillMount() {
// Unfortunately, migrations like this need to go UNSAFE_componentWillMount. As soon as there's
// migration hook for this module.ts, we can do the migrations there instead.
const { target, datasource } = this.props;
......
import React from 'react';
import { TextArea } from '@grafana/ui';
export interface Props {
onChange: (query: string) => void;
onRunQuery: () => void;
query: string;
}
export function MQLQueryEditor({ query, onChange, onRunQuery }: React.PropsWithChildren<Props>) {
const onKeyDown = (event: any) => {
if (event.key === 'Enter' && (event.shiftKey || event.ctrlKey)) {
event.preventDefault();
onRunQuery();
}
};
return (
<>
<TextArea
name="Query"
className="slate-query-field"
value={query}
rows={10}
placeholder="Enter a Cloud Monitoring MQL query (Run with Shift+Enter)"
onBlur={onRunQuery}
onChange={e => onChange(e.currentTarget.value)}
onKeyDown={onKeyDown}
/>
</>
);
}
import React, { useState, useEffect } from 'react';
import { Project, Aggregations, Metrics, LabelFilter, GroupBys, Alignments, AlignmentPeriods, AliasBy } from '.';
import { MetricQuery, MetricDescriptor } from '../types';
import { Project, VisualMetricQueryEditor, AliasBy } from '.';
import { MetricQuery, MetricDescriptor, EditorMode } from '../types';
import { getAlignmentPickerData } from '../functions';
import CloudMonitoringDatasource from '../datasource';
import { SelectableValue } from '@grafana/data';
import { MQLQueryEditor } from './MQLQueryEditor';
export interface Props {
refId: string;
......@@ -25,6 +26,7 @@ export const defaultState: State = {
};
export const defaultQuery: (dataSource: CloudMonitoringDatasource) => MetricQuery = dataSource => ({
editorMode: EditorMode.Visual,
projectName: dataSource.getDefaultProject(),
metricType: '',
metricKind: '',
......@@ -36,13 +38,15 @@ export const defaultQuery: (dataSource: CloudMonitoringDatasource) => MetricQuer
groupBys: [],
filters: [],
aliasBy: '',
query: '',
});
function Editor({
refId,
query,
datasource,
onChange,
onChange: onQueryChange,
onRunQuery,
usedAlignmentPeriod,
variableOptionGroup,
}: React.PropsWithChildren<Props>) {
......@@ -56,6 +60,11 @@ function Editor({
}
}, [query.projectName, query.groupBys, query.metricType]);
const onChange = (metricQuery: MetricQuery) => {
onQueryChange({ ...query, ...metricQuery });
onRunQuery();
};
const onMetricTypeChange = async ({ valueType, metricKind, type, unit }: MetricDescriptor) => {
const { perSeriesAligner, alignOptions } = getAlignmentPickerData(
{ valueType, metricKind, perSeriesAligner: state.perSeriesAligner },
......@@ -68,9 +77,6 @@ function Editor({
onChange({ ...query, perSeriesAligner, metricType: type, unit, valueType, metricKind });
};
const { labels } = state;
const { perSeriesAligner, alignOptions } = getAlignmentPickerData(query, datasource.templateSrv);
return (
<>
<Project
......@@ -81,58 +87,33 @@ function Editor({
onChange({ ...query, projectName });
}}
/>
<Metrics
templateSrv={datasource.templateSrv}
projectName={query.projectName}
metricType={query.metricType}
templateVariableOptions={variableOptionGroup.options}
datasource={datasource}
onChange={onMetricTypeChange}
>
{metric => (
<>
<LabelFilter
labels={labels}
filters={query.filters!}
onChange={filters => onChange({ ...query, filters })}
variableOptionGroup={variableOptionGroup}
/>
<GroupBys
groupBys={Object.keys(labels)}
values={query.groupBys!}
onChange={groupBys => onChange({ ...query, groupBys })}
{query.editorMode === EditorMode.Visual && (
<VisualMetricQueryEditor
labels={state.labels}
variableOptionGroup={variableOptionGroup}
/>
<Aggregations
metricDescriptor={metric}
templateVariableOptions={variableOptionGroup.options}
crossSeriesReducer={query.crossSeriesReducer}
groupBys={query.groupBys!}
onChange={crossSeriesReducer => onChange({ ...query, crossSeriesReducer })}
>
{displayAdvancedOptions =>
displayAdvancedOptions && (
<Alignments
alignOptions={alignOptions}
templateVariableOptions={variableOptionGroup.options}
perSeriesAligner={perSeriesAligner || ''}
onChange={perSeriesAligner => onChange({ ...query, perSeriesAligner })}
/>
)
}
</Aggregations>
<AlignmentPeriods
templateSrv={datasource.templateSrv}
templateVariableOptions={variableOptionGroup.options}
alignmentPeriod={query.alignmentPeriod || ''}
perSeriesAligner={query.perSeriesAligner || ''}
usedAlignmentPeriod={usedAlignmentPeriod}
onChange={alignmentPeriod => onChange({ ...query, alignmentPeriod })}
onMetricTypeChange={onMetricTypeChange}
onChange={onChange}
datasource={datasource}
query={query}
/>
<AliasBy value={query.aliasBy || ''} onChange={aliasBy => onChange({ ...query, aliasBy })} />
</>
)}
</Metrics>
{query.editorMode === EditorMode.MQL && (
<MQLQueryEditor
onChange={(q: string) => onQueryChange({ ...query, query: q })}
onRunQuery={onRunQuery}
query={query.query}
></MQLQueryEditor>
)}
<AliasBy
value={query.aliasBy}
onChange={aliasBy => {
onChange({ ...query, aliasBy });
}}
/>
</>
);
}
......
import React, { PureComponent } from 'react';
import appEvents from 'app/core/app_events';
import { CoreEvents } from 'app/types';
import { Help, MetricQueryEditor, QueryTypeSelector, SLOQueryEditor } from './';
import { CloudMonitoringQuery, MetricQuery, QueryType, SLOQuery } from '../types';
import { ExploreQueryFieldProps, SelectableValue } from '@grafana/data';
import { Segment } from '@grafana/ui';
import { Help, MetricQueryEditor, SLOQueryEditor } from './';
import { CloudMonitoringQuery, MetricQuery, QueryType, SLOQuery, queryTypes, EditorMode } from '../types';
import { defaultQuery } from './MetricQueryEditor';
import { defaultQuery as defaultSLOQuery } from './SLOQueryEditor';
import { formatCloudMonitoringError, toOption } from '../functions';
import CloudMonitoringDatasource from '../datasource';
import { ExploreQueryFieldProps } from '@grafana/data';
export type Props = ExploreQueryFieldProps<CloudMonitoringDatasource, CloudMonitoringQuery>;
......@@ -18,7 +19,7 @@ interface State {
export class QueryEditor extends PureComponent<Props, State> {
state: State = { lastQueryError: '' };
async UNSAFE_UNSAFE_componentWillMount() {
async UNSAFE_componentWillMount() {
const { datasource, query } = this.props;
// Unfortunately, migrations like this need to go UNSAFE_componentWillMount. As soon as there's
......@@ -76,21 +77,51 @@ export class QueryEditor extends PureComponent<Props, State> {
return (
<>
<QueryTypeSelector
value={queryType}
templateVariableOptions={variableOptionGroup.options}
onChange={(queryType: QueryType) => {
onChange({ ...query, sloQuery, queryType });
<div className="gf-form-inline">
<label className="gf-form-label query-keyword width-9">Query Type</label>
<Segment
value={[...queryTypes, ...variableOptionGroup.options].find(qt => qt.value === queryType)}
options={[
...queryTypes,
{
label: 'Template Variables',
options: variableOptionGroup.options,
},
]}
onChange={({ value }: SelectableValue<QueryType>) => {
onChange({ ...query, sloQuery, queryType: value! });
onRunQuery();
}}
></QueryTypeSelector>
/>
{query.queryType !== QueryType.SLO && (
<button
className="gf-form-label "
onClick={() =>
this.onQueryChange('metricQuery', {
...metricQuery,
editorMode: metricQuery.editorMode === EditorMode.MQL ? EditorMode.Visual : EditorMode.MQL,
})
}
>
<span className="query-keyword">{'<>'}</span>&nbsp;&nbsp;
{metricQuery.editorMode === EditorMode.MQL ? 'Switch to builder' : 'Edit MQL'}
</button>
)}
<div className="gf-form gf-form--grow">
<label className="gf-form-label gf-form-label--grow"></label>
</div>
</div>
{queryType === QueryType.METRICS && (
<MetricQueryEditor
refId={query.refId}
variableOptionGroup={variableOptionGroup}
usedAlignmentPeriod={usedAlignmentPeriod}
onChange={(query: MetricQuery) => this.onQueryChange('metricQuery', query)}
onChange={(metricQuery: MetricQuery) => {
this.props.onChange({ ...this.props.query, metricQuery });
}}
onRunQuery={onRunQuery}
datasource={datasource}
query={metricQuery}
......@@ -107,6 +138,7 @@ export class QueryEditor extends PureComponent<Props, State> {
query={sloQuery}
></SLOQueryEditor>
)}
<Help
rawQuery={decodeURIComponent(meta?.executedQueryString ?? '')}
lastQueryError={this.state.lastQueryError}
......
import React from 'react';
import { Aggregations, Metrics, LabelFilter, GroupBys, Alignments, AlignmentPeriods } from '.';
import { MetricQuery, MetricDescriptor } from '../types';
import { getAlignmentPickerData } from '../functions';
import CloudMonitoringDatasource from '../datasource';
import { SelectableValue } from '@grafana/data';
export interface Props {
usedAlignmentPeriod?: number;
variableOptionGroup: SelectableValue<string>;
onMetricTypeChange: (query: MetricDescriptor) => void;
onChange: (query: MetricQuery) => void;
query: MetricQuery;
datasource: CloudMonitoringDatasource;
labels: any;
}
function Editor({
query,
labels,
datasource,
onChange,
onMetricTypeChange,
usedAlignmentPeriod,
variableOptionGroup,
}: React.PropsWithChildren<Props>) {
const { perSeriesAligner, alignOptions } = getAlignmentPickerData(query, datasource.templateSrv);
return (
<Metrics
templateSrv={datasource.templateSrv}
projectName={query.projectName}
metricType={query.metricType}
templateVariableOptions={variableOptionGroup.options}
datasource={datasource}
onChange={onMetricTypeChange}
>
{metric => (
<>
<LabelFilter
labels={labels}
filters={query.filters!}
onChange={filters => onChange({ ...query, filters })}
variableOptionGroup={variableOptionGroup}
/>
<GroupBys
groupBys={Object.keys(labels)}
values={query.groupBys!}
onChange={groupBys => onChange({ ...query, groupBys })}
variableOptionGroup={variableOptionGroup}
/>
<Aggregations
metricDescriptor={metric}
templateVariableOptions={variableOptionGroup.options}
crossSeriesReducer={query.crossSeriesReducer}
groupBys={query.groupBys!}
onChange={crossSeriesReducer => onChange({ ...query, crossSeriesReducer })}
>
{displayAdvancedOptions =>
displayAdvancedOptions && (
<Alignments
alignOptions={alignOptions}
templateVariableOptions={variableOptionGroup.options}
perSeriesAligner={perSeriesAligner || ''}
onChange={perSeriesAligner => onChange({ ...query, perSeriesAligner })}
/>
)
}
</Aggregations>
<AlignmentPeriods
templateSrv={datasource.templateSrv}
templateVariableOptions={variableOptionGroup.options}
alignmentPeriod={query.alignmentPeriod || ''}
perSeriesAligner={query.perSeriesAligner || ''}
usedAlignmentPeriod={usedAlignmentPeriod}
onChange={alignmentPeriod => onChange({ ...query, alignmentPeriod })}
/>
</>
)}
</Metrics>
);
}
export const VisualMetricQueryEditor = React.memo(Editor);
......@@ -11,5 +11,7 @@ export { Aggregations } from './Aggregations';
export { SimpleSelect } from './SimpleSelect';
export { MetricQueryEditor } from './MetricQueryEditor';
export { SLOQueryEditor } from './SLOQueryEditor';
export { MQLQueryEditor } from './MQLQueryEditor';
export { QueryTypeSelector } from './QueryType';
export { QueryInlineField, QueryField } from './Fields';
export { VisualMetricQueryEditor } from './VisualMetricQueryEditor';
......@@ -10,7 +10,7 @@ import {
import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { CloudMonitoringOptions, CloudMonitoringQuery, Filter, MetricDescriptor, QueryType } from './types';
import { CloudMonitoringOptions, CloudMonitoringQuery, Filter, MetricDescriptor, QueryType, EditorMode } from './types';
import API from './api';
import { DataSourceWithBackend } from '@grafana/runtime';
import { CloudMonitoringVariableSupport } from './variables';
......@@ -114,6 +114,7 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend<
filters: this.interpolateFilters(metricQuery.filters || [], scopedVars),
groupBys: this.interpolateGroupBys(metricQuery.groupBys || [], scopedVars),
view: metricQuery.view || 'FULL',
editorMode: metricQuery.editorMode,
},
sloQuery: sloQuery && this.interpolateProps(sloQuery, scopedVars),
};
......@@ -324,6 +325,10 @@ export default class CloudMonitoringDatasource extends DataSourceWithBackend<
return !!selectorName && !!serviceId && !!sloId && !!projectName;
}
if (query.queryType && query.queryType === QueryType.METRICS && query.metricQuery.editorMode === EditorMode.MQL) {
return !!query.metricQuery.projectName && !!query.metricQuery.query;
}
const { metricType } = query.metricQuery;
return !!metricType;
......
......@@ -35,7 +35,7 @@
{
"path": "cloudmonitoring",
"method": "GET",
"url": "https://content-monitoring.googleapis.com",
"url": "https://monitoring.googleapis.com",
"jwtTokenAuth": {
"scopes": ["https://www.googleapis.com/auth/monitoring.read"],
"params": {
......
......@@ -58,12 +58,18 @@ export enum QueryType {
SLO = 'slo',
}
export enum EditorMode {
Visual = 'visual',
MQL = 'mql',
}
export const queryTypes = [
{ label: 'Metrics', value: QueryType.METRICS },
{ label: 'Service Level Objectives (SLO)', value: QueryType.SLO },
];
export interface MetricQuery {
editorMode: EditorMode;
projectName: string;
unit?: string;
metricType: string;
......@@ -76,6 +82,7 @@ export interface MetricQuery {
metricKind?: string;
valueType?: string;
view?: string;
query: string;
}
export interface SLOQuery {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment