Skip to content

N
nexpie-grafana-theme
Commit 376a9d35 by Kyle Brandt Committed by GitHub
Options

Azure Monitor: Change response to be dataframes (#25123) 

note: This is just Azure Monitor within the Azure Monitor datasource (not insights, insights analytics, or log analytics yet).

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>
parent 07582a8e
Showing with 126 additions and 194 deletions
packages/grafana-runtime/src/utils/DataSourceWithBackend.ts
......@@ -5,8 +5,9 @@ import {
DataSourceInstanceSettings,
DataQuery,
DataSourceJsonData,
ScopedVars,
} from '@grafana/data';
import { Observable, from } from 'rxjs';
import { Observable, from, of } from 'rxjs';
import { config } from '..';
import { getBackendSrv } from '../services';
import { toDataQueryResponse } from './queryResponse';
......@@ -53,9 +54,13 @@ export class DataSourceWithBackend<
/**
* Ideally final -- any other implementation may not work as expected
*/
query(request: DataQueryRequest): Observable<DataQueryResponse> {
const { targets, intervalMs, maxDataPoints, range, requestId } = request;
query(request: DataQueryRequest<TQuery>): Observable<DataQueryResponse> {
const { intervalMs, maxDataPoints, range, requestId } = request;
const orgId = config.bootData.user.orgId;
let targets = request.targets;
if (this.filterQuery) {
targets = targets.filter(q => this.filterQuery!(q));
}
const queries = targets.map(q => {
if (q.datasource === ExpressionDatasourceID) {
return {
......@@ -70,7 +75,7 @@ export class DataSourceWithBackend<
throw new Error('Unknown Datasource: ' + q.datasource);
}
return {
...this.applyTemplateVariables(q),
...this.applyTemplateVariables(q, request.scopedVars),
datasourceId: ds.id,
intervalMs,
maxDataPoints,
......@@ -78,6 +83,11 @@ export class DataSourceWithBackend<
};
});
// Return early if no queries exist
if (!queries.length) {
return of({ data: [] });
}
const body: any = {
queries,
};
......@@ -106,11 +116,18 @@ export class DataSourceWithBackend<
}
/**
* Override to skip executing a query
*
* @virtual
*/
filterQuery?(query: TQuery): boolean;
/**
* Override to apply template variables
*
* @virtual
*/
applyTemplateVariables(query: DataQuery) {
applyTemplateVariables(query: TQuery, scopedVars: ScopedVars): Record<string, any> {
return query;
}
......
pkg/tsdb/azuremonitor/azuremonitor-datasource.go
......@@ -12,6 +12,7 @@ import (
"strings"
"time"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/api/pluginproxy"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
......@@ -20,7 +21,6 @@ import (
opentracing "github.com/opentracing/opentracing-go"
"golang.org/x/net/context/ctxhttp"
"github.com/grafana/grafana/pkg/components/null"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/tsdb"
)
......@@ -260,25 +260,32 @@ func (e *AzureMonitorDatasource) unmarshalResponse(res *http.Response) (AzureMon
return data, nil
}
func (e *AzureMonitorDatasource) parseResponse(queryRes *tsdb.QueryResult, data AzureMonitorResponse, query *AzureMonitorQuery) error {
if len(data.Value) == 0 {
func (e *AzureMonitorDatasource) parseResponse(queryRes *tsdb.QueryResult, amr AzureMonitorResponse, query *AzureMonitorQuery) error {
if len(amr.Value) == 0 {
return nil
}
for _, series := range data.Value[0].Timeseries {
points := []tsdb.TimePoint{}
for _, series := range amr.Value[0].Timeseries {
metadataName := ""
metadataValue := ""
if len(series.Metadatavalues) > 0 {
metadataName = series.Metadatavalues[0].Name.LocalizedValue
metadataValue = series.Metadatavalues[0].Value
}
metricName := formatAzureMonitorLegendKey(query.Alias, query.UrlComponents["resourceName"], data.Value[0].Name.LocalizedValue, metadataName, metadataValue, data.Namespace, data.Value[0].ID)
metricName := formatAzureMonitorLegendKey(query.Alias, query.UrlComponents["resourceName"], amr.Value[0].Name.LocalizedValue, metadataName, metadataValue, amr.Namespace, amr.Value[0].ID)
frame := data.NewFrameOfFieldTypes("", len(series.Data), data.FieldTypeTime, data.FieldTypeFloat64)
frame.RefID = query.RefID
frame.Fields[1].Name = metricName
frame.Fields[1].SetConfig(&data.FieldConfig{
Unit: amr.Value[0].Unit,
})
for _, point := range series.Data {
requestedAgg := query.Params.Get("aggregation")
for i, point := range series.Data {
var value float64
switch query.Params.Get("aggregation") {
switch requestedAgg {
case "Average":
value = point.Average
case "Total":
......@@ -292,15 +299,17 @@ func (e *AzureMonitorDatasource) parseResponse(queryRes *tsdb.QueryResult, data
default:
value = point.Count
}
points = append(points, tsdb.NewTimePoint(null.FloatFrom(value), float64((point.TimeStamp).Unix())*1000))
frame.SetRow(i, point.TimeStamp, value)
}
queryRes.Series = append(queryRes.Series, &tsdb.TimeSeries{
Name: metricName,
Points: points,
})
encodedFrame, err := frame.MarshalArrow()
if err != nil {
queryRes.Error = fmt.Errorf("failed to encode dataframe response into arrow: %w", err)
}
queryRes.Dataframes = append(queryRes.Dataframes, encodedFrame)
}
queryRes.Meta.Set("unit", data.Value[0].Unit)
return nil
}
......
public/app/features/dashboard/state/runRequest.ts
......@@ -21,6 +21,7 @@ import {
import { toDataQueryError } from '@grafana/runtime';
import { emitDataRequestEvent } from './analyticsProcessor';
import { ExpressionDatasourceID, expressionDatasource } from 'app/features/expressions/ExpressionDatasource';
import { ExpressionQuery } from 'app/features/expressions/types';
type MapOfResponsePackets = { [str: string]: DataQueryResponse };
......@@ -145,7 +146,7 @@ export function callQueryMethod(datasource: DataSourceApi, request: DataQueryReq
// If any query has an expression, use the expression endpoint
for (const target of request.targets) {
if (target.datasource === ExpressionDatasourceID) {
return expressionDatasource.query(request);
return expressionDatasource.query(request as DataQueryRequest<ExpressionQuery>);
}
}
......
public/app/plugins/datasource/grafana-azure-monitor-datasource/azure_monitor/azure_monitor_datasource.test.ts
import AzureMonitorDatasource from '../datasource';
import { TemplateSrv } from 'app/features/templating/template_srv';
import { toUtc, DataFrame, getFrameDisplayName } from '@grafana/data';
import { DataSourceInstanceSettings } from '@grafana/data';
import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__
import { AzureDataSourceJsonData } from '../types';
const templateSrv = new TemplateSrv();
jest.mock('@grafana/runtime', () => ({
...jest.requireActual('@grafana/runtime'),
getBackendSrv: () => backendSrv,
getTemplateSrv: () => templateSrv,
}));
interface TestContext {
instanceSettings: DataSourceInstanceSettings<AzureDataSourceJsonData>;
ds: AzureMonitorDatasource;
}
describe('AzureMonitorDatasource', () => {
const ctx: any = {
templateSrv: new TemplateSrv(),
};
const ctx: TestContext = {} as TestContext;
const datasourceRequestMock = jest.spyOn(backendSrv, 'datasourceRequest');
beforeEach(() => {
jest.clearAllMocks();
ctx.instanceSettings = {
ctx.instanceSettings = ({
name: 'test',
url: 'http://azuremonitor.com',
jsonData: { subscriptionId: '9935389e-9122-4ef9-95f9-1513dd24753f' },
cloudName: 'azuremonitor',
};
ctx.ds = new AzureMonitorDatasource(ctx.instanceSettings, ctx.templateSrv);
} as unknown) as DataSourceInstanceSettings<AzureDataSourceJsonData>;
ctx.ds = new AzureMonitorDatasource(ctx.instanceSettings, templateSrv);
});
describe('When performing testDatasource', () => {
......@@ -78,74 +85,6 @@ describe('AzureMonitorDatasource', () => {
});
});
describe('When performing query', () => {
const options = {
range: {
from: toUtc('2017-08-22T20:00:00Z'),
to: toUtc('2017-08-22T23:59:00Z'),
},
targets: [
{
apiVersion: '2018-01-01',
refId: 'A',
queryType: 'Azure Monitor',
azureMonitor: {
resourceGroup: 'testRG',
resourceName: 'testRN',
metricDefinition: 'Microsoft.Compute/virtualMachines',
metricNamespace: 'default',
metricName: 'Percentage CPU',
timeGrain: 'PT1H',
alias: '{{metric}}',
},
},
],
};
const response: any = {
results: {
A: {
refId: 'A',
meta: {
rawQuery:
'aggregation=Average&api-version=2018-01-01&interval=PT1M' +
'&metricnames=Percentage+CPU&timespan=2019-05-19T15%3A11%3A37Z%2F2019-05-19T21%3A11%3A37Z',
unit: 'Percent',
},
series: [
{
name: 'Percentage CPU',
points: [
[2.2075, 1558278660000],
[2.29, 1558278720000],
],
},
],
tables: null,
},
},
};
beforeEach(() => {
datasourceRequestMock.mockImplementation((options: { url: string }) => {
expect(options.url).toContain('/api/tsdb/query');
return Promise.resolve({ data: response, status: 200 });
});
});
it('should return a list of datapoints', () => {
return ctx.ds.query(options).then((results: any) => {
expect(results.data.length).toBe(1);
const data = results.data[0] as DataFrame;
expect(getFrameDisplayName(data)).toEqual('Percentage CPU');
expect(data.fields[0].values.get(0)).toEqual(1558278660000);
expect(data.fields[1].values.get(0)).toEqual(2.2075);
expect(data.fields[0].values.get(1)).toEqual(1558278720000);
expect(data.fields[1].values.get(1)).toEqual(2.29);
});
});
});
describe('When performing metricFindQuery', () => {
describe('with a subscriptions query', () => {
const response = {
......@@ -626,7 +565,7 @@ describe('AzureMonitorDatasource', () => {
});
it('should return list of Resource Groups', () => {
return ctx.ds.getResourceGroups().then((results: Array<{ text: string; value: string }>) => {
return ctx.ds.getResourceGroups('subscriptionId').then((results: Array<{ text: string; value: string }>) => {
expect(results.length).toEqual(2);
expect(results[0].text).toEqual('grp1');
expect(results[0].value).toEqual('grp1');
......
public/app/plugins/datasource/grafana-azure-monitor-datasource/azure_monitor/azure_monitor_datasource.ts
import _ from 'lodash';
import { filter, startsWith } from 'lodash';
import UrlBuilder from './url_builder';
import ResponseParser from './response_parser';
import SupportedNamespaces from './supported_namespaces';
......@@ -9,31 +9,25 @@ import {
AzureMonitorMetricDefinitionsResponse,
AzureMonitorResourceGroupsResponse,
} from '../types';
import { DataQueryRequest, DataQueryResponseData, DataSourceInstanceSettings } from '@grafana/data';
import { DataSourceInstanceSettings, ScopedVars } from '@grafana/data';
import { getBackendSrv, DataSourceWithBackend, getTemplateSrv } from '@grafana/runtime';
import { TimeSeries, toDataFrame } from '@grafana/data';
import { TemplateSrv } from 'app/features/templating/template_srv';
import { getBackendSrv } from '@grafana/runtime';
const defaultDropdownValue = 'select';
export default class AzureMonitorDatasource {
export default class AzureMonitorDatasource extends DataSourceWithBackend<AzureMonitorQuery, AzureDataSourceJsonData> {
apiVersion = '2018-01-01';
apiPreviewVersion = '2017-12-01-preview';
id: number;
subscriptionId: string;
baseUrl: string;
resourceGroup: string;
resourceName: string;
url: string;
defaultDropdownValue = 'select';
cloudName: string;
supportedMetricNamespaces: string[] = [];
/** @ngInject */
constructor(
private instanceSettings: DataSourceInstanceSettings<AzureDataSourceJsonData>,
private templateSrv: TemplateSrv
) {
this.id = instanceSettings.id;
constructor(private instanceSettings: DataSourceInstanceSettings<AzureDataSourceJsonData>) {
super(instanceSettings);
this.subscriptionId = instanceSettings.jsonData.subscriptionId;
this.cloudName = instanceSettings.jsonData.cloudName || 'azuremonitor';
this.baseUrl = `/${this.cloudName}/subscriptions`;
......@@ -46,20 +40,21 @@ export default class AzureMonitorDatasource {
return !!this.subscriptionId && this.subscriptionId.length > 0;
}
async query(options: DataQueryRequest<AzureMonitorQuery>): Promise<DataQueryResponseData[]> {
const queries = _.filter(options.targets, item => {
filterQuery(item: AzureMonitorQuery): boolean {
return (
item.hide !== true &&
item.azureMonitor.resourceGroup &&
item.azureMonitor.resourceGroup !== this.defaultDropdownValue &&
item.azureMonitor.resourceGroup !== defaultDropdownValue &&
item.azureMonitor.resourceName &&
item.azureMonitor.resourceName !== this.defaultDropdownValue &&
item.azureMonitor.resourceName !== defaultDropdownValue &&
item.azureMonitor.metricDefinition &&
item.azureMonitor.metricDefinition !== this.defaultDropdownValue &&
item.azureMonitor.metricDefinition !== defaultDropdownValue &&
item.azureMonitor.metricName &&
item.azureMonitor.metricName !== this.defaultDropdownValue
item.azureMonitor.metricName !== defaultDropdownValue
);
}).map(target => {
}
applyTemplateVariables(target: AzureMonitorQuery, scopedVars: ScopedVars): Record<string, any> {
const item = target.azureMonitor;
// fix for timeGrainUnit which is a deprecated/removed field name
......@@ -67,80 +62,42 @@ export default class AzureMonitorDatasource {
item.timeGrain = TimegrainConverter.createISO8601Duration(item.timeGrain, item.timeGrainUnit);
}
const subscriptionId = this.templateSrv.replace(target.subscription || this.subscriptionId, options.scopedVars);
const resourceGroup = this.templateSrv.replace(item.resourceGroup, options.scopedVars);
const resourceName = this.templateSrv.replace(item.resourceName, options.scopedVars);
const metricNamespace = this.templateSrv.replace(item.metricNamespace, options.scopedVars);
const metricDefinition = this.templateSrv.replace(item.metricDefinition, options.scopedVars);
const timeGrain = this.templateSrv.replace((item.timeGrain || '').toString(), options.scopedVars);
const aggregation = this.templateSrv.replace(item.aggregation, options.scopedVars);
const top = this.templateSrv.replace(item.top || '', options.scopedVars);
const templateSrv = getTemplateSrv();
const subscriptionId = templateSrv.replace(target.subscription || this.subscriptionId, scopedVars);
const resourceGroup = templateSrv.replace(item.resourceGroup, scopedVars);
const resourceName = templateSrv.replace(item.resourceName, scopedVars);
const metricNamespace = templateSrv.replace(item.metricNamespace, scopedVars);
const metricDefinition = templateSrv.replace(item.metricDefinition, scopedVars);
const timeGrain = templateSrv.replace((item.timeGrain || '').toString(), scopedVars);
const aggregation = templateSrv.replace(item.aggregation, scopedVars);
const top = templateSrv.replace(item.top || '', scopedVars);
return {
refId: target.refId,
intervalMs: options.intervalMs,
datasourceId: this.id,
subscription: subscriptionId,
queryType: 'Azure Monitor',
type: 'timeSeriesQuery',
raw: false,
azureMonitor: {
resourceGroup: resourceGroup,
resourceName: resourceName,
metricDefinition: metricDefinition,
timeGrain: timeGrain,
resourceGroup,
resourceName,
metricDefinition,
timeGrain,
allowedTimeGrainsMs: item.allowedTimeGrainsMs,
metricName: this.templateSrv.replace(item.metricName, options.scopedVars),
metricName: templateSrv.replace(item.metricName, scopedVars),
metricNamespace:
metricNamespace && metricNamespace !== this.defaultDropdownValue ? metricNamespace : metricDefinition,
metricNamespace && metricNamespace !== defaultDropdownValue ? metricNamespace : metricDefinition,
aggregation: aggregation,
dimension: this.templateSrv.replace(item.dimension, options.scopedVars),
dimension: templateSrv.replace(item.dimension, scopedVars),
top: top || '10',
dimensionFilter: this.templateSrv.replace(item.dimensionFilter, options.scopedVars),
dimensionFilter: templateSrv.replace(item.dimensionFilter, scopedVars),
alias: item.alias,
format: target.format,
},
};
});
if (!queries || queries.length === 0) {
return Promise.resolve([]);
}
const { data } = await getBackendSrv().datasourceRequest({
url: '/api/tsdb/query',
method: 'POST',
data: {
from: options.range.from.valueOf().toString(),
to: options.range.to.valueOf().toString(),
queries,
},
});
const result: DataQueryResponseData[] = [];
if (data.results) {
Object['values'](data.results).forEach((queryRes: any) => {
if (!queryRes.series) {
return;
}
queryRes.series.forEach((series: any) => {
const timeSerie: TimeSeries = {
target: series.name,
datapoints: series.points,
refId: queryRes.refId,
meta: queryRes.meta,
};
result.push(toDataFrame(timeSerie));
});
});
return result;
}
return Promise.resolve([]);
}
annotationQuery(options: any) {}
metricFindQuery(query: string) {
const subscriptionsQuery = query.match(/^Subscriptions\(\)/i);
if (subscriptionsQuery) {
......@@ -234,7 +191,7 @@ export default class AzureMonitorDatasource {
}
toVariable(metric: string) {
return this.templateSrv.replace((metric || '').trim());
return getTemplateSrv().replace((metric || '').trim());
}
getSubscriptions(route?: string) {
......@@ -258,7 +215,7 @@ export default class AzureMonitorDatasource {
return ResponseParser.parseResponseValues(result, 'type', 'type');
})
.then((result: any) => {
return _.filter(result, t => {
return filter(result, t => {
for (let i = 0; i < this.supportedMetricNamespaces.length; i++) {
if (t.value.toLowerCase() === this.supportedMetricNamespaces[i].toLowerCase()) {
return true;
......@@ -304,7 +261,7 @@ export default class AzureMonitorDatasource {
const url = `${this.baseUrl}/${subscriptionId}/resourceGroups/${resourceGroup}/resources?api-version=${this.apiVersion}`;
return this.doRequest(url).then((result: any) => {
if (!_.startsWith(metricDefinition, 'Microsoft.Storage/storageAccounts/')) {
if (!startsWith(metricDefinition, 'Microsoft.Storage/storageAccounts/')) {
return ResponseParser.parseResourceNames(result, metricDefinition);
}
......@@ -378,19 +335,19 @@ export default class AzureMonitorDatasource {
});
}
testDatasource() {
testDatasource(): Promise<any> {
if (!this.isValidConfigField(this.instanceSettings.jsonData.tenantId)) {
return {
return Promise.resolve({
status: 'error',
message: 'The Tenant Id field is required.',
};
});
}
if (!this.isValidConfigField(this.instanceSettings.jsonData.clientId)) {
return {
return Promise.resolve({
status: 'error',
message: 'The Client Id field is required.',
};
});
}
const url = `/${this.cloudName}/subscriptions?api-version=2019-03-01`;
......
public/app/plugins/datasource/grafana-azure-monitor-datasource/datasource.ts
......@@ -3,8 +3,15 @@ import AzureMonitorDatasource from './azure_monitor/azure_monitor_datasource';
import AppInsightsDatasource from './app_insights/app_insights_datasource';
import AzureLogAnalyticsDatasource from './azure_log_analytics/azure_log_analytics_datasource';
import { AzureMonitorQuery, AzureDataSourceJsonData } from './types';
import { DataSourceApi, DataQueryRequest, DataSourceInstanceSettings } from '@grafana/data';
import {
DataSourceApi,
DataQueryRequest,
DataSourceInstanceSettings,
DataQueryResponse,
DataQueryResponseData,
} from '@grafana/data';
import { TemplateSrv } from 'app/features/templating/template_srv';
import { Observable } from 'rxjs';
export default class Datasource extends DataSourceApi<AzureMonitorQuery, AzureDataSourceJsonData> {
azureMonitorDatasource: AzureMonitorDatasource;
......@@ -14,13 +21,12 @@ export default class Datasource extends DataSourceApi<AzureMonitorQuery, AzureDa
/** @ngInject */
constructor(instanceSettings: DataSourceInstanceSettings<AzureDataSourceJsonData>, private templateSrv: TemplateSrv) {
super(instanceSettings);
this.azureMonitorDatasource = new AzureMonitorDatasource(instanceSettings, this.templateSrv);
this.azureMonitorDatasource = new AzureMonitorDatasource(instanceSettings);
this.appInsightsDatasource = new AppInsightsDatasource(instanceSettings, this.templateSrv);
this.azureLogAnalyticsDatasource = new AzureLogAnalyticsDatasource(instanceSettings, this.templateSrv);
}
async query(options: DataQueryRequest<AzureMonitorQuery>) {
query(options: DataQueryRequest<AzureMonitorQuery>): Promise<DataQueryResponse> | Observable<DataQueryResponseData> {
const promises: any[] = [];
const azureMonitorOptions = _.cloneDeep(options);
const appInsightsOptions = _.cloneDeep(options);
......@@ -30,13 +36,6 @@ export default class Datasource extends DataSourceApi<AzureMonitorQuery, AzureDa
appInsightsOptions.targets = _.filter(appInsightsOptions.targets, ['queryType', 'Application Insights']);
azureLogAnalyticsOptions.targets = _.filter(azureLogAnalyticsOptions.targets, ['queryType', 'Azure Log Analytics']);
if (azureMonitorOptions.targets.length > 0) {
const amPromise = this.azureMonitorDatasource.query(azureMonitorOptions);
if (amPromise) {
promises.push(amPromise);
}
}
if (appInsightsOptions.targets.length > 0) {
const aiPromise = this.appInsightsDatasource.query(appInsightsOptions);
if (aiPromise) {
......@@ -51,6 +50,16 @@ export default class Datasource extends DataSourceApi<AzureMonitorQuery, AzureDa
}
}
if (azureMonitorOptions.targets.length > 0) {
const obs = this.azureMonitorDatasource.query(azureMonitorOptions);
if (!promises.length) {
return obs; // return the observable directly
}
// NOTE: this only includes the data!
// When all three query types are ready to be observale, they should all use observable
promises.push(obs.toPromise().then(r => r.data));
}
if (promises.length === 0) {
return Promise.resolve({ data: [] });
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment