Annotations: Add annotations support to Loki (#18949)

docs/sources/plugins/developing/datasources.md

@@ -146,7 +146,8 @@ Request object passed to datasource.annotationQuery function:
     "datasource": "generic datasource",
     "datasource": "generic datasource",
     "enable": true,
     "enable": true,
     "name": "annotation name"
     "name": "annotation name"
-  }
+  },
+  "dashboard": DashboardModel
 }
 }
 ```
 ```
 
 

packages/grafana-data/src/utils/dataFrameView.ts

@@ -68,4 +68,10 @@ export class DataFrameView<T = any> implements Vector<T> {
   toJSON(): T[] {
   toJSON(): T[] {
     return this.toArray();
     return this.toArray();
   }
   }
+
+  forEachRow(iterator: (row: T) => void) {
+    for (let i = 0; i < this.data.length; i++) {
+      iterator(this.get(i));
+    }
+  }
 }
 }

packages/grafana-ui/src/types/datasource.ts

@@ -8,6 +8,7 @@ import {
   LogRowModel,
   LogRowModel,
   LoadingState,
   LoadingState,
   DataFrameDTO,
   DataFrameDTO,
+  AnnotationEvent,
 } from '@grafana/data';
 } from '@grafana/data';
 import { PluginMeta, GrafanaPlugin } from './plugin';
 import { PluginMeta, GrafanaPlugin } from './plugin';
 import { PanelData } from './panel';
 import { PanelData } from './panel';
@@ -276,6 +277,12 @@ export abstract class DataSourceApi<
    * Used in explore
    * Used in explore
    */
    */
   languageProvider?: any;
   languageProvider?: any;
+
+  /**
+   * Can be optionally implemented to allow datasource to be a source of annotations for dashboard. To be visible
+   * in the annotation editor `annotations` capability also needs to be enabled in plugin.json.
+   */
+  annotationQuery?(options: AnnotationQueryRequest<TQuery>): Promise<AnnotationEvent[]>;
 }
 }
 
 
 export interface QueryEditorProps<
 export interface QueryEditorProps<
@@ -542,3 +549,18 @@ export interface DataSourceSelectItem {
   meta: DataSourcePluginMeta;
   meta: DataSourcePluginMeta;
   sort: string;
   sort: string;
 }
 }
+
+/**
+ * Options passed to the datasource.annotationQuery method. See docs/plugins/developing/datasource.md
+ */
+export interface AnnotationQueryRequest<MoreOptions = {}> {
+  range: TimeRange;
+  rangeRaw: RawTimeRange;
+  // Should be DataModel but cannot import that here from the main app. Needs to be moved to package first.
+  dashboard: any;
+  annotation: {
+    datasource: string;
+    enable: boolean;
+    name: string;
+  } & MoreOptions;
+}

public/app/core/angular_wrappers.ts

@@ -12,6 +12,7 @@ import { FunctionEditor } from 'app/plugins/datasource/graphite/FunctionEditor';
 import { SearchField } from './components/search/SearchField';
 import { SearchField } from './components/search/SearchField';
 import { GraphContextMenu } from 'app/plugins/panel/graph/GraphContextMenu';
 import { GraphContextMenu } from 'app/plugins/panel/graph/GraphContextMenu';
 import ReactProfileWrapper from 'app/features/profile/ReactProfileWrapper';
 import ReactProfileWrapper from 'app/features/profile/ReactProfileWrapper';
+import { LokiAnnotationsQueryEditor } from '../plugins/datasource/loki/components/AnnotationsQueryEditor';
 
 
 export function registerAngularDirectives() {
 export function registerAngularDirectives() {
   react2AngularDirective('sidemenu', SideMenu, []);
   react2AngularDirective('sidemenu', SideMenu, []);
@@ -102,4 +103,10 @@ export function registerAngularDirectives() {
   ]);
   ]);
 
 
   react2AngularDirective('reactProfileWrapper', ReactProfileWrapper, []);
   react2AngularDirective('reactProfileWrapper', ReactProfileWrapper, []);
+
+  react2AngularDirective('lokiAnnotationsQueryEditor', LokiAnnotationsQueryEditor, [
+    'expr',
+    'onChange',
+    ['datasource', { watchDepth: 'reference' }],
+  ]);
 }
 }

public/app/features/annotations/annotations_srv.ts

@@ -15,6 +15,7 @@ import { AnnotationEvent } from '@grafana/data';
 import DatasourceSrv from '../plugins/datasource_srv';
 import DatasourceSrv from '../plugins/datasource_srv';
 import { BackendSrv } from 'app/core/services/backend_srv';
 import { BackendSrv } from 'app/core/services/backend_srv';
 import { TimeSrv } from '../dashboard/services/TimeSrv';
 import { TimeSrv } from '../dashboard/services/TimeSrv';
+import { DataSourceApi } from '@grafana/ui';
 
 
 export class AnnotationsSrv {
 export class AnnotationsSrv {
   globalAnnotationsPromise: any;
   globalAnnotationsPromise: any;
@@ -126,7 +127,7 @@ export class AnnotationsSrv {
       dsPromises.push(datasourcePromise);
       dsPromises.push(datasourcePromise);
       promises.push(
       promises.push(
         datasourcePromise
         datasourcePromise
-          .then((datasource: any) => {
+          .then((datasource: DataSourceApi) => {
             // issue query against data source
             // issue query against data source
             return datasource.annotationQuery({
             return datasource.annotationQuery({
               range: range,
               range: range,

public/app/plugins/datasource/loki/LokiAnnotationsQueryCtrl.tsx

+/**
+ * Just a simple wrapper for a react component that is actually implementing the query editor.
+ */
+export class LokiAnnotationsQueryCtrl {
+  static templateUrl = 'partials/annotations.editor.html';
+  annotation: any;
+
+  /** @ngInject */
+  constructor() {
+    this.annotation.target = this.annotation.target || {};
+    this.onQueryChange = this.onQueryChange.bind(this);
+  }
+
+  onQueryChange(expr: string) {
+    this.annotation.expr = expr;
+  }
+}

public/app/plugins/datasource/loki/components/AnnotationsQueryEditor.tsx

+// Libraries
+import React, { memo } from 'react';
+
+// Types
+import { DataSourceApi, DataSourceJsonData, DataSourceStatus } from '@grafana/ui';
+import { LokiQuery } from '../types';
+import { useLokiSyntax } from './useLokiSyntax';
+import { LokiQueryFieldForm } from './LokiQueryFieldForm';
+
+interface Props {
+  expr: string;
+  datasource: DataSourceApi<LokiQuery, DataSourceJsonData>;
+  onChange: (expr: string) => void;
+}
+
+export const LokiAnnotationsQueryEditor = memo(function LokiAnnotationQueryEditor(props: Props) {
+  const { expr, datasource, onChange } = props;
+
+  // Timerange to get existing labels from. Hard coding like this seems to be good enough right now.
+  const absolute = {
+    from: Date.now() - 10000,
+    to: Date.now(),
+  };
+
+  const { isSyntaxReady, setActiveOption, refreshLabels, ...syntaxProps } = useLokiSyntax(
+    datasource.languageProvider,
+    DataSourceStatus.Connected,
+    absolute
+  );
+
+  const query: LokiQuery = {
+    refId: '',
+    expr,
+  };
+
+  return (
+    <div className="gf-form-group">
+      <LokiQueryFieldForm
+        datasource={datasource}
+        datasourceStatus={DataSourceStatus.Connected}
+        query={query}
+        onChange={(query: LokiQuery) => onChange(query.expr)}
+        onRunQuery={() => {}}
+        history={[]}
+        panelData={null}
+        onLoadOptions={setActiveOption}
+        onLabelsRefresh={refreshLabels}
+        syntaxLoaded={isSyntaxReady}
+        absoluteRange={absolute}
+        {...syntaxProps}
+      />
+    </div>
+  );
+});

public/app/plugins/datasource/loki/datasource.test.ts

 import LokiDatasource from './datasource';
 import LokiDatasource from './datasource';
 import { LokiQuery } from './types';
 import { LokiQuery } from './types';
 import { getQueryOptions } from 'test/helpers/getQueryOptions';
 import { getQueryOptions } from 'test/helpers/getQueryOptions';
-import { DataSourceApi } from '@grafana/ui';
-import { DataFrame } from '@grafana/data';
+import { AnnotationQueryRequest, DataSourceApi } from '@grafana/ui';
+import { DataFrame, dateTime } from '@grafana/data';
 import { BackendSrv } from 'app/core/services/backend_srv';
 import { BackendSrv } from 'app/core/services/backend_srv';
 import { TemplateSrv } from 'app/features/templating/template_srv';
 import { TemplateSrv } from 'app/features/templating/template_srv';
 
 
@@ -22,15 +22,15 @@ describe('LokiDatasource', () => {
     },
     },
   };
   };
 
 
-  describe('when querying', () => {
-    const backendSrvMock = { datasourceRequest: jest.fn() };
-    const backendSrv = (backendSrvMock as unknown) as BackendSrv;
+  const backendSrvMock = { datasourceRequest: jest.fn() };
+  const backendSrv = (backendSrvMock as unknown) as BackendSrv;
 
 
-    const templateSrvMock = ({
-      getAdhocFilters: (): any[] => [],
-      replace: (a: string) => a,
-    } as unknown) as TemplateSrv;
+  const templateSrvMock = ({
+    getAdhocFilters: (): any[] => [],
+    replace: (a: string) => a,
+  } as unknown) as TemplateSrv;
 
 
+  describe('when querying', () => {
     const testLimit = makeLimitTest(instanceSettings, backendSrvMock, backendSrv, templateSrvMock, testResp);
     const testLimit = makeLimitTest(instanceSettings, backendSrvMock, backendSrv, templateSrvMock, testResp);
 
 
     test('should use default max lines when no limit given', () => {
     test('should use default max lines when no limit given', () => {
@@ -171,6 +171,37 @@ describe('LokiDatasource', () => {
       });
       });
     });
     });
   });
   });
+
+  describe('annotationQuery', () => {
+    it('should transform the loki data to annototion response', async () => {
+      const ds = new LokiDatasource(instanceSettings, backendSrv, templateSrvMock);
+      backendSrvMock.datasourceRequest = jest.fn(() =>
+        Promise.resolve({
+          data: {
+            streams: [
+              {
+                entries: [{ ts: '2019-02-01T10:27:37.498180581Z', line: 'hello' }],
+                labels: '{label="value"}',
+              },
+              {
+                entries: [{ ts: '2019-02-01T12:27:37.498180581Z', line: 'hello 2' }],
+                labels: '{label2="value2"}',
+              },
+            ],
+          },
+        })
+      );
+      const query = makeAnnotationQueryRequest();
+
+      const res = await ds.annotationQuery(query);
+      expect(res.length).toBe(2);
+      expect(res[0].text).toBe('hello');
+      expect(res[0].tags).toEqual(['value']);
+
+      expect(res[1].text).toBe('hello 2');
+      expect(res[1].tags).toEqual(['value2']);
+    });
+  });
 });
 });
 
 
 type LimitTestArgs = {
 type LimitTestArgs = {
@@ -208,3 +239,27 @@ function makeLimitTest(
     expect(backendSrvMock.datasourceRequest.mock.calls[0][0].url).toContain(`limit=${expectedLimit}`);
     expect(backendSrvMock.datasourceRequest.mock.calls[0][0].url).toContain(`limit=${expectedLimit}`);
   };
   };
 }
 }
+
+function makeAnnotationQueryRequest(): AnnotationQueryRequest<LokiQuery> {
+  const timeRange = {
+    from: dateTime(),
+    to: dateTime(),
+  };
+  return {
+    annotation: {
+      expr: '{test=test}',
+      refId: '',
+      datasource: 'loki',
+      enable: true,
+      name: 'test-annotation',
+    },
+    dashboard: {
+      id: 1,
+    } as any,
+    range: {
+      ...timeRange,
+      raw: timeRange,
+    },
+    rangeRaw: timeRange,
+  };
+}

public/app/plugins/datasource/loki/datasource.ts

 // Libraries
 // Libraries
 import _ from 'lodash';
 import _ from 'lodash';
 // Services & Utils
 // Services & Utils
-import { dateMath, DataFrame, LogRowModel, LoadingState, DateTime } from '@grafana/data';
+import {
+  dateMath,
+  DataFrame,
+  LogRowModel,
+  LoadingState,
+  DateTime,
+  AnnotationEvent,
+  DataFrameView,
+} from '@grafana/data';
 import { addLabelToSelector } from 'app/plugins/datasource/prometheus/add_label_to_query';
 import { addLabelToSelector } from 'app/plugins/datasource/prometheus/add_label_to_query';
 import LanguageProvider from './language_provider';
 import LanguageProvider from './language_provider';
 import { logStreamToDataFrame } from './result_transformer';
 import { logStreamToDataFrame } from './result_transformer';
@@ -15,6 +23,7 @@ import {
   DataQueryRequest,
   DataQueryRequest,
   DataStreamObserver,
   DataStreamObserver,
   DataQueryResponse,
   DataQueryResponse,
+  AnnotationQueryRequest,
 } from '@grafana/ui';
 } from '@grafana/ui';
 
 
 import { LokiQuery, LokiOptions, LokiLogsStream, LokiResponse } from './types';
 import { LokiQuery, LokiOptions, LokiLogsStream, LokiResponse } from './types';
@@ -193,7 +202,7 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
     }
     }
   };
   };
 
 
-  runQueries = async (options: DataQueryRequest<LokiQuery>) => {
+  runQueries = async (options: DataQueryRequest<LokiQuery>): Promise<{ data: DataFrame[] }> => {
     const queryTargets = options.targets
     const queryTargets = options.targets
       .filter(target => target.expr && !target.hide && !target.live)
       .filter(target => target.expr && !target.hide && !target.live)
       .map(target => this.prepareQueryTarget(target, options));
       .map(target => this.prepareQueryTarget(target, options));
@@ -368,6 +377,52 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
         return { status: 'error', message: message };
         return { status: 'error', message: message };
       });
       });
   }
   }
+
+  async annotationQuery(options: AnnotationQueryRequest<LokiQuery>): Promise<AnnotationEvent[]> {
+    if (!options.annotation.expr) {
+      return [];
+    }
+
+    const query = queryRequestFromAnnotationOptions(options);
+    const { data } = await this.runQueries(query);
+    const annotations: AnnotationEvent[] = [];
+    for (const frame of data) {
+      const tags = Object.values(frame.labels);
+      const view = new DataFrameView<{ ts: string; line: string }>(frame);
+      view.forEachRow(row => {
+        annotations.push({
+          time: new Date(row.ts).valueOf(),
+          text: row.line,
+          tags,
+        });
+      });
+    }
+
+    return annotations;
+  }
+}
+
+function queryRequestFromAnnotationOptions(options: AnnotationQueryRequest<LokiQuery>): DataQueryRequest<LokiQuery> {
+  const refId = `annotation-${options.annotation.name}`;
+  const target: LokiQuery = { refId, expr: options.annotation.expr };
+
+  return {
+    requestId: refId,
+    range: options.range,
+    targets: [target],
+    dashboardId: options.dashboard.id,
+    scopedVars: null,
+    startTime: Date.now(),
+
+    // This should mean the default defined on datasource is used.
+    maxDataPoints: 0,
+
+    // Dummy values, are required in type but not used here.
+    timezone: 'utc',
+    panelId: 0,
+    interval: '',
+    intervalMs: 0,
+  };
 }
 }
 
 
 export default LokiDatasource;
 export default LokiDatasource;

public/app/plugins/datasource/loki/module.ts

@@ -3,6 +3,7 @@ import Datasource from './datasource';
 import LokiStartPage from './components/LokiStartPage';
 import LokiStartPage from './components/LokiStartPage';
 import LokiQueryField from './components/LokiQueryField';
 import LokiQueryField from './components/LokiQueryField';
 import LokiQueryEditor from './components/LokiQueryEditor';
 import LokiQueryEditor from './components/LokiQueryEditor';
+import { LokiAnnotationsQueryCtrl } from './LokiAnnotationsQueryCtrl';
 
 
 export class LokiConfigCtrl {
 export class LokiConfigCtrl {
   static templateUrl = 'partials/config.html';
   static templateUrl = 'partials/config.html';
@@ -14,4 +15,5 @@ export {
   LokiConfigCtrl as ConfigCtrl,
   LokiConfigCtrl as ConfigCtrl,
   LokiQueryField as ExploreQueryField,
   LokiQueryField as ExploreQueryField,
   LokiStartPage as ExploreStartPage,
   LokiStartPage as ExploreStartPage,
+  LokiAnnotationsQueryCtrl as AnnotationsQueryCtrl,
 };
 };

public/app/plugins/datasource/loki/partials/annotations.editor.html

+<loki-annotations-query-editor
+    expr="ctrl.annotation.expr"
+    on-change="ctrl.onQueryChange"
+    datasource="ctrl.datasource"
+/>

public/app/plugins/datasource/loki/plugin.json

@@ -6,7 +6,7 @@
 
 
   "metrics": true,
   "metrics": true,
   "alerting": false,
   "alerting": false,
-  "annotations": false,
+  "annotations": true,
   "logs": true,
   "logs": true,
   "streaming": true,
   "streaming": true,