Grafana/Loki: Adds support for new Loki endpoints and metrics (#20158)

* Grafana/Loki: Adds support for new Loki endpoints and metrics

* Adds `/loki/` prefix to new loki endpoints and updates response interfaces

* Improved legacy support

* Removed changes related to plugin.json and added Loki-specific hacks

* Fixes live streaming for legacy loki datasources

packages/grafana-data/src/dataframe/DataFrameView.ts

@@ -26,22 +26,24 @@ export class DataFrameView<T = any> implements Vector<T> {
 
   constructor(private data: DataFrame) {
     const obj = ({} as unknown) as T;
+
     for (let i = 0; i < data.fields.length; i++) {
       const field = data.fields[i];
-      const getter = () => {
-        return field.values.get(this.index);
-      };
+      const getter = () => field.values.get(this.index);
+
       if (!(obj as any).hasOwnProperty(field.name)) {
         Object.defineProperty(obj, field.name, {
           enumerable: true, // Shows up as enumerable property
           get: getter,
         });
       }
+
       Object.defineProperty(obj, i, {
         enumerable: false, // Don't enumerate array index
         get: getter,
       });
     }
+
     this.obj = obj;
   }
 
@@ -59,11 +61,7 @@ export class DataFrameView<T = any> implements Vector<T> {
   }
 
   toArray(): T[] {
-    const arr: T[] = [];
-    for (let i = 0; i < this.data.length; i++) {
-      arr.push({ ...this.get(i) });
-    }
-    return arr;
+    return new Array(this.data.length).fill(0).map((_, i) => ({ ...this.get(i) }));
   }
 
   toJSON(): T[] {

packages/grafana-data/src/types/datasource.ts

@@ -261,6 +261,8 @@ export abstract class DataSourceApi<
    */
   languageProvider?: any;
 
+  getVersion?(): Promise<string>;
+
   /**
    * Can be optionally implemented to allow datasource to be a source of annotations for dashboard. To be visible
    * in the annotation editor `annotations` capability also needs to be enabled in plugin.json.
@@ -302,6 +304,7 @@ export interface ExploreQueryFieldProps<
 
 export interface ExploreStartPageProps {
   datasource?: DataSourceApi;
+  exploreMode: 'Logs' | 'Metrics';
   onClickExample: (query: DataQuery) => void;
 }
 
@@ -443,18 +446,22 @@ export interface DataQueryError {
 
 export interface DataQueryRequest<TQuery extends DataQuery = DataQuery> {
   requestId: string; // Used to identify results and optionally cancel the request in backendSrv
-  timezone: string;
-  range: TimeRange;
-  rangeRaw?: RawTimeRange;
-  timeInfo?: string; // The query time description (blue text in the upper right)
-  targets: TQuery[];
-  panelId: number;
+
   dashboardId: number;
-  cacheTimeout?: string;
   interval: string;
-  intervalMs: number;
-  maxDataPoints: number;
+  intervalMs?: number;
+  maxDataPoints?: number;
+  panelId: number;
+  range?: TimeRange;
+  reverse?: boolean;
   scopedVars: ScopedVars;
+  targets: TQuery[];
+  timezone: string;
+
+  cacheTimeout?: string;
+  exploreMode?: 'Logs' | 'Metrics';
+  rangeRaw?: RawTimeRange;
+  timeInfo?: string; // The query time description (blue text in the upper right)
 
   // Request Timing
   startTime: number;

packages/grafana-data/src/types/graph.ts

@@ -10,11 +10,10 @@ export type GraphSeriesValue = number | null;
 
 /** View model projection of a series */
 export interface GraphSeriesXY {
-  label: string;
   color: string;
   data: GraphSeriesValue[][]; // [x,y][]
-  info?: DisplayValue[]; // Legend info
   isVisible: boolean;
+  label: string;
   yAxis: YAxis;
   // Field with series' time values
   timeField: Field;
@@ -22,6 +21,8 @@ export interface GraphSeriesXY {
   valueField: Field;
   seriesIndex: number;
   timeStep: number;
+
+  info?: DisplayValue[]; // Legend info
 }
 
 export interface CreatePlotOverlay {

public/app/core/services/backend_srv.ts

@@ -10,6 +10,17 @@ import { FolderInfo, DashboardDTO, CoreEvents } from 'app/types';
 import { BackendSrv as BackendService, getBackendSrv as getBackendService, BackendSrvRequest } from '@grafana/runtime';
 import { AppEvents } from '@grafana/data';
 
+export interface DatasourceRequestOptions {
+  retry?: number;
+  method?: string;
+  requestId?: string;
+  timeout?: angular.IPromise<any>;
+  url?: string;
+  headers?: { [key: string]: any };
+  silent?: boolean;
+  data?: { [key: string]: any };
+}
+
 export class BackendSrv implements BackendService {
   private inFlightRequests: { [key: string]: Array<angular.IDeferred<any>> } = {};
   private HTTP_REQUEST_CANCELED = -1;

public/app/core/utils/explore.ts

@@ -148,6 +148,7 @@ export function buildQueryTransaction(
       __interval_ms: { text: intervalMs, value: intervalMs },
     },
     maxDataPoints: queryOptions.maxDataPoints,
+    exploreMode: queryOptions.mode,
   };
 
   return {
@@ -517,7 +518,7 @@ export const convertToWebSocketUrl = (url: string) => {
   const protocol = window.location.protocol === 'https:' ? 'wss://' : 'ws://';
   let backend = `${protocol}${window.location.host}${config.appSubUrl}`;
   if (backend.endsWith('/')) {
-    backend = backend.slice(0, backend.length - 1);
+    backend = backend.slice(0, -1);
   }
   return `${backend}${url}`;
 };

public/app/features/annotations/annotations_srv.ts

@@ -14,7 +14,7 @@ import { DashboardModel } from '../dashboard/state/DashboardModel';
 import DatasourceSrv from '../plugins/datasource_srv';
 import { BackendSrv } from 'app/core/services/backend_srv';
 import { TimeSrv } from '../dashboard/services/TimeSrv';
-import { DataSourceApi, PanelEvents, AnnotationEvent, AppEvents } from '@grafana/data';
+import { DataSourceApi, PanelEvents, AnnotationEvent, AppEvents, PanelModel, TimeRange } from '@grafana/data';
 import { GrafanaRootScope } from 'app/routes/GrafanaCtrl';
 
 export class AnnotationsSrv {
@@ -44,7 +44,7 @@ export class AnnotationsSrv {
     this.datasourcePromises = null;
   }
 
-  getAnnotations(options: any) {
+  getAnnotations(options: { dashboard: DashboardModel; panel: PanelModel; range: TimeRange }) {
     return this.$q
       .all([this.getGlobalAnnotations(options), this.getAlertStates(options)])
       .then(results => {
@@ -104,7 +104,7 @@ export class AnnotationsSrv {
     return this.alertStatesPromise;
   }
 
-  getGlobalAnnotations(options: any) {
+  getGlobalAnnotations(options: { dashboard: DashboardModel; panel: PanelModel; range: TimeRange }) {
     const dashboard = options.dashboard;
 
     if (this.globalAnnotationsPromise) {
@@ -130,7 +130,7 @@ export class AnnotationsSrv {
           .then((datasource: DataSourceApi) => {
             // issue query against data source
             return datasource.annotationQuery({
-              range: range,
+              range,
               rangeRaw: range.raw,
               annotation: annotation,
               dashboard: dashboard,

public/app/features/dashboard/state/runRequest.ts

@@ -78,7 +78,7 @@ export function processResponsePacket(packet: DataQueryResponse, state: RunningQ
  * It will
  *  * Merge multiple responses into a single DataFrame array based on the packet key
  *  * Will emit a loading state if no response after 50ms
- *  * Cancel any still runnning network requests on unsubscribe (using request.requestId)
+ *  * Cancel any still running network requests on unsubscribe (using request.requestId)
  */
 export function runRequest(datasource: DataSourceApi, request: DataQueryRequest): Observable<PanelData> {
   let state: RunningQueryState = {

public/app/features/dashboard/utils/getPanelMenu.ts

@@ -83,7 +83,7 @@ export const getPanelMenu = (dashboard: DashboardModel, panel: PanelModel) => {
   const onNavigateToExplore = (event: React.MouseEvent<any>) => {
     event.preventDefault();
     const openInNewWindow = event.ctrlKey || event.metaKey ? (url: string) => window.open(url) : undefined;
-    store.dispatch(navigateToExplore(panel, { getDataSourceSrv, getTimeSrv, getExploreUrl, openInNewWindow }));
+    store.dispatch(navigateToExplore(panel, { getDataSourceSrv, getTimeSrv, getExploreUrl, openInNewWindow }) as any);
   };
 
   const menu: PanelMenuItem[] = [];

public/app/features/explore/Explore.tsx

@@ -35,6 +35,7 @@ import {
   TimeZone,
   AbsoluteTimeRange,
 } from '@grafana/data';
+
 import {
   ExploreItemState,
   ExploreUrlState,
@@ -288,7 +289,11 @@ export class Explore extends React.PureComponent<ExploreProps> {
                     <ErrorBoundaryAlert>
                       {showingStartPage && (
                         <div className="grafana-info-box grafana-info-box--max-lg">
-                          <StartPage onClickExample={this.onClickExample} datasource={datasourceInstance} />
+                          <StartPage
+                            onClickExample={this.onClickExample}
+                            datasource={datasourceInstance}
+                            exploreMode={mode}
+                          />
                         </div>
                       )}
                       {!showingStartPage && (
@@ -373,6 +378,7 @@ function mapStateToProps(state: StoreState, { exploreId }: ExploreProps): Partia
   const initialRange = urlRange ? getTimeRangeFromUrlMemoized(urlRange, timeZone).raw : DEFAULT_RANGE;
 
   let newMode: ExploreMode;
+
   if (supportedModes.length) {
     const urlModeIsValid = supportedModes.includes(urlMode);
     const modeStateIsValid = supportedModes.includes(mode);
@@ -385,7 +391,7 @@ function mapStateToProps(state: StoreState, { exploreId }: ExploreProps): Partia
       newMode = supportedModes[0];
     }
   } else {
-    newMode = [ExploreMode.Metrics, ExploreMode.Logs].includes(mode) ? mode : ExploreMode.Metrics;
+    newMode = [ExploreMode.Metrics, ExploreMode.Logs].includes(urlMode) ? urlMode : null;
   }
 
   const initialUI = ui || DEFAULT_UI_STATE;

public/app/features/explore/ExploreToolbar.tsx

@@ -347,7 +347,9 @@ const mapStateToProps = (state: StoreState, { exploreId }: OwnProps): StateProps
     ? exploreDatasources.find(datasource => datasource.name === datasourceInstance.name)
     : undefined;
   const hasLiveOption =
-    datasourceInstance && datasourceInstance.meta && datasourceInstance.meta.streaming ? true : false;
+    datasourceInstance && datasourceInstance.meta && datasourceInstance.meta.streaming && mode === ExploreMode.Logs
+      ? true
+      : false;
 
   return {
     datasourceMissing,

public/app/features/explore/state/actionTypes.ts

@@ -186,6 +186,7 @@ export interface UpdateUIStatePayload extends Partial<ExploreUIState> {
 export interface UpdateDatasourceInstancePayload {
   exploreId: ExploreId;
   datasourceInstance: DataSourceApi;
+  version?: string;
 }
 
 export interface ToggleLogLevelPayload {

public/app/features/explore/state/actions.ts

@@ -123,8 +123,15 @@ export function changeDatasource(exploreId: ExploreId, datasource: string): Thun
     const currentDataSourceInstance = getState().explore[exploreId].datasourceInstance;
     const queries = getState().explore[exploreId].queries;
     const orgId = getState().user.orgId;
+    const datasourceVersion = newDataSourceInstance.getVersion && (await newDataSourceInstance.getVersion());
 
-    dispatch(updateDatasourceInstanceAction({ exploreId, datasourceInstance: newDataSourceInstance }));
+    dispatch(
+      updateDatasourceInstanceAction({
+        exploreId,
+        datasourceInstance: newDataSourceInstance,
+        version: datasourceVersion,
+      })
+    );
 
     await dispatch(importQueries(exploreId, queries, currentDataSourceInstance, newDataSourceInstance));
 
@@ -436,6 +443,7 @@ export function runQueries(exploreId: ExploreId): ThunkResult<void> {
       liveStreaming: live,
       showingGraph,
       showingTable,
+      mode,
     };
 
     const datasourceId = datasourceInstance.meta.id;

public/app/features/explore/state/reducers.ts

@@ -275,30 +275,46 @@ export const itemReducer = reducerFactory<ExploreItemState>({} as ExploreItemSta
   .addMapper({
     filter: updateDatasourceInstanceAction,
     mapper: (state, action): ExploreItemState => {
-      const { datasourceInstance } = action.payload;
-      const [supportedModes, mode] = getModesForDatasource(datasourceInstance, state.mode);
-
-      const originPanelId = state.urlState && state.urlState.originPanelId;
+      const { datasourceInstance, version } = action.payload;
 
       // Custom components
       const StartPage = datasourceInstance.components.ExploreStartPage;
       stopQueryState(state.querySubscription);
 
+      let newMetadata = datasourceInstance.meta;
+
+      // HACK: Temporary hack for Loki datasource. Can remove when plugin.json structure is changed.
+      if (version && version.length && datasourceInstance.meta.name === 'Loki') {
+        const lokiVersionMetadata: Record<string, { metrics: boolean }> = {
+          v0: {
+            metrics: false,
+          },
+
+          v1: {
+            metrics: true,
+          },
+        };
+        newMetadata = { ...newMetadata, ...lokiVersionMetadata[version] };
+      }
+
+      const updatedDatasourceInstance = Object.assign(datasourceInstance, { meta: newMetadata });
+      const [supportedModes, mode] = getModesForDatasource(updatedDatasourceInstance, state.mode);
+
       return {
         ...state,
-        datasourceInstance,
+        datasourceInstance: updatedDatasourceInstance,
         graphResult: null,
         tableResult: null,
         logsResult: null,
         latency: 0,
         queryResponse: createEmptyQueryResponse(),
         loading: false,
-        StartPage,
+        StartPage: datasourceInstance.components.ExploreStartPage,
         showingStartPage: Boolean(StartPage),
         queryKeys: [],
         supportedModes,
         mode,
-        originPanelId,
+        originPanelId: state.urlState && state.urlState.originPanelId,
       };
     },
   })
@@ -657,10 +673,7 @@ export const updateChildRefreshState = (
 };
 
 const getModesForDatasource = (dataSource: DataSourceApi, currentMode: ExploreMode): [ExploreMode[], ExploreMode] => {
-  // Temporary hack here. We want Loki to work in dashboards for which it needs to have metrics = true which is weird
-  // for Explore.
-  // TODO: need to figure out a better way to handle this situation
-  const supportsGraph = dataSource.meta.name === 'Loki' ? false : dataSource.meta.metrics;
+  const supportsGraph = dataSource.meta.metrics;
   const supportsLogs = dataSource.meta.logs;
 
   let mode = currentMode || ExploreMode.Metrics;
@@ -678,6 +691,12 @@ const getModesForDatasource = (dataSource: DataSourceApi, currentMode: ExploreMo
     mode = supportedModes[0];
   }
 
+  // HACK: Used to set Loki's default explore mode to Logs mode.
+  // A better solution would be to introduce a "default" or "preferred" mode to the datasource config
+  if (dataSource.meta.name === 'Loki' && !currentMode) {
+    mode = ExploreMode.Logs;
+  }
+
   return [supportedModes, mode];
 };
 

public/app/features/playlist/specs/playlist_srv.test.ts

@@ -3,13 +3,11 @@ import configureMockStore from 'redux-mock-store';
 import { PlaylistSrv } from '../playlist_srv';
 import { setStore } from 'app/store/store';
 
-const mockStore = configureMockStore();
+const mockStore = configureMockStore<any, any>();
 
-setStore(
-  mockStore({
-    location: {},
-  })
-);
+setStore(mockStore({
+  location: {},
+}) as any);
 
 const dashboards = [{ url: 'dash1' }, { url: 'dash2' }];
 
@@ -122,13 +120,11 @@ describe('PlaylistSrv', () => {
 
     srv.next();
 
-    setStore(
-      mockStore({
-        location: {
-          path: 'dash2',
-        },
-      })
-    );
+    setStore(mockStore({
+      location: {
+        path: 'dash2',
+      },
+    }) as any);
 
     expect((srv as any).validPlaylistUrl).toBe('dash2');
 

public/app/features/plugins/datasource_srv.ts

@@ -66,7 +66,7 @@ export class DatasourceSrv implements DataSourceService {
 
     const dsConfig = config.datasources[name];
     if (!dsConfig) {
-      return this.$q.reject({ message: 'Datasource named ' + name + ' was not found' });
+      return this.$q.reject({ message: `Datasource named ${name} was not found` });
     }
 
     const deferred = this.$q.defer();

public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts

@@ -285,7 +285,7 @@ describe('CloudWatchDatasource', () => {
       beforeEach(() => {
         redux.setStore({
           dispatch: jest.fn(),
-        });
+        } as any);
 
         ctx.backendSrv.datasourceRequest = jest.fn(() => {
           return Promise.reject(backendErrorResponse);

public/app/plugins/datasource/loki/components/LokiCheatSheet.tsx

@@ -2,11 +2,31 @@ import React, { PureComponent } from 'react';
 import { shuffle } from 'lodash';
 import { ExploreStartPageProps, DataQuery } from '@grafana/data';
 import LokiLanguageProvider from '../language_provider';
+import { ExploreMode } from 'app/types';
 
 const DEFAULT_EXAMPLES = ['{job="default/prometheus"}'];
 const PREFERRED_LABELS = ['job', 'app', 'k8s_app'];
 const EXAMPLES_LIMIT = 5;
 
+const LOGQL_EXAMPLES = [
+  {
+    title: 'Count over time',
+    expression: 'count_over_time({job="mysql"}[5m])',
+    label: 'This query counts all the log lines within the last five minutes for the MySQL job.',
+  },
+  {
+    title: 'Rate',
+    expression: 'rate(({job="mysql"} |= "error" != "timeout")[10s])',
+    label:
+      'This query gets the per-second rate of all non-timeout errors within the last ten seconds for the MySQL job.',
+  },
+  {
+    title: 'Aggregate, count, and group',
+    expression: 'sum(count_over_time({job="mysql"}[5m])) by (level)',
+    label: 'Get the count of logs during the last five minutes, grouping by level.',
+  },
+];
+
 export default class LokiCheatSheet extends PureComponent<ExploreStartPageProps, { userExamples: string[] }> {
   userLabelTimer: NodeJS.Timeout;
   state = {
@@ -57,7 +77,7 @@ export default class LokiCheatSheet extends PureComponent<ExploreStartPageProps,
     );
   }
 
-  render() {
+  renderLogsCheatSheet() {
     const { userExamples } = this.state;
 
     return (
@@ -98,4 +118,25 @@ export default class LokiCheatSheet extends PureComponent<ExploreStartPageProps,
       </>
     );
   }
+
+  renderMetricsCheatSheet() {
+    return (
+      <div>
+        <h2>LogQL Cheat Sheet</h2>
+        {LOGQL_EXAMPLES.map(item => (
+          <div className="cheat-sheet-item" key={item.expression}>
+            <div className="cheat-sheet-item__title">{item.title}</div>
+            {this.renderExpression(item.expression)}
+            <div className="cheat-sheet-item__label">{item.label}</div>
+          </div>
+        ))}
+      </div>
+    );
+  }
+
+  render() {
+    const { exploreMode } = this.props;
+
+    return exploreMode === ExploreMode.Logs ? this.renderLogsCheatSheet() : this.renderMetricsCheatSheet();
+  }
 }

public/app/plugins/datasource/loki/components/LokiQueryFieldForm.tsx

@@ -17,9 +17,9 @@ import {
 import { Plugin, Node } from 'slate';
 
 // Types
-import { LokiQuery } from '../types';
 import { DOMUtil } from '@grafana/ui';
 import { ExploreQueryFieldProps, AbsoluteTimeRange } from '@grafana/data';
+import { LokiQuery, LokiOptions } from '../types';
 import { Grammar } from 'prismjs';
 import LokiLanguageProvider, { LokiHistoryItem } from '../language_provider';
 import LokiDatasource from '../datasource';
@@ -61,7 +61,7 @@ function willApplySuggestion(suggestion: string, { typeaheadContext, typeaheadTe
   return suggestion;
 }
 
-export interface LokiQueryFieldFormProps extends ExploreQueryFieldProps<LokiDatasource, LokiQuery> {
+export interface LokiQueryFieldFormProps extends ExploreQueryFieldProps<LokiDatasource, LokiQuery, LokiOptions> {
   history: LokiHistoryItem[];
   syntax: Grammar;
   logLabelOptions: CascaderOption[];

public/app/plugins/datasource/loki/datasource.test.ts

 import LokiDatasource from './datasource';
-import { LokiQuery } from './types';
+import { LokiQuery, LokiResultType, LokiResponse, LokiLegacyStreamResponse } from './types';
 import { getQueryOptions } from 'test/helpers/getQueryOptions';
 import { AnnotationQueryRequest, DataSourceApi, DataFrame, dateTime } from '@grafana/data';
 import { BackendSrv } from 'app/core/services/backend_srv';
 import { TemplateSrv } from 'app/features/templating/template_srv';
 import { CustomVariable } from 'app/features/templating/custom_variable';
+import { ExploreMode } from 'app/types';
+import { of } from 'rxjs';
+import omit from 'lodash/omit';
 
 describe('LokiDatasource', () => {
   const instanceSettings: any = {
     url: 'myloggingurl',
   };
 
-  const testResp = {
+  const legacyTestResp: { data: LokiLegacyStreamResponse; status: number } = {
     data: {
       streams: [
         {
@@ -20,6 +23,22 @@ describe('LokiDatasource', () => {
         },
       ],
     },
+    status: 404, // for simulating legacy endpoint
+  };
+
+  const testResp: { data: LokiResponse } = {
+    data: {
+      data: {
+        resultType: LokiResultType.Stream,
+        result: [
+          {
+            stream: {},
+            values: [['1573646419522934000', 'hello']],
+          },
+        ],
+      },
+      status: 'success',
+    },
   };
 
   const backendSrvMock = { datasourceRequest: jest.fn() };
@@ -30,8 +49,67 @@ describe('LokiDatasource', () => {
     replace: (a: string) => a,
   } as unknown) as TemplateSrv;
 
+  describe('when running range query with fallback', () => {
+    let ds: LokiDatasource;
+    beforeEach(() => {
+      const customData = { ...(instanceSettings.jsonData || {}), maxLines: 20 };
+      const customSettings = { ...instanceSettings, jsonData: customData };
+      ds = new LokiDatasource(customSettings, backendSrv, templateSrvMock);
+      backendSrvMock.datasourceRequest = jest.fn(() => Promise.resolve(legacyTestResp));
+    });
+
+    test('should try latest endpoint but fall back to legacy endpoint if it cannot be reached', async () => {
+      const options = getQueryOptions<LokiQuery>({
+        targets: [{ expr: '{job="grafana"}', refId: 'B' }],
+        exploreMode: ExploreMode.Logs,
+      });
+
+      ds.runLegacyQuery = jest.fn();
+      await ds.runRangeQueryWithFallback(options.targets[0], options).toPromise();
+      expect(ds.runLegacyQuery).toBeCalled();
+    });
+  });
+
   describe('when querying', () => {
-    const testLimit = makeLimitTest(instanceSettings, backendSrvMock, backendSrv, templateSrvMock, testResp);
+    const testLimit = makeLimitTest(instanceSettings, backendSrvMock, backendSrv, templateSrvMock, legacyTestResp);
+    let ds: LokiDatasource;
+
+    beforeEach(() => {
+      const customData = { ...(instanceSettings.jsonData || {}), maxLines: 20 };
+      const customSettings = { ...instanceSettings, jsonData: customData };
+      ds = new LokiDatasource(customSettings, backendSrv, templateSrvMock);
+      backendSrvMock.datasourceRequest = jest.fn(() => Promise.resolve(testResp));
+    });
+
+    test('should run instant query and range query when in metrics mode', async () => {
+      const options = getQueryOptions<LokiQuery>({
+        targets: [{ expr: 'rate({job="grafana"}[5m])', refId: 'A' }],
+        exploreMode: ExploreMode.Metrics,
+      });
+
+      ds.runInstantQuery = jest.fn(() => of({ data: [] }));
+      ds.runLegacyQuery = jest.fn();
+      ds.runRangeQueryWithFallback = jest.fn(() => of({ data: [] }));
+      await ds.query(options).toPromise();
+
+      expect(ds.runInstantQuery).toBeCalled();
+      expect(ds.runLegacyQuery).not.toBeCalled();
+      expect(ds.runRangeQueryWithFallback).toBeCalled();
+    });
+
+    test('should just run range query when in logs mode', async () => {
+      const options = getQueryOptions<LokiQuery>({
+        targets: [{ expr: '{job="grafana"}', refId: 'B' }],
+        exploreMode: ExploreMode.Logs,
+      });
+
+      ds.runInstantQuery = jest.fn(() => of({ data: [] }));
+      ds.runRangeQueryWithFallback = jest.fn(() => of({ data: [] }));
+      await ds.query(options).toPromise();
+
+      expect(ds.runInstantQuery).not.toBeCalled();
+      expect(ds.runRangeQueryWithFallback).toBeCalled();
+    });
 
     test('should use default max lines when no limit given', () => {
       testLimit({
@@ -61,14 +139,17 @@ describe('LokiDatasource', () => {
       });
     });
 
-    test('should return series data', async done => {
+    test('should return series data', async () => {
       const customData = { ...(instanceSettings.jsonData || {}), maxLines: 20 };
       const customSettings = { ...instanceSettings, jsonData: customData };
       const ds = new LokiDatasource(customSettings, backendSrv, templateSrvMock);
-      backendSrvMock.datasourceRequest = jest.fn(() => Promise.resolve(testResp));
+      backendSrvMock.datasourceRequest = jest
+        .fn()
+        .mockReturnValueOnce(Promise.resolve(legacyTestResp))
+        .mockReturnValueOnce(Promise.resolve(omit(legacyTestResp, 'status')));
 
       const options = getQueryOptions<LokiQuery>({
-        targets: [{ expr: '{} foo', refId: 'B' }],
+        targets: [{ expr: '{job="grafana"} |= "foo"', refId: 'B' }],
       });
 
       const res = await ds.query(options).toPromise();
@@ -76,14 +157,13 @@ describe('LokiDatasource', () => {
       const dataFrame = res.data[0] as DataFrame;
       expect(dataFrame.fields[1].values.get(0)).toBe('hello');
       expect(dataFrame.meta.limit).toBe(20);
-      expect(dataFrame.meta.searchWords).toEqual(['(?i)foo']);
-      done();
+      expect(dataFrame.meta.searchWords).toEqual(['foo']);
     });
   });
 
   describe('When interpolating variables', () => {
-    let ds: any = {};
-    let variable: any = {};
+    let ds: LokiDatasource;
+    let variable: CustomVariable;
 
     beforeEach(() => {
       const customData = { ...(instanceSettings.jsonData || {}), maxLines: 20 };
@@ -155,23 +235,25 @@ describe('LokiDatasource', () => {
     });
 
     describe('and call fails with 401 error', () => {
-      beforeEach(async () => {
-        const backendSrv = ({
-          async datasourceRequest() {
-            return Promise.reject({
-              statusText: 'Unauthorized',
-              status: 401,
-              data: {
-                message: 'Unauthorized',
-              },
-            });
-          },
-        } as unknown) as BackendSrv;
-        ds = new LokiDatasource(instanceSettings, backendSrv, {} as TemplateSrv);
-        result = await ds.testDatasource();
+      let ds: LokiDatasource;
+      beforeEach(() => {
+        backendSrvMock.datasourceRequest = jest.fn(() =>
+          Promise.reject({
+            statusText: 'Unauthorized',
+            status: 401,
+            data: {
+              message: 'Unauthorized',
+            },
+          })
+        );
+
+        const customData = { ...(instanceSettings.jsonData || {}), maxLines: 20 };
+        const customSettings = { ...instanceSettings, jsonData: customData };
+        ds = new LokiDatasource(customSettings, backendSrv, templateSrvMock);
       });
 
-      it('should return error status and a detailed error message', () => {
+      it('should return error status and a detailed error message', async () => {
+        const result = await ds.testDatasource();
         expect(result.status).toEqual('error');
         expect(result.message).toBe('Loki: Unauthorized. 401. Unauthorized');
       });
@@ -221,24 +303,32 @@ describe('LokiDatasource', () => {
   });
 
   describe('annotationQuery', () => {
-    it('should transform the loki data to annototion response', async () => {
+    it('should transform the loki data to annotation response', async () => {
       const ds = new LokiDatasource(instanceSettings, backendSrv, templateSrvMock);
-      backendSrvMock.datasourceRequest = jest.fn(() =>
-        Promise.resolve({
-          data: {
-            streams: [
-              {
-                entries: [{ ts: '2019-02-01T10:27:37.498180581Z', line: 'hello' }],
-                labels: '{label="value"}',
-              },
-              {
-                entries: [{ ts: '2019-02-01T12:27:37.498180581Z', line: 'hello 2' }],
-                labels: '{label2="value2"}',
-              },
-            ],
-          },
-        })
-      );
+      backendSrvMock.datasourceRequest = jest
+        .fn()
+        .mockReturnValueOnce(
+          Promise.resolve({
+            data: [],
+            status: 404,
+          })
+        )
+        .mockReturnValueOnce(
+          Promise.resolve({
+            data: {
+              streams: [
+                {
+                  entries: [{ ts: '2019-02-01T10:27:37.498180581Z', line: 'hello' }],
+                  labels: '{label="value"}',
+                },
+                {
+                  entries: [{ ts: '2019-02-01T12:27:37.498180581Z', line: 'hello 2' }],
+                  labels: '{label2="value2"}',
+                },
+              ],
+            },
+          })
+        );
       const query = makeAnnotationQueryRequest();
 
       const res = await ds.annotationQuery(query);

public/app/plugins/datasource/loki/language_provider.test.ts

@@ -10,6 +10,18 @@ import { beforeEach } from 'test/lib/common';
 import { makeMockLokiDatasource } from './mocks';
 import LokiDatasource from './datasource';
 
+jest.mock('app/store/store', () => ({
+  store: {
+    getState: jest.fn().mockReturnValue({
+      explore: {
+        left: {
+          mode: 'Logs',
+        },
+      },
+    }),
+  },
+}));
+
 describe('Language completion provider', () => {
   const datasource = makeMockLokiDatasource({});
 

public/app/plugins/datasource/loki/live_streams.test.ts

@@ -36,7 +36,7 @@ describe('Live Stream Tests', () => {
     fakeSocket = new Subject<any>();
     const labels: Labels = { job: 'varlogs' };
     const target = makeTarget('fake', labels);
-    const stream = new LiveStreams().getStream(target);
+    const stream = new LiveStreams().getLegacyStream(target);
     expect.assertions(4);
 
     const tests = [
@@ -74,21 +74,21 @@ describe('Live Stream Tests', () => {
   it('returns the same subscription if the url matches existing one', () => {
     fakeSocket = new Subject<any>();
     const liveStreams = new LiveStreams();
-    const stream1 = liveStreams.getStream(makeTarget('url_to_match'));
-    const stream2 = liveStreams.getStream(makeTarget('url_to_match'));
+    const stream1 = liveStreams.getLegacyStream(makeTarget('url_to_match'));
+    const stream2 = liveStreams.getLegacyStream(makeTarget('url_to_match'));
     expect(stream1).toBe(stream2);
   });
 
   it('returns new subscription when the previous unsubscribed', () => {
     fakeSocket = new Subject<any>();
     const liveStreams = new LiveStreams();
-    const stream1 = liveStreams.getStream(makeTarget('url_to_match'));
+    const stream1 = liveStreams.getLegacyStream(makeTarget('url_to_match'));
     const subscription = stream1.subscribe({
       next: noop,
     });
     subscription.unsubscribe();
 
-    const stream2 = liveStreams.getStream(makeTarget('url_to_match'));
+    const stream2 = liveStreams.getLegacyStream(makeTarget('url_to_match'));
     expect(stream1).not.toBe(stream2);
   });
 
@@ -101,7 +101,7 @@ describe('Live Stream Tests', () => {
     spy.and.returnValue(fakeSocket);
 
     const liveStreams = new LiveStreams();
-    const stream1 = liveStreams.getStream(makeTarget('url_to_match'));
+    const stream1 = liveStreams.getLegacyStream(makeTarget('url_to_match'));
     const subscription = stream1.subscribe({
       next: noop,
     });

public/app/plugins/datasource/loki/live_streams.ts

 import { DataFrame, FieldType, parseLabels, KeyValue, CircularDataFrame } from '@grafana/data';
 import { Observable } from 'rxjs';
 import { webSocket } from 'rxjs/webSocket';
-import { LokiResponse } from './types';
+import { LokiLegacyStreamResponse, LokiTailResponse } from './types';
 import { finalize, map } from 'rxjs/operators';
-import { appendResponseToBufferedData } from './result_transformer';
+import { appendLegacyResponseToBufferedData, appendResponseToBufferedData } from './result_transformer';
 
 /**
  * Maps directly to a query in the UI (refId is key)
  */
-export interface LiveTarget {
+export interface LegacyTarget {
   query: string;
   regexp: string;
   url: string;
@@ -16,6 +16,13 @@ export interface LiveTarget {
   size: number;
 }
 
+export interface LiveTarget {
+  query: string;
+  delay_for?: string;
+  limit?: string;
+  start?: string;
+}
+
 /**
  * Cache of websocket streams that can be returned as observable. In case there already is a stream for particular
  * target it is returned and on subscription returns the latest dataFrame.
@@ -23,26 +30,59 @@ export interface LiveTarget {
 export class LiveStreams {
   private streams: KeyValue<Observable<DataFrame[]>> = {};
 
-  getStream(target: LiveTarget): Observable<DataFrame[]> {
+  getLegacyStream(target: LegacyTarget): Observable<DataFrame[]> {
+    let stream = this.streams[target.url];
+
+    if (stream) {
+      return stream;
+    }
+
+    const data = new CircularDataFrame({ capacity: target.size });
+    data.addField({ name: 'ts', type: FieldType.time, config: { title: 'Time' } });
+    data.addField({ name: 'line', type: FieldType.string }).labels = parseLabels(target.query);
+    data.addField({ name: 'labels', type: FieldType.other }); // The labels for each line
+    data.addField({ name: 'id', type: FieldType.string });
+
+    stream = webSocket(target.url).pipe(
+      finalize(() => {
+        delete this.streams[target.url];
+      }),
+
+      map((response: LokiLegacyStreamResponse) => {
+        appendLegacyResponseToBufferedData(response, data);
+        return [data];
+      })
+    );
+    this.streams[target.url] = stream;
+
+    return stream;
+  }
+
+  getStream(target: LegacyTarget): Observable<DataFrame[]> {
     let stream = this.streams[target.url];
-    if (!stream) {
-      const data = new CircularDataFrame({ capacity: target.size });
-      data.addField({ name: 'ts', type: FieldType.time, config: { title: 'Time' } });
-      data.addField({ name: 'line', type: FieldType.string }).labels = parseLabels(target.query);
-      data.addField({ name: 'labels', type: FieldType.other }); // The labels for each line
-      data.addField({ name: 'id', type: FieldType.string });
-
-      stream = webSocket(target.url).pipe(
-        finalize(() => {
-          delete this.streams[target.url];
-        }),
-        map((response: LokiResponse) => {
-          appendResponseToBufferedData(response, data);
-          return [data];
-        })
-      );
-      this.streams[target.url] = stream;
+
+    if (stream) {
+      return stream;
     }
+
+    const data = new CircularDataFrame({ capacity: target.size });
+    data.addField({ name: 'ts', type: FieldType.time, config: { title: 'Time' } });
+    data.addField({ name: 'line', type: FieldType.string }).labels = parseLabels(target.query);
+    data.addField({ name: 'labels', type: FieldType.other }); // The labels for each line
+    data.addField({ name: 'id', type: FieldType.string });
+
+    stream = webSocket(target.url).pipe(
+      finalize(() => {
+        delete this.streams[target.url];
+      }),
+
+      map((response: LokiTailResponse) => {
+        appendResponseToBufferedData(response, data);
+        return [data];
+      })
+    );
+    this.streams[target.url] = stream;
+
     return stream;
   }
 }

public/app/plugins/datasource/loki/plugin.json

@@ -4,10 +4,10 @@
   "id": "loki",
   "category": "logging",
 
+  "logs": true,
   "metrics": true,
   "alerting": false,
   "annotations": true,
-  "logs": true,
   "streaming": true,
 
   "queryOptions": {

public/app/plugins/datasource/loki/result_transformer.test.ts

-import { logStreamToDataFrame, appendResponseToBufferedData } from './result_transformer';
+import { legacyLogStreamToDataFrame, appendLegacyResponseToBufferedData } from './result_transformer';
 import { FieldType, MutableDataFrame } from '@grafana/data';
-import { LokiLogsStream } from './types';
+import { LokiLegacyStreamResult } from './types';
 
-const streams: LokiLogsStream[] = [
+const streams: LokiLegacyStreamResult[] = [
   {
     labels: '{foo="bar"}',
     entries: [
@@ -25,7 +25,7 @@ const streams: LokiLogsStream[] = [
 
 describe('logStreamToDataFrame', () => {
   it('converts streams to series', () => {
-    const data = streams.map(stream => logStreamToDataFrame(stream));
+    const data = streams.map(stream => legacyLogStreamToDataFrame(stream));
 
     expect(data.length).toBe(2);
     expect(data[0].fields[1].labels['foo']).toEqual('bar');
@@ -46,7 +46,7 @@ describe('appendResponseToBufferedData', () => {
     data.addField({ name: 'labels', type: FieldType.other });
     data.addField({ name: 'id', type: FieldType.string });
 
-    appendResponseToBufferedData({ streams }, data);
+    appendLegacyResponseToBufferedData({ streams }, data);
     expect(data.get(0)).toEqual({
       ts: '1970-01-01T00:00:00Z',
       line: "foo: 'bar'",

public/app/plugins/datasource/loki/syntax.ts

 import { Grammar } from 'prismjs';
+import { CompletionItem } from '@grafana/ui';
 
-/* tslint:disable max-line-length */
+const AGGREGATION_OPERATORS: CompletionItem[] = [
+  {
+    label: 'sum',
+    insertText: 'sum',
+    documentation: 'Calculate sum over dimensions',
+  },
+  {
+    label: 'min',
+    insertText: 'min',
+    documentation: 'Select minimum over dimensions',
+  },
+  {
+    label: 'max',
+    insertText: 'max',
+    documentation: 'Select maximum over dimensions',
+  },
+  {
+    label: 'avg',
+    insertText: 'avg',
+    documentation: 'Calculate the average over dimensions',
+  },
+  {
+    label: 'stddev',
+    insertText: 'stddev',
+    documentation: 'Calculate population standard deviation over dimensions',
+  },
+  {
+    label: 'stdvar',
+    insertText: 'stdvar',
+    documentation: 'Calculate population standard variance over dimensions',
+  },
+  {
+    label: 'count',
+    insertText: 'count',
+    documentation: 'Count number of elements in the vector',
+  },
+  {
+    label: 'bottomk',
+    insertText: 'bottomk',
+    documentation: 'Smallest k elements by sample value',
+  },
+  {
+    label: 'topk',
+    insertText: 'topk',
+    documentation: 'Largest k elements by sample value',
+  },
+];
+
+export const RANGE_VEC_FUNCTIONS = [
+  {
+    insertText: 'count_over_time',
+    label: 'count_over_time',
+    detail: 'count_over_time(range-vector)',
+    documentation: 'The count of all values in the specified interval.',
+  },
+  {
+    insertText: 'rate',
+    label: 'rate',
+    detail: 'rate(v range-vector)',
+    documentation:
+      "Calculates the per-second average rate of increase of the time series in the range vector. Breaks in monotonicity (such as counter resets due to target restarts) are automatically adjusted for. Also, the calculation extrapolates to the ends of the time range, allowing for missed scrapes or imperfect alignment of scrape cycles with the range's time period.",
+  },
+];
+
+export const FUNCTIONS = [...AGGREGATION_OPERATORS, ...RANGE_VEC_FUNCTIONS];
 
 const tokenizer: Grammar = {
   comment: {
     pattern: /(^|[^\n])#.*/,
     lookbehind: true,
   },
+  'context-aggregation': {
+    pattern: /((without|by)\s*)\([^)]*\)/, // by ()
+    lookbehind: true,
+    inside: {
+      'label-key': {
+        pattern: /[^(),\s][^,)]*[^),\s]*/,
+        alias: 'attr-name',
+      },
+      punctuation: /[()]/,
+    },
+  },
   'context-labels': {
-    pattern: /(^|\s)\{[^}]*(?=})/,
+    pattern: /\{[^}]*(?=})/,
     lookbehind: true,
     inside: {
       'label-key': {
@@ -23,9 +99,31 @@ const tokenizer: Grammar = {
       punctuation: /[{]/,
     },
   },
-  // number: /\b-?\d+((\.\d*)?([eE][+-]?\d+)?)?\b/,
+  function: new RegExp(`\\b(?:${FUNCTIONS.map(f => f.label).join('|')})(?=\\s*\\()`, 'i'),
+  'context-range': [
+    {
+      pattern: /\[[^\]]*(?=\])/, // [1m]
+      inside: {
+        'range-duration': {
+          pattern: /\b\d+[smhdwy]\b/i,
+          alias: 'number',
+        },
+      },
+    },
+    {
+      pattern: /(offset\s+)\w+/, // offset 1m
+      lookbehind: true,
+      inside: {
+        'range-duration': {
+          pattern: /\b\d+[smhdwy]\b/i,
+          alias: 'number',
+        },
+      },
+    },
+  ],
+  number: /\b-?\d+((\.\d*)?([eE][+-]?\d+)?)?\b/,
   operator: new RegExp(`/&&?|\\|?\\||!=?|<(?:=>?|<|>)?|>[>=]?`, 'i'),
-  punctuation: /[{}`,.]/,
+  punctuation: /[{}()`,.]/,
 };
 
 export default tokenizer;

public/app/plugins/datasource/loki/types.ts

 import { Labels, DataQuery, DataSourceJsonData } from '@grafana/data';
 
+export interface LokiLegacyQueryRequest {
+  query: string;
+  limit?: number;
+  start?: number;
+  end?: number;
+  direction?: 'BACKWARD' | 'FORWARD';
+  regexp?: string;
+
+  refId: string;
+}
+
+export interface LokiInstantQueryRequest {
+  query: string;
+  limit?: number;
+  time?: string;
+  direction?: 'BACKWARD' | 'FORWARD';
+}
+
+export interface LokiRangeQueryRequest {
+  query: string;
+  limit?: number;
+  start?: number;
+  end?: number;
+  step?: number;
+  direction?: 'BACKWARD' | 'FORWARD';
+}
+
+export enum LokiResultType {
+  Stream = 'streams',
+  Vector = 'vector',
+  Matrix = 'matrix',
+}
+
 export interface LokiQuery extends DataQuery {
   expr: string;
   liveStreaming?: boolean;
   query?: string;
   regexp?: string;
+  format?: string;
+  reverse?: boolean;
+  legendFormat?: string;
+  valueWithRefId?: boolean;
 }
 
 export interface LokiOptions extends DataSourceJsonData {
@@ -12,11 +49,46 @@ export interface LokiOptions extends DataSourceJsonData {
   derivedFields?: DerivedFieldConfig[];
 }
 
-export interface LokiResponse {
-  streams: LokiLogsStream[];
+export interface LokiVectorResult {
+  metric: { [label: string]: string };
+  value: [number, string];
+}
+
+export interface LokiVectorResponse {
+  status: string;
+  data: {
+    resultType: LokiResultType.Vector;
+    result: LokiVectorResult[];
+  };
 }
 
-export interface LokiLogsStream {
+export interface LokiMatrixResult {
+  metric: { [label: string]: string };
+  values: Array<[number, string]>;
+}
+
+export interface LokiMatrixResponse {
+  status: string;
+  data: {
+    resultType: LokiResultType.Matrix;
+    result: LokiMatrixResult[];
+  };
+}
+
+export interface LokiStreamResult {
+  stream: Record<string, string>;
+  values: Array<[string, string]>;
+}
+
+export interface LokiStreamResponse {
+  status: string;
+  data: {
+    resultType: LokiResultType.Stream;
+    result: LokiStreamResult[];
+  };
+}
+
+export interface LokiLegacyStreamResult {
   labels: string;
   entries: LokiLogsStreamEntry[];
   search?: string;
@@ -24,6 +96,21 @@ export interface LokiLogsStream {
   uniqueLabels?: Labels;
 }
 
+export interface LokiLegacyStreamResponse {
+  streams: LokiLegacyStreamResult[];
+}
+
+export interface LokiTailResponse {
+  streams: LokiStreamResult[];
+  dropped_entries?: Array<{
+    labels: Record<string, string>;
+    timestamp: string;
+  }>;
+}
+
+export type LokiResult = LokiVectorResult | LokiMatrixResult | LokiStreamResult | LokiLegacyStreamResult;
+export type LokiResponse = LokiVectorResponse | LokiMatrixResponse | LokiStreamResponse;
+
 export interface LokiLogsStreamEntry {
   line: string;
   ts: string;
@@ -41,3 +128,15 @@ export type DerivedFieldConfig = {
   name: string;
   url?: string;
 };
+
+export interface TransformerOptions {
+  format: string;
+  legendFormat: string;
+  step: number;
+  start: number;
+  end: number;
+  query: string;
+  responseListLength: number;
+  refId: string;
+  valueWithRefId?: boolean;
+}

public/app/store/configureStore.ts

@@ -6,6 +6,7 @@ import { setStore } from './store';
 import { StoreState } from 'app/types/store';
 import { toggleLogActionsMiddleware } from 'app/core/middlewares/application';
 import { addReducer, createRootReducer } from '../core/reducers/root';
+import { ActionOf } from 'app/core/redux';
 
 export function addRootReducer(reducers: any) {
   // this is ok now because we add reducers before configureStore is called
@@ -27,7 +28,11 @@ export function configureStore() {
       ? applyMiddleware(toggleLogActionsMiddleware, thunk, logger)
       : applyMiddleware(thunk);
 
-  const store: any = createStore(createRootReducer(), {}, composeEnhancers(storeEnhancers));
+  const store = createStore<StoreState, ActionOf<any>, any, any>(
+    createRootReducer(),
+    {},
+    composeEnhancers(storeEnhancers)
+  );
   setStore(store);
   return store;
 }

public/app/store/store.ts

-export let store: any;
+import { StoreState } from 'app/types';
+import { Store } from 'redux';
 
-export function setStore(newStore: any) {
+export let store: Store<StoreState>;
+
+export function setStore(newStore: Store<StoreState>) {
   store = newStore;
 }

public/app/types/explore.ts

@@ -228,6 +228,7 @@ export interface QueryOptions {
   liveStreaming?: boolean;
   showingGraph?: boolean;
   showingTable?: boolean;
+  mode?: ExploreMode;
 }
 
 export interface QueryTransaction {