Merge pull request #14689 from grafana/davkal/loki-limit

Loki: query limit configurable in datasource

public/app/plugins/datasource/loki/datasource.test.ts

 import LokiDatasource from './datasource';
 
 describe('LokiDatasource', () => {
-  const instanceSettings = {
+  const instanceSettings: any = {
     url: 'myloggingurl',
   };
 
+  describe('when querying', () => {
+    const backendSrvMock = { datasourceRequest: jest.fn() };
+
+    const templateSrvMock = {
+      getAdhocFilters: () => [],
+      replace: a => a,
+    };
+
+    const range = { from: 'now-6h', to: 'now' };
+
+    test('should use default max lines when no limit given', () => {
+      const ds = new LokiDatasource(instanceSettings, backendSrvMock, templateSrvMock);
+      backendSrvMock.datasourceRequest = jest.fn();
+      ds.query({ range, targets: [{ expr: 'foo' }] });
+      expect(backendSrvMock.datasourceRequest.mock.calls.length).toBe(1);
+      expect(backendSrvMock.datasourceRequest.mock.calls[0][0].url).toContain('limit=1000');
+    });
+
+    test('should use custom max lines if limit is set', () => {
+      const customData = { ...(instanceSettings.jsonData || {}), maxLines: 20 };
+      const customSettings = { ...instanceSettings, jsonData: customData };
+      const ds = new LokiDatasource(customSettings, backendSrvMock, templateSrvMock);
+      backendSrvMock.datasourceRequest = jest.fn();
+      ds.query({ range, targets: [{ expr: 'foo' }] });
+      expect(backendSrvMock.datasourceRequest.mock.calls.length).toBe(1);
+      expect(backendSrvMock.datasourceRequest.mock.calls[0][0].url).toContain('limit=20');
+    });
+  });
+
   describe('when performing testDataSource', () => {
     let ds;
     let result;

public/app/plugins/datasource/loki/datasource.ts

@@ -9,11 +9,11 @@ import LanguageProvider from './language_provider';
 import { mergeStreamsToLogs } from './result_transformer';
 import { formatQuery, parseQuery } from './query_utils';
 
-export const DEFAULT_LIMIT = 1000;
+export const DEFAULT_MAX_LINES = 1000;
 
 const DEFAULT_QUERY_PARAMS = {
   direction: 'BACKWARD',
-  limit: DEFAULT_LIMIT,
+  limit: DEFAULT_MAX_LINES,
   regexp: '',
   query: '',
 };
@@ -29,10 +29,13 @@ function serializeParams(data: any) {
 
 export default class LokiDatasource {
   languageProvider: LanguageProvider;
+  maxLines: number;
 
   /** @ngInject */
   constructor(private instanceSettings, private backendSrv, private templateSrv) {
     this.languageProvider = new LanguageProvider(this);
+    const settingsData = instanceSettings.jsonData || {};
+    this.maxLines = parseInt(settingsData.maxLines, 10) || DEFAULT_MAX_LINES;
   }
 
   _request(apiUrl: string, data?, options?: any) {
@@ -47,7 +50,7 @@ export default class LokiDatasource {
   }
 
   mergeStreams(streams: LogsStream[], intervalMs: number): LogsModel {
-    const logs = mergeStreamsToLogs(streams);
+    const logs = mergeStreamsToLogs(streams, this.maxLines);
     logs.series = makeSeriesForLogs(logs.rows, intervalMs);
     return logs;
   }
@@ -61,6 +64,7 @@ export default class LokiDatasource {
       ...parseQuery(interpolated),
       start,
       end,
+      limit: this.maxLines,
     };
   }
 
@@ -77,6 +81,9 @@ export default class LokiDatasource {
     return Promise.all(queries).then((results: any[]) => {
       // Flatten streams from multiple queries
       const allStreams: LogsStream[] = results.reduce((acc, response, i) => {
+        if (!response) {
+          return acc;
+        }
         const streams: LogsStream[] = response.data.streams || [];
         // Inject search for match highlighting
         const search: string = queryTargets[i].regexp;

public/app/plugins/datasource/loki/partials/config.html

 <datasource-http-settings current="ctrl.current" no-direct-access="true">
 </datasource-http-settings>
+
+<div class="gf-form-group">
+  <div class="gf-form-inline">
+    <div class="gf-form">
+      <span class="gf-form-label width-8">Maximum lines</span>
+      <input type="text" class="gf-form-input width-8" ng-model="ctrl.current.jsonData.maxLines" spellcheck='false' placeholder="1000"></input>
+      <info-popover mode="right-absolute">
+        Loki queries must contain a limit of the maximum number of lines returned (default: 1000).
+        Increase this limit to have a bigger result set for ad-hoc analysis.
+        Decrease this limit if your browser becomes sluggish when displaying the log results.
+      </info-popover>
+    </div>
+  </div>
+</div>

public/app/plugins/datasource/loki/result_transformer.ts

@@ -11,7 +11,7 @@ import {
   LogsStreamLabels,
   LogsMetaKind,
 } from 'app/core/logs_model';
-import { DEFAULT_LIMIT } from './datasource';
+import { DEFAULT_MAX_LINES } from './datasource';
 
 /**
  * Returns the log level of a log line.
@@ -140,7 +140,7 @@ export function processEntry(
   };
 }
 
-export function mergeStreamsToLogs(streams: LogsStream[], limit = DEFAULT_LIMIT): LogsModel {
+export function mergeStreamsToLogs(streams: LogsStream[], limit = DEFAULT_MAX_LINES): LogsModel {
   // Unique model identifier
   const id = streams.map(stream => stream.labels).join();