Logging: Log frontend errors (#28073)

* basic frontend  Sentry integration

* backend endpoint to capture sentry events

* WIP!

* log user email for frontend logs

* remove debug logging

* lint fixes

* Fix type exports & property names

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

* additional struct naming fix

* rename log endpoint, config section & interface

* add sentry sample rate to config

* refac to use EchoSrv

* log user id

* backend tests

* tests for SentryEchoBackend

* sentry echo backend tests

* CustomEndpointTransport tests

* Update pkg/api/frontend_logging_test.go

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

* Update conf/defaults.ini

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

* Update pkg/api/frontend_logging_test.go

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

* don't export unnecesasrily

* update go.sum

* get rid of Convey in tests, use stdlib

* add sentry config to sample.ini

* cleanup to set orig logging handler in test

* Apply suggestions from code review

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

* PR feedback changes

* lock sentry version

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

conf/defaults.ini

@@ -560,6 +560,19 @@ facility =
 # Syslog tag. By default, the process' argv[0] is used.
 tag =
 
+[log.frontend]
+# Should Sentry be initialized
+enabled = false
+
+# Sentry DSN if you wanna send events to Sentry. In this case, set custom_endpoint to empty
+sentry_dsn =
+
+# Custom endpoint to send Sentry events to. If this is configured, DSN will be ignored and events push to this endpoint. Default endpoint will log frontend errors to stdout.
+custom_endpoint = /log
+
+# Rate of events to be reported between 0 (none) and 1 (all), float
+sample_rate = 1.0
+
 #################################### Usage Quotas ########################
 [quota]
 enabled = false

conf/sample.ini

@@ -551,6 +551,19 @@
 # Syslog tag. By default, the process' argv[0] is used.
 ;tag =
 
+[log.frontend]
+# Should Sentry be initialized
+;enabled = false
+
+# Sentry DSN if you wanna send events to Sentry. In this case, set custom_endpoint to empty
+;sentry_dsn =
+
+# Custom endpoint to send Sentry events to. If this is configured, DSN will be ignored and events push to this endpoint. Default endpoint will log frontend errors to stdout.
+;custom_endpoint = /log
+
+# Rate of events to be reported between 0 (none) and 1 (all), float
+;sample_rate = 1.0
+
 #################################### Usage Quotas ########################
 [quota]
 ; enabled = false

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
 	github.com/fatih/color v1.9.0
 	github.com/gchaincl/sqlhooks v1.3.0
+	github.com/getsentry/sentry-go v0.7.0
 	github.com/go-macaron/binding v0.0.0-20190806013118-0b4f37bab25b
 	github.com/go-macaron/gzip v0.0.0-20160222043647-cad1c6580a07
 	github.com/go-sql-driver/mysql v1.5.0
@@ -53,7 +54,6 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0
 	github.com/jonboulle/clockwork v0.2.1 // indirect
 	github.com/jung-kurt/gofpdf v1.10.1
-	github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
 	github.com/lib/pq v1.3.0
 	github.com/linkedin/goavro/v2 v2.9.7
 	github.com/magefile/mage v1.9.0
@@ -80,8 +80,6 @@ require (
 	github.com/urfave/cli/v2 v2.1.1
 	github.com/xorcare/pointer v1.1.0
 	github.com/yudai/gojsondiff v1.0.0
-	github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
-	github.com/yudai/pp v2.0.1+incompatible // indirect
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
 	golang.org/x/net v0.0.0-20201022231255-08b38378de70
 	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43

package.json

@@ -205,6 +205,9 @@
   "dependencies": {
     "@grafana/slate-react": "0.22.9-grafana",
     "@reduxjs/toolkit": "1.3.4",
+    "@sentry/browser": "5.25.0",
+    "@sentry/types": "5.24.2",
+    "@sentry/utils": "5.24.2",
     "@torkelo/react-select": "3.0.8",
     "@types/antlr4": "^4.7.1",
     "@types/braintree__sanitize-url": "4.0.0",

packages/grafana-data/src/types/config.ts

@@ -60,6 +60,18 @@ export interface LicenseInfo {
 }
 
 /**
+ * Describes Sentry integration config
+ *
+ * @public
+ */
+export interface SentryConfig {
+  enabled: boolean;
+  dsn: string;
+  customEndpoint: string;
+  sampleRate: number;
+}
+
+/**
  * Describes all the different Grafana configuration values available for an instance.
  *
  * @public
@@ -105,4 +117,5 @@ export interface GrafanaConfig {
   licenseInfo: LicenseInfo;
   http2Enabled: boolean;
   dateFormats?: SystemDateFormatSettings;
+  sentry: SentryConfig;
 }

packages/grafana-runtime/src/config.ts

@@ -62,6 +62,12 @@ export class GrafanaBootConfig implements GrafanaConfig {
   rendererAvailable = false;
   http2Enabled = false;
   dateFormats?: SystemDateFormatSettings;
+  sentry = {
+    enabled: false,
+    dsn: '',
+    customEndpoint: '',
+    sampleRate: 1,
+  };
   marketplaceUrl?: string;
 
   constructor(options: GrafanaBootConfig) {

packages/grafana-runtime/src/services/EchoSrv.ts

@@ -78,6 +78,7 @@ export interface EchoEvent<T extends EchoEventType = any, P = any> {
 export enum EchoEventType {
   Performance = 'performance',
   MetaAnalytics = 'meta-analytics',
+  Sentry = 'sentry',
 }
 
 /**

pkg/api/api.go

@@ -445,4 +445,7 @@ func (hs *HTTPServer) registerRoutes() {
 	r.Get("/api/snapshots/:key", Wrap(GetDashboardSnapshot))
 	r.Get("/api/snapshots-delete/:deleteKey", reqSnapshotPublicModeOrSignedIn, Wrap(DeleteDashboardSnapshotByDeleteKey))
 	r.Delete("/api/snapshots/:key", reqEditorRole, Wrap(DeleteDashboardSnapshot))
+
+	// Frontend logs
+	r.Post("/log", bind(frontendSentryEvent{}), Wrap(hs.logFrontendMessage))
 }

pkg/api/dtos/index.go

 package dtos
 
+import "github.com/grafana/grafana/pkg/setting"
+
 type IndexViewData struct {
 	User                    *CurrentUser
 	Settings                map[string]interface{}
@@ -18,6 +20,7 @@ type IndexViewData struct {
 	FavIcon                 string
 	AppleTouchIcon          string
 	AppTitle                string
+	Sentry                  *setting.Sentry
 }
 
 type PluginCss struct {

pkg/api/frontend_logging.go

+package api
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/getsentry/sentry-go"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/inconshreveable/log15"
+)
+
+var frontendLogger = log.New("frontend")
+
+type frontendSentryExceptionValue struct {
+	Value      string            `json:"value,omitempty"`
+	Type       string            `json:"type,omitempty"`
+	Stacktrace sentry.Stacktrace `json:"stacktrace,omitempty"`
+}
+
+type frontendSentryException struct {
+	Values []frontendSentryExceptionValue `json:"values,omitempty"`
+}
+
+type frontendSentryEvent struct {
+	*sentry.Event
+	Exception *frontendSentryException `json:"exception,omitempty"`
+}
+
+func (value *frontendSentryExceptionValue) FmtMessage() string {
+	return fmt.Sprintf("%s: %s", value.Type, value.Value)
+}
+
+func (value *frontendSentryExceptionValue) FmtStacktrace() string {
+	var stacktrace = value.FmtMessage()
+	for _, frame := range value.Stacktrace.Frames {
+		stacktrace += fmt.Sprintf("\n  at %s (%s:%v:%v)", frame.Function, frame.Filename, frame.Lineno, frame.Colno)
+	}
+	return stacktrace
+}
+
+func (exception *frontendSentryException) FmtStacktraces() string {
+	var stacktraces []string
+	for _, value := range exception.Values {
+		stacktraces = append(stacktraces, value.FmtStacktrace())
+	}
+	return strings.Join(stacktraces, "\n\n")
+}
+
+func (event *frontendSentryEvent) ToLogContext() log15.Ctx {
+	var ctx = make(log15.Ctx)
+	ctx["url"] = event.Request.URL
+	ctx["user_agent"] = event.Request.Headers["User-Agent"]
+	ctx["event_id"] = event.EventID
+	ctx["original_timestamp"] = event.Timestamp
+	if event.Exception != nil {
+		ctx["stacktrace"] = event.Exception.FmtStacktraces()
+	}
+	if len(event.User.Email) > 0 {
+		ctx["user_email"] = event.User.Email
+		ctx["user_id"] = event.User.ID
+	}
+
+	return ctx
+}
+
+func (hs *HTTPServer) logFrontendMessage(c *models.ReqContext, event frontendSentryEvent) Response {
+	var msg = "unknown"
+
+	if len(event.Message) > 0 {
+		msg = event.Message
+	} else if event.Exception != nil && len(event.Exception.Values) > 0 {
+		msg = event.Exception.Values[0].FmtMessage()
+	}
+
+	var ctx = event.ToLogContext()
+
+	switch event.Level {
+	case sentry.LevelError:
+		frontendLogger.Error(msg, ctx)
+	case sentry.LevelWarning:
+		frontendLogger.Warn(msg, ctx)
+	case sentry.LevelDebug:
+		frontendLogger.Debug(msg, ctx)
+	default:
+		frontendLogger.Info(msg, ctx)
+	}
+
+	return Success("ok")
+}

pkg/api/frontend_logging_test.go

+package api
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/getsentry/sentry-go"
+	"github.com/grafana/grafana/pkg/models"
+	log "github.com/inconshreveable/log15"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type logScenarioFunc func(c *scenarioContext, logs []*log.Record)
+
+func logSentryEventScenario(t *testing.T, desc string, event frontendSentryEvent, fn logScenarioFunc) {
+	t.Run(desc, func(t *testing.T) {
+		logs := []*log.Record{}
+		origHandler := frontendLogger.GetHandler()
+		frontendLogger.SetHandler(log.FuncHandler(func(r *log.Record) error {
+			logs = append(logs, r)
+			return nil
+		}))
+		t.Cleanup(func() {
+			frontendLogger.SetHandler(origHandler)
+		})
+
+		sc := setupScenarioContext("/log")
+		hs := HTTPServer{}
+
+		handler := Wrap(func(w http.ResponseWriter, c *models.ReqContext) Response {
+			sc.context = c
+			return hs.logFrontendMessage(c, event)
+		})
+
+		sc.m.Post(sc.url, handler)
+		sc.fakeReqWithParams("POST", sc.url, map[string]string{}).exec()
+		fn(sc, logs)
+	})
+}
+
+func TestFrontendLoggingEndpoint(t *testing.T) {
+	ts, err := time.Parse("2006-01-02T15:04:05.000Z", "2020-10-22T06:29:29.078Z")
+	require.NoError(t, err)
+
+	t.Run("FrontendLoggingEndpoint", func(t *testing.T) {
+		request := sentry.Request{
+			URL: "http://localhost:3000/",
+			Headers: map[string]string{
+				"User-Agent": "Chrome",
+			},
+		}
+
+		user := sentry.User{
+			Email: "geralt@kaermorhen.com",
+			ID:    "45",
+		}
+
+		errorEvent := frontendSentryEvent{
+			&sentry.Event{
+				EventID:   "123",
+				Level:     sentry.LevelError,
+				Request:   &request,
+				Timestamp: ts,
+			},
+			&frontendSentryException{
+				Values: []frontendSentryExceptionValue{
+					{
+						Type:  "UserError",
+						Value: "Please replace user and try again",
+						Stacktrace: sentry.Stacktrace{
+							Frames: []sentry.Frame{
+								{
+									Function: "foofn",
+									Filename: "foo.js",
+									Lineno:   123,
+									Colno:    23,
+								},
+								{
+									Function: "barfn",
+									Filename: "bar.js",
+									Lineno:   113,
+									Colno:    231,
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+
+		logSentryEventScenario(t, "Should log received error event", errorEvent, func(sc *scenarioContext, logs []*log.Record) {
+			assert.Equal(t, 200, sc.resp.Code)
+			assert.Len(t, logs, 1)
+			assertContextContains(t, logs[0], "logger", "frontend")
+			assertContextContains(t, logs[0], "url", errorEvent.Request.URL)
+			assertContextContains(t, logs[0], "user_agent", errorEvent.Request.Headers["User-Agent"])
+			assertContextContains(t, logs[0], "event_id", errorEvent.EventID)
+			assertContextContains(t, logs[0], "original_timestamp", errorEvent.Timestamp)
+			assertContextContains(t, logs[0], "stacktrace", `UserError: Please replace user and try again
+  at foofn (foo.js:123:23)
+  at barfn (bar.js:113:231)`)
+		})
+
+		messageEvent := frontendSentryEvent{
+			&sentry.Event{
+				EventID:   "123",
+				Level:     sentry.LevelInfo,
+				Request:   &request,
+				Timestamp: ts,
+				Message:   "hello world",
+				User:      user,
+			},
+			nil,
+		}
+
+		logSentryEventScenario(t, "Should log received message event", messageEvent, func(sc *scenarioContext, logs []*log.Record) {
+			assert.Equal(t, 200, sc.resp.Code)
+			assert.Len(t, logs, 1)
+			assert.Equal(t, "hello world", logs[0].Msg)
+			assert.Equal(t, log.LvlInfo, logs[0].Lvl)
+			assertContextContains(t, logs[0], "logger", "frontend")
+			assertContextContains(t, logs[0], "url", messageEvent.Request.URL)
+			assertContextContains(t, logs[0], "user_agent", messageEvent.Request.Headers["User-Agent"])
+			assertContextContains(t, logs[0], "event_id", messageEvent.EventID)
+			assertContextContains(t, logs[0], "original_timestamp", messageEvent.Timestamp)
+			assert.NotContains(t, logs[0].Ctx, "stacktrace")
+			assertContextContains(t, logs[0], "user_email", user.Email)
+			assertContextContains(t, logs[0], "user_id", user.ID)
+		})
+	})
+}
+
+func indexOf(arr []interface{}, item string) int {
+	for i, elem := range arr {
+		if elem == item {
+			return i
+		}
+	}
+	return -1
+}
+
+func assertContextContains(t *testing.T, logRecord *log.Record, label string, value interface{}) {
+	assert.Contains(t, logRecord.Ctx, label)
+	labelIdx := indexOf(logRecord.Ctx, label)
+	assert.Equal(t, value, logRecord.Ctx[labelIdx+1])
+}

pkg/api/frontendsettings.go

@@ -240,6 +240,7 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *models.ReqContext) (map[string]i
 		"featureToggles":    hs.Cfg.FeatureToggles,
 		"rendererAvailable": hs.RenderService.IsAvailable(),
 		"http2Enabled":      hs.Cfg.Protocol == setting.HTTP2Scheme,
+		"sentry":            hs.Cfg.Sentry,
 		"marketplaceUrl":    hs.Cfg.MarketplaceURL,
 	}
 

pkg/api/index.go

@@ -403,6 +403,7 @@ func (hs *HTTPServer) setIndexViewData(c *models.ReqContext) (*dtos.IndexViewDat
 		AppleTouchIcon:          "public/img/apple-touch-icon.png",
 		AppTitle:                "Grafana",
 		NavTree:                 navTree,
+		Sentry:                  &hs.Cfg.Sentry,
 	}
 
 	if setting.DisableGravatar {

pkg/setting/setting.go

@@ -322,6 +322,9 @@ type Cfg struct {
 	AlertingAnnotationCleanupSetting   AnnotationCleanupSettings
 	DashboardAnnotationCleanupSettings AnnotationCleanupSettings
 	APIAnnotationCleanupSettings       AnnotationCleanupSettings
+
+	// Sentry config
+	Sentry Sentry
 }
 
 // IsExpressionsEnabled returns whether the expressions feature is enabled.
@@ -846,6 +849,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
 	}
 
 	cfg.readDateFormats()
+	cfg.readSentryConfig()
 
 	return nil
 }

pkg/setting/setting_sentry.go

+package setting
+
+type Sentry struct {
+	Enabled        bool    `json:"enabled"`
+	DSN            string  `json:"dsn"`
+	CustomEndpoint string  `json:"customEndpoint"`
+	SampleRate     float64 `json:"sampleRate"`
+}
+
+func (cfg *Cfg) readSentryConfig() {
+	raw := cfg.Raw.Section("log.frontend")
+	cfg.Sentry = Sentry{
+		Enabled:        raw.Key("enabled").MustBool(true),
+		DSN:            raw.Key("sentry_dsn").String(),
+		CustomEndpoint: raw.Key("custom_endpoint").String(),
+		SampleRate:     raw.Key("sample_rate").MustFloat64(),
+	}
+}

public/app/app.ts

@@ -49,6 +49,7 @@ import { getStandardFieldConfigs, getStandardOptionEditors, getScrollbarWidth }
 import { getDefaultVariableAdapters, variableAdapters } from './features/variables/adapters';
 import { initDevFeatures } from './dev';
 import { getStandardTransformers } from 'app/core/utils/standardTransformers';
+import { SentryEchoBackend } from './core/services/echo/backends/sentry/SentryBackend';
 import { monkeyPatchInjectorWithPreAssignedBindings } from './core/injectorMonkeyPatch';
 
 // add move to lodash for backward compatabiltiy
@@ -205,6 +206,13 @@ export class GrafanaApp {
     });
 
     registerEchoBackend(new PerformanceBackend({}));
+    registerEchoBackend(
+      new SentryEchoBackend({
+        ...config.sentry,
+        user: config.bootData.user,
+        buildInfo: config.buildInfo,
+      })
+    );
 
     window.addEventListener('DOMContentLoaded', () => {
       reportPerformance('dcl', Math.round(performance.now()));

public/app/core/services/echo/EchoSrv.ts

 import { getEchoSrv, EchoEventType } from '@grafana/runtime';
+import { captureException } from '@sentry/browser';
 import { PerformanceEvent } from './backends/PerformanceBackend';
 
 export const reportPerformance = (metric: string, value: number) => {
@@ -10,3 +11,7 @@ export const reportPerformance = (metric: string, value: number) => {
     },
   });
 };
+
+// Sentry will process the error, adding it's own metadata, applying any sampling rules,
+// then push it to EchoSrv as SentryEvent
+export const reportError = (error: Error) => captureException(error);

public/app/core/services/echo/backends/sentry/SentryBackend.test.ts

+import { init as initSentry, setUser as sentrySetUser, Event as SentryEvent } from '@sentry/browser';
+import { SentryEchoBackend, SentryEchoBackendOptions } from './SentryBackend';
+import { BuildInfo } from '@grafana/data';
+import { FetchTransport } from '@sentry/browser/dist/transports';
+import { CustomEndpointTransport } from './transports/CustomEndpointTransport';
+import { EchoSrvTransport } from './transports/EchoSrvTransport';
+import { SentryEchoEvent } from './types';
+import { EchoBackend, EchoEventType, EchoMeta, setEchoSrv } from '@grafana/runtime';
+import { waitFor } from '@testing-library/react';
+import { Echo } from '../../Echo';
+
+jest.mock('@sentry/browser');
+
+describe('SentryEchoBackend', () => {
+  beforeEach(() => jest.resetAllMocks());
+
+  const buildInfo: BuildInfo = {
+    version: '1.0',
+    commit: 'abcd123',
+    isEnterprise: false,
+    env: 'production',
+    edition: "Director's cut",
+    latestVersion: 'ba',
+    hasUpdate: false,
+    hideVersion: false,
+  };
+
+  const options: SentryEchoBackendOptions = {
+    enabled: true,
+    buildInfo,
+    dsn: 'https://examplePublicKey@o0.ingest.testsentry.io/0',
+    sampleRate: 1,
+    customEndpoint: '',
+    user: {
+      email: 'darth.vader@sith.glx',
+      id: 504,
+    },
+  };
+
+  it('will set up sentry`s FetchTransport if DSN is provided', async () => {
+    const backend = new SentryEchoBackend(options);
+    expect(backend.transports.length).toEqual(1);
+    expect(backend.transports[0]).toBeInstanceOf(FetchTransport);
+    expect((backend.transports[0] as FetchTransport).options.dsn).toEqual(options.dsn);
+  });
+
+  it('will set up custom endpoint transport if custom endpoint is provided', async () => {
+    const backend = new SentryEchoBackend({
+      ...options,
+      dsn: '',
+      customEndpoint: '/log',
+    });
+    expect(backend.transports.length).toEqual(1);
+    expect(backend.transports[0]).toBeInstanceOf(CustomEndpointTransport);
+    expect((backend.transports[0] as CustomEndpointTransport).options.endpoint).toEqual('/log');
+  });
+
+  it('will initialize sentry and set user', async () => {
+    new SentryEchoBackend(options);
+    expect(initSentry).toHaveBeenCalledTimes(1);
+    expect(initSentry).toHaveBeenCalledWith({
+      release: buildInfo.version,
+      environment: buildInfo.env,
+      dsn: options.dsn,
+      sampleRate: options.sampleRate,
+      transport: EchoSrvTransport,
+    });
+    expect(sentrySetUser).toHaveBeenCalledWith({
+      email: options.user?.email,
+      id: String(options.user?.id),
+    });
+  });
+
+  it('will forward events to transports', async () => {
+    const backend = new SentryEchoBackend(options);
+    backend.transports = [{ sendEvent: jest.fn() }, { sendEvent: jest.fn() }];
+    const event: SentryEchoEvent = {
+      type: EchoEventType.Sentry,
+      payload: ({ foo: 'bar' } as unknown) as SentryEvent,
+      meta: ({} as unknown) as EchoMeta,
+    };
+    backend.addEvent(event);
+    backend.transports.forEach(transport => {
+      expect(transport.sendEvent).toHaveBeenCalledTimes(1);
+      expect(transport.sendEvent).toHaveBeenCalledWith(event.payload);
+    });
+  });
+
+  it('integration test with EchoSrv, Sentry and CustomFetchTransport', async () => {
+    // sets up the whole thing between window.onerror and backend endpoint call, checks that error is reported
+
+    // use actual sentry & mock window.fetch
+    const sentry = jest.requireActual('@sentry/browser');
+    (initSentry as jest.Mock).mockImplementation(sentry.init);
+    (sentrySetUser as jest.Mock).mockImplementation(sentry.setUser);
+    const fetchSpy = (window.fetch = jest.fn());
+    fetchSpy.mockResolvedValue({ status: 200 } as Response);
+
+    // set up echo srv & sentry backend
+    const echo = new Echo({ debug: true });
+    setEchoSrv(echo);
+    const sentryBackend = new SentryEchoBackend({
+      ...options,
+      dsn: '',
+      customEndpoint: '/log',
+    });
+    echo.addBackend(sentryBackend);
+
+    // lets add another echo backend for sentry events for good measure
+    const myCustomErrorBackend: EchoBackend = {
+      supportedEvents: [EchoEventType.Sentry],
+      flush: () => {},
+      options: {},
+      addEvent: jest.fn(),
+    };
+    echo.addBackend(myCustomErrorBackend);
+
+    // fire off an error using global error handler, Sentry should pick it up
+    const error = new Error('test error');
+    window.onerror!(error.message, undefined, undefined, undefined, error);
+
+    // check that error was reported to backend
+    await waitFor(() => expect(fetchSpy).toHaveBeenCalledTimes(1));
+    const [url, reqInit]: [string, RequestInit] = fetchSpy.mock.calls[0];
+    expect(url).toEqual('/log');
+    expect((JSON.parse(reqInit.body as string) as SentryEvent).exception!.values![0].value).toEqual('test error');
+
+    // check that our custom backend got it too
+    expect(myCustomErrorBackend.addEvent).toHaveBeenCalledTimes(1);
+  });
+});

public/app/core/services/echo/backends/sentry/SentryBackend.ts

+import { EchoBackend, EchoEventType } from '@grafana/runtime';
+import { SentryConfig } from '@grafana/data/src/types/config';
+import { BrowserOptions, init as initSentry, setUser as sentrySetUser } from '@sentry/browser';
+import { FetchTransport } from '@sentry/browser/dist/transports';
+import { CustomEndpointTransport } from './transports/CustomEndpointTransport';
+import { EchoSrvTransport } from './transports/EchoSrvTransport';
+import { BuildInfo } from '@grafana/data';
+import { SentryEchoEvent, User, BaseTransport } from './types';
+
+export interface SentryEchoBackendOptions extends SentryConfig {
+  user?: User;
+  buildInfo: BuildInfo;
+}
+
+export class SentryEchoBackend implements EchoBackend<SentryEchoEvent, SentryEchoBackendOptions> {
+  supportedEvents = [EchoEventType.Sentry];
+
+  transports: BaseTransport[];
+
+  constructor(public options: SentryEchoBackendOptions) {
+    // set up transports to post events to grafana backend and/or Sentry
+    this.transports = [];
+    if (options.dsn) {
+      this.transports.push(new FetchTransport({ dsn: options.dsn }));
+    }
+    if (options.customEndpoint) {
+      this.transports.push(new CustomEndpointTransport({ endpoint: options.customEndpoint }));
+    }
+
+    // initialize Sentry so it can set up it's hooks and start collecting errors
+    const sentryOptions: BrowserOptions = {
+      release: options.buildInfo.version,
+      environment: options.buildInfo.env,
+      // seems Sentry won't attempt to send events to transport unless a valid DSN is defined :shrug:
+      dsn: options.dsn || 'https://examplePublicKey@o0.ingest.sentry.io/0',
+      sampleRate: options.sampleRate,
+      transport: EchoSrvTransport, // will dump errors to EchoSrv
+    };
+
+    if (options.user) {
+      sentrySetUser({
+        email: options.user.email,
+        id: String(options.user.id),
+      });
+    }
+
+    initSentry(sentryOptions);
+  }
+
+  addEvent = (e: SentryEchoEvent) => {
+    this.transports.forEach(t => t.sendEvent(e.payload));
+  };
+
+  // backend will log events to stdout, and at least in case of hosted grafana they will be
+  // ingested into Loki. Due to Loki limitations logs cannot be backdated,
+  // so not using buffering for this backend to make sure that events are logged as close
+  // to their context as possible
+  flush = () => {};
+}

public/app/core/services/echo/backends/sentry/transports/CustomEndpointTransport.test.ts

+import { Event, Severity } from '@sentry/browser';
+import { CustomEndpointTransport } from './CustomEndpointTransport';
+
+describe('CustomEndpointTransport', () => {
+  const fetchSpy = (window.fetch = jest.fn());
+  beforeEach(() => jest.resetAllMocks());
+  const now = new Date();
+
+  const event: Event = {
+    level: Severity.Error,
+    breadcrumbs: [],
+    exception: {
+      values: [
+        {
+          type: 'SomeError',
+          value: 'foo',
+        },
+      ],
+    },
+    timestamp: now.getTime() / 1000,
+  };
+
+  it('will send received event to backend using window.fetch', async () => {
+    fetchSpy.mockResolvedValue({ status: 200 } as Response);
+    const transport = new CustomEndpointTransport({ endpoint: '/log' });
+    await transport.sendEvent(event);
+    expect(fetchSpy).toHaveBeenCalledTimes(1);
+    const [url, reqInit]: [string, RequestInit] = fetchSpy.mock.calls[0];
+    expect(url).toEqual('/log');
+    expect(reqInit.method).toEqual('POST');
+    expect(reqInit.headers).toEqual({
+      'Content-Type': 'application/json',
+    });
+    expect(JSON.parse(reqInit.body as string)).toEqual({
+      ...event,
+      timestamp: now.toISOString(),
+    });
+  });
+
+  it('will back off if backend returns Retry-After', async () => {
+    const rateLimiterResponse = {
+      status: 429,
+      ok: false,
+      headers: (new Headers({
+        'Retry-After': '1', // 1 second
+      }) as any) as Headers,
+    } as Response;
+    fetchSpy.mockResolvedValueOnce(rateLimiterResponse).mockResolvedValueOnce({ status: 200 } as Response);
+    const transport = new CustomEndpointTransport({ endpoint: '/log' });
+
+    // first call - backend is called, rejected because of 429
+    await expect(transport.sendEvent(event)).rejects.toEqual(rateLimiterResponse);
+    expect(fetchSpy).toHaveBeenCalledTimes(1);
+
+    // second immediate call - shot circuited because retry-after time has not expired, backend not called
+    await expect(transport.sendEvent(event)).rejects.toBeTruthy();
+    expect(fetchSpy).toHaveBeenCalledTimes(1);
+
+    // wait out the retry-after and call again - great success
+    await new Promise(resolve => setTimeout(() => resolve(), 1001));
+    await expect(transport.sendEvent(event)).resolves.toBeTruthy();
+    expect(fetchSpy).toHaveBeenCalledTimes(2);
+  });
+});

public/app/core/services/echo/backends/sentry/transports/CustomEndpointTransport.ts

+import { Event, Severity } from '@sentry/browser';
+import { logger, parseRetryAfterHeader, PromiseBuffer, supportsReferrerPolicy, SyncPromise } from '@sentry/utils';
+import { Response, Status } from '@sentry/types';
+import { BaseTransport } from '../types';
+
+export interface CustomEndpointTransportOptions {
+  endpoint: string;
+  fetchParameters?: Partial<RequestInit>;
+}
+
+/**
+ * This is a copy of sentry's FetchTransport, edited to be able to push to any custom url
+ * instead of using Sentry-specific endpoint logic.
+ * Also transofrms some of the payload values to be parseable by go.
+ * Sends events sequanetially and implements back-off in case of rate limiting.
+ */
+
+export class CustomEndpointTransport implements BaseTransport {
+  /** Locks transport after receiving 429 response */
+  private _disabledUntil: Date = new Date(Date.now());
+
+  private readonly _buffer: PromiseBuffer<Response> = new PromiseBuffer(30);
+
+  constructor(public options: CustomEndpointTransportOptions) {}
+
+  sendEvent(event: Event): PromiseLike<Response> {
+    if (new Date(Date.now()) < this._disabledUntil) {
+      return Promise.reject({
+        event,
+        reason: `Transport locked till ${this._disabledUntil} due to too many requests.`,
+        status: 429,
+      });
+    }
+
+    const sentryReq = {
+      // convert all timestamps to iso string, so it's parseable by backend
+      body: JSON.stringify({
+        ...event,
+        level: event.level ?? (event.exception ? Severity.Error : Severity.Info),
+        exception: event.exception
+          ? {
+              values: event.exception.values?.map(value => ({
+                ...value,
+                // according to both typescript and go types, value is supposed to be string.
+                // but in some odd cases at runtime it turns out to be an empty object {}
+                // let's fix it here
+                value: fmtSentryErrorValue(value.value),
+              })),
+            }
+          : event.exception,
+        breadcrumbs: event.breadcrumbs?.map(breadcrumb => ({
+          ...breadcrumb,
+          timestamp: makeTimestamp(breadcrumb.timestamp),
+        })),
+        timestamp: makeTimestamp(event.timestamp),
+      }),
+      url: this.options.endpoint,
+    };
+
+    const options: RequestInit = {
+      body: sentryReq.body,
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      method: 'POST',
+      // Despite all stars in the sky saying that Edge supports old draft syntax, aka 'never', 'always', 'origin' and 'default
+      // https://caniuse.com/#feat=referrer-policy
+      // It doesn't. And it throw exception instead of ignoring this parameter...
+      // REF: https://github.com/getsentry/raven-js/issues/1233
+      referrerPolicy: (supportsReferrerPolicy() ? 'origin' : '') as ReferrerPolicy,
+    };
+
+    if (this.options.fetchParameters !== undefined) {
+      Object.assign(options, this.options.fetchParameters);
+    }
+
+    return this._buffer.add(
+      new SyncPromise<Response>((resolve, reject) => {
+        window
+          .fetch(sentryReq.url, options)
+          .then(response => {
+            const status = Status.fromHttpCode(response.status);
+
+            if (status === Status.Success) {
+              resolve({ status });
+              return;
+            }
+
+            if (status === Status.RateLimit) {
+              const now = Date.now();
+              const retryAfterHeader = response.headers.get('Retry-After');
+              this._disabledUntil = new Date(now + parseRetryAfterHeader(now, retryAfterHeader));
+              logger.warn(`Too many requests, backing off till: ${this._disabledUntil}`);
+            }
+
+            reject(response);
+          })
+          .catch(reject);
+      })
+    );
+  }
+}
+
+function makeTimestamp(time: number | undefined): string {
+  if (time) {
+    return new Date(time * 1000).toISOString();
+  }
+  return new Date().toISOString();
+}
+
+function fmtSentryErrorValue(value: unknown): string | undefined {
+  if (typeof value === 'string' || value === undefined) {
+    return value;
+  } else if (value && typeof value === 'object' && Object.keys(value).length === 0) {
+    return '';
+  }
+  return String(value);
+}

public/app/core/services/echo/backends/sentry/transports/EchoSrvTransport.ts

+import { getEchoSrv, EchoEventType } from '@grafana/runtime';
+import { BaseTransport } from '@sentry/browser/dist/transports';
+import { Event } from '@sentry/browser';
+import { Status } from '@sentry/types';
+
+export class EchoSrvTransport extends BaseTransport {
+  sendEvent(event: Event) {
+    getEchoSrv().addEvent({
+      type: EchoEventType.Sentry,
+      payload: event,
+    });
+    return Promise.resolve({ status: Status.Success, event });
+  }
+}

public/app/core/services/echo/backends/sentry/types.ts

+import { EchoEvent, EchoEventType } from '@grafana/runtime';
+import { Event as SentryEvent } from '@sentry/browser';
+import { Response } from '@sentry/types';
+
+export interface BaseTransport {
+  sendEvent(event: SentryEvent): PromiseLike<Response>;
+}
+
+export type SentryEchoEvent = EchoEvent<EchoEventType.Sentry, SentryEvent>;
+
+export interface User {
+  email: string;
+  id: number;
+}

yarn.lock

@@ -5166,6 +5166,71 @@
   dependencies:
     any-observable "^0.3.0"
 
+"@sentry/browser@5.25.0":
+  version "5.25.0"
+  resolved "https://registry.yarnpkg.com/@sentry/browser/-/browser-5.25.0.tgz#4e3d2132ba1f2e2b26f73c49cbb6977ee9c9fea9"
+  integrity sha512-QDVUbUuTu58xCdId0eUO4YzpvrPdoUw1ryVy/Yep9Es/HD0fiSyO1Js0eQVkV/EdXtyo2pomc1Bpy7dbn2EJ2w==
+  dependencies:
+    "@sentry/core" "5.25.0"
+    "@sentry/types" "5.25.0"
+    "@sentry/utils" "5.25.0"
+    tslib "^1.9.3"
+
+"@sentry/core@5.25.0":
+  version "5.25.0"
+  resolved "https://registry.yarnpkg.com/@sentry/core/-/core-5.25.0.tgz#525ad37f9e8a95603768e3b74b437d5235a51578"
+  integrity sha512-hY6Zmo7t/RV+oZuvXHP6nyAj/QnZr2jW0e7EbL5YKMV8q0vlnjcE0LgqFXme726OJemoLk67z+sQOJic/Ztehg==
+  dependencies:
+    "@sentry/hub" "5.25.0"
+    "@sentry/minimal" "5.25.0"
+    "@sentry/types" "5.25.0"
+    "@sentry/utils" "5.25.0"
+    tslib "^1.9.3"
+
+"@sentry/hub@5.25.0":
+  version "5.25.0"
+  resolved "https://registry.yarnpkg.com/@sentry/hub/-/hub-5.25.0.tgz#6932535604cafaee1ac7f361b0e7c2ce8f7e7bc3"
+  integrity sha512-kOlOiJV8wMX50lYpzMlOXBoH7MNG0Ho4RTusdZnXZBaASq5/ljngDJkLr6uylNjceZQP21wzipCQajsJMYB7EQ==
+  dependencies:
+    "@sentry/types" "5.25.0"
+    "@sentry/utils" "5.25.0"
+    tslib "^1.9.3"
+
+"@sentry/minimal@5.25.0":
+  version "5.25.0"
+  resolved "https://registry.yarnpkg.com/@sentry/minimal/-/minimal-5.25.0.tgz#447b5406b45c8c436c461abea4474d6a849ed975"
+  integrity sha512-9JFKuW7U+1vPO86k3+XRtJyooiVZsVOsFFO4GulBzepi3a0ckNyPgyjUY1saLH+cEHx18hu8fGgajvI8ANUF2g==
+  dependencies:
+    "@sentry/hub" "5.25.0"
+    "@sentry/types" "5.25.0"
+    tslib "^1.9.3"
+
+"@sentry/types@5.24.2":
+  version "5.24.2"
+  resolved "https://registry.yarnpkg.com/@sentry/types/-/types-5.24.2.tgz#e2c25d1e75d8dbec5dbbd9a309a321425b61c2ca"
+  integrity sha512-HcOK00R0tQG5vzrIrqQ0jC28+z76jWSgQCzXiessJ5SH/9uc6NzdO7sR7K8vqMP2+nweCHckFohC8G0T1DLzuQ==
+
+"@sentry/types@5.25.0":
+  version "5.25.0"
+  resolved "https://registry.yarnpkg.com/@sentry/types/-/types-5.25.0.tgz#3bcf95e118d655d3f4e8bfa5f0be2e1fe4ea5307"
+  integrity sha512-8M4PREbcar+15wrtEqcwfcU33SS+2wBSIOd/NrJPXJPTYxi49VypCN1mZBDyWkaK+I+AuQwI3XlRPCfsId3D1A==
+
+"@sentry/utils@5.24.2":
+  version "5.24.2"
+  resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-5.24.2.tgz#90b7dff939bbbf4bb8edcac6aac2d04a0552af80"
+  integrity sha512-oPGde4tNEDHKk0Cg9q2p0qX649jLDUOwzJXHKpd0X65w3A6eJByDevMr8CSzKV9sesjrUpxqAv6f9WWlz185tA==
+  dependencies:
+    "@sentry/types" "5.24.2"
+    tslib "^1.9.3"
+
+"@sentry/utils@5.25.0":
+  version "5.25.0"
+  resolved "https://registry.yarnpkg.com/@sentry/utils/-/utils-5.25.0.tgz#b132034be66d7381d30879d2a9e09216fed28342"
+  integrity sha512-Hz5spdIkMSRH5NR1YFOp5qbsY5Ud2lKhEQWlqxcVThMG5YNUc10aYv5ijL19v0YkrC2rqPjCRm7GrVtzOc7bXQ==
+  dependencies:
+    "@sentry/types" "5.25.0"
+    tslib "^1.9.3"
+
 "@sinonjs/commons@^1", "@sinonjs/commons@^1.6.0", "@sinonjs/commons@^1.7.0":
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.7.0.tgz#f90ffc52a2e519f018b13b6c4da03cbff36ebed6"