Instrument backend plugin requests (#23346)

pkg/infra/metrics/metrics.go

@@ -8,7 +8,8 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 )
 
-const exporterName = "grafana"
+// ExporterName is used as namespace for exposing prometheus metrics
+const ExporterName = "grafana"
 
 var (
 	// MInstanceStart is a metric counter for started instances
@@ -159,28 +160,28 @@ func init() {
 	MInstanceStart = prometheus.NewCounter(prometheus.CounterOpts{
 		Name:      "instance_start_total",
 		Help:      "counter for started instances",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MPageStatus = newCounterVecStartingAtZero(
 		prometheus.CounterOpts{
 			Name:      "page_response_status_total",
 			Help:      "page http response status",
-			Namespace: exporterName,
+			Namespace: ExporterName,
 		}, []string{"code"}, httpStatusCodes...)
 
 	MApiStatus = newCounterVecStartingAtZero(
 		prometheus.CounterOpts{
 			Name:      "api_response_status_total",
 			Help:      "api http response status",
-			Namespace: exporterName,
+			Namespace: ExporterName,
 		}, []string{"code"}, httpStatusCodes...)
 
 	MProxyStatus = newCounterVecStartingAtZero(
 		prometheus.CounterOpts{
 			Name:      "proxy_response_status_total",
 			Help:      "proxy http response status",
-			Namespace: exporterName,
+			Namespace: ExporterName,
 		}, []string{"code"}, httpStatusCodes...)
 
 	MHttpRequestTotal = prometheus.NewCounterVec(
@@ -203,241 +204,241 @@ func init() {
 	MApiUserSignUpStarted = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_user_signup_started_total",
 		Help:      "amount of users who started the signup flow",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MApiUserSignUpCompleted = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_user_signup_completed_total",
 		Help:      "amount of users who completed the signup flow",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MApiUserSignUpInvite = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_user_signup_invite_total",
 		Help:      "amount of users who have been invited",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MApiDashboardSave = prometheus.NewSummary(prometheus.SummaryOpts{
 		Name:       "api_dashboard_save_milliseconds",
 		Help:       "summary for dashboard save duration",
 		Objectives: objectiveMap,
-		Namespace:  exporterName,
+		Namespace:  ExporterName,
 	})
 
 	MApiDashboardGet = prometheus.NewSummary(prometheus.SummaryOpts{
 		Name:       "api_dashboard_get_milliseconds",
 		Help:       "summary for dashboard get duration",
 		Objectives: objectiveMap,
-		Namespace:  exporterName,
+		Namespace:  ExporterName,
 	})
 
 	MApiDashboardSearch = prometheus.NewSummary(prometheus.SummaryOpts{
 		Name:       "api_dashboard_search_milliseconds",
 		Help:       "summary for dashboard search duration",
 		Objectives: objectiveMap,
-		Namespace:  exporterName,
+		Namespace:  ExporterName,
 	})
 
 	MApiAdminUserCreate = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_admin_user_created_total",
 		Help:      "api admin user created counter",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MApiLoginPost = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_login_post_total",
 		Help:      "api login post counter",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MApiLoginOAuth = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_login_oauth_total",
 		Help:      "api login oauth counter",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MApiLoginSAML = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_login_saml_total",
 		Help:      "api login saml counter",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MApiOrgCreate = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_org_create_total",
 		Help:      "api org created counter",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MApiDashboardSnapshotCreate = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_dashboard_snapshot_create_total",
 		Help:      "dashboard snapshots created",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MApiDashboardSnapshotExternal = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_dashboard_snapshot_external_total",
 		Help:      "external dashboard snapshots created",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MApiDashboardSnapshotGet = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_dashboard_snapshot_get_total",
 		Help:      "loaded dashboards",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MApiDashboardInsert = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_models_dashboard_insert_total",
 		Help:      "dashboards inserted ",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MAlertingResultState = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Name:      "alerting_result_total",
 		Help:      "alert execution result counter",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	}, []string{"state"})
 
 	MAlertingNotificationSent = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Name:      "alerting_notification_sent_total",
 		Help:      "counter for how many alert notifications have been sent",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	}, []string{"type"})
 
 	MAlertingNotificationFailed = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Name:      "alerting_notification_failed_total",
 		Help:      "counter for how many alert notifications have failed",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	}, []string{"type"})
 
 	MAwsCloudWatchGetMetricStatistics = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "aws_cloudwatch_get_metric_statistics_total",
 		Help:      "counter for getting metric statistics from aws",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MAwsCloudWatchListMetrics = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "aws_cloudwatch_list_metrics_total",
 		Help:      "counter for getting list of metrics from aws",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MAwsCloudWatchGetMetricData = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "aws_cloudwatch_get_metric_data_total",
 		Help:      "counter for getting metric data time series from aws",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MDBDataSourceQueryByID = newCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "db_datasource_query_by_id_total",
 		Help:      "counter for getting datasource by id",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	LDAPUsersSyncExecutionTime = prometheus.NewSummary(prometheus.SummaryOpts{
 		Name:       "ldap_users_sync_execution_time",
 		Help:       "summary for LDAP users sync execution duration",
 		Objectives: objectiveMap,
-		Namespace:  exporterName,
+		Namespace:  ExporterName,
 	})
 
 	MDataSourceProxyReqTimer = prometheus.NewSummary(prometheus.SummaryOpts{
 		Name:       "api_dataproxy_request_all_milliseconds",
 		Help:       "summary for dataproxy request duration",
 		Objectives: objectiveMap,
-		Namespace:  exporterName,
+		Namespace:  ExporterName,
 	})
 
 	MAlertingExecutionTime = prometheus.NewSummary(prometheus.SummaryOpts{
 		Name:       "alerting_execution_time_milliseconds",
 		Help:       "summary of alert exeuction duration",
 		Objectives: objectiveMap,
-		Namespace:  exporterName,
+		Namespace:  ExporterName,
 	})
 
 	MAlertingActiveAlerts = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name:      "alerting_active_alerts",
 		Help:      "amount of active alerts",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MStatTotalDashboards = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name:      "stat_totals_dashboard",
 		Help:      "total amount of dashboards",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MStatTotalUsers = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name:      "stat_total_users",
 		Help:      "total amount of users",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MStatActiveUsers = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name:      "stat_active_users",
 		Help:      "number of active users",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MStatTotalOrgs = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name:      "stat_total_orgs",
 		Help:      "total amount of orgs",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	MStatTotalPlaylists = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name:      "stat_total_playlists",
 		Help:      "total amount of playlists",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	StatsTotalViewers = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name:      "stat_totals_viewers",
 		Help:      "total amount of viewers",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	StatsTotalEditors = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name:      "stat_totals_editors",
 		Help:      "total amount of editors",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	StatsTotalAdmins = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name:      "stat_totals_admins",
 		Help:      "total amount of admins",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	StatsTotalActiveViewers = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name:      "stat_totals_active_viewers",
 		Help:      "total amount of viewers",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	StatsTotalActiveEditors = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name:      "stat_totals_active_editors",
 		Help:      "total amount of active editors",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	StatsTotalActiveAdmins = prometheus.NewGauge(prometheus.GaugeOpts{
 		Name:      "stat_totals_active_admins",
 		Help:      "total amount of active admins",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	})
 
 	grafanaBuildVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name:      "build_info",
 		Help:      "A metric with a constant '1' value labeled by version, revision, branch, and goversion from which Grafana was built",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	}, []string{"version", "revision", "branch", "goversion", "edition"})
 
 	grafanPluginBuildInfoDesc = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Name:      "plugin_build_info",
 		Help:      "A metric with a constant '1' value labeled by pluginId, pluginType and version from which Grafana plugin was built",
-		Namespace: exporterName,
+		Namespace: ExporterName,
 	}, []string{"plugin_id", "plugin_type", "version"})
 }
 

pkg/plugins/backendplugin/backend_plugin.go

@@ -141,7 +141,14 @@ func (p *BackendPlugin) CollectMetrics(ctx context.Context) (*pluginv2.CollectMe
 		}, nil
 	}
 
-	res, err := p.diagnostics.CollectMetrics(ctx, &pluginv2.CollectMetricsRequest{})
+	var res *pluginv2.CollectMetricsResponse
+	err := InstrumentPluginRequest(p.id, "metrics", func() error {
+		var innerErr error
+		res, innerErr = p.diagnostics.CollectMetrics(ctx, &pluginv2.CollectMetricsRequest{})
+
+		return innerErr
+	})
+
 	if err != nil {
 		if st, ok := status.FromError(err); ok {
 			if st.Code() == codes.Unimplemented {
@@ -197,7 +204,13 @@ func (p *BackendPlugin) checkHealth(ctx context.Context, config *PluginConfig) (
 		}
 	}
 
-	res, err := p.diagnostics.CheckHealth(ctx, &pluginv2.CheckHealthRequest{Config: pconfig})
+	var res *pluginv2.CheckHealthResponse
+	err = InstrumentPluginRequest(p.id, "checkhealth", func() error {
+		var innerErr error
+		res, innerErr = p.diagnostics.CheckHealth(ctx, &pluginv2.CheckHealthRequest{Config: pconfig})
+		return innerErr
+	})
+
 	if err != nil {
 		if st, ok := status.FromError(err); ok {
 			if st.Code() == codes.Unimplemented {

pkg/plugins/backendplugin/instrumentation.go

+package backendplugin
+
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	pluginRequestCounter  *prometheus.CounterVec
+	pluginRequestDuration *prometheus.SummaryVec
+)
+
+func init() {
+	pluginRequestCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "grafana",
+		Name:      "plugin_request_total",
+		Help:      "The total amount of plugin requests",
+	}, []string{"plugin_id", "endpoint", "status"})
+
+	pluginRequestDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+		Namespace:  "grafana",
+		Name:       "plugin_request_duration_milliseconds",
+		Help:       "Plugin request duration",
+		Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
+	}, []string{"plugin_id", "endpoint"})
+
+	prometheus.MustRegister(pluginRequestCounter, pluginRequestDuration)
+}
+
+// InstrumentPluginRequest instruments success rate and latency of `fn`
+func InstrumentPluginRequest(pluginID string, endpoint string, fn func() error) error {
+	status := "ok"
+
+	start := time.Now()
+
+	err := fn()
+	if err != nil {
+		status = "error"
+	}
+
+	elapsed := time.Since(start) / time.Millisecond
+	pluginRequestDuration.WithLabelValues(pluginID, endpoint).Observe(float64(elapsed))
+	pluginRequestCounter.WithLabelValues(pluginID, endpoint, status).Inc()
+
+	return err
+}

pkg/plugins/backendplugin/manager.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/util/errutil"
 	"github.com/grafana/grafana/pkg/util/proxyutil"
 
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -187,17 +188,17 @@ func (m *manager) CheckHealth(ctx context.Context, pluginConfig *PluginConfig) (
 }
 
 // CallResource calls a plugin resource.
-func (m *manager) CallResource(config PluginConfig, c *models.ReqContext, path string) {
+func (m *manager) CallResource(config PluginConfig, reqCtx *models.ReqContext, path string) {
 	m.pluginsMu.RLock()
 	p, registered := m.plugins[config.PluginID]
 	m.pluginsMu.RUnlock()
 
 	if !registered {
-		c.JsonApiErr(404, "Plugin not registered", nil)
+		reqCtx.JsonApiErr(404, "Plugin not registered", nil)
 		return
 	}
 
-	clonedReq := c.Req.Clone(c.Req.Context())
+	clonedReq := reqCtx.Req.Clone(reqCtx.Req.Context())
 	keepCookieNames := []string{}
 	if config.JSONData != nil {
 		if keepCookies := config.JSONData.Get("keepCookies"); keepCookies != nil {
@@ -208,9 +209,9 @@ func (m *manager) CallResource(config PluginConfig, c *models.ReqContext, path s
 	proxyutil.ClearCookieHeader(clonedReq, keepCookieNames)
 	proxyutil.PrepareProxyRequest(clonedReq)
 
-	body, err := c.Req.Body().Bytes()
+	body, err := reqCtx.Req.Body().Bytes()
 	if err != nil {
-		c.JsonApiErr(500, "Failed to read request body", err)
+		reqCtx.JsonApiErr(500, "Failed to read request body", err)
 		return
 	}
 
@@ -221,32 +222,41 @@ func (m *manager) CallResource(config PluginConfig, c *models.ReqContext, path s
 		URL:     clonedReq.URL.String(),
 		Headers: clonedReq.Header,
 		Body:    body,
-		User:    c.SignedInUser,
+		User:    reqCtx.SignedInUser,
 	}
 
-	stream, err := p.callResource(clonedReq.Context(), req)
+	err = InstrumentPluginRequest(p.id, "resource", func() error {
+		stream, err := p.callResource(clonedReq.Context(), req)
+		if err != nil {
+			return errutil.Wrap("Failed to call resource", err)
+		}
+
+		return flushStream(p, stream, reqCtx)
+	})
+
 	if err != nil {
-		c.JsonApiErr(500, "Failed to call resource", err)
-		return
+		reqCtx.JsonApiErr(500, "Failed to ", err)
 	}
+}
 
+func flushStream(plugin *BackendPlugin, stream callResourceResultStream, reqCtx *models.ReqContext) error {
 	processedStreams := 0
 
 	for {
 		resp, err := stream.Recv()
 		if err == io.EOF {
 			if processedStreams == 0 {
-				c.JsonApiErr(500, "Received empty resource response ", nil)
+				return errors.New("Received empty resource response")
 			}
-			return
+			return nil
 		}
 		if err != nil {
 			if processedStreams == 0 {
-				c.JsonApiErr(500, "Failed to receive response from resource call", err)
-			} else {
-				p.logger.Error("Failed to receive response from resource call", "error", err)
+				return errutil.Wrap("Failed to receive response from resource call", err)
 			}
-			return
+
+			plugin.logger.Error("Failed to receive response from resource call", "error", err)
+			return nil
 		}
 
 		// Expected that headers and status are only part of first stream
@@ -264,18 +274,18 @@ func (m *manager) CallResource(config PluginConfig, c *models.ReqContext, path s
 				}
 
 				for _, v := range values {
-					c.Resp.Header().Add(k, v)
+					reqCtx.Resp.Header().Add(k, v)
 				}
 			}
 
-			c.WriteHeader(resp.Status)
+			reqCtx.WriteHeader(resp.Status)
 		}
 
-		if _, err := c.Write(resp.Body); err != nil {
-			p.logger.Error("Failed to write resource response", "error", err)
+		if _, err := reqCtx.Write(resp.Body); err != nil {
+			plugin.logger.Error("Failed to write resource response", "error", err)
 		}
 
-		c.Resp.Flush()
+		reqCtx.Resp.Flush()
 		processedStreams++
 	}
 }

pkg/plugins/datasource/wrapper/datasource_plugin_wrapper_v2.go

@@ -77,7 +77,14 @@ func (tw *DatasourcePluginWrapperV2) Query(ctx context.Context, ds *models.DataS
 		})
 	}
 
-	pbRes, err := tw.DataPlugin.QueryData(ctx, pbQuery)
+	var pbRes *pluginv2.QueryDataResponse
+	err = backendplugin.InstrumentPluginRequest(ds.Type, "dataquery", func() error {
+		var err error
+		pbRes, err = tw.DataPlugin.QueryData(ctx, pbQuery)
+
+		return err
+	})
+
 	if err != nil {
 		return nil, err
 	}