trace failed executions

pkg/services/alerting/engine.go

@@ -2,9 +2,12 @@ package alerting
 
 
 import (
 import (
 	"context"
 	"context"
+	"fmt"
 	"time"
 	"time"
 
 
 	"github.com/opentracing/opentracing-go"
 	"github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go/ext"
+	tlog "github.com/opentracing/opentracing-go/log"
 
 
 	"github.com/benbjohnson/clock"
 	"github.com/benbjohnson/clock"
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/log"
@@ -101,20 +104,21 @@ func (e *Engine) processJob(grafanaCtx context.Context, job *Job) error {
 	}()
 	}()
 
 
 	alertCtx, cancelFn := context.WithTimeout(context.Background(), alertTimeout)
 	alertCtx, cancelFn := context.WithTimeout(context.Background(), alertTimeout)
+	span := opentracing.StartSpan("alert execution")
+	alertCtx = opentracing.ContextWithSpan(alertCtx, span)
 
 
 	job.Running = true
 	job.Running = true
 	evalContext := NewEvalContext(alertCtx, job.Rule)
 	evalContext := NewEvalContext(alertCtx, job.Rule)
-
-	done := make(chan struct{})
-
-	span := opentracing.StartSpan("alerting")
-	alertCtx = opentracing.ContextWithSpan(alertCtx, span)
 	evalContext.Ctx = alertCtx
 	evalContext.Ctx = alertCtx
 
 
+	done := make(chan struct{})
 	go func() {
 	go func() {
 		defer func() {
 		defer func() {
 			if err := recover(); err != nil {
 			if err := recover(); err != nil {
 				e.log.Error("Alert Panic", "error", err, "stack", log.Stack(1))
 				e.log.Error("Alert Panic", "error", err, "stack", log.Stack(1))
+				ext.Error.Set(span, true)
+				span.LogFields(tlog.Error(fmt.Errorf("error: %v", err)))
+				span.Finish()
 				close(done)
 				close(done)
 			}
 			}
 		}()
 		}()
@@ -126,8 +130,8 @@ func (e *Engine) processJob(grafanaCtx context.Context, job *Job) error {
 		span.SetTag("dashboardId", evalContext.Rule.DashboardId)
 		span.SetTag("dashboardId", evalContext.Rule.DashboardId)
 		span.SetTag("firing", evalContext.Firing)
 		span.SetTag("firing", evalContext.Firing)
 
 
-		close(done)
 		span.Finish()
 		span.Finish()
+		close(done)
 	}()
 	}()
 
 
 	var err error = nil
 	var err error = nil

pkg/tsdb/graphite/graphite.go

@@ -81,7 +81,7 @@ func (e *GraphiteExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice,
 		return result
 		return result
 	}
 	}
 
 
-	span, ctx := opentracing.StartSpanFromContext(ctx, "alerting.graphite")
+	span, ctx := opentracing.StartSpanFromContext(ctx, "graphite query")
 	span.SetTag("target", target)
 	span.SetTag("target", target)
 	span.SetTag("from", from)
 	span.SetTag("from", from)
 	span.SetTag("until", until)
 	span.SetTag("until", until)