feat(instrumentation): work on settings model for internal metrics publishing, #4696

conf/defaults.ini

@@ -6,6 +6,9 @@
 # possible values : production, development
 app_mode = production
 
+# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
+instance_name = ${HOSTNAME}
+
 #################################### Paths ####################################
 [paths]
 # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)

conf/sample.ini

@@ -6,6 +6,9 @@
 # possible values : production, development
 ; app_mode = production
 
+# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
+; instance_name = ${HOSTNAME}
+
 #################################### Paths ####################################
 [paths]
 # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)

pkg/cmd/grafana-server/main.go

@@ -64,15 +64,12 @@ func main() {
 	social.NewOAuthService()
 	eventpublisher.Init()
 	plugins.Init()
+	metrics.Init()
 
 	if err := notifications.Init(); err != nil {
 		log.Fatal(3, "Notification service failed to initialize", err)
 	}
 
-	if setting.ReportingEnabled {
-		go metrics.StartUsageReportLoop()
-	}
-
 	StartServer()
 	exitChan <- 0
 }

pkg/cmd/grafana-server/web.go

@@ -31,7 +31,7 @@ func newMacaron() *macaron.Macaron {
 
 	for _, route := range plugins.StaticRoutes {
 		pluginRoute := path.Join("/public/plugins/", route.PluginId)
-		log.Info("Plugins: Adding route %s -> %s", pluginRoute, route.Directory)
+		log.Debug("Plugins: Adding route %s -> %s", pluginRoute, route.Directory)
 		mapStatic(m, route.Directory, "", pluginRoute)
 	}
 

pkg/metrics/send.go → pkg/metrics/publish.go

@@ -9,40 +9,36 @@ import (
 
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/log"
-	"github.com/grafana/grafana/pkg/metrics/senders"
 	m "github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
-type MetricSender interface {
-	Send(metrics map[string]interface{}) error
+func Init() {
+	go instrumentationLoop()
 }
 
-func StartUsageReportLoop() chan struct{} {
+func instrumentationLoop() chan struct{} {
 	M_Instance_Start.Inc(1)
 
-	hourTicker := time.NewTicker(time.Hour * 24)
-	secondTicker := time.NewTicker(time.Second * 10)
+	settings := readSettings()
 
-	sender := &receiver.GraphiteSender{
-		Host:     "localhost",
-		Port:     "2003",
-		Protocol: "tcp",
-		Prefix:   "grafana.",
-	}
+	onceEveryDayTick := time.NewTicker(time.Hour * 24)
+	secondTicker := time.NewTicker(time.Second * time.Duration(settings.IntervalSeconds))
 
 	for {
 		select {
-		case <-hourTicker.C:
+		case <-onceEveryDayTick.C:
 			sendUsageStats()
 		case <-secondTicker.C:
-			sendMetricUsage(sender)
+			if settings.Enabled {
+				sendMetrics(settings)
+			}
 		}
 	}
 }
 
-func sendMetricUsage(sender MetricSender) {
+func sendMetrics(settings *MetricSettings) {
 	metrics := map[string]interface{}{}
 
 	MetricStats.Each(func(name string, i interface{}) {
@@ -63,13 +59,16 @@ func sendMetricUsage(sender MetricSender) {
 		}
 	})
 
-	err := sender.Send(metrics)
-	if err != nil {
-		log.Error(1, "Failed to send metrics:", err)
+	for _, publisher := range settings.Publishers {
+		publisher.Publish(metrics)
 	}
 }
 
 func sendUsageStats() {
+	if !setting.ReportingEnabled {
+		return
+	}
+
 	log.Trace("Sending anonymous usage stats to stats.grafana.org")
 
 	version := strings.Replace(setting.BuildVersion, ".", "_", -1)

pkg/metrics/senders/graphite.go → pkg/metrics/publishers/graphite.go

-package receiver
+package publishers
 
 import (
 	"bytes"
 	"fmt"
-	"github.com/grafana/grafana/pkg/log"
 	"net"
 	"time"
+
+	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/setting"
 )
 
-type GraphiteSender struct {
-	Host     string
-	Port     string
+type GraphitePublisher struct {
+	Address  string
 	Protocol string
 	Prefix   string
 }
 
-func (this *GraphiteSender) Send(metrics map[string]interface{}) error {
-	log.Debug("GraphiteSender: Sending metrics to graphite")
+func CreateGraphitePublisher() (*GraphitePublisher, error) {
+	graphiteSection, err := setting.Cfg.GetSection("metrics.graphite")
+	if err != nil {
+		return nil, nil
+	}
 
-	address := fmt.Sprintf("%s:%s", this.Host, this.Port)
-	conn, err := net.DialTimeout(this.Protocol, address, time.Second*5)
+	graphiteReceiver := &GraphitePublisher{}
+	graphiteReceiver.Protocol = "tcp"
+	graphiteReceiver.Address = graphiteSection.Key("address").MustString("localhost:2003")
+	graphiteReceiver.Prefix = graphiteSection.Key("prefix").MustString("service.grafana.%(instance_name)s")
+
+	return graphiteReceiver, nil
+}
+
+func (this *GraphitePublisher) Publish(metrics map[string]interface{}) {
+	conn, err := net.DialTimeout(this.Protocol, this.Address, time.Second*5)
 
 	if err != nil {
-		return fmt.Errorf("Graphite Sender: Failed to connec to %s!", err)
+		log.Error(3, "Metrics: GraphitePublisher:  Failed to connect to %s!", err)
+		return
 	}
 
 	buf := bytes.NewBufferString("")
@@ -30,15 +43,13 @@ func (this *GraphiteSender) Send(metrics map[string]interface{}) error {
 	for key, value := range metrics {
 		metricName := this.Prefix + key
 		line := fmt.Sprintf("%s %d %d\n", metricName, value, now)
-		log.Debug("SendMetric: sending %s", line)
 		buf.WriteString(line)
 	}
 
+	log.Trace("Metrics: GraphitePublisher.Publish() \n%s", buf)
 	_, err = conn.Write(buf.Bytes())
 
 	if err != nil {
-		return fmt.Errorf("Graphite Sender: Failed to send metrics! %s", err)
+		log.Error(3, "Metrics: GraphitePublisher: Failed to send metrics! %s", err)
 	}
-
-	return nil
 }

pkg/metrics/settings.go

+package metrics
+
+import (
+	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/metrics/publishers"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+type MetricPublisher interface {
+	Publish(metrics map[string]interface{})
+}
+
+type MetricSettings struct {
+	Enabled         bool
+	IntervalSeconds int64
+
+	Publishers []MetricPublisher
+}
+
+func readSettings() *MetricSettings {
+	var settings = &MetricSettings{
+		Enabled:    false,
+		Publishers: make([]MetricPublisher, 0),
+	}
+
+	var section, err = setting.Cfg.GetSection("metrics")
+	if err != nil {
+		log.Fatal(3, "Unable to find metrics config section")
+		return nil
+	}
+
+	settings.Enabled = section.Key("enabled").MustBool(false)
+	settings.IntervalSeconds = section.Key("interval_seconds").MustInt64(10)
+
+	if !settings.Enabled {
+		return settings
+	}
+
+	if graphitePublisher, err := publishers.CreateGraphitePublisher(); err != nil {
+		log.Error(3, "Metrics: Failed to init Graphite metric publisher", err)
+	} else if graphitePublisher != nil {
+		log.Info("Metrics: Internal metrics publisher Graphite initialized")
+		settings.Publishers = append(settings.Publishers, graphitePublisher)
+	}
+
+	return settings
+}

pkg/setting/setting.go

@@ -40,6 +40,7 @@ var (
 	Env          string = DEV
 	AppUrl       string
 	AppSubUrl    string
+	InstanceName string
 
 	// build
 	BuildVersion string
@@ -259,6 +260,12 @@ func evalEnvVarExpression(value string) string {
 		envVar = strings.TrimPrefix(envVar, "${")
 		envVar = strings.TrimSuffix(envVar, "}")
 		envValue := os.Getenv(envVar)
+
+		// if env variable is hostname and it is emtpy use os.Hostname as default
+		if envVar == "HOSTNAME" && envValue == "" {
+			envValue, _ = os.Hostname()
+		}
+
 		return envValue
 	})
 }
@@ -395,11 +402,28 @@ func validateStaticRootPath() error {
 	return fmt.Errorf("Failed to detect generated css or javascript files in static root (%s), have you executed default grunt task?", StaticRootPath)
 }
 
+// func readInstanceName() string {
+// 	hostname, _ := os.Hostname()
+// 	if hostname == "" {
+// 		hostname = "hostname_unknown"
+// 	}
+//
+// 	instanceName := Cfg.Section("").Key("instance_name").MustString("")
+// 	if instanceName = "" {
+// 		// set value as it might be used in other places
+// 		Cfg.Section("").Key("instance_name").SetValue(hostname)
+// 		instanceName = hostname
+// 	}
+//
+// 	return
+// }
+
 func NewConfigContext(args *CommandLineArgs) error {
 	setHomePath(args)
 	loadConfiguration(args)
 
 	Env = Cfg.Section("").Key("app_mode").MustString("development")
+	InstanceName = Cfg.Section("").Key("instance_name").MustString("unknown_instance_name")
 	PluginsPath = Cfg.Section("paths").Key("plugins").String()
 
 	server := Cfg.Section("server")

pkg/setting/setting_test.go

@@ -89,5 +89,14 @@ func TestLoadingSettings(t *testing.T) {
 			So(DataPath, ShouldEqual, "/tmp/env_override")
 		})
 
+		Convey("instance_name default to hostname even if hostname env is emtpy", func() {
+			NewConfigContext(&CommandLineArgs{
+				HomePath: "../../",
+			})
+
+			hostname, _ := os.Hostname()
+			So(InstanceName, ShouldEqual, hostname)
+		})
+
 	})
 }