Skip to content

N
nexpie-grafana-theme
Commit 6e7941d3 by bergquist
Options

moves usage stats sender to new package

parent 487e7b5e
Showing with 246 additions and 202 deletions
pkg/infra/usagestats/service.go 0 → 100644
package usagestats
import (
"context"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/services/auth"
"github.com/grafana/grafana/pkg/social"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/setting"
)
var metricsLogger log.Logger = log.New("metrics")
func init() {
registry.RegisterService(&UsageStatsService{})
}
type UsageStatsService struct {
Cfg *setting.Cfg `inject:""`
TokenService *auth.UserAuthTokenService `inject:""`
Bus bus.Bus `inject:""`
oauthProviders map[string]bool
}
func (uss *UsageStatsService) Init() error {
uss.oauthProviders = social.GetOAuthProviders(uss.Cfg)
return nil
}
func (uss *UsageStatsService) Run(ctx context.Context) error {
uss.updateTotalStats()
onceEveryDayTick := time.NewTicker(time.Hour * 24)
everyMinuteTicker := time.NewTicker(time.Minute)
defer onceEveryDayTick.Stop()
defer everyMinuteTicker.Stop()
for {
select {
case <-onceEveryDayTick.C:
uss.sendUsageStats(uss.oauthProviders)
case <-everyMinuteTicker.C:
uss.updateTotalStats()
case <-ctx.Done():
return ctx.Err()
}
}
}
pkg/infra/usagestats/usage_stats.go 0 → 100644
package usagestats
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"runtime"
"strings"
"time"
"github.com/grafana/grafana/pkg/metrics"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting"
)
var usageStatsURL = "https://stats.grafana.org/grafana-usage-report"
func (uss *UsageStatsService) sendUsageStats(oauthProviders map[string]bool) {
if !setting.ReportingEnabled {
return
}
metricsLogger.Debug(fmt.Sprintf("Sending anonymous usage stats to %s", usageStatsURL))
version := strings.Replace(setting.BuildVersion, ".", "_", -1)
metrics := map[string]interface{}{}
report := map[string]interface{}{
"version": version,
"metrics": metrics,
"os": runtime.GOOS,
"arch": runtime.GOARCH,
"edition": getEdition(),
"packaging": setting.Packaging,
}
statsQuery := models.GetSystemStatsQuery{}
if err := uss.Bus.Dispatch(&statsQuery); err != nil {
metricsLogger.Error("Failed to get system stats", "error", err)
return
}
metrics["stats.dashboards.count"] = statsQuery.Result.Dashboards
metrics["stats.users.count"] = statsQuery.Result.Users
metrics["stats.orgs.count"] = statsQuery.Result.Orgs
metrics["stats.playlist.count"] = statsQuery.Result.Playlists
metrics["stats.plugins.apps.count"] = len(plugins.Apps)
metrics["stats.plugins.panels.count"] = len(plugins.Panels)
metrics["stats.plugins.datasources.count"] = len(plugins.DataSources)
metrics["stats.alerts.count"] = statsQuery.Result.Alerts
metrics["stats.active_users.count"] = statsQuery.Result.ActiveUsers
metrics["stats.datasources.count"] = statsQuery.Result.Datasources
metrics["stats.stars.count"] = statsQuery.Result.Stars
metrics["stats.folders.count"] = statsQuery.Result.Folders
metrics["stats.dashboard_permissions.count"] = statsQuery.Result.DashboardPermissions
metrics["stats.folder_permissions.count"] = statsQuery.Result.FolderPermissions
metrics["stats.provisioned_dashboards.count"] = statsQuery.Result.ProvisionedDashboards
metrics["stats.snapshots.count"] = statsQuery.Result.Snapshots
metrics["stats.teams.count"] = statsQuery.Result.Teams
dsStats := models.GetDataSourceStatsQuery{}
if err := uss.Bus.Dispatch(&dsStats); err != nil {
metricsLogger.Error("Failed to get datasource stats", "error", err)
return
}
// send counters for each data source
// but ignore any custom data sources
// as sending that name could be sensitive information
dsOtherCount := 0
for _, dsStat := range dsStats.Result {
if models.IsKnownDataSourcePlugin(dsStat.Type) {
metrics["stats.ds."+dsStat.Type+".count"] = dsStat.Count
} else {
dsOtherCount += dsStat.Count
}
}
metrics["stats.ds.other.count"] = dsOtherCount
metrics["stats.packaging."+setting.Packaging+".count"] = 1
dsAccessStats := models.GetDataSourceAccessStatsQuery{}
if err := uss.Bus.Dispatch(&dsAccessStats); err != nil {
metricsLogger.Error("Failed to get datasource access stats", "error", err)
return
}
// send access counters for each data source
// but ignore any custom data sources
// as sending that name could be sensitive information
dsAccessOtherCount := make(map[string]int64)
for _, dsAccessStat := range dsAccessStats.Result {
if dsAccessStat.Access == "" {
continue
}
access := strings.ToLower(dsAccessStat.Access)
if models.IsKnownDataSourcePlugin(dsAccessStat.Type) {
metrics["stats.ds_access."+dsAccessStat.Type+"."+access+".count"] = dsAccessStat.Count
} else {
old := dsAccessOtherCount[access]
dsAccessOtherCount[access] = old + dsAccessStat.Count
}
}
for access, count := range dsAccessOtherCount {
metrics["stats.ds_access.other."+access+".count"] = count
}
anStats := models.GetAlertNotifierUsageStatsQuery{}
if err := uss.Bus.Dispatch(&anStats); err != nil {
metricsLogger.Error("Failed to get alert notification stats", "error", err)
return
}
for _, stats := range anStats.Result {
metrics["stats.alert_notifiers."+stats.Type+".count"] = stats.Count
}
authTypes := map[string]bool{}
authTypes["anonymous"] = setting.AnonymousEnabled
authTypes["basic_auth"] = setting.BasicAuthEnabled
authTypes["ldap"] = setting.LdapEnabled
authTypes["auth_proxy"] = setting.AuthProxyEnabled
for provider, enabled := range oauthProviders {
authTypes["oauth_"+provider] = enabled
}
for authType, enabled := range authTypes {
enabledValue := 0
if enabled {
enabledValue = 1
}
metrics["stats.auth_enabled."+authType+".count"] = enabledValue
}
out, _ := json.MarshalIndent(report, "", " ")
data := bytes.NewBuffer(out)
client := http.Client{Timeout: 5 * time.Second}
go client.Post(usageStatsURL, "application/json", data)
}
func (uss *UsageStatsService) updateTotalStats() {
statsQuery := models.GetSystemStatsQuery{}
if err := uss.Bus.Dispatch(&statsQuery); err != nil {
metricsLogger.Error("Failed to get system stats", "error", err)
return
}
metrics.M_StatTotal_Dashboards.Set(float64(statsQuery.Result.Dashboards))
metrics.M_StatTotal_Users.Set(float64(statsQuery.Result.Users))
metrics.M_StatActive_Users.Set(float64(statsQuery.Result.ActiveUsers))
metrics.M_StatTotal_Playlists.Set(float64(statsQuery.Result.Playlists))
metrics.M_StatTotal_Orgs.Set(float64(statsQuery.Result.Orgs))
}
func getEdition() string {
if setting.IsEnterprise {
return "enterprise"
} else {
return "oss"
}
}
pkg/metrics/metrics_test.go → pkg/infra/usagestats/usage_stats_test.go
package metrics package usagestats
import ( import (
"bytes" "bytes"
...@@ -21,8 +21,13 @@ import ( ...@@ -21,8 +21,13 @@ import (
func TestMetrics(t *testing.T) { func TestMetrics(t *testing.T) {
Convey("Test send usage stats", t, func() { Convey("Test send usage stats", t, func() {
uss := &UsageStatsService{
Bus: bus.New(),
}
var getSystemStatsQuery *models.GetSystemStatsQuery var getSystemStatsQuery *models.GetSystemStatsQuery
bus.AddHandler("test", func(query *models.GetSystemStatsQuery) error { uss.Bus.AddHandler(func(query *models.GetSystemStatsQuery) error {
query.Result = &models.SystemStats{ query.Result = &models.SystemStats{
Dashboards: 1, Dashboards: 1,
Datasources: 2, Datasources: 2,
...@@ -44,7 +49,7 @@ func TestMetrics(t *testing.T) { ...@@ -44,7 +49,7 @@ func TestMetrics(t *testing.T) {
}) })
var getDataSourceStatsQuery *models.GetDataSourceStatsQuery var getDataSourceStatsQuery *models.GetDataSourceStatsQuery
bus.AddHandler("test", func(query *models.GetDataSourceStatsQuery) error { uss.Bus.AddHandler(func(query *models.GetDataSourceStatsQuery) error {
query.Result = []*models.DataSourceStats{ query.Result = []*models.DataSourceStats{
{ {
Type: models.DS_ES, Type: models.DS_ES,
...@@ -68,7 +73,7 @@ func TestMetrics(t *testing.T) { ...@@ -68,7 +73,7 @@ func TestMetrics(t *testing.T) {
}) })
var getDataSourceAccessStatsQuery *models.GetDataSourceAccessStatsQuery var getDataSourceAccessStatsQuery *models.GetDataSourceAccessStatsQuery
bus.AddHandler("test", func(query *models.GetDataSourceAccessStatsQuery) error { uss.Bus.AddHandler(func(query *models.GetDataSourceAccessStatsQuery) error {
query.Result = []*models.DataSourceAccessStats{ query.Result = []*models.DataSourceAccessStats{
{ {
Type: models.DS_ES, Type: models.DS_ES,
...@@ -116,7 +121,7 @@ func TestMetrics(t *testing.T) { ...@@ -116,7 +121,7 @@ func TestMetrics(t *testing.T) {
}) })
var getAlertNotifierUsageStatsQuery *models.GetAlertNotifierUsageStatsQuery var getAlertNotifierUsageStatsQuery *models.GetAlertNotifierUsageStatsQuery
bus.AddHandler("test", func(query *models.GetAlertNotifierUsageStatsQuery) error { uss.Bus.AddHandler(func(query *models.GetAlertNotifierUsageStatsQuery) error {
query.Result = []*models.NotifierUsageStats{ query.Result = []*models.NotifierUsageStats{
{ {
Type: "slack", Type: "slack",
...@@ -155,11 +160,11 @@ func TestMetrics(t *testing.T) { ...@@ -155,11 +160,11 @@ func TestMetrics(t *testing.T) {
"grafana_com": true, "grafana_com": true,
} }
sendUsageStats(oauthProviders) uss.sendUsageStats(oauthProviders)
Convey("Given reporting not enabled and sending usage stats", func() { Convey("Given reporting not enabled and sending usage stats", func() {
setting.ReportingEnabled = false setting.ReportingEnabled = false
sendUsageStats(oauthProviders) uss.sendUsageStats(oauthProviders)
Convey("Should not gather stats or call http endpoint", func() { Convey("Should not gather stats or call http endpoint", func() {
So(getSystemStatsQuery, ShouldBeNil) So(getSystemStatsQuery, ShouldBeNil)
...@@ -179,7 +184,7 @@ func TestMetrics(t *testing.T) { ...@@ -179,7 +184,7 @@ func TestMetrics(t *testing.T) {
setting.Packaging = "deb" setting.Packaging = "deb"
wg.Add(1) wg.Add(1)
sendUsageStats(oauthProviders) uss.sendUsageStats(oauthProviders)
Convey("Should gather stats and call http endpoint", func() { Convey("Should gather stats and call http endpoint", func() {
if waitTimeout(&wg, 2*time.Second) { if waitTimeout(&wg, 2*time.Second) {
......
pkg/metrics/metrics.go
package metrics package metrics
import ( import (
"bytes"
"encoding/json"
"net/http"
"runtime" "runtime"
"strings"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
) )
...@@ -68,23 +59,6 @@ var ( ...@@ -68,23 +59,6 @@ var (
grafanaBuildVersion *prometheus.GaugeVec grafanaBuildVersion *prometheus.GaugeVec
) )
func newCounterVecStartingAtZero(opts prometheus.CounterOpts, labels []string, labelValues ...string) *prometheus.CounterVec {
counter := prometheus.NewCounterVec(opts, labels)
for _, label := range labelValues {
counter.WithLabelValues(label).Add(0)
}
return counter
}
func newCounterStartingAtZero(opts prometheus.CounterOpts, labelValues ...string) prometheus.Counter {
counter := prometheus.NewCounter(opts)
counter.Add(0)
return counter
}
func init() { func init() {
M_Instance_Start = prometheus.NewCounter(prometheus.CounterOpts{ M_Instance_Start = prometheus.NewCounter(prometheus.CounterOpts{
Name: "instance_start_total", Name: "instance_start_total",
...@@ -362,154 +336,19 @@ func initMetricVars() { ...@@ -362,154 +336,19 @@ func initMetricVars() {
} }
func updateTotalStats() { func newCounterVecStartingAtZero(opts prometheus.CounterOpts, labels []string, labelValues ...string) *prometheus.CounterVec {
statsQuery := models.GetSystemStatsQuery{} counter := prometheus.NewCounterVec(opts, labels)
if err := bus.Dispatch(&statsQuery); err != nil {
metricsLogger.Error("Failed to get system stats", "error", err)
return
}
M_StatTotal_Dashboards.Set(float64(statsQuery.Result.Dashboards))
M_StatTotal_Users.Set(float64(statsQuery.Result.Users))
M_StatActive_Users.Set(float64(statsQuery.Result.ActiveUsers))
M_StatTotal_Playlists.Set(float64(statsQuery.Result.Playlists))
M_StatTotal_Orgs.Set(float64(statsQuery.Result.Orgs))
}
var usageStatsURL = "https://stats.grafana.org/grafana-usage-report"
func getEdition() string {
if setting.IsEnterprise {
return "enterprise"
} else {
return "oss"
}
}
func sendUsageStats(oauthProviders map[string]bool) {
if !setting.ReportingEnabled {
return
}
metricsLogger.Debug("Sending anonymous usage stats to stats.grafana.org")
version := strings.Replace(setting.BuildVersion, ".", "_", -1)
metrics := map[string]interface{}{}
report := map[string]interface{}{
"version": version,
"metrics": metrics,
"os": runtime.GOOS,
"arch": runtime.GOARCH,
"edition": getEdition(),
"packaging": setting.Packaging,
}
statsQuery := models.GetSystemStatsQuery{}
if err := bus.Dispatch(&statsQuery); err != nil {
metricsLogger.Error("Failed to get system stats", "error", err)
return
}
metrics["stats.dashboards.count"] = statsQuery.Result.Dashboards
metrics["stats.users.count"] = statsQuery.Result.Users
metrics["stats.orgs.count"] = statsQuery.Result.Orgs
metrics["stats.playlist.count"] = statsQuery.Result.Playlists
metrics["stats.plugins.apps.count"] = len(plugins.Apps)
metrics["stats.plugins.panels.count"] = len(plugins.Panels)
metrics["stats.plugins.datasources.count"] = len(plugins.DataSources)
metrics["stats.alerts.count"] = statsQuery.Result.Alerts
metrics["stats.active_users.count"] = statsQuery.Result.ActiveUsers
metrics["stats.datasources.count"] = statsQuery.Result.Datasources
metrics["stats.stars.count"] = statsQuery.Result.Stars
metrics["stats.folders.count"] = statsQuery.Result.Folders
metrics["stats.dashboard_permissions.count"] = statsQuery.Result.DashboardPermissions
metrics["stats.folder_permissions.count"] = statsQuery.Result.FolderPermissions
metrics["stats.provisioned_dashboards.count"] = statsQuery.Result.ProvisionedDashboards
metrics["stats.snapshots.count"] = statsQuery.Result.Snapshots
metrics["stats.teams.count"] = statsQuery.Result.Teams
dsStats := models.GetDataSourceStatsQuery{}
if err := bus.Dispatch(&dsStats); err != nil {
metricsLogger.Error("Failed to get datasource stats", "error", err)
return
}
// send counters for each data source
// but ignore any custom data sources
// as sending that name could be sensitive information
dsOtherCount := 0
for _, dsStat := range dsStats.Result {
if models.IsKnownDataSourcePlugin(dsStat.Type) {
metrics["stats.ds."+dsStat.Type+".count"] = dsStat.Count
} else {
dsOtherCount += dsStat.Count
}
}
metrics["stats.ds.other.count"] = dsOtherCount
metrics["stats.packaging."+setting.Packaging+".count"] = 1
dsAccessStats := models.GetDataSourceAccessStatsQuery{}
if err := bus.Dispatch(&dsAccessStats); err != nil {
metricsLogger.Error("Failed to get datasource access stats", "error", err)
return
}
// send access counters for each data source
// but ignore any custom data sources
// as sending that name could be sensitive information
dsAccessOtherCount := make(map[string]int64)
for _, dsAccessStat := range dsAccessStats.Result {
if dsAccessStat.Access == "" {
continue
}
access := strings.ToLower(dsAccessStat.Access)
if models.IsKnownDataSourcePlugin(dsAccessStat.Type) {
metrics["stats.ds_access."+dsAccessStat.Type+"."+access+".count"] = dsAccessStat.Count
} else {
old := dsAccessOtherCount[access]
dsAccessOtherCount[access] = old + dsAccessStat.Count
}
}
for access, count := range dsAccessOtherCount {
metrics["stats.ds_access.other."+access+".count"] = count
}
anStats := models.GetAlertNotifierUsageStatsQuery{}
if err := bus.Dispatch(&anStats); err != nil {
metricsLogger.Error("Failed to get alert notification stats", "error", err)
return
}
for _, stats := range anStats.Result {
metrics["stats.alert_notifiers."+stats.Type+".count"] = stats.Count
}
authTypes := map[string]bool{}
authTypes["anonymous"] = setting.AnonymousEnabled
authTypes["basic_auth"] = setting.BasicAuthEnabled
authTypes["ldap"] = setting.LdapEnabled
authTypes["auth_proxy"] = setting.AuthProxyEnabled
for provider, enabled := range oauthProviders { for _, label := range labelValues {
authTypes["oauth_"+provider] = enabled counter.WithLabelValues(label).Add(0)
} }
for authType, enabled := range authTypes { return counter
enabledValue := 0 }
if enabled {
enabledValue = 1
}
metrics["stats.auth_enabled."+authType+".count"] = enabledValue
}
out, _ := json.MarshalIndent(report, "", " ") func newCounterStartingAtZero(opts prometheus.CounterOpts, labelValues ...string) prometheus.Counter {
data := bytes.NewBuffer(out) counter := prometheus.NewCounter(opts)
counter.Add(0)
client := http.Client{Timeout: 5 * time.Second} return counter
go client.Post(usageStatsURL, "application/json", data)
} }
pkg/metrics/service.go
...@@ -2,7 +2,6 @@ package metrics ...@@ -2,7 +2,6 @@ package metrics
import ( import (
"context" "context"
"time"
"github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/metrics/graphitebridge" "github.com/grafana/grafana/pkg/metrics/graphitebridge"
...@@ -30,7 +29,6 @@ type InternalMetricsService struct { ...@@ -30,7 +29,6 @@ type InternalMetricsService struct {
intervalSeconds int64 intervalSeconds int64
graphiteCfg *graphitebridge.Config graphiteCfg *graphitebridge.Config
oauthProviders map[string]bool
} }
func (im *InternalMetricsService) Init() error { func (im *InternalMetricsService) Init() error {
...@@ -50,22 +48,6 @@ func (im *InternalMetricsService) Run(ctx context.Context) error { ...@@ -50,22 +48,6 @@ func (im *InternalMetricsService) Run(ctx context.Context) error {
M_Instance_Start.Inc() M_Instance_Start.Inc()
// set the total stats gauges before we publishing metrics <-ctx.Done()
updateTotalStats()
onceEveryDayTick := time.NewTicker(time.Hour * 24)
everyMinuteTicker := time.NewTicker(time.Minute)
defer onceEveryDayTick.Stop()
defer everyMinuteTicker.Stop()
for {
select {
case <-onceEveryDayTick.C:
sendUsageStats(im.oauthProviders)
case <-everyMinuteTicker.C:
updateTotalStats()
case <-ctx.Done():
return ctx.Err() return ctx.Err()
}
}
} }
pkg/metrics/settings.go
...@@ -5,8 +5,6 @@ import ( ...@@ -5,8 +5,6 @@ import (
"strings" "strings"
"time" "time"
"github.com/grafana/grafana/pkg/social"
"github.com/grafana/grafana/pkg/metrics/graphitebridge" "github.com/grafana/grafana/pkg/metrics/graphitebridge"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
...@@ -24,8 +22,6 @@ func (im *InternalMetricsService) readSettings() error { ...@@ -24,8 +22,6 @@ func (im *InternalMetricsService) readSettings() error {
return fmt.Errorf("Unable to parse metrics graphite section, %v", err) return fmt.Errorf("Unable to parse metrics graphite section, %v", err)
} }
im.oauthProviders = social.GetOAuthProviders(im.Cfg)
return nil return nil
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment