Skip to content

N
nexpie-grafana-theme
Commit 396a5a94 by Carl Bergquist Committed by GitHub
Options

Merge pull request #15300 from bergquist/token_usage_stats 

adds usage stats for sessions
parents a402a371 2be60887
Showing with 265 additions and 204 deletions
pkg/infra/usagestats/service.go 0 → 100644
package usagestats
import (
"context"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/social"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/setting"
)
var metricsLogger log.Logger = log.New("metrics")
func init() {
registry.RegisterService(&UsageStatsService{})
}
type UsageStatsService struct {
Cfg *setting.Cfg `inject:""`
Bus bus.Bus `inject:""`
SQLStore *sqlstore.SqlStore `inject:""`
oauthProviders map[string]bool
}
func (uss *UsageStatsService) Init() error {
uss.oauthProviders = social.GetOAuthProviders(uss.Cfg)
return nil
}
func (uss *UsageStatsService) Run(ctx context.Context) error {
uss.updateTotalStats()
onceEveryDayTick := time.NewTicker(time.Hour * 24)
everyMinuteTicker := time.NewTicker(time.Minute)
defer onceEveryDayTick.Stop()
defer everyMinuteTicker.Stop()
for {
select {
case <-onceEveryDayTick.C:
uss.sendUsageStats(uss.oauthProviders)
case <-everyMinuteTicker.C:
uss.updateTotalStats()
case <-ctx.Done():
return ctx.Err()
}
}
}
pkg/infra/usagestats/usage_stats.go 0 → 100644
package usagestats
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"runtime"
"strings"
"time"
"github.com/grafana/grafana/pkg/metrics"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting"
)
var usageStatsURL = "https://stats.grafana.org/grafana-usage-report"
func (uss *UsageStatsService) sendUsageStats(oauthProviders map[string]bool) {
if !setting.ReportingEnabled {
return
}
metricsLogger.Debug(fmt.Sprintf("Sending anonymous usage stats to %s", usageStatsURL))
version := strings.Replace(setting.BuildVersion, ".", "_", -1)
metrics := map[string]interface{}{}
report := map[string]interface{}{
"version": version,
"metrics": metrics,
"os": runtime.GOOS,
"arch": runtime.GOARCH,
"edition": getEdition(),
"packaging": setting.Packaging,
}
statsQuery := models.GetSystemStatsQuery{}
if err := uss.Bus.Dispatch(&statsQuery); err != nil {
metricsLogger.Error("Failed to get system stats", "error", err)
return
}
metrics["stats.dashboards.count"] = statsQuery.Result.Dashboards
metrics["stats.users.count"] = statsQuery.Result.Users
metrics["stats.orgs.count"] = statsQuery.Result.Orgs
metrics["stats.playlist.count"] = statsQuery.Result.Playlists
metrics["stats.plugins.apps.count"] = len(plugins.Apps)
metrics["stats.plugins.panels.count"] = len(plugins.Panels)
metrics["stats.plugins.datasources.count"] = len(plugins.DataSources)
metrics["stats.alerts.count"] = statsQuery.Result.Alerts
metrics["stats.active_users.count"] = statsQuery.Result.ActiveUsers
metrics["stats.datasources.count"] = statsQuery.Result.Datasources
metrics["stats.stars.count"] = statsQuery.Result.Stars
metrics["stats.folders.count"] = statsQuery.Result.Folders
metrics["stats.dashboard_permissions.count"] = statsQuery.Result.DashboardPermissions
metrics["stats.folder_permissions.count"] = statsQuery.Result.FolderPermissions
metrics["stats.provisioned_dashboards.count"] = statsQuery.Result.ProvisionedDashboards
metrics["stats.snapshots.count"] = statsQuery.Result.Snapshots
metrics["stats.teams.count"] = statsQuery.Result.Teams
metrics["stats.total_sessions.count"] = statsQuery.Result.Sessions
userCount := statsQuery.Result.Users
avgSessionsPerUser := statsQuery.Result.Sessions
if userCount != 0 {
avgSessionsPerUser = avgSessionsPerUser / userCount
}
metrics["stats.avg_sessions_per_user.count"] = avgSessionsPerUser
dsStats := models.GetDataSourceStatsQuery{}
if err := uss.Bus.Dispatch(&dsStats); err != nil {
metricsLogger.Error("Failed to get datasource stats", "error", err)
return
}
// send counters for each data source
// but ignore any custom data sources
// as sending that name could be sensitive information
dsOtherCount := 0
for _, dsStat := range dsStats.Result {
if models.IsKnownDataSourcePlugin(dsStat.Type) {
metrics["stats.ds."+dsStat.Type+".count"] = dsStat.Count
} else {
dsOtherCount += dsStat.Count
}
}
metrics["stats.ds.other.count"] = dsOtherCount
metrics["stats.packaging."+setting.Packaging+".count"] = 1
dsAccessStats := models.GetDataSourceAccessStatsQuery{}
if err := uss.Bus.Dispatch(&dsAccessStats); err != nil {
metricsLogger.Error("Failed to get datasource access stats", "error", err)
return
}
// send access counters for each data source
// but ignore any custom data sources
// as sending that name could be sensitive information
dsAccessOtherCount := make(map[string]int64)
for _, dsAccessStat := range dsAccessStats.Result {
if dsAccessStat.Access == "" {
continue
}
access := strings.ToLower(dsAccessStat.Access)
if models.IsKnownDataSourcePlugin(dsAccessStat.Type) {
metrics["stats.ds_access."+dsAccessStat.Type+"."+access+".count"] = dsAccessStat.Count
} else {
old := dsAccessOtherCount[access]
dsAccessOtherCount[access] = old + dsAccessStat.Count
}
}
for access, count := range dsAccessOtherCount {
metrics["stats.ds_access.other."+access+".count"] = count
}
anStats := models.GetAlertNotifierUsageStatsQuery{}
if err := uss.Bus.Dispatch(&anStats); err != nil {
metricsLogger.Error("Failed to get alert notification stats", "error", err)
return
}
for _, stats := range anStats.Result {
metrics["stats.alert_notifiers."+stats.Type+".count"] = stats.Count
}
authTypes := map[string]bool{}
authTypes["anonymous"] = setting.AnonymousEnabled
authTypes["basic_auth"] = setting.BasicAuthEnabled
authTypes["ldap"] = setting.LdapEnabled
authTypes["auth_proxy"] = setting.AuthProxyEnabled
for provider, enabled := range oauthProviders {
authTypes["oauth_"+provider] = enabled
}
for authType, enabled := range authTypes {
enabledValue := 0
if enabled {
enabledValue = 1
}
metrics["stats.auth_enabled."+authType+".count"] = enabledValue
}
out, _ := json.MarshalIndent(report, "", " ")
data := bytes.NewBuffer(out)
client := http.Client{Timeout: 5 * time.Second}
go client.Post(usageStatsURL, "application/json", data)
}
func (uss *UsageStatsService) updateTotalStats() {
statsQuery := models.GetSystemStatsQuery{}
if err := uss.Bus.Dispatch(&statsQuery); err != nil {
metricsLogger.Error("Failed to get system stats", "error", err)
return
}
metrics.M_StatTotal_Dashboards.Set(float64(statsQuery.Result.Dashboards))
metrics.M_StatTotal_Users.Set(float64(statsQuery.Result.Users))
metrics.M_StatActive_Users.Set(float64(statsQuery.Result.ActiveUsers))
metrics.M_StatTotal_Playlists.Set(float64(statsQuery.Result.Playlists))
metrics.M_StatTotal_Orgs.Set(float64(statsQuery.Result.Orgs))
}
func getEdition() string {
if setting.IsEnterprise {
return "enterprise"
} else {
return "oss"
}
}
pkg/metrics/metrics_test.go → pkg/infra/usagestats/usage_stats_test.go
package metrics
package usagestats
import (
"bytes"
......@@ -15,14 +15,21 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
. "github.com/smartystreets/goconvey/convey"
)
func TestMetrics(t *testing.T) {
Convey("Test send usage stats", t, func() {
uss := &UsageStatsService{
Bus: bus.New(),
SQLStore: sqlstore.InitTestDB(t),
}
var getSystemStatsQuery *models.GetSystemStatsQuery
bus.AddHandler("test", func(query *models.GetSystemStatsQuery) error {
uss.Bus.AddHandler(func(query *models.GetSystemStatsQuery) error {
query.Result = &models.SystemStats{
Dashboards: 1,
Datasources: 2,
......@@ -38,13 +45,14 @@ func TestMetrics(t *testing.T) {
ProvisionedDashboards: 12,
Snapshots: 13,
Teams: 14,
Sessions: 15,
}
getSystemStatsQuery = query
return nil
})
var getDataSourceStatsQuery *models.GetDataSourceStatsQuery
bus.AddHandler("test", func(query *models.GetDataSourceStatsQuery) error {
uss.Bus.AddHandler(func(query *models.GetDataSourceStatsQuery) error {
query.Result = []*models.DataSourceStats{
{
Type: models.DS_ES,
......@@ -68,7 +76,7 @@ func TestMetrics(t *testing.T) {
})
var getDataSourceAccessStatsQuery *models.GetDataSourceAccessStatsQuery
bus.AddHandler("test", func(query *models.GetDataSourceAccessStatsQuery) error {
uss.Bus.AddHandler(func(query *models.GetDataSourceAccessStatsQuery) error {
query.Result = []*models.DataSourceAccessStats{
{
Type: models.DS_ES,
......@@ -116,7 +124,7 @@ func TestMetrics(t *testing.T) {
})
var getAlertNotifierUsageStatsQuery *models.GetAlertNotifierUsageStatsQuery
bus.AddHandler("test", func(query *models.GetAlertNotifierUsageStatsQuery) error {
uss.Bus.AddHandler(func(query *models.GetAlertNotifierUsageStatsQuery) error {
query.Result = []*models.NotifierUsageStats{
{
Type: "slack",
......@@ -155,11 +163,11 @@ func TestMetrics(t *testing.T) {
"grafana_com": true,
}
sendUsageStats(oauthProviders)
uss.sendUsageStats(oauthProviders)
Convey("Given reporting not enabled and sending usage stats", func() {
setting.ReportingEnabled = false
sendUsageStats(oauthProviders)
uss.sendUsageStats(oauthProviders)
Convey("Should not gather stats or call http endpoint", func() {
So(getSystemStatsQuery, ShouldBeNil)
......@@ -179,7 +187,7 @@ func TestMetrics(t *testing.T) {
setting.Packaging = "deb"
wg.Add(1)
sendUsageStats(oauthProviders)
uss.sendUsageStats(oauthProviders)
Convey("Should gather stats and call http endpoint", func() {
if waitTimeout(&wg, 2*time.Second) {
......@@ -221,6 +229,8 @@ func TestMetrics(t *testing.T) {
So(metrics.Get("stats.provisioned_dashboards.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.ProvisionedDashboards)
So(metrics.Get("stats.snapshots.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Snapshots)
So(metrics.Get("stats.teams.count").MustInt(), ShouldEqual, getSystemStatsQuery.Result.Teams)
So(metrics.Get("stats.total_sessions.count").MustInt64(), ShouldEqual, 15)
So(metrics.Get("stats.avg_sessions_per_user.count").MustInt64(), ShouldEqual, 5)
So(metrics.Get("stats.ds."+models.DS_ES+".count").MustInt(), ShouldEqual, 9)
So(metrics.Get("stats.ds."+models.DS_PROMETHEUS+".count").MustInt(), ShouldEqual, 10)
......@@ -246,6 +256,7 @@ func TestMetrics(t *testing.T) {
So(metrics.Get("stats.auth_enabled.oauth_grafana_com.count").MustInt(), ShouldEqual, 1)
So(metrics.Get("stats.packaging.deb.count").MustInt(), ShouldEqual, 1)
})
})
......
pkg/metrics/metrics.go
package metrics
import (
"bytes"
"encoding/json"
"net/http"
"runtime"
"strings"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/setting"
"github.com/prometheus/client_golang/prometheus"
)
......@@ -68,23 +59,6 @@ var (
grafanaBuildVersion *prometheus.GaugeVec
)
func newCounterVecStartingAtZero(opts prometheus.CounterOpts, labels []string, labelValues ...string) *prometheus.CounterVec {
counter := prometheus.NewCounterVec(opts, labels)
for _, label := range labelValues {
counter.WithLabelValues(label).Add(0)
}
return counter
}
func newCounterStartingAtZero(opts prometheus.CounterOpts, labelValues ...string) prometheus.Counter {
counter := prometheus.NewCounter(opts)
counter.Add(0)
return counter
}
func init() {
M_Instance_Start = prometheus.NewCounter(prometheus.CounterOpts{
Name: "instance_start_total",
......@@ -362,154 +336,19 @@ func initMetricVars() {
}
func updateTotalStats() {
statsQuery := models.GetSystemStatsQuery{}
if err := bus.Dispatch(&statsQuery); err != nil {
metricsLogger.Error("Failed to get system stats", "error", err)
return
}
M_StatTotal_Dashboards.Set(float64(statsQuery.Result.Dashboards))
M_StatTotal_Users.Set(float64(statsQuery.Result.Users))
M_StatActive_Users.Set(float64(statsQuery.Result.ActiveUsers))
M_StatTotal_Playlists.Set(float64(statsQuery.Result.Playlists))
M_StatTotal_Orgs.Set(float64(statsQuery.Result.Orgs))
}
var usageStatsURL = "https://stats.grafana.org/grafana-usage-report"
func getEdition() string {
if setting.IsEnterprise {
return "enterprise"
} else {
return "oss"
}
}
func sendUsageStats(oauthProviders map[string]bool) {
if !setting.ReportingEnabled {
return
}
metricsLogger.Debug("Sending anonymous usage stats to stats.grafana.org")
version := strings.Replace(setting.BuildVersion, ".", "_", -1)
metrics := map[string]interface{}{}
report := map[string]interface{}{
"version": version,
"metrics": metrics,
"os": runtime.GOOS,
"arch": runtime.GOARCH,
"edition": getEdition(),
"packaging": setting.Packaging,
}
statsQuery := models.GetSystemStatsQuery{}
if err := bus.Dispatch(&statsQuery); err != nil {
metricsLogger.Error("Failed to get system stats", "error", err)
return
}
metrics["stats.dashboards.count"] = statsQuery.Result.Dashboards
metrics["stats.users.count"] = statsQuery.Result.Users
metrics["stats.orgs.count"] = statsQuery.Result.Orgs
metrics["stats.playlist.count"] = statsQuery.Result.Playlists
metrics["stats.plugins.apps.count"] = len(plugins.Apps)
metrics["stats.plugins.panels.count"] = len(plugins.Panels)
metrics["stats.plugins.datasources.count"] = len(plugins.DataSources)
metrics["stats.alerts.count"] = statsQuery.Result.Alerts
metrics["stats.active_users.count"] = statsQuery.Result.ActiveUsers
metrics["stats.datasources.count"] = statsQuery.Result.Datasources
metrics["stats.stars.count"] = statsQuery.Result.Stars
metrics["stats.folders.count"] = statsQuery.Result.Folders
metrics["stats.dashboard_permissions.count"] = statsQuery.Result.DashboardPermissions
metrics["stats.folder_permissions.count"] = statsQuery.Result.FolderPermissions
metrics["stats.provisioned_dashboards.count"] = statsQuery.Result.ProvisionedDashboards
metrics["stats.snapshots.count"] = statsQuery.Result.Snapshots
metrics["stats.teams.count"] = statsQuery.Result.Teams
dsStats := models.GetDataSourceStatsQuery{}
if err := bus.Dispatch(&dsStats); err != nil {
metricsLogger.Error("Failed to get datasource stats", "error", err)
return
}
// send counters for each data source
// but ignore any custom data sources
// as sending that name could be sensitive information
dsOtherCount := 0
for _, dsStat := range dsStats.Result {
if models.IsKnownDataSourcePlugin(dsStat.Type) {
metrics["stats.ds."+dsStat.Type+".count"] = dsStat.Count
} else {
dsOtherCount += dsStat.Count
}
}
metrics["stats.ds.other.count"] = dsOtherCount
metrics["stats.packaging."+setting.Packaging+".count"] = 1
dsAccessStats := models.GetDataSourceAccessStatsQuery{}
if err := bus.Dispatch(&dsAccessStats); err != nil {
metricsLogger.Error("Failed to get datasource access stats", "error", err)
return
}
// send access counters for each data source
// but ignore any custom data sources
// as sending that name could be sensitive information
dsAccessOtherCount := make(map[string]int64)
for _, dsAccessStat := range dsAccessStats.Result {
if dsAccessStat.Access == "" {
continue
}
access := strings.ToLower(dsAccessStat.Access)
if models.IsKnownDataSourcePlugin(dsAccessStat.Type) {
metrics["stats.ds_access."+dsAccessStat.Type+"."+access+".count"] = dsAccessStat.Count
} else {
old := dsAccessOtherCount[access]
dsAccessOtherCount[access] = old + dsAccessStat.Count
}
}
for access, count := range dsAccessOtherCount {
metrics["stats.ds_access.other."+access+".count"] = count
}
anStats := models.GetAlertNotifierUsageStatsQuery{}
if err := bus.Dispatch(&anStats); err != nil {
metricsLogger.Error("Failed to get alert notification stats", "error", err)
return
}
for _, stats := range anStats.Result {
metrics["stats.alert_notifiers."+stats.Type+".count"] = stats.Count
}
authTypes := map[string]bool{}
authTypes["anonymous"] = setting.AnonymousEnabled
authTypes["basic_auth"] = setting.BasicAuthEnabled
authTypes["ldap"] = setting.LdapEnabled
authTypes["auth_proxy"] = setting.AuthProxyEnabled
func newCounterVecStartingAtZero(opts prometheus.CounterOpts, labels []string, labelValues ...string) *prometheus.CounterVec {
counter := prometheus.NewCounterVec(opts, labels)
for provider, enabled := range oauthProviders {
authTypes["oauth_"+provider] = enabled
for _, label := range labelValues {
counter.WithLabelValues(label).Add(0)
}
for authType, enabled := range authTypes {
enabledValue := 0
if enabled {
enabledValue = 1
}
metrics["stats.auth_enabled."+authType+".count"] = enabledValue
}
return counter
}
out, _ := json.MarshalIndent(report, "", " ")
data := bytes.NewBuffer(out)
func newCounterStartingAtZero(opts prometheus.CounterOpts, labelValues ...string) prometheus.Counter {
counter := prometheus.NewCounter(opts)
counter.Add(0)
client := http.Client{Timeout: 5 * time.Second}
go client.Post(usageStatsURL, "application/json", data)
return counter
}
pkg/metrics/service.go
......@@ -2,7 +2,6 @@ package metrics
import (
"context"
"time"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/metrics/graphitebridge"
......@@ -30,7 +29,6 @@ type InternalMetricsService struct {
intervalSeconds int64
graphiteCfg *graphitebridge.Config
oauthProviders map[string]bool
}
func (im *InternalMetricsService) Init() error {
......@@ -50,22 +48,6 @@ func (im *InternalMetricsService) Run(ctx context.Context) error {
M_Instance_Start.Inc()
// set the total stats gauges before we publishing metrics
updateTotalStats()
onceEveryDayTick := time.NewTicker(time.Hour * 24)
everyMinuteTicker := time.NewTicker(time.Minute)
defer onceEveryDayTick.Stop()
defer everyMinuteTicker.Stop()
for {
select {
case <-onceEveryDayTick.C:
sendUsageStats(im.oauthProviders)
case <-everyMinuteTicker.C:
updateTotalStats()
case <-ctx.Done():
return ctx.Err()
}
}
<-ctx.Done()
return ctx.Err()
}
pkg/metrics/settings.go
......@@ -5,8 +5,6 @@ import (
"strings"
"time"
"github.com/grafana/grafana/pkg/social"
"github.com/grafana/grafana/pkg/metrics/graphitebridge"
"github.com/grafana/grafana/pkg/setting"
"github.com/prometheus/client_golang/prometheus"
......@@ -24,8 +22,6 @@ func (im *InternalMetricsService) readSettings() error {
return fmt.Errorf("Unable to parse metrics graphite section, %v", err)
}
im.oauthProviders = social.GetOAuthProviders(im.Cfg)
return nil
}
......
pkg/models/stats.go
......@@ -15,6 +15,7 @@ type SystemStats struct {
FolderPermissions int64
Folders int64
ProvisionedDashboards int64
Sessions int64
}
type DataSourceStats struct {
......
pkg/services/sqlstore/stats.go
......@@ -74,7 +74,8 @@ func GetSystemStats(query *m.GetSystemStatsQuery) error {
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + `) AS provisioned_dashboards,`)
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + `) AS snapshots,`)
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("team") + `) AS teams`)
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("team") + `) AS teams,`)
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("user_auth_token") + `) AS sessions`)
var stats m.SystemStats
_, err := x.SQL(sb.GetSqlString(), sb.params...).Get(&stats)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment