Merge branch 'feat-9671' of https://github.com/alexanderzobnin/grafana into…

Merge branch 'feat-9671' of https://github.com/alexanderzobnin/grafana into alexanderzobnin-feat-9671

.gitignore

@@ -45,6 +45,7 @@ profile.cov
 /pkg/cmd/grafana-cli/grafana-cli
 /pkg/cmd/grafana-cli/grafana-cli
 /pkg/cmd/grafana-server/grafana-server
 /pkg/cmd/grafana-server/grafana-server
 /pkg/cmd/grafana-server/debug
 /pkg/cmd/grafana-server/debug
+debug.test
 /examples/*/dist
 /examples/*/dist
 /packaging/**/*.rpm
 /packaging/**/*.rpm
 /packaging/**/*.deb
 /packaging/**/*.deb

conf/defaults.ini

@@ -174,6 +174,7 @@ disable_gravatar = false
 # data source proxy whitelist (ip_or_domain:port separated by spaces)
 # data source proxy whitelist (ip_or_domain:port separated by spaces)
 data_source_proxy_whitelist =
 data_source_proxy_whitelist =
 
 
+#################################### Snapshots ###########################
 [snapshots]
 [snapshots]
 # snapshot sharing options
 # snapshot sharing options
 external_enabled = true
 external_enabled = true
@@ -186,7 +187,13 @@ snapshot_remove_expired = true
 # remove snapshots after 90 days
 # remove snapshots after 90 days
 snapshot_TTL_days = 90
 snapshot_TTL_days = 90
 
 
-#################################### Users ####################################
+#################################### Dashboards History ##################
+[dashboards]
+# A setting of 20 (default) means only the last 20 versions will be stored and older versions removed.
+# To keep all dashboard versions you can set this value to 2147483647.
+versions_to_keep = 20
+
+#################################### Users ###############################
 [users]
 [users]
 # disable user signup / registration
 # disable user signup / registration
 allow_sign_up = false
 allow_sign_up = false

conf/sample.ini

@@ -162,6 +162,7 @@ log_queries =
 # data source proxy whitelist (ip_or_domain:port separated by spaces)
 # data source proxy whitelist (ip_or_domain:port separated by spaces)
 ;data_source_proxy_whitelist =
 ;data_source_proxy_whitelist =
 
 
+#################################### Snapshots ###########################
 [snapshots]
 [snapshots]
 # snapshot sharing options
 # snapshot sharing options
 ;external_enabled = true
 ;external_enabled = true
@@ -174,7 +175,13 @@ log_queries =
 # remove snapshots after 90 days
 # remove snapshots after 90 days
 ;snapshot_TTL_days = 90
 ;snapshot_TTL_days = 90
 
 
-#################################### Users ####################################
+#################################### Dashboards History ##################
+[dashboards]
+# A setting of 20 (default) means only the last 20 versions will be stored and older versions removed.
+# To keep all dashboard versions you can set this value to 2147483647.
+;versions_to_keep = 20
+
+#################################### Users ###############################
 [users]
 [users]
 # disable user signup / registration
 # disable user signup / registration
 ;allow_sign_up = true
 ;allow_sign_up = true

pkg/models/dashboard_version.go

@@ -69,3 +69,10 @@ type GetDashboardVersionsQuery struct {
 
 
 	Result []*DashboardVersionDTO
 	Result []*DashboardVersionDTO
 }
 }
+
+//
+// Commands
+//
+
+type DeleteExpiredVersionsCommand struct {
+}

pkg/services/cleanup/cleanup.go

@@ -45,6 +45,7 @@ func (service *CleanUpService) start(ctx context.Context) error {
 		case <-ticker.C:
 		case <-ticker.C:
 			service.cleanUpTmpFiles()
 			service.cleanUpTmpFiles()
 			service.deleteExpiredSnapshots()
 			service.deleteExpiredSnapshots()
+			service.deleteExpiredDashboardVersions()
 		case <-ctx.Done():
 		case <-ctx.Done():
 			return ctx.Err()
 			return ctx.Err()
 		}
 		}
@@ -83,3 +84,7 @@ func (service *CleanUpService) cleanUpTmpFiles() {
 func (service *CleanUpService) deleteExpiredSnapshots() {
 func (service *CleanUpService) deleteExpiredSnapshots() {
 	bus.Dispatch(&m.DeleteExpiredSnapshotsCommand{})
 	bus.Dispatch(&m.DeleteExpiredSnapshotsCommand{})
 }
 }
+
+func (service *CleanUpService) deleteExpiredDashboardVersions() {
+	bus.Dispatch(&m.DeleteExpiredVersionsCommand{})
+}

pkg/services/sqlstore/dashboard_version.go

 package sqlstore
 package sqlstore
 
 
 import (
 import (
+	"math"
+	"strings"
+
 	"github.com/grafana/grafana/pkg/bus"
 	"github.com/grafana/grafana/pkg/bus"
 	m "github.com/grafana/grafana/pkg/models"
 	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/setting"
 )
 )
 
 
 func init() {
 func init() {
 	bus.AddHandler("sql", GetDashboardVersion)
 	bus.AddHandler("sql", GetDashboardVersion)
 	bus.AddHandler("sql", GetDashboardVersions)
 	bus.AddHandler("sql", GetDashboardVersions)
+	bus.AddHandler("sql", DeleteExpiredVersions)
 }
 }
 
 
 // GetDashboardVersion gets the dashboard version for the given dashboard ID and version number.
 // GetDashboardVersion gets the dashboard version for the given dashboard ID and version number.
@@ -58,3 +63,71 @@ func GetDashboardVersions(query *m.GetDashboardVersionsQuery) error {
 	}
 	}
 	return nil
 	return nil
 }
 }
+
+func DeleteExpiredVersions(cmd *m.DeleteExpiredVersionsCommand) error {
+	return inTransaction(func(sess *DBSession) error {
+		var expiredCount int64 = 0
+		var versions []DashboardVersionExp
+
+		// Don't clean up if user set versions_to_keep to 2147483647 (MaxInt32)
+		if versionsToKeep := setting.DashboardVersionsToKeep; versionsToKeep < math.MaxInt32 {
+			err := sess.Table("dashboard_version").
+				Select("dashboard_version.id, dashboard_version.version, dashboard_version.dashboard_id").
+				Where(`dashboard_id IN (
+					SELECT dashboard_id FROM dashboard_version
+					GROUP BY dashboard_id HAVING COUNT(dashboard_version.id) > ?
+				)`, versionsToKeep).
+				Desc("dashboard_version.dashboard_id", "dashboard_version.version").
+				Find(&versions)
+
+			if err != nil {
+				return err
+			}
+
+			// Keep last versionsToKeep versions and delete other
+			versionIdsToDelete := getVersionIDsToDelete(versions, versionsToKeep)
+			if len(versionIdsToDelete) > 0 {
+				deleteExpiredSql := `DELETE FROM dashboard_version WHERE id IN (?` + strings.Repeat(",?", len(versionIdsToDelete)-1) + `)`
+				expiredResponse, err := sess.Exec(deleteExpiredSql, versionIdsToDelete...)
+				if err != nil {
+					return err
+				}
+				expiredCount, _ = expiredResponse.RowsAffected()
+			}
+		}
+
+		sqlog.Debug("Deleted old/expired dashboard versions", "expired", expiredCount)
+		return nil
+	})
+}
+
+// Short version of DashboardVersion for getting expired versions
+type DashboardVersionExp struct {
+	Id          int64 `json:"id"`
+	DashboardId int64 `json:"dashboardId"`
+	Version     int   `json:"version"`
+}
+
+func getVersionIDsToDelete(versions []DashboardVersionExp, versionsToKeep int) []interface{} {
+	versionIds := make([]interface{}, 0)
+
+	if len(versions) == 0 {
+		return versionIds
+	}
+
+	currentDashboard := versions[0].DashboardId
+	count := 0
+	for _, v := range versions {
+		if v.DashboardId == currentDashboard {
+			count++
+		} else {
+			count = 1
+			currentDashboard = v.DashboardId
+		}
+		if count > versionsToKeep {
+			versionIds = append(versionIds, v.Id)
+		}
+	}
+
+	return versionIds
+}

pkg/services/sqlstore/dashboard_version_test.go

@@ -8,6 +8,7 @@ import (
 
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	m "github.com/grafana/grafana/pkg/models"
 	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/setting"
 )
 )
 
 
 func updateTestDashboard(dashboard *m.Dashboard, data map[string]interface{}) {
 func updateTestDashboard(dashboard *m.Dashboard, data map[string]interface{}) {
@@ -101,3 +102,44 @@ func TestGetDashboardVersions(t *testing.T) {
 		})
 		})
 	})
 	})
 }
 }
+
+func TestDeleteExpiredVersions(t *testing.T) {
+	Convey("Testing dashboard versions clean up", t, func() {
+		InitTestDB(t)
+		versionsToKeep := 5
+		versionsToWrite := 10
+		setting.DashboardVersionsToKeep = versionsToKeep
+
+		savedDash := insertTestDashboard("test dash 53", 1, "diff-all")
+		for i := 0; i < versionsToWrite-1; i++ {
+			updateTestDashboard(savedDash, map[string]interface{}{
+				"tags": "different-tag",
+			})
+		}
+
+		Convey("Clean up old dashboard versions", func() {
+			err := DeleteExpiredVersions(&m.DeleteExpiredVersionsCommand{})
+			So(err, ShouldBeNil)
+
+			query := m.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1}
+			GetDashboardVersions(&query)
+
+			So(len(query.Result), ShouldEqual, versionsToKeep)
+			// Ensure latest versions were kept
+			So(query.Result[versionsToKeep-1].Version, ShouldEqual, versionsToWrite-versionsToKeep+1)
+			So(query.Result[0].Version, ShouldEqual, versionsToWrite)
+		})
+
+		Convey("Don't delete anything if there're no expired versions", func() {
+			setting.DashboardVersionsToKeep = versionsToWrite
+
+			err := DeleteExpiredVersions(&m.DeleteExpiredVersionsCommand{})
+			So(err, ShouldBeNil)
+
+			query := m.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1}
+			GetDashboardVersions(&query)
+
+			So(len(query.Result), ShouldEqual, versionsToWrite)
+		})
+	})
+}

pkg/setting/setting.go

@@ -90,6 +90,9 @@ var (
 	SnapShotTTLDays       int
 	SnapShotTTLDays       int
 	SnapShotRemoveExpired bool
 	SnapShotRemoveExpired bool
 
 
+	// Dashboard history
+	DashboardVersionsToKeep int
+
 	// User settings
 	// User settings
 	AllowUserSignUp         bool
 	AllowUserSignUp         bool
 	AllowUserOrgCreate      bool
 	AllowUserOrgCreate      bool
@@ -520,6 +523,10 @@ func NewConfigContext(args *CommandLineArgs) error {
 	SnapShotRemoveExpired = snapshots.Key("snapshot_remove_expired").MustBool(true)
 	SnapShotRemoveExpired = snapshots.Key("snapshot_remove_expired").MustBool(true)
 	SnapShotTTLDays = snapshots.Key("snapshot_TTL_days").MustInt(90)
 	SnapShotTTLDays = snapshots.Key("snapshot_TTL_days").MustInt(90)
 
 
+	// read dashboard settings
+	dashboards := Cfg.Section("dashboards")
+	DashboardVersionsToKeep = dashboards.Key("versions_to_keep").MustInt(20)
+
 	//  read data source proxy white list
 	//  read data source proxy white list
 	DataProxyWhiteList = make(map[string]bool)
 	DataProxyWhiteList = make(map[string]bool)
 	for _, hostAndIp := range util.SplitString(security.Key("data_source_proxy_whitelist").String()) {
 	for _, hostAndIp := range util.SplitString(security.Key("data_source_proxy_whitelist").String()) {