feat: purge old files and snapshots

closes #4087
closes #2172

conf/defaults.ini

@@ -161,6 +161,12 @@ external_enabled = true
 external_snapshot_url = https://snapshots-origin.raintank.io
 external_snapshot_name = Publish to snapshot.raintank.io
 
+# remove expired snapshot
+snapshot_remove_expired = true
+
+# remove snapshots after 90 days
+snapshot_TTL_days = 90
+
 #################################### Users ####################################
 [users]
 # disable user signup / registration
@@ -267,6 +273,9 @@ from_address = admin@grafana.localhost
 welcome_email_on_sign_up = false
 templates_pattern = emails/*.html
 
+[tmp.files]
+rendered_image_ttl_days = 14
+
 #################################### Logging ##########################
 [log]
 # Either "console", "file", "syslog". Default is console and  file

conf/sample.ini

@@ -149,6 +149,12 @@ check_for_updates = true
 ;external_snapshot_url = https://snapshots-origin.raintank.io
 ;external_snapshot_name = Publish to snapshot.raintank.io
 
+# remove expired snapshot
+;snapshot_remove_expired = true
+
+# remove snapshots after 90 days
+;snapshot_TTL_days = 90
+
 #################################### Users ####################################
 [users]
 # disable user signup / registration

pkg/cmd/grafana-server/main.go

@@ -17,6 +17,7 @@ import (
 	"github.com/grafana/grafana/pkg/metrics"
 	"github.com/grafana/grafana/pkg/plugins"
 	alertingInit "github.com/grafana/grafana/pkg/services/alerting/init"
+	"github.com/grafana/grafana/pkg/services/backgroundtasks"
 	"github.com/grafana/grafana/pkg/services/eventpublisher"
 	"github.com/grafana/grafana/pkg/services/notifications"
 	"github.com/grafana/grafana/pkg/services/search"
@@ -62,13 +63,13 @@ func main() {
 	writePIDFile()
 	initRuntime()
 	metrics.Init()
-
 	search.Init()
 	login.Init()
 	social.NewOAuthService()
 	eventpublisher.Init()
 	plugins.Init()
 	alertingInit.Init()
+	backgroundtasks.Init()
 
 	if err := notifications.Init(); err != nil {
 		log.Fatal(3, "Notification service failed to initialize", err)

pkg/models/timer.go

+package models
+
+import "time"
+
+type HourCommand struct {
+	Time time.Time
+}

pkg/services/backgroundtasks/background_tasks.go

+//"I want to be a cleaner, just like you," said Mathilda
+//"Okay," replied Leon
+
+package backgroundtasks
+
+import (
+	"time"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/log"
+	"github.com/grafana/grafana/pkg/models"
+)
+
+var (
+	tlog log.Logger = log.New("ticker")
+)
+
+func Init() {
+	go start()
+}
+
+func start() {
+	go cleanup(time.Now())
+
+	ticker := time.NewTicker(time.Hour * 1)
+	for {
+		select {
+		case tick := <-ticker.C:
+			go cleanup(tick)
+		}
+	}
+}
+
+func cleanup(now time.Time) {
+	err := bus.Publish(&models.HourCommand{Time: now})
+	if err != nil {
+		tlog.Error("Cleanup job failed", "error", err)
+	}
+}

pkg/services/backgroundtasks/remove_tmp_images.go

+package backgroundtasks
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+func init() {
+	bus.AddEventListener(CleanTmpFiles)
+}
+
+func CleanTmpFiles(cmd *models.HourCommand) error {
+	files, err := ioutil.ReadDir(setting.ImagesDir)
+
+	var toDelete []os.FileInfo
+	for _, file := range files {
+		if file.ModTime().AddDate(0, 0, setting.RenderedImageTTLDays).Before(cmd.Time) {
+			toDelete = append(toDelete, file)
+		}
+	}
+
+	for _, file := range toDelete {
+		fullPath := path.Join(setting.ImagesDir, file.Name())
+		err := os.Remove(fullPath)
+		if err != nil {
+			return err
+		}
+	}
+
+	tlog.Debug("Found old rendered image to delete", "deleted", len(toDelete), "keept", len(files))
+
+	return err
+}

pkg/services/sqlstore/dashboard_snapshot.go

@@ -5,7 +5,9 @@ import (
 
 	"github.com/go-xorm/xorm"
 	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/log"
 	m "github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/setting"
 )
 
 func init() {
@@ -13,6 +15,31 @@ func init() {
 	bus.AddHandler("sql", GetDashboardSnapshot)
 	bus.AddHandler("sql", DeleteDashboardSnapshot)
 	bus.AddHandler("sql", SearchDashboardSnapshots)
+	bus.AddEventListener(DeleteExpiredSnapshots)
+}
+
+func DeleteExpiredSnapshots(cmd *m.HourCommand) error {
+	return inTransaction(func(sess *xorm.Session) error {
+		var expiredCount int64 = 0
+		var oldCount int64 = 0
+
+		if setting.SnapShotRemoveExpired {
+			deleteExpiredSql := "DELETE FROM dashboard_snapshot WHERE expires < ?"
+			expiredResponse, err := x.Exec(deleteExpiredSql, cmd.Time)
+			if err != nil {
+				return err
+			}
+			expiredCount, _ = expiredResponse.RowsAffected()
+		}
+
+		oldSnapshotsSql := "DELETE FROM dashboard_snapshot WHERE created < ?"
+		oldResponse, err := x.Exec(oldSnapshotsSql, cmd.Time.AddDate(0, 0, setting.SnapShotTTLDays*-1))
+		oldCount, _ = oldResponse.RowsAffected()
+
+		log.Debug2("Deleted old/expired snaphots", "to old", oldCount, "expired", expiredCount)
+
+		return err
+	})
 }
 
 func CreateDashboardSnapshot(cmd *m.CreateDashboardSnapshotCommand) error {

pkg/setting/setting.go

@@ -81,6 +81,8 @@ var (
 	ExternalSnapshotUrl   string
 	ExternalSnapshotName  string
 	ExternalEnabled       bool
+	SnapShotTTLDays       int
+	SnapShotRemoveExpired bool
 
 	// User settings
 	AllowUserSignUp    bool
@@ -120,6 +122,7 @@ var (
 	// PhantomJs Rendering
 	ImagesDir            string
 	PhantomDir           string
+	RenderedImageTTLDays int
 
 	// for logging purposes
 	configFiles                  []string
@@ -495,6 +498,8 @@ func NewConfigContext(args *CommandLineArgs) error {
 	ExternalSnapshotUrl = snapshots.Key("external_snapshot_url").String()
 	ExternalSnapshotName = snapshots.Key("external_snapshot_name").String()
 	ExternalEnabled = snapshots.Key("external_enabled").MustBool(true)
+	SnapShotRemoveExpired = snapshots.Key("snapshot_remove_expired").MustBool(true)
+	SnapShotTTLDays = snapshots.Key("snapshot_TTL_days").MustInt(90)
 
 	//  read data source proxy white list
 	DataProxyWhiteList = make(map[string]bool)
@@ -535,6 +540,9 @@ func NewConfigContext(args *CommandLineArgs) error {
 	ImagesDir = filepath.Join(DataPath, "png")
 	PhantomDir = filepath.Join(HomePath, "vendor/phantomjs")
 
+	tmpFilesSection := Cfg.Section("tmp.files")
+	RenderedImageTTLDays = tmpFilesSection.Key("rendered_image_ttl_days").MustInt(14)
+
 	analytics := Cfg.Section("analytics")
 	ReportingEnabled = analytics.Key("reporting_enabled").MustBool(true)
 	CheckForUpdates = analytics.Key("check_for_updates").MustBool(true)