Merge pull request #10373 from bergquist/dashboard_folder_provisioning

Dashboards as cfg: create dashboard folder if missing

pkg/models/dashboards.go

@@ -139,8 +139,12 @@ func (dash *Dashboard) GetString(prop string, defaultValue string) string {
 
 // UpdateSlug updates the slug
 func (dash *Dashboard) UpdateSlug() {
-	title := strings.ToLower(dash.Data.Get("title").MustString())
-	dash.Slug = slug.Make(title)
+	title := dash.Data.Get("title").MustString()
+	dash.Slug = SlugifyTitle(title)
+}
+
+func SlugifyTitle(title string) string {
+	return slug.Make(strings.ToLower(title))
 }
 
 //

pkg/models/dashboards_test.go

@@ -16,6 +16,12 @@ func TestDashboardModel(t *testing.T) {
 		So(dashboard.Slug, ShouldEqual, "grafana-play-home")
 	})
 
+	Convey("Can slugify title", t, func() {
+		slug := SlugifyTitle("Grafana Play Home")
+
+		So(slug, ShouldEqual, "grafana-play-home")
+	})
+
 	Convey("Given a dashboard json", t, func() {
 		json := simplejson.New()
 		json.Set("title", "test dash")

pkg/services/provisioning/dashboards/dashboard_cache.go

+package dashboards
+
+import (
+	"github.com/grafana/grafana/pkg/services/dashboards"
+	gocache "github.com/patrickmn/go-cache"
+	"time"
+)
+
+type dashboardCache struct {
+	internalCache *gocache.Cache
+}
+
+func NewDashboardCache() *dashboardCache {
+	return &dashboardCache{internalCache: gocache.New(5*time.Minute, 30*time.Minute)}
+}
+
+func (fr *dashboardCache) addDashboardCache(key string, json *dashboards.SaveDashboardItem) {
+	fr.internalCache.Add(key, json, time.Minute*10)
+}
+
+func (fr *dashboardCache) getCache(key string) (*dashboards.SaveDashboardItem, bool) {
+	obj, exist := fr.internalCache.Get(key)
+	if !exist {
+		return nil, exist
+	}
+
+	dash, ok := obj.(*dashboards.SaveDashboardItem)
+	if !ok {
+		return nil, ok
+	}
+
+	return dash, ok
+}

pkg/services/provisioning/dashboards/file_reader.go

@@ -2,6 +2,7 @@ package dashboards
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -15,7 +16,12 @@ import (
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/log"
 	"github.com/grafana/grafana/pkg/models"
-	gocache "github.com/patrickmn/go-cache"
+)
+
+var (
+	checkDiskForChangesInterval time.Duration = time.Second * 3
+
+	ErrFolderNameMissing error = errors.New("Folder name missing")
 )
 
 type fileReader struct {
@@ -23,7 +29,8 @@ type fileReader struct {
 	Path          string
 	log           log.Logger
 	dashboardRepo dashboards.Repository
-	cache         *gocache.Cache
+	cache         *dashboardCache
+	createWalk    func(fr *fileReader, folderId int64) filepath.WalkFunc
 }
 
 func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReader, error) {
@@ -41,32 +48,15 @@ func NewDashboardFileReader(cfg *DashboardsAsConfig, log log.Logger) (*fileReade
 		Path:          path,
 		log:           log,
 		dashboardRepo: dashboards.GetRepository(),
-		cache:         gocache.New(5*time.Minute, 30*time.Minute),
+		cache:         NewDashboardCache(),
+		createWalk:    createWalkFn,
 	}, nil
 }
 
-func (fr *fileReader) addCache(key string, json *dashboards.SaveDashboardItem) {
-	fr.cache.Add(key, json, time.Minute*10)
-}
-
-func (fr *fileReader) getCache(key string) (*dashboards.SaveDashboardItem, bool) {
-	obj, exist := fr.cache.Get(key)
-	if !exist {
-		return nil, exist
-	}
-
-	dash, ok := obj.(*dashboards.SaveDashboardItem)
-	if !ok {
-		return nil, ok
-	}
-
-	return dash, ok
-}
-
 func (fr *fileReader) ReadAndListen(ctx context.Context) error {
-	ticker := time.NewTicker(time.Second * 3)
+	ticker := time.NewTicker(checkDiskForChangesInterval)
 
-	if err := fr.walkFolder(); err != nil {
+	if err := fr.startWalkingDisk(); err != nil {
 		fr.log.Error("failed to search for dashboards", "error", err)
 	}
 
@@ -78,7 +68,9 @@ func (fr *fileReader) ReadAndListen(ctx context.Context) error {
 			if !running { // avoid walking the filesystem in parallel. incase fs is very slow.
 				running = true
 				go func() {
-					fr.walkFolder()
+					if err := fr.startWalkingDisk(); err != nil {
+						fr.log.Error("failed to search for dashboards", "error", err)
+					}
 					running = false
 				}()
 			}
@@ -88,14 +80,57 @@ func (fr *fileReader) ReadAndListen(ctx context.Context) error {
 	}
 }
 
-func (fr *fileReader) walkFolder() error {
+func (fr *fileReader) startWalkingDisk() error {
 	if _, err := os.Stat(fr.Path); err != nil {
 		if os.IsNotExist(err) {
 			return err
 		}
 	}
 
-	return filepath.Walk(fr.Path, func(path string, fileInfo os.FileInfo, err error) error {
+	folderId, err := getOrCreateFolderId(fr.Cfg, fr.dashboardRepo)
+	if err != nil && err != ErrFolderNameMissing {
+		return err
+	}
+
+	return filepath.Walk(fr.Path, fr.createWalk(fr, folderId))
+}
+
+func getOrCreateFolderId(cfg *DashboardsAsConfig, repo dashboards.Repository) (int64, error) {
+	if cfg.Folder == "" {
+		return 0, ErrFolderNameMissing
+	}
+
+	cmd := &models.GetDashboardQuery{Slug: models.SlugifyTitle(cfg.Folder), OrgId: cfg.OrgId}
+	err := bus.Dispatch(cmd)
+
+	if err != nil && err != models.ErrDashboardNotFound {
+		return 0, err
+	}
+
+	// dashboard folder not found. create one.
+	if err == models.ErrDashboardNotFound {
+		dash := &dashboards.SaveDashboardItem{}
+		dash.Dashboard = models.NewDashboard(cfg.Folder)
+		dash.Dashboard.IsFolder = true
+		dash.Overwrite = true
+		dash.OrgId = cfg.OrgId
+		dbDash, err := repo.SaveDashboard(dash)
+		if err != nil {
+			return 0, err
+		}
+
+		return dbDash.Id, nil
+	}
+
+	if !cmd.Result.IsFolder {
+		return 0, fmt.Errorf("Got invalid response. Expected folder, found dashboard")
+	}
+
+	return cmd.Result.Id, nil
+}
+
+func createWalkFn(fr *fileReader, folderId int64) filepath.WalkFunc {
+	return func(path string, fileInfo os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -110,12 +145,12 @@ func (fr *fileReader) walkFolder() error {
 			return nil
 		}
 
-		cachedDashboard, exist := fr.getCache(path)
+		cachedDashboard, exist := fr.cache.getCache(path)
 		if exist && cachedDashboard.UpdatedAt == fileInfo.ModTime() {
 			return nil
 		}
 
-		dash, err := fr.readDashboardFromFile(path)
+		dash, err := fr.readDashboardFromFile(path, folderId)
 		if err != nil {
 			fr.log.Error("failed to load dashboard from ", "file", path, "error", err)
 			return nil
@@ -147,10 +182,10 @@ func (fr *fileReader) walkFolder() error {
 		fr.log.Debug("loading dashboard from disk into database.", "file", path)
 		_, err = fr.dashboardRepo.SaveDashboard(dash)
 		return err
-	})
+	}
 }
 
-func (fr *fileReader) readDashboardFromFile(path string) (*dashboards.SaveDashboardItem, error) {
+func (fr *fileReader) readDashboardFromFile(path string, folderId int64) (*dashboards.SaveDashboardItem, error) {
 	reader, err := os.Open(path)
 	if err != nil {
 		return nil, err
@@ -167,12 +202,12 @@ func (fr *fileReader) readDashboardFromFile(path string) (*dashboards.SaveDashbo
 		return nil, err
 	}
 
-	dash, err := createDashboardJson(data, stat.ModTime(), fr.Cfg)
+	dash, err := createDashboardJson(data, stat.ModTime(), fr.Cfg, folderId)
 	if err != nil {
 		return nil, err
 	}
 
-	fr.addCache(path, dash)
+	fr.cache.addDashboardCache(path, dash)
 
 	return dash, nil
 }

pkg/services/provisioning/dashboards/file_reader_test.go

@@ -2,6 +2,7 @@ package dashboards
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -22,7 +23,7 @@ var (
 )
 
 func TestDashboardFileReader(t *testing.T) {
-	Convey("Reading dashboards from disk", t, func() {
+	Convey("Dashboard file reader", t, func() {
 		bus.ClearBusHandlers()
 		fakeRepo = &fakeDashboardRepo{}
 
@@ -30,91 +31,191 @@ func TestDashboardFileReader(t *testing.T) {
 		dashboards.SetRepository(fakeRepo)
 		logger := log.New("test.logger")
 
-		cfg := &DashboardsAsConfig{
-			Name:    "Default",
-			Type:    "file",
-			OrgId:   1,
-			Folder:  "",
-			Options: map[string]interface{}{},
-		}
+		Convey("Reading dashboards from disk", func() {
 
-		Convey("Can read default dashboard", func() {
-			cfg.Options["folder"] = defaultDashboards
+			cfg := &DashboardsAsConfig{
+				Name:    "Default",
+				Type:    "file",
+				OrgId:   1,
+				Folder:  "",
+				Options: map[string]interface{}{},
+			}
 
-			reader, err := NewDashboardFileReader(cfg, logger)
-			So(err, ShouldBeNil)
+			Convey("Can read default dashboard", func() {
+				cfg.Options["folder"] = defaultDashboards
+				cfg.Folder = "Team A"
 
-			err = reader.walkFolder()
-			So(err, ShouldBeNil)
+				reader, err := NewDashboardFileReader(cfg, logger)
+				So(err, ShouldBeNil)
 
-			So(len(fakeRepo.inserted), ShouldEqual, 2)
-		})
+				err = reader.startWalkingDisk()
+				So(err, ShouldBeNil)
+
+				folders := 0
+				dashboards := 0
 
-		Convey("Should not update dashboards when db is newer", func() {
-			cfg.Options["folder"] = oneDashboard
+				for _, i := range fakeRepo.inserted {
+					if i.Dashboard.IsFolder {
+						folders++
+					} else {
+						dashboards++
+					}
+				}
 
-			fakeRepo.getDashboard = append(fakeRepo.getDashboard, &models.Dashboard{
-				Updated: time.Now().Add(time.Hour),
-				Slug:    "grafana",
+				So(dashboards, ShouldEqual, 2)
+				So(folders, ShouldEqual, 1)
 			})
 
-			reader, err := NewDashboardFileReader(cfg, logger)
-			So(err, ShouldBeNil)
+			Convey("Should not update dashboards when db is newer", func() {
+				cfg.Options["folder"] = oneDashboard
 
-			err = reader.walkFolder()
-			So(err, ShouldBeNil)
+				fakeRepo.getDashboard = append(fakeRepo.getDashboard, &models.Dashboard{
+					Updated: time.Now().Add(time.Hour),
+					Slug:    "grafana",
+				})
 
-			So(len(fakeRepo.inserted), ShouldEqual, 0)
-		})
+				reader, err := NewDashboardFileReader(cfg, logger)
+				So(err, ShouldBeNil)
+
+				err = reader.startWalkingDisk()
+				So(err, ShouldBeNil)
+
+				So(len(fakeRepo.inserted), ShouldEqual, 0)
+			})
+
+			Convey("Can read default dashboard and replace old version in database", func() {
+				cfg.Options["folder"] = oneDashboard
 
-		Convey("Can read default dashboard and replace old version in database", func() {
-			cfg.Options["folder"] = oneDashboard
+				stat, _ := os.Stat(oneDashboard + "/dashboard1.json")
 
-			stat, _ := os.Stat(oneDashboard + "/dashboard1.json")
+				fakeRepo.getDashboard = append(fakeRepo.getDashboard, &models.Dashboard{
+					Updated: stat.ModTime().AddDate(0, 0, -1),
+					Slug:    "grafana",
+				})
 
-			fakeRepo.getDashboard = append(fakeRepo.getDashboard, &models.Dashboard{
-				Updated: stat.ModTime().AddDate(0, 0, -1),
-				Slug:    "grafana",
+				reader, err := NewDashboardFileReader(cfg, logger)
+				So(err, ShouldBeNil)
+
+				err = reader.startWalkingDisk()
+				So(err, ShouldBeNil)
+
+				So(len(fakeRepo.inserted), ShouldEqual, 1)
 			})
 
-			reader, err := NewDashboardFileReader(cfg, logger)
-			So(err, ShouldBeNil)
+			Convey("Invalid configuration should return error", func() {
+				cfg := &DashboardsAsConfig{
+					Name:   "Default",
+					Type:   "file",
+					OrgId:  1,
+					Folder: "",
+				}
 
-			err = reader.walkFolder()
-			So(err, ShouldBeNil)
+				_, err := NewDashboardFileReader(cfg, logger)
+				So(err, ShouldNotBeNil)
+			})
 
-			So(len(fakeRepo.inserted), ShouldEqual, 1)
+			Convey("Broken dashboards should not cause error", func() {
+				cfg.Options["folder"] = brokenDashboards
+
+				_, err := NewDashboardFileReader(cfg, logger)
+				So(err, ShouldBeNil)
+			})
 		})
 
-		Convey("Invalid configuration should return error", func() {
+		Convey("Should not create new folder if folder name is missing", func() {
 			cfg := &DashboardsAsConfig{
 				Name:   "Default",
 				Type:   "file",
 				OrgId:  1,
 				Folder: "",
+				Options: map[string]interface{}{
+					"folder": defaultDashboards,
+				},
 			}
 
-			_, err := NewDashboardFileReader(cfg, logger)
-			So(err, ShouldNotBeNil)
+			_, err := getOrCreateFolderId(cfg, fakeRepo)
+			So(err, ShouldEqual, ErrFolderNameMissing)
 		})
 
-		Convey("Broken dashboards should not cause error", func() {
+		Convey("can get or Create dashboard folder", func() {
+			cfg := &DashboardsAsConfig{
+				Name:   "Default",
+				Type:   "file",
+				OrgId:  1,
+				Folder: "TEAM A",
+				Options: map[string]interface{}{
+					"folder": defaultDashboards,
+				},
+			}
+
+			folderId, err := getOrCreateFolderId(cfg, fakeRepo)
+			So(err, ShouldBeNil)
+			inserted := false
+			for _, d := range fakeRepo.inserted {
+				if d.Dashboard.IsFolder && d.Dashboard.Id == folderId {
+					inserted = true
+				}
+			}
+			So(len(fakeRepo.inserted), ShouldEqual, 1)
+			So(inserted, ShouldBeTrue)
+		})
+
+		Convey("Walking the folder with dashboards", func() {
 			cfg := &DashboardsAsConfig{
 				Name:   "Default",
 				Type:   "file",
 				OrgId:  1,
 				Folder: "",
 				Options: map[string]interface{}{
-					"folder": brokenDashboards,
+					"folder": defaultDashboards,
 				},
 			}
 
-			_, err := NewDashboardFileReader(cfg, logger)
+			reader, err := NewDashboardFileReader(cfg, log.New("test-logger"))
 			So(err, ShouldBeNil)
+
+			Convey("should skip dirs that starts with .", func() {
+				shouldSkip := reader.createWalk(reader, 0)("path", &FakeFileInfo{isDirectory: true, name: ".folder"}, nil)
+				So(shouldSkip, ShouldEqual, filepath.SkipDir)
+			})
+
+			Convey("should keep walking if file is not .json", func() {
+				shouldSkip := reader.createWalk(reader, 0)("path", &FakeFileInfo{isDirectory: true, name: "folder"}, nil)
+				So(shouldSkip, ShouldBeNil)
+			})
 		})
 	})
 }
 
+type FakeFileInfo struct {
+	isDirectory bool
+	name        string
+}
+
+func (ffi *FakeFileInfo) IsDir() bool {
+	return ffi.isDirectory
+}
+
+func (ffi FakeFileInfo) Size() int64 {
+	return 1
+}
+
+func (ffi FakeFileInfo) Mode() os.FileMode {
+	return 0777
+}
+
+func (ffi FakeFileInfo) Name() string {
+	return ffi.name
+}
+
+func (ffi FakeFileInfo) ModTime() time.Time {
+	return time.Time{}
+}
+
+func (ffi FakeFileInfo) Sys() interface{} {
+	return nil
+}
+
 type fakeDashboardRepo struct {
 	inserted     []*dashboards.SaveDashboardItem
 	getDashboard []*models.Dashboard

pkg/services/provisioning/dashboards/types.go

@@ -18,14 +18,16 @@ type DashboardsAsConfig struct {
 	Options  map[string]interface{} `json:"options" yaml:"options"`
 }
 
-func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig) (*dashboards.SaveDashboardItem, error) {
-
+func createDashboardJson(data *simplejson.Json, lastModified time.Time, cfg *DashboardsAsConfig, folderId int64) (*dashboards.SaveDashboardItem, error) {
 	dash := &dashboards.SaveDashboardItem{}
 	dash.Dashboard = models.NewDashboardFromJson(data)
 	dash.UpdatedAt = lastModified
 	dash.Overwrite = true
 	dash.OrgId = cfg.OrgId
-	dash.Dashboard.Data.Set("editable", cfg.Editable)
+	dash.Dashboard.FolderId = folderId
+	if !cfg.Editable {
+		dash.Dashboard.Data.Set("editable", cfg.Editable)
+	}
 
 	if dash.Dashboard.Title == "" {
 		return nil, models.ErrDashboardTitleEmpty