Skip to content

N
nexpie-grafana-theme
Commit 151b24b9 by Andrej Ocenas Committed by Carl Bergquist
Options

CLI: Add command to migrate all datasources to use encrypted password fields (#17118) 

closes: #17107
parent b9181df2
Showing with 266 additions and 25 deletions
pkg/cmd/grafana-cli/commands/commands.go
...@@ -7,14 +7,16 @@ import ( ...@@ -7,14 +7,16 @@ import (
"github.com/codegangsta/cli" "github.com/codegangsta/cli"
"github.com/fatih/color" "github.com/fatih/color"
"github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/commands/datamigrations"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/setting"
) )
func runDbCommand(command func(commandLine CommandLine) error) func(context *cli.Context) { func runDbCommand(command func(commandLine utils.CommandLine, sqlStore *sqlstore.SqlStore) error) func(context *cli.Context) {
return func(context *cli.Context) { return func(context *cli.Context) {
cmd := &contextCommandLine{context} cmd := &utils.ContextCommandLine{Context: context}
cfg := setting.NewCfg() cfg := setting.NewCfg()
cfg.Load(&setting.CommandLineArgs{ cfg.Load(&setting.CommandLineArgs{
...@@ -28,7 +30,7 @@ func runDbCommand(command func(commandLine CommandLine) error) func(context *cli ...@@ -28,7 +30,7 @@ func runDbCommand(command func(commandLine CommandLine) error) func(context *cli
engine.Bus = bus.GetBus() engine.Bus = bus.GetBus()
engine.Init() engine.Init()
if err := command(cmd); err != nil { if err := command(cmd, engine); err != nil {
logger.Errorf("\n%s: ", color.RedString("Error")) logger.Errorf("\n%s: ", color.RedString("Error"))
logger.Errorf("%s\n\n", err) logger.Errorf("%s\n\n", err)
...@@ -40,10 +42,10 @@ func runDbCommand(command func(commandLine CommandLine) error) func(context *cli ...@@ -40,10 +42,10 @@ func runDbCommand(command func(commandLine CommandLine) error) func(context *cli
} }
} }
func runPluginCommand(command func(commandLine CommandLine) error) func(context *cli.Context) { func runPluginCommand(command func(commandLine utils.CommandLine) error) func(context *cli.Context) {
return func(context *cli.Context) { return func(context *cli.Context) {
cmd := &contextCommandLine{context} cmd := &utils.ContextCommandLine{Context: context}
if err := command(cmd); err != nil { if err := command(cmd); err != nil {
logger.Errorf("\n%s: ", color.RedString("Error")) logger.Errorf("\n%s: ", color.RedString("Error"))
logger.Errorf("%s %s\n\n", color.RedString("✗"), err) logger.Errorf("%s %s\n\n", color.RedString("✗"), err)
...@@ -107,6 +109,17 @@ var adminCommands = []cli.Command{ ...@@ -107,6 +109,17 @@ var adminCommands = []cli.Command{
}, },
}, },
}, },
{
Name: "data-migration",
Usage: "Runs a script that migrates or cleanups data in your db",
Subcommands: []cli.Command{
{
Name: "encrypt-datasource-passwords",
Usage: "Migrates passwords from unsecured fields to secure_json_data field. Return ok unless there is an error. Safe to execute multiple times.",
Action: runDbCommand(datamigrations.EncryptDatasourcePaswords),
},
},
},
} }
var Commands = []cli.Command{ var Commands = []cli.Command{
......
pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords.go 0 → 100644
package datamigrations
import (
"context"
"encoding/json"
"github.com/fatih/color"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
"github.com/grafana/grafana/pkg/util/errutil"
)
var (
datasourceTypes = []string{
"mysql",
"influxdb",
"elasticsearch",
"graphite",
"prometheus",
"opentsdb",
}
)
// EncryptDatasourcePaswords migrates un-encrypted secrets on datasources
// to the secureJson Column.
func EncryptDatasourcePaswords(c utils.CommandLine, sqlStore *sqlstore.SqlStore) error {
return sqlStore.WithDbSession(context.Background(), func(session *sqlstore.DBSession) error {
passwordsUpdated, err := migrateColumn(session, "password")
if err != nil {
return err
}
basicAuthUpdated, err := migrateColumn(session, "basic_auth_password")
if err != nil {
return err
}
logger.Info("\n")
if passwordsUpdated > 0 {
logger.Infof("%s Encrypted password field for %d datasources \n", color.GreenString("✔"), passwordsUpdated)
}
if basicAuthUpdated > 0 {
logger.Infof("%s Encrypted basic_auth_password field for %d datasources \n", color.GreenString("✔"), basicAuthUpdated)
}
if passwordsUpdated == 0 && basicAuthUpdated == 0 {
logger.Infof("%s All datasources secrets are allready encrypted\n", color.GreenString("✔"))
}
logger.Info("\n")
logger.Warn("Warning: Datasource provisioning files need to be manually changed to prevent overwriting of " +
"the data during provisioning. See https://grafana.com/docs/installation/upgrading/#upgrading-to-v6-2 for " +
"details")
return nil
})
}
func migrateColumn(session *sqlstore.DBSession, column string) (int, error) {
var rows []map[string]string
session.Cols("id", column, "secure_json_data")
session.Table("data_source")
session.In("type", datasourceTypes)
session.Where(column + " IS NOT NULL AND " + column + " != ''")
err := session.Find(&rows)
if err != nil {
return 0, errutil.Wrapf(err, "failed to select column: %s", column)
}
rowsUpdated, err := updateRows(session, rows, column)
return rowsUpdated, errutil.Wrapf(err, "failed to update column: %s", column)
}
func updateRows(session *sqlstore.DBSession, rows []map[string]string, passwordFieldName string) (int, error) {
var rowsUpdated int
for _, row := range rows {
newSecureJSONData, err := getUpdatedSecureJSONData(row, passwordFieldName)
if err != nil {
return 0, err
}
data, err := json.Marshal(newSecureJSONData)
if err != nil {
return 0, errutil.Wrap("marshaling newSecureJsonData failed", err)
}
newRow := map[string]interface{}{"secure_json_data": data, passwordFieldName: ""}
session.Table("data_source")
session.Where("id = ?", row["id"])
// Setting both columns while having value only for secure_json_data should clear the [passwordFieldName] column
session.Cols("secure_json_data", passwordFieldName)
_, err = session.Update(newRow)
if err != nil {
return 0, err
}
rowsUpdated++
}
return rowsUpdated, nil
}
func getUpdatedSecureJSONData(row map[string]string, passwordFieldName string) (map[string]interface{}, error) {
encryptedPassword, err := util.Encrypt([]byte(row[passwordFieldName]), setting.SecretKey)
if err != nil {
return nil, err
}
var secureJSONData map[string]interface{}
if err := json.Unmarshal([]byte(row["secure_json_data"]), &secureJSONData); err != nil {
return nil, err
}
jsonFieldName := util.ToCamelCase(passwordFieldName)
secureJSONData[jsonFieldName] = encryptedPassword
return secureJSONData, nil
}
pkg/cmd/grafana-cli/commands/datamigrations/encrypt_datasource_passwords_test.go 0 → 100644
package datamigrations
import (
"testing"
"time"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/commands/commandstest"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/stretchr/testify/assert"
)
func TestPasswordMigrationCommand(t *testing.T) {
//setup datasources with password, basic_auth and none
sqlstore := sqlstore.InitTestDB(t)
session := sqlstore.NewSession()
defer session.Close()
datasources := []*models.DataSource{
{Type: "influxdb", Name: "influxdb", Password: "foobar"},
{Type: "graphite", Name: "graphite", BasicAuthPassword: "foobar"},
{Type: "prometheus", Name: "prometheus", SecureJsonData: securejsondata.GetEncryptedJsonData(map[string]string{})},
}
// set required default values
for _, ds := range datasources {
ds.Created = time.Now()
ds.Updated = time.Now()
ds.SecureJsonData = securejsondata.GetEncryptedJsonData(map[string]string{})
}
_, err := session.Insert(&datasources)
assert.Nil(t, err)
//run migration
err = EncryptDatasourcePaswords(&commandstest.FakeCommandLine{}, sqlstore)
assert.Nil(t, err)
//verify that no datasources still have password or basic_auth
var dss []*models.DataSource
err = session.SQL("select * from data_source").Find(&dss)
assert.Nil(t, err)
assert.Equal(t, len(dss), 3)
for _, ds := range dss {
sj := ds.SecureJsonData.Decrypt()
if ds.Name == "influxdb" {
assert.Equal(t, ds.Password, "")
v, exist := sj["password"]
assert.True(t, exist)
assert.Equal(t, v, "foobar", "expected password to be moved to securejson")
}
if ds.Name == "graphite" {
assert.Equal(t, ds.BasicAuthPassword, "")
v, exist := sj["basicAuthPassword"]
assert.True(t, exist)
assert.Equal(t, v, "foobar", "expected basic_auth_password to be moved to securejson")
}
if ds.Name == "prometheus" {
assert.Equal(t, len(sj), 0)
}
}
}
pkg/cmd/grafana-cli/commands/install_command.go
...@@ -14,13 +14,14 @@ import ( ...@@ -14,13 +14,14 @@ import (
"strings" "strings"
"github.com/fatih/color" "github.com/fatih/color"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
) )
func validateInput(c CommandLine, pluginFolder string) error { func validateInput(c utils.CommandLine, pluginFolder string) error {
arg := c.Args().First() arg := c.Args().First()
if arg == "" { if arg == "" {
return errors.New("please specify plugin to install") return errors.New("please specify plugin to install")
...@@ -46,7 +47,7 @@ func validateInput(c CommandLine, pluginFolder string) error { ...@@ -46,7 +47,7 @@ func validateInput(c CommandLine, pluginFolder string) error {
return nil return nil
} }
func installCommand(c CommandLine) error { func installCommand(c utils.CommandLine) error {
pluginFolder := c.PluginDirectory() pluginFolder := c.PluginDirectory()
if err := validateInput(c, pluginFolder); err != nil { if err := validateInput(c, pluginFolder); err != nil {
return err return err
...@@ -60,7 +61,7 @@ func installCommand(c CommandLine) error { ...@@ -60,7 +61,7 @@ func installCommand(c CommandLine) error {
// InstallPlugin downloads the plugin code as a zip file from the Grafana.com API // InstallPlugin downloads the plugin code as a zip file from the Grafana.com API
// and then extracts the zip into the plugins directory. // and then extracts the zip into the plugins directory.
func InstallPlugin(pluginName, version string, c CommandLine) error { func InstallPlugin(pluginName, version string, c utils.CommandLine) error {
pluginFolder := c.PluginDirectory() pluginFolder := c.PluginDirectory()
downloadURL := c.PluginURL() downloadURL := c.PluginURL()
if downloadURL == "" { if downloadURL == "" {
......
pkg/cmd/grafana-cli/commands/listremote_command.go
...@@ -3,9 +3,10 @@ package commands ...@@ -3,9 +3,10 @@ package commands
import ( import (
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
) )
func listremoteCommand(c CommandLine) error { func listremoteCommand(c utils.CommandLine) error {
plugin, err := s.ListAllPlugins(c.RepoDirectory()) plugin, err := s.ListAllPlugins(c.RepoDirectory())
if err != nil { if err != nil {
......
pkg/cmd/grafana-cli/commands/listversions_command.go
...@@ -5,9 +5,10 @@ import ( ...@@ -5,9 +5,10 @@ import (
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
) )
func validateVersionInput(c CommandLine) error { func validateVersionInput(c utils.CommandLine) error {
arg := c.Args().First() arg := c.Args().First()
if arg == "" { if arg == "" {
return errors.New("please specify plugin to list versions for") return errors.New("please specify plugin to list versions for")
...@@ -16,7 +17,7 @@ func validateVersionInput(c CommandLine) error { ...@@ -16,7 +17,7 @@ func validateVersionInput(c CommandLine) error {
return nil return nil
} }
func listversionsCommand(c CommandLine) error { func listversionsCommand(c utils.CommandLine) error {
if err := validateVersionInput(c); err != nil { if err := validateVersionInput(c); err != nil {
return err return err
} }
......
pkg/cmd/grafana-cli/commands/ls_command.go
...@@ -8,6 +8,7 @@ import ( ...@@ -8,6 +8,7 @@ import (
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
) )
var ls_getPlugins func(path string) []m.InstalledPlugin = s.GetLocalPlugins var ls_getPlugins func(path string) []m.InstalledPlugin = s.GetLocalPlugins
...@@ -31,7 +32,7 @@ var validateLsCommand = func(pluginDir string) error { ...@@ -31,7 +32,7 @@ var validateLsCommand = func(pluginDir string) error {
return nil return nil
} }
func lsCommand(c CommandLine) error { func lsCommand(c utils.CommandLine) error {
pluginDir := c.PluginDirectory() pluginDir := c.PluginDirectory()
if err := validateLsCommand(pluginDir); err != nil { if err := validateLsCommand(pluginDir); err != nil {
return err return err
......
pkg/cmd/grafana-cli/commands/remove_command.go
...@@ -5,12 +5,13 @@ import ( ...@@ -5,12 +5,13 @@ import (
"fmt" "fmt"
"strings" "strings"
services "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
) )
var removePlugin func(pluginPath, id string) error = services.RemoveInstalledPlugin var removePlugin func(pluginPath, id string) error = services.RemoveInstalledPlugin
func removeCommand(c CommandLine) error { func removeCommand(c utils.CommandLine) error {
pluginPath := c.PluginDirectory() pluginPath := c.PluginDirectory()
plugin := c.Args().First() plugin := c.Args().First()
......
pkg/cmd/grafana-cli/commands/reset_password_command.go
...@@ -6,13 +6,15 @@ import ( ...@@ -6,13 +6,15 @@ import (
"github.com/fatih/color" "github.com/fatih/color"
"github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/sqlstore"
"github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/util"
) )
const AdminUserId = 1 const AdminUserId = 1
func resetPasswordCommand(c CommandLine) error { func resetPasswordCommand(c utils.CommandLine, sqlStore *sqlstore.SqlStore) error {
newPassword := c.Args().First() newPassword := c.Args().First()
password := models.Password(newPassword) password := models.Password(newPassword)
......
pkg/cmd/grafana-cli/commands/upgrade_all_command.go
...@@ -4,6 +4,7 @@ import ( ...@@ -4,6 +4,7 @@ import (
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models" m "github.com/grafana/grafana/pkg/cmd/grafana-cli/models"
s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
"github.com/hashicorp/go-version" "github.com/hashicorp/go-version"
) )
...@@ -27,7 +28,7 @@ func ShouldUpgrade(installed string, remote m.Plugin) bool { ...@@ -27,7 +28,7 @@ func ShouldUpgrade(installed string, remote m.Plugin) bool {
return false return false
} }
func upgradeAllCommand(c CommandLine) error { func upgradeAllCommand(c utils.CommandLine) error {
pluginsDir := c.PluginDirectory() pluginsDir := c.PluginDirectory()
localPlugins := s.GetLocalPlugins(pluginsDir) localPlugins := s.GetLocalPlugins(pluginsDir)
......
pkg/cmd/grafana-cli/commands/upgrade_command.go
...@@ -4,9 +4,10 @@ import ( ...@@ -4,9 +4,10 @@ import (
"github.com/fatih/color" "github.com/fatih/color"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/logger" "github.com/grafana/grafana/pkg/cmd/grafana-cli/logger"
s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services" s "github.com/grafana/grafana/pkg/cmd/grafana-cli/services"
"github.com/grafana/grafana/pkg/cmd/grafana-cli/utils"
) )
func upgradeCommand(c CommandLine) error { func upgradeCommand(c utils.CommandLine) error {
pluginsDir := c.PluginDirectory() pluginsDir := c.PluginDirectory()
pluginName := c.Args().First() pluginName := c.Args().First()
......
pkg/cmd/grafana-cli/commands/command_line.go → pkg/cmd/grafana-cli/utils/command_line.go
package commands package utils
import ( import (
"github.com/codegangsta/cli" "github.com/codegangsta/cli"
...@@ -22,30 +22,30 @@ type CommandLine interface { ...@@ -22,30 +22,30 @@ type CommandLine interface {
PluginURL() string PluginURL() string
} }
type contextCommandLine struct { type ContextCommandLine struct {
*cli.Context *cli.Context
} }
func (c *contextCommandLine) ShowHelp() { func (c *ContextCommandLine) ShowHelp() {
cli.ShowCommandHelp(c.Context, c.Command.Name) cli.ShowCommandHelp(c.Context, c.Command.Name)
} }
func (c *contextCommandLine) ShowVersion() { func (c *ContextCommandLine) ShowVersion() {
cli.ShowVersion(c.Context) cli.ShowVersion(c.Context)
} }
func (c *contextCommandLine) Application() *cli.App { func (c *ContextCommandLine) Application() *cli.App {
return c.App return c.App
} }
func (c *contextCommandLine) PluginDirectory() string { func (c *ContextCommandLine) PluginDirectory() string {
return c.GlobalString("pluginsDir") return c.GlobalString("pluginsDir")
} }
func (c *contextCommandLine) RepoDirectory() string { func (c *ContextCommandLine) RepoDirectory() string {
return c.GlobalString("repo") return c.GlobalString("repo")
} }
func (c *contextCommandLine) PluginURL() string { func (c *ContextCommandLine) PluginURL() string {
return c.GlobalString("pluginUrl") return c.GlobalString("pluginUrl")
} }
pkg/util/strings.go
...@@ -4,6 +4,7 @@ import ( ...@@ -4,6 +4,7 @@ import (
"fmt" "fmt"
"math" "math"
"regexp" "regexp"
"strings"
"time" "time"
) )
...@@ -66,3 +67,19 @@ func GetAgeString(t time.Time) string { ...@@ -66,3 +67,19 @@ func GetAgeString(t time.Time) string {
return "< 1m" return "< 1m"
} }
// ToCamelCase changes kebab case, snake case or mixed strings to camel case. See unit test for examples.
func ToCamelCase(str string) string {
var finalParts []string
parts := strings.Split(str, "_")
for _, part := range parts {
finalParts = append(finalParts, strings.Split(part, "-")...)
}
for index, part := range finalParts[1:] {
finalParts[index+1] = strings.Title(part)
}
return strings.Join(finalParts, "")
}
pkg/util/strings_test.go
...@@ -37,3 +37,12 @@ func TestDateAge(t *testing.T) { ...@@ -37,3 +37,12 @@ func TestDateAge(t *testing.T) {
So(GetAgeString(time.Now().Add(-time.Hour*24*409)), ShouldEqual, "1y") So(GetAgeString(time.Now().Add(-time.Hour*24*409)), ShouldEqual, "1y")
}) })
} }
func TestToCamelCase(t *testing.T) {
Convey("ToCamelCase", t, func() {
So(ToCamelCase("kebab-case-string"), ShouldEqual, "kebabCaseString")
So(ToCamelCase("snake_case_string"), ShouldEqual, "snakeCaseString")
So(ToCamelCase("mixed-case_string"), ShouldEqual, "mixedCaseString")
So(ToCamelCase("alreadyCamelCase"), ShouldEqual, "alreadyCamelCase")
})
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment