Settings: Expand variables in configuration (#25075)

pkg/services/ldap/settings.go

@@ -125,7 +125,10 @@ func readConfig(configFile string) (*Config, error) {
 	}
 
 	// interpolate full toml string (it can contain ENV variables)
-	stringContent := setting.EvalEnvVarExpression(string(fileBytes))
+	stringContent, err := setting.ExpandVar(string(fileBytes))
+	if err != nil {
+		return nil, errutil.Wrap("Failed to expand variables", err)
+	}
 
 	_, err = toml.Decode(stringContent, result)
 	if err != nil {

pkg/setting/expanders.go

+package setting
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+
+	"gopkg.in/ini.v1"
+)
+
+type Expander interface {
+	SetupExpander(file *ini.File) error
+	Expand(string) (string, error)
+}
+
+type registeredExpander struct {
+	name     string
+	priority int64
+	expander Expander
+}
+
+var expanders = []registeredExpander{
+	{
+		name:     "env",
+		priority: -10,
+		expander: envExpander{},
+	},
+	{
+		name:     "file",
+		priority: -5,
+		expander: fileExpander{},
+	},
+}
+
+func AddExpander(name string, priority int64, e Expander) {
+	expanders = append(expanders, registeredExpander{
+		name:     name,
+		priority: priority,
+		expander: e,
+	})
+}
+
+var regex = regexp.MustCompile(`\$(|__\w+){([^}]+)}`)
+
+func expandConfig(file *ini.File) error {
+	sort.Slice(expanders, func(i, j int) bool {
+		return expanders[i].priority < expanders[j].priority
+	})
+
+	for _, expander := range expanders {
+		err := expander.expander.SetupExpander(file)
+		if err != nil {
+			return fmt.Errorf("got error during initilazation of expander '%s': %w", expander.name, err)
+		}
+
+		for _, section := range file.Sections() {
+			for _, key := range section.Keys() {
+				updated, err := applyExpander(key.Value(), expander)
+				if err != nil {
+					return fmt.Errorf("got error while expanding %s.%s with expander '%s': %w",
+						section.Name(),
+						key.Name(),
+						expander.name,
+						err)
+				}
+
+				key.SetValue(updated)
+			}
+		}
+	}
+	return nil
+}
+
+func ExpandVar(s string) (string, error) {
+	for _, expander := range expanders {
+		var err error
+		s, err = applyExpander(s, expander)
+		if err != nil {
+			return "", fmt.Errorf("got error while expanding expander %s: %w", expander.name, err)
+		}
+	}
+	return s, nil
+}
+
+func applyExpander(s string, e registeredExpander) (string, error) {
+	matches := regex.FindAllStringSubmatch(s, -1)
+
+	for _, match := range matches {
+		if len(match) < 3 {
+			return "", fmt.Errorf("regex error, got %d results back for match, expected 3", len(match))
+		}
+
+		_, isEnv := e.expander.(envExpander)
+		if match[1] == "__"+e.name || (match[1] == "" && isEnv) {
+			updated, err := e.expander.Expand(match[2])
+			if err != nil {
+				return "", err
+			}
+
+			s = strings.Replace(s, match[0], updated, 1)
+		}
+	}
+
+	return s, nil
+}
+
+type envExpander struct {
+}
+
+func (e envExpander) SetupExpander(file *ini.File) error {
+	return nil
+}
+
+func (e envExpander) Expand(s string) (string, error) {
+	envValue := os.Getenv(s)
+
+	// if env variable is hostname and it is empty use os.Hostname as default
+	if s == "HOSTNAME" && envValue == "" {
+		return os.Hostname()
+	}
+
+	return os.Getenv(s), nil
+}
+
+type fileExpander struct {
+}
+
+func (e fileExpander) SetupExpander(file *ini.File) error {
+	return nil
+}
+
+func (e fileExpander) Expand(s string) (string, error) {
+	_, err := os.Stat(s)
+	if err != nil {
+		return "", err
+	}
+
+	f, err := ioutil.ReadFile(s)
+	if err != nil {
+		return "", err
+	}
+
+	return strings.TrimSpace(string(f)), nil
+}

pkg/setting/expanders_test.go

+package setting
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExpandVar_EnvSuccessful(t *testing.T) {
+	const key = "GF_TEST_SETTING_EXPANDER_ENV"
+	const expected = "aurora borealis"
+	os.Setenv(key, expected)
+
+	// expanded format
+	{
+		got, err := ExpandVar(fmt.Sprintf("$__env{%s}", key))
+		assert.NoError(t, err)
+		assert.Equal(t, expected, got)
+	}
+
+	// short format
+	{
+		got, err := ExpandVar(fmt.Sprintf("${%s}", key))
+		assert.NoError(t, err)
+		assert.Equal(t, expected, got)
+	}
+}
+
+func TestExpandVar_FileSuccessful(t *testing.T) {
+	f, err := ioutil.TempFile(os.TempDir(), "file expansion *")
+	require.NoError(t, err)
+	file := f.Name()
+
+	defer func() {
+		os.Remove(file)
+	}()
+
+	_, err = f.WriteString("hello, world")
+	require.NoError(t, err)
+	f.Close()
+
+	got, err := ExpandVar(fmt.Sprintf("$__file{%s}", file))
+	assert.NoError(t, err)
+	assert.Equal(t, "hello, world", got)
+}
+
+func TestExpandVar_FileDoesNotExist(t *testing.T) {
+	got, err := ExpandVar(
+		fmt.Sprintf("$__file{%s%sthisisnotarealfile_%d}",
+			os.TempDir(),
+			string(os.PathSeparator),
+			rand.Int63()),
+	)
+	assert.Error(t, err)
+	assert.True(t, errors.Is(err, os.ErrNotExist))
+	assert.Empty(t, got)
+}
+
+func TestExpanderRegex(t *testing.T) {
+	tests := map[string][][]string{
+		// we should not expand variables where there are none
+		"smoketest":                          {},
+		"Pa$$word{0}":                        {},
+		"$_almost{but not quite a variable}": {},
+		// args are required
+		"$__file{}": {},
+
+		// base cases
+		"${ENV}":             {{"", "ENV"}},
+		"$__env{ENV}":        {{"__env", "ENV"}},
+		"$__file{/dev/null}": {{"__file", "/dev/null"}},
+		"$__vault{item}":     {{"__vault", "item"}},
+		// contains a space in the argument
+		"$__file{C:\\Program Files\\grafana\\something}": {{"__file", "C:\\Program Files\\grafana\\something"}},
+
+		// complex cases
+		"get variable from $__env{ENV}ironment":               {{"__env", "ENV"}},
+		"$__env{VAR1} $__file{/var/lib/grafana/secrets/var1}": {{"__env", "VAR1"}, {"__file", "/var/lib/grafana/secrets/var1"}},
+		"$__env{$__file{this is invalid}}":                    {{"__env", "$__file{this is invalid"}},
+	}
+
+	for input, expected := range tests {
+		output := regex.FindAllStringSubmatch(input, -1)
+		require.Len(t, output, len(expected))
+		for i, variable := range output {
+			assert.Equal(t, expected[i], variable[1:])
+		}
+	}
+}

pkg/setting/setting.go

@@ -12,7 +12,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"regexp"
 	"runtime"
 	"strings"
 	"time"
@@ -440,30 +439,6 @@ func makeAbsolute(path string, root string) string {
 	return filepath.Join(root, path)
 }
 
-func EvalEnvVarExpression(value string) string {
-	regex := regexp.MustCompile(`\${(\w+)}`)
-	return regex.ReplaceAllStringFunc(value, func(envVar string) string {
-		envVar = strings.TrimPrefix(envVar, "${")
-		envVar = strings.TrimSuffix(envVar, "}")
-		envValue := os.Getenv(envVar)
-
-		// if env variable is hostname and it is empty use os.Hostname as default
-		if envVar == "HOSTNAME" && envValue == "" {
-			envValue, _ = os.Hostname()
-		}
-
-		return envValue
-	})
-}
-
-func evalConfigValues(file *ini.File) {
-	for _, section := range file.Sections() {
-		for _, key := range section.Keys() {
-			key.SetValue(EvalEnvVarExpression(key.Value()))
-		}
-	}
-}
-
 func loadSpecifiedConfigFile(configFile string, masterFile *ini.File) error {
 	if configFile == "" {
 		configFile = filepath.Join(HomePath, CustomInitPath)
@@ -550,7 +525,10 @@ func (cfg *Cfg) loadConfiguration(args *CommandLineArgs) (*ini.File, error) {
 	applyCommandLineProperties(commandLineProps, parsedFile)
 
 	// evaluate config values containing environment variables
-	evalConfigValues(parsedFile)
+	err = expandConfig(parsedFile)
+	if err != nil {
+		return nil, err
+	}
 
 	// update data path and logging config
 	dataPath, err := valueAsString(parsedFile.Section("paths"), "data", "")