Provisioning: Apply settings expanders (#25692)

* Provisioning: Apply settings expanders

Adds support for the variable expanders introduced to the configuration file in #25075 to provisioning.

* Update pkg/services/provisioning/values/values_test.go

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

* Update pkg/services/provisioning/values/values.go

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>

* Provisioning: Handle errors when expanding JSON maps

pkg/services/provisioning/values/values.go

@@ -11,11 +11,14 @@
 package values
 package values
 
 
 import (
 import (
+	"fmt"
 	"os"
 	"os"
 	"reflect"
 	"reflect"
 	"strconv"
 	"strconv"
 	"strings"
 	"strings"
 
 
+	"github.com/grafana/grafana/pkg/setting"
+
 	"github.com/grafana/grafana/pkg/util/errutil"
 	"github.com/grafana/grafana/pkg/util/errutil"
 )
 )
 
 
@@ -117,7 +120,10 @@ func (val *JSONValue) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	interpolated := make(map[string]interface{})
 	interpolated := make(map[string]interface{})
 	raw := make(map[string]interface{})
 	raw := make(map[string]interface{})
 	for key, val := range unmarshaled {
 	for key, val := range unmarshaled {
-		interpolated[key], raw[key] = transformInterface(val)
+		interpolated[key], raw[key], err = transformInterface(val)
+		if err != nil {
+			return err
+		}
 	}
 	}
 
 
 	val.Raw = raw
 	val.Raw = raw
@@ -143,7 +149,10 @@ func (val *StringMapValue) UnmarshalYAML(unmarshal func(interface{}) error) erro
 	interpolated := make(map[string]string)
 	interpolated := make(map[string]string)
 	raw := make(map[string]string)
 	raw := make(map[string]string)
 	for key, val := range unmarshaled {
 	for key, val := range unmarshaled {
-		interpolated[key], raw[key] = interpolateValue(val)
+		interpolated[key], raw[key], err = interpolateValue(val)
+		if err != nil {
+			return err
+		}
 	}
 	}
 	val.Raw = raw
 	val.Raw = raw
 	val.value = interpolated
 	val.value = interpolated
@@ -158,11 +167,11 @@ func (val *StringMapValue) Value() map[string]string {
 // slices and the actual interpolation is done on all simple string values in the structure. It returns a copy of any
 // slices and the actual interpolation is done on all simple string values in the structure. It returns a copy of any
 // map or slice value instead of modifying them in place and also return value without interpolation but with converted
 // map or slice value instead of modifying them in place and also return value without interpolation but with converted
 // type as a second value.
 // type as a second value.
-func transformInterface(i interface{}) (interface{}, interface{}) {
+func transformInterface(i interface{}) (interface{}, interface{}, error) {
 	typeOf := reflect.TypeOf(i)
 	typeOf := reflect.TypeOf(i)
 
 
 	if typeOf == nil {
 	if typeOf == nil {
-		return nil, nil
+		return nil, nil, nil
 	}
 	}
 
 
 	switch typeOf.Kind() {
 	switch typeOf.Kind() {
@@ -174,43 +183,55 @@ func transformInterface(i interface{}) (interface{}, interface{}) {
 		return interpolateValue(i.(string))
 		return interpolateValue(i.(string))
 	default:
 	default:
 		// Was int, float or some other value that we do not need to do any transform on.
 		// Was int, float or some other value that we do not need to do any transform on.
-		return i, i
+		return i, i, nil
 	}
 	}
 }
 }
 
 
-func transformSlice(i []interface{}) (interface{}, interface{}) {
+func transformSlice(i []interface{}) (interface{}, interface{}, error) {
 	var transformedSlice []interface{}
 	var transformedSlice []interface{}
 	var rawSlice []interface{}
 	var rawSlice []interface{}
 	for _, val := range i {
 	for _, val := range i {
-		transformed, raw := transformInterface(val)
+		transformed, raw, err := transformInterface(val)
+		if err != nil {
+			return nil, nil, err
+		}
 		transformedSlice = append(transformedSlice, transformed)
 		transformedSlice = append(transformedSlice, transformed)
 		rawSlice = append(rawSlice, raw)
 		rawSlice = append(rawSlice, raw)
 	}
 	}
-	return transformedSlice, rawSlice
+	return transformedSlice, rawSlice, nil
 }
 }
 
 
-func transformMap(i map[interface{}]interface{}) (interface{}, interface{}) {
+func transformMap(i map[interface{}]interface{}) (interface{}, interface{}, error) {
 	transformed := make(map[string]interface{})
 	transformed := make(map[string]interface{})
 	raw := make(map[string]interface{})
 	raw := make(map[string]interface{})
 	for key, val := range i {
 	for key, val := range i {
 		stringKey, ok := key.(string)
 		stringKey, ok := key.(string)
 		if ok {
 		if ok {
-			transformed[stringKey], raw[stringKey] = transformInterface(val)
+			var err error
+			transformed[stringKey], raw[stringKey], err = transformInterface(val)
+			if err != nil {
+				return nil, nil, err
+			}
 		}
 		}
 	}
 	}
-	return transformed, raw
+	return transformed, raw, nil
 }
 }
 
 
-// interpolateValue returns final value after interpolation. At the moment only env var interpolation is done
-// here but in the future something like interpolation from file could be also done here.
+// interpolateValue returns the final value after interpolation. In addition to environment variable interpolation,
+// expanders available for the settings file are expanded here.
 // For a literal '$', '$$' can be used to avoid interpolation.
 // For a literal '$', '$$' can be used to avoid interpolation.
-func interpolateValue(val string) (string, string) {
+func interpolateValue(val string) (string, string, error) {
 	parts := strings.Split(val, "$$")
 	parts := strings.Split(val, "$$")
 	interpolated := make([]string, len(parts))
 	interpolated := make([]string, len(parts))
 	for i, v := range parts {
 	for i, v := range parts {
+		expanded, err := setting.ExpandVar(v)
+		if err != nil {
+			return val, val, fmt.Errorf("failed to interpolate value '%s': %w", val, err)
+		}
+		v = expanded
 		interpolated[i] = os.ExpandEnv(v)
 		interpolated[i] = os.ExpandEnv(v)
 	}
 	}
-	return strings.Join(interpolated, "$"), val
+	return strings.Join(interpolated, "$"), val, nil
 }
 }
 
 
 type interpolated struct {
 type interpolated struct {
@@ -228,6 +249,9 @@ func getInterpolated(unmarshal func(interface{}) error) (*interpolated, error) {
 	}
 	}
 	// We get new raw value here which can have a bit different type, as yaml types nested maps as
 	// We get new raw value here which can have a bit different type, as yaml types nested maps as
 	// map[interface{}]interface and we want it to be map[string]interface{}
 	// map[interface{}]interface and we want it to be map[string]interface{}
-	value, raw := interpolateValue(veryRaw)
+	value, raw, err := interpolateValue(veryRaw)
+	if err != nil {
+		return &interpolated{}, err
+	}
 	return &interpolated{raw: raw, value: value}, nil
 	return &interpolated{raw: raw, value: value}, nil
 }
 }

pkg/services/provisioning/values/values_test.go

 package values
 package values
 
 
 import (
 import (
+	"errors"
+	"fmt"
+	"io/ioutil"
 	"os"
 	"os"
 	"testing"
 	"testing"
 
 
+	"github.com/grafana/grafana/pkg/setting"
+	"gopkg.in/ini.v1"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	. "github.com/smartystreets/goconvey/convey"
 	. "github.com/smartystreets/goconvey/convey"
 	"gopkg.in/yaml.v2"
 	"gopkg.in/yaml.v2"
 )
 )
@@ -245,3 +254,53 @@ func unmarshalingTest(document string, out interface{}) {
 	err := yaml.Unmarshal([]byte(document), out)
 	err := yaml.Unmarshal([]byte(document), out)
 	So(err, ShouldBeNil)
 	So(err, ShouldBeNil)
 }
 }
+
+func TestValues_readFile(t *testing.T) {
+	type Data struct {
+		Val StringValue `yaml:"val"`
+	}
+
+	f, err := ioutil.TempFile(os.TempDir(), "file expansion *")
+	require.NoError(t, err)
+	file := f.Name()
+
+	defer func() {
+		require.NoError(t, os.Remove(file))
+	}()
+
+	const expected = "hello, world"
+	_, err = f.WriteString(expected)
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+
+	data := &Data{}
+	err = yaml.Unmarshal([]byte(fmt.Sprintf("val: $__file{%s}", file)), data)
+	require.NoError(t, err)
+	assert.Equal(t, expected, data.Val.Value())
+}
+
+func TestValues_expanderError(t *testing.T) {
+	type Data struct {
+		Top JSONValue `yaml:"top"`
+	}
+
+	setting.AddExpander("fail", 0, failExpander{})
+
+	data := &Data{}
+	err := yaml.Unmarshal([]byte("top:\n  val: $__fail{val}"), data)
+	require.Error(t, err)
+	require.Truef(t, errors.Is(err, errExpand), "expected error to wrap: %v\ngot: %v", errExpand, err)
+	assert.Empty(t, data)
+}
+
+var errExpand = errors.New("test error: bad expander")
+
+type failExpander struct{}
+
+func (f failExpander) SetupExpander(file *ini.File) error {
+	return nil
+}
+
+func (f failExpander) Expand(s string) (string, error) {
+	return "", errExpand
+}