OAuth: Support JMES path lookup when retrieving user email (#14683)

Add support for fetching e-mail with JMES path

Signed-off-by: Bob Shannon <bobs@dropbox.com>

conf/defaults.ini

@@ -366,6 +366,7 @@ client_id = some_id
 client_secret = some_secret
 client_secret = some_secret
 scopes = user:email
 scopes = user:email
 email_attribute_name = email:primary
 email_attribute_name = email:primary
+email_attribute_path =
 auth_url =
 auth_url =
 token_url =
 token_url =
 api_url =
 api_url =

conf/sample.ini

@@ -319,6 +319,8 @@
 ;client_id = some_id
 ;client_id = some_id
 ;client_secret = some_secret
 ;client_secret = some_secret
 ;scopes = user:email,read:org
 ;scopes = user:email,read:org
+;email_attribute_name = email:primary
+;email_attribute_path =
 ;auth_url = https://foo.bar/login/oauth/authorize
 ;auth_url = https://foo.bar/login/oauth/authorize
 ;token_url = https://foo.bar/login/oauth/access_token
 ;token_url = https://foo.bar/login/oauth/access_token
 ;api_url = https://foo.bar/user
 ;api_url = https://foo.bar/user

docs/sources/auth/generic-oauth.md

@@ -40,9 +40,11 @@ Set `api_url` to the resource that returns [OpenID UserInfo](https://connect2id.
 Grafana will attempt to determine the user's e-mail address by querying the OAuth provider as described below in the following order until an e-mail address is found:
 Grafana will attempt to determine the user's e-mail address by querying the OAuth provider as described below in the following order until an e-mail address is found:
 
 
 1. Check for the presence of an e-mail address via the `email` field encoded in the OAuth `id_token` parameter.
 1. Check for the presence of an e-mail address via the `email` field encoded in the OAuth `id_token` parameter.
-2. Check for the presence of an e-mail address in the `attributes` map encoded in the OAuth `id_token` parameter. By default Grafana will perform a lookup into the attributes map using the `email:primary` key, however, this is configurable and can be adjusted by using the `email_attribute_name` configuration option.
-3. Query the `/emails` endpoint of the OAuth provider's API (configured with `api_url`) and check for the presence of an e-mail address marked as a primary address.
-4. If no e-mail address is found in steps (1-3), then the e-mail address of the user is set to the empty string.
+2. Check for the presence of an e-mail address using the [JMES path](http://jmespath.org/examples.html) specified via the `email_attribute_path` configuration option. The JSON used for the path lookup is the HTTP response obtained from querying the UserInfo endpoint specified via the `api_url` configuration option.
+**Note**: Only available in Grafana v6.4+.
+3. Check for the presence of an e-mail address in the `attributes` map encoded in the OAuth `id_token` parameter. By default Grafana will perform a lookup into the attributes map using the `email:primary` key, however, this is configurable and can be adjusted by using the `email_attribute_name` configuration option.
+4. Query the `/emails` endpoint of the OAuth provider's API (configured with `api_url`) and check for the presence of an e-mail address marked as a primary address.
+5. If no e-mail address is found in steps (1-4), then the e-mail address of the user is set to the empty string.
 
 
 ## Set up OAuth2 with Okta
 ## Set up OAuth2 with Okta
 
 

pkg/login/social/generic_oauth.go

@@ -10,7 +10,7 @@ import (
 	"regexp"
 	"regexp"
 
 
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/models"
-
+	"github.com/jmespath/go-jmespath"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2"
 )
 )
 
 
@@ -21,6 +21,7 @@ type SocialGenericOAuth struct {
 	apiUrl               string
 	apiUrl               string
 	allowSignup          bool
 	allowSignup          bool
 	emailAttributeName   string
 	emailAttributeName   string
+	emailAttributePath   string
 	teamIds              []int
 	teamIds              []int
 }
 }
 
 
@@ -78,6 +79,37 @@ func (s *SocialGenericOAuth) IsOrganizationMember(client *http.Client) bool {
 	return false
 	return false
 }
 }
 
 
+// searchJSONForEmail searches the provided JSON response for an e-mail address
+// using the configured e-mail attribute path associated with the generic OAuth
+// provider.
+// Returns an empty string if an e-mail address is not found.
+func (s *SocialGenericOAuth) searchJSONForEmail(data []byte) string {
+	if s.emailAttributePath == "" {
+		s.log.Error("No e-mail attribute path specified")
+		return ""
+	}
+	if len(data) == 0 {
+		s.log.Error("Empty user info JSON response provided")
+		return ""
+	}
+	var buf interface{}
+	if err := json.Unmarshal(data, &buf); err != nil {
+		s.log.Error("Failed to unmarshal user info JSON response", "err", err.Error())
+		return ""
+	}
+	val, err := jmespath.Search(s.emailAttributePath, buf)
+	if err != nil {
+		s.log.Error("Failed to search user info JSON response with provided path", "emailAttributePath", s.emailAttributePath, "err", err.Error())
+		return ""
+	}
+	strVal, ok := val.(string)
+	if ok {
+		return strVal
+	}
+	s.log.Error("E-mail not found when searching JSON with provided path", "emailAttributePath", s.emailAttributePath)
+	return ""
+}
+
 func (s *SocialGenericOAuth) FetchPrivateEmail(client *http.Client) (string, error) {
 func (s *SocialGenericOAuth) FetchPrivateEmail(client *http.Client) (string, error) {
 	type Record struct {
 	type Record struct {
 		Email       string `json:"email"`
 		Email       string `json:"email"`
@@ -181,15 +213,16 @@ type UserInfoJson struct {
 
 
 func (s *SocialGenericOAuth) UserInfo(client *http.Client, token *oauth2.Token) (*BasicUserInfo, error) {
 func (s *SocialGenericOAuth) UserInfo(client *http.Client, token *oauth2.Token) (*BasicUserInfo, error) {
 	var data UserInfoJson
 	var data UserInfoJson
+	var rawUserInfoResponse HttpGetResponse
 	var err error
 	var err error
 
 
 	if !s.extractToken(&data, token) {
 	if !s.extractToken(&data, token) {
-		response, err := HttpGet(client, s.apiUrl)
+		rawUserInfoResponse, err = HttpGet(client, s.apiUrl)
 		if err != nil {
 		if err != nil {
 			return nil, fmt.Errorf("Error getting user info: %s", err)
 			return nil, fmt.Errorf("Error getting user info: %s", err)
 		}
 		}
 
 
-		err = json.Unmarshal(response.Body, &data)
+		err = json.Unmarshal(rawUserInfoResponse.Body, &data)
 		if err != nil {
 		if err != nil {
 			return nil, fmt.Errorf("Error decoding user info JSON: %s", err)
 			return nil, fmt.Errorf("Error decoding user info JSON: %s", err)
 		}
 		}
@@ -197,7 +230,7 @@ func (s *SocialGenericOAuth) UserInfo(client *http.Client, token *oauth2.Token)
 
 
 	name := s.extractName(&data)
 	name := s.extractName(&data)
 
 
-	email := s.extractEmail(&data)
+	email := s.extractEmail(&data, rawUserInfoResponse.Body)
 	if email == "" {
 	if email == "" {
 		email, err = s.FetchPrivateEmail(client)
 		email, err = s.FetchPrivateEmail(client)
 		if err != nil {
 		if err != nil {
@@ -250,8 +283,7 @@ func (s *SocialGenericOAuth) extractToken(data *UserInfoJson, token *oauth2.Toke
 		return false
 		return false
 	}
 	}
 
 
-	email := s.extractEmail(data)
-	if email == "" {
+	if email := s.extractEmail(data, payload); email == "" {
 		s.log.Debug("No email found in id_token", "json", string(payload), "data", data)
 		s.log.Debug("No email found in id_token", "json", string(payload), "data", data)
 		return false
 		return false
 	}
 	}
@@ -260,11 +292,18 @@ func (s *SocialGenericOAuth) extractToken(data *UserInfoJson, token *oauth2.Toke
 	return true
 	return true
 }
 }
 
 
-func (s *SocialGenericOAuth) extractEmail(data *UserInfoJson) string {
+func (s *SocialGenericOAuth) extractEmail(data *UserInfoJson, userInfoResp []byte) string {
 	if data.Email != "" {
 	if data.Email != "" {
 		return data.Email
 		return data.Email
 	}
 	}
 
 
+	if s.emailAttributePath != "" {
+		email := s.searchJSONForEmail(userInfoResp)
+		if email != "" {
+			return email
+		}
+	}
+
 	emails, ok := data.Attributes[s.emailAttributeName]
 	emails, ok := data.Attributes[s.emailAttributeName]
 	if ok && len(emails) != 0 {
 	if ok && len(emails) != 0 {
 		return emails[0]
 		return emails[0]
@@ -275,6 +314,7 @@ func (s *SocialGenericOAuth) extractEmail(data *UserInfoJson) string {
 		if emailErr == nil {
 		if emailErr == nil {
 			return emailAddr.Address
 			return emailAddr.Address
 		}
 		}
+		s.log.Debug("Failed to parse e-mail address", "err", emailErr.Error())
 	}
 	}
 
 
 	return ""
 	return ""

pkg/login/social/generic_oauth_test.go

+package social
+
+import (
+	"github.com/grafana/grafana/pkg/infra/log"
+	. "github.com/smartystreets/goconvey/convey"
+	"testing"
+)
+
+func TestSearchJSONForEmail(t *testing.T) {
+	Convey("Given a generic OAuth provider", t, func() {
+		provider := SocialGenericOAuth{
+			SocialBase: &SocialBase{
+				log: log.New("generic_oauth_test"),
+			},
+		}
+
+		tests := []struct {
+			Name                 string
+			UserInfoJSONResponse []byte
+			EmailAttributePath   string
+			ExpectedResult       string
+		}{
+			{
+				Name:                 "Given an invalid user info JSON response",
+				UserInfoJSONResponse: []byte("{"),
+				EmailAttributePath:   "attributes.email",
+				ExpectedResult:       "",
+			},
+			{
+				Name:                 "Given an empty user info JSON response and empty JMES path",
+				UserInfoJSONResponse: []byte{},
+				EmailAttributePath:   "",
+				ExpectedResult:       "",
+			},
+			{
+				Name:                 "Given an empty user info JSON response and valid JMES path",
+				UserInfoJSONResponse: []byte{},
+				EmailAttributePath:   "attributes.email",
+				ExpectedResult:       "",
+			},
+			{
+				Name: "Given a simple user info JSON response and valid JMES path",
+				UserInfoJSONResponse: []byte(`{
+	"attributes": {
+		"email": "grafana@localhost"
+	}
+}`),
+				EmailAttributePath: "attributes.email",
+				ExpectedResult:     "grafana@localhost",
+			},
+			{
+				Name: "Given a user info JSON response with e-mails array and valid JMES path",
+				UserInfoJSONResponse: []byte(`{
+	"attributes": {
+		"emails": ["grafana@localhost", "admin@localhost"]
+	}
+}`),
+				EmailAttributePath: "attributes.emails[0]",
+				ExpectedResult:     "grafana@localhost",
+			},
+			{
+				Name: "Given a nested user info JSON response and valid JMES path",
+				UserInfoJSONResponse: []byte(`{
+	"identities": [
+		{
+			"userId": "grafana@localhost"
+		},
+		{
+			"userId": "admin@localhost"
+		}
+	]
+}`),
+				EmailAttributePath: "identities[0].userId",
+				ExpectedResult:     "grafana@localhost",
+			},
+		}
+
+		for _, test := range tests {
+			provider.emailAttributePath = test.EmailAttributePath
+			Convey(test.Name, func() {
+				actualResult := provider.searchJSONForEmail(test.UserInfoJSONResponse)
+				So(actualResult, ShouldEqual, test.ExpectedResult)
+			})
+		}
+	})
+}

pkg/login/social/social.go

@@ -73,6 +73,7 @@ func NewOAuthService() {
 			ApiUrl:                       sec.Key("api_url").String(),
 			ApiUrl:                       sec.Key("api_url").String(),
 			Enabled:                      sec.Key("enabled").MustBool(),
 			Enabled:                      sec.Key("enabled").MustBool(),
 			EmailAttributeName:           sec.Key("email_attribute_name").String(),
 			EmailAttributeName:           sec.Key("email_attribute_name").String(),
+			EmailAttributePath:           sec.Key("email_attribute_path").String(),
 			AllowedDomains:               util.SplitString(sec.Key("allowed_domains").String()),
 			AllowedDomains:               util.SplitString(sec.Key("allowed_domains").String()),
 			HostedDomain:                 sec.Key("hosted_domain").String(),
 			HostedDomain:                 sec.Key("hosted_domain").String(),
 			AllowSignup:                  sec.Key("allow_sign_up").MustBool(),
 			AllowSignup:                  sec.Key("allow_sign_up").MustBool(),
@@ -167,6 +168,7 @@ func NewOAuthService() {
 				apiUrl:               info.ApiUrl,
 				apiUrl:               info.ApiUrl,
 				allowSignup:          info.AllowSignup,
 				allowSignup:          info.AllowSignup,
 				emailAttributeName:   info.EmailAttributeName,
 				emailAttributeName:   info.EmailAttributeName,
+				emailAttributePath:   info.EmailAttributePath,
 				teamIds:              sec.Key("team_ids").Ints(","),
 				teamIds:              sec.Key("team_ids").Ints(","),
 				allowedOrganizations: util.SplitString(sec.Key("allowed_organizations").String()),
 				allowedOrganizations: util.SplitString(sec.Key("allowed_organizations").String()),
 			}
 			}

pkg/setting/setting_oauth.go

@@ -6,6 +6,7 @@ type OAuthInfo struct {
 	AuthUrl, TokenUrl            string
 	AuthUrl, TokenUrl            string
 	Enabled                      bool
 	Enabled                      bool
 	EmailAttributeName           string
 	EmailAttributeName           string
+	EmailAttributePath           string
 	AllowedDomains               []string
 	AllowedDomains               []string
 	HostedDomain                 string
 	HostedDomain                 string
 	ApiUrl                       string
 	ApiUrl                       string