Skip to content

N
nexpie-grafana-theme
Commit 1646de45 by gastonqiu Committed by GitHub
Options

Image uploader: Fix uploading of images to GCS (#26493) 

* GCS image uploader: Re-implement with Google SDK

Signed-off-by: gaston.qiu <gaston.qiu@umbocv.com>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
parent b3a86816
Showing with 72 additions and 77 deletions
go.mod
...@@ -11,8 +11,8 @@ replace github.com/denisenkom/go-mssqldb => github.com/denisenkom/go-mssqldb v0. ...@@ -11,8 +11,8 @@ replace github.com/denisenkom/go-mssqldb => github.com/denisenkom/go-mssqldb v0.
replace k8s.io/client-go => k8s.io/client-go v0.18.8 replace k8s.io/client-go => k8s.io/client-go v0.18.8
require ( require (
cloud.google.com/go v0.60.0 // indirect cloud.google.com/go v0.70.0 // indirect
cloud.google.com/go/storage v1.8.0 cloud.google.com/go/storage v1.12.0
github.com/BurntSushi/toml v0.3.1 github.com/BurntSushi/toml v0.3.1
github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f
github.com/aws/aws-sdk-go v1.33.12 github.com/aws/aws-sdk-go v1.33.12
...@@ -37,8 +37,8 @@ require ( ...@@ -37,8 +37,8 @@ require (
github.com/go-sql-driver/mysql v1.5.0 github.com/go-sql-driver/mysql v1.5.0
github.com/go-stack/stack v1.8.0 github.com/go-stack/stack v1.8.0
github.com/gobwas/glob v0.2.3 github.com/gobwas/glob v0.2.3
github.com/golang/protobuf v1.4.2 github.com/golang/protobuf v1.4.3
github.com/google/go-cmp v0.5.0 github.com/google/go-cmp v0.5.2
github.com/gosimple/slug v1.4.2 github.com/gosimple/slug v1.4.2
github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4 github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4
github.com/grafana/grafana-plugin-sdk-go v0.78.0 github.com/grafana/grafana-plugin-sdk-go v0.78.0
...@@ -82,12 +82,15 @@ require ( ...@@ -82,12 +82,15 @@ require (
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
github.com/yudai/pp v2.0.1+incompatible // indirect github.com/yudai/pp v2.0.1+incompatible // indirect
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc golang.org/x/net v0.0.0-20201022231255-08b38378de70
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d // indirect golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd // indirect
golang.org/x/text v0.3.3 // indirect golang.org/x/tools v0.0.0-20201023150057-2f4fa188d925 // indirect
google.golang.org/grpc v1.30.0 google.golang.org/api v0.33.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20201022181438-0ff5f38871d5 // indirect
google.golang.org/grpc v1.33.1
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/ini.v1 v1.51.0 gopkg.in/ini.v1 v1.51.0
......
pkg/components/imguploader/gcsuploader.go
...@@ -5,7 +5,6 @@ import ( ...@@ -5,7 +5,6 @@ import (
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"net/http"
"os" "os"
"path" "path"
"time" "time"
...@@ -16,13 +15,7 @@ import ( ...@@ -16,13 +15,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/util"
"golang.org/x/oauth2/google" "golang.org/x/oauth2/google"
) "google.golang.org/api/option"
const (
tokenUrl string = "https://www.googleapis.com/auth/devstorage.read_write" // #nosec
uploadUrl string = "https://www.googleapis.com/upload/storage/v1/b/%s/o?uploadType=media&name=%s"
publicReadOption string = "&predefinedAcl=publicRead"
bodySizeLimit = 1 << 20
) )
type GCSUploader struct { type GCSUploader struct {
...@@ -30,12 +23,12 @@ type GCSUploader struct { ...@@ -30,12 +23,12 @@ type GCSUploader struct {
bucket string bucket string
path string path string
log log.Logger log log.Logger
enableSignedUrls bool enableSignedURLs bool
signedUrlExpiration time.Duration signedURLExpiration time.Duration
} }
func NewGCSUploader(keyFile, bucket, path string, enableSignedUrls bool, signedUrlExpiration string) (*GCSUploader, error) { func NewGCSUploader(keyFile, bucket, path string, enableSignedURLs bool, signedURLExpiration string) (*GCSUploader, error) {
expiration, err := time.ParseDuration(signedUrlExpiration) expiration, err := time.ParseDuration(signedURLExpiration)
if err != nil { if err != nil {
return nil, err return nil, err
} }
...@@ -47,11 +40,12 @@ func NewGCSUploader(keyFile, bucket, path string, enableSignedUrls bool, signedU ...@@ -47,11 +40,12 @@ func NewGCSUploader(keyFile, bucket, path string, enableSignedUrls bool, signedU
bucket: bucket, bucket: bucket,
path: path, path: path,
log: log.New("gcsuploader"), log: log.New("gcsuploader"),
enableSignedUrls: enableSignedUrls, enableSignedURLs: enableSignedURLs,
signedUrlExpiration: expiration, signedURLExpiration: expiration,
} }
uploader.log.Debug(fmt.Sprintf("Created GCSUploader key=%q bucket=%q path=%q, enable_signed_urls=%v signed_url_expiration=%q", keyFile, bucket, path, enableSignedUrls, expiration.String())) uploader.log.Debug("Created GCSUploader", "key", keyFile, "bucket", bucket, "path", path, "enableSignedUrls",
enableSignedURLs, "signedUrlExpiration", expiration.String())
return uploader, nil return uploader, nil
} }
...@@ -65,59 +59,61 @@ func (u *GCSUploader) Upload(ctx context.Context, imageDiskPath string) (string, ...@@ -65,59 +59,61 @@ func (u *GCSUploader) Upload(ctx context.Context, imageDiskPath string) (string,
fileName += pngExt fileName += pngExt
key := path.Join(u.path, fileName) key := path.Join(u.path, fileName)
var client *http.Client var keyData []byte
if u.keyFile != "" { if u.keyFile != "" {
u.log.Debug("Opening key file ", u.keyFile) u.log.Debug("Opening key file ", u.keyFile)
data, err := ioutil.ReadFile(u.keyFile) keyData, err = ioutil.ReadFile(u.keyFile)
if err != nil { if err != nil {
return "", err return "", err
} }
}
const scope = storage.ScopeReadWrite
u.log.Debug("Creating JWT conf") var client *storage.Client
conf, err := google.JWTConfigFromJSON(data, tokenUrl) if u.keyFile != "" {
u.log.Debug("Creating Google credentials from JSON")
creds, err := google.CredentialsFromJSON(ctx, keyData, scope)
if err != nil { if err != nil {
return "", err return "", err
} }
u.log.Debug("Creating HTTP client") u.log.Debug("Creating GCS client")
client = conf.Client(ctx) client, err = storage.NewClient(ctx, option.WithCredentials(creds))
if err != nil {
return "", err
}
} else { } else {
u.log.Debug("Key file is empty, trying to use application default credentials") u.log.Debug("Creating GCS client with default application credentials")
client, err = google.DefaultClient(ctx) client, err = storage.NewClient(ctx, option.WithScopes(scope))
if err != nil { if err != nil {
return "", err return "", err
} }
} }
err = u.uploadFile(client, imageDiskPath, key) if err := u.uploadFile(ctx, client, imageDiskPath, key); err != nil {
if err != nil {
return "", err return "", err
} }
if !u.enableSignedUrls { if !u.enableSignedURLs {
return fmt.Sprintf("https://storage.googleapis.com/%s/%s", u.bucket, key), nil return fmt.Sprintf("https://storage.googleapis.com/%s/%s", u.bucket, key), nil
} }
u.log.Debug("Signing GCS URL") u.log.Debug("Signing GCS URL")
var conf *jwt.Config var conf *jwt.Config
if u.keyFile != "" { if u.keyFile != "" {
jsonKey, err := ioutil.ReadFile(u.keyFile) conf, err = google.JWTConfigFromJSON(keyData)
if err != nil {
return "", fmt.Errorf("ioutil.ReadFile: %v", err)
}
conf, err = google.JWTConfigFromJSON(jsonKey)
if err != nil { if err != nil {
return "", fmt.Errorf("google.JWTConfigFromJSON: %v", err) return "", err
} }
} else { } else {
creds, err := google.FindDefaultCredentials(ctx, storage.ScopeReadWrite) creds, err := google.FindDefaultCredentials(ctx, scope)
if err != nil { if err != nil {
return "", fmt.Errorf("google.FindDefaultCredentials: %v", err) return "", fmt.Errorf("failed to find default Google credentials: %s", err)
} }
conf, err = google.JWTConfigFromJSON(creds.JSON) conf, err = google.JWTConfigFromJSON(creds.JSON)
if err != nil { if err != nil {
return "", fmt.Errorf("google.JWTConfigFromJSON: %v", err) return "", err
} }
} }
opts := &storage.SignedURLOptions{ opts := &storage.SignedURLOptions{
...@@ -125,50 +121,46 @@ func (u *GCSUploader) Upload(ctx context.Context, imageDiskPath string) (string, ...@@ -125,50 +121,46 @@ func (u *GCSUploader) Upload(ctx context.Context, imageDiskPath string) (string,
Method: "GET", Method: "GET",
GoogleAccessID: conf.Email, GoogleAccessID: conf.Email,
PrivateKey: conf.PrivateKey, PrivateKey: conf.PrivateKey,
Expires: time.Now().Add(u.signedUrlExpiration), Expires: time.Now().Add(u.signedURLExpiration),
} }
signedUrl, err := storage.SignedURL(u.bucket, key, opts) signedURL, err := storage.SignedURL(u.bucket, key, opts)
if err != nil { if err != nil {
return "", fmt.Errorf("storage.SignedURL: %v", err) return "", err
} }
return signedUrl, nil
}
func (u *GCSUploader) uploadFile(client *http.Client, imageDiskPath, key string) error { return signedURL, nil
u.log.Debug("Opening image file ", imageDiskPath) }
func (u *GCSUploader) uploadFile(
ctx context.Context,
client *storage.Client,
imageDiskPath,
key string,
) error {
u.log.Debug("Opening image file", "path", imageDiskPath)
fileReader, err := os.Open(imageDiskPath) fileReader, err := os.Open(imageDiskPath)
if err != nil { if err != nil {
return err return err
} }
defer fileReader.Close() defer fileReader.Close()
reqUrl := fmt.Sprintf(uploadUrl, u.bucket, key) // Set public access if not generating a signed URL
if !u.enableSignedUrls { pubAcc := !u.enableSignedURLs
reqUrl += publicReadOption
}
u.log.Debug("Request URL: ", reqUrl)
req, err := http.NewRequest("POST", reqUrl, fileReader) u.log.Debug("Uploading to GCS bucket using SDK", "bucket", u.bucket, "key", key, "public", pubAcc)
if err != nil {
return err
}
req.Header.Add("Content-Type", "image/png") uri := fmt.Sprintf("gs://%s/%s", u.bucket, key)
u.log.Debug("Sending POST request to GCS")
resp, err := client.Do(req) wc := client.Bucket(u.bucket).Object(key).NewWriter(ctx)
if err != nil { if pubAcc {
return err wc.ObjectAttrs.PredefinedACL = "publicRead"
} }
defer resp.Body.Close() if _, err := io.Copy(wc, fileReader); err != nil {
_ = wc.Close()
if resp.StatusCode != 200 { return fmt.Errorf("failed to upload to %s: %s", uri, err)
respBody, err := ioutil.ReadAll(io.LimitReader(resp.Body, bodySizeLimit))
if err == nil && len(respBody) > 0 {
u.log.Error(fmt.Sprintf("GCS response: url=%q status=%d, body=%q", reqUrl, resp.StatusCode, string(respBody)))
} }
return fmt.Errorf("GCS response status code %d", resp.StatusCode) if err := wc.Close(); err != nil {
return fmt.Errorf("failed to upload to %s: %s", uri, err)
} }
return nil return nil
......
pkg/components/imguploader/imguploader.go
...@@ -86,10 +86,10 @@ func NewImageUploader() (ImageUploader, error) { ...@@ -86,10 +86,10 @@ func NewImageUploader() (ImageUploader, error) {
keyFile := gcssec.Key("key_file").MustString("") keyFile := gcssec.Key("key_file").MustString("")
bucketName := gcssec.Key("bucket").MustString("") bucketName := gcssec.Key("bucket").MustString("")
path := gcssec.Key("path").MustString("") path := gcssec.Key("path").MustString("")
enableSignedUrls := gcssec.Key("enable_signed_urls").MustBool(false) enableSignedURLs := gcssec.Key("enable_signed_urls").MustBool(false)
signedUrlExpiration := gcssec.Key("signed_url_expiration").MustString(defaultSGcsSignedUrlExpiration.String()) signedURLExpiration := gcssec.Key("signed_url_expiration").MustString(defaultSGcsSignedUrlExpiration.String())
return NewGCSUploader(keyFile, bucketName, path, enableSignedUrls, signedUrlExpiration) return NewGCSUploader(keyFile, bucketName, path, enableSignedURLs, signedURLExpiration)
case "azure_blob": case "azure_blob":
azureBlobSec, err := setting.Raw.GetSection("external_image_storage.azure_blob") azureBlobSec, err := setting.Raw.GetSection("external_image_storage.azure_blob")
if err != nil { if err != nil {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment