Skip to content

N
nexpie-grafana-theme
Commit 1646de45 by gastonqiu Committed by GitHub
Options

Image uploader: Fix uploading of images to GCS (#26493) 

* GCS image uploader: Re-implement with Google SDK

Signed-off-by: gaston.qiu <gaston.qiu@umbocv.com>
Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
parent b3a86816
Showing with 72 additions and 77 deletions
go.mod
......@@ -11,8 +11,8 @@ replace github.com/denisenkom/go-mssqldb => github.com/denisenkom/go-mssqldb v0.
replace k8s.io/client-go => k8s.io/client-go v0.18.8
require (
cloud.google.com/go v0.60.0 // indirect
cloud.google.com/go/storage v1.8.0
cloud.google.com/go v0.70.0 // indirect
cloud.google.com/go/storage v1.12.0
github.com/BurntSushi/toml v0.3.1
github.com/VividCortex/mysqlerr v0.0.0-20170204212430-6c6b55f8796f
github.com/aws/aws-sdk-go v1.33.12
......@@ -37,8 +37,8 @@ require (
github.com/go-sql-driver/mysql v1.5.0
github.com/go-stack/stack v1.8.0
github.com/gobwas/glob v0.2.3
github.com/golang/protobuf v1.4.2
github.com/google/go-cmp v0.5.0
github.com/golang/protobuf v1.4.3
github.com/google/go-cmp v0.5.2
github.com/gosimple/slug v1.4.2
github.com/grafana/grafana-plugin-model v0.0.0-20190930120109-1fc953a61fb4
github.com/grafana/grafana-plugin-sdk-go v0.78.0
......@@ -82,12 +82,15 @@ require (
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
github.com/yudai/pp v2.0.1+incompatible // indirect
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d // indirect
golang.org/x/text v0.3.3 // indirect
google.golang.org/grpc v1.30.0
golang.org/x/net v0.0.0-20201022231255-08b38378de70
golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
golang.org/x/sys v0.0.0-20201022201747-fb209a7c41cd // indirect
golang.org/x/tools v0.0.0-20201023150057-2f4fa188d925 // indirect
google.golang.org/api v0.33.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20201022181438-0ff5f38871d5 // indirect
google.golang.org/grpc v1.33.1
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/ini.v1 v1.51.0
......
pkg/components/imguploader/gcsuploader.go
......@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path"
"time"
......@@ -16,13 +15,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/util"
"golang.org/x/oauth2/google"
)
const (
tokenUrl string = "https://www.googleapis.com/auth/devstorage.read_write" // #nosec
uploadUrl string = "https://www.googleapis.com/upload/storage/v1/b/%s/o?uploadType=media&name=%s"
publicReadOption string = "&predefinedAcl=publicRead"
bodySizeLimit = 1 << 20
"google.golang.org/api/option"
)
type GCSUploader struct {
......@@ -30,12 +23,12 @@ type GCSUploader struct {
bucket string
path string
log log.Logger
enableSignedUrls bool
signedUrlExpiration time.Duration
enableSignedURLs bool
signedURLExpiration time.Duration
}
func NewGCSUploader(keyFile, bucket, path string, enableSignedUrls bool, signedUrlExpiration string) (*GCSUploader, error) {
expiration, err := time.ParseDuration(signedUrlExpiration)
func NewGCSUploader(keyFile, bucket, path string, enableSignedURLs bool, signedURLExpiration string) (*GCSUploader, error) {
expiration, err := time.ParseDuration(signedURLExpiration)
if err != nil {
return nil, err
}
......@@ -47,11 +40,12 @@ func NewGCSUploader(keyFile, bucket, path string, enableSignedUrls bool, signedU
bucket: bucket,
path: path,
log: log.New("gcsuploader"),
enableSignedUrls: enableSignedUrls,
signedUrlExpiration: expiration,
enableSignedURLs: enableSignedURLs,
signedURLExpiration: expiration,
}
uploader.log.Debug(fmt.Sprintf("Created GCSUploader key=%q bucket=%q path=%q, enable_signed_urls=%v signed_url_expiration=%q", keyFile, bucket, path, enableSignedUrls, expiration.String()))
uploader.log.Debug("Created GCSUploader", "key", keyFile, "bucket", bucket, "path", path, "enableSignedUrls",
enableSignedURLs, "signedUrlExpiration", expiration.String())
return uploader, nil
}
......@@ -65,59 +59,61 @@ func (u *GCSUploader) Upload(ctx context.Context, imageDiskPath string) (string,
fileName += pngExt
key := path.Join(u.path, fileName)
var client *http.Client
var keyData []byte
if u.keyFile != "" {
u.log.Debug("Opening key file ", u.keyFile)
data, err := ioutil.ReadFile(u.keyFile)
keyData, err = ioutil.ReadFile(u.keyFile)
if err != nil {
return "", err
}
}
const scope = storage.ScopeReadWrite
u.log.Debug("Creating JWT conf")
conf, err := google.JWTConfigFromJSON(data, tokenUrl)
var client *storage.Client
if u.keyFile != "" {
u.log.Debug("Creating Google credentials from JSON")
creds, err := google.CredentialsFromJSON(ctx, keyData, scope)
if err != nil {
return "", err
}
u.log.Debug("Creating HTTP client")
client = conf.Client(ctx)
u.log.Debug("Creating GCS client")
client, err = storage.NewClient(ctx, option.WithCredentials(creds))
if err != nil {
return "", err
}
} else {
u.log.Debug("Key file is empty, trying to use application default credentials")
client, err = google.DefaultClient(ctx)
u.log.Debug("Creating GCS client with default application credentials")
client, err = storage.NewClient(ctx, option.WithScopes(scope))
if err != nil {
return "", err
}
}
err = u.uploadFile(client, imageDiskPath, key)
if err != nil {
if err := u.uploadFile(ctx, client, imageDiskPath, key); err != nil {
return "", err
}
if !u.enableSignedUrls {
if !u.enableSignedURLs {
return fmt.Sprintf("https://storage.googleapis.com/%s/%s", u.bucket, key), nil
}
u.log.Debug("Signing GCS URL")
var conf *jwt.Config
if u.keyFile != "" {
jsonKey, err := ioutil.ReadFile(u.keyFile)
if err != nil {
return "", fmt.Errorf("ioutil.ReadFile: %v", err)
}
conf, err = google.JWTConfigFromJSON(jsonKey)
conf, err = google.JWTConfigFromJSON(keyData)
if err != nil {
return "", fmt.Errorf("google.JWTConfigFromJSON: %v", err)
return "", err
}
} else {
creds, err := google.FindDefaultCredentials(ctx, storage.ScopeReadWrite)
creds, err := google.FindDefaultCredentials(ctx, scope)
if err != nil {
return "", fmt.Errorf("google.FindDefaultCredentials: %v", err)
return "", fmt.Errorf("failed to find default Google credentials: %s", err)
}
conf, err = google.JWTConfigFromJSON(creds.JSON)
if err != nil {
return "", fmt.Errorf("google.JWTConfigFromJSON: %v", err)
return "", err
}
}
opts := &storage.SignedURLOptions{
......@@ -125,50 +121,46 @@ func (u *GCSUploader) Upload(ctx context.Context, imageDiskPath string) (string,
Method: "GET",
GoogleAccessID: conf.Email,
PrivateKey: conf.PrivateKey,
Expires: time.Now().Add(u.signedUrlExpiration),
Expires: time.Now().Add(u.signedURLExpiration),
}
signedUrl, err := storage.SignedURL(u.bucket, key, opts)
signedURL, err := storage.SignedURL(u.bucket, key, opts)
if err != nil {
return "", fmt.Errorf("storage.SignedURL: %v", err)
return "", err
}
return signedUrl, nil
}
func (u *GCSUploader) uploadFile(client *http.Client, imageDiskPath, key string) error {
u.log.Debug("Opening image file ", imageDiskPath)
return signedURL, nil
}
func (u *GCSUploader) uploadFile(
ctx context.Context,
client *storage.Client,
imageDiskPath,
key string,
) error {
u.log.Debug("Opening image file", "path", imageDiskPath)
fileReader, err := os.Open(imageDiskPath)
if err != nil {
return err
}
defer fileReader.Close()
reqUrl := fmt.Sprintf(uploadUrl, u.bucket, key)
if !u.enableSignedUrls {
reqUrl += publicReadOption
}
u.log.Debug("Request URL: ", reqUrl)
// Set public access if not generating a signed URL
pubAcc := !u.enableSignedURLs
req, err := http.NewRequest("POST", reqUrl, fileReader)
if err != nil {
return err
}
u.log.Debug("Uploading to GCS bucket using SDK", "bucket", u.bucket, "key", key, "public", pubAcc)
req.Header.Add("Content-Type", "image/png")
u.log.Debug("Sending POST request to GCS")
uri := fmt.Sprintf("gs://%s/%s", u.bucket, key)
resp, err := client.Do(req)
if err != nil {
return err
wc := client.Bucket(u.bucket).Object(key).NewWriter(ctx)
if pubAcc {
wc.ObjectAttrs.PredefinedACL = "publicRead"
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
respBody, err := ioutil.ReadAll(io.LimitReader(resp.Body, bodySizeLimit))
if err == nil && len(respBody) > 0 {
u.log.Error(fmt.Sprintf("GCS response: url=%q status=%d, body=%q", reqUrl, resp.StatusCode, string(respBody)))
if _, err := io.Copy(wc, fileReader); err != nil {
_ = wc.Close()
return fmt.Errorf("failed to upload to %s: %s", uri, err)
}
return fmt.Errorf("GCS response status code %d", resp.StatusCode)
if err := wc.Close(); err != nil {
return fmt.Errorf("failed to upload to %s: %s", uri, err)
}
return nil
......
pkg/components/imguploader/imguploader.go
......@@ -86,10 +86,10 @@ func NewImageUploader() (ImageUploader, error) {
keyFile := gcssec.Key("key_file").MustString("")
bucketName := gcssec.Key("bucket").MustString("")
path := gcssec.Key("path").MustString("")
enableSignedUrls := gcssec.Key("enable_signed_urls").MustBool(false)
signedUrlExpiration := gcssec.Key("signed_url_expiration").MustString(defaultSGcsSignedUrlExpiration.String())
enableSignedURLs := gcssec.Key("enable_signed_urls").MustBool(false)
signedURLExpiration := gcssec.Key("signed_url_expiration").MustString(defaultSGcsSignedUrlExpiration.String())
return NewGCSUploader(keyFile, bucketName, path, enableSignedUrls, signedUrlExpiration)
return NewGCSUploader(keyFile, bucketName, path, enableSignedURLs, signedURLExpiration)
case "azure_blob":
azureBlobSec, err := setting.Raw.GetSection("external_image_storage.azure_blob")
if err != nil {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment