From e0a4ec1f1f72de3171920f650020f39181a2c74c Mon Sep 17 00:00:00 2001 From: DarthSim Date: Thu, 19 Nov 2020 20:47:18 +0600 Subject: [PATCH] Azure Blob Storage support --- CHANGELOG.md | 1 + azure_transport.go | 49 +++++++++++++++++++ config.go | 9 ++++ docs/_sidebar.md | 1 + docs/configuration.md | 11 +++++ docs/serving_files_from_azure_blob_storage.md | 8 +++ download.go | 8 +++ go.mod | 3 +- go.sum | 17 +++++++ 9 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 azure_transport.go create mode 100644 docs/serving_files_from_azure_blob_storage.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 5833387a..d0491d6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Unreleased] ### Added - AVIF support. +- Azure Blob Storage support. - (pro) Remove Adobe Illustrator garbage from SVGs. ### Changed diff --git a/azure_transport.go b/azure_transport.go new file mode 100644 index 00000000..ec33d70b --- /dev/null +++ b/azure_transport.go @@ -0,0 +1,49 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/Azure/azure-storage-blob-go/azblob" +) + +type azureTransport struct { + serviceURL *azblob.ServiceURL +} + +func newAzureTransport() (http.RoundTripper, error) { + credential, err := azblob.NewSharedKeyCredential(conf.ABSName, conf.ABSKey) + if err != nil { + return nil, err + } + + pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + + endpoint := conf.ABSEndpoint + if len(endpoint) == 0 { + endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", conf.ABSName) + } + endpointURL, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + + serviceURL := azblob.NewServiceURL(*endpointURL, pipeline) + + return azureTransport{&serviceURL}, nil +} + +func (t azureTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) { + containerURL := t.serviceURL.NewContainerURL(strings.ToLower(req.URL.Host)) + blobURL := containerURL.NewBlockBlobURL(strings.TrimPrefix(req.URL.Path, "/")) + + get, err := blobURL.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false) + if err != nil { + return nil, err + } + + return get.Response(), nil +} diff --git a/config.go b/config.go index 552f1996..4214cb6f 100644 --- a/config.go +++ b/config.go @@ -230,6 +230,10 @@ type config struct { S3Endpoint string GCSEnabled bool GCSKey string + ABSEnabled bool + ABSName string + ABSKey string + ABSEndpoint string ETagEnabled bool @@ -386,6 +390,11 @@ func configure() error { boolEnvConfig(&conf.GCSEnabled, "IMGPROXY_USE_GCS") strEnvConfig(&conf.GCSKey, "IMGPROXY_GCS_KEY") + boolEnvConfig(&conf.ABSEnabled, "IMGPROXY_USE_ABS") + strEnvConfig(&conf.ABSName, "IMGPROXY_ABS_NAME") + strEnvConfig(&conf.ABSKey, "IMGPROXY_ABS_KEY") + strEnvConfig(&conf.ABSEndpoint, "IMGPROXY_ABS_ENDPOINT") + boolEnvConfig(&conf.ETagEnabled, "IMGPROXY_USE_ETAG") strEnvConfig(&conf.BaseURL, "IMGPROXY_BASE_URL") diff --git a/docs/_sidebar.md b/docs/_sidebar.md index 6e7ac015..7f4d567d 100644 --- a/docs/_sidebar.md +++ b/docs/_sidebar.md @@ -10,6 +10,7 @@ * [Serving local files](serving_local_files) * [Serving files from Amazon S3](serving_files_from_s3) * [Serving files from Google Cloud Storage](serving_files_from_google_cloud_storage) +* [Serving files from Azure Blob Storage](serving_files_from_azure_blob_storage.md) * [New Relic](new_relic) * [Prometheus](prometheus) * [Image formats support](image_formats_support) diff --git a/docs/configuration.md b/docs/configuration.md index 940aa000..9f3e70b4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -247,6 +247,17 @@ imgproxy can process files from Google Cloud Storage buckets, but this feature i Check out the [Serving files from Google Cloud Storage](serving_files_from_google_cloud_storage.md) guide to learn more. +## Serving files from Azure Blob Storage + +imgproxy can process files from Azure Blob Storage containers, but this feature is disabled by default. To enable it, set `IMGPROXY_USE_ABS` to `true`: + +* `IMGPROXY_USE_ABS`: when `true`, enables image fetching from Azure Blob Storage containers. Default: false; +* `IMGPROXY_ABS_NAME`: Azure account name. Default: blank; +* `IMGPROXY_ABS_KEY`: Azure account key. Default: blank; +* `IMGPROXY_ABS_ENDPOINT`: custom Azure Blob Storage endpoint to being used by imgproxy. Default: blank. + +Check out the [Serving files from Azure Blob Storage](serving_files_from_azure_blob_storage.md) guide to learn more. + ## New Relic metrics imgproxy can send its metrics to New Relic. Specify your New Relic license key to activate this feature: diff --git a/docs/serving_files_from_azure_blob_storage.md b/docs/serving_files_from_azure_blob_storage.md new file mode 100644 index 00000000..d4061315 --- /dev/null +++ b/docs/serving_files_from_azure_blob_storage.md @@ -0,0 +1,8 @@ +# Serving files from Azure Blob Storage + +imgproxy can process images from Azure Blob Storage containers. To use this feature, do the following: + +1. Set `IMGPROXY_USE_ABS` environment variable as `true`; +2. Set `IMGPROXY_ABS_NAME` to your Azure account name and `IMGPROXY_ABS_KEY` to your Azure account key; +4. _(optional)_ Specify Azure Blob Storage endpoint with `IMGPROXY_ABS_ENDPOINT`; +4. Use `abs://%bucket_name/%file_key` as the source image URL. diff --git a/download.go b/download.go index 90d64cb2..1005b54d 100644 --- a/download.go +++ b/download.go @@ -79,6 +79,14 @@ func initDownloading() error { } } + if conf.ABSEnabled { + if t, err := newAzureTransport(); err != nil { + return err + } else { + transport.RegisterProtocol("abs", t) + } + } + downloadClient = &http.Client{ Timeout: time.Duration(conf.DownloadTimeout) * time.Second, Transport: transport, diff --git a/go.mod b/go.mod index 979617a3..e19afba7 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.11 require ( cloud.google.com/go/storage v1.10.0 + github.com/Azure/azure-storage-blob-go v0.11.0 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/aws/aws-sdk-go v1.34.0 github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b @@ -25,7 +26,7 @@ require ( github.com/stretchr/testify v1.6.1 golang.org/x/image v0.0.0-20200609002522-3f4726a040e8 golang.org/x/net v0.0.0-20200707034311-ab3426394381 - golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 + golang.org/x/sys v0.0.0-20200828194041-157a740278f4 golang.org/x/text v0.3.3 google.golang.org/api v0.30.0 ) diff --git a/go.sum b/go.sum index 8a793c61..4c56fe8e 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,15 @@ cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-storage-blob-go v0.11.0 h1:WCTHKKNkHlzm7lzUNXRSD11784LwJqdrxnwWJxsJQHg= +github.com/Azure/azure-storage-blob-go v0.11.0/go.mod h1:A0u4VjtpgZJ7Y7um/+ix2DHBuEKFC6sEIlj0xc13a4Q= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -267,6 +276,8 @@ github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/matoous/go-nanoid v1.4.1 h1:Yag04X+qPMDtYbyJsMDhoe8rP5kRl293b2QK8KRp2SE= github.com/matoous/go-nanoid v1.4.1/go.mod h1:fvGBnhcQ+zcrB3qJIG32PAN11J/y1IYkGX2/VeHzuH0= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= @@ -289,6 +300,7 @@ github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/newrelic/go-agent/v3 v3.9.0 h1:5bcTbdk/Up5cIYIkQjCG92Y+uNoett9wmhuz4kPiFlM= github.com/newrelic/go-agent/v3 v3.9.0/go.mod h1:1A1dssWBwzB7UemzRU6ZVaGDsI+cEn5/bNxI0wiYlIc= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -456,6 +468,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -515,6 +528,7 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8 h1:JA8d3MPx/IToSyXZG/RhwYEtfrKO1Fxrqe8KrkiLXKM= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -537,6 +551,8 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4 h1:kCCpuwSAoYJPkNc6x0xT9yTtV4oKtARo4RGBQWOfg9E= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -703,6 +719,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=