go/imgproxy
1
0
Fork 0
mirror of https://github.com/imgproxy/imgproxy.git synced 2025-10-30 23:08:02 +02:00
Code Releases Activity

Support OpenStack Swift Object Storage (#837)

Browse Source 
* Add OpenStack Swift support

* Fix linting errors

* Update CHANGELOG

* Update swift documentation

* Fix linting error

* Swift transport: pass req.Context down, fix parseObjectURL() implementation

* Make swift transport test a test suite

* Use swift://{container}/{object_path} as the source image URL format

* Cleanup

* Swift transport: only close object when returning 304
This commit is contained in:
Joe Cai
2022-04-06 21:00:19 +10:00
committed by GitHub
parent 078f896d21
commit 7a2296aee8
10 changed files with 289 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -4,6 +4,7 @@
### Add
- Add `IMGPROXY_FALLBACK_IMAGE_TTL` config.
- Add [watermark_size](https://docs.imgproxy.net/generating_the_url?id=watermark-size) processing option.
- Add OpenStack Object Storage ("Swift") support.
### Change
- (pro) Don't check `Content-Length` header of videos.

28
config/config.go
View File

@@ -89,6 +89,16 @@ var (
ABSName string
ABSKey string
ABSEndpoint string
SwiftEnabled bool
SwiftUsername string
SwiftAPIKey string
SwiftAuthURL string
SwiftDomain string
SwiftTenant string
SwiftAuthVersion int
SwiftConnectTimeoutSeconds int
SwiftTimeoutSeconds int
ETagEnabled bool
ETagBuster string
@@ -230,6 +240,15 @@ func Reset() {
ABSName = ""
ABSKey = ""
ABSEndpoint = ""
SwiftEnabled = false
SwiftUsername = ""
SwiftAPIKey = ""
SwiftAuthURL = ""
SwiftAuthVersion = 0
SwiftTenant = ""
SwiftDomain = ""
SwiftConnectTimeoutSeconds = 10
SwiftTimeoutSeconds = 60
ETagEnabled = false
ETagBuster = ""
@@ -384,6 +403,15 @@ func Configure() error {
configurators.String(&ABSKey, "IMGPROXY_ABS_KEY")
configurators.String(&ABSEndpoint, "IMGPROXY_ABS_ENDPOINT")
configurators.Bool(&SwiftEnabled, "IMGPROXY_USE_SWIFT")
configurators.String(&SwiftUsername, "IMGPROXY_SWIFT_USERNAME")
configurators.String(&SwiftAPIKey, "IMGPROXY_SWIFT_API_KEY")
configurators.String(&SwiftAuthURL, "IMGPROXY_SWIFT_AUTH_URL")
configurators.String(&SwiftDomain, "IMGPROXY_SWIFT_DOMAIN")
configurators.String(&SwiftTenant, "IMGPROXY_SWIFT_TENANT")
configurators.Int(&SwiftConnectTimeoutSeconds, "IMGPROXY_SWIFT_CONNECT_TIMEOUT_SECONDS")
configurators.Int(&SwiftTimeoutSeconds, "IMGPROXY_SWIFT_TIMEOUT_SECONDS")
configurators.Bool(&ETagEnabled, "IMGPROXY_USE_ETAG")
configurators.String(&ETagBuster, "IMGPROXY_ETAG_BUSTER")

1
docs/_sidebar.md
View File

@@ -13,6 +13,7 @@
* [Serving files from Amazon S3](serving_files_from_s3)
* [Serving files from Google Cloud Storage](serving_files_from_google_cloud_storage)
* [Serving files from Azure Blob Storage](serving_files_from_azure_blob_storage)
* [Serving files from OpenStack Object Storage ("Swift")](serving_files_from_openstack_swift)
* [New Relic](new_relic)
* [Prometheus](prometheus)
* [Datadog](datadog)

13
docs/configuration.md
View File

@@ -325,6 +325,19 @@ imgproxy can process files from Azure Blob Storage containers, but this feature
Check out the [Serving files from Azure Blob Storage](serving_files_from_azure_blob_storage.md) guide to learn more.
## Serving files from OpenStack Object Storage ("Swift")
imgproxy can process files from OpenStack Object Storage, but this feature is disabled by default. To enable it, set `IMGPROXY_USE_SWIFT` to `true`.
* `IMGPROXY_USE_SWIFT`: when `true`, enables image fetching from OpenStack Swift Object Storage. Default: `false`
* `IMGPROXY_SWIFT_USERNAME`: the username for Swift API access. Default: blank
* `IMGPROXY_SWIFT_API_KEY`: the API key for Swift API access. Default: blank
* `IMGPROXY_SWIFT_AUTH_URL`: the Swift Auth URL. Default: blank
* `IMGPROXY_SWIFT_AUTH_VERSION`: the Swift auth version, set to 1, 2 or 3 or leave at 0 for autodetect.
* `IMGPROXY_SWIFT_TENANT`: the tenant name (optional, v2 auth only). Default: blank
* `IMGPROXY_SWIFT_DOMAIN`: the Swift domain name (optional, v3 auth only): Default: blank
* `IMGRPOXY_SWIFT_TIMEOUT_SECONDS`: the data channel timeout in seconds. Default: 60
* `IMGRPOXY_SWIFT_CONNECT_TIMEOUT_SECONDS`: the connect channel timeout in seconds. Default: 10
## New Relic metrics
imgproxy can send its metrics to New Relic. Specify your New Relic license key to activate this feature:

16
docs/serving_files_from_openstack_swift.md Normal file
View File

@@ -0,0 +1,16 @@
# Serving files from OpenStack Object Storage ("Swift")
imgproxy can process images from OpenStack Object Storage, also known as Swift. To use this feature, do the following:
1. Set `IMGPROXY_USE_SWIFT` environment variable to `true`
2. Configure Swift authentication with the following environment variables
* `IMGPROXY_SWIFT_USERNAME`: the username for Swift API access. Default: blank
* `IMGPROXY_SWIFT_API_KEY`: the API key for Swift API access. Default: blank
* `IMGPROXY_SWIFT_AUTH_URL`: the Swift Auth URL. Default: blank
* `IMGPROXY_SWIFT_AUTH_VERSION`: the Swift auth version, set to 1, 2 or 3 or leave at 0 for autodetect.
* `IMGPROXY_SWIFT_TENANT`: the tenant name (optional, v2 auth only). Default: blank
* `IMGPROXY_SWIFT_DOMAIN`: the Swift domain name (optional, v3 auth only): Default: blank
3. Use `swift://%{container}/%{object_path}` as the source image URL. e.g. an original object storage URL in the format of
`/v1/{account}/{container}/{object_path}` such as `http://127.0.0.1:8080/v1/AUTH_test/images/flowers/rose.jpg` should
be converted to `swift://images/flowers/rose.jpg`.

3
go.mod
View File

@@ -16,6 +16,7 @@ require (
github.com/honeybadger-io/honeybadger-go v0.5.0
github.com/ianlancetaylor/cgosymbolizer v0.0.0-20220217162856-c813f11194b9 // indirect
github.com/matoous/go-nanoid/v2 v2.0.0
github.com/ncw/swift/v2 v2.0.1
github.com/newrelic/go-agent/v3 v3.15.2
github.com/prometheus/client_golang v1.12.1
github.com/sirupsen/logrus v1.8.1
@@ -32,3 +33,5 @@ require (
replace git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999
replace github.com/shirou/gopsutil => github.com/shirou/gopsutil v2.20.9+incompatible
replace github.com/go-chi/chi/v4 => github.com/go-chi/chi v4.0.0+incompatible

3
go.sum
View File

@@ -298,6 +298,7 @@ github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0
github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
github.com/go-chi/chi v1.5.0/go.mod h1:REp24E+25iKvxgeTfHmdUoL5x15kBiDBlnIl5bCwe2k=
github.com/go-chi/chi v4.0.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
github.com/go-chi/chi/v4 v4.0.0-rc1/go.mod h1:Yfiy+5nynjDc7IMJiguACIro1KxlGW2dLUqcroaEUEY=
github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs=
github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
@@ -771,6 +772,8 @@ github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5Vgl
github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0=
github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
github.com/newrelic/go-agent/v3 v3.15.2 h1:NEpksu2AhuZncbwkDqUg2IvUJst3JQ/TemYfK4WdS/Y=
github.com/newrelic/go-agent/v3 v3.15.2/go.mod h1:1A1dssWBwzB7UemzRU6ZVaGDsI+cEn5/bNxI0wiYlIc=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=

9
imagedata/download.go
View File

@@ -17,6 +17,7 @@ import (
fsTransport "github.com/imgproxy/imgproxy/v3/transport/fs"
gcsTransport "github.com/imgproxy/imgproxy/v3/transport/gcs"
s3Transport "github.com/imgproxy/imgproxy/v3/transport/s3"
swiftTransport "github.com/imgproxy/imgproxy/v3/transport/swift"
)
var (
@@ -94,6 +95,14 @@ func initDownloading() error {
}
}
if config.SwiftEnabled {
if t, err := swiftTransport.New(); err != nil {
return err
} else {
registerProtocol("swift", t)
}
}
downloadClient = &http.Client{
Timeout: time.Duration(config.DownloadTimeout) * time.Second,
Transport: transport,

88
transport/swift/swift.go Normal file
View File

@@ -0,0 +1,88 @@
package swift
import (
"context"
"fmt"
"net/http"
"strings"
"time"
"github.com/imgproxy/imgproxy/v3/config"
"github.com/ncw/swift/v2"
)
type transport struct {
con *swift.Connection
}
func New() (http.RoundTripper, error) {
c := &swift.Connection{
UserName: config.SwiftUsername,
ApiKey: config.SwiftAPIKey,
AuthUrl: config.SwiftAuthURL,
AuthVersion: config.SwiftAuthVersion,
Domain: config.SwiftDomain, // v3 auth only
Tenant: config.SwiftTenant, // v2 auth only
Timeout: time.Duration(config.SwiftTimeoutSeconds) * time.Second,
ConnectTimeout: time.Duration(config.SwiftConnectTimeoutSeconds) * time.Second,
}
ctx := context.Background()
err := c.Authenticate(ctx)
if err != nil {
return nil, fmt.Errorf("swift authentication error: %s", err)
}
return transport{con: c}, nil
}
func (t transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
// Users should have converted the object storage URL in the format of swift://{container}/{object}
container := req.URL.Host
objectName := strings.TrimPrefix(req.URL.Path, "/")
headers := make(swift.Headers)
object, headers, err := t.con.ObjectOpen(req.Context(), container, objectName, false, headers)
if err != nil {
return nil, fmt.Errorf("error opening object: %v", err)
}
header := make(http.Header)
if config.ETagEnabled {
if etag, ok := headers["Etag"]; ok {
header.Set("ETag", etag)
if len(etag) > 0 && etag == req.Header.Get("If-None-Match") {
object.Close()
return &http.Response{
StatusCode: http.StatusNotModified,
Proto: "HTTP/1.0",
ProtoMajor: 1,
ProtoMinor: 0,
Header: header,
ContentLength: 0,
Body: nil,
Close: false,
Request: req,
}, nil
}
}
}
return &http.Response{
Status: "200 OK",
StatusCode: 200,
Proto: "HTTP/1.0",
ProtoMajor: 1,
ProtoMinor: 0,
Header: header,
Body: object,
Close: true,
Request: req,
}, nil
}

127
transport/swift/swift_test.go Normal file
View File

@@ -0,0 +1,127 @@
package swift
import (
"context"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"testing"
"github.com/imgproxy/imgproxy/v3/config"
"github.com/ncw/swift/v2"
"github.com/ncw/swift/v2/swifttest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
)
const (
testContainer = "test"
testObject = "foo/test.png"
)
type SwiftTestSuite struct {
suite.Suite
server *swifttest.SwiftServer
transport http.RoundTripper
}
func (s *SwiftTestSuite) SetupSuite() {
s.server, _ = swifttest.NewSwiftServer("localhost")
config.Reset()
config.SwiftAuthURL = s.server.AuthURL
config.SwiftUsername = swifttest.TEST_ACCOUNT
config.SwiftAPIKey = swifttest.TEST_ACCOUNT
config.SwiftAuthVersion = 1
s.setupTestFile()
var err error
s.transport, err = New()
assert.Nil(s.T(), err, "failed to initialize swift transport")
assert.IsType(s.T(), transport{}, s.transport)
}
func (s *SwiftTestSuite) setupTestFile() {
t := s.T()
c := &swift.Connection{
UserName: config.SwiftUsername,
ApiKey: config.SwiftAPIKey,
AuthUrl: config.SwiftAuthURL,
AuthVersion: config.SwiftAuthVersion,
}
ctx := context.Background()
err := c.Authenticate(ctx)
assert.Nil(t, err, "failed to authenticate with test server")
err = c.ContainerCreate(ctx, testContainer, nil)
assert.Nil(t, err, "failed to create container")
f, err := c.ObjectCreate(ctx, testContainer, testObject, true, "", "image/png", nil)
assert.Nil(t, err, "failed to create object")
defer f.Close()
wd, err := os.Getwd()
assert.Nil(t, err)
data, err := ioutil.ReadFile(filepath.Join(wd, "..", "..", "testdata", "test1.png"))
assert.Nil(t, err, "failed to read testdata/test1.png")
b, err := f.Write(data)
assert.Greater(t, b, 100)
assert.Nil(t, err)
}
func (s *SwiftTestSuite) TearDownSuite() {
s.server.Close()
}
func (s *SwiftTestSuite) TestRoundTripWithETagDisabledReturns200() {
config.ETagEnabled = false
request, _ := http.NewRequest("GET", "swift://test/foo/test.png", nil)
response, err := s.transport.RoundTrip(request)
assert.Nil(s.T(), err)
assert.Equal(s.T(), 200, response.StatusCode)
}
func (s *SwiftTestSuite) TestRoundTripWithETagEnabled() {
config.ETagEnabled = true
request, _ := http.NewRequest("GET", "swift://test/foo/test.png", nil)
response, err := s.transport.RoundTrip(request)
assert.Nil(s.T(), err)
assert.Equal(s.T(), 200, response.StatusCode)
assert.Equal(s.T(), "e27ca34142be8e55220e44155c626cd0", response.Header.Get("ETag"))
}
func (s *SwiftTestSuite) TestRoundTripWithIfNoneMatchReturns304() {
config.ETagEnabled = true
request, _ := http.NewRequest("GET", "swift://test/foo/test.png", nil)
request.Header.Set("If-None-Match", "e27ca34142be8e55220e44155c626cd0")
response, err := s.transport.RoundTrip(request)
assert.Nil(s.T(), err)
assert.Equal(s.T(), http.StatusNotModified, response.StatusCode)
}
func (s *SwiftTestSuite) TestRoundTripWithUpdatedETagReturns200() {
config.ETagEnabled = true
request, _ := http.NewRequest("GET", "swift://test/foo/test.png", nil)
request.Header.Set("If-None-Match", "foobar")
response, err := s.transport.RoundTrip(request)
assert.Nil(s.T(), err)
assert.Equal(s.T(), http.StatusOK, response.StatusCode)
}
func TestSwiftTransport(t *testing.T) {
suite.Run(t, new(SwiftTestSuite))
}