Support OpenStack Swift Object Storage (#837)

* Add OpenStack Swift support

* Fix linting errors

* Update CHANGELOG

* Update swift documentation

* Fix linting error

* Swift transport: pass req.Context down, fix parseObjectURL() implementation

* Make swift transport test a test suite

* Use swift://{container}/{object_path} as the source image URL format

* Cleanup

* Swift transport: only close object when returning 304

CHANGELOG.md

@@ -4,6 +4,7 @@
 ### Add
 - Add `IMGPROXY_FALLBACK_IMAGE_TTL` config.
 - Add [watermark_size](https://docs.imgproxy.net/generating_the_url?id=watermark-size) processing option.
+- Add OpenStack Object Storage ("Swift") support.
 
 ### Change
 - (pro) Don't check `Content-Length` header of videos.

config/config.go

@@ -89,6 +89,16 @@ var (
 	ABSName             string
 	ABSKey              string
 	ABSEndpoint         string
+	SwiftEnabled        bool
+	SwiftUsername       string
+	SwiftAPIKey         string
+	SwiftAuthURL        string
+	SwiftDomain         string
+	SwiftTenant         string
+	SwiftAuthVersion    int
+
+	SwiftConnectTimeoutSeconds int
+	SwiftTimeoutSeconds        int
 
 	ETagEnabled bool
 	ETagBuster  string
@@ -230,6 +240,15 @@ func Reset() {
 	ABSName = ""
 	ABSKey = ""
 	ABSEndpoint = ""
+	SwiftEnabled = false
+	SwiftUsername = ""
+	SwiftAPIKey = ""
+	SwiftAuthURL = ""
+	SwiftAuthVersion = 0
+	SwiftTenant = ""
+	SwiftDomain = ""
+	SwiftConnectTimeoutSeconds = 10
+	SwiftTimeoutSeconds = 60
 
 	ETagEnabled = false
 	ETagBuster = ""
@@ -384,6 +403,15 @@ func Configure() error {
 	configurators.String(&ABSKey, "IMGPROXY_ABS_KEY")
 	configurators.String(&ABSEndpoint, "IMGPROXY_ABS_ENDPOINT")
 
+	configurators.Bool(&SwiftEnabled, "IMGPROXY_USE_SWIFT")
+	configurators.String(&SwiftUsername, "IMGPROXY_SWIFT_USERNAME")
+	configurators.String(&SwiftAPIKey, "IMGPROXY_SWIFT_API_KEY")
+	configurators.String(&SwiftAuthURL, "IMGPROXY_SWIFT_AUTH_URL")
+	configurators.String(&SwiftDomain, "IMGPROXY_SWIFT_DOMAIN")
+	configurators.String(&SwiftTenant, "IMGPROXY_SWIFT_TENANT")
+	configurators.Int(&SwiftConnectTimeoutSeconds, "IMGPROXY_SWIFT_CONNECT_TIMEOUT_SECONDS")
+	configurators.Int(&SwiftTimeoutSeconds, "IMGPROXY_SWIFT_TIMEOUT_SECONDS")
+
 	configurators.Bool(&ETagEnabled, "IMGPROXY_USE_ETAG")
 	configurators.String(&ETagBuster, "IMGPROXY_ETAG_BUSTER")
 

docs/_sidebar.md

@@ -13,6 +13,7 @@
 * [Serving files from Amazon S3](serving_files_from_s3)
 * [Serving files from Google Cloud Storage](serving_files_from_google_cloud_storage)
 * [Serving files from Azure Blob Storage](serving_files_from_azure_blob_storage)
+* [Serving files from OpenStack Object Storage ("Swift")](serving_files_from_openstack_swift)
 * [New Relic](new_relic)
 * [Prometheus](prometheus)
 * [Datadog](datadog)

docs/configuration.md

@@ -325,6 +325,19 @@ imgproxy can process files from Azure Blob Storage containers, but this feature
 
 Check out the [Serving files from Azure Blob Storage](serving_files_from_azure_blob_storage.md) guide to learn more.
 
+## Serving files from OpenStack Object Storage ("Swift")
+imgproxy can process files from OpenStack Object Storage, but this feature is disabled by default. To enable it, set `IMGPROXY_USE_SWIFT` to `true`.
+* `IMGPROXY_USE_SWIFT`: when `true`, enables image fetching from OpenStack Swift Object Storage. Default: `false`
+* `IMGPROXY_SWIFT_USERNAME`: the username for Swift API access. Default: blank
+* `IMGPROXY_SWIFT_API_KEY`: the API key for Swift API access. Default: blank
+* `IMGPROXY_SWIFT_AUTH_URL`: the Swift Auth URL. Default: blank
+* `IMGPROXY_SWIFT_AUTH_VERSION`: the Swift auth version, set to 1, 2 or 3 or leave at 0 for autodetect.
+* `IMGPROXY_SWIFT_TENANT`: the tenant name (optional, v2 auth only). Default: blank
+* `IMGPROXY_SWIFT_DOMAIN`: the Swift domain name (optional, v3 auth only): Default: blank
+* `IMGRPOXY_SWIFT_TIMEOUT_SECONDS`: the data channel timeout in seconds. Default: 60
+* `IMGRPOXY_SWIFT_CONNECT_TIMEOUT_SECONDS`: the connect channel timeout in seconds. Default: 10
+
+
 ## New Relic metrics
 
 imgproxy can send its metrics to New Relic. Specify your New Relic license key to activate this feature:

docs/serving_files_from_openstack_swift.md

@@ -0,0 +1,16 @@
+# Serving files from OpenStack Object Storage ("Swift")
+
+imgproxy can process images from OpenStack Object Storage, also known as Swift. To use this feature, do the following:
+
+1. Set `IMGPROXY_USE_SWIFT` environment variable to `true`
+2. Configure Swift authentication with the following environment variables
+   * `IMGPROXY_SWIFT_USERNAME`: the username for Swift API access. Default: blank
+   * `IMGPROXY_SWIFT_API_KEY`: the API key for Swift API access. Default: blank
+   * `IMGPROXY_SWIFT_AUTH_URL`: the Swift Auth URL. Default: blank
+   * `IMGPROXY_SWIFT_AUTH_VERSION`: the Swift auth version, set to 1, 2 or 3 or leave at 0 for autodetect.
+   * `IMGPROXY_SWIFT_TENANT`: the tenant name (optional, v2 auth only). Default: blank
+   * `IMGPROXY_SWIFT_DOMAIN`: the Swift domain name (optional, v3 auth only): Default: blank
+
+3. Use `swift://%{container}/%{object_path}` as the source image URL. e.g. an original object storage URL in the format of
+   `/v1/{account}/{container}/{object_path}` such as `http://127.0.0.1:8080/v1/AUTH_test/images/flowers/rose.jpg` should
+   be converted to `swift://images/flowers/rose.jpg`. 

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/honeybadger-io/honeybadger-go v0.5.0
 	github.com/ianlancetaylor/cgosymbolizer v0.0.0-20220217162856-c813f11194b9 // indirect
 	github.com/matoous/go-nanoid/v2 v2.0.0
+	github.com/ncw/swift/v2 v2.0.1
 	github.com/newrelic/go-agent/v3 v3.15.2
 	github.com/prometheus/client_golang v1.12.1
 	github.com/sirupsen/logrus v1.8.1
@@ -32,3 +33,5 @@ require (
 replace git.apache.org/thrift.git => github.com/apache/thrift v0.0.0-20180902110319-2566ecd5d999
 
 replace github.com/shirou/gopsutil => github.com/shirou/gopsutil v2.20.9+incompatible
+
+replace github.com/go-chi/chi/v4 => github.com/go-chi/chi v4.0.0+incompatible

go.sum

@@ -298,6 +298,7 @@ github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0
 github.com/go-asn1-ber/asn1-ber v1.3.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
 github.com/go-chi/chi v1.5.0/go.mod h1:REp24E+25iKvxgeTfHmdUoL5x15kBiDBlnIl5bCwe2k=
+github.com/go-chi/chi v4.0.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
 github.com/go-chi/chi/v4 v4.0.0-rc1/go.mod h1:Yfiy+5nynjDc7IMJiguACIro1KxlGW2dLUqcroaEUEY=
 github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
@@ -771,6 +772,8 @@ github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5Vgl
 github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
 github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0=
+github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
 github.com/newrelic/go-agent/v3 v3.15.2 h1:NEpksu2AhuZncbwkDqUg2IvUJst3JQ/TemYfK4WdS/Y=
 github.com/newrelic/go-agent/v3 v3.15.2/go.mod h1:1A1dssWBwzB7UemzRU6ZVaGDsI+cEn5/bNxI0wiYlIc=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=

imagedata/download.go

@@ -17,6 +17,7 @@ import (
 	fsTransport "github.com/imgproxy/imgproxy/v3/transport/fs"
 	gcsTransport "github.com/imgproxy/imgproxy/v3/transport/gcs"
 	s3Transport "github.com/imgproxy/imgproxy/v3/transport/s3"
+	swiftTransport "github.com/imgproxy/imgproxy/v3/transport/swift"
 )
 
 var (
@@ -94,6 +95,14 @@ func initDownloading() error {
 		}
 	}
 
+	if config.SwiftEnabled {
+		if t, err := swiftTransport.New(); err != nil {
+			return err
+		} else {
+			registerProtocol("swift", t)
+		}
+	}
+
 	downloadClient = &http.Client{
 		Timeout:   time.Duration(config.DownloadTimeout) * time.Second,
 		Transport: transport,

transport/swift/swift.go

@@ -0,0 +1,88 @@
+package swift
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/imgproxy/imgproxy/v3/config"
+	"github.com/ncw/swift/v2"
+)
+
+type transport struct {
+	con *swift.Connection
+}
+
+func New() (http.RoundTripper, error) {
+	c := &swift.Connection{
+		UserName:       config.SwiftUsername,
+		ApiKey:         config.SwiftAPIKey,
+		AuthUrl:        config.SwiftAuthURL,
+		AuthVersion:    config.SwiftAuthVersion,
+		Domain:         config.SwiftDomain, // v3 auth only
+		Tenant:         config.SwiftTenant, // v2 auth only
+		Timeout:        time.Duration(config.SwiftTimeoutSeconds) * time.Second,
+		ConnectTimeout: time.Duration(config.SwiftConnectTimeoutSeconds) * time.Second,
+	}
+
+	ctx := context.Background()
+
+	err := c.Authenticate(ctx)
+
+	if err != nil {
+		return nil, fmt.Errorf("swift authentication error: %s", err)
+	}
+
+	return transport{con: c}, nil
+}
+
+func (t transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
+	// Users should have converted the object storage URL in the format of swift://{container}/{object}
+	container := req.URL.Host
+	objectName := strings.TrimPrefix(req.URL.Path, "/")
+
+	headers := make(swift.Headers)
+
+	object, headers, err := t.con.ObjectOpen(req.Context(), container, objectName, false, headers)
+
+	if err != nil {
+		return nil, fmt.Errorf("error opening object: %v", err)
+	}
+
+	header := make(http.Header)
+
+	if config.ETagEnabled {
+		if etag, ok := headers["Etag"]; ok {
+			header.Set("ETag", etag)
+
+			if len(etag) > 0 && etag == req.Header.Get("If-None-Match") {
+				object.Close()
+				return &http.Response{
+					StatusCode:    http.StatusNotModified,
+					Proto:         "HTTP/1.0",
+					ProtoMajor:    1,
+					ProtoMinor:    0,
+					Header:        header,
+					ContentLength: 0,
+					Body:          nil,
+					Close:         false,
+					Request:       req,
+				}, nil
+			}
+		}
+	}
+
+	return &http.Response{
+		Status:     "200 OK",
+		StatusCode: 200,
+		Proto:      "HTTP/1.0",
+		ProtoMajor: 1,
+		ProtoMinor: 0,
+		Header:     header,
+		Body:       object,
+		Close:      true,
+		Request:    req,
+	}, nil
+}

transport/swift/swift_test.go

@@ -0,0 +1,127 @@
+package swift
+
+import (
+	"context"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/imgproxy/imgproxy/v3/config"
+	"github.com/ncw/swift/v2"
+	"github.com/ncw/swift/v2/swifttest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+)
+
+const (
+	testContainer = "test"
+	testObject    = "foo/test.png"
+)
+
+type SwiftTestSuite struct {
+	suite.Suite
+	server    *swifttest.SwiftServer
+	transport http.RoundTripper
+}
+
+func (s *SwiftTestSuite) SetupSuite() {
+	s.server, _ = swifttest.NewSwiftServer("localhost")
+
+	config.Reset()
+
+	config.SwiftAuthURL = s.server.AuthURL
+	config.SwiftUsername = swifttest.TEST_ACCOUNT
+	config.SwiftAPIKey = swifttest.TEST_ACCOUNT
+	config.SwiftAuthVersion = 1
+
+	s.setupTestFile()
+
+	var err error
+	s.transport, err = New()
+	assert.Nil(s.T(), err, "failed to initialize swift transport")
+	assert.IsType(s.T(), transport{}, s.transport)
+}
+
+func (s *SwiftTestSuite) setupTestFile() {
+	t := s.T()
+	c := &swift.Connection{
+		UserName:    config.SwiftUsername,
+		ApiKey:      config.SwiftAPIKey,
+		AuthUrl:     config.SwiftAuthURL,
+		AuthVersion: config.SwiftAuthVersion,
+	}
+
+	ctx := context.Background()
+
+	err := c.Authenticate(ctx)
+	assert.Nil(t, err, "failed to authenticate with test server")
+
+	err = c.ContainerCreate(ctx, testContainer, nil)
+	assert.Nil(t, err, "failed to create container")
+
+	f, err := c.ObjectCreate(ctx, testContainer, testObject, true, "", "image/png", nil)
+	assert.Nil(t, err, "failed to create object")
+
+	defer f.Close()
+
+	wd, err := os.Getwd()
+	assert.Nil(t, err)
+
+	data, err := ioutil.ReadFile(filepath.Join(wd, "..", "..", "testdata", "test1.png"))
+	assert.Nil(t, err, "failed to read testdata/test1.png")
+
+	b, err := f.Write(data)
+	assert.Greater(t, b, 100)
+	assert.Nil(t, err)
+}
+
+func (s *SwiftTestSuite) TearDownSuite() {
+	s.server.Close()
+}
+
+func (s *SwiftTestSuite) TestRoundTripWithETagDisabledReturns200() {
+	config.ETagEnabled = false
+	request, _ := http.NewRequest("GET", "swift://test/foo/test.png", nil)
+
+	response, err := s.transport.RoundTrip(request)
+	assert.Nil(s.T(), err)
+	assert.Equal(s.T(), 200, response.StatusCode)
+}
+
+func (s *SwiftTestSuite) TestRoundTripWithETagEnabled() {
+	config.ETagEnabled = true
+	request, _ := http.NewRequest("GET", "swift://test/foo/test.png", nil)
+
+	response, err := s.transport.RoundTrip(request)
+	assert.Nil(s.T(), err)
+	assert.Equal(s.T(), 200, response.StatusCode)
+	assert.Equal(s.T(), "e27ca34142be8e55220e44155c626cd0", response.Header.Get("ETag"))
+}
+
+func (s *SwiftTestSuite) TestRoundTripWithIfNoneMatchReturns304() {
+	config.ETagEnabled = true
+
+	request, _ := http.NewRequest("GET", "swift://test/foo/test.png", nil)
+	request.Header.Set("If-None-Match", "e27ca34142be8e55220e44155c626cd0")
+
+	response, err := s.transport.RoundTrip(request)
+	assert.Nil(s.T(), err)
+	assert.Equal(s.T(), http.StatusNotModified, response.StatusCode)
+}
+
+func (s *SwiftTestSuite) TestRoundTripWithUpdatedETagReturns200() {
+	config.ETagEnabled = true
+
+	request, _ := http.NewRequest("GET", "swift://test/foo/test.png", nil)
+	request.Header.Set("If-None-Match", "foobar")
+
+	response, err := s.transport.RoundTrip(request)
+	assert.Nil(s.T(), err)
+	assert.Equal(s.T(), http.StatusOK, response.StatusCode)
+}
+
+func TestSwiftTransport(t *testing.T) {
+	suite.Run(t, new(SwiftTestSuite))
+}