2021-04-26 13:52:50 +02:00
|
|
|
package imagedata
|
|
|
|
|
|
|
|
import (
|
|
|
|
"compress/gzip"
|
|
|
|
"crypto/tls"
|
|
|
|
"fmt"
|
|
|
|
"io/ioutil"
|
|
|
|
"net"
|
|
|
|
"net/http"
|
|
|
|
"time"
|
|
|
|
|
2021-09-30 16:23:30 +02:00
|
|
|
"github.com/imgproxy/imgproxy/v3/config"
|
|
|
|
"github.com/imgproxy/imgproxy/v3/ierrors"
|
2021-04-26 13:52:50 +02:00
|
|
|
|
2021-09-30 16:23:30 +02:00
|
|
|
azureTransport "github.com/imgproxy/imgproxy/v3/transport/azure"
|
|
|
|
fsTransport "github.com/imgproxy/imgproxy/v3/transport/fs"
|
|
|
|
gcsTransport "github.com/imgproxy/imgproxy/v3/transport/gcs"
|
|
|
|
s3Transport "github.com/imgproxy/imgproxy/v3/transport/s3"
|
2021-04-26 13:52:50 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
var (
|
|
|
|
downloadClient *http.Client
|
|
|
|
|
2021-11-01 14:13:33 +02:00
|
|
|
enabledSchemes = map[string]struct{}{
|
|
|
|
"http": {},
|
|
|
|
"https": {},
|
|
|
|
}
|
|
|
|
|
2021-04-26 13:52:50 +02:00
|
|
|
imageHeadersToStore = []string{
|
|
|
|
"Cache-Control",
|
|
|
|
"Expires",
|
2021-09-29 12:23:54 +02:00
|
|
|
"ETag",
|
2021-04-26 13:52:50 +02:00
|
|
|
}
|
2021-09-07 15:04:33 +02:00
|
|
|
|
|
|
|
// For tests
|
|
|
|
redirectAllRequestsTo string
|
2021-04-26 13:52:50 +02:00
|
|
|
)
|
|
|
|
|
|
|
|
const msgSourceImageIsUnreachable = "Source image is unreachable"
|
|
|
|
|
2021-10-13 13:59:46 +02:00
|
|
|
type ErrorNotModified struct {
|
|
|
|
Message string
|
|
|
|
Headers map[string]string
|
|
|
|
}
|
|
|
|
|
|
|
|
func (e *ErrorNotModified) Error() string {
|
|
|
|
return e.Message
|
|
|
|
}
|
|
|
|
|
2021-04-26 13:52:50 +02:00
|
|
|
func initDownloading() error {
|
|
|
|
transport := &http.Transport{
|
|
|
|
Proxy: http.ProxyFromEnvironment,
|
|
|
|
MaxIdleConns: config.Concurrency,
|
|
|
|
MaxIdleConnsPerHost: config.Concurrency,
|
|
|
|
DisableCompression: true,
|
|
|
|
DialContext: (&net.Dialer{KeepAlive: 600 * time.Second}).DialContext,
|
|
|
|
}
|
|
|
|
|
|
|
|
if config.IgnoreSslVerification {
|
|
|
|
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
|
|
|
|
}
|
|
|
|
|
2021-11-01 14:13:33 +02:00
|
|
|
registerProtocol := func(scheme string, rt http.RoundTripper) {
|
|
|
|
transport.RegisterProtocol(scheme, rt)
|
|
|
|
enabledSchemes[scheme] = struct{}{}
|
|
|
|
}
|
|
|
|
|
2021-04-26 13:52:50 +02:00
|
|
|
if config.LocalFileSystemRoot != "" {
|
2021-11-01 14:13:33 +02:00
|
|
|
registerProtocol("local", fsTransport.New())
|
2021-04-26 13:52:50 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if config.S3Enabled {
|
|
|
|
if t, err := s3Transport.New(); err != nil {
|
|
|
|
return err
|
|
|
|
} else {
|
2021-11-01 14:13:33 +02:00
|
|
|
registerProtocol("s3", t)
|
2021-04-26 13:52:50 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if config.GCSEnabled {
|
|
|
|
if t, err := gcsTransport.New(); err != nil {
|
|
|
|
return err
|
|
|
|
} else {
|
2021-11-01 14:13:33 +02:00
|
|
|
registerProtocol("gs", t)
|
2021-04-26 13:52:50 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if config.ABSEnabled {
|
|
|
|
if t, err := azureTransport.New(); err != nil {
|
|
|
|
return err
|
|
|
|
} else {
|
2021-11-01 14:13:33 +02:00
|
|
|
registerProtocol("abs", t)
|
2021-04-26 13:52:50 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
downloadClient = &http.Client{
|
|
|
|
Timeout: time.Duration(config.DownloadTimeout) * time.Second,
|
|
|
|
Transport: transport,
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2021-10-13 13:59:46 +02:00
|
|
|
func headersToStore(res *http.Response) map[string]string {
|
|
|
|
m := make(map[string]string)
|
|
|
|
|
|
|
|
for _, h := range imageHeadersToStore {
|
|
|
|
if val := res.Header.Get(h); len(val) != 0 {
|
|
|
|
m[h] = val
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return m
|
|
|
|
}
|
|
|
|
|
2021-09-29 12:23:54 +02:00
|
|
|
func requestImage(imageURL string, header http.Header) (*http.Response, error) {
|
2021-04-26 13:52:50 +02:00
|
|
|
req, err := http.NewRequest("GET", imageURL, nil)
|
|
|
|
if err != nil {
|
|
|
|
return nil, ierrors.New(404, err.Error(), msgSourceImageIsUnreachable).SetUnexpected(config.ReportDownloadingErrors)
|
|
|
|
}
|
|
|
|
|
2021-11-01 14:13:33 +02:00
|
|
|
if _, ok := enabledSchemes[req.URL.Scheme]; !ok {
|
|
|
|
return nil, ierrors.New(
|
|
|
|
404,
|
|
|
|
fmt.Sprintf("Unknown sheme: %s", req.URL.Scheme),
|
|
|
|
msgSourceImageIsUnreachable,
|
|
|
|
).SetUnexpected(config.ReportDownloadingErrors)
|
|
|
|
}
|
|
|
|
|
2021-04-26 13:52:50 +02:00
|
|
|
req.Header.Set("User-Agent", config.UserAgent)
|
|
|
|
|
2021-09-29 12:23:54 +02:00
|
|
|
for k, v := range header {
|
|
|
|
if len(v) > 0 {
|
|
|
|
req.Header.Set(k, v[0])
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-04-26 13:52:50 +02:00
|
|
|
res, err := downloadClient.Do(req)
|
|
|
|
if err != nil {
|
2021-11-01 14:13:33 +02:00
|
|
|
return nil, ierrors.New(500, checkTimeoutErr(err).Error(), msgSourceImageIsUnreachable).SetUnexpected(config.ReportDownloadingErrors)
|
2021-04-26 13:52:50 +02:00
|
|
|
}
|
|
|
|
|
2021-09-29 12:23:54 +02:00
|
|
|
if res.StatusCode == http.StatusNotModified {
|
2021-10-13 13:59:46 +02:00
|
|
|
return nil, &ErrorNotModified{Message: "Not Modified", Headers: headersToStore(res)}
|
2021-09-29 12:23:54 +02:00
|
|
|
}
|
|
|
|
|
2021-04-26 13:52:50 +02:00
|
|
|
if res.StatusCode != 200 {
|
|
|
|
body, _ := ioutil.ReadAll(res.Body)
|
|
|
|
res.Body.Close()
|
|
|
|
|
2021-11-01 14:13:33 +02:00
|
|
|
status := 404
|
|
|
|
if res.StatusCode >= 500 {
|
|
|
|
status = 500
|
|
|
|
}
|
|
|
|
|
2021-09-29 12:23:54 +02:00
|
|
|
msg := fmt.Sprintf("Status: %d; %s", res.StatusCode, string(body))
|
2021-11-01 14:13:33 +02:00
|
|
|
return nil, ierrors.New(status, msg, msgSourceImageIsUnreachable).SetUnexpected(config.ReportDownloadingErrors)
|
2021-04-26 13:52:50 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
return res, nil
|
|
|
|
}
|
|
|
|
|
2021-09-29 12:23:54 +02:00
|
|
|
func download(imageURL string, header http.Header) (*ImageData, error) {
|
2021-09-07 15:04:33 +02:00
|
|
|
// We use this for testing
|
|
|
|
if len(redirectAllRequestsTo) > 0 {
|
|
|
|
imageURL = redirectAllRequestsTo
|
|
|
|
}
|
|
|
|
|
2021-09-29 12:23:54 +02:00
|
|
|
res, err := requestImage(imageURL, header)
|
2021-04-26 13:52:50 +02:00
|
|
|
if res != nil {
|
|
|
|
defer res.Body.Close()
|
|
|
|
}
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
body := res.Body
|
|
|
|
contentLength := int(res.ContentLength)
|
|
|
|
|
|
|
|
if res.Header.Get("Content-Encoding") == "gzip" {
|
|
|
|
gzipBody, errGzip := gzip.NewReader(res.Body)
|
|
|
|
if gzipBody != nil {
|
|
|
|
defer gzipBody.Close()
|
|
|
|
}
|
|
|
|
if errGzip != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
body = gzipBody
|
|
|
|
contentLength = 0
|
|
|
|
}
|
|
|
|
|
|
|
|
imgdata, err := readAndCheckImage(body, contentLength)
|
|
|
|
if err != nil {
|
2021-11-01 14:13:33 +02:00
|
|
|
return nil, ierrors.Wrap(err, 0).SetUnexpected(config.ReportDownloadingErrors)
|
2021-04-26 13:52:50 +02:00
|
|
|
}
|
|
|
|
|
2021-10-13 13:59:46 +02:00
|
|
|
imgdata.Headers = headersToStore(res)
|
2021-04-26 13:52:50 +02:00
|
|
|
|
|
|
|
return imgdata, nil
|
|
|
|
}
|
2021-09-07 15:04:33 +02:00
|
|
|
|
|
|
|
func RedirectAllRequestsTo(u string) {
|
|
|
|
redirectAllRequestsTo = u
|
|
|
|
}
|
|
|
|
|
|
|
|
func StopRedirectingRequests() {
|
|
|
|
redirectAllRequestsTo = ""
|
|
|
|
}
|