Add support of ETag caching (#47)

* Add ETag caching support

* Update the readme file

* Typo

* Calculate ETag before image processing, rename Env Var

* Make PO struct field public

* Generate random value on server startup to be used in ETag calculation

* Minor refactoring

* Adjust boolean value parsing as discussed in GitHub PR page

* Move random value generation to the config

* Update README

* Revert changes

* Use footprint to calculate the hash of an image

README.md

@@ -144,6 +144,7 @@ $ xxd -g 2 -l 64 -p /dev/random | tr -d '\n'
 * `IMGPROXY_CONCURRENCY` — the maximum number of image requests to be processed simultaneously. Default: double number of CPU cores;
 * `IMGPROXY_MAX_CLIENTS` — the maximum number of simultaneous active connections. Default: `IMGPROXY_CONCURRENCY * 5`;
 * `IMGPROXY_TTL` — duration in seconds sent in `Expires` and `Cache-Control: max-age` headers. Default: `3600` (1 hour);
+* `IMGPROXY_USE_ETAG` — when true, enables using [ETag](https://en.wikipedia.org/wiki/HTTP_ETag) header for the cache control. Default: false; 
 * `IMGPROXY_LOCAL_FILESYSTEM_ROOT` — root of the local filesystem. See [Serving local files](#serving-local-files). Keep empty to disable serving of local files.
 
 #### Security

config.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bytes"
+	"crypto/rand"
 	"encoding/hex"
 	"flag"
 	"fmt"
@@ -30,6 +31,13 @@ func strEnvConfig(s *string, name string) {
 	}
 }
 
+func boolEnvConfig(b *bool, name string) {
+	*b = false
+	if env, err := strconv.ParseBool(os.Getenv(name)); err == nil {
+		*b = env
+	}
+}
+
 func hexEnvConfig(b *[]byte, name string) {
 	var err error
 
@@ -87,6 +95,8 @@ type config struct {
 	Secret string
 
 	LocalFileSystemRoot string
+	ETagEnabled         bool
+	RandomValue         []byte
 }
 
 var conf = config{
@@ -100,6 +110,7 @@ var conf = config{
 	MaxSrcResolution: 16800000,
 	Quality:          80,
 	GZipCompression:  5,
+	ETagEnabled:      false,
 }
 
 func init() {
@@ -135,6 +146,7 @@ func init() {
 	strEnvConfig(&conf.Secret, "IMGPROXY_SECRET")
 
 	strEnvConfig(&conf.LocalFileSystemRoot, "IMGPROXY_LOCAL_FILESYSTEM_ROOT")
+	boolEnvConfig(&conf.ETagEnabled, "IMGPROXY_USE_ETAG")
 
 	if len(conf.Key) == 0 {
 		log.Fatalln("Key is not defined")
@@ -205,6 +217,13 @@ func init() {
 		}
 	}
 
+	if conf.ETagEnabled {
+		conf.RandomValue = make([]byte, 16)
+		rand.Read(conf.RandomValue)
+		log.Printf("ETag support is activated. The random value was generated to be used for ETag calculation: %s\n",
+			fmt.Sprintf("%x", conf.RandomValue))
+	}
+
 	initVips()
 	initDownloading()
 }

etag.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"crypto/sha1"
+	"encoding/binary"
+	"fmt"
+	"net/http"
+)
+
+// checks whether client's ETag matches current response body.
+// - if the IMGPROXY_USE_ETAG env var is unset, this function always returns false
+// - if the IMGPROXY_USE_ETAG is set to "true", the function calculates current ETag and compares it
+//   with another ETag value provided by a client request
+// Note that the calculated ETag value is saved to outcoming response with "ETag" header.
+func isETagMatching(b []byte, po *processingOptions, rw *http.ResponseWriter, r *http.Request) bool {
+
+	if !conf.ETagEnabled {
+		return false
+	}
+
+	// calculate current ETag value using sha1 hashing function
+	currentEtagValue := calculateHashSumFor(b, po)
+	(*rw).Header().Set("ETag", currentEtagValue)
+	return currentEtagValue == r.Header.Get("If-None-Match")
+}
+
+// the function calculates the SHA checksum for the current image and current Processing Options.
+// The principal is very simple: if an original image is the same and POs are the same, then
+// the checksum must be always identical. But if PO has some different parameters, the
+// checksum must be different even if original images match
+func calculateHashSumFor(b []byte, po *processingOptions) string {
+
+	footprint := sha1.Sum(b)
+
+	hash := sha1.New()
+	hash.Write(footprint[:])
+	binary.Write(hash, binary.LittleEndian, *po)
+	hash.Write(conf.RandomValue)
+
+	return fmt.Sprintf("%x", hash.Sum(nil))
+}

process.go

@@ -69,12 +69,12 @@ var resizeTypes = map[string]resizeType{
 }
 
 type processingOptions struct {
-	resize  resizeType
-	width   int
-	height  int
-	gravity gravityType
-	enlarge bool
-	format  imageType
+	Resize  resizeType
+	Width   int
+	Height  int
+	Gravity gravityType
+	Enlarge bool
+	Format  imageType
 }
 
 var vipsSupportSmartcrop bool
@@ -132,7 +132,7 @@ func shutdownVips() {
 }
 
 func randomAccessRequired(po processingOptions) int {
-	if po.gravity == SMART {
+	if po.Gravity == SMART {
 		return 1
 	}
 	return 0
@@ -170,16 +170,16 @@ func extractMeta(img *C.VipsImage) (int, int, int, bool) {
 }
 
 func calcScale(width, height int, po processingOptions) float64 {
-	if (po.width == width && po.height == height) || (po.resize != FILL && po.resize != FIT) {
+	if (po.Width == width && po.Height == height) || (po.Resize != FILL && po.Resize != FIT) {
 		return 1
 	}
 
-	fsw, fsh, fow, foh := float64(width), float64(height), float64(po.width), float64(po.height)
+	fsw, fsh, fow, foh := float64(width), float64(height), float64(po.Width), float64(po.Height)
 
 	wr := fow / fsw
 	hr := foh / fsh
 
-	if po.resize == FIT {
+	if po.Resize == FIT {
 		return math.Min(wr, hr)
 	}
 
@@ -206,22 +206,22 @@ func calcShink(scale float64, imgtype imageType) int {
 }
 
 func calcCrop(width, height int, po processingOptions) (left, top int) {
-	left = (width - po.width + 1) / 2
-	top = (height - po.height + 1) / 2
+	left = (width - po.Width + 1) / 2
+	top = (height - po.Height + 1) / 2
 
-	if po.gravity == NORTH {
+	if po.Gravity == NORTH {
 		top = 0
 	}
 
-	if po.gravity == EAST {
-		left = width - po.width
+	if po.Gravity == EAST {
+		left = width - po.Width
 	}
 
-	if po.gravity == SOUTH {
-		top = height - po.height
+	if po.Gravity == SOUTH {
+		top = height - po.Height
 	}
 
-	if po.gravity == WEST {
+	if po.Gravity == WEST {
 		left = 0
 	}
 
@@ -232,7 +232,7 @@ func processImage(data []byte, imgtype imageType, po processingOptions, t *timer
 	defer C.vips_cleanup()
 	defer keepAlive(data)
 
-	if po.gravity == SMART && !vipsSupportSmartcrop {
+	if po.Gravity == SMART && !vipsSupportSmartcrop {
 		return nil, errors.New("Smart crop is not supported by used version of libvips")
 	}
 
@@ -247,18 +247,18 @@ func processImage(data []byte, imgtype imageType, po processingOptions, t *timer
 	imgWidth, imgHeight, angle, flip := extractMeta(img)
 
 	// Ensure we won't crop out of bounds
-	if !po.enlarge || po.resize == CROP {
-		if imgWidth < po.width {
-			po.width = imgWidth
+	if !po.Enlarge || po.Resize == CROP {
+		if imgWidth < po.Width {
+			po.Width = imgWidth
 		}
 
-		if imgHeight < po.height {
-			po.height = imgHeight
+		if imgHeight < po.Height {
+			po.Height = imgHeight
 		}
 	}
 
-	if po.width != imgWidth || po.height != imgHeight {
-		if po.resize == FILL || po.resize == FIT {
+	if po.Width != imgWidth || po.Height != imgHeight {
+		if po.Resize == FILL || po.Resize == FIT {
 			scale := calcScale(imgWidth, imgHeight, po)
 
 			// Do some shrink-on-load
@@ -326,17 +326,17 @@ func processImage(data []byte, imgtype imageType, po processingOptions, t *timer
 
 		t.Check()
 
-		if po.resize == FILL || po.resize == CROP {
-			if po.gravity == SMART {
+		if po.Resize == FILL || po.Resize == CROP {
+			if po.Gravity == SMART {
 				if err = vipsImageCopyMemory(&img); err != nil {
 					return nil, err
 				}
-				if err = vipsSmartCrop(&img, po.width, po.height); err != nil {
+				if err = vipsSmartCrop(&img, po.Width, po.Height); err != nil {
 					return nil, err
 				}
 			} else {
 				left, top := calcCrop(int(img.Xsize), int(img.Ysize), po)
-				if err = vipsCrop(&img, left, top, po.width, po.height); err != nil {
+				if err = vipsCrop(&img, left, top, po.Width, po.Height); err != nil {
 					return nil, err
 				}
 			}
@@ -345,7 +345,7 @@ func processImage(data []byte, imgtype imageType, po processingOptions, t *timer
 
 	t.Check()
 
-	return vipsSaveImage(img, po.format)
+	return vipsSaveImage(img, po.Format)
 }
 
 func vipsLoadImage(data []byte, imgtype imageType, shrink int) (*C.struct__VipsImage, error) {

server.go

@@ -46,38 +46,38 @@ func parsePath(r *http.Request) (string, processingOptions, error) {
 	}
 
 	if r, ok := resizeTypes[parts[1]]; ok {
-		po.resize = r
+		po.Resize = r
 	} else {
 		return "", po, fmt.Errorf("Invalid resize type: %s", parts[1])
 	}
 
-	if po.width, err = strconv.Atoi(parts[2]); err != nil {
+	if po.Width, err = strconv.Atoi(parts[2]); err != nil {
 		return "", po, fmt.Errorf("Invalid width: %s", parts[2])
 	}
 
-	if po.height, err = strconv.Atoi(parts[3]); err != nil {
+	if po.Height, err = strconv.Atoi(parts[3]); err != nil {
 		return "", po, fmt.Errorf("Invalid height: %s", parts[3])
 	}
 
 	if g, ok := gravityTypes[parts[4]]; ok {
-		po.gravity = g
+		po.Gravity = g
 	} else {
 		return "", po, fmt.Errorf("Invalid gravity: %s", parts[4])
 	}
 
-	po.enlarge = parts[5] != "0"
+	po.Enlarge = parts[5] != "0"
 
 	filenameParts := strings.Split(strings.Join(parts[6:], ""), ".")
 
 	if len(filenameParts) < 2 {
-		po.format = imageTypes["jpg"]
+		po.Format = imageTypes["jpg"]
 	} else if f, ok := imageTypes[filenameParts[1]]; ok {
-		po.format = f
+		po.Format = f
 	} else {
 		return "", po, fmt.Errorf("Invalid image format: %s", filenameParts[1])
 	}
 
-	if !vipsTypeSupportSave[po.format] {
+	if !vipsTypeSupportSave[po.Format] {
 		return "", po, errors.New("Resulting image type not supported")
 	}
 
@@ -108,7 +108,9 @@ func respondWithImage(r *http.Request, rw http.ResponseWriter, data []byte, imgU
 
 	rw.Header().Set("Expires", time.Now().Add(time.Second*time.Duration(conf.TTL)).Format(http.TimeFormat))
 	rw.Header().Set("Cache-Control", fmt.Sprintf("max-age=%d, public", conf.TTL))
-	rw.Header().Set("Content-Type", mimes[po.format])
+	rw.Header().Set("Content-Type", mimes[po.Format])
+	rw.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
+
 	if gzipped {
 		rw.Header().Set("Content-Encoding", "gzip")
 	}
@@ -176,8 +178,8 @@ func (h *httpHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 	}
 
 	if r.URL.Path == "/health" {
-		rw.WriteHeader(200);
-		rw.Write([]byte("imgproxy is running"));
+		rw.WriteHeader(200)
+		rw.Write([]byte("imgproxy is running"))
 		return
 	}
 
@@ -197,6 +199,13 @@ func (h *httpHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 
 	t.Check()
 
+	if isETagMatching(b, &procOpt, &rw, r) {
+		// if client has its own locally cached copy of this file, then return 304, no need to send it again over the network
+		rw.WriteHeader(304)
+		logResponse(304, fmt.Sprintf("Returned 'Not Modified' instead of actual image in %s: %s; %+v", t.Since(), imgURL, procOpt))
+		return
+	}
+
 	b, err = processImage(b, imgtype, procOpt, t)
 	if err != nil {
 		panic(newError(500, err.Error(), "Error occurred while processing image"))