Configurable bufPool calibration threshold

bufpool.go

@@ -30,7 +30,7 @@ func newBufPool(name string, n int, defaultSize int) *bufPool {
 		name:        name,
 		defaultSize: defaultSize,
 		buffers:     make([]*bytes.Buffer, n),
-		calls:       make(intSlice, 1024),
+		calls:       make(intSlice, conf.BufferPoolCalibrationThreshold),
 	}
 
 	for i := range pool.buffers {

config.go

@@ -195,34 +195,36 @@ type config struct {
 	SentryEnvironment string
 	SentryRelease     string
 
-	FreeMemoryInterval int
-	DownloadBufferSize int
-	GZipBufferSize     int
+	FreeMemoryInterval             int
+	DownloadBufferSize             int
+	GZipBufferSize                 int
+	BufferPoolCalibrationThreshold int
 }
 
 var conf = config{
-	Bind:                  ":8080",
-	ReadTimeout:           10,
-	WriteTimeout:          10,
-	DownloadTimeout:       5,
-	Concurrency:           runtime.NumCPU() * 2,
-	TTL:                   3600,
-	IgnoreSslVerification: false,
-	MaxSrcResolution:      16800000,
-	MaxGifFrames:          1,
-	AllowInsecure:         false,
-	SignatureSize:         32,
-	Quality:               80,
-	GZipCompression:       5,
-	UserAgent:             fmt.Sprintf("imgproxy/%s", version),
-	ETagEnabled:           false,
-	S3Enabled:             false,
-	WatermarkOpacity:      1,
-	BugsnagStage:          "production",
-	HoneybadgerEnv:        "production",
-	SentryEnvironment:     "production",
-	SentryRelease:         fmt.Sprintf("imgproxy/%s", version),
-	FreeMemoryInterval:    10,
+	Bind:                           ":8080",
+	ReadTimeout:                    10,
+	WriteTimeout:                   10,
+	DownloadTimeout:                5,
+	Concurrency:                    runtime.NumCPU() * 2,
+	TTL:                            3600,
+	IgnoreSslVerification:          false,
+	MaxSrcResolution:               16800000,
+	MaxGifFrames:                   1,
+	AllowInsecure:                  false,
+	SignatureSize:                  32,
+	Quality:                        80,
+	GZipCompression:                5,
+	UserAgent:                      fmt.Sprintf("imgproxy/%s", version),
+	ETagEnabled:                    false,
+	S3Enabled:                      false,
+	WatermarkOpacity:               1,
+	BugsnagStage:                   "production",
+	HoneybadgerEnv:                 "production",
+	SentryEnvironment:              "production",
+	SentryRelease:                  fmt.Sprintf("imgproxy/%s", version),
+	FreeMemoryInterval:             10,
+	BufferPoolCalibrationThreshold: 1024,
 }
 
 func init() {
@@ -318,6 +320,7 @@ func init() {
 	intEnvConfig(&conf.FreeMemoryInterval, "IMGPROXY_FREE_MEMORY_INTERVAL")
 	intEnvConfig(&conf.DownloadBufferSize, "IMGPROXY_DOWNLOAD_BUFFER_SIZE")
 	intEnvConfig(&conf.GZipBufferSize, "IMGPROXY_GZIP_BUFFER_SIZE")
+	intEnvConfig(&conf.BufferPoolCalibrationThreshold, "IMGPROXY_BUFFER_POOL_CALIBRATION_THRESHOLD")
 
 	if len(conf.Keys) != len(conf.Salts) {
 		logFatal("Number of keys and number of salts should be equal. Keys: %d, salts: %d", len(conf.Keys), len(conf.Salts))
@@ -388,7 +391,7 @@ func init() {
 	}
 
 	if conf.GZipCompression < 0 {
-		logFatal("GZip compression should be greater than or quual to 0, now - %d\n", conf.GZipCompression)
+		logFatal("GZip compression should be greater than or equal to 0, now - %d\n", conf.GZipCompression)
 	} else if conf.GZipCompression > 9 {
 		logFatal("GZip compression can't be greater than 9, now - %d\n", conf.GZipCompression)
 	}
@@ -430,17 +433,21 @@ func init() {
 	}
 
 	if conf.DownloadBufferSize < 0 {
-		logFatal("Download buffer size should be greater than or quual to 0")
+		logFatal("Download buffer size should be greater than or equal to 0")
 	} else if conf.DownloadBufferSize > int(^uint32(0)) {
 		logFatal("Download buffer size can't be creater than %d", ^uint32(0))
 	}
 
 	if conf.GZipBufferSize < 0 {
-		logFatal("GZip buffer size should be greater than or quual to 0")
+		logFatal("GZip buffer size should be greater than or equal to 0")
 	} else if conf.GZipBufferSize > int(^uint32(0)) {
 		logFatal("GZip buffer size can't be creater than %d", ^uint32(0))
 	}
 
+	if conf.BufferPoolCalibrationThreshold < 64 {
+		logFatal("Buffer pool calibration threshhold should be greater than or equal to 64")
+	}
+
 	initNewrelic()
 	initPrometheus()
 	initDownloading()

docs/configuration.md

@@ -193,7 +193,8 @@ imgproxy can send logs to syslog, but this feature is disabled by default. To en
 
 * `IMGPROXY_DOWNLOAD_BUFFER_SIZE`: the initial size (in bytes) of a single download buffer. When zero, initializes empty download buffers. Default: `0`;
 * `IMGPROXY_GZIP_BUFFER_SIZE`: the initial size (in bytes) of a single GZip buffer. When zero, initializes empty GZip buffers. Makes sense only when GZip compression is enabled. Default: `0`;
-* `IMGPROXY_FREE_MEMORY_INTERVAL`: the interval (in seconds) at which unused memory will be returned to the OS. Default: `10`.
+* `IMGPROXY_FREE_MEMORY_INTERVAL`: the interval (in seconds) at which unused memory will be returned to the OS. Default: `10`;
+* `IMGPROXY_BUFFER_POOL_CALIBRATION_THRESHOLD`: the number of buffers that should be returned to a pool before calibration. Default: `1024`.
 
 ### Miscellaneous
 

docs/memory_usage_tweaks.md

@@ -2,11 +2,11 @@
 
 There are some imgproxy options that can help you to optimize memory usage and decrease memory fragmentation.
 
-**Warning:** This is an advanced part. Please make sure that you know what're you doing before changing anything.
+**Warning:** This is an advanced part. Please make sure that you know what you're doing before changing anything.
 
 ### `IMGPROXY_DOWNLOAD_BUFFER_SIZE`
 
-imgproxy uses memory buffers to download source images. While these buffers are empty at the start by default, they can grow to a required size when imgproxy downloads an image. Allocating new memory to grow the buffers can cause memory fragmentation. Allocating required memory at the start can eliminate much of memory fragmentation since buffers won't grow. Setting `IMGPROXY_DOWNLOAD_BUFFER_SIZE` will tell imgproxy to initialize download buffers with _at least_ the specified size. It's recommended to use the estimates size of your biggest image as the initial download buffers size.
+imgproxy uses memory buffers to download source images. While these buffers are empty at the start by default, they can grow to a required size when imgproxy downloads an image. Allocating new memory to grow the buffers can cause memory fragmentation. Allocating required memory at the start can eliminate much of memory fragmentation since buffers won't grow. Setting `IMGPROXY_DOWNLOAD_BUFFER_SIZE` will tell imgproxy to initialize download buffers with _at least_ the specified size. It's recommended to use the estimated 95 percentile of your image sizes as the initial download buffers size.
 
 ### `IMGPROXY_GZIP_BUFFER_SIZE`
 
@@ -15,3 +15,7 @@ The same as `IMGPROXY_DOWNLOAD_BUFFER_SIZE` but for GZip buffers. If you use GZi
 ### `IMGPROXY_FREE_MEMORY_INTERVAL`
 
 Working with a large amount of data can cause allocating some memory that is not used most of the time. That's why imgproxy enforces Go's garbage collector to free as much memory as possible and return it to the OS. The default interval of this action is 10 seconds, but you can change it by setting `IMGPROXY_FREE_MEMORY_INTERVAL`. Decreasing the interval can smooth the memory usage graph but it can also slow down imgproxy a little. Increasing has the opposite effect.
+
+### `IMGPROXY_BUFFER_POOL_CALIBRATION_THRESHOLD`
+
+Buffer pools in imgproxy do self-calibration time by time. imgproxy collects stats about the sizes of the buffers returned to a pool and calculates the default buffer size and the maximum size of a buffer that can be returned to the pool. This allows dropping buffers that are too big for most of the images and save some memory. By default, imgproxy starts calibration after 1024 buffers were returned to a pool. You can change this number with `IMGPROXY_BUFFER_POOL_CALIBRATION_THRESHOLD` variable. Increasing the number will give you rarer but more accurate calibration.