go/imgproxy
1
0
Fork 0
mirror of https://github.com/imgproxy/imgproxy.git synced 2025-01-18 11:12:10 +02:00
Code Releases Activity

Load env variables from env files or secrets

Browse Source
This commit is contained in:
DarthSim 2023-02-28 19:40:56 +03:00 committed by Sergey Alexandrovich
parent 3b3720fbb5
commit 93b1dc5757
12 changed files with 437 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -3,6 +3,7 @@
## [Unreleased]
### Add
- Add [multi-region mode](https://docs.imgproxy.net/latest/serving_files_from_s3?id=multi-region-mode) to S3 integration.
- Add the ability to [load environment variables](https://docs.imgproxy.net/latest/loading_environment_variables) from a file or a cloud secret.
- Add `IMGPROXY_WORKERS` alias for the `IMGPROXY_CONCURRENCY` config.
### Change

121
config/loadenv/aws.go Normal file
View File

@ -0,0 +1,121 @@
package loadenv
import (
"fmt"
"os"
"strings"
"github.com/DarthSim/godotenv"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/secretsmanager"
"github.com/aws/aws-sdk-go/service/ssm"
)
func loadAWSSecret() error {
secretID := os.Getenv("IMGPROXY_ENV_AWS_SECRET_ID")
secretVersionID := os.Getenv("IMGPROXY_ENV_AWS_SECRET_VERSION_ID")
secretVersionStage := os.Getenv("IMGPROXY_ENV_AWS_SECRET_VERSION_STAGE")
secretRegion := os.Getenv("IMGPROXY_ENV_AWS_SECRET_REGION")
if len(secretID) == 0 {
return nil
}
sess, err := session.NewSession()
if err != nil {
return fmt.Errorf("Can't create AWS Secrets Manager session: %s", err)
}
conf := aws.NewConfig()
if len(secretRegion) != 0 {
conf.Region = aws.String(secretRegion)
}
svc := secretsmanager.New(sess, conf)
input := secretsmanager.GetSecretValueInput{SecretId: aws.String(secretID)}
if len(secretVersionID) > 0 {
input.VersionId = aws.String(secretVersionID)
} else if len(secretVersionStage) > 0 {
input.VersionStage = aws.String(secretVersionStage)
}
output, err := svc.GetSecretValue(&input)
if err != nil {
return fmt.Errorf("Can't retrieve config from AWS Secrets Manager: %s", err)
}
if output.SecretString == nil {
return nil
}
envmap, err := godotenv.Unmarshal(*output.SecretString)
if err != nil {
return fmt.Errorf("Can't parse config from AWS Secrets Manager: %s", err)
}
for k, v := range envmap {
if err = os.Setenv(k, v); err != nil {
return fmt.Errorf("Can't set %s env variable from AWS Secrets Manager: %s", k, err)
}
}
return nil
}
func loadAWSSystemManagerParams() error {
paramsPath := os.Getenv("IMGPROXY_ENV_AWS_SSM_PARAMETERS_PATH")
paramsRegion := os.Getenv("IMGPROXY_ENV_AWS_SSM_PARAMETERS_REGION")
if len(paramsPath) == 0 {
return nil
}
sess, err := session.NewSession()
if err != nil {
return fmt.Errorf("Can't create AWS SSM session: %s", err)
}
conf := aws.NewConfig()
if len(paramsRegion) != 0 {
conf.Region = aws.String(paramsRegion)
}
svc := ssm.New(sess, conf)
input := ssm.GetParametersByPathInput{
Path: aws.String(paramsPath),
WithDecryption: aws.Bool(true),
}
output, err := svc.GetParametersByPath(&input)
if err != nil {
return fmt.Errorf("Can't retrieve parameters from AWS SSM: %s", err)
}
for _, p := range output.Parameters {
if p == nil || p.Name == nil || p.Value == nil {
continue
}
if p.DataType == nil || *p.DataType != "text" {
continue
}
name := *p.Name
env := strings.ReplaceAll(
strings.TrimPrefix(strings.TrimPrefix(name, paramsPath), "/"),
"/", "_",
)
if err = os.Setenv(env, *p.Value); err != nil {
return fmt.Errorf("Can't set %s env variable from AWS SSM: %s", env, err)
}
}
return nil
}

82
config/loadenv/gcp.go Normal file
View File

@ -0,0 +1,82 @@
package loadenv
import (
"context"
"errors"
"fmt"
"os"
"time"
secretmanager "cloud.google.com/go/secretmanager/apiv1"
"cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
"github.com/DarthSim/godotenv"
"google.golang.org/api/option"
)
func loadGCPSecret() error {
secretID := os.Getenv("IMGPROXY_ENV_GCP_SECRET_ID")
secretVersion := os.Getenv("IMGPROXY_ENV_GCP_SECRET_VERSION_ID")
secretProject := os.Getenv("IMGPROXY_ENV_GCP_SECRET_PROJECT_ID")
secretKey := os.Getenv("IMGPROXY_ENV_GCP_KEY")
if len(secretID) == 0 {
return nil
}
if len(secretVersion) == 0 {
secretVersion = "latest"
}
var (
client *secretmanager.Client
err error
)
ctx, ctxcancel := context.WithTimeout(context.Background(), time.Minute)
defer ctxcancel()
opts := []option.ClientOption{}
if len(secretKey) > 0 {
opts = append(opts, option.WithCredentialsJSON([]byte(secretKey)))
}
client, err = secretmanager.NewClient(ctx, opts...)
if err != nil {
return fmt.Errorf("Can't create Google Cloud Secret Manager client: %s", err)
}
req := secretmanagerpb.AccessSecretVersionRequest{
Name: fmt.Sprintf("projects/%s/secrets/%s/versions/%s", secretProject, secretID, secretVersion),
}
resp, err := client.AccessSecretVersion(ctx, &req)
if err != nil {
return fmt.Errorf("Can't get Google Cloud Secret Manager secret: %s", err)
}
payload := resp.GetPayload()
if payload == nil {
return errors.New("Can't get Google Cloud Secret Manager secret: payload is empty")
}
data := payload.GetData()
if len(data) == 0 {
return nil
}
envmap, err := godotenv.Unmarshal(string(data))
if err != nil {
return fmt.Errorf("Can't parse config from Google Cloud Secrets Manager: %s", err)
}
for k, v := range envmap {
if err = os.Setenv(k, v); err != nil {
return fmt.Errorf("Can't set %s env variable from Google Cloud Secrets Manager: %s", k, err)
}
}
return nil
}

21
config/loadenv/loadenv.go Normal file
View File

@ -0,0 +1,21 @@
package loadenv
func Load() error {
if err := loadAWSSecret(); err != nil {
return err
}
if err := loadAWSSystemManagerParams(); err != nil {
return err
}
if err := loadGCPSecret(); err != nil {
return err
}
if err := loadLocalFile(); err != nil {
return err
}
return nil
}

38
config/loadenv/local_file.go Normal file
View File

@ -0,0 +1,38 @@
package loadenv
import (
"fmt"
"os"
"github.com/DarthSim/godotenv"
)
func loadLocalFile() error {
path := os.Getenv("IMGPROXY_ENV_LOCAL_FILE_PATH")
if len(path) == 0 {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("Can't read loacal environment file: %s", err)
}
if len(data) == 0 {
return nil
}
envmap, err := godotenv.Unmarshal(string(data))
if err != nil {
return fmt.Errorf("Can't parse config from local file: %s", err)
}
for k, v := range envmap {
if err = os.Setenv(k, v); err != nil {
return fmt.Errorf("Can't set %s env variable from local file: %s", k, err)
}
}
return nil
}

4
docs/_sidebar.md
View File

@ -1,7 +1,9 @@
* [Getting started](GETTING_STARTED.md)
* [Pro version<img src="/assets/pro.svg">](https://imgproxy.net/#pro)
* [Installation](installation.md)
* [Configuration](configuration.md)
* Configuration
* [Configuration](configuration.md)
* [Loading environment variables](loading_environment_variables.md)
* Generating the URL
* [Generating the URL](generating_the_url.md)
* [Getting the image info<img title="imgproxy Pro feature" src="/assets/pro.svg">](getting_the_image_info.md)

2
docs/cloud_watch.md
View File

@ -42,8 +42,6 @@ AWS_ACCESS_KEY_ID=my_access_key AWS_SECRET_ACCESS_KEY=my_secret_key imgproxy
docker run -e AWS_ACCESS_KEY_ID=my_access_key -e AWS_SECRET_ACCESS_KEY=my_secret_key -it darthsim/imgproxy
```
This is the recommended method when using dockerized imgproxy.
#### Shared credentials file
Alternatively, you can create the `.aws/credentials` file in your home directory with the following content:

160
docs/loading_environment_variables.md Normal file
View File

@ -0,0 +1,160 @@
# Loading environment variables
imgproxy can load environment variables from various sources such as:
* [Local file](#local-file)
* [AWS Secrets Manager](#aws-secrets-manager)
* [AWS Systems Manager Parameter Store](#aws-systems-manager-parameter-store)
* [Google Cloud Secret Manager](#google-cloud-secret-manager)
## Local file
You can create an [environment file](#environment-file-syntax) and configure imgproxy to read environment variables from it.
* `IMGPROXY_ENV_LOCAL_FILE_PATH`: the path of the environmebt file to load
## AWS Secrets Manager
You can store the content of an [environment file](#environment-file-syntax) as an AWS Secrets Manager secret and configure imgproxy to read environment variables from it.
* `IMGPROXY_ENV_AWS_SECRET_ID`: the ARN or name of the secret to load
* `IMGPROXY_ENV_AWS_SECRET_VERSION_ID`: _(optional)_ the unique identifier of the version of the secret to load
* `IMGPROXY_ENV_AWS_SECRET_VERSION_STAGE`: _(optional)_ the staging label of the version of the secret to load
* `IMGPROXY_ENV_AWS_SECRET_REGION`: _(optional)_ the region of the secret to load
**📝 Note:** If both `IMGPROXY_ENV_AWS_SECRET_VERSION_ID` and `IMGPROXY_ENV_AWS_SECRET_VERSION_STAGE` are set, `IMGPROXY_ENV_AWS_SECRET_VERSION_STAGE` will be ignored
### Set up AWS Secrets Manager credentials
There are three ways to specify your AWS credentials. The credentials policy should allow performing the `secretsmanager:GetSecretValue` and `secretsmanager:ListSecretVersionIds` actions with the specified secret:
#### IAM Roles
If you're running imgproxy on an Amazon Web Services platform, you can use IAM roles to to get the security credentials to retrieve the secret.
* **Elastic Container Service (ECS):** Assign an [IAM role to a task](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html).
* **Elastic Kubernetes Service (EKS):** Assign a [service account to a pod](https://docs.aws.amazon.com/eks/latest/userguide/pod-configuration.html).
* **Elastic Beanstalk:** Assign an [IAM role to an instance](https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/iam-instanceprofile.html).
#### Environment variables
You can specify an AWS Access Key ID and a Secret Access Key by setting the standard `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables.
``` bash
AWS_ACCESS_KEY_ID=my_access_key AWS_SECRET_ACCESS_KEY=my_secret_key imgproxy
# same for Docker
docker run -e AWS_ACCESS_KEY_ID=my_access_key -e AWS_SECRET_ACCESS_KEY=my_secret_key -it darthsim/imgproxy
```
#### Shared credentials file
Alternatively, you can create the `.aws/credentials` file in your home directory with the following content:
```ini
[default]
aws_access_key_id = %access_key_id
aws_secret_access_key = %secret_access_key
```
## AWS Systems Manager Parameter Store
You can store multiple AWS Systems Manager Parameter Store entries and configure imgproxy to load their values to separate environment variables.
* `IMGPROXY_ENV_AWS_SSM_PARAMETERS_PATH`: the [path](#aws-systems-manager-path) of the parameters to load
* `IMGPROXY_ENV_AWS_SSM_PARAMETERS_REGION`: _(optional)_ the region of the parameters to load
### AWS Systems Manager path
Let's assume that you created the following AWS Systems Manager parameters:
* `/imgproxy/prod/IMGPROXY_KEY`
* `/imgproxy/prod/IMGPROXY_SALT`
* `/imgproxy/prod/IMGPROXY_CLOUD_WATCH/SERVICE_NAME`
* `/imgproxy/prod/IMGPROXY_CLOUD_WATCH/NAMESPACE`
* `/imgproxy/staging/IMGPROXY_KEY`
If you set `IMGPROXY_ENV_AWS_SSM_PARAMETERS_PATH` to `/imgproxy/prod`, imgproxy will load these parameters the following way:
* `/imgproxy/prod/IMGPROXY_KEY` value will be loaded to `IMGPROXY_KEY`
* `/imgproxy/prod/IMGPROXY_SALT` value will be loaded to `IMGPROXY_SALT`
* `/imgproxy/prod/IMGPROXY_CLOUD_WATCH/SERVICE_NAME` value will be loaded to `IMGPROXY_CLOUD_WATCH_SERVICE_NAME`
* `/imgproxy/prod/IMGPROXY_CLOUD_WATCH/NAMESPACE` value will be loaded to `IMGPROXY_CLOUD_WATCH_NAMESPACE`
* `/imgproxy/staging/IMGPROXY_KEY` will be ignored since its path is not `/imgproxy/prod`
### Set up AWS Systems Manager credentials
There are three ways to specify your AWS credentials. The credentials policy should allow performing the `ssm:GetParametersByPath` action with the specified parameters:
#### IAM Roles
If you're running imgproxy on an Amazon Web Services platform, you can use IAM roles to to get the security credentials to retrieve the secret.
* **Elastic Container Service (ECS):** Assign an [IAM role to a task](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html).
* **Elastic Kubernetes Service (EKS):** Assign a [service account to a pod](https://docs.aws.amazon.com/eks/latest/userguide/pod-configuration.html).
* **Elastic Beanstalk:** Assign an [IAM role to an instance](https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/iam-instanceprofile.html).
#### Environment variables
You can specify an AWS Access Key ID and a Secret Access Key by setting the standard `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables.
``` bash
AWS_ACCESS_KEY_ID=my_access_key AWS_SECRET_ACCESS_KEY=my_secret_key imgproxy
# same for Docker
docker run -e AWS_ACCESS_KEY_ID=my_access_key -e AWS_SECRET_ACCESS_KEY=my_secret_key -it darthsim/imgproxy
```
#### Shared credentials file
Alternatively, you can create the `.aws/credentials` file in your home directory with the following content:
```ini
[default]
aws_access_key_id = %access_key_id
aws_secret_access_key = %secret_access_key
```
## Google Cloud Secret Manager
You can store the content of an [environment file](#environment-file-syntax) in Google Cloud Secret Manager secret and configure imgproxy to read environment variables from it.
* `IMGPROXY_ENV_GCP_SECRET_ID`: the name of the secret to load
* `IMGPROXY_ENV_GCP_SECRET_VERSION_ID`: _(optional)_ the unique identifier of the version of the secret to load
* `IMGPROXY_ENV_GCP_SECRET_PROJECT_ID`: the name or ID of the Google Cloud project that contains the secret
### Setup credentials
If you run imgproxy inside Google Cloud infrastructure (Compute Engine, Kubernetes Engine, App Engine, Cloud Functions, etc), and you have granted access to the specified secret to the service account, you probably don't need to do anything here. imgproxy will try to use the credentials provided by Google.
Otherwise, set `IMGPROXY_ENV_GCP_KEY` environment variable to the content of Google Cloud JSON key. Get more info about JSON keys: [https://cloud.google.com/iam/docs/creating-managing-service-account-keys](https://cloud.google.com/iam/docs/creating-managing-service-account-keys).
## Environment file syntax
The following syntax rules apply to environment files:
* Blank lines are ignored
* Lines beginning with `#` are processed as comments and ignored
* Each line represents a key-value pair. Values can optionally be quoted:
* `VAR=VAL` -> `VAL`
* `VAR="VAL"` -> `VAL`
* `VAR='VAL'` -> `VAL`
* Unquoted and double-quoted (`"`) values have variable substitution applied:
* `VAR=${OTHER_VAR}` -> value of `OTHER_VAR`
* `VAR=$OTHER_VAR` -> value of `OTHER_VAR`
* `VAR="$OTHER_VAR"` -> value of `OTHER_VAR`
* `VAR="${OTHER_VAR}"` -> value of `OTHER_VAR`
* Single-quoted (`'`) values are used literally:
* `VAR='$OTHER_VAR'` -> `$OTHER_VAR`
* `VAR='${OTHER_VAR}'` -> `${OTHER_VAR}`
* Double quotes in double-quoted (`"`) values can be escaped with `\`:
* `VAR="{\"hello\": \"json\"}"` -> `{"hello": "json"}`
* Slash (`\`) in double-quoted values can be escaped with another slash:
* `VAR="some\\value"` -> `some\value`
* A new line can be added to double-quoted values using `\n`:
* `VAR="some\nvalue"` ->
```
some
value
```

2
docs/serving_files_from_s3.md
View File

@ -39,8 +39,6 @@ AWS_ACCESS_KEY_ID=my_access_key AWS_SECRET_ACCESS_KEY=my_secret_key imgproxy
docker run -e AWS_ACCESS_KEY_ID=my_access_key -e AWS_SECRET_ACCESS_KEY=my_secret_key -it darthsim/imgproxy
```
This is the recommended method when using dockerized imgproxy.
#### Shared credentials file
Alternatively, you can create the `.aws/credentials` file in your home directory with the following content:

2
go.mod
View File

@ -7,6 +7,7 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
github.com/DarthSim/godotenv v1.3.1
github.com/DataDog/datadog-go/v5 v5.3.0
github.com/airbrake/gobrake/v5 v5.6.1
github.com/aws/aws-sdk-go v1.44.260
@ -58,6 +59,7 @@ require (
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v1.0.1 // indirect
cloud.google.com/go/pubsub v1.30.0 // indirect
cloud.google.com/go/secretmanager v1.10.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
github.com/DataDog/appsec-internal-go v1.0.0 // indirect

4
go.sum
View File

@ -37,6 +37,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA
cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
cloud.google.com/go/pubsub v1.30.0 h1:vCge8m7aUKBJYOgrZp7EsNDf6QMd2CAlXZqWTn3yq6s=
cloud.google.com/go/pubsub v1.30.0/go.mod h1:qWi1OPS0B+b5L+Sg6Gmc9zD1Y+HaM0MdUr7LsupY1P4=
cloud.google.com/go/secretmanager v1.10.0 h1:pu03bha7ukxF8otyPKTFdDz+rr9sE3YauS5PliDXK60=
cloud.google.com/go/secretmanager v1.10.0/go.mod h1:MfnrdvKMPNra9aZtQFvBcvRU54hbPD8/HayQdlUgJpU=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
@ -59,6 +61,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DarthSim/gofakes3 v0.0.0-20230502153341-3fc66d2bc272 h1:Gj21neabaU3DEwwVPG6/Vn0GSuDQ1n2m2z1qV33SCI0=
github.com/DarthSim/gofakes3 v0.0.0-20230502153341-3fc66d2bc272/go.mod h1:Cnosl0cRZIfKjTMuH49sQog2LeNsU5Hf4WnPIDWIDV0=
github.com/DarthSim/godotenv v1.3.1 h1:NMWdswlRx2M9uPY4Ux8p/Q/rDs7A97OG89fECiQ/Tz0=
github.com/DarthSim/godotenv v1.3.1/go.mod h1:B3ySe1HYTUFFR6+TPyHyxPWjUdh48il0Blebg9p1cCc=
github.com/DarthSim/opentelemetry-go-contrib/detectors/aws/ecs v0.0.0-20230510163401-1a377505ea6c h1:6XK2HjE3YbWRAl4nNpXFwiZ8LP+JZjxihvZw5ZUgyss=
github.com/DarthSim/opentelemetry-go-contrib/detectors/aws/ecs v0.0.0-20230510163401-1a377505ea6c/go.mod h1:OshtJzwB+6SKoFM4ovJIbsHuwg7PpLGIbpaAOHJwyUU=
github.com/DataDog/appsec-internal-go v1.0.0 h1:2u5IkF4DBj3KVeQn5Vg2vjPUtt513zxEYglcqnd500U=

5
main.go
View File

@ -13,6 +13,7 @@ import (
"go.uber.org/automaxprocs/maxprocs"
"github.com/imgproxy/imgproxy/v3/config"
"github.com/imgproxy/imgproxy/v3/config/loadenv"
"github.com/imgproxy/imgproxy/v3/errorreport"
"github.com/imgproxy/imgproxy/v3/gliblog"
"github.com/imgproxy/imgproxy/v3/imagedata"
@ -35,6 +36,10 @@ func initialize() error {
maxprocs.Set(maxprocs.Logger(log.Debugf))
if err := loadenv.Load(); err != nil {
return err
}
if err := config.Configure(); err != nil {
return err
}