From 89748feaa544131d856972e854f366837dbdd3f5 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 13 Mar 2018 20:47:29 +0000 Subject: [PATCH] s3: update docs to discourage use of v2 auth - fixes #2120 From testing it appears that CEPH no longer works properly with v2 auth and neither does Dreamhost, so update the docs anc configuration to recommend v4 auth. --- backend/s3/s3.go | 7 +- docs/content/about.md | 2 +- docs/content/s3.md | 177 +++++++++++++++++++++++++----------------- 3 files changed, 107 insertions(+), 79 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 311a38863..ef433c7df 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -74,7 +74,7 @@ func init() { Help: "AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.", }, { Name: "region", - Help: "Region to connect to.", + Help: "Region to connect to. Leave blank if you are using an S3 clone and you don't have a region.", Examples: []fs.OptionExample{{ Value: "us-east-1", Help: "The default endpoint - a good choice if you are unsure.\nUS Region, Northern Virginia or Pacific Northwest.\nLeave location constraint empty.", @@ -119,10 +119,7 @@ func init() { Help: "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.", }, { Value: "other-v2-signature", - Help: "If using an S3 clone that only understands v2 signatures\neg Ceph/Dreamhost\nset this and make sure you set the endpoint.", - }, { - Value: "other-v4-signature", - Help: "If using an S3 clone that understands v4 signatures set this\nand make sure you set the endpoint.", + Help: "Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.\nSet this and make sure you set the endpoint.", }}, }, { Name: "endpoint", diff --git a/docs/content/about.md b/docs/content/about.md index 4dd7cb91a..21506fd26 100644 --- a/docs/content/about.md +++ b/docs/content/about.md @@ -19,7 +19,7 @@ Rclone is a command line program to sync files and directories to and from: * {{< provider name="Box" home="https://www.box.com/" config="/box/" >}} * {{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}} * {{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}} -* {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/" >}} +* {{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}} * {{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}} * {{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}} * {{< provider name="Google Cloud Storage" home="https://cloud.google.com/storage/" config="/googlecloudstorage/" >}} diff --git a/docs/content/s3.md b/docs/content/s3.md index d3f6b6f78..702eb6dde 100644 --- a/docs/content/s3.md +++ b/docs/content/s3.md @@ -20,37 +20,23 @@ This will guide you through an interactive setup process. No remotes found - make a new one n) New remote s) Set configuration password -n/s> n +q) Quit config +n/s/q> n name> remote Type of storage to configure. Choose a number from below, or type in your own value - 1 / Amazon Drive + 1 / Alias for a existing remote + \ "alias" + 2 / Amazon Drive \ "amazon cloud drive" - 2 / Amazon S3 (also Dreamhost, Ceph, Minio) + 3 / Amazon S3 (also Dreamhost, Ceph, Minio) \ "s3" - 3 / Backblaze B2 + 4 / Backblaze B2 \ "b2" - 4 / Dropbox - \ "dropbox" - 5 / Encrypt/Decrypt a remote - \ "crypt" - 6 / Google Cloud Storage (this is not Google Drive) - \ "google cloud storage" - 7 / Google Drive - \ "drive" - 8 / Hubic - \ "hubic" - 9 / Local Disk - \ "local" -10 / Microsoft OneDrive - \ "onedrive" -11 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH) - \ "swift" -12 / SSH/SFTP Connection - \ "sftp" -13 / Yandex Disk - \ "yandex" -Storage> 2 +[snip] +23 / http Connection + \ "http" +Storage> s3 Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank. Choose a number from below, or type in your own value 1 / Enter AWS credentials in the next step @@ -59,80 +45,91 @@ Choose a number from below, or type in your own value \ "true" env_auth> 1 AWS Access Key ID - leave blank for anonymous access or runtime credentials. -access_key_id> access_key +access_key_id> XXX AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials. -secret_access_key> secret_key -Region to connect to. +secret_access_key> YYY +Region to connect to. Leave blank if you are using an S3 clone and you don't have a region. Choose a number from below, or type in your own value / The default endpoint - a good choice if you are unsure. 1 | US Region, Northern Virginia or Pacific Northwest. | Leave location constraint empty. \ "us-east-1" + / US East (Ohio) Region + 2 | Needs location constraint us-east-2. + \ "us-east-2" / US West (Oregon) Region - 2 | Needs location constraint us-west-2. + 3 | Needs location constraint us-west-2. \ "us-west-2" / US West (Northern California) Region - 3 | Needs location constraint us-west-1. + 4 | Needs location constraint us-west-1. \ "us-west-1" - / EU (Ireland) Region Region - 4 | Needs location constraint EU or eu-west-1. + / Canada (Central) Region + 5 | Needs location constraint ca-central-1. + \ "ca-central-1" + / EU (Ireland) Region + 6 | Needs location constraint EU or eu-west-1. \ "eu-west-1" + / EU (London) Region + 7 | Needs location constraint eu-west-2. + \ "eu-west-2" / EU (Frankfurt) Region - 5 | Needs location constraint eu-central-1. + 8 | Needs location constraint eu-central-1. \ "eu-central-1" / Asia Pacific (Singapore) Region - 6 | Needs location constraint ap-southeast-1. + 9 | Needs location constraint ap-southeast-1. \ "ap-southeast-1" / Asia Pacific (Sydney) Region - 7 | Needs location constraint ap-southeast-2. +10 | Needs location constraint ap-southeast-2. \ "ap-southeast-2" / Asia Pacific (Tokyo) Region - 8 | Needs location constraint ap-northeast-1. +11 | Needs location constraint ap-northeast-1. \ "ap-northeast-1" / Asia Pacific (Seoul) - 9 | Needs location constraint ap-northeast-2. +12 | Needs location constraint ap-northeast-2. \ "ap-northeast-2" / Asia Pacific (Mumbai) -10 | Needs location constraint ap-south-1. +13 | Needs location constraint ap-south-1. \ "ap-south-1" / South America (Sao Paulo) Region -11 | Needs location constraint sa-east-1. +14 | Needs location constraint sa-east-1. \ "sa-east-1" - / If using an S3 clone that only understands v2 signatures -12 | eg Ceph/Dreamhost - | set this and make sure you set the endpoint. + / Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH. +15 | Set this and make sure you set the endpoint. \ "other-v2-signature" - / If using an S3 clone that understands v4 signatures set this -13 | and make sure you set the endpoint. - \ "other-v4-signature" region> 1 Endpoint for S3 API. Leave blank if using AWS to use the default endpoint for the region. Specify if using an S3 clone such as Ceph. -endpoint> +endpoint> Location constraint - must be set to match the Region. Used when creating buckets only. Choose a number from below, or type in your own value 1 / Empty for US Region, Northern Virginia or Pacific Northwest. \ "" - 2 / US West (Oregon) Region. + 2 / US East (Ohio) Region. + \ "us-east-2" + 3 / US West (Oregon) Region. \ "us-west-2" - 3 / US West (Northern California) Region. + 4 / US West (Northern California) Region. \ "us-west-1" - 4 / EU (Ireland) Region. + 5 / Canada (Central) Region. + \ "ca-central-1" + 6 / EU (Ireland) Region. \ "eu-west-1" - 5 / EU Region. + 7 / EU (London) Region. + \ "eu-west-2" + 8 / EU Region. \ "EU" - 6 / Asia Pacific (Singapore) Region. + 9 / Asia Pacific (Singapore) Region. \ "ap-southeast-1" - 7 / Asia Pacific (Sydney) Region. +10 / Asia Pacific (Sydney) Region. \ "ap-southeast-2" - 8 / Asia Pacific (Tokyo) Region. +11 / Asia Pacific (Tokyo) Region. \ "ap-northeast-1" - 9 / Asia Pacific (Seoul) +12 / Asia Pacific (Seoul) \ "ap-northeast-2" -10 / Asia Pacific (Mumbai) +13 / Asia Pacific (Mumbai) \ "ap-south-1" -11 / South America (Sao Paulo) Region. +14 / South America (Sao Paulo) Region. \ "sa-east-1" location_constraint> 1 Canned ACL used when creating buckets and/or storing objects in S3. @@ -153,14 +150,14 @@ Choose a number from below, or type in your own value / Both the object owner and the bucket owner get FULL_CONTROL over the object. 6 | If you specify this canned ACL when creating a bucket, Amazon S3 ignores it. \ "bucket-owner-full-control" -acl> private +acl> 1 The server-side encryption algorithm used when storing this object in S3. Choose a number from below, or type in your own value 1 / None \ "" 2 / AES256 \ "AES256" -server_side_encryption> +server_side_encryption> 1 The storage class to use when storing objects in S3. Choose a number from below, or type in your own value 1 / Default @@ -171,19 +168,19 @@ Choose a number from below, or type in your own value \ "REDUCED_REDUNDANCY" 4 / Standard Infrequent Access storage class \ "STANDARD_IA" -storage_class> +storage_class> 1 Remote config -------------------- [remote] env_auth = false -access_key_id = access_key -secret_access_key = secret_key +access_key_id = XXX +secret_access_key = YYY region = us-east-1 -endpoint = -location_constraint = +endpoint = +location_constraint = acl = private -server_side_encryption = -storage_class = +server_side_encryption = +storage_class = -------------------- y) Yes this is OK e) Edit this remote @@ -381,16 +378,27 @@ You will be able to list and copy data but not upload it. ### Ceph ### -Ceph is an object storage system which presents an Amazon S3 interface. +[Ceph](https://ceph.com/) is an open source unified, distributed +storage system designed for excellent performance, reliability and +scalability. It has an S3 compatible object storage interface. + +To use rclone with Ceph, configure as above but leave the region blank +and set the endpoint. You should end up with something like this in +your config: -To use rclone with ceph, you need to set the following parameters in -the config. ``` -access_key_id = Whatever -secret_access_key = Whatever -endpoint = https://ceph.endpoint.goes.here/ -region = other-v2-signature +[ceph] +type = s3 +env_auth = false +access_key_id = XXX +secret_access_key = YYY +region = +endpoint = https://ceph.endpoint.example.com +location_constraint = +acl = +server_side_encryption = +storage_class = ``` Note also that Ceph sometimes puts `/` in the passwords it gives @@ -418,6 +426,29 @@ removed). Because this is a json dump, it is encoding the `/` as `\/`, so if you use the secret key as `xxxxxx/xxxx` it will work fine. +### Dreamhost ### + +Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is +an object storage system based on CEPH. + +To use rclone with Dreamhost, configure as above but leave the region blank +and set the endpoint. You should end up with something like this in +your config: + +``` +[dreamobjects] +env_auth = false +access_key_id = your_access_key +secret_access_key = your_secret_key +region = +endpoint = objects-us-west-1.dream.io +location_constraint = +acl = private +server_side_encryption = +storage_class = +``` + + ### DigitalOcean Spaces ### [Spaces](https://www.digitalocean.com/products/object-storage/) is an [S3-interoperable](https://developers.digitalocean.com/documentation/spaces/) object storage service from cloud provider DigitalOcean. @@ -429,7 +460,7 @@ When prompted for a `region` or `location_constraint`, press enter to use the de Going through the whole process of creating a new remote by running `rclone config`, each prompt should be answered as shown below: ``` -Storage> 2 +Storage> s3 env_auth> 1 access_key_id> YOUR_ACCESS_KEY secret_access_key> YOUR_SECRET_KEY