go/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-01-13 20:38:12 +02:00
Code Issues Releases Activity

qingstor: Fixes before merge

Browse Source 
* use rclone's http.Client for bwlimit, logging, etc
  * remove extraneous fmt.Sprintf from logging
  * fix icon in docs
  * add docs about --fast-list
  * hoist md5 regexp compilation out of function
  * create container if necessary on server side copy
  * keep note of whether the container has been deleted
  * build constraint not to compile for plan9
This commit is contained in:
Nick Craig-Wood 2017-08-03 14:31:55 +01:00
parent 7b9557df90
commit 8a185deefa
5 changed files with 110 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
docs/content/qingstor.md
View File

@ -4,7 +4,7 @@ description: "Rclone docs for QingStor Object Storage"
date: "2017-06-26" date: "2017-06-26"
--- ---
<i class="fa fa-qingstor"></i> QingStor <i class="fa fa-hdd-o"></i> QingStor
--------------------------------------- ---------------------------------------
Paths are specified as `remote:bucket` (or `remote:` for the `lsd` Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
@ -119,6 +119,12 @@ files in the bucket.
rclone sync /home/local/directory remote:bucket rclone sync /home/local/directory remote:bucket
### --fast-list ###
This remote supports `--fast-list` which allows you to use fewer
transactions in exchange for more memory. See the [rclone
docs](/docs/#fast-list) for more details.
### Multipart uploads ### ### Multipart uploads ###
rclone supports multipart uploads with QingStor which means that it can rclone supports multipart uploads with QingStor which means that it can
@ -134,6 +140,7 @@ you will get an error, `incorrect zone, the bucket is not in 'XXX'
zone`. zone`.
### Authentication ### ### Authentication ###
There are two ways to supply `rclone` with a set of QingStor There are two ways to supply `rclone` with a set of QingStor
credentials. In order of precedence: credentials. In order of precedence:

2
docs/layouts/chrome/navbar.html
View File

@ -61,7 +61,7 @@
<li><a href="/http/"><i class="fa fa-globe"></i> HTTP</a></li> <li><a href="/http/"><i class="fa fa-globe"></i> HTTP</a></li>
<li><a href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a></li> <li><a href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a></li>
<li><a href="/onedrive/"><i class="fa fa-windows"></i> Microsoft OneDrive</a></li> <li><a href="/onedrive/"><i class="fa fa-windows"></i> Microsoft OneDrive</a></li>
<li><a href="/qingstor/"><i class="fa fa-qingstor"></i> QingStor</a></li> <li><a href="/qingstor/"><i class="fa fa-hdd-o"></i> QingStor</a></li>
<li><a href="/swift/"><i class="fa fa-space-shuttle"></i> Openstack Swift</a></li> <li><a href="/swift/"><i class="fa fa-space-shuttle"></i> Openstack Swift</a></li>
<li><a href="/sftp/"><i class="fa fa-server"></i> SFTP</a></li> <li><a href="/sftp/"><i class="fa fa-server"></i> SFTP</a></li>
<li><a href="/yandex/"><i class="fa fa-space-shuttle"></i> Yandex Disk</a></li> <li><a href="/yandex/"><i class="fa fa-space-shuttle"></i> Yandex Disk</a></li>

2
fstest/fstests/gen_tests.go
View File

@ -162,6 +162,6 @@ func main() {
generateTestProgram(t, fns, "Sftp") generateTestProgram(t, fns, "Sftp")
generateTestProgram(t, fns, "FTP") generateTestProgram(t, fns, "FTP")
generateTestProgram(t, fns, "Box") generateTestProgram(t, fns, "Box")
generateTestProgram(t, fns, "QingStor") generateTestProgram(t, fns, "QingStor", buildConstraint("!plan9"))
log.Printf("Done") log.Printf("Done")
} }

74
qingstor/qingstor.go
View File

@ -1,5 +1,8 @@
// Package qingstor provides an interface to QingStor object storage // Package qingstor provides an interface to QingStor object storage
// Home: https://www.qingcloud.com/ // Home: https://www.qingcloud.com/
// +build !plan9
package qingstor package qingstor
import ( import (
@ -89,14 +92,15 @@ func timestampToTime(tp int64) time.Time {
// Fs represents a remote qingstor server // Fs represents a remote qingstor server
type Fs struct { type Fs struct {
name string // The name of the remote name string // The name of the remote
zone string // The zone we are working on zone string // The zone we are working on
bucket string // The bucket we are working on bucket string // The bucket we are working on
bucketOK bool // true if we have created the bucket bucketOKMu sync.Mutex // mutex to protect bucketOK and bucketDeleted
bucketMtx sync.Mutex // mutex to protect bucket bucketOK bool // true if we have created the bucket
root string // The root is a subdir, is a special object bucketDeleted bool // true if we have deleted the bucket
features *fs.Features // optional features root string // The root is a subdir, is a special object
svc *qs.Service // The connection to the qingstor server features *fs.Features // optional features
svc *qs.Service // The connection to the qingstor server
} }
// Object describes a qingstor object // Object describes a qingstor object
@ -206,6 +210,7 @@ func qsServiceConnection(name string) (*qs.Service, error) {
cf.Host = host cf.Host = host
cf.Port = port cf.Port = port
cf.ConnectionRetries = connectionRetries cf.ConnectionRetries = connectionRetries
cf.Connection = fs.Config.Client()
svc, _ := qs.Init(cf) svc, _ := qs.Init(cf)
@ -320,6 +325,10 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.
// //
// If it isn't possible then return fs.ErrorCantCopy // If it isn't possible then return fs.ErrorCantCopy
func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
err := f.Mkdir("")
if err != nil {
return nil, err
}
srcObj, ok := src.(*Object) srcObj, ok := src.(*Object)
if !ok { if !ok {
fs.Debugf(src, "Can't copy - not same remote type") fs.Debugf(src, "Can't copy - not same remote type")
@ -329,7 +338,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
key := f.root + remote key := f.root + remote
source := path.Join("/"+srcFs.bucket, srcFs.root+srcObj.remote) source := path.Join("/"+srcFs.bucket, srcFs.root+srcObj.remote)
fs.Debugf(f, fmt.Sprintf("Copied, source key is: %s, and dst key is: %s", source, key)) fs.Debugf(f, "Copied, source key is: %s, and dst key is: %s", source, key)
req := qs.PutObjectInput{ req := qs.PutObjectInput{
XQSCopySource: &source, XQSCopySource: &source,
} }
@ -340,7 +349,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
} }
_, err = bucketInit.PutObject(key, &req) _, err = bucketInit.PutObject(key, &req)
if err != nil { if err != nil {
fs.Debugf(f, fmt.Sprintf("Copied Faild, API Error: %s", err)) fs.Debugf(f, "Copied Faild, API Error: %v", err)
return nil, err return nil, err
} }
return f.NewObject(remote) return f.NewObject(remote)
@ -612,18 +621,20 @@ func (f *Fs) dirExists() (bool, error) {
// Mkdir creates the bucket if it doesn't exist // Mkdir creates the bucket if it doesn't exist
func (f *Fs) Mkdir(dir string) error { func (f *Fs) Mkdir(dir string) error {
f.bucketMtx.Lock() f.bucketOKMu.Lock()
defer f.bucketMtx.Unlock() defer f.bucketOKMu.Unlock()
if f.bucketOK { if f.bucketOK {
return nil return nil
} }
exists, err := f.dirExists() if !f.bucketDeleted {
if err == nil { exists, err := f.dirExists()
f.bucketOK = exists if err == nil {
} f.bucketOK = exists
if err != nil || exists { }
return err if err != nil || exists {
return err
}
} }
bucketInit, err := f.svc.Bucket(f.bucket, f.zone) bucketInit, err := f.svc.Bucket(f.bucket, f.zone)
@ -639,6 +650,7 @@ func (f *Fs) Mkdir(dir string) error {
if err == nil { if err == nil {
f.bucketOK = true f.bucketOK = true
f.bucketDeleted = false
} }
return err return err
@ -668,8 +680,8 @@ func (f *Fs) dirIsEmpty() (bool, error) {
// Rmdir delete a bucket // Rmdir delete a bucket
func (f *Fs) Rmdir(dir string) error { func (f *Fs) Rmdir(dir string) error {
f.bucketMtx.Lock() f.bucketOKMu.Lock()
defer f.bucketMtx.Unlock() defer f.bucketOKMu.Unlock()
if f.root != "" || dir != "" { if f.root != "" || dir != "" {
return nil return nil
} }
@ -679,11 +691,11 @@ func (f *Fs) Rmdir(dir string) error {
return err return err
} }
if !isEmpty { if !isEmpty {
fs.Debugf(f, fmt.Sprintf("The bucket %s you tried to delete not empty.", f.bucket)) fs.Debugf(f, "The bucket %s you tried to delete not empty.", f.bucket)
return errors.New("BucketNotEmpty: The bucket you tried to delete is not empty") return errors.New("BucketNotEmpty: The bucket you tried to delete is not empty")
} }
fs.Debugf(f, fmt.Sprintf("Tried to delete the bucket %s", f.bucket)) fs.Debugf(f, "Tried to delete the bucket %s", f.bucket)
bucketInit, err := f.svc.Bucket(f.bucket, f.zone) bucketInit, err := f.svc.Bucket(f.bucket, f.zone)
if err != nil { if err != nil {
return err return err
@ -691,6 +703,7 @@ func (f *Fs) Rmdir(dir string) error {
_, err = bucketInit.Delete() _, err = bucketInit.Delete()
if err == nil { if err == nil {
f.bucketOK = false f.bucketOK = false
f.bucketDeleted = true
} }
return err return err
} }
@ -705,10 +718,10 @@ func (o *Object) readMetaData() (err error) {
} }
key := o.fs.root + o.remote key := o.fs.root + o.remote
fs.Debugf(o, fmt.Sprintf("Read metadata of key: %s", key)) fs.Debugf(o, "Read metadata of key: %s", key)
resp, err := bucketInit.HeadObject(key, &qs.HeadObjectInput{}) resp, err := bucketInit.HeadObject(key, &qs.HeadObjectInput{})
if err != nil { if err != nil {
fs.Debugf(o, fmt.Sprintf("Read metadata faild, API Error: %s", err)) fs.Debugf(o, "Read metadata faild, API Error: %v", err)
if e, ok := err.(*qsErr.QingStorError); ok { if e, ok := err.(*qsErr.QingStorError); ok {
if e.StatusCode == http.StatusNotFound { if e.StatusCode == http.StatusNotFound {
return fs.ErrorObjectNotFound return fs.ErrorObjectNotFound
@ -834,10 +847,10 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
defer func() { defer func() {
if err != nil { if err != nil {
fs.Errorf(o, fmt.Sprintf("Create Object Faild, API ERROR: %s", err)) fs.Errorf(o, "Create Object Faild, API ERROR: %v", err)
// Abort Upload when init success and upload failed // Abort Upload when init success and upload failed
if uploadID != nil { if uploadID != nil {
fs.Debugf(o, fmt.Sprintf("Abort Upload Multipart, upload_id: %s, objectParts: %s", *uploadID, objectParts)) fs.Debugf(o, "Abort Upload Multipart, upload_id: %s, objectParts: %s", *uploadID, objectParts)
abortReq := qs.AbortMultipartUploadInput{ abortReq := qs.AbortMultipartUploadInput{
UploadID: uploadID, UploadID: uploadID,
} }
@ -846,7 +859,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
} }
}() }()
fs.Debugf(o, fmt.Sprintf("Initiate Upload Multipart, key: %s", key)) fs.Debugf(o, "Initiate Upload Multipart, key: %s", key)
mimeType := fs.MimeType(src) mimeType := fs.MimeType(src)
initReq := qs.InitiateMultipartUploadInput{ initReq := qs.InitiateMultipartUploadInput{
ContentType: &mimeType, ContentType: &mimeType,
@ -877,7 +890,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
ContentLength: &size, ContentLength: &size,
Body: buffer, Body: buffer,
} }
fs.Debugf(o, fmt.Sprintf("Upload Multipart, upload_id: %s, part_number: %d", *uploadID, number)) fs.Debugf(o, "Upload Multipart, upload_id: %s, part_number: %d", *uploadID, number)
_, err = bucketInit.UploadMultipart(key, &req) _, err = bucketInit.UploadMultipart(key, &req)
if err != nil { if err != nil {
return err return err
@ -891,7 +904,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
} }
// Complete Multipart Upload // Complete Multipart Upload
fs.Debugf(o, fmt.Sprintf("Complete Upload Multipart, upload_id: %s, objectParts: %d", *uploadID, objectParts)) fs.Debugf(o, "Complete Upload Multipart, upload_id: %s, objectParts: %d", *uploadID, objectParts)
completeReq := qs.CompleteMultipartUploadInput{ completeReq := qs.CompleteMultipartUploadInput{
UploadID: uploadID, UploadID: uploadID,
ObjectParts: objectParts, ObjectParts: objectParts,
@ -923,10 +936,11 @@ func (o *Object) Fs() fs.Info {
return o.fs return o.fs
} }
var matchMd5 = regexp.MustCompile(`^[0-9a-f]{32}$`)
// Hash returns the selected checksum of the file // Hash returns the selected checksum of the file
// If no checksum is available it returns "" // If no checksum is available it returns ""
func (o *Object) Hash(t fs.HashType) (string, error) { func (o *Object) Hash(t fs.HashType) (string, error) {
var matchMd5 = regexp.MustCompile(`^[0-9a-f]{32}$`)
if t != fs.HashMD5 { if t != fs.HashMD5 {
return "", fs.ErrHashUnsupported return "", fs.ErrHashUnsupported
} }

108
qingstor/qingstor_test.go
View File

@ -2,6 +2,9 @@
// //
// Automatically generated - DO NOT EDIT // Automatically generated - DO NOT EDIT
// Regenerate with: make gen_tests // Regenerate with: make gen_tests
// +build !plan9
package qingstor_test package qingstor_test
import ( import (
@ -18,55 +21,56 @@ func TestSetup(t *testing.T) {
} }
// Generic tests for the Fs // Generic tests for the Fs
func TestInit(t *testing.T) { fstests.TestInit(t) } func TestInit(t *testing.T) { fstests.TestInit(t) }
func TestFsString(t *testing.T) { fstests.TestFsString(t) } func TestFsString(t *testing.T) { fstests.TestFsString(t) }
func TestFsName(t *testing.T) { fstests.TestFsName(t) } func TestFsName(t *testing.T) { fstests.TestFsName(t) }
func TestFsRoot(t *testing.T) { fstests.TestFsRoot(t) } func TestFsRoot(t *testing.T) { fstests.TestFsRoot(t) }
func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) }
func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) }
func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) }
func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) }
func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) }
func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) }
func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) }
func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) }
func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) }
func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) }
func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) }
func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) }
func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) }
func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) }
func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) }
func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) }
func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) }
func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) }
func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) }
func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) }
func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) }
func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) }
func TestFsNewObjectDir(t *testing.T) { fstests.TestFsNewObjectDir(t) } func TestFsNewObjectDir(t *testing.T) { fstests.TestFsNewObjectDir(t) }
func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) }
func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } func TestFsMove(t *testing.T) { fstests.TestFsMove(t) }
func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) }
func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) }
func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) }
func TestFsDirChangeNotify(t *testing.T) { fstests.TestFsDirChangeNotify(t) } func TestFsDirChangeNotify(t *testing.T) { fstests.TestFsDirChangeNotify(t) }
func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } func TestObjectString(t *testing.T) { fstests.TestObjectString(t) }
func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) }
func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) }
func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) }
func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) }
func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) }
func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) }
func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) }
func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) }
func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) }
func TestObjectPartialRead(t *testing.T) { fstests.TestObjectPartialRead(t) } func TestObjectPartialRead(t *testing.T) { fstests.TestObjectPartialRead(t) }
func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) }
func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) }
func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) }
func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) } func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) }
func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) }
func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } func TestFsPutUnknownLengthFile(t *testing.T) { fstests.TestFsPutUnknownLengthFile(t) }
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) }
func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }