go/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-11-29 05:47:23 +02:00
Code Issues Releases Activity

swift: Report disk usage in segment containers

Browse Source 
Large objects are split and stored in a _segments container in Swift.
These should be included when reporting on the space used.

Fixes #8857
This commit is contained in:
Andrew Ruthven
2025-10-30 05:55:53 +13:00
committed by GitHub
parent 87b71dd6b9
commit 5420dbbe38
2 changed files with 52 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
backend/swift/swift.go
View File

@@ -943,6 +943,20 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
used = container.Bytes
objects = container.Count
total = container.QuotaBytes
if f.opt.UseSegmentsContainer.Value {
err = f.pacer.Call(func() (bool, error) {
segmentsContainer := f.rootContainer + segmentsContainerSuffix
container, _, err = f.c.Container(ctx, segmentsContainer)
return shouldRetry(ctx, err)
})
if err != nil && err != swift.ContainerNotFound {
return nil, fmt.Errorf("container info failed: %w", err)
}
if err == nil {
used += container.Bytes
}
}
} else {
var containers []swift.Container
err = f.pacer.Call(func() (bool, error) {

40
backend/swift/swift_test.go
View File

@@ -56,6 +56,11 @@ func (f *Fs) testNoChunk(t *testing.T) {
uploadHash := hash.NewMultiHasher()
in := io.TeeReader(buf, uploadHash)
// Track how much space is used before we put our object.
usage, err := f.About(ctx)
require.NoError(t, err)
usedBeforePut := *usage.Used
file.Size = -1
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
obj, err := f.Features().PutStream(ctx, in, obji)
@@ -70,6 +75,13 @@ func (f *Fs) testNoChunk(t *testing.T) {
require.NoError(t, err)
file.Check(t, obj, f.Precision())
// Check how much space is used after the upload, should match the amount we
// uploaded..
usage, err = f.About(ctx)
require.NoError(t, err)
expectedUsed := usedBeforePut + obj.Size()
require.EqualValues(t, expectedUsed, *usage.Used)
// Delete the object
assert.NoError(t, obj.Remove(ctx))
}
@@ -105,12 +117,24 @@ func (f *Fs) testWithChunk(t *testing.T) {
uploadHash := hash.NewMultiHasher()
in := io.TeeReader(buf, uploadHash)
// Track how much space is used before we put our object.
ctx := context.TODO()
usage, err := f.About(ctx)
require.NoError(t, err)
usedBeforePut := *usage.Used
file.Size = -1
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
ctx := context.TODO()
obj, err := f.Features().PutStream(ctx, in, obji)
require.NoError(t, err)
require.NotEmpty(t, obj)
// Check how much space is used after the upload, should match the amount we
// uploaded..
usage, err = f.About(ctx)
require.NoError(t, err)
expectedUsed := usedBeforePut + obj.Size()
require.EqualValues(t, expectedUsed, *usage.Used)
}
func (f *Fs) testWithChunkFail(t *testing.T) {
@@ -183,9 +207,14 @@ func (f *Fs) testCopyLargeObject(t *testing.T) {
uploadHash := hash.NewMultiHasher()
in := io.TeeReader(buf, uploadHash)
// Track how much space is used before we put our object.
ctx := context.TODO()
usage, err := f.About(ctx)
require.NoError(t, err)
usedBeforePut := *usage.Used
file.Size = -1
obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)
ctx := context.TODO()
obj, err := f.Features().PutStream(ctx, in, obji)
require.NoError(t, err)
require.NotEmpty(t, obj)
@@ -194,6 +223,13 @@ func (f *Fs) testCopyLargeObject(t *testing.T) {
require.NoError(t, err)
require.NotEmpty(t, objTarget)
require.Equal(t, obj.Size(), objTarget.Size())
// Check how much space is used after the upload, should match the amount we
// uploaded *and* the copy.
usage, err = f.About(ctx)
require.NoError(t, err)
expectedUsed := usedBeforePut + obj.Size() + objTarget.Size()
require.EqualValues(t, expectedUsed, *usage.Used)
}
func (f *Fs) testPolicyDiscovery(t *testing.T) {