diff --git a/backend/swift/swift.go b/backend/swift/swift.go index d4d5c5fc9..ac18da01a 100644 --- a/backend/swift/swift.go +++ b/backend/swift/swift.go @@ -561,6 +561,21 @@ func (f *Fs) setRoot(root string) { f.rootContainer, f.rootDirectory = bucket.Split(f.root) } +// Fetch the base container's policy to be used if/when we need to create a +// segments container to ensure we use the same policy. +func (f *Fs) fetchStoragePolicy(ctx context.Context, container string) (fs.Fs, error) { + err := f.pacer.Call(func() (bool, error) { + var rxHeaders swift.Headers + _, rxHeaders, err := f.c.Container(ctx, container) + + f.opt.StoragePolicy = rxHeaders["X-Storage-Policy"] + fs.Debugf(f, "Auto set StoragePolicy to %s", f.opt.StoragePolicy) + + return shouldRetryHeaders(ctx, rxHeaders, err) + }) + return nil, err +} + // NewFsWithConnection constructs an Fs from the path, container:path // and authenticated connection. // @@ -590,6 +605,7 @@ func NewFsWithConnection(ctx context.Context, opt *Options, name, root string, c f.opt.UseSegmentsContainer.Valid = true fs.Debugf(f, "Auto set use_segments_container to %v", f.opt.UseSegmentsContainer.Value) } + if f.rootContainer != "" && f.rootDirectory != "" { // Check to see if the object exists - ignoring directory markers var info swift.Object @@ -1132,6 +1148,13 @@ func (f *Fs) newSegmentedUpload(ctx context.Context, dstContainer string, dstPat container: dstContainer, } if f.opt.UseSegmentsContainer.Value { + if f.opt.StoragePolicy == "" { + _, err = f.fetchStoragePolicy(ctx, dstContainer) + if err != nil { + return nil, err + } + } + su.container += segmentsContainerSuffix err = f.makeContainer(ctx, su.container) if err != nil { diff --git a/backend/swift/swift_test.go b/backend/swift/swift_test.go index f58f8f4b6..9f3d2d74f 100644 --- a/backend/swift/swift_test.go +++ b/backend/swift/swift_test.go @@ -76,6 +76,7 @@ func (f *Fs) testNoChunk(t *testing.T) { // Additional tests that aren't in the framework func (f *Fs) InternalTest(t *testing.T) { + t.Run("PolicyDiscovery", f.testPolicyDiscovery) t.Run("NoChunk", f.testNoChunk) t.Run("WithChunk", f.testWithChunk) t.Run("WithChunkFail", f.testWithChunkFail) @@ -195,4 +196,50 @@ func (f *Fs) testCopyLargeObject(t *testing.T) { require.Equal(t, obj.Size(), objTarget.Size()) } +func (f *Fs) testPolicyDiscovery(t *testing.T) { + ctx := context.TODO() + container := "testPolicyDiscovery-1" + // Reset the policy so we can test if it is populated. + f.opt.StoragePolicy = "" + err := f.makeContainer(ctx, container) + require.NoError(t, err) + _, err = f.fetchStoragePolicy(ctx, container) + require.NoError(t, err) + + // Default policy for Swift is Policy-0. + assert.Equal(t, "Policy-0", f.opt.StoragePolicy) + + // Create a container using a non-default policy, and check to ensure + // that the created segments container uses the same non-default policy. + policy := "Policy-1" + container = "testPolicyDiscovery-2" + + f.opt.StoragePolicy = policy + err = f.makeContainer(ctx, container) + require.NoError(t, err) + + // Reset the policy so we can test if it is populated, and set to the + // non-default policy. + f.opt.StoragePolicy = "" + _, err = f.fetchStoragePolicy(ctx, container) + require.NoError(t, err) + assert.Equal(t, policy, f.opt.StoragePolicy) + + // Test that when a segmented upload container is made, the newly + // created container inherits the non-default policy of the base + // container. + f.opt.StoragePolicy = "" + f.opt.UseSegmentsContainer.Value = true + su, err := f.newSegmentedUpload(ctx, container, "") + require.NoError(t, err) + // The container name we expected? + segmentsContainer := container + segmentsContainerSuffix + assert.Equal(t, segmentsContainer, su.container) + // The policy we expected? + f.opt.StoragePolicy = "" + _, err = f.fetchStoragePolicy(ctx, su.container) + require.NoError(t, err) + assert.Equal(t, policy, f.opt.StoragePolicy) +} + var _ fstests.InternalTester = (*Fs)(nil) diff --git a/fstest/testserver/init.d/TestSwiftAIO b/fstest/testserver/init.d/TestSwiftAIO index 7a04d63ff..d46ad5029 100755 --- a/fstest/testserver/init.d/TestSwiftAIO +++ b/fstest/testserver/init.d/TestSwiftAIO @@ -8,10 +8,12 @@ PORT=28628 . $(dirname "$0")/docker.bash start() { + # We need to replace the remakerings in the container to create Policy-1. docker run --rm -d --name ${NAME} \ -p 127.0.0.1:${PORT}:8080 \ + -v $(dirname "$0")/TestSwiftAIO.d/remakerings:/swift/bin/remakerings:ro \ bouncestorage/swift-aio - + echo type=swift echo env_auth=false echo user=test:tester diff --git a/fstest/testserver/init.d/TestSwiftAIO.d/remakerings b/fstest/testserver/init.d/TestSwiftAIO.d/remakerings new file mode 100755 index 000000000..06785b3ab --- /dev/null +++ b/fstest/testserver/init.d/TestSwiftAIO.d/remakerings @@ -0,0 +1,33 @@ +#!/bin/bash + +cd /etc/swift + +if ! grep storage-policy swift.conf; then + cat <> swift.conf + +# Policy-0 is the default, we need two policies to test policy inheritance. +[storage-policy:0] +name = Policy-0 +default = true + +[storage-policy:1] +name = Policy-1 +EOF +fi + +rm -f *.builder *.ring.gz backups/*.builder backups/*.ring.gz + +swift-ring-builder object.builder create 10 1 1 +swift-ring-builder object.builder add r1z1-127.0.0.1:6010/sdb1 1 +swift-ring-builder object.builder rebalance +swift-ring-builder container.builder create 10 1 1 +swift-ring-builder container.builder add r1z1-127.0.0.1:6011/sdb1 1 +swift-ring-builder container.builder rebalance +swift-ring-builder account.builder create 10 1 1 +swift-ring-builder account.builder add r1z1-127.0.0.1:6012/sdb1 1 +swift-ring-builder account.builder rebalance + +# For Policy-1: +swift-ring-builder object-1.builder create 10 1 1 +swift-ring-builder object-1.builder add r1z1-127.0.0.1:6010/sdb1 1 +swift-ring-builder object-1.builder rebalance