rclone/backend/azureblob/azureblob_internal_test.go

//go:build !plan9 && !solaris && !js

package azureblob

import (
	"context"
	"encoding/base64"
	"fmt"
	"net/http"
	"strings"
	"testing"
	"time"

	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/object"
	"github.com/rclone/rclone/fstest"
	"github.com/rclone/rclone/fstest/fstests"
	"github.com/rclone/rclone/lib/random"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestBlockIDCreator(t *testing.T) {
	// Check creation and random number
	bic, err := newBlockIDCreator()
	require.NoError(t, err)
	bic2, err := newBlockIDCreator()
	require.NoError(t, err)
	assert.NotEqual(t, bic.random, bic2.random)
	assert.NotEqual(t, bic.random, [8]byte{})

	// Set random to known value for tests
	bic.random = [8]byte{1, 2, 3, 4, 5, 6, 7, 8}
	chunkNumber := uint64(0xFEDCBA9876543210)

	// Check creation of ID
	want := base64.StdEncoding.EncodeToString([]byte{0xFE, 0xDC, 0xBA, 0x98, 0x76, 0x54, 0x32, 0x10, 1, 2, 3, 4, 5, 6, 7, 8})
	assert.Equal(t, "/ty6mHZUMhABAgMEBQYHCA==", want)
	got := bic.newBlockID(chunkNumber)
	assert.Equal(t, want, got)
	assert.Equal(t, "/ty6mHZUMhABAgMEBQYHCA==", got)

	// Test checkID is working
	assert.NoError(t, bic.checkID(chunkNumber, got))
	assert.ErrorContains(t, bic.checkID(chunkNumber, "$"+got), "illegal base64")
	assert.ErrorContains(t, bic.checkID(chunkNumber, "AAAA"+got), "bad block ID length")
	assert.ErrorContains(t, bic.checkID(chunkNumber+1, got), "expecting decoded")
	assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
}

func (f *Fs) testFeatures(t *testing.T) {
	// Check first feature flags are set on this remote
	enabled := f.Features().SetTier
	assert.True(t, enabled)
	enabled = f.Features().GetTier
	assert.True(t, enabled)
}

type ReadSeekCloser struct {
	*strings.Reader
}

func (r *ReadSeekCloser) Close() error {
	return nil
}

// Stage a block at remote but don't commit it
func (f *Fs) stageBlockWithoutCommit(ctx context.Context, t *testing.T, remote string) {
	var (
		containerName, blobPath = f.split(remote)
		containerClient         = f.cntSVC(containerName)
		blobClient              = containerClient.NewBlockBlobClient(blobPath)
		data                    = "uncommitted data"
		blockID                 = "1"
		blockIDBase64           = base64.StdEncoding.EncodeToString([]byte(blockID))
	)
	r := &ReadSeekCloser{strings.NewReader(data)}
	_, err := blobClient.StageBlock(ctx, blockIDBase64, r, nil)
	require.NoError(t, err)

	// Verify the block is staged but not committed
	blockList, err := blobClient.GetBlockList(ctx, blockblob.BlockListTypeAll, nil)
	require.NoError(t, err)
	found := false
	for _, block := range blockList.UncommittedBlocks {
		if *block.Name == blockIDBase64 {
			found = true
			break
		}
	}
	require.True(t, found, "Block ID not found in uncommitted blocks")
}

// This tests uploading a blob where it has uncommitted blocks with a different ID size.
//
// https://gauravmantri.com/2013/05/18/windows-azure-blob-storage-dealing-with-the-specified-blob-or-block-content-is-invalid-error/
//
// TestIntegration/FsMkdir/FsPutFiles/Internal/WriteUncommittedBlocks
func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
	var (
		ctx    = context.Background()
		remote = "testBlob"
	)

	// Multipart copy the blob please
	oldUseCopyBlob, oldCopyCutoff := f.opt.UseCopyBlob, f.opt.CopyCutoff
	f.opt.UseCopyBlob = false
	f.opt.CopyCutoff = f.opt.ChunkSize
	defer func() {
		f.opt.UseCopyBlob, f.opt.CopyCutoff = oldUseCopyBlob, oldCopyCutoff
	}()

	// Create a blob with uncommitted blocks
	f.stageBlockWithoutCommit(ctx, t, remote)

	// Now attempt to overwrite the block with a different sized block ID to provoke this error

	// Check the object does not exist
	_, err := f.NewObject(ctx, remote)
	require.Equal(t, fs.ErrorObjectNotFound, err)

	// Upload a multipart file over the block with uncommitted chunks of a different ID size
	size := 4*int(f.opt.ChunkSize) - 1
	contents := random.String(size)
	item := fstest.NewItem(remote, contents, fstest.Time("2001-05-06T04:05:06.499Z"))
	o := fstests.PutTestContents(ctx, t, f, &item, contents, true)

	// Check size
	assert.Equal(t, int64(size), o.Size())

	// Create a new blob with uncommitted blocks
	newRemote := "testBlob2"
	f.stageBlockWithoutCommit(ctx, t, newRemote)

	// Copy over that block
	dst, err := f.Copy(ctx, o, newRemote)
	require.NoError(t, err)

	// Check basics
	assert.Equal(t, int64(size), dst.Size())
	assert.Equal(t, newRemote, dst.Remote())

	// Check contents
	gotContents := fstests.ReadObject(ctx, t, dst, -1)
	assert.Equal(t, contents, gotContents)

	// Remove the object
	require.NoError(t, dst.Remove(ctx))
}

func (f *Fs) InternalTest(t *testing.T) {
	t.Run("Features", f.testFeatures)
	t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
	t.Run("Metadata", f.testMetadataPaths)
}

// helper to read blob properties for an object
func getProps(ctx context.Context, t *testing.T, o fs.Object) *blob.GetPropertiesResponse {
	ao := o.(*Object)
	props, err := ao.readMetaDataAlways(ctx)
	require.NoError(t, err)
	return props
}

// helper to assert select headers and user metadata
func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, want map[string]string, wantUserMeta map[string]string) {
	// Headers
	get := func(p *string) string {
		if p == nil {
			return ""
		}
		return *p
	}
	if v, ok := want["content-type"]; ok {
		assert.Equal(t, v, get(props.ContentType), "content-type")
	}
	if v, ok := want["cache-control"]; ok {
		assert.Equal(t, v, get(props.CacheControl), "cache-control")
	}
	if v, ok := want["content-disposition"]; ok {
		assert.Equal(t, v, get(props.ContentDisposition), "content-disposition")
	}
	if v, ok := want["content-encoding"]; ok {
		assert.Equal(t, v, get(props.ContentEncoding), "content-encoding")
	}
	if v, ok := want["content-language"]; ok {
		assert.Equal(t, v, get(props.ContentLanguage), "content-language")
	}
	// User metadata (case-insensitive keys from service)
	norm := make(map[string]*string, len(props.Metadata))
	for kk, vv := range props.Metadata {
		norm[strings.ToLower(kk)] = vv
	}
	for k, v := range wantUserMeta {
		pv, ok := norm[strings.ToLower(k)]
		if assert.True(t, ok, fmt.Sprintf("missing user metadata key %q", k)) {
			if pv == nil {
				assert.Equal(t, v, "", k)
			} else {
				assert.Equal(t, v, *pv, k)
			}
		} else {
			// Log available keys for diagnostics
			keys := make([]string, 0, len(props.Metadata))
			for kk := range props.Metadata {
				keys = append(keys, kk)
			}
			t.Logf("available user metadata keys: %v", keys)
		}
	}
}

// helper to read blob tags for an object
func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
	ao := o.(*Object)
	blb := ao.getBlobSVC()
	resp, err := blb.GetTags(ctx, nil)
	require.NoError(t, err)
	out := make(map[string]string)
	for _, tag := range resp.BlobTagSet {
		if tag.Key != nil {
			k := *tag.Key
			v := ""
			if tag.Value != nil {
				v = *tag.Value
			}
			out[k] = v
		}
	}
	return out
}

// Test metadata across different write paths
func (f *Fs) testMetadataPaths(t *testing.T) {
	ctx := context.Background()
	if testing.Short() {
		t.Skip("skipping in short mode")
	}

	// Common expected metadata and headers
	baseMeta := fs.Metadata{
		"cache-control":       "no-cache",
		"content-disposition": "inline",
		"content-language":    "en-US",
		// Note: Don't set content-encoding here to avoid download decoding differences
		// We will set a custom user metadata key
		"potato": "royal",
		// and modtime
		"mtime": fstest.Time("2009-05-06T04:05:06.499999999Z").Format(time.RFC3339Nano),
	}

	// Singlepart upload
	t.Run("PutSinglepart", func(t *testing.T) {
		// size less than chunk size
		contents := random.String(int(f.opt.ChunkSize / 2))
		item := fstest.NewItem("meta-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
		// override content-type via metadata mapping
		meta := fs.Metadata{}
		meta.Merge(baseMeta)
		meta["content-type"] = "text/plain"
		obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", meta)
		defer func() { _ = obj.Remove(ctx) }()

		props := getProps(ctx, t, obj)
		assertHeadersAndMetadata(t, props, map[string]string{
			"content-type":        "text/plain",
			"cache-control":       "no-cache",
			"content-disposition": "inline",
			"content-language":    "en-US",
		}, map[string]string{
			"potato": "royal",
		})
		_ = http.StatusOK // keep import for parity but don't inspect RawResponse
	})

	// Multipart upload
	t.Run("PutMultipart", func(t *testing.T) {
		// size greater than chunk size to force multipart
		contents := random.String(int(f.opt.ChunkSize + 1024))
		item := fstest.NewItem("meta-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
		meta := fs.Metadata{}
		meta.Merge(baseMeta)
		meta["content-type"] = "application/json"
		obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", meta)
		defer func() { _ = obj.Remove(ctx) }()

		props := getProps(ctx, t, obj)
		assertHeadersAndMetadata(t, props, map[string]string{
			"content-type":        "application/json",
			"cache-control":       "no-cache",
			"content-disposition": "inline",
			"content-language":    "en-US",
		}, map[string]string{
			"potato": "royal",
		})

		// Tags: Singlepart upload
		t.Run("PutSinglepartTags", func(t *testing.T) {
			contents := random.String(int(f.opt.ChunkSize / 2))
			item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
			meta := fs.Metadata{
				"x-ms-tags": "env=dev,team=sync",
			}
			obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/plain", meta)
			defer func() { _ = obj.Remove(ctx) }()

			tags := getTagsMap(ctx, t, obj)
			assert.Equal(t, "dev", tags["env"])
			assert.Equal(t, "sync", tags["team"])
		})

		// Tags: Multipart upload
		t.Run("PutMultipartTags", func(t *testing.T) {
			contents := random.String(int(f.opt.ChunkSize + 2048))
			item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
			meta := fs.Metadata{
				"x-ms-tags": "project=alpha,release=2025-08",
			}
			obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "application/octet-stream", meta)
			defer func() { _ = obj.Remove(ctx) }()

			tags := getTagsMap(ctx, t, obj)
			assert.Equal(t, "alpha", tags["project"])
			assert.Equal(t, "2025-08", tags["release"])
		})
	})

	// Singlepart copy with metadata-set mapping; omit content-type to exercise fallback
	t.Run("CopySinglepart", func(t *testing.T) {
		// create small source
		contents := random.String(int(f.opt.ChunkSize / 2))
		srcItem := fstest.NewItem("meta-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
		srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
		defer func() { _ = srcObj.Remove(ctx) }()

		// set mapping via MetadataSet
		ctx2, ci := fs.AddConfig(ctx)
		ci.Metadata = true
		ci.MetadataSet = fs.Metadata{
			"cache-control":       "private, max-age=60",
			"content-disposition": "attachment; filename=foo.txt",
			"content-language":    "fr",
			// no content-type: should fallback to source
			"potato": "maris",
		}

		// do copy
		dstName := "meta-copy-single-dst.txt"
		dst, err := f.Copy(ctx2, srcObj, dstName)
		require.NoError(t, err)
		defer func() { _ = dst.Remove(ctx2) }()

		props := getProps(ctx2, t, dst)
		// content-type should fallback to source (text/plain)
		assertHeadersAndMetadata(t, props, map[string]string{
			"content-type":        "text/plain",
			"cache-control":       "private, max-age=60",
			"content-disposition": "attachment; filename=foo.txt",
			"content-language":    "fr",
		}, map[string]string{
			"potato": "maris",
		})
		// mtime should be populated on copy when --metadata is used
		// and should equal the source ModTime (RFC3339Nano)
		// Read user metadata (case-insensitive)
		m := props.Metadata
		var gotMtime string
		for k, v := range m {
			if strings.EqualFold(k, "mtime") && v != nil {
				gotMtime = *v
				break
			}
		}
		if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
			// parse and compare times ignoring formatting differences
			parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
			require.NoError(t, err)
			assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
		}
	})

	// CopySinglepart with only --metadata (no MetadataSet) must inject mtime and preserve src content-type
	t.Run("CopySinglepart_MetadataOnly", func(t *testing.T) {
		contents := random.String(int(f.opt.ChunkSize / 2))
		srcItem := fstest.NewItem("meta-copy-single-only-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
		srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
		defer func() { _ = srcObj.Remove(ctx) }()

		ctx2, ci := fs.AddConfig(ctx)
		ci.Metadata = true

		dstName := "meta-copy-single-only-dst.txt"
		dst, err := f.Copy(ctx2, srcObj, dstName)
		require.NoError(t, err)
		defer func() { _ = dst.Remove(ctx2) }()

		props := getProps(ctx2, t, dst)
		assertHeadersAndMetadata(t, props, map[string]string{
			"content-type": "text/plain",
		}, map[string]string{})
		// Assert mtime injected
		m := props.Metadata
		var gotMtime string
		for k, v := range m {
			if strings.EqualFold(k, "mtime") && v != nil {
				gotMtime = *v
				break
			}
		}
		if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
			parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
			require.NoError(t, err)
			assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
		}
	})

	// Multipart copy with metadata-set mapping; omit content-type to exercise fallback
	t.Run("CopyMultipart", func(t *testing.T) {
		// create large source to force multipart
		contents := random.String(int(f.opt.CopyCutoff + 1024))
		srcItem := fstest.NewItem("meta-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
		srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
		defer func() { _ = srcObj.Remove(ctx) }()

		// set mapping via MetadataSet
		ctx2, ci := fs.AddConfig(ctx)
		ci.Metadata = true
		ci.MetadataSet = fs.Metadata{
			"cache-control": "max-age=0, no-cache",
			// omit content-type to trigger fallback
			"content-language": "de",
			"potato":           "desiree",
		}

		dstName := "meta-copy-multi-dst.txt"
		dst, err := f.Copy(ctx2, srcObj, dstName)
		require.NoError(t, err)
		defer func() { _ = dst.Remove(ctx2) }()

		props := getProps(ctx2, t, dst)
		// content-type should fallback to source (application/octet-stream)
		assertHeadersAndMetadata(t, props, map[string]string{
			"content-type":     "application/octet-stream",
			"cache-control":    "max-age=0, no-cache",
			"content-language": "de",
		}, map[string]string{
			"potato": "desiree",
		})
		// mtime should be populated on copy when --metadata is used
		m := props.Metadata
		var gotMtime string
		for k, v := range m {
			if strings.EqualFold(k, "mtime") && v != nil {
				gotMtime = *v
				break
			}
		}
		if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
			parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
			require.NoError(t, err)
			assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
		}
	})

	// CopyMultipart with only --metadata must inject mtime and preserve src content-type
	t.Run("CopyMultipart_MetadataOnly", func(t *testing.T) {
		contents := random.String(int(f.opt.CopyCutoff + 2048))
		srcItem := fstest.NewItem("meta-copy-multi-only-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
		srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
		defer func() { _ = srcObj.Remove(ctx) }()

		ctx2, ci := fs.AddConfig(ctx)
		ci.Metadata = true

		dstName := "meta-copy-multi-only-dst.txt"
		dst, err := f.Copy(ctx2, srcObj, dstName)
		require.NoError(t, err)
		defer func() { _ = dst.Remove(ctx2) }()

		props := getProps(ctx2, t, dst)
		assertHeadersAndMetadata(t, props, map[string]string{
			"content-type": "application/octet-stream",
		}, map[string]string{})
		m := props.Metadata
		var gotMtime string
		for k, v := range m {
			if strings.EqualFold(k, "mtime") && v != nil {
				gotMtime = *v
				break
			}
		}
		if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
			parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
			require.NoError(t, err)
			assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
		}
	})

	// Tags: Singlepart copy
	t.Run("CopySinglepartTags", func(t *testing.T) {
		// create small source
		contents := random.String(int(f.opt.ChunkSize / 2))
		srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
		srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
		defer func() { _ = srcObj.Remove(ctx) }()

		// set mapping via MetadataSet including tags
		ctx2, ci := fs.AddConfig(ctx)
		ci.Metadata = true
		ci.MetadataSet = fs.Metadata{
			"x-ms-tags": "copy=single,mode=test",
		}

		dstName := "tags-copy-single-dst.txt"
		dst, err := f.Copy(ctx2, srcObj, dstName)
		require.NoError(t, err)
		defer func() { _ = dst.Remove(ctx2) }()

		tags := getTagsMap(ctx2, t, dst)
		assert.Equal(t, "single", tags["copy"])
		assert.Equal(t, "test", tags["mode"])
	})

	// Tags: Multipart copy
	t.Run("CopyMultipartTags", func(t *testing.T) {
		// create large source to force multipart
		contents := random.String(int(f.opt.CopyCutoff + 4096))
		srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
		srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
		defer func() { _ = srcObj.Remove(ctx) }()

		ctx2, ci := fs.AddConfig(ctx)
		ci.Metadata = true
		ci.MetadataSet = fs.Metadata{
			"x-ms-tags": "copy=multi,mode=test",
		}

		dstName := "tags-copy-multi-dst.txt"
		dst, err := f.Copy(ctx2, srcObj, dstName)
		require.NoError(t, err)
		defer func() { _ = dst.Remove(ctx2) }()

		tags := getTagsMap(ctx2, t, dst)
		assert.Equal(t, "multi", tags["copy"])
		assert.Equal(t, "test", tags["mode"])
	})

	// Negative: invalid x-ms-tags must error
	t.Run("InvalidXMsTags", func(t *testing.T) {
		contents := random.String(32)
		item := fstest.NewItem("tags-invalid.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
		// construct ObjectInfo with invalid x-ms-tags
		buf := strings.NewReader(contents)
		// Build obj info with metadata
		meta := fs.Metadata{
			"x-ms-tags": "badpair-without-equals",
		}
		// force metadata on
		ctx2, ci := fs.AddConfig(ctx)
		ci.Metadata = true
		obji := object.NewStaticObjectInfo(item.Path, item.ModTime, int64(len(contents)), true, nil, nil)
		obji = obji.WithMetadata(meta).WithMimeType("text/plain")
		_, err := f.Put(ctx2, buf, obji)
		require.Error(t, err)
		assert.Contains(t, err.Error(), "invalid tag")
	})
}