azureblob: add metadata and tags support across upload and copy paths

This change adds first-class metadata support to the Azure Blob backend,
including headers, user metadata, tags, and modtime overrides, and wires
it through uploads and server-side copies.

There is a behavior change in that rclone will now set the "mtime"
custom metadata when doing server side copies to azure and the
`--metadata` argument is given.

- Map standard headers: cache-control, content-disposition, content-encoding,
  content-language, content-type to corresponding x-ms-blob-* HTTP headers.
- Map user metadata: any non-reserved keys (excluding x-ms-*) are sent as
  blob user metadata. Keys are normalized to lowercase for consistency.
- Support tags: parse `x-ms-tags` as a comma-separated list of key=value
  pairs and apply them on uploads and copies.
- Support mtime override: accept `mtime` in metadata (RFC3339/RFC3339Nano)
  to override the stored modtime persisted in user metadata.

backend/azureblob/azureblob_internal_test.go

@@ -5,11 +5,16 @@ package azureblob
 import (
 	"context"
 	"encoding/base64"
+	"fmt"
+	"net/http"
 	"strings"
 	"testing"
+	"time"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/rclone/rclone/lib/random"
@@ -148,4 +153,417 @@ func (f *Fs) testWriteUncommittedBlocks(t *testing.T) {
 func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("Features", f.testFeatures)
 	t.Run("WriteUncommittedBlocks", f.testWriteUncommittedBlocks)
+	t.Run("Metadata", f.testMetadataPaths)
+}
+
+// helper to read blob properties for an object
+func getProps(ctx context.Context, t *testing.T, o fs.Object) *blob.GetPropertiesResponse {
+	ao := o.(*Object)
+	props, err := ao.readMetaDataAlways(ctx)
+	require.NoError(t, err)
+	return props
+}
+
+// helper to assert select headers and user metadata
+func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, want map[string]string, wantUserMeta map[string]string) {
+	// Headers
+	get := func(p *string) string {
+		if p == nil {
+			return ""
+		}
+		return *p
+	}
+	if v, ok := want["content-type"]; ok {
+		assert.Equal(t, v, get(props.ContentType), "content-type")
+	}
+	if v, ok := want["cache-control"]; ok {
+		assert.Equal(t, v, get(props.CacheControl), "cache-control")
+	}
+	if v, ok := want["content-disposition"]; ok {
+		assert.Equal(t, v, get(props.ContentDisposition), "content-disposition")
+	}
+	if v, ok := want["content-encoding"]; ok {
+		assert.Equal(t, v, get(props.ContentEncoding), "content-encoding")
+	}
+	if v, ok := want["content-language"]; ok {
+		assert.Equal(t, v, get(props.ContentLanguage), "content-language")
+	}
+	// User metadata (case-insensitive keys from service)
+	norm := make(map[string]*string, len(props.Metadata))
+	for kk, vv := range props.Metadata {
+		norm[strings.ToLower(kk)] = vv
+	}
+	for k, v := range wantUserMeta {
+		pv, ok := norm[strings.ToLower(k)]
+		if assert.True(t, ok, fmt.Sprintf("missing user metadata key %q", k)) {
+			if pv == nil {
+				assert.Equal(t, v, "", k)
+			} else {
+				assert.Equal(t, v, *pv, k)
+			}
+		} else {
+			// Log available keys for diagnostics
+			keys := make([]string, 0, len(props.Metadata))
+			for kk := range props.Metadata {
+				keys = append(keys, kk)
+			}
+			t.Logf("available user metadata keys: %v", keys)
+		}
+	}
+}
+
+// helper to read blob tags for an object
+func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
+	ao := o.(*Object)
+	blb := ao.getBlobSVC()
+	resp, err := blb.GetTags(ctx, nil)
+	require.NoError(t, err)
+	out := make(map[string]string)
+	for _, tag := range resp.BlobTagSet {
+		if tag.Key != nil {
+			k := *tag.Key
+			v := ""
+			if tag.Value != nil {
+				v = *tag.Value
+			}
+			out[k] = v
+		}
+	}
+	return out
+}
+
+// Test metadata across different write paths
+func (f *Fs) testMetadataPaths(t *testing.T) {
+	ctx := context.Background()
+	if testing.Short() {
+		t.Skip("skipping in short mode")
+	}
+
+	// Common expected metadata and headers
+	baseMeta := fs.Metadata{
+		"cache-control":       "no-cache",
+		"content-disposition": "inline",
+		"content-language":    "en-US",
+		// Note: Don't set content-encoding here to avoid download decoding differences
+		// We will set a custom user metadata key
+		"potato": "royal",
+		// and modtime
+		"mtime": fstest.Time("2009-05-06T04:05:06.499999999Z").Format(time.RFC3339Nano),
+	}
+
+	// Singlepart upload
+	t.Run("PutSinglepart", func(t *testing.T) {
+		// size less than chunk size
+		contents := random.String(int(f.opt.ChunkSize / 2))
+		item := fstest.NewItem("meta-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+		// override content-type via metadata mapping
+		meta := fs.Metadata{}
+		meta.Merge(baseMeta)
+		meta["content-type"] = "text/plain"
+		obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", meta)
+		defer func() { _ = obj.Remove(ctx) }()
+
+		props := getProps(ctx, t, obj)
+		assertHeadersAndMetadata(t, props, map[string]string{
+			"content-type":        "text/plain",
+			"cache-control":       "no-cache",
+			"content-disposition": "inline",
+			"content-language":    "en-US",
+		}, map[string]string{
+			"potato": "royal",
+		})
+		_ = http.StatusOK // keep import for parity but don't inspect RawResponse
+	})
+
+	// Multipart upload
+	t.Run("PutMultipart", func(t *testing.T) {
+		// size greater than chunk size to force multipart
+		contents := random.String(int(f.opt.ChunkSize + 1024))
+		item := fstest.NewItem("meta-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+		meta := fs.Metadata{}
+		meta.Merge(baseMeta)
+		meta["content-type"] = "application/json"
+		obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/html", meta)
+		defer func() { _ = obj.Remove(ctx) }()
+
+		props := getProps(ctx, t, obj)
+		assertHeadersAndMetadata(t, props, map[string]string{
+			"content-type":        "application/json",
+			"cache-control":       "no-cache",
+			"content-disposition": "inline",
+			"content-language":    "en-US",
+		}, map[string]string{
+			"potato": "royal",
+		})
+
+		// Tags: Singlepart upload
+		t.Run("PutSinglepartTags", func(t *testing.T) {
+			contents := random.String(int(f.opt.ChunkSize / 2))
+			item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+			meta := fs.Metadata{
+				"x-ms-tags": "env=dev,team=sync",
+			}
+			obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "text/plain", meta)
+			defer func() { _ = obj.Remove(ctx) }()
+
+			tags := getTagsMap(ctx, t, obj)
+			assert.Equal(t, "dev", tags["env"])
+			assert.Equal(t, "sync", tags["team"])
+		})
+
+		// Tags: Multipart upload
+		t.Run("PutMultipartTags", func(t *testing.T) {
+			contents := random.String(int(f.opt.ChunkSize + 2048))
+			item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+			meta := fs.Metadata{
+				"x-ms-tags": "project=alpha,release=2025-08",
+			}
+			obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, "application/octet-stream", meta)
+			defer func() { _ = obj.Remove(ctx) }()
+
+			tags := getTagsMap(ctx, t, obj)
+			assert.Equal(t, "alpha", tags["project"])
+			assert.Equal(t, "2025-08", tags["release"])
+		})
+	})
+
+	// Singlepart copy with metadata-set mapping; omit content-type to exercise fallback
+	t.Run("CopySinglepart", func(t *testing.T) {
+		// create small source
+		contents := random.String(int(f.opt.ChunkSize / 2))
+		srcItem := fstest.NewItem("meta-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+		srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
+		defer func() { _ = srcObj.Remove(ctx) }()
+
+		// set mapping via MetadataSet
+		ctx2, ci := fs.AddConfig(ctx)
+		ci.Metadata = true
+		ci.MetadataSet = fs.Metadata{
+			"cache-control":       "private, max-age=60",
+			"content-disposition": "attachment; filename=foo.txt",
+			"content-language":    "fr",
+			// no content-type: should fallback to source
+			"potato": "maris",
+		}
+
+		// do copy
+		dstName := "meta-copy-single-dst.txt"
+		dst, err := f.Copy(ctx2, srcObj, dstName)
+		require.NoError(t, err)
+		defer func() { _ = dst.Remove(ctx2) }()
+
+		props := getProps(ctx2, t, dst)
+		// content-type should fallback to source (text/plain)
+		assertHeadersAndMetadata(t, props, map[string]string{
+			"content-type":        "text/plain",
+			"cache-control":       "private, max-age=60",
+			"content-disposition": "attachment; filename=foo.txt",
+			"content-language":    "fr",
+		}, map[string]string{
+			"potato": "maris",
+		})
+		// mtime should be populated on copy when --metadata is used
+		// and should equal the source ModTime (RFC3339Nano)
+		// Read user metadata (case-insensitive)
+		m := props.Metadata
+		var gotMtime string
+		for k, v := range m {
+			if strings.EqualFold(k, "mtime") && v != nil {
+				gotMtime = *v
+				break
+			}
+		}
+		if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
+			// parse and compare times ignoring formatting differences
+			parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
+			require.NoError(t, err)
+			assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
+		}
+	})
+
+	// CopySinglepart with only --metadata (no MetadataSet) must inject mtime and preserve src content-type
+	t.Run("CopySinglepart_MetadataOnly", func(t *testing.T) {
+		contents := random.String(int(f.opt.ChunkSize / 2))
+		srcItem := fstest.NewItem("meta-copy-single-only-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+		srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
+		defer func() { _ = srcObj.Remove(ctx) }()
+
+		ctx2, ci := fs.AddConfig(ctx)
+		ci.Metadata = true
+
+		dstName := "meta-copy-single-only-dst.txt"
+		dst, err := f.Copy(ctx2, srcObj, dstName)
+		require.NoError(t, err)
+		defer func() { _ = dst.Remove(ctx2) }()
+
+		props := getProps(ctx2, t, dst)
+		assertHeadersAndMetadata(t, props, map[string]string{
+			"content-type": "text/plain",
+		}, map[string]string{})
+		// Assert mtime injected
+		m := props.Metadata
+		var gotMtime string
+		for k, v := range m {
+			if strings.EqualFold(k, "mtime") && v != nil {
+				gotMtime = *v
+				break
+			}
+		}
+		if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
+			parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
+			require.NoError(t, err)
+			assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
+		}
+	})
+
+	// Multipart copy with metadata-set mapping; omit content-type to exercise fallback
+	t.Run("CopyMultipart", func(t *testing.T) {
+		// create large source to force multipart
+		contents := random.String(int(f.opt.CopyCutoff + 1024))
+		srcItem := fstest.NewItem("meta-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+		srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
+		defer func() { _ = srcObj.Remove(ctx) }()
+
+		// set mapping via MetadataSet
+		ctx2, ci := fs.AddConfig(ctx)
+		ci.Metadata = true
+		ci.MetadataSet = fs.Metadata{
+			"cache-control": "max-age=0, no-cache",
+			// omit content-type to trigger fallback
+			"content-language": "de",
+			"potato":           "desiree",
+		}
+
+		dstName := "meta-copy-multi-dst.txt"
+		dst, err := f.Copy(ctx2, srcObj, dstName)
+		require.NoError(t, err)
+		defer func() { _ = dst.Remove(ctx2) }()
+
+		props := getProps(ctx2, t, dst)
+		// content-type should fallback to source (application/octet-stream)
+		assertHeadersAndMetadata(t, props, map[string]string{
+			"content-type":     "application/octet-stream",
+			"cache-control":    "max-age=0, no-cache",
+			"content-language": "de",
+		}, map[string]string{
+			"potato": "desiree",
+		})
+		// mtime should be populated on copy when --metadata is used
+		m := props.Metadata
+		var gotMtime string
+		for k, v := range m {
+			if strings.EqualFold(k, "mtime") && v != nil {
+				gotMtime = *v
+				break
+			}
+		}
+		if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
+			parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
+			require.NoError(t, err)
+			assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
+		}
+	})
+
+	// CopyMultipart with only --metadata must inject mtime and preserve src content-type
+	t.Run("CopyMultipart_MetadataOnly", func(t *testing.T) {
+		contents := random.String(int(f.opt.CopyCutoff + 2048))
+		srcItem := fstest.NewItem("meta-copy-multi-only-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+		srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
+		defer func() { _ = srcObj.Remove(ctx) }()
+
+		ctx2, ci := fs.AddConfig(ctx)
+		ci.Metadata = true
+
+		dstName := "meta-copy-multi-only-dst.txt"
+		dst, err := f.Copy(ctx2, srcObj, dstName)
+		require.NoError(t, err)
+		defer func() { _ = dst.Remove(ctx2) }()
+
+		props := getProps(ctx2, t, dst)
+		assertHeadersAndMetadata(t, props, map[string]string{
+			"content-type": "application/octet-stream",
+		}, map[string]string{})
+		m := props.Metadata
+		var gotMtime string
+		for k, v := range m {
+			if strings.EqualFold(k, "mtime") && v != nil {
+				gotMtime = *v
+				break
+			}
+		}
+		if assert.NotEmpty(t, gotMtime, "mtime not set on destination metadata") {
+			parsed, err := time.Parse(time.RFC3339Nano, gotMtime)
+			require.NoError(t, err)
+			assert.True(t, srcObj.ModTime(ctx2).Equal(parsed), "dst mtime should equal src ModTime")
+		}
+	})
+
+	// Tags: Singlepart copy
+	t.Run("CopySinglepartTags", func(t *testing.T) {
+		// create small source
+		contents := random.String(int(f.opt.ChunkSize / 2))
+		srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+		srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "text/plain", nil)
+		defer func() { _ = srcObj.Remove(ctx) }()
+
+		// set mapping via MetadataSet including tags
+		ctx2, ci := fs.AddConfig(ctx)
+		ci.Metadata = true
+		ci.MetadataSet = fs.Metadata{
+			"x-ms-tags": "copy=single,mode=test",
+		}
+
+		dstName := "tags-copy-single-dst.txt"
+		dst, err := f.Copy(ctx2, srcObj, dstName)
+		require.NoError(t, err)
+		defer func() { _ = dst.Remove(ctx2) }()
+
+		tags := getTagsMap(ctx2, t, dst)
+		assert.Equal(t, "single", tags["copy"])
+		assert.Equal(t, "test", tags["mode"])
+	})
+
+	// Tags: Multipart copy
+	t.Run("CopyMultipartTags", func(t *testing.T) {
+		// create large source to force multipart
+		contents := random.String(int(f.opt.CopyCutoff + 4096))
+		srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+		srcObj := fstests.PutTestContentsMetadata(ctx, t, f, &srcItem, true, contents, true, "application/octet-stream", nil)
+		defer func() { _ = srcObj.Remove(ctx) }()
+
+		ctx2, ci := fs.AddConfig(ctx)
+		ci.Metadata = true
+		ci.MetadataSet = fs.Metadata{
+			"x-ms-tags": "copy=multi,mode=test",
+		}
+
+		dstName := "tags-copy-multi-dst.txt"
+		dst, err := f.Copy(ctx2, srcObj, dstName)
+		require.NoError(t, err)
+		defer func() { _ = dst.Remove(ctx2) }()
+
+		tags := getTagsMap(ctx2, t, dst)
+		assert.Equal(t, "multi", tags["copy"])
+		assert.Equal(t, "test", tags["mode"])
+	})
+
+	// Negative: invalid x-ms-tags must error
+	t.Run("InvalidXMsTags", func(t *testing.T) {
+		contents := random.String(32)
+		item := fstest.NewItem("tags-invalid.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+		// construct ObjectInfo with invalid x-ms-tags
+		buf := strings.NewReader(contents)
+		// Build obj info with metadata
+		meta := fs.Metadata{
+			"x-ms-tags": "badpair-without-equals",
+		}
+		// force metadata on
+		ctx2, ci := fs.AddConfig(ctx)
+		ci.Metadata = true
+		obji := object.NewStaticObjectInfo(item.Path, item.ModTime, int64(len(contents)), true, nil, nil)
+		obji = obji.WithMetadata(meta).WithMimeType("text/plain")
+		_, err := f.Put(ctx2, buf, obji)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "invalid tag")
+	})
 }