transform: add truncate_keep_extension and truncate_bytes

This change adds a truncate_bytes mode which counts the number of bytes, as
opposed to the number of UTF-8 characters. This can be useful for ensuring that a
crypt-encoded filename will not exceed the underlying backend's length limits
(see https://forum.rclone.org/t/any-clear-file-name-length-when-using-crypt/36930 ).

This change also adds support for _keep_extension when using truncate and
truncate_bytes.

lib/transform/gen_help.go

@@ -37,6 +37,9 @@ var commandList = []commands{
 	{command: "--name-transform replace=old:new", description: "Replaces occurrences of old with new in the file name."},
 	{command: "--name-transform date={YYYYMMDD}", description: "Appends or prefixes the specified date format."},
 	{command: "--name-transform truncate=N", description: "Truncates the file name to a maximum of N characters."},
+	{command: "--name-transform truncate_keep_extension=N", description: "Truncates the file name to a maximum of N characters while preserving the original file extension."},
+	{command: "--name-transform truncate_bytes=N", description: "Truncates the file name to a maximum of N bytes (not characters)."},
+	{command: "--name-transform truncate_bytes_keep_extension=N", description: "Truncates the file name to a maximum of N bytes (not characters) while preserving the original file extension."},
 	{command: "--name-transform base64encode", description: "Encodes the file name in Base64."},
 	{command: "--name-transform base64decode", description: "Decodes a Base64-encoded file name."},
 	{command: "--name-transform encoder=ENCODING", description: "Converts the file name to the specified encoding (e.g., ISO-8859-1, Windows-1252, Macintosh)."},

lib/transform/options.go

@@ -159,6 +159,12 @@ func (t *transform) requiresValue() bool {
 		return true
 	case ConvTruncate:
 		return true
+	case ConvTruncateKeepExtension:
+		return true
+	case ConvTruncateBytes:
+		return true
+	case ConvTruncateBytesKeepExtension:
+		return true
 	case ConvEncoder:
 		return true
 	case ConvDecoder:
@@ -190,6 +196,9 @@ const (
 	ConvIndex
 	ConvDate
 	ConvTruncate
+	ConvTruncateKeepExtension
+	ConvTruncateBytes
+	ConvTruncateBytesKeepExtension
 	ConvBase64Encode
 	ConvBase64Decode
 	ConvEncoder
@@ -211,35 +220,38 @@ type transformChoices struct{}
 
 func (transformChoices) Choices() []string {
 	return []string{
-		ConvNone:                "none",
+		ConvNone:                       "none",
-		ConvToNFC:               "nfc",
+		ConvToNFC:                      "nfc",
-		ConvToNFD:               "nfd",
+		ConvToNFD:                      "nfd",
-		ConvToNFKC:              "nfkc",
+		ConvToNFKC:                     "nfkc",
-		ConvToNFKD:              "nfkd",
+		ConvToNFKD:                     "nfkd",
-		ConvFindReplace:         "replace",
+		ConvFindReplace:                "replace",
-		ConvPrefix:              "prefix",
+		ConvPrefix:                     "prefix",
-		ConvSuffix:              "suffix",
+		ConvSuffix:                     "suffix",
-		ConvSuffixKeepExtension: "suffix_keep_extension",
+		ConvSuffixKeepExtension:        "suffix_keep_extension",
-		ConvTrimPrefix:          "trimprefix",
+		ConvTrimPrefix:                 "trimprefix",
-		ConvTrimSuffix:          "trimsuffix",
+		ConvTrimSuffix:                 "trimsuffix",
-		ConvIndex:               "index",
+		ConvIndex:                      "index",
-		ConvDate:                "date",
+		ConvDate:                       "date",
-		ConvTruncate:            "truncate",
+		ConvTruncate:                   "truncate",
-		ConvBase64Encode:        "base64encode",
+		ConvTruncateKeepExtension:      "truncate_keep_extension",
-		ConvBase64Decode:        "base64decode",
+		ConvTruncateBytes:              "truncate_bytes",
-		ConvEncoder:             "encoder",
+		ConvTruncateBytesKeepExtension: "truncate_bytes_keep_extension",
-		ConvDecoder:             "decoder",
+		ConvBase64Encode:               "base64encode",
-		ConvISO8859_1:           "ISO-8859-1",
+		ConvBase64Decode:               "base64decode",
-		ConvWindows1252:         "Windows-1252",
+		ConvEncoder:                    "encoder",
-		ConvMacintosh:           "Macintosh",
+		ConvDecoder:                    "decoder",
-		ConvCharmap:             "charmap",
+		ConvISO8859_1:                  "ISO-8859-1",
-		ConvLowercase:           "lowercase",
+		ConvWindows1252:                "Windows-1252",
-		ConvUppercase:           "uppercase",
+		ConvMacintosh:                  "Macintosh",
-		ConvTitlecase:           "titlecase",
+		ConvCharmap:                    "charmap",
-		ConvASCII:               "ascii",
+		ConvLowercase:                  "lowercase",
-		ConvURL:                 "url",
+		ConvUppercase:                  "uppercase",
-		ConvRegex:               "regex",
+		ConvTitlecase:                  "titlecase",
-		ConvCommand:             "command",
+		ConvASCII:                      "ascii",
+		ConvURL:                        "url",
+		ConvRegex:                      "regex",
+		ConvCommand:                    "command",
 	}
 }
 

lib/transform/transform.go

@@ -165,14 +165,25 @@ func transformPathSegment(s string, t transform) (string, error) {
 		if err != nil {
 			return s, err
 		}
-		if max <= 0 {
+		return truncateChars(s, max, false), nil
-			return s, nil
+	case ConvTruncateKeepExtension:
+		max, err := strconv.Atoi(t.value)
+		if err != nil {
+			return s, err
 		}
-		if utf8.RuneCountInString(s) <= max {
+		return truncateChars(s, max, true), nil
-			return s, nil
+	case ConvTruncateBytes:
+		max, err := strconv.Atoi(t.value)
+		if err != nil {
+			return s, err
 		}
-		runes := []rune(s)
+		return truncateBytes(s, max, false)
-		return string(runes[:max]), nil
+	case ConvTruncateBytesKeepExtension:
+		max, err := strconv.Atoi(t.value)
+		if err != nil {
+			return s, err
+		}
+		return truncateBytes(s, max, true)
 	case ConvEncoder:
 		var enc encoder.MultiEncoder
 		err := enc.Set(t.value)
@@ -231,9 +242,13 @@ func transformPathSegment(s string, t transform) (string, error) {
 //
 // i.e. file.txt becomes file_somesuffix.txt not file.txt_somesuffix
 func SuffixKeepExtension(remote string, suffix string) string {
+	base, exts := splitExtension(remote)
+	return base + suffix + exts
+}
+
+func splitExtension(remote string) (base, exts string) {
+	base = remote
 	var (
-		base  = remote
-		exts  = ""
 		first = true
 		ext   = path.Ext(remote)
 	)
@@ -248,7 +263,45 @@ func SuffixKeepExtension(remote string, suffix string) string {
 		first = false
 		ext = path.Ext(base)
 	}
-	return base + suffix + exts
+	return base, exts
+}
+
+func truncateChars(s string, max int, keepExtension bool) string {
+	if max <= 0 {
+		return s
+	}
+	if utf8.RuneCountInString(s) <= max {
+		return s
+	}
+	exts := ""
+	if keepExtension {
+		s, exts = splitExtension(s)
+	}
+	runes := []rune(s)
+	return string(runes[:max-utf8.RuneCountInString(exts)]) + exts
+}
+
+// truncateBytes is like truncateChars but counts the number of bytes, not UTF-8 characters
+func truncateBytes(s string, max int, keepExtension bool) (string, error) {
+	if max <= 0 {
+		return s, nil
+	}
+	if len(s) <= max {
+		return s, nil
+	}
+	exts := ""
+	if keepExtension {
+		s, exts = splitExtension(s)
+	}
+
+	// ensure we don't split a multi-byte UTF-8 character
+	for i := max - len(exts); i > 0; i-- {
+		b := append([]byte(s)[:i], exts...)
+		if len(b) <= max && utf8.Valid(b) {
+			return string(b), nil
+		}
+	}
+	return "", errors.New("could not truncate to valid UTF-8")
 }
 
 // forbid transformations that add/remove path separators

lib/transform/transform_test.go

@@ -128,6 +128,12 @@ func TestVarious(t *testing.T) {
 		{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown _ Fox Went to the Caf_!.txt", []string{"all,charmap=ISO-8859-7"}},
 		{"stories/The Quick Brown Fox: A Memoir [draft].txt", "stories/The Quick Brown Fox： A Memoir ［draft］.txt", []string{"all,encoder=Colon,SquareBracket"}},
 		{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox", []string{"all,truncate=21"}},
+		{"stories/Вот русское предложение, в котором байтов больше, чем символов.txt", "stories/Вот русское предложение, в котором байтов больше, чем символов.txt", []string{"truncate=70"}},
+		{"stories/Вот русское предложение, в котором байтов больше, чем символов.txt", "stories/Вот русское предложение, в котором байтов больше, чем символ", []string{"truncate=60"}},
+		{"stories/Вот русское предложение, в котором байтов больше, чем символов.txt", "stories/Вот русское предложение, в котором байтов больше, чем символов.txt", []string{"truncate_bytes=300"}},
+		{"stories/Вот русское предложение, в котором байтов больше, чем символов.txt", "stories/Вот русское предложение, в котором бай", []string{"truncate_bytes=70"}},
+		{"stories/Вот русское предложение, в котором байтов больше, чем символов.txt", "stories/Вот русское предложение, в котором байтов больше, чем си.txt", []string{"truncate_keep_extension=60"}},
+		{"stories/Вот русское предложение, в котором байтов больше, чем символов.txt", "stories/Вот русское предложение, в котором б.txt", []string{"truncate_bytes_keep_extension=70"}},
 		{"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt", []string{"all,command=echo"}},
 		{"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt-" + time.Now().Local().Format("20060102"), []string{"date=-{YYYYMMDD}"}},
 		{"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt-" + time.Now().Local().Format("2006-01-02 0304PM"), []string{"date=-{macfriendlytime}"}},