rclone/lib/transform/transform.go

// Package transform holds functions for path name transformations
//
//go:generate go run gen_help.go transform.md
package transform

import (
	"bytes"
	"context"
	_ "embed"
	"encoding/base64"
	"errors"
	"fmt"
	"mime"
	"net/url"
	"os/exec"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/rclone/rclone/fs"
	"github.com/rclone/rclone/fs/fserrors"
	"github.com/rclone/rclone/lib/encoder"
	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/unicode/norm"
)

//go:embed transform.md
var help string

// Help returns the help string cleaned up to simplify appending
func Help() string {
	// Chop off auto generated message
	nl := strings.IndexRune(help, '\n')
	return strings.TrimSpace(help[nl:]) + "\n\n"
}

// Path transforms a path s according to the --name-transform options in use
//
// If no transforms are in use, s is returned unchanged
func Path(ctx context.Context, s string, isDir bool) string {
	if !Transforming(ctx) {
		return s
	}

	old := s
	opt, err := getOptions(ctx)
	if err != nil {
		err = fs.CountError(ctx, err)
		fs.Errorf(s, "Failed to parse transform flags: %v", err)
	}
	for _, t := range opt {
		if isDir && t.tag == file {
			continue
		}
		baseOnly := !isDir && t.tag == file
		if t.tag == dir && !isDir {
			s, err = transformDir(s, t)
		} else {
			s, err = transformPath(s, t, baseOnly)
		}
		if err != nil {
			err = fs.CountError(ctx, fserrors.NoRetryError(err))
			fs.Errorf(s, "Failed to transform: %v", err)
		}
	}
	if old != s {
		fs.Debugf(old, "transformed to: %v", s)
	}
	if strings.Count(old, "/") != strings.Count(s, "/") {
		err = fs.CountError(ctx, fserrors.NoRetryError(fmt.Errorf("number of path segments must match: %v (%v), %v (%v)", old, strings.Count(old, "/"), s, strings.Count(s, "/"))))
		fs.Errorf(old, "%v", err)
		return old
	}
	return s
}

// transformPath transforms a path string according to the chosen TransformAlgo.
// Each path segment is transformed separately, to preserve path separators.
// If baseOnly is true, only the base will be transformed (useful for renaming while walking a dir tree recursively.)
// for example, "some/nested/path" -> "some/nested/CONVERTEDPATH"
// otherwise, the entire is path is transformed.
func transformPath(s string, t transform, baseOnly bool) (string, error) {
	if s == "" || s == "/" || s == "\\" || s == "." {
		return "", nil
	}

	if baseOnly {
		transformedBase, err := transformPathSegment(path.Base(s), t)
		if err := validateSegment(transformedBase); err != nil {
			return "", err
		}
		return path.Join(path.Dir(s), transformedBase), err
	}

	segments := strings.Split(s, "/")
	transformedSegments := make([]string, len(segments))
	for _, seg := range segments {
		convSeg, err := transformPathSegment(seg, t)
		if err != nil {
			return "", err
		}
		if err := validateSegment(convSeg); err != nil {
			return "", err
		}
		transformedSegments = append(transformedSegments, convSeg)
	}
	return path.Join(transformedSegments...), nil
}

// transform all but the last path segment
func transformDir(s string, t transform) (string, error) {
	dirPath, err := transformPath(path.Dir(s), t, false)
	if err != nil {
		return "", err
	}
	return path.Join(dirPath, path.Base(s)), nil
}

// transformPathSegment transforms one path segment (or really any string) according to the chosen TransformAlgo.
// It assumes path separators have already been trimmed.
func transformPathSegment(s string, t transform) (string, error) {
	switch t.key {
	case ConvNone:
		return s, nil
	case ConvToNFC:
		return norm.NFC.String(s), nil
	case ConvToNFD:
		return norm.NFD.String(s), nil
	case ConvToNFKC:
		return norm.NFKC.String(s), nil
	case ConvToNFKD:
		return norm.NFKD.String(s), nil
	case ConvBase64Encode:
		return base64.URLEncoding.EncodeToString([]byte(s)), nil // URLEncoding to avoid slashes
	case ConvBase64Decode:
		if s == ".DS_Store" {
			return s, nil
		}
		b, err := base64.URLEncoding.DecodeString(s)
		if err != nil {
			fs.Errorf(s, "base64 error")
		}
		return string(b), err
	case ConvFindReplace:
		split := strings.Split(t.value, ":")
		if len(split) != 2 {
			return s, fmt.Errorf("wrong number of values: %v", t.value)
		}
		return strings.ReplaceAll(s, split[0], split[1]), nil
	case ConvPrefix:
		return t.value + s, nil
	case ConvSuffix:
		return s + t.value, nil
	case ConvSuffixKeepExtension:
		return SuffixKeepExtension(s, t.value), nil
	case ConvTrimPrefix:
		return strings.TrimPrefix(s, t.value), nil
	case ConvTrimSuffix:
		return strings.TrimSuffix(s, t.value), nil
	case ConvTruncate:
		max, err := strconv.Atoi(t.value)
		if err != nil {
			return s, err
		}
		return truncateChars(s, max, false), nil
	case ConvTruncateKeepExtension:
		max, err := strconv.Atoi(t.value)
		if err != nil {
			return s, err
		}
		return truncateChars(s, max, true), nil
	case ConvTruncateBytes:
		max, err := strconv.Atoi(t.value)
		if err != nil {
			return s, err
		}
		return truncateBytes(s, max, false)
	case ConvTruncateBytesKeepExtension:
		max, err := strconv.Atoi(t.value)
		if err != nil {
			return s, err
		}
		return truncateBytes(s, max, true)
	case ConvEncoder:
		var enc encoder.MultiEncoder
		err := enc.Set(t.value)
		if err != nil {
			return s, err
		}
		return enc.Encode(s), nil
	case ConvDecoder:
		var enc encoder.MultiEncoder
		err := enc.Set(t.value)
		if err != nil {
			return s, err
		}
		return enc.Decode(s), nil
	case ConvISO8859_1:
		return encodeWithReplacement(s, charmap.ISO8859_1), nil
	case ConvWindows1252:
		return encodeWithReplacement(s, charmap.Windows1252), nil
	case ConvMacintosh:
		return encodeWithReplacement(s, charmap.Macintosh), nil
	case ConvCharmap:
		var cmapType CharmapChoices
		err := cmapType.Set(t.value)
		if err != nil {
			return s, err
		}
		c := charmapByID(cmapType)
		return encodeWithReplacement(s, c), nil
	case ConvLowercase:
		return strings.ToLower(s), nil
	case ConvUppercase:
		return strings.ToUpper(s), nil
	case ConvTitlecase:
		return strings.ToTitle(s), nil
	case ConvASCII:
		return toASCII(s), nil
	case ConvURL:
		return url.QueryEscape(s), nil
	case ConvDate:
		return s + AppyTimeGlobs(t.value, time.Now()), nil
	case ConvRegex:
		split := strings.Split(t.value, "/")
		if len(split) != 2 {
			return s, fmt.Errorf("regex syntax error: %v", t.value)
		}
		re := regexp.MustCompile(split[0])
		return re.ReplaceAllString(s, split[1]), nil
	case ConvCommand:
		return mapper(s, t.value)
	default:
		return "", errors.New("this option is not yet implemented")
	}
}

// SuffixKeepExtension adds a suffix while keeping extension
//
// i.e. file.txt becomes file_somesuffix.txt not file.txt_somesuffix
func SuffixKeepExtension(remote string, suffix string) string {
	base, exts := splitExtension(remote)
	return base + suffix + exts
}

func splitExtension(remote string) (base, exts string) {
	base = remote
	var (
		first = true
		ext   = path.Ext(remote)
	)
	for ext != "" {
		// Look second and subsequent extensions in mime types.
		// If they aren't found then don't keep it as an extension.
		if !first && mime.TypeByExtension(ext) == "" {
			break
		}
		base = base[:len(base)-len(ext)]
		exts = ext + exts
		first = false
		ext = path.Ext(base)
	}
	return base, exts
}

func truncateChars(s string, max int, keepExtension bool) string {
	if max <= 0 {
		return s
	}
	if utf8.RuneCountInString(s) <= max {
		return s
	}
	exts := ""
	if keepExtension {
		s, exts = splitExtension(s)
	}
	runes := []rune(s)
	return string(runes[:max-utf8.RuneCountInString(exts)]) + exts
}

// truncateBytes is like truncateChars but counts the number of bytes, not UTF-8 characters
func truncateBytes(s string, max int, keepExtension bool) (string, error) {
	if max <= 0 {
		return s, nil
	}
	if len(s) <= max {
		return s, nil
	}
	exts := ""
	if keepExtension {
		s, exts = splitExtension(s)
	}

	// ensure we don't split a multi-byte UTF-8 character
	for i := max - len(exts); i > 0; i-- {
		b := append([]byte(s)[:i], exts...)
		if len(b) <= max && utf8.Valid(b) {
			return string(b), nil
		}
	}
	return "", errors.New("could not truncate to valid UTF-8")
}

// forbid transformations that add/remove path separators
func validateSegment(s string) error {
	if strings.TrimSpace(s) == "" {
		return errors.New("transform cannot render path segments empty")
	}
	if strings.ContainsRune(s, '/') {
		return fmt.Errorf("transform cannot add path separators: %v", s)
	}
	return nil
}

// ParseGlobs determines whether a string contains {brackets}
// and returns the substring (including both brackets) for replacing
// substring is first opening bracket to last closing bracket --
// good for {{this}} but not {this}{this}
func ParseGlobs(s string) (hasGlobs bool, substring string) {
	open := strings.Index(s, "{")
	close := strings.LastIndex(s, "}")
	if open >= 0 && close > open {
		return true, s[open : close+1]
	}
	return false, ""
}

// TrimBrackets converts {{this}} to this
func TrimBrackets(s string) string {
	return strings.Trim(s, "{}")
}

// TimeFormat converts a user-supplied string to a Go time constant, if possible
func TimeFormat(timeFormat string) string {
	switch timeFormat {
	case "Layout":
		timeFormat = time.Layout
	case "ANSIC":
		timeFormat = time.ANSIC
	case "UnixDate":
		timeFormat = time.UnixDate
	case "RubyDate":
		timeFormat = time.RubyDate
	case "RFC822":
		timeFormat = time.RFC822
	case "RFC822Z":
		timeFormat = time.RFC822Z
	case "RFC850":
		timeFormat = time.RFC850
	case "RFC1123":
		timeFormat = time.RFC1123
	case "RFC1123Z":
		timeFormat = time.RFC1123Z
	case "RFC3339":
		timeFormat = time.RFC3339
	case "RFC3339Nano":
		timeFormat = time.RFC3339Nano
	case "Kitchen":
		timeFormat = time.Kitchen
	case "Stamp":
		timeFormat = time.Stamp
	case "StampMilli":
		timeFormat = time.StampMilli
	case "StampMicro":
		timeFormat = time.StampMicro
	case "StampNano":
		timeFormat = time.StampNano
	case "DateTime":
		timeFormat = time.DateTime
	case "DateOnly":
		timeFormat = time.DateOnly
	case "TimeOnly":
		timeFormat = time.TimeOnly
	case "MacFriendlyTime", "macfriendlytime", "mac":
		timeFormat = "2006-01-02 0304PM" // not actually a Go constant -- but useful as macOS filenames can't have colons
	case "YYYYMMDD":
		timeFormat = "20060102"
	}
	return timeFormat
}

// AppyTimeGlobs converts "myfile-{DateOnly}.txt" to "myfile-2006-01-02.txt"
func AppyTimeGlobs(s string, t time.Time) string {
	hasGlobs, substring := ParseGlobs(s)
	if !hasGlobs {
		return s
	}
	timeString := t.Local().Format(TimeFormat(TrimBrackets(substring)))
	return strings.ReplaceAll(s, substring, timeString)
}

func mapper(s string, command string) (string, error) {
	out, err := exec.Command(command, s).CombinedOutput()
	if err != nil {
		out = bytes.TrimSpace(out)
		return s, fmt.Errorf("%s: error running command %q: %v", out, command+" "+s, err)
	}
	return string(bytes.TrimSpace(out)), nil
}