From dddc52a668c7ac115af5b7fc3eb96cc9da3e4bb3 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Fri, 26 Apr 2024 14:52:10 +1200 Subject: [PATCH] Feature: Set tagging filters via a config file --- cmd/root.go | 12 +++-- config/config.go | 42 +++++++----------- config/tags.go | 81 ++++++++++++++++++++++++++++++++++ internal/storage/messages.go | 9 +++- internal/storage/tagfilters.go | 38 +++++++++++++--- internal/storage/tags.go | 6 +-- internal/tools/utils.go | 11 +++++ 7 files changed, 154 insertions(+), 45 deletions(-) create mode 100644 config/tags.go create mode 100644 internal/tools/utils.go diff --git a/cmd/root.go b/cmd/root.go index 07645f5..e934408 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -127,7 +127,8 @@ func init() { rootCmd.Flags().StringVar(&config.POP3TLSKey, "pop3-tls-key", config.POP3TLSKey, "Optional TLS key for POP3 server - requires pop3-tls-cert") // Tagging - rootCmd.Flags().StringVarP(&config.SMTPCLITags, "tag", "t", config.SMTPCLITags, "Tag new messages matching filters") + rootCmd.Flags().StringVarP(&config.CLITagsArg, "tag", "t", config.CLITagsArg, "Tag new messages matching filters") + rootCmd.Flags().StringVar(&config.TagsConfig, "tags-config", config.TagsConfig, "Load tags filters from yaml configuration file") rootCmd.Flags().BoolVar(&tools.TagsTitleCase, "tags-title-case", tools.TagsTitleCase, "TitleCase new tags generated from plus-addresses and X-Tags") // Webhook @@ -283,12 +284,9 @@ func initConfigFromEnv() { config.POP3TLSKey = os.Getenv("MP_POP3_TLS_KEY") // Tagging - if len(os.Getenv("MP_TAG")) > 0 { - config.SMTPCLITags = os.Getenv("MP_TAG") - } - if getEnabledFromEnv("MP_TAGS_TITLE_CASE") { - tools.TagsTitleCase = getEnabledFromEnv("MP_TAGS_TITLE_CASE") - } + config.CLITagsArg = os.Getenv("MP_TAG") + config.TagsConfig = os.Getenv("MP_TAGS_CONFIG") + tools.TagsTitleCase = getEnabledFromEnv("MP_TAGS_TITLE_CASE") // Webhook if len(os.Getenv("MP_WEBHOOK_URL")) > 0 { diff --git a/config/config.go b/config/config.go index 949e918..b0d888c 100644 --- a/config/config.go +++ b/config/config.go @@ -15,7 +15,6 @@ import ( "github.com/axllent/mailpit/internal/auth" "github.com/axllent/mailpit/internal/logger" "github.com/axllent/mailpit/internal/spamassassin" - "github.com/axllent/mailpit/internal/tools" "gopkg.in/yaml.v3" ) @@ -86,14 +85,17 @@ var ( // BlockRemoteCSSAndFonts used to disable remote CSS & fonts BlockRemoteCSSAndFonts = false - // SMTPCLITags is used to map the CLI args - SMTPCLITags string + // CLITagsArg is used to map the CLI args + CLITagsArg string // ValidTagRegexp represents a valid tag ValidTagRegexp = regexp.MustCompile(`^([a-zA-Z0-9\-\ \_\.]){1,}$`) - // SMTPTags are expressions to apply tags to new mail - SMTPTags []autoTag + // TagsConfig is a yaml file to pre-load tags + TagsConfig string + + // TagFilters are used to apply tags to new mail + TagFilters []autoTag // SMTPRelayConfigFile to parse a yaml file and store config of relay SMTP server SMTPRelayConfigFile string @@ -163,8 +165,8 @@ var ( // AutoTag struct for auto-tagging type autoTag struct { - Tag string Match string + Tags []string } // SMTPRelayConfigStruct struct for parsing yaml & storing variables @@ -381,27 +383,13 @@ func VerifyConfig() error { } } - SMTPTags = []autoTag{} - - if SMTPCLITags != "" { - args := tools.ArgsParser(SMTPCLITags) - - for _, a := range args { - t := strings.Split(a, "=") - if len(t) > 1 { - tag := tools.CleanTag(t[0]) - if !ValidTagRegexp.MatchString(tag) || len(tag) == 0 { - return fmt.Errorf("[tag] invalid tag (%s) - can only contain spaces, letters, numbers, - & _", tag) - } - match := strings.TrimSpace(strings.ToLower(strings.Join(t[1:], "="))) - if len(match) == 0 { - return fmt.Errorf("[tag] invalid tag match (%s) - no search detected", tag) - } - SMTPTags = append(SMTPTags, autoTag{Tag: tag, Match: match}) - } else { - return fmt.Errorf("[tag] error parsing tags (%s)", a) - } - } + // load tag filters + TagFilters = []autoTag{} + if err := loadTagsFromArgs(CLITagsArg); err != nil { + return err + } + if err := loadTagsFromConfig(TagsConfig); err != nil { + return err } if SMTPAllowedRecipients != "" { diff --git a/config/tags.go b/config/tags.go new file mode 100644 index 0000000..cb46f25 --- /dev/null +++ b/config/tags.go @@ -0,0 +1,81 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/axllent/mailpit/internal/logger" + "github.com/axllent/mailpit/internal/tools" + "gopkg.in/yaml.v3" +) + +type yamlTags struct { + Filters []yamlTag `yaml:"filters"` +} + +type yamlTag struct { + Match string `yaml:"match"` + Tags string `yaml:"tags"` +} + +// Load tags from a configuration from a file, if set +func loadTagsFromConfig(c string) error { + if c == "" { + return nil // not set, ignore + } + + c = filepath.Clean(c) + + if !isFile(c) { + return fmt.Errorf("[tags] configuration file not found or unreadable: %s", c) + } + + data, err := os.ReadFile(c) + if err != nil { + return fmt.Errorf("[tags] %s", err.Error()) + } + + conf := yamlTags{} + + if err := yaml.Unmarshal(data, &conf); err != nil { + return err + } + + if conf.Filters == nil { + return fmt.Errorf("[tags] missing tag: array in %s", c) + } + + for _, t := range conf.Filters { + tags := strings.Split(t.Tags, ",") + TagFilters = append(TagFilters, autoTag{Match: t.Match, Tags: tags}) + } + + logger.Log().Debugf("[tags] loaded %s from config %s", tools.Plural(len(conf.Filters), "tag filter", "tag filters"), c) + + return nil +} + +func loadTagsFromArgs(c string) error { + if c == "" { + return nil // not set, ignore + } + + args := tools.ArgsParser(c) + + for _, a := range args { + t := strings.Split(a, "=") + if len(t) > 1 { + match := strings.TrimSpace(strings.ToLower(strings.Join(t[1:], "="))) + tags := strings.Split(t[0], ",") + TagFilters = append(TagFilters, autoTag{Match: match, Tags: tags}) + } else { + return fmt.Errorf("[tag] error parsing tags (%s)", a) + } + } + + logger.Log().Debugf("[tags] loaded %s from CLI args", tools.Plural(len(args), "tag filter", "tag filters")) + + return nil +} diff --git a/internal/storage/messages.go b/internal/storage/messages.go index 500f44a..6382672 100644 --- a/internal/storage/messages.go +++ b/internal/storage/messages.go @@ -112,14 +112,19 @@ func Store(body *[]byte) (string, error) { return "", err } - // extract tags from body matches based on --tag, plus addresses & X-Tags header + // extract tags from body matches rawTags := findTagsInRawMessage(body) + // extract plus addresses tags from enmime.Envelope plusTags := obj.tagsFromPlusAddresses() + // extract tags from X-Tags header xTags := tools.SetTagCasing(strings.Split(strings.TrimSpace(env.Root.Header.Get("X-Tags")), ",")) - searchTags := TagFilterMatches(id) + // extract tags from search matches + searchTags := tagFilterMatches(id) + // combine all tags into one slice tags := append(rawTags, plusTags...) tags = append(tags, xTags...) + // sort and extract only unique tags tags = sortedUniqueTags(append(tags, searchTags...)) if len(tags) > 0 { diff --git a/internal/storage/tagfilters.go b/internal/storage/tagfilters.go index a15d928..2990235 100644 --- a/internal/storage/tagfilters.go +++ b/internal/storage/tagfilters.go @@ -3,17 +3,19 @@ package storage import ( "context" "database/sql" + "strings" "github.com/axllent/mailpit/config" "github.com/axllent/mailpit/internal/logger" + "github.com/axllent/mailpit/internal/tools" "github.com/leporo/sqlf" ) // TagFilter struct type TagFilter struct { - Search string - SQL *sqlf.Stmt - Tags []string + Match string + SQL *sqlf.Stmt + Tags []string } var tagFilters = []TagFilter{} @@ -22,13 +24,37 @@ var tagFilters = []TagFilter{} func LoadTagFilters() { tagFilters = []TagFilter{} - for _, t := range config.SMTPTags { - tagFilters = append(tagFilters, TagFilter{Search: t.Match, Tags: []string{t.Tag}, SQL: searchQueryBuilder(t.Match, "")}) + for _, t := range config.TagFilters { + match := strings.TrimSpace(t.Match) + if match == "" { + logger.Log().Warnf("[tags] ignoring tag item with missing 'match'") + continue + } + if t.Tags == nil || len(t.Tags) == 0 { + logger.Log().Warnf("[tags] ignoring tag items with missing 'tags' array") + continue + } + + validTags := []string{} + for _, tag := range t.Tags { + tagName := tools.CleanTag(tag) + if !config.ValidTagRegexp.MatchString(tagName) || len(tagName) == 0 { + logger.Log().Warnf("[tags] invalid tag (%s) - can only contain spaces, letters, numbers, - & _", tagName) + continue + } + validTags = append(validTags, tagName) + } + + if len(validTags) == 0 { + continue + } + + tagFilters = append(tagFilters, TagFilter{Match: match, Tags: validTags, SQL: searchQueryBuilder(match, "")}) } } // TagFilterMatches returns a slice of matching tags from a message -func TagFilterMatches(id string) []string { +func tagFilterMatches(id string) []string { tags := []string{} if len(tagFilters) == 0 { diff --git a/internal/storage/tags.go b/internal/storage/tags.go index 8aeec6f..c281300 100644 --- a/internal/storage/tags.go +++ b/internal/storage/tags.go @@ -220,14 +220,14 @@ func pruneUnusedTags() error { // Returns a comma-separated string. func findTagsInRawMessage(message *[]byte) []string { tags := []string{} - if len(config.SMTPTags) == 0 { + if len(tagFilters) == 0 { return tags } str := bytes.ToLower(*message) - for _, t := range config.SMTPTags { + for _, t := range tagFilters { if bytes.Contains(str, []byte(t.Match)) { - tags = append(tags, t.Tag) + tags = append(tags, t.Tags...) } } diff --git a/internal/tools/utils.go b/internal/tools/utils.go new file mode 100644 index 0000000..f152c6e --- /dev/null +++ b/internal/tools/utils.go @@ -0,0 +1,11 @@ +package tools + +import "fmt" + +// Plural returns a singular or plural of a word together with the total +func Plural(total int, singular, plural string) string { + if total == 1 { + return fmt.Sprintf("%d %s", total, singular) + } + return fmt.Sprintf("%d %s", total, plural) +}