Merge branch 'feature/tag-filters' into develop

cmd/root.go

@@ -127,8 +127,9 @@ func init() {
 	rootCmd.Flags().StringVar(&config.POP3TLSKey, "pop3-tls-key", config.POP3TLSKey, "Optional TLS key for POP3 server - requires pop3-tls-cert")
 
 	// Tagging
-	rootCmd.Flags().StringVarP(&config.SMTPCLITags, "tag", "t", config.SMTPCLITags, "Tag new messages matching filters")
-	rootCmd.Flags().BoolVar(&tools.TagsTitleCase, "tags-title-case", tools.TagsTitleCase, "Convert new tags automatically to TitleCase")
+	rootCmd.Flags().StringVarP(&config.CLITagsArg, "tag", "t", config.CLITagsArg, "Tag new messages matching filters")
+	rootCmd.Flags().StringVar(&config.TagsConfig, "tags-config", config.TagsConfig, "Load tags filters from yaml configuration file")
+	rootCmd.Flags().BoolVar(&tools.TagsTitleCase, "tags-title-case", tools.TagsTitleCase, "TitleCase new tags generated from plus-addresses and X-Tags")
 
 	// Webhook
 	rootCmd.Flags().StringVar(&config.WebhookURL, "webhook-url", config.WebhookURL, "Send a webhook request for new messages")
@@ -283,12 +284,9 @@ func initConfigFromEnv() {
 	config.POP3TLSKey = os.Getenv("MP_POP3_TLS_KEY")
 
 	// Tagging
-	if len(os.Getenv("MP_TAG")) > 0 {
-		config.SMTPCLITags = os.Getenv("MP_TAG")
-	}
-	if getEnabledFromEnv("MP_TAGS_TITLE_CASE") {
-		tools.TagsTitleCase = getEnabledFromEnv("MP_TAGS_TITLE_CASE")
-	}
+	config.CLITagsArg = os.Getenv("MP_TAG")
+	config.TagsConfig = os.Getenv("MP_TAGS_CONFIG")
+	tools.TagsTitleCase = getEnabledFromEnv("MP_TAGS_TITLE_CASE")
 
 	// Webhook
 	if len(os.Getenv("MP_WEBHOOK_URL")) > 0 {

config/config.go

@@ -15,7 +15,6 @@ import (
 	"github.com/axllent/mailpit/internal/auth"
 	"github.com/axllent/mailpit/internal/logger"
 	"github.com/axllent/mailpit/internal/spamassassin"
-	"github.com/axllent/mailpit/internal/tools"
 	"gopkg.in/yaml.v3"
 )
 
@@ -86,14 +85,17 @@ var (
 	// BlockRemoteCSSAndFonts used to disable remote CSS & fonts
 	BlockRemoteCSSAndFonts = false
 
-	// SMTPCLITags is used to map the CLI args
-	SMTPCLITags string
+	// CLITagsArg is used to map the CLI args
+	CLITagsArg string
 
 	// ValidTagRegexp represents a valid tag
 	ValidTagRegexp = regexp.MustCompile(`^([a-zA-Z0-9\-\ \_\.]){1,}$`)
 
-	// SMTPTags are expressions to apply tags to new mail
-	SMTPTags []AutoTag
+	// TagsConfig is a yaml file to pre-load tags
+	TagsConfig string
+
+	// TagFilters are used to apply tags to new mail
+	TagFilters []autoTag
 
 	// SMTPRelayConfigFile to parse a yaml file and store config of relay SMTP server
 	SMTPRelayConfigFile string
@@ -162,9 +164,9 @@ var (
 )
 
 // AutoTag struct for auto-tagging
-type AutoTag struct {
-	Tag   string
+type autoTag struct {
 	Match string
+	Tags  []string
 }
 
 // SMTPRelayConfigStruct struct for parsing yaml & storing variables
@@ -381,27 +383,13 @@ func VerifyConfig() error {
 		}
 	}
 
-	SMTPTags = []AutoTag{}
-
-	if SMTPCLITags != "" {
-		args := tools.ArgsParser(SMTPCLITags)
-
-		for _, a := range args {
-			t := strings.Split(a, "=")
-			if len(t) > 1 {
-				tag := tools.CleanTag(t[0])
-				if !ValidTagRegexp.MatchString(tag) || len(tag) == 0 {
-					return fmt.Errorf("[tag] invalid tag (%s) - can only contain spaces, letters, numbers, - & _", tag)
-				}
-				match := strings.TrimSpace(strings.ToLower(strings.Join(t[1:], "=")))
-				if len(match) == 0 {
-					return fmt.Errorf("[tag] invalid tag match (%s) - no search detected", tag)
-				}
-				SMTPTags = append(SMTPTags, AutoTag{Tag: tag, Match: match})
-			} else {
-				return fmt.Errorf("[tag] error parsing tags (%s)", a)
-			}
-		}
+	// load tag filters
+	TagFilters = []autoTag{}
+	if err := loadTagsFromArgs(CLITagsArg); err != nil {
+		return err
+	}
+	if err := loadTagsFromConfig(TagsConfig); err != nil {
+		return err
 	}
 
 	if SMTPAllowedRecipients != "" {

config/tags.go

@@ -0,0 +1,81 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"gopkg.in/yaml.v3"
+)
+
+type yamlTags struct {
+	Filters []yamlTag `yaml:"filters"`
+}
+
+type yamlTag struct {
+	Match string `yaml:"match"`
+	Tags  string `yaml:"tags"`
+}
+
+// Load tags from a configuration from a file, if set
+func loadTagsFromConfig(c string) error {
+	if c == "" {
+		return nil // not set, ignore
+	}
+
+	c = filepath.Clean(c)
+
+	if !isFile(c) {
+		return fmt.Errorf("[tags] configuration file not found or unreadable: %s", c)
+	}
+
+	data, err := os.ReadFile(c)
+	if err != nil {
+		return fmt.Errorf("[tags] %s", err.Error())
+	}
+
+	conf := yamlTags{}
+
+	if err := yaml.Unmarshal(data, &conf); err != nil {
+		return err
+	}
+
+	if conf.Filters == nil {
+		return fmt.Errorf("[tags] missing tag: array in %s", c)
+	}
+
+	for _, t := range conf.Filters {
+		tags := strings.Split(t.Tags, ",")
+		TagFilters = append(TagFilters, autoTag{Match: t.Match, Tags: tags})
+	}
+
+	logger.Log().Debugf("[tags] loaded %s from config %s", tools.Plural(len(conf.Filters), "tag filter", "tag filters"), c)
+
+	return nil
+}
+
+func loadTagsFromArgs(c string) error {
+	if c == "" {
+		return nil // not set, ignore
+	}
+
+	args := tools.ArgsParser(c)
+
+	for _, a := range args {
+		t := strings.Split(a, "=")
+		if len(t) > 1 {
+			match := strings.TrimSpace(strings.ToLower(strings.Join(t[1:], "=")))
+			tags := strings.Split(t[0], ",")
+			TagFilters = append(TagFilters, autoTag{Match: match, Tags: tags})
+		} else {
+			return fmt.Errorf("[tag] error parsing tags (%s)", a)
+		}
+	}
+
+	logger.Log().Debugf("[tags] loaded %s from CLI args", tools.Plural(len(args), "tag filter", "tag filters"))
+
+	return nil
+}

internal/storage/database.go

@@ -108,6 +108,8 @@ func InitDB() error {
 		return err
 	}
 
+	LoadTagFilters()
+
 	dbFile = p
 	dbLastAction = time.Now()
 

internal/storage/messages.go

@@ -71,13 +71,6 @@ func Store(body *[]byte) (string, error) {
 		return "", err
 	}
 
-	// extract tags from body matches based on --tag, plus addresses & X-Tags header
-	tagStr := findTagsInRawMessage(body) + "," +
-		obj.tagsFromPlusAddresses() + "," +
-		strings.TrimSpace(env.Root.Header.Get("X-Tags"))
-
-	tagData := uniqueTagsFromString(tagStr)
-
 	// begin a transaction to ensure both the message
 	// and data are stored successfully
 	ctx := context.Background()
@@ -119,9 +112,23 @@ func Store(body *[]byte) (string, error) {
 		return "", err
 	}
 
-	if len(tagData) > 0 {
-		// set tags after tx.Commit()
-		if err := SetMessageTags(id, tagData); err != nil {
+	// extract tags from body matches
+	rawTags := findTagsInRawMessage(body)
+	// extract plus addresses tags from enmime.Envelope
+	plusTags := obj.tagsFromPlusAddresses()
+	// extract tags from X-Tags header
+	xTags := tools.SetTagCasing(strings.Split(strings.TrimSpace(env.Root.Header.Get("X-Tags")), ","))
+	// extract tags from search matches
+	searchTags := tagFilterMatches(id)
+
+	// combine all tags into one slice
+	tags := append(rawTags, plusTags...)
+	tags = append(tags, xTags...)
+	// sort and extract only unique tags
+	tags = sortedUniqueTags(append(tags, searchTags...))
+
+	if len(tags) > 0 {
+		if err := SetMessageTags(id, tags); err != nil {
 			return "", err
 		}
 	}
@@ -137,7 +144,7 @@ func Store(body *[]byte) (string, error) {
 	c.Attachments = attachments
 	c.Subject = subject
 	c.Size = size
-	c.Tags = tagData
+	c.Tags = tags
 	c.Snippet = snippet
 
 	websockets.Broadcast("new", c)

internal/storage/search.go

@@ -294,6 +294,16 @@ func searchQueryBuilder(searchString, timezone string) *sqlf.Stmt {
 					q.Where("ReplyToJSON LIKE ?", "%"+escPercentChar(w)+"%")
 				}
 			}
+		} else if strings.HasPrefix(lw, "addressed:") {
+			w = cleanString(w[10:])
+			arg := "%" + escPercentChar(w) + "%"
+			if w != "" {
+				if exclude {
+					q.Where("(ToJSON NOT LIKE ? AND FromJSON NOT LIKE ? AND CcJSON NOT LIKE ? AND BccJSON NOT LIKE ? AND ReplyToJSON NOT LIKE ?)", arg, arg, arg, arg, arg)
+				} else {
+					q.Where("(ToJSON LIKE ? OR FromJSON LIKE ? OR CcJSON LIKE ? OR BccJSON LIKE ? OR ReplyToJSON LIKE ?)", arg, arg, arg, arg, arg)
+				}
+			}
 		} else if strings.HasPrefix(lw, "subject:") {
 			w = w[8:]
 			if w != "" {

internal/storage/tagfilters.go

@@ -0,0 +1,84 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/leporo/sqlf"
+)
+
+// TagFilter struct
+type TagFilter struct {
+	Match string
+	SQL   *sqlf.Stmt
+	Tags  []string
+}
+
+var tagFilters = []TagFilter{}
+
+// LoadTagFilters loads tag filters from the config and pre-generates the SQL query
+func LoadTagFilters() {
+	tagFilters = []TagFilter{}
+
+	for _, t := range config.TagFilters {
+		match := strings.TrimSpace(t.Match)
+		if match == "" {
+			logger.Log().Warnf("[tags] ignoring tag item with missing 'match'")
+			continue
+		}
+		if t.Tags == nil || len(t.Tags) == 0 {
+			logger.Log().Warnf("[tags] ignoring tag items with missing 'tags' array")
+			continue
+		}
+
+		validTags := []string{}
+		for _, tag := range t.Tags {
+			tagName := tools.CleanTag(tag)
+			if !config.ValidTagRegexp.MatchString(tagName) || len(tagName) == 0 {
+				logger.Log().Warnf("[tags] invalid tag (%s) - can only contain spaces, letters, numbers, - & _", tagName)
+				continue
+			}
+			validTags = append(validTags, tagName)
+		}
+
+		if len(validTags) == 0 {
+			continue
+		}
+
+		tagFilters = append(tagFilters, TagFilter{Match: match, Tags: validTags, SQL: searchQueryBuilder(match, "")})
+	}
+}
+
+// TagFilterMatches returns a slice of matching tags from a message
+func tagFilterMatches(id string) []string {
+	tags := []string{}
+
+	if len(tagFilters) == 0 {
+		return tags
+	}
+
+	for _, f := range tagFilters {
+		var matchID string
+		q := f.SQL.Clone().Where("ID = ?", id)
+		if err := q.QueryAndClose(context.Background(), db, func(row *sql.Rows) {
+			var ignore sql.NullString
+
+			if err := row.Scan(&ignore, &matchID, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore); err != nil {
+				logger.Log().Errorf("[db] %s", err.Error())
+				return
+			}
+		}); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			return tags
+		}
+		if matchID == id {
+			tags = append(tags, f.Tags...)
+		}
+	}
+
+	return tags
+}

internal/storage/tags.go

@@ -1,6 +1,7 @@
 package storage
 
 import (
+	"bytes"
 	"context"
 	"database/sql"
 	"regexp"
@@ -19,7 +20,7 @@ var (
 	addTagMutex   sync.RWMutex
 )
 
-// SetMessageTags will set the tags for a given database ID
+// SetMessageTags will set the tags for a given database ID, removing any not in the array
 func SetMessageTags(id string, tags []string) error {
 	applyTags := []string{}
 	for _, t := range tags {
@@ -33,7 +34,6 @@ func SetMessageTags(id string, tags []string) error {
 	origTagCount := len(currentTags)
 
 	for _, t := range applyTags {
-		t = tools.CleanTag(t)
 		if t == "" || !config.ValidTagRegexp.MatchString(t) || inArray(t, currentTags) {
 			continue
 		}
@@ -74,14 +74,15 @@ func AddMessageTag(id, name string) error {
 		addTagMutex.Unlock()
 		// check message does not already have this tag
 		var count int
-		if _, err := sqlf.From(tenant("message_tags")).
+
+		if err := sqlf.From(tenant("message_tags")).
 			Select("COUNT(ID)").To(&count).
 			Where("ID = ?", id).
 			Where("TagID = ?", tagID).
-			ExecAndClose(context.TODO(), db); err != nil {
+			QueryRowAndClose(context.Background(), db); err != nil {
 			return err
 		}
-		if count != 0 {
+		if count > 0 {
 			// already exists
 			return nil
 		}
@@ -213,26 +214,28 @@ func pruneUnusedTags() error {
 	return nil
 }
 
-// Find tags set via --tags in raw message.
+// Find tags set via --tags in raw message, useful for matching all headers etc.
+// This function is largely superseded by the database searching, however this
+// includes literally everything and is kept for backwards compatibility.
 // Returns a comma-separated string.
-func findTagsInRawMessage(message *[]byte) string {
-	tagStr := ""
-	if len(config.SMTPTags) == 0 {
-		return tagStr
+func findTagsInRawMessage(message *[]byte) []string {
+	tags := []string{}
+	if len(tagFilters) == 0 {
+		return tags
 	}
 
-	str := strings.ToLower(string(*message))
-	for _, t := range config.SMTPTags {
-		if strings.Contains(str, t.Match) {
-			tagStr += "," + t.Tag
+	str := bytes.ToLower(*message)
+	for _, t := range tagFilters {
+		if bytes.Contains(str, []byte(t.Match)) {
+			tags = append(tags, t.Tags...)
 		}
 	}
 
-	return tagStr
+	return tags
 }
 
 // Returns tags found in email plus addresses (eg: test+tagname@example.com)
-func (d DBMailSummary) tagsFromPlusAddresses() string {
+func (d DBMailSummary) tagsFromPlusAddresses() []string {
 	tags := []string{}
 	for _, c := range d.To {
 		matches := addressPlusRe.FindAllStringSubmatch(c.Address, 1)
@@ -257,7 +260,7 @@ func (d DBMailSummary) tagsFromPlusAddresses() string {
 		tags = append(tags, strings.Split(matches[0][2], "+")...)
 	}
 
-	return strings.Join(tags, ",")
+	return tools.SetTagCasing(tags)
 }
 
 // Get message tags from the database for a given database ID
@@ -282,24 +285,27 @@ func getMessageTags(id string) []string {
 	return tags
 }
 
-// UniqueTagsFromString will split a string with commas, and extract a unique slice of formatted tags
-func uniqueTagsFromString(s string) []string {
+// SortedUniqueTags will return a unique slice of normalised tags
+func sortedUniqueTags(s []string) []string {
 	tags := []string{}
+	added := make(map[string]bool)
 
-	if s == "" {
+	if len(s) == 0 {
 		return tags
 	}
 
-	parts := strings.Split(s, ",")
-	for _, p := range parts {
+	for _, p := range s {
 		w := tools.CleanTag(p)
 		if w == "" {
 			continue
 		}
+		lc := strings.ToLower(w)
+		if _, exists := added[lc]; exists {
+			continue
+		}
 		if config.ValidTagRegexp.MatchString(w) {
-			if !inArray(w, tags) {
-				tags = append(tags, w)
-			}
+			added[lc] = true
+			tags = append(tags, w)
 		} else {
 			logger.Log().Debugf("[tags] ignoring invalid tag: %s", w)
 		}

internal/tools/tags.go

@@ -19,18 +19,29 @@ var (
 	TagsTitleCase bool
 )
 
-// CleanTag returns a clean tag, removing whitespace and invalid characters
+// CleanTag returns a clean tag, trimming whitespace and replacing invalid characters
 func CleanTag(s string) string {
-	s = strings.TrimSpace(
+	return strings.TrimSpace(
 		multiSpaceRe.ReplaceAllString(
 			tagsInvalidChars.ReplaceAllString(s, " "),
 			" ",
 		),
 	)
+}
 
-	if TagsTitleCase {
-		return cases.Title(language.Und, cases.NoLower).String(s)
+// SetTagCasing returns the slice of tags, title-casing if set
+func SetTagCasing(s []string) []string {
+	if !TagsTitleCase {
+		return s
 	}
 
-	return s
+	titleTags := []string{}
+
+	c := cases.Title(language.Und, cases.NoLower)
+
+	for _, t := range s {
+		titleTags = append(titleTags, c.String(t))
+	}
+
+	return titleTags
 }

internal/tools/utils.go

@@ -0,0 +1,11 @@
+package tools
+
+import "fmt"
+
+// Plural returns a singular or plural of a word together with the total
+func Plural(total int, singular, plural string) string {
+	if total == 1 {
+		return fmt.Sprintf("%d %s", total, singular)
+	}
+	return fmt.Sprintf("%d %s", total, plural)
+}

server/ui-src/components/Notifications.vue

@@ -64,9 +64,11 @@ export default {
 					}
 
 					for (let i in response.Data.Tags) {
-						if (mailbox.tags.indexOf(response.Data.Tags[i]) < 0) {
+						if (mailbox.tags.findIndex(e => { return e.toLowerCase() === response.Data.Tags[i].toLowerCase() }) < 0) {
 							mailbox.tags.push(response.Data.Tags[i])
-							mailbox.tags.sort()
+							mailbox.tags.sort((a, b) => {
+								return a.toLowerCase().localeCompare(b.toLowerCase())
+							})
 						}
 					}