API: Message relay / release

This enables a SMTP server to be configured, and messages to be manually "released" via the relay server. Aditionally, messages can be auto-relayed via the SMTP server do Mailpit acts as a form of caching proxy.

@see #29

cmd/root.go

@@ -98,6 +98,9 @@ func init() {
 	rootCmd.Flags().BoolVar(&config.SMTPAuthAllowInsecure, "smtp-auth-allow-insecure", config.SMTPAuthAllowInsecure, "Enable insecure PLAIN & LOGIN authentication")
 	rootCmd.Flags().StringVarP(&config.SMTPCLITags, "tag", "t", config.SMTPCLITags, "Tag new messages matching filters")
 
+	rootCmd.Flags().StringVar(&config.SMTPRelayConfigFile, "smtp-relay-config", config.SMTPRelayConfigFile, "SMTP configuration file to allow releasing messages")
+	rootCmd.Flags().BoolVar(&config.SMTPRelayAllIncoming, "smtp-relay-all", config.SMTPRelayAllIncoming, "Relay all incoming messages via external SMTP server (caution!)")
+
 	rootCmd.Flags().BoolVarP(&logger.QuietLogging, "quiet", "q", logger.QuietLogging, "Quiet logging (errors only)")
 	rootCmd.Flags().BoolVarP(&logger.VerboseLogging, "verbose", "v", logger.VerboseLogging, "Verbose logging")
 
@@ -179,6 +182,14 @@ func initConfigFromEnv() {
 		config.SMTPAuthAllowInsecure = true
 	}
 
+	// Relay server config
+	if len(os.Getenv("MP_SMTP_RELAY_CONFIG")) > 0 {
+		config.SMTPRelayConfigFile = os.Getenv("MP_SMTP_RELAY_CONFIG")
+	}
+	if getEnabledFromEnv("MP_SMTP_RELAY_ALL") {
+		config.SMTPRelayAllIncoming = true
+	}
+
 	if len(os.Getenv("MP_WEBROOT")) > 0 {
 		config.Webroot = os.Getenv("MP_WEBROOT")
 	}
@@ -189,10 +200,10 @@ func initConfigFromEnv() {
 		config.UseMessageDates = true
 	}
 	if getEnabledFromEnv("MP_QUIET") {
-		config.QuietLogging = true
+		logger.QuietLogging = true
 	}
 	if getEnabledFromEnv("MP_VERBOSE") {
-		config.VerboseLogging = true
+		logger.VerboseLogging = true
 	}
 }
 

config/config.go

@@ -10,8 +10,10 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/axllent/mailpit/utils/logger"
 	"github.com/mattn/go-shellwords"
 	"github.com/tg123/go-htpasswd"
+	"gopkg.in/yaml.v3"
 )
 
 var (
@@ -54,8 +56,8 @@ var (
 	// SMTPAuthFile for SMTP authentication
 	SMTPAuthFile string
 
-	// SMTPAuth used for euthentication
+	// SMTPAuthConfig used for authentication auto-generated from SMTPAuthFile
-	SMTPAuth *htpasswd.File
+	SMTPAuthConfig *htpasswd.File
 
 	// SMTPAuthAllowInsecure allows PLAIN & LOGIN unencrypted authentication
 	SMTPAuthAllowInsecure bool
@@ -70,7 +72,20 @@ var (
 	TagRegexp = regexp.MustCompile(`^([a-zA-Z0-9\-\ \_]){3,}$`)
 
 	// SMTPTags are expressions to apply tags to new mail
-	SMTPTags []Tag
+	SMTPTags []AutoTag
+
+	// SMTPRelayConfigFile to parse a yaml file and store config of relay SMTP server
+	SMTPRelayConfigFile string
+
+	// SMTPRelayConfig to parse a yaml file and store config of relay SMTP server
+	SMTPRelayConfig smtpRelayConfigStruct
+
+	// ReleaseEnabled is whether message releases are enabled, requires a valid SMTPRelayConfigFile
+	ReleaseEnabled = false
+
+	// SMTPRelayAllIncoming is whether to relay all incoming messages via preconfgured SMTP server.
+	// Use with extreme caution!
+	SMTPRelayAllIncoming = false
 
 	// ContentSecurityPolicy for HTTP server
 	ContentSecurityPolicy = "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; frame-src 'self'; img-src * data: blob:; font-src 'self' data:; media-src 'self'; connect-src 'self' ws: wss:; object-src 'none'; base-uri 'self';"
@@ -85,12 +100,25 @@ var (
 	RepoBinaryName = "mailpit"
 )
 
-// Tag struct
+// AutoTag struct for auto-tagging
-type Tag struct {
+type AutoTag struct {
 	Tag   string
 	Match string
 }
 
+// SMTPRelayConfigStruct struct for parsing yaml & storing variables
+type smtpRelayConfigStruct struct {
+	Host          string `yaml:"host"`
+	Port          int    `yaml:"port"`
+	STARTTLS      bool   `yaml:"starttls"`
+	AllowInsecure bool   `yaml:"allow-insecure"`
+	Auth          string `yaml:"auth"`        // none, plain, cram-md5
+	Username      string `yaml:"username"`    // plain & cram-md5
+	Password      string `yaml:"password"`    // plain
+	Secret        string `yaml:"secret"`      // cram-md5
+	ReturnPath    string `yaml:"return-path"` // allows overriding the boune address
+}
+
 // VerifyConfig wil do some basic checking
 func VerifyConfig() error {
 	if DataFile != "" && isDir(DataFile) {
@@ -158,7 +186,7 @@ func VerifyConfig() error {
 		if err != nil {
 			return err
 		}
-		SMTPAuth = a
+		SMTPAuthConfig = a
 	}
 
 	if SMTPTLSCert == "" && (SMTPAuthFile != "" || SMTPAuthAcceptAny) && !SMTPAuthAllowInsecure {
@@ -173,7 +201,7 @@ func VerifyConfig() error {
 	s := strings.TrimRight(path.Join("/", Webroot, "/"), "/") + "/"
 	Webroot = s
 
-	SMTPTags = []Tag{}
+	SMTPTags = []AutoTag{}
 
 	p := shellwords.NewParser()
 
@@ -194,14 +222,77 @@ func VerifyConfig() error {
 				if len(match) == 0 {
 					return fmt.Errorf("Invalid tag match (%s) - no search detected", tag)
 				}
-				SMTPTags = append(SMTPTags, Tag{Tag: tag, Match: match})
+				SMTPTags = append(SMTPTags, AutoTag{Tag: tag, Match: match})
 			} else {
 				return fmt.Errorf("Error parsing tags (%s)", a)
 			}
 		}
-
 	}
 
+	if err := parseRelayConfig(SMTPRelayConfigFile); err != nil {
+		return err
+	}
+
+	if !ReleaseEnabled && SMTPRelayAllIncoming {
+		return errors.New("SMTP relay config must be set to relay all messages")
+	}
+
+	if SMTPRelayAllIncoming {
+		// this deserves a warning
+		logger.Log().Warnf("[smtp] enabling automatic relay of all new messages via %s:%d", SMTPRelayConfig.Host, SMTPRelayConfig.Port)
+	}
+
+	return nil
+}
+
+// Parse & validate the SMTPRelayConfigFile (if set)
+func parseRelayConfig(c string) error {
+	if c == "" {
+		return nil
+	}
+
+	if !isFile(c) {
+		return fmt.Errorf("SMTP relay configuration not found: %s", SMTPRelayConfigFile)
+	}
+
+	data, err := os.ReadFile(c)
+	if err != nil {
+		return err
+	}
+
+	if err := yaml.Unmarshal(data, &SMTPRelayConfig); err != nil {
+		return err
+	}
+
+	if SMTPRelayConfig.Host == "" {
+		return errors.New("SMTP relay host not set")
+	}
+
+	if SMTPRelayConfig.Port == 0 {
+		SMTPRelayConfig.Port = 25 // default
+	}
+
+	SMTPRelayConfig.Auth = strings.ToLower(SMTPRelayConfig.Auth)
+
+	if SMTPRelayConfig.Auth == "" || SMTPRelayConfig.Auth == "none" || SMTPRelayConfig.Auth == "false" {
+		SMTPRelayConfig.Auth = "none"
+	} else if SMTPRelayConfig.Auth == "plain" {
+		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Password == "" {
+			return fmt.Errorf("SMTP relay host username or password not set for PLAIN authentication (%s)", c)
+		}
+	} else if strings.HasPrefix(SMTPRelayConfig.Auth, "cram") {
+		SMTPRelayConfig.Auth = "cram-md5"
+		if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Secret == "" {
+			return fmt.Errorf("SMTP relay host username or secret not set for CRAM-MD5 authentication (%s)", c)
+		}
+	} else {
+		return fmt.Errorf("SMTP relay authentication method not supported: %s", SMTPRelayConfig.Auth)
+	}
+
+	ReleaseEnabled = true
+
+	logger.Log().Infof("[smtp] enabling message relaying via %s:%d", SMTPRelayConfig.Host, SMTPRelayConfig.Port)
+
 	return nil
 }
 

server/apiv1/api.go

@@ -10,8 +10,12 @@ import (
 	"strings"
 
 	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/server/smtpd"
 	"github.com/axllent/mailpit/storage"
+	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/utils/tools"
 	"github.com/gorilla/mux"
+	uuid "github.com/satori/go.uuid"
 )
 
 // GetMessages returns a paginated list of messages as JSON
@@ -491,6 +495,121 @@ func SetTags(w http.ResponseWriter, r *http.Request) {
 	_, _ = w.Write([]byte("ok"))
 }
 
+// ReleaseMessage (method: POST) will release a message via a preconfigured external SMTP server.
+// If no IDs are provided then all messages are updated.
+func ReleaseMessage(w http.ResponseWriter, r *http.Request) {
+	// swagger:route POST /api/v1/message/{ID}/release message Release
+	//
+	// # Release message
+	//
+	// Release a message via a preconfigured external SMTP server..
+	//
+	//	Consumes:
+	//	- application/json
+	//
+	//	Produces:
+	//	- text/plain
+	//
+	//	Schemes: http, https
+	//
+	//	Parameters:
+	//	  + name: ID
+	//	    in: path
+	//	    description: message id
+	//	    required: true
+	//	    type: string
+	//		+ name: To
+	//	    in: body
+	//	    description: Array of email addresses to release message to
+	//	    required: true
+	//	    type: ReleaseMessageRequest
+	//
+	//	Responses:
+	//		200: OKResponse
+	//		default: ErrorResponse
+
+	vars := mux.Vars(r)
+
+	id := vars["id"]
+
+	msg, err := storage.GetMessageRaw(id)
+	if err != nil {
+		fourOFour(w)
+		return
+	}
+
+	decoder := json.NewDecoder(r.Body)
+
+	data := releaseMessageRequest{}
+
+	if err := decoder.Decode(&data); err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	tos := data.To
+	if len(tos) == 0 {
+		httpError(w, "No valid addresses found")
+		return
+	}
+
+	for _, to := range tos {
+		if _, err := mail.ParseAddress(to); err != nil {
+			httpError(w, "Invalid email address: "+to)
+			return
+		}
+	}
+
+	reader := bytes.NewReader(msg)
+	m, err := mail.ReadMessage(reader)
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	froms, err := m.Header.AddressList("From")
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	from := froms[0].Address
+
+	// if sender is used, then change from to the sender
+	if senders, err := m.Header.AddressList("Sender"); err == nil {
+		from = senders[0].Address
+	}
+
+	msg, err = tools.RemoveMessageHeaders(msg, []string{"Bcc", "Message-Id"})
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
+	if config.SMTPRelayConfig.ReturnPath != "" && m.Header.Get("Return-Path") != "<"+config.SMTPRelayConfig.ReturnPath+">" {
+		msg, err = tools.RemoveMessageHeaders(msg, []string{"Return-Path"})
+		if err != nil {
+			httpError(w, err.Error())
+			return
+		}
+		msg = append([]byte("Return-Path: <"+config.SMTPRelayConfig.ReturnPath+">\r\n"), msg...)
+	}
+
+	// generate unique ID
+	uid := uuid.NewV4().String() + "@mailpit"
+	// add unique ID
+	msg = append([]byte("Message-Id: <"+uid+">\r\n"), msg...)
+
+	if err := smtpd.Send(from, tos, msg); err != nil {
+		logger.Log().Errorf("[smtp] error sending message: %s", err.Error())
+		httpError(w, "SMTP error: "+err.Error())
+		return
+	}
+
+	w.Header().Add("Content-Type", "text/plain")
+	_, _ = w.Write([]byte("ok"))
+}
+
 // FourOFour returns a basic 404 message
 func fourOFour(w http.ResponseWriter) {
 	w.Header().Set("Referrer-Policy", "no-referrer")

server/apiv1/swagger.go

@@ -47,11 +47,19 @@ type setTagsRequest struct {
 	// in:body
 	Tags []string `json:"tags"`
 
-	// ids
+	// IDs
 	// in:body
 	IDs []string `json:"ids"`
 }
 
+// Release request
+// swagger:model ReleaseMessageRequest
+type releaseMessageRequest struct {
+	// To
+	// in:body
+	To []string `json:"to"`
+}
+
 // Binary data reponse inherits the attachment's content type
 // swagger:response BinaryResponse
 type binaryResponse struct {

server/server.go

@@ -1,3 +1,4 @@
+// Package server is the HTTP daemon
 package server
 
 import (

server/smtpd/smtp.go

@@ -0,0 +1,71 @@
+package smtpd
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/smtp"
+
+	"github.com/axllent/mailpit/config"
+)
+
+// Send will connect to a pre-configured SMTP server and send a message to one or more recipients.
+func Send(from string, to []string, msg []byte) error {
+	addr := fmt.Sprintf("%s:%d", config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port)
+
+	c, err := smtp.Dial(addr)
+	if err != nil {
+		return err
+	}
+
+	defer c.Close()
+
+	if config.SMTPRelayConfig.STARTTLS {
+		conf := &tls.Config{ServerName: config.SMTPRelayConfig.Host}
+
+		conf.InsecureSkipVerify = config.SMTPRelayConfig.AllowInsecure
+
+		if err = c.StartTLS(conf); err != nil {
+			return err
+		}
+	}
+
+	var a smtp.Auth
+
+	if config.SMTPRelayConfig.Auth == "plain" {
+		a = smtp.PlainAuth("", config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Password, config.SMTPRelayConfig.Host)
+	}
+
+	if config.SMTPRelayConfig.Auth == "cram-md5" {
+		a = smtp.CRAMMD5Auth(config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Secret)
+	}
+
+	if a != nil {
+		if err = c.Auth(a); err != nil {
+			return err
+		}
+	}
+	if err = c.Mail(from); err != nil {
+		return err
+	}
+
+	for _, addr := range to {
+		if err = c.Rcpt(addr); err != nil {
+			return err
+		}
+	}
+
+	w, err := c.Data()
+	if err != nil {
+		return err
+	}
+
+	if _, err := w.Write(msg); err != nil {
+		return err
+	}
+
+	if err := w.Close(); err != nil {
+		return err
+	}
+
+	return c.Quit()
+}

server/smtpd/smtpd.go

@@ -13,16 +13,34 @@ import (
 	"github.com/axllent/mailpit/storage"
 	"github.com/axllent/mailpit/utils/logger"
 	"github.com/mhale/smtpd"
+	uuid "github.com/satori/go.uuid"
 )
 
 func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
 	msg, err := mail.ReadMessage(bytes.NewReader(data))
 	if err != nil {
-		logger.Log().Errorf("[smtp] error parsing message: %s", err.Error())
+		logger.Log().Errorf("[smtpd] error parsing message: %s", err.Error())
 
 		return err
 	}
 
+	// add a message ID if not set
+	if msg.Header.Get("Message-Id") == "" {
+		// generate unique ID
+		uid := uuid.NewV4().String() + "@mailpit"
+		// add unique ID
+		data = append([]byte("Message-Id: <"+uid+">\r\n"), data...)
+	}
+
+	// if enabled, this will route the email 1:1 through to the preconfigured smtp server
+	if config.SMTPRelayAllIncoming {
+		if err := Send(from, to, data); err != nil {
+			logger.Log().Errorf("[smtp] error relaying message: %s", err.Error())
+		} else {
+			logger.Log().Debugf("[smtp] relayed message from %s via %s:%d", from, config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port)
+		}
+	}
+
 	// build array of all addresses in the header to compare to the []to array
 	emails, hasBccHeader := scanAddressesInHeader(msg.Header)
 
@@ -35,7 +53,7 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
 				missingAddresses = append(missingAddresses, a)
 			}
 		} else {
-			logger.Log().Warnf("[smtp] ignoring invalid email address: %s", a)
+			logger.Log().Warnf("[smtpd] ignoring invalid email address: %s", a)
 		}
 	}
 
@@ -60,7 +78,7 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
 			data = append(bcc, data...)
 		}
 
-		logger.Log().Debugf("[smtp] added missing addresses to Bcc header: %s", strings.Join(missingAddresses, ", "))
+		logger.Log().Debugf("[smtpd] added missing addresses to Bcc header: %s", strings.Join(missingAddresses, ", "))
 	}
 
 	if _, err := storage.Store(data); err != nil {
@@ -70,17 +88,17 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
 	}
 
 	subject := msg.Header.Get("Subject")
-	logger.Log().Debugf("[smtp] received (%s) from:%s to:%s subject:%q", cleanIP(origin), from, to[0], subject)
+	logger.Log().Debugf("[smtpd] received (%s) from:%s subject:%q", cleanIP(origin), from, subject)
 
 	return nil
 }
 
 func authHandler(remoteAddr net.Addr, mechanism string, username []byte, password []byte, shared []byte) (bool, error) {
-	allow := config.SMTPAuth.Match(string(username), string(password))
+	allow := config.SMTPAuthConfig.Match(string(username), string(password))
 	if allow {
-		logger.Log().Debugf("[smtp] allow %s login:%q from:%s", mechanism, string(username), cleanIP(remoteAddr))
+		logger.Log().Debugf("[smtpd] allow %s login:%q from:%s", mechanism, string(username), cleanIP(remoteAddr))
 	} else {
-		logger.Log().Warnf("[smtp] deny %s login:%q from:%s", mechanism, string(username), cleanIP(remoteAddr))
+		logger.Log().Warnf("[smtpd] deny %s login:%q from:%s", mechanism, string(username), cleanIP(remoteAddr))
 	}
 
 	return allow, nil
@@ -88,7 +106,7 @@ func authHandler(remoteAddr net.Addr, mechanism string, username []byte, passwor
 
 // Allow any username and password
 func authHandlerAny(remoteAddr net.Addr, mechanism string, username []byte, password []byte, shared []byte) (bool, error) {
-	logger.Log().Debugf("[smtp] allow %s login %q from %s", mechanism, string(username), cleanIP(remoteAddr))
+	logger.Log().Debugf("[smtpd] allow %s login %q from %s", mechanism, string(username), cleanIP(remoteAddr))
 
 	return true, nil
 }
@@ -97,19 +115,19 @@ func authHandlerAny(remoteAddr net.Addr, mechanism string, username []byte, pass
 func Listen() error {
 	if config.SMTPAuthAllowInsecure {
 		if config.SMTPAuthFile != "" {
-			logger.Log().Infof("[smtp] enabling login auth via %s (insecure)", config.SMTPAuthFile)
+			logger.Log().Infof("[smtpd] enabling login auth via %s (insecure)", config.SMTPAuthFile)
 		} else if config.SMTPAuthAcceptAny {
-			logger.Log().Info("[smtp] enabling all auth (insecure)")
+			logger.Log().Info("[smtpd] enabling all auth (insecure)")
 		}
 	} else {
 		if config.SMTPAuthFile != "" {
-			logger.Log().Infof("[smtp] enabling login auth via %s (TLS)", config.SMTPAuthFile)
+			logger.Log().Infof("[smtpd] enabling login auth via %s (TLS)", config.SMTPAuthFile)
 		} else if config.SMTPAuthAcceptAny {
-			logger.Log().Info("[smtp] enabling any auth (TLS)")
+			logger.Log().Info("[smtpd] enabling any auth (TLS)")
 		}
 	}
 
-	logger.Log().Infof("[smtp] starting on %s", logger.CleanIP(config.SMTPListen))
+	logger.Log().Infof("[smtpd] starting on %s", logger.CleanIP(config.SMTPListen))
 
 	return listenAndServe(config.SMTPListen, mailHandler, authHandler)
 }

utils/tools/message.go

@@ -0,0 +1,59 @@
+// Package tools provides various methods for variouws things
+package tools
+
+import (
+	"bufio"
+	"bytes"
+	"net/mail"
+	"regexp"
+
+	"github.com/axllent/mailpit/utils/logger"
+)
+
+// RemoveMessageHeaders scans a message for headers, if found them removes them.
+// It will only remove a single instance of any header, and is intended to remove
+// Bcc & Message-Id.
+func RemoveMessageHeaders(msg []byte, headers []string) ([]byte, error) {
+	reader := bytes.NewReader(msg)
+	m, err := mail.ReadMessage(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	reBlank := regexp.MustCompile(`^\s+`)
+
+	for _, hdr := range headers {
+		// case-insentitive
+		reHdr := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(hdr+":"))
+
+		// header := []byte(hdr + ":")
+		if m.Header.Get(hdr) != "" {
+			scanner := bufio.NewScanner(bytes.NewReader(msg))
+			found := false
+			hdr := []byte("")
+			for scanner.Scan() {
+				line := scanner.Bytes()
+				if !found && reHdr.Match(line) {
+					// add the first line starting with <header>:
+					hdr = append(hdr, line...)
+					hdr = append(hdr, []byte("\r\n")...)
+					found = true
+				} else if found && reBlank.Match(line) {
+					// add any following lines starting with a whitespace (tab or space)
+					hdr = append(hdr, line...)
+					hdr = append(hdr, []byte("\r\n")...)
+				} else if found {
+					// stop scanning, we have the full <header>
+					break
+				}
+			}
+
+			if len(hdr) > 0 {
+				logger.Log().Debugf("[release] removing %s header", hdr)
+				msg = bytes.Replace(msg, hdr, []byte(""), 1)
+			}
+		}
+	}
+
+	return msg, nil
+}