From 04462f76c687934eb8239912a7b65428400c37b1 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Fri, 21 Apr 2023 12:10:13 +1200 Subject: [PATCH] API: Message relay / release This enables a SMTP server to be configured, and messages to be manually "released" via the relay server. Aditionally, messages can be auto-relayed via the SMTP server do Mailpit acts as a form of caching proxy. @see #29 --- cmd/root.go | 15 ++++- config/config.go | 109 +++++++++++++++++++++++++++++++++--- server/apiv1/api.go | 119 ++++++++++++++++++++++++++++++++++++++++ server/apiv1/swagger.go | 10 +++- server/server.go | 1 + server/smtpd/smtp.go | 71 ++++++++++++++++++++++++ server/smtpd/smtpd.go | 44 ++++++++++----- utils/tools/message.go | 59 ++++++++++++++++++++ 8 files changed, 403 insertions(+), 25 deletions(-) create mode 100644 server/smtpd/smtp.go create mode 100644 utils/tools/message.go diff --git a/cmd/root.go b/cmd/root.go index 2e917ed..c7c6a11 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -98,6 +98,9 @@ func init() { rootCmd.Flags().BoolVar(&config.SMTPAuthAllowInsecure, "smtp-auth-allow-insecure", config.SMTPAuthAllowInsecure, "Enable insecure PLAIN & LOGIN authentication") rootCmd.Flags().StringVarP(&config.SMTPCLITags, "tag", "t", config.SMTPCLITags, "Tag new messages matching filters") + rootCmd.Flags().StringVar(&config.SMTPRelayConfigFile, "smtp-relay-config", config.SMTPRelayConfigFile, "SMTP configuration file to allow releasing messages") + rootCmd.Flags().BoolVar(&config.SMTPRelayAllIncoming, "smtp-relay-all", config.SMTPRelayAllIncoming, "Relay all incoming messages via external SMTP server (caution!)") + rootCmd.Flags().BoolVarP(&logger.QuietLogging, "quiet", "q", logger.QuietLogging, "Quiet logging (errors only)") rootCmd.Flags().BoolVarP(&logger.VerboseLogging, "verbose", "v", logger.VerboseLogging, "Verbose logging") @@ -179,6 +182,14 @@ func initConfigFromEnv() { config.SMTPAuthAllowInsecure = true } + // Relay server config + if len(os.Getenv("MP_SMTP_RELAY_CONFIG")) > 0 { + config.SMTPRelayConfigFile = os.Getenv("MP_SMTP_RELAY_CONFIG") + } + if getEnabledFromEnv("MP_SMTP_RELAY_ALL") { + config.SMTPRelayAllIncoming = true + } + if len(os.Getenv("MP_WEBROOT")) > 0 { config.Webroot = os.Getenv("MP_WEBROOT") } @@ -189,10 +200,10 @@ func initConfigFromEnv() { config.UseMessageDates = true } if getEnabledFromEnv("MP_QUIET") { - config.QuietLogging = true + logger.QuietLogging = true } if getEnabledFromEnv("MP_VERBOSE") { - config.VerboseLogging = true + logger.VerboseLogging = true } } diff --git a/config/config.go b/config/config.go index 3501aca..75aac98 100644 --- a/config/config.go +++ b/config/config.go @@ -10,8 +10,10 @@ import ( "regexp" "strings" + "github.com/axllent/mailpit/utils/logger" "github.com/mattn/go-shellwords" "github.com/tg123/go-htpasswd" + "gopkg.in/yaml.v3" ) var ( @@ -54,8 +56,8 @@ var ( // SMTPAuthFile for SMTP authentication SMTPAuthFile string - // SMTPAuth used for euthentication - SMTPAuth *htpasswd.File + // SMTPAuthConfig used for authentication auto-generated from SMTPAuthFile + SMTPAuthConfig *htpasswd.File // SMTPAuthAllowInsecure allows PLAIN & LOGIN unencrypted authentication SMTPAuthAllowInsecure bool @@ -70,7 +72,20 @@ var ( TagRegexp = regexp.MustCompile(`^([a-zA-Z0-9\-\ \_]){3,}$`) // SMTPTags are expressions to apply tags to new mail - SMTPTags []Tag + SMTPTags []AutoTag + + // SMTPRelayConfigFile to parse a yaml file and store config of relay SMTP server + SMTPRelayConfigFile string + + // SMTPRelayConfig to parse a yaml file and store config of relay SMTP server + SMTPRelayConfig smtpRelayConfigStruct + + // ReleaseEnabled is whether message releases are enabled, requires a valid SMTPRelayConfigFile + ReleaseEnabled = false + + // SMTPRelayAllIncoming is whether to relay all incoming messages via preconfgured SMTP server. + // Use with extreme caution! + SMTPRelayAllIncoming = false // ContentSecurityPolicy for HTTP server ContentSecurityPolicy = "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; frame-src 'self'; img-src * data: blob:; font-src 'self' data:; media-src 'self'; connect-src 'self' ws: wss:; object-src 'none'; base-uri 'self';" @@ -85,12 +100,25 @@ var ( RepoBinaryName = "mailpit" ) -// Tag struct -type Tag struct { +// AutoTag struct for auto-tagging +type AutoTag struct { Tag string Match string } +// SMTPRelayConfigStruct struct for parsing yaml & storing variables +type smtpRelayConfigStruct struct { + Host string `yaml:"host"` + Port int `yaml:"port"` + STARTTLS bool `yaml:"starttls"` + AllowInsecure bool `yaml:"allow-insecure"` + Auth string `yaml:"auth"` // none, plain, cram-md5 + Username string `yaml:"username"` // plain & cram-md5 + Password string `yaml:"password"` // plain + Secret string `yaml:"secret"` // cram-md5 + ReturnPath string `yaml:"return-path"` // allows overriding the boune address +} + // VerifyConfig wil do some basic checking func VerifyConfig() error { if DataFile != "" && isDir(DataFile) { @@ -158,7 +186,7 @@ func VerifyConfig() error { if err != nil { return err } - SMTPAuth = a + SMTPAuthConfig = a } if SMTPTLSCert == "" && (SMTPAuthFile != "" || SMTPAuthAcceptAny) && !SMTPAuthAllowInsecure { @@ -173,7 +201,7 @@ func VerifyConfig() error { s := strings.TrimRight(path.Join("/", Webroot, "/"), "/") + "/" Webroot = s - SMTPTags = []Tag{} + SMTPTags = []AutoTag{} p := shellwords.NewParser() @@ -194,14 +222,77 @@ func VerifyConfig() error { if len(match) == 0 { return fmt.Errorf("Invalid tag match (%s) - no search detected", tag) } - SMTPTags = append(SMTPTags, Tag{Tag: tag, Match: match}) + SMTPTags = append(SMTPTags, AutoTag{Tag: tag, Match: match}) } else { return fmt.Errorf("Error parsing tags (%s)", a) } } - } + if err := parseRelayConfig(SMTPRelayConfigFile); err != nil { + return err + } + + if !ReleaseEnabled && SMTPRelayAllIncoming { + return errors.New("SMTP relay config must be set to relay all messages") + } + + if SMTPRelayAllIncoming { + // this deserves a warning + logger.Log().Warnf("[smtp] enabling automatic relay of all new messages via %s:%d", SMTPRelayConfig.Host, SMTPRelayConfig.Port) + } + + return nil +} + +// Parse & validate the SMTPRelayConfigFile (if set) +func parseRelayConfig(c string) error { + if c == "" { + return nil + } + + if !isFile(c) { + return fmt.Errorf("SMTP relay configuration not found: %s", SMTPRelayConfigFile) + } + + data, err := os.ReadFile(c) + if err != nil { + return err + } + + if err := yaml.Unmarshal(data, &SMTPRelayConfig); err != nil { + return err + } + + if SMTPRelayConfig.Host == "" { + return errors.New("SMTP relay host not set") + } + + if SMTPRelayConfig.Port == 0 { + SMTPRelayConfig.Port = 25 // default + } + + SMTPRelayConfig.Auth = strings.ToLower(SMTPRelayConfig.Auth) + + if SMTPRelayConfig.Auth == "" || SMTPRelayConfig.Auth == "none" || SMTPRelayConfig.Auth == "false" { + SMTPRelayConfig.Auth = "none" + } else if SMTPRelayConfig.Auth == "plain" { + if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Password == "" { + return fmt.Errorf("SMTP relay host username or password not set for PLAIN authentication (%s)", c) + } + } else if strings.HasPrefix(SMTPRelayConfig.Auth, "cram") { + SMTPRelayConfig.Auth = "cram-md5" + if SMTPRelayConfig.Username == "" || SMTPRelayConfig.Secret == "" { + return fmt.Errorf("SMTP relay host username or secret not set for CRAM-MD5 authentication (%s)", c) + } + } else { + return fmt.Errorf("SMTP relay authentication method not supported: %s", SMTPRelayConfig.Auth) + } + + ReleaseEnabled = true + + logger.Log().Infof("[smtp] enabling message relaying via %s:%d", SMTPRelayConfig.Host, SMTPRelayConfig.Port) + return nil } diff --git a/server/apiv1/api.go b/server/apiv1/api.go index ba9da91..0b718e2 100644 --- a/server/apiv1/api.go +++ b/server/apiv1/api.go @@ -10,8 +10,12 @@ import ( "strings" "github.com/axllent/mailpit/config" + "github.com/axllent/mailpit/server/smtpd" "github.com/axllent/mailpit/storage" + "github.com/axllent/mailpit/utils/logger" + "github.com/axllent/mailpit/utils/tools" "github.com/gorilla/mux" + uuid "github.com/satori/go.uuid" ) // GetMessages returns a paginated list of messages as JSON @@ -491,6 +495,121 @@ func SetTags(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) } +// ReleaseMessage (method: POST) will release a message via a preconfigured external SMTP server. +// If no IDs are provided then all messages are updated. +func ReleaseMessage(w http.ResponseWriter, r *http.Request) { + // swagger:route POST /api/v1/message/{ID}/release message Release + // + // # Release message + // + // Release a message via a preconfigured external SMTP server.. + // + // Consumes: + // - application/json + // + // Produces: + // - text/plain + // + // Schemes: http, https + // + // Parameters: + // + name: ID + // in: path + // description: message id + // required: true + // type: string + // + name: To + // in: body + // description: Array of email addresses to release message to + // required: true + // type: ReleaseMessageRequest + // + // Responses: + // 200: OKResponse + // default: ErrorResponse + + vars := mux.Vars(r) + + id := vars["id"] + + msg, err := storage.GetMessageRaw(id) + if err != nil { + fourOFour(w) + return + } + + decoder := json.NewDecoder(r.Body) + + data := releaseMessageRequest{} + + if err := decoder.Decode(&data); err != nil { + httpError(w, err.Error()) + return + } + + tos := data.To + if len(tos) == 0 { + httpError(w, "No valid addresses found") + return + } + + for _, to := range tos { + if _, err := mail.ParseAddress(to); err != nil { + httpError(w, "Invalid email address: "+to) + return + } + } + + reader := bytes.NewReader(msg) + m, err := mail.ReadMessage(reader) + if err != nil { + httpError(w, err.Error()) + return + } + + froms, err := m.Header.AddressList("From") + if err != nil { + httpError(w, err.Error()) + return + } + + from := froms[0].Address + + // if sender is used, then change from to the sender + if senders, err := m.Header.AddressList("Sender"); err == nil { + from = senders[0].Address + } + + msg, err = tools.RemoveMessageHeaders(msg, []string{"Bcc", "Message-Id"}) + if err != nil { + httpError(w, err.Error()) + return + } + + if config.SMTPRelayConfig.ReturnPath != "" && m.Header.Get("Return-Path") != "<"+config.SMTPRelayConfig.ReturnPath+">" { + msg, err = tools.RemoveMessageHeaders(msg, []string{"Return-Path"}) + if err != nil { + httpError(w, err.Error()) + return + } + msg = append([]byte("Return-Path: <"+config.SMTPRelayConfig.ReturnPath+">\r\n"), msg...) + } + + // generate unique ID + uid := uuid.NewV4().String() + "@mailpit" + // add unique ID + msg = append([]byte("Message-Id: <"+uid+">\r\n"), msg...) + + if err := smtpd.Send(from, tos, msg); err != nil { + logger.Log().Errorf("[smtp] error sending message: %s", err.Error()) + httpError(w, "SMTP error: "+err.Error()) + return + } + + w.Header().Add("Content-Type", "text/plain") + _, _ = w.Write([]byte("ok")) +} + // FourOFour returns a basic 404 message func fourOFour(w http.ResponseWriter) { w.Header().Set("Referrer-Policy", "no-referrer") diff --git a/server/apiv1/swagger.go b/server/apiv1/swagger.go index e27b026..7bb037e 100644 --- a/server/apiv1/swagger.go +++ b/server/apiv1/swagger.go @@ -47,11 +47,19 @@ type setTagsRequest struct { // in:body Tags []string `json:"tags"` - // ids + // IDs // in:body IDs []string `json:"ids"` } +// Release request +// swagger:model ReleaseMessageRequest +type releaseMessageRequest struct { + // To + // in:body + To []string `json:"to"` +} + // Binary data reponse inherits the attachment's content type // swagger:response BinaryResponse type binaryResponse struct { diff --git a/server/server.go b/server/server.go index 2dd6ff4..63c87d3 100644 --- a/server/server.go +++ b/server/server.go @@ -1,3 +1,4 @@ +// Package server is the HTTP daemon package server import ( diff --git a/server/smtpd/smtp.go b/server/smtpd/smtp.go new file mode 100644 index 0000000..34eaac3 --- /dev/null +++ b/server/smtpd/smtp.go @@ -0,0 +1,71 @@ +package smtpd + +import ( + "crypto/tls" + "fmt" + "net/smtp" + + "github.com/axllent/mailpit/config" +) + +// Send will connect to a pre-configured SMTP server and send a message to one or more recipients. +func Send(from string, to []string, msg []byte) error { + addr := fmt.Sprintf("%s:%d", config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port) + + c, err := smtp.Dial(addr) + if err != nil { + return err + } + + defer c.Close() + + if config.SMTPRelayConfig.STARTTLS { + conf := &tls.Config{ServerName: config.SMTPRelayConfig.Host} + + conf.InsecureSkipVerify = config.SMTPRelayConfig.AllowInsecure + + if err = c.StartTLS(conf); err != nil { + return err + } + } + + var a smtp.Auth + + if config.SMTPRelayConfig.Auth == "plain" { + a = smtp.PlainAuth("", config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Password, config.SMTPRelayConfig.Host) + } + + if config.SMTPRelayConfig.Auth == "cram-md5" { + a = smtp.CRAMMD5Auth(config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Secret) + } + + if a != nil { + if err = c.Auth(a); err != nil { + return err + } + } + if err = c.Mail(from); err != nil { + return err + } + + for _, addr := range to { + if err = c.Rcpt(addr); err != nil { + return err + } + } + + w, err := c.Data() + if err != nil { + return err + } + + if _, err := w.Write(msg); err != nil { + return err + } + + if err := w.Close(); err != nil { + return err + } + + return c.Quit() +} diff --git a/server/smtpd/smtpd.go b/server/smtpd/smtpd.go index b31ee8d..0e26373 100644 --- a/server/smtpd/smtpd.go +++ b/server/smtpd/smtpd.go @@ -13,16 +13,34 @@ import ( "github.com/axllent/mailpit/storage" "github.com/axllent/mailpit/utils/logger" "github.com/mhale/smtpd" + uuid "github.com/satori/go.uuid" ) func mailHandler(origin net.Addr, from string, to []string, data []byte) error { msg, err := mail.ReadMessage(bytes.NewReader(data)) if err != nil { - logger.Log().Errorf("[smtp] error parsing message: %s", err.Error()) + logger.Log().Errorf("[smtpd] error parsing message: %s", err.Error()) return err } + // add a message ID if not set + if msg.Header.Get("Message-Id") == "" { + // generate unique ID + uid := uuid.NewV4().String() + "@mailpit" + // add unique ID + data = append([]byte("Message-Id: <"+uid+">\r\n"), data...) + } + + // if enabled, this will route the email 1:1 through to the preconfigured smtp server + if config.SMTPRelayAllIncoming { + if err := Send(from, to, data); err != nil { + logger.Log().Errorf("[smtp] error relaying message: %s", err.Error()) + } else { + logger.Log().Debugf("[smtp] relayed message from %s via %s:%d", from, config.SMTPRelayConfig.Host, config.SMTPRelayConfig.Port) + } + } + // build array of all addresses in the header to compare to the []to array emails, hasBccHeader := scanAddressesInHeader(msg.Header) @@ -35,7 +53,7 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error { missingAddresses = append(missingAddresses, a) } } else { - logger.Log().Warnf("[smtp] ignoring invalid email address: %s", a) + logger.Log().Warnf("[smtpd] ignoring invalid email address: %s", a) } } @@ -60,7 +78,7 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error { data = append(bcc, data...) } - logger.Log().Debugf("[smtp] added missing addresses to Bcc header: %s", strings.Join(missingAddresses, ", ")) + logger.Log().Debugf("[smtpd] added missing addresses to Bcc header: %s", strings.Join(missingAddresses, ", ")) } if _, err := storage.Store(data); err != nil { @@ -70,17 +88,17 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error { } subject := msg.Header.Get("Subject") - logger.Log().Debugf("[smtp] received (%s) from:%s to:%s subject:%q", cleanIP(origin), from, to[0], subject) + logger.Log().Debugf("[smtpd] received (%s) from:%s subject:%q", cleanIP(origin), from, subject) return nil } func authHandler(remoteAddr net.Addr, mechanism string, username []byte, password []byte, shared []byte) (bool, error) { - allow := config.SMTPAuth.Match(string(username), string(password)) + allow := config.SMTPAuthConfig.Match(string(username), string(password)) if allow { - logger.Log().Debugf("[smtp] allow %s login:%q from:%s", mechanism, string(username), cleanIP(remoteAddr)) + logger.Log().Debugf("[smtpd] allow %s login:%q from:%s", mechanism, string(username), cleanIP(remoteAddr)) } else { - logger.Log().Warnf("[smtp] deny %s login:%q from:%s", mechanism, string(username), cleanIP(remoteAddr)) + logger.Log().Warnf("[smtpd] deny %s login:%q from:%s", mechanism, string(username), cleanIP(remoteAddr)) } return allow, nil @@ -88,7 +106,7 @@ func authHandler(remoteAddr net.Addr, mechanism string, username []byte, passwor // Allow any username and password func authHandlerAny(remoteAddr net.Addr, mechanism string, username []byte, password []byte, shared []byte) (bool, error) { - logger.Log().Debugf("[smtp] allow %s login %q from %s", mechanism, string(username), cleanIP(remoteAddr)) + logger.Log().Debugf("[smtpd] allow %s login %q from %s", mechanism, string(username), cleanIP(remoteAddr)) return true, nil } @@ -97,19 +115,19 @@ func authHandlerAny(remoteAddr net.Addr, mechanism string, username []byte, pass func Listen() error { if config.SMTPAuthAllowInsecure { if config.SMTPAuthFile != "" { - logger.Log().Infof("[smtp] enabling login auth via %s (insecure)", config.SMTPAuthFile) + logger.Log().Infof("[smtpd] enabling login auth via %s (insecure)", config.SMTPAuthFile) } else if config.SMTPAuthAcceptAny { - logger.Log().Info("[smtp] enabling all auth (insecure)") + logger.Log().Info("[smtpd] enabling all auth (insecure)") } } else { if config.SMTPAuthFile != "" { - logger.Log().Infof("[smtp] enabling login auth via %s (TLS)", config.SMTPAuthFile) + logger.Log().Infof("[smtpd] enabling login auth via %s (TLS)", config.SMTPAuthFile) } else if config.SMTPAuthAcceptAny { - logger.Log().Info("[smtp] enabling any auth (TLS)") + logger.Log().Info("[smtpd] enabling any auth (TLS)") } } - logger.Log().Infof("[smtp] starting on %s", logger.CleanIP(config.SMTPListen)) + logger.Log().Infof("[smtpd] starting on %s", logger.CleanIP(config.SMTPListen)) return listenAndServe(config.SMTPListen, mailHandler, authHandler) } diff --git a/utils/tools/message.go b/utils/tools/message.go new file mode 100644 index 0000000..bd78243 --- /dev/null +++ b/utils/tools/message.go @@ -0,0 +1,59 @@ +// Package tools provides various methods for variouws things +package tools + +import ( + "bufio" + "bytes" + "net/mail" + "regexp" + + "github.com/axllent/mailpit/utils/logger" +) + +// RemoveMessageHeaders scans a message for headers, if found them removes them. +// It will only remove a single instance of any header, and is intended to remove +// Bcc & Message-Id. +func RemoveMessageHeaders(msg []byte, headers []string) ([]byte, error) { + reader := bytes.NewReader(msg) + m, err := mail.ReadMessage(reader) + if err != nil { + return nil, err + } + + reBlank := regexp.MustCompile(`^\s+`) + + for _, hdr := range headers { + // case-insentitive + reHdr := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(hdr+":")) + + // header := []byte(hdr + ":") + if m.Header.Get(hdr) != "" { + scanner := bufio.NewScanner(bytes.NewReader(msg)) + found := false + hdr := []byte("") + for scanner.Scan() { + line := scanner.Bytes() + if !found && reHdr.Match(line) { + // add the first line starting with
: + hdr = append(hdr, line...) + hdr = append(hdr, []byte("\r\n")...) + found = true + } else if found && reBlank.Match(line) { + // add any following lines starting with a whitespace (tab or space) + hdr = append(hdr, line...) + hdr = append(hdr, []byte("\r\n")...) + } else if found { + // stop scanning, we have the full
+ break + } + } + + if len(hdr) > 0 { + logger.Log().Debugf("[release] removing %s header", hdr) + msg = bytes.Replace(msg, hdr, []byte(""), 1) + } + } + } + + return msg, nil +}