Merge branch 'release/v1.9.9'

.github/workflows/tests.yml

@@ -25,8 +25,8 @@ jobs:
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-
-    - run: go test ./internal/storage ./server ./internal/tools ./internal/tools/html2text -v
-    - run: go test ./internal/storage ./internal/tools/html2text -bench=.
+    - run: go test ./internal/storage ./server ./internal/tools ./internal/html2text -v
+    - run: go test ./internal/storage ./internal/html2text -bench=.
     
     # build the assets
     - name: Build web UI

CHANGELOG.md

@@ -2,6 +2,20 @@
 
 Notable changes to Mailpit will be documented in this file.
 
+## [v1.9.9]
+
+### Chore
+- Move html2text module to internal/html2text
+
+### Feature
+- Set optional webhook for received messages ([#195](https://github.com/axllent/mailpit/issues/195))
+- Reset message date on release ([#194](https://github.com/axllent/mailpit/issues/194))
+
+### Libs
+- update node modules
+- Update Go modules
+
+
 ## [v1.9.8]
 
 ### Chore

README.md

@@ -37,6 +37,7 @@ Mailpit was originally **inspired** by MailHog which is now [no longer maintaine
 - Optional HTTPS for web UI ([see wiki](https://github.com/axllent/mailpit/wiki/HTTPS))
 - Optional basic authentication for web UI ([see wiki](https://github.com/axllent/mailpit/wiki/Basic-authentication))
 - A simple REST API ([see docs](docs/apiv1/README.md))
+- Optional webhook for received messages ([see docs](https://github.com/axllent/mailpit/wiki/Webhook))
 - Multi-architecture [Docker images](https://github.com/axllent/mailpit/wiki/Docker-images)
 
 
@@ -47,11 +48,11 @@ The Mailpit web UI listens by default on `http://0.0.0.0:8025`, and the SMTP por
 Mailpit runs as a single binary and can be installed in different ways:
 
 
-### Install via Brew (Mac)
+### Install via package managers
 
-Install Mailpit with `brew install mailpit`.
-
-To run automatically run mailpit in the background, run `brew services start mailpit`.
+- **Mac**: `brew install mailpit` (to run automatically in background `brew services start mailpit`)
+- **Arch Linux**: available in the AUR as `mailpit`
+- **FreeBSD**: `pkg install mailpit`
 
 
 ### Install via bash script (Linux & Mac)
@@ -84,6 +85,7 @@ Run `mailpit -h` to see options. More information can be seen in [the docs](http
 
 If installed using homebrew, you may run `brew services start mailpit` to run always run mailpit automatically.
 
+
 ### Testing Mailpit
 
 Please refer to [the documentation](https://github.com/axllent/mailpit/wiki/Testing-Mailpit) of how to easily test email delivery to Mailpit.

cmd/root.go

@@ -13,6 +13,7 @@ import (
 	"github.com/axllent/mailpit/internal/storage"
 	"github.com/axllent/mailpit/server"
 	"github.com/axllent/mailpit/server/smtpd"
+	"github.com/axllent/mailpit/server/webhook"
 	"github.com/spf13/cobra"
 )
 
@@ -105,6 +106,8 @@ func init() {
 
 	rootCmd.Flags().StringVar(&config.SMTPRelayConfigFile, "smtp-relay-config", config.SMTPRelayConfigFile, "SMTP configuration file to allow releasing messages")
 	rootCmd.Flags().BoolVar(&config.SMTPRelayAllIncoming, "smtp-relay-all", config.SMTPRelayAllIncoming, "Relay all incoming messages via external SMTP server (caution!)")
+	rootCmd.Flags().StringVar(&config.WebhookURL, "webhook-url", config.WebhookURL, "Send a webhook request for new messages")
+	rootCmd.Flags().IntVar(&webhook.RateLimit, "webhook-limit", webhook.RateLimit, "Limit webhook requests per second")
 
 	rootCmd.Flags().StringVarP(&config.SMTPCLITags, "tag", "t", config.SMTPCLITags, "Tag new messages matching filters")
 	rootCmd.Flags().BoolVarP(&logger.QuietLogging, "quiet", "q", logger.QuietLogging, "Quiet logging (errors only)")
@@ -169,6 +172,14 @@ func initConfigFromEnv() {
 		config.SMTPRelayAllIncoming = true
 	}
 
+	// Webhook
+	if len(os.Getenv("MP_WEBHOOK_URL")) > 0 {
+		config.WebhookURL = os.Getenv("MP_WEBHOOK_URL")
+	}
+	if len(os.Getenv("MP_WEBHOOK_LIMIT")) > 0 {
+		webhook.RateLimit, _ = strconv.Atoi(os.Getenv("MP_WEBHOOK_LIMIT"))
+	}
+
 	// Misc options
 	if len(os.Getenv("MP_WEBROOT")) > 0 {
 		config.Webroot = os.Getenv("MP_WEBROOT")

config/config.go

@@ -4,6 +4,7 @@ package config
 import (
 	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
@@ -94,6 +95,9 @@ var (
 	// Use with extreme caution!
 	SMTPRelayAllIncoming = false
 
+	// WebhookURL for calling
+	WebhookURL string
+
 	// ContentSecurityPolicy for HTTP server - set via VerifyConfig()
 	ContentSecurityPolicy string
 
@@ -223,6 +227,10 @@ func VerifyConfig() error {
 	s := strings.TrimRight(path.Join("/", Webroot, "/"), "/") + "/"
 	Webroot = s
 
+	if WebhookURL != "" && !isValidURL(WebhookURL) {
+		return fmt.Errorf("Webhook URL does not appear to be a valid URL (%s)", WebhookURL)
+	}
+
 	SMTPTags = []AutoTag{}
 
 	if SMTPCLITags != "" {
@@ -349,3 +357,12 @@ func isDir(path string) bool {
 
 	return true
 }
+
+func isValidURL(s string) bool {
+	u, err := url.ParseRequestURI(s)
+	if err != nil {
+		return false
+	}
+
+	return strings.HasPrefix(u.Scheme, "http")
+}

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/vanng822/go-premailer v1.20.2
 	golang.org/x/net v0.17.0
 	golang.org/x/text v0.13.0
+	golang.org/x/time v0.3.0
 	gopkg.in/yaml.v3 v3.0.1
 	modernc.org/sqlite v1.26.0
 )
@@ -40,7 +41,7 @@ require (
 	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect

go.sum

@@ -82,8 +82,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leporo/sqlf v1.4.0 h1:SyWnX/8GSGOzVmanG0Ub1c04mR9nNl6Tq3IeFKX2/4c=
 github.com/leporo/sqlf v1.4.0/go.mod h1:pgN9yKsAnQ+2ewhbZogr98RcasUjPsHF3oXwPPhHvBw=
-github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
-github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
@@ -192,6 +192,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
+golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

internal/storage/database.go

@@ -21,6 +21,7 @@ import (
 	"github.com/axllent/mailpit/config"
 	"github.com/axllent/mailpit/internal/logger"
 	"github.com/axllent/mailpit/internal/tools"
+	"github.com/axllent/mailpit/server/webhook"
 	"github.com/axllent/mailpit/server/websockets"
 	"github.com/google/uuid"
 	"github.com/jhillyerd/enmime"
@@ -234,6 +235,7 @@ func Store(body []byte) (string, error) {
 	c.Snippet = snippet
 
 	websockets.Broadcast("new", c)
+	webhook.Send(c)
 
 	dbLastAction = time.Now()
 

internal/storage/utils.go

@@ -10,8 +10,8 @@ import (
 	"time"
 
 	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/html2text"
 	"github.com/axllent/mailpit/internal/logger"
-	"github.com/axllent/mailpit/internal/tools/html2text"
 	"github.com/axllent/mailpit/server/websockets"
 	"github.com/jhillyerd/enmime"
 	"github.com/leporo/sqlf"

internal/tools/message.go

@@ -11,8 +11,7 @@ import (
 )
 
 // RemoveMessageHeaders scans a message for headers, if found them removes them.
-// It will only remove a single instance of any header, and is intended to remove
-// Bcc & Message-Id.
+// It will only remove a single instance of any given message header.
 func RemoveMessageHeaders(msg []byte, headers []string) ([]byte, error) {
 	reader := bytes.NewReader(msg)
 	m, err := mail.ReadMessage(reader)
@@ -49,7 +48,7 @@ func RemoveMessageHeaders(msg []byte, headers []string) ([]byte, error) {
 			}
 
 			if len(hdr) > 0 {
-				logger.Log().Debugf("[release] removing %s header", hdr)
+				logger.Log().Debugf("[release] removed %s header", hdr)
 				msg = bytes.Replace(msg, hdr, []byte(""), 1)
 			}
 		}
@@ -57,3 +56,44 @@ func RemoveMessageHeaders(msg []byte, headers []string) ([]byte, error) {
 
 	return msg, nil
 }
+
+// UpdateMessageHeader scans a message for a header and updates its value if found.
+func UpdateMessageHeader(msg []byte, header, value string) ([]byte, error) {
+	reader := bytes.NewReader(msg)
+	m, err := mail.ReadMessage(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	if m.Header.Get(header) != "" {
+		reBlank := regexp.MustCompile(`^\s+`)
+		reHdr := regexp.MustCompile(`(?i)^` + regexp.QuoteMeta(header+":"))
+
+		scanner := bufio.NewScanner(bytes.NewReader(msg))
+		found := false
+		hdr := []byte("")
+		for scanner.Scan() {
+			line := scanner.Bytes()
+			if !found && reHdr.Match(line) {
+				// add the first line starting with <header>:
+				hdr = append(hdr, line...)
+				hdr = append(hdr, []byte("\r\n")...)
+				found = true
+			} else if found && reBlank.Match(line) {
+				// add any following lines starting with a whitespace (tab or space)
+				hdr = append(hdr, line...)
+				hdr = append(hdr, []byte("\r\n")...)
+			} else if found {
+				// stop scanning, we have the full <header>
+				break
+			}
+		}
+
+		if len(hdr) > 0 {
+			logger.Log().Debugf("[release] replaced %s header", hdr)
+			msg = bytes.Replace(msg, hdr, []byte(header+": "+value+"\r\n"), 1)
+		}
+	}
+
+	return msg, nil
+}

internal/tools/snippets.go

@@ -4,7 +4,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/axllent/mailpit/internal/tools/html2text"
+	"github.com/axllent/mailpit/internal/html2text"
 )
 
 // CreateSnippet returns a message snippet. It will use the HTML version (if it exists)

server/apiv1/api.go

@@ -9,6 +9,7 @@ import (
 	"net/mail"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/axllent/mailpit/config"
 	"github.com/axllent/mailpit/internal/htmlcheck"
@@ -632,7 +633,7 @@ func ReleaseMessage(w http.ResponseWriter, r *http.Request) {
 		from = senders[0].Address
 	}
 
-	msg, err = tools.RemoveMessageHeaders(msg, []string{"Bcc", "Message-Id"})
+	msg, err = tools.RemoveMessageHeaders(msg, []string{"Bcc"})
 	if err != nil {
 		httpError(w, err.Error())
 		return
@@ -652,10 +653,21 @@ func ReleaseMessage(w http.ResponseWriter, r *http.Request) {
 		from = config.SMTPRelayConfig.ReturnPath
 	}
 
+	// update message date
+	msg, err = tools.UpdateMessageHeader(msg, "Date", time.Now().Format(time.RFC1123Z))
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
+
 	// generate unique ID
 	uid := uuid.New().String() + "@mailpit"
-	// add unique ID
-	msg = append([]byte("Message-Id: <"+uid+">\r\n"), msg...)
+	// update Message-Id with unique ID
+	msg, err = tools.UpdateMessageHeader(msg, "Message-Id", "<"+uid+">")
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
 
 	if err := smtpd.Send(from, tos, msg); err != nil {
 		logger.Log().Errorf("[smtp] error sending message: %s", err.Error())

server/webhook/webhook.go

@@ -0,0 +1,72 @@
+// Package webhook will optionally call a preconfigured endpoint
+package webhook
+
+import (
+	"bytes"
+	"encoding/json"
+	"net/http"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"golang.org/x/time/rate"
+)
+
+var (
+	// RateLimit is the minimum number of seconds between requests
+	RateLimit = 1
+
+	rl rate.Sometimes
+
+	rateLimiterSet bool
+)
+
+// Send will post the MessageSummary to a webhook (if configured)
+func Send(msg interface{}) {
+	if config.WebhookURL == "" {
+		return
+	}
+
+	if !rateLimiterSet {
+		if RateLimit > 0 {
+			rl = rate.Sometimes{Interval: time.Duration(RateLimit) * time.Second}
+		} else {
+			// run 1000 per second - ie: do not limit
+			rl = rate.Sometimes{First: 1000, Interval: time.Second}
+		}
+		rateLimiterSet = true
+	}
+
+	go func() {
+		rl.Do(func() {
+			b, err := json.Marshal(msg)
+			if err != nil {
+				logger.Log().Errorf("[webhook] invalid data: %s", err)
+				return
+			}
+
+			req, err := http.NewRequest("POST", config.WebhookURL, bytes.NewBuffer(b))
+			if err != nil {
+				logger.Log().Errorf("[webhook] error: %s", err)
+				return
+			}
+
+			req.Header.Set("User-Agent", "Mailpit/"+config.Version)
+			req.Header.Set("Content-Type", "application/json")
+
+			client := &http.Client{}
+			resp, err := client.Do(req)
+			if err != nil {
+				logger.Log().Errorf("[webhook] error sending data: %s", err)
+				return
+			}
+
+			if resp.StatusCode < 200 || resp.StatusCode > 299 {
+				logger.Log().Warningf("[webhook] %s returned a %d status", config.WebhookURL, resp.StatusCode)
+				return
+			}
+
+			defer resp.Body.Close()
+		})
+	}()
+}

server/websockets/hub.go

@@ -80,6 +80,7 @@ func Broadcast(t string, msg interface{}) {
 
 	if err != nil {
 		logger.Log().Errorf("[websocket] broadcast received invalid data: %s", err)
+		return
 	}
 
 	go func() { MessageHub.Broadcast <- b }()