1
0
mirror of https://github.com/axllent/mailpit.git synced 2025-01-10 00:43:53 +02:00

Merge branch 'release/v1.12.1'

This commit is contained in:
Ralph Slooten 2024-01-03 15:03:16 +13:00
commit fa8b398afc
23 changed files with 179 additions and 85 deletions

View File

@ -9,12 +9,13 @@ jobs:
strategy:
matrix:
go-version: [1.21.x]
os: [ubuntu-latest]
os: [ubuntu-latest, windows-latest, macos-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
cache: false
- uses: actions/checkout@v4
- name: Run Go tests
uses: actions/cache@v3
@ -30,15 +31,19 @@ jobs:
# build the assets
- name: Build web UI
if: startsWith(matrix.os, 'ubuntu') == true
uses: actions/setup-node@v4
with:
node-version: 18
cache: 'npm'
- run: npm install
- run: npm run package
- if: startsWith(matrix.os, 'ubuntu') == true
run: npm install
- if: startsWith(matrix.os, 'ubuntu') == true
run: npm run package
# validate the swagger file
- name: Validate OpenAPI definition
if: startsWith(matrix.os, 'ubuntu') == true
uses: char0n/swagger-editor-validate@v1
with:
definition-file: server/ui/api/v1/swagger.json

View File

@ -2,6 +2,30 @@
Notable changes to Mailpit will be documented in this file.
## [v1.12.1]
### Chore
- Significantly increase database performance using WAL (Write-Ahead-Log)
- Standardize error logging & formatting
### Feature
- Add option to only allow SMTP recipients matching a regular expression (disable open-relay behaviour [#219](https://github.com/axllent/mailpit/issues/219))
### Fix
- Log total deleted messages when auto-pruning messages (--max)
- Prevent rare error from websocket connection (unexpected non-whitespace character)
- Log total deleted messages when deleting all messages from search
### Libs
- Update node modules
### Tests
- Run tests on Linux, Windows & Mac
### UI
- Automatically refresh connected browsers if Mailpit is upgraded (version change)
## [v1.12.0]
### Chore

View File

@ -103,6 +103,7 @@ func init() {
rootCmd.Flags().BoolVar(&config.SMTPAuthAllowInsecure, "smtp-auth-allow-insecure", config.SMTPAuthAllowInsecure, "Enable insecure PLAIN & LOGIN authentication")
rootCmd.Flags().BoolVar(&config.SMTPStrictRFCHeaders, "smtp-strict-rfc-headers", config.SMTPStrictRFCHeaders, "Return SMTP error if message headers contain <CR><CR><LF>")
rootCmd.Flags().IntVar(&config.SMTPMaxRecipients, "smtp-max-recipients", config.SMTPMaxRecipients, "Maximum SMTP recipients allowed")
rootCmd.Flags().StringVar(&config.SMTPAllowedRecipients, "smtp-allowed-recipients", config.SMTPAllowedRecipients, "Only allow SMTP recipients matching a regular expression (default allow all)")
rootCmd.Flags().StringVar(&config.SMTPRelayConfigFile, "smtp-relay-config", config.SMTPRelayConfigFile, "SMTP configuration file to allow releasing messages")
rootCmd.Flags().BoolVar(&config.SMTPRelayAllIncoming, "smtp-relay-all", config.SMTPRelayAllIncoming, "Relay all incoming messages via external SMTP server (caution!)")
@ -170,6 +171,9 @@ func initConfigFromEnv() {
if len(os.Getenv("MP_SMTP_MAX_RECIPIENTS")) > 0 {
config.SMTPMaxRecipients, _ = strconv.Atoi(os.Getenv("MP_SMTP_MAX_RECIPIENTS"))
}
if len(os.Getenv("MP_SMTP_ALLOWED_RECIPIENTS")) > 0 {
config.SMTPAllowedRecipients = os.Getenv("MP_SMTP_ALLOWED_RECIPIENTS")
}
// Relay server config
config.SMTPRelayConfigFile = os.Getenv("MP_SMTP_RELAY_CONFIG")

View File

@ -93,6 +93,12 @@ var (
// @see https://github.com/axllent/mailpit/issues/87 & https://github.com/axllent/mailpit/issues/153
SMTPStrictRFCHeaders bool
// SMTPAllowedRecipients if set, will only accept recipients matching this regular expression
SMTPAllowedRecipients string
// SMTPAllowedRecipientsRegexp is the compiled version of SMTPAllowedRecipients
SMTPAllowedRecipientsRegexp *regexp.Regexp
// ReleaseEnabled is whether message releases are enabled, requires a valid SMTPRelayConfigFile
ReleaseEnabled = false
@ -262,6 +268,16 @@ func VerifyConfig() error {
}
}
if SMTPAllowedRecipients != "" {
restrictRegexp, err := regexp.Compile(SMTPAllowedRecipients)
if err != nil {
return fmt.Errorf("Failed to compile smtp-allowed-recipients regexp: %s", err.Error())
}
SMTPAllowedRecipientsRegexp = restrictRegexp
logger.Log().Infof("[smtp] only allowing recipients matching the following regexp: %s", SMTPAllowedRecipients)
}
if err := parseRelayConfig(SMTPRelayConfigFile); err != nil {
return err
}
@ -335,11 +351,11 @@ func parseRelayConfig(c string) error {
if SMTPRelayConfig.RecipientAllowlist != "" {
if err != nil {
return fmt.Errorf("failed to compile recipient allowlist regexp: %e", err)
return fmt.Errorf("Failed to compile relay recipient allowlist regexp: %s", err.Error())
}
SMTPRelayConfig.RecipientAllowlistRegexp = allowlistRegexp
logger.Log().Infof("[smtp] recipient allowlist is active with the following regexp: %s", SMTPRelayConfig.RecipientAllowlist)
logger.Log().Infof("[smtp] relay recipient allowlist is active with the following regexp: %s", SMTPRelayConfig.RecipientAllowlist)
}

View File

@ -25,14 +25,12 @@ func runCSSTests(html string) ([]Warning, int, error) {
inlined, err := inlineRemoteCSS(html)
if err != nil {
// logger.Log().Warn(err)
inlined = html
}
// merge all CSS inline
merged, err := mergeInlineCSS(inlined)
if err != nil {
// logger.Log().Warn(err)
merged = inlined
}
@ -157,7 +155,7 @@ func inlineRemoteCSS(h string) (string, error) {
resp, err := downloadToBytes(a.Val)
if err != nil {
logger.Log().Warningf("html check failed to download %s", a.Val)
logger.Log().Warnf("[html-check] failed to download %s", a.Val)
continue
}
@ -179,7 +177,7 @@ func inlineRemoteCSS(h string) (string, error) {
newDoc, err := doc.Html()
if err != nil {
logger.Log().Warning(err)
logger.Log().Warnf("[html-check] failed to download %s", err.Error())
return h, err
}

View File

@ -79,7 +79,7 @@ func doHead(link string, followRedirects bool) (int, error) {
req, err := http.NewRequest("HEAD", link, nil)
if err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[link-check] %s", err.Error())
return 0, err
}

View File

@ -21,9 +21,9 @@ var (
mu sync.RWMutex
smtpReceived int
smtpReceivedSize int
smtpErrors int
smtpAccepted int
smtpAcceptedSize int
smtpRejected int
smtpIgnored int
)
@ -50,15 +50,15 @@ type AppInformation struct {
Uptime int
// Current memory usage in bytes
Memory uint64
// Messages deleted
// Database runtime messages deleted
MessagesDeleted int
// SMTP messages received via since run
SMTPReceived int
// Total size in bytes of received messages since run
SMTPReceivedSize int
// SMTP errors since run
SMTPErrors int
// SMTP messages ignored since run (duplicate IDs)
// Accepted runtime SMTP messages
SMTPAccepted int
// Total runtime accepted messages size in bytes
SMTPAcceptedSize int
// Rejected runtime SMTP messages
SMTPRejected int
// Ignored runtime SMTP messages (when using --ignore-duplicate-ids)
SMTPIgnored int
}
}
@ -75,9 +75,9 @@ func Load() AppInformation {
info.RuntimeStats.Uptime = int(time.Since(startedAt).Seconds())
info.RuntimeStats.MessagesDeleted = storage.StatsDeleted
info.RuntimeStats.SMTPReceived = smtpReceived
info.RuntimeStats.SMTPReceivedSize = smtpReceivedSize
info.RuntimeStats.SMTPErrors = smtpErrors
info.RuntimeStats.SMTPAccepted = smtpAccepted
info.RuntimeStats.SMTPAcceptedSize = smtpAcceptedSize
info.RuntimeStats.SMTPRejected = smtpRejected
info.RuntimeStats.SMTPIgnored = smtpIgnored
if latestVersionCache != "" {
@ -116,18 +116,18 @@ func Track() {
startedAt = time.Now()
}
// LogSMTPReceived logs a successfully SMTP transaction
func LogSMTPReceived(size int) {
// LogSMTPAccepted logs a successful SMTP transaction
func LogSMTPAccepted(size int) {
mu.Lock()
smtpReceived = smtpReceived + 1
smtpReceivedSize = smtpReceivedSize + size
smtpAccepted = smtpAccepted + 1
smtpAcceptedSize = smtpAcceptedSize + size
mu.Unlock()
}
// LogSMTPError logs a failed SMTP transaction
func LogSMTPError() {
// LogSMTPRejected logs a rejected SMTP transaction
func LogSMTPRejected() {
mu.Lock()
smtpErrors = smtpErrors + 1
smtpRejected = smtpRejected + 1
mu.Unlock()
}

View File

@ -76,6 +76,12 @@ func InitDB() error {
// @see https://github.com/mattn/go-sqlite3#faq
db.SetMaxOpenConns(1)
// SQLite performance tuning (https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)
_, err = db.Exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = normal;")
if err != nil {
return err
}
// create tables if necessary & apply migrations
if err := dbApplyMigrations(); err != nil {
return err
@ -110,7 +116,7 @@ func InitDB() error {
func Close() {
if db != nil {
if err := db.Close(); err != nil {
logger.Log().Warning("[db] error closing database, ignoring")
logger.Log().Warn("[db] error closing database, ignoring")
}
}
@ -128,7 +134,7 @@ func Store(body *[]byte) (string, error) {
// Parse message body with enmime
env, err := enmime.ReadEnvelope(bytes.NewReader(*body))
if err != nil {
logger.Log().Warningf("[db] %s", err.Error())
logger.Log().Warnf("[message] %s", err.Error())
return "", nil
}
@ -271,12 +277,12 @@ func List(start, limit int) ([]MessageSummary, error) {
em := MessageSummary{}
if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet); err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[db] %s", err.Error())
return
}
if err := json.Unmarshal([]byte(metadata), &em); err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[json] %s", err.Error())
return
}
@ -349,7 +355,7 @@ func GetMessage(id string) (*Message, error) {
var created int64
if err := row.Scan(&created); err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[db] %s", err.Error())
return
}
@ -357,7 +363,7 @@ func GetMessage(id string) (*Message, error) {
date = time.UnixMilli(created)
}); err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[db] %s", err.Error())
}
}
@ -686,11 +692,11 @@ func DeleteAllMessages() error {
logger.Log().Debugf("[db] deleted %d messages in %s", total, elapsed)
}
logMessagesDeleted(total)
dbLastAction = time.Now()
dbDataDeleted = false
logMessagesDeleted(total)
websockets.Broadcast("prune", nil)
BroadcastMailboxStats()

View File

@ -36,7 +36,7 @@ func migrateTagsToManyMany() {
tags := []string{}
if err := json.Unmarshal([]byte(jsonTags), &tags); err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[json] %s", err.Error())
return
}

View File

@ -3,6 +3,7 @@ package storage
import (
"time"
"github.com/axllent/mailpit/config"
"github.com/axllent/mailpit/server/websockets"
)
@ -25,9 +26,11 @@ func BroadcastMailboxStats() {
b := struct {
Total int
Unread int
Version string
}{
Total: CountTotal(),
Unread: CountUnread(),
Version: config.Version,
}
websockets.Broadcast("stats", b)

View File

@ -29,7 +29,7 @@ func ReindexAll() {
})
if err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[db] %s", err.Error())
os.Exit(1)
}
@ -59,7 +59,7 @@ func ReindexAll() {
env, err := enmime.ReadEnvelope(r)
if err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[message] %s", err.Error())
continue
}
@ -77,7 +77,7 @@ func ReindexAll() {
ctx := context.Background()
tx, err := db.BeginTx(ctx, nil)
if err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[db] %s", err.Error())
continue
}
@ -88,13 +88,13 @@ func ReindexAll() {
for _, u := range updates {
_, err = tx.Exec("UPDATE mailbox SET SearchText = ?, Snippet = ? WHERE ID = ?", u.SearchText, u.Snippet, u.ID)
if err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[db] %s", err.Error())
continue
}
}
if err := tx.Commit(); err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[db] %s", err.Error())
continue
}

View File

@ -43,12 +43,12 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) {
em := MessageSummary{}
if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet, &ignore, &ignore, &ignore, &ignore); err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[db] %s", err.Error())
return
}
if err := json.Unmarshal([]byte(metadata), &em); err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[db] %s", err.Error())
return
}
@ -114,7 +114,7 @@ func DeleteSearch(search string) error {
var ignore string
if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet, &ignore, &ignore, &ignore, &ignore); err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[db] %s", err.Error())
return
}
@ -193,6 +193,8 @@ func DeleteSearch(search string) error {
dbLastAction = time.Now()
dbDataDeleted = true
logMessagesDeleted(total)
BroadcastMailboxStats()
}

View File

@ -149,7 +149,7 @@ func GetAllTags() []string {
QueryAndClose(nil, db, func(row *sql.Rows) {
tags = append(tags, name)
}); err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[db] %s", err.Error())
}
return tags
@ -172,7 +172,7 @@ func GetAllTagsCount() map[string]int64 {
tags[name] = total
// tags = append(tags, name)
}); err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[db] %s", err.Error())
}
return tags
@ -193,7 +193,7 @@ func pruneUnusedTags() error {
var c int
if err := row.Scan(&id, &n, &c); err != nil {
logger.Log().Error("[tags]", err)
logger.Log().Errorf("[tags] %s", err.Error())
return
}

View File

@ -171,6 +171,8 @@ func dbCron() {
elapsed := time.Since(start)
logger.Log().Debugf("[db] auto-pruned %d messages in %s", len(ids), elapsed)
logMessagesDeleted(len(ids))
websockets.Broadcast("prune", nil)
}
}

6
package-lock.json generated
View File

@ -2218,9 +2218,9 @@
"optional": true
},
"node_modules/sass": {
"version": "1.69.6",
"resolved": "https://registry.npmjs.org/sass/-/sass-1.69.6.tgz",
"integrity": "sha512-qbRr3k9JGHWXCvZU77SD2OTwUlC+gNT+61JOLcmLm+XqH4h/5D+p4IIsxvpkB89S9AwJOyb5+rWNpIucaFxSFQ==",
"version": "1.69.7",
"resolved": "https://registry.npmjs.org/sass/-/sass-1.69.7.tgz",
"integrity": "sha512-rzj2soDeZ8wtE2egyLXgOOHQvaC2iosZrkF6v3EUG+tBwEvhqUCzm0VP3k9gHF9LXbSrRhT5SksoI56Iw8NPnQ==",
"dev": true,
"dependencies": {
"chokidar": ">=3.0.0 <4.0.0",

View File

@ -77,7 +77,7 @@ func Thumbnail(w http.ResponseWriter, r *http.Request) {
img, err := imaging.Decode(buf)
if err != nil {
// it's not an image, return default
logger.Log().Warning(err)
logger.Log().Warnf("[image] %s", err.Error())
blankImage(a, w)
return
}
@ -99,7 +99,7 @@ func Thumbnail(w http.ResponseWriter, r *http.Request) {
dst = imaging.OverlayCenter(dst, dstImageFill, 1.0)
if err := jpeg.Encode(foo, dst, &jpeg.Options{Quality: 70}); err != nil {
logger.Log().Warning(err)
logger.Log().Warnf("[image] %s", err.Error())
blankImage(a, w)
return
}
@ -120,7 +120,7 @@ func blankImage(a *enmime.Part, w http.ResponseWriter) {
dstImageFill := imaging.Fill(img, thumbWidth, thumbHeight, imaging.Center, imaging.Lanczos)
if err := jpeg.Encode(foo, dstImageFill, &jpeg.Options{Quality: 70}); err != nil {
logger.Log().Warning(err)
logger.Log().Warnf("[image] %s", err.Error())
}
fileName := a.FileName

View File

@ -95,7 +95,7 @@ func ProxyHandler(w http.ResponseWriter, r *http.Request) {
address, err := absoluteURL(parts[3], uri)
if err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[proxy] %s", err.Error())
return []byte(parts[3])
}

View File

@ -290,7 +290,7 @@ func index(w http.ResponseWriter, _ *http.Request) {
</head>
<body class="h-100">
<div class="container-fluid h-100 d-flex flex-column" id="app" data-webroot="{{ .Webroot }}">
<div class="container-fluid h-100 d-flex flex-column" id="app" data-webroot="{{ .Webroot }}" data-version="{{ .Version }}">
<noscript>You require JavaScript to use this app.</noscript>
</div>

View File

@ -28,7 +28,7 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
msg, err := mail.ReadMessage(bytes.NewReader(data))
if err != nil {
logger.Log().Errorf("[smtpd] error parsing message: %s", err.Error())
stats.LogSMTPError()
stats.LogSMTPRejected()
return err
}
@ -121,11 +121,10 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
_, err = storage.Store(&data)
if err != nil {
logger.Log().Errorf("[db] error storing message: %s", err.Error())
stats.LogSMTPError()
return err
}
stats.LogSMTPReceived(len(data))
stats.LogSMTPAccepted(len(data))
data = nil // avoid memory leaks
@ -153,6 +152,22 @@ func authHandlerAny(remoteAddr net.Addr, mechanism string, username []byte, _ []
return true, nil
}
// HandlerRcpt used to optionally restrict recipients based on `--smtp-allowed-recipients`
func handlerRcpt(remoteAddr net.Addr, from string, to string) bool {
if config.SMTPAllowedRecipientsRegexp == nil {
return true
}
result := config.SMTPAllowedRecipientsRegexp.MatchString(to)
if !result {
logger.Log().Warnf("[smtpd] rejected message to %s from %s (%s)", to, from, cleanIP(remoteAddr))
stats.LogSMTPRejected()
}
return result
}
// Listen starts the SMTPD server
func Listen() error {
if config.SMTPAuthAllowInsecure {
@ -178,6 +193,7 @@ func listenAndServe(addr string, handler smtpd.Handler, authHandler smtpd.AuthHa
srv := &smtpd.Server{
Addr: addr,
Handler: handler,
HandlerRcpt: handlerRcpt,
Appname: "Mailpit",
Hostname: "",
AuthHandler: nil,

View File

@ -240,19 +240,23 @@ export default {
</tr>
<tr>
<td>
SMTP messages received
SMTP messages accepted
</td>
<td>
{{ formatNumber(mailbox.appInfo.RuntimeStats.SMTPReceived) }}
({{ getFileSize(mailbox.appInfo.RuntimeStats.SMTPReceivedSize) }})
{{ formatNumber(mailbox.appInfo.RuntimeStats.SMTPAccepted) }}
<small class="text-secondary">
({{
getFileSize(mailbox.appInfo.RuntimeStats.SMTPAcceptedSize)
}})
</small>
</td>
</tr>
<tr>
<td>
SMTP errors
SMTP messages rejected
</td>
<td>
{{ formatNumber(mailbox.appInfo.RuntimeStats.SMTPErrors) }}
{{ formatNumber(mailbox.appInfo.RuntimeStats.SMTPRejected) }}
</td>
</tr>
<tr>

View File

@ -15,10 +15,16 @@ export default {
reconnectRefresh: false,
socketURI: false,
pauseNotifications: false, // prevent spamming
version: false
}
},
mounted() {
let d = document.getElementById('app')
if (d) {
this.version = d.dataset.version
}
let proto = location.protocol == 'https:' ? 'wss' : 'ws'
this.socketURI = proto + "://" + document.location.host + this.resolve(`/api/events`)
@ -35,10 +41,13 @@ export default {
let ws = new WebSocket(this.socketURI)
let self = this
ws.onmessage = function (e) {
let response = JSON.parse(e.data)
if (!response) {
let response
try {
response = JSON.parse(e.data)
} catch (e) {
return
}
// new messages
if (response.Type == "new" && response.Data) {
if (!mailbox.searching) {
@ -79,6 +88,11 @@ export default {
// refresh mailbox stats
mailbox.total = response.Data.Total
mailbox.unread = response.Data.Unread
// detect version updated, refresh is needed
if (self.version != response.Data.Version) {
location.reload()
}
}
}

View File

@ -758,27 +758,27 @@
"format": "uint64"
},
"MessagesDeleted": {
"description": "Messages deleted",
"description": "Database runtime messages deleted",
"type": "integer",
"format": "int64"
},
"SMTPErrors": {
"description": "SMTP errors since run",
"SMTPAccepted": {
"description": "Accepted runtime SMTP messages",
"type": "integer",
"format": "int64"
},
"SMTPAcceptedSize": {
"description": "Total runtime accepted messages size in bytes",
"type": "integer",
"format": "int64"
},
"SMTPIgnored": {
"description": "SMTP messages ignored since run (duplicate IDs)",
"description": "Ignored runtime SMTP messages (when using --ignore-duplicate-ids)",
"type": "integer",
"format": "int64"
},
"SMTPReceived": {
"description": "SMTP messages received via since run",
"type": "integer",
"format": "int64"
},
"SMTPReceivedSize": {
"description": "Total size in bytes of received messages since run",
"SMTPRejected": {
"description": "Rejected runtime SMTP messages",
"type": "integer",
"format": "int64"
},

View File

@ -132,7 +132,7 @@ func ServeWs(hub *Hub, w http.ResponseWriter, r *http.Request) {
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
logger.Log().Error(err)
logger.Log().Errorf("[websocket] %s", err.Error())
return
}