Merge branch 'release/0.1.4'

.github/workflows/tests.yml

@@ -1,9 +1,9 @@
-name: Test
+name: Tests
 on:
   pull_request:
     branches: [ develop ]
   push:
-    branches: [ develop ]
+    branches: [ develop, 'feature/**' ]
 jobs:
   test:
     strategy:

CHANGELOG.md

@@ -3,6 +3,16 @@
 Notable changes to Mailpit will be documented in this file.
 
 
+## 0.1.4
+
+### Feature
+- Email compression in storage
+
+### Testing
+- Enable testing on feature branches
+- Database total/unread statistics tests
+
+
 ## 0.1.3
 
 ### Feature

README.md

@@ -1,5 +1,11 @@
 # Mailpit
 
+![Tests](https://github.com/axllent/mailpit/actions/workflows/tests.yml/badge.svg)
+![Build status](https://github.com/axllent/mailpit/actions/workflows/release-build.yml/badge.svg)
+![Docker builds](https://github.com/axllent/mailpit/actions/workflows/build-docker.yml/badge.svg)
+![CodeQL](https://github.com/axllent/mailpit/actions/workflows/codeql-analysis.yml/badge.svg)
+[![Go Report Card](https://goreportcard.com/badge/github.com/axllent/mailpit)](https://goreportcard.com/report/github.com/axllent/mailpit)
+
 Mailpit is an email testing tool for developers.
 
 It acts as both an SMTP server, and provides a web interface to view all captured emails.

cmd/root.go

@@ -116,7 +116,7 @@ func init() {
 		config.UISSLKey = os.Getenv("MP_SSL_KEY")
 	}
 
-	rootCmd.Flags().StringVarP(&config.DataDir, "data", "d", config.DataDir, "Optional path to store peristent data")
+	rootCmd.Flags().StringVarP(&config.DataDir, "data", "d", config.DataDir, "Optional path to store persistent data")
 	rootCmd.Flags().StringVarP(&config.SMTPListen, "smtp", "s", config.SMTPListen, "SMTP bind interface and port")
 	rootCmd.Flags().StringVarP(&config.HTTPListen, "listen", "l", config.HTTPListen, "HTTP bind interface and port for UI")
 	rootCmd.Flags().IntVarP(&config.MaxMessages, "max", "m", config.MaxMessages, "Max number of messages to store")

data/message.go

@@ -17,7 +17,6 @@ type Message struct {
 	Bcc         []*mail.Address
 	Subject     string
 	Date        time.Time
-	Created     time.Time
 	Text        string
 	HTML        string
 	Size        int

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/gorilla/websocket v1.5.0
 	github.com/jhillyerd/enmime v0.10.0
 	github.com/k3a/html2text v1.0.8
+	github.com/klauspost/compress v1.15.9
 	github.com/mhale/smtpd v0.8.0
 	github.com/ostafen/clover/v2 v2.0.0-alpha.2
 	github.com/sirupsen/logrus v1.9.0
@@ -35,7 +36,6 @@ require (
 	github.com/google/orderedcode v0.0.1 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba // indirect
-	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect

server/ui-src/App.vue

@@ -436,13 +436,13 @@ export default {
 								{{ getRelativeCreated(message) }}
 							</div>
 
-							<div class="text-truncate d-md-none">
+							<div class="text-truncate d-md-none privacy">
 								<span v-if="message.From" :title="message.From.Address">{{ message.From.Name ? message.From.Name : message.From.Address }}</span>
 							</div> 
-							<div class="text-truncate d-none d-md-block">
+							<div class="text-truncate d-none d-md-block privacy">
 								<b v-if="message.From" :title="message.From.Address">{{ message.From.Name ? message.From.Name : message.From.Address }}</b>
 							</div>
-							<div class="d-none d-md-block text-truncate text-muted small">
+							<div class="d-none d-md-block text-truncate text-muted small privacy">
 								{{ getPrimaryEmailTo(message) }}
 								<span v-if="message.To && message.To.length > 1">
 									[+{{message.To.length - 1}}]

server/ui-src/assets/styles.scss

@@ -57,3 +57,9 @@
 .list-group-item:first-child {
     border-top: 0;
 }
+
+body.blur {
+    .privacy {
+        filter: blur(3px);
+    }
+}

server/ui-src/templates/Message.vue

@@ -78,7 +78,7 @@ export default {
 			<tbody>
 				<tr class="small">
 					<th>From</th>
-					<td>
+					<td class="privacy">
 						<span v-if="message.From">
 							<span v-if="message.From.Name">{{ message.From.Name + " " }}</span>
 							<span v-if="message.From.Address">&lt;{{ message.From.Address }}&gt;</span>
@@ -90,7 +90,7 @@ export default {
 				</tr>
 				<tr class="small">
 					<th>To</th>
-					<td>
+					<td class="privacy">
 						<span v-for="(t, i) in message.To">
 							<template v-if="i > 0">,</template>
 							{{ t.Name + " <" + t.Address +">" }}
@@ -99,7 +99,7 @@ export default {
 				</tr>
 				<tr v-if="message.Cc" class="small">
 					<th>CC</th>
-					<td>
+					<td class="privacy">
 						<span v-for="(t, i) in message.Cc">
 							<template v-if="i > 0">,</template>
 							{{ t.Name + " <" + t.Address +">" }}
@@ -108,7 +108,7 @@ export default {
 				</tr>
 				<tr v-if="message.Bcc" class="small">
 					<th>CC</th>
-					<td>
+					<td class="privacy">
 						<span v-for="(t, i) in message.Bcc">
 							<template v-if="i > 0">,</template>
 							{{ t.Name + " <" + t.Address +">" }}

storage/database.go

@@ -17,6 +17,7 @@ import (
 	"github.com/axllent/mailpit/logger"
 	"github.com/axllent/mailpit/server/websockets"
 	"github.com/jhillyerd/enmime"
+	"github.com/klauspost/compress/zstd"
 	"github.com/ostafen/clover/v2"
 )
 
@@ -28,6 +29,10 @@ var (
 
 	count       int
 	per100start = time.Now()
+
+	// zstd encoder & decoder
+	encoder, _ = zstd.NewWriter(nil)
+	decoder, _ = zstd.NewReader(nil)
 )
 
 // CloverStore struct
@@ -65,7 +70,7 @@ func InitDB() error {
 		// method invoked upon seeing signal
 		go func() {
 			s := <-sigs
-			logger.Log().Infof("[db] got %s signal, saving persistant data & shutting down", s)
+			logger.Log().Infof("[db] got %s signal, saving persistent data & shutting down", s)
 			if err := db.Close(); err != nil {
 				logger.Log().Errorf("[db] %s", err.Error())
 			}
@@ -219,7 +224,10 @@ func Store(mailbox string, b []byte) (string, error) {
 	raw := clover.NewDocument()
 	raw.Set("_id", id)
 	raw.Set("Created", time.Now())
-	raw.Set("Data", string(b))
+
+	compressed := encoder.EncodeAll(b, make([]byte, 0, len(b)))
+	raw.Set("Email", string(compressed))
+
 	_, err = db.InsertOne(mailbox+"_data", raw)
 	if err != nil {
 		// delete the summary because the data insert failed
@@ -369,18 +377,12 @@ func CountUnread(mailbox string) (int, error) {
 func GetMessage(mailbox, id string) (*data.Message, error) {
 	mailbox = sanitizeMailboxName(mailbox)
 
-	q, err := db.FindById(mailbox+"_data", id)
-	if err != nil {
-		return nil, err
-	}
-
-	if q == nil {
+	raw, err := GetMessageRaw(mailbox, id)
+	if err != nil || raw == nil {
 		return nil, errors.New("message not found")
 	}
 
-	raw := q.Get("Data").(string)
-
-	r := bytes.NewReader([]byte(raw))
+	r := bytes.NewReader(raw)
 
 	env, err := enmime.ReadEnvelope(r)
 	if err != nil {
@@ -398,9 +400,8 @@ func GetMessage(mailbox, id string) (*data.Message, error) {
 	date, _ := env.Date()
 
 	obj := data.Message{
-		ID:      q.ObjectId(),
+		ID:      id,
 		Read:    true,
-		Created: q.Get("Created").(time.Time),
 		From:    from,
 		Date:    date,
 		To:      addressToSlice(env, "To"),
@@ -456,12 +457,12 @@ func GetMessage(mailbox, id string) (*data.Message, error) {
 func GetAttachmentPart(mailbox, id, partID string) (*enmime.Part, error) {
 	mailbox = sanitizeMailboxName(mailbox)
 
-	data, err := GetMessageRaw(mailbox, id)
+	raw, err := GetMessageRaw(mailbox, id)
 	if err != nil {
 		return nil, err
 	}
 
-	r := bytes.NewReader(data)
+	r := bytes.NewReader(raw)
 
 	env, err := enmime.ReadEnvelope(r)
 	if err != nil {
@@ -502,9 +503,20 @@ func GetMessageRaw(mailbox, id string) ([]byte, error) {
 		return nil, errors.New("message not found")
 	}
 
-	data := q.Get("Data").(string)
+	var raw []byte
 
-	return []byte(data), err
+	if q.Has("Email") {
+		msg := q.Get("Email").(string)
+		raw, err = decoder.DecodeAll([]byte(msg), nil)
+		if err != nil {
+			return nil, fmt.Errorf("error decompressing message: %s", err.Error())
+		}
+	} else {
+		// deprecated 2022/08/10 - can be eventually removed
+		raw = []byte(q.Get("Data").(string))
+	}
+
+	return raw, err
 }
 
 // UnreadMessage will delete all messages from a mailbox

storage/database_test.go

@@ -12,11 +12,13 @@ import (
 
 	"github.com/axllent/mailpit/config"
 	"github.com/jhillyerd/enmime"
+	"github.com/ostafen/clover/v2"
 )
 
 var (
 	testTextEmail []byte
 	testMimeEmail []byte
+	testRuns      = 1000
 )
 
 func TestTextEmailInserts(t *testing.T) {
@@ -25,7 +27,10 @@ func TestTextEmailInserts(t *testing.T) {
 
 RepeatTest:
 	start := time.Now()
-	for i := 0; i < 1000; i++ {
+
+	assertEqualStats(t, 0, 0)
+
+	for i := 0; i < testRuns; i++ {
 		if _, err := Store(DefaultMailbox, testTextEmail); err != nil {
 			t.Log("error ", err)
 			t.Fail()
@@ -38,9 +43,11 @@ RepeatTest:
 		t.Fail()
 	}
 
-	assertEqual(t, count, 1000, "incorrect number of text emails stored")
+	assertEqual(t, count, testRuns, "incorrect number of text emails stored")
 
-	t.Logf("inserted 1,000 text emails in %s\n", time.Since(start))
+	t.Logf("inserted %d text emails in %s", testRuns, time.Since(start))
+
+	assertEqualStats(t, testRuns, testRuns)
 
 	delStart := time.Now()
 	if err := DeleteAllMessages(DefaultMailbox); err != nil {
@@ -56,7 +63,9 @@ RepeatTest:
 
 	assertEqual(t, count, 0, "incorrect number of text emails deleted")
 
-	t.Logf("deleted 1,000 text emails in %s\n", time.Since(delStart))
+	t.Logf("deleted %d text emails in %s", testRuns, time.Since(delStart))
+
+	assertEqualStats(t, 0, 0)
 
 	db.Close()
 	if config.DataDir == "" {
@@ -74,7 +83,10 @@ func TestMimeEmailInserts(t *testing.T) {
 
 RepeatTest:
 	start := time.Now()
-	for i := 0; i < 1000; i++ {
+
+	assertEqualStats(t, 0, 0)
+
+	for i := 0; i < testRuns; i++ {
 		if _, err := Store(DefaultMailbox, testMimeEmail); err != nil {
 			t.Log("error ", err)
 			t.Fail()
@@ -87,9 +99,11 @@ RepeatTest:
 		t.Fail()
 	}
 
-	assertEqual(t, count, 1000, "incorrect number of mime emails stored")
+	assertEqual(t, count, testRuns, "incorrect number of emails with mime attachments stored")
 
-	t.Logf("inserted 1,000 emails with mime attachments in %s\n", time.Since(start))
+	t.Logf("inserted %d emails with mime attachments in %s", testRuns, time.Since(start))
+
+	assertEqualStats(t, testRuns, testRuns)
 
 	delStart := time.Now()
 	if err := DeleteAllMessages(DefaultMailbox); err != nil {
@@ -103,9 +117,11 @@ RepeatTest:
 		t.Fail()
 	}
 
-	assertEqual(t, count, 0, "incorrect number of mime emails deleted")
+	assertEqual(t, count, 0, "incorrect number of emails with mime attachments deleted")
 
-	t.Logf("deleted 1,000 mime emails in %s\n", time.Since(delStart))
+	t.Logf("deleted %d emails with mime attachments in %s", testRuns, time.Since(delStart))
+
+	assertEqualStats(t, 0, 0)
 
 	db.Close()
 	if config.DataDir == "" {
@@ -158,12 +174,56 @@ RepeatTest:
 	}
 }
 
+func TestDatabaseStats(t *testing.T) {
+	setup(false)
+	t.Log("Testing database stats")
+	assertEqualStats(t, 0, 0)
+
+	for i := 0; i < 100; i++ {
+		if _, err := Store(DefaultMailbox, testTextEmail); err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+
+	assertEqualStats(t, 100, 100)
+
+	// mark 10 as read
+	docs, err := db.FindAll(
+		clover.NewQuery(DefaultMailbox).
+			Limit(10),
+	)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	for _, d := range docs {
+		_, err := GetMessage(DefaultMailbox, d.ObjectId())
+		if err != nil {
+			t.Log("error ", err)
+			t.Fail()
+		}
+	}
+
+	assertEqualStats(t, 100, 90)
+
+	if err := MarkAllRead(DefaultMailbox); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqualStats(t, 100, 0)
+
+	db.Close()
+}
+
 func TestSearch(t *testing.T) {
 	setup(false)
 	t.Log("Testing memory storage")
 
 RepeatTest:
-	for i := 0; i < 1000; i++ {
+	for i := 0; i < testRuns; i++ {
 		msg := enmime.Builder().
 			From(fmt.Sprintf("From %d", i), fmt.Sprintf("from-%d@example.com", i)).
 			Subject(fmt.Sprintf("Subject line %d end", i)).
@@ -189,7 +249,7 @@ RepeatTest:
 		}
 	}
 
-	for i := 1; i < 101; i++ {
+	for i := 1; i < 51; i++ {
 		// search a random something that will return a single result
 		searchIndx := rand.Intn(4) + 1
 		var search string
@@ -204,7 +264,7 @@ RepeatTest:
 			search = fmt.Sprintf("the email body %d jdsauk dwqmdqw", i)
 		}
 
-		summaries, err := Search(DefaultMailbox, search, 0, 200)
+		summaries, err := Search(DefaultMailbox, search, 0, 10)
 		if err != nil {
 			t.Log("error ", err)
 			t.Fail()
@@ -220,12 +280,12 @@ RepeatTest:
 	}
 
 	// search something that will return 200 rsults
-	summaries, err := Search(DefaultMailbox, "This is the email body", 0, 200)
+	summaries, err := Search(DefaultMailbox, "This is the email body", 0, 50)
 	if err != nil {
 		t.Log("error ", err)
 		t.Fail()
 	}
-	assertEqual(t, len(summaries), 200, "200 search results expected")
+	assertEqual(t, len(summaries), 50, "50 search results expected")
 
 	db.Close()
 
@@ -296,3 +356,14 @@ func assertEqual(t *testing.T, a interface{}, b interface{}, message string) {
 	message = fmt.Sprintf("%s: \"%v\" != \"%v\"", message, a, b)
 	t.Fatal(message)
 }
+
+func assertEqualStats(t *testing.T, total int, unread int) {
+	s := StatsGet(DefaultMailbox)
+	if total != s.Total {
+		t.Fatal(fmt.Sprintf("Incorrect total mailbox stats: \"%d\" != \"%d\"", total, s.Total))
+	}
+
+	if unread != s.Unread {
+		t.Fatal(fmt.Sprintf("Incorrect unread mailbox stats: \"%d\" != \"%d\"", unread, s.Unread))
+	}
+}

storage/utils.go

@@ -56,11 +56,10 @@ func cleanString(str string) string {
 	return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(str)), " "))
 }
 
-// Auto-prune runs every 5 minutes to automatically delete oldest messages
+// Auto-prune runs every minute to automatically delete oldest messages
 // if total is greater than the threshold
 func pruneCron() {
 	for {
-		// time.Sleep(5 * 60 * time.Second)
 		time.Sleep(60 * time.Second)
 		mailboxes, err := db.ListCollections()
 		if err != nil {
@@ -94,7 +93,7 @@ func pruneCron() {
 }
 
 // SanitizeMailboxName returns a clean mailbox name
-// allowing only `alphanumeric` characters and `-``
+// allowing only `alphanumeric` characters and `-“
 func sanitizeMailboxName(mailbox string) string {
 	re := regexp.MustCompile(`[^a-zA-Z0-9\-]`)