diff --git a/.github/workflows/test.yml b/.github/workflows/tests.yml similarity index 93% rename from .github/workflows/test.yml rename to .github/workflows/tests.yml index 5dfbd6a..237ff5a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/tests.yml @@ -1,9 +1,9 @@ -name: Test +name: Tests on: pull_request: branches: [ develop ] push: - branches: [ develop ] + branches: [ develop, 'feature/**' ] jobs: test: strategy: diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f5311d..f1b9da5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,16 @@ Notable changes to Mailpit will be documented in this file. +## 0.1.4 + +### Feature +- Email compression in storage + +### Testing +- Enable testing on feature branches +- Database total/unread statistics tests + + ## 0.1.3 ### Feature diff --git a/README.md b/README.md index 1dc3a32..74a5d97 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # Mailpit +![Tests](https://github.com/axllent/mailpit/actions/workflows/tests.yml/badge.svg) +![Build status](https://github.com/axllent/mailpit/actions/workflows/release-build.yml/badge.svg) +![Docker builds](https://github.com/axllent/mailpit/actions/workflows/build-docker.yml/badge.svg) +![CodeQL](https://github.com/axllent/mailpit/actions/workflows/codeql-analysis.yml/badge.svg) +[![Go Report Card](https://goreportcard.com/badge/github.com/axllent/mailpit)](https://goreportcard.com/report/github.com/axllent/mailpit) + Mailpit is an email testing tool for developers. It acts as both an SMTP server, and provides a web interface to view all captured emails. diff --git a/cmd/root.go b/cmd/root.go index da09ca3..30fe20c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -116,7 +116,7 @@ func init() { config.UISSLKey = os.Getenv("MP_SSL_KEY") } - rootCmd.Flags().StringVarP(&config.DataDir, "data", "d", config.DataDir, "Optional path to store peristent data") + rootCmd.Flags().StringVarP(&config.DataDir, "data", "d", config.DataDir, "Optional path to store persistent data") rootCmd.Flags().StringVarP(&config.SMTPListen, "smtp", "s", config.SMTPListen, "SMTP bind interface and port") rootCmd.Flags().StringVarP(&config.HTTPListen, "listen", "l", config.HTTPListen, "HTTP bind interface and port for UI") rootCmd.Flags().IntVarP(&config.MaxMessages, "max", "m", config.MaxMessages, "Max number of messages to store") diff --git a/data/message.go b/data/message.go index b08724a..b193402 100644 --- a/data/message.go +++ b/data/message.go @@ -17,7 +17,6 @@ type Message struct { Bcc []*mail.Address Subject string Date time.Time - Created time.Time Text string HTML string Size int diff --git a/go.mod b/go.mod index bdf7a07..2cf68d8 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/jhillyerd/enmime v0.10.0 github.com/k3a/html2text v1.0.8 + github.com/klauspost/compress v1.15.9 github.com/mhale/smtpd v0.8.0 github.com/ostafen/clover/v2 v2.0.0-alpha.2 github.com/sirupsen/logrus v1.9.0 @@ -35,7 +36,6 @@ require ( github.com/google/orderedcode v0.0.1 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba // indirect - github.com/klauspost/compress v1.15.9 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect diff --git a/screenshot.png b/screenshot.png index ef3a4fb..2c8a065 100644 Binary files a/screenshot.png and b/screenshot.png differ diff --git a/server/ui-src/App.vue b/server/ui-src/App.vue index 83cc588..ace30a2 100644 --- a/server/ui-src/App.vue +++ b/server/ui-src/App.vue @@ -436,13 +436,13 @@ export default { {{ getRelativeCreated(message) }} -
+
{{ message.From.Name ? message.From.Name : message.From.Address }}
-
+
{{ message.From.Name ? message.From.Name : message.From.Address }}
-
+
{{ getPrimaryEmailTo(message) }} [+{{message.To.length - 1}}] diff --git a/server/ui-src/assets/styles.scss b/server/ui-src/assets/styles.scss index 15c78b0..10f35a4 100644 --- a/server/ui-src/assets/styles.scss +++ b/server/ui-src/assets/styles.scss @@ -57,3 +57,9 @@ .list-group-item:first-child { border-top: 0; } + +body.blur { + .privacy { + filter: blur(3px); + } +} diff --git a/server/ui-src/templates/Message.vue b/server/ui-src/templates/Message.vue index 77b2f3a..1370014 100644 --- a/server/ui-src/templates/Message.vue +++ b/server/ui-src/templates/Message.vue @@ -78,7 +78,7 @@ export default { From - + {{ message.From.Name + " " }} <{{ message.From.Address }}> @@ -90,7 +90,7 @@ export default { To - + {{ t.Name + " <" + t.Address +">" }} @@ -99,7 +99,7 @@ export default { CC - + {{ t.Name + " <" + t.Address +">" }} @@ -108,7 +108,7 @@ export default { CC - + {{ t.Name + " <" + t.Address +">" }} diff --git a/storage/database.go b/storage/database.go index 7b2c31b..063e4ab 100644 --- a/storage/database.go +++ b/storage/database.go @@ -17,6 +17,7 @@ import ( "github.com/axllent/mailpit/logger" "github.com/axllent/mailpit/server/websockets" "github.com/jhillyerd/enmime" + "github.com/klauspost/compress/zstd" "github.com/ostafen/clover/v2" ) @@ -28,6 +29,10 @@ var ( count int per100start = time.Now() + + // zstd encoder & decoder + encoder, _ = zstd.NewWriter(nil) + decoder, _ = zstd.NewReader(nil) ) // CloverStore struct @@ -65,7 +70,7 @@ func InitDB() error { // method invoked upon seeing signal go func() { s := <-sigs - logger.Log().Infof("[db] got %s signal, saving persistant data & shutting down", s) + logger.Log().Infof("[db] got %s signal, saving persistent data & shutting down", s) if err := db.Close(); err != nil { logger.Log().Errorf("[db] %s", err.Error()) } @@ -219,7 +224,10 @@ func Store(mailbox string, b []byte) (string, error) { raw := clover.NewDocument() raw.Set("_id", id) raw.Set("Created", time.Now()) - raw.Set("Data", string(b)) + + compressed := encoder.EncodeAll(b, make([]byte, 0, len(b))) + raw.Set("Email", string(compressed)) + _, err = db.InsertOne(mailbox+"_data", raw) if err != nil { // delete the summary because the data insert failed @@ -369,18 +377,12 @@ func CountUnread(mailbox string) (int, error) { func GetMessage(mailbox, id string) (*data.Message, error) { mailbox = sanitizeMailboxName(mailbox) - q, err := db.FindById(mailbox+"_data", id) - if err != nil { - return nil, err - } - - if q == nil { + raw, err := GetMessageRaw(mailbox, id) + if err != nil || raw == nil { return nil, errors.New("message not found") } - raw := q.Get("Data").(string) - - r := bytes.NewReader([]byte(raw)) + r := bytes.NewReader(raw) env, err := enmime.ReadEnvelope(r) if err != nil { @@ -398,9 +400,8 @@ func GetMessage(mailbox, id string) (*data.Message, error) { date, _ := env.Date() obj := data.Message{ - ID: q.ObjectId(), + ID: id, Read: true, - Created: q.Get("Created").(time.Time), From: from, Date: date, To: addressToSlice(env, "To"), @@ -456,12 +457,12 @@ func GetMessage(mailbox, id string) (*data.Message, error) { func GetAttachmentPart(mailbox, id, partID string) (*enmime.Part, error) { mailbox = sanitizeMailboxName(mailbox) - data, err := GetMessageRaw(mailbox, id) + raw, err := GetMessageRaw(mailbox, id) if err != nil { return nil, err } - r := bytes.NewReader(data) + r := bytes.NewReader(raw) env, err := enmime.ReadEnvelope(r) if err != nil { @@ -502,9 +503,20 @@ func GetMessageRaw(mailbox, id string) ([]byte, error) { return nil, errors.New("message not found") } - data := q.Get("Data").(string) + var raw []byte - return []byte(data), err + if q.Has("Email") { + msg := q.Get("Email").(string) + raw, err = decoder.DecodeAll([]byte(msg), nil) + if err != nil { + return nil, fmt.Errorf("error decompressing message: %s", err.Error()) + } + } else { + // deprecated 2022/08/10 - can be eventually removed + raw = []byte(q.Get("Data").(string)) + } + + return raw, err } // UnreadMessage will delete all messages from a mailbox diff --git a/storage/database_test.go b/storage/database_test.go index 5f58e43..e38b45a 100644 --- a/storage/database_test.go +++ b/storage/database_test.go @@ -12,11 +12,13 @@ import ( "github.com/axllent/mailpit/config" "github.com/jhillyerd/enmime" + "github.com/ostafen/clover/v2" ) var ( testTextEmail []byte testMimeEmail []byte + testRuns = 1000 ) func TestTextEmailInserts(t *testing.T) { @@ -25,7 +27,10 @@ func TestTextEmailInserts(t *testing.T) { RepeatTest: start := time.Now() - for i := 0; i < 1000; i++ { + + assertEqualStats(t, 0, 0) + + for i := 0; i < testRuns; i++ { if _, err := Store(DefaultMailbox, testTextEmail); err != nil { t.Log("error ", err) t.Fail() @@ -38,9 +43,11 @@ RepeatTest: t.Fail() } - assertEqual(t, count, 1000, "incorrect number of text emails stored") + assertEqual(t, count, testRuns, "incorrect number of text emails stored") - t.Logf("inserted 1,000 text emails in %s\n", time.Since(start)) + t.Logf("inserted %d text emails in %s", testRuns, time.Since(start)) + + assertEqualStats(t, testRuns, testRuns) delStart := time.Now() if err := DeleteAllMessages(DefaultMailbox); err != nil { @@ -56,7 +63,9 @@ RepeatTest: assertEqual(t, count, 0, "incorrect number of text emails deleted") - t.Logf("deleted 1,000 text emails in %s\n", time.Since(delStart)) + t.Logf("deleted %d text emails in %s", testRuns, time.Since(delStart)) + + assertEqualStats(t, 0, 0) db.Close() if config.DataDir == "" { @@ -74,7 +83,10 @@ func TestMimeEmailInserts(t *testing.T) { RepeatTest: start := time.Now() - for i := 0; i < 1000; i++ { + + assertEqualStats(t, 0, 0) + + for i := 0; i < testRuns; i++ { if _, err := Store(DefaultMailbox, testMimeEmail); err != nil { t.Log("error ", err) t.Fail() @@ -87,9 +99,11 @@ RepeatTest: t.Fail() } - assertEqual(t, count, 1000, "incorrect number of mime emails stored") + assertEqual(t, count, testRuns, "incorrect number of emails with mime attachments stored") - t.Logf("inserted 1,000 emails with mime attachments in %s\n", time.Since(start)) + t.Logf("inserted %d emails with mime attachments in %s", testRuns, time.Since(start)) + + assertEqualStats(t, testRuns, testRuns) delStart := time.Now() if err := DeleteAllMessages(DefaultMailbox); err != nil { @@ -103,9 +117,11 @@ RepeatTest: t.Fail() } - assertEqual(t, count, 0, "incorrect number of mime emails deleted") + assertEqual(t, count, 0, "incorrect number of emails with mime attachments deleted") - t.Logf("deleted 1,000 mime emails in %s\n", time.Since(delStart)) + t.Logf("deleted %d emails with mime attachments in %s", testRuns, time.Since(delStart)) + + assertEqualStats(t, 0, 0) db.Close() if config.DataDir == "" { @@ -158,12 +174,56 @@ RepeatTest: } } +func TestDatabaseStats(t *testing.T) { + setup(false) + t.Log("Testing database stats") + assertEqualStats(t, 0, 0) + + for i := 0; i < 100; i++ { + if _, err := Store(DefaultMailbox, testTextEmail); err != nil { + t.Log("error ", err) + t.Fail() + } + } + + assertEqualStats(t, 100, 100) + + // mark 10 as read + docs, err := db.FindAll( + clover.NewQuery(DefaultMailbox). + Limit(10), + ) + if err != nil { + t.Log("error ", err) + t.Fail() + } + + for _, d := range docs { + _, err := GetMessage(DefaultMailbox, d.ObjectId()) + if err != nil { + t.Log("error ", err) + t.Fail() + } + } + + assertEqualStats(t, 100, 90) + + if err := MarkAllRead(DefaultMailbox); err != nil { + t.Log("error ", err) + t.Fail() + } + + assertEqualStats(t, 100, 0) + + db.Close() +} + func TestSearch(t *testing.T) { setup(false) t.Log("Testing memory storage") RepeatTest: - for i := 0; i < 1000; i++ { + for i := 0; i < testRuns; i++ { msg := enmime.Builder(). From(fmt.Sprintf("From %d", i), fmt.Sprintf("from-%d@example.com", i)). Subject(fmt.Sprintf("Subject line %d end", i)). @@ -189,7 +249,7 @@ RepeatTest: } } - for i := 1; i < 101; i++ { + for i := 1; i < 51; i++ { // search a random something that will return a single result searchIndx := rand.Intn(4) + 1 var search string @@ -204,7 +264,7 @@ RepeatTest: search = fmt.Sprintf("the email body %d jdsauk dwqmdqw", i) } - summaries, err := Search(DefaultMailbox, search, 0, 200) + summaries, err := Search(DefaultMailbox, search, 0, 10) if err != nil { t.Log("error ", err) t.Fail() @@ -220,12 +280,12 @@ RepeatTest: } // search something that will return 200 rsults - summaries, err := Search(DefaultMailbox, "This is the email body", 0, 200) + summaries, err := Search(DefaultMailbox, "This is the email body", 0, 50) if err != nil { t.Log("error ", err) t.Fail() } - assertEqual(t, len(summaries), 200, "200 search results expected") + assertEqual(t, len(summaries), 50, "50 search results expected") db.Close() @@ -296,3 +356,14 @@ func assertEqual(t *testing.T, a interface{}, b interface{}, message string) { message = fmt.Sprintf("%s: \"%v\" != \"%v\"", message, a, b) t.Fatal(message) } + +func assertEqualStats(t *testing.T, total int, unread int) { + s := StatsGet(DefaultMailbox) + if total != s.Total { + t.Fatal(fmt.Sprintf("Incorrect total mailbox stats: \"%d\" != \"%d\"", total, s.Total)) + } + + if unread != s.Unread { + t.Fatal(fmt.Sprintf("Incorrect unread mailbox stats: \"%d\" != \"%d\"", unread, s.Unread)) + } +} diff --git a/storage/utils.go b/storage/utils.go index 2cd2c19..7b5f20d 100644 --- a/storage/utils.go +++ b/storage/utils.go @@ -56,11 +56,10 @@ func cleanString(str string) string { return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(str)), " ")) } -// Auto-prune runs every 5 minutes to automatically delete oldest messages +// Auto-prune runs every minute to automatically delete oldest messages // if total is greater than the threshold func pruneCron() { for { - // time.Sleep(5 * 60 * time.Second) time.Sleep(60 * time.Second) mailboxes, err := db.ListCollections() if err != nil { @@ -94,7 +93,7 @@ func pruneCron() { } // SanitizeMailboxName returns a clean mailbox name -// allowing only `alphanumeric` characters and `-`` +// allowing only `alphanumeric` characters and `-“ func sanitizeMailboxName(mailbox string) string { re := regexp.MustCompile(`[^a-zA-Z0-9\-]`)