go/mailpit
1
0
Fork 0
mirror of https://github.com/axllent/mailpit.git synced 2025-03-17 21:18:19 +02:00
Code Issues Releases Activity

Feature: Email compression in storage

Browse Source 
Reduces storage requirements +-25% & speeds up database read & writes by between 25-33%, depending on email content (attachments).
This commit is contained in:
Ralph Slooten 2022-08-10 14:33:16 +12:00
parent cc15ada304
commit 86cc237c78
5 changed files with 48 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
data/message.go
View File

@ -17,7 +17,6 @@ type Message struct {
Bcc []*mail.Address
Subject string
Date time.Time
Created time.Time
Text string
HTML string
Size int

2
go.mod
View File

@ -8,6 +8,7 @@ require (
github.com/gorilla/websocket v1.5.0
github.com/jhillyerd/enmime v0.10.0
github.com/k3a/html2text v1.0.8
github.com/klauspost/compress v1.15.9
github.com/mhale/smtpd v0.8.0
github.com/ostafen/clover/v2 v2.0.0-alpha.2
github.com/sirupsen/logrus v1.9.0
@ -35,7 +36,6 @@ require (
github.com/google/orderedcode v0.0.1 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba // indirect
github.com/klauspost/compress v1.15.9 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect

44
storage/database.go
View File

@ -17,6 +17,7 @@ import (
"github.com/axllent/mailpit/logger"
"github.com/axllent/mailpit/server/websockets"
"github.com/jhillyerd/enmime"
"github.com/klauspost/compress/zstd"
"github.com/ostafen/clover/v2"
)
@ -28,6 +29,10 @@ var (
count int
per100start = time.Now()
// zstd encoder & decoder
encoder, _ = zstd.NewWriter(nil)
decoder, _ = zstd.NewReader(nil)
)
// CloverStore struct
@ -219,7 +224,10 @@ func Store(mailbox string, b []byte) (string, error) {
raw := clover.NewDocument()
raw.Set("_id", id)
raw.Set("Created", time.Now())
raw.Set("Data", string(b))
compressed := encoder.EncodeAll(b, make([]byte, 0, len(b)))
raw.Set("Email", string(compressed))
_, err = db.InsertOne(mailbox+"_data", raw)
if err != nil {
// delete the summary because the data insert failed
@ -369,18 +377,12 @@ func CountUnread(mailbox string) (int, error) {
func GetMessage(mailbox, id string) (*data.Message, error) {
mailbox = sanitizeMailboxName(mailbox)
q, err := db.FindById(mailbox+"_data", id)
if err != nil {
return nil, err
}
if q == nil {
raw, err := GetMessageRaw(mailbox, id)
if err != nil || raw == nil {
return nil, errors.New("message not found")
}
raw := q.Get("Data").(string)
r := bytes.NewReader([]byte(raw))
r := bytes.NewReader(raw)
env, err := enmime.ReadEnvelope(r)
if err != nil {
@ -398,9 +400,8 @@ func GetMessage(mailbox, id string) (*data.Message, error) {
date, _ := env.Date()
obj := data.Message{
ID: q.ObjectId(),
ID: id,
Read: true,
Created: q.Get("Created").(time.Time),
From: from,
Date: date,
To: addressToSlice(env, "To"),
@ -456,12 +457,12 @@ func GetMessage(mailbox, id string) (*data.Message, error) {
func GetAttachmentPart(mailbox, id, partID string) (*enmime.Part, error) {
mailbox = sanitizeMailboxName(mailbox)
data, err := GetMessageRaw(mailbox, id)
raw, err := GetMessageRaw(mailbox, id)
if err != nil {
return nil, err
}
r := bytes.NewReader(data)
r := bytes.NewReader(raw)
env, err := enmime.ReadEnvelope(r)
if err != nil {
@ -502,9 +503,20 @@ func GetMessageRaw(mailbox, id string) ([]byte, error) {
return nil, errors.New("message not found")
}
data := q.Get("Data").(string)
var raw []byte
return []byte(data), err
if q.Has("Email") {
msg := q.Get("Email").(string)
raw, err = decoder.DecodeAll([]byte(msg), nil)
if err != nil {
return nil, fmt.Errorf("error decompressing message: %s", err.Error())
}
} else {
// deprecated 2022/08/10 - can be eventually removed
raw = []byte(q.Get("Data").(string))
}
return raw, err
}
// UnreadMessage will delete all messages from a mailbox

33
storage/database_test.go
View File

@ -18,6 +18,7 @@ import (
var (
testTextEmail []byte
testMimeEmail []byte
testRuns = 1000
)
func TestTextEmailInserts(t *testing.T) {
@ -29,7 +30,7 @@ RepeatTest:
assertEqualStats(t, 0, 0)
for i := 0; i < 1000; i++ {
for i := 0; i < testRuns; i++ {
if _, err := Store(DefaultMailbox, testTextEmail); err != nil {
t.Log("error ", err)
t.Fail()
@ -42,11 +43,11 @@ RepeatTest:
t.Fail()
}
assertEqual(t, count, 1000, "incorrect number of text emails stored")
assertEqual(t, count, testRuns, "incorrect number of text emails stored")
t.Logf("inserted 1,000 text emails in %s\n", time.Since(start))
t.Logf("inserted %d text emails in %s", testRuns, time.Since(start))
assertEqualStats(t, 1000, 1000)
assertEqualStats(t, testRuns, testRuns)
delStart := time.Now()
if err := DeleteAllMessages(DefaultMailbox); err != nil {
@ -62,7 +63,7 @@ RepeatTest:
assertEqual(t, count, 0, "incorrect number of text emails deleted")
t.Logf("deleted 1,000 text emails in %s\n", time.Since(delStart))
t.Logf("deleted %d text emails in %s", testRuns, time.Since(delStart))
assertEqualStats(t, 0, 0)
@ -85,7 +86,7 @@ RepeatTest:
assertEqualStats(t, 0, 0)
for i := 0; i < 1000; i++ {
for i := 0; i < testRuns; i++ {
if _, err := Store(DefaultMailbox, testMimeEmail); err != nil {
t.Log("error ", err)
t.Fail()
@ -98,11 +99,11 @@ RepeatTest:
t.Fail()
}
assertEqual(t, count, 1000, "incorrect number of mime emails stored")
assertEqual(t, count, testRuns, "incorrect number of emails with mime attachments stored")
t.Logf("inserted 1,000 emails with mime attachments in %s\n", time.Since(start))
t.Logf("inserted %d emails with mime attachments in %s", testRuns, time.Since(start))
assertEqualStats(t, 1000, 1000)
assertEqualStats(t, testRuns, testRuns)
delStart := time.Now()
if err := DeleteAllMessages(DefaultMailbox); err != nil {
@ -116,9 +117,9 @@ RepeatTest:
t.Fail()
}
assertEqual(t, count, 0, "incorrect number of mime emails deleted")
assertEqual(t, count, 0, "incorrect number of emails with mime attachments deleted")
t.Logf("deleted 1,000 mime emails in %s\n", time.Since(delStart))
t.Logf("deleted %d emails with mime attachments in %s", testRuns, time.Since(delStart))
assertEqualStats(t, 0, 0)
@ -222,7 +223,7 @@ func TestSearch(t *testing.T) {
t.Log("Testing memory storage")
RepeatTest:
for i := 0; i < 1000; i++ {
for i := 0; i < testRuns; i++ {
msg := enmime.Builder().
From(fmt.Sprintf("From %d", i), fmt.Sprintf("from-%d@example.com", i)).
Subject(fmt.Sprintf("Subject line %d end", i)).
@ -248,7 +249,7 @@ RepeatTest:
}
}
for i := 1; i < 101; i++ {
for i := 1; i < 51; i++ {
// search a random something that will return a single result
searchIndx := rand.Intn(4) + 1
var search string
@ -263,7 +264,7 @@ RepeatTest:
search = fmt.Sprintf("the email body %d jdsauk dwqmdqw", i)
}
summaries, err := Search(DefaultMailbox, search, 0, 200)
summaries, err := Search(DefaultMailbox, search, 0, 10)
if err != nil {
t.Log("error ", err)
t.Fail()
@ -279,12 +280,12 @@ RepeatTest:
}
// search something that will return 200 rsults
summaries, err := Search(DefaultMailbox, "This is the email body", 0, 200)
summaries, err := Search(DefaultMailbox, "This is the email body", 0, 50)
if err != nil {
t.Log("error ", err)
t.Fail()
}
assertEqual(t, len(summaries), 200, "200 search results expected")
assertEqual(t, len(summaries), 50, "50 search results expected")
db.Close()

5
storage/utils.go
View File

@ -56,11 +56,10 @@ func cleanString(str string) string {
return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(str)), " "))
}
// Auto-prune runs every 5 minutes to automatically delete oldest messages
// Auto-prune runs every minute to automatically delete oldest messages
// if total is greater than the threshold
func pruneCron() {
for {
// time.Sleep(5 * 60 * time.Second)
time.Sleep(60 * time.Second)
mailboxes, err := db.ListCollections()
if err != nil {
@ -94,7 +93,7 @@ func pruneCron() {
}
// SanitizeMailboxName returns a clean mailbox name
// allowing only `alphanumeric` characters and `-``
// allowing only `alphanumeric` characters and `-“
func sanitizeMailboxName(mailbox string) string {
re := regexp.MustCompile(`[^a-zA-Z0-9\-]`)