diff --git a/data/message.go b/data/message.go index b08724a..b193402 100644 --- a/data/message.go +++ b/data/message.go @@ -17,7 +17,6 @@ type Message struct { Bcc []*mail.Address Subject string Date time.Time - Created time.Time Text string HTML string Size int diff --git a/go.mod b/go.mod index bdf7a07..2cf68d8 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/gorilla/websocket v1.5.0 github.com/jhillyerd/enmime v0.10.0 github.com/k3a/html2text v1.0.8 + github.com/klauspost/compress v1.15.9 github.com/mhale/smtpd v0.8.0 github.com/ostafen/clover/v2 v2.0.0-alpha.2 github.com/sirupsen/logrus v1.9.0 @@ -35,7 +36,6 @@ require ( github.com/google/orderedcode v0.0.1 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba // indirect - github.com/klauspost/compress v1.15.9 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect diff --git a/storage/database.go b/storage/database.go index cd1c516..063e4ab 100644 --- a/storage/database.go +++ b/storage/database.go @@ -17,6 +17,7 @@ import ( "github.com/axllent/mailpit/logger" "github.com/axllent/mailpit/server/websockets" "github.com/jhillyerd/enmime" + "github.com/klauspost/compress/zstd" "github.com/ostafen/clover/v2" ) @@ -28,6 +29,10 @@ var ( count int per100start = time.Now() + + // zstd encoder & decoder + encoder, _ = zstd.NewWriter(nil) + decoder, _ = zstd.NewReader(nil) ) // CloverStore struct @@ -219,7 +224,10 @@ func Store(mailbox string, b []byte) (string, error) { raw := clover.NewDocument() raw.Set("_id", id) raw.Set("Created", time.Now()) - raw.Set("Data", string(b)) + + compressed := encoder.EncodeAll(b, make([]byte, 0, len(b))) + raw.Set("Email", string(compressed)) + _, err = db.InsertOne(mailbox+"_data", raw) if err != nil { // delete the summary because the data insert failed @@ -369,18 +377,12 @@ func CountUnread(mailbox string) (int, error) { func GetMessage(mailbox, id string) (*data.Message, error) { mailbox = sanitizeMailboxName(mailbox) - q, err := db.FindById(mailbox+"_data", id) - if err != nil { - return nil, err - } - - if q == nil { + raw, err := GetMessageRaw(mailbox, id) + if err != nil || raw == nil { return nil, errors.New("message not found") } - raw := q.Get("Data").(string) - - r := bytes.NewReader([]byte(raw)) + r := bytes.NewReader(raw) env, err := enmime.ReadEnvelope(r) if err != nil { @@ -398,9 +400,8 @@ func GetMessage(mailbox, id string) (*data.Message, error) { date, _ := env.Date() obj := data.Message{ - ID: q.ObjectId(), + ID: id, Read: true, - Created: q.Get("Created").(time.Time), From: from, Date: date, To: addressToSlice(env, "To"), @@ -456,12 +457,12 @@ func GetMessage(mailbox, id string) (*data.Message, error) { func GetAttachmentPart(mailbox, id, partID string) (*enmime.Part, error) { mailbox = sanitizeMailboxName(mailbox) - data, err := GetMessageRaw(mailbox, id) + raw, err := GetMessageRaw(mailbox, id) if err != nil { return nil, err } - r := bytes.NewReader(data) + r := bytes.NewReader(raw) env, err := enmime.ReadEnvelope(r) if err != nil { @@ -502,9 +503,20 @@ func GetMessageRaw(mailbox, id string) ([]byte, error) { return nil, errors.New("message not found") } - data := q.Get("Data").(string) + var raw []byte - return []byte(data), err + if q.Has("Email") { + msg := q.Get("Email").(string) + raw, err = decoder.DecodeAll([]byte(msg), nil) + if err != nil { + return nil, fmt.Errorf("error decompressing message: %s", err.Error()) + } + } else { + // deprecated 2022/08/10 - can be eventually removed + raw = []byte(q.Get("Data").(string)) + } + + return raw, err } // UnreadMessage will delete all messages from a mailbox diff --git a/storage/database_test.go b/storage/database_test.go index ac23dc2..e38b45a 100644 --- a/storage/database_test.go +++ b/storage/database_test.go @@ -18,6 +18,7 @@ import ( var ( testTextEmail []byte testMimeEmail []byte + testRuns = 1000 ) func TestTextEmailInserts(t *testing.T) { @@ -29,7 +30,7 @@ RepeatTest: assertEqualStats(t, 0, 0) - for i := 0; i < 1000; i++ { + for i := 0; i < testRuns; i++ { if _, err := Store(DefaultMailbox, testTextEmail); err != nil { t.Log("error ", err) t.Fail() @@ -42,11 +43,11 @@ RepeatTest: t.Fail() } - assertEqual(t, count, 1000, "incorrect number of text emails stored") + assertEqual(t, count, testRuns, "incorrect number of text emails stored") - t.Logf("inserted 1,000 text emails in %s\n", time.Since(start)) + t.Logf("inserted %d text emails in %s", testRuns, time.Since(start)) - assertEqualStats(t, 1000, 1000) + assertEqualStats(t, testRuns, testRuns) delStart := time.Now() if err := DeleteAllMessages(DefaultMailbox); err != nil { @@ -62,7 +63,7 @@ RepeatTest: assertEqual(t, count, 0, "incorrect number of text emails deleted") - t.Logf("deleted 1,000 text emails in %s\n", time.Since(delStart)) + t.Logf("deleted %d text emails in %s", testRuns, time.Since(delStart)) assertEqualStats(t, 0, 0) @@ -85,7 +86,7 @@ RepeatTest: assertEqualStats(t, 0, 0) - for i := 0; i < 1000; i++ { + for i := 0; i < testRuns; i++ { if _, err := Store(DefaultMailbox, testMimeEmail); err != nil { t.Log("error ", err) t.Fail() @@ -98,11 +99,11 @@ RepeatTest: t.Fail() } - assertEqual(t, count, 1000, "incorrect number of mime emails stored") + assertEqual(t, count, testRuns, "incorrect number of emails with mime attachments stored") - t.Logf("inserted 1,000 emails with mime attachments in %s\n", time.Since(start)) + t.Logf("inserted %d emails with mime attachments in %s", testRuns, time.Since(start)) - assertEqualStats(t, 1000, 1000) + assertEqualStats(t, testRuns, testRuns) delStart := time.Now() if err := DeleteAllMessages(DefaultMailbox); err != nil { @@ -116,9 +117,9 @@ RepeatTest: t.Fail() } - assertEqual(t, count, 0, "incorrect number of mime emails deleted") + assertEqual(t, count, 0, "incorrect number of emails with mime attachments deleted") - t.Logf("deleted 1,000 mime emails in %s\n", time.Since(delStart)) + t.Logf("deleted %d emails with mime attachments in %s", testRuns, time.Since(delStart)) assertEqualStats(t, 0, 0) @@ -222,7 +223,7 @@ func TestSearch(t *testing.T) { t.Log("Testing memory storage") RepeatTest: - for i := 0; i < 1000; i++ { + for i := 0; i < testRuns; i++ { msg := enmime.Builder(). From(fmt.Sprintf("From %d", i), fmt.Sprintf("from-%d@example.com", i)). Subject(fmt.Sprintf("Subject line %d end", i)). @@ -248,7 +249,7 @@ RepeatTest: } } - for i := 1; i < 101; i++ { + for i := 1; i < 51; i++ { // search a random something that will return a single result searchIndx := rand.Intn(4) + 1 var search string @@ -263,7 +264,7 @@ RepeatTest: search = fmt.Sprintf("the email body %d jdsauk dwqmdqw", i) } - summaries, err := Search(DefaultMailbox, search, 0, 200) + summaries, err := Search(DefaultMailbox, search, 0, 10) if err != nil { t.Log("error ", err) t.Fail() @@ -279,12 +280,12 @@ RepeatTest: } // search something that will return 200 rsults - summaries, err := Search(DefaultMailbox, "This is the email body", 0, 200) + summaries, err := Search(DefaultMailbox, "This is the email body", 0, 50) if err != nil { t.Log("error ", err) t.Fail() } - assertEqual(t, len(summaries), 200, "200 search results expected") + assertEqual(t, len(summaries), 50, "50 search results expected") db.Close() diff --git a/storage/utils.go b/storage/utils.go index 2cd2c19..7b5f20d 100644 --- a/storage/utils.go +++ b/storage/utils.go @@ -56,11 +56,10 @@ func cleanString(str string) string { return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(str)), " ")) } -// Auto-prune runs every 5 minutes to automatically delete oldest messages +// Auto-prune runs every minute to automatically delete oldest messages // if total is greater than the threshold func pruneCron() { for { - // time.Sleep(5 * 60 * time.Second) time.Sleep(60 * time.Second) mailboxes, err := db.ListCollections() if err != nil { @@ -94,7 +93,7 @@ func pruneCron() { } // SanitizeMailboxName returns a clean mailbox name -// allowing only `alphanumeric` characters and `-`` +// allowing only `alphanumeric` characters and `-“ func sanitizeMailboxName(mailbox string) string { re := regexp.MustCompile(`[^a-zA-Z0-9\-]`)