From 254b2dd8ecebfcd3b4c3fdc9b2caf29804d7284c Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Fri, 5 Apr 2024 15:48:32 +1300 Subject: [PATCH] Feature: Option to use rqlite database storage (#254) --- .github/workflows/tests.yml | 4 +- go.mod | 1 + go.sum | 2 + internal/stats/stats.go | 40 ++++------ internal/storage/cron.go | 23 +++--- internal/storage/database.go | 68 +++++++++++++---- internal/storage/messages.go | 122 +++++++++++++++++++++--------- internal/storage/messages_test.go | 18 ++--- internal/storage/notifications.go | 4 +- internal/storage/search.go | 13 ++-- internal/storage/settings.go | 12 +-- internal/storage/structs.go | 12 +-- internal/storage/tags.go | 23 +----- internal/storage/testing.go | 15 ++-- internal/storage/utils.go | 4 +- server/apiv1/api.go | 22 +++--- server/apiv1/structs.go | 8 +- server/pop3/pop3.go | 15 ++-- server/server_test.go | 13 +++- server/ui/api/v1/swagger.json | 60 +++++++-------- 20 files changed, 276 insertions(+), 203 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9d7c08b..6c188bb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,8 +26,8 @@ jobs: key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go- - - run: go test ./internal/storage ./server ./internal/tools ./internal/html2text -v - - run: go test ./internal/storage ./internal/html2text -bench=. + - run: go test -p 1 ./internal/storage ./server ./internal/tools ./internal/html2text -v + - run: go test -p 1 ./internal/storage ./internal/html2text -bench=. # build the assets - name: Build web UI diff --git a/go.mod b/go.mod index b59df55..dbd8f41 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/lithammer/shortuuid/v4 v4.0.0 github.com/mhale/smtpd v0.8.2 github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e + github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index c5dbcc0..dfd85eb 100644 --- a/go.sum +++ b/go.sum @@ -112,6 +112,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418 h1:gYUQqzapdN4PQF5j0zDFI9ANQVAVFoJivNp5bTZEZMo= +github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= diff --git a/internal/stats/stats.go b/internal/stats/stats.go index cca3f7e..cfbc5b8 100644 --- a/internal/stats/stats.go +++ b/internal/stats/stats.go @@ -2,7 +2,6 @@ package stats import ( - "os" "runtime" "sync" "time" @@ -21,10 +20,10 @@ var ( mu sync.RWMutex - smtpAccepted int - smtpAcceptedSize int - smtpRejected int - smtpIgnored int + smtpAccepted float64 + smtpAcceptedSize float64 + smtpRejected float64 + smtpIgnored float64 ) // AppInformation struct @@ -37,29 +36,29 @@ type AppInformation struct { // Database path Database string // Database size in bytes - DatabaseSize int64 + DatabaseSize float64 // Total number of messages in the database - Messages int + Messages float64 // Total number of messages in the database - Unread int + Unread float64 // Tags and message totals per tag Tags map[string]int64 // Runtime statistics RuntimeStats struct { // Mailpit server uptime in seconds - Uptime int + Uptime float64 // Current memory usage in bytes Memory uint64 // Database runtime messages deleted - MessagesDeleted int + MessagesDeleted float64 // Accepted runtime SMTP messages - SMTPAccepted int + SMTPAccepted float64 // Total runtime accepted messages size in bytes - SMTPAcceptedSize int + SMTPAcceptedSize float64 // Rejected runtime SMTP messages - SMTPRejected int + SMTPRejected float64 // Ignored runtime SMTP messages (when using --ignore-duplicate-ids) - SMTPIgnored int + SMTPIgnored float64 } } @@ -72,8 +71,7 @@ func Load() AppInformation { runtime.ReadMemStats(&m) info.RuntimeStats.Memory = m.Sys - m.HeapReleased - - info.RuntimeStats.Uptime = int(time.Since(startedAt).Seconds()) + info.RuntimeStats.Uptime = time.Since(startedAt).Seconds() info.RuntimeStats.MessagesDeleted = storage.StatsDeleted info.RuntimeStats.SMTPAccepted = smtpAccepted info.RuntimeStats.SMTPAcceptedSize = smtpAcceptedSize @@ -97,15 +95,9 @@ func Load() AppInformation { } info.Database = config.DataFile - - db, err := os.Stat(info.Database) - if err == nil { - info.DatabaseSize = db.Size() - } - + info.DatabaseSize = storage.DbSize() info.Messages = storage.CountTotal() info.Unread = storage.CountUnread() - info.Tags = storage.GetAllTagsCount() return info @@ -120,7 +112,7 @@ func Track() { func LogSMTPAccepted(size int) { mu.Lock() smtpAccepted = smtpAccepted + 1 - smtpAcceptedSize = smtpAcceptedSize + size + smtpAcceptedSize = smtpAcceptedSize + float64(size) mu.Unlock() } diff --git a/internal/storage/cron.go b/internal/storage/cron.go index f2d5ca1..c0f170c 100644 --- a/internal/storage/cron.go +++ b/internal/storage/cron.go @@ -27,7 +27,7 @@ func dbCron() { if deletedSize > 0 { total := totalMessagesSize() - var deletedPercent int64 + var deletedPercent float64 if total == 0 { deletedPercent = 100 } else { @@ -35,7 +35,7 @@ func dbCron() { } // only vacuum the DB if at least 1% of mail storage size has been deleted if deletedPercent >= 1 { - logger.Log().Debugf("[db] deleted messages is %d%% of total size, reclaim space", deletedPercent) + logger.Log().Debugf("[db] deleted messages is %f%% of total size, reclaim space", deletedPercent) vacuumDb() } } @@ -62,8 +62,8 @@ func pruneMessages() { ids := []string{} var prunedSize int64 - var size int - if err := q.Query(context.TODO(), db, func(row *sql.Rows) { + var size float64 + if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) { var id string if err := row.Scan(&id, &size); err != nil { @@ -93,19 +93,19 @@ func pruneMessages() { args[i] = id } - _, err = tx.Query(`DELETE FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec + _, err = tx.Exec(`DELETE FROM mailbox_data WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec if err != nil { logger.Log().Errorf("[db] %s", err.Error()) return } - _, err = tx.Query(`DELETE FROM mailbox_data WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec + _, err = tx.Exec(`DELETE FROM message_tags WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec if err != nil { logger.Log().Errorf("[db] %s", err.Error()) return } - _, err = tx.Query(`DELETE FROM message_tags WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec + _, err = tx.Exec(`DELETE FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec if err != nil { logger.Log().Errorf("[db] %s", err.Error()) return @@ -137,6 +137,11 @@ func pruneMessages() { // Vacuum the database to reclaim space from deleted messages func vacuumDb() { + if sqlDriver == "rqlite" { + // let rqlite handle vacuuming + return + } + start := time.Now() // set WAL file checkpoint @@ -147,7 +152,7 @@ func vacuumDb() { // vacuum database if _, err := db.Exec("VACUUM"); err != nil { - logger.Log().Errorf("[db] %s", err.Error()) + logger.Log().Errorf("[db] VACUUM: %s", err.Error()) return } @@ -162,5 +167,5 @@ func vacuumDb() { } elapsed := time.Since(start) - logger.Log().Debugf("[db] vacuumed database in %s", elapsed) + logger.Log().Debugf("[db] vacuum completed in %s", elapsed) } diff --git a/internal/storage/database.go b/internal/storage/database.go index 583cffb..b04c891 100644 --- a/internal/storage/database.go +++ b/internal/storage/database.go @@ -9,6 +9,7 @@ import ( "os/signal" "path" "path/filepath" + "strings" "syscall" "time" @@ -17,14 +18,18 @@ import ( "github.com/klauspost/compress/zstd" "github.com/leporo/sqlf" - // sqlite (native) - https://gitlab.com/cznic/sqlite + // sqlite - https://gitlab.com/cznic/sqlite _ "modernc.org/sqlite" + + // rqlite - https://github.com/rqlite/gorqlite | https://rqlite.io/ + _ "github.com/rqlite/gorqlite/stdlib" ) var ( db *sql.DB dbFile string dbIsTemp bool + sqlDriver string dbLastAction time.Time // zstd compression encoder & decoder @@ -35,38 +40,55 @@ var ( // InitDB will initialise the database func InitDB() error { p := config.DataFile + var dsn string if p == "" { // when no path is provided then we create a temporary file // which will get deleted on Close(), SIGINT or SIGTERM p = fmt.Sprintf("%s-%d.db", path.Join(os.TempDir(), "mailpit"), time.Now().UnixNano()) dbIsTemp = true + sqlDriver = "sqlite" logger.Log().Debugf("[db] using temporary database: %s", p) + } else if strings.HasPrefix(p, "http://") || strings.HasPrefix(p, "https://") { + sqlDriver = "rqlite" + dsn = p + logger.Log().Debugf("[db] opening rqlite database %s", p) } else { p = filepath.Clean(p) + sqlDriver = "sqlite" + dsn = fmt.Sprintf("file:%s?cache=shared", p) + logger.Log().Debugf("[db] opening database %s", p) } config.DataFile = p - logger.Log().Debugf("[db] opening database %s", p) - var err error - dsn := fmt.Sprintf("file:%s?cache=shared", p) - - db, err = sql.Open("sqlite", dsn) + db, err = sql.Open(sqlDriver, dsn) if err != nil { return err } + for i := 1; i < 6; i++ { + if err := Ping(); err != nil { + logger.Log().Errorf("[db] %s", err.Error()) + logger.Log().Infof("[db] reconnecting in 5 seconds (%d/5)", i) + time.Sleep(5 * time.Second) + } else { + continue + } + } + // prevent "database locked" errors // @see https://github.com/mattn/go-sqlite3#faq db.SetMaxOpenConns(1) - // SQLite performance tuning (https://phiresky.github.io/blog/2020/sqlite-performance-tuning/) - _, err = db.Exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = normal;") - if err != nil { - return err + if sqlDriver == "sqlite" { + // SQLite performance tuning (https://phiresky.github.io/blog/2020/sqlite-performance-tuning/) + _, err = db.Exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = normal;") + if err != nil { + return err + } } // create tables if necessary & apply migrations @@ -138,8 +160,8 @@ func StatsGet() MailboxStats { } // CountTotal returns the number of emails in the database -func CountTotal() int { - var total int +func CountTotal() float64 { + var total float64 _ = sqlf.From("mailbox"). Select("COUNT(*)").To(&total). @@ -149,8 +171,8 @@ func CountTotal() int { } // CountUnread returns the number of emails in the database that are unread. -func CountUnread() int { - var total int +func CountUnread() float64 { + var total float64 _ = sqlf.From("mailbox"). Select("COUNT(*)").To(&total). @@ -161,8 +183,8 @@ func CountUnread() int { } // CountRead returns the number of emails in the database that are read. -func CountRead() int { - var total int +func CountRead() float64 { + var total float64 _ = sqlf.From("mailbox"). Select("COUNT(*)").To(&total). @@ -172,6 +194,20 @@ func CountRead() int { return total } +// DbSize returns the size of the SQLite database. +func DbSize() float64 { + var total sql.NullFloat64 + + err := db.QueryRow("SELECT page_count * page_size AS size FROM pragma_page_count(), pragma_page_size()").Scan(&total) + + if err != nil { + logger.Log().Errorf("[db] %s", err.Error()) + return total.Float64 + } + + return total.Float64 +} + // IsUnread returns whether a message is unread or not. func IsUnread(id string) bool { var unread int diff --git a/internal/storage/messages.go b/internal/storage/messages.go index 2228f06..5310a84 100644 --- a/internal/storage/messages.go +++ b/internal/storage/messages.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "database/sql" + "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -88,21 +90,22 @@ func Store(body *[]byte) (string, error) { defer tx.Rollback() subject := env.GetHeader("Subject") - size := len(*body) + size := float64(len(*body)) inline := len(env.Inlines) attachments := len(env.Attachments) snippet := tools.CreateSnippet(env.Text, env.HTML) // insert mail summary data - _, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Snippet) values(?,?,?,?,?,?,?,?,?,0,?)", + _, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Snippet) VALUES(?,?,?,?,?,?,?,?,?,0,?)", created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, snippet) if err != nil { return "", err } // insert compressed raw message - compressed := dbEncoder.EncodeAll(*body, make([]byte, 0, size)) - _, err = tx.Exec("INSERT INTO mailbox_data(ID, Email) values(?,?)", id, string(compressed)) + encoded := dbEncoder.EncodeAll(*body, make([]byte, 0, int(size))) + hexStr := hex.EncodeToString(encoded) + _, err = tx.Exec("INSERT INTO mailbox_data(ID, Email) VALUES(?, x'"+hexStr+"')", id) if err != nil { return "", err } @@ -155,12 +158,12 @@ func List(start, limit int) ([]MessageSummary, error) { Offset(start) if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) { - var created int64 + var created float64 var id string var messageID string var subject string var metadata string - var size int + var size float64 var attachments int var read int var snippet string @@ -176,7 +179,7 @@ func List(start, limit int) ([]MessageSummary, error) { return } - em.Created = time.UnixMilli(created) + em.Created = time.UnixMilli(int64(created)) em.ID = id em.MessageID = messageID em.Subject = subject @@ -246,7 +249,7 @@ func GetMessage(id string) (*Message, error) { Where(`ID = ?`, id) if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) { - var created int64 + var created float64 if err := row.Scan(&created); err != nil { logger.Log().Errorf("[db] %s", err.Error()) @@ -255,7 +258,7 @@ func GetMessage(id string) (*Message, error) { logger.Log().Debugf("[db] %s does not contain a date header, using received datetime", id) - date = time.UnixMilli(created) + date = time.UnixMilli(int64(created)) }); err != nil { logger.Log().Errorf("[db] %s", err.Error()) } @@ -273,7 +276,7 @@ func GetMessage(id string) (*Message, error) { ReturnPath: returnPath, Subject: env.GetHeader("Subject"), Tags: getMessageTags(id), - Size: len(raw), + Size: float64(len(raw)), Text: env.Text, } @@ -331,7 +334,6 @@ func GetMessageRaw(id string) ([]byte, error) { Select(`ID`).To(&i). Select(`Email`).To(&msg). Where(`ID = ?`, id) - err := q.QueryRowAndClose(context.Background(), db) if err != nil { return nil, err @@ -341,7 +343,17 @@ func GetMessageRaw(id string) ([]byte, error) { return nil, errors.New("message not found") } - raw, err := dbDecoder.DecodeAll([]byte(msg), nil) + var data []byte + if sqlDriver == "rqlite" { + data, err = base64.StdEncoding.DecodeString(msg) + if err != nil { + return nil, fmt.Errorf("error decoding base64 message: %w", err) + } + } else { + data = []byte(msg) + } + + raw, err := dbDecoder.DecodeAll(data, nil) if err != nil { return nil, fmt.Errorf("error decompressing message: %s", err.Error()) } @@ -451,7 +463,7 @@ func MarkAllRead() error { } elapsed := time.Since(start) - logger.Log().Debugf("[db] marked %d messages as read in %s", total, elapsed) + logger.Log().Debugf("[db] marked %v messages as read in %s", total, elapsed) BroadcastMailboxStats() @@ -476,7 +488,7 @@ func MarkAllUnread() error { } elapsed := time.Since(start) - logger.Log().Debugf("[db] marked %d messages as unread in %s", total, elapsed) + logger.Log().Debugf("[db] marked %v messages as unread in %s", total, elapsed) BroadcastMailboxStats() @@ -507,52 +519,92 @@ func MarkUnread(id string) error { return err } -// DeleteOneMessage will delete a single message from a mailbox -func DeleteOneMessage(id string) error { - m, err := GetMessageRaw(id) +// DeleteMessages deletes one or more messages in bulk +func DeleteMessages(ids []string) error { + if len(ids) == 0 { + return nil + } + + start := time.Now() + + args := make([]interface{}, len(ids)) + for i, id := range ids { + args[i] = id + } + + rows, err := db.Query(`SELECT ID, Size FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(args)-1)+`)`, args...) if err != nil { return err } + defer rows.Close() + + toDelete := []string{} + var totalSize float64 + + for rows.Next() { + var id string + var size float64 + if err := rows.Scan(&id, &size); err != nil { + return err + } + toDelete = append(toDelete, id) + totalSize = totalSize + size + } + + if err = rows.Err(); err != nil { + return err + } + + if len(toDelete) == 0 { + return nil // nothing to delete + } - size := len(m) - // begin a transaction to ensure both the message - // and data are deleted successfully tx, err := db.BeginTx(context.Background(), nil) if err != nil { return err } - // roll back if it fails - defer tx.Rollback() + args = make([]interface{}, len(toDelete)) + for i, id := range toDelete { + args[i] = id + } - _, err = tx.Exec("DELETE FROM mailbox WHERE ID = ?", id) + _, err = tx.Exec(`DELETE FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec if err != nil { return err } - _, err = tx.Exec("DELETE FROM mailbox_data WHERE ID = ?", id) + _, err = tx.Exec(`DELETE FROM mailbox_data WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec + if err != nil { + return err + } + + _, err = tx.Exec(`DELETE FROM message_tags WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec if err != nil { return err } err = tx.Commit() - if err == nil { - logger.Log().Debugf("[db] deleted message %s", id) - } - - if err := DeleteAllMessageTags(id); err != nil { - return err - } - dbLastAction = time.Now() - addDeletedSize(int64(size)) + addDeletedSize(int64(totalSize)) - logMessagesDeleted(1) + logMessagesDeleted(len(toDelete)) + + pruneUnusedTags() + + elapsed := time.Since(start) + + messages := "messages" + if len(toDelete) == 1 { + messages = "message" + } + + logger.Log().Debugf("[db] deleted %d %s in %s", len(toDelete), messages, elapsed) BroadcastMailboxStats() - return err + return nil } // DeleteAllMessages will delete all messages from a mailbox diff --git a/internal/storage/messages_test.go b/internal/storage/messages_test.go index 07dd10b..b088bd1 100644 --- a/internal/storage/messages_test.go +++ b/internal/storage/messages_test.go @@ -13,8 +13,6 @@ func TestTextEmailInserts(t *testing.T) { start := time.Now() - assertEqualStats(t, 0, 0) - for i := 0; i < testRuns; i++ { if _, err := Store(&testTextEmail); err != nil { t.Log("error ", err) @@ -22,19 +20,17 @@ func TestTextEmailInserts(t *testing.T) { } } - assertEqual(t, CountTotal(), testRuns, "Incorrect number of text emails stored") + assertEqual(t, CountTotal(), float64(testRuns), "Incorrect number of text emails stored") t.Logf("Inserted %d text emails in %s", testRuns, time.Since(start)) - assertEqualStats(t, testRuns, testRuns) - delStart := time.Now() if err := DeleteAllMessages(); err != nil { t.Log("error ", err) t.Fail() } - assertEqual(t, CountTotal(), 0, "incorrect number of text emails deleted") + assertEqual(t, CountTotal(), float64(0), "incorrect number of text emails deleted") t.Logf("deleted %d text emails in %s", testRuns, time.Since(delStart)) @@ -56,19 +52,17 @@ func TestMimeEmailInserts(t *testing.T) { } } - assertEqual(t, CountTotal(), testRuns, "Incorrect number of mime emails stored") + assertEqual(t, CountTotal(), float64(testRuns), "Incorrect number of mime emails stored") t.Logf("Inserted %d text emails in %s", testRuns, time.Since(start)) - assertEqualStats(t, testRuns, testRuns) - delStart := time.Now() if err := DeleteAllMessages(); err != nil { t.Log("error ", err) t.Fail() } - assertEqual(t, CountTotal(), 0, "incorrect number of mime emails deleted") + assertEqual(t, CountTotal(), float64(0), "incorrect number of mime emails deleted") t.Logf("Deleted %d mime emails in %s", testRuns, time.Since(delStart)) } @@ -107,14 +101,14 @@ func TestRetrieveMimeEmail(t *testing.T) { t.Log("error ", err) t.Fail() } - assertEqual(t, len(attachmentData.Content), msg.Attachments[0].Size, "attachment size does not match") + assertEqual(t, float64(len(attachmentData.Content)), msg.Attachments[0].Size, "attachment size does not match") inlineData, err := GetAttachmentPart(id, msg.Inline[0].PartID) if err != nil { t.Log("error ", err) t.Fail() } - assertEqual(t, len(inlineData.Content), msg.Inline[0].Size, "inline attachment size does not match") + assertEqual(t, float64(len(inlineData.Content)), msg.Inline[0].Size, "inline attachment size does not match") } func TestMessageSummary(t *testing.T) { diff --git a/internal/storage/notifications.go b/internal/storage/notifications.go index 8c59c2f..c54e122 100644 --- a/internal/storage/notifications.go +++ b/internal/storage/notifications.go @@ -24,8 +24,8 @@ func BroadcastMailboxStats() { time.Sleep(250 * time.Millisecond) bcStatsDelay = false b := struct { - Total int - Unread int + Total float64 + Unread float64 Version string }{ Total: CountTotal(), diff --git a/internal/storage/search.go b/internal/storage/search.go index 3c869f6..30e67ff 100644 --- a/internal/storage/search.go +++ b/internal/storage/search.go @@ -30,12 +30,12 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) { var err error if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) { - var created int64 + var created float64 var id string var messageID string var subject string var metadata string - var size int + var size float64 var attachments int var snippet string var read int @@ -52,7 +52,7 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) { return } - em.Created = time.UnixMilli(created) + em.Created = time.UnixMilli(int64(created)) em.ID = id em.MessageID = messageID em.Subject = subject @@ -99,17 +99,16 @@ func DeleteSearch(search string) error { q := searchQueryBuilder(search) ids := []string{} - deleteSize := 0 + deleteSize := float64(0) if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) { - var created int64 + var created float64 var id string var messageID string var subject string var metadata string - var size int + var size float64 var attachments int - // var tags string var read int var snippet string var ignore string diff --git a/internal/storage/settings.go b/internal/storage/settings.go index c81da5e..02f91a9 100644 --- a/internal/storage/settings.go +++ b/internal/storage/settings.go @@ -35,8 +35,8 @@ func SettingPut(k, v string) error { } // The total deleted message size as an int64 value -func getDeletedSize() int64 { - var result sql.NullInt64 +func getDeletedSize() float64 { + var result sql.NullFloat64 err := sqlf.From("settings"). Select("Value").To(&result). Where("Key = ?", "DeletedSize"). @@ -47,12 +47,12 @@ func getDeletedSize() int64 { return 0 } - return result.Int64 + return result.Float64 } // The total raw non-compressed messages size in bytes of all messages in the database -func totalMessagesSize() int64 { - var result sql.NullInt64 +func totalMessagesSize() float64 { + var result sql.NullFloat64 err := sqlf.From("mailbox"). Select("SUM(Size)").To(&result). QueryAndClose(context.TODO(), db, func(row *sql.Rows) {}) @@ -61,7 +61,7 @@ func totalMessagesSize() int64 { return 0 } - return result.Int64 + return result.Float64 } // AddDeletedSize will add the value to the DeletedSize setting diff --git a/internal/storage/structs.go b/internal/storage/structs.go index 8503d73..1742e78 100644 --- a/internal/storage/structs.go +++ b/internal/storage/structs.go @@ -41,7 +41,7 @@ type Message struct { // Message body HTML HTML string // Message size in bytes - Size int + Size float64 // Inline message attachments Inline []Attachment // Message attachments @@ -61,7 +61,7 @@ type Attachment struct { // Content ID ContentID string // Size in bytes - Size int + Size float64 } // MessageSummary struct for frontend messages @@ -91,7 +91,7 @@ type MessageSummary struct { // Message tags Tags []string // Message size in bytes (total) - Size int + Size float64 // Whether the message has any attachments Attachments int // Message snippet includes up to 250 characters @@ -100,8 +100,8 @@ type MessageSummary struct { // MailboxStats struct for quick mailbox total/read lookups type MailboxStats struct { - Total int - Unread int + Total float64 + Unread float64 Tags []string } @@ -124,7 +124,7 @@ func AttachmentSummary(a *enmime.Part) Attachment { } o.ContentType = a.ContentType o.ContentID = a.ContentID - o.Size = len(a.Content) + o.Size = float64(len(a.Content)) return o } diff --git a/internal/storage/tags.go b/internal/storage/tags.go index ab5b2e4..bf03a7f 100644 --- a/internal/storage/tags.go +++ b/internal/storage/tags.go @@ -92,32 +92,13 @@ func AddMessageTag(id, name string) error { logger.Log().Debugf("[tags] adding tag \"%s\" to %s", name, id) // tag dos not exist, add new one - if err := sqlf.InsertInto("tags"). + if _, err := sqlf.InsertInto("tags"). Set("Name", name). - Returning("ID").To(&tagID). - QueryRowAndClose(context.TODO(), db); err != nil { - return err - } - - // check message does not already have this tag - var count int - if _, err := sqlf.From("message_tags"). - Select("COUNT(ID)").To(&count). - Where("ID = ?", id). - Where("TagID = ?", tagID). ExecAndClose(context.TODO(), db); err != nil { return err } - if count != 0 { - return nil // already exists - } - // add tag to message - _, err := sqlf.InsertInto("message_tags"). - Set("ID", id). - Set("TagID", tagID). - ExecAndClose(context.TODO(), db) - return err + return AddMessageTag(id, name) } // DeleteMessageTag deleted a tag from a message diff --git a/internal/storage/testing.go b/internal/storage/testing.go index 4b767f0..8665d14 100644 --- a/internal/storage/testing.go +++ b/internal/storage/testing.go @@ -19,7 +19,7 @@ var ( func setup() { logger.NoLogging = true config.MaxMessages = 0 - config.DataFile = "" + config.DataFile = os.Getenv("MP_DATA_FILE") if err := InitDB(); err != nil { panic(err) @@ -27,6 +27,11 @@ func setup() { var err error + // ensure DB is empty + if err := DeleteAllMessages(); err != nil { + panic(err) + } + testTextEmail, err = os.ReadFile("testdata/plain-text.eml") if err != nil { panic(err) @@ -53,11 +58,11 @@ func assertEqual(t *testing.T, a interface{}, b interface{}, message string) { func assertEqualStats(t *testing.T, total int, unread int) { s := StatsGet() - if total != s.Total { - t.Fatalf("Incorrect total mailbox stats: \"%d\" != \"%d\"", total, s.Total) + if float64(total) != s.Total { + t.Fatalf("Incorrect total mailbox stats: \"%v\" != \"%v\"", total, s.Total) } - if unread != s.Unread { - t.Fatalf("Incorrect unread mailbox stats: \"%d\" != \"%d\"", unread, s.Unread) + if float64(unread) != s.Unread { + t.Fatalf("Incorrect unread mailbox stats: \"%v\" != \"%v\"", unread, s.Unread) } } diff --git a/internal/storage/utils.go b/internal/storage/utils.go index f57001e..64583c8 100644 --- a/internal/storage/utils.go +++ b/internal/storage/utils.go @@ -15,7 +15,7 @@ var ( // for stats to prevent import cycle mu sync.RWMutex // StatsDeleted for counting the number of messages deleted - StatsDeleted int + StatsDeleted float64 ) // Return a header field as a []*mail.Address, or "null" is not found/empty @@ -73,7 +73,7 @@ func cleanString(str string) string { // LogMessagesDeleted logs the number of messages deleted func logMessagesDeleted(n int) { mu.Lock() - StatsDeleted = StatsDeleted + n + StatsDeleted = StatsDeleted + float64(n) mu.Unlock() } diff --git a/server/apiv1/api.go b/server/apiv1/api.go index 9c30abe..8cc56e3 100644 --- a/server/apiv1/api.go +++ b/server/apiv1/api.go @@ -67,7 +67,7 @@ func GetMessages(w http.ResponseWriter, r *http.Request) { res.Start = start res.Messages = messages - res.Count = len(messages) // legacy - now undocumented in API specs + res.Count = float64(len(messages)) // legacy - now undocumented in API specs res.Total = stats.Total res.Unread = stats.Unread res.Tags = stats.Tags @@ -133,9 +133,9 @@ func Search(w http.ResponseWriter, r *http.Request) { res.Start = start res.Messages = messages - res.Count = len(messages) // legacy - now undocumented in API specs - res.Total = stats.Total // total messages in mailbox - res.MessagesCount = results + res.Count = float64(len(messages)) // legacy - now undocumented in API specs + res.Total = stats.Total // total messages in mailbox + res.MessagesCount = float64(results) res.Unread = stats.Unread res.Tags = stats.Tags @@ -337,7 +337,11 @@ func GetHeaders(w http.ResponseWriter, r *http.Request) { return } - bytes, _ := json.Marshal(m.Header) + bytes, err := json.Marshal(m.Header) + if err != nil { + httpError(w, err.Error()) + return + } w.Header().Add("Content-Type", "application/json") _, _ = w.Write(bytes) @@ -428,11 +432,9 @@ func DeleteMessages(w http.ResponseWriter, r *http.Request) { return } } else { - for _, id := range data.IDs { - if err := storage.DeleteOneMessage(id); err != nil { - httpError(w, err.Error()) - return - } + if err := storage.DeleteMessages(data.IDs); err != nil { + httpError(w, err.Error()) + return } } diff --git a/server/apiv1/structs.go b/server/apiv1/structs.go index dfe9e0a..7055afc 100644 --- a/server/apiv1/structs.go +++ b/server/apiv1/structs.go @@ -10,18 +10,18 @@ import ( // MessagesSummary is a summary of a list of messages type MessagesSummary struct { // Total number of messages in mailbox - Total int `json:"total"` + Total float64 `json:"total"` // Total number of unread messages in mailbox - Unread int `json:"unread"` + Unread float64 `json:"unread"` // Legacy - now undocumented in API specs but left for backwards compatibility. // Removed from API documentation 2023-07-12 // swagger:ignore - Count int `json:"count"` + Count float64 `json:"count"` // Total number of messages matching current query - MessagesCount int `json:"messages_count"` + MessagesCount float64 `json:"messages_count"` // Pagination offset Start int `json:"start"` diff --git a/server/pop3/pop3.go b/server/pop3/pop3.go index bc70db3..caeb142 100644 --- a/server/pop3/pop3.go +++ b/server/pop3/pop3.go @@ -78,11 +78,10 @@ func Run() { type message struct { ID string - Size int + Size float64 } func handleClient(conn net.Conn) { - var ( user = "" state = 1 @@ -92,7 +91,7 @@ func handleClient(conn net.Conn) { defer func() { if state == UPDATE { for _, id := range toDelete { - _ = storage.DeleteOneMessage(id) + _ = storage.DeleteMessages([]string{id}) } if len(toDelete) > 0 { // update web UI to remove deleted messages @@ -178,19 +177,19 @@ func handleClient(conn net.Conn) { } } else if cmd == "STAT" && state == TRANSACTION { - totalSize := 0 + totalSize := float64(0) for _, m := range messages { totalSize = totalSize + m.Size } - sendResponse(conn, fmt.Sprintf("+OK %d %d", len(messages), totalSize)) + sendResponse(conn, fmt.Sprintf("+OK %d %d", len(messages), int64(totalSize))) } else if cmd == "LIST" && state == TRANSACTION { - totalSize := 0 + totalSize := float64(0) for _, m := range messages { totalSize = totalSize + m.Size } - sendData(conn, fmt.Sprintf("+OK %d messages (%d octets)", len(messages), totalSize)) + sendData(conn, fmt.Sprintf("+OK %d messages (%d octets)", len(messages), int64(totalSize))) // print all sizes for row, m := range messages { @@ -200,7 +199,7 @@ func handleClient(conn net.Conn) { sendData(conn, ".") } else if cmd == "UIDL" && state == TRANSACTION { - totalSize := 0 + totalSize := float64(0) for _, m := range messages { totalSize = totalSize + m.Size } diff --git a/server/server_test.go b/server/server_test.go index c65c830..2a7ebbc 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "os" "strings" "testing" @@ -204,11 +205,15 @@ func TestAPIv1Search(t *testing.T) { func setup() { logger.NoLogging = true config.MaxMessages = 0 - config.DataFile = "" + config.DataFile = os.Getenv("MP_DATA_FILE") if err := storage.InitDB(); err != nil { panic(err) } + + if err := storage.DeleteAllMessages(); err != nil { + panic(err) + } } func assertStatsEqual(t *testing.T, uri string, unread, total int) { @@ -225,8 +230,8 @@ func assertStatsEqual(t *testing.T, uri string, unread, total int) { return } - assertEqual(t, unread, m.Unread, "wrong unread count") - assertEqual(t, total, m.Total, "wrong total count") + assertEqual(t, float64(unread), m.Unread, "wrong unread count") + assertEqual(t, float64(total), m.Total, "wrong total count") } func assertSearchEqual(t *testing.T, uri, query string, count int) { @@ -246,7 +251,7 @@ func assertSearchEqual(t *testing.T, uri, query string, count int) { return } - assertEqual(t, count, m.MessagesCount, "wrong search results count") + assertEqual(t, float64(count), m.MessagesCount, "wrong search results count") } func insertEmailData(t *testing.T) { diff --git a/server/ui/api/v1/swagger.json b/server/ui/api/v1/swagger.json index 54dd822..90a7396 100644 --- a/server/ui/api/v1/swagger.json +++ b/server/ui/api/v1/swagger.json @@ -773,8 +773,8 @@ }, "DatabaseSize": { "description": "Database size in bytes", - "type": "integer", - "format": "int64" + "type": "number", + "format": "double" }, "LatestVersion": { "description": "Latest Mailpit version", @@ -782,8 +782,8 @@ }, "Messages": { "description": "Total number of messages in the database", - "type": "integer", - "format": "int64" + "type": "number", + "format": "double" }, "RuntimeStats": { "description": "Runtime statistics", @@ -796,33 +796,33 @@ }, "MessagesDeleted": { "description": "Database runtime messages deleted", - "type": "integer", - "format": "int64" + "type": "number", + "format": "double" }, "SMTPAccepted": { "description": "Accepted runtime SMTP messages", - "type": "integer", - "format": "int64" + "type": "number", + "format": "double" }, "SMTPAcceptedSize": { "description": "Total runtime accepted messages size in bytes", - "type": "integer", - "format": "int64" + "type": "number", + "format": "double" }, "SMTPIgnored": { "description": "Ignored runtime SMTP messages (when using --ignore-duplicate-ids)", - "type": "integer", - "format": "int64" + "type": "number", + "format": "double" }, "SMTPRejected": { "description": "Rejected runtime SMTP messages", - "type": "integer", - "format": "int64" + "type": "number", + "format": "double" }, "Uptime": { "description": "Mailpit server uptime in seconds", - "type": "integer", - "format": "int64" + "type": "number", + "format": "double" } } }, @@ -836,8 +836,8 @@ }, "Unread": { "description": "Total number of messages in the database", - "type": "integer", - "format": "int64" + "type": "number", + "format": "double" }, "Version": { "description": "Current Mailpit version", @@ -868,8 +868,8 @@ }, "Size": { "description": "Size in bytes", - "type": "integer", - "format": "int64" + "type": "number", + "format": "double" } }, "x-go-package": "github.com/axllent/mailpit/internal/storage" @@ -1176,8 +1176,8 @@ }, "Size": { "description": "Message size in bytes", - "type": "integer", - "format": "int64" + "type": "number", + "format": "double" }, "Subject": { "description": "Message subject", @@ -1268,8 +1268,8 @@ }, "Size": { "description": "Message size in bytes (total)", - "type": "integer", - "format": "int64" + "type": "number", + "format": "double" }, "Snippet": { "description": "Message snippet includes up to 250 characters", @@ -1310,8 +1310,8 @@ }, "messages_count": { "description": "Total number of messages matching current query", - "type": "integer", - "format": "int64", + "type": "number", + "format": "double", "x-go-name": "MessagesCount" }, "start": { @@ -1330,14 +1330,14 @@ }, "total": { "description": "Total number of messages in mailbox", - "type": "integer", - "format": "int64", + "type": "number", + "format": "double", "x-go-name": "Total" }, "unread": { "description": "Total number of unread messages in mailbox", - "type": "integer", - "format": "int64", + "type": "number", + "format": "double", "x-go-name": "Unread" } },