go/mailpit
1
0
Fork 0
mirror of https://github.com/axllent/mailpit.git synced 2025-02-03 13:12:03 +02:00
Code Issues Releases Activity

Feature: Option to use rqlite database storage (#254)

Browse Source
This commit is contained in:
Ralph Slooten 2024-04-05 15:48:32 +13:00
parent 5166a761ec
commit 254b2dd8ec
20 changed files with 276 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/tests.yml vendored
View File

@ -26,8 +26,8 @@ jobs:
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- run: go test ./internal/storage ./server ./internal/tools ./internal/html2text -v
- run: go test ./internal/storage ./internal/html2text -bench=.
- run: go test -p 1 ./internal/storage ./server ./internal/tools ./internal/html2text -v
- run: go test -p 1 ./internal/storage ./internal/html2text -bench=.
# build the assets
- name: Build web UI

1
go.mod
View File

@ -16,6 +16,7 @@ require (
github.com/lithammer/shortuuid/v4 v4.0.0
github.com/mhale/smtpd v0.8.2
github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e
github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.8.0
github.com/spf13/pflag v1.0.5

2
go.sum
View File

@ -112,6 +112,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418 h1:gYUQqzapdN4PQF5j0zDFI9ANQVAVFoJivNp5bTZEZMo=
github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=

40
internal/stats/stats.go
View File

@ -2,7 +2,6 @@
package stats
import (
"os"
"runtime"
"sync"
"time"
@ -21,10 +20,10 @@ var (
mu sync.RWMutex
smtpAccepted int
smtpAcceptedSize int
smtpRejected int
smtpIgnored int
smtpAccepted float64
smtpAcceptedSize float64
smtpRejected float64
smtpIgnored float64
)
// AppInformation struct
@ -37,29 +36,29 @@ type AppInformation struct {
// Database path
Database string
// Database size in bytes
DatabaseSize int64
DatabaseSize float64
// Total number of messages in the database
Messages int
Messages float64
// Total number of messages in the database
Unread int
Unread float64
// Tags and message totals per tag
Tags map[string]int64
// Runtime statistics
RuntimeStats struct {
// Mailpit server uptime in seconds
Uptime int
Uptime float64
// Current memory usage in bytes
Memory uint64
// Database runtime messages deleted
MessagesDeleted int
MessagesDeleted float64
// Accepted runtime SMTP messages
SMTPAccepted int
SMTPAccepted float64
// Total runtime accepted messages size in bytes
SMTPAcceptedSize int
SMTPAcceptedSize float64
// Rejected runtime SMTP messages
SMTPRejected int
SMTPRejected float64
// Ignored runtime SMTP messages (when using --ignore-duplicate-ids)
SMTPIgnored int
SMTPIgnored float64
}
}
@ -72,8 +71,7 @@ func Load() AppInformation {
runtime.ReadMemStats(&m)
info.RuntimeStats.Memory = m.Sys - m.HeapReleased
info.RuntimeStats.Uptime = int(time.Since(startedAt).Seconds())
info.RuntimeStats.Uptime = time.Since(startedAt).Seconds()
info.RuntimeStats.MessagesDeleted = storage.StatsDeleted
info.RuntimeStats.SMTPAccepted = smtpAccepted
info.RuntimeStats.SMTPAcceptedSize = smtpAcceptedSize
@ -97,15 +95,9 @@ func Load() AppInformation {
}
info.Database = config.DataFile
db, err := os.Stat(info.Database)
if err == nil {
info.DatabaseSize = db.Size()
}
info.DatabaseSize = storage.DbSize()
info.Messages = storage.CountTotal()
info.Unread = storage.CountUnread()
info.Tags = storage.GetAllTagsCount()
return info
@ -120,7 +112,7 @@ func Track() {
func LogSMTPAccepted(size int) {
mu.Lock()
smtpAccepted = smtpAccepted + 1
smtpAcceptedSize = smtpAcceptedSize + size
smtpAcceptedSize = smtpAcceptedSize + float64(size)
mu.Unlock()
}

23
internal/storage/cron.go
View File

@ -27,7 +27,7 @@ func dbCron() {
if deletedSize > 0 {
total := totalMessagesSize()
var deletedPercent int64
var deletedPercent float64
if total == 0 {
deletedPercent = 100
} else {
@ -35,7 +35,7 @@ func dbCron() {
}
// only vacuum the DB if at least 1% of mail storage size has been deleted
if deletedPercent >= 1 {
logger.Log().Debugf("[db] deleted messages is %d%% of total size, reclaim space", deletedPercent)
logger.Log().Debugf("[db] deleted messages is %f%% of total size, reclaim space", deletedPercent)
vacuumDb()
}
}
@ -62,8 +62,8 @@ func pruneMessages() {
ids := []string{}
var prunedSize int64
var size int
if err := q.Query(context.TODO(), db, func(row *sql.Rows) {
var size float64
if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
var id string
if err := row.Scan(&id, &size); err != nil {
@ -93,19 +93,19 @@ func pruneMessages() {
args[i] = id
}
_, err = tx.Query(`DELETE FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
_, err = tx.Exec(`DELETE FROM mailbox_data WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
if err != nil {
logger.Log().Errorf("[db] %s", err.Error())
return
}
_, err = tx.Query(`DELETE FROM mailbox_data WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
_, err = tx.Exec(`DELETE FROM message_tags WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
if err != nil {
logger.Log().Errorf("[db] %s", err.Error())
return
}
_, err = tx.Query(`DELETE FROM message_tags WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
_, err = tx.Exec(`DELETE FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
if err != nil {
logger.Log().Errorf("[db] %s", err.Error())
return
@ -137,6 +137,11 @@ func pruneMessages() {
// Vacuum the database to reclaim space from deleted messages
func vacuumDb() {
if sqlDriver == "rqlite" {
// let rqlite handle vacuuming
return
}
start := time.Now()
// set WAL file checkpoint
@ -147,7 +152,7 @@ func vacuumDb() {
// vacuum database
if _, err := db.Exec("VACUUM"); err != nil {
logger.Log().Errorf("[db] %s", err.Error())
logger.Log().Errorf("[db] VACUUM: %s", err.Error())
return
}
@ -162,5 +167,5 @@ func vacuumDb() {
}
elapsed := time.Since(start)
logger.Log().Debugf("[db] vacuumed database in %s", elapsed)
logger.Log().Debugf("[db] vacuum completed in %s", elapsed)
}

68
internal/storage/database.go
View File

@ -9,6 +9,7 @@ import (
"os/signal"
"path"
"path/filepath"
"strings"
"syscall"
"time"
@ -17,14 +18,18 @@ import (
"github.com/klauspost/compress/zstd"
"github.com/leporo/sqlf"
// sqlite (native) - https://gitlab.com/cznic/sqlite
// sqlite - https://gitlab.com/cznic/sqlite
_ "modernc.org/sqlite"
// rqlite - https://github.com/rqlite/gorqlite | https://rqlite.io/
_ "github.com/rqlite/gorqlite/stdlib"
)
var (
db *sql.DB
dbFile string
dbIsTemp bool
sqlDriver string
dbLastAction time.Time
// zstd compression encoder & decoder
@ -35,38 +40,55 @@ var (
// InitDB will initialise the database
func InitDB() error {
p := config.DataFile
var dsn string
if p == "" {
// when no path is provided then we create a temporary file
// which will get deleted on Close(), SIGINT or SIGTERM
p = fmt.Sprintf("%s-%d.db", path.Join(os.TempDir(), "mailpit"), time.Now().UnixNano())
dbIsTemp = true
sqlDriver = "sqlite"
logger.Log().Debugf("[db] using temporary database: %s", p)
} else if strings.HasPrefix(p, "http://") || strings.HasPrefix(p, "https://") {
sqlDriver = "rqlite"
dsn = p
logger.Log().Debugf("[db] opening rqlite database %s", p)
} else {
p = filepath.Clean(p)
sqlDriver = "sqlite"
dsn = fmt.Sprintf("file:%s?cache=shared", p)
logger.Log().Debugf("[db] opening database %s", p)
}
config.DataFile = p
logger.Log().Debugf("[db] opening database %s", p)
var err error
dsn := fmt.Sprintf("file:%s?cache=shared", p)
db, err = sql.Open("sqlite", dsn)
db, err = sql.Open(sqlDriver, dsn)
if err != nil {
return err
}
for i := 1; i < 6; i++ {
if err := Ping(); err != nil {
logger.Log().Errorf("[db] %s", err.Error())
logger.Log().Infof("[db] reconnecting in 5 seconds (%d/5)", i)
time.Sleep(5 * time.Second)
} else {
continue
}
}
// prevent "database locked" errors
// @see https://github.com/mattn/go-sqlite3#faq
db.SetMaxOpenConns(1)
// SQLite performance tuning (https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)
_, err = db.Exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = normal;")
if err != nil {
return err
if sqlDriver == "sqlite" {
// SQLite performance tuning (https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)
_, err = db.Exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = normal;")
if err != nil {
return err
}
}
// create tables if necessary & apply migrations
@ -138,8 +160,8 @@ func StatsGet() MailboxStats {
}
// CountTotal returns the number of emails in the database
func CountTotal() int {
var total int
func CountTotal() float64 {
var total float64
_ = sqlf.From("mailbox").
Select("COUNT(*)").To(&total).
@ -149,8 +171,8 @@ func CountTotal() int {
}
// CountUnread returns the number of emails in the database that are unread.
func CountUnread() int {
var total int
func CountUnread() float64 {
var total float64
_ = sqlf.From("mailbox").
Select("COUNT(*)").To(&total).
@ -161,8 +183,8 @@ func CountUnread() int {
}
// CountRead returns the number of emails in the database that are read.
func CountRead() int {
var total int
func CountRead() float64 {
var total float64
_ = sqlf.From("mailbox").
Select("COUNT(*)").To(&total).
@ -172,6 +194,20 @@ func CountRead() int {
return total
}
// DbSize returns the size of the SQLite database.
func DbSize() float64 {
var total sql.NullFloat64
err := db.QueryRow("SELECT page_count * page_size AS size FROM pragma_page_count(), pragma_page_size()").Scan(&total)
if err != nil {
logger.Log().Errorf("[db] %s", err.Error())
return total.Float64
}
return total.Float64
}
// IsUnread returns whether a message is unread or not.
func IsUnread(id string) bool {
var unread int

122
internal/storage/messages.go
View File

@ -4,6 +4,8 @@ import (
"bytes"
"context"
"database/sql"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
@ -88,21 +90,22 @@ func Store(body *[]byte) (string, error) {
defer tx.Rollback()
subject := env.GetHeader("Subject")
size := len(*body)
size := float64(len(*body))
inline := len(env.Inlines)
attachments := len(env.Attachments)
snippet := tools.CreateSnippet(env.Text, env.HTML)
// insert mail summary data
_, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Snippet) values(?,?,?,?,?,?,?,?,?,0,?)",
_, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Snippet) VALUES(?,?,?,?,?,?,?,?,?,0,?)",
created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, snippet)
if err != nil {
return "", err
}
// insert compressed raw message
compressed := dbEncoder.EncodeAll(*body, make([]byte, 0, size))
_, err = tx.Exec("INSERT INTO mailbox_data(ID, Email) values(?,?)", id, string(compressed))
encoded := dbEncoder.EncodeAll(*body, make([]byte, 0, int(size)))
hexStr := hex.EncodeToString(encoded)
_, err = tx.Exec("INSERT INTO mailbox_data(ID, Email) VALUES(?, x'"+hexStr+"')", id)
if err != nil {
return "", err
}
@ -155,12 +158,12 @@ func List(start, limit int) ([]MessageSummary, error) {
Offset(start)
if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
var created int64
var created float64
var id string
var messageID string
var subject string
var metadata string
var size int
var size float64
var attachments int
var read int
var snippet string
@ -176,7 +179,7 @@ func List(start, limit int) ([]MessageSummary, error) {
return
}
em.Created = time.UnixMilli(created)
em.Created = time.UnixMilli(int64(created))
em.ID = id
em.MessageID = messageID
em.Subject = subject
@ -246,7 +249,7 @@ func GetMessage(id string) (*Message, error) {
Where(`ID = ?`, id)
if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
var created int64
var created float64
if err := row.Scan(&created); err != nil {
logger.Log().Errorf("[db] %s", err.Error())
@ -255,7 +258,7 @@ func GetMessage(id string) (*Message, error) {
logger.Log().Debugf("[db] %s does not contain a date header, using received datetime", id)
date = time.UnixMilli(created)
date = time.UnixMilli(int64(created))
}); err != nil {
logger.Log().Errorf("[db] %s", err.Error())
}
@ -273,7 +276,7 @@ func GetMessage(id string) (*Message, error) {
ReturnPath: returnPath,
Subject: env.GetHeader("Subject"),
Tags: getMessageTags(id),
Size: len(raw),
Size: float64(len(raw)),
Text: env.Text,
}
@ -331,7 +334,6 @@ func GetMessageRaw(id string) ([]byte, error) {
Select(`ID`).To(&i).
Select(`Email`).To(&msg).
Where(`ID = ?`, id)
err := q.QueryRowAndClose(context.Background(), db)
if err != nil {
return nil, err
@ -341,7 +343,17 @@ func GetMessageRaw(id string) ([]byte, error) {
return nil, errors.New("message not found")
}
raw, err := dbDecoder.DecodeAll([]byte(msg), nil)
var data []byte
if sqlDriver == "rqlite" {
data, err = base64.StdEncoding.DecodeString(msg)
if err != nil {
return nil, fmt.Errorf("error decoding base64 message: %w", err)
}
} else {
data = []byte(msg)
}
raw, err := dbDecoder.DecodeAll(data, nil)
if err != nil {
return nil, fmt.Errorf("error decompressing message: %s", err.Error())
}
@ -451,7 +463,7 @@ func MarkAllRead() error {
}
elapsed := time.Since(start)
logger.Log().Debugf("[db] marked %d messages as read in %s", total, elapsed)
logger.Log().Debugf("[db] marked %v messages as read in %s", total, elapsed)
BroadcastMailboxStats()
@ -476,7 +488,7 @@ func MarkAllUnread() error {
}
elapsed := time.Since(start)
logger.Log().Debugf("[db] marked %d messages as unread in %s", total, elapsed)
logger.Log().Debugf("[db] marked %v messages as unread in %s", total, elapsed)
BroadcastMailboxStats()
@ -507,52 +519,92 @@ func MarkUnread(id string) error {
return err
}
// DeleteOneMessage will delete a single message from a mailbox
func DeleteOneMessage(id string) error {
m, err := GetMessageRaw(id)
// DeleteMessages deletes one or more messages in bulk
func DeleteMessages(ids []string) error {
if len(ids) == 0 {
return nil
}
start := time.Now()
args := make([]interface{}, len(ids))
for i, id := range ids {
args[i] = id
}
rows, err := db.Query(`SELECT ID, Size FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(args)-1)+`)`, args...)
if err != nil {
return err
}
defer rows.Close()
toDelete := []string{}
var totalSize float64
for rows.Next() {
var id string
var size float64
if err := rows.Scan(&id, &size); err != nil {
return err
}
toDelete = append(toDelete, id)
totalSize = totalSize + size
}
if err = rows.Err(); err != nil {
return err
}
if len(toDelete) == 0 {
return nil // nothing to delete
}
size := len(m)
// begin a transaction to ensure both the message
// and data are deleted successfully
tx, err := db.BeginTx(context.Background(), nil)
if err != nil {
return err
}
// roll back if it fails
defer tx.Rollback()
args = make([]interface{}, len(toDelete))
for i, id := range toDelete {
args[i] = id
}
_, err = tx.Exec("DELETE FROM mailbox WHERE ID = ?", id)
_, err = tx.Exec(`DELETE FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
if err != nil {
return err
}
_, err = tx.Exec("DELETE FROM mailbox_data WHERE ID = ?", id)
_, err = tx.Exec(`DELETE FROM mailbox_data WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
if err != nil {
return err
}
_, err = tx.Exec(`DELETE FROM message_tags WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
if err != nil {
return err
}
err = tx.Commit()
if err == nil {
logger.Log().Debugf("[db] deleted message %s", id)
}
if err := DeleteAllMessageTags(id); err != nil {
return err
}
dbLastAction = time.Now()
addDeletedSize(int64(size))
addDeletedSize(int64(totalSize))
logMessagesDeleted(1)
logMessagesDeleted(len(toDelete))
pruneUnusedTags()
elapsed := time.Since(start)
messages := "messages"
if len(toDelete) == 1 {
messages = "message"
}
logger.Log().Debugf("[db] deleted %d %s in %s", len(toDelete), messages, elapsed)
BroadcastMailboxStats()
return err
return nil
}
// DeleteAllMessages will delete all messages from a mailbox

18
internal/storage/messages_test.go
View File

@ -13,8 +13,6 @@ func TestTextEmailInserts(t *testing.T) {
start := time.Now()
assertEqualStats(t, 0, 0)
for i := 0; i < testRuns; i++ {
if _, err := Store(&testTextEmail); err != nil {
t.Log("error ", err)
@ -22,19 +20,17 @@ func TestTextEmailInserts(t *testing.T) {
}
}
assertEqual(t, CountTotal(), testRuns, "Incorrect number of text emails stored")
assertEqual(t, CountTotal(), float64(testRuns), "Incorrect number of text emails stored")
t.Logf("Inserted %d text emails in %s", testRuns, time.Since(start))
assertEqualStats(t, testRuns, testRuns)
delStart := time.Now()
if err := DeleteAllMessages(); err != nil {
t.Log("error ", err)
t.Fail()
}
assertEqual(t, CountTotal(), 0, "incorrect number of text emails deleted")
assertEqual(t, CountTotal(), float64(0), "incorrect number of text emails deleted")
t.Logf("deleted %d text emails in %s", testRuns, time.Since(delStart))
@ -56,19 +52,17 @@ func TestMimeEmailInserts(t *testing.T) {
}
}
assertEqual(t, CountTotal(), testRuns, "Incorrect number of mime emails stored")
assertEqual(t, CountTotal(), float64(testRuns), "Incorrect number of mime emails stored")
t.Logf("Inserted %d text emails in %s", testRuns, time.Since(start))
assertEqualStats(t, testRuns, testRuns)
delStart := time.Now()
if err := DeleteAllMessages(); err != nil {
t.Log("error ", err)
t.Fail()
}
assertEqual(t, CountTotal(), 0, "incorrect number of mime emails deleted")
assertEqual(t, CountTotal(), float64(0), "incorrect number of mime emails deleted")
t.Logf("Deleted %d mime emails in %s", testRuns, time.Since(delStart))
}
@ -107,14 +101,14 @@ func TestRetrieveMimeEmail(t *testing.T) {
t.Log("error ", err)
t.Fail()
}
assertEqual(t, len(attachmentData.Content), msg.Attachments[0].Size, "attachment size does not match")
assertEqual(t, float64(len(attachmentData.Content)), msg.Attachments[0].Size, "attachment size does not match")
inlineData, err := GetAttachmentPart(id, msg.Inline[0].PartID)
if err != nil {
t.Log("error ", err)
t.Fail()
}
assertEqual(t, len(inlineData.Content), msg.Inline[0].Size, "inline attachment size does not match")
assertEqual(t, float64(len(inlineData.Content)), msg.Inline[0].Size, "inline attachment size does not match")
}
func TestMessageSummary(t *testing.T) {

4
internal/storage/notifications.go
View File

@ -24,8 +24,8 @@ func BroadcastMailboxStats() {
time.Sleep(250 * time.Millisecond)
bcStatsDelay = false
b := struct {
Total int
Unread int
Total float64
Unread float64
Version string
}{
Total: CountTotal(),

13
internal/storage/search.go
View File

@ -30,12 +30,12 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) {
var err error
if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
var created int64
var created float64
var id string
var messageID string
var subject string
var metadata string
var size int
var size float64
var attachments int
var snippet string
var read int
@ -52,7 +52,7 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) {
return
}
em.Created = time.UnixMilli(created)
em.Created = time.UnixMilli(int64(created))
em.ID = id
em.MessageID = messageID
em.Subject = subject
@ -99,17 +99,16 @@ func DeleteSearch(search string) error {
q := searchQueryBuilder(search)
ids := []string{}
deleteSize := 0
deleteSize := float64(0)
if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
var created int64
var created float64
var id string
var messageID string
var subject string
var metadata string
var size int
var size float64
var attachments int
// var tags string
var read int
var snippet string
var ignore string

12
internal/storage/settings.go
View File

@ -35,8 +35,8 @@ func SettingPut(k, v string) error {
}
// The total deleted message size as an int64 value
func getDeletedSize() int64 {
var result sql.NullInt64
func getDeletedSize() float64 {
var result sql.NullFloat64
err := sqlf.From("settings").
Select("Value").To(&result).
Where("Key = ?", "DeletedSize").
@ -47,12 +47,12 @@ func getDeletedSize() int64 {
return 0
}
return result.Int64
return result.Float64
}
// The total raw non-compressed messages size in bytes of all messages in the database
func totalMessagesSize() int64 {
var result sql.NullInt64
func totalMessagesSize() float64 {
var result sql.NullFloat64
err := sqlf.From("mailbox").
Select("SUM(Size)").To(&result).
QueryAndClose(context.TODO(), db, func(row *sql.Rows) {})
@ -61,7 +61,7 @@ func totalMessagesSize() int64 {
return 0
}
return result.Int64
return result.Float64
}
// AddDeletedSize will add the value to the DeletedSize setting

12
internal/storage/structs.go
View File

@ -41,7 +41,7 @@ type Message struct {
// Message body HTML
HTML string
// Message size in bytes
Size int
Size float64
// Inline message attachments
Inline []Attachment
// Message attachments
@ -61,7 +61,7 @@ type Attachment struct {
// Content ID
ContentID string
// Size in bytes
Size int
Size float64
}
// MessageSummary struct for frontend messages
@ -91,7 +91,7 @@ type MessageSummary struct {
// Message tags
Tags []string
// Message size in bytes (total)
Size int
Size float64
// Whether the message has any attachments
Attachments int
// Message snippet includes up to 250 characters
@ -100,8 +100,8 @@ type MessageSummary struct {
// MailboxStats struct for quick mailbox total/read lookups
type MailboxStats struct {
Total int
Unread int
Total float64
Unread float64
Tags []string
}
@ -124,7 +124,7 @@ func AttachmentSummary(a *enmime.Part) Attachment {
}
o.ContentType = a.ContentType
o.ContentID = a.ContentID
o.Size = len(a.Content)
o.Size = float64(len(a.Content))
return o
}

23
internal/storage/tags.go
View File

@ -92,32 +92,13 @@ func AddMessageTag(id, name string) error {
logger.Log().Debugf("[tags] adding tag \"%s\" to %s", name, id)
// tag dos not exist, add new one
if err := sqlf.InsertInto("tags").
if _, err := sqlf.InsertInto("tags").
Set("Name", name).
Returning("ID").To(&tagID).
QueryRowAndClose(context.TODO(), db); err != nil {
return err
}
// check message does not already have this tag
var count int
if _, err := sqlf.From("message_tags").
Select("COUNT(ID)").To(&count).
Where("ID = ?", id).
Where("TagID = ?", tagID).
ExecAndClose(context.TODO(), db); err != nil {
return err
}
if count != 0 {
return nil // already exists
}
// add tag to message
_, err := sqlf.InsertInto("message_tags").
Set("ID", id).
Set("TagID", tagID).
ExecAndClose(context.TODO(), db)
return err
return AddMessageTag(id, name)
}
// DeleteMessageTag deleted a tag from a message

15
internal/storage/testing.go
View File

@ -19,7 +19,7 @@ var (
func setup() {
logger.NoLogging = true
config.MaxMessages = 0
config.DataFile = ""
config.DataFile = os.Getenv("MP_DATA_FILE")
if err := InitDB(); err != nil {
panic(err)
@ -27,6 +27,11 @@ func setup() {
var err error
// ensure DB is empty
if err := DeleteAllMessages(); err != nil {
panic(err)
}
testTextEmail, err = os.ReadFile("testdata/plain-text.eml")
if err != nil {
panic(err)
@ -53,11 +58,11 @@ func assertEqual(t *testing.T, a interface{}, b interface{}, message string) {
func assertEqualStats(t *testing.T, total int, unread int) {
s := StatsGet()
if total != s.Total {
t.Fatalf("Incorrect total mailbox stats: \"%d\" != \"%d\"", total, s.Total)
if float64(total) != s.Total {
t.Fatalf("Incorrect total mailbox stats: \"%v\" != \"%v\"", total, s.Total)
}
if unread != s.Unread {
t.Fatalf("Incorrect unread mailbox stats: \"%d\" != \"%d\"", unread, s.Unread)
if float64(unread) != s.Unread {
t.Fatalf("Incorrect unread mailbox stats: \"%v\" != \"%v\"", unread, s.Unread)
}
}

4
internal/storage/utils.go
View File

@ -15,7 +15,7 @@ var (
// for stats to prevent import cycle
mu sync.RWMutex
// StatsDeleted for counting the number of messages deleted
StatsDeleted int
StatsDeleted float64
)
// Return a header field as a []*mail.Address, or "null" is not found/empty
@ -73,7 +73,7 @@ func cleanString(str string) string {
// LogMessagesDeleted logs the number of messages deleted
func logMessagesDeleted(n int) {
mu.Lock()
StatsDeleted = StatsDeleted + n
StatsDeleted = StatsDeleted + float64(n)
mu.Unlock()
}

22
server/apiv1/api.go
View File

@ -67,7 +67,7 @@ func GetMessages(w http.ResponseWriter, r *http.Request) {
res.Start = start
res.Messages = messages
res.Count = len(messages) // legacy - now undocumented in API specs
res.Count = float64(len(messages)) // legacy - now undocumented in API specs
res.Total = stats.Total
res.Unread = stats.Unread
res.Tags = stats.Tags
@ -133,9 +133,9 @@ func Search(w http.ResponseWriter, r *http.Request) {
res.Start = start
res.Messages = messages
res.Count = len(messages) // legacy - now undocumented in API specs
res.Total = stats.Total // total messages in mailbox
res.MessagesCount = results
res.Count = float64(len(messages)) // legacy - now undocumented in API specs
res.Total = stats.Total // total messages in mailbox
res.MessagesCount = float64(results)
res.Unread = stats.Unread
res.Tags = stats.Tags
@ -337,7 +337,11 @@ func GetHeaders(w http.ResponseWriter, r *http.Request) {
return
}
bytes, _ := json.Marshal(m.Header)
bytes, err := json.Marshal(m.Header)
if err != nil {
httpError(w, err.Error())
return
}
w.Header().Add("Content-Type", "application/json")
_, _ = w.Write(bytes)
@ -428,11 +432,9 @@ func DeleteMessages(w http.ResponseWriter, r *http.Request) {
return
}
} else {
for _, id := range data.IDs {
if err := storage.DeleteOneMessage(id); err != nil {
httpError(w, err.Error())
return
}
if err := storage.DeleteMessages(data.IDs); err != nil {
httpError(w, err.Error())
return
}
}

8
server/apiv1/structs.go
View File

@ -10,18 +10,18 @@ import (
// MessagesSummary is a summary of a list of messages
type MessagesSummary struct {
// Total number of messages in mailbox
Total int `json:"total"`
Total float64 `json:"total"`
// Total number of unread messages in mailbox
Unread int `json:"unread"`
Unread float64 `json:"unread"`
// Legacy - now undocumented in API specs but left for backwards compatibility.
// Removed from API documentation 2023-07-12
// swagger:ignore
Count int `json:"count"`
Count float64 `json:"count"`
// Total number of messages matching current query
MessagesCount int `json:"messages_count"`
MessagesCount float64 `json:"messages_count"`
// Pagination offset
Start int `json:"start"`

15
server/pop3/pop3.go
View File

@ -78,11 +78,10 @@ func Run() {
type message struct {
ID string
Size int
Size float64
}
func handleClient(conn net.Conn) {
var (
user = ""
state = 1
@ -92,7 +91,7 @@ func handleClient(conn net.Conn) {
defer func() {
if state == UPDATE {
for _, id := range toDelete {
_ = storage.DeleteOneMessage(id)
_ = storage.DeleteMessages([]string{id})
}
if len(toDelete) > 0 {
// update web UI to remove deleted messages
@ -178,19 +177,19 @@ func handleClient(conn net.Conn) {
}
} else if cmd == "STAT" && state == TRANSACTION {
totalSize := 0
totalSize := float64(0)
for _, m := range messages {
totalSize = totalSize + m.Size
}
sendResponse(conn, fmt.Sprintf("+OK %d %d", len(messages), totalSize))
sendResponse(conn, fmt.Sprintf("+OK %d %d", len(messages), int64(totalSize)))
} else if cmd == "LIST" && state == TRANSACTION {
totalSize := 0
totalSize := float64(0)
for _, m := range messages {
totalSize = totalSize + m.Size
}
sendData(conn, fmt.Sprintf("+OK %d messages (%d octets)", len(messages), totalSize))
sendData(conn, fmt.Sprintf("+OK %d messages (%d octets)", len(messages), int64(totalSize)))
// print all sizes
for row, m := range messages {
@ -200,7 +199,7 @@ func handleClient(conn net.Conn) {
sendData(conn, ".")
} else if cmd == "UIDL" && state == TRANSACTION {
totalSize := 0
totalSize := float64(0)
for _, m := range messages {
totalSize = totalSize + m.Size
}

13
server/server_test.go
View File

@ -8,6 +8,7 @@ import (
"net/http"
"net/http/httptest"
"net/url"
"os"
"strings"
"testing"
@ -204,11 +205,15 @@ func TestAPIv1Search(t *testing.T) {
func setup() {
logger.NoLogging = true
config.MaxMessages = 0
config.DataFile = ""
config.DataFile = os.Getenv("MP_DATA_FILE")
if err := storage.InitDB(); err != nil {
panic(err)
}
if err := storage.DeleteAllMessages(); err != nil {
panic(err)
}
}
func assertStatsEqual(t *testing.T, uri string, unread, total int) {
@ -225,8 +230,8 @@ func assertStatsEqual(t *testing.T, uri string, unread, total int) {
return
}
assertEqual(t, unread, m.Unread, "wrong unread count")
assertEqual(t, total, m.Total, "wrong total count")
assertEqual(t, float64(unread), m.Unread, "wrong unread count")
assertEqual(t, float64(total), m.Total, "wrong total count")
}
func assertSearchEqual(t *testing.T, uri, query string, count int) {
@ -246,7 +251,7 @@ func assertSearchEqual(t *testing.T, uri, query string, count int) {
return
}
assertEqual(t, count, m.MessagesCount, "wrong search results count")
assertEqual(t, float64(count), m.MessagesCount, "wrong search results count")
}
func insertEmailData(t *testing.T) {

60
server/ui/api/v1/swagger.json
View File

@ -773,8 +773,8 @@
},
"DatabaseSize": {
"description": "Database size in bytes",
"type": "integer",
"format": "int64"
"type": "number",
"format": "double"
},
"LatestVersion": {
"description": "Latest Mailpit version",
@ -782,8 +782,8 @@
},
"Messages": {
"description": "Total number of messages in the database",
"type": "integer",
"format": "int64"
"type": "number",
"format": "double"
},
"RuntimeStats": {
"description": "Runtime statistics",
@ -796,33 +796,33 @@
},
"MessagesDeleted": {
"description": "Database runtime messages deleted",
"type": "integer",
"format": "int64"
"type": "number",
"format": "double"
},
"SMTPAccepted": {
"description": "Accepted runtime SMTP messages",
"type": "integer",
"format": "int64"
"type": "number",
"format": "double"
},
"SMTPAcceptedSize": {
"description": "Total runtime accepted messages size in bytes",
"type": "integer",
"format": "int64"
"type": "number",
"format": "double"
},
"SMTPIgnored": {
"description": "Ignored runtime SMTP messages (when using --ignore-duplicate-ids)",
"type": "integer",
"format": "int64"
"type": "number",
"format": "double"
},
"SMTPRejected": {
"description": "Rejected runtime SMTP messages",
"type": "integer",
"format": "int64"
"type": "number",
"format": "double"
},
"Uptime": {
"description": "Mailpit server uptime in seconds",
"type": "integer",
"format": "int64"
"type": "number",
"format": "double"
}
}
},
@ -836,8 +836,8 @@
},
"Unread": {
"description": "Total number of messages in the database",
"type": "integer",
"format": "int64"
"type": "number",
"format": "double"
},
"Version": {
"description": "Current Mailpit version",
@ -868,8 +868,8 @@
},
"Size": {
"description": "Size in bytes",
"type": "integer",
"format": "int64"
"type": "number",
"format": "double"
}
},
"x-go-package": "github.com/axllent/mailpit/internal/storage"
@ -1176,8 +1176,8 @@
},
"Size": {
"description": "Message size in bytes",
"type": "integer",
"format": "int64"
"type": "number",
"format": "double"
},
"Subject": {
"description": "Message subject",
@ -1268,8 +1268,8 @@
},
"Size": {
"description": "Message size in bytes (total)",
"type": "integer",
"format": "int64"
"type": "number",
"format": "double"
},
"Snippet": {
"description": "Message snippet includes up to 250 characters",
@ -1310,8 +1310,8 @@
},
"messages_count": {
"description": "Total number of messages matching current query",
"type": "integer",
"format": "int64",
"type": "number",
"format": "double",
"x-go-name": "MessagesCount"
},
"start": {
@ -1330,14 +1330,14 @@
},
"total": {
"description": "Total number of messages in mailbox",
"type": "integer",
"format": "int64",
"type": "number",
"format": "double",
"x-go-name": "Total"
},
"unread": {
"description": "Total number of unread messages in mailbox",
"type": "integer",
"format": "int64",
"type": "number",
"format": "double",
"x-go-name": "Unread"
}
},