Feature: Option to use rqlite database storage (#254)

.github/workflows/tests.yml

@@ -26,8 +26,8 @@ jobs:
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-
-    - run: go test ./internal/storage ./server ./internal/tools ./internal/html2text -v
+    - run: go test -p 1 ./internal/storage ./server ./internal/tools ./internal/html2text -v
-    - run: go test ./internal/storage ./internal/html2text -bench=.
+    - run: go test -p 1 ./internal/storage ./internal/html2text -bench=.
     
     # build the assets
     - name: Build web UI

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/lithammer/shortuuid/v4 v4.0.0
 	github.com/mhale/smtpd v0.8.2
 	github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e
+	github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -112,6 +112,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418 h1:gYUQqzapdN4PQF5j0zDFI9ANQVAVFoJivNp5bTZEZMo=
+github.com/rqlite/gorqlite v0.0.0-20240227123050-397b03f02418/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=

internal/stats/stats.go

@@ -2,7 +2,6 @@
 package stats
 
 import (
-	"os"
 	"runtime"
 	"sync"
 	"time"
@@ -21,10 +20,10 @@ var (
 
 	mu sync.RWMutex
 
-	smtpAccepted     int
+	smtpAccepted     float64
-	smtpAcceptedSize int
+	smtpAcceptedSize float64
-	smtpRejected     int
+	smtpRejected     float64
-	smtpIgnored      int
+	smtpIgnored      float64
 )
 
 // AppInformation struct
@@ -37,29 +36,29 @@ type AppInformation struct {
 	// Database path
 	Database string
 	// Database size in bytes
-	DatabaseSize int64
+	DatabaseSize float64
 	// Total number of messages in the database
-	Messages int
+	Messages float64
 	// Total number of messages in the database
-	Unread int
+	Unread float64
 	// Tags and message totals per tag
 	Tags map[string]int64
 	// Runtime statistics
 	RuntimeStats struct {
 		// Mailpit server uptime in seconds
-		Uptime int
+		Uptime float64
 		// Current memory usage in bytes
 		Memory uint64
 		// Database runtime messages deleted
-		MessagesDeleted int
+		MessagesDeleted float64
 		// Accepted runtime SMTP messages
-		SMTPAccepted int
+		SMTPAccepted float64
 		// Total runtime accepted messages size in bytes
-		SMTPAcceptedSize int
+		SMTPAcceptedSize float64
 		// Rejected runtime SMTP messages
-		SMTPRejected int
+		SMTPRejected float64
 		// Ignored runtime SMTP messages (when using --ignore-duplicate-ids)
-		SMTPIgnored int
+		SMTPIgnored float64
 	}
 }
 
@@ -72,8 +71,7 @@ func Load() AppInformation {
 	runtime.ReadMemStats(&m)
 
 	info.RuntimeStats.Memory = m.Sys - m.HeapReleased
-
+	info.RuntimeStats.Uptime = time.Since(startedAt).Seconds()
-	info.RuntimeStats.Uptime = int(time.Since(startedAt).Seconds())
 	info.RuntimeStats.MessagesDeleted = storage.StatsDeleted
 	info.RuntimeStats.SMTPAccepted = smtpAccepted
 	info.RuntimeStats.SMTPAcceptedSize = smtpAcceptedSize
@@ -97,15 +95,9 @@ func Load() AppInformation {
 	}
 
 	info.Database = config.DataFile
-
+	info.DatabaseSize = storage.DbSize()
-	db, err := os.Stat(info.Database)
-	if err == nil {
-		info.DatabaseSize = db.Size()
-	}
-
 	info.Messages = storage.CountTotal()
 	info.Unread = storage.CountUnread()
-
 	info.Tags = storage.GetAllTagsCount()
 
 	return info
@@ -120,7 +112,7 @@ func Track() {
 func LogSMTPAccepted(size int) {
 	mu.Lock()
 	smtpAccepted = smtpAccepted + 1
-	smtpAcceptedSize = smtpAcceptedSize + size
+	smtpAcceptedSize = smtpAcceptedSize + float64(size)
 	mu.Unlock()
 }
 

internal/storage/cron.go

@@ -27,7 +27,7 @@ func dbCron() {
 
 			if deletedSize > 0 {
 				total := totalMessagesSize()
-				var deletedPercent int64
+				var deletedPercent float64
 				if total == 0 {
 					deletedPercent = 100
 				} else {
@@ -35,7 +35,7 @@ func dbCron() {
 				}
 				// only vacuum the DB if at least 1% of mail storage size has been deleted
 				if deletedPercent >= 1 {
-					logger.Log().Debugf("[db] deleted messages is %d%% of total size, reclaim space", deletedPercent)
+					logger.Log().Debugf("[db] deleted messages is %f%% of total size, reclaim space", deletedPercent)
 					vacuumDb()
 				}
 			}
@@ -62,8 +62,8 @@ func pruneMessages() {
 
 	ids := []string{}
 	var prunedSize int64
-	var size int
+	var size float64
-	if err := q.Query(context.TODO(), db, func(row *sql.Rows) {
+	if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
 		var id string
 
 		if err := row.Scan(&id, &size); err != nil {
@@ -93,19 +93,19 @@ func pruneMessages() {
 		args[i] = id
 	}
 
-	_, err = tx.Query(`DELETE FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
+	_, err = tx.Exec(`DELETE FROM mailbox_data WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
 	if err != nil {
 		logger.Log().Errorf("[db] %s", err.Error())
 		return
 	}
 
-	_, err = tx.Query(`DELETE FROM mailbox_data WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
+	_, err = tx.Exec(`DELETE FROM message_tags WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
 	if err != nil {
 		logger.Log().Errorf("[db] %s", err.Error())
 		return
 	}
 
-	_, err = tx.Query(`DELETE FROM message_tags WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
+	_, err = tx.Exec(`DELETE FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
 	if err != nil {
 		logger.Log().Errorf("[db] %s", err.Error())
 		return
@@ -137,6 +137,11 @@ func pruneMessages() {
 
 // Vacuum the database to reclaim space from deleted messages
 func vacuumDb() {
+	if sqlDriver == "rqlite" {
+		// let rqlite handle vacuuming
+		return
+	}
+
 	start := time.Now()
 
 	// set WAL file checkpoint
@@ -147,7 +152,7 @@ func vacuumDb() {
 
 	// vacuum database
 	if _, err := db.Exec("VACUUM"); err != nil {
-		logger.Log().Errorf("[db] %s", err.Error())
+		logger.Log().Errorf("[db] VACUUM: %s", err.Error())
 		return
 	}
 
@@ -162,5 +167,5 @@ func vacuumDb() {
 	}
 
 	elapsed := time.Since(start)
-	logger.Log().Debugf("[db] vacuumed database in %s", elapsed)
+	logger.Log().Debugf("[db] vacuum completed in %s", elapsed)
 }

internal/storage/database.go

@@ -9,6 +9,7 @@ import (
 	"os/signal"
 	"path"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"time"
 
@@ -17,14 +18,18 @@ import (
 	"github.com/klauspost/compress/zstd"
 	"github.com/leporo/sqlf"
 
-	// sqlite (native) - https://gitlab.com/cznic/sqlite
+	// sqlite - https://gitlab.com/cznic/sqlite
 	_ "modernc.org/sqlite"
+
+	// rqlite - https://github.com/rqlite/gorqlite | https://rqlite.io/
+	_ "github.com/rqlite/gorqlite/stdlib"
 )
 
 var (
 	db           *sql.DB
 	dbFile       string
 	dbIsTemp     bool
+	sqlDriver    string
 	dbLastAction time.Time
 
 	// zstd compression encoder & decoder
@@ -35,38 +40,55 @@ var (
 // InitDB will initialise the database
 func InitDB() error {
 	p := config.DataFile
+	var dsn string
 
 	if p == "" {
 		// when no path is provided then we create a temporary file
 		// which will get deleted on Close(), SIGINT or SIGTERM
 		p = fmt.Sprintf("%s-%d.db", path.Join(os.TempDir(), "mailpit"), time.Now().UnixNano())
 		dbIsTemp = true
+		sqlDriver = "sqlite"
 		logger.Log().Debugf("[db] using temporary database: %s", p)
+	} else if strings.HasPrefix(p, "http://") || strings.HasPrefix(p, "https://") {
+		sqlDriver = "rqlite"
+		dsn = p
+		logger.Log().Debugf("[db] opening rqlite database %s", p)
 	} else {
 		p = filepath.Clean(p)
+		sqlDriver = "sqlite"
+		dsn = fmt.Sprintf("file:%s?cache=shared", p)
+		logger.Log().Debugf("[db] opening database %s", p)
 	}
 
 	config.DataFile = p
 
-	logger.Log().Debugf("[db] opening database %s", p)
-
 	var err error
 
-	dsn := fmt.Sprintf("file:%s?cache=shared", p)
+	db, err = sql.Open(sqlDriver, dsn)
-
-	db, err = sql.Open("sqlite", dsn)
 	if err != nil {
 		return err
 	}
 
+	for i := 1; i < 6; i++ {
+		if err := Ping(); err != nil {
+			logger.Log().Errorf("[db] %s", err.Error())
+			logger.Log().Infof("[db] reconnecting in 5 seconds (%d/5)", i)
+			time.Sleep(5 * time.Second)
+		} else {
+			continue
+		}
+	}
+
 	// prevent "database locked" errors
 	// @see https://github.com/mattn/go-sqlite3#faq
 	db.SetMaxOpenConns(1)
 
-	// SQLite performance tuning (https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)
+	if sqlDriver == "sqlite" {
-	_, err = db.Exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = normal;")
+		// SQLite performance tuning (https://phiresky.github.io/blog/2020/sqlite-performance-tuning/)
-	if err != nil {
+		_, err = db.Exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = normal;")
-		return err
+		if err != nil {
+			return err
+		}
 	}
 
 	// create tables if necessary & apply migrations
@@ -138,8 +160,8 @@ func StatsGet() MailboxStats {
 }
 
 // CountTotal returns the number of emails in the database
-func CountTotal() int {
+func CountTotal() float64 {
-	var total int
+	var total float64
 
 	_ = sqlf.From("mailbox").
 		Select("COUNT(*)").To(&total).
@@ -149,8 +171,8 @@ func CountTotal() int {
 }
 
 // CountUnread returns the number of emails in the database that are unread.
-func CountUnread() int {
+func CountUnread() float64 {
-	var total int
+	var total float64
 
 	_ = sqlf.From("mailbox").
 		Select("COUNT(*)").To(&total).
@@ -161,8 +183,8 @@ func CountUnread() int {
 }
 
 // CountRead returns the number of emails in the database that are read.
-func CountRead() int {
+func CountRead() float64 {
-	var total int
+	var total float64
 
 	_ = sqlf.From("mailbox").
 		Select("COUNT(*)").To(&total).
@@ -172,6 +194,20 @@ func CountRead() int {
 	return total
 }
 
+// DbSize returns the size of the SQLite database.
+func DbSize() float64 {
+	var total sql.NullFloat64
+
+	err := db.QueryRow("SELECT page_count * page_size AS size FROM pragma_page_count(), pragma_page_size()").Scan(&total)
+
+	if err != nil {
+		logger.Log().Errorf("[db] %s", err.Error())
+		return total.Float64
+	}
+
+	return total.Float64
+}
+
 // IsUnread returns whether a message is unread or not.
 func IsUnread(id string) bool {
 	var unread int

internal/storage/messages.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"context"
 	"database/sql"
+	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -88,21 +90,22 @@ func Store(body *[]byte) (string, error) {
 	defer tx.Rollback()
 
 	subject := env.GetHeader("Subject")
-	size := len(*body)
+	size := float64(len(*body))
 	inline := len(env.Inlines)
 	attachments := len(env.Attachments)
 	snippet := tools.CreateSnippet(env.Text, env.HTML)
 
 	// insert mail summary data
-	_, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Snippet) values(?,?,?,?,?,?,?,?,?,0,?)",
+	_, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Snippet) VALUES(?,?,?,?,?,?,?,?,?,0,?)",
 		created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, snippet)
 	if err != nil {
 		return "", err
 	}
 
 	// insert compressed raw message
-	compressed := dbEncoder.EncodeAll(*body, make([]byte, 0, size))
+	encoded := dbEncoder.EncodeAll(*body, make([]byte, 0, int(size)))
-	_, err = tx.Exec("INSERT INTO mailbox_data(ID, Email) values(?,?)", id, string(compressed))
+	hexStr := hex.EncodeToString(encoded)
+	_, err = tx.Exec("INSERT INTO mailbox_data(ID, Email) VALUES(?, x'"+hexStr+"')", id)
 	if err != nil {
 		return "", err
 	}
@@ -155,12 +158,12 @@ func List(start, limit int) ([]MessageSummary, error) {
 		Offset(start)
 
 	if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
-		var created int64
+		var created float64
 		var id string
 		var messageID string
 		var subject string
 		var metadata string
-		var size int
+		var size float64
 		var attachments int
 		var read int
 		var snippet string
@@ -176,7 +179,7 @@ func List(start, limit int) ([]MessageSummary, error) {
 			return
 		}
 
-		em.Created = time.UnixMilli(created)
+		em.Created = time.UnixMilli(int64(created))
 		em.ID = id
 		em.MessageID = messageID
 		em.Subject = subject
@@ -246,7 +249,7 @@ func GetMessage(id string) (*Message, error) {
 			Where(`ID = ?`, id)
 
 		if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
-			var created int64
+			var created float64
 
 			if err := row.Scan(&created); err != nil {
 				logger.Log().Errorf("[db] %s", err.Error())
@@ -255,7 +258,7 @@ func GetMessage(id string) (*Message, error) {
 
 			logger.Log().Debugf("[db] %s does not contain a date header, using received datetime", id)
 
-			date = time.UnixMilli(created)
+			date = time.UnixMilli(int64(created))
 		}); err != nil {
 			logger.Log().Errorf("[db] %s", err.Error())
 		}
@@ -273,7 +276,7 @@ func GetMessage(id string) (*Message, error) {
 		ReturnPath: returnPath,
 		Subject:    env.GetHeader("Subject"),
 		Tags:       getMessageTags(id),
-		Size:       len(raw),
+		Size:       float64(len(raw)),
 		Text:       env.Text,
 	}
 
@@ -331,7 +334,6 @@ func GetMessageRaw(id string) ([]byte, error) {
 		Select(`ID`).To(&i).
 		Select(`Email`).To(&msg).
 		Where(`ID = ?`, id)
-
 	err := q.QueryRowAndClose(context.Background(), db)
 	if err != nil {
 		return nil, err
@@ -341,7 +343,17 @@ func GetMessageRaw(id string) ([]byte, error) {
 		return nil, errors.New("message not found")
 	}
 
-	raw, err := dbDecoder.DecodeAll([]byte(msg), nil)
+	var data []byte
+	if sqlDriver == "rqlite" {
+		data, err = base64.StdEncoding.DecodeString(msg)
+		if err != nil {
+			return nil, fmt.Errorf("error decoding base64 message: %w", err)
+		}
+	} else {
+		data = []byte(msg)
+	}
+
+	raw, err := dbDecoder.DecodeAll(data, nil)
 	if err != nil {
 		return nil, fmt.Errorf("error decompressing message: %s", err.Error())
 	}
@@ -451,7 +463,7 @@ func MarkAllRead() error {
 	}
 
 	elapsed := time.Since(start)
-	logger.Log().Debugf("[db] marked %d messages as read in %s", total, elapsed)
+	logger.Log().Debugf("[db] marked %v messages as read in %s", total, elapsed)
 
 	BroadcastMailboxStats()
 
@@ -476,7 +488,7 @@ func MarkAllUnread() error {
 	}
 
 	elapsed := time.Since(start)
-	logger.Log().Debugf("[db] marked %d messages as unread in %s", total, elapsed)
+	logger.Log().Debugf("[db] marked %v messages as unread in %s", total, elapsed)
 
 	BroadcastMailboxStats()
 
@@ -507,52 +519,92 @@ func MarkUnread(id string) error {
 	return err
 }
 
-// DeleteOneMessage will delete a single message from a mailbox
+// DeleteMessages deletes one or more messages in bulk
-func DeleteOneMessage(id string) error {
+func DeleteMessages(ids []string) error {
-	m, err := GetMessageRaw(id)
+	if len(ids) == 0 {
+		return nil
+	}
+
+	start := time.Now()
+
+	args := make([]interface{}, len(ids))
+	for i, id := range ids {
+		args[i] = id
+	}
+
+	rows, err := db.Query(`SELECT ID, Size FROM mailbox WHERE  ID IN (?`+strings.Repeat(",?", len(args)-1)+`)`, args...)
 	if err != nil {
 		return err
 	}
+	defer rows.Close()
+
+	toDelete := []string{}
+	var totalSize float64
+
+	for rows.Next() {
+		var id string
+		var size float64
+		if err := rows.Scan(&id, &size); err != nil {
+			return err
+		}
+		toDelete = append(toDelete, id)
+		totalSize = totalSize + size
+	}
+
+	if err = rows.Err(); err != nil {
+		return err
+	}
+
+	if len(toDelete) == 0 {
+		return nil // nothing to delete
+	}
 
-	size := len(m)
-	// begin a transaction to ensure both the message
-	// and data are deleted successfully
 	tx, err := db.BeginTx(context.Background(), nil)
 	if err != nil {
 		return err
 	}
 
-	// roll back if it fails
+	args = make([]interface{}, len(toDelete))
-	defer tx.Rollback()
+	for i, id := range toDelete {
+		args[i] = id
+	}
 
-	_, err = tx.Exec("DELETE FROM mailbox WHERE ID  = ?", id)
+	_, err = tx.Exec(`DELETE FROM mailbox WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
 	if err != nil {
 		return err
 	}
 
-	_, err = tx.Exec("DELETE FROM mailbox_data WHERE ID  = ?", id)
+	_, err = tx.Exec(`DELETE FROM mailbox_data WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.Exec(`DELETE FROM message_tags WHERE ID IN (?`+strings.Repeat(",?", len(ids)-1)+`)`, args...) // #nosec
 	if err != nil {
 		return err
 	}
 
 	err = tx.Commit()
 
-	if err == nil {
-		logger.Log().Debugf("[db] deleted message %s", id)
-	}
-
-	if err := DeleteAllMessageTags(id); err != nil {
-		return err
-	}
-
 	dbLastAction = time.Now()
-	addDeletedSize(int64(size))
+	addDeletedSize(int64(totalSize))
 
-	logMessagesDeleted(1)
+	logMessagesDeleted(len(toDelete))
+
+	pruneUnusedTags()
+
+	elapsed := time.Since(start)
+
+	messages := "messages"
+	if len(toDelete) == 1 {
+		messages = "message"
+	}
+
+	logger.Log().Debugf("[db] deleted %d %s in %s", len(toDelete), messages, elapsed)
 
 	BroadcastMailboxStats()
 
-	return err
+	return nil
 }
 
 // DeleteAllMessages will delete all messages from a mailbox

internal/storage/messages_test.go

@@ -13,8 +13,6 @@ func TestTextEmailInserts(t *testing.T) {
 
 	start := time.Now()
 
-	assertEqualStats(t, 0, 0)
-
 	for i := 0; i < testRuns; i++ {
 		if _, err := Store(&testTextEmail); err != nil {
 			t.Log("error ", err)
@@ -22,19 +20,17 @@ func TestTextEmailInserts(t *testing.T) {
 		}
 	}
 
-	assertEqual(t, CountTotal(), testRuns, "Incorrect number of text emails stored")
+	assertEqual(t, CountTotal(), float64(testRuns), "Incorrect number of text emails stored")
 
 	t.Logf("Inserted %d text emails in %s", testRuns, time.Since(start))
 
-	assertEqualStats(t, testRuns, testRuns)
-
 	delStart := time.Now()
 	if err := DeleteAllMessages(); err != nil {
 		t.Log("error ", err)
 		t.Fail()
 	}
 
-	assertEqual(t, CountTotal(), 0, "incorrect number of text emails deleted")
+	assertEqual(t, CountTotal(), float64(0), "incorrect number of text emails deleted")
 
 	t.Logf("deleted %d text emails in %s", testRuns, time.Since(delStart))
 
@@ -56,19 +52,17 @@ func TestMimeEmailInserts(t *testing.T) {
 		}
 	}
 
-	assertEqual(t, CountTotal(), testRuns, "Incorrect number of mime emails stored")
+	assertEqual(t, CountTotal(), float64(testRuns), "Incorrect number of mime emails stored")
 
 	t.Logf("Inserted %d text emails in %s", testRuns, time.Since(start))
 
-	assertEqualStats(t, testRuns, testRuns)
-
 	delStart := time.Now()
 	if err := DeleteAllMessages(); err != nil {
 		t.Log("error ", err)
 		t.Fail()
 	}
 
-	assertEqual(t, CountTotal(), 0, "incorrect number of mime emails deleted")
+	assertEqual(t, CountTotal(), float64(0), "incorrect number of mime emails deleted")
 
 	t.Logf("Deleted %d mime emails in %s", testRuns, time.Since(delStart))
 }
@@ -107,14 +101,14 @@ func TestRetrieveMimeEmail(t *testing.T) {
 		t.Log("error ", err)
 		t.Fail()
 	}
-	assertEqual(t, len(attachmentData.Content), msg.Attachments[0].Size, "attachment size does not match")
+	assertEqual(t, float64(len(attachmentData.Content)), msg.Attachments[0].Size, "attachment size does not match")
 
 	inlineData, err := GetAttachmentPart(id, msg.Inline[0].PartID)
 	if err != nil {
 		t.Log("error ", err)
 		t.Fail()
 	}
-	assertEqual(t, len(inlineData.Content), msg.Inline[0].Size, "inline attachment size does not match")
+	assertEqual(t, float64(len(inlineData.Content)), msg.Inline[0].Size, "inline attachment size does not match")
 }
 
 func TestMessageSummary(t *testing.T) {

internal/storage/notifications.go

@@ -24,8 +24,8 @@ func BroadcastMailboxStats() {
 		time.Sleep(250 * time.Millisecond)
 		bcStatsDelay = false
 		b := struct {
-			Total   int
+			Total   float64
-			Unread  int
+			Unread  float64
 			Version string
 		}{
 			Total:   CountTotal(),

internal/storage/search.go

@@ -30,12 +30,12 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) {
 	var err error
 
 	if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
-		var created int64
+		var created float64
 		var id string
 		var messageID string
 		var subject string
 		var metadata string
-		var size int
+		var size float64
 		var attachments int
 		var snippet string
 		var read int
@@ -52,7 +52,7 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) {
 			return
 		}
 
-		em.Created = time.UnixMilli(created)
+		em.Created = time.UnixMilli(int64(created))
 		em.ID = id
 		em.MessageID = messageID
 		em.Subject = subject
@@ -99,17 +99,16 @@ func DeleteSearch(search string) error {
 	q := searchQueryBuilder(search)
 
 	ids := []string{}
-	deleteSize := 0
+	deleteSize := float64(0)
 
 	if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) {
-		var created int64
+		var created float64
 		var id string
 		var messageID string
 		var subject string
 		var metadata string
-		var size int
+		var size float64
 		var attachments int
-		// var tags string
 		var read int
 		var snippet string
 		var ignore string

internal/storage/settings.go

@@ -35,8 +35,8 @@ func SettingPut(k, v string) error {
 }
 
 // The total deleted message size as an int64 value
-func getDeletedSize() int64 {
+func getDeletedSize() float64 {
-	var result sql.NullInt64
+	var result sql.NullFloat64
 	err := sqlf.From("settings").
 		Select("Value").To(&result).
 		Where("Key = ?", "DeletedSize").
@@ -47,12 +47,12 @@ func getDeletedSize() int64 {
 		return 0
 	}
 
-	return result.Int64
+	return result.Float64
 }
 
 // The total raw non-compressed messages size in bytes of all messages in the database
-func totalMessagesSize() int64 {
+func totalMessagesSize() float64 {
-	var result sql.NullInt64
+	var result sql.NullFloat64
 	err := sqlf.From("mailbox").
 		Select("SUM(Size)").To(&result).
 		QueryAndClose(context.TODO(), db, func(row *sql.Rows) {})
@@ -61,7 +61,7 @@ func totalMessagesSize() int64 {
 		return 0
 	}
 
-	return result.Int64
+	return result.Float64
 }
 
 // AddDeletedSize will add the value to the DeletedSize setting

internal/storage/structs.go

@@ -41,7 +41,7 @@ type Message struct {
 	// Message body HTML
 	HTML string
 	// Message size in bytes
-	Size int
+	Size float64
 	// Inline message attachments
 	Inline []Attachment
 	// Message attachments
@@ -61,7 +61,7 @@ type Attachment struct {
 	// Content ID
 	ContentID string
 	// Size in bytes
-	Size int
+	Size float64
 }
 
 // MessageSummary struct for frontend messages
@@ -91,7 +91,7 @@ type MessageSummary struct {
 	// Message tags
 	Tags []string
 	// Message size in bytes (total)
-	Size int
+	Size float64
 	// Whether the message has any attachments
 	Attachments int
 	// Message snippet includes up to 250 characters
@@ -100,8 +100,8 @@ type MessageSummary struct {
 
 // MailboxStats struct for quick mailbox total/read lookups
 type MailboxStats struct {
-	Total  int
+	Total  float64
-	Unread int
+	Unread float64
 	Tags   []string
 }
 
@@ -124,7 +124,7 @@ func AttachmentSummary(a *enmime.Part) Attachment {
 	}
 	o.ContentType = a.ContentType
 	o.ContentID = a.ContentID
-	o.Size = len(a.Content)
+	o.Size = float64(len(a.Content))
 
 	return o
 }

internal/storage/tags.go

@@ -92,32 +92,13 @@ func AddMessageTag(id, name string) error {
 	logger.Log().Debugf("[tags] adding tag \"%s\" to %s", name, id)
 
 	// tag dos not exist, add new one
-	if err := sqlf.InsertInto("tags").
+	if _, err := sqlf.InsertInto("tags").
 		Set("Name", name).
-		Returning("ID").To(&tagID).
-		QueryRowAndClose(context.TODO(), db); err != nil {
-		return err
-	}
-
-	// check message does not already have this tag
-	var count int
-	if _, err := sqlf.From("message_tags").
-		Select("COUNT(ID)").To(&count).
-		Where("ID = ?", id).
-		Where("TagID = ?", tagID).
 		ExecAndClose(context.TODO(), db); err != nil {
 		return err
 	}
-	if count != 0 {
-		return nil // already exists
-	}
 
-	// add tag to message
+	return AddMessageTag(id, name)
-	_, err := sqlf.InsertInto("message_tags").
-		Set("ID", id).
-		Set("TagID", tagID).
-		ExecAndClose(context.TODO(), db)
-	return err
 }
 
 // DeleteMessageTag deleted a tag from a message

internal/storage/testing.go

@@ -19,7 +19,7 @@ var (
 func setup() {
 	logger.NoLogging = true
 	config.MaxMessages = 0
-	config.DataFile = ""
+	config.DataFile = os.Getenv("MP_DATA_FILE")
 
 	if err := InitDB(); err != nil {
 		panic(err)
@@ -27,6 +27,11 @@ func setup() {
 
 	var err error
 
+	// ensure DB is empty
+	if err := DeleteAllMessages(); err != nil {
+		panic(err)
+	}
+
 	testTextEmail, err = os.ReadFile("testdata/plain-text.eml")
 	if err != nil {
 		panic(err)
@@ -53,11 +58,11 @@ func assertEqual(t *testing.T, a interface{}, b interface{}, message string) {
 
 func assertEqualStats(t *testing.T, total int, unread int) {
 	s := StatsGet()
-	if total != s.Total {
+	if float64(total) != s.Total {
-		t.Fatalf("Incorrect total mailbox stats: \"%d\" != \"%d\"", total, s.Total)
+		t.Fatalf("Incorrect total mailbox stats: \"%v\" != \"%v\"", total, s.Total)
 	}
 
-	if unread != s.Unread {
+	if float64(unread) != s.Unread {
-		t.Fatalf("Incorrect unread mailbox stats: \"%d\" != \"%d\"", unread, s.Unread)
+		t.Fatalf("Incorrect unread mailbox stats: \"%v\" != \"%v\"", unread, s.Unread)
 	}
 }

internal/storage/utils.go

@@ -15,7 +15,7 @@ var (
 	// for stats to prevent import cycle
 	mu sync.RWMutex
 	// StatsDeleted for counting the number of messages deleted
-	StatsDeleted int
+	StatsDeleted float64
 )
 
 // Return a header field as a []*mail.Address, or "null" is not found/empty
@@ -73,7 +73,7 @@ func cleanString(str string) string {
 // LogMessagesDeleted logs the number of messages deleted
 func logMessagesDeleted(n int) {
 	mu.Lock()
-	StatsDeleted = StatsDeleted + n
+	StatsDeleted = StatsDeleted + float64(n)
 	mu.Unlock()
 }
 

server/apiv1/api.go

@@ -67,7 +67,7 @@ func GetMessages(w http.ResponseWriter, r *http.Request) {
 
 	res.Start = start
 	res.Messages = messages
-	res.Count = len(messages) // legacy - now undocumented in API specs
+	res.Count = float64(len(messages)) // legacy - now undocumented in API specs
 	res.Total = stats.Total
 	res.Unread = stats.Unread
 	res.Tags = stats.Tags
@@ -133,9 +133,9 @@ func Search(w http.ResponseWriter, r *http.Request) {
 
 	res.Start = start
 	res.Messages = messages
-	res.Count = len(messages) // legacy - now undocumented in API specs
+	res.Count = float64(len(messages)) // legacy - now undocumented in API specs
-	res.Total = stats.Total   // total messages in mailbox
+	res.Total = stats.Total            // total messages in mailbox
-	res.MessagesCount = results
+	res.MessagesCount = float64(results)
 	res.Unread = stats.Unread
 	res.Tags = stats.Tags
 
@@ -337,7 +337,11 @@ func GetHeaders(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	bytes, _ := json.Marshal(m.Header)
+	bytes, err := json.Marshal(m.Header)
+	if err != nil {
+		httpError(w, err.Error())
+		return
+	}
 
 	w.Header().Add("Content-Type", "application/json")
 	_, _ = w.Write(bytes)
@@ -428,11 +432,9 @@ func DeleteMessages(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	} else {
-		for _, id := range data.IDs {
+		if err := storage.DeleteMessages(data.IDs); err != nil {
-			if err := storage.DeleteOneMessage(id); err != nil {
+			httpError(w, err.Error())
-				httpError(w, err.Error())
+			return
-				return
-			}
 		}
 	}
 

server/apiv1/structs.go

@@ -10,18 +10,18 @@ import (
 // MessagesSummary is a summary of a list of messages
 type MessagesSummary struct {
 	// Total number of messages in mailbox
-	Total int `json:"total"`
+	Total float64 `json:"total"`
 
 	// Total number of unread messages in mailbox
-	Unread int `json:"unread"`
+	Unread float64 `json:"unread"`
 
 	// Legacy - now undocumented in API specs but left for backwards compatibility.
 	// Removed from API documentation 2023-07-12
 	// swagger:ignore
-	Count int `json:"count"`
+	Count float64 `json:"count"`
 
 	// Total number of messages matching current query
-	MessagesCount int `json:"messages_count"`
+	MessagesCount float64 `json:"messages_count"`
 
 	// Pagination offset
 	Start int `json:"start"`

server/pop3/pop3.go

@@ -78,11 +78,10 @@ func Run() {
 
 type message struct {
 	ID   string
-	Size int
+	Size float64
 }
 
 func handleClient(conn net.Conn) {
-
 	var (
 		user     = ""
 		state    = 1
@@ -92,7 +91,7 @@ func handleClient(conn net.Conn) {
 	defer func() {
 		if state == UPDATE {
 			for _, id := range toDelete {
-				_ = storage.DeleteOneMessage(id)
+				_ = storage.DeleteMessages([]string{id})
 			}
 			if len(toDelete) > 0 {
 				// update web UI to remove deleted messages
@@ -178,19 +177,19 @@ func handleClient(conn net.Conn) {
 			}
 
 		} else if cmd == "STAT" && state == TRANSACTION {
-			totalSize := 0
+			totalSize := float64(0)
 			for _, m := range messages {
 				totalSize = totalSize + m.Size
 			}
 
-			sendResponse(conn, fmt.Sprintf("+OK %d %d", len(messages), totalSize))
+			sendResponse(conn, fmt.Sprintf("+OK %d %d", len(messages), int64(totalSize)))
 
 		} else if cmd == "LIST" && state == TRANSACTION {
-			totalSize := 0
+			totalSize := float64(0)
 			for _, m := range messages {
 				totalSize = totalSize + m.Size
 			}
-			sendData(conn, fmt.Sprintf("+OK %d messages (%d octets)", len(messages), totalSize))
+			sendData(conn, fmt.Sprintf("+OK %d messages (%d octets)", len(messages), int64(totalSize)))
 
 			// print all sizes
 			for row, m := range messages {
@@ -200,7 +199,7 @@ func handleClient(conn net.Conn) {
 			sendData(conn, ".")
 
 		} else if cmd == "UIDL" && state == TRANSACTION {
-			totalSize := 0
+			totalSize := float64(0)
 			for _, m := range messages {
 				totalSize = totalSize + m.Size
 			}

server/server_test.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os"
 	"strings"
 	"testing"
 
@@ -204,11 +205,15 @@ func TestAPIv1Search(t *testing.T) {
 func setup() {
 	logger.NoLogging = true
 	config.MaxMessages = 0
-	config.DataFile = ""
+	config.DataFile = os.Getenv("MP_DATA_FILE")
 
 	if err := storage.InitDB(); err != nil {
 		panic(err)
 	}
+
+	if err := storage.DeleteAllMessages(); err != nil {
+		panic(err)
+	}
 }
 
 func assertStatsEqual(t *testing.T, uri string, unread, total int) {
@@ -225,8 +230,8 @@ func assertStatsEqual(t *testing.T, uri string, unread, total int) {
 		return
 	}
 
-	assertEqual(t, unread, m.Unread, "wrong unread count")
+	assertEqual(t, float64(unread), m.Unread, "wrong unread count")
-	assertEqual(t, total, m.Total, "wrong total count")
+	assertEqual(t, float64(total), m.Total, "wrong total count")
 }
 
 func assertSearchEqual(t *testing.T, uri, query string, count int) {
@@ -246,7 +251,7 @@ func assertSearchEqual(t *testing.T, uri, query string, count int) {
 		return
 	}
 
-	assertEqual(t, count, m.MessagesCount, "wrong search results count")
+	assertEqual(t, float64(count), m.MessagesCount, "wrong search results count")
 }
 
 func insertEmailData(t *testing.T) {

server/ui/api/v1/swagger.json

@@ -773,8 +773,8 @@
         },
         "DatabaseSize": {
           "description": "Database size in bytes",
-          "type": "integer",
+          "type": "number",
-          "format": "int64"
+          "format": "double"
         },
         "LatestVersion": {
           "description": "Latest Mailpit version",
@@ -782,8 +782,8 @@
         },
         "Messages": {
           "description": "Total number of messages in the database",
-          "type": "integer",
+          "type": "number",
-          "format": "int64"
+          "format": "double"
         },
         "RuntimeStats": {
           "description": "Runtime statistics",
@@ -796,33 +796,33 @@
             },
             "MessagesDeleted": {
               "description": "Database runtime messages deleted",
-              "type": "integer",
+              "type": "number",
-              "format": "int64"
+              "format": "double"
             },
             "SMTPAccepted": {
               "description": "Accepted runtime SMTP messages",
-              "type": "integer",
+              "type": "number",
-              "format": "int64"
+              "format": "double"
             },
             "SMTPAcceptedSize": {
               "description": "Total runtime accepted messages size in bytes",
-              "type": "integer",
+              "type": "number",
-              "format": "int64"
+              "format": "double"
             },
             "SMTPIgnored": {
               "description": "Ignored runtime SMTP messages (when using --ignore-duplicate-ids)",
-              "type": "integer",
+              "type": "number",
-              "format": "int64"
+              "format": "double"
             },
             "SMTPRejected": {
               "description": "Rejected runtime SMTP messages",
-              "type": "integer",
+              "type": "number",
-              "format": "int64"
+              "format": "double"
             },
             "Uptime": {
               "description": "Mailpit server uptime in seconds",
-              "type": "integer",
+              "type": "number",
-              "format": "int64"
+              "format": "double"
             }
           }
         },
@@ -836,8 +836,8 @@
         },
         "Unread": {
           "description": "Total number of messages in the database",
-          "type": "integer",
+          "type": "number",
-          "format": "int64"
+          "format": "double"
         },
         "Version": {
           "description": "Current Mailpit version",
@@ -868,8 +868,8 @@
         },
         "Size": {
           "description": "Size in bytes",
-          "type": "integer",
+          "type": "number",
-          "format": "int64"
+          "format": "double"
         }
       },
       "x-go-package": "github.com/axllent/mailpit/internal/storage"
@@ -1176,8 +1176,8 @@
         },
         "Size": {
           "description": "Message size in bytes",
-          "type": "integer",
+          "type": "number",
-          "format": "int64"
+          "format": "double"
         },
         "Subject": {
           "description": "Message subject",
@@ -1268,8 +1268,8 @@
         },
         "Size": {
           "description": "Message size in bytes (total)",
-          "type": "integer",
+          "type": "number",
-          "format": "int64"
+          "format": "double"
         },
         "Snippet": {
           "description": "Message snippet includes up to 250 characters",
@@ -1310,8 +1310,8 @@
         },
         "messages_count": {
           "description": "Total number of messages matching current query",
-          "type": "integer",
+          "type": "number",
-          "format": "int64",
+          "format": "double",
           "x-go-name": "MessagesCount"
         },
         "start": {
@@ -1330,14 +1330,14 @@
         },
         "total": {
           "description": "Total number of messages in mailbox",
-          "type": "integer",
+          "type": "number",
-          "format": "int64",
+          "format": "double",
           "x-go-name": "Total"
         },
         "unread": {
           "description": "Total number of unread messages in mailbox",
-          "type": "integer",
+          "type": "number",
-          "format": "int64",
+          "format": "double",
           "x-go-name": "Unread"
         }
       },