Chore: Compress database only when >= 1% of total message size has been deleted

internal/storage/database.go

@@ -33,12 +33,12 @@ import (
 )
 
 var (
-	db            *sql.DB
-	dbFile        string
-	dbIsTemp      bool
-	dbLastAction  time.Time
-	dbIsIdle      bool
-	dbDataDeleted bool
+	db           *sql.DB
+	dbFile       string
+	dbIsTemp     bool
+	dbLastAction time.Time
+	dbIsIdle     bool
+	deletedSize  int64
 
 	// zstd compression encoder & decoder
 	dbEncoder, _ = zstd.NewWriter(nil)
@@ -615,6 +615,10 @@ func MarkUnread(id string) error {
 
 // DeleteOneMessage will delete a single message from a mailbox
 func DeleteOneMessage(id string) error {
+	m, err := GetMessage(id)
+	if err != nil {
+		return err
+	}
 	// begin a transaction to ensure both the message
 	// and data are deleted successfully
 	tx, err := db.BeginTx(context.Background(), nil)
@@ -646,7 +650,7 @@ func DeleteOneMessage(id string) error {
 	}
 
 	dbLastAction = time.Now()
-	dbDataDeleted = true
+	deletedSize = deletedSize + int64(m.Size)
 
 	logMessagesDeleted(1)
 
@@ -707,7 +711,7 @@ func DeleteAllMessages() error {
 	}
 
 	dbLastAction = time.Now()
-	dbDataDeleted = false
+	deletedSize = 0
 
 	logMessagesDeleted(total)
 

internal/storage/search.go

@@ -99,6 +99,7 @@ func DeleteSearch(search string) error {
 	q := searchQueryBuilder(search)
 
 	ids := []string{}
+	deleteSize := 0
 
 	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
 		var created int64
@@ -119,6 +120,7 @@ func DeleteSearch(search string) error {
 		}
 
 		ids = append(ids, id)
+		deleteSize = deleteSize + size
 	}); err != nil {
 		return err
 	}
@@ -191,7 +193,7 @@ func DeleteSearch(search string) error {
 		}
 
 		dbLastAction = time.Now()
-		dbDataDeleted = true
+		deletedSize = deletedSize + int64(deleteSize)
 
 		logMessagesDeleted(total)
 

internal/storage/utils.go

@@ -88,32 +88,49 @@ func dbCron() {
 		// for 5 minutes, if so VACUUM
 		currentTime := time.Now()
 		diff := currentTime.Sub(dbLastAction)
-		if dbDataDeleted && diff.Minutes() > 5 {
-			dbDataDeleted = false
+
+		// get DB file size
+		fileInfo, err := os.Stat(config.DataFile)
+		if err != nil {
+			logger.Log().Errorf("[db] unable to stat database %s: %s", config.DataFile, err.Error())
+			continue
+		}
+
+		deletedPercent := deletedSize * 100 / fileInfo.Size()
+
+		// only vacuum DB when at least 2% of mail storage size has been deleted
+		// as this saves a lot of CPU on large databases
+		if deletedPercent >= 1 && diff.Minutes() > 5 {
+			logger.Log().Debugf("[db] compressing database as %d%% has been deleted", deletedPercent)
+			deletedSize = 0
 			_, err := db.Exec("VACUUM")
 			if err == nil {
 				elapsed := time.Since(start)
 				logger.Log().Debugf("[db] compressed idle database in %s", elapsed)
 			}
+
 			continue
 		}
 
 		if config.MaxMessages > 0 {
-			q := sqlf.Select("ID").
+			q := sqlf.Select("ID, Size").
 				From("mailbox").
 				OrderBy("Created DESC").
 				Limit(5000).
 				Offset(config.MaxMessages)
 
 			ids := []string{}
+			var prunedSize int64
+			var size int
 			if err := q.Query(nil, db, func(row *sql.Rows) {
 				var id string
 
-				if err := row.Scan(&id); err != nil {
+				if err := row.Scan(&id, &size); err != nil {
 					logger.Log().Errorf("[db] %s", err.Error())
 					return
 				}
 				ids = append(ids, id)
+				prunedSize = prunedSize + int64(size)
 
 			}); err != nil {
 				logger.Log().Errorf("[db] %s", err.Error())
@@ -166,7 +183,7 @@ func dbCron() {
 				logger.Log().Errorf("[db] %s", err.Error())
 			}
 
-			dbDataDeleted = true
+			deletedSize = deletedSize + prunedSize
 
 			elapsed := time.Since(start)
 			logger.Log().Debugf("[db] auto-pruned %d messages in %s", len(ids), elapsed)