Merge branch 'release/v1.9.5'

.github/workflows/build-docker.yml

@@ -8,16 +8,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
+      uses: docker/setup-qemu-action@v3
 
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
+      uses: docker/setup-buildx-action@v3
 
     - name: Login to DockerHub
-      uses: docker/login-action@v2
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKER_USERNAME }}
         password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
@@ -30,7 +30,7 @@ jobs:
         version_extractor_regex: 'v(.*)$'
 
     - name: Build and push
-      uses: docker/build-push-action@v4
+      uses: docker/build-push-action@v5
       with:
         context: .
         # platforms: linux/386,linux/amd64,linux/arm,linux/arm64

.github/workflows/codeql-analysis.yml

@@ -38,7 +38,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/release-build.yml

@@ -21,7 +21,7 @@ jobs:
           - goarch: arm
             goos: windows
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # build the assets
     - uses: actions/setup-node@v3

.github/workflows/tests.yml

@@ -15,7 +15,7 @@ jobs:
     - uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/cache@v3
       with:
         path: |

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 Notable changes to Mailpit will be documented in this file.
 
+## [v1.9.5]
+
+### Feature
+- Add `reindex` subcommand to reindex all messages
+- Display email previews ([#175](https://github.com/axllent/mailpit/issues/175))
+
+### Fix
+- HTML message preview background color when switching themes in Chrome
+- Correctly detect tags in search (UI)
+
+### Tests
+- Add message summary tests
+- Add snippet tests
+
+
 ## [v1.9.4]
 
 ### Chore

cmd/reindex.go

@@ -0,0 +1,40 @@
+/*
+Copyright © 2022-Now() Ralph Slooten
+This file is part of a CLI application.
+*/
+package cmd
+
+import (
+	"os"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/spf13/cobra"
+)
+
+// reindexCmd represents the reindex command
+var reindexCmd = &cobra.Command{
+	Use:   "reindex <database>",
+	Short: "Reindex the database",
+	Long: `This will reindex all messages in the entire database.
+
+If you have several thousand messages in your mailbox, then it is advised to shut down
+Mailpit while you reindex as this process will likely result in database locking issues.`,
+	Args: cobra.ExactArgs(1),
+	Run: func(cmd *cobra.Command, args []string) {
+		config.DataFile = args[0]
+		config.MaxMessages = 0
+
+		if err := storage.InitDB(); err != nil {
+			logger.Log().Error(err)
+			os.Exit(1)
+		}
+
+		storage.ReindexAll()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(reindexCmd)
+}

go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/klauspost/compress v1.17.0
 	github.com/leporo/sqlf v1.4.0
 	github.com/mhale/smtpd v0.8.0
+	github.com/microcosm-cc/bluemonday v1.0.25
 	github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e
 	github.com/satori/go.uuid v1.2.0
 	github.com/sirupsen/logrus v1.9.3
@@ -32,6 +33,7 @@ require (
 	github.com/DATA-DOG/go-sqlmock v1.5.0 // indirect
 	github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 // indirect
 	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect
 	github.com/cznic/ql v1.2.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect

go.sum

@@ -13,6 +13,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
 github.com/axllent/semver v0.0.1 h1:QqF+KSGxgj8QZzSXAvKFqjGWE5792ksOnQhludToK8E=
 github.com/axllent/semver v0.0.1/go.mod h1:2xSPzvG8n9mRfdtxSvWvfTfQGWfHsMsHO1iZnKATMSc=
+github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
+github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI=
 github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@@ -97,6 +99,8 @@ github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwp
 github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0=
 github.com/mhale/smtpd v0.8.0/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4=
+github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg=
+github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

internal/storage/database.go

@@ -18,9 +18,9 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/GuiaBolso/darwin"
 	"github.com/axllent/mailpit/config"
 	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
 	"github.com/axllent/mailpit/server/websockets"
 	"github.com/jhillyerd/enmime"
 	"github.com/klauspost/compress/zstd"
@@ -42,81 +42,8 @@ var (
 	// zstd compression encoder & decoder
 	dbEncoder, _ = zstd.NewWriter(nil)
 	dbDecoder, _ = zstd.NewReader(nil)
-
-	dbMigrations = []darwin.Migration{
-		{
-			Version:     1.0,
-			Description: "Creating tables",
-			Script: `CREATE TABLE IF NOT EXISTS mailbox (
-				Sort INTEGER PRIMARY KEY AUTOINCREMENT,
-				ID TEXT NOT NULL,
-				Data BLOB,
-				Search TEXT,
-				Read INTEGER
-			);
-			CREATE INDEX IF NOT EXISTS idx_sort ON mailbox (Sort);
-			CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID);
-			CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read);
-			
-			CREATE TABLE IF NOT EXISTS mailbox_data (
-				ID TEXT KEY NOT NULL,
-				Email BLOB
-			);
-			CREATE UNIQUE INDEX IF NOT EXISTS idx_data_id ON mailbox_data (ID);`,
-		},
-		{
-			Version:     1.1,
-			Description: "Create tags column",
-			Script: `ALTER TABLE mailbox ADD COLUMN Tags Text  NOT NULL DEFAULT '[]';
-			CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
-		},
-		{
-			Version:     1.2,
-			Description: "Creating new mailbox format",
-			Script: `CREATE TABLE IF NOT EXISTS mailboxtmp (
-				Created INTEGER NOT NULL,
-				ID TEXT NOT NULL,
-				MessageID TEXT NOT NULL,
-				Subject TEXT NOT NULL,
-				Metadata TEXT,
-				Size INTEGER NOT NULL,
-				Inline INTEGER NOT NULL,
-				Attachments INTEGER NOT NULL,
-				Read INTEGER,
-				Tags TEXT,
-				SearchText TEXT
-			);
-			INSERT INTO mailboxtmp 
-				(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags) 
-			SELECT 
-				Sort, ID, '', json_extract(Data, '$.Subject'),Data, 
-				json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'), 
-				Search, Read, Tags
-			FROM mailbox;
-
-			DROP TABLE IF EXISTS mailbox;
-			ALTER TABLE mailboxtmp RENAME TO mailbox;
-			CREATE INDEX IF NOT EXISTS idx_created ON mailbox (Created);
-			CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID);
-			CREATE INDEX IF NOT EXISTS idx_message_id ON mailbox (MessageID);
-			CREATE INDEX IF NOT EXISTS idx_subject ON mailbox (Subject);
-			CREATE INDEX IF NOT EXISTS idx_size ON mailbox (Size);
-			CREATE INDEX IF NOT EXISTS idx_inline ON mailbox (Inline);
-			CREATE INDEX IF NOT EXISTS idx_attachments ON mailbox (Attachments);
-			CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read);
-			CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
-		},
-	}
 )
 
-// DBMailSummary struct for storing mail summary
-type DBMailSummary struct {
-	From *mail.Address
-	To   []*mail.Address
-	Cc   []*mail.Address
-	Bcc  []*mail.Address
-}
-
 // InitDB will initialise the database
 func InitDB() error {
 	p := config.DataFile
@@ -178,15 +105,6 @@ func InitDB() error {
 	return nil
 }
 
-// Create tables and apply migrations if required
-func dbApplyMigrations() error {
-	driver := darwin.NewGenericDriver(db, darwin.SqliteDialect{})
-
-	d := darwin.New(driver, dbMigrations, nil)
-
-	return d.Migrate()
-}
-
 // Close will close the database, and delete if a temporary table
 func Close() {
 	if db != nil {
@@ -281,10 +199,11 @@ func Store(body []byte) (string, error) {
 	size := len(body)
 	inline := len(env.Inlines)
 	attachments := len(env.Attachments)
+	snippet := tools.CreateSnippet(env.Text, env.HTML)
 
 	// insert mail summary data
-	_, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Tags, Read) values(?,?,?,?,?,?,?,?,?,?,0)",
+	_, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Tags, Read, Snippet) values(?,?,?,?,?,?,?,?,?,?,0, ?)",
-		created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, string(tagJSON))
+		created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, string(tagJSON), snippet)
 	if err != nil {
 		return "", err
 	}
@@ -312,6 +231,7 @@ func Store(body []byte) (string, error) {
 	c.Subject = subject
 	c.Size = size
 	c.Tags = tagData
+	c.Snippet = snippet
 
 	websockets.Broadcast("new", c)
 
@@ -328,7 +248,7 @@ func List(start, limit int) ([]MessageSummary, error) {
 	results := []MessageSummary{}
 
 	q := sqlf.From("mailbox").
-		Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags`).
+		Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags, Snippet`).
 		OrderBy("Created DESC").
 		Limit(limit).
 		Offset(start)
@@ -343,9 +263,10 @@ func List(start, limit int) ([]MessageSummary, error) {
 		var attachments int
 		var tags string
 		var read int
+		var snippet string
 		em := MessageSummary{}
 
-		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags); err != nil {
+		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &snippet); err != nil {
 			logger.Log().Error(err)
 			return
 		}
@@ -367,6 +288,7 @@ func List(start, limit int) ([]MessageSummary, error) {
 		em.Size = size
 		em.Attachments = attachments
 		em.Read = read == 1
+		em.Snippet = snippet
 
 		results = append(results, em)
 	}); err != nil {

internal/storage/database_test.go

@@ -117,6 +117,38 @@ func TestRetrieveMimeEmail(t *testing.T) {
 	assertEqual(t, len(inlineData.Content), msg.Inline[0].Size, "inline attachment size does not match")
 }
 
+func TestMessageSummary(t *testing.T) {
+	setup()
+	defer Close()
+
+	t.Log("Testing message summary")
+
+	if _, err := Store(testMimeEmail); err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	summaries, err := List(0, 1)
+	if err != nil {
+		t.Log("error ", err)
+		t.Fail()
+	}
+
+	assertEqual(t, len(summaries), 1, "Expected 1 result")
+
+	msg := summaries[0]
+
+	assertEqual(t, msg.From.Name, "Sender Smith", "\"From\" name does not match")
+	assertEqual(t, msg.From.Address, "sender2@example.com", "\"From\" address does not match")
+	assertEqual(t, msg.Subject, "inline + attachment", "subject does not match")
+	assertEqual(t, len(msg.To), 1, "incorrect number of recipients")
+	assertEqual(t, msg.To[0].Name, "Recipient Ross", "\"To\" name does not match")
+	assertEqual(t, msg.To[0].Address, "recipient2@example.com", "\"To\" address does not match")
+	assertEqual(t, msg.Snippet, "Message with inline image and attachment:", "\"Snippet\" does does not match")
+	assertEqual(t, msg.Attachments, 1, "Expected 1 attachment")
+	assertEqual(t, msg.MessageID, "33af2ac1-c33d-9738-35e3-a6daf90bbd89@gmail.com", "\"MessageID\" does not match")
+}
+
 func BenchmarkImportText(b *testing.B) {
 	setup()
 	defer Close()

internal/storage/migrations.go

@@ -0,0 +1,84 @@
+package storage
+
+import "github.com/GuiaBolso/darwin"
+
+var (
+	dbMigrations = []darwin.Migration{
+		{
+			Version:     1.0,
+			Description: "Creating tables",
+			Script: `CREATE TABLE IF NOT EXISTS mailbox (
+				Sort INTEGER PRIMARY KEY AUTOINCREMENT,
+				ID TEXT NOT NULL,
+				Data BLOB,
+				Search TEXT,
+				Read INTEGER
+			);
+			CREATE INDEX IF NOT EXISTS idx_sort ON mailbox (Sort);
+			CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID);
+			CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read);
+			
+			CREATE TABLE IF NOT EXISTS mailbox_data (
+				ID TEXT KEY NOT NULL,
+				Email BLOB
+			);
+			CREATE UNIQUE INDEX IF NOT EXISTS idx_data_id ON mailbox_data (ID);`,
+		},
+		{
+			Version:     1.1,
+			Description: "Create tags column",
+			Script: `ALTER TABLE mailbox ADD COLUMN Tags Text  NOT NULL DEFAULT '[]';
+			CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
+		},
+		{
+			Version:     1.2,
+			Description: "Creating new mailbox format",
+			Script: `CREATE TABLE IF NOT EXISTS mailboxtmp (
+				Created INTEGER NOT NULL,
+				ID TEXT NOT NULL,
+				MessageID TEXT NOT NULL,
+				Subject TEXT NOT NULL,
+				Metadata TEXT,
+				Size INTEGER NOT NULL,
+				Inline INTEGER NOT NULL,
+				Attachments INTEGER NOT NULL,
+				Read INTEGER,
+				Tags TEXT,
+				SearchText TEXT
+			);
+			INSERT INTO mailboxtmp 
+				(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags) 
+			SELECT 
+				Sort, ID, '', json_extract(Data, '$.Subject'),Data, 
+				json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'), 
+				Search, Read, Tags
+			FROM mailbox;
+
+			DROP TABLE IF EXISTS mailbox;
+			ALTER TABLE mailboxtmp RENAME TO mailbox;
+			CREATE INDEX IF NOT EXISTS idx_created ON mailbox (Created);
+			CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID);
+			CREATE INDEX IF NOT EXISTS idx_message_id ON mailbox (MessageID);
+			CREATE INDEX IF NOT EXISTS idx_subject ON mailbox (Subject);
+			CREATE INDEX IF NOT EXISTS idx_size ON mailbox (Size);
+			CREATE INDEX IF NOT EXISTS idx_inline ON mailbox (Inline);
+			CREATE INDEX IF NOT EXISTS idx_attachments ON mailbox (Attachments);
+			CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read);
+			CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
+		},
+		{
+			Version:     1.3,
+			Description: "Create snippet column",
+			Script:      `ALTER TABLE mailbox ADD COLUMN Snippet Text NOT NULL DEFAULT '';`,
+		},
+	}
+)
+
+// Create tables and apply migrations if required
+func dbApplyMigrations() error {
+	driver := darwin.NewGenericDriver(db, darwin.SqliteDialect{})
+
+	d := darwin.New(driver, dbMigrations, nil)
+
+	return d.Migrate()
+}

internal/storage/reindex.go

@@ -0,0 +1,184 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"os"
+
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
+	"github.com/jhillyerd/enmime"
+	"github.com/leporo/sqlf"
+)
+
+// ReindexAll will regenerate the search text and snippet for a message
+// and update the database.
+func ReindexAll() {
+	ids := []string{}
+	var i string
+	chunkSize := 1000
+
+	finished := 0
+
+	err := sqlf.Select("ID").To(&i).
+		From("mailbox").
+		OrderBy("Created DESC").
+		QueryAndClose(nil, db, func(row *sql.Rows) {
+			ids = append(ids, i)
+		})
+
+	if err != nil {
+		logger.Log().Error(err)
+		os.Exit(1)
+	}
+
+	total := len(ids)
+
+	chunks := chunkBy(ids, chunkSize)
+
+	logger.Log().Infof("Reindexing %d messages", total)
+
+	// fmt.Println(len(ids), " = ", len(chunks), "chunks")
+
+	type updateStruct struct {
+		ID         string
+		SearchText string
+		Snippet    string
+	}
+
+	for _, ids := range chunks {
+		updates := []updateStruct{}
+
+		for _, id := range ids {
+			raw, err := GetMessageRaw(id)
+			if err != nil {
+				logger.Log().Error(err)
+				continue
+			}
+
+			r := bytes.NewReader(raw)
+
+			env, err := enmime.ReadEnvelope(r)
+			if err != nil {
+				logger.Log().Error(err)
+				continue
+			}
+
+			searchText := createSearchText(env)
+			snippet := tools.CreateSnippet(env.Text, env.HTML)
+
+			u := updateStruct{}
+			u.ID = id
+			u.SearchText = searchText
+			u.Snippet = snippet
+
+			updates = append(updates, u)
+		}
+
+		ctx := context.Background()
+		tx, err := db.BeginTx(ctx, nil)
+		if err != nil {
+			logger.Log().Error(err)
+			continue
+		}
+
+		// roll back if it fails
+		defer tx.Rollback()
+
+		// insert mail summary data
+		for _, u := range updates {
+			_, err = tx.Exec("UPDATE mailbox SET SearchText = ?, Snippet = ? WHERE ID = ?", u.SearchText, u.Snippet, u.ID)
+			if err != nil {
+				logger.Log().Error(err)
+				continue
+			}
+		}
+
+		if err := tx.Commit(); err != nil {
+			logger.Log().Error(err)
+			continue
+		}
+
+		finished += len(updates)
+
+		logger.Log().Printf("Reindexed: %d / %d (%d%%)", finished, total, finished*100/total)
+	}
+}
+
+// Reindex will regenerate the search text and snippet for a message
+// and update the database.
+func Reindex(id string) error {
+	// ids := []string{}
+	// var i string
+	// // chunkSize := 100
+
+	// err := sqlf.Select("ID").To(&i).From("mailbox_data").QueryAndClose(nil, db, func(row *sql.Rows) {
+	// 	ids = append(ids, id)
+	// })
+
+	// if err != nil {
+	// 	return err
+	// }
+
+	// chunks := chunkBy(ids, 100)
+
+	// fmt.Println(len(ids), " = ", len(chunks), "chunks")
+
+	// return nil
+
+	raw, err := GetMessageRaw(id)
+	if err != nil {
+		return err
+	}
+
+	r := bytes.NewReader(raw)
+
+	env, err := enmime.ReadEnvelope(r)
+	if err != nil {
+		return err
+	}
+
+	searchText := createSearchText(env)
+	snippet := tools.CreateSnippet(env.Text, env.HTML)
+
+	// return nil
+
+	// ctx := context.Background()
+	// tx, err := db.BeginTx(ctx, nil)
+	// if err != nil {
+	// 	return err
+	// }
+
+	// // roll back if it fails
+	// defer tx.Rollback()
+
+	// // insert mail summary data
+	// _, err = tx.Exec("UPDATE mailbox SET SearchText = ?, Snippet = ? WHERE ID = ?", searchText, snippet, id)
+	// if err != nil {
+	// 	return err
+	// }
+
+	// return tx.Commit()
+
+	_, err = sqlf.Update("mailbox").
+		Set("SearchText", searchText).
+		Set("Snippet", snippet).
+		Where("ID = ?", id).
+		ExecAndClose(context.Background(), db)
+
+	return err
+}
+
+// ctx := context.Background()
+// tx, err := db.BeginTx(ctx, nil)
+// if err != nil {
+// 	return "", err
+// }
+
+func chunkBy[T any](items []T, chunkSize int) (chunks [][]T) {
+	for chunkSize < len(items) {
+		items, chunks = items[chunkSize:], append(chunks, items[0:chunkSize:chunkSize])
+	}
+	return append(chunks, items)
+}

internal/storage/search.go

@@ -38,11 +38,12 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) {
 		var size int
 		var attachments int
 		var tags string
+		var snippet string
 		var read int
 		var ignore string
 		em := MessageSummary{}
 
-		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil {
+		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &snippet, &ignore, &ignore, &ignore, &ignore); err != nil {
 			logger.Log().Error(err)
 			return
 		}
@@ -64,6 +65,7 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) {
 		em.Size = size
 		em.Attachments = attachments
 		em.Read = read == 1
+		em.Snippet = snippet
 
 		allResults = append(allResults, em)
 	}); err != nil {
@@ -109,9 +111,10 @@ func DeleteSearch(search string) error {
 		var attachments int
 		var tags string
 		var read int
+		var snippet string
 		var ignore string
 
-		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil {
+		if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &snippet, &ignore, &ignore, &ignore, &ignore); err != nil {
 			logger.Log().Error(err)
 			return
 		}
@@ -193,7 +196,7 @@ func searchQueryBuilder(searchString string) *sqlf.Stmt {
 	args := tools.ArgsParser(searchString)
 
 	q := sqlf.From("mailbox").
-		Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags,
+		Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags, Snippet,
 			IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON,
 			IFNULL(json_extract(Metadata, '$.From'), '{}') as FromJSON,
 			IFNULL(json_extract(Metadata, '$.Cc'), '{}') as CcJSON,

internal/storage/structs.go

@@ -89,6 +89,8 @@ type MessageSummary struct {
 	Size int
 	// Whether the message has any attachments
 	Attachments int
+	// Message snippet includes up to 250 characters
+	Snippet string
 }
 
 // MailboxStats struct for quick mailbox total/read lookups
@@ -98,6 +100,14 @@ type MailboxStats struct {
 	Tags   []string
 }
 
+// DBMailSummary struct for storing mail summary
+type DBMailSummary struct {
+	From *mail.Address
+	To   []*mail.Address
+	Cc   []*mail.Address
+	Bcc  []*mail.Address
+}
+
 // AttachmentSummary returns a summary of the attachment without any binary data
 func AttachmentSummary(a *enmime.Part) Attachment {
 	o := Attachment{}

internal/storage/testdata/mime-attachment.eml

@@ -108,10 +108,9 @@ Content-Transfer-Encoding: 7bit
     <meta http-equiv="content-type" content="text/html; charset=UTF-8">
   </head>
   <body>
-    Message with inline image and attachment:<br>
+    <h1>Message with inline image and attachment:</h1>
     <br>
-    <img src="cid:part1.845LaYlX.wtWMpWwa@gmail.com"
+    <p><img src="cid:part1.845LaYlX.wtWMpWwa@gmail.com"></p>
-      moz-do-not-send="false"><br>
     <br>
     <br>
   </body>

internal/tools/html.go

@@ -2,7 +2,9 @@ package tools
 
 import (
 	"fmt"
+	"strings"
 
+	"github.com/microcosm-cc/bluemonday"
 	"golang.org/x/net/html"
 )
 
@@ -17,3 +19,12 @@ func GetHTMLAttributeVal(e *html.Node, key string) (string, error) {
 
 	return "", fmt.Errorf("%s not found", key)
 }
+
+// StripHTML returns text from an HTML string
+func stripHTML(h string) string {
+	p := bluemonday.StrictPolicy()
+	// // ensure joining html elements are spaced apart, eg table cells etc
+	h = strings.ReplaceAll(h, "><", "> <")
+	// return p.Sanitize(h)
+	return html.UnescapeString(p.Sanitize(h))
+}

internal/tools/snippets.go

@@ -0,0 +1,42 @@
+package tools
+
+import (
+	"regexp"
+	"strings"
+)
+
+// CreateSnippet returns a message snippet. It will use the HTML version (if it exists)
+// and fall back to the text version.
+func CreateSnippet(text, html string) string {
+	text = strings.TrimSpace(text)
+	html = strings.TrimSpace(html)
+	characters := 200
+	spaceRe := regexp.MustCompile(`\s+`)
+	nlRe := regexp.MustCompile(`\r?\n`)
+
+	if text == "" && html == "" {
+		return ""
+	}
+
+	if html != "" {
+		data := nlRe.ReplaceAllString(stripHTML(html), " ")
+		data = strings.TrimSpace(spaceRe.ReplaceAllString(data, " "))
+
+		if len(data) <= characters {
+			return data
+		}
+
+		return data[0:characters] + "..."
+	}
+
+	if text != "" {
+		text = spaceRe.ReplaceAllString(text, " ")
+		if len(text) <= characters {
+			return text
+		}
+
+		return text[0:characters] + "..."
+	}
+
+	return ""
+}

internal/tools/tools_test.go

@@ -43,3 +43,29 @@ func TestCleanTag(t *testing.T) {
 		}
 	}
 }
+
+func TestSnippets(t *testing.T) {
+	tests := map[string]string{}
+	tests["this is a  test"] = "this is a test"
+	tests["thiS IS a Test"] = "thiS IS a Test"
+	tests["thiS IS a Test :-)"] = "thiS IS a Test :-)"
+	tests["<h1>This is a test.</h1> "] = "This is a test."
+	tests["this_is-a     test "] = "this_is-a test"
+	tests["this_is-a&^%%(*)@ test"] = "this_is-a&^%%(*)@ test"
+	tests["<h1>Heading</h1><p>Paragraph</p>"] = "Heading Paragraph"
+	tests[`<h1>Heading</h1>
+		<p>Paragraph</p>`] = "Heading Paragraph"
+	tests[`<h1>Heading</h1><p>   <a href="https://github.com">linked text</a></p>`] = "Heading linked text"
+	// broken html
+	tests[`<h1>Heading</h3><p>   <a href="https://github.com">linked text.`] = "Heading linked text."
+	// truncation to 200 chars + ...
+	tests["abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789"] = "abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmno..."
+
+	for str, expected := range tests {
+		res := CreateSnippet(str, str)
+		if res != expected {
+			t.Log("CreateSnippet error:", res, "!=", expected)
+			t.Fail()
+		}
+	}
+}

server/ui-src/assets/styles.scss

@@ -94,11 +94,33 @@
 }
 
 .message {
+	.subject {
+		display: -webkit-box;
+		-webkit-box-orient: vertical;
+		-webkit-line-clamp: 1;
+		overflow: hidden;
+		color: $text-muted;
+
+		b {
+			color: $list-group-color;
+		}
+
+		small {
+			opacity: 0.5;
+		}
+	}
+
 	&.read {
 		color: $text-muted;
 
 		b {
+			opacity: 0.7;
 			font-weight: normal;
+			color: $list-group-color;
+		}
+
+		small {
+			opacity: 0.5;
 		}
 	}
 	&.selected {

server/ui-src/components/ListMessages.vue

@@ -141,7 +141,10 @@ export default {
 					</div>
 				</div>
 				<div class="col-lg-6 col-xxl-7 mt-2 mt-lg-0">
-					<div><b>{{ message.Subject != "" ? message.Subject : "[ no subject ]" }}</b></div>
+					<div class="subject">
+						<b>{{ message.Subject != "" ? message.Subject : "[ no subject ]" }}</b>
+						<small v-if="message.Snippet != ''" class="small">&nbsp; {{ message.Snippet }}</small>
+					</div>
 					<div>
 						<RouterLink class="badge me-1" v-for="t in message.Tags" :to="'/search?q=' + tagEncodeURI(t)"
 							:style="mailbox.showTagColors ? { backgroundColor: colorHash(t) } : { backgroundColor: '#6c757d' }"

server/ui-src/components/NavTags.vue

@@ -19,7 +19,7 @@ export default {
 				return false
 			}
 
-			let re = new RegExp(`\\b[^\-!]tag:"?${tag}"?\\b`, 'i')
+			let re = new RegExp(`(^|\\s)tag:"?${tag}"?($|\\s)`, 'i')
 			return query.match(re)
 		}
 	}

server/ui-src/components/message/Message.vue

@@ -443,7 +443,7 @@ export default {
 				aria-labelledby="nav-html-tab" tabindex="0">
 				<div id="responsive-view" :class="scaleHTMLPreview" :style="responsiveSizes[scaleHTMLPreview]">
 					<iframe target-blank="" class="tab-pane d-block" id="preview-html" :srcdoc="sanitizeHTML(message.HTML)"
-						v-on:load="resizeIframe" frameborder="0" style="width: 100%; height: 100%;">
+						v-on:load="resizeIframe" frameborder="0" style="width: 100%; height: 100%; background: #fff;">
 					</iframe>
 				</div>
 				<Attachments v-if="allAttachments(message).length" :message="message"