go/mailpit
1
0
Fork 0
mirror of https://github.com/axllent/mailpit.git synced 2025-03-17 21:18:19 +02:00
Code Issues Releases Activity

Merge branch 'release/v1.9.5'

Browse Source
This commit is contained in:
Ralph Slooten 2023-10-05 17:39:19 +13:00
commit af93444374
23 changed files with 503 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.github/workflows/build-docker.yml vendored
View File

@ -8,16 +8,16 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
uses: docker/login-action@v2
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
@ -30,7 +30,7 @@ jobs:
version_extractor_regex: 'v(.*)$'
- name: Build and push
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
context: .
# platforms: linux/386,linux/amd64,linux/arm,linux/arm64

2
.github/workflows/codeql-analysis.yml vendored
View File

@ -38,7 +38,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v4
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL

2
.github/workflows/release-build.yml vendored
View File

@ -21,7 +21,7 @@ jobs:
- goarch: arm
goos: windows
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
# build the assets
- uses: actions/setup-node@v3

2
.github/workflows/tests.yml vendored
View File

@ -15,7 +15,7 @@ jobs:
- uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/cache@v3
with:
path: |

15
CHANGELOG.md
View File

@ -2,6 +2,21 @@
Notable changes to Mailpit will be documented in this file.
## [v1.9.5]
### Feature
- Add `reindex` subcommand to reindex all messages
- Display email previews ([#175](https://github.com/axllent/mailpit/issues/175))
### Fix
- HTML message preview background color when switching themes in Chrome
- Correctly detect tags in search (UI)
### Tests
- Add message summary tests
- Add snippet tests
## [v1.9.4]
### Chore

40
cmd/reindex.go Normal file
View File

@ -0,0 +1,40 @@
/*
Copyright © 2022-Now() Ralph Slooten
This file is part of a CLI application.
*/
package cmd
import (
"os"
"github.com/axllent/mailpit/config"
"github.com/axllent/mailpit/internal/logger"
"github.com/axllent/mailpit/internal/storage"
"github.com/spf13/cobra"
)
// reindexCmd represents the reindex command
var reindexCmd = &cobra.Command{
Use: "reindex <database>",
Short: "Reindex the database",
Long: `This will reindex all messages in the entire database.
If you have several thousand messages in your mailbox, then it is advised to shut down
Mailpit while you reindex as this process will likely result in database locking issues.`,
Args: cobra.ExactArgs(1),
Run: func(cmd *cobra.Command, args []string) {
config.DataFile = args[0]
config.MaxMessages = 0
if err := storage.InitDB(); err != nil {
logger.Log().Error(err)
os.Exit(1)
}
storage.ReindexAll()
},
}
func init() {
rootCmd.AddCommand(reindexCmd)
}

2
go.mod
View File

@ -15,6 +15,7 @@ require (
github.com/klauspost/compress v1.17.0
github.com/leporo/sqlf v1.4.0
github.com/mhale/smtpd v0.8.0
github.com/microcosm-cc/bluemonday v1.0.25
github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e
github.com/satori/go.uuid v1.2.0
github.com/sirupsen/logrus v1.9.3
@ -32,6 +33,7 @@ require (
github.com/DATA-DOG/go-sqlmock v1.5.0 // indirect
github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 // indirect
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect
github.com/cznic/ql v1.2.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect

4
go.sum
View File

@ -13,6 +13,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/axllent/semver v0.0.1 h1:QqF+KSGxgj8QZzSXAvKFqjGWE5792ksOnQhludToK8E=
github.com/axllent/semver v0.0.1/go.mod h1:2xSPzvG8n9mRfdtxSvWvfTfQGWfHsMsHO1iZnKATMSc=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI=
github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
@ -97,6 +99,8 @@ github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwp
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0=
github.com/mhale/smtpd v0.8.0/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4=
github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg=
github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

96
internal/storage/database.go
View File

@ -18,9 +18,9 @@ import (
"syscall"
"time"
"github.com/GuiaBolso/darwin"
"github.com/axllent/mailpit/config"
"github.com/axllent/mailpit/internal/logger"
"github.com/axllent/mailpit/internal/tools"
"github.com/axllent/mailpit/server/websockets"
"github.com/jhillyerd/enmime"
"github.com/klauspost/compress/zstd"
@ -42,81 +42,8 @@ var (
// zstd compression encoder & decoder
dbEncoder, _ = zstd.NewWriter(nil)
dbDecoder, _ = zstd.NewReader(nil)
dbMigrations = []darwin.Migration{
{
Version: 1.0,
Description: "Creating tables",
Script: `CREATE TABLE IF NOT EXISTS mailbox (
Sort INTEGER PRIMARY KEY AUTOINCREMENT,
ID TEXT NOT NULL,
Data BLOB,
Search TEXT,
Read INTEGER
);
CREATE INDEX IF NOT EXISTS idx_sort ON mailbox (Sort);
CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID);
CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read);
CREATE TABLE IF NOT EXISTS mailbox_data (
ID TEXT KEY NOT NULL,
Email BLOB
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_data_id ON mailbox_data (ID);`,
},
{
Version: 1.1,
Description: "Create tags column",
Script: `ALTER TABLE mailbox ADD COLUMN Tags Text NOT NULL DEFAULT '[]';
CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
},
{
Version: 1.2,
Description: "Creating new mailbox format",
Script: `CREATE TABLE IF NOT EXISTS mailboxtmp (
Created INTEGER NOT NULL,
ID TEXT NOT NULL,
MessageID TEXT NOT NULL,
Subject TEXT NOT NULL,
Metadata TEXT,
Size INTEGER NOT NULL,
Inline INTEGER NOT NULL,
Attachments INTEGER NOT NULL,
Read INTEGER,
Tags TEXT,
SearchText TEXT
);
INSERT INTO mailboxtmp
(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags)
SELECT
Sort, ID, '', json_extract(Data, '$.Subject'),Data,
json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'),
Search, Read, Tags
FROM mailbox;
DROP TABLE IF EXISTS mailbox;
ALTER TABLE mailboxtmp RENAME TO mailbox;
CREATE INDEX IF NOT EXISTS idx_created ON mailbox (Created);
CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID);
CREATE INDEX IF NOT EXISTS idx_message_id ON mailbox (MessageID);
CREATE INDEX IF NOT EXISTS idx_subject ON mailbox (Subject);
CREATE INDEX IF NOT EXISTS idx_size ON mailbox (Size);
CREATE INDEX IF NOT EXISTS idx_inline ON mailbox (Inline);
CREATE INDEX IF NOT EXISTS idx_attachments ON mailbox (Attachments);
CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read);
CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
},
}
)
// DBMailSummary struct for storing mail summary
type DBMailSummary struct {
From *mail.Address
To []*mail.Address
Cc []*mail.Address
Bcc []*mail.Address
}
// InitDB will initialise the database
func InitDB() error {
p := config.DataFile
@ -178,15 +105,6 @@ func InitDB() error {
return nil
}
// Create tables and apply migrations if required
func dbApplyMigrations() error {
driver := darwin.NewGenericDriver(db, darwin.SqliteDialect{})
d := darwin.New(driver, dbMigrations, nil)
return d.Migrate()
}
// Close will close the database, and delete if a temporary table
func Close() {
if db != nil {
@ -281,10 +199,11 @@ func Store(body []byte) (string, error) {
size := len(body)
inline := len(env.Inlines)
attachments := len(env.Attachments)
snippet := tools.CreateSnippet(env.Text, env.HTML)
// insert mail summary data
_, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Tags, Read) values(?,?,?,?,?,?,?,?,?,?,0)",
created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, string(tagJSON))
_, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Tags, Read, Snippet) values(?,?,?,?,?,?,?,?,?,?,0, ?)",
created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, string(tagJSON), snippet)
if err != nil {
return "", err
}
@ -312,6 +231,7 @@ func Store(body []byte) (string, error) {
c.Subject = subject
c.Size = size
c.Tags = tagData
c.Snippet = snippet
websockets.Broadcast("new", c)
@ -328,7 +248,7 @@ func List(start, limit int) ([]MessageSummary, error) {
results := []MessageSummary{}
q := sqlf.From("mailbox").
Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags`).
Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags, Snippet`).
OrderBy("Created DESC").
Limit(limit).
Offset(start)
@ -343,9 +263,10 @@ func List(start, limit int) ([]MessageSummary, error) {
var attachments int
var tags string
var read int
var snippet string
em := MessageSummary{}
if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags); err != nil {
if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &snippet); err != nil {
logger.Log().Error(err)
return
}
@ -367,6 +288,7 @@ func List(start, limit int) ([]MessageSummary, error) {
em.Size = size
em.Attachments = attachments
em.Read = read == 1
em.Snippet = snippet
results = append(results, em)
}); err != nil {

32
internal/storage/database_test.go
View File

@ -117,6 +117,38 @@ func TestRetrieveMimeEmail(t *testing.T) {
assertEqual(t, len(inlineData.Content), msg.Inline[0].Size, "inline attachment size does not match")
}
func TestMessageSummary(t *testing.T) {
setup()
defer Close()
t.Log("Testing message summary")
if _, err := Store(testMimeEmail); err != nil {
t.Log("error ", err)
t.Fail()
}
summaries, err := List(0, 1)
if err != nil {
t.Log("error ", err)
t.Fail()
}
assertEqual(t, len(summaries), 1, "Expected 1 result")
msg := summaries[0]
assertEqual(t, msg.From.Name, "Sender Smith", "\"From\" name does not match")
assertEqual(t, msg.From.Address, "sender2@example.com", "\"From\" address does not match")
assertEqual(t, msg.Subject, "inline + attachment", "subject does not match")
assertEqual(t, len(msg.To), 1, "incorrect number of recipients")
assertEqual(t, msg.To[0].Name, "Recipient Ross", "\"To\" name does not match")
assertEqual(t, msg.To[0].Address, "recipient2@example.com", "\"To\" address does not match")
assertEqual(t, msg.Snippet, "Message with inline image and attachment:", "\"Snippet\" does does not match")
assertEqual(t, msg.Attachments, 1, "Expected 1 attachment")
assertEqual(t, msg.MessageID, "33af2ac1-c33d-9738-35e3-a6daf90bbd89@gmail.com", "\"MessageID\" does not match")
}
func BenchmarkImportText(b *testing.B) {
setup()
defer Close()

84
internal/storage/migrations.go Normal file
View File

@ -0,0 +1,84 @@
package storage
import "github.com/GuiaBolso/darwin"
var (
dbMigrations = []darwin.Migration{
{
Version: 1.0,
Description: "Creating tables",
Script: `CREATE TABLE IF NOT EXISTS mailbox (
Sort INTEGER PRIMARY KEY AUTOINCREMENT,
ID TEXT NOT NULL,
Data BLOB,
Search TEXT,
Read INTEGER
);
CREATE INDEX IF NOT EXISTS idx_sort ON mailbox (Sort);
CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID);
CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read);
CREATE TABLE IF NOT EXISTS mailbox_data (
ID TEXT KEY NOT NULL,
Email BLOB
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_data_id ON mailbox_data (ID);`,
},
{
Version: 1.1,
Description: "Create tags column",
Script: `ALTER TABLE mailbox ADD COLUMN Tags Text NOT NULL DEFAULT '[]';
CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
},
{
Version: 1.2,
Description: "Creating new mailbox format",
Script: `CREATE TABLE IF NOT EXISTS mailboxtmp (
Created INTEGER NOT NULL,
ID TEXT NOT NULL,
MessageID TEXT NOT NULL,
Subject TEXT NOT NULL,
Metadata TEXT,
Size INTEGER NOT NULL,
Inline INTEGER NOT NULL,
Attachments INTEGER NOT NULL,
Read INTEGER,
Tags TEXT,
SearchText TEXT
);
INSERT INTO mailboxtmp
(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags)
SELECT
Sort, ID, '', json_extract(Data, '$.Subject'),Data,
json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'),
Search, Read, Tags
FROM mailbox;
DROP TABLE IF EXISTS mailbox;
ALTER TABLE mailboxtmp RENAME TO mailbox;
CREATE INDEX IF NOT EXISTS idx_created ON mailbox (Created);
CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID);
CREATE INDEX IF NOT EXISTS idx_message_id ON mailbox (MessageID);
CREATE INDEX IF NOT EXISTS idx_subject ON mailbox (Subject);
CREATE INDEX IF NOT EXISTS idx_size ON mailbox (Size);
CREATE INDEX IF NOT EXISTS idx_inline ON mailbox (Inline);
CREATE INDEX IF NOT EXISTS idx_attachments ON mailbox (Attachments);
CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read);
CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
},
{
Version: 1.3,
Description: "Create snippet column",
Script: `ALTER TABLE mailbox ADD COLUMN Snippet Text NOT NULL DEFAULT '';`,
},
}
)
// Create tables and apply migrations if required
func dbApplyMigrations() error {
driver := darwin.NewGenericDriver(db, darwin.SqliteDialect{})
d := darwin.New(driver, dbMigrations, nil)
return d.Migrate()
}

184
internal/storage/reindex.go Normal file
View File

@ -0,0 +1,184 @@
package storage
import (
"bytes"
"context"
"database/sql"
"os"
"github.com/axllent/mailpit/internal/logger"
"github.com/axllent/mailpit/internal/tools"
"github.com/jhillyerd/enmime"
"github.com/leporo/sqlf"
)
// ReindexAll will regenerate the search text and snippet for a message
// and update the database.
func ReindexAll() {
ids := []string{}
var i string
chunkSize := 1000
finished := 0
err := sqlf.Select("ID").To(&i).
From("mailbox").
OrderBy("Created DESC").
QueryAndClose(nil, db, func(row *sql.Rows) {
ids = append(ids, i)
})
if err != nil {
logger.Log().Error(err)
os.Exit(1)
}
total := len(ids)
chunks := chunkBy(ids, chunkSize)
logger.Log().Infof("Reindexing %d messages", total)
// fmt.Println(len(ids), " = ", len(chunks), "chunks")
type updateStruct struct {
ID string
SearchText string
Snippet string
}
for _, ids := range chunks {
updates := []updateStruct{}
for _, id := range ids {
raw, err := GetMessageRaw(id)
if err != nil {
logger.Log().Error(err)
continue
}
r := bytes.NewReader(raw)
env, err := enmime.ReadEnvelope(r)
if err != nil {
logger.Log().Error(err)
continue
}
searchText := createSearchText(env)
snippet := tools.CreateSnippet(env.Text, env.HTML)
u := updateStruct{}
u.ID = id
u.SearchText = searchText
u.Snippet = snippet
updates = append(updates, u)
}
ctx := context.Background()
tx, err := db.BeginTx(ctx, nil)
if err != nil {
logger.Log().Error(err)
continue
}
// roll back if it fails
defer tx.Rollback()
// insert mail summary data
for _, u := range updates {
_, err = tx.Exec("UPDATE mailbox SET SearchText = ?, Snippet = ? WHERE ID = ?", u.SearchText, u.Snippet, u.ID)
if err != nil {
logger.Log().Error(err)
continue
}
}
if err := tx.Commit(); err != nil {
logger.Log().Error(err)
continue
}
finished += len(updates)
logger.Log().Printf("Reindexed: %d / %d (%d%%)", finished, total, finished*100/total)
}
}
// Reindex will regenerate the search text and snippet for a message
// and update the database.
func Reindex(id string) error {
// ids := []string{}
// var i string
// // chunkSize := 100
// err := sqlf.Select("ID").To(&i).From("mailbox_data").QueryAndClose(nil, db, func(row *sql.Rows) {
// ids = append(ids, id)
// })
// if err != nil {
// return err
// }
// chunks := chunkBy(ids, 100)
// fmt.Println(len(ids), " = ", len(chunks), "chunks")
// return nil
raw, err := GetMessageRaw(id)
if err != nil {
return err
}
r := bytes.NewReader(raw)
env, err := enmime.ReadEnvelope(r)
if err != nil {
return err
}
searchText := createSearchText(env)
snippet := tools.CreateSnippet(env.Text, env.HTML)
// return nil
// ctx := context.Background()
// tx, err := db.BeginTx(ctx, nil)
// if err != nil {
// return err
// }
// // roll back if it fails
// defer tx.Rollback()
// // insert mail summary data
// _, err = tx.Exec("UPDATE mailbox SET SearchText = ?, Snippet = ? WHERE ID = ?", searchText, snippet, id)
// if err != nil {
// return err
// }
// return tx.Commit()
_, err = sqlf.Update("mailbox").
Set("SearchText", searchText).
Set("Snippet", snippet).
Where("ID = ?", id).
ExecAndClose(context.Background(), db)
return err
}
// ctx := context.Background()
// tx, err := db.BeginTx(ctx, nil)
// if err != nil {
// return "", err
// }
func chunkBy[T any](items []T, chunkSize int) (chunks [][]T) {
for chunkSize < len(items) {
items, chunks = items[chunkSize:], append(chunks, items[0:chunkSize:chunkSize])
}
return append(chunks, items)
}

9
internal/storage/search.go
View File

@ -38,11 +38,12 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) {
var size int
var attachments int
var tags string
var snippet string
var read int
var ignore string
em := MessageSummary{}
if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil {
if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &snippet, &ignore, &ignore, &ignore, &ignore); err != nil {
logger.Log().Error(err)
return
}
@ -64,6 +65,7 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) {
em.Size = size
em.Attachments = attachments
em.Read = read == 1
em.Snippet = snippet
allResults = append(allResults, em)
}); err != nil {
@ -109,9 +111,10 @@ func DeleteSearch(search string) error {
var attachments int
var tags string
var read int
var snippet string
var ignore string
if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil {
if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &snippet, &ignore, &ignore, &ignore, &ignore); err != nil {
logger.Log().Error(err)
return
}
@ -193,7 +196,7 @@ func searchQueryBuilder(searchString string) *sqlf.Stmt {
args := tools.ArgsParser(searchString)
q := sqlf.From("mailbox").
Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags,
Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags, Snippet,
IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON,
IFNULL(json_extract(Metadata, '$.From'), '{}') as FromJSON,
IFNULL(json_extract(Metadata, '$.Cc'), '{}') as CcJSON,

10
internal/storage/structs.go
View File

@ -89,6 +89,8 @@ type MessageSummary struct {
Size int
// Whether the message has any attachments
Attachments int
// Message snippet includes up to 250 characters
Snippet string
}
// MailboxStats struct for quick mailbox total/read lookups
@ -98,6 +100,14 @@ type MailboxStats struct {
Tags []string
}
// DBMailSummary struct for storing mail summary
type DBMailSummary struct {
From *mail.Address
To []*mail.Address
Cc []*mail.Address
Bcc []*mail.Address
}
// AttachmentSummary returns a summary of the attachment without any binary data
func AttachmentSummary(a *enmime.Part) Attachment {
o := Attachment{}

5
internal/storage/testdata/mime-attachment.eml vendored
View File

@ -108,10 +108,9 @@ Content-Transfer-Encoding: 7bit
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
</head>
<body>
Message with inline image and attachment:<br>
<h1>Message with inline image and attachment:</h1>
<br>
<img src="cid:part1.845LaYlX.wtWMpWwa@gmail.com"
moz-do-not-send="false"><br>
<p><img src="cid:part1.845LaYlX.wtWMpWwa@gmail.com"></p>
<br>
<br>
</body>

0
internal/storage/test_shared.go → internal/storage/testing.go
View File

11
internal/tools/html.go
View File

@ -2,7 +2,9 @@ package tools
import (
"fmt"
"strings"
"github.com/microcosm-cc/bluemonday"
"golang.org/x/net/html"
)
@ -17,3 +19,12 @@ func GetHTMLAttributeVal(e *html.Node, key string) (string, error) {
return "", fmt.Errorf("%s not found", key)
}
// StripHTML returns text from an HTML string
func stripHTML(h string) string {
p := bluemonday.StrictPolicy()
// // ensure joining html elements are spaced apart, eg table cells etc
h = strings.ReplaceAll(h, "><", "> <")
// return p.Sanitize(h)
return html.UnescapeString(p.Sanitize(h))
}

42
internal/tools/snippets.go Normal file
View File

@ -0,0 +1,42 @@
package tools
import (
"regexp"
"strings"
)
// CreateSnippet returns a message snippet. It will use the HTML version (if it exists)
// and fall back to the text version.
func CreateSnippet(text, html string) string {
text = strings.TrimSpace(text)
html = strings.TrimSpace(html)
characters := 200
spaceRe := regexp.MustCompile(`\s+`)
nlRe := regexp.MustCompile(`\r?\n`)
if text == "" && html == "" {
return ""
}
if html != "" {
data := nlRe.ReplaceAllString(stripHTML(html), " ")
data = strings.TrimSpace(spaceRe.ReplaceAllString(data, " "))
if len(data) <= characters {
return data
}
return data[0:characters] + "..."
}
if text != "" {
text = spaceRe.ReplaceAllString(text, " ")
if len(text) <= characters {
return text
}
return text[0:characters] + "..."
}
return ""
}

26
internal/tools/tools_test.go
View File

@ -43,3 +43,29 @@ func TestCleanTag(t *testing.T) {
}
}
}
func TestSnippets(t *testing.T) {
tests := map[string]string{}
tests["this is a test"] = "this is a test"
tests["thiS IS a Test"] = "thiS IS a Test"
tests["thiS IS a Test :-)"] = "thiS IS a Test :-)"
tests["<h1>This is a test.</h1> "] = "This is a test."
tests["this_is-a test "] = "this_is-a test"
tests["this_is-a&^%%(*)@ test"] = "this_is-a&^%%(*)@ test"
tests["<h1>Heading</h1><p>Paragraph</p>"] = "Heading Paragraph"
tests[`<h1>Heading</h1>
<p>Paragraph</p>`] = "Heading Paragraph"
tests[`<h1>Heading</h1><p> <a href="https://github.com">linked text</a></p>`] = "Heading linked text"
// broken html
tests[`<h1>Heading</h3><p> <a href="https://github.com">linked text.`] = "Heading linked text."
// truncation to 200 chars + ...
tests["abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789"] = "abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmno..."
for str, expected := range tests {
res := CreateSnippet(str, str)
if res != expected {
t.Log("CreateSnippet error:", res, "!=", expected)
t.Fail()
}
}
}

22
server/ui-src/assets/styles.scss
View File

@ -94,11 +94,33 @@
}
.message {
.subject {
display: -webkit-box;
-webkit-box-orient: vertical;
-webkit-line-clamp: 1;
overflow: hidden;
color: $text-muted;
b {
color: $list-group-color;
}
small {
opacity: 0.5;
}
}
&.read {
color: $text-muted;
b {
opacity: 0.7;
font-weight: normal;
color: $list-group-color;
}
small {
opacity: 0.5;
}
}
&.selected {

5
server/ui-src/components/ListMessages.vue
View File

@ -141,7 +141,10 @@ export default {
</div>
</div>
<div class="col-lg-6 col-xxl-7 mt-2 mt-lg-0">
<div><b>{{ message.Subject != "" ? message.Subject : "[ no subject ]" }}</b></div>
<div class="subject">
<b>{{ message.Subject != "" ? message.Subject : "[ no subject ]" }}</b>
<small v-if="message.Snippet != ''" class="small">&nbsp; {{ message.Snippet }}</small>
</div>
<div>
<RouterLink class="badge me-1" v-for="t in message.Tags" :to="'/search?q=' + tagEncodeURI(t)"
:style="mailbox.showTagColors ? { backgroundColor: colorHash(t) } : { backgroundColor: '#6c757d' }"

2
server/ui-src/components/NavTags.vue
View File

@ -19,7 +19,7 @@ export default {
return false
}
let re = new RegExp(`\\b[^\-!]tag:"?${tag}"?\\b`, 'i')
let re = new RegExp(`(^|\\s)tag:"?${tag}"?($|\\s)`, 'i')
return query.match(re)
}
}

2
server/ui-src/components/message/Message.vue
View File

@ -443,7 +443,7 @@ export default {
aria-labelledby="nav-html-tab" tabindex="0">
<div id="responsive-view" :class="scaleHTMLPreview" :style="responsiveSizes[scaleHTMLPreview]">
<iframe target-blank="" class="tab-pane d-block" id="preview-html" :srcdoc="sanitizeHTML(message.HTML)"
v-on:load="resizeIframe" frameborder="0" style="width: 100%; height: 100%;">
v-on:load="resizeIframe" frameborder="0" style="width: 100%; height: 100%; background: #fff;">
</iframe>
</div>
<Attachments v-if="allAttachments(message).length" :message="message"