From ec3dd0c196ef89d6a0c0d14165091a7ea24072eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 21:22:09 +1300 Subject: [PATCH 01/12] Bump docker/login-action from 2 to 3 (#181) Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. --- .github/workflows/build-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index d340f7f..972f489 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -17,7 +17,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_ACCESS_TOKEN }} From 3b43a803af8276c95424109d5968971f4a6352a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 21:22:37 +1300 Subject: [PATCH 02/12] Bump docker/build-push-action from 4 to 5 (#180) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 4 to 5. --- .github/workflows/build-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 972f489..2999c42 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -30,7 +30,7 @@ jobs: version_extractor_regex: 'v(.*)$' - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . # platforms: linux/386,linux/amd64,linux/arm,linux/arm64 From 66660b9074dcd97cfa715e1413c7cab034891198 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 21:23:52 +1300 Subject: [PATCH 03/12] Bump docker/setup-buildx-action from 2 to 3 (#179) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2 to 3. --- .github/workflows/build-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 2999c42..5f680f3 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -14,7 +14,7 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub uses: docker/login-action@v3 From 0af6850d34116bd79742901e6911cd3414b7330b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 21:24:24 +1300 Subject: [PATCH 04/12] Bump actions/checkout from 3 to 4 (#178) Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. --- .github/workflows/build-docker.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/release-build.yml | 2 +- .github/workflows/tests.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 5f680f3..321da74 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 25b283e..0952583 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index 728d52d..949bfcb 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -21,7 +21,7 @@ jobs: - goarch: arm goos: windows steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # build the assets - uses: actions/setup-node@v3 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6fbfbe0..e26ce52 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/setup-go@v4 with: go-version: ${{ matrix.go-version }} - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/cache@v3 with: path: | From 3fb926f015e428e8978fe1501caf60090aa8fe81 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 21:27:39 +1300 Subject: [PATCH 05/12] Bump docker/setup-qemu-action from 2 to 3 (#177) Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 2 to 3. --- .github/workflows/build-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 321da74..7016761 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -11,7 +11,7 @@ jobs: uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 636918dd0ec75378fa67a3adadfd57acf0f30de2 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Thu, 5 Oct 2023 17:01:13 +1300 Subject: [PATCH 06/12] Feature: Display email previews (#175) --- go.mod | 2 + go.sum | 4 + internal/storage/database.go | 96 ++----------------- internal/storage/migrations.go | 84 ++++++++++++++++ internal/storage/search.go | 9 +- internal/storage/structs.go | 10 ++ .../storage/{test_shared.go => testing.go} | 0 internal/tools/html.go | 11 +++ internal/tools/snippets.go | 42 ++++++++ server/ui-src/assets/styles.scss | 22 +++++ server/ui-src/components/ListMessages.vue | 5 +- 11 files changed, 194 insertions(+), 91 deletions(-) create mode 100644 internal/storage/migrations.go rename internal/storage/{test_shared.go => testing.go} (100%) create mode 100644 internal/tools/snippets.go diff --git a/go.mod b/go.mod index f38cd5e..59b13dd 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/klauspost/compress v1.17.0 github.com/leporo/sqlf v1.4.0 github.com/mhale/smtpd v0.8.0 + github.com/microcosm-cc/bluemonday v1.0.25 github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.9.3 @@ -32,6 +33,7 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.0 // indirect github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect github.com/cznic/ql v1.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/go.sum b/go.sum index 16b2c2b..df39836 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/axllent/semver v0.0.1 h1:QqF+KSGxgj8QZzSXAvKFqjGWE5792ksOnQhludToK8E= github.com/axllent/semver v0.0.1/go.mod h1:2xSPzvG8n9mRfdtxSvWvfTfQGWfHsMsHO1iZnKATMSc= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI= github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -97,6 +99,8 @@ github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwp github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0= github.com/mhale/smtpd v0.8.0/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4= +github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= +github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/internal/storage/database.go b/internal/storage/database.go index f7cbbee..c6d55ed 100644 --- a/internal/storage/database.go +++ b/internal/storage/database.go @@ -18,9 +18,9 @@ import ( "syscall" "time" - "github.com/GuiaBolso/darwin" "github.com/axllent/mailpit/config" "github.com/axllent/mailpit/internal/logger" + "github.com/axllent/mailpit/internal/tools" "github.com/axllent/mailpit/server/websockets" "github.com/jhillyerd/enmime" "github.com/klauspost/compress/zstd" @@ -42,81 +42,8 @@ var ( // zstd compression encoder & decoder dbEncoder, _ = zstd.NewWriter(nil) dbDecoder, _ = zstd.NewReader(nil) - - dbMigrations = []darwin.Migration{ - { - Version: 1.0, - Description: "Creating tables", - Script: `CREATE TABLE IF NOT EXISTS mailbox ( - Sort INTEGER PRIMARY KEY AUTOINCREMENT, - ID TEXT NOT NULL, - Data BLOB, - Search TEXT, - Read INTEGER - ); - CREATE INDEX IF NOT EXISTS idx_sort ON mailbox (Sort); - CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID); - CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read); - - CREATE TABLE IF NOT EXISTS mailbox_data ( - ID TEXT KEY NOT NULL, - Email BLOB - ); - CREATE UNIQUE INDEX IF NOT EXISTS idx_data_id ON mailbox_data (ID);`, - }, - { - Version: 1.1, - Description: "Create tags column", - Script: `ALTER TABLE mailbox ADD COLUMN Tags Text NOT NULL DEFAULT '[]'; - CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`, - }, - { - Version: 1.2, - Description: "Creating new mailbox format", - Script: `CREATE TABLE IF NOT EXISTS mailboxtmp ( - Created INTEGER NOT NULL, - ID TEXT NOT NULL, - MessageID TEXT NOT NULL, - Subject TEXT NOT NULL, - Metadata TEXT, - Size INTEGER NOT NULL, - Inline INTEGER NOT NULL, - Attachments INTEGER NOT NULL, - Read INTEGER, - Tags TEXT, - SearchText TEXT - ); - INSERT INTO mailboxtmp - (Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags) - SELECT - Sort, ID, '', json_extract(Data, '$.Subject'),Data, - json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'), - Search, Read, Tags - FROM mailbox; - - DROP TABLE IF EXISTS mailbox; - ALTER TABLE mailboxtmp RENAME TO mailbox; - CREATE INDEX IF NOT EXISTS idx_created ON mailbox (Created); - CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID); - CREATE INDEX IF NOT EXISTS idx_message_id ON mailbox (MessageID); - CREATE INDEX IF NOT EXISTS idx_subject ON mailbox (Subject); - CREATE INDEX IF NOT EXISTS idx_size ON mailbox (Size); - CREATE INDEX IF NOT EXISTS idx_inline ON mailbox (Inline); - CREATE INDEX IF NOT EXISTS idx_attachments ON mailbox (Attachments); - CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read); - CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`, - }, - } ) -// DBMailSummary struct for storing mail summary -type DBMailSummary struct { - From *mail.Address - To []*mail.Address - Cc []*mail.Address - Bcc []*mail.Address -} - // InitDB will initialise the database func InitDB() error { p := config.DataFile @@ -178,15 +105,6 @@ func InitDB() error { return nil } -// Create tables and apply migrations if required -func dbApplyMigrations() error { - driver := darwin.NewGenericDriver(db, darwin.SqliteDialect{}) - - d := darwin.New(driver, dbMigrations, nil) - - return d.Migrate() -} - // Close will close the database, and delete if a temporary table func Close() { if db != nil { @@ -281,10 +199,11 @@ func Store(body []byte) (string, error) { size := len(body) inline := len(env.Inlines) attachments := len(env.Attachments) + snippet := tools.CreateSnippet(env.Text, env.HTML) // insert mail summary data - _, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Tags, Read) values(?,?,?,?,?,?,?,?,?,?,0)", - created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, string(tagJSON)) + _, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Tags, Read, Snippet) values(?,?,?,?,?,?,?,?,?,?,0, ?)", + created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, string(tagJSON), snippet) if err != nil { return "", err } @@ -312,6 +231,7 @@ func Store(body []byte) (string, error) { c.Subject = subject c.Size = size c.Tags = tagData + c.Snippet = snippet websockets.Broadcast("new", c) @@ -328,7 +248,7 @@ func List(start, limit int) ([]MessageSummary, error) { results := []MessageSummary{} q := sqlf.From("mailbox"). - Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags`). + Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags, Snippet`). OrderBy("Created DESC"). Limit(limit). Offset(start) @@ -343,9 +263,10 @@ func List(start, limit int) ([]MessageSummary, error) { var attachments int var tags string var read int + var snippet string em := MessageSummary{} - if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags); err != nil { + if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &snippet); err != nil { logger.Log().Error(err) return } @@ -367,6 +288,7 @@ func List(start, limit int) ([]MessageSummary, error) { em.Size = size em.Attachments = attachments em.Read = read == 1 + em.Snippet = snippet results = append(results, em) }); err != nil { diff --git a/internal/storage/migrations.go b/internal/storage/migrations.go new file mode 100644 index 0000000..ac5937b --- /dev/null +++ b/internal/storage/migrations.go @@ -0,0 +1,84 @@ +package storage + +import "github.com/GuiaBolso/darwin" + +var ( + dbMigrations = []darwin.Migration{ + { + Version: 1.0, + Description: "Creating tables", + Script: `CREATE TABLE IF NOT EXISTS mailbox ( + Sort INTEGER PRIMARY KEY AUTOINCREMENT, + ID TEXT NOT NULL, + Data BLOB, + Search TEXT, + Read INTEGER + ); + CREATE INDEX IF NOT EXISTS idx_sort ON mailbox (Sort); + CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID); + CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read); + + CREATE TABLE IF NOT EXISTS mailbox_data ( + ID TEXT KEY NOT NULL, + Email BLOB + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_data_id ON mailbox_data (ID);`, + }, + { + Version: 1.1, + Description: "Create tags column", + Script: `ALTER TABLE mailbox ADD COLUMN Tags Text NOT NULL DEFAULT '[]'; + CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`, + }, + { + Version: 1.2, + Description: "Creating new mailbox format", + Script: `CREATE TABLE IF NOT EXISTS mailboxtmp ( + Created INTEGER NOT NULL, + ID TEXT NOT NULL, + MessageID TEXT NOT NULL, + Subject TEXT NOT NULL, + Metadata TEXT, + Size INTEGER NOT NULL, + Inline INTEGER NOT NULL, + Attachments INTEGER NOT NULL, + Read INTEGER, + Tags TEXT, + SearchText TEXT + ); + INSERT INTO mailboxtmp + (Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags) + SELECT + Sort, ID, '', json_extract(Data, '$.Subject'),Data, + json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'), + Search, Read, Tags + FROM mailbox; + + DROP TABLE IF EXISTS mailbox; + ALTER TABLE mailboxtmp RENAME TO mailbox; + CREATE INDEX IF NOT EXISTS idx_created ON mailbox (Created); + CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID); + CREATE INDEX IF NOT EXISTS idx_message_id ON mailbox (MessageID); + CREATE INDEX IF NOT EXISTS idx_subject ON mailbox (Subject); + CREATE INDEX IF NOT EXISTS idx_size ON mailbox (Size); + CREATE INDEX IF NOT EXISTS idx_inline ON mailbox (Inline); + CREATE INDEX IF NOT EXISTS idx_attachments ON mailbox (Attachments); + CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read); + CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`, + }, + { + Version: 1.3, + Description: "Create snippet column", + Script: `ALTER TABLE mailbox ADD COLUMN Snippet Text NOT NULL DEFAULT '';`, + }, + } +) + +// Create tables and apply migrations if required +func dbApplyMigrations() error { + driver := darwin.NewGenericDriver(db, darwin.SqliteDialect{}) + + d := darwin.New(driver, dbMigrations, nil) + + return d.Migrate() +} diff --git a/internal/storage/search.go b/internal/storage/search.go index 9bd844d..abb8a6a 100644 --- a/internal/storage/search.go +++ b/internal/storage/search.go @@ -38,11 +38,12 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) { var size int var attachments int var tags string + var snippet string var read int var ignore string em := MessageSummary{} - if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil { + if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &snippet, &ignore, &ignore, &ignore, &ignore); err != nil { logger.Log().Error(err) return } @@ -64,6 +65,7 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) { em.Size = size em.Attachments = attachments em.Read = read == 1 + em.Snippet = snippet allResults = append(allResults, em) }); err != nil { @@ -109,9 +111,10 @@ func DeleteSearch(search string) error { var attachments int var tags string var read int + var snippet string var ignore string - if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil { + if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &snippet, &ignore, &ignore, &ignore, &ignore); err != nil { logger.Log().Error(err) return } @@ -193,7 +196,7 @@ func searchQueryBuilder(searchString string) *sqlf.Stmt { args := tools.ArgsParser(searchString) q := sqlf.From("mailbox"). - Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags, + Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags, Snippet, IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON, IFNULL(json_extract(Metadata, '$.From'), '{}') as FromJSON, IFNULL(json_extract(Metadata, '$.Cc'), '{}') as CcJSON, diff --git a/internal/storage/structs.go b/internal/storage/structs.go index e3e05df..2ad9721 100644 --- a/internal/storage/structs.go +++ b/internal/storage/structs.go @@ -89,6 +89,8 @@ type MessageSummary struct { Size int // Whether the message has any attachments Attachments int + // Message snippet includes up to 250 characters + Snippet string } // MailboxStats struct for quick mailbox total/read lookups @@ -98,6 +100,14 @@ type MailboxStats struct { Tags []string } +// DBMailSummary struct for storing mail summary +type DBMailSummary struct { + From *mail.Address + To []*mail.Address + Cc []*mail.Address + Bcc []*mail.Address +} + // AttachmentSummary returns a summary of the attachment without any binary data func AttachmentSummary(a *enmime.Part) Attachment { o := Attachment{} diff --git a/internal/storage/test_shared.go b/internal/storage/testing.go similarity index 100% rename from internal/storage/test_shared.go rename to internal/storage/testing.go diff --git a/internal/tools/html.go b/internal/tools/html.go index 6e5e883..06de636 100644 --- a/internal/tools/html.go +++ b/internal/tools/html.go @@ -2,7 +2,9 @@ package tools import ( "fmt" + "strings" + "github.com/microcosm-cc/bluemonday" "golang.org/x/net/html" ) @@ -17,3 +19,12 @@ func GetHTMLAttributeVal(e *html.Node, key string) (string, error) { return "", fmt.Errorf("%s not found", key) } + +// StripHTML returns text from an HTML string +func stripHTML(h string) string { + p := bluemonday.StrictPolicy() + // // ensure joining html elements are spaced apart, eg table cells etc + h = strings.ReplaceAll(h, "><", "> <") + // return p.Sanitize(h) + return html.UnescapeString(p.Sanitize(h)) +} diff --git a/internal/tools/snippets.go b/internal/tools/snippets.go new file mode 100644 index 0000000..6dee2ad --- /dev/null +++ b/internal/tools/snippets.go @@ -0,0 +1,42 @@ +package tools + +import ( + "regexp" + "strings" +) + +// CreateSnippet returns a message snippet. It will use the HTML version (if it exists) +// and fall back to the text version. +func CreateSnippet(text, html string) string { + text = strings.TrimSpace(text) + html = strings.TrimSpace(html) + characters := 200 + spaceRe := regexp.MustCompile(`\s+`) + nlRe := regexp.MustCompile(`\r?\n`) + + if text == "" && html == "" { + return "" + } + + if html != "" { + data := nlRe.ReplaceAllString(stripHTML(html), " ") + data = strings.TrimSpace(spaceRe.ReplaceAllString(data, " ")) + + if len(data) <= characters { + return data + } + + return data[0:characters] + "..." + } + + if text != "" { + text = spaceRe.ReplaceAllString(text, " ") + if len(text) <= characters { + return text + } + + return text[0:characters] + "..." + } + + return "" +} diff --git a/server/ui-src/assets/styles.scss b/server/ui-src/assets/styles.scss index cadff15..bbfadea 100644 --- a/server/ui-src/assets/styles.scss +++ b/server/ui-src/assets/styles.scss @@ -94,11 +94,33 @@ } .message { + .subject { + display: -webkit-box; + -webkit-box-orient: vertical; + -webkit-line-clamp: 1; + overflow: hidden; + color: $text-muted; + + b { + color: $list-group-color; + } + + small { + opacity: 0.5; + } + } + &.read { color: $text-muted; b { + opacity: 0.7; font-weight: normal; + color: $list-group-color; + } + + small { + opacity: 0.5; } } &.selected { diff --git a/server/ui-src/components/ListMessages.vue b/server/ui-src/components/ListMessages.vue index 1d8c8e1..2028f18 100644 --- a/server/ui-src/components/ListMessages.vue +++ b/server/ui-src/components/ListMessages.vue @@ -141,7 +141,10 @@ export default {
-
{{ message.Subject != "" ? message.Subject : "[ no subject ]" }}
+
+ {{ message.Subject != "" ? message.Subject : "[ no subject ]" }} +   {{ message.Snippet }} +
Date: Thu, 5 Oct 2023 17:02:12 +1300 Subject: [PATCH 07/12] Tests: Add snippet tests --- internal/storage/testdata/mime-attachment.eml | 5 ++-- internal/tools/tools_test.go | 26 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/internal/storage/testdata/mime-attachment.eml b/internal/storage/testdata/mime-attachment.eml index add00c7..f998414 100644 --- a/internal/storage/testdata/mime-attachment.eml +++ b/internal/storage/testdata/mime-attachment.eml @@ -108,10 +108,9 @@ Content-Transfer-Encoding: 7bit - Message with inline image and attachment:
+

Message with inline image and attachment:


-
+



diff --git a/internal/tools/tools_test.go b/internal/tools/tools_test.go index edfb947..ad2321d 100644 --- a/internal/tools/tools_test.go +++ b/internal/tools/tools_test.go @@ -43,3 +43,29 @@ func TestCleanTag(t *testing.T) { } } } + +func TestSnippets(t *testing.T) { + tests := map[string]string{} + tests["this is a test"] = "this is a test" + tests["thiS IS a Test"] = "thiS IS a Test" + tests["thiS IS a Test :-)"] = "thiS IS a Test :-)" + tests["

This is a test.

"] = "This is a test." + tests["this_is-a test "] = "this_is-a test" + tests["this_is-a&^%%(*)@ test"] = "this_is-a&^%%(*)@ test" + tests["

Heading

Paragraph

"] = "Heading Paragraph" + tests[`

Heading

+

Paragraph

`] = "Heading Paragraph" + tests[`

Heading

linked text

`] = "Heading linked text" + // broken html + tests[`

Heading

linked text.`] = "Heading linked text." + // truncation to 200 chars + ... + tests["abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789"] = "abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmnopqrstuvwxyx0123456789 abcdefghijklmno..." + + for str, expected := range tests { + res := CreateSnippet(str, str) + if res != expected { + t.Log("CreateSnippet error:", res, "!=", expected) + t.Fail() + } + } +} From a3f83ea5ceb8087295a5c26bc472663c3503070a Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Thu, 5 Oct 2023 17:02:35 +1300 Subject: [PATCH 08/12] Tests: Add message summary tests --- internal/storage/database_test.go | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/internal/storage/database_test.go b/internal/storage/database_test.go index 8d284df..c6fc33e 100644 --- a/internal/storage/database_test.go +++ b/internal/storage/database_test.go @@ -117,6 +117,38 @@ func TestRetrieveMimeEmail(t *testing.T) { assertEqual(t, len(inlineData.Content), msg.Inline[0].Size, "inline attachment size does not match") } +func TestMessageSummary(t *testing.T) { + setup() + defer Close() + + t.Log("Testing message summary") + + if _, err := Store(testMimeEmail); err != nil { + t.Log("error ", err) + t.Fail() + } + + summaries, err := List(0, 1) + if err != nil { + t.Log("error ", err) + t.Fail() + } + + assertEqual(t, len(summaries), 1, "Expected 1 result") + + msg := summaries[0] + + assertEqual(t, msg.From.Name, "Sender Smith", "\"From\" name does not match") + assertEqual(t, msg.From.Address, "sender2@example.com", "\"From\" address does not match") + assertEqual(t, msg.Subject, "inline + attachment", "subject does not match") + assertEqual(t, len(msg.To), 1, "incorrect number of recipients") + assertEqual(t, msg.To[0].Name, "Recipient Ross", "\"To\" name does not match") + assertEqual(t, msg.To[0].Address, "recipient2@example.com", "\"To\" address does not match") + assertEqual(t, msg.Snippet, "Message with inline image and attachment:", "\"Snippet\" does does not match") + assertEqual(t, msg.Attachments, 1, "Expected 1 attachment") + assertEqual(t, msg.MessageID, "33af2ac1-c33d-9738-35e3-a6daf90bbd89@gmail.com", "\"MessageID\" does not match") +} + func BenchmarkImportText(b *testing.B) { setup() defer Close() From 2b18b1bee16b1c3a7afd1623728abb734fb4de7b Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Thu, 5 Oct 2023 17:04:05 +1300 Subject: [PATCH 09/12] Feature: Add `reindex` subcommand to reindex all messages --- cmd/reindex.go | 40 ++++++++ internal/storage/reindex.go | 184 ++++++++++++++++++++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 cmd/reindex.go create mode 100644 internal/storage/reindex.go diff --git a/cmd/reindex.go b/cmd/reindex.go new file mode 100644 index 0000000..97ad990 --- /dev/null +++ b/cmd/reindex.go @@ -0,0 +1,40 @@ +/* +Copyright © 2022-Now() Ralph Slooten +This file is part of a CLI application. +*/ +package cmd + +import ( + "os" + + "github.com/axllent/mailpit/config" + "github.com/axllent/mailpit/internal/logger" + "github.com/axllent/mailpit/internal/storage" + "github.com/spf13/cobra" +) + +// reindexCmd represents the reindex command +var reindexCmd = &cobra.Command{ + Use: "reindex ", + Short: "Reindex the database", + Long: `This will reindex all messages in the entire database. + +If you have several thousand messages in your mailbox, then it is advised to shut down +Mailpit while you reindex as this process will likely result in database locking issues.`, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + config.DataFile = args[0] + config.MaxMessages = 0 + + if err := storage.InitDB(); err != nil { + logger.Log().Error(err) + os.Exit(1) + } + + storage.ReindexAll() + }, +} + +func init() { + rootCmd.AddCommand(reindexCmd) +} diff --git a/internal/storage/reindex.go b/internal/storage/reindex.go new file mode 100644 index 0000000..7b14324 --- /dev/null +++ b/internal/storage/reindex.go @@ -0,0 +1,184 @@ +package storage + +import ( + "bytes" + "context" + "database/sql" + "os" + + "github.com/axllent/mailpit/internal/logger" + "github.com/axllent/mailpit/internal/tools" + "github.com/jhillyerd/enmime" + "github.com/leporo/sqlf" +) + +// ReindexAll will regenerate the search text and snippet for a message +// and update the database. +func ReindexAll() { + ids := []string{} + var i string + chunkSize := 1000 + + finished := 0 + + err := sqlf.Select("ID").To(&i). + From("mailbox"). + OrderBy("Created DESC"). + QueryAndClose(nil, db, func(row *sql.Rows) { + ids = append(ids, i) + }) + + if err != nil { + logger.Log().Error(err) + os.Exit(1) + } + + total := len(ids) + + chunks := chunkBy(ids, chunkSize) + + logger.Log().Infof("Reindexing %d messages", total) + + // fmt.Println(len(ids), " = ", len(chunks), "chunks") + + type updateStruct struct { + ID string + SearchText string + Snippet string + } + + for _, ids := range chunks { + updates := []updateStruct{} + + for _, id := range ids { + raw, err := GetMessageRaw(id) + if err != nil { + logger.Log().Error(err) + continue + } + + r := bytes.NewReader(raw) + + env, err := enmime.ReadEnvelope(r) + if err != nil { + logger.Log().Error(err) + continue + } + + searchText := createSearchText(env) + snippet := tools.CreateSnippet(env.Text, env.HTML) + + u := updateStruct{} + u.ID = id + u.SearchText = searchText + u.Snippet = snippet + + updates = append(updates, u) + } + + ctx := context.Background() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + logger.Log().Error(err) + continue + } + + // roll back if it fails + defer tx.Rollback() + + // insert mail summary data + for _, u := range updates { + _, err = tx.Exec("UPDATE mailbox SET SearchText = ?, Snippet = ? WHERE ID = ?", u.SearchText, u.Snippet, u.ID) + if err != nil { + logger.Log().Error(err) + continue + } + } + + if err := tx.Commit(); err != nil { + logger.Log().Error(err) + continue + } + + finished += len(updates) + + logger.Log().Printf("Reindexed: %d / %d (%d%%)", finished, total, finished*100/total) + } +} + +// Reindex will regenerate the search text and snippet for a message +// and update the database. +func Reindex(id string) error { + // ids := []string{} + // var i string + // // chunkSize := 100 + + // err := sqlf.Select("ID").To(&i).From("mailbox_data").QueryAndClose(nil, db, func(row *sql.Rows) { + // ids = append(ids, id) + // }) + + // if err != nil { + // return err + // } + + // chunks := chunkBy(ids, 100) + + // fmt.Println(len(ids), " = ", len(chunks), "chunks") + + // return nil + + raw, err := GetMessageRaw(id) + if err != nil { + return err + } + + r := bytes.NewReader(raw) + + env, err := enmime.ReadEnvelope(r) + if err != nil { + return err + } + + searchText := createSearchText(env) + snippet := tools.CreateSnippet(env.Text, env.HTML) + + // return nil + + // ctx := context.Background() + // tx, err := db.BeginTx(ctx, nil) + // if err != nil { + // return err + // } + + // // roll back if it fails + // defer tx.Rollback() + + // // insert mail summary data + // _, err = tx.Exec("UPDATE mailbox SET SearchText = ?, Snippet = ? WHERE ID = ?", searchText, snippet, id) + // if err != nil { + // return err + // } + + // return tx.Commit() + + _, err = sqlf.Update("mailbox"). + Set("SearchText", searchText). + Set("Snippet", snippet). + Where("ID = ?", id). + ExecAndClose(context.Background(), db) + + return err +} + +// ctx := context.Background() +// tx, err := db.BeginTx(ctx, nil) +// if err != nil { +// return "", err +// } + +func chunkBy[T any](items []T, chunkSize int) (chunks [][]T) { + for chunkSize < len(items) { + items, chunks = items[chunkSize:], append(chunks, items[0:chunkSize:chunkSize]) + } + return append(chunks, items) +} From 7446f52205bf53081be91e0f1582ee806d330802 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Thu, 5 Oct 2023 17:23:22 +1300 Subject: [PATCH 10/12] Fix: Correctly detect tags in search (UI) --- server/ui-src/components/NavTags.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/ui-src/components/NavTags.vue b/server/ui-src/components/NavTags.vue index 05d3de8..5632d72 100644 --- a/server/ui-src/components/NavTags.vue +++ b/server/ui-src/components/NavTags.vue @@ -19,7 +19,7 @@ export default { return false } - let re = new RegExp(`\\b[^\-!]tag:"?${tag}"?\\b`, 'i') + let re = new RegExp(`(^|\\s)tag:"?${tag}"?($|\\s)`, 'i') return query.match(re) } } From 4e2d4d6365d3ac66074329e6bc95fb0eca8f9aa5 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Thu, 5 Oct 2023 17:38:26 +1300 Subject: [PATCH 11/12] Fix: HTML message preview background color when switching themes in Chrome Fixes #182 --- server/ui-src/components/message/Message.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/ui-src/components/message/Message.vue b/server/ui-src/components/message/Message.vue index cc08b7e..57616f4 100644 --- a/server/ui-src/components/message/Message.vue +++ b/server/ui-src/components/message/Message.vue @@ -443,7 +443,7 @@ export default { aria-labelledby="nav-html-tab" tabindex="0">

Date: Thu, 5 Oct 2023 17:39:19 +1300 Subject: [PATCH 12/12] Release v1.9.5 --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 80ff5fa..9698471 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ Notable changes to Mailpit will be documented in this file. +## [v1.9.5] + +### Feature +- Add `reindex` subcommand to reindex all messages +- Display email previews ([#175](https://github.com/axllent/mailpit/issues/175)) + +### Fix +- HTML message preview background color when switching themes in Chrome +- Correctly detect tags in search (UI) + +### Tests +- Add message summary tests +- Add snippet tests + + ## [v1.9.4] ### Chore