From 4b707537b9334641c37fb1d1aab96fe708e507ec Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 30 Jul 2022 22:01:30 +1200 Subject: [PATCH 1/9] Bugfix: Update to clover-v2.0.0-alpha.2 to fix sorting --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 8d6e91e..3f666fe 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/jhillyerd/enmime v0.10.0 github.com/k3a/html2text v1.0.8 github.com/mhale/smtpd v0.8.0 - github.com/ostafen/clover/v2 v2.0.0-alpha.1 + github.com/ostafen/clover/v2 v2.0.0-alpha.2 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 github.com/spf13/pflag v1.0.5 @@ -38,7 +38,7 @@ require ( github.com/mattn/go-runewidth v0.0.13 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/rivo/uniseg v0.3.0 // indirect + github.com/rivo/uniseg v0.3.1 // indirect github.com/satori/go.uuid v1.2.0 // indirect github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect github.com/stretchr/testify v1.7.2 // indirect diff --git a/go.sum b/go.sum index 5cc9e8b..a9d33be 100644 --- a/go.sum +++ b/go.sum @@ -128,8 +128,8 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/ostafen/clover/v2 v2.0.0-alpha.1 h1:Oa+N1xXvEXirUUmwssN+ASd/Xf9DikITT8Id+zc0l+o= -github.com/ostafen/clover/v2 v2.0.0-alpha.1/go.mod h1:7UyIG46NglzTDRKB4LJiS/enXpuo67Lj05eM8mdhL6M= +github.com/ostafen/clover/v2 v2.0.0-alpha.2 h1:PgOWohvpj4qNCyASJ7Q8Ke8ld/wsoi+dQJ05b1ebwus= +github.com/ostafen/clover/v2 v2.0.0-alpha.2/go.mod h1:7UyIG46NglzTDRKB4LJiS/enXpuo67Lj05eM8mdhL6M= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -138,8 +138,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.3.0 h1:eyC18g7xB83Dv/xlJXLgNkRidVoR7nqFZBJvqo/K188= -github.com/rivo/uniseg v0.3.0/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.3.1 h1:SDPP7SHNl1L7KrEFCSJslJ/DM9DT02Nq2C61XrfHMmk= +github.com/rivo/uniseg v0.3.1/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= From ad1037c02b654b7e582c2c644841ea7477fcf9b4 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 30 Jul 2022 22:02:42 +1200 Subject: [PATCH 2/9] 0.0.3 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a07dd5..dc8a8ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [0.0.3] + +- Bugfix: Update to clover-v2.0.0-alpha.2 to fix sorting + + ## [0.0.2] - Unread message statistics & updates From 154b2342056d3adb1be716e16f8919319d3c6dba Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 30 Jul 2022 22:33:20 +1200 Subject: [PATCH 3/9] UI: Minor UI & logging changes --- server/ui-src/App.vue | 3 +-- storage/database.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/server/ui-src/App.vue b/server/ui-src/App.vue index 7de51a1..139076e 100644 --- a/server/ui-src/App.vue +++ b/server/ui-src/App.vue @@ -408,7 +408,7 @@ export default {
- This will permanently delete all messages. + This will permanently delete {{ formatNumber(total) }} messages.
@@ -418,5 +418,4 @@ export default {
- diff --git a/storage/database.go b/storage/database.go index 78845b2..d2ab5df 100644 --- a/storage/database.go +++ b/storage/database.go @@ -224,7 +224,7 @@ func Store(mailbox string, b []byte) (string, error) { count++ if count%100 == 0 { - logger.Log().Infof("%d messages added (%s per 100)", count, time.Since(per100start)) + logger.Log().Infof("100 messages added in %s", time.Since(per100start)) per100start = time.Now() } From 41c7c2a93af86acb48e5dd8b3d9406820c547b13 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 30 Jul 2022 23:00:34 +1200 Subject: [PATCH 4/9] UI: Cater for messages without From email address --- server/ui-src/templates/Message.vue | 3 ++- storage/database.go | 6 +++++- storage/utils.go | 17 ++++++++++------- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/server/ui-src/templates/Message.vue b/server/ui-src/templates/Message.vue index bee7fd1..6e7ef3a 100644 --- a/server/ui-src/templates/Message.vue +++ b/server/ui-src/templates/Message.vue @@ -80,7 +80,8 @@ export default { From - {{ message.From.Name + " <" + message.From.Address +">" }} + {{ message.From.Name }} + <{{ message.From.Address }}> [ Unknown ] diff --git a/storage/database.go b/storage/database.go index d2ab5df..edc299a 100644 --- a/storage/database.go +++ b/storage/database.go @@ -185,6 +185,8 @@ func Store(mailbox string, b []byte) (string, error) { fromData := addressToSlice(env, "From") if len(fromData) > 0 { from = fromData[0] + } else if env.GetHeader("From") != "" { + from = &mail.Address{Name: env.GetHeader("From")} } obj := CloverStore{ @@ -311,7 +313,7 @@ func List(mailbox string, start, limit int) ([]data.Summary, error) { // Search returns a summary of items mathing a search. It searched the SearchText field. func Search(mailbox, search string, start, limit int) ([]data.Summary, error) { - sq := fmt.Sprintf("(?i)%s", regexp.QuoteMeta(search)) + sq := fmt.Sprintf("(?i)%s", cleanString(regexp.QuoteMeta(search))) q, err := db.FindAll(clover.NewQuery(mailbox). Skip(start). Limit(limit). @@ -393,6 +395,8 @@ func GetMessage(mailbox, id string) (*data.Message, error) { fromData := addressToSlice(env, "From") if len(fromData) > 0 { from = fromData[0] + } else if env.GetHeader("From") != "" { + from = &mail.Address{Name: env.GetHeader("From")} } date, err := env.Date() diff --git a/storage/utils.go b/storage/utils.go index 8cfca72..f2bbb18 100644 --- a/storage/utils.go +++ b/storage/utils.go @@ -42,17 +42,20 @@ func createSearchText(env *enmime.Envelope) string { b.WriteString(a.FileName + " ") } - d := b.String() - - // remove/replace new lines - re := regexp.MustCompile(`(\r?\n|\t|>|<|"|:|\,|;)`) - d = re.ReplaceAllString(d, " ") - // remove duplicate whitespace and trim - d = strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(d)), " ")) + d := cleanString(b.String()) return d } +// cleanString removed unwanted characters from stored search text and search queries +func cleanString(str string) string { + // remove/replace new lines + re := regexp.MustCompile(`(\r?\n|\t|>|<|"|:|\,|;)`) + str = re.ReplaceAllString(str, " ") + // remove duplicate whitespace and trim + return strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(str)), " ")) +} + // Auto-prune runs every 5 minutes to automatically delete oldest messages // if total is greater than the threshold func pruneCron() { From 2944c2a32f6e1e3cbb7eb9603188b339f6d74700 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 30 Jul 2022 23:48:57 +1200 Subject: [PATCH 5/9] Tests: Add search tests --- storage/database_test.go | 75 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/storage/database_test.go b/storage/database_test.go index 9d473bc..c2152ff 100644 --- a/storage/database_test.go +++ b/storage/database_test.go @@ -1,12 +1,15 @@ package storage import ( + "bytes" "fmt" "io/ioutil" + "math/rand" "testing" "time" "github.com/axllent/mailpit/config" + "github.com/jhillyerd/enmime" ) var ( @@ -123,6 +126,78 @@ func TestRetrieveMimeEmail(t *testing.T) { assertEqual(t, len(attachmentData.Content), msg.Attachments[0].Size, "attachment size does not match") inlineData, err := GetAttachmentPart(DefaultMailbox, id, msg.Inline[0].PartID) assertEqual(t, len(inlineData.Content), msg.Inline[0].Size, "inline attachment size does not match") + + db.Close() +} + +func TestSearch(t *testing.T) { + setup() + + for i := 0; i < 1000; i++ { + msg := enmime.Builder(). + From(fmt.Sprintf("From %d", i), fmt.Sprintf("from-%d@example.com", i)). + Subject(fmt.Sprintf("Subject line %d end", i)). + Text([]byte(fmt.Sprintf("This is the email body %d .", i))). + To(fmt.Sprintf("To %d", i), fmt.Sprintf("to-%d@example.com", i)) + + env, err := msg.Build() + if err != nil { + t.Log("error ", err) + t.Fail() + } + + buf := new(bytes.Buffer) + + if err := env.Encode(buf); err != nil { + t.Log("error ", err) + t.Fail() + } + + if _, err := Store(DefaultMailbox, buf.Bytes()); err != nil { + t.Log("error ", err) + t.Fail() + } + } + + for i := 1; i < 101; i++ { + // search a random something that will return a single result + searchIndx := rand.Intn(4) + 1 + var search string + switch searchIndx { + case 1: + search = fmt.Sprintf("from-%d@example.com", i) + case 2: + search = fmt.Sprintf("to-%d@example.com", i) + case 3: + search = fmt.Sprintf("Subject line %d end", i) + default: + search = fmt.Sprintf("the email body %d jdsauk dwqmdqw", i) + } + + summaries, err := Search(DefaultMailbox, search, 0, 200) + if err != nil { + t.Log("error ", err) + t.Fail() + } + + assertEqual(t, len(summaries), 1, "1 search result expected") + + assertEqual(t, summaries[0].From.Name, fmt.Sprintf("From %d", i), "\"From\" name does not match") + assertEqual(t, summaries[0].From.Address, fmt.Sprintf("from-%d@example.com", i), "\"From\" address does not match") + assertEqual(t, summaries[0].To[0].Name, fmt.Sprintf("To %d", i), "\"To\" name does not match") + assertEqual(t, summaries[0].To[0].Address, fmt.Sprintf("to-%d@example.com", i), "\"To\" address does not match") + assertEqual(t, summaries[0].Subject, fmt.Sprintf("Subject line %d end", i), "\"Subject\" does not match") + } + + // search something that will return 200 rsults + summaries, err := Search(DefaultMailbox, "This is the email body", 0, 200) + if err != nil { + t.Log("error ", err) + t.Fail() + } + assertEqual(t, len(summaries), 200, "200 search results expected") + + db.Close() } func BenchmarkImportText(b *testing.B) { From 00d254d7c41be358cc0a99d388e796703c3ee5b9 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 30 Jul 2022 23:52:57 +1200 Subject: [PATCH 6/9] Add test cache --- .github/workflows/test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 839d03f..870f7cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,6 +16,14 @@ jobs: with: go-version: ${{ matrix.go-version }} - uses: actions/checkout@v3 + - uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- - run: go test ./storage -v - run: go test ./storage -bench=. From 8d308a6776436ce1189e45268e4eceb8b5a1011a Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 30 Jul 2022 23:53:55 +1200 Subject: [PATCH 7/9] Add test cache --- .github/workflows/test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 870f7cc..5dfbd6a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,13 +17,13 @@ jobs: go-version: ${{ matrix.go-version }} - uses: actions/checkout@v3 - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - ~/go - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go- + with: + path: | + ~/.cache/go-build + ~/go + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- - run: go test ./storage -v - run: go test ./storage -bench=. From 3103b50f0857d18c48ec60ee622fabf4938c938f Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sun, 31 Jul 2022 00:05:07 +1200 Subject: [PATCH 8/9] UI: Add space in To fields --- server/ui-src/templates/Message.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/ui-src/templates/Message.vue b/server/ui-src/templates/Message.vue index 6e7ef3a..77b2f3a 100644 --- a/server/ui-src/templates/Message.vue +++ b/server/ui-src/templates/Message.vue @@ -80,7 +80,7 @@ export default { From - {{ message.From.Name }} + {{ message.From.Name + " " }} <{{ message.From.Address }}> From 8f474bc31303882270112cdfafeaedc908938707 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sun, 31 Jul 2022 08:46:41 +1200 Subject: [PATCH 9/9] Add changelog generator config --- .chglog/CHANGELOG.tpl.md | 48 ++++++++++++++++++++++++++++++++++++++++ .chglog/RELEASE.tpl.md | 12 ++++++++++ .chglog/config.yml | 28 +++++++++++++++++++++++ .chglog/release-info.md | 0 4 files changed, 88 insertions(+) create mode 100755 .chglog/CHANGELOG.tpl.md create mode 100755 .chglog/RELEASE.tpl.md create mode 100755 .chglog/config.yml create mode 100644 .chglog/release-info.md diff --git a/.chglog/CHANGELOG.tpl.md b/.chglog/CHANGELOG.tpl.md new file mode 100755 index 0000000..5abaff8 --- /dev/null +++ b/.chglog/CHANGELOG.tpl.md @@ -0,0 +1,48 @@ +# Changelog + +Notable changes to Mailpit will be documented in this file. + + +{{ if .Versions -}} +{{ if .Unreleased.CommitGroups -}} +## [Unreleased] + +{{ if .Unreleased.CommitGroups -}} +{{ range .Unreleased.CommitGroups -}} +### {{ .Title }} +{{ range .Commits -}} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +{{ end }} +{{ end }} +{{ end -}} +{{ end -}} +{{ end -}} + +{{ range .Versions }} +{{- if .CommitGroups -}} +## {{ .Tag.Name }} + +{{ range .CommitGroups -}} +### {{ .Title }} +{{ range .Commits -}} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +{{ end }} +{{ end }} +{{ end -}} + +{{- if .MergeCommits -}} +### Pull Requests +{{ range .MergeCommits -}} +- {{ .Header }} +{{ end }} +{{ end -}} + +{{- if .NoteGroups -}} +{{ range .NoteGroups -}} +### {{ .Title }} +{{ range .Notes }} +{{ .Body }} +{{ end }} +{{ end -}} +{{ end -}} +{{ end -}} diff --git a/.chglog/RELEASE.tpl.md b/.chglog/RELEASE.tpl.md new file mode 100755 index 0000000..a8ce8ae --- /dev/null +++ b/.chglog/RELEASE.tpl.md @@ -0,0 +1,12 @@ +{{ if .Versions -}} +{{ range .Versions }} +{{- if .CommitGroups -}} +{{ range .CommitGroups -}} +### {{ .Title }} +{{ range .Commits -}} +- {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }} +{{ end }} +{{ end -}} +{{ end -}} +{{ end -}} +{{ end -}} diff --git a/.chglog/config.yml b/.chglog/config.yml new file mode 100755 index 0000000..65597c4 --- /dev/null +++ b/.chglog/config.yml @@ -0,0 +1,28 @@ +style: github +template: CHANGELOG.tpl.md +info: + title: CHANGELOG + repository_url: https://github.com/axllent/mailpit +options: + commits: + # filters: + # Type: + # - feat + # - fix + # - perf + # - refactor + commit_groups: + title_maps: + feature: Feature + fix: Fix + # perf: Performance Improvements + # refactor: Code Refactoring + header: + pattern: "^(\\w*)(?:\\(([\\w\\$\\.\\-\\*\\s]*)\\))?\\:\\s(.*)$" + pattern_maps: + - Type + - Scope + - Subject + notes: + keywords: + - BREAKING CHANGE diff --git a/.chglog/release-info.md b/.chglog/release-info.md new file mode 100644 index 0000000..e69de29