Merge branch 'release/v1.9.3'

.github/workflows/tests.yml

@@ -1,7 +1,7 @@
 name: Tests
 on:
   pull_request:
-    branches: [ develop ]
+    branches: [ develop, 'feature/**' ]
   push:
     branches: [ develop, 'feature/**' ]
 jobs:
@@ -24,8 +24,8 @@ jobs:
         key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
         restore-keys: |
           ${{ runner.os }}-go-
-    - run: go test ./storage ./server -v
-    - run: go test ./storage -bench=.
+    - run: go test ./internal/storage ./server ./internal/tools -v
+    - run: go test ./internal/storage -bench=.
     
     # build the assets
     - uses: actions/setup-node@v3

CHANGELOG.md

@@ -2,6 +2,27 @@
 
 Notable changes to Mailpit will be documented in this file.
 
+## [v1.9.3]
+
+### Chore
+- Update internal/storage import paths
+- Move storage package to internal/storage
+- Update internal import paths
+- Move utils/* packages to internal/*
+
+### Testing
+- Add endpoints for integration tests
+
+### Tests
+- Add more API tests
+- Add tests for ArgsParser & CleanTag
+
+### UI
+- Do not show excluded search tags as "current" in nav
+- Display "Loading messages" instead of "No results" while loading results
+- Only queue broadcast events if clients are connected
+
+
 ## [v1.9.2]
 
 ### Fix

cmd/root.go

@@ -8,10 +8,10 @@ import (
 	"strings"
 
 	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
 	"github.com/axllent/mailpit/server"
 	"github.com/axllent/mailpit/server/smtpd"
-	"github.com/axllent/mailpit/storage"
-	"github.com/axllent/mailpit/utils/logger"
 	"github.com/spf13/cobra"
 )
 
@@ -142,7 +142,7 @@ func init() {
 
 // Load settings from environment
 func initConfigFromEnv() {
-	// defaults from envars if provided
+	// inherit from environment if provided
 	if len(os.Getenv("MP_DATA_FILE")) > 0 {
 		config.DataFile = os.Getenv("MP_DATA_FILE")
 	}

cmd/version.go

@@ -6,7 +6,7 @@ import (
 	"runtime"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/utils/updater"
+	"github.com/axllent/mailpit/internal/updater"
 	"github.com/spf13/cobra"
 )
 

config/config.go

@@ -10,8 +10,8 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/axllent/mailpit/utils/logger"
-	"github.com/axllent/mailpit/utils/tools"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
 	"github.com/tg123/go-htpasswd"
 	"gopkg.in/yaml.v3"
 )

internal/htmlcheck/css.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/utils/logger"
-	"github.com/axllent/mailpit/utils/tools"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
 	"github.com/vanng822/go-premailer/premailer"
 	"golang.org/x/net/html"
 	"golang.org/x/net/html/atom"

internal/htmlcheck/html.go

@@ -5,7 +5,7 @@ import (
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/axllent/mailpit/utils/tools"
+	"github.com/axllent/mailpit/internal/tools"
 )
 
 // HTML tests

internal/linkcheck/main.go

@@ -6,8 +6,8 @@ import (
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
-	"github.com/axllent/mailpit/storage"
-	"github.com/axllent/mailpit/utils/tools"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/tools"
 )
 
 var linkRe = regexp.MustCompile(`(?m)\b(http|ftp|https):\/\/([\w_-]+(?:(?:\.[\w_-]+)+))([\w.,@?^=%&:'!\/~+#-]*[\w@?^=%&\/~+#-])`)

internal/linkcheck/status.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/internal/logger"
 )
 
 func getHTTPStatuses(links []string, followRedirects bool) []Link {

internal/storage/database.go

@@ -20,8 +20,8 @@ import (
 
 	"github.com/GuiaBolso/darwin"
 	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
 	"github.com/axllent/mailpit/server/websockets"
-	"github.com/axllent/mailpit/utils/logger"
 	"github.com/jhillyerd/enmime"
 	"github.com/klauspost/compress/zstd"
 	"github.com/leporo/sqlf"

internal/storage/migrationTasks.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/internal/logger"
 	"github.com/jhillyerd/enmime"
 	"github.com/leporo/sqlf"
 	"golang.org/x/text/language"

internal/storage/search.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/axllent/mailpit/utils/logger"
-	"github.com/axllent/mailpit/utils/tools"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
 	"github.com/leporo/sqlf"
 )
 
@@ -26,7 +26,7 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) {
 		limit = 50
 	}
 
-	q := searchParser(search)
+	q := searchQueryBuilder(search)
 	var err error
 
 	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
@@ -95,7 +95,7 @@ func Search(search string, start, limit int) ([]MessageSummary, int, error) {
 // is:read, is:unread, has:attachment, to:<term>, from:<term> & subject:<term>
 // Negative searches also also included by prefixing the search term with a `-` or `!`
 func DeleteSearch(search string) error {
-	q := searchParser(search)
+	q := searchQueryBuilder(search)
 
 	ids := []string{}
 
@@ -187,7 +187,7 @@ func DeleteSearch(search string) error {
 }
 
 // SearchParser returns the SQL syntax for the database search based on the search arguments
-func searchParser(searchString string) *sqlf.Stmt {
+func searchQueryBuilder(searchString string) *sqlf.Stmt {
 	searchString = strings.ToLower(searchString)
 	// group strings with quotes as a single argument and remove quotes
 	args := tools.ArgsParser(searchString)

internal/storage/tags.go

@@ -7,8 +7,8 @@ import (
 	"strings"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/utils/logger"
-	"github.com/axllent/mailpit/utils/tools"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/tools"
 	"github.com/leporo/sqlf"
 )
 

internal/storage/test_shared.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/internal/logger"
 )
 
 var (

internal/storage/utils.go

@@ -10,8 +10,8 @@ import (
 	"time"
 
 	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
 	"github.com/axllent/mailpit/server/websockets"
-	"github.com/axllent/mailpit/utils/logger"
 	"github.com/jhillyerd/enmime"
 	"github.com/k3a/html2text"
 	"github.com/leporo/sqlf"

internal/tools/message.go

@@ -7,7 +7,7 @@ import (
 	"net/mail"
 	"regexp"
 
-	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/internal/logger"
 )
 
 // RemoveMessageHeaders scans a message for headers, if found them removes them.

internal/tools/tools_test.go

@@ -0,0 +1,45 @@
+package tools
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestArgsParser(t *testing.T) {
+	tests := map[string][]string{}
+	tests["this is a test"] = []string{"this", "is", "a", "test"}
+	tests["\"this is\" a test"] = []string{"this is", "a", "test"}
+	tests["!\"this is\" a test"] = []string{"!this is", "a", "test"}
+	tests["subject:this is a test"] = []string{"subject:this", "is", "a", "test"}
+	tests["subject:\"this is\" a test"] = []string{"subject:this is", "a", "test"}
+	tests["subject:\"this is\" \"a test\""] = []string{"subject:this is", "a test"}
+	tests["subject:\"this 'is\" \"a test\""] = []string{"subject:this 'is", "a test"}
+	tests["subject:\"this 'is a test"] = []string{"subject:this 'is a test"}
+	tests["\"this is a test\"=\"this is a test\""] = []string{"this is a test=this is a test"}
+
+	for search, expected := range tests {
+		res := ArgsParser(search)
+		if !reflect.DeepEqual(res, expected) {
+			t.Log("Args parser error:", res, "!=", expected)
+			t.Fail()
+		}
+	}
+}
+
+func TestCleanTag(t *testing.T) {
+	tests := map[string]string{}
+	tests["this is a test"] = "this is a test"
+	tests["thiS IS a Test"] = "thiS IS a Test"
+	tests["thiS IS a Test :-)"] = "thiS IS a Test -"
+	tests["  thiS 99     IS a Test :-)"] = "thiS 99 IS a Test -"
+	tests["this_is-a test "] = "this_is-a test"
+	tests["this_is-a&^%%(*)@ test"] = "this_is-a test"
+
+	for search, expected := range tests {
+		res := CleanTag(search)
+		if res != expected {
+			t.Log("CleanTags error:", res, "!=", expected)
+			t.Fail()
+		}
+	}
+}

internal/updater/updater.go

@@ -13,7 +13,7 @@ import (
 	"path/filepath"
 	"runtime"
 
-	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/internal/logger"
 	"github.com/axllent/semver"
 )
 

sendmail/cmd/cmd.go

@@ -24,7 +24,7 @@ import (
 	"strings"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/internal/logger"
 	"github.com/reiver/go-telnet"
 	flag "github.com/spf13/pflag"
 )

server/apiv1/api.go

@@ -11,12 +11,12 @@ import (
 	"strings"
 
 	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/htmlcheck"
+	"github.com/axllent/mailpit/internal/linkcheck"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/tools"
 	"github.com/axllent/mailpit/server/smtpd"
-	"github.com/axllent/mailpit/storage"
-	"github.com/axllent/mailpit/utils/htmlcheck"
-	"github.com/axllent/mailpit/utils/linkcheck"
-	"github.com/axllent/mailpit/utils/logger"
-	"github.com/axllent/mailpit/utils/tools"
 	"github.com/gorilla/mux"
 	uuid "github.com/satori/go.uuid"
 )

server/apiv1/info.go

@@ -7,8 +7,8 @@ import (
 	"runtime"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/storage"
-	"github.com/axllent/mailpit/utils/updater"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/updater"
 )
 
 // Response includes the current and latest Mailpit version, database info, and memory usage

server/apiv1/structs.go

@@ -1,9 +1,9 @@
 package apiv1
 
 import (
-	"github.com/axllent/mailpit/storage"
-	"github.com/axllent/mailpit/utils/htmlcheck"
-	"github.com/axllent/mailpit/utils/linkcheck"
+	"github.com/axllent/mailpit/internal/htmlcheck"
+	"github.com/axllent/mailpit/internal/linkcheck"
+	"github.com/axllent/mailpit/internal/storage"
 )
 
 // MessagesSummary is a summary of a list of messages

server/apiv1/swagger.go

@@ -81,6 +81,13 @@ type textResponse struct {
 	Body string
 }
 
+// HTML response
+// swagger:response HTMLResponse
+type htmlResponse struct {
+	// in: body
+	Body string
+}
+
 // Error response
 // swagger:response ErrorResponse
 type errorResponse struct {

server/apiv1/thumbnails.go

@@ -10,8 +10,8 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/axllent/mailpit/storage"
-	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
 	"github.com/disintegration/imaging"
 	"github.com/gorilla/mux"
 	"github.com/jhillyerd/enmime"

server/handlers/message-rendered.go

@@ -0,0 +1,163 @@
+package handlers
+
+import (
+	"fmt"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/gorilla/mux"
+)
+
+// GetMessageHTML (method: GET) returns a rendered version of a message's HTML part
+func GetMessageHTML(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /view/{ID}.html testing GetMessageHTML
+	//
+	// # Render message HTML part
+	//
+	// Renders just the message's HTML part which can be used for UI integration testing.
+	// Attached inline images are modified to link to the API provided they exist.
+	// Note that is the message does not contain a HTML part then an 404 error is returned.
+	//
+	// The ID can be set to `latest` to return the latest message.
+	//
+	//	Produces:
+	//	- text/html
+	//
+	//	Schemes: http, https
+	//
+	//	Parameters:
+	//	  + name: ID
+	//	    in: path
+	//	    description: Database ID or latest
+	//	    required: true
+	//	    type: string
+	//
+	//	Responses:
+	//		200: HTMLResponse
+	//		default: ErrorResponse
+
+	vars := mux.Vars(r)
+
+	id := vars["id"]
+
+	if id == "latest" {
+		messages, err := storage.List(0, 1)
+		if err != nil {
+			httpError(w, err.Error())
+			return
+		}
+
+		if len(messages) == 0 {
+			w.WriteHeader(404)
+			fmt.Fprint(w, "Message not found")
+			return
+		}
+
+		id = messages[0].ID
+	}
+
+	msg, err := storage.GetMessage(id)
+	if err != nil {
+		w.WriteHeader(404)
+		fmt.Fprint(w, "Message not found")
+		return
+	}
+	if msg.HTML == "" {
+		w.WriteHeader(404)
+		fmt.Fprint(w, "This message does not contain a HTML part")
+		return
+	}
+
+	html := linkInlinedImages(msg)
+	w.Header().Add("Content-Type", "text/html; charset=utf-8")
+	_, _ = w.Write([]byte(html))
+}
+
+// GetMessageText (method: GET) returns a message's text part
+func GetMessageText(w http.ResponseWriter, r *http.Request) {
+	// swagger:route GET /view/{ID}.txt testing GetMessageText
+	//
+	// # Render message text part
+	//
+	// Renders just the message's text part which can be used for UI integration testing.
+	//
+	// The ID can be set to `latest` to return the latest message.
+	//
+	//	Produces:
+	//	- text/plain
+	//
+	//	Schemes: http, https
+	//
+	//	Parameters:
+	//	  + name: ID
+	//	    in: path
+	//	    description: Database ID or latest
+	//	    required: true
+	//	    type: string
+	//
+	//	Responses:
+	//		200: TextResponse
+	//		default: ErrorResponse
+
+	vars := mux.Vars(r)
+
+	id := vars["id"]
+
+	if id == "latest" {
+		messages, err := storage.List(0, 1)
+		if err != nil {
+			httpError(w, err.Error())
+			return
+		}
+
+		if len(messages) == 0 {
+			w.WriteHeader(404)
+			fmt.Fprint(w, "Message not found")
+			return
+		}
+
+		id = messages[0].ID
+	}
+
+	msg, err := storage.GetMessage(id)
+	if err != nil {
+		w.WriteHeader(404)
+		fmt.Fprint(w, "Message not found")
+		return
+	}
+
+	w.Header().Add("Content-Type", "text/plain; charset=utf-8")
+	_, _ = w.Write([]byte(msg.Text))
+}
+
+// This will remap all attachment images with relative paths
+func linkInlinedImages(msg *storage.Message) string {
+	html := msg.HTML
+
+	for _, a := range msg.Inline {
+		if a.ContentID != "" {
+			re := regexp.MustCompile(`(?i)(=["\']?)(cid:` + regexp.QuoteMeta(a.ContentID) + `)(["|\'|\\s|\\/|>|;])`)
+			u := config.Webroot + "api/v1/message/" + msg.ID + "/part/" + a.PartID
+			matches := re.FindAllStringSubmatch(html, -1)
+			for _, m := range matches {
+				html = strings.ReplaceAll(html, m[0], m[1]+u+m[3])
+			}
+		}
+	}
+
+	for _, a := range msg.Attachments {
+		if a.ContentID != "" {
+			re := regexp.MustCompile(`(?i)(=["\']?)(cid:` + regexp.QuoteMeta(a.ContentID) + `)(["|\'|\\s|\\/|>|;])`)
+			u := config.Webroot + "api/v1/message/" + msg.ID + "/part/" + a.PartID
+			matches := re.FindAllStringSubmatch(html, -1)
+			for _, m := range matches {
+				html = strings.ReplaceAll(html, m[0], m[1]+u+m[3])
+			}
+		}
+	}
+
+	return html
+}

server/handlers/proxy.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/internal/logger"
 )
 
 var linkRe = regexp.MustCompile(`(?i)^https?:\/\/`)

server/server.go

@@ -15,11 +15,11 @@ import (
 	"text/template"
 
 	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
 	"github.com/axllent/mailpit/server/apiv1"
 	"github.com/axllent/mailpit/server/handlers"
 	"github.com/axllent/mailpit/server/websockets"
-	"github.com/axllent/mailpit/storage"
-	"github.com/axllent/mailpit/utils/logger"
 	"github.com/gorilla/mux"
 )
 
@@ -67,8 +67,14 @@ func Listen() {
 		r.HandleFunc(redirect, middleWareFunc(addSlashToWebroot)).Methods("GET")
 	}
 
-	// handle everything else with the virtual index.html
-	r.PathPrefix(config.Webroot).Handler(middleWareFunc(index)).Methods("GET")
+	// frontend testing
+	r.HandleFunc(config.Webroot+"view/{id}.html", handlers.GetMessageHTML).Methods("GET")
+	r.HandleFunc(config.Webroot+"view/{id}.txt", handlers.GetMessageText).Methods("GET")
+
+	// web UI via virtual index.html
+	r.PathPrefix(config.Webroot + "view/").Handler(middleWareFunc(index)).Methods("GET")
+	r.Path(config.Webroot + "search").Handler(middleWareFunc(index)).Methods("GET")
+	r.Path(config.Webroot).Handler(middleWareFunc(index)).Methods("GET")
 
 	// put it all together
 	http.Handle("/", r)
@@ -293,10 +299,6 @@ func index(w http.ResponseWriter, _ *http.Request) {
 
 	buff.Bytes()
 
-	// f, err := embeddedFS.ReadFile("public/index.html")
-	// if err != nil {
-	// 	panic(err)
-	// }
 	w.Header().Add("Content-Type", "text/html")
 	_, _ = w.Write(buff.Bytes())
 }

server/server_test.go

@@ -12,9 +12,9 @@ import (
 	"testing"
 
 	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
 	"github.com/axllent/mailpit/server/apiv1"
-	"github.com/axllent/mailpit/storage"
-	"github.com/axllent/mailpit/utils/logger"
 	"github.com/jhillyerd/enmime"
 )
 
@@ -25,7 +25,7 @@ var (
 	}
 )
 
-func Test_APIv1(t *testing.T) {
+func TestAPIv1Messages(t *testing.T) {
 	setup()
 	defer storage.Close()
 
@@ -54,7 +54,7 @@ func Test_APIv1(t *testing.T) {
 		t.Errorf(err.Error())
 	}
 
-	// read first 10
+	// read first 10 messages
 	t.Log("Read first 10 messages including raw & headers")
 	putIDS := []string{}
 	for idx, msg := range m.Messages {
@@ -66,12 +66,12 @@ func Test_APIv1(t *testing.T) {
 			t.Errorf(err.Error())
 		}
 
-		// test RAW
+		// get RAW
 		if _, err := clientGet(ts.URL + "/api/v1/message/" + msg.ID + "/raw"); err != nil {
 			t.Errorf(err.Error())
 		}
 
-		// test headers
+		// het headers
 		if _, err := clientGet(ts.URL + "/api/v1/message/" + msg.ID + "/headers"); err != nil {
 			t.Errorf(err.Error())
 		}
@@ -79,11 +79,63 @@ func Test_APIv1(t *testing.T) {
 		// store for later
 		putIDS = append(putIDS, msg.ID)
 	}
+
+	// 10 should be marked as read
 	assertStatsEqual(t, ts.URL+"/api/v1/messages", 90, 100)
 
+	// delete all
+	t.Log("Delete all messages")
+	_, err = clientDelete(ts.URL+"/api/v1/messages", "{}")
+	if err != nil {
+		t.Errorf("Expected nil, received %s", err.Error())
+	}
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 0, 0)
+}
+
+func TestAPIv1ToggleReadStatus(t *testing.T) {
+	setup()
+	defer storage.Close()
+
+	r := apiRoutes()
+
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	m, err := fetchMessages(ts.URL + "/api/v1/messages")
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	// check count of empty database
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 0, 0)
+
+	// insert 100
+	t.Log("Insert 100 messages")
+	insertEmailData(t)
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 100, 100)
+
+	m, err = fetchMessages(ts.URL + "/api/v1/messages")
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	// read first 10 IDs
+	t.Log("Get first 10 IDs")
+	putIDS := []string{}
+	for idx, msg := range m.Messages {
+		if idx == 10 {
+			break
+		}
+
+		// store for later
+		putIDS = append(putIDS, msg.ID)
+	}
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 100, 100)
+
 	// mark first 10 as unread
-	t.Log("Mark first 10 as unread")
+	t.Log("Mark first 10 as read")
 	putData := putDataStruct
+	putData.Read = true
 	putData.IDs = putIDS
 	j, err := json.Marshal(putData)
 	if err != nil {
@@ -93,11 +145,11 @@ func Test_APIv1(t *testing.T) {
 	if err != nil {
 		t.Errorf(err.Error())
 	}
-	assertStatsEqual(t, ts.URL+"/api/v1/messages", 100, 100)
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 90, 100)
 
 	// mark first 10 as read
-	t.Log("Mark first 10 as read")
-	putData.Read = true
+	t.Log("Mark first 10 as unread")
+	putData.Read = false
 	j, err = json.Marshal(putData)
 	if err != nil {
 		t.Errorf(err.Error())
@@ -106,25 +158,7 @@ func Test_APIv1(t *testing.T) {
 	if err != nil {
 		t.Errorf(err.Error())
 	}
-	assertStatsEqual(t, ts.URL+"/api/v1/messages", 90, 100)
-
-	// search
-	assertSearchEqual(t, ts.URL+"/api/v1/search", "from-1@example.com", 1)
-	assertSearchEqual(t, ts.URL+"/api/v1/search", "to:from-1@example.com", 0)
-	assertSearchEqual(t, ts.URL+"/api/v1/search", "from:@example.com", 100)
-	assertSearchEqual(t, ts.URL+"/api/v1/search", "subject:\"Subject line\"", 100)
-	assertSearchEqual(t, ts.URL+"/api/v1/search", "subject:\"Subject line 17 end\"", 1)
-	assertSearchEqual(t, ts.URL+"/api/v1/search", "!thisdoesnotexist", 100)
-	assertSearchEqual(t, ts.URL+"/api/v1/search", "-thisdoesnotexist", 100)
-	assertSearchEqual(t, ts.URL+"/api/v1/search", "thisdoesnotexist", 0)
-
-	// delete first 10
-	t.Log("Delete first 10")
-	_, err = clientDelete(ts.URL+"/api/v1/messages", string(j))
-	if err != nil {
-		t.Errorf(err.Error())
-	}
-	assertStatsEqual(t, ts.URL+"/api/v1/messages", 90, 90)
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 100, 100)
 
 	// mark all as read
 	putData.Read = true
@@ -139,15 +173,34 @@ func Test_APIv1(t *testing.T) {
 	if err != nil {
 		t.Errorf(err.Error())
 	}
-	assertStatsEqual(t, ts.URL+"/api/v1/messages", 0, 90)
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 0, 100)
+}
 
-	// delete all
-	t.Log("Delete all messages")
-	_, err = clientDelete(ts.URL+"/api/v1/messages", "{}")
-	if err != nil {
-		t.Errorf("Expected nil, received %s", err.Error())
-	}
-	assertStatsEqual(t, ts.URL+"/api/v1/messages", 0, 0)
+func TestAPIv1Search(t *testing.T) {
+	setup()
+	defer storage.Close()
+
+	r := apiRoutes()
+
+	ts := httptest.NewServer(r)
+	defer ts.Close()
+
+	// insert 100
+	t.Log("Insert 100 messages")
+	insertEmailData(t)
+	assertStatsEqual(t, ts.URL+"/api/v1/messages", 100, 100)
+
+	// search
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "from-1@example.com", 1)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "from:from-1@example.com", 1)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "-from:from-1@example.com", 99)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "to:from-1@example.com", 0)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "from:@example.com", 100)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "subject:\"Subject line\"", 100)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "subject:\"Subject line 17 end\"", 1)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "!thisdoesnotexist", 100)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "-thisdoesnotexist", 100)
+	assertSearchEqual(t, ts.URL+"/api/v1/search", "thisdoesnotexist", 0)
 }
 
 func setup() {

server/smtpd/smtp.go

@@ -8,7 +8,7 @@ import (
 	"net/smtp"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/internal/logger"
 )
 
 func allowedRecipients(to []string) []string {
@@ -63,22 +63,10 @@ func Send(from string, to []string, msg []byte) error {
 		}
 	}
 
-	var a smtp.Auth
+	auth := relayAuthFromConfig()
 
-	if config.SMTPRelayConfig.Auth == "plain" {
-		a = smtp.PlainAuth("", config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Password, config.SMTPRelayConfig.Host)
-	}
-
-	if config.SMTPRelayConfig.Auth == "login" {
-		a = LoginAuth(config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Password)
-	}
-
-	if config.SMTPRelayConfig.Auth == "cram-md5" {
-		a = smtp.CRAMMD5Auth(config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Secret)
-	}
-
-	if a != nil {
-		if err = c.Auth(a); err != nil {
+	if auth != nil {
+		if err = c.Auth(auth); err != nil {
 			return fmt.Errorf("error response to AUTH command: %s", err.Error())
 		}
 	}
@@ -109,6 +97,25 @@ func Send(from string, to []string, msg []byte) error {
 	return c.Quit()
 }
 
+// Return the SMTP relay authentication based on config
+func relayAuthFromConfig() smtp.Auth {
+	var a smtp.Auth
+
+	if config.SMTPRelayConfig.Auth == "plain" {
+		a = smtp.PlainAuth("", config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Password, config.SMTPRelayConfig.Host)
+	}
+
+	if config.SMTPRelayConfig.Auth == "login" {
+		a = LoginAuth(config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Password)
+	}
+
+	if config.SMTPRelayConfig.Auth == "cram-md5" {
+		a = smtp.CRAMMD5Auth(config.SMTPRelayConfig.Username, config.SMTPRelayConfig.Secret)
+	}
+
+	return a
+}
+
 // Custom implementation of LOGIN SMTP authentication
 // @see https://gist.github.com/andelf/5118732
 type loginAuth struct {

server/smtpd/smtpd.go

@@ -10,8 +10,8 @@ import (
 	"strings"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/storage"
-	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/storage"
 	"github.com/mhale/smtpd"
 	uuid "github.com/satori/go.uuid"
 )

server/ui-src/components/ListMessages.vue

@@ -8,6 +8,10 @@ export default {
 		CommonMixins
 	],
 
+	props: {
+		loadingMessages: Number, // use different name to `loading` as that is already in use in CommonMixins
+	},
+
 	data() {
 		return {
 			mailbox,
@@ -153,13 +157,15 @@ export default {
 				<div class="d-none d-lg-block col-2 col-xxl-1 small text-end text-muted">
 					{{ getRelativeCreated(message) }}
 				</div>
-				<!-- </a> -->
 			</RouterLink>
 		</div>
 	</template>
 	<template v-else>
 		<p class="text-center mt-5">
-			<template v-if="getSearch()">No results for <code>{{ getSearch() }}</code></template>
+			<span v-if="loadingMessages > 0" class="text-secondary">
+				Loading messages...
+			</span>
+			<template v-else-if="getSearch()">No results for <code>{{ getSearch() }}</code></template>
 			<template v-else>No messages in your mailbox</template>
 		</p>
 	</template>

server/ui-src/components/NavTags.vue

@@ -19,7 +19,7 @@ export default {
 				return false
 			}
 
-			let re = new RegExp(`\\btag:"?${tag}"?\\b`, 'i')
+			let re = new RegExp(`\\b[^\-!]tag:"?${tag}"?\\b`, 'i')
 			return query.match(re)
 		}
 	}

server/ui-src/views/MailboxView.vue

@@ -83,7 +83,7 @@ export default {
 
 		<div class="col-xl-10 col-md-9 mh-100 ps-0 ps-md-2 pe-0">
 			<div class="mh-100" style="overflow-y: auto;" id="message-page">
-				<ListMessages />
+				<ListMessages :loading-messages="loading" />
 			</div>
 		</div>
 	</div>

server/ui-src/views/SearchView.vue

@@ -111,7 +111,7 @@ export default {
 
 		<div class="col-lg-10 col-md-9 mh-100 ps-0 ps-md-2 pe-0">
 			<div class="mh-100" style="overflow-y: auto;" id="message-page">
-				<ListMessages />
+				<ListMessages :loading-messages="loading" />
 			</div>
 		</div>
 	</div>

server/ui/api/v1/swagger.json

@@ -654,6 +654,74 @@
           }
         }
       }
+    },
+    "/view/{ID}.html": {
+      "get": {
+        "description": "Renders just the message's HTML part which can be used for UI integration testing.\nAttached inline images are modified to link to the API provided they exist.\nNote that is the message does not contain a HTML part then an 404 error is returned.\n\nThe ID can be set to `latest` to return the latest message.",
+        "produces": [
+          "text/html"
+        ],
+        "schemes": [
+          "http",
+          "https"
+        ],
+        "tags": [
+          "testing"
+        ],
+        "summary": "Render message HTML part",
+        "operationId": "GetMessageHTML",
+        "parameters": [
+          {
+            "type": "string",
+            "description": "Database ID or latest",
+            "name": "ID",
+            "in": "path",
+            "required": true
+          }
+        ],
+        "responses": {
+          "200": {
+            "$ref": "#/responses/HTMLResponse"
+          },
+          "default": {
+            "$ref": "#/responses/ErrorResponse"
+          }
+        }
+      }
+    },
+    "/view/{ID}.txt": {
+      "get": {
+        "description": "Renders just the message's text part which can be used for UI integration testing.\n\nThe ID can be set to `latest` to return the latest message.",
+        "produces": [
+          "text/plain"
+        ],
+        "schemes": [
+          "http",
+          "https"
+        ],
+        "tags": [
+          "testing"
+        ],
+        "summary": "Render message text part",
+        "operationId": "GetMessageText",
+        "parameters": [
+          {
+            "type": "string",
+            "description": "Database ID or latest",
+            "name": "ID",
+            "in": "path",
+            "required": true
+          }
+        ],
+        "responses": {
+          "200": {
+            "$ref": "#/responses/TextResponse"
+          },
+          "default": {
+            "$ref": "#/responses/ErrorResponse"
+          }
+        }
+      }
     }
   },
   "definitions": {
@@ -732,7 +800,7 @@
           "format": "int64"
         }
       },
-      "x-go-package": "github.com/axllent/mailpit/storage"
+      "x-go-package": "github.com/axllent/mailpit/internal/storage"
     },
     "DeleteRequest": {
       "description": "Delete request",
@@ -776,7 +844,7 @@
         }
       },
       "x-go-name": "Response",
-      "x-go-package": "github.com/axllent/mailpit/utils/htmlcheck"
+      "x-go-package": "github.com/axllent/mailpit/internal/htmlcheck"
     },
     "HTMLCheckResult": {
       "description": "Result struct",
@@ -808,7 +876,7 @@
         }
       },
       "x-go-name": "Result",
-      "x-go-package": "github.com/axllent/mailpit/utils/htmlcheck"
+      "x-go-package": "github.com/axllent/mailpit/internal/htmlcheck"
     },
     "HTMLCheckScore": {
       "description": "Score struct",
@@ -836,7 +904,7 @@
         }
       },
       "x-go-name": "Score",
-      "x-go-package": "github.com/axllent/mailpit/utils/htmlcheck"
+      "x-go-package": "github.com/axllent/mailpit/internal/htmlcheck"
     },
     "HTMLCheckTotal": {
       "description": "Total weighted result for all scores",
@@ -869,7 +937,7 @@
         }
       },
       "x-go-name": "Total",
-      "x-go-package": "github.com/axllent/mailpit/utils/htmlcheck"
+      "x-go-package": "github.com/axllent/mailpit/internal/htmlcheck"
     },
     "HTMLCheckWarning": {
       "description": "Warning represents a failed test",
@@ -925,7 +993,7 @@
         }
       },
       "x-go-name": "Warning",
-      "x-go-package": "github.com/axllent/mailpit/utils/htmlcheck"
+      "x-go-package": "github.com/axllent/mailpit/internal/htmlcheck"
     },
     "Link": {
       "description": "Link struct",
@@ -945,7 +1013,7 @@
           "type": "string"
         }
       },
-      "x-go-package": "github.com/axllent/mailpit/utils/linkcheck"
+      "x-go-package": "github.com/axllent/mailpit/internal/linkcheck"
     },
     "LinkCheckResponse": {
       "description": "Response represents the Link check response",
@@ -965,7 +1033,7 @@
         }
       },
       "x-go-name": "Response",
-      "x-go-package": "github.com/axllent/mailpit/utils/linkcheck"
+      "x-go-package": "github.com/axllent/mailpit/internal/linkcheck"
     },
     "Message": {
       "description": "Message data excluding physical attachments",
@@ -1058,7 +1126,7 @@
           }
         }
       },
-      "x-go-package": "github.com/axllent/mailpit/storage"
+      "x-go-package": "github.com/axllent/mailpit/internal/storage"
     },
     "MessageHeaders": {
       "description": "Message headers",
@@ -1139,7 +1207,7 @@
           }
         }
       },
-      "x-go-package": "github.com/axllent/mailpit/storage"
+      "x-go-package": "github.com/axllent/mailpit/internal/storage"
     },
     "MessagesSummary": {
       "description": "MessagesSummary is a summary of a list of messages",
@@ -1300,6 +1368,9 @@
     "ErrorResponse": {
       "description": "Error response"
     },
+    "HTMLResponse": {
+      "description": "HTML response"
+    },
     "InfoResponse": {
       "description": "Application information",
       "schema": {

server/websockets/client.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/axllent/mailpit/config"
-	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/internal/logger"
 	"github.com/gorilla/websocket"
 )
 

server/websockets/hub.go

@@ -7,7 +7,7 @@ package websockets
 import (
 	"encoding/json"
 
-	"github.com/axllent/mailpit/utils/logger"
+	"github.com/axllent/mailpit/internal/logger"
 )
 
 // Hub maintains the set of active clients and broadcasts messages to the
@@ -69,7 +69,7 @@ func (h *Hub) Run() {
 
 // Broadcast will spawn a broadcast message to all connected clients
 func Broadcast(t string, msg interface{}) {
-	if MessageHub == nil {
+	if MessageHub == nil || len(MessageHub.Clients) == 0 {
 		return
 	}