Merge branch 'feature/search-pagination' into develop

server/apiv1/api.go

@@ -62,10 +62,11 @@ func GetMessages(w http.ResponseWriter, r *http.Request) {
 
 	res.Start = start
 	res.Messages = messages
-	res.Count = len(messages)
+	res.Count = len(messages) // legacy - now undocumented in API specs
 	res.Total = stats.Total
 	res.Unread = stats.Unread
 	res.Tags = stats.Tags
+	res.MessagesCount = stats.Total
 
 	bytes, _ := json.Marshal(res)
 	w.Header().Add("Content-Type", "application/json")
@@ -109,7 +110,7 @@ func Search(w http.ResponseWriter, r *http.Request) {
 
 	start, limit := getStartLimit(r)
 
-	messages, err := storage.Search(search, start, limit)
+	messages, results, err := storage.Search(search, start, limit)
 	if err != nil {
 		httpError(w, err.Error())
 		return
@@ -121,8 +122,9 @@ func Search(w http.ResponseWriter, r *http.Request) {
 
 	res.Start = start
 	res.Messages = messages
-	res.Count = len(messages)
+	res.Count = results // legacy - now undocumented in API specs
 	res.Total = stats.Total
+	res.MessagesCount = results
 	res.Unread = stats.Unread
 	res.Tags = stats.Tags
 

server/apiv1/structs.go

@@ -12,9 +12,17 @@ type MessagesSummary struct {
 	// Total number of unread messages in mailbox
 	Unread int `json:"unread"`
 
-	// Number of results returned
+	// Legacy - now undocumented in API specs but left for backwards compatibility.
+	// Removed from API documentation 2023-07-12
+	// swagger:ignore
 	Count int `json:"count"`
 
+	// Total number of messages matching current query
+	MessagesCount int `json:"messages_count"`
+
+	// // Number of results returned on current page
+	// Count int `json:"count"`
+
 	// Pagination offset
 	Start int `json:"start"`
 

server/apiv1/webui.go

@@ -32,6 +32,7 @@ func WebUIConfig(w http.ResponseWriter, r *http.Request) {
 	// # Get web UI configuration
 	//
 	// Returns configuration settings for the web UI.
+	// Intended for web UI only!
 	//
 	//	Produces:
 	//	- application/json

server/server_test.go

@@ -195,7 +195,7 @@ func assertSearchEqual(t *testing.T, uri, query string, count int) {
 		return
 	}
 
-	assertEqual(t, count, m.Count, "wrong search results count")
+	assertEqual(t, count, m.MessagesCount, "wrong search results count")
 }
 
 func insertEmailData(t *testing.T) {

server/ui-src/App.vue

@@ -26,6 +26,7 @@ export default {
 			limit: 50,
 			total: 0,
 			unread: 0,
+			messagesCount: 0,
 			start: 0,
 			count: 0,
 			tags: [],
@@ -75,7 +76,7 @@ export default {
 			return this.start > 0
 		},
 		canNext: function () {
-			return this.total > (this.start + this.count)
+			return this.messagesCount > (this.start + this.count)
 		},
 		unreadInSearch: function () {
 			if (!this.searching) {
@@ -146,11 +147,12 @@ export default {
 			let uri = 'api/v1/messages'
 			if (self.search) {
 				self.searching = true
-				self.items = []
 				uri = 'api/v1/search'
-				self.start = 0 // search is displayed on one page
 				params['query'] = self.search
-				params['limit'] = 200
+				params['limit'] = self.limit
+				if (self.start > 0) {
+					params['start'] = self.start
+				}
 			} else {
 				self.searching = false
 				params['limit'] = self.limit
@@ -162,7 +164,8 @@ export default {
 			self.get(uri, params, function (response) {
 				self.total = response.data.total
 				self.unread = response.data.unread
-				self.count = response.data.count
+				self.count = response.data.messages.length
+				self.messagesCount = response.data.messages_count
 				self.start = response.data.start
 				self.items = response.data.messages
 				self.tags = response.data.tags
@@ -194,6 +197,7 @@ export default {
 
 		doSearch: function (e) {
 			e.preventDefault()
+			this.start = 0
 			this.loadMessages()
 		},
 
@@ -204,13 +208,14 @@ export default {
 			}
 			this.search = 'tag:' + tag
 			window.location.hash = ""
+			this.start = 0
 			this.loadMessages()
 		},
 
 		resetSearch: function (e) {
 			e.preventDefault()
 			this.search = ''
-			this.scrollInPlace = true
+			this.start = 0
 			this.loadMessages()
 		},
 
@@ -459,9 +464,9 @@ export default {
 
 		// websocket connect
 		connect: function () {
-			let wsproto = location.protocol == 'https:' ? 'wss' : 'ws'
+			let proto = location.protocol == 'https:' ? 'wss' : 'ws'
 			let ws = new WebSocket(
-				wsproto + "://" + document.location.host + document.location.pathname + "api/events"
+				proto + "://" + document.location.host + document.location.pathname + "api/events"
 			)
 			let self = this
 			ws.onmessage = function (e) {
@@ -809,7 +814,7 @@ export default {
 				<i class="bi bi-check2-square"></i>
 			</button>
 
-			<select v-model="limit" v-on:change="loadMessages" v-if="!searching"
+			<select v-model="limit" v-on:change="loadMessages"
 				class="form-select form-select-sm d-none d-md-inline w-auto me-2">
 				<option value="25">25</option>
 				<option value="50">50</option>
@@ -817,23 +822,18 @@ export default {
 				<option value="200">200</option>
 			</select>
 
-			<span v-if="searching">
+			<small>
-				<b>{{ formatNumber(items.length) }} result<template v-if="items.length != 1">s</template></b>
+				{{ formatNumber(start + 1) }}-{{ formatNumber(start + items.length) }} <small>of</small>
-			</span>
+				{{ formatNumber(messagesCount) }}
-			<span v-else>
+			</small>
-				<small>
+			<button class="btn btn-outline-light ms-2 me-1" :disabled="!canPrev" v-on:click="viewPrev"
-					{{ formatNumber(start + 1) }}-{{ formatNumber(start + items.length) }} <small>of</small>
+				:title="'View previous ' + limit + ' messages'">
-					{{ formatNumber(total) }}
+				<i class="bi bi-caret-left-fill"></i>
-				</small>
+			</button>
-				<button class="btn btn-outline-light ms-2 me-1" :disabled="!canPrev" v-on:click="viewPrev" v-if="!searching"
+			<button class="btn btn-outline-light" :disabled="!canNext" v-on:click="viewNext"
-					:title="'View previous ' + limit + ' messages'">
+				:title="'View next ' + limit + ' messages'">
-					<i class="bi bi-caret-left-fill"></i>
+				<i class="bi bi-caret-right-fill"></i>
-				</button>
+			</button>
-				<button class="btn btn-outline-light" :disabled="!canNext" v-on:click="viewNext" v-if="!searching"
-					:title="'View next ' + limit + ' messages'">
-					<i class="bi bi-caret-right-fill"></i>
-				</button>
-			</span>
 		</div>
 	</div>
 	<div class="row flex-fill" style="min-height:0">

server/ui-src/templates/Message.vue

@@ -23,11 +23,12 @@ export default {
 		return {
 			srcURI: false,
 			iframes: [], // for resizing
-			showTags: false, // to force rerendering of component
+			showTags: false, // to force re-rendering of component
+			canSaveTags: false, // prevent auto-saving tags on render
 			messageTags: [],
 			allTags: [],
 			loadHeaders: false,
-			showMobileBtns: false,
+			showMobileButtons: false,
 			scaleHTMLPreview: 'display',
 			// keys names match bootstrap icon names 
 			responsiveSizes: {
@@ -39,20 +40,25 @@ export default {
 	},
 
 	watch: {
+		// handle changes to the URL messageID
 		message: {
 			handler() {
 				let self = this
 				self.showTags = false
+				self.canSaveTags = false
 				self.messageTags = self.message.Tags
 				self.allTags = self.existingTags
 				self.loadHeaders = false
-				self.scaleHTMLPreview = 'display';// default view
+				self.scaleHTMLPreview = 'display' // default view
 				// delay to select first tab and add HTML highlighting (prev/next)
 				self.$nextTick(function () {
 					self.renderUI()
 					self.showTags = true
 					self.$nextTick(function () {
 						Tags.init("select[multiple]")
+						window.setTimeout(function () {
+							self.canSaveTags = true
+						}, 200)
 					})
 				})
 			},
@@ -60,8 +66,8 @@ export default {
 			immediate: true
 		},
 		messageTags() {
-			// save changed to tags
+			// save changes to tags
-			if (this.showTags) {
+			if (this.canSaveTags) {
 				this.saveTags()
 			}
 		},
@@ -78,6 +84,7 @@ export default {
 	mounted() {
 		let self = this
 		self.showTags = false
+		self.canSaveTags = false
 		self.allTags = self.existingTags
 		window.addEventListener("resize", self.resizeIframes)
 		self.renderUI()
@@ -95,7 +102,12 @@ export default {
 
 		self.showTags = true
 		self.$nextTick(function () {
-			Tags.init("select[multiple]")
+			self.$nextTick(function () {
+				Tags.init('select[multiple]')
+				window.setTimeout(function () {
+					self.canSaveTags = true
+				}, 200)
+			})
 		})
 	},
 
@@ -104,6 +116,10 @@ export default {
 	},
 
 	methods: {
+		isHTMLTabSelected: function () {
+			this.showMobileButtons = this.$refs.navhtml
+				&& this.$refs.navhtml.classList.contains('active')
+		},
 		renderUI: function () {
 			let self = this
 			// click the first non-disabled tab
@@ -111,6 +127,14 @@ export default {
 			document.activeElement.blur() // blur focus
 			document.getElementById('message-view').scrollTop = 0
 
+			self.isHTMLTabSelected()
+
+			document.querySelectorAll('button[data-bs-toggle="tab"]').forEach(function (listObj) {
+				listObj.addEventListener('shown.bs.tab', function (event) {
+					self.isHTMLTabSelected()
+				})
+			})
+
 			// delay 0.2s until vue has rendered the iframe content
 			window.setTimeout(function () {
 				let p = document.getElementById('preview-html')
@@ -311,29 +335,30 @@ export default {
 		<nav>
 			<div class="nav nav-tabs my-3" id="nav-tab" role="tablist">
 				<button class="nav-link" id="nav-html-tab" data-bs-toggle="tab" data-bs-target="#nav-html" type="button"
-					role="tab" aria-controls="nav-html" aria-selected="true" v-if="message.HTML"
+					role="tab" aria-controls="nav-html" aria-selected="true" v-if="message.HTML" ref="navhtml"
-					v-on:click="showMobileBtns = true; resizeIframes()">HTML</button>
+					v-on:click="resizeIframes()">HTML</button>
 				<button class="nav-link" id="nav-html-source-tab" data-bs-toggle="tab" data-bs-target="#nav-html-source"
-					type="button" role="tab" aria-controls="nav-html-source" aria-selected="false" v-if="message.HTML"
+					type="button" role="tab" aria-controls="nav-html-source" aria-selected="false" v-if="message.HTML">
-					v-on:click=" showMobileBtns = false">
 					HTML <span class="d-sm-none">Src</span><span class="d-none d-sm-inline">Source</span>
 				</button>
 				<button class="nav-link" id="nav-plain-text-tab" data-bs-toggle="tab" data-bs-target="#nav-plain-text"
 					type="button" role="tab" aria-controls="nav-plain-text" aria-selected="false"
-					:class="message.HTML == '' ? 'show' : ''" v-on:click=" showMobileBtns = false">Text</button>
+					:class="message.HTML == '' ? 'show' : ''">
+					Text
+				</button>
 				<button class="nav-link" id="nav-headers-tab" data-bs-toggle="tab" data-bs-target="#nav-headers"
-					type="button" role="tab" aria-controls="nav-headers" aria-selected="false"
+					type="button" role="tab" aria-controls="nav-headers" aria-selected="false">
-					v-on:click=" showMobileBtns = false">
 					<span class="d-sm-none">Hdrs</span><span class="d-none d-sm-inline">Headers</span>
 				</button>
 				<button class="nav-link" id="nav-raw-tab" data-bs-toggle="tab" data-bs-target="#nav-raw" type="button"
-					role="tab" aria-controls="nav-raw" aria-selected="false"
+					role="tab" aria-controls="nav-raw" aria-selected="false">
-					v-on:click=" showMobileBtns = false">Raw</button>
+					Raw
+				</button>
 
-				<div class="d-none d-lg-block ms-auto me-2" v-if="showMobileBtns">
+				<div class="d-none d-lg-block ms-auto me-2" v-if="showMobileButtons">
-					<template v-for="  vals, key   in   responsiveSizes  ">
+					<template v-for="vals, key in responsiveSizes">
 						<button class="btn" :disabled="scaleHTMLPreview == key" :title="'Switch to ' + key + ' view'"
-							v-on:click=" scaleHTMLPreview = key">
+							v-on:click="scaleHTMLPreview = key">
 							<i class="bi" :class="'bi-' + key"></i>
 						</button>
 					</template>

server/ui/api/v1/swagger.json

@@ -489,7 +489,7 @@
     },
     "/api/v1/webui": {
       "get": {
-        "description": "Returns configuration settings for the web UI.",
+        "description": "Returns configuration settings for the web UI.\nIntended for web UI only!",
         "produces": [
           "application/json"
         ],
@@ -789,12 +789,6 @@
       "description": "MessagesSummary is a summary of a list of messages",
       "type": "object",
       "properties": {
-        "count": {
-          "description": "Number of results returned",
-          "type": "integer",
-          "format": "int64",
-          "x-go-name": "Count"
-        },
         "messages": {
           "description": "Messages summary\nin:body",
           "type": "array",
@@ -803,6 +797,12 @@
           },
           "x-go-name": "Messages"
         },
+        "messages_count": {
+          "description": "Total number of messages matching current query",
+          "type": "integer",
+          "format": "int64",
+          "x-go-name": "MessagesCount"
+        },
         "start": {
           "description": "Pagination offset",
           "type": "integer",
@@ -926,10 +926,10 @@
   },
   "responses": {
     "BinaryResponse": {
-      "description": "Binary data reponse inherits the attachment's content type"
+      "description": "Binary data response inherits the attachment's content type"
     },
     "ErrorResponse": {
-      "description": "Error reponse"
+      "description": "Error response"
     },
     "InfoResponse": {
       "description": "Application information",
@@ -949,7 +949,7 @@
       }
     },
     "OKResponse": {
-      "description": "Plain text \"ok\" reponse"
+      "description": "Plain text \"ok\" response"
     },
     "TextResponse": {
       "description": "Plain text response"

storage/database.go

@@ -384,9 +384,14 @@ func List(start, limit int) ([]MessageSummary, error) {
 // The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as:
 // is:read, is:unread, has:attachment, to:<term>, from:<term> & subject:<term>
 // Negative searches also also included by prefixing the search term with a `-` or `!`
-func Search(search string, start, limit int) ([]MessageSummary, error) {
+func Search(search string, start, limit int) ([]MessageSummary, int, error) {
 	results := []MessageSummary{}
+	allResults := []MessageSummary{}
 	tsStart := time.Now()
+	nrResults := 0
+	if limit < 0 {
+		limit = 50
+	}
 
 	s := strings.ToLower(search)
 	// add another quote if missing closing quote
@@ -398,11 +403,11 @@ func Search(search string, start, limit int) ([]MessageSummary, error) {
 	p := shellwords.NewParser()
 	args, err := p.Parse(s)
 	if err != nil {
-		return results, errors.New("Your search contains invalid characters")
+		return results, nrResults, errors.New("Your search contains invalid characters")
 	}
 
 	// generate the SQL based on arguments
-	q := searchParser(args, start, limit)
+	q := searchParser(args)
 
 	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
 		var created int64
@@ -440,18 +445,29 @@ func Search(search string, start, limit int) ([]MessageSummary, error) {
 		em.Attachments = attachments
 		em.Read = read == 1
 
-		results = append(results, em)
+		allResults = append(allResults, em)
 	}); err != nil {
-		return results, err
+		return results, nrResults, err
+	}
+
+	dbLastAction = time.Now()
+
+	nrResults = len(allResults)
+
+	if nrResults > start {
+		end := nrResults
+		if nrResults >= start+limit {
+			end = start + limit
+		}
+
+		results = allResults[start:end]
 	}
 
 	elapsed := time.Since(tsStart)
 
 	logger.Log().Debugf("[db] search for \"%s\" in %s", search, elapsed)
 
-	dbLastAction = time.Now()
+	return results, nrResults, err
-
-	return results, err
 }
 
 // GetMessage returns a Message generated from the mailbox_data collection.

storage/database_test.go

@@ -168,9 +168,9 @@ func TestSearch(t *testing.T) {
 
 	for i := 1; i < 51; i++ {
 		// search a random something that will return a single result
-		searchIndx := rand.Intn(4) + 1
+		searchIdx := rand.Intn(4) + 1
 		var search string
-		switch searchIndx {
+		switch searchIdx {
 		case 1:
 			search = fmt.Sprintf("from-%d@example.com", i)
 		case 2:
@@ -181,7 +181,7 @@ func TestSearch(t *testing.T) {
 			search = fmt.Sprintf("\"the email body %d jdsauk dwqmdqw\"", i)
 		}
 
-		summaries, err := Search(search, 0, 100)
+		summaries, _, err := Search(search, 0, 100)
 		if err != nil {
 			t.Log("error ", err)
 			t.Fail()
@@ -196,8 +196,8 @@ func TestSearch(t *testing.T) {
 		assertEqual(t, summaries[0].Subject, fmt.Sprintf("Subject line %d end", i), "\"Subject\" does not match")
 	}
 
-	// search something that will return 200 rsults
+	// search something that will return 200 results
-	summaries, err := Search("This is the email body", 0, testRuns)
+	summaries, _, err := Search("This is the email body", 0, testRuns)
 	if err != nil {
 		t.Log("error ", err)
 		t.Fail()

storage/search.go

@@ -8,25 +8,14 @@ import (
 )
 
 // SearchParser returns the SQL syntax for the database search based on the search arguments
-func searchParser(args []string, start, limit int) *sqlf.Stmt {
+func searchParser(args []string) *sqlf.Stmt {
-	if limit == 0 {
-		limit = 50
-	}
-
 	q := sqlf.From("mailbox").
 		Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags,
 			IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON,
 			IFNULL(json_extract(Metadata, '$.From'), '{}') as FromJSON,
 			IFNULL(json_extract(Metadata, '$.Cc'), '{}') as CcJSON,
 			IFNULL(json_extract(Metadata, '$.Bcc'), '{}') as BccJSON
-		`).
+		`).OrderBy("Created DESC")
-		OrderBy("Created DESC").
-		Limit(limit).
-		Offset(start)
-
-	if limit > 0 {
-		q = q.Limit(limit)
-	}
 
 	for _, w := range args {
 		if cleanString(w) == "" {