Merge branch 'release/v1.6.6'

.github/workflows/release-build.yml

@@ -33,7 +33,7 @@ jobs:
     - run: npm run package
 
     # build the binaries
-    - uses: wangyoucao577/go-release-action@v1.37
+    - uses: wangyoucao577/go-release-action@v1.38
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
         goos: ${{ matrix.goos }}

CHANGELOG.md

@@ -2,6 +2,26 @@
 
 Notable changes to Mailpit will be documented in this file.
 
+## [v1.6.6]
+
+### API
+- Set Access-Control-Allow-Headers when --api-cors is set
+- Include correct start value in search reponse
+
+### Feature
+- Option to ignore duplicate Message-IDs
+
+### Libs
+- Update node modules
+- Update Go modules
+
+### Swagger
+- Update swagger field descriptions
+
+### UI
+- Style Undisclosed recipients in message view
+
+
 ## [v1.6.5]
 
 ### Feature

README.md

@@ -72,20 +72,14 @@ See [Docker instructions](https://github.com/axllent/mailpit/wiki/Docker-images)
 To build Mailpit from source see [building from source](https://github.com/axllent/mailpit/wiki/Building-from-source).
 
 
+### Testing Mailpit
+
+Please refer to [the documentation](https://github.com/axllent/mailpit/wiki/Testing-Mailpit) of how to easily test email delivery to Mailpit.
+
+
 ### Configuring sendmail
 
-There are several different options available:
+Mailpit's SMTP server (by default on port 1025), so you will likely need to configure your sending application to deliver mail via that port. A common MTA (Mail Transfer Agent) that delivers system emails to a SMTP server is `sendmail`, used by many applications including PHP. Mailpit can also act as substitute for sendmail. For instructions of how to set this up, please refer to the [sendmail documentation](https://github.com/axllent/mailpit/wiki/Configuring-sendmail).
-
-You can use `mailpit sendmail` as your sendmail configuration in `php.ini`:
-```
-sendmail_path = /usr/local/bin/mailpit sendmail
-```
-
-If Mailpit is found on the same host as sendmail, you can symlink the Mailpit binary to sendmail, eg: `ln -s /usr/local/bin/mailpit /usr/sbin/sendmail`  (only if Mailpit is running on default 1025 port).
-
-You can use your default system `sendmail` binary to route directly to port `1025` (configurable) by calling `/usr/sbin/sendmail -S localhost:1025`.
-
-You can build a Mailpit-specific sendmail binary from source (see [building from source](https://github.com/axllent/mailpit/wiki/Building-from-source)).
 
 
 ## Why rewrite MailHog?

cmd/root.go

@@ -87,6 +87,7 @@ func init() {
 	rootCmd.Flags().StringVar(&config.Webroot, "webroot", config.Webroot, "Set the webroot for web UI & API")
 	rootCmd.Flags().StringVar(&server.AccessControlAllowOrigin, "api-cors", server.AccessControlAllowOrigin, "Set API CORS Access-Control-Allow-Origin header")
 	rootCmd.Flags().BoolVar(&config.UseMessageDates, "use-message-dates", config.UseMessageDates, "Use message dates as the received dates")
+	rootCmd.Flags().BoolVar(&config.IgnoreDuplicateIDs, "ignore-duplicate-ids", config.IgnoreDuplicateIDs, "Ignore duplicate messages (by Message-Id)")
 
 	rootCmd.Flags().StringVar(&config.UIAuthFile, "ui-auth-file", config.UIAuthFile, "A password file for web UI authentication")
 	rootCmd.Flags().StringVar(&config.UITLSCert, "ui-tls-cert", config.UITLSCert, "TLS certificate for web UI (HTTPS) - requires ui-tls-key")
@@ -97,11 +98,11 @@ func init() {
 	rootCmd.Flags().StringVar(&config.SMTPTLSCert, "smtp-tls-cert", config.SMTPTLSCert, "TLS certificate for SMTP (STARTTLS) - requires smtp-tls-key")
 	rootCmd.Flags().StringVar(&config.SMTPTLSKey, "smtp-tls-key", config.SMTPTLSKey, "TLS key for SMTP (STARTTLS) - requires smtp-tls-cert")
 	rootCmd.Flags().BoolVar(&config.SMTPAuthAllowInsecure, "smtp-auth-allow-insecure", config.SMTPAuthAllowInsecure, "Enable insecure PLAIN & LOGIN authentication")
-	rootCmd.Flags().StringVarP(&config.SMTPCLITags, "tag", "t", config.SMTPCLITags, "Tag new messages matching filters")
 
 	rootCmd.Flags().StringVar(&config.SMTPRelayConfigFile, "smtp-relay-config", config.SMTPRelayConfigFile, "SMTP configuration file to allow releasing messages")
 	rootCmd.Flags().BoolVar(&config.SMTPRelayAllIncoming, "smtp-relay-all", config.SMTPRelayAllIncoming, "Relay all incoming messages via external SMTP server (caution!)")
 
+	rootCmd.Flags().StringVarP(&config.SMTPCLITags, "tag", "t", config.SMTPCLITags, "Tag new messages matching filters")
 	rootCmd.Flags().BoolVarP(&logger.QuietLogging, "quiet", "q", logger.QuietLogging, "Quiet logging (errors only)")
 	rootCmd.Flags().BoolVarP(&logger.VerboseLogging, "verbose", "v", logger.VerboseLogging, "Verbose logging")
 
@@ -201,8 +202,8 @@ func initConfigFromEnv() {
 	if getEnabledFromEnv("MP_USE_MESSAGE_DATES") {
 		config.UseMessageDates = true
 	}
-	if getEnabledFromEnv("MP_USE_MESSAGE_DATES") {
+	if getEnabledFromEnv("MP_IGNORE_DUPLICATE_IDS") {
-		config.UseMessageDates = true
+		config.IgnoreDuplicateIDs = true
 	}
 	if getEnabledFromEnv("MP_QUIET") {
 		logger.QuietLogging = true

config/config.go

@@ -65,6 +65,9 @@ var (
 	// SMTPAuthAcceptAny accepts any username/password including none
 	SMTPAuthAcceptAny bool
 
+	// IgnoreDuplicateIDs will skip messages with the same ID
+	IgnoreDuplicateIDs bool
+
 	// SMTPCLITags is used to map the CLI args
 	SMTPCLITags string
 

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/tg123/go-htpasswd v1.2.1
 	golang.org/x/text v0.9.0
 	gopkg.in/yaml.v3 v3.0.1
-	modernc.org/sqlite v1.21.2
+	modernc.org/sqlite v1.22.1
 )
 
 require (
@@ -54,7 +54,7 @@ require (
 	lukechampine.com/uint128 v1.3.0 // indirect
 	modernc.org/cc/v3 v3.40.0 // indirect
 	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.22.4 // indirect
+	modernc.org/libc v1.22.5 // indirect
 	modernc.org/mathutil v1.5.0 // indirect
 	modernc.org/memory v1.5.0 // indirect
 	modernc.org/opt v0.1.3 // indirect

go.sum

@@ -196,6 +196,8 @@ modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
 modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
 modernc.org/libc v1.22.4 h1:wymSbZb0AlrjdAVX3cjreCHTPCpPARbQXNz6BHPzdwQ=
 modernc.org/libc v1.22.4/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
+modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
+modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
 modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
 modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
 modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
@@ -204,9 +206,13 @@ modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
 modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
 modernc.org/sqlite v1.21.2 h1:ixuUG0QS413Vfzyx6FWx6PYTmHaOegTY+hjzhn7L+a0=
 modernc.org/sqlite v1.21.2/go.mod h1:cxbLkB5WS32DnQqeH4h4o1B0eMr8W/y8/RGuxQ3JsC0=
+modernc.org/sqlite v1.22.1 h1:P2+Dhp5FR1RlVRkQ3dDfCiv3Ok8XPxqpe70IjYVA9oE=
+modernc.org/sqlite v1.22.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
 modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
 modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
 modernc.org/tcl v1.15.1 h1:mOQwiEK4p7HruMZcwKTZPw/aqtGM4aY00uzWhlKKYws=
+modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY=
 modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
 modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
 modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE=
+modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY=

server/apiv1/api.go

@@ -34,13 +34,13 @@ func GetMessages(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: start
 	//	    in: query
-	//	    description: pagination offset
+	//	    description: Pagination offset
 	//	    required: false
 	//	    type: integer
 	//	    default: 0
 	//	  + name: limit
 	//	    in: query
-	//	    description: limit results
+	//	    description: Limit results
 	//	    required: false
 	//	    type: integer
 	//	    default: 50
@@ -88,12 +88,12 @@ func Search(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: query
 	//	    in: query
-	//	    description: search query
+	//	    description: Search query
 	//	    required: true
 	//	    type: string
 	//	  + name: limit
 	//	    in: query
-	//	    description: limit results
+	//	    description: Limit results
 	//	    required: false
 	//	    type: integer
 	//	    default: 50
@@ -119,7 +119,7 @@ func Search(w http.ResponseWriter, r *http.Request) {
 
 	var res MessagesSummary
 
-	res.Start = 0
+	res.Start = start
 	res.Messages = messages
 	res.Count = len(messages)
 	res.Total = stats.Total
@@ -147,7 +147,7 @@ func GetMessage(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ID
 	//	    in: path
-	//	    description: message id
+	//	    description: Message ID
 	//	    required: true
 	//	    type: string
 	//
@@ -188,12 +188,12 @@ func DownloadAttachment(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ID
 	//	    in: path
-	//	    description: message id
+	//	    description: Message ID
 	//	    required: true
 	//	    type: string
 	//	  + name: PartID
 	//	    in: path
-	//	    description: attachment part id
+	//	    description: Attachment part ID
 	//	    required: true
 	//	    type: string
 	//
@@ -237,7 +237,7 @@ func GetHeaders(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ID
 	//	    in: path
-	//	    description: message id
+	//	    description: Message ID
 	//	    required: true
 	//	    type: string
 	//
@@ -284,7 +284,7 @@ func DownloadRaw(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ID
 	//	    in: path
-	//	    description: message id
+	//	    description: Message ID
 	//	    required: true
 	//	    type: string
 	//
@@ -330,7 +330,7 @@ func DeleteMessages(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ids
 	//	    in: body
-	//	    description: Message ids to delete
+	//	    description: Message IDs to delete
 	//	    required: false
 	//	    type: DeleteRequest
 	//
@@ -381,7 +381,7 @@ func SetReadStatus(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ids
 	//	    in: body
-	//	    description: Message ids to update
+	//	    description: Message IDs to update
 	//	    required: false
 	//	    type: SetReadStatusRequest
 	//
@@ -459,7 +459,7 @@ func SetTags(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ids
 	//	    in: body
-	//	    description: Message ids to update
+	//	    description: Message IDs to update
 	//	    required: true
 	//	    type: SetTagsRequest
 	//
@@ -515,10 +515,10 @@ func ReleaseMessage(w http.ResponseWriter, r *http.Request) {
 	//	Parameters:
 	//	  + name: ID
 	//	    in: path
-	//	    description: message id
+	//	    description: Message ID
 	//	    required: true
 	//	    type: string
-	//		+ name: To
+	//		+ name: to
 	//	    in: body
 	//	    description: Array of email addresses to release message to
 	//	    required: true

server/server.go

@@ -123,6 +123,7 @@ func middleWareFunc(fn http.HandlerFunc) http.HandlerFunc {
 		if AccessControlAllowOrigin != "" && strings.HasPrefix(r.RequestURI, config.Webroot+"api/") {
 			w.Header().Set("Access-Control-Allow-Origin", AccessControlAllowOrigin)
 			w.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, PUT")
+			w.Header().Set("Access-Control-Allow-Headers", "*")
 		}
 
 		if config.UIAuthFile != "" {
@@ -161,6 +162,7 @@ func middlewareHandler(h http.Handler) http.Handler {
 		if AccessControlAllowOrigin != "" && strings.HasPrefix(r.RequestURI, config.Webroot+"api/") {
 			w.Header().Set("Access-Control-Allow-Origin", AccessControlAllowOrigin)
 			w.Header().Set("Access-Control-Allow-Methods", "GET, POST, DELETE, PUT")
+			w.Header().Set("Access-Control-Allow-Headers", "*")
 		}
 
 		if config.UIAuthFile != "" {

server/smtpd/smtpd.go

@@ -24,12 +24,19 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
 		return err
 	}
 
+	messageID := strings.Trim(msg.Header.Get("Message-Id"), "<>")
+
 	// add a message ID if not set
-	if msg.Header.Get("Message-Id") == "" {
+	if messageID == "" {
 		// generate unique ID
-		uid := uuid.NewV4().String() + "@mailpit"
+		messageID = uuid.NewV4().String() + "@mailpit"
 		// add unique ID
-		data = append([]byte("Message-Id: <"+uid+">\r\n"), data...)
+		data = append([]byte("Message-Id: <"+messageID+">\r\n"), data...)
+	} else if config.IgnoreDuplicateIDs {
+		if storage.MessageIDExists(messageID) {
+			logger.Log().Debugf("[smtpd] duplicate message found, ignoring %s", messageID)
+			return nil
+		}
 	}
 
 	// if enabled, this will route the email 1:1 through to the preconfigured smtp server
@@ -81,7 +88,8 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
 		logger.Log().Debugf("[smtpd] added missing addresses to Bcc header: %s", strings.Join(missingAddresses, ", "))
 	}
 
-	if _, err := storage.Store(data); err != nil {
+	_, err = storage.Store(data)
+	if err != nil {
 		logger.Log().Errorf("[db] error storing message: %d", err.Error())
 
 		return err

server/ui-src/templates/Message.vue

@@ -195,7 +195,7 @@ export default {
 									<template v-if="i > 0">, </template>
 									<span class="text-nowrap">{{ t.Name + " &lt;" + t.Address + "&gt;" }}</span>
 								</span>
-								<span v-else>Undisclosed recipients</span>
+								<span v-else class="text-muted">[Undisclosed recipients]</span>
 							</td>
 						</tr>
 						<tr v-if="message.Cc && message.Cc.length" class="small">
@@ -272,28 +272,28 @@ export default {
 					role="tab" aria-controls="nav-html" aria-selected="true" v-if="message.HTML"
 					v-on:click="showMobileBtns = true; resizeIframes()">HTML</button>
 				<button class="nav-link" id="nav-html-source-tab" data-bs-toggle="tab" data-bs-target="#nav-html-source"
-					type="button" role="tab" aria-controls="nav-html-source" aria-selected="false" v-if="message.HTML"
+					type="button" role="tab" aria-controls="nav-html-source" aria-selected="false" v-if=" message.HTML "
-					v-on:click="showMobileBtns = false">
+					v-on:click=" showMobileBtns = false ">
 					HTML <span class="d-sm-none">Src</span><span class="d-none d-sm-inline">Source</span>
 				</button>
 				<button class="nav-link" id="nav-plain-text-tab" data-bs-toggle="tab" data-bs-target="#nav-plain-text"
 					type="button" role="tab" aria-controls="nav-plain-text" aria-selected="false"
-					:class="message.HTML == '' ? 'show' : ''" v-on:click="showMobileBtns = false">Text</button>
+					:class=" message.HTML == '' ? 'show' : '' " v-on:click=" showMobileBtns = false ">Text</button>
 				<button class="nav-link" id="nav-headers-tab" data-bs-toggle="tab" data-bs-target="#nav-headers"
 					type="button" role="tab" aria-controls="nav-headers" aria-selected="false"
-					v-on:click="showMobileBtns = false">
+					v-on:click=" showMobileBtns = false ">
 					<span class="d-sm-none">Hdrs</span><span class="d-none d-sm-inline">Headers</span>
 				</button>
 				<button class="nav-link" id="nav-raw-tab" data-bs-toggle="tab" data-bs-target="#nav-raw" type="button"
 					role="tab" aria-controls="nav-raw" aria-selected="false"
-					v-on:click="showMobileBtns = false">Raw</button>
+					v-on:click=" showMobileBtns = false ">Raw</button>
 
-				<div class="d-none d-lg-block ms-auto me-2" v-if="showMobileBtns">
+				<div class="d-none d-lg-block ms-auto me-2" v-if=" showMobileBtns ">
-					<template v-for="vals, key in responsiveSizes">
+					<template v-for="  vals, key   in   responsiveSizes  ">
-						<button class="btn" :class="scaleHTMLPreview == key ? 'btn-outline-primary' : ''"
+						<button class="btn" :class=" scaleHTMLPreview == key ? 'btn-outline-primary' : '' "
-							:disabled="scaleHTMLPreview == key" :title="'Switch to ' + key + ' view'"
+							:disabled=" scaleHTMLPreview == key " :title=" 'Switch to ' + key + ' view' "
-							v-on:click="scaleHTMLPreview = key">
+							v-on:click=" scaleHTMLPreview = key ">
-							<i class="bi" :class="'bi-' + key"></i>
+							<i class="bi" :class=" 'bi-' + key "></i>
 						</button>
 					</template>
 				</div>
@@ -301,31 +301,31 @@ export default {
 		</nav>
 
 		<div class="tab-content mb-5" id="nav-tabContent">
-			<div v-if="message.HTML != ''" class="tab-pane fade show" id="nav-html" role="tabpanel"
+			<div v-if=" message.HTML != '' " class="tab-pane fade show" id="nav-html" role="tabpanel"
 				aria-labelledby="nav-html-tab" tabindex="0">
-				<div id="responsive-view" :class="scaleHTMLPreview" :style="responsiveSizes[scaleHTMLPreview]">
+				<div id="responsive-view" :class=" scaleHTMLPreview " :style=" responsiveSizes[scaleHTMLPreview] ">
-					<iframe target-blank="" class="tab-pane d-block" id="preview-html" :srcdoc="message.HTML"
+					<iframe target-blank="" class="tab-pane d-block" id="preview-html" :srcdoc=" message.HTML "
-						v-on:load="resizeIframe" seamless frameborder="0" style="width: 100%; height: 100%;">
+						v-on:load=" resizeIframe " seamless frameborder="0" style="width: 100%; height: 100%;">
 					</iframe>
 				</div>
-				<Attachments v-if="allAttachments(message).length" :message="message"
+				<Attachments v-if=" allAttachments(message).length " :message=" message "
-					:attachments="allAttachments(message)"></Attachments>
+					:attachments=" allAttachments(message) "></Attachments>
 			</div>
 			<div class="tab-pane fade" id="nav-html-source" role="tabpanel" aria-labelledby="nav-html-source-tab"
-				tabindex="0" v-if="message.HTML">
+				tabindex="0" v-if=" message.HTML ">
 				<pre><code class="language-html">{{ message.HTML }}</code></pre>
 			</div>
 			<div class="tab-pane fade" id="nav-plain-text" role="tabpanel" aria-labelledby="nav-plain-text-tab" tabindex="0"
-				:class="message.HTML == '' ? 'show' : ''">
+				:class=" message.HTML == '' ? 'show' : '' ">
 				<div class="text-view">{{ message.Text }}</div>
-				<Attachments v-if="allAttachments(message).length" :message="message"
+				<Attachments v-if=" allAttachments(message).length " :message=" message "
-					:attachments="allAttachments(message)"></Attachments>
+					:attachments=" allAttachments(message) "></Attachments>
 			</div>
 			<div class="tab-pane fade" id="nav-headers" role="tabpanel" aria-labelledby="nav-headers-tab" tabindex="0">
-				<Headers v-if="loadHeaders" :message="message"></Headers>
+				<Headers v-if=" loadHeaders " :message=" message "></Headers>
 			</div>
 			<div class="tab-pane fade" id="nav-raw" role="tabpanel" aria-labelledby="nav-raw-tab" tabindex="0">
-				<iframe v-if="srcURI" :src="srcURI" v-on:load="resizeIframe" seamless frameborder="0"
+				<iframe v-if=" srcURI " :src=" srcURI " v-on:load=" resizeIframe " seamless frameborder="0"
 					style="width: 100%; height: 300px;" id="message-src"></iframe>
 			</div>
 		</div>

server/ui/api/v1/swagger.json

@@ -66,7 +66,7 @@
         "parameters": [
           {
             "type": "string",
-            "description": "message id",
+            "description": "Message ID",
             "name": "ID",
             "in": "path",
             "required": true
@@ -103,7 +103,7 @@
         "parameters": [
           {
             "type": "string",
-            "description": "message id",
+            "description": "Message ID",
             "name": "ID",
             "in": "path",
             "required": true
@@ -142,14 +142,14 @@
         "parameters": [
           {
             "type": "string",
-            "description": "message id",
+            "description": "Message ID",
             "name": "ID",
             "in": "path",
             "required": true
           },
           {
             "type": "string",
-            "description": "attachment part id",
+            "description": "Attachment part ID",
             "name": "PartID",
             "in": "path",
             "required": true
@@ -224,7 +224,7 @@
         "parameters": [
           {
             "type": "string",
-            "description": "message id",
+            "description": "Message ID",
             "name": "ID",
             "in": "path",
             "required": true
@@ -261,14 +261,14 @@
         "parameters": [
           {
             "type": "string",
-            "description": "message id",
+            "description": "Message ID",
             "name": "ID",
             "in": "path",
             "required": true
           },
           {
             "description": "Array of email addresses to release message to",
-            "name": "To",
+            "name": "to",
             "in": "body",
             "required": true,
             "schema": {
@@ -307,14 +307,14 @@
           {
             "type": "integer",
             "default": 0,
-            "description": "pagination offset",
+            "description": "Pagination offset",
             "name": "start",
             "in": "query"
           },
           {
             "type": "integer",
             "default": 50,
-            "description": "limit results",
+            "description": "Limit results",
             "name": "limit",
             "in": "query"
           }
@@ -347,11 +347,11 @@
         "operationId": "SetReadStatus",
         "parameters": [
           {
-            "description": "Message ids to update",
+            "description": "Message IDs to update",
             "name": "ids",
             "in": "body",
             "schema": {
-              "description": "Message ids to update",
+              "description": "Message IDs to update",
               "type": "object",
               "$ref": "#/definitions/SetReadStatusRequest"
             }
@@ -385,11 +385,11 @@
         "operationId": "Delete",
         "parameters": [
           {
-            "description": "Message ids to delete",
+            "description": "Message IDs to delete",
             "name": "ids",
             "in": "body",
             "schema": {
-              "description": "Message ids to delete",
+              "description": "Message IDs to delete",
               "type": "object",
               "$ref": "#/definitions/DeleteRequest"
             }
@@ -423,7 +423,7 @@
         "parameters": [
           {
             "type": "string",
-            "description": "search query",
+            "description": "Search query",
             "name": "query",
             "in": "query",
             "required": true
@@ -431,7 +431,7 @@
           {
             "type": "integer",
             "default": 50,
-            "description": "limit results",
+            "description": "Limit results",
             "name": "limit",
             "in": "query"
           }
@@ -466,12 +466,12 @@
         "operationId": "SetTags",
         "parameters": [
           {
-            "description": "Message ids to update",
+            "description": "Message IDs to update",
             "name": "ids",
             "in": "body",
             "required": true,
             "schema": {
-              "description": "Message ids to update",
+              "description": "Message IDs to update",
               "type": "object",
               "$ref": "#/definitions/SetTagsRequest"
             }
@@ -568,23 +568,23 @@
       "type": "object",
       "properties": {
         "ContentID": {
-          "description": "content id",
+          "description": "Content ID",
           "type": "string"
         },
         "ContentType": {
-          "description": "content type",
+          "description": "Content type",
           "type": "string"
         },
         "FileName": {
-          "description": "file name",
+          "description": "File name",
           "type": "string"
         },
         "PartID": {
-          "description": "attachment part id",
+          "description": "Attachment part ID",
           "type": "string"
         },
         "Size": {
-          "description": "size in bytes",
+          "description": "Size in bytes",
           "type": "integer",
           "format": "int64"
         }
@@ -645,7 +645,7 @@
           "type": "string"
         },
         "ID": {
-          "description": "Unique message database id",
+          "description": "Database ID",
           "type": "string"
         },
         "Inline": {
@@ -655,6 +655,10 @@
             "$ref": "#/definitions/Attachment"
           }
         },
+        "MessageID": {
+          "description": "Message ID",
+          "type": "string"
+        },
         "Read": {
           "description": "Read status",
           "type": "boolean"
@@ -667,7 +671,7 @@
           }
         },
         "ReturnPath": {
-          "description": "ReturnPath is the Return-Path",
+          "description": "Return-Path",
           "type": "string"
         },
         "Size": {
@@ -744,7 +748,7 @@
           "$ref": "#/definitions/Address"
         },
         "ID": {
-          "description": "Unique message database id",
+          "description": "Database ID",
           "type": "string"
         },
         "Read": {

storage/database.go

@@ -72,20 +72,55 @@ var (
 			Script: `ALTER TABLE mailbox ADD COLUMN Tags Text  NOT NULL DEFAULT '[]';
 			CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
 		},
+		{
+			Version:     1.2,
+			Description: "Creating new mailbox format",
+			Script: `CREATE TABLE IF NOT EXISTS mailboxtmp (
+				Created INTEGER NOT NULL,
+				ID TEXT NOT NULL,
+				MessageID TEXT NOT NULL,
+				Subject TEXT NOT NULL,
+				Metadata TEXT,
+				Size INTEGER NOT NULL,
+				Inline INTEGER NOT NULL,
+				Attachments INTEGER NOT NULL,
+				Read INTEGER,
+				Tags TEXT,
+				SearchText TEXT
+			);
+			INSERT INTO mailboxtmp 
+				(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Read, Tags) 
+			SELECT 
+				Sort, ID, '', json_extract(Data, '$.Subject'),Data, 
+				json_extract(Data, '$.Size'), json_extract(Data, '$.Inline'), json_extract(Data, '$.Attachments'), 
+				Search, Read, Tags
+			FROM mailbox;
+
+			DROP TABLE IF EXISTS mailbox;
+			ALTER TABLE mailboxtmp RENAME TO mailbox;
+			CREATE INDEX IF NOT EXISTS idx_created ON mailbox (Created);
+			CREATE UNIQUE INDEX IF NOT EXISTS idx_id ON mailbox (ID);
+			CREATE INDEX IF NOT EXISTS idx_message_id ON mailbox (MessageID);
+			CREATE INDEX IF NOT EXISTS idx_subject ON mailbox (Subject);
+			CREATE INDEX IF NOT EXISTS idx_size ON mailbox (Size);
+			CREATE INDEX IF NOT EXISTS idx_inline ON mailbox (Inline);
+			CREATE INDEX IF NOT EXISTS idx_attachments ON mailbox (Attachments);
+			CREATE INDEX IF NOT EXISTS idx_read ON mailbox (Read);
+			CREATE INDEX IF NOT EXISTS idx_tags ON mailbox (Tags);`,
+		},
 	}
 )
 
 // DBMailSummary struct for storing mail summary
 type DBMailSummary struct {
-	Created     time.Time
+	From *mail.Address
-	From        *mail.Address
+	To   []*mail.Address
-	To          []*mail.Address
+	Cc   []*mail.Address
-	Cc          []*mail.Address
+	Bcc  []*mail.Address
-	Bcc         []*mail.Address
+	// Subject     string
-	Subject     string
+	// Size        int
-	Size        int
+	// Inline      int
-	Inline      int
+	// Attachments int
-	Attachments int
 }
 
 // InitDB will initialise the database
@@ -144,6 +179,8 @@ func InitDB() error {
 	// auto-prune & delete
 	go dbCron()
 
+	go dataMigrations()
+
 	return nil
 }
 
@@ -189,22 +226,21 @@ func Store(body []byte) (string, error) {
 		from = &mail.Address{Name: env.GetHeader("From")}
 	}
 
+	messageID := strings.Trim(env.Root.Header.Get("Message-ID"), "<>")
+
 	obj := DBMailSummary{
-		Created:     time.Now(),
+		From: from,
-		From:        from,
+		To:   addressToSlice(env, "To"),
-		To:          addressToSlice(env, "To"),
+		Cc:   addressToSlice(env, "Cc"),
-		Cc:          addressToSlice(env, "Cc"),
+		Bcc:  addressToSlice(env, "Bcc"),
-		Bcc:         addressToSlice(env, "Bcc"),
-		Subject:     env.GetHeader("Subject"),
-		Size:        len(body),
-		Inline:      len(env.Inlines),
-		Attachments: len(env.Attachments),
 	}
 
+	created := time.Now()
+
 	// use message date instead of created date
 	if config.UseMessageDates {
 		if mDate, err := env.Date(); err == nil {
-			obj.Created = mDate
+			created = mDate
 		}
 	}
 
@@ -237,8 +273,14 @@ func Store(body []byte) (string, error) {
 	// roll back if it fails
 	defer tx.Rollback()
 
+	subject := env.GetHeader("Subject")
+	size := len(body)
+	inline := len(env.Inlines)
+	attachments := len(env.Attachments)
+
 	// insert mail summary data
-	_, err = tx.Exec("INSERT INTO mailbox(ID, Data, Search, Tags, Read) values(?,?,?,?,0)", id, string(summaryJSON), searchText, string(tagJSON))
+	_, err = tx.Exec("INSERT INTO mailbox(Created, ID, MessageID, Subject, Metadata, Size, Inline, Attachments, SearchText, Tags, Read) values(?,?,?,?,?,?,?,?,?,?,0)",
+		created.UnixMilli(), id, messageID, subject, string(summaryJSON), size, inline, attachments, searchText, string(tagJSON))
 	if err != nil {
 		return "", err
 	}
@@ -259,9 +301,12 @@ func Store(body []byte) (string, error) {
 		return "", err
 	}
 
-	c.Tags = tagData
+	c.Created = created
-
 	c.ID = id
+	c.Attachments = attachments
+	c.Subject = subject
+	c.Size = size
+	c.Tags = tagData
 
 	websockets.Broadcast("new", c)
 
@@ -276,24 +321,28 @@ func List(start, limit int) ([]MessageSummary, error) {
 	results := []MessageSummary{}
 
 	q := sqlf.From("mailbox").
-		Select(`ID, Data, Tags, Read`).
+		Select(`Created, ID, Subject, Metadata, Size, Attachments, Read, Tags`).
-		OrderBy("Sort DESC").
+		OrderBy("Created DESC").
 		Limit(limit).
 		Offset(start)
 
 	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+		var created int64
 		var id string
-		var summary string
+		var subject string
+		var metadata string
+		var size int
+		var attachments int
 		var tags string
 		var read int
 		em := MessageSummary{}
 
-		if err := row.Scan(&id, &summary, &tags, &read); err != nil {
+		if err := row.Scan(&created, &id, &subject, &metadata, &size, &attachments, &read, &tags); err != nil {
 			logger.Log().Error(err)
 			return
 		}
 
-		if err := json.Unmarshal([]byte(summary), &em); err != nil {
+		if err := json.Unmarshal([]byte(metadata), &em); err != nil {
 			logger.Log().Error(err)
 			return
 		}
@@ -303,11 +352,17 @@ func List(start, limit int) ([]MessageSummary, error) {
 			return
 		}
 
+		em.Created = time.UnixMilli(created)
 		em.ID = id
+		em.Subject = subject
+		em.Size = size
+		em.Attachments = attachments
 		em.Read = read == 1
 
 		results = append(results, em)
 
+		// logger.PrettyPrint(em)
+
 	}); err != nil {
 		return results, err
 	}
@@ -342,19 +397,23 @@ func Search(search string, start, limit int) ([]MessageSummary, error) {
 	q := searchParser(args, start, limit)
 
 	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+		var created int64
 		var id string
-		var summary string
+		var subject string
+		var metadata string
+		var size int
+		var attachments int
 		var tags string
 		var read int
 		var ignore string
 		em := MessageSummary{}
 
-		if err := row.Scan(&id, &summary, &tags, &read, &ignore, &ignore, &ignore, &ignore, &ignore, &ignore); err != nil {
+		if err := row.Scan(&created, &id, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil {
 			logger.Log().Error(err)
 			return
 		}
 
-		if err := json.Unmarshal([]byte(summary), &em); err != nil {
+		if err := json.Unmarshal([]byte(metadata), &em); err != nil {
 			logger.Log().Error(err)
 			return
 		}
@@ -364,7 +423,11 @@ func Search(search string, start, limit int) ([]MessageSummary, error) {
 			return
 		}
 
+		em.Created = time.UnixMilli(created)
 		em.ID = id
+		em.Subject = subject
+		em.Size = size
+		em.Attachments = attachments
 		em.Read = read == 1
 
 		results = append(results, em)
@@ -404,6 +467,8 @@ func GetMessage(id string) (*Message, error) {
 		from = &mail.Address{Name: env.GetHeader("From")}
 	}
 
+	messageID := strings.Trim(env.GetHeader("Message-ID"), "<>")
+
 	returnPath := strings.Trim(env.GetHeader("Return-Path"), "<>")
 	if returnPath == "" {
 		returnPath = from.Address
@@ -413,27 +478,20 @@ func GetMessage(id string) (*Message, error) {
 	if err != nil {
 		// return received datetime when message does not contain a date header
 		q := sqlf.From("mailbox").
-			Select(`Data`).
+			Select(`Created`).
-			OrderBy("Sort DESC").
 			Where(`ID = ?`, id)
 
 		if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
-			var summary string
+			var created int64
-			em := MessageSummary{}
 
-			if err := row.Scan(&summary); err != nil {
+			if err := row.Scan(&created); err != nil {
-				logger.Log().Error(err)
-				return
-			}
-
-			if err := json.Unmarshal([]byte(summary), &em); err != nil {
 				logger.Log().Error(err)
 				return
 			}
 
 			logger.Log().Debugf("[db] %s does not contain a date header, using received datetime", id)
 
-			date = em.Created
+			date = time.UnixMicro(created)
 		}); err != nil {
 			logger.Log().Error(err)
 		}
@@ -441,6 +499,7 @@ func GetMessage(id string) (*Message, error) {
 
 	obj := Message{
 		ID:         id,
+		MessageID:  messageID,
 		Read:       true,
 		From:       from,
 		Date:       date,
@@ -821,3 +880,16 @@ func IsUnread(id string) bool {
 
 	return unread == 1
 }
+
+// MessageIDExists blaah
+func MessageIDExists(id string) bool {
+	var total int
+
+	q := sqlf.From("mailbox").
+		Select("COUNT(*)").To(&total).
+		Where("MessageID = ?", id)
+
+	_ = q.QueryRowAndClose(nil, db)
+
+	return total != 0
+}

storage/migrationTasks.go

@@ -0,0 +1,200 @@
+package storage
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"strings"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/utils/logger"
+	"github.com/jhillyerd/enmime"
+	"github.com/leporo/sqlf"
+	"golang.org/x/text/language"
+	"golang.org/x/text/message"
+)
+
+func dataMigrations() {
+	updateSortByCreatedTask()
+	assignMessageIDsTask()
+}
+
+// Update Sort column using Created datetime <= v1.6.5
+// Migration task implemented 05/2023 - can be removed end 2023
+func updateSortByCreatedTask() {
+	q := sqlf.From("mailbox").
+		Select("ID").
+		Select(`json_extract(Metadata, '$.Created') as Created`).
+		Where("Created < ?", 1155000600)
+
+	toUpdate := make(map[string]int64)
+	p := message.NewPrinter(language.English)
+
+	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+		var id string
+		var ts sql.NullString
+		if err := row.Scan(&id, &ts); err != nil {
+			logger.Log().Error("[migration]", err)
+			return
+		}
+
+		if !ts.Valid {
+			logger.Log().Errorf("[migration] cannot get Created timestamp from %s", id)
+			return
+		}
+
+		t, _ := time.Parse(time.RFC3339Nano, ts.String)
+		toUpdate[id] = t.UnixMilli()
+	}); err != nil {
+		logger.Log().Error("[migration]", err)
+		return
+	}
+
+	total := len(toUpdate)
+
+	if total == 0 {
+		return
+	}
+
+	logger.Log().Infof("[migration] updating timestamp for %s messages", p.Sprintf("%d", len(toUpdate)))
+
+	// begin a transaction
+	ctx := context.Background()
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		logger.Log().Error("[migration]", err)
+		return
+	}
+
+	// roll back if it fails
+	defer tx.Rollback()
+
+	var blockTime = time.Now()
+
+	count := 0
+	for id, ts := range toUpdate {
+		count++
+		_, err := tx.Exec(`UPDATE mailbox SET Created = ? WHERE ID = ?`, ts, id)
+		if err != nil {
+			logger.Log().Error("[migration]", err)
+		}
+
+		if count%1000 == 0 {
+			percent := (100 * count) / total
+			logger.Log().Infof("[migration] updated timestamp for 1,000 messages [%d%%] in %s", percent, time.Since(blockTime))
+			blockTime = time.Now()
+		}
+	}
+
+	logger.Log().Infof("[migration] commit %s changes", p.Sprintf("%d", count))
+
+	if err := tx.Commit(); err != nil {
+		logger.Log().Error("[migration]", err)
+		return
+	}
+
+	logger.Log().Infof("[migration] complete")
+}
+
+// Find any messages without a stored Message-ID and update it <= v1.6.5
+// Migration task implemented 05/2023 - can be removed end 2023
+func assignMessageIDsTask() {
+	if !config.IgnoreDuplicateIDs {
+		return
+	}
+
+	q := sqlf.From("mailbox").
+		Select("ID").
+		Where("MessageID = ''")
+
+	missingIDS := make(map[string]string)
+
+	if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
+		var id string
+		if err := row.Scan(&id); err != nil {
+			logger.Log().Error("[migration]", err)
+			return
+		}
+		missingIDS[id] = ""
+	}); err != nil {
+		logger.Log().Error("[migration]", err)
+	}
+
+	if len(missingIDS) == 0 {
+		return
+	}
+
+	var count int
+	var blockTime = time.Now()
+	p := message.NewPrinter(language.English)
+
+	total := len(missingIDS)
+
+	logger.Log().Infof("[migration] extracting Message-IDs for %s messages", p.Sprintf("%d", total))
+
+	for id := range missingIDS {
+		raw, err := GetMessageRaw(id)
+		if err != nil {
+			logger.Log().Error("[migration]", err)
+			continue
+		}
+
+		r := bytes.NewReader(raw)
+
+		env, err := enmime.ReadEnvelope(r)
+		if err != nil {
+			logger.Log().Error("[migration]", err)
+			continue
+		}
+
+		messageID := strings.Trim(env.GetHeader("Message-ID"), "<>")
+
+		missingIDS[id] = messageID
+
+		count++
+
+		if count%1000 == 0 {
+			percent := (100 * count) / total
+			logger.Log().Infof("[migration] extracted 1,000 Message-IDs [%d%%] in %s", percent, time.Since(blockTime))
+			blockTime = time.Now()
+		}
+	}
+
+	// begin a transaction
+	ctx := context.Background()
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		logger.Log().Error("[migration]", err)
+		return
+	}
+
+	// roll back if it fails
+	defer tx.Rollback()
+
+	count = 0
+
+	for id, mid := range missingIDS {
+		_, err = tx.Exec(`UPDATE mailbox SET MessageID = ? WHERE ID = ?`, mid, id)
+		if err != nil {
+			logger.Log().Error("[migration]", err)
+		}
+
+		count++
+
+		if count%1000 == 0 {
+			percent := (100 * count) / total
+			logger.Log().Infof("[migration] stored 1,000 Message-IDs [%d%%] in %s", percent, time.Since(blockTime))
+			blockTime = time.Now()
+		}
+	}
+
+	logger.Log().Infof("[migration] commit %s changes", p.Sprintf("%d", count))
+
+	if err := tx.Commit(); err != nil {
+		logger.Log().Error("[migration]", err)
+		return
+	}
+
+	logger.Log().Infof("[migration] complete")
+}

storage/search.go

@@ -14,15 +14,13 @@ func searchParser(args []string, start, limit int) *sqlf.Stmt {
 	}
 
 	q := sqlf.From("mailbox").
-		Select(`ID, Data, Tags, Read,
+		Select(`Created, ID, Subject, Metadata, Size, Attachments, Read, Tags,
-			json_extract(Data, '$.To') as ToJSON,
+			IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON,
-			json_extract(Data, '$.From') as FromJSON,
+			IFNULL(json_extract(Metadata, '$.From'), '{}') as FromJSON,
-			IFNULL(json_extract(Data, '$.Cc'), '{}') as CcJSON,
+			IFNULL(json_extract(Metadata, '$.Cc'), '{}') as CcJSON,
-			IFNULL(json_extract(Data, '$.Bcc'), '{}') as BccJSON,
+			IFNULL(json_extract(Metadata, '$.Bcc'), '{}') as BccJSON
-			json_extract(Data, '$.Subject') as Subject,
-			json_extract(Data, '$.Attachments') as Attachments
 		`).
-		OrderBy("Sort DESC").
+		OrderBy("Created DESC").
 		Limit(limit).
 		Offset(start)
 
@@ -92,6 +90,15 @@ func searchParser(args []string, start, limit int) *sqlf.Stmt {
 					q.Where("Subject LIKE ?", "%"+escPercentChar(w)+"%")
 				}
 			}
+		} else if strings.HasPrefix(w, "message-id:") {
+			w = cleanString(w[11:])
+			if w != "" {
+				if exclude {
+					q.Where("MessageID NOT LIKE ?", "%"+escPercentChar(w)+"%")
+				} else {
+					q.Where("MessageID LIKE ?", "%"+escPercentChar(w)+"%")
+				}
+			}
 		} else if strings.HasPrefix(w, "tag:") {
 			w = cleanString(w[4:])
 			if w != "" {
@@ -122,9 +129,9 @@ func searchParser(args []string, start, limit int) *sqlf.Stmt {
 		} else {
 			// search text
 			if exclude {
-				q.Where("search NOT LIKE ?", "%"+cleanString(escPercentChar(w))+"%")
+				q.Where("SearchText NOT LIKE ?", "%"+cleanString(escPercentChar(w))+"%")
 			} else {
-				q.Where("search LIKE ?", "%"+cleanString(escPercentChar(w))+"%")
+				q.Where("SearchText LIKE ?", "%"+cleanString(escPercentChar(w))+"%")
 			}
 		}
 	}

storage/structs.go

@@ -11,8 +11,10 @@ import (
 //
 // swagger:model Message
 type Message struct {
-	// Unique message database id
+	// Database ID
 	ID string
+	// Message ID
+	MessageID string
 	// Read status
 	Read bool
 	// From address
@@ -25,7 +27,7 @@ type Message struct {
 	Bcc []*mail.Address
 	// ReplyTo addresses
 	ReplyTo []*mail.Address
-	// ReturnPath is the Return-Path
+	// Return-Path
 	ReturnPath string
 	// Message subject
 	Subject string
@@ -49,15 +51,15 @@ type Message struct {
 //
 // swagger:model Attachment
 type Attachment struct {
-	// attachment part id
+	// Attachment part ID
 	PartID string
-	// file name
+	// File name
 	FileName string
-	// content type
+	// Content type
 	ContentType string
-	// content id
+	// Content ID
 	ContentID string
-	// size in bytes
+	// Size in bytes
 	Size int
 }
 
@@ -65,7 +67,7 @@ type Attachment struct {
 //
 // swagger:model MessageSummary
 type MessageSummary struct {
-	// Unique message database id
+	// Database ID
 	ID string
 	// Read status
 	Read bool