Chore: Include runtime statistics in API (info) & UI (About)

Resolves #218

internal/stats/stats.go

@@ -0,0 +1,139 @@
+// Package stats stores and returns Mailpit statistics
+package stats
+
+import (
+	"os"
+	"runtime"
+	"sync"
+	"time"
+
+	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/storage"
+	"github.com/axllent/mailpit/internal/updater"
+)
+
+var (
+	// to prevent hammering Github for latest version
+	latestVersionCache string
+
+	// StartedAt is set to the current ime when Mailpit starts
+	startedAt time.Time
+
+	mu sync.RWMutex
+
+	smtpReceived     int
+	smtpReceivedSize int
+	smtpErrors       int
+	smtpIgnored      int
+)
+
+// AppInformation struct
+// swagger:model AppInformation
+type AppInformation struct {
+	// Current Mailpit version
+	Version string
+	// Latest Mailpit version
+	LatestVersion string
+	// Database path
+	Database string
+	// Database size in bytes
+	DatabaseSize int64
+	// Total number of messages in the database
+	Messages int
+	// Total number of messages in the database
+	Unread int
+	// Tags and message totals per tag
+	Tags map[string]int64
+	// Runtime statistics
+	RuntimeStats struct {
+		// Mailpit server uptime in seconds
+		Uptime int
+		// Current memory usage in bytes
+		Memory uint64
+		// Messages deleted
+		MessagesDeleted int
+		// SMTP messages received via since run
+		SMTPReceived int
+		// Total size in bytes of received messages since run
+		SMTPReceivedSize int
+		// SMTP errors since run
+		SMTPErrors int
+		// SMTP messages ignored since run (duplicate IDs)
+		SMTPIgnored int
+	}
+}
+
+// Load the current statistics
+func Load() AppInformation {
+	info := AppInformation{}
+	info.Version = config.Version
+
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+
+	info.RuntimeStats.Memory = m.Sys - m.HeapReleased
+
+	info.RuntimeStats.Uptime = int(time.Since(startedAt).Seconds())
+	info.RuntimeStats.MessagesDeleted = storage.StatsDeleted
+	info.RuntimeStats.SMTPReceived = smtpReceived
+	info.RuntimeStats.SMTPReceivedSize = smtpReceivedSize
+	info.RuntimeStats.SMTPErrors = smtpErrors
+	info.RuntimeStats.SMTPIgnored = smtpIgnored
+
+	if latestVersionCache != "" {
+		info.LatestVersion = latestVersionCache
+	} else {
+		latest, _, _, err := updater.GithubLatest(config.Repo, config.RepoBinaryName)
+		if err == nil {
+			info.LatestVersion = latest
+			latestVersionCache = latest
+
+			// clear latest version cache after 5 minutes
+			go func() {
+				time.Sleep(5 * time.Minute)
+				latestVersionCache = ""
+			}()
+		}
+	}
+
+	info.Database = config.DataFile
+
+	db, err := os.Stat(info.Database)
+	if err == nil {
+		info.DatabaseSize = db.Size()
+	}
+
+	info.Messages = storage.CountTotal()
+	info.Unread = storage.CountUnread()
+
+	info.Tags = storage.GetAllTagsCount()
+
+	return info
+}
+
+// Track will start the statistics logging in memory
+func Track() {
+	startedAt = time.Now()
+}
+
+// LogSMTPReceived logs a successfully SMTP transaction
+func LogSMTPReceived(size int) {
+	mu.Lock()
+	smtpReceived = smtpReceived + 1
+	smtpReceivedSize = smtpReceivedSize + size
+	mu.Unlock()
+}
+
+// LogSMTPError logs a failed SMTP transaction
+func LogSMTPError() {
+	mu.Lock()
+	smtpErrors = smtpErrors + 1
+	mu.Unlock()
+}
+
+// LogSMTPIgnored logs an ignored SMTP transaction
+func LogSMTPIgnored() {
+	mu.Lock()
+	smtpIgnored = smtpIgnored + 1
+	mu.Unlock()
+}

internal/storage/database.go

@@ -628,6 +628,8 @@ func DeleteOneMessage(id string) error {
 	dbLastAction = time.Now()
 	dbDataDeleted = true
 
+	logMessagesDeleted(1)
+
 	BroadcastMailboxStats()
 
 	return err
@@ -684,6 +686,8 @@ func DeleteAllMessages() error {
 		logger.Log().Debugf("[db] deleted %d messages in %s", total, elapsed)
 	}
 
+	logMessagesDeleted(total)
+
 	dbLastAction = time.Now()
 	dbDataDeleted = false
 
@@ -693,24 +697,6 @@ func DeleteAllMessages() error {
 	return err
 }
 
-// GetAllTags returns all used tags
-func GetAllTags() []string {
-	var tags = []string{}
-	var name string
-
-	if err := sqlf.
-		Select(`DISTINCT Name`).
-		From("tags").To(&name).
-		OrderBy("Name").
-		QueryAndClose(nil, db, func(row *sql.Rows) {
-			tags = append(tags, name)
-		}); err != nil {
-		logger.Log().Error(err)
-	}
-
-	return tags
-}
-
 // StatsGet returns the total/unread statistics for a mailbox
 func StatsGet() MailboxStats {
 	var (

internal/storage/tags.go

@@ -137,6 +137,47 @@ func DeleteAllMessageTags(id string) error {
 	return pruneUnusedTags()
 }
 
+// GetAllTags returns all used tags
+func GetAllTags() []string {
+	var tags = []string{}
+	var name string
+
+	if err := sqlf.
+		Select(`DISTINCT Name`).
+		From("tags").To(&name).
+		OrderBy("Name").
+		QueryAndClose(nil, db, func(row *sql.Rows) {
+			tags = append(tags, name)
+		}); err != nil {
+		logger.Log().Error(err)
+	}
+
+	return tags
+}
+
+// GetAllTagsCount returns all used tags with their total messages
+func GetAllTagsCount() map[string]int64 {
+	var tags = make(map[string]int64)
+	var name string
+	var total int64
+
+	if err := sqlf.
+		Select(`Name`).To(&name).
+		Select(`COUNT(message_tags.TagID) as total`).To(&total).
+		From("tags").
+		LeftJoin("message_tags", "tags.ID = message_tags.TagID").
+		GroupBy("message_tags.TagID").
+		OrderBy("Name").
+		QueryAndClose(nil, db, func(row *sql.Rows) {
+			tags[name] = total
+			// tags = append(tags, name)
+		}); err != nil {
+		logger.Log().Error(err)
+	}
+
+	return tags
+}
+
 // PruneUnusedTags will delete all unused tags from the database
 func pruneUnusedTags() error {
 	q := sqlf.From("tags").

internal/storage/utils.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/axllent/mailpit/config"
@@ -17,6 +18,13 @@ import (
 	"github.com/leporo/sqlf"
 )
 
+var (
+	// for stats to prevent import cycle
+	mu sync.RWMutex
+	// StatsDeleted for counting the number of messages deleted
+	StatsDeleted int
+)
+
 // Return a header field as a []*mail.Address, or "null" is not found/empty
 func addressToSlice(env *enmime.Envelope, key string) []*mail.Address {
 	data, err := env.AddressList(key)
@@ -168,6 +176,13 @@ func dbCron() {
 	}
 }
 
+// LogMessagesDeleted logs the number of messages deleted
+func logMessagesDeleted(n int) {
+	mu.Lock()
+	StatsDeleted = StatsDeleted + n
+	mu.Unlock()
+}
+
 // IsFile returns whether a path is a file
 func isFile(path string) bool {
 	info, err := os.Stat(path)

server/apiv1/info.go

@@ -3,32 +3,10 @@ package apiv1
 import (
 	"encoding/json"
 	"net/http"
-	"os"
-	"runtime"
 
-	"github.com/axllent/mailpit/config"
+	"github.com/axllent/mailpit/internal/stats"
-	"github.com/axllent/mailpit/internal/storage"
-	"github.com/axllent/mailpit/internal/updater"
 )
 
-// Response includes the current and latest Mailpit version, database info, and memory usage
-//
-// swagger:model AppInformation
-type appInformation struct {
-	// Current Mailpit version
-	Version string
-	// Latest Mailpit version
-	LatestVersion string
-	// Database path
-	Database string
-	// Database size in bytes
-	DatabaseSize int64
-	// Total number of messages in the database
-	Messages int
-	// Current memory usage in bytes
-	Memory uint64
-}
-
 // AppInfo returns some basic details about the running app, and latest release.
 func AppInfo(w http.ResponseWriter, _ *http.Request) {
 	// swagger:route GET /api/v1/info application AppInformation
@@ -45,27 +23,8 @@ func AppInfo(w http.ResponseWriter, _ *http.Request) {
 	//	Responses:
 	//		200: InfoResponse
 	//		default: ErrorResponse
-	info := appInformation{}
-	info.Version = config.Version
 
-	var m runtime.MemStats
+	info := stats.Load()
-	runtime.ReadMemStats(&m)
-
-	info.Memory = m.Sys - m.HeapReleased
-
-	latest, _, _, err := updater.GithubLatest(config.Repo, config.RepoBinaryName)
-	if err == nil {
-		info.LatestVersion = latest
-	}
-
-	info.Database = config.DataFile
-
-	db, err := os.Stat(info.Database)
-	if err == nil {
-		info.DatabaseSize = db.Size()
-	}
-
-	info.Messages = storage.CountTotal()
 
 	bytes, _ := json.Marshal(info)
 

server/apiv1/swagger.go

@@ -1,5 +1,7 @@
 package apiv1
 
+import "github.com/axllent/mailpit/internal/stats"
+
 // These structs are for the purpose of defining swagger HTTP parameters & responses
 
 // Application information
@@ -8,7 +10,7 @@ type infoResponse struct {
 	// Application information
 	//
 	// in: body
-	Body appInformation
+	Body stats.AppInformation
 }
 
 // Web UI configuration

server/server.go

@@ -17,6 +17,7 @@ import (
 	"github.com/axllent/mailpit/config"
 	"github.com/axllent/mailpit/internal/auth"
 	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/stats"
 	"github.com/axllent/mailpit/internal/storage"
 	"github.com/axllent/mailpit/server/apiv1"
 	"github.com/axllent/mailpit/server/handlers"
@@ -34,6 +35,7 @@ var AccessControlAllowOrigin string
 func Listen() {
 	isReady := &atomic.Value{}
 	isReady.Store(false)
+	stats.Track()
 
 	serverRoot, err := fs.Sub(embeddedFS, "ui")
 	if err != nil {

server/smtpd/smtpd.go

@@ -12,6 +12,7 @@ import (
 	"github.com/axllent/mailpit/config"
 	"github.com/axllent/mailpit/internal/auth"
 	"github.com/axllent/mailpit/internal/logger"
+	"github.com/axllent/mailpit/internal/stats"
 	"github.com/axllent/mailpit/internal/storage"
 	"github.com/google/uuid"
 	"github.com/mhale/smtpd"
@@ -27,7 +28,7 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
 	msg, err := mail.ReadMessage(bytes.NewReader(data))
 	if err != nil {
 		logger.Log().Errorf("[smtpd] error parsing message: %s", err.Error())
-
+		stats.LogSMTPError()
 		return err
 	}
 
@@ -63,6 +64,7 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
 	} else if config.IgnoreDuplicateIDs {
 		if storage.MessageIDExists(messageID) {
 			logger.Log().Debugf("[smtpd] duplicate message found, ignoring %s", messageID)
+			stats.LogSMTPIgnored()
 			return nil
 		}
 	}
@@ -116,13 +118,17 @@ func mailHandler(origin net.Addr, from string, to []string, data []byte) error {
 		logger.Log().Debugf("[smtpd] added missing addresses to Bcc header: %s", strings.Join(missingAddresses, ", "))
 	}
 
-	_, err = storage.Store(data)
+	_, err = storage.Store(&data)
 	if err != nil {
 		logger.Log().Errorf("[db] error storing message: %s", err.Error())
-
+		stats.LogSMTPError()
 		return err
 	}
 
+	stats.LogSMTPReceived(len(data))
+
+	data = nil // avoid memory leaks
+
 	subject := msg.Header.Get("Subject")
 	logger.Log().Debugf("[smtpd] received (%s) from:%s subject:%q", cleanIP(origin), from, subject)
 

server/ui-src/assets/_bootstrap.scss

@@ -16,7 +16,7 @@
 @import "bootstrap/scss/images";
 @import "bootstrap/scss/containers";
 @import "bootstrap/scss/grid";
-// @import "bootstrap/scss/tables";
+@import "bootstrap/scss/tables";
 @import "bootstrap/scss/forms";
 @import "bootstrap/scss/buttons";
 @import "bootstrap/scss/transitions";

server/ui-src/components/AboutMailpit.vue

@@ -148,10 +148,11 @@ export default {
 
 	<template v-else>
 		<!-- Modals -->
-		<div class="modal fade" id="AppInfoModal" tabindex="-1" aria-labelledby="AppInfoModalLabel" aria-hidden="true">
+		<div class="modal modal-xl fade" id="AppInfoModal" tabindex="-1" aria-labelledby="AppInfoModalLabel"
+			aria-hidden="true">
 			<div class="modal-dialog">
-				<div class="modal-content">
+				<div class="modal-content" v-if="mailbox.appInfo.RuntimeStats">
-					<div class="modal-header" v-if="mailbox.appInfo">
+					<div class="modal-header">
 						<h5 class="modal-title" id="AppInfoModalLabel">
 							Mailpit
 							<code>({{ mailbox.appInfo.Version }})</code>
@@ -170,40 +171,107 @@ export default {
 						</a>
 
 						<div class="row g-3">
-							<div class="col-12">
+							<div class="col-xl-6">
-								<RouterLink to="/api/v1/" class="btn btn-primary w-100" target="_blank">
+								<div class="row g-3">
-									<i class="bi bi-braces"></i>
+									<div class="col-12">
-									OpenAPI / Swagger API documentation
+										<RouterLink to="/api/v1/" class="btn btn-primary w-100" target="_blank">
-								</RouterLink>
+											<i class="bi bi-braces"></i>
-							</div>
+											OpenAPI / Swagger API documentation
-							<div class="col-sm-6">
+										</RouterLink>
-								<a class="btn btn-primary w-100" href="https://github.com/axllent/mailpit" target="_blank">
+									</div>
-									<i class="bi bi-github"></i>
+									<div class="col-sm-6">
-									Github
+										<a class="btn btn-primary w-100" href="https://github.com/axllent/mailpit"
-								</a>
+											target="_blank">
-							</div>
+											<i class="bi bi-github"></i>
-							<div class="col-sm-6">
+											Github
-								<a class="btn btn-primary w-100" href="https://mailpit.axllent.org/docs/" target="_blank">
+										</a>
-									Documentation
+									</div>
-								</a>
+									<div class="col-sm-6">
-							</div>
+										<a class="btn btn-primary w-100" href="https://mailpit.axllent.org/docs/"
-							<div class="col-6">
+											target="_blank">
-								<div class="card border-secondary text-center">
+											Documentation
-									<div class="card-header">Database size</div>
+										</a>
-									<div class="card-body text-secondary">
+									</div>
-										<h5 class="card-title">{{ getFileSize(mailbox.appInfo.DatabaseSize) }} </h5>
+									<div class="col-6">
+										<div class="card border-secondary text-center">
+											<div class="card-header">Database size</div>
+											<div class="card-body text-secondary">
+												<h5 class="card-title">{{ getFileSize(mailbox.appInfo.DatabaseSize) }} </h5>
+											</div>
+										</div>
+									</div>
+									<div class="col-6">
+										<div class="card border-secondary text-center">
+											<div class="card-header">RAM usage</div>
+											<div class="card-body text-secondary">
+												<h5 class="card-title">{{ getFileSize(mailbox.appInfo.RuntimeStats.Memory)
+												}} </h5>
+											</div>
+										</div>
 									</div>
 								</div>
 							</div>
-							<div class="col-6">
+							<div class="col-xl-6">
-								<div class="card border-secondary text-center">
+								<div class="card border-secondary">
-									<div class="card-header">RAM usage</div>
+									<div class="card-header h4">
-									<div class="card-body text-secondary">
+										Runtime statistics
-										<h5 class="card-title">{{ getFileSize(mailbox.appInfo.Memory) }} </h5>
+										<button class="btn btn-sm btn-outline-secondary float-end" v-on:click="loadInfo">
+											Refresh
+										</button>
 									</div>
+									<div class="card-body text-secondary">
+										<table class="table table-sm table-borderless mb-0">
+											<tbody>
+												<tr>
+													<td>
+														Mailpit uptime
+													</td>
+													<td>
+														{{ secondsToRelative(mailbox.appInfo.RuntimeStats.Uptime) }}
+													</td>
+												</tr>
+												<tr>
+													<td>
+														Messages deleted
+													</td>
+													<td>
+														{{ formatNumber(mailbox.appInfo.RuntimeStats.MessagesDeleted) }}
+													</td>
+												</tr>
+												<tr>
+													<td>
+														SMTP messages received
+													</td>
+													<td>
+														{{ formatNumber(mailbox.appInfo.RuntimeStats.SMTPReceived) }}
+														({{ getFileSize(mailbox.appInfo.RuntimeStats.SMTPReceivedSize) }})
+													</td>
+												</tr>
+												<tr>
+													<td>
+														SMTP errors
+													</td>
+													<td>
+														{{ formatNumber(mailbox.appInfo.RuntimeStats.SMTPErrors) }}
+													</td>
+												</tr>
+												<tr>
+													<td>
+														SMTP messages ignored
+													</td>
+													<td>
+														{{ formatNumber(mailbox.appInfo.RuntimeStats.SMTPIgnored) }}
+													</td>
+												</tr>
+											</tbody>
+										</table>
+									</div>
+
 								</div>
+
 							</div>
 						</div>
+
 					</div>
 					<div class="modal-footer">
 						<button type="button" class="btn btn-outline-secondary" data-bs-dismiss="modal">Close</button>

server/ui-src/mixins/CommonMixins.js

@@ -33,6 +33,9 @@ export default {
 		},
 
 		getFileSize: function (bytes) {
+			if (bytes == 0) {
+				return '0B'
+			}
 			var i = Math.floor(Math.log(bytes) / Math.log(1024))
 			return (bytes / Math.pow(1024, i)).toFixed(1) * 1 + ' ' + ['B', 'kB', 'MB', 'GB', 'TB'][i]
 		},
@@ -45,6 +48,10 @@ export default {
 			return moment(d).format('ddd, D MMM YYYY, h:mm a')
 		},
 
+		secondsToRelative: function (d) {
+			return moment().subtract(d, 'seconds').fromNow()
+		},
+
 		tagEncodeURI: function (tag) {
 			if (tag.match(/ /)) {
 				tag = `"${tag}"`

server/ui/api/v1/swagger.json

@@ -727,7 +727,7 @@
       "x-go-package": "net/mail"
     },
     "AppInformation": {
-      "description": "Response includes the current and latest Mailpit version, database info, and memory usage",
+      "description": "AppInformation struct",
       "type": "object",
       "properties": {
         "Database": {
@@ -743,23 +743,71 @@
           "description": "Latest Mailpit version",
           "type": "string"
         },
-        "Memory": {
-          "description": "Current memory usage in bytes",
-          "type": "integer",
-          "format": "uint64"
-        },
         "Messages": {
           "description": "Total number of messages in the database",
           "type": "integer",
           "format": "int64"
         },
+        "RuntimeStats": {
+          "description": "Runtime statistics",
+          "type": "object",
+          "properties": {
+            "Memory": {
+              "description": "Current memory usage in bytes",
+              "type": "integer",
+              "format": "uint64"
+            },
+            "MessagesDeleted": {
+              "description": "Messages deleted",
+              "type": "integer",
+              "format": "int64"
+            },
+            "SMTPErrors": {
+              "description": "SMTP errors since run",
+              "type": "integer",
+              "format": "int64"
+            },
+            "SMTPIgnored": {
+              "description": "SMTP messages ignored since run (duplicate IDs)",
+              "type": "integer",
+              "format": "int64"
+            },
+            "SMTPReceived": {
+              "description": "SMTP messages received via since run",
+              "type": "integer",
+              "format": "int64"
+            },
+            "SMTPReceivedSize": {
+              "description": "Total size in bytes of received messages since run",
+              "type": "integer",
+              "format": "int64"
+            },
+            "Uptime": {
+              "description": "Mailpit server uptime in seconds",
+              "type": "integer",
+              "format": "int64"
+            }
+          }
+        },
+        "Tags": {
+          "description": "Tags and message totals per tag",
+          "type": "object",
+          "additionalProperties": {
+            "type": "integer",
+            "format": "int64"
+          }
+        },
+        "Unread": {
+          "description": "Total number of messages in the database",
+          "type": "integer",
+          "format": "int64"
+        },
         "Version": {
           "description": "Current Mailpit version",
           "type": "string"
         }
       },
-      "x-go-name": "appInformation",
+      "x-go-package": "github.com/axllent/mailpit/internal/stats"
-      "x-go-package": "github.com/axllent/mailpit/server/apiv1"
     },
     "Attachment": {
       "description": "Attachment struct for inline and attachments",