go/mailpit
1
0
Fork 0
mirror of https://github.com/axllent/mailpit.git synced 2025-03-17 21:18:19 +02:00
Code Issues Releases Activity

Feature: Improved search parser

Browse Source
This commit is contained in:
Ralph Slooten 2023-09-22 06:46:23 +12:00
parent aa0af5de32
commit 0d084cfa1d
6 changed files with 213 additions and 101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -14,7 +14,6 @@ require (
github.com/k3a/html2text v1.2.1
github.com/klauspost/compress v1.16.7
github.com/leporo/sqlf v1.4.0
github.com/mattn/go-shellwords v1.0.12
github.com/mhale/smtpd v0.8.0
github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e
github.com/satori/go.uuid v1.2.0

2
go.sum
View File

@ -93,8 +93,6 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0=

94
storage/database.go
View File

@ -25,7 +25,6 @@ import (
"github.com/jhillyerd/enmime"
"github.com/klauspost/compress/zstd"
"github.com/leporo/sqlf"
"github.com/mattn/go-shellwords"
uuid "github.com/satori/go.uuid"
// sqlite (native) - https://gitlab.com/cznic/sqlite
@ -367,9 +366,6 @@ func List(start, limit int) ([]MessageSummary, error) {
em.Read = read == 1
results = append(results, em)
// logger.PrettyPrint(em)
}); err != nil {
return results, err
}
@ -379,96 +375,6 @@ func List(start, limit int) ([]MessageSummary, error) {
return results, nil
}
// Search will search a mailbox for search terms.
// The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as:
// is:read, is:unread, has:attachment, to:<term>, from:<term> & subject:<term>
// Negative searches also also included by prefixing the search term with a `-` or `!`
func Search(search string, start, limit int) ([]MessageSummary, int, error) {
results := []MessageSummary{}
allResults := []MessageSummary{}
tsStart := time.Now()
nrResults := 0
if limit < 0 {
limit = 50
}
s := escSearch(strings.ToLower(search))
// add another quote if missing closing quote
quotes := strings.Count(s, `"`)
if quotes%2 != 0 {
s += `"`
}
p := shellwords.NewParser()
args, err := p.Parse(s)
if err != nil {
return results, nrResults, errors.New("Your search contains invalid characters")
}
// generate the SQL based on arguments
q := searchParser(args)
if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
var created int64
var id string
var messageID string
var subject string
var metadata string
var size int
var attachments int
var tags string
var read int
var ignore string
em := MessageSummary{}
if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil {
logger.Log().Error(err)
return
}
if err := json.Unmarshal([]byte(metadata), &em); err != nil {
logger.Log().Error(err)
return
}
if err := json.Unmarshal([]byte(tags), &em.Tags); err != nil {
logger.Log().Error(err)
return
}
em.Created = time.UnixMilli(created)
em.ID = id
em.MessageID = messageID
em.Subject = subject
em.Size = size
em.Attachments = attachments
em.Read = read == 1
allResults = append(allResults, em)
}); err != nil {
return results, nrResults, err
}
dbLastAction = time.Now()
nrResults = len(allResults)
if nrResults > start {
end := nrResults
if nrResults >= start+limit {
end = start + limit
}
results = allResults[start:end]
}
elapsed := time.Since(tsStart)
logger.Log().Debugf("[db] search for \"%s\" in %s", search, elapsed)
return results, nrResults, err
}
// GetMessage returns a Message generated from the mailbox_data collection.
// If the message lacks a date header, then the received datetime is used.
func GetMessage(id string) (*Message, error) {

183
storage/search.go
View File

@ -1,14 +1,193 @@
package storage
import (
"context"
"database/sql"
"encoding/json"
"regexp"
"strings"
"time"
"github.com/axllent/mailpit/utils/logger"
"github.com/axllent/mailpit/utils/tools"
"github.com/leporo/sqlf"
)
// Search will search a mailbox for search terms.
// The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as:
// is:read, is:unread, has:attachment, to:<term>, from:<term> & subject:<term>
// Negative searches also also included by prefixing the search term with a `-` or `!`
func Search(search string, start, limit int) ([]MessageSummary, int, error) {
results := []MessageSummary{}
allResults := []MessageSummary{}
tsStart := time.Now()
nrResults := 0
if limit < 0 {
limit = 50
}
q := searchParser(search)
var err error
if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
var created int64
var id string
var messageID string
var subject string
var metadata string
var size int
var attachments int
var tags string
var read int
var ignore string
em := MessageSummary{}
if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil {
logger.Log().Error(err)
return
}
if err := json.Unmarshal([]byte(metadata), &em); err != nil {
logger.Log().Error(err)
return
}
if err := json.Unmarshal([]byte(tags), &em.Tags); err != nil {
logger.Log().Error(err)
return
}
em.Created = time.UnixMilli(created)
em.ID = id
em.MessageID = messageID
em.Subject = subject
em.Size = size
em.Attachments = attachments
em.Read = read == 1
allResults = append(allResults, em)
}); err != nil {
return results, nrResults, err
}
dbLastAction = time.Now()
nrResults = len(allResults)
if nrResults > start {
end := nrResults
if nrResults >= start+limit {
end = start + limit
}
results = allResults[start:end]
}
elapsed := time.Since(tsStart)
logger.Log().Debugf("[db] search for \"%s\" in %s", search, elapsed)
return results, nrResults, err
}
// DeleteSearch will delete all messages for search terms.
// The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as:
// is:read, is:unread, has:attachment, to:<term>, from:<term> & subject:<term>
// Negative searches also also included by prefixing the search term with a `-` or `!`
func DeleteSearch(search string) error {
q := searchParser(search)
ids := []string{}
if err := q.QueryAndClose(nil, db, func(row *sql.Rows) {
var created int64
var id string
var messageID string
var subject string
var metadata string
var size int
var attachments int
var tags string
var read int
var ignore string
if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil {
logger.Log().Error(err)
return
}
ids = append(ids, id)
}); err != nil {
return err
}
if len(ids) > 0 {
total := len(ids)
// split ids into chunks
var chunks [][]string
if total > 1000 {
chunkSize := 1000
chunks = make([][]string, 0, (len(ids)+chunkSize-1)/chunkSize)
for chunkSize < len(ids) {
ids, chunks = ids[chunkSize:], append(chunks, ids[0:chunkSize:chunkSize])
}
} else {
chunks = append(chunks, ids)
}
// begin a transaction to ensure both the message
// and data are deleted successfully
tx, err := db.BeginTx(context.Background(), nil)
if err != nil {
return err
}
// roll back if it fails
defer tx.Rollback()
for _, ids := range chunks {
delIDs := make([]interface{}, len(ids))
for i, id := range ids {
delIDs[i] = id
}
sqlDelete1 := `DELETE FROM mailbox WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)`
_, err = tx.Exec(sqlDelete1, delIDs...)
if err != nil {
return err
}
sqlDelete2 := `DELETE FROM mailbox_data WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)`
_, err = tx.Exec(sqlDelete2, delIDs...)
if err != nil {
return err
}
}
err = tx.Commit()
if err == nil {
logger.Log().Debugf("[db] deleted %d messages matching %s", total, search)
}
dbLastAction = time.Now()
dbDataDeleted = true
BroadcastMailboxStats()
}
return nil
}
// SearchParser returns the SQL syntax for the database search based on the search arguments
func searchParser(args []string) *sqlf.Stmt {
func searchParser(searchString string) *sqlf.Stmt {
searchString = strings.ToLower(searchString)
// group strings with quotes as a single argument and remove quotes
args := tools.ArgsParser(searchString)
q := sqlf.From("mailbox").
Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags,
IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON,
@ -71,7 +250,7 @@ func searchParser(args []string) *sqlf.Stmt {
}
}
} else if strings.HasPrefix(w, "subject:") {
w = cleanString(w[8:])
w = w[8:]
if w != "" {
if exclude {
q.Where("Subject NOT LIKE ?", "%"+escPercentChar(w)+"%")

2
storage/utils.go
View File

@ -186,8 +186,6 @@ func escPercentChar(s string) string {
// Escape certain characters in search phrases
func escSearch(str string) string {
str = strings.ReplaceAll(str, "(", " ")
str = strings.ReplaceAll(str, ")", " ")
dest := make([]byte, 0, 2*len(str))
var escape byte
for i := 0; i < len(str); i++ {

32
utils/tools/args.go Normal file
View File

@ -0,0 +1,32 @@
package tools
import "strings"
// ArgsParser will split a string by new words and quotes phrases
func ArgsParser(s string) []string {
args := []string{}
sb := &strings.Builder{}
quoted := false
for _, r := range s {
if r == '"' {
quoted = !quoted
sb.WriteRune(r) // keep '"' otherwise comment this line
} else if !quoted && r == ' ' {
v := strings.TrimSpace(strings.ReplaceAll(sb.String(), "\"", ""))
if v != "" {
args = append(args, v)
}
sb.Reset()
} else {
sb.WriteRune(r)
}
}
if sb.Len() > 0 {
v := strings.TrimSpace(strings.ReplaceAll(sb.String(), "\"", ""))
if v != "" {
args = append(args, v)
}
}
return args
}