diff --git a/go.mod b/go.mod index 75c663d..d9ac94b 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,6 @@ require ( github.com/k3a/html2text v1.2.1 github.com/klauspost/compress v1.16.7 github.com/leporo/sqlf v1.4.0 - github.com/mattn/go-shellwords v1.0.12 github.com/mhale/smtpd v0.8.0 github.com/reiver/go-telnet v0.0.0-20180421082511-9ff0b2ab096e github.com/satori/go.uuid v1.2.0 diff --git a/go.sum b/go.sum index 0d54902..03667ce 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,6 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= -github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0= diff --git a/storage/database.go b/storage/database.go index 40aa219..88bd5b2 100644 --- a/storage/database.go +++ b/storage/database.go @@ -25,7 +25,6 @@ import ( "github.com/jhillyerd/enmime" "github.com/klauspost/compress/zstd" "github.com/leporo/sqlf" - "github.com/mattn/go-shellwords" uuid "github.com/satori/go.uuid" // sqlite (native) - https://gitlab.com/cznic/sqlite @@ -367,9 +366,6 @@ func List(start, limit int) ([]MessageSummary, error) { em.Read = read == 1 results = append(results, em) - - // logger.PrettyPrint(em) - }); err != nil { return results, err } @@ -379,96 +375,6 @@ func List(start, limit int) ([]MessageSummary, error) { return results, nil } -// Search will search a mailbox for search terms. -// The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as: -// is:read, is:unread, has:attachment, to:, from: & subject: -// Negative searches also also included by prefixing the search term with a `-` or `!` -func Search(search string, start, limit int) ([]MessageSummary, int, error) { - results := []MessageSummary{} - allResults := []MessageSummary{} - tsStart := time.Now() - nrResults := 0 - if limit < 0 { - limit = 50 - } - - s := escSearch(strings.ToLower(search)) - // add another quote if missing closing quote - quotes := strings.Count(s, `"`) - if quotes%2 != 0 { - s += `"` - } - - p := shellwords.NewParser() - args, err := p.Parse(s) - if err != nil { - return results, nrResults, errors.New("Your search contains invalid characters") - } - - // generate the SQL based on arguments - q := searchParser(args) - - if err := q.QueryAndClose(nil, db, func(row *sql.Rows) { - var created int64 - var id string - var messageID string - var subject string - var metadata string - var size int - var attachments int - var tags string - var read int - var ignore string - em := MessageSummary{} - - if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil { - logger.Log().Error(err) - return - } - - if err := json.Unmarshal([]byte(metadata), &em); err != nil { - logger.Log().Error(err) - return - } - - if err := json.Unmarshal([]byte(tags), &em.Tags); err != nil { - logger.Log().Error(err) - return - } - - em.Created = time.UnixMilli(created) - em.ID = id - em.MessageID = messageID - em.Subject = subject - em.Size = size - em.Attachments = attachments - em.Read = read == 1 - - allResults = append(allResults, em) - }); err != nil { - return results, nrResults, err - } - - dbLastAction = time.Now() - - nrResults = len(allResults) - - if nrResults > start { - end := nrResults - if nrResults >= start+limit { - end = start + limit - } - - results = allResults[start:end] - } - - elapsed := time.Since(tsStart) - - logger.Log().Debugf("[db] search for \"%s\" in %s", search, elapsed) - - return results, nrResults, err -} - // GetMessage returns a Message generated from the mailbox_data collection. // If the message lacks a date header, then the received datetime is used. func GetMessage(id string) (*Message, error) { diff --git a/storage/search.go b/storage/search.go index c65e20c..c4398f7 100644 --- a/storage/search.go +++ b/storage/search.go @@ -1,14 +1,193 @@ package storage import ( + "context" + "database/sql" + "encoding/json" "regexp" "strings" + "time" + "github.com/axllent/mailpit/utils/logger" + "github.com/axllent/mailpit/utils/tools" "github.com/leporo/sqlf" ) +// Search will search a mailbox for search terms. +// The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as: +// is:read, is:unread, has:attachment, to:, from: & subject: +// Negative searches also also included by prefixing the search term with a `-` or `!` +func Search(search string, start, limit int) ([]MessageSummary, int, error) { + results := []MessageSummary{} + allResults := []MessageSummary{} + tsStart := time.Now() + nrResults := 0 + if limit < 0 { + limit = 50 + } + + q := searchParser(search) + var err error + + if err := q.QueryAndClose(nil, db, func(row *sql.Rows) { + var created int64 + var id string + var messageID string + var subject string + var metadata string + var size int + var attachments int + var tags string + var read int + var ignore string + em := MessageSummary{} + + if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil { + logger.Log().Error(err) + return + } + + if err := json.Unmarshal([]byte(metadata), &em); err != nil { + logger.Log().Error(err) + return + } + + if err := json.Unmarshal([]byte(tags), &em.Tags); err != nil { + logger.Log().Error(err) + return + } + + em.Created = time.UnixMilli(created) + em.ID = id + em.MessageID = messageID + em.Subject = subject + em.Size = size + em.Attachments = attachments + em.Read = read == 1 + + allResults = append(allResults, em) + }); err != nil { + return results, nrResults, err + } + + dbLastAction = time.Now() + + nrResults = len(allResults) + + if nrResults > start { + end := nrResults + if nrResults >= start+limit { + end = start + limit + } + + results = allResults[start:end] + } + + elapsed := time.Since(tsStart) + + logger.Log().Debugf("[db] search for \"%s\" in %s", search, elapsed) + + return results, nrResults, err +} + +// DeleteSearch will delete all messages for search terms. +// The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as: +// is:read, is:unread, has:attachment, to:, from: & subject: +// Negative searches also also included by prefixing the search term with a `-` or `!` +func DeleteSearch(search string) error { + q := searchParser(search) + + ids := []string{} + + if err := q.QueryAndClose(nil, db, func(row *sql.Rows) { + var created int64 + var id string + var messageID string + var subject string + var metadata string + var size int + var attachments int + var tags string + var read int + var ignore string + + if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &tags, &ignore, &ignore, &ignore, &ignore); err != nil { + logger.Log().Error(err) + return + } + + ids = append(ids, id) + }); err != nil { + return err + } + + if len(ids) > 0 { + total := len(ids) + + // split ids into chunks + var chunks [][]string + if total > 1000 { + chunkSize := 1000 + chunks = make([][]string, 0, (len(ids)+chunkSize-1)/chunkSize) + for chunkSize < len(ids) { + ids, chunks = ids[chunkSize:], append(chunks, ids[0:chunkSize:chunkSize]) + } + } else { + chunks = append(chunks, ids) + } + + // begin a transaction to ensure both the message + // and data are deleted successfully + tx, err := db.BeginTx(context.Background(), nil) + if err != nil { + return err + } + + // roll back if it fails + defer tx.Rollback() + + for _, ids := range chunks { + delIDs := make([]interface{}, len(ids)) + for i, id := range ids { + delIDs[i] = id + } + + sqlDelete1 := `DELETE FROM mailbox WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` + + _, err = tx.Exec(sqlDelete1, delIDs...) + if err != nil { + return err + } + + sqlDelete2 := `DELETE FROM mailbox_data WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` + + _, err = tx.Exec(sqlDelete2, delIDs...) + if err != nil { + return err + } + } + + err = tx.Commit() + + if err == nil { + logger.Log().Debugf("[db] deleted %d messages matching %s", total, search) + } + + dbLastAction = time.Now() + dbDataDeleted = true + + BroadcastMailboxStats() + } + + return nil +} + // SearchParser returns the SQL syntax for the database search based on the search arguments -func searchParser(args []string) *sqlf.Stmt { +func searchParser(searchString string) *sqlf.Stmt { + searchString = strings.ToLower(searchString) + // group strings with quotes as a single argument and remove quotes + args := tools.ArgsParser(searchString) + q := sqlf.From("mailbox"). Select(`Created, ID, MessageID, Subject, Metadata, Size, Attachments, Read, Tags, IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON, @@ -71,7 +250,7 @@ func searchParser(args []string) *sqlf.Stmt { } } } else if strings.HasPrefix(w, "subject:") { - w = cleanString(w[8:]) + w = w[8:] if w != "" { if exclude { q.Where("Subject NOT LIKE ?", "%"+escPercentChar(w)+"%") diff --git a/storage/utils.go b/storage/utils.go index 220aaab..6ca394e 100644 --- a/storage/utils.go +++ b/storage/utils.go @@ -186,8 +186,6 @@ func escPercentChar(s string) string { // Escape certain characters in search phrases func escSearch(str string) string { - str = strings.ReplaceAll(str, "(", " ") - str = strings.ReplaceAll(str, ")", " ") dest := make([]byte, 0, 2*len(str)) var escape byte for i := 0; i < len(str); i++ { diff --git a/utils/tools/args.go b/utils/tools/args.go new file mode 100644 index 0000000..a2cf1fa --- /dev/null +++ b/utils/tools/args.go @@ -0,0 +1,32 @@ +package tools + +import "strings" + +// ArgsParser will split a string by new words and quotes phrases +func ArgsParser(s string) []string { + args := []string{} + sb := &strings.Builder{} + quoted := false + for _, r := range s { + if r == '"' { + quoted = !quoted + sb.WriteRune(r) // keep '"' otherwise comment this line + } else if !quoted && r == ' ' { + v := strings.TrimSpace(strings.ReplaceAll(sb.String(), "\"", "")) + if v != "" { + args = append(args, v) + } + sb.Reset() + } else { + sb.WriteRune(r) + } + } + if sb.Len() > 0 { + v := strings.TrimSpace(strings.ReplaceAll(sb.String(), "\"", "")) + if v != "" { + args = append(args, v) + } + } + + return args +}