package storage import ( "context" "database/sql" "encoding/json" "regexp" "strings" "time" "github.com/araddon/dateparse" "github.com/axllent/mailpit/internal/logger" "github.com/axllent/mailpit/internal/tools" "github.com/leporo/sqlf" ) // Search will search a mailbox for search terms. // The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as: // is:read, is:unread, has:attachment, to:, from: & subject: // Negative searches also also included by prefixing the search term with a `-` or `!` func Search(search, timezone string, start, limit int) ([]MessageSummary, int, error) { results := []MessageSummary{} allResults := []MessageSummary{} tsStart := time.Now() nrResults := 0 if limit < 0 { limit = 50 } q := searchQueryBuilder(search, timezone) var err error if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) { var created float64 var id string var messageID string var subject string var metadata string var size float64 var attachments int var snippet string var read int var ignore string em := MessageSummary{} if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet, &ignore, &ignore, &ignore, &ignore, &ignore); err != nil { logger.Log().Errorf("[db] %s", err.Error()) return } if err := json.Unmarshal([]byte(metadata), &em); err != nil { logger.Log().Errorf("[db] %s", err.Error()) return } em.Created = time.UnixMilli(int64(created)) em.ID = id em.MessageID = messageID em.Subject = subject em.Size = size em.Attachments = attachments em.Read = read == 1 em.Snippet = snippet allResults = append(allResults, em) }); err != nil { return results, nrResults, err } dbLastAction = time.Now() nrResults = len(allResults) if nrResults > start { end := nrResults if nrResults >= start+limit { end = start + limit } results = allResults[start:end] } // set tags for listed messages only for i, m := range results { results[i].Tags = getMessageTags(m.ID) } elapsed := time.Since(tsStart) logger.Log().Debugf("[db] search for \"%s\" in %s", search, elapsed) return results, nrResults, err } // DeleteSearch will delete all messages for search terms. // The search is broken up by segments (exact phrases can be quoted), and interprets specific terms such as: // is:read, is:unread, has:attachment, to:, from: & subject: // Negative searches also also included by prefixing the search term with a `-` or `!` func DeleteSearch(search, timezone string) error { q := searchQueryBuilder(search, timezone) ids := []string{} deleteSize := float64(0) if err := q.QueryAndClose(context.TODO(), db, func(row *sql.Rows) { var created float64 var id string var messageID string var subject string var metadata string var size float64 var attachments int var read int var snippet string var ignore string if err := row.Scan(&created, &id, &messageID, &subject, &metadata, &size, &attachments, &read, &snippet, &ignore, &ignore, &ignore, &ignore, &ignore); err != nil { logger.Log().Errorf("[db] %s", err.Error()) return } ids = append(ids, id) deleteSize = deleteSize + size }); err != nil { return err } if len(ids) > 0 { total := len(ids) // split ids into chunks of 1000 ids var chunks [][]string if total > 1000 { chunkSize := 1000 chunks = make([][]string, 0, (len(ids)+chunkSize-1)/chunkSize) for chunkSize < len(ids) { ids, chunks = ids[chunkSize:], append(chunks, ids[0:chunkSize:chunkSize]) } if len(ids) > 0 { // add remaining ids <= 1000 chunks = append(chunks, ids) } } else { chunks = append(chunks, ids) } // begin a transaction to ensure both the message // and data are deleted successfully tx, err := db.BeginTx(context.Background(), nil) if err != nil { return err } // roll back if it fails defer tx.Rollback() for _, ids := range chunks { delIDs := make([]interface{}, len(ids)) for i, id := range ids { delIDs[i] = id } sqlDelete1 := `DELETE FROM ` + tenant("mailbox") + ` WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec _, err = tx.Exec(sqlDelete1, delIDs...) if err != nil { return err } sqlDelete2 := `DELETE FROM ` + tenant("mailbox_data") + ` WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec _, err = tx.Exec(sqlDelete2, delIDs...) if err != nil { return err } sqlDelete3 := `DELETE FROM ` + tenant("message_tags") + ` WHERE ID IN (?` + strings.Repeat(",?", len(ids)-1) + `)` // #nosec _, err = tx.Exec(sqlDelete3, delIDs...) if err != nil { return err } } err = tx.Commit() if err := pruneUnusedTags(); err != nil { return err } if err == nil { logger.Log().Debugf("[db] deleted %d messages matching %s", total, search) } dbLastAction = time.Now() addDeletedSize(int64(deleteSize)) logMessagesDeleted(total) BroadcastMailboxStats() } return nil } // SearchParser returns the SQL syntax for the database search based on the search arguments func searchQueryBuilder(searchString, timezone string) *sqlf.Stmt { // group strings with quotes as a single argument and remove quotes args := tools.ArgsParser(searchString) if timezone != "" { loc, err := time.LoadLocation(timezone) if err != nil { logger.Log().Warnf("ignoring invalid timezone:\"%s\"", timezone) } else { time.Local = loc } } q := sqlf.From(tenant("mailbox") + " m"). Select(`m.Created, m.ID, m.MessageID, m.Subject, m.Metadata, m.Size, m.Attachments, m.Read, m.Snippet, IFNULL(json_extract(Metadata, '$.To'), '{}') as ToJSON, IFNULL(json_extract(Metadata, '$.From'), '{}') as FromJSON, IFNULL(json_extract(Metadata, '$.Cc'), '{}') as CcJSON, IFNULL(json_extract(Metadata, '$.Bcc'), '{}') as BccJSON, IFNULL(json_extract(Metadata, '$.ReplyTo'), '{}') as ReplyToJSON `). OrderBy("m.Created DESC") for _, w := range args { if cleanString(w) == "" { continue } // lowercase search to try match search prefixes lw := strings.ToLower(w) exclude := false // search terms starting with a `-` or `!` imply an exclude if len(w) > 1 && (strings.HasPrefix(w, "-") || strings.HasPrefix(w, "!")) { exclude = true w = w[1:] lw = lw[1:] } re := regexp.MustCompile(`[a-zA-Z0-9]+`) if !re.MatchString(w) { continue } if strings.HasPrefix(lw, "to:") { w = cleanString(w[3:]) if w != "" { if exclude { q.Where("ToJSON NOT LIKE ?", "%"+escPercentChar(w)+"%") } else { q.Where("ToJSON LIKE ?", "%"+escPercentChar(w)+"%") } } } else if strings.HasPrefix(lw, "from:") { w = cleanString(w[5:]) if w != "" { if exclude { q.Where("FromJSON NOT LIKE ?", "%"+escPercentChar(w)+"%") } else { q.Where("FromJSON LIKE ?", "%"+escPercentChar(w)+"%") } } } else if strings.HasPrefix(lw, "cc:") { w = cleanString(w[3:]) if w != "" { if exclude { q.Where("CcJSON NOT LIKE ?", "%"+escPercentChar(w)+"%") } else { q.Where("CcJSON LIKE ?", "%"+escPercentChar(w)+"%") } } } else if strings.HasPrefix(lw, "bcc:") { w = cleanString(w[4:]) if w != "" { if exclude { q.Where("BccJSON NOT LIKE ?", "%"+escPercentChar(w)+"%") } else { q.Where("BccJSON LIKE ?", "%"+escPercentChar(w)+"%") } } } else if strings.HasPrefix(lw, "reply-to:") { w = cleanString(w[9:]) if w != "" { if exclude { q.Where("ReplyToJSON NOT LIKE ?", "%"+escPercentChar(w)+"%") } else { q.Where("ReplyToJSON LIKE ?", "%"+escPercentChar(w)+"%") } } } else if strings.HasPrefix(lw, "subject:") { w = w[8:] if w != "" { if exclude { q.Where("Subject NOT LIKE ?", "%"+escPercentChar(w)+"%") } else { q.Where("Subject LIKE ?", "%"+escPercentChar(w)+"%") } } } else if strings.HasPrefix(lw, "message-id:") { w = cleanString(w[11:]) if w != "" { if exclude { q.Where("MessageID NOT LIKE ?", "%"+escPercentChar(w)+"%") } else { q.Where("MessageID LIKE ?", "%"+escPercentChar(w)+"%") } } } else if strings.HasPrefix(lw, "tag:") { w = cleanString(w[4:]) if w != "" { if exclude { q.Where(`m.ID NOT IN (SELECT mt.ID FROM `+tenant("message_tags")+` mt JOIN `+tenant("tags")+` t ON mt.TagID = t.ID WHERE t.Name = ?)`, w) } else { q.Where(`m.ID IN (SELECT mt.ID FROM `+tenant("message_tags")+` mt JOIN `+tenant("tags")+` t ON mt.TagID = t.ID WHERE t.Name = ?)`, w) } } } else if lw == "is:read" { if exclude { q.Where("Read = 0") } else { q.Where("Read = 1") } } else if lw == "is:unread" { if exclude { q.Where("Read = 1") } else { q.Where("Read = 0") } } else if lw == "is:tagged" { if exclude { q.Where(`m.ID NOT IN (SELECT DISTINCT mt.ID FROM ` + tenant("message_tags") + ` mt JOIN tags t ON mt.TagID = t.ID)`) } else { q.Where(`m.ID IN (SELECT DISTINCT mt.ID FROM ` + tenant("message_tags") + ` mt JOIN tags t ON mt.TagID = t.ID)`) } } else if lw == "has:attachment" || lw == "has:attachments" { if exclude { q.Where("Attachments = 0") } else { q.Where("Attachments > 0") } } else if strings.HasPrefix(lw, "after:") { w = cleanString(w[6:]) if w != "" { t, err := dateparse.ParseLocal(w) if err != nil { logger.Log().Warnf("ignoring invalid after: date \"%s\"", w) } else { timestamp := t.UnixMilli() if exclude { q.Where(`m.Created <= ?`, timestamp) } else { q.Where(`m.Created >= ?`, timestamp) } } } } else if strings.HasPrefix(lw, "before:") { w = cleanString(w[7:]) if w != "" { t, err := dateparse.ParseLocal(w) if err != nil { logger.Log().Warnf("ignoring invalid before: date \"%s\"", w) } else { timestamp := t.UnixMilli() if exclude { q.Where(`m.Created >= ?`, timestamp) } else { q.Where(`m.Created <= ?`, timestamp) } } } } else { // search text if exclude { q.Where("SearchText NOT LIKE ?", "%"+cleanString(escPercentChar(strings.ToLower(w)))+"%") } else { q.Where("SearchText LIKE ?", "%"+cleanString(escPercentChar(strings.ToLower(w)))+"%") } } } return q }