From 8878ece19f67aafc4c301ed6f2c74e07dea3c639 Mon Sep 17 00:00:00 2001 From: Ralph Slooten Date: Sat, 15 Feb 2025 14:33:11 +1300 Subject: [PATCH] Feature: Add dump feature to export all raw messages to a local directory (#443) --- cmd/dump.go | 36 ++++++++ internal/dump/dump.go | 163 +++++++++++++++++++++++++++++++++++ internal/storage/messages.go | 8 +- internal/tools/fs.go | 23 +++++ 4 files changed, 227 insertions(+), 3 deletions(-) create mode 100644 cmd/dump.go create mode 100644 internal/dump/dump.go create mode 100644 internal/tools/fs.go diff --git a/cmd/dump.go b/cmd/dump.go new file mode 100644 index 0000000..f8f5305 --- /dev/null +++ b/cmd/dump.go @@ -0,0 +1,36 @@ +package cmd + +import ( + "github.com/axllent/mailpit/config" + "github.com/axllent/mailpit/internal/dump" + "github.com/axllent/mailpit/internal/logger" + "github.com/spf13/cobra" +) + +// dumpCmd represents the dump command +var dumpCmd = &cobra.Command{ + Use: "dump ", + Short: "Dump all messages from a database to a directory", + Long: `Dump all messages stored in Mailpit into a local directory as individual files. + +The database can either be the database file (eg: --database /var/lib/mailpit/mailpit.db) or a +URL of a running Mailpit instance (eg: --http http://127.0.0.1/). If dumping over HTTP, the URL +should be the base URL of your running Mailpit instance, not the link to the API itself.`, + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + if err := dump.Sync(args[0]); err != nil { + logger.Log().Fatal(err) + } + }, +} + +func init() { + rootCmd.AddCommand(dumpCmd) + + dumpCmd.Flags().SortFlags = false + + dumpCmd.Flags().StringVar(&config.Database, "database", config.Database, "Dump messages directly from a database file") + dumpCmd.Flags().StringVar(&config.TenantID, "tenant-id", config.TenantID, "Database tenant ID to isolate data (optional)") + dumpCmd.Flags().StringVar(&dump.URL, "http", dump.URL, "Dump messages via HTTP API (base URL of running Mailpit instance)") + dumpCmd.Flags().BoolVarP(&logger.VerboseLogging, "verbose", "v", logger.VerboseLogging, "Verbose logging") +} diff --git a/internal/dump/dump.go b/internal/dump/dump.go new file mode 100644 index 0000000..60c43a9 --- /dev/null +++ b/internal/dump/dump.go @@ -0,0 +1,163 @@ +// Package dump is used to export all messages from mailpit into a directory +package dump + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "os" + "path" + "regexp" + "strings" + + "github.com/axllent/mailpit/config" + "github.com/axllent/mailpit/internal/logger" + "github.com/axllent/mailpit/internal/storage" + "github.com/axllent/mailpit/internal/tools" + "github.com/axllent/mailpit/server/apiv1" +) + +var ( + linkRe = regexp.MustCompile(`(?i)^https?:\/\/`) + + outDir string + + // Base URL of mailpit instance + base string + + // URL is the base URL of a remove Mailpit instance + URL string + + summary = []storage.MessageSummary{} +) + +// Sync will sync all messages from the specified database or API to the specified output directory +func Sync(d string) error { + + outDir = path.Clean(d) + + if URL != "" { + if !linkRe.MatchString(URL) { + return errors.New("Invalid URL") + } + + base = strings.TrimRight(URL, "/") + "/" + } + + if base == "" && config.Database == "" { + return errors.New("No database or API URL specified") + } + + if !tools.IsDir(outDir) { + if err := os.MkdirAll(outDir, 0755); err != nil { + return err + } + } + + if err := loadIDs(); err != nil { + return err + } + + if err := saveMessages(); err != nil { + return err + } + + return nil +} + +// LoadIDs will load all message IDs from the specified database or API +func loadIDs() error { + if base != "" { + // remote + logger.Log().Debugf("Fetching messages summary from %s", base) + res, err := http.Get(base + "api/v1/messages?limit=0") + + if err != nil { + return err + } + + body, err := io.ReadAll(res.Body) + + if err != nil { + return err + } + + var data apiv1.MessagesSummary + if err := json.Unmarshal(body, &data); err != nil { + return err + } + + summary = data.Messages + + } else { + // make sure the database isn't pruned while open + config.MaxMessages = 0 + + var err error + // local database + if err = storage.InitDB(); err != nil { + return err + } + + logger.Log().Debugf("Fetching messages summary from %s", config.Database) + + summary, err = storage.List(0, 0, 0) + if err != nil { + return err + } + } + + if len(summary) == 0 { + return errors.New("No messages found") + } + + return nil +} + +func saveMessages() error { + for _, m := range summary { + out := path.Join(outDir, m.ID+".eml") + + // skip if message exists + if tools.IsFile(out) { + continue + } + + var b []byte + + if base != "" { + res, err := http.Get(base + "api/v1/message/" + m.ID + "/raw") + + if err != nil { + logger.Log().Errorf("Error fetching message %s: %s", m.ID, err.Error()) + continue + } + + b, err = io.ReadAll(res.Body) + + if err != nil { + logger.Log().Errorf("Error fetching message %s: %s", m.ID, err.Error()) + continue + } + } else { + var err error + b, err = storage.GetMessageRaw(m.ID) + if err != nil { + logger.Log().Errorf("Error fetching message %s: %s", m.ID, err.Error()) + continue + } + } + + if err := os.WriteFile(out, b, 0644); err != nil { + logger.Log().Errorf("Error writing message %s: %s", m.ID, err.Error()) + continue + } + + _ = os.Chtimes(out, m.Created, m.Created) + + logger.Log().Debugf("Saved message %s to %s", m.ID, out) + } + + return nil +} diff --git a/internal/storage/messages.go b/internal/storage/messages.go index 77faa23..752d7fc 100644 --- a/internal/storage/messages.go +++ b/internal/storage/messages.go @@ -175,9 +175,11 @@ func List(start int, beforeTS int64, limit int) ([]MessageSummary, error) { q := sqlf.From(tenant("mailbox") + " m"). Select(`m.Created, m.ID, m.MessageID, m.Subject, m.Metadata, m.Size, m.Attachments, m.Read, m.Snippet`). - OrderBy("m.Created DESC"). - Limit(limit). - Offset(start) + OrderBy("m.Created DESC") + + if limit > 0 { + q = q.Limit(limit).Offset(start) + } if beforeTS > 0 { q = q.Where("Created < ?", beforeTS) diff --git a/internal/tools/fs.go b/internal/tools/fs.go new file mode 100644 index 0000000..411b02e --- /dev/null +++ b/internal/tools/fs.go @@ -0,0 +1,23 @@ +package tools + +import ( + "os" + "path/filepath" +) + +// IsFile returns whether a file exists and is readable +func IsFile(path string) bool { + f, err := os.Open(filepath.Clean(path)) + defer f.Close() + return err == nil +} + +// IsDir returns whether a path is a directory +func IsDir(path string) bool { + info, err := os.Stat(path) + if err != nil || os.IsNotExist(err) || !info.IsDir() { + return false + } + + return true +}