self-hosted/docker-volume-backup
1
0
Fork 0
mirror of https://github.com/offen/docker-volume-backup.git synced 2025-11-23 21:44:40 +02:00
Code Issues Releases Activity

Support Nextcloud / WebDav (#48)

Browse Source 
* add studio-b12/gowebdav to be able to upload to webdav server

* make sure all env variables are present for webdav upload

* implement file upload to WebDav server

directory defaults to the base directory

* docs: add the new feature to the documentation

* if no WebDav env variable are given throw no error

* docs: use more elegant english :D

Co-authored-by: Frederik Ring <frederik.ring@gmail.com>

* docs: use official spelling of "WebDAV"

* perf: golang likes to return early instead of having an else block

* use WEBDAV_PATH instead of WEBDAV_DIRECTORY

* use split_words for more convenience

like shown here: https://github.com/kelseyhightower/envconfig#struct-tag-support

* simplify

* feat: add pruning of files in WebDAV remote

Based on / Inspired by the minio/S3 implementation of pruning remote files.

* remove logging from the development

* test: first try implementing tests

Sandly I have to use the remote pipeline -- local wont work for me.

* test: adapt used volume names

* test: specify image only once!

* test: minio AND webdav data should be present

* test: backups on WebDAV remote are laying in the root directory

* test: the webdav server stores date in /var/lib/dav

* trying with data subfolder

* test: 1 container was added so the number raised from 3 to 4

* webdav  subfolder is "data" not "backup"

* fix: password AND username must be defined

not password OR username

* improve logging

* feat: if the given path on the server isnt preset it will be created

* test: add creation of new folder for webdav to tests

Co-authored-by: Frederik Ring <frederik.ring@gmail.com>
This commit is contained in:
Kaerbr
2022-01-22 13:29:21 +01:00
committed by GitHub
parent 6b79f1914b
commit 6ded00aa06
6 changed files with 145 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
cmd/backup/main.go
View File

@@ -9,6 +9,7 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
@@ -32,6 +33,7 @@ import (
"github.com/otiai10/copy"
"github.com/sirupsen/logrus"
"golang.org/x/crypto/openpgp"
"github.com/studio-b12/gowebdav"
)
func main() {
@@ -86,12 +88,13 @@ func main() {
// script holds all the stateful information required to orchestrate a
// single backup run.
type script struct {
cli *client.Client
mc *minio.Client
logger *logrus.Logger
sender *router.ServiceRouter
hooks []hook
hookLevel hookLevel
cli *client.Client
mc *minio.Client
webdavClient *gowebdav.Client
logger *logrus.Logger
sender *router.ServiceRouter
hooks []hook
hookLevel hookLevel
start time.Time
file string
@@ -127,6 +130,10 @@ type config struct {
EmailSMTPPort int `envconfig:"EMAIL_SMTP_PORT" default:"587"`
EmailSMTPUsername string `envconfig:"EMAIL_SMTP_USERNAME"`
EmailSMTPPassword string `envconfig:"EMAIL_SMTP_PASSWORD"`
WebdavUrl string `split_words:"true"`
WebdavPath string `split_words:"true" default:"/"`
WebdavUsername string `split_words:"true"`
WebdavPassword string `split_words:"true"`
}
var msgBackupFailed = "backup run failed"
@@ -209,6 +216,17 @@ func newScript() (*script, error) {
s.mc = mc
}
// WebDAV check for env variables
// WebDAV instanciate client
if s.c.WebdavUrl != "" {
if s.c.WebdavUsername == "" || s.c.WebdavPassword == "" {
return nil, errors.New("newScript: WEBDAV_URL is defined, but no credentials were provided")
} else {
webdavClient := gowebdav.NewClient(s.c.WebdavUrl, s.c.WebdavUsername, s.c.WebdavPassword)
s.webdavClient = webdavClient
}
}
if s.c.EmailNotificationRecipient != "" {
emailURL := fmt.Sprintf(
"smtp://%s:%s@%s:%d/?from=%s&to=%s",
@@ -517,6 +535,21 @@ func (s *script) copyBackup() error {
s.logger.Infof("Uploaded a copy of backup `%s` to bucket `%s`.", s.file, s.c.AwsS3BucketName)
}
// WebDAV file upload
if s.webdavClient != nil {
bytes, err := os.ReadFile(s.file)
if err != nil {
return fmt.Errorf("copyBackup: error reading the file to be uploaded: %w", err)
}
if err := s.webdavClient.MkdirAll(s.c.WebdavPath, 0644); err != nil {
return fmt.Errorf("copyBackup: error creating directory '%s' on WebDAV server: %w", s.c.WebdavPath, err)
}
if err := s.webdavClient.Write(filepath.Join(s.c.WebdavPath, name), bytes, 0644); err != nil {
return fmt.Errorf("copyBackup: error uploading the file to WebDAV server: %w", err)
}
s.logger.Infof("Uploaded a copy of backup `%s` to WebDAV-URL '%s' at path '%s'.", s.file, s.c.WebdavUrl, s.c.WebdavPath)
}
if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
if err := copyFile(s.file, path.Join(s.c.BackupArchive, name)); err != nil {
return fmt.Errorf("copyBackup: error copying file to local archive: %w", err)
@@ -551,6 +584,7 @@ func (s *script) pruneOldBackups() error {
deadline := time.Now().AddDate(0, 0, -int(s.c.BackupRetentionDays))
// Prune minio/S3 backups
if s.mc != nil {
candidates := s.mc.ListObjects(context.Background(), s.c.AwsS3BucketName, minio.ListObjectsOptions{
WithMetadata: true,
@@ -612,6 +646,38 @@ func (s *script) pruneOldBackups() error {
}
}
// Prune WebDAV backups
if s.webdavClient != nil {
candidates, err := s.webdavClient.ReadDir(s.c.WebdavPath)
if err != nil {
return fmt.Errorf("pruneOldBackups: error looking up candidates from remote storage: %w", err)
}
var matches []fs.FileInfo
var lenCandidates int
for _, candidate := range candidates {
lenCandidates++
if candidate.ModTime().Before(deadline) {
matches = append(matches, candidate)
}
}
if len(matches) != 0 && len(matches) != lenCandidates {
for _, match := range matches {
if err := s.webdavClient.Remove(filepath.Join(s.c.WebdavPath, match.Name())); err != nil {
return fmt.Errorf("pruneOldBackups: error removing a file from remote storage: %w", err)
}
s.logger.Infof("Pruned %s from WebDAV: %s", match.Name(), filepath.Join(s.c.WebdavUrl, s.c.WebdavPath))
}
s.logger.Infof("Pruned %d out of %d remote backup(s) as their age exceeded the configured retention period of %d days.", len(matches), lenCandidates, s.c.BackupRetentionDays)
} else if len(matches) != 0 && len(matches) == lenCandidates {
s.logger.Warnf("The current configuration would delete all %d remote backup copies.", len(matches))
s.logger.Warn("Refusing to do so, please check your configuration.")
} else {
s.logger.Infof("None of %d remote backup(s) were pruned.", lenCandidates)
}
}
// Prune local backups
if _, err := os.Stat(s.c.BackupArchive); !os.IsNotExist(err) {
globPattern := path.Join(
s.c.BackupArchive,