From 7fd0e5187bdf7c5d91c7886a5f9ab2c11c6754a1 Mon Sep 17 00:00:00 2001 From: Lee Brown Date: Sun, 14 Jul 2019 19:16:13 -0800 Subject: [PATCH] delete config_sync.go that impliments fsnotify to watch for local files to be created/updated and uploads them to S3. --- internal/platform/devops/config_sync.go | 270 ------------------------ internal/platform/devops/ecs_service.go | 131 ------------ 2 files changed, 401 deletions(-) delete mode 100644 internal/platform/devops/config_sync.go diff --git a/internal/platform/devops/config_sync.go b/internal/platform/devops/config_sync.go deleted file mode 100644 index 833dcd6..0000000 --- a/internal/platform/devops/config_sync.go +++ /dev/null @@ -1,270 +0,0 @@ -package devops - -import ( - "io/ioutil" - "log" - "os" - "path/filepath" - "strings" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/secretsmanager" - "github.com/fsnotify/fsnotify" - "github.com/pkg/errors" -) - -// SyncCfgInit provides the functionality to keep config files sync'd between running tasks and across deployments. -func SyncCfgInit(log *log.Logger, awsSession *session.Session, secretPrefix, watchDir string, syncInterval time.Duration) (func(), error) { - - localfiles := make(map[string]time.Time) - - // Do the initial sync before starting file watch to download any existing configs. - err := SyncCfgDir(log, awsSession, secretPrefix, watchDir, localfiles) - if err != nil { - return nil, err - } - - // Create a new file watcher. - watcher, err := fsnotify.NewWatcher() - if err != nil { - return nil, errors.WithStack(err) - } - - // Return function that will should be run in the back ground via a go routine that will watch for new files created - // locally and updated in AWS Secrets Manager. - f := func() { - defer watcher.Close() - - // Init the watch to wait for sync local files to Secret Manager. - WatchCfgDir(log, awsSession, secretPrefix, watchDir, watcher, localfiles) - - // Init ticker to sync remote files from Secret Manager locally at the defined interval. - if syncInterval.Seconds() > 0 { - ticker := time.NewTicker(syncInterval) - defer ticker.Stop() - - go func() { - for _ = range ticker.C { - log.Println("AWS Secrets Manager : Checking for remote updates") - - // Do the initial sync before starting file watch to download any existing configs. - err := SyncCfgDir(log, awsSession, secretPrefix, watchDir, localfiles) - if err != nil { - log.Printf("AWS Secrets Manager : Remote sync error - %+v", err) - } - } - }() - } - } - - log.Printf("AWS Secrets Manager : Watching config dir %s", watchDir) - - // Note: Out of the box fsnotify can watch a single file, or a single directory. - if err := watcher.Add(watchDir); err != nil { - return nil, errors.Wrapf(err, "failed to add file watcher to %s", watchDir) - } - - return f, nil -} - -// SyncCfgDir lists all the Secrets from AWS Secrets Manager for a provided prefix and downloads them locally. -func SyncCfgDir(log *log.Logger, awsSession *session.Session, secretPrefix, watchDir string, localfiles map[string]time.Time) error { - - svc := secretsmanager.New(awsSession) - - // Get a list of secrets for the prefix when the time they were last changed. - secretIDs := make(map[string]time.Time) - err := svc.ListSecretsPages(&secretsmanager.ListSecretsInput{}, func(res *secretsmanager.ListSecretsOutput, lastPage bool) bool { - for _, s := range res.SecretList { - - // Skip any secret that does not have a matching prefix. - if !strings.HasPrefix(*s.Name, secretPrefix) { - continue - } - - secretIDs[*s.Name] = s.LastChangedDate.UTC() - } - - return !lastPage - }) - - if err != nil { - return errors.Wrap(err, "failed to list secrets") - } - - for id, curChanged := range secretIDs { - - // Load the secret by ID from Secrets Manager. - res, err := svc.GetSecretValue(&secretsmanager.GetSecretValueInput{ - SecretId: aws.String(id), - }) - if err != nil { - return errors.Wrapf(err, "failed to get secret value for id %s", id) - } - - filename := filepath.Base(id) - localpath := filepath.Join(watchDir, filename) - - // Ensure the secret exists locally. - if exists(localpath) { - // If the secret was previously downloaded and current last changed time is less than or equal to the time - // the secret was last downloaded, then no need to update. - if lastChanged, ok := localfiles[id]; ok && curChanged.UTC().Unix() <= lastChanged.UTC().Unix() { - continue - } - - } - - log.Printf("AWS Secrets Manager : Writing Config %s", filename) - err = ioutil.WriteFile(localpath, res.SecretBinary, 0644) - if err != nil { - return errors.Wrapf(err, "failed to write secret value for id %s to %s", id, localpath) - } - - // Only mark that the secret was updated when the file was successfully saved locally. - localfiles[id] = curChanged - } - - return nil -} - -// WatchCfgDir watches for new/updated files locally and uploads them to in AWS Secrets Manager. -func WatchCfgDir(log *log.Logger, awsSession *session.Session, secretPrefix, dir string, watcher *fsnotify.Watcher, localfiles map[string]time.Time) error { - - for { - select { - // watch for events - case event, ok := <-watcher.Events: - if !ok { - return nil - } - - err := handleWatchCfgEvent(log, awsSession, secretPrefix, event) - if err != nil { - log.Printf("AWS Secrets Manager : Watcher Error - %+v", err) - } - - // watch for errors - case err, ok := <-watcher.Errors: - if !ok { - return nil - } - if err != nil { - log.Printf("AWS Secrets Manager : Watcher Error - %+v", err) - } - } - } - - return nil -} - -// handleWatchCfgEvent handles a fsnotify event. For new files, secrets are created, for updated files, the secret is -// updated. For deleted files the secret is removed. -func handleWatchCfgEvent(log *log.Logger, awsSession *session.Session, secretPrefix string, event fsnotify.Event) error { - - svc := secretsmanager.New(awsSession) - - fname := filepath.Base(event.Name) - secretID := filepath.Join(secretPrefix, fname) - - if event.Op&fsnotify.Create == fsnotify.Create || event.Op&fsnotify.Write == fsnotify.Write { - - dat, err := ioutil.ReadFile(event.Name) - if err != nil { - return errors.Wrapf(err, "file watcher failed to read file %s", event.Name) - } - - // Create the new entry in AWS Secret Manager for the file. - _, err = svc.CreateSecret(&secretsmanager.CreateSecretInput{ - Name: aws.String(secretID), - SecretString: aws.String(string(dat)), - }) - if err != nil { - if aerr, ok := err.(awserr.Error); !ok { - - if aerr.Code() == secretsmanager.ErrCodeInvalidRequestException { - // InvalidRequestException: You can't create this secret because a secret with this - // name is already scheduled for deletion. - - // Restore secret after it was already previously deleted. - _, err = svc.RestoreSecret(&secretsmanager.RestoreSecretInput{ - SecretId: aws.String(secretID), - }) - if err != nil { - return errors.Wrapf(err, "file watcher failed to restore secret %s for %s", secretID, event.Name) - } - - } else if aerr.Code() != secretsmanager.ErrCodeResourceExistsException { - return errors.Wrapf(err, "file watcher failed to create secret %s for %s", secretID, event.Name) - } - } - - // If where was a resource exists error for create, then need to update the secret instead. - _, err = svc.UpdateSecret(&secretsmanager.UpdateSecretInput{ - SecretId: aws.String(secretID), - SecretString: aws.String(string(dat)), - }) - if err != nil { - return errors.Wrapf(err, "file watcher failed to update secret %s for %s", secretID, event.Name) - } - - log.Printf("AWS Secrets Manager : Secret %s updated for %s", secretID, event.Name) - } else { - log.Printf("AWS Secrets Manager : Secret %s created for %s", secretID, event.Name) - } - - } else if event.Op&fsnotify.Remove == fsnotify.Remove || event.Op&fsnotify.Rename == fsnotify.Rename { - // Delay delete to ensure the file is really deleted. - //delCheck := time.NewTimer(time.Minute) - - //<-delCheck.C - - // Create the new entry in AWS Secret Manager for the file. - _, err := svc.DeleteSecret(&secretsmanager.DeleteSecretInput{ - SecretId: aws.String(secretID), - - // (Optional) Specifies that the secret is to be deleted without any recovery - // window. You can't use both this parameter and the RecoveryWindowInDays parameter - // in the same API call. - // - // An asynchronous background process performs the actual deletion, so there - // can be a short delay before the operation completes. If you write code to - // delete and then immediately recreate a secret with the same name, ensure - // that your code includes appropriate back off and retry logic. - // - // Use this parameter with caution. This parameter causes the operation to skip - // the normal waiting period before the permanent deletion that AWS would normally - // impose with the RecoveryWindowInDays parameter. If you delete a secret with - // the ForceDeleteWithouRecovery parameter, then you have no opportunity to - // recover the secret. It is permanently lost. - ForceDeleteWithoutRecovery: aws.Bool(false), - - // (Optional) Specifies the number of days that Secrets Manager waits before - // it can delete the secret. You can't use both this parameter and the ForceDeleteWithoutRecovery - // parameter in the same API call. - // - // This value can range from 7 to 30 days. - RecoveryWindowInDays: aws.Int64(30), - }) - if err != nil { - return errors.Wrapf(err, "file watcher failed to delete secret %s for %s", secretID, event.Name) - } - - log.Printf("AWS Secrets Manager : Secret %s deleted for %s", secretID, event.Name) - } - - return nil -} - -// Exists reports whether the named file or directory exists. -func exists(name string) bool { - if _, err := os.Stat(name); err != nil { - if os.IsNotExist(err) { - return false - } - } - return true -} diff --git a/internal/platform/devops/ecs_service.go b/internal/platform/devops/ecs_service.go index 2297e6d..eb04f92 100644 --- a/internal/platform/devops/ecs_service.go +++ b/internal/platform/devops/ecs_service.go @@ -227,134 +227,3 @@ func RegisterEcsServiceTasksRoute53(log *log.Logger, awsSession *session.Session return nil } - -/* -res, err := pester.Get("http://169.254.170.2/v2/metadata") -if err != nil { - fmt.Println("http://169.254.170.2/v2/metadata failed", err.Error()) -} else { - dat, _ := ioutil.ReadAll(res.Body) - res.Body.Close() - fmt.Println("http://169.254.170.2/v2/metadata, OK", string(dat)) -} - -http://169.254.170.2/v2/metadata, - -{ - "Cluster": "arn:aws:ecs:us-west-2:888955683113:cluster/example-project-dev", - "TaskARN": "arn:aws:ecs:us-west-2:888955683113:task/700e38dd-dec5-4201-b711-c04a51feef8a", - "Family": "web-api", - "Revision": "113", - "DesiredStatus": "RUNNING", - "KnownStatus": "RUNNING", - "Containers": [{ - "DockerId": "c786dfdf6510b20294832ccbc3d66e6f1f915a4a79ead2588aa760a6365c839a", - "Name": "datadog-agent", - "DockerName": "ecs-web-api-113-datadog-agent-d884dee0c79af1fb6400", - "Image": "datadog/agent:latest", - "ImageID": "sha256:233c75f21f71838a59d478472d021be7006e752da6a70a11f77cf185c1050737", - "Labels": { - "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:888955683113:cluster/example-project-dev", - "com.amazonaws.ecs.container-name": "datadog-agent", - "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:888955683113:task/700e38dd-dec5-4201-b711-c04a51feef8a", - "com.amazonaws.ecs.task-definition-family": "web-api", - "com.amazonaws.ecs.task-definition-version": "113" - }, - "DesiredStatus": "RUNNING", - "KnownStatus": "STOPPED", - "ExitCode": 1, - "Limits": { - "CPU": 128, - "Memory": 0 - }, - "CreatedAt": "2019-07-11T05:36:54.135666318Z", - "StartedAt": "2019-07-11T05:36:54.481305866Z", - "FinishedAt": "2019-07-11T05:36:54.863742829Z", - "Type": "NORMAL", - "Networks": [{ - "NetworkMode": "awsvpc", - "IPv4Addresses": ["172.31.62.204"] - }], - "Volumes": [{ - "DockerName": "0960558c657c6e79d43e0e55f4ff259a97d78f58d9ad0d738e74495f4ba3cb06", - "Source": "/var/lib/docker/volumes/0960558c657c6e79d43e0e55f4ff259a97d78f58d9ad0d738e74495f4ba3cb06/_data", - "Destination": "/etc/datadog-agent" - }, { - "DockerName": "7a103f880857a1c2947e4a1bfff48efd25d24943a2d6a6e4dd86fa9dab3f10f0", - "Source": "/var/lib/docker/volumes/7a103f880857a1c2947e4a1bfff48efd25d24943a2d6a6e4dd86fa9dab3f10f0/_data", - "Destination": "/tmp" - }, { - "DockerName": "c88c03366eadb5d9da27708919e77ac5f8e0877c3dbb32c80580cb22e5811c00", - "Source": "/var/lib/docker/volumes/c88c03366eadb5d9da27708919e77ac5f8e0877c3dbb32c80580cb22e5811c00/_data", - "Destination": "/var/log/datadog" - }, { - "DockerName": "df97387f6ccc34c023055ef8a34a41e9d1edde4715c1849f1460683d31749539", - "Source": "/var/lib/docker/volumes/df97387f6ccc34c023055ef8a34a41e9d1edde4715c1849f1460683d31749539/_data", - "Destination": "/var/run/s6" - }] - }, { - "DockerId": "ab6bd869e675f64122a33a74da9183b304bbc60b649a15d0d83ebc48eeafdd76", - "Name": "~internal~ecs~pause", - "DockerName": "ecs-web-api-113-internalecspause-aab99b88b9ddadb0c701", - "Image": "fg-proxy:tinyproxy", - "ImageID": "", - "Labels": { - "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:888955683113:cluster/example-project-dev", - "com.amazonaws.ecs.container-name": "~internal~ecs~pause", - "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:888955683113:task/700e38dd-dec5-4201-b711-c04a51feef8a", - "com.amazonaws.ecs.task-definition-family": "web-api", - "com.amazonaws.ecs.task-definition-version": "113" - }, - "DesiredStatus": "RESOURCES_PROVISIONED", - "KnownStatus": "RESOURCES_PROVISIONED", - "Limits": { - "CPU": 0, - "Memory": 0 - }, - "CreatedAt": "2019-07-11T05:36:34.896093577Z", - "StartedAt": "2019-07-11T05:36:35.302359045Z", - "Type": "CNI_PAUSE", - "Networks": [{ - "NetworkMode": "awsvpc", - "IPv4Addresses": ["172.31.62.204"] - }] - }, { - "DockerId": "07bce50839fc992393799457811e4a0ac56979b2164c7aec6e66b40162ae3119", - "Name": "web-api-dev", - "DockerName": "ecs-web-api-113-web-api-dev-ceefbfb4dba2a6e05900", - "Image": "888955683113.dkr.ecr.us-west-2.amazonaws.com/example-project:dev-web-api", - "ImageID": "sha256:cf793de01311ac4e5e32c76cb4625f6600ec8017c726e99e28ec2199d4af599b", - "Labels": { - "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:888955683113:cluster/example-project-dev", - "com.amazonaws.ecs.container-name": "web-api-dev", - "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:888955683113:task/700e38dd-dec5-4201-b711-c04a51feef8a", - "com.amazonaws.ecs.task-definition-family": "web-api", - "com.amazonaws.ecs.task-definition-version": "113", - "com.datadoghq.ad.check_names": "[\"web-api-dev\"]", - "com.datadoghq.ad.init_configs": "[{}]", - "com.datadoghq.ad.instances": "[{\"host\": \"%%host%%\", \"port\": 80}]", - "com.datadoghq.ad.logs": "[{\"source\": \"docker\", \"service\": \"web-api-dev\", \"service_name\": \"web-api\", \"cluster\": \"example-project-dev\", \"env\": \"dev\"}]" - }, - "DesiredStatus": "RUNNING", - "KnownStatus": "RUNNING", - "Limits": { - "CPU": 128, - "Memory": 0 - }, - "CreatedAt": "2019-07-11T05:36:42.417547421Z", - "StartedAt": "2019-07-11T05:36:53.88095717Z", - "Type": "NORMAL", - "Networks": [{ - "NetworkMode": "awsvpc", - "IPv4Addresses": ["172.31.62.204"] - }], - "Health": {} - }], - "Limits": { - "CPU": 0.5, - "Memory": 2048 - }, - "PullStartedAt": "2019-07-11T05:36:35.407114703Z", - "PullStoppedAt": "2019-07-11T05:36:54.128398742Z" -} -*/