Added support for building lambda functions

cmd/web-api/docs/docs.go

@@ -1,6 +1,6 @@
 // GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
 // This file was generated by geeks-accelerator/swag at
-// 2019-08-07 20:20:04.308889 -0800 AKDT m=+439.591671736
+// 2019-08-08 01:10:01.332719 -0800 AKDT m=+446.639667515
 
 package docs
 
@@ -139,14 +139,9 @@ var doc = `{
         },
         "/oauth/token": {
             "post": {
-                "security": [
-                    {
-                        "BasicAuth": []
-                    }
-                ],
                 "description": "Token generates an oauth2 accessToken using Basic Auth with a user's email and password.",
                 "consumes": [
-                    "application/json"
+                    "application/x-www-form-urlencoded"
                 ],
                 "produces": [
                     "application/json"
@@ -156,6 +151,26 @@ var doc = `{
                 ],
                 "summary": "Token handles a request to authenticate a user.",
                 "parameters": [
+                    {
+                        "type": "string",
+                        "description": "Email",
+                        "name": "username",
+                        "in": "formData",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Password",
+                        "name": "password",
+                        "in": "formData",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Account ID",
+                        "name": "account_id",
+                        "in": "formData"
+                    },
                     {
                         "enum": [
                             "user",
@@ -164,7 +179,7 @@ var doc = `{
                         "type": "string",
                         "description": "Scope",
                         "name": "scope",
-                        "in": "query"
+                        "in": "formData"
                     }
                 ],
                 "responses": {
@@ -190,7 +205,7 @@ var doc = `{
                 }
             }
         },
-        "/project": {
+        "/projects": {
             "get": {
                 "security": [
                     {
@@ -269,9 +284,7 @@ var doc = `{
                         }
                     }
                 }
-            }
+            },
-        },
-        "/projects": {
             "post": {
                 "security": [
                     {

cmd/web-api/docs/swagger.json

@@ -119,14 +119,9 @@
         },
         "/oauth/token": {
             "post": {
-                "security": [
-                    {
-                        "BasicAuth": []
-                    }
-                ],
                 "description": "Token generates an oauth2 accessToken using Basic Auth with a user's email and password.",
                 "consumes": [
-                    "application/json"
+                    "application/x-www-form-urlencoded"
                 ],
                 "produces": [
                     "application/json"
@@ -136,6 +131,26 @@
                 ],
                 "summary": "Token handles a request to authenticate a user.",
                 "parameters": [
+                    {
+                        "type": "string",
+                        "description": "Email",
+                        "name": "username",
+                        "in": "formData",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Password",
+                        "name": "password",
+                        "in": "formData",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "description": "Account ID",
+                        "name": "account_id",
+                        "in": "formData"
+                    },
                     {
                         "enum": [
                             "user",
@@ -144,7 +159,7 @@
                         "type": "string",
                         "description": "Scope",
                         "name": "scope",
-                        "in": "query"
+                        "in": "formData"
                     }
                 ],
                 "responses": {
@@ -170,7 +185,7 @@
                 }
             }
         },
-        "/project": {
+        "/projects": {
             "get": {
                 "security": [
                     {
@@ -249,9 +264,7 @@
                         }
                     }
                 }
-            }
+            },
-        },
-        "/projects": {
             "post": {
                 "security": [
                     {

cmd/web-api/docs/swagger.yaml

@@ -596,15 +596,29 @@ paths:
   /oauth/token:
     post:
       consumes:
-      - application/json
+      - application/x-www-form-urlencoded
       description: Token generates an oauth2 accessToken using Basic Auth with a user's
         email and password.
       parameters:
+      - description: Email
+        in: formData
+        name: username
+        required: true
+        type: string
+      - description: Password
+        in: formData
+        name: password
+        required: true
+        type: string
+      - description: Account ID
+        in: formData
+        name: account_id
+        type: string
       - description: Scope
         enum:
         - user
         - admin
-        in: query
+        in: formData
         name: scope
         type: string
       produces:
@@ -623,12 +637,10 @@ paths:
           description: Internal Server Error
           schema:
             $ref: '#/definitions/weberror.ErrorResponse'
-      security:
-      - BasicAuth: []
       summary: Token handles a request to authenticate a user.
       tags:
       - user
-  /project:
+  /projects:
     get:
       consumes:
       - application/json
@@ -680,7 +692,6 @@ paths:
       summary: List projects
       tags:
       - project
-  /projects:
     patch:
       consumes:
       - application/json

cmd/web-api/handlers/project.go

@@ -40,7 +40,7 @@ type Project struct {
 // @Failure 400 {object} weberror.ErrorResponse
 // @Failure 403 {object} weberror.ErrorResponse
 // @Failure 500 {object} weberror.ErrorResponse
-// @Router /project [get]
+// @Router /projects [get]
 func (p *Project) Find(ctx context.Context, w http.ResponseWriter, r *http.Request, params map[string]string) error {
 	claims, ok := ctx.Value(auth.Key).(auth.Claims)
 	if !ok {

cmd/web-api/handlers/user.go

@@ -470,13 +470,10 @@ func (u *User) SwitchAccount(ctx context.Context, w http.ResponseWriter, r *http
 // @Tags user
 // @Accept  x-www-form-urlencoded
 // @Produce  json
-// @Security BasicAuth
 // @Param username formData string true "Email"
 // @Param password formData string true "Password"
 // @Param account_id formData string false "Account ID"
-// @Param scope formData {array} string true "Scope" Enums(user, admin)
+// @Param scope formData string false "Scope" Enums(user, admin)
-// @Param scope query {array}  string false "Scope" Enums(user, admin)
-// @Param account_id query string false "Account ID"
 // @Success 200
 // @Failure 400 {object} weberror.ErrorResponse
 // @Failure 401 {object} weberror.ErrorResponse

tools/devops/cmd/cicd/aws.go

@@ -303,6 +303,41 @@ func EcsReadTaskDefinition(serviceDir, targetEnv string) ([]byte, error) {
 	return dat, nil
 }
 
+// LambdaReadFuncDefinition reads a task definition file and json decodes it.
+func LambdaReadFuncDefinition(serviceDir, targetEnv string) ([]byte, error) {
+	checkPaths := []string{
+		filepath.Join(serviceDir, fmt.Sprintf("lambda-func-definition-%s.json", targetEnv)),
+		filepath.Join(serviceDir, "lambda-func-definition.json"),
+	}
+
+	var defFile string
+	for _, tf := range checkPaths {
+		ok, _ := exists(tf)
+		if ok {
+			defFile = tf
+			break
+		}
+	}
+
+	if defFile == "" {
+		return nil, errors.Errorf("failed to locate task definition - checked %s", strings.Join(checkPaths, ", "))
+	}
+
+	dat, err := ioutil.ReadFile(defFile)
+	if err != nil {
+		return nil, errors.WithMessagef(err, "failed to read file %s", defFile)
+	}
+
+	return dat, nil
+}
+
+// LambdaS3KeyFromReleaseImage generates an S3 key from a release image.
+func LambdaS3KeyFromReleaseImage(releaseImage string) string {
+	it := filepath.Base(releaseImage)
+	it = strings.Replace(it, ":", "/", -1)
+	return filepath.Join("src/aws/lambda/", it+".zip")
+}
+
 // parseTaskDefinition json decodes it.
 func parseTaskDefinitionInput(dat []byte) (*ecs.RegisterTaskDefinitionInput, error) {
 	dat = convertKeys(dat)

tools/devops/cmd/cicd/service_build.go

@@ -15,6 +15,8 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/ecr"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/pborman/uuid"
 	"github.com/pkg/errors"
 	"gopkg.in/go-playground/validator.v9"
 )
@@ -26,12 +28,15 @@ type ServiceBuildFlags struct {
 	Env         string `validate:"oneof=dev stage prod" example:"dev"`
 
 	// Optional flags.
-	ProjectRoot string `validate:"omitempty" example:"."`
+	ProjectRoot         string `validate:"omitempty" example:"."`
-	ProjectName string ` validate:"omitempty" example:"example-project"`
+	ProjectName         string ` validate:"omitempty" example:"example-project"`
-	DockerFile  string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"`
+	DockerFile          string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"`
-	CommitRef   string `validate:"omitempty" example:"master@1ecfd275"`
+	CommitRef           string `validate:"omitempty" example:"master@1ecfd275"`
-	NoCache     bool   `validate:"omitempty" example:"false"`
+	S3BucketPrivateName string `validate:"omitempty" example:"saas-example-project-private"`
-	NoPush      bool   `validate:"omitempty" example:"false"`
+	BuildDir            string `validate:"omitempty" example:"."`
+	NoCache             bool   `validate:"omitempty" example:"false"`
+	NoPush              bool   `validate:"omitempty" example:"false"`
+	IsLambda            bool   `validate:"omitempty" example:"false"`
 }
 
 // serviceBuildRequest defines the details needed to execute a service build.
@@ -42,9 +47,12 @@ type serviceBuildRequest struct {
 	EcrRepository          *ecr.CreateRepositoryInput
 	EcrRepositoryMaxImages int `validate:"omitempty"`
 
-	CommitRef string `validate:"omitempty"`
+	BuildDir            string `validate:"omitempty" example:""`
-	NoCache   bool   `validate:"omitempty"`
+	CommitRef           string `validate:"omitempty"`
-	NoPush    bool   `validate:"omitempty"`
+	S3BucketPrivateName string `validate:"omitempty"`
+	NoCache             bool   `validate:"omitempty"`
+	NoPush              bool   `validate:"omitempty"`
+	IsLambda            bool   `validate:"omitempty"`
 
 	flags ServiceBuildFlags
 }
@@ -81,13 +89,20 @@ func NewServiceBuildRequest(log *log.Logger, flags ServiceBuildFlags) (*serviceB
 		req = serviceBuildRequest{
 			serviceRequest: sr,
 
-			CommitRef: flags.CommitRef,
+			CommitRef:           flags.CommitRef,
-			NoCache:   flags.NoCache,
+			S3BucketPrivateName: flags.S3BucketPrivateName,
-			NoPush:    flags.NoPush,
+			BuildDir:            flags.BuildDir,
+			NoCache:             flags.NoCache,
+			NoPush:              flags.NoPush,
+			IsLambda:            flags.IsLambda,
 
 			flags: flags,
 		}
 
+		if req.BuildDir == "" {
+			req.BuildDir = req.ProjectRoot
+		}
+
 		// Set default AWS ECR Repository Name.
 		req.EcrRepositoryName = ecrRepositoryName(req.ProjectName)
 		req.EcrRepository = &ecr.CreateRepositoryInput{
@@ -210,9 +225,18 @@ func ServiceBuild(log *log.Logger, req *serviceBuildRequest) error {
 	// Once we can access the repository in ECR, do the docker build.
 	{
 		log.Printf("Starting docker build %s\n", req.ReleaseImage)
-		dockerFile, err := filepath.Rel(req.ProjectRoot, req.DockerFile)
+
-		if err != nil {
+		var dockerFile string
-			return errors.Wrapf(err, "Failed parse relative path for %s from %s", req.DockerFile, req.ProjectRoot)
+		dockerPath := filepath.Join(req.BuildDir, req.DockerFile)
+		if _, err := os.Stat(dockerPath); err == nil {
+			dockerFile = req.DockerFile
+		} else {
+			dockerPath = req.DockerFile
+
+			dockerFile, err = filepath.Rel(req.BuildDir, dockerPath)
+			if err != nil {
+				return errors.Wrapf(err, "Failed parse relative path for %s from %s", dockerPath, req.BuildDir)
+			}
 		}
 
 		// Name of the first build stage declared in the docckerFile.
@@ -222,7 +246,7 @@ func ServiceBuild(log *log.Logger, req *serviceBuildRequest) error {
 		// FROM golang:1.12.6-alpine3.9 AS build_base
 		var buildBaseImageTag string
 		{
-			file, err := os.Open(req.DockerFile)
+			file, err := os.Open(dockerPath)
 			if err != nil {
 				log.Fatal(err)
 			}
@@ -349,9 +373,32 @@ func ServiceBuild(log *log.Logger, req *serviceBuildRequest) error {
 
 		cmds = append(cmds, buildCmd)
 
+		s3Files := make(map[string]*s3manager.UploadInput)
 		if req.NoPush == false {
-			cmds = append(cmds, dockerLoginCmd)
+			if req.IsLambda {
-			cmds = append(cmds, []string{"docker", "push", req.ReleaseImage})
+
+				lambdaS3Key := LambdaS3KeyFromReleaseImage(req.ReleaseImage)
+
+				tmpDir := os.TempDir()
+				lambdaZip := filepath.Join(tmpDir, filepath.Base(lambdaS3Key))
+
+				containerName := uuid.NewRandom().String()
+
+				cmds = append(cmds, []string{"docker", "create", "-ti", "--name", containerName, req.ReleaseImage, "bash"})
+				cmds = append(cmds, []string{"docker", "cp", containerName + ":/var/task", tmpDir})
+				cmds = append(cmds, []string{"docker", "rm", containerName})
+				cmds = append(cmds, []string{"cd", tmpDir + "/task"})
+				cmds = append(cmds, []string{"zip", "-r", lambdaZip, "."})
+
+				s3Files[lambdaZip] = &s3manager.UploadInput{
+					Bucket: &req.S3BucketPrivateName,
+					Key:    &lambdaS3Key,
+				}
+
+			} else {
+				cmds = append(cmds, dockerLoginCmd)
+				cmds = append(cmds, []string{"docker", "push", req.ReleaseImage})
+			}
 		}
 
 		for _, cmd := range cmds {
@@ -364,7 +411,7 @@ func ServiceBuild(log *log.Logger, req *serviceBuildRequest) error {
 
 			log.Printf("\t\t%s\n", logCmd)
 
-			err = execCmds(log, req.ProjectRoot, cmd)
+			err := execCmds(log, req.BuildDir, cmd)
 			if err != nil {
 				if len(cmd) > 2 && cmd[1] == "pull" {
 					log.Printf("\t\t\tSkipping pull - %s\n", err.Error())
@@ -374,6 +421,27 @@ func ServiceBuild(log *log.Logger, req *serviceBuildRequest) error {
 			}
 		}
 
+		if s3Files != nil && len(s3Files) > 0 {
+			// Create an uploader with the session and default options
+			uploader := s3manager.NewUploader(req.awsSession())
+
+			// Perform an upload.
+			for lf, upParams := range s3Files {
+				f, err := os.Open(lf)
+				if err != nil {
+					return errors.Wrapf(err, "Failed open file to %s", lf)
+				}
+				upParams.Body = f
+
+				_, err = uploader.Upload(upParams)
+				if err != nil {
+					return errors.Wrapf(err, "Failed upload file to %s", *upParams.Key)
+				}
+
+				log.Printf("\t\tUploaded %s to s3://%s/%s\n", lf, *upParams.Bucket, *upParams.Key)
+			}
+		}
+
 		log.Printf("\t%s\tbuild complete.\n", tests.Success)
 	}
 

tools/devops/cmd/cicd/service_deploy.go

@@ -66,6 +66,7 @@ type ServiceDeployFlags struct {
 	DockerFile      string `validate:"omitempty" example:"./cmd/web-api/Dockerfile"`
 	EnableLambdaVPC bool   `validate:"omitempty" example:"false"`
 	EnableEcsElb    bool   `validate:"omitempty" example:"false"`
+	IsLambda            bool   `validate:"omitempty" example:"false"`
 
 	StaticFilesS3Enable        bool `validate:"omitempty" example:"false"`
 	StaticFilesImgResizeEnable bool `validate:"omitempty" example:"false"`
@@ -136,6 +137,7 @@ type serviceDeployRequest struct {
 	VpcPublicSubnets []*ec2.CreateSubnetInput
 
 	EnableLambdaVPC bool `validate:"omitempty"`
+	IsLambda            bool   `validate:"omitempty"`
 	RecreateService bool `validate:"omitempty"`
 
 	SDNamepsace *servicediscovery.CreatePrivateDnsNamespaceInput
@@ -192,6 +194,7 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 			S3BucketPrivateName: flags.S3BucketPrivateName,
 			S3BucketPublicName:  flags.S3BucketPublicName,
 
+			IsLambda: flags.IsLambda,
 			EnableLambdaVPC: flags.EnableLambdaVPC,
 			EnableEcsElb:    flags.EnableEcsElb,
 			RecreateService: flags.RecreateService,
@@ -436,6 +439,12 @@ func NewServiceDeployRequest(log *log.Logger, flags ServiceDeployFlags) (*servic
 			req.EcrRepositoryName = ecrRepositoryName(req.ProjectName)
 			log.Printf("\t\t\tSet ECR Repository Name to '%s'.", req.EcrRepositoryName)
 
+			if req.IsLambda {
+
+			}	else {
+
+			}
+
 			// Set default AWS ECS Cluster Name.
 			req.EcsClusterName = req.ProjectName + "-" + req.Env
 			req.EcsCluster = &ecs.CreateClusterInput{

tools/devops/lambda-funcs/ddlogscollector/Dockerfile

@@ -0,0 +1,8 @@
+FROM lambci/lambda-base:raw
+
+LABEL maintainer="lee@geeksinthewoods.com"
+
+COPY . /var/task
+
+WORKDIR /var/task
+

tools/devops/lambda-funcs/ddlogscollector/lambda-func-definition.json

@@ -0,0 +1,18 @@
+{
+    "FunctionName": "{LAMBDA_FUNC}",
+    "MemorySize": 512,
+    "Role": "",
+    "Timeout": 300,
+    "Runtime": "python2.7",
+    "TracingConfig": {
+        "Mode": "PassThrough"
+    },
+    "Description": "Ship logs from cloudwatch to datadog",
+    "Handler": "lambda_function.lambda_handler",
+    "Environment": {
+        "Variables": {
+            "DD_API_KEY": "{DATADOG_APIKEY}",
+            "LAMBDA_FUNC": "{LAMBDA_FUNC}"
+        }
+    }
+}

tools/devops/lambda-funcs/ddlogscollector/lambda_function.py

@@ -0,0 +1,505 @@
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2018 Datadog, Inc.
+
+from __future__ import print_function
+
+import base64
+import gzip
+import json
+import os
+import re
+import socket
+import ssl
+import urllib
+from io import BytesIO, BufferedReader
+
+import boto3
+
+# Proxy
+# Define the proxy endpoint to forward the logs to
+DD_SITE = os.getenv("DD_SITE", default="datadoghq.com")
+DD_URL = os.getenv("DD_URL", default="lambda-intake.logs." + DD_SITE)
+
+# Define the proxy port to forward the logs to
+try:
+    if "DD_SITE" in os.environ and DD_SITE == "datadoghq.eu":
+        DD_PORT = int(os.environ.get("DD_PORT", 443))
+    else:
+        DD_PORT = int(os.environ.get("DD_PORT", 10516))
+except Exception:
+    DD_PORT = 10516
+
+# Scrubbing sensitive data
+# Option to redact all pattern that looks like an ip address / email address
+try:
+    is_ipscrubbing = os.environ["REDACT_IP"]
+except Exception:
+    is_ipscrubbing = False
+try:
+    is_emailscrubbing = os.environ["REDACT_EMAIL"]
+except Exception:
+    is_emailscrubbing = False
+
+# DD_API_KEY: Datadog API Key
+DD_API_KEY = "<your_api_key>"
+if "DD_KMS_API_KEY" in os.environ:
+    ENCRYPTED = os.environ["DD_KMS_API_KEY"]
+    DD_API_KEY = boto3.client("kms").decrypt(
+        CiphertextBlob=base64.b64decode(ENCRYPTED)
+    )["Plaintext"]
+elif "DD_API_KEY" in os.environ:
+    DD_API_KEY = os.environ["DD_API_KEY"]
+
+# Strip any trailing and leading whitespace from the API key
+DD_API_KEY = DD_API_KEY.strip()
+
+cloudtrail_regex = re.compile(
+    "\d+_CloudTrail_\w{2}-\w{4,9}-\d_\d{8}T\d{4}Z.+.json.gz$", re.I
+)
+ip_regex = re.compile("\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", re.I)
+email_regex = re.compile("[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+", re.I)
+
+DD_SOURCE = "ddsource"
+DD_CUSTOM_TAGS = "ddtags"
+DD_SERVICE = "service"
+DD_HOST = "host"
+DD_FORWARDER_VERSION = "1.2.3"
+
+# Pass custom tags as environment variable, ensure comma separated, no trailing comma in envvar!
+DD_TAGS = os.environ.get("DD_TAGS", "")
+
+class DatadogConnection(object):
+    def __init__(self, host, port, ddApiKey):
+        self.host = host
+        self.port = port
+        self.api_key = ddApiKey
+        self._sock = None
+
+    def _connect(self):
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s = ssl.wrap_socket(s)
+        s.connect((self.host, self.port))
+        return s
+
+    def safe_submit_log(self, log, metadata):
+        try:
+            self.send_entry(log, metadata)
+        except Exception as e:
+            # retry once
+            if self._sock:
+                # make sure we don't keep old connections open
+                self._sock.close()
+            self._sock = self._connect()
+            self.send_entry(log, metadata)
+        return self
+
+    def send_entry(self, log_entry, metadata):
+        # The log_entry can only be a string or a dict
+        if isinstance(log_entry, str):
+            log_entry = {"message": log_entry}
+        elif not isinstance(log_entry, dict):
+            raise Exception(
+                "Cannot send the entry as it must be either a string or a dict. Provided entry: "
+                + str(log_entry)
+            )
+
+        # Merge with metadata
+        log_entry = merge_dicts(log_entry, metadata)
+
+        # Send to Datadog
+        str_entry = json.dumps(log_entry)
+
+        # Scrub ip addresses if activated
+        if is_ipscrubbing:
+            try:
+                str_entry = ip_regex.sub("xxx.xxx.xxx.xx", str_entry)
+            except Exception as e:
+                print(
+                    "Unexpected exception while scrubbing logs: {} for event {}".format(
+                        str(e), str_entry
+                    )
+                )
+        # Scrub email addresses if activated
+        if is_emailscrubbing:
+            try:
+                str_entry = email_regex.sub("xxxxx@xxxxx.com", str_entry)
+            except Exception as e:
+                print(
+                    "Unexpected exception while scrubbing logs: {} for event {}".format(
+                        str(e), str_entry
+                    )
+                )
+
+        # For debugging purpose uncomment the following line
+        # print(str_entry)
+        prefix = "%s " % self.api_key
+        return self._sock.send((prefix + str_entry + "\n").encode("UTF-8"))
+
+    def __enter__(self):
+        self._sock = self._connect()
+        return self
+
+    def __exit__(self, ex_type, ex_value, traceback):
+        if self._sock:
+            self._sock.close()
+        if ex_type is not None:
+            print("DatadogConnection exit: ", ex_type, ex_value, traceback)
+
+
+def lambda_handler(event, context):
+    # Check prerequisites
+    if DD_API_KEY == "<your_api_key>" or DD_API_KEY == "":
+        raise Exception(
+            "You must configure your API key before starting this lambda function (see #Parameters section)"
+        )
+    # Check if the API key is the correct number of characters
+    if len(DD_API_KEY) != 32:
+        raise Exception(
+            "The API key is not the expected length. Please confirm that your API key is correct"
+        )
+
+    metadata = {"ddsourcecategory": "aws"}
+
+    # create socket
+    with DatadogConnection(DD_URL, DD_PORT, DD_API_KEY) as con:
+        # Add the context to meta
+        if "aws" not in metadata:
+            metadata["aws"] = {}
+        aws_meta = metadata["aws"]
+        aws_meta["function_version"] = context.function_version
+        aws_meta["invoked_function_arn"] = context.invoked_function_arn
+        # Add custom tags here by adding new value with the following format "key1:value1, key2:value2"  - might be subject to modifications
+        dd_custom_tags_data = {
+            "forwardername": context.function_name.lower(),
+            "memorysize": context.memory_limit_in_mb,
+            "forwarder_version": DD_FORWARDER_VERSION,
+        }
+        metadata[DD_CUSTOM_TAGS] = ",".join(
+            filter(
+                None,
+                [
+                    DD_TAGS,
+                    ",".join(
+                        [
+                            "{}:{}".format(k, v)
+                            for k, v in dd_custom_tags_data.iteritems()
+                        ]
+                    ),
+                ],
+            )
+        )
+
+        try:
+            logs = generate_logs(event, context, metadata)
+            for log in logs:
+                con = con.safe_submit_log(log, metadata)
+        except Exception as e:
+            print("Unexpected exception: {} for event {}".format(str(e), event))
+
+
+def generate_logs(event, context, metadata):
+    try:
+        # Route to the corresponding parser
+        event_type = parse_event_type(event)
+        if event_type == "s3":
+            logs = s3_handler(event, context, metadata)
+        elif event_type == "awslogs":
+            logs = awslogs_handler(event, context, metadata)
+        elif event_type == "events":
+            logs = cwevent_handler(event, metadata)
+        elif event_type == "sns":
+            logs = sns_handler(event, metadata)
+    except Exception as e:
+        # Logs through the socket the error
+        err_message = "Error parsing the object. Exception: {} for event {}".format(
+            str(e), event
+        )
+        logs = [err_message]
+    return logs
+
+
+# Utility functions
+
+
+def parse_event_type(event):
+    if "Records" in event and len(event["Records"]) > 0:
+        if "s3" in event["Records"][0]:
+            return "s3"
+        elif "Sns" in event["Records"][0]:
+            return "sns"
+
+    elif "awslogs" in event:
+        return "awslogs"
+
+    elif "detail" in event:
+        return "events"
+    raise Exception("Event type not supported (see #Event supported section)")
+
+
+# Handle S3 events
+def s3_handler(event, context, metadata):
+    s3 = boto3.client("s3")
+
+    # Get the object from the event and show its content type
+    bucket = event["Records"][0]["s3"]["bucket"]["name"]
+    key = urllib.unquote_plus(event["Records"][0]["s3"]["object"]["key"]).decode("utf8")
+
+    keyMetadata = parse_key_metadata(key)
+    for k in keyMetadata :
+        metadata[k] = keyMetadata[k]
+
+    source = parse_event_source(event, key)
+    metadata[DD_SOURCE] = source
+
+    ##Get the ARN of the service and set it as the hostname
+    if DD_HOST not in metadata.keys() :
+        hostname = parse_service_arn(source, key, bucket, context)
+        if hostname:
+            metadata[DD_HOST] = hostname
+
+    ##default service to source value
+    if DD_SERVICE not in metadata.keys() :
+        metadata[DD_SERVICE] = source
+
+    # Extract the S3 object
+    response = s3.get_object(Bucket=bucket, Key=key)
+    body = response["Body"]
+    data = body.read()
+
+    # If the name has a .gz extension, then decompress the data
+    if key[-3:] == ".gz":
+        with gzip.GzipFile(fileobj=BytesIO(data)) as decompress_stream:
+            # Reading line by line avoid a bug where gzip would take a very long time (>5min) for
+            # file around 60MB gzipped
+            data = "".join(BufferedReader(decompress_stream))
+
+    if is_cloudtrail(str(key)):
+        cloud_trail = json.loads(data)
+        for event in cloud_trail["Records"]:
+            # Create structured object and send it
+            structured_line = merge_dicts(
+                event, {"aws": {"s3": {"bucket": bucket, "key": key}}}
+            )
+            yield structured_line
+    else:
+        # Send lines to Datadog
+        for line in data.splitlines():
+            # Create structured object and send it
+            structured_line = {
+                "aws": {"s3": {"bucket": bucket, "key": key}},
+                "message": line,
+            }
+            yield structured_line
+
+
+# Handle CloudWatch logs
+def awslogs_handler(event, context, metadata):
+    # Get logs
+    with gzip.GzipFile(
+            fileobj=BytesIO(base64.b64decode(event["awslogs"]["data"]))
+    ) as decompress_stream:
+        # Reading line by line avoid a bug where gzip would take a very long
+        # time (>5min) for file around 60MB gzipped
+        data = "".join(BufferedReader(decompress_stream))
+    logs = json.loads(str(data))
+
+    # Set the source on the logs
+    source = logs.get("logGroup", "cloudwatch")
+    metadata[DD_SOURCE] = parse_event_source(event, source)
+
+    # Default service to source value
+    metadata[DD_SERVICE] = metadata[DD_SOURCE]
+
+    # Build aws attributes
+    aws_attributes = {
+        "aws": {
+            "awslogs": {
+                "logGroup": logs["logGroup"],
+                "logStream": logs["logStream"],
+                "owner": logs["owner"],
+            }
+        }
+    }
+
+    # For Lambda logs we want to extract the function name,
+    # then rebuild the arn of the monitored lambda using that name.
+    # Start by splitting the log group to get the function name
+    if metadata[DD_SOURCE] == "lambda":
+        log_group_parts = logs["logGroup"].split("/lambda/")
+        if len(log_group_parts) > 0:
+            function_name = log_group_parts[1].lower()
+            # Split the arn of the forwarder to extract the prefix
+            arn_parts = context.invoked_function_arn.split("function:")
+            if len(arn_parts) > 0:
+                arn_prefix = arn_parts[0]
+                # Rebuild the arn by replacing the function name
+                arn = arn_prefix + "function:" + function_name
+                # Add the arn as a log attribute
+                arn_attributes = {"lambda": {"arn": arn}}
+                aws_attributes = merge_dicts(aws_attributes, arn_attributes)
+                # Add the function name as tag
+                metadata[DD_CUSTOM_TAGS] += ",functionname:" + function_name
+                # Set the arn as the hostname
+                metadata[DD_HOST] = arn
+
+    # Create and send structured logs to Datadog
+    for log in logs["logEvents"]:
+        yield merge_dicts(log, aws_attributes)
+
+
+# Handle Cloudwatch Events
+def cwevent_handler(event, metadata):
+
+    data = event
+
+    # Set the source on the log
+    source = data.get("source", "cloudwatch")
+    service = source.split(".")
+    if len(service) > 1:
+        metadata[DD_SOURCE] = service[1]
+    else:
+        metadata[DD_SOURCE] = "cloudwatch"
+    ##default service to source value
+    metadata[DD_SERVICE] = metadata[DD_SOURCE]
+
+    yield data
+
+
+# Handle Sns events
+def sns_handler(event, metadata):
+
+    data = event
+    # Set the source on the log
+    metadata[DD_SOURCE] = parse_event_source(event, "sns")
+
+    for ev in data["Records"]:
+        # Create structured object and send it
+        structured_line = ev
+        yield structured_line
+
+
+def merge_dicts(a, b, path=None):
+    if path is None:
+        path = []
+    for key in b:
+        if key in a:
+            if isinstance(a[key], dict) and isinstance(b[key], dict):
+                merge_dicts(a[key], b[key], path + [str(key)])
+            elif a[key] == b[key]:
+                pass  # same leaf value
+            else:
+                raise Exception(
+                    "Conflict while merging metadatas and the log entry at %s"
+                    % ".".join(path + [str(key)])
+                )
+        else:
+            a[key] = b[key]
+    return a
+
+
+def is_cloudtrail(key):
+    match = cloudtrail_regex.search(key)
+    return bool(match)
+
+
+def parse_event_source(event, key):
+    if "elasticloadbalancing" in key:
+        return "elb"
+    for source in [
+        "lambda",
+        "redshift",
+        "cloudfront",
+        "kinesis",
+        "mariadb",
+        "mysql",
+        "apigateway",
+        "route53",
+        "vpc",
+        "rds",
+        "sns",
+        "waf",
+        "docdb",
+        "ecs",
+    ]:
+        if source in key:
+            return source
+        if "API-Gateway" in key:
+            return "apigateway"
+        if is_cloudtrail(str(key)):
+            return "cloudtrail"
+        if "awslogs" in event:
+            return "cloudwatch"
+        if "Records" in event and len(event["Records"]) > 0:
+            if "s3" in event["Records"][0]:
+                return "s3"
+    return "aws"
+
+def parse_service_arn(source, key, bucket, context):
+    if source == "elb":
+        #For ELB logs we parse the filename to extract parameters in order to rebuild the ARN
+        #1. We extract the region from the filename
+        #2. We extract the loadbalancer name and replace the "." by "/" to match the ARN format
+        #3. We extract the id of the loadbalancer
+        #4. We build the arn
+        keysplit = key.split("_")
+        idsplit = key.split("/")
+        if len(keysplit) > 3:
+            region = keysplit[2].lower()
+            name = keysplit[3]
+            elbname = name.replace(".", "/")
+            if len(idsplit) > 1:
+                idvalue = idsplit[1]
+                return "arn:aws:elasticloadbalancing:" + region + ":" + idvalue + ":loadbalancer/" + elbname
+    if source == "s3":
+        #For S3 access logs we use the bucket name to rebuild the arn
+        if bucket:
+            return "arn:aws:s3:::" + bucket
+    if source == "cloudfront":
+        #For Cloudfront logs we need to get the account and distribution id from the lambda arn and the filename
+        #1. We extract the cloudfront id  from the filename
+        #2. We extract the AWS account id from the lambda arn
+        #3. We build the arn
+        namesplit = key.split("/")
+        if len(namesplit) > 0:
+            filename = namesplit[len(namesplit)-1]
+            #(distribution-ID.YYYY-MM-DD-HH.unique-ID.gz)
+            filenamesplit = filename.split(".")
+            if len(filenamesplit) > 3:
+                distributionID = filenamesplit[len(filenamesplit)-4].lower()
+                arn = context.invoked_function_arn
+                arnsplit = arn.split(":")
+                if len(arnsplit) == 7:
+                    awsaccountID = arnsplit[4].lower()
+                    return "arn:aws:cloudfront::" + awsaccountID+":distribution/" + distributionID
+    if source == "redshift":
+        #For redshift logs we leverage the filename to extract the relevant information
+        #1. We extract the region from the filename
+        #2. We extract the account-id from the filename
+        #3. We extract the name of the cluster
+        #4. We build the arn: arn:aws:redshift:region:account-id:cluster:cluster-name
+        namesplit = key.split("/")
+        if len(namesplit) == 8:
+            region = namesplit[3].lower()
+            accountID = namesplit[1].lower()
+            filename = namesplit[7]
+            filesplit = filename.split("_")
+            if len(filesplit) == 6:
+                clustername = filesplit[3]
+                return "arn:aws:redshift:" + region + ":" + accountID + ":cluster:" + clustername
+    return
+
+def parse_key_metadata(key):
+    metadata = {}
+    keysplit = key.split("/")
+    for k in keysplit :
+        if "=" in k :
+            prt = key.split("=")
+            metadata[prt[0]] = prt[0]
+        elif "_" in k :
+            kn = key.split("_")[0]
+            if kn in ["source", "cluster", "service", "env", "region", "host"]:
+                metadata[kn] = k.replace(kn+"_", "")
+
+    return metadata

tools/devops/main.go

@@ -55,6 +55,9 @@ func main() {
 				cli.StringFlag{Name: "dockerfile", Usage: "DockerFile for service", Destination: &buildFlags.DockerFile},
 				cli.StringFlag{Name: "root", Usage: "project root directory", Destination: &buildFlags.ProjectRoot},
 				cli.StringFlag{Name: "project", Usage: "name of project", Destination: &buildFlags.ProjectName},
+				cli.StringFlag{Name: "build_dir", Usage: "build context directory", Destination: &buildFlags.BuildDir},
+				cli.StringFlag{Name: "private_bucket", Usage: "dev, stage, or prod", Destination: &buildFlags.S3BucketPrivateName},
+				cli.BoolFlag{Name: "lambda", Usage: "build as lambda function", Destination: &buildFlags.IsLambda},
 				cli.BoolFlag{Name: "no_cache", Usage: "skip docker cache", Destination: &buildFlags.NoCache},
 				cli.BoolFlag{Name: "no_push", Usage: "skip docker push after build", Destination: &buildFlags.NoPush},
 			},