Create pipeline with separate builds

deployments/pipeline-new.yml

@@ -45,6 +45,9 @@ Resources:
 
   Pipeline:
     Type: AWS::CodePipeline::Pipeline
+    DependsOn:
+      - SSMArtifactBucket
+      - SSMCodePipelineServiceRole
     Properties:
       RoleArn: !GetAtt CodePipelineServiceRole.Arn
       ArtifactStore:
@@ -73,36 +76,99 @@ Resources:
                 - Name: Source
               RunOrder: 1
 
+        # #-----------------------------------------------------------------------------#
+        # # Infrastructure Resources
+        # #-----------------------------------------------------------------------------#
+        # - Name: Network_Resources
+        #   Actions:
+        #     - Name: Deploy
+        #       ActionTypeId:
+        #         Category: Deploy
+        #         Owner: AWS
+        #         Version: 1
+        #         Provider: CloudFormation
+        #       Configuration:
+        #         ActionMode: CREATE_UPDATE
+        #         RoleArn: !GetAtt CloudFormationDeployRole.Arn
+        #         StackName: !Sub ${EnvironmentName}-Network
+        #         TemplatePath: Source::deployments/network.yml
+        #         Capabilities: CAPABILITY_IAM
+        #         ParameterOverrides: !Sub |
+        #           {
+        #             "EnvironmentName": "${EnvironmentName}"
+        #           }
+        #       InputArtifacts:
+        #         - Name: Source
+
+        # - Name: Base_Resources
+        #   Actions:
+        #     # Rabbit, Redis, and Postgres
+        #     - Name: Resources
+        #       ActionTypeId:
+        #         Category: Deploy
+        #         Owner: AWS
+        #         Version: 1
+        #         Provider: CloudFormation
+        #       Configuration:
+        #         ActionMode: CREATE_UPDATE
+        #         RoleArn: !GetAtt CloudFormationDeployRole.Arn
+        #         StackName: !Sub ${EnvironmentName}-Resources
+        #         TemplatePath: Source::deployments/resources.yml
+        #         Capabilities: CAPABILITY_IAM
+        #         ParameterOverrides: !Sub |
+        #           {
+        #             "EnvironmentName": "${EnvironmentName}"
+        #           }
+        #       InputArtifacts:
+        #         - Name: Source
+
+        #     # Application load balancer
+        #     - Name: Load_Balancer
+        #       ActionTypeId:
+        #         Category: Deploy
+        #         Owner: AWS
+        #         Version: 1
+        #         Provider: CloudFormation
+        #       Configuration:
+        #         ActionMode: CREATE_UPDATE
+        #         RoleArn: !GetAtt CloudFormationDeployRole.Arn
+        #         StackName: !Sub ${EnvironmentName}-LoadBalancer
+        #         TemplatePath: Source::deployments/alb.yml
+        #         Capabilities: CAPABILITY_IAM
+        #         ParameterOverrides: !Sub |
+        #           {
+        #             "EnvironmentName": "${EnvironmentName}"
+        #           }
+        #       InputArtifacts:
+        #         - Name: Source
+
+        #     # ECS/Fargate cluster
+        #     - Name: Cluster
+        #       ActionTypeId:
+        #         Category: Deploy
+        #         Owner: AWS
+        #         Version: 1
+        #         Provider: CloudFormation
+        #       Configuration:
+        #         ActionMode: CREATE_UPDATE
+        #         RoleArn: !GetAtt CloudFormationDeployRole.Arn
+        #         StackName: !Sub ${EnvironmentName}-Cluster
+        #         TemplatePath: !Sub Source::deployments/cluster-${DeploymentType}.yml
+        #         Capabilities: CAPABILITY_IAM
+        #         ParameterOverrides: !Sub |
+        #           {
+        #             "EnvironmentName": "${EnvironmentName}"
+        #           }
+        #       InputArtifacts:
+        #         - Name: Source
+
         #-----------------------------------------------------------------------------#
-        # Infrastructure Resources
+        # Services
         #-----------------------------------------------------------------------------#
-        - Name: Network_Resources
+        - Name: Services
           Actions:
-            - Name: Deploy
-              ActionTypeId:
-                Category: Deploy
-                Owner: AWS
-                Version: 1
-                Provider: CloudFormation
-              Configuration:
-                ActionMode: CREATE_UPDATE
-                RoleArn: !GetAtt CloudFormationDeployRole.Arn
-                StackName: !Sub ${EnvironmentName}-Network
-                TemplatePath: Source::deployments/network.yml
-                Capabilities: CAPABILITY_IAM
-                ParameterOverrides: !Sub |
-                  {
-                    "EnvironmentName": "${EnvironmentName}"
-                  }
-              InputArtifacts:
-                - Name: Source
-              OutputArtifacts:
-                - Name: Network
 
-        - Name: Base_Resources
+            - Name: Server
-          Actions:
-            # Rabbit, Redis, and Postgres
-            - Name: Resources
               ActionTypeId:
                 Category: Deploy
                 Owner: AWS
@@ -111,20 +177,18 @@ Resources:
               Configuration:
                 ActionMode: CREATE_UPDATE
                 RoleArn: !GetAtt CloudFormationDeployRole.Arn
-                StackName: !Sub ${EnvironmentName}-Resources
+                StackName: !Sub ${EnvironmentName}-Server
-                TemplatePath: Source::deployments/resources.yml
+                TemplatePath: !Sub Source::deployments/services-${DeploymentType}/server.yml
                 Capabilities: CAPABILITY_IAM
                 ParameterOverrides: !Sub |
                   {
-                    "EnvironmentName": "${EnvironmentName}"
+                    "EnvironmentName": "${EnvironmentName}",
+                    "ServiceName": "server"
                   }
               InputArtifacts:
                 - Name: Source
-              OutputArtifacts:
-                - Name: Resources
 
-            # Application load balancer
+            - Name: Cache
-            - Name: Load_Balancer
               ActionTypeId:
                 Category: Deploy
                 Owner: AWS
@@ -133,20 +197,18 @@ Resources:
               Configuration:
                 ActionMode: CREATE_UPDATE
                 RoleArn: !GetAtt CloudFormationDeployRole.Arn
-                StackName: !Sub ${EnvironmentName}-LoadBalancer
+                StackName: !Sub ${EnvironmentName}-Cache
-                TemplatePath: Source::deployments/alb.yml
+                TemplatePath: !Sub Source::deployments/services-${DeploymentType}/cache.yml
                 Capabilities: CAPABILITY_IAM
                 ParameterOverrides: !Sub |
                   {
-                    "EnvironmentName": "${EnvironmentName}"
+                    "EnvironmentName": "${EnvironmentName}",
+                    "ServiceName": "cache"
                   }
               InputArtifacts:
                 - Name: Source
-              OutputArtifacts:
-                - Name: LoadBalancer
 
-            # ECS/Fargate cluster
+            - Name: Database
-            - Name: Cluster
               ActionTypeId:
                 Category: Deploy
                 Owner: AWS
@@ -155,17 +217,17 @@ Resources:
               Configuration:
                 ActionMode: CREATE_UPDATE
                 RoleArn: !GetAtt CloudFormationDeployRole.Arn
-                StackName: !Sub ${EnvironmentName}-Cluster
+                StackName: !Sub ${EnvironmentName}-Database
-                TemplatePath: !Sub Source::deployments/cluster-${DeploymentType}.yml
+                TemplatePath: !Sub Source::deployments/services-${DeploymentType}/database.yml
                 Capabilities: CAPABILITY_IAM
                 ParameterOverrides: !Sub |
                   {
-                    "EnvironmentName": "${EnvironmentName}"
+                    "EnvironmentName": "${EnvironmentName}",
+                    "ServiceName": "database"
                   }
               InputArtifacts:
                 - Name: Source
-              OutputArtifacts:
+
-                - Name: Cluster
 
         #-----------------------------------------------------------------------------#
         # Service Pipelines
@@ -182,7 +244,7 @@ Resources:
               Configuration:
                 ActionMode: CREATE_UPDATE
                 RoleArn: !GetAtt CloudFormationDeployRole.Arn
-                StackName: !Sub ${EnvironmentName}-Server
+                StackName: !Sub ${EnvironmentName}-ServerPipeline
                 TemplatePath: Source::deployments/pipeline-service.yml
                 Capabilities: CAPABILITY_IAM
                 ParameterOverrides: !Sub |
@@ -208,7 +270,7 @@ Resources:
               Configuration:
                 ActionMode: CREATE_UPDATE
                 RoleArn: !GetAtt CloudFormationDeployRole.Arn
-                StackName: !Sub ${EnvironmentName}-Cache
+                StackName: !Sub ${EnvironmentName}-CachePipeline
                 TemplatePath: Source::deployments/pipeline-service.yml
                 Capabilities: CAPABILITY_IAM
                 ParameterOverrides: !Sub |
@@ -234,7 +296,7 @@ Resources:
               Configuration:
                 ActionMode: CREATE_UPDATE
                 RoleArn: !GetAtt CloudFormationDeployRole.Arn
-                StackName: !Sub ${EnvironmentName}-Database
+                StackName: !Sub ${EnvironmentName}-DatabasePipeline
                 TemplatePath: Source::deployments/pipeline-service.yml
                 Capabilities: CAPABILITY_IAM
                 ParameterOverrides: !Sub |
@@ -300,6 +362,22 @@ Resources:
                   - cloudformation:ValidateTemplate
                   - cloudformation:ExecuteChangeSet
                 Resource: "*"
+              # Allow codepipeline to get images from ECR
+              - Effect: Allow
+                Action:
+                  - ecr:GetAuthorizationToken
+                  - ecr:BatchCheckLayerAvailability
+                  - ecr:GetDownloadUrlForLayer
+                  - ecr:GetRepositoryPolicy
+                  - ecr:DescribeRepositories
+                  - ecr:ListImages
+                  - ecr:DescribeImages
+                  - ecr:BatchGetImage
+                  - ecr:GetLifecyclePolicy
+                  - ecr:GetLifecyclePolicyPreview
+                  - ecr:ListTagsForResource
+                  - ecr:DescribeImageScanFindings
+                Resource: "*"
 
   # This role is passed by CodePipeline to CloudFormation to use
   # when setting up resources in the pipeline
@@ -337,54 +415,22 @@ Resources:
                 Resource: "*"
 
   #-----------------------------------------------------------------------------#
-  # Role for CodeBuild service
+  # Parameter Store
   #-----------------------------------------------------------------------------#
-  CodeBuildServiceRole:
+  SSMArtifactBucket:
-    Type: AWS::IAM::Role
+    Type: AWS::SSM::Parameter
     Properties:
-      Path: /
+      Name: /microservices/artifact_bucket
-      AssumeRolePolicyDocument:
+      Type: String
-        Version: 2012-10-17
+      Value: !Ref ArtifactBucket
-        Statement:
+  SSMCodePipelineServiceRole:
-          - Effect: Allow
+    Type: AWS::SSM::Parameter
-            Principal:
+    Properties:
-              Service: codebuild.amazonaws.com
+      Name: /microservices/codepipeline_service_role
-            Action: sts:AssumeRole
+      Type: String
-      Policies:
+      Value: !Ref CodePipelineServiceRole
-        - PolicyName: root
+
-          PolicyDocument:
-            Version: 2012-10-17
-            Statement:
-              - Resource: "*"
-                Effect: Allow
-                Action:
-                  - logs:CreateLogGroup
-                  - logs:CreateLogStream
-                  - logs:PutLogEvents
-                  - ecr:GetAuthorizationToken
-              - Resource: !Sub arn:aws:s3:::${ArtifactBucket}/*
-                Effect: Allow
-                Action:
-                  - s3:GetObject
-                  - s3:PutObject
-                  - s3:GetObjectVersion
-              - Resource: "*"
-                Effect: Allow
-                Action:
-                  - ecr:GetDownloadUrlForLayer
-                  - ecr:BatchGetImage
-                  - ecr:BatchCheckLayerAvailability
-                  - ecr:PutImage
-                  - ecr:InitiateLayerUpload
-                  - ecr:UploadLayerPart
-                  - ecr:CompleteLayerUpload
 
 Outputs:
   PipelineUrl:
     Value: !Sub https://console.aws.amazon.com/codepipeline/home?region=${AWS::Region}#/view/${Pipeline}
-  ArtifactBucket:
-    Description: The bucket to store pipeline artifacts
-    Value: !Ref ArtifactBucket
-    Export:
-      Name: !Sub ${EnvironmentName}:ArtifactBucket
-  

deployments/pipeline-service.yml

@@ -23,6 +23,12 @@ Parameters:
     Type: String
     Description: A commit message that triggers the build process
     Default: \[BuildAll\]
+  ArtifactBucket:
+    Type: AWS::SSM::Parameter::Value<String>
+    Default: /microservices/artifact_bucket
+  CodePipelineServiceRole:
+    Type: AWS::SSM::Parameter::Value<String>
+    Default: /microservices/codepipeline_service_role
 
 Resources:
 
@@ -34,14 +40,6 @@ Resources:
     Properties:
       RepositoryName: !Sub ${EnvironmentName}/${ServiceName}
 
-  #-----------------------------------------------------------------------------#
-  # Artifact Bucket
-  #-----------------------------------------------------------------------------#
-  ArtifactBucket:
-    Type: AWS::S3::Bucket
-    Properties:
-      BucketName: !Sub microservices-services-${AWS::AccountId}
-
   #-----------------------------------------------------------------------------#
   # CodeBuild Project
   #-----------------------------------------------------------------------------#
@@ -135,7 +133,7 @@ Resources:
   Pipeline:
     Type: AWS::CodePipeline::Pipeline
     Properties:
-      RoleArn: !GetAtt CodePipelineServiceRole.Arn
+      RoleArn: !Ref CodePipelineServiceRole
       ArtifactStore:
         Type: S3
         Location: !Ref ArtifactBucket
@@ -146,8 +144,9 @@ Resources:
         #-----------------------------------------------------------------------------#
         - Name: Source
           Actions:
-            - Name: Source
+
-              Namespace: SourceVariables
+            - Name: Image
+              Namespace: ImageVariables
               ActionTypeId:
                 Category: Source
                 Owner: AWS
@@ -157,7 +156,7 @@ Resources:
                 RepositoryName: !Ref ServerRepository
                 ImageTag: latest
               OutputArtifacts:
-                - Name: SourceImage
+                - Name: Image
               RunOrder: 1
 
             - Name: Code
@@ -173,7 +172,7 @@ Resources:
                 OAuthToken: !Ref GitHubToken
                 PollForSourceChanges: false
               OutputArtifacts:
-                - Name: SourceCode
+                - Name: Code
               RunOrder: 1
 
         #-----------------------------------------------------------------------------#
@@ -186,122 +185,37 @@ Resources:
                 Category: Deploy
                 Owner: AWS
                 Version: 1
-                Provider: CloudFormation
+                Provider: ECS
               Configuration:
-                ActionMode: CREATE_UPDATE
+                ClusterName:
-                RoleArn: !GetAtt CloudFormationDeployRole.Arn
+                  Fn::ImportValue: !Sub ${EnvironmentName}:ClusterName
-                StackName: !Sub ${EnvironmentName}-ServerService
+                ServiceName: !Ref ServiceName
-                TemplatePath: !Sub SourceCode::deployments/services-${DeploymentType}/server.yml
+                FileName: imageDetail.json
-                Capabilities: CAPABILITY_IAM
-                ParameterOverrides: !Sub |
-                  {
-                    "EnvironmentName": "${EnvironmentName}",
-                    "ImageUrl": "${ServerRepository.RepositoryUri}@#{SourceVariables.ImageDigest}"
-                  }
               InputArtifacts:
-                - Name: SourceImage
+                - Name: Image
-                - Name: SourceCode
 
-  #-----------------------------------------------------------------------------#
+        # - Name: Deploy
-  # Role for CodePipeline service
+        #   Actions:
-  #-----------------------------------------------------------------------------#
+        #     - Name: Deploy
-  CodePipelineServiceRole:
+        #       ActionTypeId:
-    Type: AWS::IAM::Role
+        #         Category: Deploy
-    Properties:
+        #         Owner: AWS
-      Path: /
+        #         Version: 1
-      AssumeRolePolicyDocument:
+        #         Provider: CloudFormation
-        Version: 2012-10-17
+        #       Configuration:
-        Statement:
+        #         ActionMode: CREATE_UPDATE
-          - Effect: Allow
+        #         RoleArn: !GetAtt CloudFormationDeployRole.Arn
-            Principal:
+        #         StackName: !Sub ${EnvironmentName}-ServerService
-              Service: codepipeline.amazonaws.com
+        #         TemplatePath: !Sub Code::deployments/services-${DeploymentType}/server.yml
-            Action: sts:AssumeRole
+        #         Capabilities: CAPABILITY_IAM
-      Policies:
+        #         ParameterOverrides: !Sub |
-        - PolicyName: root
+        #           {
-          PolicyDocument:
+        #             "EnvironmentName": "${EnvironmentName}",
-            Version: 2012-10-17
+        #             "ImageUrl": "${ServerRepository.RepositoryUri}@#{ImageVariables.ImageDigest}"
-            Statement:
+        #           }
-              # Allow codepipeline to put artifacts in the S3 bucket
+        #       InputArtifacts:
-              # as well as get artifacts back out of it.
+        #         - Name: Image
-              - Resource:
+        #         - Name: Code
-                  - !Sub arn:aws:s3:::${ArtifactBucket}/*
-                Effect: Allow
-                Action:
-                  - s3:PutObject
-                  - s3:GetObject
-                  - s3:GetObjectVersion
-                  - s3:GetBucketVersioning
-              # Allow codepipeline to build code builds
-              - Resource: "*"
-                Effect: Allow
-                Action:
-                  - codebuild:StartBuild
-                  - codebuild:BatchGetBuilds
-                  - iam:PassRole
-              # Allow codepipeline to deploy cloudformation stacks
-              - Effect: Allow
-                Action:
-                  - cloudformation:CreateChangeSet
-                  - cloudformation:CreateStack
-                  - cloudformation:CreateUploadBucket
-                  - cloudformation:DeleteStack
-                  - cloudformation:Describe*
-                  - cloudformation:List*
-                  - cloudformation:UpdateStack
-                  - cloudformation:ValidateTemplate
-                  - cloudformation:ExecuteChangeSet
-                Resource: "*"
-              # Allow codepipeline to get images from ECR
-              - Effect: Allow
-                Action:
-                  - ecr:GetAuthorizationToken
-                  - ecr:BatchCheckLayerAvailability
-                  - ecr:GetDownloadUrlForLayer
-                  - ecr:GetRepositoryPolicy
-                  - ecr:DescribeRepositories
-                  - ecr:ListImages
-                  - ecr:DescribeImages
-                  - ecr:BatchGetImage
-                  - ecr:GetLifecyclePolicy
-                  - ecr:GetLifecyclePolicyPreview
-                  - ecr:ListTagsForResource
-                  - ecr:DescribeImageScanFindings
-                Resource: "*"
-
-  # This role is passed by CodePipeline to CloudFormation to use
-  # when setting up resources in the pipeline
-  CloudFormationDeployRole:
-    Type: AWS::IAM::Role
-    Properties:
-      Path: /
-      AssumeRolePolicyDocument:
-        Version: 2012-10-17
-        Statement:
-          - Effect: Allow
-            Principal:
-              Service: cloudformation.amazonaws.com
-            Action: sts:AssumeRole
-      Policies:
-        - PolicyName: deploy-stack
-          PolicyDocument:
-            Statement:
-              - Effect: Allow
-                Action:
-                  - "iam:*"
-                  - "ec2:*"
-                  - "ecs:*"
-                  - "elasticloadbalancing:*"
-                  - "autoscaling:*"
-                  - "elasticache:*"
-                  - "logs:*"
-                  - "application-autoscaling:*"
-                  - "cloudwatch:*"
-                  - "route53:*"
-                  - "rds:*"
-                  - "mq:*"
-                  # - "secretsmanager:*"
-                  - "ssm:*"
-                Resource: "*"
 
   #-----------------------------------------------------------------------------#
   # Role for CodeBuild service

deployments/services-fargate/cache.yml

@@ -64,7 +64,7 @@ Resources:
           Cpu: !Ref 'ContainerCpu'
           Memory: !Ref 'ContainerMemory'
           # Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/microservices/${ServiceName}:latest
-          Image: !Ref ImageUrl
+          # Image: !Ref ImageUrl
           LogConfiguration:
             LogDriver: 'awslogs'
             Options:

deployments/services-fargate/database.yml

@@ -64,7 +64,7 @@ Resources:
           Cpu: !Ref 'ContainerCpu'
           Memory: !Ref 'ContainerMemory'
           # Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/microservices/${ServiceName}:latest
-          Image: !Ref ImageUrl
+          # Image: !Ref ImageUrl
           LogConfiguration:
             LogDriver: 'awslogs'
             Options:

deployments/services-fargate/server.yml

@@ -80,7 +80,7 @@ Resources:
           Cpu: !Ref 'ContainerCpu'
           Memory: !Ref 'ContainerMemory'
           # Image: !Sub ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/microservices/${ServiceName}:latest
-          Image: !Ref ImageUrl
+          # Image: !Ref ImageUrl
           PortMappings:
             - ContainerPort: !Ref 'ContainerPort'
           LogConfiguration: