Automated patching for arclight

infrastructure/cinco/config/dags-deploy.yaml

@@ -2,11 +2,13 @@ template:
   path: codebuild.j2
   type: file
 parameters:
-  GitHubRepo: https://github.com/ucldc/cinco
   Namespace: cinco-dags-deploy
-  Branch: main
 
 sceptre_user_data:
+  Source:
+    Type: GITHUB
+    GitHubRepo: https://github.com/ucldc/cinco
+    Branch: main
   CodeBuildPolicies:
     - arn:aws:iam::777968769372:policy/cinco-dags-sync
   TriggerFilterGroups:

infrastructure/cinco/config/dev/arclight/build.yaml

@@ -0,0 +1,39 @@
+template:
+  path: codebuild.j2
+  type: file
+parameters:
+  Namespace: cinco-arclight-dev
+
+sceptre_user_data:
+  ECRRepositoryName: cinco-arclight
+  BuildEnvironment:
+    - SUBNET_IDS: {{ var.dev.SUBNET_IDS }}
+  Source:
+    Type: CODEPIPELINE
+  build_spec: |-
+    !Sub >-
+              version: 0.2
+
+              phases:
+                build:
+                  commands:
+                    - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${AWS::AccountId}.dkr.ecr.us-west-2.amazonaws.com
+                    - docker build -t cinco-arclight arclight --file arclight/Dockerfile
+                    - docker tag cinco-arclight:latest ${AWS::AccountId}.dkr.ecr.us-west-2.amazonaws.com/cinco-arclight:latest
+                    - docker push ${AWS::AccountId}.dkr.ecr.us-west-2.amazonaws.com/cinco-arclight:latest
+                post_build:
+                  commands:
+                    - docker run -d --name arclight-tail-dev-null cinco-arclight tail -f /dev/null
+                    - docker cp arclight-tail-dev-null:/rails/public/assets ./public_assets
+                    - aws s3 cp --recursive ./public_assets s3://cinco-dev-tmp/arclight_assets/assets
+                    - docker stop arclight-tail-dev-null
+  additional_policy_statements: |-
+    - Effect: Allow
+                Action:
+                  - s3:PutObject
+                  - s3:GetObject
+                  - s3:GetObjectVersion
+                  - s3:GetBucketAcl
+                  - s3:GetBucketLocation
+                Resource:
+                  - !Sub arn:aws:s3:::cinco-dev*

infrastructure/cinco/config/dev/arclight/code-pipeline.yaml

@@ -0,0 +1,41 @@
+template:
+  path: code-pipeline.yaml.j2
+  type: file
+parameters:
+  Namespace: cinco-arclight-dev
+  SourceRepositoryId: ucldc/cinco
+  ChangeDetectionBranchName: infrastructure-arclight-auto-patching
+  CodeConnectionArn: arn:aws:codeconnections:us-west-2:866216109762:connection/fa425c08-dc3a-40c2-b45d-adb3bc6f91c7
+  CodeBuildRoleName: !stack_output_external cinco-dev-arclight-build::CodeBuildRole
+  CodeBuildProjectName: cinco-arclight-dev
+
+sceptre_user_data:
+  GitPushFilter:
+    - Branches:
+        Includes:
+          - infrastructure-arclight-auto-patching
+      FilePaths:
+        Includes:
+          - arclight/**
+  ### prod git push filter
+  # there is no native release event for CodePipeline, so we trigger on tags starting with 'v' for version
+  # GitPushFilter:
+  #   - Tags:
+  #       Includes:
+  #         - v*
+  ### stage git push filter
+  # GitPushFilter:
+  #   - Branches:
+  #       Includes:
+  #         - main
+  #     FilePaths:
+  #       Includes:
+  #         - arclight/**
+
+# For patching, we want to rebuild and deploy the container using the same dockerfile, code version, or env vars
+# Steps:
+#   - use describeServiceDeployments API to get info on what's currently deployed: stage==Completed, state==SUCCESSFUL
+#   - get currently deployed task definition id (targetServiceRevision/arn)
+#   - get image build details for that task definition using describe tasks API
+#   - we need to use the git hash in our tags for stage
+# https://aws.amazon.com/blogs/containers/improving-deployment-visibility-for-amazon-ecs-services/

infrastructure/cinco/config/dev/arclight/patching-scheduler.yaml

@@ -0,0 +1,10 @@
+template:
+  path: lambda-scheduler.yaml
+  type: file
+parameters:
+  Namespace: cinco-arclight-dev-patching
+  TargetLambdaArn: !stack_output_external cinco-patching-lambda-function::LambdaFunctionArn
+  Description: Scheduler for cinco arclight dev regular patching
+  # Tuesdays at 10:30am
+  ScheduleExpression: "cron(30 10 ? * 3 *)"
+  Input: '{"pipeline": "cinco-arclight-dev-pipeline", "cluster": "cinco-stage", "service": "cinco-arclight-dev-service", "gitrepo": "ucldc/cinco"}'

infrastructure/cinco/config/dev/s3.yaml

@@ -0,0 +1,5 @@
+template:
+  path: public-s3.yaml
+  type: file
+parameters:
+  BucketName: {{ var.dev.S3_BUCKET_NAME }}

infrastructure/cinco/config/patching-lambda-codebuild.yaml

@@ -0,0 +1,48 @@
+template:
+  path: codebuild.j2
+  type: file
+parameters:
+  Namespace: cinco-trigger-ecs-patching-pipeline-codebuild
+
+sceptre_user_data:
+  Source:
+    Type: GITHUB
+    GitHubRepo: https://github.com/ucldc/cinco
+    Branch: main
+  ECRRepositoryName: cinco-trigger-ecs-patching-pipeline
+  BuildEnvironment:
+    - SUBNET_IDS: {{ var.stage.SUBNET_IDS }}
+  TriggerFilterGroups:
+    - - Type: EVENT
+        Pattern: PUSH
+      - Type: HEAD_REF
+        Pattern: ^refs/heads/main$
+        ExcludeMatchedPattern: false
+      - Type: FILE_PATH
+        Pattern: "infrastructure/lambdas/trigger-ecs-patching-pipeline/*"
+        ExcludeMatchedPattern: false
+  build_spec: |-
+    !Sub >-
+              version: 0.2
+
+              phases:
+                pre_build:
+                  commands:
+                    - cd $CODEBUILD_SRC_DIR/lambdas/trigger-ecs-patching-pipeline
+                build:
+                  commands:
+                    # login to ECR public so we can pull lambda image
+                    - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
+                    - docker build --platform linux/amd64 -t ${Namespace} .
+                    # login to ECR private so we can push new image
+                    - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${AWS::AccountId}.dkr.ecr.us-west-2.amazonaws.com
+                    - docker tag ${Namespace}:latest ${AWS::AccountId}.dkr.ecr.us-west-2.amazonaws.com/${Namespace}:latest
+                    - docker push ${AWS::AccountId}.dkr.ecr.us-west-2.amazonaws.com/${Namespace}:latest
+                    # update lambda function with new image
+                    - aws lambda update-function-code --function-name trigger-ecs-patching-pipeline-lambda-function --image-uri ${AWS::AccountId}.dkr.ecr.us-west-2.amazonaws.com/${Namespace}:latest
+  additional_policy_statements: |-
+    - Effect: Allow
+                Action:
+                  - lambda:GetLayerVersion
+                  - lambda:UpdateFunctionCode
+                Resource: "*"

infrastructure/cinco/config/patching-lambda-function.yaml

@@ -0,0 +1,26 @@
+template:
+  path: lambda-function.j2
+  type: file
+parameters:
+  Namespace: cinco-trigger-ecs-patching-pipeline
+  VpcId: {{ var.prd.VPC_ID }}
+  FunctionDescription: Trigger patching pipeline for an ECS Service
+  ECRRepositoryName: cinco-trigger-ecs-patching-pipeline
+  SubnetId1: subnet-08a31a6d2aff9ac32
+  SubnetId2: subnet-09e65806b80ebad6b
+  Timeout: '20'
+
+sceptre_user_data:
+  additional_policy_statements: |-
+    - Effect: Allow
+                Action:
+                  - ecs:ListServiceDeployments
+                  - ecs:DescribeServiceRevisions
+                  - ecs:DescribeTaskDefinition
+                Resource:
+                  - !Sub arn:aws:ecs:${AWS::Region}:${AWS::AccountId}:service/*
+              - Effect: Allow
+                Action:
+                  - codepipeline:StartPipelineExecution
+                Resource:
+                  - !Sub arn:aws:codepipeline:${AWS::Region}:${AWS::AccountId}:*

infrastructure/cinco/config/prd/arclight/build.yaml

@@ -2,12 +2,13 @@ template:
   path: codebuild.j2
   type: file
 parameters:
-  GitHubRepo: https://github.com/ucldc/cinco
   Namespace: cinco-arclight-prd
-  Branch: ''
   ECRRepositoryURI: 777968769372.dkr.ecr.us-west-2.amazonaws.com/cinco-arclight
 
 sceptre_user_data:
+  Source:
+    Type: GITHUB
+    GitHubRepo: https://github.com/ucldc/cinco
   BuildEnvironment:
     - SUBNET_IDS: {{ var.prd.SUBNET_IDS }}
     - SLACK_HOOK: {{ var.prd.SLACK_HOOK }}
@@ -48,6 +49,14 @@ sceptre_user_data:
                     - aws ecs update-service --cluster cinco-prd --service cinco-arclight-prd-service --task-definition $TASK_FAMILY:$TD_REVISION --force-new-deployment
   additional_policy_statements: |-
     - Effect: Allow
+                Action:
+                  - ecs:RunTask
+                  - ecs:DescribeTasks
+                  - ecs:DescribeTaskDefinition
+                  - ecs:RegisterTaskDefinition
+                  - ecs:UpdateService
+                Resource: "*"
+              - Effect: Allow
                 Action:
                   - s3:PutObject
                   - s3:GetObject

infrastructure/cinco/config/prd/cincoctrl/build.yaml

@@ -2,20 +2,21 @@ template:
   path: codebuild.j2
   type: file
 parameters:
-  GitHubRepo: https://github.com/ucldc/cinco
   Namespace: cinco-ctrl-prd
-  Branch: ''
   ECRRepositoryURI: 777968769372.dkr.ecr.us-west-2.amazonaws.com/cinco-ctrl
 
 sceptre_user_data:
+  Source:
+    Type: GITHUB
+    GitHubRepo: https://github.com/ucldc/cinco
   BuildEnvironment:
-    - SUBNET_IDS: {{ var.prd.SUBNET_IDS }}
-    - SLACK_HOOK: {{ var.prd.SLACK_HOOK }}
+  - SUBNET_IDS: {{ var.prd.SUBNET_IDS }}
+  - SLACK_HOOK: {{ var.prd.SLACK_HOOK }}
   TriggerFilterGroups:
   # A build is triggered if any filter group evaluates to true, which
   # occurs when all the filters in the group evaluate to true.
-    - - Type: EVENT
-        Pattern: RELEASED
+  - - Type: EVENT
+      Pattern: RELEASED
   build_spec: |-
     !Sub >-
               version: 0.2
@@ -49,3 +50,12 @@ sceptre_user_data:
                     - echo "Forcing a new deployment of the service to apply changes"
                     - python $CODEBUILD_SRC_DIR/infrastructure/cinco/scripts/message_slack.py --application cincoctrl --cinco_version $TAG --td_family_revisions "$TASK_FAMILY:$TD_REVISION"
                     - aws ecs update-service --cluster cinco-prd --service cinco-ctrl-prd-service --task-definition $TASK_FAMILY:$TD_REVISION --force-new-deployment
+  additional_policy_statements: |-
+    - Effect: Allow
+                Action:
+                  - ecs:RunTask
+                  - ecs:DescribeTasks
+                  - ecs:DescribeTaskDefinition
+                  - ecs:RegisterTaskDefinition
+                  - ecs:UpdateService
+                Resource: "*"

infrastructure/cinco/config/prd/solr/build.yaml

@@ -2,12 +2,13 @@ template:
   path: codebuild.j2
   type: file
 parameters:
-  GitHubRepo: https://github.com/ucldc/cinco
   Namespace: cinco-solr-prd
-  Branch: ''
   ECRRepositoryURI: !stack_output stage/solr/build.yaml::ECRRepository
 
 sceptre_user_data:
+  Source:
+    Type: GITHUB
+    GitHubRepo: https://github.com/ucldc/cinco
   TriggerFilterGroups:
   # A build is triggered if any filter group evaluates to true, which
   # occurs when all the filters in the group evaluate to true.
@@ -71,3 +72,13 @@ sceptre_user_data:
                     # - aws ecs update-service --cluster cinco-prd --service cinco-solr-follower-2-prd-service --task-definition $TASK_FAMILY:$TD_REVISION --force-new-deployment
                     ###### Message Slack #####
                     - python $CODEBUILD_SRC_DIR/infrastructure/cinco/scripts/message_slack.py --application solr --cinco_version $TAG --td_family_revisions $LEADER $FOLLOWER_1 $FOLLOWER_2
+  additional_policy_statements: |-
+    - Effect: Allow
+                Action:
+                  - ecs:RunTask
+                  - ecs:DescribeTasks
+                  - ecs:DescribeTaskDefinition
+                  - ecs:RegisterTaskDefinition
+                  - ecs:UpdateService
+
+                Resource: "*"

infrastructure/cinco/config/stage/arclight/build.yaml

@@ -2,11 +2,13 @@ template:
   path: codebuild.j2
   type: file
 parameters:
-  GitHubRepo: https://github.com/ucldc/cinco
   Namespace: cinco-arclight-stage
-  Branch: main
 
 sceptre_user_data:
+  Source:
+    Type: GITHUB
+    GitHubRepo: https://github.com/ucldc/cinco
+    Branch: main
   ECRRepositoryName: cinco-arclight
   BuildEnvironment:
     - SUBNET_IDS: {{ var.stage.SUBNET_IDS }}
@@ -39,6 +41,14 @@ sceptre_user_data:
                     - aws ecs update-service --cluster cinco-stage --service cinco-arclight-stage-service --force-new-deployment
   additional_policy_statements: |-
     - Effect: Allow
+                Action:
+                  - ecs:RunTask
+                  - ecs:DescribeTasks
+                  - ecs:DescribeTaskDefinition
+                  - ecs:RegisterTaskDefinition
+                  - ecs:UpdateService
+                Resource: "*"
+              - Effect: Allow
                 Action:
                   - s3:PutObject
                   - s3:GetObject

infrastructure/cinco/config/stage/cincoctrl/build.yaml

@@ -2,11 +2,13 @@ template:
   path: codebuild.j2
   type: file
 parameters:
-  GitHubRepo: https://github.com/ucldc/cinco
   Namespace: cinco-ctrl-stage
-  Branch: main
 
 sceptre_user_data:
+  Source:
+    Type: GITHUB
+    GitHubRepo: https://github.com/ucldc/cinco
+    Branch: main
   ECRRepositoryName: cinco-ctrl
   BuildEnvironment:
     - SUBNET_IDS: {{ var.stage.SUBNET_IDS }}
@@ -39,3 +41,13 @@ sceptre_user_data:
                     - python $CODEBUILD_SRC_DIR/infrastructure/cinco/scripts/ecs_manage.py collectstatic --no-input
                     - python $CODEBUILD_SRC_DIR/infrastructure/cinco/scripts/ecs_manage.py migrate --no-input
                     - aws ecs update-service --cluster cinco-stage --service cinco-ctrl-stage-service --force-new-deployment
+  additional_policy_statements: |-
+    - Effect: Allow
+                Action:
+                  - ecs:RunTask
+                  - ecs:DescribeTasks
+                  - ecs:DescribeTaskDefinition
+                  - ecs:RegisterTaskDefinition
+                  - ecs:UpdateService
+
+                Resource: "*"

infrastructure/cinco/config/stage/solr/build.yaml

@@ -2,11 +2,13 @@ template:
   path: codebuild.j2
   type: file
 parameters:
-  GitHubRepo: https://github.com/ucldc/cinco
   Namespace: cinco-solr-stage
-  Branch: main
 
 sceptre_user_data:
+  Source:
+    Type: GITHUB
+    GitHubRepo: https://github.com/ucldc/cinco
+    Branch: main
   ECRRepositoryName: cinco-solr
   TriggerFilterGroups:
     - - Type: EVENT
@@ -34,3 +36,11 @@ sceptre_user_data:
                 post_build:
                   commands:
                     - aws ecs update-service --cluster cinco-stage --service cinco-solr-stage-service --force-new-deployment
+  additional_policy_statements: |-
+    - Effect: Allow
+                Action:
+                  - ecs:RunTask
+                  - ecs:DescribeTasks
+                  - ecs:DescribeTaskDefinition
+                  - ecs:RegisterTaskDefinition
+                  - ecs:UpdateService