Deploy CloudFront distributions in front of CopWatch

.github/workflows/deploy.yaml

@@ -2,7 +2,7 @@ name: deploy
 
 on:
   push:
-    branches: [main, develop, CU-8686edfh0-performance]
+    branches: [main, develop, CU-86897a79g-cache-misses]
 
 jobs:
   deploy:

Dockerfile

@@ -15,14 +15,12 @@ ARG APP_USER=appuser
 RUN groupadd -r ${APP_USER} && useradd --no-log-init -r -g ${APP_USER} ${APP_USER}
 
 # Install packages needed to run your application (not build deps):
-#   mime-support -- for mime types when serving static files
 #   postgresql-client -- for running database commands
 # We need to recreate the /usr/share/man/man{1..8} directories first because
 # they were clobbered by a parent image.
 RUN set -ex \
     && RUN_DEPS=" \
     libpcre3 \
-    mime-support \
     postgresql-client \
     vim \
     " \
@@ -127,7 +125,6 @@ RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/
     jq \
     libpcre3 \
     libpq-dev \
-    mime-support \
     nodejs \
     openssh-client \
     postgresql-client-12 \

deploy/deploy-cf-stack.yml

@@ -15,3 +15,24 @@
           unexpectedly), execute the changeset. When complete, hit RETURN to continue or Control-C,
           then A to abort.
       when: cloudformation_stack_result is changed
+
+- name: kubernetes cluster management
+  hosts: cluster
+  gather_facts: false
+  tags: cdn
+  vars:
+    ansible_connection: local
+    ansible_python_interpreter: "{{ ansible_playbook_python }}"
+  vars_files:
+    - group_vars/cdn.yml
+  roles:
+    - role: caktus.aws-web-stacks
+  tasks:
+    - name: Prompt if CloudFormation stack changed
+      pause:
+        prompt: >
+          A changeset was created for the CloudFormation stack. You MUST login to the AWS console,
+          inspect the changeset, and if everything looks good (i.e., it's not DELETING anything
+          unexpectedly), execute the changeset. When complete, hit RETURN to continue or Control-C,
+          then A to abort.
+      when: cloudformation_stack_result is changed

deploy/group_vars/all.yml

@@ -25,39 +25,39 @@ admin_database_password: !vault |
 
 stack_name: "{{ app_name }}-stack"
 
-cloudformation_stack:
-  region: "{{ aws_region }}"
-  stack_name: "{{ stack_name }}"
-  template_bucket: "aws-web-stacks-{{ app_name }}"
-  # Webstacks required variable
-  template_local_path: '{{ playbook_dir + "/stack/eks-no-nat.yml" }}'
-  create_changeset: true
-  termination_protection: true
-
-  template_parameters:
-    PrimaryAZ: "{{ aws_region }}a"
-    SecondaryAZ: "{{ aws_region }}b"
-    DesiredScale: 2
-    MaxScale: 4
-    UseAES256Encryption: "true"
-    CustomerManagedCmkArn: ""
-    ContainerInstanceType: t3a.large
-    ContainerVolumeSize: 40
-    DatabaseAllocatedStorage: 100
-    DatabaseClass: db.t3.large
-    DatabaseEngineVersion: "12"
-    DatabaseParameterGroupFamily: postgres12
-    DatabaseMultiAZ: "false"
-    DatabaseUser: "{{ app_name }}_admin"
-    DatabasePassword: "{{ admin_database_password }}"
-    DatabaseName: "{{ app_name }}"
-    DomainName: nccopwatch.org
-    DomainNameAlternates: ""
-    AssetsCloudFrontDomain: files.nccopwatch.org
-    AssetsCloudFrontCertArn: arn:aws:acm:us-east-1:606178775542:certificate/379950bb-4b29-4308-8418-122674fe1076
-    AssetsUseCloudFront: "true"
-  tags:
-    Environment: "{{ app_name }}"
+# cloudformation_stack:
+#   region: "{{ aws_region }}"
+#   stack_name: "{{ stack_name }}"
+#   template_bucket: "aws-web-stacks-{{ app_name }}"
+#   # Webstacks required variable
+#   template_local_path: '{{ playbook_dir + "/stack/eks-no-nat.yml" }}'
+#   create_changeset: true
+#   termination_protection: true
+
+#   template_parameters:
+#     PrimaryAZ: "{{ aws_region }}a"
+#     SecondaryAZ: "{{ aws_region }}b"
+#     DesiredScale: 2
+#     MaxScale: 4
+#     UseAES256Encryption: "true"
+#     CustomerManagedCmkArn: ""
+#     ContainerInstanceType: t3a.large
+#     ContainerVolumeSize: 40
+#     DatabaseAllocatedStorage: 100
+#     DatabaseClass: db.t3.large
+#     DatabaseEngineVersion: "12"
+#     DatabaseParameterGroupFamily: postgres12
+#     DatabaseMultiAZ: "false"
+#     DatabaseUser: "{{ app_name }}_admin"
+#     DatabasePassword: "{{ admin_database_password }}"
+#     DatabaseName: "{{ app_name }}"
+#     DomainName: nccopwatch.org
+#     DomainNameAlternates: ""
+#     AssetsCloudFrontDomain: files.nccopwatch.org
+#     AssetsCloudFrontCertArn: arn:aws:acm:us-east-1:606178775542:certificate/379950bb-4b29-4308-8418-122674fe1076
+#     AssetsUseCloudFront: "true"
+#   tags:
+#     Environment: "{{ app_name }}"
 
 # Install Descheduler to attempt to spread out pods again after node failures
 k8s_install_descheduler: yes

deploy/group_vars/cdn.yml

@@ -0,0 +1,18 @@
+cloudformation_stack_state: present
+cloudformation_stack_profile: '{{ aws_profile }}'
+cloudformation_stack_region: '{{ aws_region }}'
+cloudformation_stack_name: 'cdn-staging'
+cloudformation_stack_template_bucket: 'aws-web-stacks-trafficstops'
+cloudformation_stack_template_local_path: '{{ playbook_dir + "/stack/cloudfront.yml" }}'
+cloudformation_stack_template_parameters:
+  AppCloudFrontCertArn: arn:aws:acm:us-east-1:606178775542:certificate/379950bb-4b29-4308-8418-122674fe1076
+  AppCloudFrontForwardedHeaders: "Host,Authorization"
+  DomainName: staging.nccopwatch.org
+  AppCloudFrontOriginDomainName: staging-origin.nccopwatch.org
+  AppCloudFrontRoleArn: cdn-staging
+  # Required for the CloudFront template
+  DomainNameAlternates: ""
+  CustomAppCertificateArn: ""
+  CertificateValidationMethod: "(none)"
+cloudformation_stack_tags:
+  Environment: staging

deploy/stack/cloudfront.yml

@@ -0,0 +1,226 @@
+# This Cloudformation stack template was generated by
+# https://github.com/caktus/aws-web-stacks
+# at 2024-10-03 10:18:42.500933
+# with parameters:
+#	USE_CLOUDFRONT = on
+
+Conditions:
+  AppCloudFrontCertArnCondition: !Not
+    - !Equals
+      - !Ref 'AppCloudFrontCertArn'
+      - ''
+  AppCloudFrontForwardedHeadersCondition: !Not
+    - !Equals
+      - !Join
+        - ''
+        - !Ref 'AppCloudFrontForwardedHeaders'
+      - ''
+  CertificateCondition: !Or
+    - !Not
+      - !Equals
+        - !Ref 'CustomAppCertificateArn'
+        - ''
+    - !Not
+      - !Equals
+        - !Ref 'CertificateValidationMethod'
+        - (none)
+  CustomAppCertArnCondition: !Not
+    - !Equals
+      - !Ref 'CustomAppCertificateArn'
+      - ''
+  NoAlternateDomains: !Equals
+    - !Join
+      - ''
+      - !Ref 'DomainNameAlternates'
+    - ''
+  StackCertificateCondition: !Not
+    - !Equals
+      - !Ref 'CertificateValidationMethod'
+      - (none)
+  UsEast1Condition: !Equals
+    - !Ref 'AWS::Region'
+    - us-east-1
+Metadata:
+  AWS::CloudFormation::Interface:
+    ParameterGroups:
+      - Label:
+          default: Global
+        Parameters:
+          - DomainName
+          - DomainNameAlternates
+          - CertificateValidationMethod
+          - CustomAppCertificateArn
+      - Label:
+          default: Application Server
+        Parameters:
+          - AppCloudFrontOriginDomainName
+          - AppCloudFrontRoleArn
+          - AppCloudFrontProtocolPolicy
+          - AppCloudFrontForwardedHeaders
+          - AppCloudFrontCertArn
+    ParameterLabels:
+      AppCloudFrontCertArn:
+        default: CloudFront SSL Certificate ARN
+      AppCloudFrontForwardedHeaders:
+        default: CloudFront Forwarded Headers
+      AppCloudFrontOriginDomainName:
+        default: CloudFront Origin Domain Name
+      AppCloudFrontProtocolPolicy:
+        default: CloudFront Protocol Policy
+      AppCloudFrontRoleArn:
+        default: CloudFront Role ARN
+      CertificateValidationMethod:
+        default: Certificate Validation Method
+      CustomAppCertificateArn:
+        default: Custom App Certificate ARN
+      DomainName:
+        default: Domain Name
+      DomainNameAlternates:
+        default: Alternate Domain Names
+Outputs:
+  AppCloudFrontDomainName:
+    Description: The app CDN domain name
+    Value: !GetAtt 'AppCloudFrontDistribution.DomainName'
+Parameters:
+  AppCloudFrontCertArn:
+    Description: If your stack is NOT in the us-east-1 you must manually create an ACM certificate for your application domain in the us-east-1 region and provide its ARN here.
+    Type: String
+  AppCloudFrontForwardedHeaders:
+    Default: ''
+    Description: The headers that will be forwarded to the origin and used in the cache key. The 'Host' header is required for SSL on an Elastic Load Balancer, but it should NOT be passed to a Lambda Function
+      URL.
+    Type: CommaDelimitedList
+  AppCloudFrontOriginDomainName:
+    Default: ''
+    Description: Domain name of the origin server
+    Type: String
+  AppCloudFrontProtocolPolicy:
+    AllowedValues:
+      - redirect-to-https
+      - https-only
+      - allow-all
+    Default: redirect-to-https
+    Description: 'The protocols allowed by the application server''s CloudFront distribution. See: http://docs.aws.amazon.com/cloudfront/latest/APIReference/API_DefaultCacheBehavior.html'
+    Type: String
+  AppCloudFrontRoleArn:
+    Default: ''
+    Description: ARN of the role to add IAM permissions for invalidating this distribution
+    Type: String
+  CertificateValidationMethod:
+    AllowedValues:
+      - (none)
+      - DNS
+      - Email
+    Default: DNS
+    Description: >-
+      How to validate domain ownership for issuing an SSL certificate - highly recommend DNS. DNS and Email will pause stack creation until you do something to complete the validation. If omitted, an HTTPS
+      listener can be manually attached to the load balancer after stack creation.
+    Type: String
+  CustomAppCertificateArn:
+    Description: An existing ACM certificate ARN to be used by the application ELB. DNS and Email validation will not work with this option.
+    Type: String
+  DomainName:
+    Description: The fully-qualified domain name for the application.
+    Type: String
+  DomainNameAlternates:
+    Description: A comma-separated list of Alternate FQDNs to be included in the Subject Alternative Name extension of the SSL certificate.
+    Type: CommaDelimitedList
+Resources:
+  AppCloudFrontDistribution:
+    Properties:
+      DistributionConfig:
+        Aliases: !Split
+          - ;
+          - !Join
+            - ''
+            - - !Ref 'DomainName'
+              - !If
+                - NoAlternateDomains
+                - ''
+                - ;
+              - !Join
+                - ;
+                - !Ref 'DomainNameAlternates'
+        DefaultCacheBehavior:
+          AllowedMethods:
+            - DELETE
+            - GET
+            - HEAD
+            - OPTIONS
+            - PATCH
+            - POST
+            - PUT
+          CachedMethods:
+            - HEAD
+            - GET
+          Compress: true
+          ForwardedValues:
+            Cookies:
+              Forward: all
+            Headers: !If
+              - AppCloudFrontForwardedHeadersCondition
+              - !Ref 'AppCloudFrontForwardedHeaders'
+              - !Ref 'AWS::NoValue'
+            QueryString: true
+          TargetOriginId: ApplicationServer
+          ViewerProtocolPolicy: !Ref 'AppCloudFrontProtocolPolicy'
+        Enabled: true
+        HttpVersion: http2
+        Origins:
+          - CustomOriginConfig:
+              OriginProtocolPolicy: https-only
+            DomainName: !Ref 'AppCloudFrontOriginDomainName'
+            Id: ApplicationServer
+        ViewerCertificate: !If
+          - UsEast1Condition
+          - AcmCertificateArn: !If
+              - CustomAppCertArnCondition
+              - !Ref 'CustomAppCertificateArn'
+              - !Ref 'Certificate'
+            MinimumProtocolVersion: TLSv1.2_2021
+            SslSupportMethod: sni-only
+          - !If
+            - AppCloudFrontCertArnCondition
+            - AcmCertificateArn: !Ref 'AppCloudFrontCertArn'
+              MinimumProtocolVersion: TLSv1.2_2021
+              SslSupportMethod: sni-only
+            - !Ref 'AWS::NoValue'
+      Tags:
+        - Key: aws-web-stacks:stack-name
+          Value: !Ref 'AWS::StackName'
+    Type: AWS::CloudFront::Distribution
+  AppCloudFrontInvalidationPolicy:
+    Properties:
+      PolicyDocument:
+        Statement:
+          - Action:
+              - cloudfront:GetDistribution
+              - cloudfront:GetDistributionConfig
+              - cloudfront:ListDistributions
+              - cloudfront:ListCloudFrontOriginAccessIdentities
+              - cloudfront:CreateInvalidation
+              - cloudfront:GetInvalidation
+              - cloudfront:ListInvalidations
+            Effect: Allow
+            Resource: '*'
+      PolicyName: AppCloudFrontInvalidationPolicy
+      Roles:
+        - !Ref 'AppCloudFrontRoleArn'
+    Type: AWS::IAM::Policy
+  Certificate:
+    Condition: StackCertificateCondition
+    Properties:
+      DomainName: !Ref 'DomainName'
+      DomainValidationOptions:
+        - DomainName: !Ref 'DomainName'
+          ValidationDomain: !Ref 'DomainName'
+      SubjectAlternativeNames: !If
+        - NoAlternateDomains
+        - !Ref 'AWS::NoValue'
+        - !Ref 'DomainNameAlternates'
+      Tags:
+        - Key: aws-web-stacks:stack-name
+          Value: !Ref 'AWS::StackName'
+      ValidationMethod: !Ref 'CertificateValidationMethod'
+    Type: AWS::CertificateManager::Certificate
+

docs/data-import.rst

@@ -25,8 +25,8 @@ To load an existing database dump on S3, run:
 .. code-block:: bash
 
     dropdb traffic_stops_nc && createdb -E UTF-8 traffic_stops_nc
-    aws s3 cp s3://traffic-stops/traffic_stops_nc.dump .
-    pg_restore -Ox -d traffic_stops_nc traffic_stops_nc.dump
+    aws s3 cp s3://forwardjustice-trafficstops/trafficstops-staging_database.dump .
+    pg_restore -Ox -d traffic_stops_nc trafficstops-staging_database.dump
 
 
 Raw NC Data (slower)

docs/dev-setup.rst

@@ -66,7 +66,7 @@ Getting Started (Manual)
 
 Run PostgreSQL in Docker::
 
-    docker-compose up -d
+    docker compose up -d db redis
 
 This will create a PostgreSQL server with multiple databases (see
 ``docker-entrypoint.postgres.sql``).