Merge branch 'release/0.18.0'

CHANGELOG.md

@@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Upcoming release]
 
+## [0.18.0] - 2023-05-16
+
+- Strip backslash characters from OSM file
+- Increase memory and resource allocations for analysis
+
 ## [0.17.0] - 2023-01-19
 
 ####  Added
@@ -192,7 +197,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [0.1.0] - 2017-04-21
 
 
-[Upcoming release]: https://github.com/azavea/pfb-network-connectivity/compare/0.17.0...HEAD
+[Upcoming release]: https://github.com/azavea/pfb-network-connectivity/compare/0.18.0...HEAD
+[0.18.0]: https://github.com/azavea/pfb-network-connectivity/compare/0.17.0...0.18.0
 [0.17.0]: https://github.com/azavea/pfb-network-connectivity/compare/0.16.1...0.17.0
 [0.16.1]: https://github.com/azavea/pfb-network-connectivity/compare/0.16.0...0.16.1
 [0.16.0]: https://github.com/azavea/pfb-network-connectivity/compare/0.15.1...0.16.0

deployment/aws-batch/job-definitions/production-pfb-analysis-run-job.json

@@ -5,7 +5,7 @@
     "containerProperties": {
         "image": "{image}",
         "vcpus": 8,
-        "memory": 30720,
+        "memory": 61000,
         "command": [
             "/pfb/scripts/entrypoint.sh"
         ],
@@ -26,19 +26,19 @@
             "value": "2680"
         }, {
             "name": "PFB_MAINTENANCE_WORK_MEM",
-            "value": "4092MB"
+            "value": "8GB"
         }, {
             "name": "PFB_MAX_WAL_SIZE",
-            "value": "4096MB"
+            "value": "8GB"
         }, {
             "name": "PFB_SHARED_BUFFERS",
-            "value": "7168MB"
+            "value": "24GB"
         }, {
             "name": "PFB_TEMP_FILE_LIMIT",
-            "value": "26214400"
+            "value": "100GB"
         }, {
             "name": "PFB_WORK_MEM",
-            "value": "3072MB"
+            "value": "3GB"
         }],
         "mountPoints": [{
             "sourceVolume": "pgdata",
@@ -52,6 +52,9 @@
         "readonlyRootFilesystem": false,
         "privileged": true,
         "ulimits": [],
-        "user": ""
+        "user": "",
+        "linuxParameters": {
+            "sharedMemorySize": 4096
+        }
     }
 }

deployment/aws-batch/job-definitions/staging-pfb-analysis-run-job.json

@@ -4,8 +4,8 @@
     "parameters": {},
     "containerProperties": {
         "image": "{image}",
-        "vcpus": 2,
-        "memory": 12288,
+        "vcpus": 8,
+        "memory": 61000,
         "command": [
             "/pfb/scripts/entrypoint.sh"
         ],
@@ -26,19 +26,19 @@
             "value": "2680"
         }, {
             "name": "PFB_MAINTENANCE_WORK_MEM",
-            "value": "2048MB"
+            "value": "8GB"
         }, {
             "name": "PFB_MAX_WAL_SIZE",
-            "value": "4096MB"
+            "value": "8GB"
         }, {
             "name": "PFB_SHARED_BUFFERS",
-            "value": "3072MB"
+            "value": "24GB"
         }, {
             "name": "PFB_TEMP_FILE_LIMIT",
-            "value": "104857600"
+            "value": "100GB"
         }, {
             "name": "PFB_WORK_MEM",
-            "value": "1536MB"
+            "value": "3GB"
         }],
         "mountPoints": [{
             "sourceVolume": "pgdata",
@@ -52,6 +52,9 @@
         "readonlyRootFilesystem": false,
         "privileged": true,
         "ulimits": [],
-        "user": ""
+        "user": "",
+        "linuxParameters": {
+            "sharedMemorySize": 4096
+        }
     }
 }

deployment/aws-batch/requirements.txt

@@ -1 +1 @@
-boto3==1.4.4
+boto3==1.26.132

scripts/infra

@@ -52,10 +52,10 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
     if [[ -n "${PFB_SETTINGS_BUCKET}" ]]; then
         pushd "${TERRAFORM_DIR}"
 
-        aws s3 cp "s3://${PFB_SETTINGS_BUCKET}/terraform/terraform.tfvars" "${PFB_SETTINGS_BUCKET}.tfvars"
-
         case "${1}" in
             plan)
+                aws s3 cp "s3://${PFB_SETTINGS_BUCKET}/terraform/terraform.tfvars" "${PFB_SETTINGS_BUCKET}.tfvars"
+
                 update_batch_definitions
 
                 docker-compose run --rm \
@@ -112,6 +112,10 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
             apply-mgmt)
                 docker-compose run --rm terraform apply "${PFB_SETTINGS_BUCKET}-mgmt.tfplan"
                 ;;
+            update-tfvars)
+                aws s3 cp "${PFB_SETTINGS_BUCKET}.tfvars" \
+                    "s3://${PFB_SETTINGS_BUCKET}/terraform/terraform.tfvars" --sse
+                ;;
             *)
                 echo "ERROR: I don't have support for that Terraform subcommand!"
                 exit 1

src/analysis/import/import_osm.sh

@@ -61,11 +61,15 @@ OSM_TEMPDIR="${NB_TEMPDIR:-$(mktemp -d)}/import_osm"
 mkdir -p "${OSM_TEMPDIR}"
 
 update_status "IMPORTING" "Clipping provided OSM file"
+OSM_DATA_FILE="${OSM_TEMPDIR}/converted.osm"
 osmconvert "${1}" \
   --drop-broken-refs \
   -b="${BBOX}" \
-  -o="${OSM_TEMPDIR}/converted.osm"
-OSM_DATA_FILE="${OSM_TEMPDIR}/converted.osm"
+  -o="${OSM_DATA_FILE}"
+# If the OSM file contains "\" as a segment name, osm2pgrouting chokes on those segments and drops
+# everything that happens to be in the same processing chunk. So strip them out.
+sed 's/\\/backslash/' $OSM_DATA_FILE > "${OSM_DATA_FILE}-cleaned"
+mv "${OSM_DATA_FILE}-cleaned" $OSM_DATA_FILE
 
 # import the osm with highways
 update_status "IMPORTING" "Importing OSM data"

src/analysis/scripts/setup_database.sh

@@ -39,7 +39,7 @@ ALTER SYSTEM SET maintenance_work_mem TO '${PFB_MAINTENANCE_WORK_MEM}';
 ALTER SYSTEM SET max_wal_size TO '${PFB_MAX_WAL_SIZE}';
 ALTER SYSTEM SET shared_buffers TO '${PFB_SHARED_BUFFERS}';
 ALTER SYSTEM SET synchronous_commit TO ${PFB_SYNCHRONOUS_COMMIT};
-ALTER SYSTEM SET temp_file_limit TO ${PFB_TEMP_FILE_LIMIT};
+ALTER SYSTEM SET temp_file_limit TO '${PFB_TEMP_FILE_LIMIT}';
 ALTER SYSTEM SET wal_buffers TO ${PFB_WAL_BUFFERS};
 ALTER SYSTEM SET work_mem TO '${PFB_WORK_MEM}';
 EOF

src/angularjs/.bowerrc

@@ -1,3 +1,5 @@
 {
-  "directory": "bower_components"
+  "directory": "bower_components",
+  "registry": "https://registry.bower.io",
+  "strict-ssl": false
 }

src/angularjs/Dockerfile

@@ -3,6 +3,9 @@ FROM node:6.17-stretch
 MAINTAINER Azavea
 
 ENV ANGULAR_DIR /opt/pfb/angularjs
+ENV NODE_OPTIONS --use-openssl-ca
+
+RUN echo "deb http://archive.debian.org/debian stretch main" > /etc/apt/sources.list
 
 RUN apt-get update && apt-get install -y rsync \
     && rm -rf /var/lib/apt/lists/*

src/tilegarden/Dockerfile

@@ -1,10 +1,9 @@
-FROM node:12.19-stretch-slim
+FROM node:12.19-buster-slim
 
 ENV BASE_DIR /opt/pfb/tilegarden
 
 # Install git for git dependencies
-RUN apt-get update -y
-RUN apt-get install git jq -y
+RUN apt-get update -y && apt-get install -y git jq python2 && apt-get clean
 RUN yarn global add carto
 
 # Copy files needed for installing packages first