Add IDOT data to data-lake

dbt/models/spatial/docs.md

@@ -493,6 +493,22 @@ Includes townships within the City of Chicago, which are technically defunct.
 **Geometry:** `MULTIPOLYGON`
 {% enddocs %}
 
+# traffic
+
+{% docs table_traffic %}
+
+Illinois Department of Transportation data source from
+[https://apps1.dot.illinois.gov/gist2/](https://apps1.dot.illinois.gov/gist2/).
+Data focuses on five features; lanes, speed limits, traffic count, road type,
+and surface type. Some columns are not present in all years of data (for example
+speed limit in 2012) Data for columns is not universally present so we average
+numeric values for roads which overlap and have a matching name. For example,
+if segment B touches segment A and C with speed limits of 25 and 30, the speed
+limit for segment B will be 27.5.
+
+**Geometry:** `MULTILINESTRING`
+{% enddocs %}
+
 # transit_dict
 
 {% docs table_transit_dict %}

dbt/models/spatial/schema.yml

@@ -174,6 +174,9 @@ sources:
       - name: township
         description: '{{ doc("table_township") }}'
 
+      - name: traffic
+        description: '{{ doc("table_traffic") }}'
+
       - name: transit_dict
         description: '{{ doc("table_transit_dict") }}'
 

etl/renv.lock

@@ -142,6 +142,20 @@
       ],
       "Hash": "ae4a925e0f6bb1b7e5fa96b739c5221a"
     },
+    "RSocrata": {
+      "Package": "RSocrata",
+      "Version": "1.7.15-1",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Requirements": [
+        "R",
+        "httr",
+        "jsonlite",
+        "mime",
+        "plyr"
+      ],
+      "Hash": "435ebea3fa736ab1317c79a5fa34fa55"
+    },
     "Rcpp": {
       "Package": "Rcpp",
       "Version": "1.0.12",
@@ -1926,8 +1940,13 @@
     "noctua": {
       "Package": "noctua",
       "Version": "2.6.2",
-      "Source": "Repository",
-      "Repository": "CRAN",
+      "Source": "GitHub",
+      "RemoteType": "github",
+      "RemoteHost": "api.github.com",
+      "RemoteUsername": "dyfanjones",
+      "RemoteRepo": "noctua",
+      "RemoteRef": "master",
+      "RemoteSha": "23a4cfbf537407c7a1547fc13ba771ba2eb098e0",
       "Requirements": [
         "DBI",
         "R",
@@ -1938,7 +1957,7 @@
         "utils",
         "uuid"
       ],
-      "Hash": "c03d73125d695e80b35b4bb3eacf0358"
+      "Hash": "b3fc482d0ae2f51ed324fd3da66471b4"
     },
     "numDeriv": {
       "Package": "numDeriv",
@@ -2276,6 +2295,17 @@
       "Repository": "CRAN",
       "Hash": "09eb987710984fc2905c7129c7d85e65"
     },
+    "plyr": {
+      "Package": "plyr",
+      "Version": "1.8.9",
+      "Source": "Repository",
+      "Repository": "CRAN",
+      "Requirements": [
+        "R",
+        "Rcpp"
+      ],
+      "Hash": "6b8177fd19982f0020743fadbfdbd933"
+    },
     "png": {
       "Package": "png",
       "Version": "0.1-8",

etl/scripts-ccao-data-raw-us-east-1/spatial/spatial-environment-traffic.R

@@ -0,0 +1,85 @@
+library(aws.s3)
+library(dplyr)
+library(httr)
+library(lubridate)
+library(purrr)
+library(sf)
+library(arrow)
+
+# Define S3 bucket and paths
+AWS_S3_RAW_BUCKET <- Sys.getenv("AWS_S3_RAW_BUCKET")
+output_bucket <- file.path(
+  AWS_S3_RAW_BUCKET,
+  "spatial", "environment", "traffic"
+)
+
+# Get list of available files
+years <- map(2012:year(Sys.Date()), \(x) {
+  if (HEAD(paste0(
+    "https://apps1.dot.illinois.gov/gist2/gisdata/all",
+    x, ".zip"
+  ))$status_code == 200) {
+    x
+  }
+}) %>%
+  unlist()
+
+# Function to process each year and upload shapefiles for
+# that specific year to S3
+process_shapefiles_for_year <- map(years, \(x) {
+  remote_file_path <- file.path(output_bucket, paste0(x, ".parquet"))
+
+  # Skip everything if file already exists
+  if (!object_exists(remote_file_path)) {
+    # Define the URL for the shapefile ZIP file, dynamically for each year
+    url <- paste0(
+      "https://apps1.dot.illinois.gov/gist2/gisdata/all", x, ".zip"
+    )
+
+    # Create a temporary file to store the downloaded ZIP
+    temp_zip <- tempfile(fileext = ".zip")
+    temp_dir <- tempdir()
+
+    # Download the ZIP file to a temporary location
+    download.file(url = url, destfile = temp_zip)
+
+    message(paste("Shapefile ZIP for year", x, "downloaded successfully."))
+
+    # Unzip the file into a temporary directory
+    unzip(temp_zip, exdir = temp_dir)
+    message(paste(
+      "Shapefile for year", x,
+      "unzipped into temporary directory."
+    ))
+
+    # List files in the unzipped directory and look for the .shp files
+    unzipped_files <- list.files(temp_dir, recursive = TRUE, full.names = TRUE)
+    shp_file_for_year <- unzipped_files[grepl(
+      paste0(
+        "HWY",
+        x
+      ),
+      unzipped_files,
+      ignore.case = TRUE
+    ) &
+      grepl("\\.shp$", unzipped_files)]
+
+    # Process only the shapefile that matches the current year
+    if (length(shp_file_for_year) == 1) {
+      # Read the shapefile into the environment using sf::st_read
+      shapefile_data <- sf::st_read(shp_file_for_year) %>%
+        # Add filter for Cook County. The name changes in different years.
+        filter(if ("COUNTY" %in% names(.)) {
+          COUNTY == "016"
+        } else {
+          INV_CO == "016"
+        }) %>%
+        mutate(year = as.character(x))
+
+      # Save the shapefile as a GeoParquet file
+      geoarrow::write_geoparquet(shapefile_data, remote_file_path)
+    } else {
+      message(paste("No shapefile found for year", x, "."))
+    }
+  }
+})