From acb0154db7c28ecaa99c0b3645ca75d5062192cc Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Wed, 9 Oct 2024 21:30:11 +0000 Subject: [PATCH 01/61] Initial test --- dbt/models/proximity/docs.md | 8 +++++ ...mity.dist_pin_to_traffic_daily_traffic.sql | 30 +++++++++++++++++++ dbt/models/spatial/docs.md | 8 +++++ dbt/models/spatial/schema.yml | 3 ++ 4 files changed, 49 insertions(+) create mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql diff --git a/dbt/models/proximity/docs.md b/dbt/models/proximity/docs.md index 8faaa1b5d..55c5430e8 100644 --- a/dbt/models/proximity/docs.md +++ b/dbt/models/proximity/docs.md @@ -217,6 +217,14 @@ Distance from each PIN to the nearest sports stadium. **Primary Key**: `pin10`, `year` {% enddocs %} +# dist_pin_to_traffic_daily_traffic + +{% docs table_dist_pin_to_traffic_daily_traffic %} +Distance from each PIN to the valid value of daily traffic. + +**Primary Key**: `pin10`, `year` +{% enddocs %} + # dist_pin_to_university {% docs table_dist_pin_to_university %} diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql new file mode 100644 index 000000000..6803d1e05 --- /dev/null +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql @@ -0,0 +1,30 @@ +-- CTAS to create a table of distance to the nearest Metra route for each PIN +{{ + config( + materialized='table', + partitioned_by=['year'], + bucketed_by=['pin10'], + bucket_count=1 + ) +}} + +WITH traffic AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE annual_traffic > 0 + AND annual_traffic IS NOT NULL +) + +SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_road_name, + ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, + ARBITRARY(xy.year) AS nearest_road_data_year, + ARBITRARY(xy.annual_traffic) AS nearest_road_annual_traffic, + pcl.year +FROM {{ source('spatial', 'parcel') }} AS pcl +INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year +GROUP BY pcl.pin10, pcl.year diff --git a/dbt/models/spatial/docs.md b/dbt/models/spatial/docs.md index 096221bb8..cd3528c15 100644 --- a/dbt/models/spatial/docs.md +++ b/dbt/models/spatial/docs.md @@ -499,6 +499,14 @@ Includes townships within the City of Chicago, which are technically defunct. Dictionary to cleanup transit route and stop names. {% enddocs %} +# traffic + +{% docs table_traffic %} +Traffic data derived from Illinois Department of Transportation + +**Geometry:** `MULTILINESTRING` +{% enddocs %} + # transit_route {% docs table_transit_route %} diff --git a/dbt/models/spatial/schema.yml b/dbt/models/spatial/schema.yml index 0982b8106..513900073 100644 --- a/dbt/models/spatial/schema.yml +++ b/dbt/models/spatial/schema.yml @@ -174,6 +174,9 @@ sources: - name: township description: '{{ doc("table_township") }}' + - name: traffic + description: '{{ doc("table_traffic") }}' + - name: transit_dict description: '{{ doc("table_transit_dict") }}' From 0652b3a3639fdc3cc474957201b802c6fb2dfa50 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Wed, 9 Oct 2024 21:32:08 +0000 Subject: [PATCH 02/61] Sort headings --- dbt/models/spatial/docs.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dbt/models/spatial/docs.md b/dbt/models/spatial/docs.md index cd3528c15..a0d2cd9d9 100644 --- a/dbt/models/spatial/docs.md +++ b/dbt/models/spatial/docs.md @@ -493,12 +493,6 @@ Includes townships within the City of Chicago, which are technically defunct. **Geometry:** `MULTIPOLYGON` {% enddocs %} -# transit_dict - -{% docs table_transit_dict %} -Dictionary to cleanup transit route and stop names. -{% enddocs %} - # traffic {% docs table_traffic %} @@ -507,6 +501,12 @@ Traffic data derived from Illinois Department of Transportation **Geometry:** `MULTILINESTRING` {% enddocs %} +# transit_dict + +{% docs table_transit_dict %} +Dictionary to cleanup transit route and stop names. +{% enddocs %} + # transit_route {% docs table_transit_route %} From 5191e58c99a4741bb3b2b7262d05a325af338b6c Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 10 Oct 2024 14:27:43 +0000 Subject: [PATCH 03/61] New test --- ...mity.dist_pin_to_traffic_daily_traffic.sql | 4 +++ ...ist_pin_to_traffic_daily_traffic_Minor.sql | 31 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor.sql diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql index 6803d1e05..8aa822f25 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql @@ -13,6 +13,10 @@ WITH traffic AS ( -- noqa: ST03 FROM {{ source('spatial', 'traffic') }} WHERE annual_traffic > 0 AND annual_traffic IS NOT NULL + AND ( + road_type = 'Interstate' + OR road_type = 'Freeway And Expressway' + ) ) SELECT diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor.sql new file mode 100644 index 000000000..127016c05 --- /dev/null +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor.sql @@ -0,0 +1,31 @@ +-- CTAS to create a table of distance to the nearest Metra route for each PIN +{{ + config( + materialized='table', + partitioned_by=['year'], + bucketed_by=['pin10'], + bucket_count=1 + ) +}} + +WITH traffic AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE annual_traffic > 0 + AND annual_traffic IS NOT NULL + AND road_type = 'Minor Arterial' +) + +SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_road_name, + ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, + ARBITRARY(xy.year) AS nearest_road_data_year, + ARBITRARY(xy.annual_traffic) AS nearest_road_annual_traffic, + pcl.year +FROM {{ source('spatial', 'parcel') }} AS pcl +INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year +GROUP BY pcl.pin10, pcl.year From c4807fb7897a064260ec1f2b697981c85dfa09af Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 10 Oct 2024 14:58:22 +0000 Subject: [PATCH 04/61] local --- ...ist_pin_to_traffic_daily_traffic_local.sql | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql new file mode 100644 index 000000000..c586f328f --- /dev/null +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql @@ -0,0 +1,31 @@ +-- CTAS to create a table of distance to the nearest Metra route for each PIN +{{ + config( + materialized='table', + partitioned_by=['year'], + bucketed_by=['pin10'], + bucket_count=1 + ) +}} + +WITH traffic AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE annual_traffic > 0 + AND annual_traffic IS NOT NULL + AND road_type = 'Local Road or Street' +) + +SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_road_name, + ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, + ARBITRARY(xy.year) AS nearest_road_data_year, + ARBITRARY(xy.annual_traffic) AS nearest_road_annual_traffic, + pcl.year +FROM {{ source('spatial', 'parcel') }} AS pcl +INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year +GROUP BY pcl.pin10, pcl.year From 6e2a088ec272f28fbf17109d3f2513b2b8d3434d Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 10 Oct 2024 15:00:49 +0000 Subject: [PATCH 05/61] surface width --- ...dist_pin_to_traffic_daily_traffic copy.sql | 32 +++++++++++++++++++ ...in_to_traffic_daily_traffic_Minor copy.sql | 29 +++++++++++++++++ ...in_to_traffic_daily_traffic_local copy.sql | 29 +++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic copy.sql create mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor copy.sql create mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local copy.sql diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic copy.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic copy.sql new file mode 100644 index 000000000..c64d9ea5a --- /dev/null +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic copy.sql @@ -0,0 +1,32 @@ +-- CTAS to create a table of distance to the nearest Metra route for each PIN +{{ + config( + materialized='table', + partitioned_by=['year'], + bucketed_by=['pin10'], + bucket_count=1 + ) +}} + +WITH traffic AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE ( + road_type = 'Interstate' + OR road_type = 'Freeway And Expressway' + ) +) + +SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_road_name, + ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, + ARBITRARY(xy.year) AS nearest_road_data_year, + ARBITRARY(xy.surface_width) AS nearest_surface_width, + pcl.year +FROM {{ source('spatial', 'parcel') }} AS pcl +INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year +GROUP BY pcl.pin10, pcl.year diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor copy.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor copy.sql new file mode 100644 index 000000000..56810b3a1 --- /dev/null +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor copy.sql @@ -0,0 +1,29 @@ +-- CTAS to create a table of distance to the nearest Metra route for each PIN +{{ + config( + materialized='table', + partitioned_by=['year'], + bucketed_by=['pin10'], + bucket_count=1 + ) +}} + +WITH traffic AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Minor Arterial' +) + +SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_road_name, + ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, + ARBITRARY(xy.year) AS nearest_road_data_year, + ARBITRARY(xy.surface_width) AS nearest_surface_width, + pcl.year +FROM {{ source('spatial', 'parcel') }} AS pcl +INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year +GROUP BY pcl.pin10, pcl.year diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local copy.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local copy.sql new file mode 100644 index 000000000..8ad53d3d3 --- /dev/null +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local copy.sql @@ -0,0 +1,29 @@ +-- CTAS to create a table of distance to the nearest Metra route for each PIN +{{ + config( + materialized='table', + partitioned_by=['year'], + bucketed_by=['pin10'], + bucket_count=1 + ) +}} + +WITH traffic AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Local Road or Street' +) + +SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_road_name, + ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, + ARBITRARY(xy.year) AS nearest_road_data_year, + ARBITRARY(xy.surface_width) AS nearest_surface_width, + pcl.year +FROM {{ source('spatial', 'parcel') }} AS pcl +INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year +GROUP BY pcl.pin10, pcl.year From ccc67fac0f77ad7dc722e8e3a53f0b05045a3e52 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 10 Oct 2024 16:08:49 +0000 Subject: [PATCH 06/61] More tests --- ... => proximity.dist_pin_to_traffic_width.sql} | 0 ...oximity.dist_pin_to_traffic_width_minor.sql} | 17 +++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) rename dbt/models/proximity/{proximity.dist_pin_to_traffic_daily_traffic_local copy.sql => proximity.dist_pin_to_traffic_width.sql} (100%) rename dbt/models/proximity/{proximity.dist_pin_to_traffic_daily_traffic_Minor copy.sql => proximity.dist_pin_to_traffic_width_minor.sql} (74%) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local copy.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_width.sql similarity index 100% rename from dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local copy.sql rename to dbt/models/proximity/proximity.dist_pin_to_traffic_width.sql diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor copy.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_width_minor.sql similarity index 74% rename from dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor copy.sql rename to dbt/models/proximity/proximity.dist_pin_to_traffic_width_minor.sql index 56810b3a1..7b4c49bf7 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor copy.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_width_minor.sql @@ -1,17 +1,25 @@ --- CTAS to create a table of distance to the nearest Metra route for each PIN -{{ +{{ config( materialized='table', partitioned_by=['year'], bucketed_by=['pin10'], bucket_count=1 - ) + ) }} WITH traffic AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} WHERE road_type = 'Minor Arterial' +), + +distinct_pins AS ( + SELECT DISTINCT + x_3435, + y_3435, + pin10, + year + FROM {{ source('spatial', 'parcel') }} ) SELECT @@ -20,8 +28,9 @@ SELECT ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, ARBITRARY(xy.year) AS nearest_road_data_year, ARBITRARY(xy.surface_width) AS nearest_surface_width, + ARBITRARY(xy.surface_type) AS nearest_surface_type, pcl.year -FROM {{ source('spatial', 'parcel') }} AS pcl +FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 From 6a9c130a58b33dd5a35587c9f61658bd7c31207b Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 10 Oct 2024 18:42:56 +0000 Subject: [PATCH 07/61] Query improvements --- ...ist_pin_to_traffic_daily_traffic_minor.sql | 31 +++++++++++++++++++ ...ity.dist_pin_to_traffic_width_highway.sql} | 0 2 files changed, 31 insertions(+) create mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql rename dbt/models/proximity/{proximity.dist_pin_to_traffic_daily_traffic copy.sql => proximity.dist_pin_to_traffic_width_highway.sql} (100%) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql new file mode 100644 index 000000000..127016c05 --- /dev/null +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql @@ -0,0 +1,31 @@ +-- CTAS to create a table of distance to the nearest Metra route for each PIN +{{ + config( + materialized='table', + partitioned_by=['year'], + bucketed_by=['pin10'], + bucket_count=1 + ) +}} + +WITH traffic AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE annual_traffic > 0 + AND annual_traffic IS NOT NULL + AND road_type = 'Minor Arterial' +) + +SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_road_name, + ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, + ARBITRARY(xy.year) AS nearest_road_data_year, + ARBITRARY(xy.annual_traffic) AS nearest_road_annual_traffic, + pcl.year +FROM {{ source('spatial', 'parcel') }} AS pcl +INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year +GROUP BY pcl.pin10, pcl.year diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic copy.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_width_highway.sql similarity index 100% rename from dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic copy.sql rename to dbt/models/proximity/proximity.dist_pin_to_traffic_width_highway.sql From 12b2ebf4dfce2eb300a0713e87c4bc546bee9833 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 10 Oct 2024 19:19:04 +0000 Subject: [PATCH 08/61] Make all distinct_pins --- ...mity.dist_pin_to_traffic_daily_traffic.sql | 21 +++++++++---- ...ist_pin_to_traffic_daily_traffic_Minor.sql | 31 ------------------- ...ist_pin_to_traffic_daily_traffic_local.sql | 11 ++++++- ...ist_pin_to_traffic_daily_traffic_minor.sql | 11 ++++++- ...mity.dist_pin_to_traffic_width_highway.sql | 11 ++++++- 5 files changed, 45 insertions(+), 40 deletions(-) delete mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor.sql diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql index 8aa822f25..b9d119b2a 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql @@ -17,18 +17,27 @@ WITH traffic AS ( -- noqa: ST03 road_type = 'Interstate' OR road_type = 'Freeway And Expressway' ) +), + +distinct_pins AS ( + SELECT DISTINCT + x_3435, + y_3435, + pin10, + year + FROM {{ source('spatial', 'parcel') }} ) SELECT pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_road_name, - ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, - ARBITRARY(xy.year) AS nearest_road_data_year, - ARBITRARY(xy.annual_traffic) AS nearest_road_annual_traffic, + MIN(xy.road_name) AS nearest_road_name, -- Using MIN() as a placeholder + MIN(xy.dist_ft) AS nearest_road_dist_ft, + MIN(xy.year) AS nearest_road_data_year, + MIN(xy.annual_traffic) AS nearest_road_annual_traffic, pcl.year -FROM {{ source('spatial', 'parcel') }} AS pcl +FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 AND pcl.year = xy.pin_year -GROUP BY pcl.pin10, pcl.year +GROUP BY pcl.pin10, pcl.year; diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor.sql deleted file mode 100644 index 127016c05..000000000 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_Minor.sql +++ /dev/null @@ -1,31 +0,0 @@ --- CTAS to create a table of distance to the nearest Metra route for each PIN -{{ - config( - materialized='table', - partitioned_by=['year'], - bucketed_by=['pin10'], - bucket_count=1 - ) -}} - -WITH traffic AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE annual_traffic > 0 - AND annual_traffic IS NOT NULL - AND road_type = 'Minor Arterial' -) - -SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_road_name, - ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, - ARBITRARY(xy.year) AS nearest_road_data_year, - ARBITRARY(xy.annual_traffic) AS nearest_road_annual_traffic, - pcl.year -FROM {{ source('spatial', 'parcel') }} AS pcl -INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year -GROUP BY pcl.pin10, pcl.year diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql index c586f328f..ef8923d8f 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql @@ -14,6 +14,15 @@ WITH traffic AS ( -- noqa: ST03 WHERE annual_traffic > 0 AND annual_traffic IS NOT NULL AND road_type = 'Local Road or Street' +), + +distinct_pins AS ( + SELECT DISTINCT + x_3435, + y_3435, + pin10, + year + FROM {{ source('spatial', 'parcel') }} ) SELECT @@ -23,7 +32,7 @@ SELECT ARBITRARY(xy.year) AS nearest_road_data_year, ARBITRARY(xy.annual_traffic) AS nearest_road_annual_traffic, pcl.year -FROM {{ source('spatial', 'parcel') }} AS pcl +FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql index 127016c05..6e3251a64 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql @@ -14,6 +14,15 @@ WITH traffic AS ( -- noqa: ST03 WHERE annual_traffic > 0 AND annual_traffic IS NOT NULL AND road_type = 'Minor Arterial' +), + +distinct_pins AS ( + SELECT DISTINCT + x_3435, + y_3435, + pin10, + year + FROM {{ source('spatial', 'parcel') }} ) SELECT @@ -23,7 +32,7 @@ SELECT ARBITRARY(xy.year) AS nearest_road_data_year, ARBITRARY(xy.annual_traffic) AS nearest_road_annual_traffic, pcl.year -FROM {{ source('spatial', 'parcel') }} AS pcl +FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_width_highway.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_width_highway.sql index c64d9ea5a..692c62bc7 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_width_highway.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_width_highway.sql @@ -15,6 +15,15 @@ WITH traffic AS ( -- noqa: ST03 road_type = 'Interstate' OR road_type = 'Freeway And Expressway' ) +), + +distinct_pins AS ( + SELECT DISTINCT + x_3435, + y_3435, + pin10, + year + FROM {{ source('spatial', 'parcel') }} ) SELECT @@ -24,7 +33,7 @@ SELECT ARBITRARY(xy.year) AS nearest_road_data_year, ARBITRARY(xy.surface_width) AS nearest_surface_width, pcl.year -FROM {{ source('spatial', 'parcel') }} AS pcl +FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 From a22e31463bed0a5062d2bc643d7ac3d09a31838e Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Fri, 11 Oct 2024 13:56:50 +0000 Subject: [PATCH 09/61] Make traffic_width unique --- .../proximity.dist_pin_to_traffic_width.sql | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_width.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_width.sql index 8ad53d3d3..8376475e4 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_width.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_width.sql @@ -12,6 +12,15 @@ WITH traffic AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} WHERE road_type = 'Local Road or Street' +), + +distinct_pins AS ( + SELECT DISTINCT + x_3435, + y_3435, + pin10, + year + FROM {{ source('spatial', 'parcel') }} ) SELECT @@ -20,8 +29,9 @@ SELECT ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, ARBITRARY(xy.year) AS nearest_road_data_year, ARBITRARY(xy.surface_width) AS nearest_surface_width, + ARBITRARY(xy.surface_type) AS nearest_surface_type, pcl.year -FROM {{ source('spatial', 'parcel') }} AS pcl +FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 From d835273ca8ec2869a00e7276886e95bc9b96517b Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Fri, 11 Oct 2024 14:19:57 +0000 Subject: [PATCH 10/61] switch to daily_traffic --- .../proximity.dist_pin_to_traffic_daily_traffic.sql | 6 +++--- .../proximity.dist_pin_to_traffic_daily_traffic_local.sql | 6 +++--- .../proximity.dist_pin_to_traffic_daily_traffic_minor.sql | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql index b9d119b2a..7ef57d384 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql @@ -11,8 +11,8 @@ WITH traffic AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} - WHERE annual_traffic > 0 - AND annual_traffic IS NOT NULL + WHERE daily_traffic > 0 + AND daily_traffic IS NOT NULL AND ( road_type = 'Interstate' OR road_type = 'Freeway And Expressway' @@ -33,7 +33,7 @@ SELECT MIN(xy.road_name) AS nearest_road_name, -- Using MIN() as a placeholder MIN(xy.dist_ft) AS nearest_road_dist_ft, MIN(xy.year) AS nearest_road_data_year, - MIN(xy.annual_traffic) AS nearest_road_annual_traffic, + MIN(xy.daily_traffic) AS nearest_road_daily_traffic, pcl.year FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql index ef8923d8f..b2172abcf 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql @@ -11,8 +11,8 @@ WITH traffic AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} - WHERE annual_traffic > 0 - AND annual_traffic IS NOT NULL + WHERE daily_traffic > 0 + AND daily_traffic IS NOT NULL AND road_type = 'Local Road or Street' ), @@ -30,7 +30,7 @@ SELECT ARBITRARY(xy.road_name) AS nearest_road_name, ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, ARBITRARY(xy.year) AS nearest_road_data_year, - ARBITRARY(xy.annual_traffic) AS nearest_road_annual_traffic, + ARBITRARY(xy.daily_traffic) AS nearest_road_daily_traffic, pcl.year FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql index 6e3251a64..0daa3982d 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql @@ -11,8 +11,8 @@ WITH traffic AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} - WHERE annual_traffic > 0 - AND annual_traffic IS NOT NULL + WHERE daily_traffic > 0 + AND daily_traffic IS NOT NULL AND road_type = 'Minor Arterial' ), @@ -30,7 +30,7 @@ SELECT ARBITRARY(xy.road_name) AS nearest_road_name, ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, ARBITRARY(xy.year) AS nearest_road_data_year, - ARBITRARY(xy.annual_traffic) AS nearest_road_annual_traffic, + ARBITRARY(xy.daily_traffic) AS nearest_road_daily_traffic, pcl.year FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy From 7bd08b84192473b2a61ce8f930339e42d7082618 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Fri, 11 Oct 2024 16:12:27 +0000 Subject: [PATCH 11/61] Try master --- .../proximity.dist_pin_to_traffic_master.sql | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql new file mode 100644 index 000000000..b2237d39f --- /dev/null +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -0,0 +1,63 @@ +{{ + config( + materialized='table', + partitioned_by=['year'], + bucketed_by=['pin10'], + bucket_count=1 + ) +}} + +WITH traffic_highway AS ( + -- Local Road or Street traffic data + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE daily_traffic > 0 + AND daily_traffic IS NOT NULL + AND road_type = 'Freeway And Expressway' +), + +traffic_minor_arterial AS ( + -- Minor Arterial traffic data + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE daily_traffic > 0 + AND daily_traffic IS NOT NULL + AND road_type = 'Minor Arterial' +), + +distinct_pins AS ( + -- Select distinct pins from the parcel dataset + SELECT DISTINCT + x_3435, + y_3435, + pin10, + year + FROM {{ source('spatial', 'parcel') }} +) + +SELECT + pcl.pin10, + -- Nearest local road values + ARBITRARY(local_xy.road_name) AS nearest_local_road_name, + ARBITRARY(local_xy.dist_ft) AS nearest_local_road_dist_ft, + ARBITRARY(local_xy.year) AS nearest_local_road_data_year, + ARBITRARY(local_xy.daily_traffic) AS nearest_local_road_daily_traffic, + -- Nearest minor arterial values + ARBITRARY(arterial_xy.road_name) AS nearest_arterial_road_name, + ARBITRARY(arterial_xy.dist_ft) AS nearest_arterial_road_dist_ft, + ARBITRARY(arterial_xy.year) AS nearest_arterial_road_data_year, + ARBITRARY(arterial_xy.daily_traffic) AS nearest_arterial_road_daily_traffic, + pcl.year +FROM distinct_pins AS pcl +-- Join for nearest local road +LEFT JOIN traffic_highway AS local_xy + ON pcl.x_3435 = local_xy.x_3435 + AND pcl.y_3435 = local_xy.y_3435 + AND pcl.year = local_xy.pin_year +-- Join for nearest minor arterial +LEFT JOIN + traffic_minor_arterial AS arterial_xy + ON pcl.x_3435 = arterial_xy.x_3435 + AND pcl.y_3435 = arterial_xy.y_3435 + AND pcl.year = arterial_xy.pin_year +GROUP BY pcl.pin10, pcl.year From ca769428e2ca5e2fd182e4eee41f6e9ad0139083 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Fri, 11 Oct 2024 17:21:52 +0000 Subject: [PATCH 12/61] Fix master? --- ..._pin_to_traffic_daily_traffic_highway.sql} | 8 +- .../proximity.dist_pin_to_traffic_master.sql | 84 +++++++++++-------- 2 files changed, 55 insertions(+), 37 deletions(-) rename dbt/models/proximity/{proximity.dist_pin_to_traffic_daily_traffic.sql => proximity.dist_pin_to_traffic_daily_traffic_highway.sql} (79%) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_highway.sql similarity index 79% rename from dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql rename to dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_highway.sql index 7ef57d384..755846c57 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_highway.sql @@ -30,10 +30,10 @@ distinct_pins AS ( SELECT pcl.pin10, - MIN(xy.road_name) AS nearest_road_name, -- Using MIN() as a placeholder - MIN(xy.dist_ft) AS nearest_road_dist_ft, - MIN(xy.year) AS nearest_road_data_year, - MIN(xy.daily_traffic) AS nearest_road_daily_traffic, + ARBITRARY(xy.road_name) AS nearest_road_name, + ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, + ARBITRARY(xy.year) AS nearest_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_road_daily_traffic, pcl.year FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index b2237d39f..92f8e4507 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -1,3 +1,5 @@ +-- CTAS to create a table of distance to the nearest road type +-- (minor arterial and highway) for each PIN {{ config( materialized='table', @@ -7,57 +9,73 @@ ) }} -WITH traffic_highway AS ( - -- Local Road or Street traffic data +WITH traffic_minor AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} WHERE daily_traffic > 0 AND daily_traffic IS NOT NULL - AND road_type = 'Freeway And Expressway' + AND road_type = 'Minor Arterial' ), -traffic_minor_arterial AS ( - -- Minor Arterial traffic data +traffic_highway AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} WHERE daily_traffic > 0 AND daily_traffic IS NOT NULL - AND road_type = 'Minor Arterial' + AND ( + road_type = 'Interstate' + OR road_type = 'Freeway And Expressway' + ) ), distinct_pins AS ( - -- Select distinct pins from the parcel dataset SELECT DISTINCT x_3435, y_3435, pin10, year FROM {{ source('spatial', 'parcel') }} +), + +-- Select nearest road from Minor Arterial +nearest_minor AS ( + SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_road_name, + ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, + ARBITRARY(xy.year) AS nearest_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_road_daily_traffic, + 'Minor Arterial' AS road_type, + pcl.year + FROM distinct_pins AS pcl + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_minor') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year + GROUP BY pcl.pin10, pcl.year +), + +-- Select nearest road from Highway (Interstate, Freeway, Expressway) +nearest_highway AS ( + SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_road_name, + ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, + ARBITRARY(xy.year) AS nearest_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_road_daily_traffic, + 'Highway' AS road_type, + pcl.year + FROM distinct_pins AS pcl + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_highway') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year + GROUP BY pcl.pin10, pcl.year ) -SELECT - pcl.pin10, - -- Nearest local road values - ARBITRARY(local_xy.road_name) AS nearest_local_road_name, - ARBITRARY(local_xy.dist_ft) AS nearest_local_road_dist_ft, - ARBITRARY(local_xy.year) AS nearest_local_road_data_year, - ARBITRARY(local_xy.daily_traffic) AS nearest_local_road_daily_traffic, - -- Nearest minor arterial values - ARBITRARY(arterial_xy.road_name) AS nearest_arterial_road_name, - ARBITRARY(arterial_xy.dist_ft) AS nearest_arterial_road_dist_ft, - ARBITRARY(arterial_xy.year) AS nearest_arterial_road_data_year, - ARBITRARY(arterial_xy.daily_traffic) AS nearest_arterial_road_daily_traffic, - pcl.year -FROM distinct_pins AS pcl --- Join for nearest local road -LEFT JOIN traffic_highway AS local_xy - ON pcl.x_3435 = local_xy.x_3435 - AND pcl.y_3435 = local_xy.y_3435 - AND pcl.year = local_xy.pin_year --- Join for nearest minor arterial -LEFT JOIN - traffic_minor_arterial AS arterial_xy - ON pcl.x_3435 = arterial_xy.x_3435 - AND pcl.y_3435 = arterial_xy.y_3435 - AND pcl.year = arterial_xy.pin_year -GROUP BY pcl.pin10, pcl.year +-- Combine the two results with UNION ALL +SELECT * +FROM nearest_minor +UNION ALL +SELECT * +FROM nearest_highway From 8ba251d6aab56c7f71e537d64506cfaab09ac8f0 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Fri, 11 Oct 2024 19:37:28 +0000 Subject: [PATCH 13/61] Another master test --- .../proximity.dist_pin_to_traffic_master.sql | 78 +++++++------------ ...imity.dist_pin_to_traffic_width_local.sql} | 0 2 files changed, 30 insertions(+), 48 deletions(-) rename dbt/models/proximity/{proximity.dist_pin_to_traffic_width.sql => proximity.dist_pin_to_traffic_width_local.sql} (100%) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 92f8e4507..dc00de67d 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -1,14 +1,3 @@ --- CTAS to create a table of distance to the nearest road type --- (minor arterial and highway) for each PIN -{{ - config( - materialized='table', - partitioned_by=['year'], - bucketed_by=['pin10'], - bucket_count=1 - ) -}} - WITH traffic_minor AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} @@ -37,45 +26,38 @@ distinct_pins AS ( FROM {{ source('spatial', 'parcel') }} ), --- Select nearest road from Minor Arterial -nearest_minor AS ( - SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_road_name, - ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, - ARBITRARY(xy.year) AS nearest_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_road_daily_traffic, - 'Minor Arterial' AS road_type, - pcl.year - FROM distinct_pins AS pcl - INNER JOIN ( {{ dist_to_nearest_geometry('traffic_minor') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year - GROUP BY pcl.pin10, pcl.year -), - --- Select nearest road from Highway (Interstate, Freeway, Expressway) -nearest_highway AS ( +-- Select nearest road from Minor Arterial and Highway +nearest_road AS ( SELECT pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_road_name, - ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, - ARBITRARY(xy.year) AS nearest_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_road_daily_traffic, - 'Highway' AS road_type, - pcl.year + pcl.year, + xy_minor.road_name AS nearest_minor_road_name, + xy_minor.dist_ft AS nearest_minor_road_dist_ft, + xy_minor.daily_traffic AS nearest_minor_daily_traffic, + xy_highway.road_name AS nearest_highway_road_name, + xy_highway.dist_ft AS nearest_highway_road_dist_ft, + xy_highway.daily_traffic AS nearest_highway_daily_traffic FROM distinct_pins AS pcl - INNER JOIN ( {{ dist_to_nearest_geometry('traffic_highway') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year - GROUP BY pcl.pin10, pcl.year + LEFT JOIN ( {{ dist_to_nearest_geometry('traffic_minor') }} ) AS xy_minor + ON pcl.x_3435 = xy_minor.x_3435 + AND pcl.y_3435 = xy_minor.y_3435 + AND pcl.year = xy_minor.pin_year + LEFT JOIN + ( {{ dist_to_nearest_geometry('traffic_highway') }} ) AS xy_highway + ON pcl.x_3435 = xy_highway.x_3435 + AND pcl.y_3435 = xy_highway.y_3435 + AND pcl.year = xy_highway.pin_year ) --- Combine the two results with UNION ALL -SELECT * -FROM nearest_minor -UNION ALL -SELECT * -FROM nearest_highway +-- Combine the data into a single row for each PIN +SELECT + pin10, + year, + ARBITRARY(nearest_minor_road_name) AS nearest_minor_road_name, + ARBITRARY(nearest_minor_road_dist_ft) AS nearest_minor_road_dist_ft, + ARBITRARY(nearest_minor_daily_traffic) AS nearest_minor_daily_traffic, + ARBITRARY(nearest_highway_road_name) AS nearest_highway_road_name, + ARBITRARY(nearest_highway_road_dist_ft) AS nearest_highway_road_dist_ft, + ARBITRARY(nearest_highway_daily_traffic) AS nearest_highway_daily_traffic +FROM nearest_road +GROUP BY pin10, year diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_width.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_width_local.sql similarity index 100% rename from dbt/models/proximity/proximity.dist_pin_to_traffic_width.sql rename to dbt/models/proximity/proximity.dist_pin_to_traffic_width_local.sql From 1cf60f27f7aa58c0e2c175bc4dff4e9907d6e5ac Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Fri, 11 Oct 2024 21:04:43 +0000 Subject: [PATCH 14/61] Another master test --- .../proximity.dist_pin_to_traffic_master.sql | 73 +++++++++++-------- 1 file changed, 44 insertions(+), 29 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index dc00de67d..d711cb01c 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -26,38 +26,53 @@ distinct_pins AS ( FROM {{ source('spatial', 'parcel') }} ), --- Select nearest road from Minor Arterial and Highway -nearest_road AS ( +-- Select nearest road from Minor Arterial +nearest_minor AS ( SELECT pcl.pin10, - pcl.year, - xy_minor.road_name AS nearest_minor_road_name, - xy_minor.dist_ft AS nearest_minor_road_dist_ft, - xy_minor.daily_traffic AS nearest_minor_daily_traffic, - xy_highway.road_name AS nearest_highway_road_name, - xy_highway.dist_ft AS nearest_highway_road_dist_ft, - xy_highway.daily_traffic AS nearest_highway_daily_traffic + ARBITRARY(xy.road_name) AS nearest_minor_road_name, + ARBITRARY(xy.dist_ft) AS nearest_minor_road_dist_ft, + ARBITRARY(xy.year) AS nearest_minor_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_minor_road_daily_traffic, + pcl.year FROM distinct_pins AS pcl - LEFT JOIN ( {{ dist_to_nearest_geometry('traffic_minor') }} ) AS xy_minor - ON pcl.x_3435 = xy_minor.x_3435 - AND pcl.y_3435 = xy_minor.y_3435 - AND pcl.year = xy_minor.pin_year - LEFT JOIN - ( {{ dist_to_nearest_geometry('traffic_highway') }} ) AS xy_highway - ON pcl.x_3435 = xy_highway.x_3435 - AND pcl.y_3435 = xy_highway.y_3435 - AND pcl.year = xy_highway.pin_year + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_minor') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year + GROUP BY pcl.pin10, pcl.year +), + +-- Select nearest road from Highway (Interstate, Freeway, Expressway) +nearest_highway AS ( + SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_highway_road_name, + ARBITRARY(xy.dist_ft) AS nearest_highway_road_dist_ft, + ARBITRARY(xy.year) AS nearest_highway_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_highway_road_daily_traffic, + pcl.year + FROM distinct_pins AS pcl + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_highway') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year + GROUP BY pcl.pin10, pcl.year ) --- Combine the data into a single row for each PIN +-- Join the results based on pin10 and year SELECT - pin10, - year, - ARBITRARY(nearest_minor_road_name) AS nearest_minor_road_name, - ARBITRARY(nearest_minor_road_dist_ft) AS nearest_minor_road_dist_ft, - ARBITRARY(nearest_minor_daily_traffic) AS nearest_minor_daily_traffic, - ARBITRARY(nearest_highway_road_name) AS nearest_highway_road_name, - ARBITRARY(nearest_highway_road_dist_ft) AS nearest_highway_road_dist_ft, - ARBITRARY(nearest_highway_daily_traffic) AS nearest_highway_daily_traffic -FROM nearest_road -GROUP BY pin10, year + COALESCE(minor.pin10, highway.pin10) AS pin10, + COALESCE(minor.year, highway.year) AS year, + minor.nearest_minor_road_name, + minor.nearest_minor_road_dist_ft, + minor.nearest_minor_road_data_year, + minor.nearest_minor_road_daily_traffic, + highway.nearest_highway_road_name, + highway.nearest_highway_road_dist_ft, + highway.nearest_highway_road_data_year, + highway.nearest_highway_road_daily_traffic +FROM nearest_minor AS minor +FULL OUTER JOIN nearest_highway AS highway + ON minor.pin10 = highway.pin10 + AND minor.year = highway.year From d3ba2590718b9dc064888db4ffec7c8cac8409ed Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Fri, 11 Oct 2024 21:13:42 +0000 Subject: [PATCH 15/61] Add config --- .../proximity/proximity.dist_pin_to_traffic_master.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index d711cb01c..fe43b6091 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -1,3 +1,13 @@ +-- CTAS to create a table of distance to the nearest Metra route for each PIN +{{ + config( + materialized='table', + partitioned_by=['year'], + bucketed_by=['pin10'], + bucket_count=1 + ) +}} + WITH traffic_minor AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} From e34b8c89904142bcbad0a2d0b6d2c452009e15ee Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Fri, 11 Oct 2024 21:40:52 +0000 Subject: [PATCH 16/61] switch to width --- .../proximity.dist_pin_to_traffic_master.sql | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index fe43b6091..de09bb3c5 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -11,20 +11,16 @@ WITH traffic_minor AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} - WHERE daily_traffic > 0 - AND daily_traffic IS NOT NULL - AND road_type = 'Minor Arterial' + WHERE road_type = 'Minor Arterial' ), traffic_highway AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} - WHERE daily_traffic > 0 - AND daily_traffic IS NOT NULL - AND ( - road_type = 'Interstate' - OR road_type = 'Freeway And Expressway' - ) + WHERE ( + road_type = 'Interstate' + OR road_type = 'Freeway And Expressway' + ) ), distinct_pins AS ( @@ -43,7 +39,7 @@ nearest_minor AS ( ARBITRARY(xy.road_name) AS nearest_minor_road_name, ARBITRARY(xy.dist_ft) AS nearest_minor_road_dist_ft, ARBITRARY(xy.year) AS nearest_minor_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_minor_road_daily_traffic, + ARBITRARY(xy.surface_width) AS nearest_minor_road_surface_width, pcl.year FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_minor') }} ) AS xy @@ -60,7 +56,7 @@ nearest_highway AS ( ARBITRARY(xy.road_name) AS nearest_highway_road_name, ARBITRARY(xy.dist_ft) AS nearest_highway_road_dist_ft, ARBITRARY(xy.year) AS nearest_highway_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_highway_road_daily_traffic, + ARBITRARY(xy.surface_width) AS nearest_highway_road_surface_width, pcl.year FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_highway') }} ) AS xy @@ -77,11 +73,11 @@ SELECT minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, minor.nearest_minor_road_data_year, - minor.nearest_minor_road_daily_traffic, + minor.nearest_minor_road_surface_width, highway.nearest_highway_road_name, highway.nearest_highway_road_dist_ft, highway.nearest_highway_road_data_year, - highway.nearest_highway_road_daily_traffic + highway.nearest_highway_road_surface_width FROM nearest_minor AS minor FULL OUTER JOIN nearest_highway AS highway ON minor.pin10 = highway.pin10 From 4ae5554ac626266c4ec15db9b633d419533071c4 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Fri, 11 Oct 2024 21:44:29 +0000 Subject: [PATCH 17/61] Make year last column --- dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index de09bb3c5..4f3141e85 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -69,7 +69,6 @@ nearest_highway AS ( -- Join the results based on pin10 and year SELECT COALESCE(minor.pin10, highway.pin10) AS pin10, - COALESCE(minor.year, highway.year) AS year, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, minor.nearest_minor_road_data_year, @@ -77,7 +76,8 @@ SELECT highway.nearest_highway_road_name, highway.nearest_highway_road_dist_ft, highway.nearest_highway_road_data_year, - highway.nearest_highway_road_surface_width + highway.nearest_highway_road_surface_width, + COALESCE(minor.year, highway.year) AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_highway AS highway ON minor.pin10 = highway.pin10 From d2a45faf2b6bfc81685bb76c8a432f69ebcfa3e2 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 15 Oct 2024 20:19:24 +0000 Subject: [PATCH 18/61] Merge into master --- ...t_pin_to_traffic_daily_traffic_highway.sql | 43 ----- ...ist_pin_to_traffic_daily_traffic_local.sql | 40 ---- ...ist_pin_to_traffic_daily_traffic_minor.sql | 40 ---- .../proximity.dist_pin_to_traffic_master.sql | 174 +++++++++++++++--- ...mity.dist_pin_to_traffic_width_highway.sql | 41 ----- ...ximity.dist_pin_to_traffic_width_local.sql | 39 ---- ...ximity.dist_pin_to_traffic_width_minor.sql | 38 ---- 7 files changed, 152 insertions(+), 263 deletions(-) delete mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_highway.sql delete mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql delete mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql delete mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_width_highway.sql delete mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_width_local.sql delete mode 100644 dbt/models/proximity/proximity.dist_pin_to_traffic_width_minor.sql diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_highway.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_highway.sql deleted file mode 100644 index 755846c57..000000000 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_highway.sql +++ /dev/null @@ -1,43 +0,0 @@ --- CTAS to create a table of distance to the nearest Metra route for each PIN -{{ - config( - materialized='table', - partitioned_by=['year'], - bucketed_by=['pin10'], - bucket_count=1 - ) -}} - -WITH traffic AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE daily_traffic > 0 - AND daily_traffic IS NOT NULL - AND ( - road_type = 'Interstate' - OR road_type = 'Freeway And Expressway' - ) -), - -distinct_pins AS ( - SELECT DISTINCT - x_3435, - y_3435, - pin10, - year - FROM {{ source('spatial', 'parcel') }} -) - -SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_road_name, - ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, - ARBITRARY(xy.year) AS nearest_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_road_daily_traffic, - pcl.year -FROM distinct_pins AS pcl -INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year -GROUP BY pcl.pin10, pcl.year; diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql deleted file mode 100644 index b2172abcf..000000000 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_local.sql +++ /dev/null @@ -1,40 +0,0 @@ --- CTAS to create a table of distance to the nearest Metra route for each PIN -{{ - config( - materialized='table', - partitioned_by=['year'], - bucketed_by=['pin10'], - bucket_count=1 - ) -}} - -WITH traffic AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE daily_traffic > 0 - AND daily_traffic IS NOT NULL - AND road_type = 'Local Road or Street' -), - -distinct_pins AS ( - SELECT DISTINCT - x_3435, - y_3435, - pin10, - year - FROM {{ source('spatial', 'parcel') }} -) - -SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_road_name, - ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, - ARBITRARY(xy.year) AS nearest_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_road_daily_traffic, - pcl.year -FROM distinct_pins AS pcl -INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year -GROUP BY pcl.pin10, pcl.year diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql deleted file mode 100644 index 0daa3982d..000000000 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_daily_traffic_minor.sql +++ /dev/null @@ -1,40 +0,0 @@ --- CTAS to create a table of distance to the nearest Metra route for each PIN -{{ - config( - materialized='table', - partitioned_by=['year'], - bucketed_by=['pin10'], - bucket_count=1 - ) -}} - -WITH traffic AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE daily_traffic > 0 - AND daily_traffic IS NOT NULL - AND road_type = 'Minor Arterial' -), - -distinct_pins AS ( - SELECT DISTINCT - x_3435, - y_3435, - pin10, - year - FROM {{ source('spatial', 'parcel') }} -) - -SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_road_name, - ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, - ARBITRARY(xy.year) AS nearest_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_road_daily_traffic, - pcl.year -FROM distinct_pins AS pcl -INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year -GROUP BY pcl.pin10, pcl.year diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 4f3141e85..00a56edda 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -1,4 +1,4 @@ --- CTAS to create a table of distance to the nearest Metra route for each PIN +-- CTAS to create a table of distance to the nearest road for each PIN {{ config( materialized='table', @@ -14,13 +14,34 @@ WITH traffic_minor AS ( -- noqa: ST03 WHERE road_type = 'Minor Arterial' ), -traffic_highway AS ( -- noqa: ST03 +traffic_interstate AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} - WHERE ( - road_type = 'Interstate' - OR road_type = 'Freeway And Expressway' - ) + WHERE road_type = 'Interstate' +), + +traffic_freeway AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Freeway and Expressway' +), + +traffic_principal AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Other Principal Arterial' +), + +traffic_major_collector AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Major Collector' +), + +traffic_minor_collector AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Minor Collector' ), distinct_pins AS ( @@ -49,17 +70,89 @@ nearest_minor AS ( GROUP BY pcl.pin10, pcl.year ), --- Select nearest road from Highway (Interstate, Freeway, Expressway) -nearest_highway AS ( +-- Select nearest road from Interstate +nearest_interstate AS ( + SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_interstate_road_name, + ARBITRARY(xy.dist_ft) AS nearest_interstate_road_dist_ft, + ARBITRARY(xy.year) AS nearest_interstate_road_data_year, + ARBITRARY(xy.surface_width) AS nearest_interstate_road_surface_width, + pcl.year + FROM distinct_pins AS pcl + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_interstate') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year + GROUP BY pcl.pin10, pcl.year +), + +-- Select nearest road from Freeway and Expressway +nearest_freeway AS ( + SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_freeway_road_name, + ARBITRARY(xy.dist_ft) AS nearest_freeway_road_dist_ft, + ARBITRARY(xy.year) AS nearest_freeway_road_data_year, + ARBITRARY(xy.surface_width) AS nearest_freeway_road_surface_width, + pcl.year + FROM distinct_pins AS pcl + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_freeway') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year + GROUP BY pcl.pin10, pcl.year +), + +-- Select nearest road from Other Principal Arterial +nearest_principal AS ( + SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_principal_road_name, + ARBITRARY(xy.dist_ft) AS nearest_principal_road_dist_ft, + ARBITRARY(xy.year) AS nearest_principal_road_data_year, + ARBITRARY(xy.surface_width) AS nearest_principal_road_surface_width, + pcl.year + FROM distinct_pins AS pcl + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_principal') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year + GROUP BY pcl.pin10, pcl.year +), + +-- Select nearest road from Major Collector +nearest_major_collector AS ( + SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_major_collector_road_name, + ARBITRARY(xy.dist_ft) AS nearest_major_collector_road_dist_ft, + ARBITRARY(xy.year) AS nearest_major_collector_road_data_year, + ARBITRARY(xy.surface_width) + AS nearest_major_collector_road_surface_width, + pcl.year + FROM distinct_pins AS pcl + INNER JOIN + ( {{ dist_to_nearest_geometry('traffic_major_collector') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year + GROUP BY pcl.pin10, pcl.year +), + +-- Select nearest road from Minor Collector +nearest_minor_collector AS ( SELECT pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_highway_road_name, - ARBITRARY(xy.dist_ft) AS nearest_highway_road_dist_ft, - ARBITRARY(xy.year) AS nearest_highway_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_highway_road_surface_width, + ARBITRARY(xy.road_name) AS nearest_minor_collector_road_name, + ARBITRARY(xy.dist_ft) AS nearest_minor_collector_road_dist_ft, + ARBITRARY(xy.year) AS nearest_minor_collector_road_data_year, + ARBITRARY(xy.surface_width) + AS nearest_minor_collector_road_surface_width, pcl.year FROM distinct_pins AS pcl - INNER JOIN ( {{ dist_to_nearest_geometry('traffic_highway') }} ) AS xy + INNER JOIN + ( {{ dist_to_nearest_geometry('traffic_minor_collector') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 AND pcl.year = xy.pin_year @@ -68,17 +161,54 @@ nearest_highway AS ( -- Join the results based on pin10 and year SELECT - COALESCE(minor.pin10, highway.pin10) AS pin10, + COALESCE( + minor.pin10, + interstate.pin10, + freeway.pin10, + principal.pin10, + major_collector.pin10, + minor_collector.pin10 + ) AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, minor.nearest_minor_road_data_year, minor.nearest_minor_road_surface_width, - highway.nearest_highway_road_name, - highway.nearest_highway_road_dist_ft, - highway.nearest_highway_road_data_year, - highway.nearest_highway_road_surface_width, - COALESCE(minor.year, highway.year) AS year + interstate.nearest_interstate_road_name, + interstate.nearest_interstate_road_dist_ft, + interstate.nearest_interstate_road_data_year, + interstate.nearest_interstate_road_surface_width, + freeway.nearest_freeway_road_name, + freeway.nearest_freeway_road_dist_ft, + freeway.nearest_freeway_road_data_year, + freeway.nearest_freeway_road_surface_width, + principal.nearest_principal_road_name, + principal.nearest_principal_road_dist_ft, + principal.nearest_principal_road_data_year, + principal.nearest_principal_road_surface_width, + major_collector.nearest_major_collector_road_name, + major_collector.nearest_major_collector_road_dist_ft, + major_collector.nearest_major_collector_road_data_year, + major_collector.nearest_major_collector_road_surface_width, + minor_collector.nearest_minor_collector_road_name, + minor_collector.nearest_minor_collector_road_dist_ft, + minor_collector.nearest_minor_collector_road_data_year, + minor_collector.nearest_minor_collector_road_surface_width, + COALESCE( + minor.year, + interstate.year, + freeway.year, + principal.year, + major_collector.year, + minor_collector.year + ) AS year FROM nearest_minor AS minor -FULL OUTER JOIN nearest_highway AS highway - ON minor.pin10 = highway.pin10 - AND minor.year = highway.year +FULL OUTER JOIN nearest_interstate AS interstate + ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year +FULL OUTER JOIN nearest_freeway AS freeway + ON minor.pin10 = freeway.pin10 AND minor.year = freeway.year +FULL OUTER JOIN nearest_principal AS principal + ON minor.pin10 = principal.pin10 AND minor.year = principal.year +FULL OUTER JOIN nearest_major_collector AS major_collector + ON minor.pin10 = major_collector.pin10 AND minor.year = major_collector.year +FULL OUTER JOIN nearest_minor_collector AS minor_collector + ON minor.pin10 = minor_collector.pin10 AND minor.year = minor_collector.year diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_width_highway.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_width_highway.sql deleted file mode 100644 index 692c62bc7..000000000 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_width_highway.sql +++ /dev/null @@ -1,41 +0,0 @@ --- CTAS to create a table of distance to the nearest Metra route for each PIN -{{ - config( - materialized='table', - partitioned_by=['year'], - bucketed_by=['pin10'], - bucket_count=1 - ) -}} - -WITH traffic AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE ( - road_type = 'Interstate' - OR road_type = 'Freeway And Expressway' - ) -), - -distinct_pins AS ( - SELECT DISTINCT - x_3435, - y_3435, - pin10, - year - FROM {{ source('spatial', 'parcel') }} -) - -SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_road_name, - ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, - ARBITRARY(xy.year) AS nearest_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_surface_width, - pcl.year -FROM distinct_pins AS pcl -INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year -GROUP BY pcl.pin10, pcl.year diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_width_local.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_width_local.sql deleted file mode 100644 index 8376475e4..000000000 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_width_local.sql +++ /dev/null @@ -1,39 +0,0 @@ --- CTAS to create a table of distance to the nearest Metra route for each PIN -{{ - config( - materialized='table', - partitioned_by=['year'], - bucketed_by=['pin10'], - bucket_count=1 - ) -}} - -WITH traffic AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Local Road or Street' -), - -distinct_pins AS ( - SELECT DISTINCT - x_3435, - y_3435, - pin10, - year - FROM {{ source('spatial', 'parcel') }} -) - -SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_road_name, - ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, - ARBITRARY(xy.year) AS nearest_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_surface_width, - ARBITRARY(xy.surface_type) AS nearest_surface_type, - pcl.year -FROM distinct_pins AS pcl -INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year -GROUP BY pcl.pin10, pcl.year diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_width_minor.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_width_minor.sql deleted file mode 100644 index 7b4c49bf7..000000000 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_width_minor.sql +++ /dev/null @@ -1,38 +0,0 @@ -{{ - config( - materialized='table', - partitioned_by=['year'], - bucketed_by=['pin10'], - bucket_count=1 - ) -}} - -WITH traffic AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Minor Arterial' -), - -distinct_pins AS ( - SELECT DISTINCT - x_3435, - y_3435, - pin10, - year - FROM {{ source('spatial', 'parcel') }} -) - -SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_road_name, - ARBITRARY(xy.dist_ft) AS nearest_road_dist_ft, - ARBITRARY(xy.year) AS nearest_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_surface_width, - ARBITRARY(xy.surface_type) AS nearest_surface_type, - pcl.year -FROM distinct_pins AS pcl -INNER JOIN ( {{ dist_to_nearest_geometry('traffic') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year -GROUP BY pcl.pin10, pcl.year From aace136d52b43e4d11dde619b1b61be4a3814a20 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 15 Oct 2024 20:52:21 +0000 Subject: [PATCH 19/61] Try to remove 2014 --- dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 00a56edda..d90bc1e64 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -42,6 +42,7 @@ traffic_minor_collector AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} WHERE road_type = 'Minor Collector' + AND year >= 2014 ), distinct_pins AS ( From 3f070dea0044a975737dd3f10319e8b65cac7bea Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 15 Oct 2024 20:53:42 +0000 Subject: [PATCH 20/61] Remove minor_collector --- .../proximity.dist_pin_to_traffic_master.sql | 38 +------------------ 1 file changed, 2 insertions(+), 36 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index d90bc1e64..db30a32af 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -38,13 +38,6 @@ traffic_major_collector AS ( -- noqa: ST03 WHERE road_type = 'Major Collector' ), -traffic_minor_collector AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Minor Collector' - AND year >= 2014 -), - distinct_pins AS ( SELECT DISTINCT x_3435, @@ -141,25 +134,6 @@ nearest_major_collector AS ( GROUP BY pcl.pin10, pcl.year ), --- Select nearest road from Minor Collector -nearest_minor_collector AS ( - SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_minor_collector_road_name, - ARBITRARY(xy.dist_ft) AS nearest_minor_collector_road_dist_ft, - ARBITRARY(xy.year) AS nearest_minor_collector_road_data_year, - ARBITRARY(xy.surface_width) - AS nearest_minor_collector_road_surface_width, - pcl.year - FROM distinct_pins AS pcl - INNER JOIN - ( {{ dist_to_nearest_geometry('traffic_minor_collector') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year - GROUP BY pcl.pin10, pcl.year -) - -- Join the results based on pin10 and year SELECT COALESCE( @@ -167,8 +141,7 @@ SELECT interstate.pin10, freeway.pin10, principal.pin10, - major_collector.pin10, - minor_collector.pin10 + major_collector.pin10 ) AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, @@ -190,17 +163,12 @@ SELECT major_collector.nearest_major_collector_road_dist_ft, major_collector.nearest_major_collector_road_data_year, major_collector.nearest_major_collector_road_surface_width, - minor_collector.nearest_minor_collector_road_name, - minor_collector.nearest_minor_collector_road_dist_ft, - minor_collector.nearest_minor_collector_road_data_year, - minor_collector.nearest_minor_collector_road_surface_width, COALESCE( minor.year, interstate.year, freeway.year, principal.year, - major_collector.year, - minor_collector.year + major_collector.year ) AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate @@ -211,5 +179,3 @@ FULL OUTER JOIN nearest_principal AS principal ON minor.pin10 = principal.pin10 AND minor.year = principal.year FULL OUTER JOIN nearest_major_collector AS major_collector ON minor.pin10 = major_collector.pin10 AND minor.year = major_collector.year -FULL OUTER JOIN nearest_minor_collector AS minor_collector - ON minor.pin10 = minor_collector.pin10 AND minor.year = minor_collector.year From b50e8133f5c0584de0ae2eaf71598334575d9a4a Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 15 Oct 2024 20:55:01 +0000 Subject: [PATCH 21/61] Remove parcel.year = pin.year --- .../proximity.dist_pin_to_traffic_master.sql | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index db30a32af..13a864793 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -38,6 +38,12 @@ traffic_major_collector AS ( -- noqa: ST03 WHERE road_type = 'Major Collector' ), +traffic_minor_collector AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Minor Collector' +), + distinct_pins AS ( SELECT DISTINCT x_3435, @@ -134,6 +140,24 @@ nearest_major_collector AS ( GROUP BY pcl.pin10, pcl.year ), +-- Select nearest road from Minor Collector +nearest_minor_collector AS ( + SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_minor_collector_road_name, + ARBITRARY(xy.dist_ft) AS nearest_minor_collector_road_dist_ft, + ARBITRARY(xy.year) AS nearest_minor_collector_road_data_year, + ARBITRARY(xy.surface_width) + AS nearest_minor_collector_road_surface_width, + pcl.year + FROM distinct_pins AS pcl + INNER JOIN + ( {{ dist_to_nearest_geometry('traffic_minor_collector') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + GROUP BY pcl.pin10, pcl.year +) + -- Join the results based on pin10 and year SELECT COALESCE( @@ -141,7 +165,8 @@ SELECT interstate.pin10, freeway.pin10, principal.pin10, - major_collector.pin10 + major_collector.pin10, + minor_collector.pin10 ) AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, @@ -163,12 +188,17 @@ SELECT major_collector.nearest_major_collector_road_dist_ft, major_collector.nearest_major_collector_road_data_year, major_collector.nearest_major_collector_road_surface_width, + minor_collector.nearest_minor_collector_road_name, + minor_collector.nearest_minor_collector_road_dist_ft, + minor_collector.nearest_minor_collector_road_data_year, + minor_collector.nearest_minor_collector_road_surface_width, COALESCE( minor.year, interstate.year, freeway.year, principal.year, - major_collector.year + major_collector.year, + minor_collector.year ) AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate @@ -179,3 +209,5 @@ FULL OUTER JOIN nearest_principal AS principal ON minor.pin10 = principal.pin10 AND minor.year = principal.year FULL OUTER JOIN nearest_major_collector AS major_collector ON minor.pin10 = major_collector.pin10 AND minor.year = major_collector.year +FULL OUTER JOIN nearest_minor_collector AS minor_collector + ON minor.pin10 = minor_collector.pin10 AND minor.year = minor_collector.year From ebbc52ef67d6c448d20eb35dc09102331b922062 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 15 Oct 2024 21:02:52 +0000 Subject: [PATCH 22/61] Remove minor again --- .../proximity.dist_pin_to_traffic_master.sql | 36 ++----------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 13a864793..db30a32af 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -38,12 +38,6 @@ traffic_major_collector AS ( -- noqa: ST03 WHERE road_type = 'Major Collector' ), -traffic_minor_collector AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Minor Collector' -), - distinct_pins AS ( SELECT DISTINCT x_3435, @@ -140,24 +134,6 @@ nearest_major_collector AS ( GROUP BY pcl.pin10, pcl.year ), --- Select nearest road from Minor Collector -nearest_minor_collector AS ( - SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_minor_collector_road_name, - ARBITRARY(xy.dist_ft) AS nearest_minor_collector_road_dist_ft, - ARBITRARY(xy.year) AS nearest_minor_collector_road_data_year, - ARBITRARY(xy.surface_width) - AS nearest_minor_collector_road_surface_width, - pcl.year - FROM distinct_pins AS pcl - INNER JOIN - ( {{ dist_to_nearest_geometry('traffic_minor_collector') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - GROUP BY pcl.pin10, pcl.year -) - -- Join the results based on pin10 and year SELECT COALESCE( @@ -165,8 +141,7 @@ SELECT interstate.pin10, freeway.pin10, principal.pin10, - major_collector.pin10, - minor_collector.pin10 + major_collector.pin10 ) AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, @@ -188,17 +163,12 @@ SELECT major_collector.nearest_major_collector_road_dist_ft, major_collector.nearest_major_collector_road_data_year, major_collector.nearest_major_collector_road_surface_width, - minor_collector.nearest_minor_collector_road_name, - minor_collector.nearest_minor_collector_road_dist_ft, - minor_collector.nearest_minor_collector_road_data_year, - minor_collector.nearest_minor_collector_road_surface_width, COALESCE( minor.year, interstate.year, freeway.year, principal.year, - major_collector.year, - minor_collector.year + major_collector.year ) AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate @@ -209,5 +179,3 @@ FULL OUTER JOIN nearest_principal AS principal ON minor.pin10 = principal.pin10 AND minor.year = principal.year FULL OUTER JOIN nearest_major_collector AS major_collector ON minor.pin10 = major_collector.pin10 AND minor.year = major_collector.year -FULL OUTER JOIN nearest_minor_collector AS minor_collector - ON minor.pin10 = minor_collector.pin10 AND minor.year = minor_collector.year From e8671f71179737f31e6bf1a805b0d87cf949c6d1 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 15 Oct 2024 21:05:44 +0000 Subject: [PATCH 23/61] Remove Freeway --- .../proximity.dist_pin_to_traffic_master.sql | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index db30a32af..aabdc2296 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -20,12 +20,6 @@ traffic_interstate AS ( -- noqa: ST03 WHERE road_type = 'Interstate' ), -traffic_freeway AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Freeway and Expressway' -), - traffic_principal AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} @@ -81,23 +75,6 @@ nearest_interstate AS ( GROUP BY pcl.pin10, pcl.year ), --- Select nearest road from Freeway and Expressway -nearest_freeway AS ( - SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_freeway_road_name, - ARBITRARY(xy.dist_ft) AS nearest_freeway_road_dist_ft, - ARBITRARY(xy.year) AS nearest_freeway_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_freeway_road_surface_width, - pcl.year - FROM distinct_pins AS pcl - INNER JOIN ( {{ dist_to_nearest_geometry('traffic_freeway') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year - GROUP BY pcl.pin10, pcl.year -), - -- Select nearest road from Other Principal Arterial nearest_principal AS ( SELECT @@ -139,7 +116,6 @@ SELECT COALESCE( minor.pin10, interstate.pin10, - freeway.pin10, principal.pin10, major_collector.pin10 ) AS pin10, @@ -151,10 +127,6 @@ SELECT interstate.nearest_interstate_road_dist_ft, interstate.nearest_interstate_road_data_year, interstate.nearest_interstate_road_surface_width, - freeway.nearest_freeway_road_name, - freeway.nearest_freeway_road_dist_ft, - freeway.nearest_freeway_road_data_year, - freeway.nearest_freeway_road_surface_width, principal.nearest_principal_road_name, principal.nearest_principal_road_dist_ft, principal.nearest_principal_road_data_year, @@ -166,15 +138,12 @@ SELECT COALESCE( minor.year, interstate.year, - freeway.year, principal.year, major_collector.year ) AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year -FULL OUTER JOIN nearest_freeway AS freeway - ON minor.pin10 = freeway.pin10 AND minor.year = freeway.year FULL OUTER JOIN nearest_principal AS principal ON minor.pin10 = principal.pin10 AND minor.year = principal.year FULL OUTER JOIN nearest_major_collector AS major_collector From 967114c997d58daf5741bd80e3d40b708a8e1514 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 15 Oct 2024 21:11:48 +0000 Subject: [PATCH 24/61] Start from begining --- .../proximity.dist_pin_to_traffic_master.sql | 112 ++++-------------- 1 file changed, 23 insertions(+), 89 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index aabdc2296..4f3141e85 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -1,4 +1,4 @@ --- CTAS to create a table of distance to the nearest road for each PIN +-- CTAS to create a table of distance to the nearest Metra route for each PIN {{ config( materialized='table', @@ -14,22 +14,13 @@ WITH traffic_minor AS ( -- noqa: ST03 WHERE road_type = 'Minor Arterial' ), -traffic_interstate AS ( -- noqa: ST03 +traffic_highway AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Interstate' -), - -traffic_principal AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Other Principal Arterial' -), - -traffic_major_collector AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Major Collector' + WHERE ( + road_type = 'Interstate' + OR road_type = 'Freeway And Expressway' + ) ), distinct_pins AS ( @@ -58,93 +49,36 @@ nearest_minor AS ( GROUP BY pcl.pin10, pcl.year ), --- Select nearest road from Interstate -nearest_interstate AS ( - SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_interstate_road_name, - ARBITRARY(xy.dist_ft) AS nearest_interstate_road_dist_ft, - ARBITRARY(xy.year) AS nearest_interstate_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_interstate_road_surface_width, - pcl.year - FROM distinct_pins AS pcl - INNER JOIN ( {{ dist_to_nearest_geometry('traffic_interstate') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year - GROUP BY pcl.pin10, pcl.year -), - --- Select nearest road from Other Principal Arterial -nearest_principal AS ( +-- Select nearest road from Highway (Interstate, Freeway, Expressway) +nearest_highway AS ( SELECT pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_principal_road_name, - ARBITRARY(xy.dist_ft) AS nearest_principal_road_dist_ft, - ARBITRARY(xy.year) AS nearest_principal_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_principal_road_surface_width, + ARBITRARY(xy.road_name) AS nearest_highway_road_name, + ARBITRARY(xy.dist_ft) AS nearest_highway_road_dist_ft, + ARBITRARY(xy.year) AS nearest_highway_road_data_year, + ARBITRARY(xy.surface_width) AS nearest_highway_road_surface_width, pcl.year FROM distinct_pins AS pcl - INNER JOIN ( {{ dist_to_nearest_geometry('traffic_principal') }} ) AS xy + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_highway') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 AND pcl.year = xy.pin_year GROUP BY pcl.pin10, pcl.year -), - --- Select nearest road from Major Collector -nearest_major_collector AS ( - SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_major_collector_road_name, - ARBITRARY(xy.dist_ft) AS nearest_major_collector_road_dist_ft, - ARBITRARY(xy.year) AS nearest_major_collector_road_data_year, - ARBITRARY(xy.surface_width) - AS nearest_major_collector_road_surface_width, - pcl.year - FROM distinct_pins AS pcl - INNER JOIN - ( {{ dist_to_nearest_geometry('traffic_major_collector') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year - GROUP BY pcl.pin10, pcl.year -), +) -- Join the results based on pin10 and year SELECT - COALESCE( - minor.pin10, - interstate.pin10, - principal.pin10, - major_collector.pin10 - ) AS pin10, + COALESCE(minor.pin10, highway.pin10) AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, minor.nearest_minor_road_data_year, minor.nearest_minor_road_surface_width, - interstate.nearest_interstate_road_name, - interstate.nearest_interstate_road_dist_ft, - interstate.nearest_interstate_road_data_year, - interstate.nearest_interstate_road_surface_width, - principal.nearest_principal_road_name, - principal.nearest_principal_road_dist_ft, - principal.nearest_principal_road_data_year, - principal.nearest_principal_road_surface_width, - major_collector.nearest_major_collector_road_name, - major_collector.nearest_major_collector_road_dist_ft, - major_collector.nearest_major_collector_road_data_year, - major_collector.nearest_major_collector_road_surface_width, - COALESCE( - minor.year, - interstate.year, - principal.year, - major_collector.year - ) AS year + highway.nearest_highway_road_name, + highway.nearest_highway_road_dist_ft, + highway.nearest_highway_road_data_year, + highway.nearest_highway_road_surface_width, + COALESCE(minor.year, highway.year) AS year FROM nearest_minor AS minor -FULL OUTER JOIN nearest_interstate AS interstate - ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year -FULL OUTER JOIN nearest_principal AS principal - ON minor.pin10 = principal.pin10 AND minor.year = principal.year -FULL OUTER JOIN nearest_major_collector AS major_collector - ON minor.pin10 = major_collector.pin10 AND minor.year = major_collector.year +FULL OUTER JOIN nearest_highway AS highway + ON minor.pin10 = highway.pin10 + AND minor.year = highway.year From ba0b0a1608361f0f8d17083f921befee2cf350b3 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 15 Oct 2024 21:23:09 +0000 Subject: [PATCH 25/61] Separate freeway --- .../proximity.dist_pin_to_traffic_master.sql | 70 +++++++++++++------ 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 4f3141e85..5d7c57946 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -1,4 +1,4 @@ --- CTAS to create a table of distance to the nearest Metra route for each PIN +-- CTAS to create a table of distance to the nearest road for each PIN {{ config( materialized='table', @@ -14,13 +14,16 @@ WITH traffic_minor AS ( -- noqa: ST03 WHERE road_type = 'Minor Arterial' ), -traffic_highway AS ( -- noqa: ST03 +traffic_interstate AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} - WHERE ( - road_type = 'Interstate' - OR road_type = 'Freeway And Expressway' - ) + WHERE road_type = 'Interstate' +), + +traffic_freeway AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Freeway And Expressway' ), distinct_pins AS ( @@ -49,17 +52,34 @@ nearest_minor AS ( GROUP BY pcl.pin10, pcl.year ), --- Select nearest road from Highway (Interstate, Freeway, Expressway) -nearest_highway AS ( +-- Select nearest road from Interstate +nearest_interstate AS ( + SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_interstate_road_name, + ARBITRARY(xy.dist_ft) AS nearest_interstate_road_dist_ft, + ARBITRARY(xy.year) AS nearest_interstate_road_data_year, + ARBITRARY(xy.surface_width) AS nearest_interstate_road_surface_width, + pcl.year + FROM distinct_pins AS pcl + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_interstate') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year + GROUP BY pcl.pin10, pcl.year +), + +-- Select nearest road from Freeway And Expressway +nearest_freeway AS ( SELECT pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_highway_road_name, - ARBITRARY(xy.dist_ft) AS nearest_highway_road_dist_ft, - ARBITRARY(xy.year) AS nearest_highway_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_highway_road_surface_width, + ARBITRARY(xy.road_name) AS nearest_freeway_road_name, + ARBITRARY(xy.dist_ft) AS nearest_freeway_road_dist_ft, + ARBITRARY(xy.year) AS nearest_freeway_road_data_year, + ARBITRARY(xy.surface_width) AS nearest_freeway_road_surface_width, pcl.year FROM distinct_pins AS pcl - INNER JOIN ( {{ dist_to_nearest_geometry('traffic_highway') }} ) AS xy + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_freeway') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 AND pcl.year = xy.pin_year @@ -68,17 +88,23 @@ nearest_highway AS ( -- Join the results based on pin10 and year SELECT - COALESCE(minor.pin10, highway.pin10) AS pin10, + COALESCE(minor.pin10, interstate.pin10, freeway.pin10) AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, minor.nearest_minor_road_data_year, minor.nearest_minor_road_surface_width, - highway.nearest_highway_road_name, - highway.nearest_highway_road_dist_ft, - highway.nearest_highway_road_data_year, - highway.nearest_highway_road_surface_width, - COALESCE(minor.year, highway.year) AS year + interstate.nearest_interstate_road_name, + interstate.nearest_interstate_road_dist_ft, + interstate.nearest_interstate_road_data_year, + interstate.nearest_interstate_road_surface_width, + freeway.nearest_freeway_road_name, + freeway.nearest_freeway_road_dist_ft, + freeway.nearest_freeway_road_data_year, + freeway.nearest_freeway_road_surface_width, + COALESCE(minor.year, interstate.year, freeway.year) AS year FROM nearest_minor AS minor -FULL OUTER JOIN nearest_highway AS highway - ON minor.pin10 = highway.pin10 - AND minor.year = highway.year +FULL OUTER JOIN nearest_interstate AS interstate + ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year +FULL OUTER JOIN nearest_freeway AS freeway + ON COALESCE(minor.pin10, interstate.pin10) = freeway.pin10 + AND COALESCE(minor.year, interstate.year) = freeway.year From aa91cf82a49ae9292d14454e0e40fc374e08d0e9 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 15 Oct 2024 21:24:25 +0000 Subject: [PATCH 26/61] Add principal --- .../proximity.dist_pin_to_traffic_master.sql | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 5d7c57946..c514abf4e 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -26,6 +26,12 @@ traffic_freeway AS ( -- noqa: ST03 WHERE road_type = 'Freeway And Expressway' ), +traffic_principal AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Other Principal Arterial' +), + distinct_pins AS ( SELECT DISTINCT x_3435, @@ -84,11 +90,29 @@ nearest_freeway AS ( AND pcl.y_3435 = xy.y_3435 AND pcl.year = xy.pin_year GROUP BY pcl.pin10, pcl.year +), + +-- Select nearest road from Other Principal Arterial +nearest_principal AS ( + SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_principal_road_name, + ARBITRARY(xy.dist_ft) AS nearest_principal_road_dist_ft, + ARBITRARY(xy.year) AS nearest_principal_road_data_year, + ARBITRARY(xy.surface_width) AS nearest_principal_road_surface_width, + pcl.year + FROM distinct_pins AS pcl + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_principal') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year + GROUP BY pcl.pin10, pcl.year ) -- Join the results based on pin10 and year SELECT - COALESCE(minor.pin10, interstate.pin10, freeway.pin10) AS pin10, + COALESCE(minor.pin10, interstate.pin10, freeway.pin10, principal.pin10) + AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, minor.nearest_minor_road_data_year, @@ -101,10 +125,17 @@ SELECT freeway.nearest_freeway_road_dist_ft, freeway.nearest_freeway_road_data_year, freeway.nearest_freeway_road_surface_width, - COALESCE(minor.year, interstate.year, freeway.year) AS year + principal.nearest_principal_road_name, + principal.nearest_principal_road_dist_ft, + principal.nearest_principal_road_data_year, + principal.nearest_principal_road_surface_width, + COALESCE(minor.year, interstate.year, freeway.year, principal.year) AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year FULL OUTER JOIN nearest_freeway AS freeway ON COALESCE(minor.pin10, interstate.pin10) = freeway.pin10 AND COALESCE(minor.year, interstate.year) = freeway.year +FULL OUTER JOIN nearest_principal AS principal + ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) = principal.pin10 + AND COALESCE(minor.year, interstate.year, freeway.year) = principal.year From 639468cd3ab3ed025e803e57f23f600656e163e5 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 15 Oct 2024 22:16:47 +0000 Subject: [PATCH 27/61] Try with major --- .../proximity.dist_pin_to_traffic_master.sql | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index c514abf4e..37fd15af3 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -26,10 +26,10 @@ traffic_freeway AS ( -- noqa: ST03 WHERE road_type = 'Freeway And Expressway' ), -traffic_principal AS ( -- noqa: ST03 +traffic_major_collector AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Other Principal Arterial' + WHERE road_type = 'Major Collector' ), distinct_pins AS ( @@ -92,17 +92,19 @@ nearest_freeway AS ( GROUP BY pcl.pin10, pcl.year ), --- Select nearest road from Other Principal Arterial -nearest_principal AS ( +-- Select nearest road from Major Collector +nearest_major_collector AS ( SELECT pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_principal_road_name, - ARBITRARY(xy.dist_ft) AS nearest_principal_road_dist_ft, - ARBITRARY(xy.year) AS nearest_principal_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_principal_road_surface_width, + ARBITRARY(xy.road_name) AS nearest_major_collector_road_name, + ARBITRARY(xy.dist_ft) AS nearest_major_collector_road_dist_ft, + ARBITRARY(xy.year) AS nearest_major_collector_road_data_year, + ARBITRARY(xy.surface_width) + AS nearest_major_collector_road_surface_width, pcl.year FROM distinct_pins AS pcl - INNER JOIN ( {{ dist_to_nearest_geometry('traffic_principal') }} ) AS xy + INNER JOIN + ( {{ dist_to_nearest_geometry('traffic_major_collector') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 AND pcl.year = xy.pin_year @@ -111,8 +113,9 @@ nearest_principal AS ( -- Join the results based on pin10 and year SELECT - COALESCE(minor.pin10, interstate.pin10, freeway.pin10, principal.pin10) - AS pin10, + COALESCE( + minor.pin10, interstate.pin10, freeway.pin10, major_collector.pin10 + ) AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, minor.nearest_minor_road_data_year, @@ -125,17 +128,20 @@ SELECT freeway.nearest_freeway_road_dist_ft, freeway.nearest_freeway_road_data_year, freeway.nearest_freeway_road_surface_width, - principal.nearest_principal_road_name, - principal.nearest_principal_road_dist_ft, - principal.nearest_principal_road_data_year, - principal.nearest_principal_road_surface_width, - COALESCE(minor.year, interstate.year, freeway.year, principal.year) AS year + major_collector.nearest_major_collector_road_name, + major_collector.nearest_major_collector_road_dist_ft, + major_collector.nearest_major_collector_road_data_year, + major_collector.nearest_major_collector_road_surface_width, + COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) + AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year FULL OUTER JOIN nearest_freeway AS freeway ON COALESCE(minor.pin10, interstate.pin10) = freeway.pin10 AND COALESCE(minor.year, interstate.year) = freeway.year -FULL OUTER JOIN nearest_principal AS principal - ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) = principal.pin10 - AND COALESCE(minor.year, interstate.year, freeway.year) = principal.year +FULL OUTER JOIN nearest_major_collector AS major_collector + ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) + = major_collector.pin10 + AND COALESCE(minor.year, interstate.year, freeway.year) + = major_collector.year From ed4ba5f05600f28c4ffd4e6bb6f6f165598a0d47 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Wed, 16 Oct 2024 14:09:23 +0000 Subject: [PATCH 28/61] Add other --- .../proximity.dist_pin_to_traffic_master.sql | 49 +++++++++++++++++-- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 37fd15af3..391287389 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -26,6 +26,12 @@ traffic_freeway AS ( -- noqa: ST03 WHERE road_type = 'Freeway And Expressway' ), +traffic_principal AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Other Principal Arterial' +), + traffic_major_collector AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} @@ -92,6 +98,23 @@ nearest_freeway AS ( GROUP BY pcl.pin10, pcl.year ), +-- Select nearest road from Other Principal Arterial +nearest_principal AS ( + SELECT + pcl.pin10, + ARBITRARY(xy.road_name) AS nearest_principal_road_name, + ARBITRARY(xy.dist_ft) AS nearest_principal_road_dist_ft, + ARBITRARY(xy.year) AS nearest_principal_road_data_year, + ARBITRARY(xy.surface_width) AS nearest_principal_road_surface_width, + pcl.year + FROM distinct_pins AS pcl + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_principal') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + AND pcl.year = xy.pin_year + GROUP BY pcl.pin10, pcl.year +), + -- Select nearest road from Major Collector nearest_major_collector AS ( SELECT @@ -114,7 +137,11 @@ nearest_major_collector AS ( -- Join the results based on pin10 and year SELECT COALESCE( - minor.pin10, interstate.pin10, freeway.pin10, major_collector.pin10 + minor.pin10, + interstate.pin10, + freeway.pin10, + principal.pin10, + major_collector.pin10 ) AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, @@ -128,20 +155,32 @@ SELECT freeway.nearest_freeway_road_dist_ft, freeway.nearest_freeway_road_data_year, freeway.nearest_freeway_road_surface_width, + principal.nearest_principal_road_name, + principal.nearest_principal_road_dist_ft, + principal.nearest_principal_road_data_year, + principal.nearest_principal_road_surface_width, major_collector.nearest_major_collector_road_name, major_collector.nearest_major_collector_road_dist_ft, major_collector.nearest_major_collector_road_data_year, major_collector.nearest_major_collector_road_surface_width, - COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) - AS year + COALESCE( + minor.year, + interstate.year, + freeway.year, + principal.year, + major_collector.year + ) AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year FULL OUTER JOIN nearest_freeway AS freeway ON COALESCE(minor.pin10, interstate.pin10) = freeway.pin10 AND COALESCE(minor.year, interstate.year) = freeway.year +FULL OUTER JOIN nearest_principal AS principal + ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) = principal.pin10 + AND COALESCE(minor.year, interstate.year, freeway.year) = principal.year FULL OUTER JOIN nearest_major_collector AS major_collector - ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) + ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10, principal.pin10) = major_collector.pin10 - AND COALESCE(minor.year, interstate.year, freeway.year) + AND COALESCE(minor.year, interstate.year, freeway.year, principal.year) = major_collector.year From 0c41137fc1ecd2a047d249827a802f27f4699bbf Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Wed, 16 Oct 2024 14:14:31 +0000 Subject: [PATCH 29/61] re-add major --- .../proximity/proximity.dist_pin_to_traffic_master.sql | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 391287389..0cf9f61e1 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -180,7 +180,9 @@ FULL OUTER JOIN nearest_principal AS principal ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) = principal.pin10 AND COALESCE(minor.year, interstate.year, freeway.year) = principal.year FULL OUTER JOIN nearest_major_collector AS major_collector - ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10, principal.pin10) - = major_collector.pin10 - AND COALESCE(minor.year, interstate.year, freeway.year, principal.year) - = major_collector.year + ON COALESCE( + minor.pin10, interstate.pin10, freeway.pin10, principal.pin10 + ) = major_collector.pin10 + AND COALESCE( + minor.year, interstate.year, freeway.year, principal.year + ) = major_collector.year From 7cc2b22d37bc1e6d8764b04a605c17ee9457c688 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Wed, 16 Oct 2024 15:25:54 +0000 Subject: [PATCH 30/61] remove year --- .../proximity.dist_pin_to_traffic_master.sql | 58 +++---------------- 1 file changed, 8 insertions(+), 50 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 0cf9f61e1..0b2c149a5 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -26,12 +26,6 @@ traffic_freeway AS ( -- noqa: ST03 WHERE road_type = 'Freeway And Expressway' ), -traffic_principal AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Other Principal Arterial' -), - traffic_major_collector AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} @@ -42,8 +36,7 @@ distinct_pins AS ( SELECT DISTINCT x_3435, y_3435, - pin10, - year + pin10 FROM {{ source('spatial', 'parcel') }} ), @@ -98,23 +91,6 @@ nearest_freeway AS ( GROUP BY pcl.pin10, pcl.year ), --- Select nearest road from Other Principal Arterial -nearest_principal AS ( - SELECT - pcl.pin10, - ARBITRARY(xy.road_name) AS nearest_principal_road_name, - ARBITRARY(xy.dist_ft) AS nearest_principal_road_dist_ft, - ARBITRARY(xy.year) AS nearest_principal_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_principal_road_surface_width, - pcl.year - FROM distinct_pins AS pcl - INNER JOIN ( {{ dist_to_nearest_geometry('traffic_principal') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year - GROUP BY pcl.pin10, pcl.year -), - -- Select nearest road from Major Collector nearest_major_collector AS ( SELECT @@ -137,11 +113,7 @@ nearest_major_collector AS ( -- Join the results based on pin10 and year SELECT COALESCE( - minor.pin10, - interstate.pin10, - freeway.pin10, - principal.pin10, - major_collector.pin10 + minor.pin10, interstate.pin10, freeway.pin10, major_collector.pin10 ) AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, @@ -155,34 +127,20 @@ SELECT freeway.nearest_freeway_road_dist_ft, freeway.nearest_freeway_road_data_year, freeway.nearest_freeway_road_surface_width, - principal.nearest_principal_road_name, - principal.nearest_principal_road_dist_ft, - principal.nearest_principal_road_data_year, - principal.nearest_principal_road_surface_width, major_collector.nearest_major_collector_road_name, major_collector.nearest_major_collector_road_dist_ft, major_collector.nearest_major_collector_road_data_year, major_collector.nearest_major_collector_road_surface_width, - COALESCE( - minor.year, - interstate.year, - freeway.year, - principal.year, - major_collector.year - ) AS year + COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) + AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year FULL OUTER JOIN nearest_freeway AS freeway ON COALESCE(minor.pin10, interstate.pin10) = freeway.pin10 AND COALESCE(minor.year, interstate.year) = freeway.year -FULL OUTER JOIN nearest_principal AS principal - ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) = principal.pin10 - AND COALESCE(minor.year, interstate.year, freeway.year) = principal.year FULL OUTER JOIN nearest_major_collector AS major_collector - ON COALESCE( - minor.pin10, interstate.pin10, freeway.pin10, principal.pin10 - ) = major_collector.pin10 - AND COALESCE( - minor.year, interstate.year, freeway.year, principal.year - ) = major_collector.year + ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) + = major_collector.pin10 + AND COALESCE(minor.year, interstate.year, freeway.year) + = major_collector.year From 03bdf619f053041eee392c67fdcf7dc667d7162a Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Wed, 16 Oct 2024 15:27:43 +0000 Subject: [PATCH 31/61] Remove year from join --- dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 0b2c149a5..92c2e6208 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -53,7 +53,6 @@ nearest_minor AS ( INNER JOIN ( {{ dist_to_nearest_geometry('traffic_minor') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year GROUP BY pcl.pin10, pcl.year ), @@ -70,7 +69,6 @@ nearest_interstate AS ( INNER JOIN ( {{ dist_to_nearest_geometry('traffic_interstate') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year GROUP BY pcl.pin10, pcl.year ), @@ -87,7 +85,6 @@ nearest_freeway AS ( INNER JOIN ( {{ dist_to_nearest_geometry('traffic_freeway') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year GROUP BY pcl.pin10, pcl.year ), @@ -106,7 +103,6 @@ nearest_major_collector AS ( ( {{ dist_to_nearest_geometry('traffic_major_collector') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 - AND pcl.year = xy.pin_year GROUP BY pcl.pin10, pcl.year ) From 09ce316c6d4825fe63c63f08447d07b0c3de3beb Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Wed, 16 Oct 2024 15:30:05 +0000 Subject: [PATCH 32/61] Remove all pcl.year --- .../proximity.dist_pin_to_traffic_master.sql | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 92c2e6208..ec2db3396 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -47,13 +47,12 @@ nearest_minor AS ( ARBITRARY(xy.road_name) AS nearest_minor_road_name, ARBITRARY(xy.dist_ft) AS nearest_minor_road_dist_ft, ARBITRARY(xy.year) AS nearest_minor_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_minor_road_surface_width, - pcl.year + ARBITRARY(xy.surface_width) AS nearest_minor_road_surface_width FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_minor') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 - GROUP BY pcl.pin10, pcl.year + GROUP BY pcl.pin10 ), -- Select nearest road from Interstate @@ -63,13 +62,12 @@ nearest_interstate AS ( ARBITRARY(xy.road_name) AS nearest_interstate_road_name, ARBITRARY(xy.dist_ft) AS nearest_interstate_road_dist_ft, ARBITRARY(xy.year) AS nearest_interstate_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_interstate_road_surface_width, - pcl.year + ARBITRARY(xy.surface_width) AS nearest_interstate_road_surface_width FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_interstate') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 - GROUP BY pcl.pin10, pcl.year + GROUP BY pcl.pin10 ), -- Select nearest road from Freeway And Expressway @@ -79,13 +77,12 @@ nearest_freeway AS ( ARBITRARY(xy.road_name) AS nearest_freeway_road_name, ARBITRARY(xy.dist_ft) AS nearest_freeway_road_dist_ft, ARBITRARY(xy.year) AS nearest_freeway_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_freeway_road_surface_width, - pcl.year + ARBITRARY(xy.surface_width) AS nearest_freeway_road_surface_width FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_freeway') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 - GROUP BY pcl.pin10, pcl.year + GROUP BY pcl.pin10 ), -- Select nearest road from Major Collector @@ -96,14 +93,13 @@ nearest_major_collector AS ( ARBITRARY(xy.dist_ft) AS nearest_major_collector_road_dist_ft, ARBITRARY(xy.year) AS nearest_major_collector_road_data_year, ARBITRARY(xy.surface_width) - AS nearest_major_collector_road_surface_width, - pcl.year + AS nearest_major_collector_road_surface_width FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_major_collector') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 - GROUP BY pcl.pin10, pcl.year + GROUP BY pcl.pin10 ) -- Join the results based on pin10 and year From 7291c2db8e49c7f1a889a04d3631a540b256621e Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Wed, 16 Oct 2024 15:34:08 +0000 Subject: [PATCH 33/61] remove year from joins --- .../proximity/proximity.dist_pin_to_traffic_master.sql | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index ec2db3396..da83c67b6 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -122,17 +122,12 @@ SELECT major_collector.nearest_major_collector_road_name, major_collector.nearest_major_collector_road_dist_ft, major_collector.nearest_major_collector_road_data_year, - major_collector.nearest_major_collector_road_surface_width, - COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) - AS year + major_collector.nearest_major_collector_road_surface_width FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate - ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year + ON minor.pin10 = interstate.pin10 FULL OUTER JOIN nearest_freeway AS freeway ON COALESCE(minor.pin10, interstate.pin10) = freeway.pin10 - AND COALESCE(minor.year, interstate.year) = freeway.year FULL OUTER JOIN nearest_major_collector AS major_collector ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) = major_collector.pin10 - AND COALESCE(minor.year, interstate.year, freeway.year) - = major_collector.year From f0ca25fbeccea282ab71fc4299fafa1b4c5bb4a1 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Wed, 16 Oct 2024 15:57:40 +0000 Subject: [PATCH 34/61] Try a bit of refactor --- .../proximity.dist_pin_to_traffic_master.sql | 48 ++++++++++++------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index da83c67b6..1cda71b2c 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -1,4 +1,3 @@ --- CTAS to create a table of distance to the nearest road for each PIN {{ config( materialized='table', @@ -40,58 +39,68 @@ distinct_pins AS ( FROM {{ source('spatial', 'parcel') }} ), --- Select nearest road from Minor Arterial +distinct_years_rhs AS ( + SELECT DISTINCT year + FROM {{ source('spatial', 'traffic') }} + WHERE road_type IS NOT NULL +), + +-- Calculate nearest Minor Arterial road per pin nearest_minor AS ( SELECT pcl.pin10, + xy.year, ARBITRARY(xy.road_name) AS nearest_minor_road_name, ARBITRARY(xy.dist_ft) AS nearest_minor_road_dist_ft, - ARBITRARY(xy.year) AS nearest_minor_road_data_year, + ARBITRARY(xy.year) AS data_year, ARBITRARY(xy.surface_width) AS nearest_minor_road_surface_width FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_minor') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 - GROUP BY pcl.pin10 + GROUP BY pcl.pin10, xy.year ), --- Select nearest road from Interstate +-- Calculate nearest Interstate road per pin nearest_interstate AS ( SELECT pcl.pin10, + xy.year, ARBITRARY(xy.road_name) AS nearest_interstate_road_name, ARBITRARY(xy.dist_ft) AS nearest_interstate_road_dist_ft, - ARBITRARY(xy.year) AS nearest_interstate_road_data_year, + ARBITRARY(xy.year) AS data_year, ARBITRARY(xy.surface_width) AS nearest_interstate_road_surface_width FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_interstate') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 - GROUP BY pcl.pin10 + GROUP BY pcl.pin10, xy.year ), --- Select nearest road from Freeway And Expressway +-- Calculate nearest Freeway or Expressway road per pin nearest_freeway AS ( SELECT pcl.pin10, + xy.year, ARBITRARY(xy.road_name) AS nearest_freeway_road_name, ARBITRARY(xy.dist_ft) AS nearest_freeway_road_dist_ft, - ARBITRARY(xy.year) AS nearest_freeway_road_data_year, + ARBITRARY(xy.year) AS data_year, ARBITRARY(xy.surface_width) AS nearest_freeway_road_surface_width FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_freeway') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 - GROUP BY pcl.pin10 + GROUP BY pcl.pin10, xy.year ), --- Select nearest road from Major Collector +-- Calculate nearest Major Collector road per pin nearest_major_collector AS ( SELECT pcl.pin10, + xy.year, ARBITRARY(xy.road_name) AS nearest_major_collector_road_name, ARBITRARY(xy.dist_ft) AS nearest_major_collector_road_dist_ft, - ARBITRARY(xy.year) AS nearest_major_collector_road_data_year, + ARBITRARY(xy.year) AS data_year, ARBITRARY(xy.surface_width) AS nearest_major_collector_road_surface_width FROM distinct_pins AS pcl @@ -99,10 +108,10 @@ nearest_major_collector AS ( ( {{ dist_to_nearest_geometry('traffic_major_collector') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 - GROUP BY pcl.pin10 + GROUP BY pcl.pin10, xy.year ) --- Join the results based on pin10 and year +-- Join all nearest roads by pin10 and year SELECT COALESCE( minor.pin10, interstate.pin10, freeway.pin10, major_collector.pin10 @@ -122,12 +131,19 @@ SELECT major_collector.nearest_major_collector_road_name, major_collector.nearest_major_collector_road_dist_ft, major_collector.nearest_major_collector_road_data_year, - major_collector.nearest_major_collector_road_surface_width + major_collector.nearest_major_collector_road_surface_width, + COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) + AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate - ON minor.pin10 = interstate.pin10 + ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year FULL OUTER JOIN nearest_freeway AS freeway ON COALESCE(minor.pin10, interstate.pin10) = freeway.pin10 + AND COALESCE(minor.year, interstate.year) = freeway.year FULL OUTER JOIN nearest_major_collector AS major_collector ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) = major_collector.pin10 + AND COALESCE(minor.year, interstate.year, freeway.year) + = major_collector.year +WHERE COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) + >= (SELECT MIN(year) FROM distinct_years_rhs) From 205e5c541b77338bb6828562044d4e59be363604 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Wed, 16 Oct 2024 16:02:05 +0000 Subject: [PATCH 35/61] rename data_year --- .../proximity/proximity.dist_pin_to_traffic_master.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 1cda71b2c..cc9db6b21 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -52,7 +52,7 @@ nearest_minor AS ( xy.year, ARBITRARY(xy.road_name) AS nearest_minor_road_name, ARBITRARY(xy.dist_ft) AS nearest_minor_road_dist_ft, - ARBITRARY(xy.year) AS data_year, + ARBITRARY(xy.year) AS nearest_minor_road_data_year, ARBITRARY(xy.surface_width) AS nearest_minor_road_surface_width FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_minor') }} ) AS xy @@ -68,7 +68,7 @@ nearest_interstate AS ( xy.year, ARBITRARY(xy.road_name) AS nearest_interstate_road_name, ARBITRARY(xy.dist_ft) AS nearest_interstate_road_dist_ft, - ARBITRARY(xy.year) AS data_year, + ARBITRARY(xy.year) AS nearest_interstate_road_data_year, ARBITRARY(xy.surface_width) AS nearest_interstate_road_surface_width FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_interstate') }} ) AS xy @@ -84,7 +84,7 @@ nearest_freeway AS ( xy.year, ARBITRARY(xy.road_name) AS nearest_freeway_road_name, ARBITRARY(xy.dist_ft) AS nearest_freeway_road_dist_ft, - ARBITRARY(xy.year) AS data_year, + ARBITRARY(xy.year) AS nearest_freeway_road_data_year, ARBITRARY(xy.surface_width) AS nearest_freeway_road_surface_width FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_freeway') }} ) AS xy @@ -100,7 +100,7 @@ nearest_major_collector AS ( xy.year, ARBITRARY(xy.road_name) AS nearest_major_collector_road_name, ARBITRARY(xy.dist_ft) AS nearest_major_collector_road_dist_ft, - ARBITRARY(xy.year) AS data_year, + ARBITRARY(xy.year) AS nearest_major_collector_road_data_year, ARBITRARY(xy.surface_width) AS nearest_major_collector_road_surface_width FROM distinct_pins AS pcl From 33e1993cd69062734c41801625768e9c57171515 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Wed, 16 Oct 2024 16:26:19 +0000 Subject: [PATCH 36/61] Try to add minor collector --- .../proximity.dist_pin_to_traffic_master.sql | 45 ++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index cc9db6b21..b20715a11 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -93,6 +93,23 @@ nearest_freeway AS ( GROUP BY pcl.pin10, xy.year ), +nearest_minor_collector AS ( + SELECT + pcl.pin10, + xy.year, + ARBITRARY(xy.road_name) AS nearest_minor_collector_road_name, + ARBITRARY(xy.dist_ft) AS nearest_minor_collector_road_dist_ft, + ARBITRARY(xy.year) AS nearest_minor_collector_road_data_year, + ARBITRARY(xy.surface_width) + AS nearest_minor_collector_road_surface_width + FROM distinct_pins AS pcl + INNER JOIN + ( {{ dist_to_nearest_geometry('traffic_minor_collector') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + GROUP BY pcl.pin10, xy.year +), + -- Calculate nearest Major Collector road per pin nearest_major_collector AS ( SELECT @@ -114,7 +131,8 @@ nearest_major_collector AS ( -- Join all nearest roads by pin10 and year SELECT COALESCE( - minor.pin10, interstate.pin10, freeway.pin10, major_collector.pin10 + minor.pin10, interstate.pin10, freeway.pin10, major_collector.pin10, + minor_collector.pin10 ) AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, @@ -132,8 +150,14 @@ SELECT major_collector.nearest_major_collector_road_dist_ft, major_collector.nearest_major_collector_road_data_year, major_collector.nearest_major_collector_road_surface_width, - COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) - AS year + minor_collector.nearest_minor_collector_road_name, + minor_collector.nearest_minor_collector_road_dist_ft, + minor_collector.nearest_minor_collector_road_data_year, + minor_collector.nearest_minor_collector_road_surface_width, + COALESCE( + minor.year, interstate.year, freeway.year, major_collector.year, + minor_collector.year + ) AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year @@ -145,5 +169,16 @@ FULL OUTER JOIN nearest_major_collector AS major_collector = major_collector.pin10 AND COALESCE(minor.year, interstate.year, freeway.year) = major_collector.year -WHERE COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) - >= (SELECT MIN(year) FROM distinct_years_rhs) +FULL OUTER JOIN nearest_minor_collector AS minor_collector + ON COALESCE( + minor.pin10, interstate.pin10, freeway.pin10, major_collector.pin10 + ) + = minor_collector.pin10 + AND COALESCE( + minor.year, interstate.year, freeway.year, major_collector.year + ) + = minor_collector.year +WHERE COALESCE( + minor.year, interstate.year, freeway.year, major_collector.year, + minor_collector.year + ) >= (SELECT MIN(year) FROM distinct_years_rhs) From ce34f6f0d3d3c406b0395664888d79f3a23acd7b Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Wed, 16 Oct 2024 16:32:02 +0000 Subject: [PATCH 37/61] Add CTE --- .../proximity/proximity.dist_pin_to_traffic_master.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index b20715a11..183ffb06d 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -13,6 +13,12 @@ WITH traffic_minor AS ( -- noqa: ST03 WHERE road_type = 'Minor Arterial' ), +traffic_minor_collector AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Minor Collector' +), + traffic_interstate AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} From 6469e296c36177060f436975cd0cb0d8b29895bf Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 24 Oct 2024 16:34:24 +0000 Subject: [PATCH 38/61] Add lands and surface type --- .../proximity.dist_pin_to_traffic_master.sql | 77 +++++++------------ 1 file changed, 26 insertions(+), 51 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 183ffb06d..9ed1d4d26 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -13,12 +13,6 @@ WITH traffic_minor AS ( -- noqa: ST03 WHERE road_type = 'Minor Arterial' ), -traffic_minor_collector AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Minor Collector' -), - traffic_interstate AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} @@ -45,7 +39,7 @@ distinct_pins AS ( FROM {{ source('spatial', 'parcel') }} ), -distinct_years_rhs AS ( +distinct_years AS ( SELECT DISTINCT year FROM {{ source('spatial', 'traffic') }} WHERE road_type IS NOT NULL @@ -59,7 +53,9 @@ nearest_minor AS ( ARBITRARY(xy.road_name) AS nearest_minor_road_name, ARBITRARY(xy.dist_ft) AS nearest_minor_road_dist_ft, ARBITRARY(xy.year) AS nearest_minor_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_minor_road_surface_width + ARBITRARY(xy.surface_width) AS nearest_minor_road_surface_width, + ARBITRARY(xy.surface_type) AS nearest_minor_surface_type, + ARBITRARY(xy.lanes) AS nearest_minor_lanes FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_minor') }} ) AS xy ON pcl.x_3435 = xy.x_3435 @@ -75,7 +71,9 @@ nearest_interstate AS ( ARBITRARY(xy.road_name) AS nearest_interstate_road_name, ARBITRARY(xy.dist_ft) AS nearest_interstate_road_dist_ft, ARBITRARY(xy.year) AS nearest_interstate_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_interstate_road_surface_width + ARBITRARY(xy.surface_width) AS nearest_interstate_road_surface_width, + ARBITRARY(xy.surface_type) AS nearest_interstate_surface_type, + ARBITRARY(xy.lanes) AS nearest_interstate_lanes FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_interstate') }} ) AS xy ON pcl.x_3435 = xy.x_3435 @@ -91,7 +89,9 @@ nearest_freeway AS ( ARBITRARY(xy.road_name) AS nearest_freeway_road_name, ARBITRARY(xy.dist_ft) AS nearest_freeway_road_dist_ft, ARBITRARY(xy.year) AS nearest_freeway_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_freeway_road_surface_width + ARBITRARY(xy.surface_width) AS nearest_freeway_road_surface_width, + ARBITRARY(xy.surface_type) AS nearest_freeway_surface_type, + ARBITRARY(xy.lanes) AS nearest_freeway_lanes FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_freeway') }} ) AS xy ON pcl.x_3435 = xy.x_3435 @@ -99,23 +99,6 @@ nearest_freeway AS ( GROUP BY pcl.pin10, xy.year ), -nearest_minor_collector AS ( - SELECT - pcl.pin10, - xy.year, - ARBITRARY(xy.road_name) AS nearest_minor_collector_road_name, - ARBITRARY(xy.dist_ft) AS nearest_minor_collector_road_dist_ft, - ARBITRARY(xy.year) AS nearest_minor_collector_road_data_year, - ARBITRARY(xy.surface_width) - AS nearest_minor_collector_road_surface_width - FROM distinct_pins AS pcl - INNER JOIN - ( {{ dist_to_nearest_geometry('traffic_minor_collector') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - GROUP BY pcl.pin10, xy.year -), - -- Calculate nearest Major Collector road per pin nearest_major_collector AS ( SELECT @@ -125,7 +108,9 @@ nearest_major_collector AS ( ARBITRARY(xy.dist_ft) AS nearest_major_collector_road_dist_ft, ARBITRARY(xy.year) AS nearest_major_collector_road_data_year, ARBITRARY(xy.surface_width) - AS nearest_major_collector_road_surface_width + AS nearest_major_collector_road_surface_width, + ARBITRARY(xy.surface_type) AS nearest_major_collector_surface_type, + ARBITRARY(xy.lanes) AS nearest_major_collector_lanes FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_major_collector') }} ) AS xy @@ -137,33 +122,34 @@ nearest_major_collector AS ( -- Join all nearest roads by pin10 and year SELECT COALESCE( - minor.pin10, interstate.pin10, freeway.pin10, major_collector.pin10, - minor_collector.pin10 + minor.pin10, interstate.pin10, freeway.pin10, major_collector.pin10 ) AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, minor.nearest_minor_road_data_year, minor.nearest_minor_road_surface_width, + minor.nearest_minor_surface_type, + minor.nearest_minor_lanes, interstate.nearest_interstate_road_name, interstate.nearest_interstate_road_dist_ft, interstate.nearest_interstate_road_data_year, interstate.nearest_interstate_road_surface_width, + interstate.nearest_interstate_surface_type, + interstate.nearest_interstate_lanes, freeway.nearest_freeway_road_name, freeway.nearest_freeway_road_dist_ft, freeway.nearest_freeway_road_data_year, freeway.nearest_freeway_road_surface_width, + freeway.nearest_freeway_surface_type, + freeway.nearest_freeway_lanes, major_collector.nearest_major_collector_road_name, major_collector.nearest_major_collector_road_dist_ft, major_collector.nearest_major_collector_road_data_year, major_collector.nearest_major_collector_road_surface_width, - minor_collector.nearest_minor_collector_road_name, - minor_collector.nearest_minor_collector_road_dist_ft, - minor_collector.nearest_minor_collector_road_data_year, - minor_collector.nearest_minor_collector_road_surface_width, - COALESCE( - minor.year, interstate.year, freeway.year, major_collector.year, - minor_collector.year - ) AS year + major_collector.nearest_major_collector_surface_type, + major_collector.nearest_major_collector_lanes, + COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) + AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year @@ -175,16 +161,5 @@ FULL OUTER JOIN nearest_major_collector AS major_collector = major_collector.pin10 AND COALESCE(minor.year, interstate.year, freeway.year) = major_collector.year -FULL OUTER JOIN nearest_minor_collector AS minor_collector - ON COALESCE( - minor.pin10, interstate.pin10, freeway.pin10, major_collector.pin10 - ) - = minor_collector.pin10 - AND COALESCE( - minor.year, interstate.year, freeway.year, major_collector.year - ) - = minor_collector.year -WHERE COALESCE( - minor.year, interstate.year, freeway.year, major_collector.year, - minor_collector.year - ) >= (SELECT MIN(year) FROM distinct_years_rhs) +WHERE COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) + >= (SELECT MIN(year) FROM distinct_years) From 0e8dc1f028864585643fdcc565b6111e7cbcc899 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 24 Oct 2024 16:40:50 +0000 Subject: [PATCH 39/61] Switch to speed limit --- .../proximity.dist_pin_to_traffic_master.sql | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 9ed1d4d26..24528bdf6 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -53,7 +53,7 @@ nearest_minor AS ( ARBITRARY(xy.road_name) AS nearest_minor_road_name, ARBITRARY(xy.dist_ft) AS nearest_minor_road_dist_ft, ARBITRARY(xy.year) AS nearest_minor_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_minor_road_surface_width, + ARBITRARY(xy.speed_limit) AS nearest_minor_road_speed_limit, ARBITRARY(xy.surface_type) AS nearest_minor_surface_type, ARBITRARY(xy.lanes) AS nearest_minor_lanes FROM distinct_pins AS pcl @@ -71,7 +71,7 @@ nearest_interstate AS ( ARBITRARY(xy.road_name) AS nearest_interstate_road_name, ARBITRARY(xy.dist_ft) AS nearest_interstate_road_dist_ft, ARBITRARY(xy.year) AS nearest_interstate_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_interstate_road_surface_width, + ARBITRARY(xy.speed_limit) AS nearest_interstate_road_speed_limit, ARBITRARY(xy.surface_type) AS nearest_interstate_surface_type, ARBITRARY(xy.lanes) AS nearest_interstate_lanes FROM distinct_pins AS pcl @@ -89,7 +89,7 @@ nearest_freeway AS ( ARBITRARY(xy.road_name) AS nearest_freeway_road_name, ARBITRARY(xy.dist_ft) AS nearest_freeway_road_dist_ft, ARBITRARY(xy.year) AS nearest_freeway_road_data_year, - ARBITRARY(xy.surface_width) AS nearest_freeway_road_surface_width, + ARBITRARY(xy.speed_limit) AS nearest_freeway_road_speed_limit, ARBITRARY(xy.surface_type) AS nearest_freeway_surface_type, ARBITRARY(xy.lanes) AS nearest_freeway_lanes FROM distinct_pins AS pcl @@ -107,8 +107,8 @@ nearest_major_collector AS ( ARBITRARY(xy.road_name) AS nearest_major_collector_road_name, ARBITRARY(xy.dist_ft) AS nearest_major_collector_road_dist_ft, ARBITRARY(xy.year) AS nearest_major_collector_road_data_year, - ARBITRARY(xy.surface_width) - AS nearest_major_collector_road_surface_width, + ARBITRARY(xy.speed_limit) + AS nearest_major_collector_road_speed_limit, ARBITRARY(xy.surface_type) AS nearest_major_collector_surface_type, ARBITRARY(xy.lanes) AS nearest_major_collector_lanes FROM distinct_pins AS pcl @@ -127,25 +127,25 @@ SELECT minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, minor.nearest_minor_road_data_year, - minor.nearest_minor_road_surface_width, + minor.nearest_minor_road_speed_limit, minor.nearest_minor_surface_type, minor.nearest_minor_lanes, interstate.nearest_interstate_road_name, interstate.nearest_interstate_road_dist_ft, interstate.nearest_interstate_road_data_year, - interstate.nearest_interstate_road_surface_width, + interstate.nearest_interstate_road_speed_limit, interstate.nearest_interstate_surface_type, interstate.nearest_interstate_lanes, freeway.nearest_freeway_road_name, freeway.nearest_freeway_road_dist_ft, freeway.nearest_freeway_road_data_year, - freeway.nearest_freeway_road_surface_width, + freeway.nearest_freeway_road_speed_limit, freeway.nearest_freeway_surface_type, freeway.nearest_freeway_lanes, major_collector.nearest_major_collector_road_name, major_collector.nearest_major_collector_road_dist_ft, major_collector.nearest_major_collector_road_data_year, - major_collector.nearest_major_collector_road_surface_width, + major_collector.nearest_major_collector_road_speed_limit, major_collector.nearest_major_collector_surface_type, major_collector.nearest_major_collector_lanes, COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) From b4351bc51ab12a29f60fe161f265a2c227481e4a Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 24 Oct 2024 18:02:28 +0000 Subject: [PATCH 40/61] Lowercase and --- dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 24528bdf6..3da54e238 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -22,7 +22,7 @@ traffic_interstate AS ( -- noqa: ST03 traffic_freeway AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Freeway And Expressway' + WHERE road_type = 'Freeway and Expressway' ), traffic_major_collector AS ( -- noqa: ST03 From 6bc3d826d1dfb21f685d4c205fca5484e67da54a Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 24 Oct 2024 18:11:51 +0000 Subject: [PATCH 41/61] Add dailiy_traffic --- .../proximity/proximity.dist_pin_to_traffic_master.sql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 3da54e238..39b1acb30 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -53,6 +53,7 @@ nearest_minor AS ( ARBITRARY(xy.road_name) AS nearest_minor_road_name, ARBITRARY(xy.dist_ft) AS nearest_minor_road_dist_ft, ARBITRARY(xy.year) AS nearest_minor_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_minor_road_daily_traffic, ARBITRARY(xy.speed_limit) AS nearest_minor_road_speed_limit, ARBITRARY(xy.surface_type) AS nearest_minor_surface_type, ARBITRARY(xy.lanes) AS nearest_minor_lanes @@ -71,6 +72,7 @@ nearest_interstate AS ( ARBITRARY(xy.road_name) AS nearest_interstate_road_name, ARBITRARY(xy.dist_ft) AS nearest_interstate_road_dist_ft, ARBITRARY(xy.year) AS nearest_interstate_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_interstate_daily_traffic, ARBITRARY(xy.speed_limit) AS nearest_interstate_road_speed_limit, ARBITRARY(xy.surface_type) AS nearest_interstate_surface_type, ARBITRARY(xy.lanes) AS nearest_interstate_lanes @@ -89,6 +91,7 @@ nearest_freeway AS ( ARBITRARY(xy.road_name) AS nearest_freeway_road_name, ARBITRARY(xy.dist_ft) AS nearest_freeway_road_dist_ft, ARBITRARY(xy.year) AS nearest_freeway_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_freeway_daily_traffic, ARBITRARY(xy.speed_limit) AS nearest_freeway_road_speed_limit, ARBITRARY(xy.surface_type) AS nearest_freeway_surface_type, ARBITRARY(xy.lanes) AS nearest_freeway_lanes @@ -109,6 +112,7 @@ nearest_major_collector AS ( ARBITRARY(xy.year) AS nearest_major_collector_road_data_year, ARBITRARY(xy.speed_limit) AS nearest_major_collector_road_speed_limit, + ARBITRARY(xy.daily_traffic) AS nearest_major_collector_daily_traffic, ARBITRARY(xy.surface_type) AS nearest_major_collector_surface_type, ARBITRARY(xy.lanes) AS nearest_major_collector_lanes FROM distinct_pins AS pcl @@ -127,24 +131,28 @@ SELECT minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, minor.nearest_minor_road_data_year, + minor.nearest_minor_road_daily_traffic, minor.nearest_minor_road_speed_limit, minor.nearest_minor_surface_type, minor.nearest_minor_lanes, interstate.nearest_interstate_road_name, interstate.nearest_interstate_road_dist_ft, interstate.nearest_interstate_road_data_year, + interstate.nearest_interstate_daily_traffic, interstate.nearest_interstate_road_speed_limit, interstate.nearest_interstate_surface_type, interstate.nearest_interstate_lanes, freeway.nearest_freeway_road_name, freeway.nearest_freeway_road_dist_ft, freeway.nearest_freeway_road_data_year, + freeway.nearest_freeway_daily_traffic, freeway.nearest_freeway_road_speed_limit, freeway.nearest_freeway_surface_type, freeway.nearest_freeway_lanes, major_collector.nearest_major_collector_road_name, major_collector.nearest_major_collector_road_dist_ft, major_collector.nearest_major_collector_road_data_year, + major_collector.nearest_major_collector_daily_traffic, major_collector.nearest_major_collector_road_speed_limit, major_collector.nearest_major_collector_surface_type, major_collector.nearest_major_collector_lanes, From 373ec21dbf9b14bf9603ea0d8fd1f9c88e72a63d Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 24 Oct 2024 19:40:26 +0000 Subject: [PATCH 42/61] Test local in wrong query --- dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 39b1acb30..3e7c1fad6 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -28,7 +28,7 @@ traffic_freeway AS ( -- noqa: ST03 traffic_major_collector AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Major Collector' + WHERE road_type = 'Local Road or Street' ), distinct_pins AS ( From 3ba546f9bfaf03d9fa20d4d6c9c90ba48305b152 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 24 Oct 2024 19:46:26 +0000 Subject: [PATCH 43/61] Try different join syntax --- .../proximity.dist_pin_to_traffic_master.sql | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 3e7c1fad6..f68c99e55 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -158,16 +158,17 @@ SELECT major_collector.nearest_major_collector_lanes, COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) AS year -FROM nearest_minor AS minor +FROM distinct_pins AS distinct_pins +FULL OUTER JOIN nearest_minor AS minor + ON distinct_pins.pin10 = minor.pin10 + AND distinct_pins.year = minor.year FULL OUTER JOIN nearest_interstate AS interstate - ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year + ON distinct_pins.pin10 = interstate.pin10 + AND distinct_pins.year = interstate.year FULL OUTER JOIN nearest_freeway AS freeway - ON COALESCE(minor.pin10, interstate.pin10) = freeway.pin10 - AND COALESCE(minor.year, interstate.year) = freeway.year + ON distinct_pins.pin10 = freeway.pin10 + AND distinct_pins.year = freeway.year FULL OUTER JOIN nearest_major_collector AS major_collector - ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) - = major_collector.pin10 - AND COALESCE(minor.year, interstate.year, freeway.year) - = major_collector.year -WHERE COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) - >= (SELECT MIN(year) FROM distinct_years) + ON distinct_pins.pin10 = major_collector.pin10 + AND distinct_pins.year = major_collector.year +WHERE minor.year >= (SELECT MIN(year) FROM distinct_years) From 6d15e6620839ed632679b7f9d8513dc08ce0192a Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 24 Oct 2024 20:19:42 +0000 Subject: [PATCH 44/61] Add local --- .../proximity.dist_pin_to_traffic_master.sql | 69 +++++++++++++++---- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index f68c99e55..d4968829f 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -25,12 +25,18 @@ traffic_freeway AS ( -- noqa: ST03 WHERE road_type = 'Freeway and Expressway' ), -traffic_major_collector AS ( -- noqa: ST03 +traffic_local AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} WHERE road_type = 'Local Road or Street' ), +traffic_major_collector AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Major Collector' +), + distinct_pins AS ( SELECT DISTINCT x_3435, @@ -102,6 +108,24 @@ nearest_freeway AS ( GROUP BY pcl.pin10, xy.year ), +nearest_local AS ( + SELECT + pcl.pin10, + xy.year, + ARBITRARY(xy.road_name) AS nearest_local_road_name, + ARBITRARY(xy.dist_ft) AS nearest_local_road_dist_ft, + ARBITRARY(xy.year) AS nearest_local_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_local_daily_traffic, + ARBITRARY(xy.speed_limit) AS nearest_local_road_speed_limit, + ARBITRARY(xy.surface_type) AS nearest_local_surface_type, + ARBITRARY(xy.lanes) AS nearest_local_lanes + FROM distinct_pins AS pcl + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_local') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + GROUP BY pcl.pin10, xy.year +), + -- Calculate nearest Major Collector road per pin nearest_major_collector AS ( SELECT @@ -126,7 +150,8 @@ nearest_major_collector AS ( -- Join all nearest roads by pin10 and year SELECT COALESCE( - minor.pin10, interstate.pin10, freeway.pin10, major_collector.pin10 + minor.pin10, interstate.pin10, freeway.pin10, + local_road.pin10, major_collector.pin10 ) AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, @@ -149,6 +174,13 @@ SELECT freeway.nearest_freeway_road_speed_limit, freeway.nearest_freeway_surface_type, freeway.nearest_freeway_lanes, + local_road.nearest_local_road_name, + local_road.nearest_local_road_dist_ft, + local_road.nearest_local_road_data_year, + local_road.nearest_local_daily_traffic, + local_road.nearest_local_road_speed_limit, + local_road.nearest_local_surface_type, + local_road.nearest_local_lanes, major_collector.nearest_major_collector_road_name, major_collector.nearest_major_collector_road_dist_ft, major_collector.nearest_major_collector_road_data_year, @@ -156,19 +188,28 @@ SELECT major_collector.nearest_major_collector_road_speed_limit, major_collector.nearest_major_collector_surface_type, major_collector.nearest_major_collector_lanes, - COALESCE(minor.year, interstate.year, freeway.year, major_collector.year) + COALESCE( + minor.year, interstate.year, freeway.year, + local_road.year, major_collector.year) AS year -FROM distinct_pins AS distinct_pins -FULL OUTER JOIN nearest_minor AS minor - ON distinct_pins.pin10 = minor.pin10 - AND distinct_pins.year = minor.year +FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate - ON distinct_pins.pin10 = interstate.pin10 - AND distinct_pins.year = interstate.year + ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year FULL OUTER JOIN nearest_freeway AS freeway - ON distinct_pins.pin10 = freeway.pin10 - AND distinct_pins.year = freeway.year + ON COALESCE(minor.pin10, interstate.pin10) = freeway.pin10 + AND COALESCE(minor.year, interstate.year) = freeway.year +FULL OUTER JOIN nearest_local AS local_road + ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) + = local_road.pin10 + AND COALESCE(minor.year, interstate.year, freeway.year) + = local_road.year FULL OUTER JOIN nearest_major_collector AS major_collector - ON distinct_pins.pin10 = major_collector.pin10 - AND distinct_pins.year = major_collector.year -WHERE minor.year >= (SELECT MIN(year) FROM distinct_years) + ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10, local_road.pin10) + = major_collector.pin10 + AND COALESCE(minor.year, interstate.year, freeway.year, local_road.year) + = major_collector.year +WHERE COALESCE( + minor.year, interstate.year, freeway.year, + local_road.year, major_collector.year + ) + >= (SELECT MIN(year) FROM distinct_years) From fdae40188fcfb7c186cd3505facfbe1436880eeb Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 24 Oct 2024 20:24:11 +0000 Subject: [PATCH 45/61] Try to add other --- .../proximity.dist_pin_to_traffic_master.sql | 54 ++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index d4968829f..4a117a63b 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -37,6 +37,12 @@ traffic_major_collector AS ( -- noqa: ST03 WHERE road_type = 'Major Collector' ), +traffic_other AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Other Principal Arterial' +), + distinct_pins AS ( SELECT DISTINCT x_3435, @@ -145,6 +151,25 @@ nearest_major_collector AS ( ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 GROUP BY pcl.pin10, xy.year +), + +-- Calculate nearest Other Principal Arterial road per pin +nearest_other AS ( + SELECT + pcl.pin10, + xy.year, + ARBITRARY(xy.road_name) AS nearest_other_road_name, + ARBITRARY(xy.dist_ft) AS nearest_other_road_dist_ft, + ARBITRARY(xy.year) AS nearest_other_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_other_daily_traffic, + ARBITRARY(xy.speed_limit) AS nearest_other_road_speed_limit, + ARBITRARY(xy.surface_type) AS nearest_other_surface_type, + ARBITRARY(xy.lanes) AS nearest_other_lanes + FROM distinct_pins AS pcl + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_other') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + GROUP BY pcl.pin10, xy.year ) -- Join all nearest roads by pin10 and year @@ -188,9 +213,16 @@ SELECT major_collector.nearest_major_collector_road_speed_limit, major_collector.nearest_major_collector_surface_type, major_collector.nearest_major_collector_lanes, + other.nearest_other_road_name, + other.nearest_other_road_dist_ft, + other.nearest_other_road_data_year, + other.nearest_other_daily_traffic, + other.nearest_other_road_speed_limit, + other.nearest_other_surface_type, + other.nearest_other_lanes, COALESCE( minor.year, interstate.year, freeway.year, - local_road.year, major_collector.year) + local_road.year, major_collector.year, other.year) AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate @@ -208,8 +240,26 @@ FULL OUTER JOIN nearest_major_collector AS major_collector = major_collector.pin10 AND COALESCE(minor.year, interstate.year, freeway.year, local_road.year) = major_collector.year +FULL OUTER JOIN nearest_other AS other + ON COALESCE( + minor.pin10, + interstate.pin10, + freeway.pin10, + local_road.pin10, + major_collector.pin10 + ) + = other.pin10 + AND COALESCE( + minor.year, + interstate.year, + freeway.year, + local_road.year, + major_collector.year + ) + = other.year WHERE COALESCE( minor.year, interstate.year, freeway.year, - local_road.year, major_collector.year + local_road.year, major_collector.year, + other.year ) >= (SELECT MIN(year) FROM distinct_years) From fe8a1cce8941a9fe09881c7511bd64893f6d7bdd Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 24 Oct 2024 21:31:40 +0000 Subject: [PATCH 46/61] Add minor collector --- .../proximity.dist_pin_to_traffic_master.sql | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 4a117a63b..55e2fe9d2 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -13,6 +13,12 @@ WITH traffic_minor AS ( -- noqa: ST03 WHERE road_type = 'Minor Arterial' ), +traffic_minor_collector AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Minor Collector' +), + traffic_interstate AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} @@ -76,6 +82,26 @@ nearest_minor AS ( GROUP BY pcl.pin10, xy.year ), +-- Calculate nearest Minor Collector road per pin +nearest_minor_collector AS ( + SELECT + pcl.pin10, + xy.year, + ARBITRARY(xy.road_name) AS nearest_minor_collector_road_name, + ARBITRARY(xy.dist_ft) AS nearest_minor_collector_road_dist_ft, + ARBITRARY(xy.year) AS nearest_minor_collector_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_minor_collector_daily_traffic, + ARBITRARY(xy.speed_limit) AS nearest_minor_collector_road_speed_limit, + ARBITRARY(xy.surface_type) AS nearest_minor_collector_surface_type, + ARBITRARY(xy.lanes) AS nearest_minor_collector_lanes + FROM distinct_pins AS pcl + INNER JOIN + ( {{ dist_to_nearest_geometry('traffic_minor_collector') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + GROUP BY pcl.pin10, xy.year +), + -- Calculate nearest Interstate road per pin nearest_interstate AS ( SELECT @@ -175,7 +201,7 @@ nearest_other AS ( -- Join all nearest roads by pin10 and year SELECT COALESCE( - minor.pin10, interstate.pin10, freeway.pin10, + minor.pin10, minor_collector.pin10, interstate.pin10, freeway.pin10, local_road.pin10, major_collector.pin10 ) AS pin10, minor.nearest_minor_road_name, @@ -185,6 +211,13 @@ SELECT minor.nearest_minor_road_speed_limit, minor.nearest_minor_surface_type, minor.nearest_minor_lanes, + minor_collector.nearest_minor_collector_road_name, + minor_collector.nearest_minor_collector_road_dist_ft, + minor_collector.nearest_minor_collector_road_data_year, + minor_collector.nearest_minor_collector_daily_traffic, + minor_collector.nearest_minor_collector_road_speed_limit, + minor_collector.nearest_minor_collector_surface_type, + minor_collector.nearest_minor_collector_lanes, interstate.nearest_interstate_road_name, interstate.nearest_interstate_road_dist_ft, interstate.nearest_interstate_road_data_year, @@ -221,7 +254,7 @@ SELECT other.nearest_other_surface_type, other.nearest_other_lanes, COALESCE( - minor.year, interstate.year, freeway.year, + minor.year, minor_collector.year interstate.year, freeway.year, local_road.year, major_collector.year, other.year) AS year FROM nearest_minor AS minor @@ -257,8 +290,19 @@ FULL OUTER JOIN nearest_other AS other major_collector.year ) = other.year -WHERE COALESCE( +FULL OUTER JOIN nearest_minor_collector AS minor_collector + ON COALESCE( + minor.pin10, interstate.pin10, freeway.pin10, + local_road.pin10, major_collector.pin10, other.pin10 + ) + = minor_collector.pin10 + AND COALESCE( minor.year, interstate.year, freeway.year, + local_road.year, major_collector.year, other.year + ) + = minor_collector.year +WHERE COALESCE( + minor.year, minor_collector.year, interstate.year, freeway.year, local_road.year, major_collector.year, other.year ) From 601292ab79c1a4033067df94cfb1ac1fb91a3124 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 24 Oct 2024 21:40:33 +0000 Subject: [PATCH 47/61] Add year to join for minor_collector --- .../proximity/proximity.dist_pin_to_traffic_master.sql | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 55e2fe9d2..9fbfbbdfd 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -294,13 +294,12 @@ FULL OUTER JOIN nearest_minor_collector AS minor_collector ON COALESCE( minor.pin10, interstate.pin10, freeway.pin10, local_road.pin10, major_collector.pin10, other.pin10 - ) - = minor_collector.pin10 + ) = minor_collector.pin10 AND COALESCE( minor.year, interstate.year, freeway.year, local_road.year, major_collector.year, other.year - ) - = minor_collector.year + ) = minor_collector.year + AND minor_collector.year >= 2014 WHERE COALESCE( minor.year, minor_collector.year, interstate.year, freeway.year, local_road.year, major_collector.year, From 35bfcf9c664ae5bd254710ed3e27236db26a5ca2 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Thu, 24 Oct 2024 21:56:11 +0000 Subject: [PATCH 48/61] Add year to join for minor_collector --- dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 9fbfbbdfd..dbf66a371 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -99,6 +99,7 @@ nearest_minor_collector AS ( ( {{ dist_to_nearest_geometry('traffic_minor_collector') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 + AND pcl.year >= 2014 GROUP BY pcl.pin10, xy.year ), From ce877905bd9855c1c6c9cfe079995ba564b59b21 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Fri, 25 Oct 2024 14:06:51 +0000 Subject: [PATCH 49/61] Test with only valid daily traffic for local --- .../proximity.dist_pin_to_traffic_master.sql | 102 ++++++------------ 1 file changed, 30 insertions(+), 72 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index dbf66a371..58ae4908b 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -13,12 +13,6 @@ WITH traffic_minor AS ( -- noqa: ST03 WHERE road_type = 'Minor Arterial' ), -traffic_minor_collector AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Minor Collector' -), - traffic_interstate AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} @@ -35,12 +29,14 @@ traffic_local AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} WHERE road_type = 'Local Road or Street' + AND daily_traffic IS NOT NULL ), -traffic_major_collector AS ( -- noqa: ST03 +traffic_collector AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} WHERE road_type = 'Major Collector' + OR road_type = 'Minor Collector' ), traffic_other AS ( -- noqa: ST03 @@ -82,27 +78,6 @@ nearest_minor AS ( GROUP BY pcl.pin10, xy.year ), --- Calculate nearest Minor Collector road per pin -nearest_minor_collector AS ( - SELECT - pcl.pin10, - xy.year, - ARBITRARY(xy.road_name) AS nearest_minor_collector_road_name, - ARBITRARY(xy.dist_ft) AS nearest_minor_collector_road_dist_ft, - ARBITRARY(xy.year) AS nearest_minor_collector_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_minor_collector_daily_traffic, - ARBITRARY(xy.speed_limit) AS nearest_minor_collector_road_speed_limit, - ARBITRARY(xy.surface_type) AS nearest_minor_collector_surface_type, - ARBITRARY(xy.lanes) AS nearest_minor_collector_lanes - FROM distinct_pins AS pcl - INNER JOIN - ( {{ dist_to_nearest_geometry('traffic_minor_collector') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - AND pcl.year >= 2014 - GROUP BY pcl.pin10, xy.year -), - -- Calculate nearest Interstate road per pin nearest_interstate AS ( SELECT @@ -160,21 +135,21 @@ nearest_local AS ( ), -- Calculate nearest Major Collector road per pin -nearest_major_collector AS ( +nearest_collector AS ( SELECT pcl.pin10, xy.year, - ARBITRARY(xy.road_name) AS nearest_major_collector_road_name, - ARBITRARY(xy.dist_ft) AS nearest_major_collector_road_dist_ft, - ARBITRARY(xy.year) AS nearest_major_collector_road_data_year, + ARBITRARY(xy.road_name) AS nearest_collector_road_name, + ARBITRARY(xy.dist_ft) AS nearest_collector_road_dist_ft, + ARBITRARY(xy.year) AS nearest_collector_road_data_year, ARBITRARY(xy.speed_limit) - AS nearest_major_collector_road_speed_limit, - ARBITRARY(xy.daily_traffic) AS nearest_major_collector_daily_traffic, - ARBITRARY(xy.surface_type) AS nearest_major_collector_surface_type, - ARBITRARY(xy.lanes) AS nearest_major_collector_lanes + AS nearest_collector_road_speed_limit, + ARBITRARY(xy.daily_traffic) AS nearest_collector_daily_traffic, + ARBITRARY(xy.surface_type) AS nearest_collector_surface_type, + ARBITRARY(xy.lanes) AS nearest_collector_lanes FROM distinct_pins AS pcl INNER JOIN - ( {{ dist_to_nearest_geometry('traffic_major_collector') }} ) AS xy + ( {{ dist_to_nearest_geometry('traffic_collector') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 GROUP BY pcl.pin10, xy.year @@ -202,8 +177,8 @@ nearest_other AS ( -- Join all nearest roads by pin10 and year SELECT COALESCE( - minor.pin10, minor_collector.pin10, interstate.pin10, freeway.pin10, - local_road.pin10, major_collector.pin10 + minor.pin10, interstate.pin10, freeway.pin10, + local_road.pin10, collector.pin10 ) AS pin10, minor.nearest_minor_road_name, minor.nearest_minor_road_dist_ft, @@ -212,13 +187,6 @@ SELECT minor.nearest_minor_road_speed_limit, minor.nearest_minor_surface_type, minor.nearest_minor_lanes, - minor_collector.nearest_minor_collector_road_name, - minor_collector.nearest_minor_collector_road_dist_ft, - minor_collector.nearest_minor_collector_road_data_year, - minor_collector.nearest_minor_collector_daily_traffic, - minor_collector.nearest_minor_collector_road_speed_limit, - minor_collector.nearest_minor_collector_surface_type, - minor_collector.nearest_minor_collector_lanes, interstate.nearest_interstate_road_name, interstate.nearest_interstate_road_dist_ft, interstate.nearest_interstate_road_data_year, @@ -240,13 +208,13 @@ SELECT local_road.nearest_local_road_speed_limit, local_road.nearest_local_surface_type, local_road.nearest_local_lanes, - major_collector.nearest_major_collector_road_name, - major_collector.nearest_major_collector_road_dist_ft, - major_collector.nearest_major_collector_road_data_year, - major_collector.nearest_major_collector_daily_traffic, - major_collector.nearest_major_collector_road_speed_limit, - major_collector.nearest_major_collector_surface_type, - major_collector.nearest_major_collector_lanes, + collector.nearest_collector_road_name, + collector.nearest_collector_road_dist_ft, + collector.nearest_collector_road_data_year, + collector.nearest_collector_daily_traffic, + collector.nearest_collector_road_speed_limit, + collector.nearest_collector_surface_type, + collector.nearest_collector_lanes, other.nearest_other_road_name, other.nearest_other_road_dist_ft, other.nearest_other_road_data_year, @@ -255,8 +223,8 @@ SELECT other.nearest_other_surface_type, other.nearest_other_lanes, COALESCE( - minor.year, minor_collector.year interstate.year, freeway.year, - local_road.year, major_collector.year, other.year) + minor.year, interstate.year, freeway.year, + local_road.year, collector.year, other.year) AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate @@ -269,18 +237,18 @@ FULL OUTER JOIN nearest_local AS local_road = local_road.pin10 AND COALESCE(minor.year, interstate.year, freeway.year) = local_road.year -FULL OUTER JOIN nearest_major_collector AS major_collector +FULL OUTER JOIN nearest_collector AS collector ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10, local_road.pin10) - = major_collector.pin10 + = collector.pin10 AND COALESCE(minor.year, interstate.year, freeway.year, local_road.year) - = major_collector.year + = collector.year FULL OUTER JOIN nearest_other AS other ON COALESCE( minor.pin10, interstate.pin10, freeway.pin10, local_road.pin10, - major_collector.pin10 + collector.pin10 ) = other.pin10 AND COALESCE( @@ -288,22 +256,12 @@ FULL OUTER JOIN nearest_other AS other interstate.year, freeway.year, local_road.year, - major_collector.year + collector.year ) = other.year -FULL OUTER JOIN nearest_minor_collector AS minor_collector - ON COALESCE( - minor.pin10, interstate.pin10, freeway.pin10, - local_road.pin10, major_collector.pin10, other.pin10 - ) = minor_collector.pin10 - AND COALESCE( - minor.year, interstate.year, freeway.year, - local_road.year, major_collector.year, other.year - ) = minor_collector.year - AND minor_collector.year >= 2014 WHERE COALESCE( - minor.year, minor_collector.year, interstate.year, freeway.year, - local_road.year, major_collector.year, + minor.year, interstate.year, freeway.year, + local_road.year, collector.year, other.year ) >= (SELECT MIN(year) FROM distinct_years) From 40ddfd7884b776d3b2a4497065724fcd594f6250 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Fri, 25 Oct 2024 15:00:41 +0000 Subject: [PATCH 50/61] Test macro upload --- dbt/macros/nearest_feature_aggregation.sql | 50 ++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 dbt/macros/nearest_feature_aggregation.sql diff --git a/dbt/macros/nearest_feature_aggregation.sql b/dbt/macros/nearest_feature_aggregation.sql new file mode 100644 index 000000000..c0d592976 --- /dev/null +++ b/dbt/macros/nearest_feature_aggregation.sql @@ -0,0 +1,50 @@ +{% macro aggregate_lowest_distance(table_name) %} + with + distances as ( + -- Unpivot all "dist_ft" columns for comparison + select + *, + unnest( + array[ + {% for column in dbt_utils.get_filtered_columns_in_relation( + table_name, "dist_ft" + ) %} + ({{ column }}, '{{ column }}') + {% endfor %} + ] + ) as (distance_value, distance_column) + from {{ table_name }} + ), + min_distances as ( + -- Identify the lowest distance and corresponding column for each row + select + *, + min(distance_value) over (partition by id) as min_distance, + array_agg(distance_column) filter ( + where distance_value = min(distance_value) over (partition by id) + ) as matching_columns + from distances + ), + aggregated as ( + -- Extract the prefix before the first underscore and rename it with + -- "aggregated_" + select + id, + min_distance, + array_agg( + distinct regexp_extract(column_name, '^(.*?)_') + ) as original_prefixes, + array_agg( + distinct 'aggregated_' || regexp_extract(column_name, '^(.*?)_') + ) as aggregated_prefixes + from + ( + select id, unnest(matching_columns) as column_name + from min_distances + ) prefix_agg + group by id, min_distance + ) + select * + from aggregated + ; +{% endmacro %} From a3131c6b0b22a2a12cb53e3dd7defec959f76cc1 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Fri, 25 Oct 2024 17:18:14 +0000 Subject: [PATCH 51/61] Test macro --- dbt/macros/nearest_feature_aggregation.sql | 48 ++++++++++--------- .../proximity.dist_pin_to_traffic_master.sql | 37 +++++--------- 2 files changed, 38 insertions(+), 47 deletions(-) diff --git a/dbt/macros/nearest_feature_aggregation.sql b/dbt/macros/nearest_feature_aggregation.sql index c0d592976..ee2271cf2 100644 --- a/dbt/macros/nearest_feature_aggregation.sql +++ b/dbt/macros/nearest_feature_aggregation.sql @@ -1,36 +1,41 @@ -{% macro aggregate_lowest_distance(table_name) %} +{% macro aggregate_smallest_feature(table_name, feature_suffix="dist_ft") %} with - distances as ( - -- Unpivot all "dist_ft" columns for comparison + columns as ( + -- Get column names that contain the suffix + select column_name + from information_schema.columns + where + table_name = lower('{{ table_name }}') + and column_name like '%' || '{{ feature_suffix }}' + ), + feature as ( + -- Unpivot all matching columns for comparison select - *, + t.*, unnest( array[ - {% for column in dbt_utils.get_filtered_columns_in_relation( - table_name, "dist_ft" - ) %} - ({{ column }}, '{{ column }}') + {% for column in columns %} + (t.{{ column.column_name }}, '{{ column.column_name }}') {% endfor %} ] - ) as (distance_value, distance_column) - from {{ table_name }} + ) as (feature_value, feature_column) + from {{ table_name }} t ), - min_distances as ( - -- Identify the lowest distance and corresponding column for each row + min_feature as ( + -- Identify the lowest feature and corresponding column for each row select *, - min(distance_value) over (partition by id) as min_distance, - array_agg(distance_column) filter ( - where distance_value = min(distance_value) over (partition by id) + min(feature_value) over (partition by id) as min_feature, + array_agg(feature_column) filter ( + where feature_value = min(feature_value) over (partition by id) ) as matching_columns - from distances + from feature ), aggregated as ( - -- Extract the prefix before the first underscore and rename it with - -- "aggregated_" + -- Extract prefix before first underscore and rename with "aggregated_" select id, - min_distance, + min_feature, array_agg( distinct regexp_extract(column_name, '^(.*?)_') ) as original_prefixes, @@ -39,10 +44,9 @@ ) as aggregated_prefixes from ( - select id, unnest(matching_columns) as column_name - from min_distances + select id, unnest(matching_columns) as column_name from min_feature ) prefix_agg - group by id, min_distance + group by id, min_feature ) select * from aggregated diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 58ae4908b..804cb5e99 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -174,7 +174,6 @@ nearest_other AS ( GROUP BY pcl.pin10, xy.year ) --- Join all nearest roads by pin10 and year SELECT COALESCE( minor.pin10, interstate.pin10, freeway.pin10, @@ -224,8 +223,8 @@ SELECT other.nearest_other_lanes, COALESCE( minor.year, interstate.year, freeway.year, - local_road.year, collector.year, other.year) - AS year + local_road.year, collector.year, other.year + ) AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year @@ -233,10 +232,8 @@ FULL OUTER JOIN nearest_freeway AS freeway ON COALESCE(minor.pin10, interstate.pin10) = freeway.pin10 AND COALESCE(minor.year, interstate.year) = freeway.year FULL OUTER JOIN nearest_local AS local_road - ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) - = local_road.pin10 - AND COALESCE(minor.year, interstate.year, freeway.year) - = local_road.year + ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) = local_road.pin10 + AND COALESCE(minor.year, interstate.year, freeway.year) = local_road.year FULL OUTER JOIN nearest_collector AS collector ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10, local_road.pin10) = collector.pin10 @@ -244,24 +241,14 @@ FULL OUTER JOIN nearest_collector AS collector = collector.year FULL OUTER JOIN nearest_other AS other ON COALESCE( - minor.pin10, - interstate.pin10, - freeway.pin10, - local_road.pin10, - collector.pin10 - ) - = other.pin10 + minor.pin10, interstate.pin10, freeway.pin10, + local_road.pin10, collector.pin10 + ) = other.pin10 AND COALESCE( - minor.year, - interstate.year, - freeway.year, - local_road.year, - collector.year - ) - = other.year + minor.year, interstate.year, freeway.year, + local_road.year, collector.year + ) = other.year WHERE COALESCE( minor.year, interstate.year, freeway.year, - local_road.year, collector.year, - other.year - ) - >= (SELECT MIN(year) FROM distinct_years) + local_road.year, collector.year, other.year + ) >= (SELECT MIN(year) FROM distinct_years) From 364fafb7853d4acb9e3f2b92628148a0eedf74ba Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Mon, 28 Oct 2024 15:13:13 +0000 Subject: [PATCH 52/61] Revert to traffic not null --- .../proximity/proximity.dist_pin_to_traffic_master.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 804cb5e99..29267f725 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -29,7 +29,6 @@ traffic_local AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} WHERE road_type = 'Local Road or Street' - AND daily_traffic IS NOT NULL ), traffic_collector AS ( -- noqa: ST03 @@ -128,7 +127,9 @@ nearest_local AS ( ARBITRARY(xy.surface_type) AS nearest_local_surface_type, ARBITRARY(xy.lanes) AS nearest_local_lanes FROM distinct_pins AS pcl - INNER JOIN ( {{ dist_to_nearest_geometry('traffic_local') }} ) AS xy + INNER JOIN ( + {{ nearest_pin_neighbors('traffic_local', 1, 100) }} + ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 GROUP BY pcl.pin10, xy.year From 28cbeb2fd22916440cbc5ee3f3077386c7d68c31 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Mon, 28 Oct 2024 15:20:15 +0000 Subject: [PATCH 53/61] Test nearest_local new format --- .../proximity.dist_pin_to_traffic_master.sql | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 29267f725..9325c2d1a 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -115,7 +115,7 @@ nearest_freeway AS ( GROUP BY pcl.pin10, xy.year ), -nearest_local AS ( +nearest_local_step_1 AS ( SELECT pcl.pin10, xy.year, @@ -135,6 +135,39 @@ nearest_local AS ( GROUP BY pcl.pin10, xy.year ), +missing_matches AS ( + SELECT + pcl.pin10, + pcl.year, + pcl.x_3435, + pcl.y_3435 + FROM {{ source('spatial', 'parcel') }} AS pcl + LEFT JOIN nearest_local_step_1 AS nl + ON pcl.pin10 = nl.pin10 + AND pcl.year = nl.nearest_local_road_data_year + WHERE nl.pin10 IS NULL +), + +nearest_local AS ( + SELECT + mm.pin10, + xy.year, + ARBITRARY(xy.road_name) AS nearest_local_road_name, + ARBITRARY(xy.dist_ft) AS nearest_local_road_dist_ft, + ARBITRARY(xy.year) AS nearest_local_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_local_daily_traffic, + ARBITRARY(xy.speed_limit) AS nearest_local_road_speed_limit, + ARBITRARY(xy.surface_type) AS nearest_local_surface_type, + ARBITRARY(xy.lanes) AS nearest_local_lanes + FROM missing_matches AS mm + INNER JOIN ( + {{ nearest_pin_neighbors('missing_matches', 1, 10000) }} + ) AS xy + ON mm.x_3435 = xy.x_3435 + AND mm.y_3435 = xy.y_3435 + GROUP BY mm.pin10, xy.year +), + -- Calculate nearest Major Collector road per pin nearest_collector AS ( SELECT From 387731c9a1e4a1491e1bc30e6b623054a86392ff Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Mon, 28 Oct 2024 15:42:31 +0000 Subject: [PATCH 54/61] Revert --- .../proximity.dist_pin_to_traffic_master.sql | 56 ++++--------------- 1 file changed, 12 insertions(+), 44 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 9325c2d1a..f888ca3a2 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -29,6 +29,7 @@ traffic_local AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} WHERE road_type = 'Local Road or Street' + AND daily_traffic IS NOT NULL ), traffic_collector AS ( -- noqa: ST03 @@ -115,59 +116,26 @@ nearest_freeway AS ( GROUP BY pcl.pin10, xy.year ), -nearest_local_step_1 AS ( +nearest_collector AS ( SELECT pcl.pin10, xy.year, - ARBITRARY(xy.road_name) AS nearest_local_road_name, - ARBITRARY(xy.dist_ft) AS nearest_local_road_dist_ft, - ARBITRARY(xy.year) AS nearest_local_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_local_daily_traffic, - ARBITRARY(xy.speed_limit) AS nearest_local_road_speed_limit, - ARBITRARY(xy.surface_type) AS nearest_local_surface_type, - ARBITRARY(xy.lanes) AS nearest_local_lanes + ARBITRARY(xy.road_name) AS nearest_collector_road_name, + ARBITRARY(xy.dist_ft) AS nearest_collector_road_dist_ft, + ARBITRARY(xy.year) AS nearest_collector_road_data_year, + ARBITRARY(xy.speed_limit) + AS nearest_collector_road_speed_limit, + ARBITRARY(xy.daily_traffic) AS nearest_collector_daily_traffic, + ARBITRARY(xy.surface_type) AS nearest_collector_surface_type, + ARBITRARY(xy.lanes) AS nearest_collector_lanes FROM distinct_pins AS pcl - INNER JOIN ( - {{ nearest_pin_neighbors('traffic_local', 1, 100) }} - ) AS xy + INNER JOIN + ( {{ dist_to_nearest_geometry('traffic_collector') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 GROUP BY pcl.pin10, xy.year ), -missing_matches AS ( - SELECT - pcl.pin10, - pcl.year, - pcl.x_3435, - pcl.y_3435 - FROM {{ source('spatial', 'parcel') }} AS pcl - LEFT JOIN nearest_local_step_1 AS nl - ON pcl.pin10 = nl.pin10 - AND pcl.year = nl.nearest_local_road_data_year - WHERE nl.pin10 IS NULL -), - -nearest_local AS ( - SELECT - mm.pin10, - xy.year, - ARBITRARY(xy.road_name) AS nearest_local_road_name, - ARBITRARY(xy.dist_ft) AS nearest_local_road_dist_ft, - ARBITRARY(xy.year) AS nearest_local_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_local_daily_traffic, - ARBITRARY(xy.speed_limit) AS nearest_local_road_speed_limit, - ARBITRARY(xy.surface_type) AS nearest_local_surface_type, - ARBITRARY(xy.lanes) AS nearest_local_lanes - FROM missing_matches AS mm - INNER JOIN ( - {{ nearest_pin_neighbors('missing_matches', 1, 10000) }} - ) AS xy - ON mm.x_3435 = xy.x_3435 - AND mm.y_3435 = xy.y_3435 - GROUP BY mm.pin10, xy.year -), - -- Calculate nearest Major Collector road per pin nearest_collector AS ( SELECT From e70ea40624ce617da7aa218c3b1ce603bfced9e9 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Mon, 28 Oct 2024 15:50:54 +0000 Subject: [PATCH 55/61] Revert --- dbt/macros/nearest_feature_aggregation.sql | 108 +++++++++--------- .../proximity.dist_pin_to_traffic_master.sql | 57 +++++---- 2 files changed, 90 insertions(+), 75 deletions(-) diff --git a/dbt/macros/nearest_feature_aggregation.sql b/dbt/macros/nearest_feature_aggregation.sql index ee2271cf2..4bf6c0959 100644 --- a/dbt/macros/nearest_feature_aggregation.sql +++ b/dbt/macros/nearest_feature_aggregation.sql @@ -1,54 +1,58 @@ {% macro aggregate_smallest_feature(table_name, feature_suffix="dist_ft") %} - with - columns as ( - -- Get column names that contain the suffix - select column_name - from information_schema.columns - where - table_name = lower('{{ table_name }}') - and column_name like '%' || '{{ feature_suffix }}' - ), - feature as ( - -- Unpivot all matching columns for comparison - select - t.*, - unnest( - array[ - {% for column in columns %} - (t.{{ column.column_name }}, '{{ column.column_name }}') - {% endfor %} - ] - ) as (feature_value, feature_column) - from {{ table_name }} t - ), - min_feature as ( - -- Identify the lowest feature and corresponding column for each row - select - *, - min(feature_value) over (partition by id) as min_feature, - array_agg(feature_column) filter ( - where feature_value = min(feature_value) over (partition by id) - ) as matching_columns - from feature - ), - aggregated as ( - -- Extract prefix before first underscore and rename with "aggregated_" - select - id, - min_feature, - array_agg( - distinct regexp_extract(column_name, '^(.*?)_') - ) as original_prefixes, - array_agg( - distinct 'aggregated_' || regexp_extract(column_name, '^(.*?)_') - ) as aggregated_prefixes - from - ( - select id, unnest(matching_columns) as column_name from min_feature - ) prefix_agg - group by id, min_feature - ) - select * - from aggregated - ; + {%- set macro_content -%} + {% raw %} + {% macro aggregate_smallest_feature(table_name, feature_suffix="dist_ft") %} + with + columns as ( + select column_name + from information_schema.columns + where + table_name = lower('{{ table_name }}') + and column_name like '%' || '{{ feature_suffix }}' + ), + feature as ( + select + t.*, + unnest( + array[ + {% for column in columns %} + (t.{{ column.column_name }}, '{{ column.column_name }}') + {% endfor %} + ] + ) as (feature_value, feature_column) + from {{ table_name }} t + ), + min_feature as ( + select + *, + min(feature_value) over (partition by id) as min_feature, + array_agg(feature_column) filter ( + where feature_value = min(feature_value) over (partition by id) + ) as matching_columns + from feature + ), + aggregated as ( + select + id, + min_feature, + array_agg( + distinct regexp_extract(column_name, '^(.*?)_') + ) as original_prefixes, + array_agg( + distinct 'aggregated_' || regexp_extract(column_name, '^(.*?)_') + ) as aggregated_prefixes + from + ( + select id, unnest(matching_columns) as column_name from min_feature + ) prefix_agg + group by id, min_feature + ) + select * + from aggregated + ; + {% endmacro %} + {% endraw %} + {%- endset -%} + + {{ macro_content }} {% endmacro %} diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index f888ca3a2..58ae4908b 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -116,21 +116,19 @@ nearest_freeway AS ( GROUP BY pcl.pin10, xy.year ), -nearest_collector AS ( +nearest_local AS ( SELECT pcl.pin10, xy.year, - ARBITRARY(xy.road_name) AS nearest_collector_road_name, - ARBITRARY(xy.dist_ft) AS nearest_collector_road_dist_ft, - ARBITRARY(xy.year) AS nearest_collector_road_data_year, - ARBITRARY(xy.speed_limit) - AS nearest_collector_road_speed_limit, - ARBITRARY(xy.daily_traffic) AS nearest_collector_daily_traffic, - ARBITRARY(xy.surface_type) AS nearest_collector_surface_type, - ARBITRARY(xy.lanes) AS nearest_collector_lanes + ARBITRARY(xy.road_name) AS nearest_local_road_name, + ARBITRARY(xy.dist_ft) AS nearest_local_road_dist_ft, + ARBITRARY(xy.year) AS nearest_local_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_local_daily_traffic, + ARBITRARY(xy.speed_limit) AS nearest_local_road_speed_limit, + ARBITRARY(xy.surface_type) AS nearest_local_surface_type, + ARBITRARY(xy.lanes) AS nearest_local_lanes FROM distinct_pins AS pcl - INNER JOIN - ( {{ dist_to_nearest_geometry('traffic_collector') }} ) AS xy + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_local') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 GROUP BY pcl.pin10, xy.year @@ -176,6 +174,7 @@ nearest_other AS ( GROUP BY pcl.pin10, xy.year ) +-- Join all nearest roads by pin10 and year SELECT COALESCE( minor.pin10, interstate.pin10, freeway.pin10, @@ -225,8 +224,8 @@ SELECT other.nearest_other_lanes, COALESCE( minor.year, interstate.year, freeway.year, - local_road.year, collector.year, other.year - ) AS year + local_road.year, collector.year, other.year) + AS year FROM nearest_minor AS minor FULL OUTER JOIN nearest_interstate AS interstate ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year @@ -234,8 +233,10 @@ FULL OUTER JOIN nearest_freeway AS freeway ON COALESCE(minor.pin10, interstate.pin10) = freeway.pin10 AND COALESCE(minor.year, interstate.year) = freeway.year FULL OUTER JOIN nearest_local AS local_road - ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) = local_road.pin10 - AND COALESCE(minor.year, interstate.year, freeway.year) = local_road.year + ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) + = local_road.pin10 + AND COALESCE(minor.year, interstate.year, freeway.year) + = local_road.year FULL OUTER JOIN nearest_collector AS collector ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10, local_road.pin10) = collector.pin10 @@ -243,14 +244,24 @@ FULL OUTER JOIN nearest_collector AS collector = collector.year FULL OUTER JOIN nearest_other AS other ON COALESCE( - minor.pin10, interstate.pin10, freeway.pin10, - local_road.pin10, collector.pin10 - ) = other.pin10 + minor.pin10, + interstate.pin10, + freeway.pin10, + local_road.pin10, + collector.pin10 + ) + = other.pin10 AND COALESCE( - minor.year, interstate.year, freeway.year, - local_road.year, collector.year - ) = other.year + minor.year, + interstate.year, + freeway.year, + local_road.year, + collector.year + ) + = other.year WHERE COALESCE( minor.year, interstate.year, freeway.year, - local_road.year, collector.year, other.year - ) >= (SELECT MIN(year) FROM distinct_years) + local_road.year, collector.year, + other.year + ) + >= (SELECT MIN(year) FROM distinct_years) From 43937d385f210b0e020c6d4a2fde802d02c7041a Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Mon, 28 Oct 2024 20:16:53 +0000 Subject: [PATCH 56/61] test --- dbt/macros/nearest_feature_aggregation.sql | 67 +---- .../proximity.dist_pin_to_traffic_master.sql | 249 ++++++++++-------- 2 files changed, 152 insertions(+), 164 deletions(-) diff --git a/dbt/macros/nearest_feature_aggregation.sql b/dbt/macros/nearest_feature_aggregation.sql index 4bf6c0959..82125083e 100644 --- a/dbt/macros/nearest_feature_aggregation.sql +++ b/dbt/macros/nearest_feature_aggregation.sql @@ -1,58 +1,11 @@ -{% macro aggregate_smallest_feature(table_name, feature_suffix="dist_ft") %} - {%- set macro_content -%} - {% raw %} - {% macro aggregate_smallest_feature(table_name, feature_suffix="dist_ft") %} - with - columns as ( - select column_name - from information_schema.columns - where - table_name = lower('{{ table_name }}') - and column_name like '%' || '{{ feature_suffix }}' - ), - feature as ( - select - t.*, - unnest( - array[ - {% for column in columns %} - (t.{{ column.column_name }}, '{{ column.column_name }}') - {% endfor %} - ] - ) as (feature_value, feature_column) - from {{ table_name }} t - ), - min_feature as ( - select - *, - min(feature_value) over (partition by id) as min_feature, - array_agg(feature_column) filter ( - where feature_value = min(feature_value) over (partition by id) - ) as matching_columns - from feature - ), - aggregated as ( - select - id, - min_feature, - array_agg( - distinct regexp_extract(column_name, '^(.*?)_') - ) as original_prefixes, - array_agg( - distinct 'aggregated_' || regexp_extract(column_name, '^(.*?)_') - ) as aggregated_prefixes - from - ( - select id, unnest(matching_columns) as column_name from min_feature - ) prefix_agg - group by id, min_feature - ) - select * - from aggregated - ; - {% endmacro %} - {% endraw %} - {%- endset -%} - - {{ macro_content }} +{% macro nearest_feature_aggregation(base_columns, characteristics) %} + {%- for characteristic in characteristics %} + case + {%- for base in base_columns %} + when {{ base }} = least({{ base_columns | join(", ") }}) + then {{ base.replace("dist_ft", characteristic) }} + {%- endfor %} + else null + end as closest_{{ characteristic }}, + {%- endfor %} {% endmacro %} diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 58ae4908b..e194bb68a 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -69,8 +69,8 @@ nearest_minor AS ( ARBITRARY(xy.year) AS nearest_minor_road_data_year, ARBITRARY(xy.daily_traffic) AS nearest_minor_road_daily_traffic, ARBITRARY(xy.speed_limit) AS nearest_minor_road_speed_limit, - ARBITRARY(xy.surface_type) AS nearest_minor_surface_type, - ARBITRARY(xy.lanes) AS nearest_minor_lanes + ARBITRARY(xy.surface_type) AS nearest_minor_road_surface_type, + ARBITRARY(xy.lanes) AS nearest_minor_road_lanes FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_minor') }} ) AS xy ON pcl.x_3435 = xy.x_3435 @@ -86,10 +86,10 @@ nearest_interstate AS ( ARBITRARY(xy.road_name) AS nearest_interstate_road_name, ARBITRARY(xy.dist_ft) AS nearest_interstate_road_dist_ft, ARBITRARY(xy.year) AS nearest_interstate_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_interstate_daily_traffic, + ARBITRARY(xy.daily_traffic) AS nearest_interstate_road_daily_traffic, ARBITRARY(xy.speed_limit) AS nearest_interstate_road_speed_limit, - ARBITRARY(xy.surface_type) AS nearest_interstate_surface_type, - ARBITRARY(xy.lanes) AS nearest_interstate_lanes + ARBITRARY(xy.surface_type) AS nearest_interstate_road_surface_type, + ARBITRARY(xy.lanes) AS nearest_interstate_road_lanes FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_interstate') }} ) AS xy ON pcl.x_3435 = xy.x_3435 @@ -105,10 +105,10 @@ nearest_freeway AS ( ARBITRARY(xy.road_name) AS nearest_freeway_road_name, ARBITRARY(xy.dist_ft) AS nearest_freeway_road_dist_ft, ARBITRARY(xy.year) AS nearest_freeway_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_freeway_daily_traffic, + ARBITRARY(xy.daily_traffic) AS nearest_freeway_road_daily_traffic, ARBITRARY(xy.speed_limit) AS nearest_freeway_road_speed_limit, - ARBITRARY(xy.surface_type) AS nearest_freeway_surface_type, - ARBITRARY(xy.lanes) AS nearest_freeway_lanes + ARBITRARY(xy.surface_type) AS nearest_freeway_road_surface_type, + ARBITRARY(xy.lanes) AS nearest_freeway_road_lanes FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_freeway') }} ) AS xy ON pcl.x_3435 = xy.x_3435 @@ -123,10 +123,10 @@ nearest_local AS ( ARBITRARY(xy.road_name) AS nearest_local_road_name, ARBITRARY(xy.dist_ft) AS nearest_local_road_dist_ft, ARBITRARY(xy.year) AS nearest_local_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_local_daily_traffic, + ARBITRARY(xy.daily_traffic) AS nearest_local_road_daily_traffic, ARBITRARY(xy.speed_limit) AS nearest_local_road_speed_limit, - ARBITRARY(xy.surface_type) AS nearest_local_surface_type, - ARBITRARY(xy.lanes) AS nearest_local_lanes + ARBITRARY(xy.surface_type) AS nearest_local_road_surface_type, + ARBITRARY(xy.lanes) AS nearest_local_road_lanes FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_local') }} ) AS xy ON pcl.x_3435 = xy.x_3435 @@ -144,9 +144,9 @@ nearest_collector AS ( ARBITRARY(xy.year) AS nearest_collector_road_data_year, ARBITRARY(xy.speed_limit) AS nearest_collector_road_speed_limit, - ARBITRARY(xy.daily_traffic) AS nearest_collector_daily_traffic, - ARBITRARY(xy.surface_type) AS nearest_collector_surface_type, - ARBITRARY(xy.lanes) AS nearest_collector_lanes + ARBITRARY(xy.daily_traffic) AS nearest_collector_road_daily_traffic, + ARBITRARY(xy.surface_type) AS nearest_collector_road_surface_type, + ARBITRARY(xy.lanes) AS nearest_collector_road_lanes FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_collector') }} ) AS xy @@ -163,105 +163,140 @@ nearest_other AS ( ARBITRARY(xy.road_name) AS nearest_other_road_name, ARBITRARY(xy.dist_ft) AS nearest_other_road_dist_ft, ARBITRARY(xy.year) AS nearest_other_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_other_daily_traffic, + ARBITRARY(xy.daily_traffic) AS nearest_other_road_daily_traffic, ARBITRARY(xy.speed_limit) AS nearest_other_road_speed_limit, - ARBITRARY(xy.surface_type) AS nearest_other_surface_type, - ARBITRARY(xy.lanes) AS nearest_other_lanes + ARBITRARY(xy.surface_type) AS nearest_other_road_surface_type, + ARBITRARY(xy.lanes) AS nearest_other_road_lanes FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_other') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 GROUP BY pcl.pin10, xy.year -) +), -- Join all nearest roads by pin10 and year +final_aggregation AS ( + SELECT + COALESCE( + minor.pin10, interstate.pin10, freeway.pin10, + local_road.pin10, collector.pin10 + ) AS pin10, + minor.nearest_minor_road_name, + minor.nearest_minor_road_dist_ft, + minor.nearest_minor_road_data_year, + minor.nearest_minor_road_daily_traffic, + minor.nearest_minor_road_speed_limit, + minor.nearest_minor_road_surface_type, + minor.nearest_minor_road_lanes, + interstate.nearest_interstate_road_name, + interstate.nearest_interstate_road_dist_ft, + interstate.nearest_interstate_road_data_year, + interstate.nearest_interstate_road_daily_traffic, + interstate.nearest_interstate_road_speed_limit, + interstate.nearest_interstate_road_surface_type, + interstate.nearest_interstate_road_lanes, + freeway.nearest_freeway_road_name, + freeway.nearest_freeway_road_dist_ft, + freeway.nearest_freeway_road_data_year, + freeway.nearest_freeway_road_daily_traffic, + freeway.nearest_freeway_road_speed_limit, + freeway.nearest_freeway_road_surface_type, + freeway.nearest_freeway_road_lanes, + local_road.nearest_local_road_name, + local_road.nearest_local_road_dist_ft, + local_road.nearest_local_road_data_year, + local_road.nearest_local_road_daily_traffic, + local_road.nearest_local_road_speed_limit, + local_road.nearest_local_road_surface_type, + local_road.nearest_local_road_lanes, + collector.nearest_collector_road_name, + collector.nearest_collector_road_dist_ft, + collector.nearest_collector_road_data_year, + collector.nearest_collector_road_daily_traffic, + collector.nearest_collector_road_speed_limit, + collector.nearest_collector_road_surface_type, + collector.nearest_collector_road_lanes, + other.nearest_other_road_name, + other.nearest_other_road_dist_ft, + other.nearest_other_road_data_year, + other.nearest_other_road_daily_traffic, + other.nearest_other_road_speed_limit, + other.nearest_other_road_surface_type, + other.nearest_other_road_lanes, + COALESCE( + minor.year, interstate.year, freeway.year, + local_road.year, collector.year, other.year) + AS year + FROM nearest_minor AS minor + FULL OUTER JOIN nearest_interstate AS interstate + ON minor.pin10 = interstate.pin10 + AND minor.year = interstate.year + FULL OUTER JOIN nearest_freeway AS freeway + ON COALESCE(minor.pin10, interstate.pin10) = freeway.pin10 + AND COALESCE(minor.year, interstate.year) = freeway.year + FULL OUTER JOIN nearest_local AS local_road + ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) + = local_road.pin10 + AND COALESCE(minor.year, interstate.year, freeway.year) + = local_road.year + FULL OUTER JOIN nearest_collector AS collector + ON COALESCE( + minor.pin10, interstate.pin10, + freeway.pin10, local_road.pin10 + ) + = collector.pin10 + AND COALESCE(minor.year, interstate.year, freeway.year, local_road.year) + = collector.year + FULL OUTER JOIN nearest_other AS other + ON COALESCE( + minor.pin10, + interstate.pin10, + freeway.pin10, + local_road.pin10, + collector.pin10 + ) + = other.pin10 + AND COALESCE( + minor.year, + interstate.year, + freeway.year, + local_road.year, + collector.year + ) + = other.year + WHERE COALESCE( + minor.year, interstate.year, freeway.year, + local_road.year, collector.year, + other.year + ) + >= (SELECT MIN(year) FROM distinct_years) +) + + SELECT - COALESCE( - minor.pin10, interstate.pin10, freeway.pin10, - local_road.pin10, collector.pin10 - ) AS pin10, - minor.nearest_minor_road_name, - minor.nearest_minor_road_dist_ft, - minor.nearest_minor_road_data_year, - minor.nearest_minor_road_daily_traffic, - minor.nearest_minor_road_speed_limit, - minor.nearest_minor_surface_type, - minor.nearest_minor_lanes, - interstate.nearest_interstate_road_name, - interstate.nearest_interstate_road_dist_ft, - interstate.nearest_interstate_road_data_year, - interstate.nearest_interstate_daily_traffic, - interstate.nearest_interstate_road_speed_limit, - interstate.nearest_interstate_surface_type, - interstate.nearest_interstate_lanes, - freeway.nearest_freeway_road_name, - freeway.nearest_freeway_road_dist_ft, - freeway.nearest_freeway_road_data_year, - freeway.nearest_freeway_daily_traffic, - freeway.nearest_freeway_road_speed_limit, - freeway.nearest_freeway_surface_type, - freeway.nearest_freeway_lanes, - local_road.nearest_local_road_name, - local_road.nearest_local_road_dist_ft, - local_road.nearest_local_road_data_year, - local_road.nearest_local_daily_traffic, - local_road.nearest_local_road_speed_limit, - local_road.nearest_local_surface_type, - local_road.nearest_local_lanes, - collector.nearest_collector_road_name, - collector.nearest_collector_road_dist_ft, - collector.nearest_collector_road_data_year, - collector.nearest_collector_daily_traffic, - collector.nearest_collector_road_speed_limit, - collector.nearest_collector_surface_type, - collector.nearest_collector_lanes, - other.nearest_other_road_name, - other.nearest_other_road_dist_ft, - other.nearest_other_road_data_year, - other.nearest_other_daily_traffic, - other.nearest_other_road_speed_limit, - other.nearest_other_surface_type, - other.nearest_other_lanes, - COALESCE( - minor.year, interstate.year, freeway.year, - local_road.year, collector.year, other.year) - AS year -FROM nearest_minor AS minor -FULL OUTER JOIN nearest_interstate AS interstate - ON minor.pin10 = interstate.pin10 AND minor.year = interstate.year -FULL OUTER JOIN nearest_freeway AS freeway - ON COALESCE(minor.pin10, interstate.pin10) = freeway.pin10 - AND COALESCE(minor.year, interstate.year) = freeway.year -FULL OUTER JOIN nearest_local AS local_road - ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10) - = local_road.pin10 - AND COALESCE(minor.year, interstate.year, freeway.year) - = local_road.year -FULL OUTER JOIN nearest_collector AS collector - ON COALESCE(minor.pin10, interstate.pin10, freeway.pin10, local_road.pin10) - = collector.pin10 - AND COALESCE(minor.year, interstate.year, freeway.year, local_road.year) - = collector.year -FULL OUTER JOIN nearest_other AS other - ON COALESCE( - minor.pin10, - interstate.pin10, - freeway.pin10, - local_road.pin10, - collector.pin10 - ) - = other.pin10 - AND COALESCE( - minor.year, - interstate.year, - freeway.year, - local_road.year, - collector.year - ) - = other.year -WHERE COALESCE( - minor.year, interstate.year, freeway.year, - local_road.year, collector.year, - other.year - ) - >= (SELECT MIN(year) FROM distinct_years) + pin10, + nearest_minor_road_dist_ft, + nearest_other_road_dist_ft, + nearest_interstate_road_dist_ft, + + {{ nearest_feature_aggregation( + [ + "nearest_minor_road_dist_ft", + "nearest_interstate_road_dist_ft", + "nearest_other_road_dist_ft", + "nearest_freeway_road_dist_ft", + "nearest_local_road_dist_ft", + "nearest_collector_road_dist_ft" + ], + [ + "name", + "dist_ft", + "data_year", + "daily_traffic", + "speed_limit", + "surface_type", + "lanes" + ] + ) }} + year +FROM final_aggregation; From 35dafbf3e7a76edf52858dcf938fb3da32887c8b Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Mon, 28 Oct 2024 20:23:23 +0000 Subject: [PATCH 57/61] Remove semicolon --- .../proximity.dist_pin_to_traffic_master.sql | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index e194bb68a..04f18448d 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -275,10 +275,48 @@ final_aggregation AS ( SELECT pin10, + nearest_minor_road_name, nearest_minor_road_dist_ft, - nearest_other_road_dist_ft, + nearest_minor_road_data_year, + nearest_minor_road_daily_traffic, + nearest_minor_road_speed_limit, + nearest_minor_road_surface_type, + nearest_minor_road_lanes, + nearest_interstate_road_name, nearest_interstate_road_dist_ft, - + nearest_interstate_road_data_year, + nearest_interstate_road_daily_traffic, + nearest_interstate_road_speed_limit, + nearest_interstate_road_surface_type, + nearest_interstate_road_lanes, + nearest_freeway_road_name, + nearest_freeway_road_dist_ft, + nearest_freeway_road_data_year, + nearest_freeway_road_daily_traffic, + nearest_freeway_road_speed_limit, + nearest_freeway_road_surface_type, + nearest_freeway_road_lanes, + nearest_local_road_name, + nearest_local_road_dist_ft, + nearest_local_road_data_year, + nearest_local_road_daily_traffic, + nearest_local_road_speed_limit, + nearest_local_road_surface_type, + nearest_local_road_lanes, + nearest_collector_road_name, + nearest_collector_road_dist_ft, + nearest_collector_road_data_year, + nearest_collector_road_daily_traffic, + nearest_collector_road_speed_limit, + nearest_collector_road_surface_type, + nearest_collector_road_lanes, + nearest_other_road_name, + nearest_other_road_dist_ft, + nearest_other_road_data_year, + nearest_other_road_daily_traffic, + nearest_other_road_speed_limit, + nearest_other_road_surface_type, + nearest_other_road_lanes, {{ nearest_feature_aggregation( [ "nearest_minor_road_dist_ft", @@ -299,4 +337,4 @@ SELECT ] ) }} year -FROM final_aggregation; +FROM final_aggregation From fe29661fa5cde784811b1d0eef5b5aeef8f46f6d Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 29 Oct 2024 14:29:52 +0000 Subject: [PATCH 58/61] Try null and all for local --- .../proximity.dist_pin_to_traffic_master.sql | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 04f18448d..aac1e72f5 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -32,6 +32,13 @@ traffic_local AS ( -- noqa: ST03 AND daily_traffic IS NOT NULL ), +traffic_local_null AS ( -- noqa: ST03 + SELECT * + FROM {{ source('spatial', 'traffic') }} + WHERE road_type = 'Local Road or Street' + AND daily_traffic IS NOT NULL +), + traffic_collector AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} @@ -121,17 +128,35 @@ nearest_local AS ( pcl.pin10, xy.year, ARBITRARY(xy.road_name) AS nearest_local_road_name, - ARBITRARY(xy.dist_ft) AS nearest_local_road_dist_ft, + ARBITRARY(xy.dist_ft) AS nearest_local_dist_ft, ARBITRARY(xy.year) AS nearest_local_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_local_road_daily_traffic, - ARBITRARY(xy.speed_limit) AS nearest_local_road_speed_limit, - ARBITRARY(xy.surface_type) AS nearest_local_road_surface_type, - ARBITRARY(xy.lanes) AS nearest_local_road_lanes + ARBITRARY(xy.daily_traffic) AS nearest_local_daily_traffic, + ARBITRARY(xy.speed_limit) AS nearest_local_speed_limit, + ARBITRARY(xy.surface_type) AS nearest_local_surface_type, + ARBITRARY(xy.lanes) AS nearest_local_lanes FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_local') }} ) AS xy ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 GROUP BY pcl.pin10, xy.year + + UNION ALL + + SELECT + pcl.pin10, + xy.year, + ARBITRARY(xy.road_name) AS nearest_local_road_name, + ARBITRARY(xy.dist_ft) AS nearest_local_dist_ft, + ARBITRARY(xy.year) AS nearest_local_road_data_year, + ARBITRARY(xy.daily_traffic) AS nearest_local_daily_traffic, + ARBITRARY(xy.speed_limit) AS nearest_local_speed_limit, + ARBITRARY(xy.surface_type) AS nearest_local_surface_type, + ARBITRARY(xy.lanes) AS nearest_local_lanes + FROM distinct_pins AS pcl + INNER JOIN ( {{ dist_to_nearest_geometry('traffic_local_null') }} ) AS xy + ON pcl.x_3435 = xy.x_3435 + AND pcl.y_3435 = xy.y_3435 + GROUP BY pcl.pin10, xy.year ), -- Calculate nearest Major Collector road per pin @@ -272,7 +297,6 @@ final_aggregation AS ( >= (SELECT MIN(year) FROM distinct_years) ) - SELECT pin10, nearest_minor_road_name, From 1d118f4305f73ef494316f604cd6b4695d1ac026 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 29 Oct 2024 14:31:36 +0000 Subject: [PATCH 59/61] typo --- dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index aac1e72f5..b7efc0c1f 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -36,7 +36,7 @@ traffic_local_null AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} WHERE road_type = 'Local Road or Street' - AND daily_traffic IS NOT NULL + AND daily_traffic IS NULL ), traffic_collector AS ( -- noqa: ST03 From 3f5f221c2020dbddc16813e20fdadd263bcbee7a Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 29 Oct 2024 14:33:26 +0000 Subject: [PATCH 60/61] add road to query --- .../proximity.dist_pin_to_traffic_master.sql | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index b7efc0c1f..8a08e6c8e 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -128,12 +128,12 @@ nearest_local AS ( pcl.pin10, xy.year, ARBITRARY(xy.road_name) AS nearest_local_road_name, - ARBITRARY(xy.dist_ft) AS nearest_local_dist_ft, + ARBITRARY(xy.dist_ft) AS nearest_local_road_dist_ft, ARBITRARY(xy.year) AS nearest_local_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_local_daily_traffic, - ARBITRARY(xy.speed_limit) AS nearest_local_speed_limit, - ARBITRARY(xy.surface_type) AS nearest_local_surface_type, - ARBITRARY(xy.lanes) AS nearest_local_lanes + ARBITRARY(xy.daily_traffic) AS nearest_local_road_daily_traffic, + ARBITRARY(xy.speed_limit) AS nearest_local_road_speed_limit, + ARBITRARY(xy.surface_type) AS nearest_local_road_surface_type, + ARBITRARY(xy.lanes) AS nearest_local_road_lanes FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_local') }} ) AS xy ON pcl.x_3435 = xy.x_3435 @@ -146,12 +146,12 @@ nearest_local AS ( pcl.pin10, xy.year, ARBITRARY(xy.road_name) AS nearest_local_road_name, - ARBITRARY(xy.dist_ft) AS nearest_local_dist_ft, + ARBITRARY(xy.dist_ft) AS nearest_local_road_dist_ft, ARBITRARY(xy.year) AS nearest_local_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_local_daily_traffic, - ARBITRARY(xy.speed_limit) AS nearest_local_speed_limit, - ARBITRARY(xy.surface_type) AS nearest_local_surface_type, - ARBITRARY(xy.lanes) AS nearest_local_lanes + ARBITRARY(xy.daily_traffic) AS nearest_local_road_daily_traffic, + ARBITRARY(xy.speed_limit) AS nearest_local_road_speed_limit, + ARBITRARY(xy.surface_type) AS nearest_local_road_surface_type, + ARBITRARY(xy.lanes) AS nearest_local_road_lanes FROM distinct_pins AS pcl INNER JOIN ( {{ dist_to_nearest_geometry('traffic_local_null') }} ) AS xy ON pcl.x_3435 = xy.x_3435 From f6bdb9a10f309a811078d5d4ed24f2b0552295c1 Mon Sep 17 00:00:00 2001 From: Damonamajor Date: Tue, 29 Oct 2024 20:56:54 +0000 Subject: [PATCH 61/61] Revert to working query --- .../proximity.dist_pin_to_traffic_master.sql | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql index 8a08e6c8e..04f18448d 100644 --- a/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql +++ b/dbt/models/proximity/proximity.dist_pin_to_traffic_master.sql @@ -32,13 +32,6 @@ traffic_local AS ( -- noqa: ST03 AND daily_traffic IS NOT NULL ), -traffic_local_null AS ( -- noqa: ST03 - SELECT * - FROM {{ source('spatial', 'traffic') }} - WHERE road_type = 'Local Road or Street' - AND daily_traffic IS NULL -), - traffic_collector AS ( -- noqa: ST03 SELECT * FROM {{ source('spatial', 'traffic') }} @@ -139,24 +132,6 @@ nearest_local AS ( ON pcl.x_3435 = xy.x_3435 AND pcl.y_3435 = xy.y_3435 GROUP BY pcl.pin10, xy.year - - UNION ALL - - SELECT - pcl.pin10, - xy.year, - ARBITRARY(xy.road_name) AS nearest_local_road_name, - ARBITRARY(xy.dist_ft) AS nearest_local_road_dist_ft, - ARBITRARY(xy.year) AS nearest_local_road_data_year, - ARBITRARY(xy.daily_traffic) AS nearest_local_road_daily_traffic, - ARBITRARY(xy.speed_limit) AS nearest_local_road_speed_limit, - ARBITRARY(xy.surface_type) AS nearest_local_road_surface_type, - ARBITRARY(xy.lanes) AS nearest_local_road_lanes - FROM distinct_pins AS pcl - INNER JOIN ( {{ dist_to_nearest_geometry('traffic_local_null') }} ) AS xy - ON pcl.x_3435 = xy.x_3435 - AND pcl.y_3435 = xy.y_3435 - GROUP BY pcl.pin10, xy.year ), -- Calculate nearest Major Collector road per pin @@ -297,6 +272,7 @@ final_aggregation AS ( >= (SELECT MIN(year) FROM distinct_years) ) + SELECT pin10, nearest_minor_road_name,