From 8cd5474fb022a872a583b080c5b01ca7b12c22d6 Mon Sep 17 00:00:00 2001
From: JoeJimFlood <joejimflood@gmail.com>
Date: Fri, 21 Jun 2024 14:57:18 -0700
Subject: [PATCH] Changed firm synthesis to be at the TAZ level instead of the
 MGRA level

---
 src/main/python/MGRAEmpByEstSize2.py      |  2 +-
 src/main/python/aggregateEmpData.py       | 50 +++++++++++++++++++++++
 src/main/python/recodeSynthEstabToMGRA.py | 20 +++++++++
 src/main/resources/cvmEst.bat             | 10 ++++-
 4 files changed, 80 insertions(+), 2 deletions(-)
 create mode 100644 src/main/python/aggregateEmpData.py
 create mode 100644 src/main/python/recodeSynthEstabToMGRA.py

diff --git a/src/main/python/MGRAEmpByEstSize2.py b/src/main/python/MGRAEmpByEstSize2.py
index d6e035ae5..039a08a9f 100644
--- a/src/main/python/MGRAEmpByEstSize2.py
+++ b/src/main/python/MGRAEmpByEstSize2.py
@@ -71,7 +71,7 @@
 #                                      r"Land_Use\mgra15_based_input_2022_02_cvm.csv"))
 
 #df_MGRA = pd.read_csv(f"{root_dir}\\input\\mgra15_based_input2022.csv")
-df_MGRA = pd.read_csv(f"{root_dir}\\input\\land_use.csv")
+df_MGRA = pd.read_csv(f"{root_dir}\\input\\land_use_taz.csv")
 df_MGRA = df_MGRA.rename(columns={'luz_id': 'LUZ', 'emp_total': 'emp_tot'})
 df_MGRA = df_MGRA.loc[:, df_MGRA.columns != 'TAZ']
 
diff --git a/src/main/python/aggregateEmpData.py b/src/main/python/aggregateEmpData.py
new file mode 100644
index 000000000..cdb5a8126
--- /dev/null
+++ b/src/main/python/aggregateEmpData.py
@@ -0,0 +1,50 @@
+import pandas as pd
+import numpy as np
+import sys
+
+infile = sys.argv[1]
+outfile = sys.argv[2]
+
+emp_fields = [
+    "emp_gov",
+    "emp_mil",
+    "emp_ag_min",
+    "emp_bus_svcs",
+    "emp_fin_res_mgm",
+    "emp_educ",
+    "emp_hlth",
+    "emp_ret",
+    "emp_trn_wrh",
+    "emp_con",
+    "emp_utl",
+    "emp_mnf",
+    "emp_whl",
+    "emp_ent",
+    "emp_accm",
+    "emp_food",
+    "emp_oth",
+    "emp_non_ws_wfh",
+    "emp_non_ws_oth",
+]
+
+print("Reading Data")
+mgra_data = pd.read_csv(infile)
+
+print("Creating Maps")
+maz2taz = pd.get_dummies(mgra_data.set_index("mgra")["taz"]) # Matrix to aggregate MAZ-level data by TAZ
+taz2luz = mgra_data.groupby("taz").first()['luz_id'] # Series for mapping TAZ to LUZ
+
+print("Aggregating Data")
+emp_by_taz = pd.DataFrame(
+    maz2taz.T.values.dot(mgra_data[emp_fields].values),
+    index = maz2taz.columns,
+    columns = emp_fields
+    )
+emp_by_taz["emp_total"] = emp_by_taz.sum(1)
+emp_by_taz.index.name = "taz"
+emp_by_taz = emp_by_taz.reset_index()
+emp_by_taz["mgra"] = emp_by_taz["taz"].copy()
+emp_by_taz["luz_id"] = emp_by_taz["taz"].map(taz2luz)
+
+print("Writing Results")
+emp_by_taz[["mgra", "taz", "luz_id"] + emp_fields + ["emp_total"]].to_csv(outfile, index = False)
\ No newline at end of file
diff --git a/src/main/python/recodeSynthEstabToMGRA.py b/src/main/python/recodeSynthEstabToMGRA.py
new file mode 100644
index 000000000..f8c1e0c2e
--- /dev/null
+++ b/src/main/python/recodeSynthEstabToMGRA.py
@@ -0,0 +1,20 @@
+import pandas as pd
+import sys
+from shutil import copy
+
+est_file = sys.argv[1]
+lu_file = sys.argv[2]
+
+print("Creating backup of synthetic establishments file")
+copy(est_file, est_file.replace(".csv", "_backup.csv"))
+
+print("Reading Data")
+est = pd.read_csv(est_file)
+lu = pd.read_csv(lu_file)
+
+print("Recoding MGRAs")
+taz2mgra = lu.groupby("taz").first()["mgra"]
+est["MGRA"] = est["MGRA"].map(taz2mgra)
+
+print("Writing Data")
+est.to_csv(est_file, index = False)
\ No newline at end of file
diff --git a/src/main/resources/cvmEst.bat b/src/main/resources/cvmEst.bat
index b93063724..f5207f283 100644
--- a/src/main/resources/cvmEst.bat
+++ b/src/main/resources/cvmEst.bat
@@ -42,10 +42,18 @@ SET MODEL_DIR=%PROJECT_DRIVE%%PROJECT_DIRECTORY%
 SET OUTPUT_DIR=%PROJECT_DRIVE%%PROJECT_DIRECTORY%\input\CVM
 SET luz_data=%PROJECT_DRIVE%%PROJECT_DIRECTORY%\%luz_data_file%
 
+:: Aggregate employment data to TAZ level
+ECHO Aggregate employment data to TAZ level
+CD /d %PROJECT_DRIVE%%PROJECT_DIRECTORY%\python\
+%PYTHON3% aggregateEmpData.py %PROJECT_DRIVE%%PROJECT_DIRECTORY%\input\land_use.csv %PROJECT_DRIVE%%PROJECT_DIRECTORY%\input\land_use_taz.csv
+
 :: run MGRAEmpByEstSize2.py
 ECHO Run CVMEstablishmentSynthesis
-CD /d %PROJECT_DRIVE%%PROJECT_DIRECTORY%\python\
 %PYTHON3% MGRAEmpByEstSize2.py %MODEL_DIR% %OUTPUT_DIR% %luz_data% 2>>%PROJECT_DRIVE%%PROJECT_DIRECTORY%\logFiles\event-cvmEmp.txt
+
+:: Recode establishment zone IDs from TAZ to MGRA
+ECHO Recode establishment zone IDs from TAZ to MGRA
+%PYTHON3% recodeSynthEstabToMGRA.py %PROJECT_DRIVE%%PROJECT_DIRECTORY%\input\CVM\SynthEstablishments.csv %PROJECT_DRIVE%%PROJECT_DIRECTORY%\input\land_use.csv
 IF %ERRORLEVEL% NEQ 0 (GOTO :ERROR) else (GOTO :SUCCESS)
 
 :SUCCESS