Merge pull request #158 from SANDAG/taz_level_firm_synthesis

Changed firm synthesis to be at the TAZ level instead of the MGRA level. The original process was trying to match employee size category distributions at the MGRA level that was challenging and time consuming.

src/main/python/MGRAEmpByEstSize2.py

@@ -71,7 +71,7 @@
 #                                      r"Land_Use\mgra15_based_input_2022_02_cvm.csv"))
 
 #df_MGRA = pd.read_csv(f"{root_dir}\\input\\mgra15_based_input2022.csv")
-df_MGRA = pd.read_csv(f"{root_dir}\\input\\land_use.csv")
+df_MGRA = pd.read_csv(f"{root_dir}\\input\\land_use_taz.csv")
 df_MGRA = df_MGRA.rename(columns={'luz_id': 'LUZ', 'emp_total': 'emp_tot'})
 df_MGRA = df_MGRA.loc[:, df_MGRA.columns != 'TAZ']
 

src/main/python/aggregateEmpData.py

@@ -0,0 +1,50 @@
+import pandas as pd
+import numpy as np
+import sys
+
+infile = sys.argv[1]
+outfile = sys.argv[2]
+
+emp_fields = [
+    "emp_gov",
+    "emp_mil",
+    "emp_ag_min",
+    "emp_bus_svcs",
+    "emp_fin_res_mgm",
+    "emp_educ",
+    "emp_hlth",
+    "emp_ret",
+    "emp_trn_wrh",
+    "emp_con",
+    "emp_utl",
+    "emp_mnf",
+    "emp_whl",
+    "emp_ent",
+    "emp_accm",
+    "emp_food",
+    "emp_oth",
+    "emp_non_ws_wfh",
+    "emp_non_ws_oth",
+]
+
+print("Reading Data")
+mgra_data = pd.read_csv(infile)
+
+print("Creating Maps")
+maz2taz = pd.get_dummies(mgra_data.set_index("mgra")["taz"]) # Matrix to aggregate MAZ-level data by TAZ
+taz2luz = mgra_data.groupby("taz").first()['luz_id'] # Series for mapping TAZ to LUZ
+
+print("Aggregating Data")
+emp_by_taz = pd.DataFrame(
+    maz2taz.T.values.dot(mgra_data[emp_fields].values),
+    index = maz2taz.columns,
+    columns = emp_fields
+    )
+emp_by_taz["emp_total"] = emp_by_taz.sum(1)
+emp_by_taz.index.name = "taz"
+emp_by_taz = emp_by_taz.reset_index()
+emp_by_taz["mgra"] = emp_by_taz["taz"].copy()
+emp_by_taz["luz_id"] = emp_by_taz["taz"].map(taz2luz)
+
+print("Writing Results")
+emp_by_taz[["mgra", "taz", "luz_id"] + emp_fields + ["emp_total"]].to_csv(outfile, index = False)

src/main/python/recodeSynthEstabToMGRA.py

@@ -0,0 +1,20 @@
+import pandas as pd
+import sys
+from shutil import copy
+
+est_file = sys.argv[1]
+lu_file = sys.argv[2]
+
+print("Creating backup of synthetic establishments file")
+copy(est_file, est_file.replace(".csv", "_backup.csv"))
+
+print("Reading Data")
+est = pd.read_csv(est_file)
+lu = pd.read_csv(lu_file)
+
+print("Recoding MGRAs")
+taz2mgra = lu.groupby("taz").first()["mgra"]
+est["MGRA"] = est["MGRA"].map(taz2mgra)
+
+print("Writing Data")
+est.to_csv(est_file, index = False)

src/main/resources/cvmEst.bat

@@ -42,10 +42,18 @@ SET MODEL_DIR=%PROJECT_DRIVE%%PROJECT_DIRECTORY%
 SET OUTPUT_DIR=%PROJECT_DRIVE%%PROJECT_DIRECTORY%\input\CVM
 SET luz_data=%PROJECT_DRIVE%%PROJECT_DIRECTORY%\%luz_data_file%
 
+:: Aggregate employment data to TAZ level
+ECHO Aggregate employment data to TAZ level
+CD /d %PROJECT_DRIVE%%PROJECT_DIRECTORY%\python\
+%PYTHON3% aggregateEmpData.py %PROJECT_DRIVE%%PROJECT_DIRECTORY%\input\land_use.csv %PROJECT_DRIVE%%PROJECT_DIRECTORY%\input\land_use_taz.csv
+
 :: run MGRAEmpByEstSize2.py
 ECHO Run CVMEstablishmentSynthesis
-CD /d %PROJECT_DRIVE%%PROJECT_DIRECTORY%\python\
 %PYTHON3% MGRAEmpByEstSize2.py %MODEL_DIR% %OUTPUT_DIR% %luz_data% 2>>%PROJECT_DRIVE%%PROJECT_DIRECTORY%\logFiles\event-cvmEmp.txt
+
+:: Recode establishment zone IDs from TAZ to MGRA
+ECHO Recode establishment zone IDs from TAZ to MGRA
+%PYTHON3% recodeSynthEstabToMGRA.py %PROJECT_DRIVE%%PROJECT_DIRECTORY%\input\CVM\SynthEstablishments.csv %PROJECT_DRIVE%%PROJECT_DIRECTORY%\input\land_use.csv
 IF %ERRORLEVEL% NEQ 0 (GOTO :ERROR) else (GOTO :SUCCESS)
 
 :SUCCESS