From cfb730d96c86a41c131d60ce80cb2c0adaf59764 Mon Sep 17 00:00:00 2001 From: Lachlan Perrier Date: Mon, 3 Jun 2024 16:46:46 -0400 Subject: [PATCH 01/12] slight modification to initialise network --- scripts/compile_model_runs.py | 5 +++++ tm2py/components/network/create_tod_scenarios.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 scripts/compile_model_runs.py diff --git a/scripts/compile_model_runs.py b/scripts/compile_model_runs.py new file mode 100644 index 00000000..8e21750b --- /dev/null +++ b/scripts/compile_model_runs.py @@ -0,0 +1,5 @@ +import geopandas as gpd +import pandas as pd +from pathlib import Path + +input_dir = "" \ No newline at end of file diff --git a/tm2py/components/network/create_tod_scenarios.py b/tm2py/components/network/create_tod_scenarios.py index e9777d14..09e137fd 100644 --- a/tm2py/components/network/create_tod_scenarios.py +++ b/tm2py/components/network/create_tod_scenarios.py @@ -46,7 +46,7 @@ def run(self): # emme_app = self._emme_manager.project(project_path) # self._emme_manager.init_modeller(emme_app) with self._setup(): - # self._create_highway_scenarios() + self._create_highway_scenarios() self._create_transit_scenarios() @_context From 93804498225242a09cd3f3ecd2129930295aaba8 Mon Sep 17 00:00:00 2001 From: Sijia Wang Date: Mon, 3 Jun 2024 18:34:08 -0400 Subject: [PATCH 02/12] uncomment create highway scenarios step --- tm2py/components/network/create_tod_scenarios.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tm2py/components/network/create_tod_scenarios.py b/tm2py/components/network/create_tod_scenarios.py index e9777d14..09e137fd 100644 --- a/tm2py/components/network/create_tod_scenarios.py +++ b/tm2py/components/network/create_tod_scenarios.py @@ -46,7 +46,7 @@ def run(self): # emme_app = self._emme_manager.project(project_path) # self._emme_manager.init_modeller(emme_app) with self._setup(): - # self._create_highway_scenarios() + self._create_highway_scenarios() self._create_transit_scenarios() @_context From eb0785849aa2fe9c9d2d8237c11b59f2c6acfa87 Mon Sep 17 00:00:00 2001 From: Sijia Wang Date: Tue, 4 Jun 2024 18:12:54 -0400 Subject: [PATCH 03/12] separate TAZ and MAZ network --- tm2py/components/demand/commercial.py | 2 +- tm2py/components/demand/demand.py | 2 +- tm2py/components/demand/prepare_demand.py | 4 +- .../network/create_tod_scenarios.py | 33 +++++--- .../network/highway/drive_access_skims.py | 2 +- .../network/highway/highway_assign.py | 78 ++++++++++++++++++- .../components/network/highway/highway_maz.py | 11 ++- .../network/highway/highway_network.py | 2 +- .../network/transit/transit_network.py | 2 +- tm2py/config.py | 6 +- tm2py/emme/manager.py | 24 ++++-- 11 files changed, 134 insertions(+), 32 deletions(-) diff --git a/tm2py/components/demand/commercial.py b/tm2py/components/demand/commercial.py index 65af8fde..128ce623 100644 --- a/tm2py/components/demand/commercial.py +++ b/tm2py/components/demand/commercial.py @@ -144,7 +144,7 @@ def emmebank(self): This should really be in the controller? Or part of network.skims? """ - self._emmebank = self.controller.emme_manager.highway_emmebank + self._emmebank = self.controller.emme_manager.highway_taz_emmebank return self._emmebank @property diff --git a/tm2py/components/demand/demand.py b/tm2py/components/demand/demand.py index 24da7202..86bdad0d 100644 --- a/tm2py/components/demand/demand.py +++ b/tm2py/components/demand/demand.py @@ -84,7 +84,7 @@ def __init__(self, controller: RunController): # @LogStartEnd("prepare highway demand") def run(self): """Open combined demand OMX files from demand models and prepare for assignment.""" - self._emmebank_path = self.get_abs_path(self.config.emme.highway_database_path) + self._emmebank_path = self.get_abs_path(self.config.emme.highway_taz_database_path) self._emmebank = self.controller.emme_manager.emmebank(self._emmebank_path) self._create_zero_matrix() for time in self.time_period_names(): diff --git a/tm2py/components/demand/prepare_demand.py b/tm2py/components/demand/prepare_demand.py index b6d8f66d..c134b3bb 100644 --- a/tm2py/components/demand/prepare_demand.py +++ b/tm2py/components/demand/prepare_demand.py @@ -164,7 +164,7 @@ def validate_inputs(self): @property def highway_emmebank(self): if self._highway_emmebank == None: - self._highway_emmebank = self.controller.emme_manager.highway_emmebank + self._highway_emmebank = self.controller.emme_manager.highway_taz_emmebank self._emmebank = self._highway_emmebank return self._highway_emmebank @@ -584,7 +584,7 @@ def num_internal_zones(self): @property def num_total_zones(self): self._emmebank_path = self.controller.get_abs_path( - self.controller.config.emme.highway_database_path + self.controller.config.emme.highway_taz_database_path ) self._emmebank = self.controller.emme_manager.emmebank(self._emmebank_path) time_period = self.controller.config.time_periods[0].name diff --git a/tm2py/components/network/create_tod_scenarios.py b/tm2py/components/network/create_tod_scenarios.py index 09e137fd..b2dd6387 100644 --- a/tm2py/components/network/create_tod_scenarios.py +++ b/tm2py/components/network/create_tod_scenarios.py @@ -46,7 +46,8 @@ def run(self): # emme_app = self._emme_manager.project(project_path) # self._emme_manager.init_modeller(emme_app) with self._setup(): - self._create_highway_scenarios() + self._create_highway_scenarios(zone_flag="taz") + self._create_highway_scenarios(zone_flag="maz") self._create_transit_scenarios() @_context @@ -90,20 +91,32 @@ def _project_coordinates(self, ref_scenario): emme_app.project.save() @LogStartEnd("Create highway time of day scenarios.") - def _create_highway_scenarios(self): - emmebank = self.controller.emme_manager.highway_emmebank.emmebank + def _create_highway_scenarios(self, zone_flag = "taz"): + if zone_flag == "taz": + emmebank = self.controller.emme_manager.highway_taz_emmebank.emmebank + else: + emmebank = self.controller.emme_manager.highway_maz_emmebank.emmebank ref_scenario = emmebank.scenario( self.controller.config.emme.all_day_scenario_id ) self._ref_auto_network = ref_scenario.get_network() n_time_periods = len(self.controller.config.time_periods) - self.controller.emme_manager.highway_emmebank.change_dimensions( - { - "scenarios": 1 + n_time_periods, - "full_matrices": 9999, - "extra_attribute_values": 60000000, - } - ) + if zone_flag == "taz": + self.controller.emme_manager.highway_taz_emmebank.change_dimensions( + { + "scenarios": 1 + n_time_periods, + "full_matrices": 9999, + "extra_attribute_values": 60000000, + } + ) + else: + self.controller.emme_manager.highway_maz_emmebank.change_dimensions( + { + "scenarios": 1 + n_time_periods, + "full_matrices": 9999, + "extra_attribute_values": 60000000, + } + ) # create VDFs & set cross-reference function parameters emmebank.extra_function_parameters.el1 = "@free_flow_time" emmebank.extra_function_parameters.el2 = "@capacity" diff --git a/tm2py/components/network/highway/drive_access_skims.py b/tm2py/components/network/highway/drive_access_skims.py index 59eb82b1..d96bc0f4 100644 --- a/tm2py/components/network/highway/drive_access_skims.py +++ b/tm2py/components/network/highway/drive_access_skims.py @@ -199,7 +199,7 @@ def process_stops(stops): def _get_drive_costs(self, period: TimePeriodConfig) -> pd.DataFrame: """Load the drive costs from OMX matrix files, return as pandas dataframe.""" - emmebank = self.controller.emme_manager.highway_emmebank.emmebank + emmebank = self.controller.emme_manager.highway_taz_emmebank.emmebank scenario = emmebank.scenario(period.emme_scenario_id) zone_numbers = scenario.zone_numbers network = self.controller.emme_manager.get_network( diff --git a/tm2py/components/network/highway/highway_assign.py b/tm2py/components/network/highway/highway_assign.py index 16fe772f..852175de 100644 --- a/tm2py/components/network/highway/highway_assign.py +++ b/tm2py/components/network/highway/highway_assign.py @@ -103,11 +103,13 @@ def __init__(self, controller: RunController): self._class_config = None self._scenario = None self._highway_emmebank = None + self._highway_maz_scenarios = None + self._highway_maz_emmebank = None @property def highway_emmebank(self): if not self._highway_emmebank: - self._highway_emmebank = self.controller.emme_manager.highway_emmebank + self._highway_emmebank = self.controller.emme_manager.highway_taz_emmebank return self._highway_emmebank @property @@ -122,6 +124,20 @@ def class_config(self): self._class_config = {c.name: c for c in self.config.classes} return self._class_config + + @property + def highway_maz_emmebank(self): + if not self._highway_maz_emmebank: + self._highway_maz_emmebank = self.controller.emme_manager.highway_maz_emmebank + return self._highway_maz_emmebank + + @property + def highway_maz_scenarios(self): + if self._highway_maz_scenarios is None: + self._highway_maz_scenarios = { + tp: self.highway_maz_emmebank.scenario(tp) for tp in self.time_period_names + } + return self._highway_maz_scenarios def validate_inputs(self): """Validate inputs files are correct, raise if an error is found.""" @@ -144,7 +160,7 @@ def run(self): AssignmentClass(c, time, iteration) for c in self.config.classes ] if iteration > 0: - self._copy_maz_flow(scenario) + self._copy_maz_flow(scenario, time) else: self._reset_background_traffic(scenario) self._create_skim_matrices(scenario, assign_classes) @@ -232,16 +248,72 @@ def _setup(self, scenario: EmmeScenario, time_period: str): self._matrix_cache.clear() self._matrix_cache = None self._skim_matrices = [] + + def _get_maz_links( + self, + time_period: str, + ): + """Create dictionary of link ids mapped to maz network. + + Args: + time_period (str): time period abbreviation + """ + _highway_maz_scenario = self.highway_maz_scenarios[time_period] + if not _highway_maz_scenario.has_traffic_results: + return {} + _highway_maz_net = _highway_maz_scenario.get_partial_network( + ["LINK"], include_attributes=False + ) + + highway_attributes = { + "LINK": ["#link_id", "@maz_flow"] + } - def _copy_maz_flow(self, scenario: EmmeScenario): + self.emme_manager.copy_attribute_values( + _highway_maz_scenario, _highway_maz_net, highway_attributes + ) + + # TODO can we just get the link attributes as a DataFrame and merge them? + maz_link_dict = { + maz_link["#link_id"]: maz_link for maz_link in _highway_maz_net.links() + } + + return maz_link_dict + + def _copy_maz_flow(self, scenario: EmmeScenario, time_period: str): """Copy maz_flow from MAZ demand assignment to ul1 for background traffic. Args: scenario: Emme scenario object + time_period (str): time period abbreviation """ + + _highway_net = scenario.get_partial_network( + ["LINK"], include_attributes=False + ) + highway_attributes = { + "LINK": ["#link_id", "@maz_flow"] + } + self.emme_manager.copy_attribute_values( + scenario, _highway_net, highway_attributes + ) + + _maz_link_dict = self._get_maz_links(time_period) + self.logger.log( "Copy @maz_flow to ul1 for background traffic", indent=True, level="DETAIL" ) + for link in _highway_net.links(): + if link["#link_id"] in _maz_link_dict.keys(): + link["@maz_flow"] = _maz_link_dict[link["#link_id"]]["@maz_flow"] + + _update_attributes = { + "LINK": ["@maz_flow"], + } + self.emme_manager.copy_attribute_values( + _highway_net, scenario, _update_attributes + ) + net_calc = NetworkCalculator(self.controller, scenario) net_calc("ul1", "@maz_flow") diff --git a/tm2py/components/network/highway/highway_maz.py b/tm2py/components/network/highway/highway_maz.py index a43c0c3d..28180a2c 100644 --- a/tm2py/components/network/highway/highway_maz.py +++ b/tm2py/components/network/highway/highway_maz.py @@ -99,7 +99,7 @@ def __init__(self, controller: RunController): @property def highway_emmebank(self): if self._highway_emmebank is None: - self._highway_emmebank = self.controller.emme_manager.highway_emmebank + self._highway_emmebank = self.controller.emme_manager.highway_maz_emmebank return self._highway_emmebank @property @@ -122,6 +122,11 @@ def run(self): county_groups[group.number] = group.counties for time in self.time_period_names: self._scenario = self.highway_emmebank.scenario(time) + if self.controller.iteration == 0: + create_attribute = self.controller.emme_manager.tool( + "inro.emme.data.extra_attribute.create_extra_attribute" + ) + create_attribute("LINK","@maz_flow","Assigned MAZ-to-MAZ flow",overwrite=True,scenario=self._scenario) with self._setup(time): self._prepare_network() for i, names in county_groups.items(): @@ -518,7 +523,7 @@ def _assign_flow_text( ) self.controller.emme_manager.copy_attribute_values( - self._network, self._scenario, {"LINK": ["temp_flow"]}, {"LINK": ["data1"]} + self._network, self._scenario, {"LINK": ["temp_flow"]}, {"LINK": ["@maz_flow"]} ) def _load_text_format_paths( @@ -689,7 +694,7 @@ def __init__(self, controller: RunController): @property def highway_emmebank(self): if self._highway_emmebank is None: - self._highway_emmebank = self.controller.emme_manager.highway_emmebank + self._highway_emmebank = self.controller.emme_manager.highway_maz_emmebank return self._highway_emmebank @property diff --git a/tm2py/components/network/highway/highway_network.py b/tm2py/components/network/highway/highway_network.py index f456b604..bf292b58 100644 --- a/tm2py/components/network/highway/highway_network.py +++ b/tm2py/components/network/highway/highway_network.py @@ -97,7 +97,7 @@ def run(self): @property def highway_emmebank(self): if not self._highway_emmebank: - self._highway_emmebank = self.controller.emme_manager.highway_emmebank + self._highway_emmebank = self.controller.emme_manager.highway_taz_emmebank return self._highway_emmebank @property diff --git a/tm2py/components/network/transit/transit_network.py b/tm2py/components/network/transit/transit_network.py index edee3844..49148f45 100644 --- a/tm2py/components/network/transit/transit_network.py +++ b/tm2py/components/network/transit/transit_network.py @@ -114,7 +114,7 @@ def transit_emmebank(self): @property def highway_emmebank(self): if not self._highway_emmebank: - self._highway_emmebank = self.controller.emme_manager.highway_emmebank + self._highway_emmebank = self.controller.emme_manager.highway_taz_emmebank return self._highway_emmebank @property diff --git a/tm2py/config.py b/tm2py/config.py index 72ab3387..f7e29f99 100644 --- a/tm2py/config.py +++ b/tm2py/config.py @@ -1262,7 +1262,8 @@ class EmmeConfig(ConfigItem): all_day_scenario_id: scenario ID to use for all day (initial imported) scenario with all time period data project_path: relative path from run_dir to Emme desktop project (.emp) - highway_database_path: relative path to highway Emmebank + highway_taz_database_path: relative path to TAZ highway Emmebank + highway_maz_database_path: relative path to MAZ highway Emmebank active_north_database_path: relative paths to active mode Emmebank for north bay active_south_database_path: relative paths to active mode Emmebank for south bay transit_database_path: relative path to transit Emmebank @@ -1274,7 +1275,8 @@ class EmmeConfig(ConfigItem): all_day_scenario_id: int project_path: pathlib.Path - highway_database_path: pathlib.Path + highway_taz_database_path: pathlib.Path + highway_maz_database_path: pathlib.Path active_north_database_path: pathlib.Path active_south_database_path: pathlib.Path transit_database_path: pathlib.Path diff --git a/tm2py/emme/manager.py b/tm2py/emme/manager.py index dabc65fa..0d2ad3a8 100644 --- a/tm2py/emme/manager.py +++ b/tm2py/emme/manager.py @@ -146,8 +146,11 @@ def __init__(self, controller, emme_config: "EmmeConfig"): self.project_path = self.controller.get_abs_path(self.config.project_path) # see if works without os.path.normcase(os.path.realpath(project_path)) - self.highway_database_path = self.controller.get_abs_path( - self.config.highway_database_path + self.highway_taz_database_path = self.controller.get_abs_path( + self.config.highway_taz_database_path + ) + self.highway_maz_database_path = self.controller.get_abs_path( + self.config.highway_maz_database_path ) self.transit_database_path = self.controller.get_abs_path( self.config.transit_database_path @@ -163,7 +166,8 @@ def __init__(self, controller, emme_config: "EmmeConfig"): self._project = None self._modeller = None - self._highway_emmebank = None + self._highway_taz_emmebank = None + self._highway_maz_emmebank = None self._transit_emmebank = None self._active_north_emmebank = None self._active_south_emmebank = None @@ -221,10 +225,16 @@ def modeller(self) -> EmmeModeller: return self._modeller @property - def highway_emmebank(self) -> EmmeBank: - if self._highway_emmebank is None: - self._highway_emmebank = EmmeBank(self, self.highway_database_path) - return self._highway_emmebank + def highway_taz_emmebank(self) -> EmmeBank: + if self._highway_taz_emmebank is None: + self._highway_taz_emmebank = EmmeBank(self, self.highway_taz_database_path) + return self._highway_taz_emmebank + + @property + def highway_maz_emmebank(self) -> EmmeBank: + if self._highway_maz_emmebank is None: + self._highway_maz_emmebank = EmmeBank(self, self.highway_maz_database_path) + return self._highway_maz_emmebank @property def transit_emmebank(self) -> EmmeBank: From 305b48370141ee7c787d96fe9a3efefe58bab33f Mon Sep 17 00:00:00 2001 From: Lachlan Perrier Date: Wed, 12 Jun 2024 16:36:47 -0400 Subject: [PATCH 04/12] added scripts for comparing model outputs + running with dropping model ft 6 --- scripts/compile_model_runs.py | 58 ++++++++++++++++++- .../network/create_tod_scenarios.py | 2 +- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/scripts/compile_model_runs.py b/scripts/compile_model_runs.py index 8e21750b..53d7c267 100644 --- a/scripts/compile_model_runs.py +++ b/scripts/compile_model_runs.py @@ -1,5 +1,61 @@ +print("running") import geopandas as gpd import pandas as pd +from sqlalchemy import create_engine from pathlib import Path +from tqdm import tqdm -input_dir = "" \ No newline at end of file +input_dir = Path(r"\\corp.pbwan.net\us\CentralData\DCCLDA00\Standard\sag\projects\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result") +output_dir = input_dir / "consolidated" + + +in_file = next(input_dir.rglob('emme_links.shp')) +print("reading") +input = gpd.read_file(in_file) +print("writing") +input[["geometry"]].to_file(output_dir / "test_geom.geojson") + +# def read_file_and_tag(path: Path) -> gpd.GeoDataFrame: +# scenario = file.parent.stem +# scenario_number = int(scenario.split("_")[-1]) + +# run = file.parent.parent.stem +# run_number = int(run.split("_")[-1]) + +# return_gdf = gpd.read_file(path) + +# return_gdf["scenario"] = scenario +# return_gdf["scenario_number"] = scenario_number +# return_gdf["run"] = run +# return_gdf["run_number"] = run_number + +# return return_gdf + + + + +# print("Reading Links...", end="") +# all_links = [] +# x = 0 +# for file in tqdm(input_dir.rglob('emme_links.shp')): +# all_links.append(read_file_and_tag(file)) +# if x == 0: +# all_links[-1][["geometry"]].to_file(output_dir / "test_min_geom.geojson") +# x = x + 1 +# links_table = pd.concat(all_links) +# links_table = gpd.GeoDataFrame(links_table, geometry="geometry", crs=all_links[0].crs) +# print("done") + +# print("reading Nodes...", end="") +# all_nodes = [] +# for file in tqdm(input_dir.rglob('emme_nodes.shp')): +# all_nodes.append(read_file_and_tag(file)) + +# nodes_table = pd.concat(all_nodes) +# nodes_table = gpd.GeoDataFrame(nodes_table, geometry="geometry", crs=all_links[0].crs) +# print("done") + +# print("outputting files...") + +# links_table.to_file(output_dir/"links.geojson") +# nodes_table.to_file(output_dir/"nodes.geojson") \ No newline at end of file diff --git a/tm2py/components/network/create_tod_scenarios.py b/tm2py/components/network/create_tod_scenarios.py index 09e137fd..7cf5a9c5 100644 --- a/tm2py/components/network/create_tod_scenarios.py +++ b/tm2py/components/network/create_tod_scenarios.py @@ -628,7 +628,7 @@ def _set_capclass(network): area_type = link["@area_type"] if area_type < 0: link["@capclass"] = -1 - elif (link["@ft"] == 99) & (link["@assignable"] == 1): + elif (link["@ft"] == 99): link["@capclass"] = 10 * area_type + 7 else: link["@capclass"] = 10 * area_type + link["@ft"] From 8848c5df217bdd37c3dd9bd6b0287a9a7404d799 Mon Sep 17 00:00:00 2001 From: Lachlan Perrier Date: Wed, 12 Jun 2024 16:37:15 -0400 Subject: [PATCH 05/12] added script for comparing skims --- scripts/compare_skims.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 scripts/compare_skims.py diff --git a/scripts/compare_skims.py b/scripts/compare_skims.py new file mode 100644 index 00000000..bef2b39c --- /dev/null +++ b/scripts/compare_skims.py @@ -0,0 +1,27 @@ +#%% +import pandas as pd +import openmatrix as omx +from pathlib import Path + +import numpy as np + +network_fid_path = Path(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result") + +#%% + +def read_matrix_as_long_df(path: Path, run_name): + run = omx.open_file(path, "r") + am_time = np.array(run["AM_da_time"]) + index_lables = list(range(am_time.shape[0])) + return pd.DataFrame(am_time, index=index_lables, columns=index_lables).stack().rename(run_name).to_frame() + +all_skims = [] +for skim_matrix_path in network_fid_path.rglob("*AM_taz.omx"): + run_name = skim_matrix_path.parts[6] + all_skims.append(read_matrix_as_long_df(skim_matrix_path, run_name)) + +all_skims = pd.concat(all_skims, axis=1) +# %% +#%%% +all_skims.to_csv(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result\consolidated\skims.csv") +# %% From d5d948b83449eee8ec201f8e97223f0d2830ffb3 Mon Sep 17 00:00:00 2001 From: Sijia Wang Date: Thu, 13 Jun 2024 18:49:24 -0400 Subject: [PATCH 06/12] drop @assignable --- tm2py/components/network/create_tod_scenarios.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tm2py/components/network/create_tod_scenarios.py b/tm2py/components/network/create_tod_scenarios.py index b2dd6387..4b718f42 100644 --- a/tm2py/components/network/create_tod_scenarios.py +++ b/tm2py/components/network/create_tod_scenarios.py @@ -641,7 +641,7 @@ def _set_capclass(network): area_type = link["@area_type"] if area_type < 0: link["@capclass"] = -1 - elif (link["@ft"] == 99) & (link["@assignable"] == 1): + elif (link["@ft"] == 99): link["@capclass"] = 10 * area_type + 7 else: link["@capclass"] = 10 * area_type + link["@ft"] From 202cb682109c058f0b454e93a3b6d0375ae80d8a Mon Sep 17 00:00:00 2001 From: Lachlan Perrier Date: Tue, 18 Jun 2024 11:39:52 -0400 Subject: [PATCH 07/12] minor_changes --- scripts/compare_skims.py | 27 ++++ scripts/compile_model_runs.py | 274 ++++++++++++++++++++++++++++++---- 2 files changed, 270 insertions(+), 31 deletions(-) diff --git a/scripts/compare_skims.py b/scripts/compare_skims.py index bef2b39c..103ae28c 100644 --- a/scripts/compare_skims.py +++ b/scripts/compare_skims.py @@ -6,6 +6,7 @@ import numpy as np network_fid_path = Path(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result") +# network_fid_path = Path(r"D:\TEMP\TM2.2.1.1-0.05") #%% @@ -17,6 +18,7 @@ def read_matrix_as_long_df(path: Path, run_name): all_skims = [] for skim_matrix_path in network_fid_path.rglob("*AM_taz.omx"): + print(skim_matrix_path) run_name = skim_matrix_path.parts[6] all_skims.append(read_matrix_as_long_df(skim_matrix_path, run_name)) @@ -25,3 +27,28 @@ def read_matrix_as_long_df(path: Path, run_name): #%%% all_skims.to_csv(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result\consolidated\skims.csv") # %% +# %% +import geopandas as gpd +from importlib import Path +import pandas as pd +#%% +output_paths_to_consolidate = Path(r"D:\TEMP\output_summaries") +all_files = [] +for file in output_paths_to_consolidate.glob("*_roadway_network.geojson"): + run_name = file.name[0:5] + print(run_name) + specific_run = gpd.read_file(file) + specific_run["run_number"] = run_name + all_files.append(specific_run) +#%% +all_files = pd.concat(all_files) +#%% +all_files.to_file(output_paths_to_consolidate / "all_runs_concat.gdb") + +#%% + +all_files.drop(columns="geometry").to_csv(output_paths_to_consolidate / "data.csv") +#%% +to_be_shape = all_files[["geometry", "model_link_id"]].drop_duplicates() +print("outputting") +to_be_shape.to_file(output_paths_to_consolidate / "geom_package") \ No newline at end of file diff --git a/scripts/compile_model_runs.py b/scripts/compile_model_runs.py index 53d7c267..d4326156 100644 --- a/scripts/compile_model_runs.py +++ b/scripts/compile_model_runs.py @@ -1,61 +1,273 @@ -print("running") +#%% import geopandas as gpd import pandas as pd -from sqlalchemy import create_engine +import numpy as np from pathlib import Path from tqdm import tqdm +from shapely.geometry import LineString -input_dir = Path(r"\\corp.pbwan.net\us\CentralData\DCCLDA00\Standard\sag\projects\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result") -output_dir = input_dir / "consolidated" +input_dir = Path(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result") +output_dir = input_dir / "consolidated_3" -in_file = next(input_dir.rglob('emme_links.shp')) -print("reading") -input = gpd.read_file(in_file) -print("writing") -input[["geometry"]].to_file(output_dir / "test_geom.geojson") -# def read_file_and_tag(path: Path) -> gpd.GeoDataFrame: +# in_file = next(input_dir.rglob('emme_links.shp')) +# print("reading", in_file) +# input2 = gpd.read_file(in_file, engine="pyogrio", use_arrow=True) +# #%% +# print("writing") +# input[["#link_id", "geometry"]].to_file(output_dir / "test_geom.geojson") + +scenarios_to_consolidate = (12, 14) +#%% + +def read_file_and_tag(path: Path, columns_to_filter = ("@ft", "VOLAU", "@capacity", "run_number", "scenario_number", "#link_id", "geometry")) -> pd.DataFrame: + + scenario = file.parent.stem + scenario_number = int(scenario.split("_")[-1]) + if scenario_number not in scenarios_to_consolidate: + return None + + run = file.parent.parent.stem + run_number = int(run.split("_")[-1]) + + return_gdf = gpd.read_file(path, engine="pyogrio") + + return_gdf["scenario"] = scenario + return_gdf["scenario_number"] = scenario_number + return_gdf["run"] = run + return_gdf["run_number"] = run_number + + if "VOLAU" not in return_gdf.columns: + print(return_gdf.columns) + print("... No VOLAU, filling with zero") + return_gdf["VOLAU" ] = 0 + + + return_gdf = return_gdf[list(columns_to_filter)] + + # assert return_gdf["#link_id"].is_unique + + return return_gdf + +def get_linestring_direction(linestring: LineString) -> str: + if not isinstance(linestring, LineString) or len(linestring.coords) < 2: + raise ValueError("Input must be a LineString with at least two coordinates") + + start_point = linestring.coords[0] + end_point = linestring.coords[-1] + + delta_x = end_point[0] - start_point[0] + delta_y = end_point[1] - start_point[1] + + if abs(delta_x) > abs(delta_y): + if delta_x > 0: + return "East" + else: + return "West" + else: + if delta_y > 0: + return "North" + else: + return "South" +#%% + +print("Reading Links...", end="") +all_links = [] +for file in tqdm(input_dir.rglob('run_*/Scenario_*/emme_links.shp')): + print(file) + all_links.append(read_file_and_tag(file)) +links_table = pd.concat(all_links) + +print("done") +#%% +scen_map = { + 11: "EA", + 12: "AM", + 13: "MD", + 14: "PM", + 15: "EV" +} + +def get_return_first_gem(row): + geom_columns = [col for col in row.index if "geometry" in col] + return [row[col] for col in geom_columns if (row[col] is not None) and (row[col] != np.NAN)][0] + +def combine_tables(dfs, columns_same): + + return_frame = dfs[0][columns_same] + + for df in dfs: + run_number = df["run_number"].iloc[0] + + scen_number = df["scenario_number"].iloc[0] + scen_number = scen_map[scen_number] + df["saturation"] = df["VOLAU"] / df["@capacity"] + + df = df[["#link_id", "@capacity", "VOLAU", "geometry", "@ft"]].rename(columns = { + "@capacity": f"capacity_run{run_number}_scen{scen_number}", + "VOLAU": f"@volau_run{run_number}_scen{scen_number}", + "saturation": f"@saturation_run{run_number}_scen{scen_number}", + "geometry": f"geometry_run{run_number}_scen{scen_number}", + "@ft": f"ft_run{run_number}_scen{scen_number}" + } + ) + # if there are link_ids that are not in the right frame + return_frame = return_frame.merge(df, how="outer", on="#link_id", validate="1:1") + geometry = return_frame.apply(get_return_first_gem, axis=1) + # remove geometries that are not main geometry + return_frame = return_frame.drop(columns=[col for col in return_frame.columns if "geometry_" in col]) + return_frame["geometry"] = geometry + + return return_frame +all_links_no_none = [links for links in all_links if (links is not None) and (links["#link_id"].is_unique)] +links_wide_table = combine_tables(all_links_no_none, ["#link_id", "geometry"]) + +links_wide_table["direction"] = links_wide_table["geometry"].apply(get_linestring_direction) +#%% +ft_cols = [col for col in links_wide_table.columns if "ft_" in col] + +links_wide_table["ft"] = links_wide_table[ft_cols].max(axis=1) +links_wide_table = links_wide_table.drop(columns=ft_cols) + +#%% +links_wide_table.to_file( + Path(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\output_summaries\all_links_data") + / "all_data_wide.geojson") + + +#%% +import matplotlib.pyplot as plt +data = [links_wide_table[col] for col in links_wide_table.iloc[:, 2:].columns] + +fig = plt.boxplot(links_wide_table, y="total_bill") +fig.show() + +# -------------------------------------------------------------------------- +#%% +links_table["direction"] = links_table["geometry"].apply(get_linestring_direction) +# %% +links_table.to_file(output_dir / "all_data.geojson", index=False) +#%% +def get_link_counts(df: pd.DataFrame): + ret_val = df.value_counts("@ft").sort_index().to_frame().T + total = ret_val.sum(axis=1) + total_minus_8 = total - ret_val[8.0].iloc[0] + ret_val["total"] = total + ret_val["total_minus_8"] = total_minus_8 + + ret_val["run_number"] = df["run_number"].iloc[0] + ret_val["scenario_number"] = df["scenario_number"].iloc[0] + return ret_val + +pd.concat( + [get_link_counts(df) for df in all_links] +).sort_values(by=["run_number", "scenario_number"]) +# #%% +# import geopandas as gpd +# import pandas as pd +# from pathlib import Path +# from tqdm import tqdm + +# input_dir = Path(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result") +# output_dir = input_dir / "consolidated" + + +# # in_file = next(input_dir.rglob('emme_links.shp')) +# # print("reading", in_file) +# # input2 = gpd.read_file(in_file, engine="pyogrio", use_arrow=True) +# # #%% +# # print("writing") +# # input[["#link_id", "geometry"]].to_file(output_dir / "test_geom.geojson") + +# #%% + +# def process_frame(return_gdf: pd.DataFrame, path: Path, columns_to_filter = ("@ft", "VOLAU", "@capacity", "run_number", "scenario_number", "#link_id")) -> pd.DataFrame: + # scenario = file.parent.stem # scenario_number = int(scenario.split("_")[-1]) # run = file.parent.parent.stem # run_number = int(run.split("_")[-1]) -# return_gdf = gpd.read_file(path) +# # return_gdf = #gpd.read_file(path, engine="pyogrio") # return_gdf["scenario"] = scenario # return_gdf["scenario_number"] = scenario_number # return_gdf["run"] = run # return_gdf["run_number"] = run_number -# return return_gdf +# return_gdf = return_gdf[list(columns_to_filter)] +# # assert return_gdf["#link_id"].is_unique - +# return return_gdf +# #%% +# geom_file = next(input_dir.rglob('run_*/Scenario*/emme_links.shp')) +# geometry = gpd.read_file(geom_file, engine="pyogrio") +# #%% +# # geometry[["#link_id", "geometry"]].to_file(output_dir / "test_geom.geojson") # print("Reading Links...", end="") -# all_links = [] +# all_links = {} # x = 0 -# for file in tqdm(input_dir.rglob('emme_links.shp')): -# all_links.append(read_file_and_tag(file)) -# if x == 0: -# all_links[-1][["geometry"]].to_file(output_dir / "test_min_geom.geojson") -# x = x + 1 -# links_table = pd.concat(all_links) -# links_table = gpd.GeoDataFrame(links_table, geometry="geometry", crs=all_links[0].crs) -# print("done") +# for file in tqdm(input_dir.rglob('run_*/Scenario*/emme_links.shp')): +# print(file) +# all_links[file] = gpd.read_file(file) + -# print("reading Nodes...", end="") -# all_nodes = [] -# for file in tqdm(input_dir.rglob('emme_nodes.shp')): -# all_nodes.append(read_file_and_tag(file)) +# #%% +# processed_values = {path: process_frame(df, path) for path, df in all_links.items()} -# nodes_table = pd.concat(all_nodes) -# nodes_table = gpd.GeoDataFrame(nodes_table, geometry="geometry", crs=all_links[0].crs) -# print("done") +# temp_iter = iter(processed_values) +# wide_data = +# for path, frame in process_name.values(): -# print("outputting files...") + +# #%% +# links_iter = iter(all_links) +# for frame in +# # %% +# links_table = pd.concat(all_links) +# #%% +# links_table.to_file("links_attr.csv", index=False) +# # #%% +# # print("reading Nodes...", end="") +# # all_nodes = [] +# # for file in tqdm(input_dir.rglob('emme_nodes.shp')): +# # all_nodes.append(read_file_and_tag(file)) + +# # nodes_table = pd.concat(all_nodes) +# # nodes_table = gpd.GeoDataFrame(nodes_table, geometry="geometry", crs=all_links[0].crs) +# # print("done") + +# # print("outputting files...") # links_table.to_file(output_dir/"links.geojson") -# nodes_table.to_file(output_dir/"nodes.geojson") \ No newline at end of file +# nodes_table.to_file(output_dir/"nodes.geojson") + + +# #%% +# from pathlib import Path +# import os +# import geopandas as gpd +# import pandas as pd +# #%% +# output_paths_to_consolidate = Path(r"D:\TEMP\output_summaries\ss") +# all_files = [] +# for file in output_paths_to_consolidate.glob("*_roadway_network.geojson"): +# run_name = file.name[0:5] +# print(run_name) +# specific_run = gpd.read_file(file) +# specific_run["run_number"] = run_name +# all_files.append(specific_run) +# #%% +# all_files = pd.concat(all_files) +# #%% + +# all_files.drop(columns="geometry").to_csv(output_paths_to_consolidate / "data.csv") +# #%% +# to_be_shape = all_files[["geometry", "model_link_id"]].drop_duplicates() +# print("outputting") +# to_be_shape.to_file(output_paths_to_consolidate / "geom_package") +# # %% From 7d4aa90e39cdeec65e282fe19fc11893f7609a83 Mon Sep 17 00:00:00 2001 From: Lachlan Perrier Date: Thu, 20 Jun 2024 10:21:24 -0400 Subject: [PATCH 08/12] minor changes to scripting,. just looking at am convergence (look at highway_assign.py line 139) --- scripts/compare_skims.py | 4 +- scripts/compile_model_runs.py | 151 +++++------------- .../network/highway/highway_assign.py | 2 +- 3 files changed, 44 insertions(+), 113 deletions(-) diff --git a/scripts/compare_skims.py b/scripts/compare_skims.py index 103ae28c..079d1edb 100644 --- a/scripts/compare_skims.py +++ b/scripts/compare_skims.py @@ -16,6 +16,8 @@ def read_matrix_as_long_df(path: Path, run_name): index_lables = list(range(am_time.shape[0])) return pd.DataFrame(am_time, index=index_lables, columns=index_lables).stack().rename(run_name).to_frame() +a = read_matrix_as_long_df(r"D:\TEMP\TM2.2.1.1-New_network_rerun\TM2.2.1.1_new_taz\skim_matrices\highway\HWYSKMAM_taz.omx", "test") +#%% all_skims = [] for skim_matrix_path in network_fid_path.rglob("*AM_taz.omx"): print(skim_matrix_path) @@ -25,7 +27,7 @@ def read_matrix_as_long_df(path: Path, run_name): all_skims = pd.concat(all_skims, axis=1) # %% #%%% -all_skims.to_csv(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result\consolidated\skims.csv") +all_skims.to_csv(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\output_summaries\skim_data\skims.csv") # %% # %% import geopandas as gpd diff --git a/scripts/compile_model_runs.py b/scripts/compile_model_runs.py index d4326156..963f29b2 100644 --- a/scripts/compile_model_runs.py +++ b/scripts/compile_model_runs.py @@ -18,7 +18,8 @@ # print("writing") # input[["#link_id", "geometry"]].to_file(output_dir / "test_geom.geojson") -scenarios_to_consolidate = (12, 14) +scenarios_to_consolidate = (11, 12, 13, 14, 15) +runs_to_consolidate = (3, 4) #%% def read_file_and_tag(path: Path, columns_to_filter = ("@ft", "VOLAU", "@capacity", "run_number", "scenario_number", "#link_id", "geometry")) -> pd.DataFrame: @@ -30,6 +31,8 @@ def read_file_and_tag(path: Path, columns_to_filter = ("@ft", "VOLAU", "@capacit run = file.parent.parent.stem run_number = int(run.split("_")[-1]) + if run_number not in runs_to_consolidate: + return None return_gdf = gpd.read_file(path, engine="pyogrio") @@ -136,11 +139,45 @@ def combine_tables(dfs, columns_same): / "all_data_wide.geojson") +#%% +num_iter = { + (3,11): 3, + (3,12): 10, + (3,13): 10, + (3,14): 19, + (3,15): 4, + (4,12): 20 +} +#%% +all_links_no_none = [links for links in all_links if (links is not None)] #and (links["#link_id"].is_unique)] +for df in all_links_no_none: + df["saturation"] = df["VOLAU"] / df["@capacity"] +ft6_sat = [(link["run_number"].iloc[0], link["scenario_number"].iloc[0], (link.loc[link["@ft"] == 6, "saturation"] > 1).mean()) for link in all_links_no_none] + +y = [val for val in num_iter.values()] +x = [x[-1] for x in ft6_sat] +col = [val[0] for val in num_iter.keys()] + +#%% +import matplotlib.pyplot as plt +plt.scatter(x, y, c=col) + +# Calculate the trendline +z = np.polyfit(x, y, 1) +p = np.poly1d(z) + +# Plot the trendline +plt.plot(x, p(x), color='red') + +plt.xlabel('proportion of ft 6 with saturation > 1') +plt.ylabel('number of iterations to solve') +plt.title('Number of iterations to solve (relative gap = 0.05)') +plt.show() #%% import matplotlib.pyplot as plt data = [links_wide_table[col] for col in links_wide_table.iloc[:, 2:].columns] -fig = plt.boxplot(links_wide_table, y="total_bill") +fig = plt.boxplot(data) fig.show() # -------------------------------------------------------------------------- @@ -162,112 +199,4 @@ def get_link_counts(df: pd.DataFrame): pd.concat( [get_link_counts(df) for df in all_links] -).sort_values(by=["run_number", "scenario_number"]) -# #%% -# import geopandas as gpd -# import pandas as pd -# from pathlib import Path -# from tqdm import tqdm - -# input_dir = Path(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result") -# output_dir = input_dir / "consolidated" - - -# # in_file = next(input_dir.rglob('emme_links.shp')) -# # print("reading", in_file) -# # input2 = gpd.read_file(in_file, engine="pyogrio", use_arrow=True) -# # #%% -# # print("writing") -# # input[["#link_id", "geometry"]].to_file(output_dir / "test_geom.geojson") - -# #%% - -# def process_frame(return_gdf: pd.DataFrame, path: Path, columns_to_filter = ("@ft", "VOLAU", "@capacity", "run_number", "scenario_number", "#link_id")) -> pd.DataFrame: - -# scenario = file.parent.stem -# scenario_number = int(scenario.split("_")[-1]) - -# run = file.parent.parent.stem -# run_number = int(run.split("_")[-1]) - -# # return_gdf = #gpd.read_file(path, engine="pyogrio") - -# return_gdf["scenario"] = scenario -# return_gdf["scenario_number"] = scenario_number -# return_gdf["run"] = run -# return_gdf["run_number"] = run_number - -# return_gdf = return_gdf[list(columns_to_filter)] - -# # assert return_gdf["#link_id"].is_unique - -# return return_gdf -# #%% -# geom_file = next(input_dir.rglob('run_*/Scenario*/emme_links.shp')) -# geometry = gpd.read_file(geom_file, engine="pyogrio") -# #%% -# # geometry[["#link_id", "geometry"]].to_file(output_dir / "test_geom.geojson") - -# print("Reading Links...", end="") -# all_links = {} -# x = 0 -# for file in tqdm(input_dir.rglob('run_*/Scenario*/emme_links.shp')): -# print(file) -# all_links[file] = gpd.read_file(file) - - -# #%% -# processed_values = {path: process_frame(df, path) for path, df in all_links.items()} - -# temp_iter = iter(processed_values) -# wide_data = -# for path, frame in process_name.values(): - - -# #%% -# links_iter = iter(all_links) -# for frame in -# # %% -# links_table = pd.concat(all_links) -# #%% -# links_table.to_file("links_attr.csv", index=False) -# # #%% -# # print("reading Nodes...", end="") -# # all_nodes = [] -# # for file in tqdm(input_dir.rglob('emme_nodes.shp')): -# # all_nodes.append(read_file_and_tag(file)) - -# # nodes_table = pd.concat(all_nodes) -# # nodes_table = gpd.GeoDataFrame(nodes_table, geometry="geometry", crs=all_links[0].crs) -# # print("done") - -# # print("outputting files...") - -# links_table.to_file(output_dir/"links.geojson") -# nodes_table.to_file(output_dir/"nodes.geojson") - - -# #%% -# from pathlib import Path -# import os -# import geopandas as gpd -# import pandas as pd -# #%% -# output_paths_to_consolidate = Path(r"D:\TEMP\output_summaries\ss") -# all_files = [] -# for file in output_paths_to_consolidate.glob("*_roadway_network.geojson"): -# run_name = file.name[0:5] -# print(run_name) -# specific_run = gpd.read_file(file) -# specific_run["run_number"] = run_name -# all_files.append(specific_run) -# #%% -# all_files = pd.concat(all_files) -# #%% - -# all_files.drop(columns="geometry").to_csv(output_paths_to_consolidate / "data.csv") -# #%% -# to_be_shape = all_files[["geometry", "model_link_id"]].drop_duplicates() -# print("outputting") -# to_be_shape.to_file(output_paths_to_consolidate / "geom_package") -# # %% +).sort_values(by=["run_number", "scenario_number"]) \ No newline at end of file diff --git a/tm2py/components/network/highway/highway_assign.py b/tm2py/components/network/highway/highway_assign.py index 16fe772f..0f75044f 100644 --- a/tm2py/components/network/highway/highway_assign.py +++ b/tm2py/components/network/highway/highway_assign.py @@ -136,7 +136,7 @@ def run(self): demand.run() else: self.highway_emmebank.zero_matrix - for time in self.time_period_names: + for time in ["AM"]: #self.time_period_names: scenario = self.highway_emmebank.scenario(time) with self._setup(scenario, time): iteration = self.controller.iteration From 99325468f8ac3862b0aadee7b056461a9af39ec4 Mon Sep 17 00:00:00 2001 From: Sijia Wang Date: Wed, 10 Jul 2024 10:37:19 -0400 Subject: [PATCH 09/12] revert temp change --- tm2py/components/network/highway/highway_assign.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tm2py/components/network/highway/highway_assign.py b/tm2py/components/network/highway/highway_assign.py index 1f7808d1..852175de 100644 --- a/tm2py/components/network/highway/highway_assign.py +++ b/tm2py/components/network/highway/highway_assign.py @@ -152,7 +152,7 @@ def run(self): demand.run() else: self.highway_emmebank.zero_matrix - for time in ["AM"]: #self.time_period_names: + for time in self.time_period_names: scenario = self.highway_emmebank.scenario(time) with self._setup(scenario, time): iteration = self.controller.iteration From 2f454413b27bf1183883d4d94dfdeaf9bc4bab39 Mon Sep 17 00:00:00 2001 From: Sijia Wang Date: Wed, 10 Jul 2024 10:39:17 -0400 Subject: [PATCH 10/12] Revert "separate TAZ and MAZ network" This reverts commit eb0785849aa2fe9c9d2d8237c11b59f2c6acfa87. --- tm2py/components/demand/commercial.py | 2 +- tm2py/components/demand/demand.py | 2 +- tm2py/components/demand/prepare_demand.py | 4 +- .../network/create_tod_scenarios.py | 33 +++----- .../network/highway/drive_access_skims.py | 2 +- .../network/highway/highway_assign.py | 78 +------------------ .../components/network/highway/highway_maz.py | 11 +-- .../network/highway/highway_network.py | 2 +- .../network/transit/transit_network.py | 2 +- tm2py/config.py | 6 +- tm2py/emme/manager.py | 24 ++---- 11 files changed, 32 insertions(+), 134 deletions(-) diff --git a/tm2py/components/demand/commercial.py b/tm2py/components/demand/commercial.py index 128ce623..65af8fde 100644 --- a/tm2py/components/demand/commercial.py +++ b/tm2py/components/demand/commercial.py @@ -144,7 +144,7 @@ def emmebank(self): This should really be in the controller? Or part of network.skims? """ - self._emmebank = self.controller.emme_manager.highway_taz_emmebank + self._emmebank = self.controller.emme_manager.highway_emmebank return self._emmebank @property diff --git a/tm2py/components/demand/demand.py b/tm2py/components/demand/demand.py index 86bdad0d..24da7202 100644 --- a/tm2py/components/demand/demand.py +++ b/tm2py/components/demand/demand.py @@ -84,7 +84,7 @@ def __init__(self, controller: RunController): # @LogStartEnd("prepare highway demand") def run(self): """Open combined demand OMX files from demand models and prepare for assignment.""" - self._emmebank_path = self.get_abs_path(self.config.emme.highway_taz_database_path) + self._emmebank_path = self.get_abs_path(self.config.emme.highway_database_path) self._emmebank = self.controller.emme_manager.emmebank(self._emmebank_path) self._create_zero_matrix() for time in self.time_period_names(): diff --git a/tm2py/components/demand/prepare_demand.py b/tm2py/components/demand/prepare_demand.py index c134b3bb..b6d8f66d 100644 --- a/tm2py/components/demand/prepare_demand.py +++ b/tm2py/components/demand/prepare_demand.py @@ -164,7 +164,7 @@ def validate_inputs(self): @property def highway_emmebank(self): if self._highway_emmebank == None: - self._highway_emmebank = self.controller.emme_manager.highway_taz_emmebank + self._highway_emmebank = self.controller.emme_manager.highway_emmebank self._emmebank = self._highway_emmebank return self._highway_emmebank @@ -584,7 +584,7 @@ def num_internal_zones(self): @property def num_total_zones(self): self._emmebank_path = self.controller.get_abs_path( - self.controller.config.emme.highway_taz_database_path + self.controller.config.emme.highway_database_path ) self._emmebank = self.controller.emme_manager.emmebank(self._emmebank_path) time_period = self.controller.config.time_periods[0].name diff --git a/tm2py/components/network/create_tod_scenarios.py b/tm2py/components/network/create_tod_scenarios.py index 4b718f42..7cf5a9c5 100644 --- a/tm2py/components/network/create_tod_scenarios.py +++ b/tm2py/components/network/create_tod_scenarios.py @@ -46,8 +46,7 @@ def run(self): # emme_app = self._emme_manager.project(project_path) # self._emme_manager.init_modeller(emme_app) with self._setup(): - self._create_highway_scenarios(zone_flag="taz") - self._create_highway_scenarios(zone_flag="maz") + self._create_highway_scenarios() self._create_transit_scenarios() @_context @@ -91,32 +90,20 @@ def _project_coordinates(self, ref_scenario): emme_app.project.save() @LogStartEnd("Create highway time of day scenarios.") - def _create_highway_scenarios(self, zone_flag = "taz"): - if zone_flag == "taz": - emmebank = self.controller.emme_manager.highway_taz_emmebank.emmebank - else: - emmebank = self.controller.emme_manager.highway_maz_emmebank.emmebank + def _create_highway_scenarios(self): + emmebank = self.controller.emme_manager.highway_emmebank.emmebank ref_scenario = emmebank.scenario( self.controller.config.emme.all_day_scenario_id ) self._ref_auto_network = ref_scenario.get_network() n_time_periods = len(self.controller.config.time_periods) - if zone_flag == "taz": - self.controller.emme_manager.highway_taz_emmebank.change_dimensions( - { - "scenarios": 1 + n_time_periods, - "full_matrices": 9999, - "extra_attribute_values": 60000000, - } - ) - else: - self.controller.emme_manager.highway_maz_emmebank.change_dimensions( - { - "scenarios": 1 + n_time_periods, - "full_matrices": 9999, - "extra_attribute_values": 60000000, - } - ) + self.controller.emme_manager.highway_emmebank.change_dimensions( + { + "scenarios": 1 + n_time_periods, + "full_matrices": 9999, + "extra_attribute_values": 60000000, + } + ) # create VDFs & set cross-reference function parameters emmebank.extra_function_parameters.el1 = "@free_flow_time" emmebank.extra_function_parameters.el2 = "@capacity" diff --git a/tm2py/components/network/highway/drive_access_skims.py b/tm2py/components/network/highway/drive_access_skims.py index d96bc0f4..59eb82b1 100644 --- a/tm2py/components/network/highway/drive_access_skims.py +++ b/tm2py/components/network/highway/drive_access_skims.py @@ -199,7 +199,7 @@ def process_stops(stops): def _get_drive_costs(self, period: TimePeriodConfig) -> pd.DataFrame: """Load the drive costs from OMX matrix files, return as pandas dataframe.""" - emmebank = self.controller.emme_manager.highway_taz_emmebank.emmebank + emmebank = self.controller.emme_manager.highway_emmebank.emmebank scenario = emmebank.scenario(period.emme_scenario_id) zone_numbers = scenario.zone_numbers network = self.controller.emme_manager.get_network( diff --git a/tm2py/components/network/highway/highway_assign.py b/tm2py/components/network/highway/highway_assign.py index 852175de..16fe772f 100644 --- a/tm2py/components/network/highway/highway_assign.py +++ b/tm2py/components/network/highway/highway_assign.py @@ -103,13 +103,11 @@ def __init__(self, controller: RunController): self._class_config = None self._scenario = None self._highway_emmebank = None - self._highway_maz_scenarios = None - self._highway_maz_emmebank = None @property def highway_emmebank(self): if not self._highway_emmebank: - self._highway_emmebank = self.controller.emme_manager.highway_taz_emmebank + self._highway_emmebank = self.controller.emme_manager.highway_emmebank return self._highway_emmebank @property @@ -124,20 +122,6 @@ def class_config(self): self._class_config = {c.name: c for c in self.config.classes} return self._class_config - - @property - def highway_maz_emmebank(self): - if not self._highway_maz_emmebank: - self._highway_maz_emmebank = self.controller.emme_manager.highway_maz_emmebank - return self._highway_maz_emmebank - - @property - def highway_maz_scenarios(self): - if self._highway_maz_scenarios is None: - self._highway_maz_scenarios = { - tp: self.highway_maz_emmebank.scenario(tp) for tp in self.time_period_names - } - return self._highway_maz_scenarios def validate_inputs(self): """Validate inputs files are correct, raise if an error is found.""" @@ -160,7 +144,7 @@ def run(self): AssignmentClass(c, time, iteration) for c in self.config.classes ] if iteration > 0: - self._copy_maz_flow(scenario, time) + self._copy_maz_flow(scenario) else: self._reset_background_traffic(scenario) self._create_skim_matrices(scenario, assign_classes) @@ -248,72 +232,16 @@ def _setup(self, scenario: EmmeScenario, time_period: str): self._matrix_cache.clear() self._matrix_cache = None self._skim_matrices = [] - - def _get_maz_links( - self, - time_period: str, - ): - """Create dictionary of link ids mapped to maz network. - - Args: - time_period (str): time period abbreviation - """ - _highway_maz_scenario = self.highway_maz_scenarios[time_period] - if not _highway_maz_scenario.has_traffic_results: - return {} - _highway_maz_net = _highway_maz_scenario.get_partial_network( - ["LINK"], include_attributes=False - ) - - highway_attributes = { - "LINK": ["#link_id", "@maz_flow"] - } - self.emme_manager.copy_attribute_values( - _highway_maz_scenario, _highway_maz_net, highway_attributes - ) - - # TODO can we just get the link attributes as a DataFrame and merge them? - maz_link_dict = { - maz_link["#link_id"]: maz_link for maz_link in _highway_maz_net.links() - } - - return maz_link_dict - - def _copy_maz_flow(self, scenario: EmmeScenario, time_period: str): + def _copy_maz_flow(self, scenario: EmmeScenario): """Copy maz_flow from MAZ demand assignment to ul1 for background traffic. Args: scenario: Emme scenario object - time_period (str): time period abbreviation """ - - _highway_net = scenario.get_partial_network( - ["LINK"], include_attributes=False - ) - highway_attributes = { - "LINK": ["#link_id", "@maz_flow"] - } - self.emme_manager.copy_attribute_values( - scenario, _highway_net, highway_attributes - ) - - _maz_link_dict = self._get_maz_links(time_period) - self.logger.log( "Copy @maz_flow to ul1 for background traffic", indent=True, level="DETAIL" ) - for link in _highway_net.links(): - if link["#link_id"] in _maz_link_dict.keys(): - link["@maz_flow"] = _maz_link_dict[link["#link_id"]]["@maz_flow"] - - _update_attributes = { - "LINK": ["@maz_flow"], - } - self.emme_manager.copy_attribute_values( - _highway_net, scenario, _update_attributes - ) - net_calc = NetworkCalculator(self.controller, scenario) net_calc("ul1", "@maz_flow") diff --git a/tm2py/components/network/highway/highway_maz.py b/tm2py/components/network/highway/highway_maz.py index 28180a2c..a43c0c3d 100644 --- a/tm2py/components/network/highway/highway_maz.py +++ b/tm2py/components/network/highway/highway_maz.py @@ -99,7 +99,7 @@ def __init__(self, controller: RunController): @property def highway_emmebank(self): if self._highway_emmebank is None: - self._highway_emmebank = self.controller.emme_manager.highway_maz_emmebank + self._highway_emmebank = self.controller.emme_manager.highway_emmebank return self._highway_emmebank @property @@ -122,11 +122,6 @@ def run(self): county_groups[group.number] = group.counties for time in self.time_period_names: self._scenario = self.highway_emmebank.scenario(time) - if self.controller.iteration == 0: - create_attribute = self.controller.emme_manager.tool( - "inro.emme.data.extra_attribute.create_extra_attribute" - ) - create_attribute("LINK","@maz_flow","Assigned MAZ-to-MAZ flow",overwrite=True,scenario=self._scenario) with self._setup(time): self._prepare_network() for i, names in county_groups.items(): @@ -523,7 +518,7 @@ def _assign_flow_text( ) self.controller.emme_manager.copy_attribute_values( - self._network, self._scenario, {"LINK": ["temp_flow"]}, {"LINK": ["@maz_flow"]} + self._network, self._scenario, {"LINK": ["temp_flow"]}, {"LINK": ["data1"]} ) def _load_text_format_paths( @@ -694,7 +689,7 @@ def __init__(self, controller: RunController): @property def highway_emmebank(self): if self._highway_emmebank is None: - self._highway_emmebank = self.controller.emme_manager.highway_maz_emmebank + self._highway_emmebank = self.controller.emme_manager.highway_emmebank return self._highway_emmebank @property diff --git a/tm2py/components/network/highway/highway_network.py b/tm2py/components/network/highway/highway_network.py index bf292b58..f456b604 100644 --- a/tm2py/components/network/highway/highway_network.py +++ b/tm2py/components/network/highway/highway_network.py @@ -97,7 +97,7 @@ def run(self): @property def highway_emmebank(self): if not self._highway_emmebank: - self._highway_emmebank = self.controller.emme_manager.highway_taz_emmebank + self._highway_emmebank = self.controller.emme_manager.highway_emmebank return self._highway_emmebank @property diff --git a/tm2py/components/network/transit/transit_network.py b/tm2py/components/network/transit/transit_network.py index 49148f45..edee3844 100644 --- a/tm2py/components/network/transit/transit_network.py +++ b/tm2py/components/network/transit/transit_network.py @@ -114,7 +114,7 @@ def transit_emmebank(self): @property def highway_emmebank(self): if not self._highway_emmebank: - self._highway_emmebank = self.controller.emme_manager.highway_taz_emmebank + self._highway_emmebank = self.controller.emme_manager.highway_emmebank return self._highway_emmebank @property diff --git a/tm2py/config.py b/tm2py/config.py index f7e29f99..72ab3387 100644 --- a/tm2py/config.py +++ b/tm2py/config.py @@ -1262,8 +1262,7 @@ class EmmeConfig(ConfigItem): all_day_scenario_id: scenario ID to use for all day (initial imported) scenario with all time period data project_path: relative path from run_dir to Emme desktop project (.emp) - highway_taz_database_path: relative path to TAZ highway Emmebank - highway_maz_database_path: relative path to MAZ highway Emmebank + highway_database_path: relative path to highway Emmebank active_north_database_path: relative paths to active mode Emmebank for north bay active_south_database_path: relative paths to active mode Emmebank for south bay transit_database_path: relative path to transit Emmebank @@ -1275,8 +1274,7 @@ class EmmeConfig(ConfigItem): all_day_scenario_id: int project_path: pathlib.Path - highway_taz_database_path: pathlib.Path - highway_maz_database_path: pathlib.Path + highway_database_path: pathlib.Path active_north_database_path: pathlib.Path active_south_database_path: pathlib.Path transit_database_path: pathlib.Path diff --git a/tm2py/emme/manager.py b/tm2py/emme/manager.py index 0d2ad3a8..dabc65fa 100644 --- a/tm2py/emme/manager.py +++ b/tm2py/emme/manager.py @@ -146,11 +146,8 @@ def __init__(self, controller, emme_config: "EmmeConfig"): self.project_path = self.controller.get_abs_path(self.config.project_path) # see if works without os.path.normcase(os.path.realpath(project_path)) - self.highway_taz_database_path = self.controller.get_abs_path( - self.config.highway_taz_database_path - ) - self.highway_maz_database_path = self.controller.get_abs_path( - self.config.highway_maz_database_path + self.highway_database_path = self.controller.get_abs_path( + self.config.highway_database_path ) self.transit_database_path = self.controller.get_abs_path( self.config.transit_database_path @@ -166,8 +163,7 @@ def __init__(self, controller, emme_config: "EmmeConfig"): self._project = None self._modeller = None - self._highway_taz_emmebank = None - self._highway_maz_emmebank = None + self._highway_emmebank = None self._transit_emmebank = None self._active_north_emmebank = None self._active_south_emmebank = None @@ -225,16 +221,10 @@ def modeller(self) -> EmmeModeller: return self._modeller @property - def highway_taz_emmebank(self) -> EmmeBank: - if self._highway_taz_emmebank is None: - self._highway_taz_emmebank = EmmeBank(self, self.highway_taz_database_path) - return self._highway_taz_emmebank - - @property - def highway_maz_emmebank(self) -> EmmeBank: - if self._highway_maz_emmebank is None: - self._highway_maz_emmebank = EmmeBank(self, self.highway_maz_database_path) - return self._highway_maz_emmebank + def highway_emmebank(self) -> EmmeBank: + if self._highway_emmebank is None: + self._highway_emmebank = EmmeBank(self, self.highway_database_path) + return self._highway_emmebank @property def transit_emmebank(self) -> EmmeBank: From d088122d52ef401c79bfec288d589ebae95325ce Mon Sep 17 00:00:00 2001 From: Sijia Wang Date: Wed, 10 Jul 2024 10:46:20 -0400 Subject: [PATCH 11/12] blacken --- scripts/compare_skims.py | 49 ++++-- scripts/compile_model_runs.py | 158 +++++++++++------- .../network/create_tod_scenarios.py | 2 +- 3 files changed, 130 insertions(+), 79 deletions(-) diff --git a/scripts/compare_skims.py b/scripts/compare_skims.py index 079d1edb..82c95ca6 100644 --- a/scripts/compare_skims.py +++ b/scripts/compare_skims.py @@ -1,23 +1,35 @@ -#%% +# %% import pandas as pd import openmatrix as omx from pathlib import Path import numpy as np -network_fid_path = Path(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result") +network_fid_path = Path( + r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result" +) # network_fid_path = Path(r"D:\TEMP\TM2.2.1.1-0.05") -#%% +# %% + def read_matrix_as_long_df(path: Path, run_name): run = omx.open_file(path, "r") am_time = np.array(run["AM_da_time"]) index_lables = list(range(am_time.shape[0])) - return pd.DataFrame(am_time, index=index_lables, columns=index_lables).stack().rename(run_name).to_frame() + return ( + pd.DataFrame(am_time, index=index_lables, columns=index_lables) + .stack() + .rename(run_name) + .to_frame() + ) + -a = read_matrix_as_long_df(r"D:\TEMP\TM2.2.1.1-New_network_rerun\TM2.2.1.1_new_taz\skim_matrices\highway\HWYSKMAM_taz.omx", "test") -#%% +a = read_matrix_as_long_df( + r"D:\TEMP\TM2.2.1.1-New_network_rerun\TM2.2.1.1_new_taz\skim_matrices\highway\HWYSKMAM_taz.omx", + "test", +) +# %% all_skims = [] for skim_matrix_path in network_fid_path.rglob("*AM_taz.omx"): print(skim_matrix_path) @@ -26,14 +38,17 @@ def read_matrix_as_long_df(path: Path, run_name): all_skims = pd.concat(all_skims, axis=1) # %% -#%%% -all_skims.to_csv(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\output_summaries\skim_data\skims.csv") +# %%% +all_skims.to_csv( + r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\output_summaries\skim_data\skims.csv" +) # %% # %% import geopandas as gpd from importlib import Path -import pandas as pd -#%% +import pandas as pd + +# %% output_paths_to_consolidate = Path(r"D:\TEMP\output_summaries") all_files = [] for file in output_paths_to_consolidate.glob("*_roadway_network.geojson"): @@ -42,15 +57,15 @@ def read_matrix_as_long_df(path: Path, run_name): specific_run = gpd.read_file(file) specific_run["run_number"] = run_name all_files.append(specific_run) -#%% +# %% all_files = pd.concat(all_files) -#%% +# %% all_files.to_file(output_paths_to_consolidate / "all_runs_concat.gdb") - -#%% - + +# %% + all_files.drop(columns="geometry").to_csv(output_paths_to_consolidate / "data.csv") -#%% +# %% to_be_shape = all_files[["geometry", "model_link_id"]].drop_duplicates() print("outputting") -to_be_shape.to_file(output_paths_to_consolidate / "geom_package") \ No newline at end of file +to_be_shape.to_file(output_paths_to_consolidate / "geom_package") diff --git a/scripts/compile_model_runs.py b/scripts/compile_model_runs.py index 963f29b2..553a3656 100644 --- a/scripts/compile_model_runs.py +++ b/scripts/compile_model_runs.py @@ -1,4 +1,4 @@ -#%% +# %% import geopandas as gpd import pandas as pd import numpy as np @@ -6,11 +6,12 @@ from tqdm import tqdm from shapely.geometry import LineString -input_dir = Path(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result") +input_dir = Path( + r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result" +) output_dir = input_dir / "consolidated_3" - # in_file = next(input_dir.rglob('emme_links.shp')) # print("reading", in_file) # input2 = gpd.read_file(in_file, engine="pyogrio", use_arrow=True) @@ -20,10 +21,21 @@ scenarios_to_consolidate = (11, 12, 13, 14, 15) runs_to_consolidate = (3, 4) -#%% +# %% -def read_file_and_tag(path: Path, columns_to_filter = ("@ft", "VOLAU", "@capacity", "run_number", "scenario_number", "#link_id", "geometry")) -> pd.DataFrame: - + +def read_file_and_tag( + path: Path, + columns_to_filter=( + "@ft", + "VOLAU", + "@capacity", + "run_number", + "scenario_number", + "#link_id", + "geometry", + ), +) -> pd.DataFrame: scenario = file.parent.stem scenario_number = int(scenario.split("_")[-1]) if scenario_number not in scenarios_to_consolidate: @@ -40,12 +52,11 @@ def read_file_and_tag(path: Path, columns_to_filter = ("@ft", "VOLAU", "@capacit return_gdf["scenario_number"] = scenario_number return_gdf["run"] = run return_gdf["run_number"] = run_number - + if "VOLAU" not in return_gdf.columns: print(return_gdf.columns) print("... No VOLAU, filling with zero") - return_gdf["VOLAU" ] = 0 - + return_gdf["VOLAU"] = 0 return_gdf = return_gdf[list(columns_to_filter)] @@ -53,6 +64,7 @@ def read_file_and_tag(path: Path, columns_to_filter = ("@ft", "VOLAU", "@capacit return return_gdf + def get_linestring_direction(linestring: LineString) -> str: if not isinstance(linestring, LineString) or len(linestring.coords) < 2: raise ValueError("Input must be a LineString with at least two coordinates") @@ -73,31 +85,32 @@ def get_linestring_direction(linestring: LineString) -> str: return "North" else: return "South" -#%% + + +# %% print("Reading Links...", end="") all_links = [] -for file in tqdm(input_dir.rglob('run_*/Scenario_*/emme_links.shp')): +for file in tqdm(input_dir.rglob("run_*/Scenario_*/emme_links.shp")): print(file) all_links.append(read_file_and_tag(file)) links_table = pd.concat(all_links) print("done") -#%% -scen_map = { - 11: "EA", - 12: "AM", - 13: "MD", - 14: "PM", - 15: "EV" -} +# %% +scen_map = {11: "EA", 12: "AM", 13: "MD", 14: "PM", 15: "EV"} + def get_return_first_gem(row): geom_columns = [col for col in row.index if "geometry" in col] - return [row[col] for col in geom_columns if (row[col] is not None) and (row[col] != np.NAN)][0] + return [ + row[col] + for col in geom_columns + if (row[col] is not None) and (row[col] != np.NAN) + ][0] + def combine_tables(dfs, columns_same): - return_frame = dfs[0][columns_same] for df in dfs: @@ -106,60 +119,79 @@ def combine_tables(dfs, columns_same): scen_number = df["scenario_number"].iloc[0] scen_number = scen_map[scen_number] df["saturation"] = df["VOLAU"] / df["@capacity"] - - df = df[["#link_id", "@capacity", "VOLAU", "geometry", "@ft"]].rename(columns = { - "@capacity": f"capacity_run{run_number}_scen{scen_number}", - "VOLAU": f"@volau_run{run_number}_scen{scen_number}", - "saturation": f"@saturation_run{run_number}_scen{scen_number}", - "geometry": f"geometry_run{run_number}_scen{scen_number}", - "@ft": f"ft_run{run_number}_scen{scen_number}" + + df = df[["#link_id", "@capacity", "VOLAU", "geometry", "@ft"]].rename( + columns={ + "@capacity": f"capacity_run{run_number}_scen{scen_number}", + "VOLAU": f"@volau_run{run_number}_scen{scen_number}", + "saturation": f"@saturation_run{run_number}_scen{scen_number}", + "geometry": f"geometry_run{run_number}_scen{scen_number}", + "@ft": f"ft_run{run_number}_scen{scen_number}", } ) # if there are link_ids that are not in the right frame - return_frame = return_frame.merge(df, how="outer", on="#link_id", validate="1:1") + return_frame = return_frame.merge( + df, how="outer", on="#link_id", validate="1:1" + ) geometry = return_frame.apply(get_return_first_gem, axis=1) # remove geometries that are not main geometry - return_frame = return_frame.drop(columns=[col for col in return_frame.columns if "geometry_" in col]) + return_frame = return_frame.drop( + columns=[col for col in return_frame.columns if "geometry_" in col] + ) return_frame["geometry"] = geometry return return_frame -all_links_no_none = [links for links in all_links if (links is not None) and (links["#link_id"].is_unique)] + + +all_links_no_none = [ + links + for links in all_links + if (links is not None) and (links["#link_id"].is_unique) +] links_wide_table = combine_tables(all_links_no_none, ["#link_id", "geometry"]) -links_wide_table["direction"] = links_wide_table["geometry"].apply(get_linestring_direction) -#%% +links_wide_table["direction"] = links_wide_table["geometry"].apply( + get_linestring_direction +) +# %% ft_cols = [col for col in links_wide_table.columns if "ft_" in col] links_wide_table["ft"] = links_wide_table[ft_cols].max(axis=1) links_wide_table = links_wide_table.drop(columns=ft_cols) -#%% +# %% links_wide_table.to_file( - Path(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\output_summaries\all_links_data") - / "all_data_wide.geojson") - - -#%% -num_iter = { - (3,11): 3, - (3,12): 10, - (3,13): 10, - (3,14): 19, - (3,15): 4, - (4,12): 20 -} -#%% -all_links_no_none = [links for links in all_links if (links is not None)] #and (links["#link_id"].is_unique)] + Path( + r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\output_summaries\all_links_data" + ) + / "all_data_wide.geojson" +) + + +# %% +num_iter = {(3, 11): 3, (3, 12): 10, (3, 13): 10, (3, 14): 19, (3, 15): 4, (4, 12): 20} +# %% +all_links_no_none = [ + links for links in all_links if (links is not None) +] # and (links["#link_id"].is_unique)] for df in all_links_no_none: df["saturation"] = df["VOLAU"] / df["@capacity"] -ft6_sat = [(link["run_number"].iloc[0], link["scenario_number"].iloc[0], (link.loc[link["@ft"] == 6, "saturation"] > 1).mean()) for link in all_links_no_none] +ft6_sat = [ + ( + link["run_number"].iloc[0], + link["scenario_number"].iloc[0], + (link.loc[link["@ft"] == 6, "saturation"] > 1).mean(), + ) + for link in all_links_no_none +] y = [val for val in num_iter.values()] x = [x[-1] for x in ft6_sat] col = [val[0] for val in num_iter.keys()] -#%% +# %% import matplotlib.pyplot as plt + plt.scatter(x, y, c=col) # Calculate the trendline @@ -167,25 +199,28 @@ def combine_tables(dfs, columns_same): p = np.poly1d(z) # Plot the trendline -plt.plot(x, p(x), color='red') +plt.plot(x, p(x), color="red") -plt.xlabel('proportion of ft 6 with saturation > 1') -plt.ylabel('number of iterations to solve') -plt.title('Number of iterations to solve (relative gap = 0.05)') +plt.xlabel("proportion of ft 6 with saturation > 1") +plt.ylabel("number of iterations to solve") +plt.title("Number of iterations to solve (relative gap = 0.05)") plt.show() -#%% +# %% import matplotlib.pyplot as plt + data = [links_wide_table[col] for col in links_wide_table.iloc[:, 2:].columns] fig = plt.boxplot(data) fig.show() # -------------------------------------------------------------------------- -#%% +# %% links_table["direction"] = links_table["geometry"].apply(get_linestring_direction) # %% links_table.to_file(output_dir / "all_data.geojson", index=False) -#%% + + +# %% def get_link_counts(df: pd.DataFrame): ret_val = df.value_counts("@ft").sort_index().to_frame().T total = ret_val.sum(axis=1) @@ -197,6 +232,7 @@ def get_link_counts(df: pd.DataFrame): ret_val["scenario_number"] = df["scenario_number"].iloc[0] return ret_val -pd.concat( - [get_link_counts(df) for df in all_links] -).sort_values(by=["run_number", "scenario_number"]) \ No newline at end of file + +pd.concat([get_link_counts(df) for df in all_links]).sort_values( + by=["run_number", "scenario_number"] +) diff --git a/tm2py/components/network/create_tod_scenarios.py b/tm2py/components/network/create_tod_scenarios.py index 7cf5a9c5..19d5f330 100644 --- a/tm2py/components/network/create_tod_scenarios.py +++ b/tm2py/components/network/create_tod_scenarios.py @@ -628,7 +628,7 @@ def _set_capclass(network): area_type = link["@area_type"] if area_type < 0: link["@capclass"] = -1 - elif (link["@ft"] == 99): + elif link["@ft"] == 99: link["@capclass"] = 10 * area_type + 7 else: link["@capclass"] = 10 * area_type + link["@ft"] From 0a0b63252b94760344d0897783c3dad355a11cf4 Mon Sep 17 00:00:00 2001 From: Sijia Wang Date: Tue, 30 Jul 2024 11:11:35 -0400 Subject: [PATCH 12/12] increase extra attribute values --- tm2py/components/network/create_tod_scenarios.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tm2py/components/network/create_tod_scenarios.py b/tm2py/components/network/create_tod_scenarios.py index 19d5f330..1df6284e 100644 --- a/tm2py/components/network/create_tod_scenarios.py +++ b/tm2py/components/network/create_tod_scenarios.py @@ -101,7 +101,7 @@ def _create_highway_scenarios(self): { "scenarios": 1 + n_time_periods, "full_matrices": 9999, - "extra_attribute_values": 60000000, + "extra_attribute_values": 100000000, } ) # create VDFs & set cross-reference function parameters