From d088122d52ef401c79bfec288d589ebae95325ce Mon Sep 17 00:00:00 2001 From: Sijia Wang Date: Wed, 10 Jul 2024 10:46:20 -0400 Subject: [PATCH] blacken --- scripts/compare_skims.py | 49 ++++-- scripts/compile_model_runs.py | 158 +++++++++++------- .../network/create_tod_scenarios.py | 2 +- 3 files changed, 130 insertions(+), 79 deletions(-) diff --git a/scripts/compare_skims.py b/scripts/compare_skims.py index 079d1edb..82c95ca6 100644 --- a/scripts/compare_skims.py +++ b/scripts/compare_skims.py @@ -1,23 +1,35 @@ -#%% +# %% import pandas as pd import openmatrix as omx from pathlib import Path import numpy as np -network_fid_path = Path(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result") +network_fid_path = Path( + r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result" +) # network_fid_path = Path(r"D:\TEMP\TM2.2.1.1-0.05") -#%% +# %% + def read_matrix_as_long_df(path: Path, run_name): run = omx.open_file(path, "r") am_time = np.array(run["AM_da_time"]) index_lables = list(range(am_time.shape[0])) - return pd.DataFrame(am_time, index=index_lables, columns=index_lables).stack().rename(run_name).to_frame() + return ( + pd.DataFrame(am_time, index=index_lables, columns=index_lables) + .stack() + .rename(run_name) + .to_frame() + ) + -a = read_matrix_as_long_df(r"D:\TEMP\TM2.2.1.1-New_network_rerun\TM2.2.1.1_new_taz\skim_matrices\highway\HWYSKMAM_taz.omx", "test") -#%% +a = read_matrix_as_long_df( + r"D:\TEMP\TM2.2.1.1-New_network_rerun\TM2.2.1.1_new_taz\skim_matrices\highway\HWYSKMAM_taz.omx", + "test", +) +# %% all_skims = [] for skim_matrix_path in network_fid_path.rglob("*AM_taz.omx"): print(skim_matrix_path) @@ -26,14 +38,17 @@ def read_matrix_as_long_df(path: Path, run_name): all_skims = pd.concat(all_skims, axis=1) # %% -#%%% -all_skims.to_csv(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\output_summaries\skim_data\skims.csv") +# %%% +all_skims.to_csv( + r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\output_summaries\skim_data\skims.csv" +) # %% # %% import geopandas as gpd from importlib import Path -import pandas as pd -#%% +import pandas as pd + +# %% output_paths_to_consolidate = Path(r"D:\TEMP\output_summaries") all_files = [] for file in output_paths_to_consolidate.glob("*_roadway_network.geojson"): @@ -42,15 +57,15 @@ def read_matrix_as_long_df(path: Path, run_name): specific_run = gpd.read_file(file) specific_run["run_number"] = run_name all_files.append(specific_run) -#%% +# %% all_files = pd.concat(all_files) -#%% +# %% all_files.to_file(output_paths_to_consolidate / "all_runs_concat.gdb") - -#%% - + +# %% + all_files.drop(columns="geometry").to_csv(output_paths_to_consolidate / "data.csv") -#%% +# %% to_be_shape = all_files[["geometry", "model_link_id"]].drop_duplicates() print("outputting") -to_be_shape.to_file(output_paths_to_consolidate / "geom_package") \ No newline at end of file +to_be_shape.to_file(output_paths_to_consolidate / "geom_package") diff --git a/scripts/compile_model_runs.py b/scripts/compile_model_runs.py index 963f29b2..553a3656 100644 --- a/scripts/compile_model_runs.py +++ b/scripts/compile_model_runs.py @@ -1,4 +1,4 @@ -#%% +# %% import geopandas as gpd import pandas as pd import numpy as np @@ -6,11 +6,12 @@ from tqdm import tqdm from shapely.geometry import LineString -input_dir = Path(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result") +input_dir = Path( + r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\run_result" +) output_dir = input_dir / "consolidated_3" - # in_file = next(input_dir.rglob('emme_links.shp')) # print("reading", in_file) # input2 = gpd.read_file(in_file, engine="pyogrio", use_arrow=True) @@ -20,10 +21,21 @@ scenarios_to_consolidate = (11, 12, 13, 14, 15) runs_to_consolidate = (3, 4) -#%% +# %% -def read_file_and_tag(path: Path, columns_to_filter = ("@ft", "VOLAU", "@capacity", "run_number", "scenario_number", "#link_id", "geometry")) -> pd.DataFrame: - + +def read_file_and_tag( + path: Path, + columns_to_filter=( + "@ft", + "VOLAU", + "@capacity", + "run_number", + "scenario_number", + "#link_id", + "geometry", + ), +) -> pd.DataFrame: scenario = file.parent.stem scenario_number = int(scenario.split("_")[-1]) if scenario_number not in scenarios_to_consolidate: @@ -40,12 +52,11 @@ def read_file_and_tag(path: Path, columns_to_filter = ("@ft", "VOLAU", "@capacit return_gdf["scenario_number"] = scenario_number return_gdf["run"] = run return_gdf["run_number"] = run_number - + if "VOLAU" not in return_gdf.columns: print(return_gdf.columns) print("... No VOLAU, filling with zero") - return_gdf["VOLAU" ] = 0 - + return_gdf["VOLAU"] = 0 return_gdf = return_gdf[list(columns_to_filter)] @@ -53,6 +64,7 @@ def read_file_and_tag(path: Path, columns_to_filter = ("@ft", "VOLAU", "@capacit return return_gdf + def get_linestring_direction(linestring: LineString) -> str: if not isinstance(linestring, LineString) or len(linestring.coords) < 2: raise ValueError("Input must be a LineString with at least two coordinates") @@ -73,31 +85,32 @@ def get_linestring_direction(linestring: LineString) -> str: return "North" else: return "South" -#%% + + +# %% print("Reading Links...", end="") all_links = [] -for file in tqdm(input_dir.rglob('run_*/Scenario_*/emme_links.shp')): +for file in tqdm(input_dir.rglob("run_*/Scenario_*/emme_links.shp")): print(file) all_links.append(read_file_and_tag(file)) links_table = pd.concat(all_links) print("done") -#%% -scen_map = { - 11: "EA", - 12: "AM", - 13: "MD", - 14: "PM", - 15: "EV" -} +# %% +scen_map = {11: "EA", 12: "AM", 13: "MD", 14: "PM", 15: "EV"} + def get_return_first_gem(row): geom_columns = [col for col in row.index if "geometry" in col] - return [row[col] for col in geom_columns if (row[col] is not None) and (row[col] != np.NAN)][0] + return [ + row[col] + for col in geom_columns + if (row[col] is not None) and (row[col] != np.NAN) + ][0] + def combine_tables(dfs, columns_same): - return_frame = dfs[0][columns_same] for df in dfs: @@ -106,60 +119,79 @@ def combine_tables(dfs, columns_same): scen_number = df["scenario_number"].iloc[0] scen_number = scen_map[scen_number] df["saturation"] = df["VOLAU"] / df["@capacity"] - - df = df[["#link_id", "@capacity", "VOLAU", "geometry", "@ft"]].rename(columns = { - "@capacity": f"capacity_run{run_number}_scen{scen_number}", - "VOLAU": f"@volau_run{run_number}_scen{scen_number}", - "saturation": f"@saturation_run{run_number}_scen{scen_number}", - "geometry": f"geometry_run{run_number}_scen{scen_number}", - "@ft": f"ft_run{run_number}_scen{scen_number}" + + df = df[["#link_id", "@capacity", "VOLAU", "geometry", "@ft"]].rename( + columns={ + "@capacity": f"capacity_run{run_number}_scen{scen_number}", + "VOLAU": f"@volau_run{run_number}_scen{scen_number}", + "saturation": f"@saturation_run{run_number}_scen{scen_number}", + "geometry": f"geometry_run{run_number}_scen{scen_number}", + "@ft": f"ft_run{run_number}_scen{scen_number}", } ) # if there are link_ids that are not in the right frame - return_frame = return_frame.merge(df, how="outer", on="#link_id", validate="1:1") + return_frame = return_frame.merge( + df, how="outer", on="#link_id", validate="1:1" + ) geometry = return_frame.apply(get_return_first_gem, axis=1) # remove geometries that are not main geometry - return_frame = return_frame.drop(columns=[col for col in return_frame.columns if "geometry_" in col]) + return_frame = return_frame.drop( + columns=[col for col in return_frame.columns if "geometry_" in col] + ) return_frame["geometry"] = geometry return return_frame -all_links_no_none = [links for links in all_links if (links is not None) and (links["#link_id"].is_unique)] + + +all_links_no_none = [ + links + for links in all_links + if (links is not None) and (links["#link_id"].is_unique) +] links_wide_table = combine_tables(all_links_no_none, ["#link_id", "geometry"]) -links_wide_table["direction"] = links_wide_table["geometry"].apply(get_linestring_direction) -#%% +links_wide_table["direction"] = links_wide_table["geometry"].apply( + get_linestring_direction +) +# %% ft_cols = [col for col in links_wide_table.columns if "ft_" in col] links_wide_table["ft"] = links_wide_table[ft_cols].max(axis=1) links_wide_table = links_wide_table.drop(columns=ft_cols) -#%% +# %% links_wide_table.to_file( - Path(r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\output_summaries\all_links_data") - / "all_data_wide.geojson") - - -#%% -num_iter = { - (3,11): 3, - (3,12): 10, - (3,13): 10, - (3,14): 19, - (3,15): 4, - (4,12): 20 -} -#%% -all_links_no_none = [links for links in all_links if (links is not None)] #and (links["#link_id"].is_unique)] + Path( + r"Z:\MTC\US0024934.9168\Task_3_runtime_improvements\3.1_network_fidelity\output_summaries\all_links_data" + ) + / "all_data_wide.geojson" +) + + +# %% +num_iter = {(3, 11): 3, (3, 12): 10, (3, 13): 10, (3, 14): 19, (3, 15): 4, (4, 12): 20} +# %% +all_links_no_none = [ + links for links in all_links if (links is not None) +] # and (links["#link_id"].is_unique)] for df in all_links_no_none: df["saturation"] = df["VOLAU"] / df["@capacity"] -ft6_sat = [(link["run_number"].iloc[0], link["scenario_number"].iloc[0], (link.loc[link["@ft"] == 6, "saturation"] > 1).mean()) for link in all_links_no_none] +ft6_sat = [ + ( + link["run_number"].iloc[0], + link["scenario_number"].iloc[0], + (link.loc[link["@ft"] == 6, "saturation"] > 1).mean(), + ) + for link in all_links_no_none +] y = [val for val in num_iter.values()] x = [x[-1] for x in ft6_sat] col = [val[0] for val in num_iter.keys()] -#%% +# %% import matplotlib.pyplot as plt + plt.scatter(x, y, c=col) # Calculate the trendline @@ -167,25 +199,28 @@ def combine_tables(dfs, columns_same): p = np.poly1d(z) # Plot the trendline -plt.plot(x, p(x), color='red') +plt.plot(x, p(x), color="red") -plt.xlabel('proportion of ft 6 with saturation > 1') -plt.ylabel('number of iterations to solve') -plt.title('Number of iterations to solve (relative gap = 0.05)') +plt.xlabel("proportion of ft 6 with saturation > 1") +plt.ylabel("number of iterations to solve") +plt.title("Number of iterations to solve (relative gap = 0.05)") plt.show() -#%% +# %% import matplotlib.pyplot as plt + data = [links_wide_table[col] for col in links_wide_table.iloc[:, 2:].columns] fig = plt.boxplot(data) fig.show() # -------------------------------------------------------------------------- -#%% +# %% links_table["direction"] = links_table["geometry"].apply(get_linestring_direction) # %% links_table.to_file(output_dir / "all_data.geojson", index=False) -#%% + + +# %% def get_link_counts(df: pd.DataFrame): ret_val = df.value_counts("@ft").sort_index().to_frame().T total = ret_val.sum(axis=1) @@ -197,6 +232,7 @@ def get_link_counts(df: pd.DataFrame): ret_val["scenario_number"] = df["scenario_number"].iloc[0] return ret_val -pd.concat( - [get_link_counts(df) for df in all_links] -).sort_values(by=["run_number", "scenario_number"]) \ No newline at end of file + +pd.concat([get_link_counts(df) for df in all_links]).sort_values( + by=["run_number", "scenario_number"] +) diff --git a/tm2py/components/network/create_tod_scenarios.py b/tm2py/components/network/create_tod_scenarios.py index 7cf5a9c5..19d5f330 100644 --- a/tm2py/components/network/create_tod_scenarios.py +++ b/tm2py/components/network/create_tod_scenarios.py @@ -628,7 +628,7 @@ def _set_capclass(network): area_type = link["@area_type"] if area_type < 0: link["@capclass"] = -1 - elif (link["@ft"] == 99): + elif link["@ft"] == 99: link["@capclass"] = 10 * area_type + 7 else: link["@capclass"] = 10 * area_type + link["@ft"]