From de9525455bb41d4bb484c0057d7fa3d899977a73 Mon Sep 17 00:00:00 2001 From: romer8 Date: Sun, 9 Jun 2024 17:17:48 -0600 Subject: [PATCH 01/24] added troute to the nexus metadata --- .../components/hydroFabricLinePlot.js | 46 ++++++++++++- .../hydroFabric/hooks/useHydroFabric.js | 5 ++ .../hydroFabric/store/actions/actionsTypes.js | 5 ++ .../store/reducers/hydroFabricReducer.js | 65 +++++++++++++++++++ reactapp/services/api/app.js | 8 ++- reactapp/views/ngiab/hydroFabricView.js | 60 +++++++++++++++++ tethysapp/ngiab/controllers.py | 60 ++++++++++++++++- tethysapp/ngiab/utils.py | 31 +++++++++ 8 files changed, 276 insertions(+), 4 deletions(-) diff --git a/reactapp/features/hydroFabric/components/hydroFabricLinePlot.js b/reactapp/features/hydroFabric/components/hydroFabricLinePlot.js index 851e69f..74b1abc 100644 --- a/reactapp/features/hydroFabric/components/hydroFabricLinePlot.js +++ b/reactapp/features/hydroFabric/components/hydroFabricLinePlot.js @@ -11,8 +11,9 @@ import '../css/chart.css'; const chartOptions = { axisX: { type: FixedScaleAxis, - divisor: 5, + divisor: 10, labelInterpolationFnc: function(value) { + console.log(value) return new Date(value).toLocaleDateString(); } }, @@ -29,6 +30,8 @@ const chartOptions = { }; + + const HydroFabricLinePlot = (props) => { // Reference to the chart container const chartRef = useRef(null); @@ -39,6 +42,7 @@ const HydroFabricLinePlot = (props) => { useEffect(() => { if (!state.nexus.series) return; const nexusSeries = state.nexus.series.map(point => ({x: new Date(point.x), y: point.y})); + const chartData = { series: [ { name: 'Nexus', data: nexusSeries }, @@ -53,6 +57,7 @@ const HydroFabricLinePlot = (props) => { return () => { + console.log("retuirnin") if(chartInstance && props.singleRowOn){ actions.reset_nexus(); chartInstance.current.detach(); @@ -100,6 +105,45 @@ const HydroFabricLinePlot = (props) => { }; }, [state.catchment.series]); // Re-run effect if series data changes + + + useEffect(() => { + if (!state.troute.series) return; + if (chartRef.current) { + const trouteSeries = state.troute.series.map(point => ({x: new Date(point.x), y: point.y})); + const chartData = { + series: [ + { name: 'Troute', data: trouteSeries }, + ] + }; + + chartInstance.current = new LineChart(chartRef.current, chartData, chartOptions); + + addAnimationToLineChart(chartInstance.current, easings) + makeAxis( + chartRef.current, + 'Time (Date)', + `${state.troute.variable ? state.troute.variable.toLowerCase() : state.troute.variable_list ? state.troute.variable_list[0].label : null}` + ) + + makeTitle( + chartRef.current, + `${state.troute.variable ? state.troute.variable.toLowerCase() : state.troute.variable_list ? state.troute.variable_list[0].label : null}: ${state.troute.id} `) + } + + return () => { + if(props.singleRowOn){ + console.log(props.singleRowOn) + actions.reset_troute(); + chartRef.current.detach(); + document.getElementById('x-axis-title')?.remove(); + document.getElementById('y-axis-title')?.remove(); + } + + }; + }, [state.troute.series]); // Re-run effect if series data changes + + return (
); diff --git a/reactapp/features/hydroFabric/hooks/useHydroFabric.js b/reactapp/features/hydroFabric/hooks/useHydroFabric.js index aa945db..53bcd21 100644 --- a/reactapp/features/hydroFabric/hooks/useHydroFabric.js +++ b/reactapp/features/hydroFabric/hooks/useHydroFabric.js @@ -16,6 +16,11 @@ const useHydroFabric = () => { set_nexus_list: (list) => dispatch({ type: hydroFabricActionsTypes.set_nexus_list, payload: list}), set_catchment_list: (list) => dispatch({ type: hydroFabricActionsTypes.set_catchment_list, payload: list}), set_catchment_variable_list: (list) => dispatch({ type: hydroFabricActionsTypes.set_catchment_variable_list, payload: list}), + set_troute_series: (series) => dispatch({ type: hydroFabricActionsTypes.set_troute_series, payload: series }), + set_troute_id: (id) => dispatch({ type: hydroFabricActionsTypes.set_troute_id, payload: id }), + set_troute_variable: (variable) => dispatch({ type: hydroFabricActionsTypes.set_troute_variable, payload: variable }), + set_troute_variable_list: (list) => dispatch({ type: hydroFabricActionsTypes.set_troute_variable_list, payload: list }), + reset_troute: () => dispatch({ type: hydroFabricActionsTypes.reset_troute }), reset_nexus: () => dispatch({ type: hydroFabricActionsTypes.reset_nexus }), reset_catchment: () => dispatch({ type: hydroFabricActionsTypes.reset_catchment }), reset: () => dispatch({ type: hydroFabricActionsTypes.reset }), diff --git a/reactapp/features/hydroFabric/store/actions/actionsTypes.js b/reactapp/features/hydroFabric/store/actions/actionsTypes.js index 57228f0..6425a3c 100644 --- a/reactapp/features/hydroFabric/store/actions/actionsTypes.js +++ b/reactapp/features/hydroFabric/store/actions/actionsTypes.js @@ -7,8 +7,13 @@ const hydroFabricActionsTypes = { set_catchment_id: 'SET_CATCHMENT_ID', set_catchment_variable: 'SET_CATCHMENT_VARIABLE', set_catchment_variable_list: 'SET_CATCHMENT_VARIABLE_LIST', + set_troute_series: 'SET_TROUTE_SERIES', + set_troute_id: 'SET_TROUTE_ID', + set_troute_variable: 'SET_TROUTE_VARIABLE', + set_troute_variable_list: 'SET_TROUTE_VARIABLE_LIST', reset_nexus: 'RESET_NEXUS', reset_catchment: 'RESET_CATCHMENT', + reset_troute: 'RESET_TROUTE', reset: 'RESET', }; diff --git a/reactapp/features/hydroFabric/store/reducers/hydroFabricReducer.js b/reactapp/features/hydroFabric/store/reducers/hydroFabricReducer.js index 63de112..288b0a7 100644 --- a/reactapp/features/hydroFabric/store/reducers/hydroFabricReducer.js +++ b/reactapp/features/hydroFabric/store/reducers/hydroFabricReducer.js @@ -15,6 +15,13 @@ const hydroFabricInitialStore = { list:null, variable_list:null }, + troute:{ + series:null, + variable:null, + id:null, + list:null, + variable_list:null + } }, actions:{} @@ -124,6 +131,64 @@ const hydroFabricReducer = (state, action) => { } } }; + case hydroFabricActionsTypes.set_troute_series: + return { + ...state, + state: { + ...state.state, + troute: { + ...state.state.troute, + series: action.payload + } + } + }; + case hydroFabricActionsTypes.set_troute_id: + return { + ...state, + state: { + ...state.state, + troute: { + ...state.state.troute, + id: action.payload + } + } + }; + case hydroFabricActionsTypes.set_troute_variable: + return { + ...state, + state: { + ...state.state, + troute: { + ...state.state.troute, + variable: action.payload + } + } + }; + case hydroFabricActionsTypes.set_troute_variable_list: + return { + ...state, + state: { + ...state.state, + troute: { + ...state.state.troute, + variable_list: action.payload + } + } + }; + case hydroFabricActionsTypes.reset_troute: + return { + ...state, + state: { + ...state.state, + troute: { + series:null, + variable:null, + id:null, + list:null, + variable_list:null + } + } + }; case hydroFabricActionsTypes.reset_nexus: return { ...state, diff --git a/reactapp/services/api/app.js b/reactapp/services/api/app.js index d418436..fd68e99 100644 --- a/reactapp/services/api/app.js +++ b/reactapp/services/api/app.js @@ -12,6 +12,12 @@ const appAPI = { getCatchmentTimeSeries: (params) => { return apiClient.get(`${APP_ROOT_URL}getCatchmentTimeSeries/`, { params }); }, -}; + getTrouteVariables: (params) => { + return apiClient.get(`${APP_ROOT_URL}getTrouteVariables/`, { params }); + }, + getTrouteTimeSeries: (params) => { + return apiClient.get(`${APP_ROOT_URL}getTrouteTimeSeries/`, { params }); + } +} export default appAPI; \ No newline at end of file diff --git a/reactapp/views/ngiab/hydroFabricView.js b/reactapp/views/ngiab/hydroFabricView.js index bdafe7c..4e45029 100644 --- a/reactapp/views/ngiab/hydroFabricView.js +++ b/reactapp/views/ngiab/hydroFabricView.js @@ -22,6 +22,7 @@ const HydroFabricView = (props) => { appAPI.getNexusTimeSeries(params).then((response) => { actions.set_nexus_series(response.data); actions.set_nexus_list(response.nexus_ids); + actions.set_troute_id(state.nexus.id); props.toggleSingleRow(false); }).catch((error) => { console.log("Error fetching nexus time series", error); @@ -33,6 +34,51 @@ const HydroFabricView = (props) => { }, [state.nexus.id]); + useEffect(() => { + if (!state.troute.id) return; + props.setIsLoading(true); + + var params = { + troute_id: state.troute.id + } + appAPI.getTrouteVariables(params).then((response) => { + actions.set_troute_variable_list(response.troute_variables); + props.toggleSingleRow(false); + props.setIsLoading(false); + }).catch((error) => { + props.setIsLoading(false); + console.log("Error fetching troute variables", error); + }) + return () => { + if (state.troute.id) return; + actions.reset_troute(); + } + },[state.troute.id]); + + + useEffect(() => { + if (!state.troute.variable || !state.troute.id) return; + // actions.reset_nexus(); + props.setIsLoading(true); + var params = { + troute_id: state.troute.id, + troute_variable: state.troute.variable + } + appAPI.getTrouteTimeSeries(params).then((response) => { + actions.set_troute_series(response.data); + props.toggleSingleRow(false); + props.setIsLoading(false); + }).catch((error) => { + props.setIsLoading(false); + console.log("Error fetching troute time series", error); + }) + return () => { + if (state.troute.id) return; + actions.reset_troute(); + } + },[state.troute.variable]); + + useEffect(() => { if (!state.catchment.id) return; @@ -132,8 +178,22 @@ const HydroFabricView = (props) => { } /> + + } + { + state.troute.id && + +
Troute
+ + + } + + }> diff --git a/tethysapp/ngiab/controllers.py b/tethysapp/ngiab/controllers.py index 80eb0a0..74e0d73 100644 --- a/tethysapp/ngiab/controllers.py +++ b/tethysapp/ngiab/controllers.py @@ -4,7 +4,15 @@ import json import geopandas as gpd from tethys_sdk.routing import controller -from .utils import get_base_output, getCatchmentsIds, getNexusIDs +from .utils import ( + get_base_output, + getCatchmentsIds, + getNexusIDs, + check_troute_id, + get_troute_vars, + get_troute_df, +) + from .app import App @@ -95,4 +103,52 @@ def getNexusTimeSeries(request, app_workspace): for time, streamflow in zip(time_col.tolist(), streamflow_cms_col.tolist()) ] - return JsonResponse({"data": data, "nexus_ids": getNexusIDs(app_workspace)}) + return JsonResponse( + { + "data": data, + "nexus_ids": getNexusIDs(app_workspace), + } + ) + + +@controller(app_workspace=True) +def getTrouteVariables(request, app_workspace): + troute_id = request.GET.get("troute_id") + clean_troute_id = troute_id.split("-")[1] + df = get_troute_df(app_workspace) + try: + if check_troute_id(df, clean_troute_id): + vars = get_troute_vars(df) + else: + vars = [] + except Exception as e: + vars = [] + + return JsonResponse({"troute_variables": vars}) + + +@controller(app_workspace=True) +def getTrouteTimeSeries(request, app_workspace): + troute_id = request.GET.get("troute_id") + clean_troute_id = troute_id.split("-")[1] + variable_column = request.GET.get("troute_variable") + df = get_troute_df(app_workspace) + df_sliced_by_id = df[df["feature_id"] == int(clean_troute_id)] + + df_sliced_by_id["t0"] = pd.to_datetime(df_sliced_by_id["t0"].iloc[0]) + + # Convert the time_offset column to timedelta + df_sliced_by_id["time"] = pd.to_timedelta(df_sliced_by_id["time"]) + + df_sliced_by_id["t1"] = df_sliced_by_id["t0"] + df_sliced_by_id["time"] + try: + time_col = df_sliced_by_id["t1"] + var_col = df_sliced_by_id[variable_column] + data = [ + {"x": time, "y": val} + for time, val in zip(time_col.tolist(), var_col.tolist()) + ] + except Exception as e: + data = [] + + return JsonResponse({"data": data}) diff --git a/tethysapp/ngiab/utils.py b/tethysapp/ngiab/utils.py index f2570c0..3f524b1 100644 --- a/tethysapp/ngiab/utils.py +++ b/tethysapp/ngiab/utils.py @@ -1,5 +1,36 @@ import os import json +import pandas as pd +import glob + + +def _get_base_troute_output(app_workspace): + base_output_path = os.path.join( + app_workspace.path, "ngen-data", "outputs", "troute" + ) + return base_output_path + + +def get_troute_df(app_workspace): + base_output_path = _get_base_troute_output(app_workspace) + troute_output_files = glob.glob(base_output_path + "/*.csv") + df = pd.read_csv(troute_output_files[0]) + return df + + +def check_troute_id(df, id): + if int(id) in df["feature_id"].values: + return True + return False + + +def get_troute_vars(df): + list_variables = df.columns.tolist()[3:] # remove feature, time and t0 + + variables = [ + {"value": variable, "label": variable.lower()} for variable in list_variables + ] + return variables def get_base_output(app_workspace): From 04bbdcac90d4d342159eab4540cf9355a9c3f66f Mon Sep 17 00:00:00 2001 From: romer8 Date: Sun, 9 Jun 2024 19:22:38 -0600 Subject: [PATCH 02/24] update the cli --- cli/convert_geom.py | 88 ++++++++++++++++++++++----------------------- cli/utils.py | 45 +++++++++++++++++++++++ 2 files changed, 87 insertions(+), 46 deletions(-) create mode 100644 cli/utils.py diff --git a/cli/convert_geom.py b/cli/convert_geom.py index 3c4f1ef..5cdb109 100644 --- a/cli/convert_geom.py +++ b/cli/convert_geom.py @@ -1,55 +1,28 @@ -import geopandas as gpd import argparse from geo.Geoserver import Geoserver -import os -import zipfile +from .utils import _read_and_transform, _create_and_publish_shp -def read_and_transform(gpkg_path, layer_name): - gdf = gpd.read_file(gpkg_path, layer=layer_name) - if gdf.crs.to_string() != "EPSG:5070": - gdf = gdf.to_crs(epsg=5070) - return gdf - - -def create_and_publish_shp( - gdf, - shp_path, - geoserver_host, - geoserver_port, - geoserver_username, - geoserver_password, +def create_workspace( + geoserver_host, geoserver_port, geoserver_username, geoserver_password ): - gdf.to_file(f"{shp_path}.shp", driver="ESRI Shapefile") - shp_files = [f"{shp_path}.{ext}" for ext in ["shp", "shx", "dbf", "prj"]] - - with zipfile.ZipFile(f"{shp_path}.zip", "w") as zipf: - for file in shp_files: - zipf.write(file, arcname=os.path.basename(file)) - - # remove shapefiles - for file in shp_files: - os.remove(file) - - geo = Geoserver( - f"http://{geoserver_host}:{geoserver_port}/geoserver", - username=geoserver_username, - password=geoserver_password, - ) - geo.create_workspace(workspace="nextgen") + try: + geo = Geoserver( + f"http://{geoserver_host}:{geoserver_port}/geoserver", + username=geoserver_username, + password=geoserver_password, + ) + geo.create_workspace(workspace="nextgen") - geo.create_shp_datastore( - path=f"{shp_path}.zip", - store_name="hydrofabrics", - workspace="nextgen", - ) - # remove zip file - os.remove(f"{shp_path}.zip") + except Exception as e: + print(f"Error in create_workspace: {e}") + return 1 + return 0 def convert_gpkg_to_geojson(gpkg_path, layer_name, output_path): try: - gdf = read_and_transform(gpkg_path, layer_name) + gdf = _read_and_transform(gpkg_path, layer_name) gdf.to_file(output_path, driver="GeoJSON") return 0 except Exception as e: @@ -67,8 +40,8 @@ def publish_gpkg_layer_to_geoserver( geoserver_password, ): try: - gdf = read_and_transform(gpkg_path, layer_name) - create_and_publish_shp( + gdf = _read_and_transform(gpkg_path, layer_name) + _create_and_publish_shp( gdf, shp_path, geoserver_host, @@ -91,8 +64,8 @@ def publish_geojson_layer_to_geoserver( geoserver_password, ): try: - gdf = read_and_transform(geojson_path, None) - create_and_publish_shp( + gdf = _read_and_transform(geojson_path, None) + _create_and_publish_shp( gdf, shp_path, geoserver_host, @@ -123,6 +96,11 @@ def main(): action="store_true", help="Publish the GeoJSON layer to GeoServer", ) + parser.add_argument( + "--create_workspace", + action="store_true", + help="Create a workspace in GeoServer", + ) # Optional Arguments parser.add_argument( @@ -192,6 +170,24 @@ def main(): ) exit(result) + if args.create_workspace: + if ( + not args.geoserver_host + or not args.geoserver_port + or not args.geoserver_username + or not args.geoserver_password + ): + parser.error( + "--create_workspace requires --geoserver_host, --geoserver_port, --geoserver_username, and --geoserver_password." + ) + result = create_workspace( + args.geoserver_host, + args.geoserver_port, + args.geoserver_username, + args.geoserver_password, + ) + exit(result) + if __name__ == "__main__": main() diff --git a/cli/utils.py b/cli/utils.py new file mode 100644 index 0000000..0fd1182 --- /dev/null +++ b/cli/utils.py @@ -0,0 +1,45 @@ +import geopandas as gpd +from geo.Geoserver import Geoserver +import zipfile +import os + + +def _read_and_transform(gpkg_path, layer_name): + gdf = gpd.read_file(gpkg_path, layer=layer_name) + if gdf.crs.to_string() != "EPSG:5070": + gdf = gdf.to_crs(epsg=5070) + return gdf + + +def _create_and_publish_shp( + gdf, + shp_path, + geoserver_host, + geoserver_port, + geoserver_username, + geoserver_password, +): + gdf.to_file(f"{shp_path}.shp", driver="ESRI Shapefile") + shp_files = [f"{shp_path}.{ext}" for ext in ["shp", "shx", "dbf", "prj"]] + + with zipfile.ZipFile(f"{shp_path}.zip", "w") as zipf: + for file in shp_files: + zipf.write(file, arcname=os.path.basename(file)) + + # remove shapefiles + for file in shp_files: + os.remove(file) + + geo = Geoserver( + f"http://{geoserver_host}:{geoserver_port}/geoserver", + username=geoserver_username, + password=geoserver_password, + ) + + geo.create_shp_datastore( + path=f"{shp_path}.zip", + store_name="hydrofabrics", + workspace="nextgen", + ) + # remove zip file + os.remove(f"{shp_path}.zip") From 14792c9106e0c8e1499a5f4192da047e6ef3423a Mon Sep 17 00:00:00 2001 From: romer8 Date: Sun, 9 Jun 2024 19:48:34 -0600 Subject: [PATCH 03/24] adding init.py --- cli/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 cli/__init__.py diff --git a/cli/__init__.py b/cli/__init__.py new file mode 100644 index 0000000..e69de29 From d130197525ae6b311297d846b3023e6968ff4174 Mon Sep 17 00:00:00 2001 From: romer8 Date: Sun, 9 Jun 2024 20:14:09 -0600 Subject: [PATCH 04/24] one file for the cli --- cli/convert_geom.py | 45 ++++++++++++++++++++++++++++++++++++++++++++- cli/utils.py | 45 --------------------------------------------- 2 files changed, 44 insertions(+), 46 deletions(-) delete mode 100644 cli/utils.py diff --git a/cli/convert_geom.py b/cli/convert_geom.py index 5cdb109..8524c55 100644 --- a/cli/convert_geom.py +++ b/cli/convert_geom.py @@ -1,6 +1,49 @@ import argparse from geo.Geoserver import Geoserver -from .utils import _read_and_transform, _create_and_publish_shp +import geopandas as gpd +import zipfile +import os + + +def _read_and_transform(gpkg_path, layer_name): + gdf = gpd.read_file(gpkg_path, layer=layer_name) + if gdf.crs.to_string() != "EPSG:5070": + gdf = gdf.to_crs(epsg=5070) + return gdf + + +def _create_and_publish_shp( + gdf, + shp_path, + geoserver_host, + geoserver_port, + geoserver_username, + geoserver_password, +): + gdf.to_file(f"{shp_path}.shp", driver="ESRI Shapefile") + shp_files = [f"{shp_path}.{ext}" for ext in ["shp", "shx", "dbf", "prj"]] + + with zipfile.ZipFile(f"{shp_path}.zip", "w") as zipf: + for file in shp_files: + zipf.write(file, arcname=os.path.basename(file)) + + # remove shapefiles + for file in shp_files: + os.remove(file) + + geo = Geoserver( + f"http://{geoserver_host}:{geoserver_port}/geoserver", + username=geoserver_username, + password=geoserver_password, + ) + + geo.create_shp_datastore( + path=f"{shp_path}.zip", + store_name="hydrofabrics", + workspace="nextgen", + ) + # remove zip file + os.remove(f"{shp_path}.zip") def create_workspace( diff --git a/cli/utils.py b/cli/utils.py deleted file mode 100644 index 0fd1182..0000000 --- a/cli/utils.py +++ /dev/null @@ -1,45 +0,0 @@ -import geopandas as gpd -from geo.Geoserver import Geoserver -import zipfile -import os - - -def _read_and_transform(gpkg_path, layer_name): - gdf = gpd.read_file(gpkg_path, layer=layer_name) - if gdf.crs.to_string() != "EPSG:5070": - gdf = gdf.to_crs(epsg=5070) - return gdf - - -def _create_and_publish_shp( - gdf, - shp_path, - geoserver_host, - geoserver_port, - geoserver_username, - geoserver_password, -): - gdf.to_file(f"{shp_path}.shp", driver="ESRI Shapefile") - shp_files = [f"{shp_path}.{ext}" for ext in ["shp", "shx", "dbf", "prj"]] - - with zipfile.ZipFile(f"{shp_path}.zip", "w") as zipf: - for file in shp_files: - zipf.write(file, arcname=os.path.basename(file)) - - # remove shapefiles - for file in shp_files: - os.remove(file) - - geo = Geoserver( - f"http://{geoserver_host}:{geoserver_port}/geoserver", - username=geoserver_username, - password=geoserver_password, - ) - - geo.create_shp_datastore( - path=f"{shp_path}.zip", - store_name="hydrofabrics", - workspace="nextgen", - ) - # remove zip file - os.remove(f"{shp_path}.zip") From 0f2c0c3dc99dfd4a8daa6680e2c532b1d9c73212 Mon Sep 17 00:00:00 2001 From: romer8 Date: Sun, 9 Jun 2024 21:23:38 -0600 Subject: [PATCH 05/24] added store name to publish --- cli/convert_geom.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cli/convert_geom.py b/cli/convert_geom.py index 8524c55..246348e 100644 --- a/cli/convert_geom.py +++ b/cli/convert_geom.py @@ -15,6 +15,7 @@ def _read_and_transform(gpkg_path, layer_name): def _create_and_publish_shp( gdf, shp_path, + store_name, geoserver_host, geoserver_port, geoserver_username, @@ -39,7 +40,7 @@ def _create_and_publish_shp( geo.create_shp_datastore( path=f"{shp_path}.zip", - store_name="hydrofabrics", + store_name=f"{store_name}", workspace="nextgen", ) # remove zip file @@ -77,6 +78,7 @@ def publish_gpkg_layer_to_geoserver( gpkg_path, layer_name, shp_path, + store_name, geoserver_host, geoserver_port, geoserver_username, @@ -87,6 +89,7 @@ def publish_gpkg_layer_to_geoserver( _create_and_publish_shp( gdf, shp_path, + store_name, geoserver_host, geoserver_port, geoserver_username, @@ -101,6 +104,7 @@ def publish_gpkg_layer_to_geoserver( def publish_geojson_layer_to_geoserver( geojson_path, shp_path, + store_name, geoserver_host, geoserver_port, geoserver_username, @@ -111,6 +115,7 @@ def publish_geojson_layer_to_geoserver( _create_and_publish_shp( gdf, shp_path, + store_name, geoserver_host, geoserver_port, geoserver_username, @@ -163,6 +168,7 @@ def main(): parser.add_argument( "--shp_path", help="Path to save the shapefile for GeoServer", required=False ) + parser.add_argument("--store_name", help="Shapefile store name", required=False) parser.add_argument("--geoserver_host", help="GeoServer host", required=False) parser.add_argument("--geoserver_port", help="GeoServer port", required=False) parser.add_argument( @@ -193,6 +199,7 @@ def main(): args.gpkg_path, args.layer_name, args.shp_path, + args.store_name, args.geoserver_host, args.geoserver_port, args.geoserver_username, @@ -206,6 +213,7 @@ def main(): result = publish_geojson_layer_to_geoserver( args.geojson_path, args.shp_path, + args.store_name, args.geoserver_host, args.geoserver_port, args.geoserver_username, From b2aabc149de517206b0e223d94222b8660e1b5ff Mon Sep 17 00:00:00 2001 From: romer8 Date: Sun, 9 Jun 2024 21:40:02 -0600 Subject: [PATCH 06/24] added flowlines --- reactapp/lib/layersData.js | 27 ++++++++++++++++++++++++++- reactapp/lib/mapUtils.js | 31 ++++++++++++++++++++++++++++++- reactapp/views/ngiab/map_view.js | 10 ++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/reactapp/lib/layersData.js b/reactapp/lib/layersData.js index b5926ff..f94b102 100644 --- a/reactapp/lib/layersData.js +++ b/reactapp/lib/layersData.js @@ -48,5 +48,30 @@ const sldCatchmentStyle = ` ` -export { initialLayersArray,sldCatchmentStyle } +const sldFlowPathsStyle = ` + + + nextgen:flowpaths + + + + + + #87CEEB + 2 + + + + + + +` + + +export { initialLayersArray,sldCatchmentStyle,sldFlowPathsStyle } diff --git a/reactapp/lib/mapUtils.js b/reactapp/lib/mapUtils.js index 608059c..24d15e6 100644 --- a/reactapp/lib/mapUtils.js +++ b/reactapp/lib/mapUtils.js @@ -8,7 +8,7 @@ import { Text, } from 'ol/style'; import { VectorSourceLayer } from 'features/Map/lib/source/sources'; -import { sldCatchmentStyle } from './layersData'; +import { sldCatchmentStyle,sldFlowPathsStyle } from './layersData'; const image = new CircleStyle({ radius: 5, @@ -202,6 +202,34 @@ let makeCatchmentLayer = (catchmentLayersURL) =>{ } } +let makeFlowPathsLayer = (flowPathsLayersURL) =>{ + return { + layerType: 'OlTileLayer', + options: + { + sourceType: 'WMSTile', + url: flowPathsLayersURL, + // all the params for the source goes here + params: { + LAYERS: 'nextgen:flowpaths', + Tiled: true, + SLD_BODY: sldFlowPathsStyle + }, + // the rest of the attributes are for the definition of the layer + name: "Flowpaths Layer", + zIndex: 2, + source:{ + serverType: 'geoserver', + crossOrigin: 'anonymous' + } + + }, + + } +} + + + const customForEachLayerAtPixelLayerFilter = (layer) =>{ return layer.get('name') !== 'baseMapLayer' // return layer.get('events') && layer.get('events').length > 0 && layer.get('events').findIndex(event => event.type === 'click') > -1 @@ -313,6 +341,7 @@ const displayFeatureInfo = (event,layer,hydroFabricActions) => { export { makeNexusLayerParams, makeCatchmentLayer, + makeFlowPathsLayer, displayFeatureInfo, displayFeatureInfoWMS, customForEachLayerAtPixelLayerFilter, diff --git a/reactapp/views/ngiab/map_view.js b/reactapp/views/ngiab/map_view.js index 5c1df5d..cdd74d3 100644 --- a/reactapp/views/ngiab/map_view.js +++ b/reactapp/views/ngiab/map_view.js @@ -13,6 +13,7 @@ import { initialLayersArray } from 'lib/layersData'; import { makeNexusLayerParams, makeCatchmentLayer, + makeFlowPathsLayer, createClusterVectorLayer } from 'lib/mapUtils'; import LoadingAnimation from 'components/loader/LoadingAnimation'; @@ -30,6 +31,12 @@ const MapView = (props) => { const catchmentLayersURL = 'http://localhost:8181/geoserver/wms'; return makeCatchmentLayer(catchmentLayersURL); },[]) + + const flowPathsLayerCallBack = useCallback(() => { + const flowPathsLayersURL = 'http://localhost:8181/geoserver/wms'; + return makeFlowPathsLayer(flowPathsLayersURL); + },[]) + const onPointerMoveLayersEventCallback = useCallback((event) => { return onPointerMoveLayersEvent(event) },[]); @@ -61,16 +68,19 @@ const MapView = (props) => { // Define the parameters for the layer var nexusLayerParams = nexusLayerParamsCallBack(); var catchmentLayer = catchmentLayerCallBack(); + var flowPathsLayer = flowPathsLayerCallBack(); nexusLayerParams['geojsonLayer']=response.geojson; const nexusClusterLayer = createClusterVectorLayer(nexusLayerParams); + const NexusExtent = nexusClusterLayer.options.params.source.getExtent(); mapActions.addLayer(nexusClusterLayer); mapActions.set_extent(NexusExtent); mapActions.addLayer(catchmentLayer); + mapActions.addLayer(flowPathsLayer); }).catch(error => { console.log(error) From 2b9a55095c04550ce98cd80cfefb78cf72c258ba Mon Sep 17 00:00:00 2001 From: romer8 Date: Sun, 9 Jun 2024 22:21:00 -0600 Subject: [PATCH 07/24] added troute in catchment as well --- reactapp/features/Map/providers/MapProvider.js | 1 - .../features/hydroFabric/components/hydroFabricLinePlot.js | 3 --- reactapp/views/ngiab/hydroFabricView.js | 3 +-- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/reactapp/features/Map/providers/MapProvider.js b/reactapp/features/Map/providers/MapProvider.js index 5c18715..44bdff2 100644 --- a/reactapp/features/Map/providers/MapProvider.js +++ b/reactapp/features/Map/providers/MapProvider.js @@ -96,7 +96,6 @@ export const MapProvider = ({ children, layers= [] }) => { useEffect(() => { if (!state.state.extent) return; - console.log(state.state.extent) state.state.mapObject.getView().fit(state.state.extent, {duration: 1300, padding: [50, 50, 50, 50]}); }, [state.state.extent]); diff --git a/reactapp/features/hydroFabric/components/hydroFabricLinePlot.js b/reactapp/features/hydroFabric/components/hydroFabricLinePlot.js index 74b1abc..43b5947 100644 --- a/reactapp/features/hydroFabric/components/hydroFabricLinePlot.js +++ b/reactapp/features/hydroFabric/components/hydroFabricLinePlot.js @@ -13,7 +13,6 @@ const chartOptions = { type: FixedScaleAxis, divisor: 10, labelInterpolationFnc: function(value) { - console.log(value) return new Date(value).toLocaleDateString(); } }, @@ -57,7 +56,6 @@ const HydroFabricLinePlot = (props) => { return () => { - console.log("retuirnin") if(chartInstance && props.singleRowOn){ actions.reset_nexus(); chartInstance.current.detach(); @@ -133,7 +131,6 @@ const HydroFabricLinePlot = (props) => { return () => { if(props.singleRowOn){ - console.log(props.singleRowOn) actions.reset_troute(); chartRef.current.detach(); document.getElementById('x-axis-title')?.remove(); diff --git a/reactapp/views/ngiab/hydroFabricView.js b/reactapp/views/ngiab/hydroFabricView.js index 4e45029..cda0412 100644 --- a/reactapp/views/ngiab/hydroFabricView.js +++ b/reactapp/views/ngiab/hydroFabricView.js @@ -37,7 +37,6 @@ const HydroFabricView = (props) => { useEffect(() => { if (!state.troute.id) return; props.setIsLoading(true); - var params = { troute_id: state.troute.id } @@ -58,7 +57,6 @@ const HydroFabricView = (props) => { useEffect(() => { if (!state.troute.variable || !state.troute.id) return; - // actions.reset_nexus(); props.setIsLoading(true); var params = { troute_id: state.troute.id, @@ -92,6 +90,7 @@ const HydroFabricView = (props) => { actions.set_catchment_variable_list(response.variables); actions.set_catchment_variable(null); actions.set_catchment_list(response.catchment_ids); + actions.set_troute_id(state.catchment.id); props.toggleSingleRow(false); props.setIsLoading(false); }).catch((error) => { From 995f200115ddc511d6edf4786a4d2a5b49721f92 Mon Sep 17 00:00:00 2001 From: romer8 Date: Mon, 28 Oct 2024 17:06:15 -0600 Subject: [PATCH 08/24] changes to make the dockerfile faster --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bb54a03..832fc5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,7 @@ COPY . ${TETHYS_HOME}/apps/ngiab # ENVIRONMENT # ############### ENV TETHYS_DB_ENGINE=django.db.backends.sqlite3 +ENV SKIP_DB_SETUP=True ENV TETHYS_DB_NAME= ENV TETHYS_DB_USERNAME= ENV TETHYS_DB_PASSWORD= @@ -48,4 +49,4 @@ EXPOSE 80 # RUN # ####### -CMD bash run.sh \ No newline at end of file +CMD ["bash", "run.sh", "--skip-perm"] \ No newline at end of file From 34cfc9c4ac23a8c2828ac57190d22bccc683bbf0 Mon Sep 17 00:00:00 2001 From: romer8 Date: Mon, 28 Oct 2024 19:05:02 -0600 Subject: [PATCH 09/24] solve libquadmath0 problem --- Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 832fc5a..5803f37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,6 @@ ARG MICRO_TETHYS=true \ MAMBA_DOCKERFILE_ACTIVATE=1 - ######################### # ADD APPLICATION FILES # ######################### @@ -31,6 +30,11 @@ ENV PORTAL_SUPERUSER_NAME=admin ENV PORTAL_SUPERUSER_PASSWORD=pass ENV PROJ_LIB=/opt/conda/envs/tethys/share/proj + +RUN apt-get update \ + && apt-get -y install libquadmath0 \ + && rm -rf /var/lib/apt/lists/* + ####################### # INSTALL APPLICATION # ####################### From b91125def5d08ecdebe061dd30dbf96a7fbcc7de Mon Sep 17 00:00:00 2001 From: romer8 Date: Mon, 28 Oct 2024 19:21:13 -0600 Subject: [PATCH 10/24] gfortran contains libquadmath0 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5803f37..fcd7f2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ ENV PROJ_LIB=/opt/conda/envs/tethys/share/proj RUN apt-get update \ - && apt-get -y install libquadmath0 \ + && apt-get -y install gfortran \ && rm -rf /var/lib/apt/lists/* ####################### From 4ac6ef579d7f76740d4ed6207f8ab2eedcb91869 Mon Sep 17 00:00:00 2001 From: romer8 Date: Mon, 28 Oct 2024 19:30:35 -0600 Subject: [PATCH 11/24] added pyarrow --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1ebfcb6..631949a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ geopandas nodejs geoserver-rest -zip \ No newline at end of file +zip +pyarrow \ No newline at end of file From 6c97f4b06ee67c205b4d4f4e457e0bfb0c829402 Mon Sep 17 00:00:00 2001 From: romer8 Date: Mon, 28 Oct 2024 19:54:56 -0600 Subject: [PATCH 12/24] fix pyarrow --- Dockerfile | 1 + requirements.txt | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fcd7f2a..b30db05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,7 @@ RUN apt-get update \ ####################### RUN cd ${TETHYS_HOME}/apps/ngiab && \ micromamba install --yes -c conda-forge --file requirements.txt && \ + micromamba remove pyarrow && micromamba install --yes -c conda-forge pyarrow && \ micromamba clean --all --yes && \ npm install && npm run build && \ tethys install -d -N diff --git a/requirements.txt b/requirements.txt index 631949a..1ebfcb6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ geopandas nodejs geoserver-rest -zip -pyarrow \ No newline at end of file +zip \ No newline at end of file From 03f4c718c786b1b933554a116199a254743e4183 Mon Sep 17 00:00:00 2001 From: romer8 Date: Mon, 28 Oct 2024 20:48:00 -0600 Subject: [PATCH 13/24] Added permissions --- Dockerfile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b30db05..93d38bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ ENV PORTAL_SUPERUSER_NAME=admin ENV PORTAL_SUPERUSER_PASSWORD=pass ENV PROJ_LIB=/opt/conda/envs/tethys/share/proj - +# fix error for numpy not beign imported RUN apt-get update \ && apt-get -y install gfortran \ && rm -rf /var/lib/apt/lists/* @@ -38,12 +38,20 @@ RUN apt-get update \ ####################### # INSTALL APPLICATION # ####################### + RUN cd ${TETHYS_HOME}/apps/ngiab && \ micromamba install --yes -c conda-forge --file requirements.txt && \ micromamba remove pyarrow && micromamba install --yes -c conda-forge pyarrow && \ micromamba clean --all --yes && \ npm install && npm run build && \ - tethys install -d -N + tethys install -d -N && \ + export NGINX_USER=$(grep 'user .*;' /etc/nginx/nginx.conf | awk '{print $2}' | awk -F';' '{print $1}') && \ + chown -R ${NGINX_USER}: ${TETHYS_PERSIST} && \ + chown -R ${NGINX_USER}: ${STATIC_ROOT} && \ + chown -R ${NGINX_USER}: ${WORKSPACE_ROOT} && \ + chown -R ${NGINX_USER}: ${MEDIA_ROOT} && \ + chown -R ${NGINX_USER}: ${TETHYSAPP_DIR} && \ + chown -R ${NGINX_USER}: ${TETHYS_HOME} ######### # PORTS # From 736190b4f520037150da17993e778414c46cd1f1 Mon Sep 17 00:00:00 2001 From: romer8 Date: Mon, 28 Oct 2024 21:20:15 -0600 Subject: [PATCH 14/24] added fix --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 93d38bf..e2a9085 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ ENV PORTAL_SUPERUSER_NAME=admin ENV PORTAL_SUPERUSER_PASSWORD=pass ENV PROJ_LIB=/opt/conda/envs/tethys/share/proj -# fix error for numpy not beign imported +# fix error for numpy not being imported. libquadmath not found RUN apt-get update \ && apt-get -y install gfortran \ && rm -rf /var/lib/apt/lists/* @@ -50,7 +50,7 @@ RUN cd ${TETHYS_HOME}/apps/ngiab && \ chown -R ${NGINX_USER}: ${STATIC_ROOT} && \ chown -R ${NGINX_USER}: ${WORKSPACE_ROOT} && \ chown -R ${NGINX_USER}: ${MEDIA_ROOT} && \ - chown -R ${NGINX_USER}: ${TETHYSAPP_DIR} && \ + chown -R ${NGINX_USER}: ${TETHYS_HOME}/apps/ngiab && \ chown -R ${NGINX_USER}: ${TETHYS_HOME} ######### From 9680ac07088953f308913b30317a3ce1dfb69a1a Mon Sep 17 00:00:00 2001 From: romer8 Date: Mon, 28 Oct 2024 21:56:25 -0600 Subject: [PATCH 15/24] removed permissions --- Dockerfile | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index e2a9085..89f0d60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,14 +44,7 @@ RUN cd ${TETHYS_HOME}/apps/ngiab && \ micromamba remove pyarrow && micromamba install --yes -c conda-forge pyarrow && \ micromamba clean --all --yes && \ npm install && npm run build && \ - tethys install -d -N && \ - export NGINX_USER=$(grep 'user .*;' /etc/nginx/nginx.conf | awk '{print $2}' | awk -F';' '{print $1}') && \ - chown -R ${NGINX_USER}: ${TETHYS_PERSIST} && \ - chown -R ${NGINX_USER}: ${STATIC_ROOT} && \ - chown -R ${NGINX_USER}: ${WORKSPACE_ROOT} && \ - chown -R ${NGINX_USER}: ${MEDIA_ROOT} && \ - chown -R ${NGINX_USER}: ${TETHYS_HOME}/apps/ngiab && \ - chown -R ${NGINX_USER}: ${TETHYS_HOME} + tethys install -d -N ######### # PORTS # From 10fa6de064d6757a7081a03dd8d9516b94ff53ca Mon Sep 17 00:00:00 2001 From: romer8 Date: Mon, 28 Oct 2024 22:12:59 -0600 Subject: [PATCH 16/24] overwrite run.sh --- Dockerfile | 2 +- simplified_run.sh | 59 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 simplified_run.sh diff --git a/Dockerfile b/Dockerfile index 89f0d60..83e9441 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,4 +55,4 @@ EXPOSE 80 # RUN # ####### -CMD ["bash", "run.sh", "--skip-perm"] \ No newline at end of file +CMD ["bash", "simplified_run.sh"] \ No newline at end of file diff --git a/simplified_run.sh b/simplified_run.sh new file mode 100644 index 0000000..0fa9817 --- /dev/null +++ b/simplified_run.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +tail_file() { + echo "tailing file $1" + ALIGN=27 + LENGTH=`echo $1 | wc -c` + PADDING=`expr ${ALIGN} - ${LENGTH}` + PREFIX=$1`perl -e "print ' ' x $PADDING;"` + file="/var/log/$1" + # each tail runs in the background but prints to stdout + # sed outputs each line from tail prepended with the filename+padding + tail -qF $file | sed --unbuffered "s|^|${PREFIX}:|g" & +} + +echo_status() { + local args="${@}" + tput setaf 4 + tput bold + echo -e "- $args" + tput sgr0 +} + + +echo_status "Starting up..." + + +echo_status "Enforcing start state... (This might take a bit)" +salt-call --local state.apply + + +echo_status "Changing permissions" +chown -R www: /usr/lib/tethys +chmod -R 777 /var/lib/tethys_persist + +echo_status "Starting supervisor" + +# Start Supervisor +/usr/bin/supervisord + +echo_status "Done!" + +# Watch Logs +echo_status "Watching logs. You can ignore errors from either apache (httpd) or nginx depending on which one you are using." + +log_files=("httpd/access_log" +"httpd/error_log" +"nginx/access.log" +"nginx/error.log" +"supervisor/supervisord.log" +"tethys/tethys.log") + +# When this exits, exit all background tail processes +trap 'kill $(jobs -p)' EXIT +for log_file in "${log_files[@]}"; do +tail_file "${log_file}" +done + +# Read output from tail; wait for kill or stop command (docker waits here) +wait From dacce4b3e8535cd31d01f75c7cbf3c13c267a394 Mon Sep 17 00:00:00 2001 From: romer8 Date: Mon, 28 Oct 2024 22:22:37 -0600 Subject: [PATCH 17/24] added to DockerFile --- Dockerfile | 4 +++- simplified_run.sh => run.sh | 0 2 files changed, 3 insertions(+), 1 deletion(-) rename simplified_run.sh => run.sh (100%) diff --git a/Dockerfile b/Dockerfile index 83e9441..1516b62 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,8 @@ ARG MICRO_TETHYS=true \ # ADD APPLICATION FILES # ######################### COPY . ${TETHYS_HOME}/apps/ngiab +COPY run.sh ${TETHYS_HOME}/run.sh + ############### @@ -55,4 +57,4 @@ EXPOSE 80 # RUN # ####### -CMD ["bash", "simplified_run.sh"] \ No newline at end of file +CMD ["bash", "run.sh"] \ No newline at end of file diff --git a/simplified_run.sh b/run.sh similarity index 100% rename from simplified_run.sh rename to run.sh From 615ae1191d46152de2b4e16a362bb85c2f4bf9c3 Mon Sep 17 00:00:00 2001 From: romer8 Date: Tue, 29 Oct 2024 10:21:46 -0600 Subject: [PATCH 18/24] reduced dockerfile healthcheck start-period and also added permissions to the run.sh --- Dockerfile | 5 ++++- run.sh | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1516b62..f0b9c71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,4 +57,7 @@ EXPOSE 80 # RUN # ####### -CMD ["bash", "run.sh"] \ No newline at end of file +CMD bash run.sh + +HEALTHCHECK --start-period=1s \ + CMD ./liveness-probe.sh \ No newline at end of file diff --git a/run.sh b/run.sh index 0fa9817..53888a0 100644 --- a/run.sh +++ b/run.sh @@ -30,7 +30,8 @@ salt-call --local state.apply echo_status "Changing permissions" chown -R www: /usr/lib/tethys -chmod -R 777 /var/lib/tethys_persist +chmod -R www: /var/lib/tethys_persist +chmod -R www: /var/logs/tethys echo_status "Starting supervisor" From 8cd15c4f3c221975f1e29fe895b9b993e951d489 Mon Sep 17 00:00:00 2001 From: romer8 Date: Tue, 29 Oct 2024 11:03:53 -0600 Subject: [PATCH 19/24] fixed errors --- run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.sh b/run.sh index 53888a0..2783d5e 100644 --- a/run.sh +++ b/run.sh @@ -30,8 +30,8 @@ salt-call --local state.apply echo_status "Changing permissions" chown -R www: /usr/lib/tethys -chmod -R www: /var/lib/tethys_persist -chmod -R www: /var/logs/tethys +chown -R www: /var/lib/tethys_persist +chown -R www: /var/logs/tethys echo_status "Starting supervisor" From 81b88cd632d2ded2832a11456a7def73832582f7 Mon Sep 17 00:00:00 2001 From: romer8 Date: Tue, 29 Oct 2024 11:50:21 -0600 Subject: [PATCH 20/24] changed to not v8 anymore --- .github/workflows/build_and_push_dev_image.yml | 2 +- run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_push_dev_image.yml b/.github/workflows/build_and_push_dev_image.yml index 97483a7..3d5c97f 100644 --- a/.github/workflows/build_and_push_dev_image.yml +++ b/.github/workflows/build_and_push_dev_image.yml @@ -23,7 +23,7 @@ jobs: platform: - linux/amd64 - linux/arm64 - - linux/arm64/v8 + # - linux/arm64/v8 steps: - name: Prepare run: | diff --git a/run.sh b/run.sh index 2783d5e..f69b3ae 100644 --- a/run.sh +++ b/run.sh @@ -28,7 +28,7 @@ echo_status "Enforcing start state... (This might take a bit)" salt-call --local state.apply -echo_status "Changing permissions" +echo_status "Fixing permissions" chown -R www: /usr/lib/tethys chown -R www: /var/lib/tethys_persist chown -R www: /var/logs/tethys From 27dae42a7e6429f85077178c5768e96642690970 Mon Sep 17 00:00:00 2001 From: romer8 Date: Tue, 29 Oct 2024 12:03:59 -0600 Subject: [PATCH 21/24] no need to install devs --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f0b9c71..fce0699 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ RUN cd ${TETHYS_HOME}/apps/ngiab && \ micromamba remove pyarrow && micromamba install --yes -c conda-forge pyarrow && \ micromamba clean --all --yes && \ npm install && npm run build && \ - tethys install -d -N + tethys install -d -N -w ######### # PORTS # From 287fdf6a511e3d428d6d1086962461190d2fa21f Mon Sep 17 00:00:00 2001 From: romer8 Date: Tue, 29 Oct 2024 12:35:47 -0600 Subject: [PATCH 22/24] updated file --- run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.sh b/run.sh index f69b3ae..79849ef 100644 --- a/run.sh +++ b/run.sh @@ -31,7 +31,7 @@ salt-call --local state.apply echo_status "Fixing permissions" chown -R www: /usr/lib/tethys chown -R www: /var/lib/tethys_persist -chown -R www: /var/logs/tethys +chown -R www: /var/log/tethys echo_status "Starting supervisor" From f950f2a17d5238d8e0c61303afed5a5f1b22b532 Mon Sep 17 00:00:00 2001 From: romer8 Date: Tue, 29 Oct 2024 16:22:34 -0600 Subject: [PATCH 23/24] fixing new troute file --- tethysapp/ngiab/controllers.py | 15 ++++----------- tethysapp/ngiab/utils.py | 2 +- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/tethysapp/ngiab/controllers.py b/tethysapp/ngiab/controllers.py index 74e0d73..ff550ad 100644 --- a/tethysapp/ngiab/controllers.py +++ b/tethysapp/ngiab/controllers.py @@ -121,7 +121,7 @@ def getTrouteVariables(request, app_workspace): vars = get_troute_vars(df) else: vars = [] - except Exception as e: + except Exception: vars = [] return JsonResponse({"troute_variables": vars}) @@ -133,22 +133,15 @@ def getTrouteTimeSeries(request, app_workspace): clean_troute_id = troute_id.split("-")[1] variable_column = request.GET.get("troute_variable") df = get_troute_df(app_workspace) - df_sliced_by_id = df[df["feature_id"] == int(clean_troute_id)] - - df_sliced_by_id["t0"] = pd.to_datetime(df_sliced_by_id["t0"].iloc[0]) - - # Convert the time_offset column to timedelta - df_sliced_by_id["time"] = pd.to_timedelta(df_sliced_by_id["time"]) - - df_sliced_by_id["t1"] = df_sliced_by_id["t0"] + df_sliced_by_id["time"] + df_sliced_by_id = df[df["featureID"] == int(clean_troute_id)] try: - time_col = df_sliced_by_id["t1"] + time_col = df_sliced_by_id["current_time"] var_col = df_sliced_by_id[variable_column] data = [ {"x": time, "y": val} for time, val in zip(time_col.tolist(), var_col.tolist()) ] - except Exception as e: + except Exception: data = [] return JsonResponse({"data": data}) diff --git a/tethysapp/ngiab/utils.py b/tethysapp/ngiab/utils.py index 3f524b1..893edb0 100644 --- a/tethysapp/ngiab/utils.py +++ b/tethysapp/ngiab/utils.py @@ -19,7 +19,7 @@ def get_troute_df(app_workspace): def check_troute_id(df, id): - if int(id) in df["feature_id"].values: + if int(id) in df["featureID"].values: return True return False From 0e9594029d8082b073b7b6a16e31281108a514c3 Mon Sep 17 00:00:00 2001 From: romer8 Date: Tue, 29 Oct 2024 16:23:18 -0600 Subject: [PATCH 24/24] refactoring code --- .../components/catchmentsSelect.js | 102 ++++++++++++++++++ .../hydroFabric/components/nexusSelect.js | 57 ++++++++++ .../hydroFabric/components/trouteSelect.js | 74 +++++++++++++ reactapp/views/ngiab/hydroFabricView.js | 2 +- reactapp/views/ngiab/hydroFabricView2.js | 60 +++++++++++ reactapp/views/ngiab/ngiab_view.js | 2 +- 6 files changed, 295 insertions(+), 2 deletions(-) create mode 100644 reactapp/features/hydroFabric/components/catchmentsSelect.js create mode 100644 reactapp/features/hydroFabric/components/nexusSelect.js create mode 100644 reactapp/features/hydroFabric/components/trouteSelect.js create mode 100644 reactapp/views/ngiab/hydroFabricView2.js diff --git a/reactapp/features/hydroFabric/components/catchmentsSelect.js b/reactapp/features/hydroFabric/components/catchmentsSelect.js new file mode 100644 index 0000000..ab0f514 --- /dev/null +++ b/reactapp/features/hydroFabric/components/catchmentsSelect.js @@ -0,0 +1,102 @@ + + +import {useEffect, Suspense, Fragment,lazy} from 'react'; +import { useHydroFabricContext } from 'features/hydroFabric/hooks/useHydroFabricContext'; +import appAPI from 'services/api/app'; + +import SelectComponent from './selectComponent'; + +// const SelectComponent = lazy(() => import('selectComponent')); + +const CatchmentSelect = (props) => { + const {state,actions} = useHydroFabricContext(); + + useEffect(() => { + if (!state.catchment.id) return; + actions.reset_nexus(); + props.setIsLoading(true); + var params = { + catchment_id: state.catchment.id + } + appAPI.getCatchmentTimeSeries(params).then((response) => { + actions.set_catchment_series(response.data); + actions.set_catchment_variable_list(response.variables); + actions.set_catchment_variable(null); + actions.set_catchment_list(response.catchment_ids); + actions.set_troute_id(state.catchment.id); + props.toggleSingleRow(false); + props.setIsLoading(false); + }).catch((error) => { + props.setIsLoading(false); + console.log("Error fetching catchment time series", error); + }) + return () => { + if (state.catchment.id) return; + actions.reset_catchment(); + } + + }, [state.catchment.id]); + + + useEffect(() => { + if (!state.catchment.variable) return; + props.setIsLoading(true); + + var params = { + catchment_id: state.catchment.id, + variable_column: state.catchment.variable + } + appAPI.getCatchmentTimeSeries(params).then((response) => { + actions.set_catchment_series(response.data); + props.toggleSingleRow(false); + props.setIsLoading(false); + }).catch((error) => { + props.setIsLoading(false); + console.log("Error fetching nexus time series", error); + }) + return () => { + + } + + }, [state.catchment.variable]); + + + return ( + + {state.catchment.id && + +
Catchment Metadata
+

ID: {state.catchment.id}

+ + + + + + } + + + + + ); +}; + +export default CatchmentSelect; \ No newline at end of file diff --git a/reactapp/features/hydroFabric/components/nexusSelect.js b/reactapp/features/hydroFabric/components/nexusSelect.js new file mode 100644 index 0000000..cfaba04 --- /dev/null +++ b/reactapp/features/hydroFabric/components/nexusSelect.js @@ -0,0 +1,57 @@ + + +import {useEffect, Fragment,lazy} from 'react'; +import { useHydroFabricContext } from 'features/hydroFabric/hooks/useHydroFabricContext'; +import appAPI from 'services/api/app'; +import SelectComponent from './selectComponent'; +// const SelectComponent = lazy(() => import('selectComponent')); + +const NexusSelect = (props) => { + const {state,actions} = useHydroFabricContext(); + + useEffect(() => { + if (!state.nexus.id) return; + actions.reset_catchment(); + var params = { + nexus_id: state.nexus.id + } + appAPI.getNexusTimeSeries(params).then((response) => { + actions.set_nexus_series(response.data); + actions.set_nexus_list(response.nexus_ids); + actions.set_troute_id(state.nexus.id); + props.toggleSingleRow(false); + }).catch((error) => { + console.log("Error fetching nexus time series", error); + }) + return () => { + if(state.nexus.id) return + actions.reset_nexus(); + } + + }, [state.nexus.id]); + + + return ( + + {state.nexus.id && + +
Nexus Metadata
+

ID: {state.nexus.id}

+ + + + } + + ); +}; + +export default NexusSelect; \ No newline at end of file diff --git a/reactapp/features/hydroFabric/components/trouteSelect.js b/reactapp/features/hydroFabric/components/trouteSelect.js new file mode 100644 index 0000000..d591b8d --- /dev/null +++ b/reactapp/features/hydroFabric/components/trouteSelect.js @@ -0,0 +1,74 @@ + + +import {useEffect,Fragment,lazy} from 'react'; +import { useHydroFabricContext } from 'features/hydroFabric/hooks/useHydroFabricContext'; +import appAPI from 'services/api/app'; +import SelectComponent from './selectComponent'; + +// const SelectComponent = lazy(() => import('../../features/hydroFabric/components/selectComponent')); + +const TRouteSelect = (props) => { + const {state,actions} = useHydroFabricContext(); + + + useEffect(() => { + if (!state.troute.id) return; + props.setIsLoading(true); + var params = { + troute_id: state.troute.id + } + appAPI.getTrouteVariables(params).then((response) => { + actions.set_troute_variable_list(response.troute_variables); + props.toggleSingleRow(false); + props.setIsLoading(false); + }).catch((error) => { + props.setIsLoading(false); + console.log("Error fetching troute variables", error); + }) + return () => { + if (state.troute.id) return; + actions.reset_troute(); + } + },[state.troute.id]); + + + useEffect(() => { + if (!state.troute.variable || !state.troute.id) return; + props.setIsLoading(true); + var params = { + troute_id: state.troute.id, + troute_variable: state.troute.variable + } + appAPI.getTrouteTimeSeries(params).then((response) => { + actions.set_troute_series(response.data); + props.toggleSingleRow(false); + props.setIsLoading(false); + }).catch((error) => { + props.setIsLoading(false); + console.log("Error fetching troute time series", error); + }) + return () => { + if (state.troute.id) return; + actions.reset_troute(); + } + },[state.troute.variable]); + + + return ( + + { + state.troute.id && + +
Troute
+ + + + } + + ); +}; + +export default TRouteSelect; \ No newline at end of file diff --git a/reactapp/views/ngiab/hydroFabricView.js b/reactapp/views/ngiab/hydroFabricView.js index cda0412..fb55239 100644 --- a/reactapp/views/ngiab/hydroFabricView.js +++ b/reactapp/views/ngiab/hydroFabricView.js @@ -35,6 +35,7 @@ const HydroFabricView = (props) => { }, [state.nexus.id]); useEffect(() => { + console.log("here") if (!state.troute.id) return; props.setIsLoading(true); var params = { @@ -77,7 +78,6 @@ const HydroFabricView = (props) => { },[state.troute.variable]); - useEffect(() => { if (!state.catchment.id) return; actions.reset_nexus(); diff --git a/reactapp/views/ngiab/hydroFabricView2.js b/reactapp/views/ngiab/hydroFabricView2.js new file mode 100644 index 0000000..1cea049 --- /dev/null +++ b/reactapp/views/ngiab/hydroFabricView2.js @@ -0,0 +1,60 @@ + + +import {useEffect, Suspense, Fragment,lazy} from 'react'; +import { useHydroFabricContext } from 'features/hydroFabric/hooks/useHydroFabricContext'; +import appAPI from 'services/api/app'; +import { SelectContainer,HydroFabricPlotContainer } from './containers'; +import LoadingAnimation from 'components/loader/LoadingAnimation'; + + +const HydroFabricLinePlot = lazy(() => import('../../features/hydroFabric/components/hydroFabricLinePlot')); +const CatchmentSelectComponent = lazy(() => import('../../features/hydroFabric/components/catchmentsSelect')); +const NexusSelectComponent = lazy(() => import('../../features/hydroFabric/components/nexusSelect')); +const TrouteSelectComponent = lazy(() => import('../../features/hydroFabric/components/trouteSelect')); + +const HydroFabricView = (props) => { + const {state,actions} = useHydroFabricContext(); + + return ( + + + }> + + + + }> + + + + }> + + + + + + }> + + + + + + + + + + ); +}; + +export default HydroFabricView; \ No newline at end of file diff --git a/reactapp/views/ngiab/ngiab_view.js b/reactapp/views/ngiab/ngiab_view.js index 12c836a..087a7c4 100644 --- a/reactapp/views/ngiab/ngiab_view.js +++ b/reactapp/views/ngiab/ngiab_view.js @@ -3,7 +3,7 @@ import { HydroFabricProvider } from 'features/hydroFabric/providers/hydroFabricP import { MapProvider } from 'features/Map/providers/MapProvider'; import { HydroFabricContainer, MapContainer } from './containers'; import LoadingAnimation from 'components/loader/LoadingAnimation'; -const HydroFabricView = lazy(() => import('./hydroFabricView.js')); +const HydroFabricView = lazy(() => import('./hydroFabricView2.js')); const MapView = lazy(() => import('./map_view.js')); const NGIABView = () => {