Merge pull request #203 from SANDAG/ABM3_develop_max_disk_space

Get maximum disk usage, update disk space check

src/asim/configs/resident/settings_mp.yaml

@@ -77,6 +77,7 @@ models:
   # - write_to_datalake
   - update_tables
   - write_tables
+  # - check_disk_usage # Uncomment to get max disk usage at end of iteration 3 resident model
 
 
 multiprocess_steps:

src/asim/extensions/__init__.py

@@ -5,4 +5,5 @@
 from . import airport_returns
 # from . import write_to_datalake
 from . import update_tables
-from . import adjust_auto_operating_cost
+from . import adjust_auto_operating_cost
+from . import check_disk_usage

src/asim/extensions/check_disk_usage.py

@@ -0,0 +1,16 @@
+# ActivitySim
+# See full license in LICENSE.txt.
+import os
+import logging
+import win32com.client as win32
+
+from activitysim.core import inject
+
+logger = logging.getLogger("activitysim")
+
+@inject.step()
+def check_disk_usage():
+    output_dir = inject.get_injectable("output_dir")
+    path = os.path.abspath(os.path.join(output_dir, "..", ".."))
+    disk_usage = win32.Dispatch('Scripting.FileSystemObject').GetFolder(path).Size
+    logger.info("Disk space usage: %f GB" % (disk_usage / (1024 ** 3)))

src/main/emme/toolbox/master_run.py

@@ -861,6 +861,10 @@ def __call__(self, main_directory, scenario_id, scenario_title, emmebank_title,
 
         # UPLOAD DATA AND SWITCH PATHS
         if useLocalDrive:
+            # # Uncomment to get disk usage at end of run
+            # # Note that max disk usage occurs in resident model, not at end of run
+            # disk_usage = win32.Dispatch('Scripting.FileSystemObject').GetFolder(self._path).Size
+            # _m.logbook_write("Disk space usage: %f GB" % (disk_usage / (1024 ** 3)))
             file_manager("UPLOAD", main_directory, username, scenario_id,
                          delete_local_files=not skipDeleteIntermediateFiles)
             self._path = main_directory

src/main/resources/sandag_abm.properties

@@ -65,7 +65,7 @@ RunModel.skipDataLoadRequest = true
 RunModel.skipDeleteIntermediateFiles = false
 RunModel.MatrixPrecision = 0.0005
 # minimual space (MB) on C drive
-RunModel.minSpaceOnC = 400
+RunModel.minSpaceOnC = 430
 
 TNC.totalThreads = 10