Compatible version with bdai merge including HW experiment code (#19)

Co-authored-by: Katrina Ashton <kashton@theaiinstitute.com>

.pre-commit-config.yaml

@@ -1,46 +1,81 @@
-# Copyright [2023] Boston Dynamics AI Institute, Inc.
+# Copyright (c) 2023 Boston Dynamics AI Institute LLC. All rights reserved.
 
 repos:
--   repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.263'
-    hooks:
-    -   id: ruff
-        args: ['--fix', '--config', 'pyproject.toml']
+- repo: https://github.com/charliermarsh/ruff-pre-commit
+  # Ruff version.
+  rev: 'v0.1.0'
+  hooks:
+    - id: ruff
+      args: ['--fix', '--config', 'pyproject.toml'] # we want this to refer to `bdai/pyproject.toml`
 
+- repo: https://github.com/psf/black
+  rev: 23.10.0
+  hooks:
+    - id: black
+      language_version: python3.10
+      args: ['--config', 'pyproject.toml'] # we want this to refer to `bdai/pyproject.toml`
+      verbose: true
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.5.0
   hooks:
   -   id: check-yaml
-      args: ['--unsafe']
+      args: ['--unsafe'] # details about the unsafe flag:
+      # https://github.com/pre-commit/pre-commit-hooks#check-yaml
+      # This is the solution proposed to prevent `check-yaml` from failing on custom tags: 
+      # https://github.com/pre-commit/pre-commit-hooks/issues/701
   -   id: check-added-large-files
       args: ['--enforce-all', '--maxkb', '200']
+      # For the `exclude` argument, see https://pre-commit.com/#regular-expressions
+      # Make sure to escape strings correctly to ensure literal matches, for example, using Python
+      #
+      # >>> print(re.escape("path/to-some/file.ext"))
+      # path/to\-some/file\.ext
+      #
+      # `path/to\-some/file\.ext` is the correct regex pattern to match `path/to-some/file.ext` literally.
+      # The inverse operation is more cumbersome: https://stackoverflow.com/a/54840284
+      exclude: |
+        (?x)^(
+          docker/ros/web/static/novnc/vendor/browser\-es\-module\-loader/dist/babel\-worker\.js|
+          docker/ros/rootfs/usr/local/share/doro\-lxde\-wallpapers/bg.*\.jpg|
+          docker/ros/web/yarn\.lock|
+          src/modelzoo/detic/lvis_v1_train_cat_info\.json|
+          src/modelzoo/edge_grasp_serve/example_pc\.npy
+        )$
   -   id: check-toml
   -   id: end-of-file-fixer
   -   id: check-merge-conflict
   -   id: check-executables-have-shebangs
   -   id: check-shebang-scripts-are-executable
-
--   repo: https://github.com/psf/black
-    rev: 23.3.0
+-   repo: https://github.com/ssciwr/clang-format-hook.git
+    rev: v12.0.1  # Use the sha / tag you want to point at
     hooks:
-    -   id: black
-        language_version: python3.9
-        args: ['--config', 'pyproject.toml']
-        verbose: true
-
--   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.2.0
+    -   id: clang-format
+        types_or: [c++, c, cuda]
+-   repo: https://github.com/cpplint/cpplint.git
+    rev: 1.6.1
     hooks:
-    -   id: mypy
-        pass_filenames: false
-        additional_dependencies:
-        - types-protobuf
-        - types-requests
-        - types-simplejson
-        - types-ujson
-        - types-PyYAML
-        - types-toml
-        - types-six
+    -   id: cpplint
+        args: ['--quiet']
+        exclude_types: [cuda]
+#-   repo: https://github.com/leoll2/copyright_notice_precommit
+#    rev: 0.1.1
+#    hooks:
+#    -   id: copyright-notice
+#        args: [--notice=copyright.txt]
+
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.6.1
+  hooks:
+  -   id: mypy
+      pass_filenames: false
+      additional_dependencies:
+      - types-protobuf
+      - types-requests
+      - types-simplejson
+      - types-ujson
+      - types-PyYAML
+      - types-toml
+      - types-six
 
 - repo: https://github.com/jumanjihouse/pre-commit-hooks
   rev: 3.0.0

config/experiments/reality.yaml

@@ -10,7 +10,7 @@ policy:
     name: "RealityITMPolicyV2"
     pointnav_policy_path: "data/pointnav_weights.pth"
     depth_image_shape: [212, 240]  # height, width
-    pointnav_stop_radius: 1.2
+    pointnav_stop_radius: 0.9
     use_max_confidence: False
     object_map_erosion_size: 5
     exploration_thresh: 0.0
@@ -21,7 +21,7 @@ policy:
     agent_radius: 0.2
 
 env:
-  max_body_cam_depth: 3.5
+  max_body_cam_depth: 2.5
   max_gripper_cam_depth: 5.0
   max_lin_dist: 0.2
   max_ang_dist: 0.523599

pyproject.toml

@@ -12,14 +12,14 @@ authors = [
     {name = "Naoki Yokoyama", email = "nyokoyama@theaiinstitute.com"},
 ]
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 dependencies = [
     "torch >= 1.10.1",
     "numpy >= 1.22.4",
     "flask >= 2.3.2",
     "seaborn >= 0.12.2",  # required by yolov7
     "open3d >= 0.17.0",
-    "transformers == 4.26.0",  # higher versions break BLIP-2
+    "transformers >= 4.28.1",  # higher versions than 4.26.0 "break" BLIP-2 but need 4.28.1 for integration
     "salesforce-lavis >= 1.0.2",  # for BLIP-2
     "frontier_exploration @ git+https://github.com/naokiyokoyama/frontier_exploration.git",
     "mobile_sam @ git+https://github.com/ChaoningZhang/MobileSAM.git",
@@ -49,8 +49,8 @@ reality = [
 "Homepage" = "theaiinstitute.com"
 "GitHub" = "https://github.com/bdaiinstitute/vlfm"
 
-[tool.setuptools]
-packages = ["vlfm", "config"]
+[tool.setuptools.packages.find]
+where = ["vlfm"]
 
 [tool.ruff]
 # Enable pycodestyle (`E`), Pyflakes (`F`), and import sorting (`I`)
@@ -92,8 +92,8 @@ line-length = 120
 # Allow unused variables when underscore-prefixed.
 dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
 
-# Assume Python 3.9.
-target-version = "py39"
+# Assume Python 3.10
+target-version = "py310"
 
 [tool.ruff.per-file-ignores]
 "__init__.py" = ["F401"]
@@ -103,8 +103,8 @@ target-version = "py39"
 max-complexity = 10
 
 [tool.black]
-line-length = 88
-target-version = ['py39']
+line-length = 120
+target-version = ['py310']
 include = '\.pyi?$'
 # `extend-exclude` is not honored when `black` is passed a file path explicitly,
 # as is typical when `black` is invoked via `pre-commit`.
@@ -116,9 +116,13 @@ force-exclude = '''
 
 preview = true
 
+[tool.coverage.run]
+relative_files = true
+
+
 # mypy configuration
 [tool.mypy]
-python_version = "3.9"
+python_version = "3.10"
 disallow_untyped_defs = true
 ignore_missing_imports = true
 explicit_package_bases = true
@@ -127,5 +131,5 @@ strict_equality = true
 warn_unreachable = true
 warn_redundant_casts = true
 no_implicit_optional = true
-files = ['vlfm']
-exclude = '^(docker|.*external|.*thirdparty|.*install|.*build|.*_experimental)/'
+files = ['vlfm','test','scripts']
+exclude = "^(docker|.*external|.*thirdparty|.*install|.*build|.*_experimental|.*_pb2.py|.*_pb2_grpc.py)"

scripts/parse_jsons.py

@@ -46,9 +46,7 @@ def calculate_frequencies(failure_causes: List[str]) -> None:
     for cause, count in counter.most_common():
         percentage = (count / total) * 100
         # Add each row to the table
-        table.add_row(
-            [cause.replace("did_not_fail", "succeeded!"), count, f"{percentage:.2f}%"]
-        )
+        table.add_row([cause.replace("did_not_fail", "succeeded!"), count, f"{percentage:.2f}%"])
 
     print(table)
 
@@ -60,10 +58,7 @@ def calculate_avg_performance(stats: List[Dict[str, Any]]) -> None:
     Args:
         stats (List[Dict[str, Any]]): A list of stats for each episode.
     """
-    success, spl, soft_spl = [
-        [episode.get(k, -1) for episode in stats]
-        for k in ["success", "spl", "soft_spl"]
-    ]
+    success, spl, soft_spl = [[episode.get(k, -1) for episode in stats] for k in ["success", "spl", "soft_spl"]]
 
     # Create a table with headers
     table = PrettyTable(["Metric", "Average"])
@@ -101,28 +96,23 @@ def calculate_avg_fail_per_category(stats: List[Dict[str, Any]]) -> None:
     table = PrettyTable(["Category", "Average Failure Rate"])
 
     # Add each row to the table
-    for category, stats in sorted(
+    for category, c_stats in sorted(
         category_stats.items(),
         key=lambda x: x[1]["fail_count"],
         reverse=True,
     ):
-        avg_failure_rate = (stats["fail_count"] / stats["total_count"]) * 100
+        avg_failure_rate = (c_stats["fail_count"] / c_stats["total_count"]) * 100
         table.add_row(
             [
                 category,
-                (
-                    f"{avg_failure_rate:.2f}% ({stats['fail_count']}/"
-                    f"{stats['total_count']})"
-                ),
+                f"{avg_failure_rate:.2f}% ({c_stats['fail_count']}/{c_stats['total_count']})",
             ]
         )
 
     print(table)
 
 
-def calculate_avg_fail_rate_per_category(
-    stats: List[Dict[str, Any]], failure_cause: str
-) -> None:
+def calculate_avg_fail_rate_per_category(stats: List[Dict[str, Any]], failure_cause: str) -> None:
     """
     For each possible "target_object", count the number of times the agent failed due to
     the given failure cause. Then, sum the counts across all categories and use it to
@@ -147,9 +137,7 @@ def calculate_avg_fail_rate_per_category(
     table = PrettyTable(["Category", f"% Occurrence for {failure_cause}"])
 
     # Sort the categories by their failure count in descending order
-    sorted_categories = sorted(
-        category_to_fail_count.items(), key=lambda x: x[1], reverse=True
-    )
+    sorted_categories = sorted(category_to_fail_count.items(), key=lambda x: x[1], reverse=True)
 
     # Add each row to the table
     for category, count in sorted_categories:

test/test_setup.py

@@ -7,7 +7,7 @@
 from vlfm.utils.generate_dummy_policy import save_dummy_policy
 
 
-def test_load_and_save_config():
+def test_load_and_save_config() -> None:
     if not os.path.exists("data"):
         os.makedirs("data")
 

test/test_visualization.py

@@ -7,7 +7,7 @@
 from vlfm.utils.visualization import generate_text_image
 
 
-def test_visualization():
+def test_visualization() -> None:
     if not os.path.exists("build"):
         os.makedirs("build")
 

vlfm/mapping/base_map.py

@@ -12,9 +12,7 @@ class BaseMap:
     _last_camera_yaw: float = 0.0
     _map_dtype: np.dtype = np.dtype(np.float32)
 
-    def __init__(
-        self, size: int = 1000, pixels_per_meter: int = 20, *args: Any, **kwargs: Any
-    ):
+    def __init__(self, size: int = 1000, pixels_per_meter: int = 20, *args: Any, **kwargs: Any):
         """
         Args:
             size: The size of the map in pixels.
@@ -23,16 +21,12 @@ def __init__(
         self.size = size
         self._map = np.zeros((size, size), dtype=self._map_dtype)
         self._episode_pixel_origin = np.array([size // 2, size // 2])
-        self._traj_vis = TrajectoryVisualizer(
-            self._episode_pixel_origin, self.pixels_per_meter
-        )
+        self._traj_vis = TrajectoryVisualizer(self._episode_pixel_origin, self.pixels_per_meter)
 
     def reset(self) -> None:
         self._map.fill(0)
         self._camera_positions = []
-        self._traj_vis = TrajectoryVisualizer(
-            self._episode_pixel_origin, self.pixels_per_meter
-        )
+        self._traj_vis = TrajectoryVisualizer(self._episode_pixel_origin, self.pixels_per_meter)
 
     def update_agent_traj(self, robot_xy: np.ndarray, robot_heading: float) -> None:
         self._camera_positions.append(robot_xy)
@@ -47,10 +41,7 @@ def _xy_to_px(self, points: np.ndarray) -> np.ndarray:
         Returns:
             The array of (x, y) pixel coordinates.
         """
-        px = (
-            np.rint(points[:, ::-1] * self.pixels_per_meter)
-            + self._episode_pixel_origin
-        )
+        px = np.rint(points[:, ::-1] * self.pixels_per_meter) + self._episode_pixel_origin
         px[:, 0] = self._map.shape[0] - px[:, 0]
         return px.astype(int)
 

vlfm/mapping/frontier_map.py

@@ -22,9 +22,7 @@ def __init__(self, encoding_type: str = "cosine"):
     def reset(self) -> None:
         self.frontiers = []
 
-    def update(
-        self, frontier_locations: List[np.ndarray], curr_image: np.ndarray, text: str
-    ) -> None:
+    def update(self, frontier_locations: List[np.ndarray], curr_image: np.ndarray, text: str) -> None:
         """
         Takes in a list of frontier coordinates and the current image observation from
         the robot. Any stored frontiers that are not present in the given list are
@@ -41,19 +39,14 @@ def update(
         self.frontiers = [
             frontier
             for frontier in self.frontiers
-            if any(
-                np.array_equal(frontier.xyz, location)
-                for location in frontier_locations
-            )
+            if any(np.array_equal(frontier.xyz, location) for location in frontier_locations)
         ]
 
         # Add any frontiers that are not already stored. Set their image field to the
         # given image.
         cosine = None
         for location in frontier_locations:
-            if not any(
-                np.array_equal(frontier.xyz, location) for frontier in self.frontiers
-            ):
+            if not any(np.array_equal(frontier.xyz, location) for frontier in self.frontiers):
                 if cosine is None:
                     cosine = self._encode(curr_image, text)
                 self.frontiers.append(Frontier(location, cosine))