fix breaking change

bdaiinstitute · Aug 30, 2023 · 584c922 · 584c922
1 parent ef3ae41
commit 584c922
Show file tree

Hide file tree

Showing 4 changed files with 87 additions and 2 deletions.
diff --git a/config/experiments/llm_objectnav_hm3d.yaml b/config/experiments/llm_objectnav_hm3d.yaml
@@ -29,6 +29,7 @@ habitat_baselines:
   evaluate: True
   eval_ckpt_path_dir: dummy_policy.pth
   num_environments: 1
+  load_resume_state_config: False
 
   torch_gpu_id: 0
   tensorboard_dir: "tb"

diff --git a/scripts/launch_vlm_servers.sh b/scripts/launch_vlm_servers.sh
@@ -12,7 +12,7 @@ export OS_PYTHON=${OS_PYTHON:-`which python`}
 export MOBILE_SAM_CHECKPOINT=${MOBILE_SAM_CHECKPOINT:-data/mobile_sam.pt}
 export GROUNDING_DINO_CONFIG=${GROUNDING_DINO_CONFIG:-GroundingDINO/groundingdino/config/GroundingDINO_SwinT_OGC.py}
 export GROUNDING_DINO_WEIGHTS=${GROUNDING_DINO_WEIGHTS:-data/groundingdino_swint_ogc.pth}
-
+export CLASSES_PATH=${CLASSES_PATH:-zsos/vlm/classes.txt}
 
 # Create a detached tmux session
 tmux new-session -d -s vlm_servers

diff --git a/zsos/policy/habitat_policies.py b/zsos/policy/habitat_policies.py
@@ -145,6 +145,7 @@ def _cache_observations(
         depth = filter_depth(depth.reshape(depth.shape[:2]), blur_type=None)
         # Habitat GPS makes west negative, so flip y
         camera_position = np.array([x, -y, self._camera_height])
+        robot_xy = camera_position[:2]
         tf_camera_to_episodic = xyz_yaw_to_tf_matrix(camera_position, camera_yaw)
 
         self._obstacle_map: ObstacleMap
@@ -157,6 +158,7 @@ def _cache_observations(
                 self._fx,
                 self._fy,
                 self._camera_fov,
+                robot_xy,
             )
             frontiers = self._obstacle_map.frontiers
         else:
@@ -168,7 +170,7 @@ def _cache_observations(
         self._observations_cache = {
             "frontier_sensor": frontiers,
             "nav_depth": observations["depth"],  # for pointnav
-            "robot_xy": camera_position[:2],
+            "robot_xy": robot_xy,
             "robot_heading": camera_yaw,
             "object_map_rgbd": [
                 (

diff --git a/zsos/vlm/coco_classes.py b/zsos/vlm/coco_classes.py
@@ -0,0 +1,82 @@
+COCO_CLASSES = [
+    "person",
+    "bicycle",
+    "car",
+    "motorcycle",
+    "airplane",
+    "bus",
+    "train",
+    "truck",
+    "boat",
+    "traffic",
+    "fire",
+    "stop",
+    "parking",
+    "bench",
+    "bird",
+    "cat",
+    "dog",
+    "horse",
+    "sheep",
+    "cow",
+    "elephant",
+    "bear",
+    "zebra",
+    "giraffe",
+    "backpack",
+    "umbrella",
+    "handbag",
+    "tie",
+    "suitcase",
+    "frisbee",
+    "skis",
+    "snowboard",
+    "sports",
+    "kite",
+    "baseball",
+    "baseball",
+    "skateboard",
+    "surfboard",
+    "tennis",
+    "bottle",
+    "wine",
+    "cup",
+    "fork",
+    "knife",
+    "spoon",
+    "bowl",
+    "banana",
+    "apple",
+    "sandwich",
+    "orange",
+    "broccoli",
+    "carrot",
+    "hot",
+    "pizza",
+    "donut",
+    "cake",
+    "chair",
+    "couch",
+    "potted",
+    "bed",
+    "dining",
+    "toilet",
+    "tv",
+    "laptop",
+    "mouse",
+    "remote",
+    "keyboard",
+    "cell",
+    "microwave",
+    "oven",
+    "toaster",
+    "sink",
+    "refrigerator",
+    "book",
+    "clock",
+    "vase",
+    "scissors",
+    "teddy",
+    "hair",
+    "toothbrush",
+]