diff --git a/config/experiments/llm_objectnav_hm3d.yaml b/config/experiments/llm_objectnav_hm3d.yaml index 5f69722..0f1b436 100644 --- a/config/experiments/llm_objectnav_hm3d.yaml +++ b/config/experiments/llm_objectnav_hm3d.yaml @@ -29,6 +29,7 @@ habitat_baselines: evaluate: True eval_ckpt_path_dir: dummy_policy.pth num_environments: 1 + load_resume_state_config: False torch_gpu_id: 0 tensorboard_dir: "tb" diff --git a/scripts/launch_vlm_servers.sh b/scripts/launch_vlm_servers.sh index be3a9a6..8ff882f 100644 --- a/scripts/launch_vlm_servers.sh +++ b/scripts/launch_vlm_servers.sh @@ -12,7 +12,7 @@ export OS_PYTHON=${OS_PYTHON:-`which python`} export MOBILE_SAM_CHECKPOINT=${MOBILE_SAM_CHECKPOINT:-data/mobile_sam.pt} export GROUNDING_DINO_CONFIG=${GROUNDING_DINO_CONFIG:-GroundingDINO/groundingdino/config/GroundingDINO_SwinT_OGC.py} export GROUNDING_DINO_WEIGHTS=${GROUNDING_DINO_WEIGHTS:-data/groundingdino_swint_ogc.pth} - +export CLASSES_PATH=${CLASSES_PATH:-zsos/vlm/classes.txt} # Create a detached tmux session tmux new-session -d -s vlm_servers diff --git a/zsos/policy/habitat_policies.py b/zsos/policy/habitat_policies.py index 8349d96..f4affa5 100644 --- a/zsos/policy/habitat_policies.py +++ b/zsos/policy/habitat_policies.py @@ -145,6 +145,7 @@ def _cache_observations( depth = filter_depth(depth.reshape(depth.shape[:2]), blur_type=None) # Habitat GPS makes west negative, so flip y camera_position = np.array([x, -y, self._camera_height]) + robot_xy = camera_position[:2] tf_camera_to_episodic = xyz_yaw_to_tf_matrix(camera_position, camera_yaw) self._obstacle_map: ObstacleMap @@ -157,6 +158,7 @@ def _cache_observations( self._fx, self._fy, self._camera_fov, + robot_xy, ) frontiers = self._obstacle_map.frontiers else: @@ -168,7 +170,7 @@ def _cache_observations( self._observations_cache = { "frontier_sensor": frontiers, "nav_depth": observations["depth"], # for pointnav - "robot_xy": camera_position[:2], + "robot_xy": robot_xy, "robot_heading": camera_yaw, "object_map_rgbd": [ ( diff --git a/zsos/vlm/coco_classes.py b/zsos/vlm/coco_classes.py new file mode 100644 index 0000000..e5f4df8 --- /dev/null +++ b/zsos/vlm/coco_classes.py @@ -0,0 +1,82 @@ +COCO_CLASSES = [ + "person", + "bicycle", + "car", + "motorcycle", + "airplane", + "bus", + "train", + "truck", + "boat", + "traffic", + "fire", + "stop", + "parking", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports", + "kite", + "baseball", + "baseball", + "skateboard", + "surfboard", + "tennis", + "bottle", + "wine", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot", + "pizza", + "donut", + "cake", + "chair", + "couch", + "potted", + "bed", + "dining", + "toilet", + "tv", + "laptop", + "mouse", + "remote", + "keyboard", + "cell", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy", + "hair", + "toothbrush", +]