Fix Voq chassis orchagent crash with 34K routes

orchagent/neighorch.cpp

@@ -1750,6 +1750,15 @@ void NeighOrch::doVoqSystemNeighTask(Consumer &consumer)
                 it++;
                 continue;
             }
+            if (m_syncdNeighbors.find(neighbor_entry) == m_syncdNeighbors.end())
+            {
+                NextHopKey nexthop = { ip_address, ibif.m_alias};
+                if (hasNextHop(nexthop))
+                {
+                   it++;
+                   continue;
+                }
+            }
 
             if (m_syncdNeighbors.find(neighbor_entry) == m_syncdNeighbors.end() ||
                     m_syncdNeighbors[neighbor_entry].mac != mac_address ||
@@ -2251,4 +2260,4 @@ bool NeighOrch::ifChangeInformRemoteNextHop(const string &alias, bool if_up)
         }
     }
     return rc;
-}
+}

tests/test_virtual_chassis.py

@@ -1121,6 +1121,125 @@ def test_remote_port_down(self, vct):
         # Cleanup inband if configuration
         self.del_inbandif_port(vct, inband_port)
 
+    def test_remote_neighbor_add(self, vct):
+        # test params
+        local_lc_switch_id = '0'
+        remote_lc_switch_id = '2'
+        test_prefix = "14.14.0.0/16"
+        inband_port = "Ethernet0"
+        test_neigh_ip_1 = "10.8.105.10"
+        test_neigh_dev_1 = "Ethernet4"
+        test_neigh_mac_1 = "00:02:03:04:05:06"
+        test_neigh_dev_2 = "Ethernet8"
+
+        print("Starting test_remote_neighbor_add")
+        local_lc_dvs = self.get_lc_dvs(vct, local_lc_switch_id)
+        remote_lc_dvs = self.get_lc_dvs(vct, remote_lc_switch_id)
+        # config inband port
+        self.config_inbandif_port(vct, inband_port)
+
+        # add neighbor
+        self.configure_neighbor(local_lc_dvs, "add", test_neigh_ip_1, test_neigh_mac_1, test_neigh_dev_1)
+
+        time.sleep(10)
+
+        asic_db = remote_lc_dvs.get_asic_db()
+        asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEIGHBOR_ENTRY", 1)
+        neighkeys = asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEIGHBOR_ENTRY")
+        print("1: Remote System Neigbors {}".format(neighkeys))
+        assert len(neighkeys), "No neigh entries in ASIC_DB"
+
+        # Check for presence of the remote neighbor in ASIC_DB
+        remote_neigh = ""
+        for nkey in neighkeys:
+            ne = ast.literal_eval(nkey)
+            if ne['ip'] == test_neigh_ip_1:
+               remote_neigh = nkey
+               break
+
+        assert remote_neigh != "", "Remote neigh not found in ASIC_DB"
+
+        # Preserve remote neigh asic db neigh key for delete verification later
+        test_remote_neigh_asic_db_key = remote_neigh
+
+
+        asic_db = remote_lc_dvs.get_asic_db()
+        nexthop_keys = asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", 1)
+        print("nexthop_keys:{}".format(nexthop_keys())
+        assert len(nexthop_keys), "No Nexthop entries in ASIC_DB"
+
+        nexthop_entry = asic_db.get_entry("ASIC_STATE:SAI_OBJECT_TYPE_NEXT_HOP", nexthop_keys[0])
+        ip = nexthop_entry.get("SAI_NEXT_HOP_ATTR_IP")
+        print("Ip address {}".format(ip)
+        assert ip != "", "Ip address not found for nexthop entry in asic db"
+        #assert ip == test_neigh_ip_1, "Remote nexthop programmed with wrong neighbor"
+
+        rif  = nexthop_entry.get("SAI_NEXT_HOP_ATTR_ROUTER_INTERFACE_ID")
+        print("RIF {}".format(rif)
+
+        # add route of LC1(pretend learnt via bgp)
+        _, res = remote_lc_dvs.runcmd(['sh', '-c', f"ip route add {test_prefix} nexthop via {test_neigh_ip_1}"])
+        assert res == "", "Error configuring route"
+        time.sleep(10)
+        _, output = remote_lc_dvs.runcmd("ip route show")
+        print("21: ip rotue show:".format(output))
+        #assert f"{test_prefix} dev {inband_port} lladdr {mac_address}" in output, "Kernel neigh not found for remote neighbor"
+
+        # del neighbor on first port and add it on second port
+        self.configure_neighbor(local_lc_dvs, "del", test_neigh_ip_1, test_neigh_mac_1, test_neigh_dev_1)
+        time.sleep(5)
+        self.configure_neighbor(local_lc_dvs, "add", test_neigh_ip_1, test_neigh_mac_1, test_neigh_dev_2)
+
+        time.sleep(10)
+
+        asic_db = remote_lc_dvs.get_asic_db()
+        asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEIGHBOR_ENTRY", 1)
+        neighkeys = asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEIGHBOR_ENTRY")
+        print("2: Remote System Neigbors {}".format(neighkeys))
+        assert len(neighkeys), "No neigh entries in ASIC_DB"
+
+        # Check for presence of the remote neighbor in ASIC_DB
+        remote_neigh = ""
+        for nkey in neighkeys:
+            ne = ast.literal_eval(nkey)
+            if ne['ip'] == test_neigh_ip_1:
+               remote_neigh = nkey
+               break
+
+        assert remote_neigh != "", "Remote neigh not found in ASIC_DB"
+
+        _, res = remote_lc_dvs.runcmd(['sh', '-c', f"ip route del {test_prefix} nexthop via {test_neigh_ip_1} "])
+        assert res == "", "Error configuring route"
+
+        time.sleep(10)
+
+        asic_db = remote_lc_dvs.get_asic_db()
+        asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEIGHBOR_ENTRY", 1)
+        neighkeys = asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEIGHBOR_ENTRY")
+        print("3: Remote System Neigbors {}".format(neighkeys))
+        assert len(neighkeys), "No neigh entries in ASIC_DB"
+
+        # Check for presence of the remote neighbor in ASIC_DB
+        remote_neigh = ""
+        for nkey in neighkeys:
+            ne = ast.literal_eval(nkey)
+            if ne['ip'] == test_neigh_ip_1:
+               remote_neigh = nkey
+               break
+
+        assert remote_neigh != "", "Remote neigh not found in ASIC_DB"
+
+        self.configure_neighbor(local_lc_dvs, "del", test_neigh_ip_1, test_neigh_mac_1, test_neigh_dev_2)
+        time.sleep(10)
+        asic_db = remote_lc_dvs.get_asic_db()
+        asic_db.wait_for_n_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEIGHBOR_ENTRY", 1)
+        neighkeys = asic_db.get_keys("ASIC_STATE:SAI_OBJECT_TYPE_NEIGHBOR_ENTRY")
+        print("4: Remote System Neigbors {}".format(neighkeys))
+        assert len(neighkeys) == 0, "No neigh entries in ASIC_DB"
+
+        # Cleanup inband if configuration
+        self.del_inbandif_port(vct, inband_port)
+
     def test_voq_drop_counters(self, vct):
         """Test VOQ switch drop counters.