ofctrl: Introduce ecmp_nexthop_monitor.

Introduce ecmp_nexthop_monitor in ovn-controller in order to track and
flush ecmp-symmetric reply ct entires when requested by the CMS (e.g
removing the related static ecmp routes). CT entries are flushed using
the ethernet mac address stored in ct_label.

Signed-off-by: Lorenzo Bianconi <[email protected]>

NEWS

@@ -17,6 +17,8 @@ Post v24.09.0
      ECMP-nexthop.
      By default ovn-controller continuously sends ARP/ND packets for
      ECMP-nexthop.
+   - Introduce ovn-controller ECMP_nexthop monitor in order to flush stale ct
+     entries when related ecmp routes are removed by the CMS.
 
 OVN v24.09.0 - 13 Sep 2024
 --------------------------

controller/automake.mk

@@ -51,7 +51,9 @@ controller_ovn_controller_SOURCES = \
 	controller/ct-zone.h \
 	controller/ct-zone.c \
 	controller/ovn-dns.c \
-	controller/ovn-dns.h
+	controller/ovn-dns.h \
+	controller/ecmp-next-hop-monitor.h \
+	controller/ecmp-next-hop-monitor.c
 
 controller_ovn_controller_LDADD = lib/libovn.la $(OVS_LIBDIR)/libopenvswitch.la
 man_MANS += controller/ovn-controller.8

controller/ecmp-next-hop-monitor.c

@@ -0,0 +1,184 @@
+/* Copyright (c) 2024, Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <config.h>
+#include "ct-zone.h"
+#include "lib/ovn-util.h"
+#include "lib/simap.h"
+#include "openvswitch/hmap.h"
+#include "openvswitch/ofp-ct.h"
+#include "openvswitch/rconn.h"
+#include "openvswitch/vlog.h"
+#include "ovn/logical-fields.h"
+#include "ovn-sb-idl.h"
+#include "controller/ecmp-next-hop-monitor.h"
+
+VLOG_DEFINE_THIS_MODULE(ecmp_next_hop_monitor);
+
+static struct hmap ecmp_nexthop;
+
+struct ecmp_nexthop_data {
+    struct hmap_node hmap_node;
+    uint16_t zone_id;
+    char *nexthop;
+    char *mac;
+};
+
+void ecmp_nexthop_init(void)
+{
+    hmap_init(&ecmp_nexthop);
+}
+
+static void
+ecmp_nexthop_erase_entry(struct ecmp_nexthop_data *e)
+{
+    free(e->nexthop);
+    free(e->mac);
+    free(e);
+}
+
+static void
+ecmp_nexthop_destroy_map(struct hmap *map)
+{
+    struct ecmp_nexthop_data *e;
+    HMAP_FOR_EACH_POP (e, hmap_node, map) {
+        ecmp_nexthop_erase_entry(e);
+    }
+    hmap_destroy(map);
+}
+
+void ecmp_nexthop_destroy(void)
+{
+    ecmp_nexthop_destroy_map(&ecmp_nexthop);
+}
+
+static struct ecmp_nexthop_data *
+ecmp_nexthop_alloc_entry(const char *nexthop, const char *mac,
+                         const uint16_t zone_id, struct hmap *map)
+{
+    struct ecmp_nexthop_data *e = xmalloc(sizeof *e);
+    e->nexthop = xstrdup(nexthop);
+    e->mac = xstrdup(mac);
+    e->zone_id = zone_id;
+
+    uint32_t hash = hash_string(nexthop, 0);
+    hash = hash_add(hash, hash_string(mac, 0));
+    hash = hash_add(hash, zone_id);
+    hmap_insert(map, &e->hmap_node, hash);
+
+    return e;
+}
+
+static struct ecmp_nexthop_data *
+ecmp_nexthop_find_entry(const char *nexthop, const char *mac,
+                        const uint16_t zone_id, struct hmap *map)
+{
+    uint32_t hash = hash_string(nexthop, 0);
+    hash = hash_add(hash, hash_string(mac, 0));
+    hash = hash_add(hash, zone_id);
+
+    struct ecmp_nexthop_data *e;
+    HMAP_FOR_EACH_WITH_HASH (e, hmap_node, hash, map) {
+        if (!strcmp(e->nexthop, nexthop) &&
+            !strcmp(e->mac, mac) && e->zone_id == zone_id) {
+            return e;
+        }
+    }
+    return NULL;
+}
+
+static void
+ecmp_nexthop_monitor_flush_ct_entry(const struct rconn *swconn,
+                                    const char *mac, uint16_t zone_id,
+                                    struct ovs_list *msgs)
+{
+    struct eth_addr ea;
+    if (!ovs_scan(mac, ETH_ADDR_SCAN_FMT, ETH_ADDR_SCAN_ARGS(ea))) {
+        return;
+    }
+
+    ovs_u128 mask = {
+        /* ct_label.ecmp_reply_eth BITS[32-79] */
+        .u64.hi = OVN_CT_ECMP_ETH_HIGH,
+        .u64.lo = OVN_CT_ECMP_ETH_LOW,
+    };
+
+    ovs_be32 lo = get_unaligned_be32((void *)&ea.be16[1]);
+    ovs_u128 nexthop = {
+        .u64.hi = ntohs(ea.be16[0]),
+        .u64.lo = (uint64_t) ntohl(lo) << 32,
+    };
+
+    struct ofp_ct_match match = {
+        .labels = nexthop,
+        .labels_mask = mask,
+    };
+    struct ofpbuf *msg = ofp_ct_match_encode(&match, &zone_id,
+                                             rconn_get_version(swconn));
+    ovs_list_push_back(msgs, &msg->list_node);
+}
+
+void
+ecmp_nexthop_monitor_run(const struct sbrec_ecmp_nexthop_table *enh_table,
+                         const struct shash *current_ct_zones,
+                         const struct rconn *swconn, struct ovs_list *msgs)
+{
+    struct hmap sb_ecmp_nexthop = HMAP_INITIALIZER(&sb_ecmp_nexthop);
+
+    const struct sbrec_ecmp_nexthop *sbrec_ecmp_nexthop;
+    SBREC_ECMP_NEXTHOP_TABLE_FOR_EACH (sbrec_ecmp_nexthop, enh_table) {
+        struct sbrec_port_binding *pb = sbrec_ecmp_nexthop->port;
+        if (!pb) {
+            continue;
+        }
+
+        const char *dp_name = smap_get(&pb->datapath->external_ids, "name");
+        if (!dp_name) {
+            continue;
+        }
+
+        char *name = xasprintf("%s_dnat", dp_name);
+        struct ct_zone *ct_zone = shash_find_data(current_ct_zones, name);
+        free(name);
+
+        if (!ct_zone) {
+            continue;
+        }
+
+        if (!ecmp_nexthop_find_entry(sbrec_ecmp_nexthop->nexthop,
+                                     sbrec_ecmp_nexthop->mac, ct_zone->zone,
+                                     &ecmp_nexthop)) {
+            ecmp_nexthop_alloc_entry(sbrec_ecmp_nexthop->nexthop,
+                                     sbrec_ecmp_nexthop->mac,
+                                     ct_zone->zone, &ecmp_nexthop);
+        }
+        ecmp_nexthop_alloc_entry(sbrec_ecmp_nexthop->nexthop,
+                                 sbrec_ecmp_nexthop->mac, ct_zone->zone,
+                                 &sb_ecmp_nexthop);
+    }
+
+    struct ecmp_nexthop_data *e;
+    HMAP_FOR_EACH_SAFE (e, hmap_node, &ecmp_nexthop) {
+        if (!ecmp_nexthop_find_entry(e->nexthop, e->mac, e->zone_id,
+                                     &sb_ecmp_nexthop)) {
+            ecmp_nexthop_monitor_flush_ct_entry(swconn, e->mac,
+                                                e->zone_id, msgs);
+            hmap_remove(&ecmp_nexthop, &e->hmap_node);
+            ecmp_nexthop_erase_entry(e);
+        }
+    }
+
+    ecmp_nexthop_destroy_map(&sb_ecmp_nexthop);
+}

controller/ecmp-next-hop-monitor.h

@@ -0,0 +1,25 @@
+/* Copyright (c) 2024, Red Hat, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef OVN_CMP_NEXT_HOP_MONITOR_H
+#define OVN_CMP_NEXT_HOP_MONITOR_H
+
+void ecmp_nexthop_init(void);
+void ecmp_nexthop_destroy(void);
+void ecmp_nexthop_monitor_run(const struct sbrec_ecmp_nexthop_table *enh_table,
+                              const struct shash *current_ct_zones,
+                              const struct rconn *swconn,
+                              struct ovs_list *msgs);
+#endif /* OVN_CMP_NEXT_HOP_MONITOR_H */

controller/ofctrl.c

@@ -54,6 +54,7 @@
 #include "vswitch-idl.h"
 #include "ovn-sb-idl.h"
 #include "ct-zone.h"
+#include "ecmp-next-hop-monitor.h"
 
 VLOG_DEFINE_THIS_MODULE(ofctrl);
 
@@ -425,6 +426,7 @@ ofctrl_init(struct ovn_extend_table *group_table,
     tx_counter = rconn_packet_counter_create();
     hmap_init(&installed_lflows);
     hmap_init(&installed_pflows);
+    ecmp_nexthop_init();
     ovs_list_init(&flow_updates);
     ovn_init_symtab(&symtab);
     groups = group_table;
@@ -877,6 +879,7 @@ ofctrl_destroy(void)
     expr_symtab_destroy(&symtab);
     shash_destroy(&symtab);
     ofctrl_meter_bands_destroy();
+    ecmp_nexthop_destroy();
 }
 
 uint64_t
@@ -2662,8 +2665,10 @@ void
 ofctrl_put(struct ovn_desired_flow_table *lflow_table,
            struct ovn_desired_flow_table *pflow_table,
            struct shash *pending_ct_zones,
+           struct shash *current_ct_zones,
            struct hmap *pending_lb_tuples,
            struct ovsdb_idl_index *sbrec_meter_by_name,
+           const struct sbrec_ecmp_nexthop_table *enh_table,
            uint64_t req_cfg,
            bool lflows_changed,
            bool pflows_changed)
@@ -2704,6 +2709,8 @@ ofctrl_put(struct ovn_desired_flow_table *lflow_table,
     /* OpenFlow messages to send to the switch to bring it up-to-date. */
     struct ovs_list msgs = OVS_LIST_INITIALIZER(&msgs);
 
+    ecmp_nexthop_monitor_run(enh_table, current_ct_zones, swconn, &msgs);
+
     /* Iterate through ct zones that need to be flushed. */
     struct shash_node *iter;
     SHASH_FOR_EACH(iter, pending_ct_zones) {

controller/ofctrl.h

@@ -31,6 +31,7 @@ struct ofpbuf;
 struct ovsrec_bridge;
 struct ovsrec_open_vswitch_table;
 struct sbrec_meter_table;
+struct sbrec_ecmp_nexthop_table;
 struct shash;
 
 struct ovn_desired_flow_table {
@@ -57,8 +58,10 @@ enum mf_field_id ofctrl_get_mf_field_id(void);
 void ofctrl_put(struct ovn_desired_flow_table *lflow_table,
                 struct ovn_desired_flow_table *pflow_table,
                 struct shash *pending_ct_zones,
+                struct shash *current_ct_zones,
                 struct hmap *pending_lb_tuples,
                 struct ovsdb_idl_index *sbrec_meter_by_name,
+                const struct sbrec_ecmp_nexthop_table *enh_table,
                 uint64_t nb_cfg,
                 bool lflow_changed,
                 bool pflow_changed);

controller/ovn-controller.c

@@ -5827,8 +5827,11 @@ main(int argc, char *argv[])
                         ofctrl_put(&lflow_output_data->flow_table,
                                    &pflow_output_data->flow_table,
                                    &ct_zones_data->ctx.pending,
+                                   &ct_zones_data->ctx.current,
                                    &lb_data->removed_tuples,
                                    sbrec_meter_by_name,
+                                   sbrec_ecmp_nexthop_table_get(
+                                        ovnsb_idl_loop.idl),
                                    ofctrl_seqno_get_req_cfg(),
                                    engine_node_changed(&en_lflow_output),
                                    engine_node_changed(&en_pflow_output));

include/ovn/logical-fields.h

@@ -213,6 +213,9 @@ const struct ovn_field *ovn_field_from_name(const char *name);
 #define OVN_CT_ECMP_ETH_1ST_BIT 32
 #define OVN_CT_ECMP_ETH_END_BIT 79
 
+#define OVN_CT_ECMP_ETH_LOW     (((1ULL << OVN_CT_ECMP_ETH_1ST_BIT) - 1) << 32)
+#define OVN_CT_ECMP_ETH_HIGH    ((1ULL << (OVN_CT_ECMP_ETH_END_BIT - 63)) - 1)
+
 #define OVN_CT_STR(LABEL_VALUE) OVS_STRINGIZE(LABEL_VALUE)
 #define OVN_CT_MASKED_STR(LABEL_VALUE) \
     OVS_STRINGIZE(LABEL_VALUE) "/" OVS_STRINGIZE(LABEL_VALUE)