From e1075499422bba574702d0cfea7cc85e340293b6 Mon Sep 17 00:00:00 2001 From: anamehra <54692434+anamehra@users.noreply.github.com> Date: Wed, 12 Apr 2023 00:07:42 -0700 Subject: [PATCH] chassis-packet: resolve the missing static routes (#14593) Why I did it Fixes #14179 chassis-packet: missing arp entries for static routes causing high orchagent cpu usage It is observed that some sonic-mgmt test case calls sonic-clear arp, which clears the static arp entries as well. Orchagent or arp_update process does not try to resolve the missing arp entries after clear. How I did it arp_update should resolve the missing arp/ndp static route entries. Added code to check for missing entries and try ping if any found to resolve it. How to verify it After boot or config reload, check ipv4 and ipv4 neigh entries to make sure all static route entries are present manual validation: Use sonic-clear arp and sonic-clear ndp to clear all neighbor entries run arp_update Check for neigh entries. All entries should be present. Testing on T0 setup route/for test_static_route.py The test set the STATIC_ROUTE entry in conifg db without ifname: sonic-db-cli CONFIG_DB hmset 'STATIC_ROUTE|2.2.2.0/24' nexthop 192.168.0.18,192.168.0.25,192.168.0.23 "STATIC_ROUTE": { "2.2.2.0/24": { "nexthop": "192.168.0.18,192.168.0.25,192.168.0.23" } }, Validate that the arp_update gets the proper ARP_UPDATE_VARDS using arp_update_vars.j2 template from config db and does not crash: { "switch_type": "", "interface": "", "pc_interface" : "PortChannel101 PortChannel102 PortChannel103 PortChannel104 ", "vlan_sub_interface": "", "vlan" : "Vlan1000", "static_route_nexthops": "192.168.0.18 192.168.0.25 192.168.0.23 ", "static_route_ifnames": "" } validate route/test_static_route.py testcase pass. --- files/build_templates/arp_update_vars.j2 | 3 ++- files/scripts/arp_update | 27 +++++++++++++++++------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/files/build_templates/arp_update_vars.j2 b/files/build_templates/arp_update_vars.j2 index 91992e781a..d17f17ac88 100644 --- a/files/build_templates/arp_update_vars.j2 +++ b/files/build_templates/arp_update_vars.j2 @@ -4,5 +4,6 @@ "pc_interface" : "{% for (name, prefix) in PORTCHANNEL_INTERFACE|pfx_filter %}{% if prefix|ipv6 %}{{ name }} {% endif %}{% endfor %}", "vlan_sub_interface": "{% for (name, prefix) in VLAN_SUB_INTERFACE|pfx_filter %}{% if prefix|ipv6 %}{{ name }} {% endif %}{% endfor %}", "vlan" : "{% if VLAN %}{{ VLAN.keys() | join(' ') }}{% endif %}", - "static_route_nexthops": "{% if STATIC_ROUTE %}{% for static_route_prefix, static_route_attr in STATIC_ROUTE.items() %}{%- if static_route_prefix -%}{{ static_route_attr['nexthop'].split(',') | join(' ') | lower + " " }}{%- endif -%}{% endfor %}{% endif %}" + "static_route_nexthops": "{% if STATIC_ROUTE %}{% for static_route_prefix, static_route_attr in STATIC_ROUTE.items() %}{%- if static_route_prefix -%}{{ static_route_attr['nexthop'].split(',') | join(' ') | lower + " " }}{%- endif -%}{% endfor %}{% endif %}", + "static_route_ifnames": "{% if STATIC_ROUTE %}{% for static_route_prefix, static_route_attr in STATIC_ROUTE.items() %}{%- if static_route_prefix and 'ifname' in static_route_attr -%}{{ static_route_attr['ifname'].split(',') | join(' ') + " " }}{%- endif -%}{% endfor %}{% endif %}" } diff --git a/files/scripts/arp_update b/files/scripts/arp_update index b686e1a93e..2725f03466 100755 --- a/files/scripts/arp_update +++ b/files/scripts/arp_update @@ -14,13 +14,16 @@ while /bin/true; do ARP_UPDATE_VARS=$(sonic-cfggen -d -t ${ARP_UPDATE_VARS_FILE}) SWITCH_TYPE=$(echo $ARP_UPDATE_VARS | jq -r '.switch_type') if [[ "$SWITCH_TYPE" == "chassis-packet" ]]; then - STATIC_ROUTE_NEXTHOPS=$(echo $ARP_UPDATE_VARS | jq -r '.static_route_nexthops') + # Get array of Nexthops and ifnames. Nexthops and ifnames are mapped one to one + STATIC_ROUTE_NEXTHOPS=($(echo $ARP_UPDATE_VARS | jq -r '.static_route_nexthops')) + STATIC_ROUTE_IFNAMES=($(echo $ARP_UPDATE_VARS | jq -r '.static_route_ifnames')) # on supervisor/rp exit the script gracefully - if [[ -z "$STATIC_ROUTE_NEXTHOPS" ]]; then + if [[ -z "$STATIC_ROUTE_NEXTHOPS" ]] || [[ -z "$STATIC_ROUTE_IFNAMES" ]]; then logger "arp_update: exiting as no static route in packet based chassis" exit 0 fi - for nexthop in $STATIC_ROUTE_NEXTHOPS; do + for i in ${!STATIC_ROUTE_NEXTHOPS[@]}; do + nexthop="${STATIC_ROUTE_NEXTHOPS[i]}" if [[ $nexthop == *"."* ]]; then neigh_state=( $(ip -4 neigh show | grep -w $nexthop | tr -s ' ' | cut -d ' ' -f 3,4) ) ping_prefix=ping @@ -28,11 +31,19 @@ while /bin/true; do neigh_state=( $(ip -6 neigh show | grep -w $nexthop | tr -s ' ' | cut -d ' ' -f 3,4) ) ping_prefix=ping6 fi - - if [[ "${neigh_state[1]}" == "INCOMPLETE" ]] || [[ "${neigh_state[1]}" == "FAILED" ]]; then - pingcmd="timeout 0.2 $ping_prefix -I ${neigh_state[0]} -n -q -i 0 -c 1 -W 1 $nexthop >/dev/null" - eval $pingcmd - logger "arp_update: sttaic route nexthop not resolved, pinging $nexthop on ${neigh_state[0]}" + if [[ -z "${neigh_state}" ]] || [[ "${neigh_state[1]}" == "INCOMPLETE" ]] || [[ "${neigh_state[1]}" == "FAILED" ]]; then + interface="${STATIC_ROUTE_IFNAMES[i]}" + if [[ -z "$interface" ]]; then + # should never be here, handling just in case + logger "ERR: arp_update: missing interface entry for static route $nexthop" + interface=${neigh_state[0]} + fi + intf_up=$(ip link show $interface | grep "state UP") + if [[ -n "$intf_up" ]]; then + pingcmd="timeout 0.2 $ping_prefix -I ${interface} -n -q -i 0 -c 1 -W 1 $nexthop >/dev/null" + eval $pingcmd + logger "arp_update: static route nexthop not resolved, pinging $nexthop on ${neigh_state[0]}" + fi fi done