7260cx3 DualToR config.bcm support based on DualToR setting in device metadata at boot time (#7168)

* 7260cx3 DualToR config.bcm support based on DualToR setting in device metadata at boot time. 
For HWSKU Arista-7260CX3-C64 the MMU setting SOC for T0/T1 is also combined into the config.bcm.j2 logic so use just one config file and adding delta based on Switch Roles.

device/arista/x86_64-arista_7260cx3_64/Arista-7260CX3-C64/config.bcm.j2

@@ -1,3 +1,24 @@
+{# Construct config.bcm to include additional soc properties per specific device metadata requirement #}
+{%- set mmu_sock = 'mmu_init_config="MSFT-TH2-Tier1"' -%}
+{%- set IPinIP_sock = '' -%}
+{%- if DEVICE_METADATA is defined and DEVICE_METADATA['localhost'] is defined -%}
+{%-     if DEVICE_METADATA['localhost']['type'] is defined -%}
+{%-         set switch_role = DEVICE_METADATA['localhost']['type'] -%}
+{%-         if 'torrouter' in switch_role.lower() or 'torswitch' in switch_role.lower() %}
+{%-             set mmu_sock = 'mmu_init_config="MSFT-TH2-Tier0"' -%}
+{%-         endif %}
+{%-     endif %}
+{%-     if DEVICE_METADATA['localhost']['subtype'] is defined -%}
+{%-         set switch_subtype = DEVICE_METADATA['localhost']['subtype'] -%}
+{%-         if 'dualtor' in switch_subtype.lower() %}
+{%-             set IPinIP_sock = 'sai_tunnel_support=1
+                                   host_as_route_disable=1
+                                   l3_ecmp_levels=2' -%}
+{%-         endif %}
+{%-     endif %}
+{%- endif %}
+{# The following is the common soc properties that used to be named "th2-a7260cx3-64-64x100G-t1.config.bcm" #}
+
 PHY_AN_ALLOW_PLL_CHANGE=1
 arl_clean_timeout_usec=15000000
 asf_mem_profile=2
@@ -1012,4 +1033,5 @@ serdes_preemphasis_115=0x184606
 serdes_preemphasis_116=0x103706
 serdes_preemphasis_117=0x133c06
 
-mmu_init_config="MSFT-TH2-Tier0"
+{{ mmu_sock }}
+{{ IPinIP_sock }}

device/arista/x86_64-arista_7260cx3_64/Arista-7260CX3-C64/sai.profile

@@ -0,0 +1,2 @@
+SAI_INIT_CONFIG_FILE=/etc/sai.d/config.bcm
+SAI_NUM_ECMP_MEMBERS=64

device/arista/x86_64-arista_7260cx3_64/Arista-7260CX3-C64/sai.profile.j2

@@ -1,14 +0,0 @@
-{# Get sai.profile based on switch_role #}
-{%- if DEVICE_METADATA is defined -%}
-{%-     set switch_role = DEVICE_METADATA['localhost']['type'] -%}
-{%-     if 'torrouter' in switch_role.lower() or 'torswitch' in switch_role.lower() %}
-{%         set sai_profile_contents = 'SAI_INIT_CONFIG_FILE=/usr/share/sonic/hwsku/th2-a7260cx3-64-64x100G-t0.config.bcm' -%}
-{%-     else %}
-{%         set sai_profile_contents = 'SAI_INIT_CONFIG_FILE=/usr/share/sonic/hwsku/th2-a7260cx3-64-64x100G-t1.config.bcm' -%}
-{%-     endif %}
-{%- else %}
-{%      set sai_profile_contents = 'SAI_INIT_CONFIG_FILE=/usr/share/sonic/hwsku/th2-a7260cx3-64-64x100G-t1.config.bcm' -%}
-{%- endif %}
-{# Write the contents of sai_ profile_filename to sai.profile file #}
-{{ sai_profile_contents }}
-SAI_NUM_ECMP_MEMBERS=64

device/arista/x86_64-arista_7260cx3_64/Arista-7260CX3-D108C8/config.bcm.j2

@@ -1,3 +1,14 @@
+{# Construct config.bcm to include additional soc properties per specific device metadata requirement #}
+{%- set IPinIP_sock = '' -%}
+{%- if DEVICE_METADATA is defined and DEVICE_METADATA['localhost'] is defined and DEVICE_METADATA['localhost']['subtype'] is defined -%}
+{%-     set switch_subtype = DEVICE_METADATA['localhost']['subtype'] -%}
+{%-     if 'dualtor' in switch_subtype.lower() %}
+{%-         set IPinIP_sock = 'sai_tunnel_support=1
+                               host_as_route_disable=1
+                               l3_ecmp_levels=2' -%}
+{%-     endif %}
+{%- endif %}
+{# The following is the common soc properties that used to be named "th2-a7260cx3-64-112x50G+8x100G.config.bcm" #}
 PHY_AN_ALLOW_PLL_CHANGE=1
 arl_clean_timeout_usec=15000000
 asf_mem_profile=2
@@ -934,3 +945,4 @@ serdes_preemphasis_130=0x580c
 serdes_preemphasis_131=0x580c
 
 mmu_init_config="MSFT-TH2-Tier0"
+{{ IPinIP_sock }}

device/arista/x86_64-arista_7260cx3_64/Arista-7260CX3-D108C8/sai.profile

@@ -1,2 +1,2 @@
-SAI_INIT_CONFIG_FILE=/usr/share/sonic/hwsku/th2-a7260cx3-64-112x50G+8x100G.config.bcm
+SAI_INIT_CONFIG_FILE=/etc/sai.d/config.bcm
 SAI_NUM_ECMP_MEMBERS=64

device/arista/x86_64-arista_7260cx3_64/Arista-7260CX3-Q64/config.bcm.j2

@@ -1,3 +1,14 @@
+{# Construct config.bcm to include additional soc properties per specific device metadata requirement #}
+{%- set IPinIP_sock = '' -%}
+{%- if DEVICE_METADATA is defined and DEVICE_METADATA['localhost'] is defined and DEVICE_METADATA['localhost']['subtype'] is defined -%}
+{%-     set switch_subtype = DEVICE_METADATA['localhost']['subtype'] -%}
+{%-     if 'dualtor' in switch_subtype.lower() %}
+{%-         set IPinIP_sock = 'sai_tunnel_support=1
+                               host_as_route_disable=1
+                               l3_ecmp_levels=2' -%}
+{%-     endif %}
+{%- endif %}
+{# The following is the common soc properties that used to be named "th2-a7260cx3-64-64x40G.config.bcm" #}
 PHY_AN_ALLOW_PLL_CHANGE=1
 arl_clean_timeout_usec=15000000
 asf_mem_profile=2
@@ -1013,3 +1024,4 @@ serdes_preemphasis_116=0x105004
 serdes_preemphasis_117=0x105004
 
 mmu_init_config="MSFT-TH2-Tier0"
+{{ IPinIP_sock }}

device/arista/x86_64-arista_7260cx3_64/Arista-7260CX3-Q64/sai.profile

@@ -1,2 +1,2 @@
-SAI_INIT_CONFIG_FILE=/usr/share/sonic/hwsku/th2-a7260cx3-64-64x40G.config.bcm
+SAI_INIT_CONFIG_FILE=/etc/sai.d/config.bcm
 SAI_NUM_ECMP_MEMBERS=64

platform/broadcom/docker-syncd-brcm/start.sh

@@ -9,10 +9,22 @@ rm -f ${SYNCD_SOCKET_FILE}
 
 mkdir -p /etc/sai.d/
 
+# There are two ways to specify the contents of the SAI_INIT_CONFIG_FILE and they are mutually exclusive
+# via current method (sai.profile.j2) or new method (config.bcm.j2)
+# If delta is large, use sai.profile.j2 which basically require the user to select which config file to use
+# If delta is small, use config.bcm.j2 where additional SAI INIT config properties are added
+#   based on specific device metadata requirement
+#   in this case sai.profile should have been modified to use the path /etc/sai.d/config.bcm
+# There is also a possibility that both sai.profile.j2 and config.bcm.j2 are absent.  in that cacse just copy
+# sai.profile to the new /etc/said directory.
+
 # Create/Copy the sai.profile to /etc/sai.d/sai.profile
 if [ -f $HWSKU_DIR/sai.profile.j2 ]; then
     sonic-cfggen -d -t $HWSKU_DIR/sai.profile.j2 > /etc/sai.d/sai.profile
 else
+    if [ -f $HWSKU_DIR/config.bcm.j2 ]; then
+        sonic-cfggen -d -t $HWSKU_DIR/config.bcm.j2 > /etc/sai.d/config.bcm
+    fi
     if [ -f $HWSKU_DIR/sai.profile ]; then
         cp $HWSKU_DIR/sai.profile /etc/sai.d/sai.profile
     fi