[asan] Add address sanitizer support. (#9857)

Implement infrastructure that allows enabling address sanitizer
for docker containers. Enable address sanitizer for SWSS container.

- Why I did it
To add a possibility to compile SONiC applications with address sanitizer (ASAN).
ASAN is a memory error detector for C/C++. It finds:
1. Use after free (dangling pointer dereference)
2. Heap buffer overflow
3. Stack buffer overflow
4. Global buffer overflow
5. Use after return
6. Use after the scope
7. Initialization order bugs
8. Memory leaks

- How I did it
By adding new ENABLE_ASAN configuration option.

- How to verify it
By default ASAN is disabled and the SONiC image is not affected.
When ASAN is enabled it inspects all allocation, deallocation, and memory usage that the application does in run time. To verify whether the application has memory errors tests that trigger memory usage of the application should be run. Ideally, the whole regression tests should be run. Memory leaks reports will be placed in /var/log/asan/ directory of SONiC host OS.

Signed-off-by: Oleksandr Ivantsiv <oivantsiv@nvidia.com>

Makefile.work

@@ -295,6 +295,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SLAVE_DIR=$(SLAVE_DIR) \
                            ENABLE_AUTO_TECH_SUPPORT=$(ENABLE_AUTO_TECH_SUPPORT) \
                            BUILD_MULTIASIC_KVM=$(BUILD_MULTIASIC_KVM) \
+                           ENABLE_ASAN=$(ENABLE_ASAN) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset

dockers/docker-orchagent/Dockerfile.j2

@@ -70,10 +70,14 @@ COPY ["files/arp_update", "/usr/bin"]
 COPY ["arp_update.conf", "files/arp_update_vars.j2", "/usr/share/sonic/templates/"]
 COPY ["ndppd.conf", "/usr/share/sonic/templates/"]
 COPY ["enable_counters.py", "tunnel_packet_handler.py", "/usr/bin/"]
-COPY ["docker-init.sh", "orchagent.sh", "swssconfig.sh", "buffermgrd.sh", "/usr/bin/"]
+COPY ["orchagent.sh", "swssconfig.sh", "buffermgrd.sh", "/usr/bin/"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
 
 # Copy all Jinja2 template files into the templates folder
 COPY ["*.j2", "/usr/share/sonic/templates/"]
 
+RUN sonic-cfggen -a "{\"ENABLE_ASAN\":\"{{ENABLE_ASAN}}\"}" -t /usr/share/sonic/templates/docker-init.j2 > /usr/bin/docker-init.sh
+RUN rm -f /usr/share/sonic/templates/docker-init.j2
+RUN chmod 755 /usr/bin/docker-init.sh
+
 ENTRYPOINT ["/usr/bin/docker-init.sh"]

dockers/docker-orchagent/docker-init.j2

@@ -6,6 +6,9 @@ mkdir -p /etc/supervisor/conf.d/
 
 CFGGEN_PARAMS=" \
     -d \
+{% if ENABLE_ASAN == "y" %}
+     -a "{\"ENABLE_ASAN\":\"{{ENABLE_ASAN}}\"}" \
+{% endif %}
     -y /etc/sonic/constants.yml \
     -t /usr/share/sonic/templates/switch.json.j2,/etc/swss/config.d/switch.json \
     -t /usr/share/sonic/templates/vxlan.json.j2,/etc/swss/config.d/vxlan.json \

dockers/docker-orchagent/supervisord.conf.j2

@@ -49,6 +49,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=rsyslogd:running
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/gearsyncd-asan.log"
+{% endif %}
 {%- endif %}
 
 {% if is_fabric_asic == 0 %}
@@ -61,6 +64,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=rsyslogd:running
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/portsyncd-asan.log"
+{% endif %}
 {%- endif %}
 
 [program:orchagent]
@@ -72,6 +78,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for={{ orchagent_dependent_startup_wait_for }}
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/orchagent-asan.log"
+{% endif %}
 
 [program:swssconfig]
 command=/usr/bin/swssconfig.sh
@@ -84,6 +93,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=orchagent:running
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/swssconfig-asan.log"
+{% endif %}
 
 {% if is_fabric_asic == 0 %}
 [program:restore_neighbors]
@@ -111,6 +123,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=orchagent:running
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/coppmgrd-asan.log"
+{% endif %}
 {%- endif %}
 
 {% if is_fabric_asic == 0 %}
@@ -123,6 +138,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=swssconfig:exited
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/neighsyncd-asan.log"
+{% endif %}
 {%- endif %}
 
 {% if is_fabric_asic == 0 %}
@@ -135,6 +153,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=swssconfig:exited
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/vlanmgrd-asan.log"
+{% endif %}
 {%- endif %}
 
 {% if is_fabric_asic == 0 %}
@@ -147,6 +168,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=swssconfig:exited
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/intfmgrd-asan.log"
+{% endif %}
 {%- endif %}
 
 {% if is_fabric_asic == 0 %}
@@ -159,6 +183,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=swssconfig:exited
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/portmgrd-asan.log"
+{% endif %}
 {%- endif %}
 
 {% if is_fabric_asic == 0 %}
@@ -171,6 +198,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=swssconfig:exited
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/buffermgrd-asan.log"
+{% endif %}
 {%- endif %}
 
 {% if is_fabric_asic == 0 %}
@@ -183,6 +213,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=swssconfig:exited
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/vrfmgrd-asan.log"
+{% endif %}
 {%- endif %}
 
 {% if is_fabric_asic == 0 %}
@@ -195,6 +228,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=swssconfig:exited
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/nbrmgrd-asan.log"
+{% endif %}
 {%- endif %}
 
 {% if is_fabric_asic == 0 %}
@@ -207,6 +243,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=swssconfig:exited
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/vxlanmgrd-asan.log"
+{% endif %}
 {%- endif %}
 
 {% if is_fabric_asic == 0 %}
@@ -219,6 +258,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=swssconfig:exited
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/tunnelmgrd-asan.log"
+{% endif %}
 {%- endif %}
 
 [program:enable_counters]
@@ -241,6 +283,9 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 dependent_startup=true
 dependent_startup_wait_for=swssconfig:exited
+{% if ENABLE_ASAN == "y" %}
+environment=ASAN_OPTIONS="log_path=/var/log/asan/fdbsyncd-asan.log"
+{% endif %}
 {%- endif %}
 
 {% if is_fabric_asic == 0 %}

files/build_templates/docker_image_ctl.j2

@@ -302,6 +302,10 @@ start() {
     if [ ! -f /etc/sonic/zero_profiles.json ] && [ -f /usr/share/sonic/templates/zero_profiles.j2 ]; then
         sonic-cfggen -d -t /usr/share/sonic/device/$PLATFORM/zero_profiles.j2 > /etc/sonic/zero_profiles.json
     fi
+
+    {%- if enable_asan == "y" %}
+    mkdir -p /var/log/asan
+    {%- endif %}
     {%- endif %}
 
     # In Multi ASIC platforms the global database config file database_global.json will exist.
@@ -425,6 +429,9 @@ start() {
 {%- endif %}
 {%- if docker_container_name == "swss" %}
         -e ASIC_VENDOR={{ sonic_asic_platform }} \
+{%- if enable_asan == "y" %}
+        -v /var/log/asan/:/var/log/asan \
+{%- endif -%}
 {%- endif -%}
 {%- if docker_container_name == "bgp" %}
         -v /etc/sonic/frr/$DEV:/etc/frr:rw \
@@ -486,6 +493,8 @@ stop() {
     {%- elif docker_container_name == "teamd" %}
     # Longer timeout of 60 sec to wait for Portchannels to be cleaned.
     /usr/local/bin/container stop -t 60 $DOCKERNAME
+    {%- elif docker_container_name == "swss" and enable_asan == "y" %}
+    /usr/local/bin/container stop -t 60 $DOCKERNAME
     {%- else %}
     /usr/local/bin/container stop $DOCKERNAME
     {%- endif %}

rules/config

@@ -208,3 +208,6 @@ BUILD_MULTIASIC_KVM = n
 
 # INCLUDE_MUX - build docker-mux for dual ToR (Gemini)
 INCLUDE_MUX = y
+
+# ENABLE_ASAN - enable address sanitizer
+ENABLE_ASAN ?= n

slave.mk

@@ -189,6 +189,13 @@ ifeq ($(SONIC_INCLUDE_MUX),y)
 INCLUDE_MUX = y
 endif
 
+ifeq ($(ENABLE_ASAN),y)
+	ifneq ($(CONFIGURED_ARCH),amd64)
+		@echo "Disabling SWSS address sanitizer due to incompatible CPU architecture: $(CONFIGURED_ARCH)"
+		override ENABLE_ASAN = n
+	endif
+endif
+
 include $(RULES_PATH)/functions
 
 ifeq ($(SONIC_USE_PDDF_FRAMEWORK),y)
@@ -318,6 +325,7 @@ $(info "ENABLE_AUTO_TECH_SUPPORT"        : "$(ENABLE_AUTO_TECH_SUPPORT)")
 $(info "PDDF_SUPPORT"                    : "$(PDDF_SUPPORT)")
 $(info "MULTIARCH_QEMU_ENVIRON"          : "$(MULTIARCH_QEMU_ENVIRON)")
 $(info "SONIC_VERSION_CONTROL_COMPONENTS": "$(SONIC_VERSION_CONTROL_COMPONENTS)")
+$(info "ENABLE_ASAN"                     : "$(ENABLE_ASAN)")
 ifeq ($(CONFIGURED_PLATFORM),vs)
 $(info "BUILD_MULTIASIC_KVM"             : "$(BUILD_MULTIASIC_KVM)")
 endif
@@ -1017,6 +1025,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
 	export include_p4rt="$(INCLUDE_P4RT)"
 	export include_sflow="$(INCLUDE_SFLOW)"
 	export enable_auto_tech_support="$(ENABLE_AUTO_TECH_SUPPORT)"
+	export enable_asan="$(ENABLE_ASAN)"
 	export include_macsec="$(INCLUDE_MACSEC)"
 	export include_mgmt_framework="$(INCLUDE_MGMT_FRAMEWORK)"
 	export include_iccpd="$(INCLUDE_ICCPD)"