Add health check probe for k8s upgrade containers. (#15223)

#### Why I did it
After k8s upgrade a container, k8s can only know the container is running, don't know the service's status inside container. So we need a probe inside container, k8s will call the probe to check whether the container is really ready.
##### Work item tracking
- Microsoft ADO **(number only)**: 22453004
#### How I did it
Add a health check probe inside config engine container, the probe will check whether the start service exit normally or not if the start service exists and call the python script to do container self-related specific checks if the script is there. The python script should be implemented by feature owner if it's needed.

more details: [design doc](https://github.com/sonic-net/SONiC/blob/master/doc/kubernetes/health-check.md)
#### How to verify it
Check path /usr/bin/readiness_probe.sh inside container.

#### Which release branch to backport (provide reason below if selected)

- [ ] 201811
- [ ] 201911
- [ ] 202006
- [ ] 202012
- [ ] 202106
- [ ] 202111
- [x] 202205
- [x] 202211

#### Tested branch (Please provide the tested image version)
- [x] 20220531.28

dockers/docker-config-engine-bullseye/Dockerfile.j2

@@ -44,6 +44,7 @@ RUN pip3 install redis==4.5.4
 
 # Copy files
 COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"]
+COPY ["files/readiness_probe.sh", "/usr/bin/"]
 COPY ["files/container_startup.py", "/usr/share/sonic/scripts/"]
 
 ## Clean up

dockers/docker-config-engine-buster/Dockerfile.j2

@@ -44,6 +44,7 @@ RUN pip3 install redis==4.5.4
 
 # Copy files
 COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"]
+COPY ["files/readiness_probe.sh", "/usr/bin/"]
 COPY ["files/container_startup.py", "/usr/share/sonic/scripts/"]
 
 ## Clean up

rules/docker-config-engine-bullseye.mk

@@ -19,6 +19,7 @@ $(DOCKER_CONFIG_ENGINE_BULLSEYE)_LOAD_DOCKERS += $(DOCKER_BASE_BULLSEYE)
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $(SWSS_VARS_TEMPLATE)
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $(RSYSLOG_PLUGIN_CONF_J2)
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT)
+$(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $($(SONIC_CTRMGRD)_HEALTH_PROBE)
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT)
 
 $(DOCKER_CONFIG_ENGINE_BULLSEYE)_DBG_DEPENDS = $($(DOCKER_BASE_BULLSEYE)_DBG_DEPENDS) \

rules/docker-config-engine-buster.mk

@@ -18,6 +18,7 @@ $(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(SWSS_VARS_TEMPLATE)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(RSYSLOG_PLUGIN_CONF_J2)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT)
+$(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_HEALTH_PROBE)
 $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT)
 
 $(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS = $($(DOCKER_BASE_BUSTER)_DBG_DEPENDS) \

rules/sonic-ctrmgrd.mk

@@ -20,12 +20,16 @@ $($(SONIC_CTRMGRD)_CFG_JSON)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH)
 $(SONIC_CTRMGRD)_SERVICE = ctrmgrd.service
 $($(SONIC_CTRMGRD)_SERVICE)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH)
 
+$(SONIC_CTRMGRD)_HEALTH_PROBE = readiness_probe.sh
+$($(SONIC_CTRMGRD)_HEALTH_PROBE)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH)
+
 SONIC_PYTHON_WHEELS += $(SONIC_CTRMGRD)
 
 $(SONIC_CTRMGRD)_FILES = $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT)
 $(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT)
 $(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_CFG_JSON)
 $(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_SERVICE)
+$(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_HEALTH_PROBE)
 
 SONIC_COPY_FILES += $($(SONIC_CTRMGRD)_FILES)
 

src/sonic-ctrmgrd/ctrmgr/readiness_probe.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# This script is used by k8s to check the readiness of containers
+# Check if the container is readiness or not, exit code 0 means readiness, others mean not readiness
+
+#### exit code contract, k8s only cares zero or not none-zero, but we want to use none-zero code to indicate different error
+# 0: readiness
+# 1: if the hook script is python code, the default crash exit code is 1
+# 2: supervisor start service doesn't exit normally
+# other exit code: returned by post_check_script, define in the post_check_script, should not include 1,2
+
+# check if the start service exists
+# if the start service doesn't exist, do nothing
+# if the start service exists, check if it exits normally
+# if the start service doesn't exit normally, exit with code 2
+pre_check_service_name="start"
+no_process_string="ERROR (no such process)"
+service_status=$(supervisorctl status $pre_check_service_name)
+if [[ $service_status != *"$no_process_string"* ]] && [[ $(echo $service_status |awk '{print $2}') != 'EXITED' ]]; then
+    exit 2
+fi
+
+# feature owner can add their own readiness check script
+# check if the post_check_script exists
+# if the post_check_script exists, run it
+# if the post_check_script exits with non-zero code, exit with the code
+post_check_script="/usr/bin/readiness_probe_hook"
+if [ -x $post_check_script ]; then
+    $post_check_script
+    post_check_result=$?
+    if [ $post_check_result != 0 ]; then
+        exit $post_check_result
+    fi
+fi
+
+exit 0