sonic-buildimage/files/build_templates/docker_image_ctl.j2
Marty Y. Lok cb4c66ae98
[chassis][multiasic] fixed rsyslogd FATAL issue in the database container in multi-asic box (#8390)
Why I did it
Fix for issue #8389

How I did it
The /etc/rsyslog.conf is empty file which cause the FATAL of the process rsyslogd in the global instance database container. The function updateSyslogConf() should only generate the rsyslog.conf for containers in the namespace. it should not do it for the containers in the global instance. Instead, default rsyslog.conf should be used. Especially for database container, updateSyslogConf() is called before the database container is created. The result cause the sonic-cfggen failed to generate the rsyslog.conf.Why I did it
Fix for issue #8389

How I did it
The /etc/rsyslog.conf is empty file which cause the FATAL of the process rsyslogd in the global instance database container. The function updateSyslogConf() should only generate the rsyslog.conf for containers in the namespace. it should not do it for the containers in the global instance. Instead, default rsyslog.conf should be used. Especially for database container, updateSyslogConf() is called before the database container is created. The result cause the sonic-cfggen failed to generate the rsyslog.conf.

Signed-off-by: mlok <marty.lok@nokia.com>
2021-12-01 07:16:49 -08:00

528 lines
21 KiB
Django/Jinja

#!/bin/bash
# single instance containers are still supported (even though it might not look like it)
# if no instance number is passed to this script, $DEV will simply be unset, resulting in docker
# commands being sent to the base container name. E.g. `docker start database$DEV` simply starts
# the container `database` if no instance number is passed since `$DEV` is not defined
{%- if docker_container_name == "database" %}
link_namespace() {
# Makes namespace of a docker container available in
# /var/run/netns so it can be managed with iproute2
mkdir -p /var/run/netns
PID="$(docker inspect -f {{"'{{.State.Pid}}'"}} "${DOCKERNAME}")"
PIDS=`ip netns pids "$NET_NS" 2>/dev/null`
if [ "$?" -eq "0" ]; then # namespace exists
if `echo $PIDS | grep --quiet -w $PID`; then # namespace is correctly linked
return 0
else # if it's incorrectly linked remove it
ip netns delete $NET_NS
fi
fi
ln -s /proc/$PID/ns/net /var/run/netns/$NET_NS
}
{%- endif %}
function updateSyslogConf()
{
# On multiNPU platforms, change the syslog target ip to docker0 ip to allow logs from containers
# running on the namespace to reach the rsyslog service running on the host
# Also update the container name
if [[ ($NUM_ASIC -gt 1) ]]; then
TARGET_IP=$(docker network inspect bridge --format={{ "'{{(index .IPAM.Config 0).Gateway}}'" }})
CONTAINER_NAME="$DOCKERNAME"
TMP_FILE="/tmp/rsyslog.$CONTAINER_NAME.conf"
{%- if docker_container_name == "database" %}
python -c "import jinja2, os; paths=['/usr/share/sonic/templates']; loader = jinja2.FileSystemLoader(paths); env = jinja2.Environment(loader=loader, trim_blocks=True); template_file='/usr/share/sonic/templates/rsyslog-container.conf.j2'; template = env.get_template(os.path.basename(template_file)); data=template.render({\"target_ip\":\"$TARGET_IP\",\"container_name\":\"$CONTAINER_NAME\"}); print(data)" > $TMP_FILE
{%- else %}
sonic-cfggen -t /usr/share/sonic/templates/rsyslog-container.conf.j2 -a "{\"target_ip\": \"$TARGET_IP\", \"container_name\": \"$CONTAINER_NAME\" }" > $TMP_FILE
{%- endif %}
docker cp $TMP_FILE ${DOCKERNAME}:/etc/rsyslog.conf
rm -rf $TMP_FILE
fi
}
function ebtables_config()
{
if [ "$DEV" ]; then
# Install ebtables filter in namespaces on multi-asic.
ip netns exec $NET_NS ebtables-restore < /etc/ebtables.filter.cfg
else
if [[ ! ($NUM_ASIC -gt 1) ]]; then
# Install ebtables filter in host for single asic.
ebtables-restore < /etc/ebtables.filter.cfg
fi
fi
}
function getMountPoint()
{
echo $1 | python -c "import sys, json, os; mnts = [x for x in json.load(sys.stdin)[0]['Mounts'] if x['Destination'] == '/usr/share/sonic/hwsku']; print('' if len(mnts) == 0 else os.path.abspath(mnts[0]['Source']))" 2>/dev/null
}
function getBootType()
{
# same code snippet in files/scripts/syncd.sh
case "$(cat /proc/cmdline)" in
*SONIC_BOOT_TYPE=warm*)
TYPE='warm'
;;
*SONIC_BOOT_TYPE=fastfast*)
TYPE='fastfast'
;;
*SONIC_BOOT_TYPE=fast*|*fast-reboot*)
TYPE='fast'
;;
*)
TYPE='cold'
esac
echo "${TYPE}"
}
function preStartAction()
{
{%- if docker_container_name == "database" %}
WARM_DIR=/host/warmboot
if [ "$DATABASE_TYPE" != "chassisdb" ]; then
if [[ ("$BOOT_TYPE" == "warm" || "$BOOT_TYPE" == "fastfast") && -f $WARM_DIR/dump.rdb ]]; then
# Load redis content from /host/warmboot/dump.rdb
docker cp $WARM_DIR/dump.rdb database$DEV:/var/lib/redis/dump.rdb
else
# Create an emtpy file and overwrite any RDB if already there
echo -n > /tmp/dump.rdb
docker cp /tmp/dump.rdb database$DEV:/var/lib/redis/
fi
fi
{%- elif docker_container_name == "pde" %}
if [[ $(/bin/systemctl status swss | grep -c running) -gt 0 ]]; then
echo "Stopping SWSS before starting PDE"
systemctl stop swss
fi
if [[ $(/bin/systemctl status watchdog-control.service | grep -c running) -gt 0 ]]; then
echo "Stopping watchdog-control.service before starting PDE"
systemctl stop watchdog-control.service
fi
{%- elif docker_container_name == "snmp" %}
$SONIC_DB_CLI STATE_DB HSET 'DEVICE_METADATA|localhost' chassis_serial_number $(decode-syseeprom -s)
{%- else %}
: # nothing
{%- endif %}
updateSyslogConf
}
{%- if docker_container_name == "database" %}
function setPlatformLagIdBoundaries()
{
CHASSIS_CONF=/usr/share/sonic/device/$PLATFORM/chassisdb.conf
if [ -f "$CHASSIS_CONF" ]; then
source $CHASSIS_CONF
docker exec -i ${DOCKERNAME} $SONIC_DB_CLI CHASSIS_APP_DB SET "SYSTEM_LAG_ID_START" "$lag_id_start"
docker exec -i ${DOCKERNAME} $SONIC_DB_CLI CHASSIS_APP_DB SET "SYSTEM_LAG_ID_END" "$lag_id_end"
fi
}
{%- endif %}
function postStartAction()
{
{%- if docker_container_name == "database" %}
if [ "$DEV" ]; then
# Enable the forwarding on eth0 interface in namespace.
SYSCTL_NET_CONFIG="/etc/sysctl.d/sysctl-net.conf"
docker exec -i database$DEV sed -i -e "s/^net.ipv4.conf.eth0.forwarding=0/net.ipv4.conf.eth0.forwarding=1/;
s/^net.ipv6.conf.eth0.forwarding=0/net.ipv6.conf.eth0.forwarding=1/" $SYSCTL_NET_CONFIG
docker exec -i database$DEV sysctl --system -e
link_namespace $DEV
fi
# Setup ebtables configuration
ebtables_config
# chassisdb starts before database starts, bypass the PING check since other
# databases are not availbale until database container is ready.
# also chassisdb doesn't support warm/fast reboot, its dump.rdb is deleted
# at service startup time, nothing need to be done here.
if [ "$DATABASE_TYPE" != "chassisdb" ]; then
# Wait until supervisord and redis starts. This change is needed
# because now database_config.json is jinja2 templated based
# and by the time file gets generated if we do redis ping
# then we catch python exception of file not valid
# that comes to syslog which is unwanted so wait till database
# config is ready and then ping
until [[ ($(docker exec -i database$DEV pgrep -x -c supervisord) -gt 0) && ($($SONIC_DB_CLI PING | grep -c PONG) -gt 0) ]]; do
sleep 1;
done
if [[ ("$BOOT_TYPE" == "warm" || "$BOOT_TYPE" == "fastfast") && -f $WARM_DIR/dump.rdb ]]; then
# retain the dump file from last boot for debugging purposes
mv $WARM_DIR/dump.rdb $WARM_DIR/dump.rdb.old
else
# If there is a config_db.json dump file, load it.
if [ -r /etc/sonic/config_db$DEV.json ]; then
if [ -r /etc/sonic/init_cfg.json ]; then
$SONIC_CFGGEN -j /etc/sonic/init_cfg.json -j /etc/sonic/config_db$DEV.json --write-to-db
else
$SONIC_CFGGEN -j /etc/sonic/config_db$DEV.json --write-to-db
fi
fi
if [[ "$BOOT_TYPE" == "fast" ]]; then
# set the key to expire in 3 minutes
$SONIC_DB_CLI STATE_DB SET "FAST_REBOOT|system" "1" "EX" "180"
fi
$SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"
fi
if [[ -x /usr/local/bin/db_migrator.py ]]; then
# Migrate the DB to the latest schema version if needed
if [ -z "$DEV" ]; then
/usr/local/bin/db_migrator.py -o migrate
fi
fi
# Add redis UDS to the redis group and give read/write access to the group
REDIS_SOCK="/var/run/redis${DEV}/redis.sock"
else
until [[ ($(docker exec -i ${DOCKERNAME} pgrep -x -c supervisord) -gt 0) &&
($(docker exec -i ${DOCKERNAME} $SONIC_DB_CLI CHASSIS_APP_DB PING | grep -c True) -gt 0) ]]; do
sleep 1
done
setPlatformLagIdBoundaries
REDIS_SOCK="/var/run/redis-chassis/redis_chassis.sock"
fi
chgrp -f redis $REDIS_SOCK && chmod -f 0760 $REDIS_SOCK
{%- elif docker_container_name == "swss" %}
docker exec swss$DEV rm -f /ready # remove cruft
if [[ "$BOOT_TYPE" == "fast" ]] && [[ -d /host/fast-reboot ]]; then
test -e /host/fast-reboot/fdb.json && docker cp /host/fast-reboot/fdb.json swss$DEV:/
test -e /host/fast-reboot/arp.json && docker cp /host/fast-reboot/arp.json swss$DEV:/
test -e /host/fast-reboot/default_routes.json && docker cp /host/fast-reboot/default_routes.json swss$DEV:/
test -e /host/fast-reboot/media_config.json && docker cp /host/fast-reboot/media_config.json swss$DEV:/
rm -fr /host/fast-reboot
fi
docker exec swss$DEV touch /ready # signal swssconfig.sh to go
{%- elif docker_container_name == "pmon" %}
DEVPATH="/usr/share/sonic/device"
REBOOT="platform_reboot"
PSENSOR="/usr/local/bin/platform_sensors.py"
if [ -d ${DEVPATH}/${PLATFORM} ] && [ -f $PSENSOR ]; then
exist=`docker exec -i pmon ls /usr/bin/platform_sensors.py "$@" 2>/dev/null`
if [ -z "$exist" ]; then
docker cp $PSENSOR pmon:/usr/bin/
fi
fi
{%- else %}
: # nothing
{%- endif %}
}
start() {
# Obtain boot type from kernel arguments
BOOT_TYPE=`getBootType`
# Obtain our platform as we will mount directories with these names in each docker
PLATFORM=${PLATFORM:-`$SONIC_CFGGEN -H -v DEVICE_METADATA.localhost.platform`}
# Parse the device specific asic conf file, if it exists
ASIC_CONF=/usr/share/sonic/device/$PLATFORM/asic.conf
if [ -f "$ASIC_CONF" ]; then
source $ASIC_CONF
fi
{%- if docker_container_name == "database" %}
# Don't mount HWSKU in {{docker_container_name}} container.
HWSKU=""
MOUNTPATH=""
{%- else %}
# Obtain our HWSKU as we will mount directories with these names in each docker
HWSKU=${HWSKU:-`$SONIC_CFGGEN -d -v 'DEVICE_METADATA["localhost"]["hwsku"]'`}
MOUNTPATH="/usr/share/sonic/device/$PLATFORM/$HWSKU"
if [ "$DEV" ]; then
MOUNTPATH="$MOUNTPATH/$DEV"
fi
{%- endif %}
DOCKERCHECK=`docker inspect --type container ${DOCKERNAME} 2>/dev/null`
if [ "$?" -eq "0" ]; then
{%- if docker_container_name == "database" %}
DOCKERMOUNT=""
{%- else %}
DOCKERMOUNT=`getMountPoint "$DOCKERCHECK"`
{%- endif %}
if [ x"$DOCKERMOUNT" == x"$MOUNTPATH" ]; then
preStartAction
{%- if docker_container_name == "database" %}
echo "Starting existing ${DOCKERNAME} container"
docker start ${DOCKERNAME}
{%- else %}
echo "Starting existing ${DOCKERNAME} container with HWSKU $HWSKU"
/usr/local/bin/container start ${DOCKERNAME}
{%- endif %}
postStartAction
exit $?
fi
# docker created with a different HWSKU, remove and recreate
echo "Removing obsolete ${DOCKERNAME} container with HWSKU $DOCKERMOUNT"
docker rm -f ${DOCKERNAME}
fi
{%- if docker_container_name == "database" %}
echo "Creating new ${DOCKERNAME} container"
if [ "$DATABASE_TYPE" != "chassisdb" ]; then
if [ -z "$DEV" ]; then
# if database_global exists in old_config, use it; otherwise use the default one in new image
if [ -f /etc/sonic/old_config/database_global.json ]; then
echo "Use database_global.json from old system..."
mv /etc/sonic/old_config/database_global.json /etc/sonic/
fi
fi
# if database_config exists in old_config, use it; otherwise use the default one in new image
if [ -f /etc/sonic/old_config/database_config$DEV.json ]; then
echo "Use database_config.json from old system..."
mv /etc/sonic/old_config/database_config$DEV.json /etc/sonic/
fi
fi
{%- else %}
echo "Creating new ${DOCKERNAME} container with HWSKU $HWSKU"
{%- endif %}
{%- if docker_container_name == "swss" %}
# Generate the asic_table.json and peripheral_table.json
if [ ! -f /etc/sonic/asic_table.json ] && [ -f /usr/share/sonic/templates/asic_table.j2 ]; then
sonic-cfggen -d -t /usr/share/sonic/templates/asic_table.j2 > /etc/sonic/asic_table.json
fi
if [ ! -f /etc/sonic/peripheral_table.json ] && [ -f /usr/share/sonic/device/$PLATFORM/port_peripheral_config.j2 ]; then
sonic-cfggen -d -t /usr/share/sonic/device/$PLATFORM/port_peripheral_config.j2 > /etc/sonic/peripheral_table.json
fi
if [ ! -f /etc/sonic/zero_profiles.json ] && [ -f /usr/share/sonic/templates/zero_profiles.j2 ]; then
sonic-cfggen -d -t /usr/share/sonic/device/$PLATFORM/zero_profiles.j2 > /etc/sonic/zero_profiles.json
fi
{%- endif %}
# In Multi ASIC platforms the global database config file database_global.json will exist.
# Parse the file and get the include path for the database_config.json files used in
# various namesapces. The database_config paths are relative to the DIR of SONIC_DB_GLOBAL_JSON.
SONIC_DB_GLOBAL_JSON="/var/run/redis/sonic-db/database_global.json"
if [ -f "$SONIC_DB_GLOBAL_JSON" ]; then
# TODO Create a separate python script with the below logic and invoke it here.
redis_dir_list=`/usr/bin/python -c "import sys; import os; import json; f=open(sys.argv[1]); \
global_db_dir = os.path.dirname(sys.argv[1]); data=json.load(f); \
print(\" \".join([os.path.normpath(global_db_dir+'/'+elem['include']).partition('sonic-db')[0]\
for elem in data['INCLUDES'] if 'namespace' in elem])); f.close()" $SONIC_DB_GLOBAL_JSON`
fi
{%- if docker_container_name == "database" %}
start_chassis_db=0
chassis_db_address=""
chassisdb_config="/usr/share/sonic/device/$PLATFORM/chassisdb.conf"
[ -f $chassisdb_config ] && source $chassisdb_config
DB_OPT=" -v /var/run/redis-chassis:/var/run/redis-chassis:ro "
if [[ "$start_chassis_db" != "1" ]] && [[ -z "$chassis_db_address" ]]; then
DB_OPT=""
else
DB_OPT=$DB_OPT" --add-host=redis_chassis.server:$chassis_db_address "
fi
{%- endif %}
if [ -z "$DEV" ]; then
NET="host"
# For Multi-ASIC platform we have to mount the redis paths for database instances running in different
# namespaces, into the single instance dockers like snmp, pmon on linux host. These global dockers
# will need to get/set tables from databases in different namespaces.
# /var/run/redis0 ---> mounted as --> /var/run/redis0
# /var/run/redis1 ---> mounted as --> /var/run/redis1 .. etc
# The below logic extracts the base DIR's where database_config.json's for various namespaces exist.
# redis_dir_list is a string of form "/var/run/redis0/ /var/run/redis1/ /var/run/redis2/"
{%- if docker_container_name != "database" %}
if [ -n "$redis_dir_list" ]; then
for redis_dir in $redis_dir_list
do
REDIS_MNT=$REDIS_MNT" -v $redis_dir:$redis_dir:rw "
done
fi
{%- else %}
if [ "$DATABASE_TYPE" == "chassisdb" ]; then
DB_OPT=${DB_OPT/redis-chassis:ro/redis-chassis:rw}
DB_OPT=$DB_OPT" -v /var/run/redis-chassis:/var/run/redis:rw "
DB_OPT=$DB_OPT" --env DATABASE_TYPE=$DATABASE_TYPE"
else
DB_OPT=$DB_OPT" -v /var/run/redis$DEV:/var/run/redis:rw "
fi
{%- endif %}
else
# This part of code is applicable for Multi-ASIC platforms. Here we mount the namespace specific
# redis directory into the docker running in that namespace. Below eg: is for namespace "asic1"
# /var/run/redis1 ---> mounted as --> /var/run/redis1
# redis_dir_list is a string of form "/var/run/redis0/ /var/run/redis1/ /var/run/redis2/"
if [ -n "$redis_dir_list" ]; then
id=`expr $DEV + 1`
redis_dir=`echo $redis_dir_list | cut -d " " -f $id`
REDIS_MNT=" -v $redis_dir:$redis_dir:rw "
fi
{%- if docker_container_name == "database" %}
DB_OPT=$DB_OPT" -v /var/run/redis$DEV:/var/run/redis:rw "
{%- else %}
NET="container:database$DEV"
DB_OPT=""
{%- endif %}
fi
{%- if docker_container_name == "bgp" %}
if [ "$DEV" ]; then
if [ ! -d "/etc/sonic/frr/$DEV" ]; then
mkdir /etc/sonic/frr/$DEV
cp -r /etc/sonic/frr/*.conf /etc/sonic/frr/$DEV
fi
fi
{%- endif %}
{%- if sonic_asic_platform == "mellanox" %}
# TODO: Mellanox will remove the --tmpfs exception after SDK socket path changed in new SDK version
{%- endif %}
docker create {{docker_image_run_opt}} \
--net=$NET \
-e RUNTIME_OWNER=local \
--uts=host \{# W/A: this should be set per-docker, for those dockers which really need host's UTS namespace #}
{%- if install_debug_image == "y" %}
-v /src:/src:ro -v /debug:/debug:rw \
{%- endif %}
{%- if '--log-driver=json-file' in docker_image_run_opt or '--log-driver' not in docker_image_run_opt %}
--log-opt max-size=2M --log-opt max-file=5 \
{%- endif %}
{%- if sonic_asic_platform == "mellanox" %}
{%- if docker_container_name == "syncd" %}
-v /var/log/mellanox:/var/log/mellanox:rw \
-v mlnx_sdk_socket:/var/run/sx_sdk \
-v mlnx_sdk_ready:/tmp \
-v /dev/shm:/dev/shm:rw \
-e SX_API_SOCKET_FILE=/var/run/sx_sdk/sx_api.sock \
{%- elif docker_container_name == "pmon" %}
-v /var/run/hw-management:/var/run/hw-management:rw \
-v mlnx_sdk_socket:/var/run/sx_sdk \
-v mlnx_sdk_ready:/tmp \
-e SX_API_SOCKET_FILE=/var/run/sx_sdk/sx_api.sock \
-v /dev/shm:/dev/shm:rw \
{%- else %}
{%- if mount_default_tmpfs|default("n") == "y" %}
--tmpfs /tmp \
{%- endif %}
{%- endif %}
{%- endif %}
{%- if sonic_asic_platform == "broadcom" %}
{%- if docker_container_name == "syncd" %}
-v /var/run/docker-syncd$DEV:/var/run/sswsyncd \
{%- endif %}
{%- endif %}
{%- if docker_container_name == "pmon" %}
-v /usr/share/sonic/firmware:/usr/share/sonic/firmware:rw \
{%- endif %}
{%- if docker_container_name == "swss" %}
-e ASIC_VENDOR={{ sonic_asic_platform }} \
{%- endif -%}
{%- if docker_container_name == "bgp" %}
-v /etc/sonic/frr/$DEV:/etc/frr:rw \
{%- endif %}
{%- if docker_container_name == "database" %}
$DB_OPT \
{%- else %}
-v /var/run/redis$DEV:/var/run/redis:rw \
-v /var/run/redis-chassis:/var/run/redis-chassis:ro \
-v /usr/share/sonic/device/$PLATFORM/$HWSKU/$DEV:/usr/share/sonic/hwsku:ro \
{%- endif %}
$REDIS_MNT \
-v /usr/share/sonic/device/$PLATFORM:/usr/share/sonic/platform:ro \
{%- if sonic_asic_platform != "mellanox" %}
{%- if mount_default_tmpfs|default("n") == "y" %}
--tmpfs /tmp \
{%- endif %}
{%- endif %}
{%- if mount_default_tmpfs|default("n") == "y" %}
--tmpfs /var/tmp \
{%- endif %}
--env "NAMESPACE_ID"="$DEV" \
--env "NAMESPACE_PREFIX"="$NAMESPACE_PREFIX" \
--env "NAMESPACE_COUNT"=$NUM_ASIC \
--name=$DOCKERNAME \
{%- if docker_image_name is defined %}
{{docker_image_name}}:latest \
{%- else %}
{{docker_image_id}} \
{%- endif %}
|| {
echo "Failed to docker run" >&1
exit 4
}
preStartAction
{%- if docker_container_name == "database" %}
docker start $DOCKERNAME
{%- else %}
/usr/local/bin/container start ${DOCKERNAME}
{%- endif %}
postStartAction
}
wait() {
{%- if docker_container_name == "database" %}
docker wait $DOCKERNAME
{%- else %}
/usr/local/bin/container wait $DOCKERNAME
{%- endif %}
}
stop() {
{%- if docker_container_name == "database" %}
docker stop $DOCKERNAME
if [ "$DEV" ]; then
ip netns delete "$NET_NS"
fi
{%- elif docker_container_name == "teamd" %}
# Longer timeout of 60 sec to wait for Portchannels to be cleaned.
/usr/local/bin/container stop -t 60 $DOCKERNAME
{%- else %}
/usr/local/bin/container stop $DOCKERNAME
{%- endif %}
}
DOCKERNAME={{docker_container_name}}
OP=$1
DEV=$2 # namespace/device number to operate on
{%- if docker_container_name == "database" %}
if [ "$DEV" == "chassisdb" ]; then
DATABASE_TYPE="chassisdb"
DOCKERNAME=$DOCKERNAME"-chassis"
unset DEV
fi
{%- endif %}
NAMESPACE_PREFIX="asic"
DOCKERNAME=$DOCKERNAME$DEV
if [ "$DEV" ]; then
NET_NS="$NAMESPACE_PREFIX$DEV" #name of the network namespace
SONIC_CFGGEN="sonic-cfggen -n $NET_NS"
SONIC_DB_CLI="sonic-db-cli -n $NET_NS"
else
NET_NS=""
SONIC_CFGGEN="sonic-cfggen"
SONIC_DB_CLI="sonic-db-cli"
fi
# read SONiC immutable variables
[ -f /etc/sonic/sonic-environment ] && . /etc/sonic/sonic-environment
case "$1" in
start|wait|stop)
$1
;;
*)
echo "Usage: $0 {start namespace(optional)|wait namespace(optional)|stop namespace(optional)}"
exit 1
;;
esac